diff --git a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy index b2eaefd9bc498..6a681d219f439 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy +++ b/Documentation/ABI/testing/configfs-usb-gadget-uac1_legacy @@ -5,8 +5,5 @@ Description: The attributes: audio_buf_size - audio buffer size - fn_cap - capture pcm device file name - fn_cntl - control device file name - fn_play - playback pcm device file name req_buf_size - ISO OUT endpoint request buffer size req_count - ISO OUT endpoint request count diff --git a/Documentation/ABI/testing/sysfs-bus-nvdimm b/Documentation/ABI/testing/sysfs-bus-nvdimm index 64eb8f4c6a41f..46dafd8482b9e 100644 --- a/Documentation/ABI/testing/sysfs-bus-nvdimm +++ b/Documentation/ABI/testing/sysfs-bus-nvdimm @@ -48,7 +48,8 @@ What: /sys/bus/nd/devices/nmemX/cxl/id Date: November 2022 KernelVersion: 6.2 Contact: Dave Jiang -Description: (RO) Show the id (serial) of the device. This is CXL specific. +Description: (RO) Show the id (serial) of the device, formatted as an + unsigned 64-bit decimal value. This is CXL specific. What: /sys/bus/nd/devices/nmemX/cxl/provider Date: November 2022 diff --git a/Documentation/ABI/testing/sysfs-fs-erofs b/Documentation/ABI/testing/sysfs-fs-erofs index b9243c7f28d71..0b8b4354e40b8 100644 --- a/Documentation/ABI/testing/sysfs-fs-erofs +++ b/Documentation/ABI/testing/sysfs-fs-erofs @@ -3,9 +3,9 @@ Date: November 2021 Contact: "Huang Jianan" Description: Shows all enabled kernel features. Supported features: - zero_padding, compr_cfgs, big_pcluster, chunked_file, - device_table, compr_head2, sb_chksum, ztailpacking, - dedupe, fragments, 48bit, metabox. + compr_cfgs, big_pcluster, chunked_file, device_table, + compr_head2, sb_chksum, ztailpacking, dedupe, fragments, + xattr_prefixes, 48bit, metabox. What: /sys/fs/erofs//sync_decompress Date: November 2021 diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon index b6b71db36ca72..ee111f632ed85 100644 --- a/Documentation/ABI/testing/sysfs-kernel-mm-damon +++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon @@ -105,7 +105,7 @@ Description: Writing a value to this file sets the update interval of the DAMON context in microseconds as the value. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/access_bp +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/access_bp Date: Feb 2025 Contact: SeongJae Park Description: Writing a value to this file sets the monitoring intervals @@ -113,7 +113,7 @@ Description: Writing a value to this file sets the monitoring intervals the given time interval (aggrs in same directory), in bp (1/10,000). Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/aggrs +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/aggrs Date: Feb 2025 Contact: SeongJae Park Description: Writing a value to this file sets the time interval to achieve @@ -121,14 +121,14 @@ Description: Writing a value to this file sets the time interval to achieve access events ratio (access_bp in same directory) within. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/min_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/min_sample_us Date: Feb 2025 Contact: SeongJae Park Description: Writing a value to this file sets the minimum value of auto-tuned sampling interval in microseconds. Reading this file returns the value. -What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intrvals_goal/max_sample_us +What: /sys/kernel/mm/damon/admin/kdamonds//contexts//monitoring_attrs/intervals/intervals_goal/max_sample_us Date: Feb 2025 Contact: SeongJae Park Description: Writing a value to this file sets the maximum value of diff --git a/Documentation/admin-guide/blockdev/zoned_loop.rst b/Documentation/admin-guide/blockdev/zoned_loop.rst index 64dcfde7450af..ae0eff6169907 100644 --- a/Documentation/admin-guide/blockdev/zoned_loop.rst +++ b/Documentation/admin-guide/blockdev/zoned_loop.rst @@ -30,11 +30,10 @@ indicates the position of the write pointer of the zone. When resetting a sequential zone, its backing file size is truncated to zero. Conversely, for a zone finish operation, the backing file is truncated to the -zone size. With this, the maximum capacity of a zloop zoned block device created -can be larger configured to be larger than the storage space available on the -backing file system. Of course, for such configuration, writing more data than -the storage space available on the backing file system will result in write -errors. +zone capacity. With this, a zloop zoned block device can be configured with a +larger capacity than the storage space available on the backing file system. Of +course, for such configuration, writing more data than the storage space +available on the backing file system will result in write errors. The zoned loop block device driver implements a complete zone transition state machine. That is, zones can be empty, implicitly opened, explicitly opened, diff --git a/Documentation/admin-guide/cgroup-v1/memory.rst b/Documentation/admin-guide/cgroup-v1/memory.rst index d6b1db8cc7ebb..4312f6785652a 100644 --- a/Documentation/admin-guide/cgroup-v1/memory.rst +++ b/Documentation/admin-guide/cgroup-v1/memory.rst @@ -47,7 +47,6 @@ Features: - pages are linked to per-memcg LRU exclusively, and there is no global LRU. - optionally, memory+swap usage can be accounted and limited. - hierarchical accounting - - soft limit - moving (recharging) account at moving a task is selectable. - usage threshold notifier - memory pressure notifier @@ -76,10 +75,9 @@ Brief summary of control files. memory.memsw.failcnt show the number of memory+Swap hits limits memory.max_usage_in_bytes show max memory usage recorded memory.memsw.max_usage_in_bytes show max memory+Swap usage recorded - memory.soft_limit_in_bytes set/show soft limit of memory usage - This knob is not available on CONFIG_PREEMPT_RT systems. - This knob is deprecated and shouldn't be - used. + memory.soft_limit_in_bytes This knob is deprecated and has no effect. + Writes are ignored and reads always + return the maximum value. memory.stat show various statistics memory.use_hierarchy set/show hierarchical account enabled This knob is deprecated and shouldn't be @@ -341,9 +339,6 @@ memory.kmem.usage_in_bytes, or in a separate counter when it makes sense. The main "kmem" counter is fed into the main counter, so kmem charges will also be visible from the user counter. -Currently no soft limit is implemented for kernel memory. It is future work -to trigger slab reclaim when those limits are reached. - 2.7.1 Current Kernel Memory resources accounted ----------------------------------------------- @@ -711,42 +706,10 @@ For compatibility reasons writing 1 to memory.use_hierarchy will always pass:: THIS IS DEPRECATED! -Soft limits allow for greater sharing of memory. The idea behind soft limits -is to allow control groups to use as much of the memory as needed, provided - -a. There is no memory contention -b. They do not exceed their hard limit - -When the system detects memory contention or low memory, control groups -are pushed back to their soft limits. If the soft limit of each control -group is very high, they are pushed back as much as possible to make -sure that one control group does not starve the others of memory. - -Please note that soft limits is a best-effort feature; it comes with -no guarantees, but it does its best to make sure that when memory is -heavily contended for, memory is allocated based on the soft limit -hints/setup. Currently soft limit based reclaim is set up such that -it gets invoked from balance_pgdat (kswapd). - -7.1 Interface -------------- - -Soft limits can be setup by using the following commands (in this example we -assume a soft limit of 256 MiB):: - - # echo 256M > memory.soft_limit_in_bytes - -If we want to change this to 1G, we can at any time use:: +Writing to memory.soft_limit_in_bytes has no effect and reading it will +always return the maximum value. - # echo 1G > memory.soft_limit_in_bytes - -.. note:: - Soft limits take effect over a long period of time, since they involve - reclaiming memory for balancing between memory cgroups - -.. note:: - It is recommended to set the soft limit always below the hard limit, - otherwise the hard limit will take precedence. +Use memory.low and memory.min in cgroup v2 instead. .. _cgroup-v1-memory-move-charges: diff --git a/Documentation/admin-guide/device-mapper/dm-ima.rst b/Documentation/admin-guide/device-mapper/dm-ima.rst index a4aa50a828e00..2a3b50ffbee4e 100644 --- a/Documentation/admin-guide/device-mapper/dm-ima.rst +++ b/Documentation/admin-guide/device-mapper/dm-ima.rst @@ -424,7 +424,8 @@ section above) has the following data format for 'integrity' target. target_attributes := "," "," "," "," "," [ ","] [ ","] "," - "," "," "," "," + "," "," "," "," + "," "," "," ";" target_name := "target_name=integrity" @@ -438,6 +439,7 @@ section above) has the following data format for 'integrity' target. block_size := "block_size=" recalculate := "recalculate=" allow_discards := "allow_discards=" + allow_discards_keyed := "allow_discards_keyed=" fix_padding := "fix_padding=" fix_hmac := "fix_hmac=" legacy_recalculate := "legacy_recalculate=" @@ -455,7 +457,8 @@ section above) has the following data format for 'integrity' target. dm_version=4.45.0; name=integrity1,uuid=,major=253,minor=1,minor_count=1,num_targets=1; target_index=0,target_begin=0,target_len=7856,target_name=integrity,target_version=1.10.0, - dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n,fix_padding=n, + dev_name=253:0,start=0,tag_size=32,mode=J,recalculate=n,allow_discards=n, + allow_discards_keyed=n,fix_padding=n, fix_hmac=n,legacy_recalculate=n,journal_sectors=88,interleave_sectors=32768,buffer_sectors=128; diff --git a/Documentation/admin-guide/device-mapper/dm-integrity.rst b/Documentation/admin-guide/device-mapper/dm-integrity.rst index c2e18ecc065c9..9c21301423c9e 100644 --- a/Documentation/admin-guide/device-mapper/dm-integrity.rst +++ b/Documentation/admin-guide/device-mapper/dm-integrity.rst @@ -190,6 +190,19 @@ allow_discards Allow block discard requests (a.k.a. TRIM) for the integrity device. Discards are only allowed to devices using internal hash. + A discarded block is marked with a constant filler tag that anyone + with raw write access to the backing device can forge without the + key. Use allow_discards_keyed instead on new volumes. + +allow_discards_keyed + Like allow_discards, but marks a discarded block with a keyed + checksum of the sector number, HMAC_key(salt || sector), instead of + the constant filler tag, so it can't be forged without the + integrity key. + + Not compatible with volumes that already have discarded blocks + marked the old way; only use on a freshly formatted volume. + fix_padding Use a smaller padding of the tag area that is more space-efficient. If this option is not present, large padding is diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index ab4e03f91e744..362ff32252209 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7339,6 +7339,10 @@ See Documentation/admin-guide/mm/transhuge.rst for more details. + tlbi= [X86-64] + Format: {ipi} + ipi: switch to IPI-based TLB flushing + topology= [S390,EARLY] Format: {off | on} Specify if the kernel should make use of the cpu @@ -7928,6 +7932,11 @@ q = USB_QUIRK_FORCE_ONE_CONFIG (Device claims zero configurations, forcing to 1); + r = USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE (Device + fails during initialization when asked for + 9-bytes configuration descriptor request. + Ask for 255-bytes request instead to mirror + Windows' behavior); Example: quirks=0781:5580:bk,0a5c:5834:gij usbhid.mousepoll= diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst index e1771f2225d5f..8384b3b9d47bb 100644 --- a/Documentation/admin-guide/pm/amd-pstate.rst +++ b/Documentation/admin-guide/pm/amd-pstate.rst @@ -280,16 +280,22 @@ A list of all the supported EPP preferences that could be used for These profiles represent different hints that are provided to the low-level firmware about the user's desired energy vs efficiency tradeoff. ``default`` represents the epp value is set by platform -firmware. This attribute is read-only. +firmware. ``custom`` designates that integer values 0-255 may be written +as well. This attribute is read-only. ``energy_performance_preference`` The current energy performance preference can be read from this attribute. and user can change current preference according to energy or performance needs -Please get all support profiles list from -``energy_performance_available_preferences`` attribute, all the profiles are -integer values defined between 0 to 255 when EPP feature is enabled by platform -firmware, if EPP feature is disabled, driver will ignore the written value +Coarse named profiles are available in the attribute +``energy_performance_available_preferences``. +Users can also write individual integer values between 0 to 255. +When dynamic EPP is enabled, writes to energy_performance_preference are blocked +even when EPP feature is enabled by platform firmware. Lower epp values shift the bias +towards improved performance while a higher epp value shifts the bias towards +power-savings. The exact impact can change from one platform to the other. +If a valid integer was last written, then a number will be returned on future reads. +If a valid string was last written then a string will be returned on future reads. This attribute is read-write. ``boost`` @@ -311,6 +317,24 @@ boost or `1` to enable it, for the respective CPU using the sysfs path Other performance and frequency values can be read back from ``/sys/devices/system/cpu/cpuX/acpi_cppc/``, see :ref:`cppc_sysfs`. +Dynamic energy performance profile +================================== +The amd-pstate driver supports dynamically selecting the energy performance +profile based on whether the machine is running on AC or DC power. + +Whether this behavior is enabled by default depends on the kernel +config option `CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`. This behavior can also be overridden +at runtime by the sysfs file ``/sys/devices/system/cpu/cpufreq/policyX/dynamic_epp``. + +When set to enabled, the driver will select a different energy performance +profile when the machine is running on battery or AC power. The driver will +also register with the platform profile handler to receive notifications of +user desired power state and react to those. +When set to disabled, the driver will not change the energy performance profile +based on the power source and will not react to user desired power state. + +Attempting to manually write to the ``energy_performance_preference`` sysfs +file will fail when ``dynamic_epp`` is enabled. ``amd-pstate`` vs ``acpi-cpufreq`` ====================================== diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml index 08fe6a707a371..70aa5193800e5 100644 --- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml @@ -27,6 +27,16 @@ properties: maxItems: 1 clocks: + minItems: 1 + maxItems: 2 + + clock-names: + minItems: 1 + items: + - const: core + - const: iface + + power-domains: maxItems: 1 required: @@ -44,6 +54,10 @@ examples: compatible = "qcom,sm8550-inline-crypto-engine", "qcom,inline-crypto-engine"; reg = <0x01d88000 0x8000>; - clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>; + clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>, + <&gcc GCC_UFS_PHY_AHB_CLK>; + clock-names = "core", + "iface"; + power-domains = <&gcc UFS_PHY_GDSC>; }; ... diff --git a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml index 3c071e6fbea61..58b1f9e85a9d1 100644 --- a/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml +++ b/Documentation/devicetree/bindings/media/i2c/st,vd55g1.yaml @@ -25,7 +25,12 @@ allOf: properties: compatible: - const: st,vd55g1 + enum: + - st,vd55g1 + - st,vd55g4 + - st,vd65g4 + description: + VD55G1 and VD55G4 are monochrome variants, while VD65G4 is a color one. reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml b/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml index f43b91984f015..d5e631f339a08 100644 --- a/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml +++ b/Documentation/devicetree/bindings/media/nxp,imx8-isi.yaml @@ -66,7 +66,6 @@ required: - interrupts - clocks - clock-names - - fsl,blk-ctrl - ports allOf: @@ -109,6 +108,16 @@ allOf: - port@0 - port@1 + - if: + properties: + compatible: + not: + contains: + const: fsl,imx8ulp-isi + then: + required: + - fsl,blk-ctrl + additionalProperties: false examples: diff --git a/Documentation/filesystems/fscrypt.rst b/Documentation/filesystems/fscrypt.rst index 696a5844bfa3b..480252ec8c781 100644 --- a/Documentation/filesystems/fscrypt.rst +++ b/Documentation/filesystems/fscrypt.rst @@ -1245,6 +1245,10 @@ astute users may notice some differences in behavior: - DAX (Direct Access) is not supported on encrypted files. +- Encrypted files cannot be used directly as swap files. To swap to + an encrypted file, set up a loopback device on top of it. + Alternatively, encrypted swap can use a dm-crypt device. + - The maximum length of an encrypted symlink is 2 bytes shorter than the maximum length of an unencrypted symlink. For example, on an EXT4 filesystem with a 4K block size, unencrypted symlinks can be up diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst index d55751cad67cf..8e0d31230b29b 100644 --- a/Documentation/gpu/drm-mm.rst +++ b/Documentation/gpu/drm-mm.rst @@ -509,8 +509,14 @@ DRM GPUVM Function References DRM Buddy Allocator =================== -DRM Buddy Function References ------------------------------ +Buddy Allocator Function References (GPU buddy) +----------------------------------------------- + +.. kernel-doc:: drivers/gpu/buddy.c + :export: + +DRM Buddy Specific Logging Function References +---------------------------------------------- .. kernel-doc:: drivers/gpu/drm/drm_buddy.c :export: diff --git a/Documentation/hwmon/cros_ec_hwmon.rst b/Documentation/hwmon/cros_ec_hwmon.rst index 6db812708325f..47a85cd40937e 100644 --- a/Documentation/hwmon/cros_ec_hwmon.rst +++ b/Documentation/hwmon/cros_ec_hwmon.rst @@ -23,9 +23,23 @@ ChromeOS embedded controller used in Chromebooks and other devices. The channel labels exposed via hwmon are retrieved from the EC itself. -Fan and temperature readings are supported. PWM fan control is also supported if -the EC also supports setting fan PWM values and fan mode. Note that EC will -switch fan control mode back to auto when suspended. This driver will restore -the fan state to what they were before suspended when resumed. -If a fan is controllable, this driver will register that fan as a cooling device -in the thermal framework as well. +Supported features +------------------ + +Fan readings + Always supported. + +Temperature readings + Always supported. + +Temperature thresholds + If supported by the EC. + +PWM fan control + If the EC also supports setting fan PWM values and fan mode. + + Note that EC will switch fan control mode back to auto when suspended. + This driver will restore the fan state to what they were before suspended when resumed. + + If a fan is controllable, this driver will register that fan as a cooling device + in the thermal framework as well. diff --git a/Documentation/hwmon/emc1403.rst b/Documentation/hwmon/emc1403.rst index 57f833b1a800e..ebf2435a76a62 100644 --- a/Documentation/hwmon/emc1403.rst +++ b/Documentation/hwmon/emc1403.rst @@ -71,10 +71,10 @@ and EMC14x8 support eight sensors (one internal, seven external). The chips implement three limits for each sensor: low (tempX_min), high (tempX_max) and critical (tempX_crit.) The chips also implement an -hysteresis mechanism which applies to all limits. The relative difference -is stored in a single register on the chip, which means that the relative -difference between the limit and its hysteresis is always the same for -all three limits. +hysteresis mechanism which applies to high and critical limits. The relative +difference is stored in a single register on the chip, which means that the +relative difference between the limit and its hysteresis is always the same +for high and critical limits. This implementation detail implies the following: diff --git a/Documentation/hwmon/gpd-fan.rst b/Documentation/hwmon/gpd-fan.rst index 0b56b70e6264d..3aecda2d82d82 100644 --- a/Documentation/hwmon/gpd-fan.rst +++ b/Documentation/hwmon/gpd-fan.rst @@ -64,7 +64,7 @@ pwm1_enable at full speed. Write "1" to set to manual, write "2" to let the EC control decide fan speed. Read this attribute to see current status. - NB：In consideration of the safety of the device, when setting to manual mode, + NB: In consideration of the safety of the device, when setting to manual mode, the pwm speed will be set to the maximum value (255) by default. You can set a different value by writing pwm1 later. diff --git a/Documentation/hwmon/hwmon-kernel-api.rst b/Documentation/hwmon/hwmon-kernel-api.rst index 1d7f1397a8274..c3eb433a78f61 100644 --- a/Documentation/hwmon/hwmon-kernel-api.rst +++ b/Documentation/hwmon/hwmon-kernel-api.rst @@ -42,6 +42,9 @@ register/unregister functions:: char *devm_hwmon_sanitize_name(struct device *dev, const char *name); + int hwmon_notify_event(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel); + void hwmon_lock(struct device *dev); void hwmon_unlock(struct device *dev); @@ -85,9 +88,22 @@ removal. When using ``[devm_]hwmon_device_register_with_info()`` to register the hardware monitoring device, accesses using the associated access functions are serialised by the hardware monitoring core. If a driver needs locking -for other functions such as interrupt handlers or for attributes which are -fully implemented in the driver, hwmon_lock() and hwmon_unlock() can be used -to ensure that calls to those functions are serialized. +for other functions such as interrupt handlers, attributes which are fully +implemented in the driver, or debugfs functions, hwmon_lock() and hwmon_unlock() +can be used to ensure that calls to those functions are serialized. Those +functions also support guard() and scoped_guard() variants. + +Drivers can call hwmon_notify_event() to notify userspace and the thermal +subsystem when a hardware monitoring event (such as an alarm or a fault +condition) occurs or clears. The parameters are the hwmon device, the sensor +type, the attribute identifier associated with the event (such as +hwmon_temp_max_alarm or hwmon_fan_fault), and the sensor channel number. +hwmon_notify_event() generates a sysfs event (calling sysfs_notify()) and a +udev event with the attribute name passed in the NAME environment property +(e.g., "NAME=temp1_max_alarm"). If the event is for a temperature sensor and +the sensor is attached to a thermal zone, it also notifies the thermal +subsystem to update the thermal zone. hwmon_notify_event() returns 0 on +success or a negative error code on failure. Using devm_hwmon_device_register_with_info() -------------------------------------------- diff --git a/Documentation/hwmon/ina2xx.rst b/Documentation/hwmon/ina2xx.rst index a3860aae444c0..d64e7af46a124 100644 --- a/Documentation/hwmon/ina2xx.rst +++ b/Documentation/hwmon/ina2xx.rst @@ -74,6 +74,16 @@ Supported chips: https://us1.silergy.com/ + * Texas Instruments INA234 + + Prefix: 'ina234' + + Addresses: I2C 0x40 - 0x43 + + Datasheet: Publicly available at the Texas Instruments website + + https://www.ti.com/ + Author: Lothar Felten Description @@ -89,7 +99,7 @@ interface. The INA220 monitors both shunt drop and supply voltage. The INA226 is a current shunt and power monitor with an I2C interface. The INA226 monitors both a shunt voltage drop and bus supply voltage. -INA230 and INA231 are high or low side current shunt and power monitors +INA230, INA231, and INA234 are high or low side current shunt and power monitors with an I2C interface. The chips monitor both a shunt voltage drop and bus supply voltage. @@ -124,8 +134,17 @@ power1_input Power(uW) measurement channel shunt_resistor Shunt resistance(uOhm) channel (not for ina260) ======================= =============================================== -Additional sysfs entries for ina226, ina230, ina231, ina260, and sy24655 ------------------------------------------------------------------------- +Additional sysfs entries +------------------------ + +Additional entries are available for the following chips: + + * ina226 + * ina230 + * ina231 + * ina234 + * ina260 + * sy24655 ======================= ==================================================== curr1_lcrit Critical low current diff --git a/Documentation/leds/leds-st1202.rst b/Documentation/leds/leds-st1202.rst index 1a09fbfcedcff..a2353549469ee 100644 --- a/Documentation/leds/leds-st1202.rst +++ b/Documentation/leds/leds-st1202.rst @@ -17,7 +17,7 @@ To be compatible with the hardware pattern format, maximum 8 tuples of brightness (PWM) and duration must be written to hw_pattern. - Min pattern duration: 22 ms -- Max pattern duration: 5660 ms +- Max pattern duration: 5610 ms The format of the hardware pattern values should be: "brightness duration brightness duration ..." diff --git a/Documentation/netlink/specs/devlink.yaml b/Documentation/netlink/specs/devlink.yaml index 3db59c9658694..426d5aa7d9551 100644 --- a/Documentation/netlink/specs/devlink.yaml +++ b/Documentation/netlink/specs/devlink.yaml @@ -99,6 +99,8 @@ definitions: name: legacy - name: switchdev + - + name: switchdev-inactive - type: enum name: eswitch-inline-mode diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml index ae4db7e032770..64db9c0b446e4 100644 --- a/Documentation/netlink/specs/rt-link.yaml +++ b/Documentation/netlink/specs/rt-link.yaml @@ -881,6 +881,8 @@ attribute-sets: - name: txqlen type: u32 + checks: + max: 32767 - name: map type: binary @@ -1564,31 +1566,31 @@ attribute-sets: type: u32 - name: mode - type: flag + type: u8 - name: guard - type: flag + type: u8 - name: protect - type: flag + type: u8 - name: fast-leave - type: flag + type: u8 - name: learning - type: flag + type: u8 - name: unicast-flood - type: flag + type: u8 - name: proxyarp - type: flag + type: u8 - name: learning-sync - type: flag + type: u8 - name: proxyarp-wifi - type: flag + type: u8 - name: root-id type: binary @@ -1635,34 +1637,34 @@ attribute-sets: type: pad - name: mcast-flood - type: flag + type: u8 - name: mcast-to-ucast - type: flag + type: u8 - name: vlan-tunnel - type: flag + type: u8 - name: bcast-flood - type: flag + type: u8 - name: group-fwd-mask type: u16 - name: neigh-suppress - type: flag + type: u8 - name: isolated - type: flag + type: u8 - name: backup-port type: u32 - name: mrp-ring-open - type: flag + type: u8 - name: mrp-in-open - type: flag + type: u8 - name: mcast-eht-hosts-limit type: u32 @@ -1671,10 +1673,10 @@ attribute-sets: type: u32 - name: locked - type: flag + type: u8 - name: mab - type: flag + type: u8 - name: mcast-n-groups type: u32 @@ -1683,7 +1685,7 @@ attribute-sets: type: u32 - name: neigh-vlan-suppress - type: flag + type: u8 - name: backup-nhid type: u32 diff --git a/Documentation/networking/devlink/devlink-eswitch-attr.rst b/Documentation/networking/devlink/devlink-eswitch-attr.rst index 08bb39ab15286..eafe09abc40c2 100644 --- a/Documentation/networking/devlink/devlink-eswitch-attr.rst +++ b/Documentation/networking/devlink/devlink-eswitch-attr.rst @@ -39,6 +39,10 @@ The following is a list of E-Switch attributes. rules. * ``switchdev`` allows for more advanced offloading capabilities of the E-Switch to hardware. + * ``switchdev_inactive`` switchdev mode but starts inactive, doesn't allow traffic + until explicitly activated. This mode is useful for orchestrators that + want to prepare the device in switchdev mode but only activate it when + all configurations are done. * - ``inline-mode`` - enum - Some HWs need the VF driver to put part of the packet @@ -74,3 +78,12 @@ Example Usage # enable encap-mode with legacy mode $ devlink dev eswitch set pci/0000:08:00.0 mode legacy inline-mode none encap-mode basic + + # start switchdev mode in inactive state + $ devlink dev eswitch set pci/0000:08:00.0 mode switchdev_inactive + + # setup switchdev configurations, representors, FDB entries, etc.. + ... + + # activate switchdev mode to allow traffic + $ devlink dev eswitch set pci/0000:08:00.0 mode switchdev diff --git a/Documentation/networking/net_cachelines/inet_connection_sock.rst b/Documentation/networking/net_cachelines/inet_connection_sock.rst index 8fae85ebb7730..cc2000f55c298 100644 --- a/Documentation/networking/net_cachelines/inet_connection_sock.rst +++ b/Documentation/networking/net_cachelines/inet_connection_sock.rst @@ -12,8 +12,8 @@ struct inet_sock icsk_inet read_mostly r struct request_sock_queue icsk_accept_queue struct inet_bind_bucket icsk_bind_hash read_mostly tcp_set_state struct inet_bind2_bucket icsk_bind2_hash read_mostly tcp_set_state,inet_put_port -struct timer_list icsk_retransmit_timer read_write inet_csk_reset_xmit_timer,tcp_connect struct timer_list icsk_delack_timer read_mostly inet_csk_reset_xmit_timer,tcp_connect +struct timer_list icsk_keepalive_timer u32 icsk_rto read_write tcp_cwnd_validate,tcp_schedule_loss_probe,tcp_connect_init,tcp_connect,tcp_write_xmit,tcp_push_one u32 icsk_rto_min u32 icsk_rto_max read_mostly tcp_reset_xmit_timer diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst index a72461b1c9305..09f28db3bc8d2 100644 --- a/Documentation/scheduler/sched-ext.rst +++ b/Documentation/scheduler/sched-ext.rst @@ -181,7 +181,7 @@ optional. The following modified excerpt is from void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei) { - exit_type = ei->type; + exit_type = ei->kind; } SEC(".struct_ops") diff --git a/Documentation/trace/hisi-ptt.rst b/Documentation/trace/hisi-ptt.rst index 6eef28ebb0c7b..f6a2655f99e57 100644 --- a/Documentation/trace/hisi-ptt.rst +++ b/Documentation/trace/hisi-ptt.rst @@ -285,20 +285,20 @@ according to the format described previously (take 8DW as an example): [...perf headers and other information] . ... HISI PTT data: size 4194304 bytes . 00000000: 00 00 00 00 Prefix - . 00000004: 01 00 00 60 Header DW0 - . 00000008: 0f 1e 00 01 Header DW1 - . 0000000c: 04 00 00 00 Header DW2 - . 00000010: 40 00 81 02 Header DW3 - . 00000014: 33 c0 04 00 Time + . 00000004: 60 00 00 01 Header DW0 + . 00000008: 01 00 1e 0f Header DW1 + . 0000000c: 00 00 00 04 Header DW2 + . 00000010: 02 81 00 40 Header DW3 + . 00000014: 00 04 c0 33 Time . 00000020: 00 00 00 00 Prefix - . 00000024: 01 00 00 60 Header DW0 - . 00000028: 0f 1e 00 01 Header DW1 - . 0000002c: 04 00 00 00 Header DW2 - . 00000030: 40 00 81 02 Header DW3 - . 00000034: 02 00 00 00 Time + . 00000024: 60 00 00 01 Header DW0 + . 00000028: 01 00 1e 0f Header DW1 + . 0000002c: 00 00 00 04 Header DW2 + . 00000030: 02 81 00 40 Header DW3 + . 00000034: 00 00 00 02 Time . 00000040: 00 00 00 00 Prefix - . 00000044: 01 00 00 60 Header DW0 - . 00000048: 0f 1e 00 01 Header DW1 - . 0000004c: 04 00 00 00 Header DW2 - . 00000050: 40 00 81 02 Header DW3 + . 00000044: 60 00 00 01 Header DW0 + . 00000048: 01 00 1e 0f Header DW1 + . 0000004c: 00 00 00 04 Header DW2 + . 00000050: 02 81 00 40 Header DW3 [...] diff --git a/Documentation/usb/gadget-testing.rst b/Documentation/usb/gadget-testing.rst index 5f90af1fb5732..edc13740ff6b7 100644 --- a/Documentation/usb/gadget-testing.rst +++ b/Documentation/usb/gadget-testing.rst @@ -712,9 +712,6 @@ The uac1 function provides these attributes in its function directory: =============== ==================================== audio_buf_size audio buffer size - fn_cap capture pcm device file name - fn_cntl control device file name - fn_play playback pcm device file name req_buf_size ISO OUT endpoint request buffer size req_count ISO OUT endpoint request count =============== ==================================== diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst index 1e7265155715b..fb0fe51ad0f5e 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-colorimetry.rst @@ -79,15 +79,25 @@ Colorimetry Control IDs - ``white_point_x`` - Specifies the normalized x chromaticity coordinate of the white point of the mastering display in increments of 0.00002. + When both ``white_point_x`` and ``white_point_y`` are zero, + the white point chromaticity is unknown. If either coordinate is + non-zero, both coordinates shall be within their valid ranges. * - __u16 - ``white_point_y`` - Specifies the normalized y chromaticity coordinate of the white point of the mastering display in increments of 0.00002. + When both ``white_point_x`` and ``white_point_y`` are zero, + the white point chromaticity is unknown. If either coordinate is + non-zero, both coordinates shall be within their valid ranges. * - __u32 - ``max_luminance`` - Specifies the nominal maximum display luminance of the mastering display in units of 0.0001 cd/m\ :sup:`2`. + A value of zero indicates that the nominal maximum display + luminance is unknown. * - __u32 - ``min_luminance`` - - specifies the nominal minimum display luminance of the mastering + - Specifies the nominal minimum display luminance of the mastering display in units of 0.0001 cd/m\ :sup:`2`. + A value of zero indicates that the nominal minimum display + luminance is unknown. diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 9f835f68b4fb9..10d8a366b2072 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -8259,6 +8259,12 @@ When this capability is enabled all memory in memslots must be mapped as attempts to create a memslot with an invalid mmap will result in an -EINVAL return. +``guest_memfd``, even though it is an anonymous file, is not supported with MTE. +Attempting to create a memslot backed by ``guest_memfd`` when the MTE capability +is enabled, or attempting to enable the MTE capability after +``guest_memfd``-backed memslots have been created, will result in an -EINVAL +return. + When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to perform a bulk copy of tags to/from the guest. diff --git a/MAINTAINERS b/MAINTAINERS index 23cce9b75986d..3cb7bd5762592 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -8688,6 +8688,19 @@ T: git https://gitlab.freedesktop.org/drm/misc/kernel.git F: drivers/gpu/drm/ttm/ F: include/drm/ttm/ +GPU BUDDY ALLOCATOR +M: Matthew Auld +M: Arun Pravin +R: Christian Koenig +L: dri-devel@lists.freedesktop.org +S: Maintained +T: git https://gitlab.freedesktop.org/drm/misc/kernel.git +F: drivers/gpu/drm_buddy.c +F: drivers/gpu/buddy.c +F: drivers/gpu/tests/gpu_buddy_test.c +F: include/linux/gpu_buddy.h +F: include/drm/drm_buddy.h + DRM AUTOMATED TESTING M: Helen Koike M: Vignesh Raman @@ -9739,6 +9752,11 @@ F: drivers/base/firmware_loader/ F: rust/kernel/firmware.rs F: include/linux/firmware.h +FIXED-LAYOUT NVMEM LAYOUT DRIVER +M: Mathieu Dubois-Briand +S: Maintained +F: drivers/nvmem/layouts/fixed-layout.c + FLEXTIMER FTM-QUADDEC DRIVER M: Patrick Havelange L: linux-iio@vger.kernel.org @@ -10547,6 +10565,7 @@ S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers/vdso F: include/asm-generic/vdso/vsyscall.h F: include/vdso/ +F: kernel/time/namespace_vdso.c F: kernel/time/vsyscall.c F: lib/vdso/ F: tools/testing/selftests/vDSO/ @@ -12402,6 +12421,7 @@ R: Eric Snowberg L: linux-integrity@vger.kernel.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity.git +F: include/linux/secure_boot.h F: security/integrity/ F: security/integrity/ima/ @@ -20516,6 +20536,7 @@ F: include/trace/events/timer* F: kernel/time/itimer.c F: kernel/time/posix-* F: kernel/time/namespace.c +F: kernel/time/namespace_vdso.c POWER MANAGEMENT CORE M: "Rafael J. Wysocki" @@ -25037,6 +25058,7 @@ R: Jan Dabros L: linux-i2c@vger.kernel.org S: Supported F: drivers/i2c/busses/i2c-designware-* +F: include/linux/designware_i2c.h SYNOPSYS DESIGNWARE I2C DRIVER - AMDISP M: Nirujogi Pratap diff --git a/Makefile b/Makefile index aa1a53c513b41..f1562b84fb2e3 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 VERSION = 6 PATCHLEVEL = 18 -SUBLEVEL = 40 +SUBLEVEL = 53 EXTRAVERSION = NAME = Baby Opossum Posse @@ -470,6 +470,10 @@ KBUILD_USERLDFLAGS := $(USERLDFLAGS) # These flags apply to all Rust code in the tree, including the kernel and # host programs. +# +# `-Aclippy::unwrap_or_default`: the lint is buggy [1] and ignores our +# MSRV. It can trigger depending on the optimization level. +# [1] https://github.com/rust-lang/rust-clippy/issues/17379 export rust_common_flags := --edition=2021 \ -Zbinary_dep_depinfo=y \ -Astable_features \ @@ -497,6 +501,7 @@ export rust_common_flags := --edition=2021 \ -Aclippy::uninlined_format_args \ -Wclippy::unnecessary_safety_comment \ -Wclippy::unnecessary_safety_doc \ + -Aclippy::unwrap_or_default \ -Wrustdoc::missing_crate_level_docs \ -Wrustdoc::unescaped_backticks @@ -1247,7 +1252,7 @@ PHONY += vmlinux_o vmlinux_o: vmlinux.a $(KBUILD_VMLINUX_LIBS) $(Q)$(MAKE) -f $(srctree)/scripts/Makefile.vmlinux_o -vmlinux.o modules.builtin.modinfo modules.builtin: vmlinux_o +vmlinux.o: vmlinux_o @: PHONY += vmlinux diff --git a/arch/Kconfig b/arch/Kconfig index 61130b88964b9..ee9fd0150f487 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -1841,4 +1841,7 @@ config ARCH_WANTS_PRE_LINK_VMLINUX config ARCH_HAS_CPU_ATTACK_VECTORS bool +config HAVE_ARCH_GET_SECUREBOOT + def_bool EFI + endmenu diff --git a/arch/alpha/include/uapi/asm/fpu.h b/arch/alpha/include/uapi/asm/fpu.h index cea9eafa056fc..d28dc36786e27 100644 --- a/arch/alpha/include/uapi/asm/fpu.h +++ b/arch/alpha/include/uapi/asm/fpu.h @@ -101,7 +101,12 @@ ieee_swcr_to_fpcr(unsigned long sw) | IEEE_TRAP_ENABLE_OVF)) << 48; fp |= (~sw & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE)) << 57; fp |= (sw & IEEE_MAP_UMZ ? FPCR_UNDZ | FPCR_UNFD : 0); - fp |= (~sw & IEEE_TRAP_ENABLE_DNO) << 41; + /* + * Disable denormal operand traps only when denormal inputs are to be + * flushed to zero. Otherwise they must keep trapping, so that /S + * instructions reach the kernel emulation handler. + */ + fp |= (sw & IEEE_MAP_DMZ ? FPCR_DNOD : 0); return fp; } @@ -116,7 +121,6 @@ ieee_fpcr_to_swcr(unsigned long fp) | IEEE_TRAP_ENABLE_OVF); sw |= (~fp >> 57) & (IEEE_TRAP_ENABLE_UNF | IEEE_TRAP_ENABLE_INE); sw |= (fp >> 47) & IEEE_MAP_UMZ; - sw |= (~fp >> 41) & IEEE_TRAP_ENABLE_DNO; return sw; } diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c index 21833d1c867d7..2e9870ee21c62 100644 --- a/arch/alpha/kernel/pci-sysfs.c +++ b/arch/alpha/kernel/pci-sysfs.c @@ -364,17 +364,17 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) switch(size) { case 1: - outb(port, val); + outb(val, port); return 1; case 2: if (port & 1) return -EINVAL; - outw(port, val); + outw(val, port); return 2; case 4: if (port & 3) return -EINVAL; - outl(port, val); + outl(val, port); return 4; } return -EINVAL; diff --git a/arch/alpha/kernel/sys_marvel.c b/arch/alpha/kernel/sys_marvel.c index 1f99b03effc2e..a37707e05e34e 100644 --- a/arch/alpha/kernel/sys_marvel.c +++ b/arch/alpha/kernel/sys_marvel.c @@ -263,6 +263,18 @@ init_io7_irqs(struct io7 *io7, */ printk(" Interrupts reported to CPU at PE %u\n", boot_cpuid); + /* Set up the lsi irqs. */ + for (i = 0; i < 128; ++i) { + irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + + /* Set up the msi irqs. */ + for (i = 128; i < (128 + 512); ++i) { + irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); + irq_set_status_flags(base + i, IRQ_LEVEL); + } + raw_spin_lock(&io7->irq_lock); /* set up the error irqs */ @@ -272,12 +284,6 @@ init_io7_irqs(struct io7 *io7, io7_redirect_irq(io7, &io7->csrs->STV_CTL.csr, boot_cpuid); io7_redirect_irq(io7, &io7->csrs->HEI_CTL.csr, boot_cpuid); - /* Set up the lsi irqs. */ - for (i = 0; i < 128; ++i) { - irq_set_chip_and_handler(base + i, lsi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - /* Disable the implemented irqs in hardware. */ for (i = 0; i < 0x60; ++i) init_one_io7_lsi(io7, i, boot_cpuid); @@ -285,13 +291,6 @@ init_io7_irqs(struct io7 *io7, init_one_io7_lsi(io7, 0x74, boot_cpuid); init_one_io7_lsi(io7, 0x75, boot_cpuid); - - /* Set up the msi irqs. */ - for (i = 128; i < (128 + 512); ++i) { - irq_set_chip_and_handler(base + i, msi_ops, handle_level_irq); - irq_set_status_flags(i, IRQ_LEVEL); - } - for (i = 0; i < 16; ++i) init_one_io7_msi(io7, i, boot_cpuid); diff --git a/arch/alpha/kernel/traps.c b/arch/alpha/kernel/traps.c index 7004397937cfd..7cd20e5f9ee03 100644 --- a/arch/alpha/kernel/traps.c +++ b/arch/alpha/kernel/traps.c @@ -166,12 +166,12 @@ static long dummy_emul(void) { return 0; } long (*alpha_fp_emul_imprecise)(struct pt_regs *regs, unsigned long writemask) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul_imprecise); -long (*alpha_fp_emul) (unsigned long pc) +long (*alpha_fp_emul) (unsigned long pc, unsigned long summary) = (void *)dummy_emul; EXPORT_SYMBOL_GPL(alpha_fp_emul); #else long alpha_fp_emul_imprecise(struct pt_regs *regs, unsigned long writemask); -long alpha_fp_emul (unsigned long pc); +long alpha_fp_emul (unsigned long pc, unsigned long summary); #endif asmlinkage void @@ -185,7 +185,7 @@ do_entArith(unsigned long summary, unsigned long write_mask, emulate the instruction. If the processor supports precise exceptions, we don't have to search. */ if (!amask(AMASK_PRECISE_TRAP)) - si_code = alpha_fp_emul(regs->pc - 4); + si_code = alpha_fp_emul(regs->pc - 4, summary); else si_code = alpha_fp_emul_imprecise(regs, write_mask); if (si_code == 0) diff --git a/arch/alpha/math-emu/math.c b/arch/alpha/math-emu/math.c index 68d420bfd3c0d..e3f2df3729e3a 100644 --- a/arch/alpha/math-emu/math.c +++ b/arch/alpha/math-emu/math.c @@ -52,13 +52,13 @@ MODULE_DESCRIPTION("FP Software completion module"); MODULE_LICENSE("GPL v2"); extern long (*alpha_fp_emul_imprecise)(struct pt_regs *, unsigned long); -extern long (*alpha_fp_emul) (unsigned long pc); +extern long (*alpha_fp_emul) (unsigned long pc, unsigned long summary); static long (*save_emul_imprecise)(struct pt_regs *, unsigned long); -static long (*save_emul) (unsigned long pc); +static long (*save_emul) (unsigned long pc, unsigned long summary); long do_alpha_fp_emul_imprecise(struct pt_regs *, unsigned long); -long do_alpha_fp_emul(unsigned long); +long do_alpha_fp_emul(unsigned long, unsigned long); static int alpha_fp_emul_init_module(void) { @@ -86,7 +86,22 @@ module_exit(alpha_fp_emul_cleanup_module); /* - * Emulate the floating point instruction at address PC. Returns -1 if the + * Exception bits of the exception summary register (EXC_SUM). Bit 0 is the + * software completion bit; bits 1 through 5 report the exceptions the + * hardware attributed to the trapping instruction, and lie at the same + * positions as the corresponding IEEE_TRAP_ENABLE_* bits. + */ +#define EXC_SUM_INV (1UL << 1) +#define EXC_SUM_DZE (1UL << 2) +#define EXC_SUM_OVF (1UL << 3) +#define EXC_SUM_UNF (1UL << 4) +#define EXC_SUM_INE (1UL << 5) +#define EXC_SUM_MASK (EXC_SUM_INV | EXC_SUM_DZE | EXC_SUM_OVF \ + | EXC_SUM_UNF | EXC_SUM_INE) + +/* + * Emulate the floating point instruction at address PC. SUMMARY is the + * exception summary register the trap was delivered with. Returns -1 if the * instruction to be emulated is illegal (such as with the opDEC trap), else * the SI_CODE for a SIGFPE signal, else 0 if everything's ok. * @@ -95,7 +110,7 @@ module_exit(alpha_fp_emul_cleanup_module); * stick the result of the operation into the appropriate register. */ long -alpha_fp_emul (unsigned long pc) +alpha_fp_emul (unsigned long pc, unsigned long summary) { FP_DECL_EX; FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); @@ -300,12 +315,56 @@ alpha_fp_emul (unsigned long pc) swcr |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); current_thread_info()->ieee_state |= (_fex << IEEE_STATUS_TO_EXCSUM_SHIFT); + } - /* Update hardware control register. */ - fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); - fpcr |= ieee_swcr_to_fpcr(swcr); - wrfpcr(fpcr); + /* + * EV6 records exception status bits in the FPCR before delivering the + * software completion trap, and swcr_update_status() above merged them + * into SWCR. Some can be wrong for the instruction we just emulated: + * a CVTTS of a value exactly representable as a subnormal sets FPCR_UNF + * even though the result is exact. Clear the exceptions the trap + * reported but that soft-fp did not raise. + */ + if (implver() == IMPLVER_EV6) { + unsigned long spurious = summary & EXC_SUM_MASK; + + if (spurious & (EXC_SUM_UNF | EXC_SUM_OVF)) { + /* + * EXC_SUM reports only the underflow or overflow, + * but the hardware sets INE alongside it in the FPCR. + */ + spurious |= EXC_SUM_INE; + } else if (!spurious) { + /* + * No exception reported, so this was a denormal + * operand trap, for which INE and UNF can be + * fabricated as well. + */ + spurious = EXC_SUM_INE | EXC_SUM_UNF; + } + /* + * Never clear an exception software has confirmed. Every + * instruction that genuinely raises one traps for software + * completion and is recorded in ieee_state above, so a bit + * found there -- including one just set from _fex -- belongs + * to this or an earlier instruction and must survive. + */ + spurious &= ~(current_thread_info()->ieee_state + >> IEEE_STATUS_TO_EXCSUM_SHIFT); + + swcr &= ~(spurious << IEEE_STATUS_TO_EXCSUM_SHIFT); + } + + /* + * Update hardware control register. This has to happen even when + * soft-fp raised nothing, to clear any fabricated bits. + */ + fpcr &= (~FPCR_MASK | FPCR_DYN_MASK); + fpcr |= ieee_swcr_to_fpcr(swcr); + wrfpcr(fpcr); + + if (_fex) { /* Do we generate a signal? */ _fex = _fex & swcr & IEEE_TRAP_ENABLE_MASK; si_code = 0; @@ -387,9 +446,16 @@ alpha_fp_emul_imprecise (struct pt_regs *regs, unsigned long write_mask) break; } if (!write_mask) { - /* Re-execute insns in the trap-shadow. */ + /* + * Re-execute insns in the trap-shadow. Pass no + * exception summary: it describes the trap, which + * was taken anywhere in the shadow, and so is not + * attribution for this instruction. Nothing is + * lost, since only EV6 -- which traps precisely and + * never comes this way -- needs it. + */ regs->pc = trigger_pc + 4; - si_code = alpha_fp_emul(trigger_pc); + si_code = alpha_fp_emul(trigger_pc, 0); goto egress; } trigger_pc -= 4; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e481254e36455..b4ef6a27e8cee 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -96,7 +96,7 @@ config ARM select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE if ARM_LPAE select HAVE_ARM_SMCCC if CPU_V7 - select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 + select HAVE_EBPF_JIT if !CPU_ENDIAN_BE32 && !CPU_32v3 select HAVE_CONTEXT_TRACKING_USER select HAVE_C_RECORDMCOUNT select HAVE_BUILDTIME_MCOUNT_SORT diff --git a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi index 51a6464aab9a3..cabf619c2e217 100644 --- a/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi +++ b/arch/arm/boot/dts/allwinner/sun4i-a10.dtsi @@ -185,7 +185,7 @@ pmu { compatible = "arm,cortex-a8-pmu"; - interrupts = <3>; + interrupts = <66>; }; reserved-memory { diff --git a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi index d0d5f7e52a917..f08d0a27de0b4 100644 --- a/arch/arm/boot/dts/broadcom/bcm-ns.dtsi +++ b/arch/arm/boot/dts/broadcom/bcm-ns.dtsi @@ -138,7 +138,7 @@ /* PCIe Controller 2 */ <0x00014000 0 &gic GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>, - <0x00014000 1 &gic GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>, + <0x00014000 1 &gic GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 2 &gic GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 3 &gic GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>, <0x00014000 4 &gic GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>, diff --git a/arch/arm/boot/dts/marvell/armada-388-helios4.dts b/arch/arm/boot/dts/marvell/armada-388-helios4.dts index ec134e22bae3e..003b0708e9656 100644 --- a/arch/arm/boot/dts/marvell/armada-388-helios4.dts +++ b/arch/arm/boot/dts/marvell/armada-388-helios4.dts @@ -169,6 +169,7 @@ gpio-controller; #gpio-cells = <2>; reg = <0x20>; + vcc-supply = <®_3p3v>; pinctrl-names = "default"; pinctrl-0 = <&pca0_pins>; interrupt-parent = <&gpio0>; @@ -201,6 +202,10 @@ reg = <0x4c>; vcc-supply = <®_3p3v>; }; + + eeprom@53 { + vcc-supply = <®_3p3v>; + }; }; i2c@11100 { @@ -217,13 +222,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata0: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata1: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; @@ -231,13 +240,17 @@ status = "okay"; #address-cells = <1>; #size-cells = <0>; + ahci-supply = <®_3p3v>; + phy-supply = <®_3p3v>; sata2: sata-port@0 { reg = <0>; + target-supply = <®_5p0v_hdd>; }; sata3: sata-port@1 { reg = <1>; + target-supply = <®_5p0v_hdd>; }; }; diff --git a/arch/arm/boot/dts/nvidia/tegra30-asus-tf600t.dts b/arch/arm/boot/dts/nvidia/tegra30-asus-tf600t.dts index 5d9e23a438204..856b1bb8b6f37 100644 --- a/arch/arm/boot/dts/nvidia/tegra30-asus-tf600t.dts +++ b/arch/arm/boot/dts/nvidia/tegra30-asus-tf600t.dts @@ -1040,9 +1040,9 @@ vdd-supply = <&vdd_3v3_sys>; vddio-supply = <&vdd_1v8_vio>; - mount-matrix = "0", "-1", "0", - "-1", "0", "0", - "0", "0", "-1"; + mount-matrix = "0", "1", "0", + "1", "0", "0", + "0", "0", "1"; /* External I2C interface */ i2c-gate { diff --git a/arch/arm/boot/dts/nvidia/tegra30-lg-p880.dts b/arch/arm/boot/dts/nvidia/tegra30-lg-p880.dts index c6ef0a20c19f3..4f57e43af6428 100644 --- a/arch/arm/boot/dts/nvidia/tegra30-lg-p880.dts +++ b/arch/arm/boot/dts/nvidia/tegra30-lg-p880.dts @@ -486,4 +486,17 @@ nvidia,int-mic-en-gpios = <&gpio TEGRA_GPIO(I, 6) GPIO_ACTIVE_HIGH>; }; + + thermal-zones { + cpu-thermal { + trips { + cpu-alert { + /* throttle at 60C until temperature drops to 59.8C */ + temperature = <60000>; + hysteresis = <200>; + type = "passive"; + }; + }; + }; + }; }; diff --git a/arch/arm/boot/dts/overlays/Makefile b/arch/arm/boot/dts/overlays/Makefile index f195f08747958..3ca704516da0c 100644 --- a/arch/arm/boot/dts/overlays/Makefile +++ b/arch/arm/boot/dts/overlays/Makefile @@ -63,6 +63,8 @@ dtbo-$(CONFIG_ARCH_BCM2835) += \ draws-pi5.dtbo \ dwc-otg-deprecated.dtbo \ dwc2.dtbo \ + ed-backlight-pwm.dtbo \ + ed-backlight-pwm-pi5.dtbo \ ed-ipc.dtbo \ edt-ft5406.dtbo \ enc28j60.dtbo \ @@ -76,6 +78,8 @@ dtbo-$(CONFIG_ARCH_BCM2835) += \ ghost-amp.dtbo \ goodix.dtbo \ googlevoicehat-soundcard.dtbo \ + gpclk.dtbo \ + gpclk-pi5.dtbo \ gpio-charger.dtbo \ gpio-fan.dtbo \ gpio-hog.dtbo \ @@ -236,6 +240,7 @@ dtbo-$(CONFIG_ARCH_BCM2835) += \ proto-codec.dtbo \ pwm.dtbo \ pwm-2chan.dtbo \ + pwm-fan.dtbo \ pwm-gpio.dtbo \ pwm-gpio-fan.dtbo \ pwm-ir-tx.dtbo \ @@ -264,6 +269,7 @@ dtbo-$(CONFIG_ARCH_BCM2835) += \ rpi-sense-v2.dtbo \ rpi-tv.dtbo \ rra-digidac1-wm8741-audio.dtbo \ + rt5616.dtbo \ sainsmart18.dtbo \ sc16is750-i2c.dtbo \ sc16is752-i2c.dtbo \ diff --git a/arch/arm/boot/dts/overlays/README b/arch/arm/boot/dts/overlays/README index 3823fdf118cbb..52bc515e0a5ad 100644 --- a/arch/arm/boot/dts/overlays/README +++ b/arch/arm/boot/dts/overlays/README @@ -1278,6 +1278,20 @@ Params: dr_mode Dual role mode: "host", "peripheral" or "otg" mode +Name: ed-backlight-pwm +Info: Configures the EDATEC PWM backlight for CM4 based boards + (BCM2711). Creates a pwm-backlight device with a 16-step gamma + brightness table and an optional pca9535 GPIO expander at 0x20 + for the backlight enable signal. +Load: dtoverlay=ed-backlight-pwm,= +Params: sg215 Invert the brightness curve for 21.5" + SG215 panels + + +Name: ed-backlight-pwm-pi5 +Info: See ed-backlight-pwm (this is the CM5 version) + + Name: ed-ipc Info: Configures the EDATEC IPC/EXP series hardware peripherals. This overlay provides support for PCA9535 GPIO expanders on @@ -1570,6 +1584,73 @@ Load: dtoverlay=googlevoicehat-soundcard Params: +Name: gpclk +Info: Configure a general purpose clock (GPCLK0, GPCLK1 or GPCLK2) output + on a GPIO pin, and set its rate. The clock is held open by a + clk-hog device (see drivers/clk/clk-hog.c, CONFIG_CLK_HOG), so it + stays enabled for as long as the overlay is loaded, with no other + consumer required. + Legal pin,function combinations, and the override that selects + each one directly: + GPCLK0: 4,4(Alt0)/gpclk0_4 20,2(Alt5)/gpclk0_20 + 32,4(Alt0)/gpclk0_32 34,4(Alt0)/gpclk0_34 + GPCLK1: 5,4(Alt0)/gpclk1_5 21,2(Alt5)/gpclk1_21 + 42,4(Alt0)/gpclk1_42 44,4(Alt0)/gpclk1_44 + GPCLK2: 6,4(Alt0)/gpclk2_6 43,4(Alt0)/gpclk2_43 + N.B.: Check the chosen pin and clock aren't already used by the + firmware, other overlays, or onboard hardware. +Load: dtoverlay=gpclk,= +Params: freq Output frequency in Hz (default 500000) + gpclk0_4 Output GPCLK0 on GPIO4 (Alt0) - the default + gpclk0_20 Output GPCLK0 on GPIO20 (Alt5) + gpclk0_32 Output GPCLK0 on GPIO32 (Alt0) + gpclk0_34 Output GPCLK0 on GPIO34 (Alt0) + gpclk1_5 Output GPCLK1 on GPIO5 (Alt0) + gpclk1_21 Output GPCLK1 on GPIO21 (Alt5) + gpclk1_42 Output GPCLK1 on GPIO42 (Alt0) + gpclk1_44 Output GPCLK1 on GPIO44 (Alt0) + gpclk2_6 Output GPCLK2 on GPIO6 (Alt0) + gpclk2_43 Output GPCLK2 on GPIO43 (Alt0) + + +Name: gpclk-pi5 +Info: Configure an RP1 general purpose clock (GPCLK0-GPCLK5) output on a + GPIO pin, and set its rate. This is the Pi 5 counterpart of gpclk, + selected automatically when gpclk is requested on a Pi 5. As with + gpclk, the clock is held open by a clk-hog device (see + drivers/clk/clk-hog.c, CONFIG_CLK_HOG), so it stays enabled for as + long as the overlay is loaded, with no other consumer required. + Legal pin,clock combinations, and the override that selects each + one directly: + GPCLK0: gpio4/gpclk0_4 gpio20/gpclk0_20 + GPCLK1: gpio5/gpclk1_5 gpio18/gpclk1_18 gpio21/gpclk1_21 + GPCLK2: gpio6/gpclk2_6 + GPCLK3: gpio32/gpclk3_32 gpio34/gpclk3_34 gpio46/gpclk3_46 + GPCLK4: gpio33/gpclk4_33 gpio43/gpclk4_43 + GPCLK5: gpio42/gpclk5_42 gpio44/gpclk5_44 gpio47/gpclk5_47 + GPCLK0-GPCLK2 on pins 4/20, 5/21 and 6 are the combinations in + common with gpclk (see above); the rest (GPCLK1 on pin 18, and + GPCLK3-GPCLK5) only exist on RP1, so aren't offered by gpclk. + N.B.: Check the chosen pin and clock aren't already used by the + firmware, other overlays, or onboard hardware. +Load: dtoverlay=gpclk-pi5,= +Params: freq Output frequency in Hz (default 500000) + gpclk0_4 Output GPCLK0 on GPIO4 - the default + gpclk0_20 Output GPCLK0 on GPIO20 + gpclk1_5 Output GPCLK1 on GPIO5 + gpclk1_18 Output GPCLK1 on GPIO18 + gpclk1_21 Output GPCLK1 on GPIO21 + gpclk2_6 Output GPCLK2 on GPIO6 + gpclk3_32 Output GPCLK3 on GPIO32 + gpclk3_34 Output GPCLK3 on GPIO34 + gpclk3_46 Output GPCLK3 on GPIO46 + gpclk4_33 Output GPCLK4 on GPIO33 + gpclk4_43 Output GPCLK4 on GPIO43 + gpclk5_42 Output GPCLK5 on GPIO42 + gpclk5_44 Output GPCLK5 on GPIO44 + gpclk5_47 Output GPCLK5 on GPIO47 + + Name: gpio-charger Info: This is a generic overlay for detecting charger with GPIO. Load: dtoverlay=gpio-charger,= @@ -3489,8 +3570,8 @@ Params: s08-spi--present 4-bit integer, bitmap indicating MCP23S08 Name: mcp2515 -Info: Configures the MCP2515 CAN controller on spi0/1/2/3/5 - For devices on spi1, spi2, spi3 or spi5, the interfaces should be +Info: Configures the MCP2515 CAN controller on spi0/1/2/3/5/6 + For devices on spi1, spi2, spi3, spi5 or spi6, the interfaces should be enabled with one of the spi-1/2/3cs overlays. Load: dtoverlay=mcp2515,= Params: spi- Configure device at spi, cs @@ -4355,6 +4436,39 @@ Params: pin Output pin (default 18) - see table clock PWM clock frequency (informational) +Name: pwm-fan +Info: Configure a PWM GPIO pin to control a cooling fan. +Load: dtoverlay=pwm-fan,= +Params: fan_gpio_12 Use GPIO12 (PWM0) to control the fan. + fan_gpio_18 Use GPIO18 (PWM0) to control the fan (default). + fan_gpio_13 Use GPIO13 (PWM1) to control the fan. + fan_gpio_19 Use GPIO19 (PWM1) to control the fan. + fan_temp0 Temperature threshold (in millicelsius) for + 1st cooling level (default 50000). + fan_temp0_hyst Temperature hysteresis (in millicelsius) for + 1st cooling level (default 5000). + fan_temp0_speed Fan PWM setting for 1st cooling level (0-255, + default 75). + fan_temp1 Temperature threshold (in millicelsius) for + 2nd cooling level (default 60000). + fan_temp1_hyst Temperature hysteresis (in millicelsius) for + 2nd cooling level (default 5000). + fan_temp1_speed Fan PWM setting for 2nd cooling level (0-255, + default 125). + fan_temp2 Temperature threshold (in millicelsius) for + 3rd cooling level (default 67500). + fan_temp2_hyst Temperature hysteresis (in millicelsius) for + 3rd cooling level (default 5000). + fan_temp2_speed Fan PWM setting for 3rd cooling level (0-255, + default 175). + fan_temp3 Temperature threshold (in millicelsius) for + 4th cooling level (default 75000). + fan_temp3_hyst Temperature hysteresis (in millicelsius) for + 4th cooling level (default 5000). + fan_temp3_speed Fan PWM setting for 4th cooling level (0-255, + default 250). + + Name: pwm-gpio Info: Configures the software PWM GPIO driver Load: dtoverlay=pwm-gpio,= @@ -4738,6 +4852,14 @@ Load: dtoverlay=rra-digidac1-wm8741-audio Params: +Name: rt5616 +Info: Overlay for the Realtek RT5616 I2S audio codec (HAT with the RT5616 + codec on I2C1 at 0x1b, MCLK 12.288 MHz, I2S connected to the I2S + consumer controller on GPIO 18-21). +Load: dtoverlay=rt5616,= +Params: alsaname Changes the card name in ALSA + + Name: sainsmart18 Info: Overlay for the SPI-connected Sainsmart 1.8" display (based on the ST7735R chip). @@ -4830,6 +4952,10 @@ Params: overclock_50 Clock (in MHz) to use when the MMC framework custom carrier boards (e.g. CM4 baseboard). Applies ALT0 function and appropriate pulls. + preserve_mmc Don't disable the other MMC interfaces + (mmc/mmcnr). Useful on Compute Module + when using gpios_22_27 with WiFi. + Name: sdio Info: Selects the bcm2835-sdhost SD/MMC driver, optionally with overclock, @@ -6069,6 +6195,8 @@ Params: 2_8_inch 2.8" 480x640 11_9_inch 11.9" 320x1480 13_3_inch_4lane 13.3" 1920x1080 4lane 13_3_inch_2lane 13.3" 1920x1080 2lane + 10_1_inchE_4lane 10.1" E 1920x1200 60fps 4lane + 10_1_inchE_2lane 10.1" E 1920x1200 30fps 2lane i2c1 Use i2c-1 with jumper wires from GPIOs 2&3 disable_touch Disable the touch controller rotation Set the panel orientation property @@ -6375,6 +6503,13 @@ Info: Overlay for the Waveshare wm8960 soundcard Load: dtoverlay=wm8960-soundcard,= Params: alsaname Changes the card name in ALSA compatible Changes the codec compatibility + mclk Sets the MCLK clock frequency in Hz (default + 12.288 MHz for the standard Waveshare HAT, + 24 MHz when the media parameter is used) + media Uses the media board configuration: 24 MHz + MCLK, enables headphone jack detection + (wlf,hp-cfg) and an on-board microphone / + dual speaker sound card Name: ws2812-pio diff --git a/arch/arm/boot/dts/overlays/ed-backlight-pwm-overlay.dts b/arch/arm/boot/dts/overlays/ed-backlight-pwm-overlay.dts new file mode 100644 index 0000000000000..cac9750a1241b --- /dev/null +++ b/arch/arm/boot/dts/overlays/ed-backlight-pwm-overlay.dts @@ -0,0 +1,62 @@ +/* + * EDATEC PWM backlight overlay (CM4/BCM2711). + */ +/dts-v1/; +/plugin/; + +/ { + compatible = "brcm,bcm2711"; + + fragment@0 { + target = <&pwm>; + __overlay__ { + pinctrl-names = "default"; + pinctrl-0 = <&pwm0_1_gpio45>; + status = "okay"; + }; + }; + + fragment@1 { + target-path = "/"; + __overlay__ { + backlight_pwm: backlight_pwm { + compatible = "pwm-backlight"; + brightness-levels = <0 6 12 24 32 48 64 96 + 128 150 170 190 210 220 240 255>; + default-brightness-level = <15>; + pwms = <&pwm 1 2000000 0>; + enable-gpios = <&pca 0 1>; + status = "okay"; + }; + }; + }; + + fragment@2 { + target = <&i2c_arm>; + __overlay__ { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + pca: pca@20 { + compatible = "nxp,pca9535"; + reg = <0x20>; + gpio-controller; + #gpio-cells = <2>; + status = "okay"; + }; + }; + }; + + fragment@3 { + target = <&backlight_pwm>; + __dormant__ { + brightness-levels = <255 240 220 210 190 170 + 150 128 96 64 48 32 24 12 6 0>; + }; + }; + + __overrides__ { + sg215 = <0>, "+3"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/ed-backlight-pwm-pi5-overlay.dts b/arch/arm/boot/dts/overlays/ed-backlight-pwm-pi5-overlay.dts new file mode 100644 index 0000000000000..c4b9d22d6f017 --- /dev/null +++ b/arch/arm/boot/dts/overlays/ed-backlight-pwm-pi5-overlay.dts @@ -0,0 +1,62 @@ +/* + * EDATEC PWM backlight overlay (CM5/Pi5/BCM2712). + */ +/dts-v1/; +/plugin/; + +/ { + compatible = "brcm,bcm2712"; + + fragment@0 { + target = <&pwm1>; + __overlay__ { + pinctrl-names = "default"; + pinctrl-0 = <&rp1_pwm1_gpio45>; + status = "okay"; + }; + }; + + fragment@1 { + target-path = "/"; + __overlay__ { + backlight_pwm: backlight_pwm { + compatible = "pwm-backlight"; + brightness-levels = <0 6 12 24 32 48 64 96 + 128 150 170 190 210 220 240 255>; + default-brightness-level = <15>; + pwms = <&pwm1 3 2000000 0>; + enable-gpios = <&pca 0 1>; + status = "okay"; + }; + }; + }; + + fragment@2 { + target = <&i2c_arm>; + __overlay__ { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + pca: pca@20 { + compatible = "nxp,pca9535"; + reg = <0x20>; + gpio-controller; + #gpio-cells = <2>; + status = "okay"; + }; + }; + }; + + fragment@3 { + target = <&backlight_pwm>; + __dormant__ { + brightness-levels = <255 240 220 210 190 170 + 150 128 96 64 48 32 24 12 6 0>; + }; + }; + + __overrides__ { + sg215 = <0>, "+3"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/gpclk-overlay.dts b/arch/arm/boot/dts/overlays/gpclk-overlay.dts new file mode 100644 index 0000000000000..0caaf63bcec44 --- /dev/null +++ b/arch/arm/boot/dts/overlays/gpclk-overlay.dts @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gpclk-overlay.dts + * + * Route one of the BCM283x/BCM2711 general-purpose clocks (GPCLK0/1/2) to + * a GPIO pin, set its output frequency, and keep it running via a clk-hog + * device (see clk-hog.c) so it stays enabled with no other consumer. + * + * Legal pin,function combinations, and the override that selects each one + * directly (see gpclk-pi5-overlay.dts for the RP1/Pi 5 equivalents - the + * combinations in common between the two use identical override names): + * GPCLK0: 4(gpclk0_4) 20(gpclk0_20) 32(gpclk0_32) 34(gpclk0_34) + * GPCLK1: 5(gpclk1_5) 21(gpclk1_21) 42(gpclk1_42) 44(gpclk1_44) + * GPCLK2: 6(gpclk2_6) 43(gpclk2_43) + * + * N.B. Some pins/GPCLKs may already be in use by the firmware, other + * overlays, or onboard hardware (e.g. pin 4 defaults to GPCLK0 and is + * used by several HATs) - check for conflicts before choosing a pin. + */ + +/dts-v1/; +/plugin/; + +#include +#include + +/ { + compatible = "brcm,bcm2835"; + + fragment@0 { + target-path = "/"; + __overlay__ { + clk_hog: gpclk_hog@4 { + compatible = "clk-hog"; + + clocks = <&clocks BCM2835_CLOCK_GP0>; + assigned-clocks = <&clocks BCM2835_CLOCK_GP0>; + assigned-clock-rates = <500000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gpclk0_4>; + }; + }; + }; + + /* Each pin's pinctrl node lives in its own fragment, numbered by + * pin number, so that only one is ever instantiated at a time - + * the kernel rejects an overlay that tries to create a gpio node + * that already exists in the live Device Tree, which would happen + * if two gpclk overlay instances both carried, say, the gpclk1_5 + * node. gpclk0_4 is the default and so is the only one enabled + * (__overlay__) from the start; the rest start __dormant__ and are + * swapped in by the relevant override, which also disables + * gpclk0_4's fragment. + */ + fragment@4 { + target = <&gpio>; + __overlay__ { + gpclk0_4: gpclk0_4 { + brcm,pins = <4>; + brcm,function = ; + }; + }; + }; + + fragment@5 { + target = <&gpio>; + __dormant__ { + gpclk1_5: gpclk1_5 { + brcm,pins = <5>; + brcm,function = ; + }; + }; + }; + + fragment@6 { + target = <&gpio>; + __dormant__ { + gpclk2_6: gpclk2_6 { + brcm,pins = <6>; + brcm,function = ; + }; + }; + }; + + fragment@20 { + target = <&gpio>; + __dormant__ { + gpclk0_20: gpclk0_20 { + brcm,pins = <20>; + brcm,function = ; + }; + }; + }; + + fragment@21 { + target = <&gpio>; + __dormant__ { + gpclk1_21: gpclk1_21 { + brcm,pins = <21>; + brcm,function = ; + }; + }; + }; + + fragment@32 { + target = <&gpio>; + __dormant__ { + gpclk0_32: gpclk0_32 { + brcm,pins = <32>; + brcm,function = ; + }; + }; + }; + + fragment@34 { + target = <&gpio>; + __dormant__ { + gpclk0_34: gpclk0_34 { + brcm,pins = <34>; + brcm,function = ; + }; + }; + }; + + fragment@42 { + target = <&gpio>; + __dormant__ { + gpclk1_42: gpclk1_42 { + brcm,pins = <42>; + brcm,function = ; + }; + }; + }; + + fragment@43 { + target = <&gpio>; + __dormant__ { + gpclk2_43: gpclk2_43 { + brcm,pins = <43>; + brcm,function = ; + }; + }; + }; + + fragment@44 { + target = <&gpio>; + __dormant__ { + gpclk1_44: gpclk1_44 { + brcm,pins = <44>; + brcm,function = ; + }; + }; + }; + + __overrides__ { + freq = <&clk_hog>,"assigned-clock-rates:0"; + + /* Named pin+clock combinations: each sets the pin mux and + * the clock together, in one go. + */ + gpclk0_4 = <&clk_hog>,"pinctrl-0:0=",<&gpclk0_4>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=4"; + gpclk0_20 = <0>,"-4+20", + <&clk_hog>,"pinctrl-0:0=",<&gpclk0_20>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=20"; + gpclk0_32 = <0>,"-4+32", + <&clk_hog>,"pinctrl-0:0=",<&gpclk0_32>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=32"; + gpclk0_34 = <0>,"-4+34", + <&clk_hog>,"pinctrl-0:0=",<&gpclk0_34>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=34"; + gpclk1_5 = <0>,"-4+5", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_5>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=5"; + gpclk1_21 = <0>,"-4+21", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_21>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=21"; + gpclk1_42 = <0>,"-4+42", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_42>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=42"; + gpclk1_44 = <0>,"-4+44", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_44>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=44"; + gpclk2_6 = <0>,"-4+6", + <&clk_hog>,"pinctrl-0:0=",<&gpclk2_6>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=6"; + gpclk2_43 = <0>,"-4+43", + <&clk_hog>,"pinctrl-0:0=",<&gpclk2_43>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=43"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/gpclk-pi5-overlay.dts b/arch/arm/boot/dts/overlays/gpclk-pi5-overlay.dts new file mode 100644 index 0000000000000..6a3d01115c9b4 --- /dev/null +++ b/arch/arm/boot/dts/overlays/gpclk-pi5-overlay.dts @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * gpclk-pi5-overlay.dts + * + * Route one of the RP1 general-purpose clocks (GPCLK0-GPCLK5) to a GPIO + * pin, set its output frequency, and keep it running via a clk-hog + * device (see clk-hog.c) so it stays enabled with no other consumer. + * This is the Pi 5 / RP1 counterpart of gpclk-overlay.dts (which targets + * the BCM2835 GPCLKs on earlier Pis). + * + * The RP1 pinctrl driver still accepts the old-style numeric brcm,pins + * property for pin selection, but functions are selected by name rather + * than by BCM283x ALT number (RP1 has three more GPCLKs, and not every + * pin's gpclkN alternate function lines up with a BCM283x ALT number + * anyway), so rather than trying to make a single numeric "func" value + * mean the same thing on both chips, each legal pin+clock combination is + * exposed as its own named override, using the same names as + * gpclk-overlay.dts for the combinations the two chips have in common. + * That means a line such as "dtoverlay=gpclk,gpclk0_20" has the same + * effect on any Pi, whichever of the two overlays it resolves to. + * + * Legal pin,clock combinations, and the override that selects each one: + * GPCLK0: gpio4(gpclk0_4) gpio20(gpclk0_20) + * GPCLK1: gpio5(gpclk1_5) gpio18(gpclk1_18) gpio21(gpclk1_21) + * GPCLK2: gpio6(gpclk2_6) + * GPCLK3: gpio32(gpclk3_32) gpio34(gpclk3_34) gpio46(gpclk3_46) + * GPCLK4: gpio33(gpclk4_33) gpio43(gpclk4_43) + * GPCLK5: gpio42(gpclk5_42) gpio44(gpclk5_44) gpio47(gpclk5_47) + * + * GPCLK0-GPCLK2 on pins 4/20, 5/21 and 6 are the combinations in common + * with gpclk-overlay.dts; the rest (GPCLK1 on pin18, and GPCLK3-GPCLK5) + * only exist on RP1. + * + * N.B. Some pins/GPCLKs may already be in use by the firmware, other + * overlays, or onboard hardware - check for conflicts before choosing a + * pin. + */ + +/dts-v1/; +/plugin/; + +#include + +/ { + compatible = "brcm,bcm2712"; + + fragment@0 { + target-path = "/"; + __overlay__ { + clk_hog: gpclk_hog@4 { + compatible = "clk-hog"; + + clocks = <&rp1_clocks RP1_CLK_GP0>; + assigned-clocks = <&rp1_clocks RP1_CLK_GP0>; + assigned-clock-rates = <500000>; + + pinctrl-names = "default"; + pinctrl-0 = <&gpclk0_4>; + }; + }; + }; + + /* Each pin's pinctrl node lives in its own fragment, numbered by + * pin number, so that only one is ever instantiated at a time - + * the kernel rejects an overlay that tries to create a gpio node + * that already exists in the live Device Tree, which would happen + * if two gpclk overlay instances both carried, say, the gpclk1_5 + * node. gpclk0_4 is the default and so is the only one enabled + * (__overlay__) from the start; the rest start __dormant__ and are + * swapped in by the relevant override, which also disables + * gpclk0_4's fragment. + */ + + fragment@4 { + target = <&gpio>; + __overlay__ { + gpclk0_4: gpclk0_4 { + brcm,pins = <4>; + function = "gpclk0"; + }; + }; + }; + + fragment@5 { + target = <&gpio>; + __dormant__ { + gpclk1_5: gpclk1_5 { + brcm,pins = <5>; + function = "gpclk1"; + }; + }; + }; + + fragment@6 { + target = <&gpio>; + __dormant__ { + gpclk2_6: gpclk2_6 { + brcm,pins = <6>; + function = "gpclk2"; + }; + }; + }; + + fragment@20 { + target = <&gpio>; + __dormant__ { + gpclk0_20: gpclk0_20 { + brcm,pins = <20>; + function = "gpclk0"; + }; + }; + }; + + fragment@21 { + target = <&gpio>; + __dormant__ { + gpclk1_21: gpclk1_21 { + brcm,pins = <21>; + function = "gpclk1"; + }; + }; + }; + + fragment@18 { + target = <&gpio>; + __dormant__ { + gpclk1_18: gpclk1_18 { + brcm,pins = <18>; + function = "gpclk1"; + }; + }; + }; + + fragment@32 { + target = <&gpio>; + __dormant__ { + gpclk3_32: gpclk3_32 { + brcm,pins = <32>; + function = "gpclk3"; + }; + }; + }; + + fragment@33 { + target = <&gpio>; + __dormant__ { + gpclk4_33: gpclk4_33 { + brcm,pins = <33>; + function = "gpclk4"; + }; + }; + }; + + fragment@34 { + target = <&gpio>; + __dormant__ { + gpclk3_34: gpclk3_34 { + brcm,pins = <34>; + function = "gpclk3"; + }; + }; + }; + + fragment@42 { + target = <&gpio>; + __dormant__ { + gpclk5_42: gpclk5_42 { + brcm,pins = <42>; + function = "gpclk5"; + }; + }; + }; + + fragment@43 { + target = <&gpio>; + __dormant__ { + gpclk4_43: gpclk4_43 { + brcm,pins = <43>; + function = "gpclk4"; + }; + }; + }; + + fragment@44 { + target = <&gpio>; + __dormant__ { + gpclk5_44: gpclk5_44 { + brcm,pins = <44>; + function = "gpclk5"; + }; + }; + }; + + fragment@46 { + target = <&gpio>; + __dormant__ { + gpclk3_46: gpclk3_46 { + brcm,pins = <46>; + function = "gpclk3"; + }; + }; + }; + + fragment@47 { + target = <&gpio>; + __dormant__ { + gpclk5_47: gpclk5_47 { + brcm,pins = <47>; + function = "gpclk5"; + }; + }; + }; + + __overrides__ { + /* Named pin+clock combinations - the recommended way to + * select a legal combination in one go. + */ + gpclk0_4 = <&clk_hog>,"pinctrl-0:0=",<&gpclk0_4>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=4"; + gpclk0_20 = <0>,"-4+20", + <&clk_hog>,"pinctrl-0:0=",<&gpclk0_20>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=20"; + gpclk1_5 = <0>,"-4+5", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_5>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=5"; + gpclk1_18 = <0>,"-4+18", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_18>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=18"; + gpclk1_21 = <0>,"-4+21", + <&clk_hog>,"pinctrl-0:0=",<&gpclk1_21>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=21"; + gpclk2_6 = <0>,"-4+6", + <&clk_hog>,"pinctrl-0:0=",<&gpclk2_6>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=6"; + gpclk3_32 = <0>,"-4+32", + <&clk_hog>,"pinctrl-0:0=",<&gpclk3_32>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=32"; + gpclk3_34 = <0>,"-4+34", + <&clk_hog>,"pinctrl-0:0=",<&gpclk3_34>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=34"; + gpclk3_46 = <0>,"-4+46", + <&clk_hog>,"pinctrl-0:0=",<&gpclk3_46>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=46"; + gpclk4_33 = <0>,"-4+33", + <&clk_hog>,"pinctrl-0:0=",<&gpclk4_33>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=33"; + gpclk4_43 = <0>,"-4+43", + <&clk_hog>,"pinctrl-0:0=",<&gpclk4_43>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=43"; + gpclk5_42 = <0>,"-4+42", + <&clk_hog>,"pinctrl-0:0=",<&gpclk5_42>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=42"; + gpclk5_44 = <0>,"-4+44", + <&clk_hog>,"pinctrl-0:0=",<&gpclk5_44>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=44"; + gpclk5_47 = <0>,"-4+47", + <&clk_hog>,"pinctrl-0:0=",<&gpclk5_47>, + <&clk_hog>,"clocks:4=", , + <&clk_hog>,"assigned-clocks:4=", , + <&clk_hog>,"reg:0=47"; + + freq = <&clk_hog>,"assigned-clock-rates:0"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/mcp2515-overlay.dts b/arch/arm/boot/dts/overlays/mcp2515-overlay.dts index 36b5c46888715..292296dd0593c 100644 --- a/arch/arm/boot/dts/overlays/mcp2515-overlay.dts +++ b/arch/arm/boot/dts/overlays/mcp2515-overlay.dts @@ -95,16 +95,30 @@ }; fragment@12 { + target-path = "spi6/spidev@0"; + __dormant__ { + status = "disabled"; + }; + }; + + fragment@13 { + target-path = "spi6/spidev@1"; + __dormant__ { + status = "disabled"; + }; + }; + + fragment@100 { target = <&gpio>; __overlay__ { - mcp2515_pins: mcp2515_pins { + mcp2515_pins: mcp2515-pins { brcm,pins = <25>; brcm,function = ; }; }; }; - fragment@13 { + fragment@101 { target-path = "/clocks"; __overlay__ { clk_mcp2515_osc: mcp2515-osc { @@ -115,7 +129,7 @@ }; }; - mcp2515_frag: fragment@14 { + mcp2515_frag: fragment@102 { target = <&spi0>; __overlay__ { status = "okay"; @@ -139,72 +153,85 @@ spi0-0 = <0>, "+0", <&mcp2515_frag>, "target:0=", <&spi0>, <&mcp2515>, "reg:0=0", - <&mcp2515_pins>, "name=mcp2515_spi0_0_pins", + <&mcp2515_pins>, "name=mcp2515-spi0-0-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi0-0-osc"; spi0-1 = <0>, "+1", <&mcp2515_frag>, "target:0=", <&spi0>, <&mcp2515>, "reg:0=1", - <&mcp2515_pins>, "name=mcp2515_spi0_1_pins", + <&mcp2515_pins>, "name=mcp2515-spi0-1-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi0-1-osc"; spi1-0 = <0>, "+2", <&mcp2515_frag>, "target:0=", <&spi1>, <&mcp2515>, "reg:0=0", - <&mcp2515_pins>, "name=mcp2515_spi1_0_pins", + <&mcp2515_pins>, "name=mcp2515-spi1-0-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi1-0-osc"; spi1-1 = <0>, "+3", <&mcp2515_frag>, "target:0=", <&spi1>, <&mcp2515>, "reg:0=1", - <&mcp2515_pins>, "name=mcp2515_spi1_1_pins", + <&mcp2515_pins>, "name=mcp2515-spi1-1-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi1-1-osc"; spi1-2 = <0>, "+4", <&mcp2515_frag>, "target:0=", <&spi1>, <&mcp2515>, "reg:0=2", - <&mcp2515_pins>, "name=mcp2515_spi1_2_pins", + <&mcp2515_pins>, "name=mcp2515-spi1-2-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi1-2-osc"; spi2-0 = <0>, "+5", <&mcp2515_frag>, "target:0=", <&spi2>, <&mcp2515>, "reg:0=0", - <&mcp2515_pins>, "name=mcp2515_spi2_0_pins", + <&mcp2515_pins>, "name=mcp2515-spi2-0-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi2-0-osc"; spi2-1 = <0>, "+6", <&mcp2515_frag>, "target:0=", <&spi2>, <&mcp2515>, "reg:0=1", - <&mcp2515_pins>, "name=mcp2515_spi2_1_pins", + <&mcp2515_pins>, "name=mcp2515-spi2-1-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi2-1-osc"; spi2-2 = <0>, "+7", <&mcp2515_frag>, "target:0=", <&spi2>, <&mcp2515>, "reg:0=2", - <&mcp2515_pins>, "name=mcp2515_spi2_2_pins", + <&mcp2515_pins>, "name=mcp2515-spi2-2-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi2-2-osc"; spi3-0 = <0>, "+8", <&mcp2515_frag>, "target?=0", <&mcp2515_frag>, "target-path=spi3", <&mcp2515>, "reg:0=0", - <&mcp2515_pins>, "name=mcp2515_spi3_0_pins", + <&mcp2515_pins>, "name=mcp2515-spi3-0-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi3-0-osc"; spi3-1 = <0>, "+9", <&mcp2515_frag>, "target?=0", <&mcp2515_frag>, "target-path=spi3", <&mcp2515>, "reg:0=1", - <&mcp2515_pins>, "name=mcp2515_spi3_1_pins", + <&mcp2515_pins>, "name=mcp2515-spi3-1-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi3-1-osc"; spi5-0 = <0>, "+10", <&mcp2515_frag>, "target?=0", <&mcp2515_frag>, "target-path=spi5", <&mcp2515>, "reg:0=0", - <&mcp2515_pins>, "name=mcp2515_spi5_0_pins", + <&mcp2515_pins>, "name=mcp2515-spi5-0-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi5-0-osc"; spi5-1 = <0>, "+11", <&mcp2515_frag>, "target?=0", <&mcp2515_frag>, "target-path=spi5", <&mcp2515>, "reg:0=1", - <&mcp2515_pins>, "name=mcp2515_spi5_1_pins", + <&mcp2515_pins>, "name=mcp2515-spi5-1-pins", <&clk_mcp2515_osc>, "name=mcp2515-spi5-1-osc"; + spi6-0 = <0>, "+12", + <&mcp2515_frag>, "target?=0", + <&mcp2515_frag>, "target-path=spi6", + <&mcp2515>, "reg:0=0", + <&mcp2515_pins>, "name=mcp2515-spi6-0-pins", + <&clk_mcp2515_osc>, "name=mcp2515-spi6-0-osc"; + spi6-1 = <0>, "+13", + <&mcp2515_frag>, "target?=0", + <&mcp2515_frag>, "target-path=spi6", + <&mcp2515>, "reg:0=1", + <&mcp2515_pins>, "name=mcp2515-spi6-1-pins", + <&clk_mcp2515_osc>, "name=mcp2515-spi6-1-osc"; + oscillator = <&clk_mcp2515_osc>, "clock-frequency:0"; speed = <&mcp2515>, "spi-max-frequency:0"; interrupt = <&mcp2515_pins>, "brcm,pins:0", diff --git a/arch/arm/boot/dts/overlays/overlay_map.dts b/arch/arm/boot/dts/overlays/overlay_map.dts index 3138cef81f534..38fc435f68bd0 100644 --- a/arch/arm/boot/dts/overlays/overlay_map.dts +++ b/arch/arm/boot/dts/overlays/overlay_map.dts @@ -63,10 +63,29 @@ bcm2712; }; + ed-backlight-pwm { + bcm2711; + bcm2712 = "ed-backlight-pwm-pi5"; + }; + + ed-backlight-pwm-pi5 { + bcm2712; + }; + dwc-otg { renamed = "dwc-otg-deprecated"; }; + gpclk { + bcm2835; + bcm2711; + bcm2712 = "gpclk-pi5"; + }; + + gpclk-pi5 { + bcm2712; + }; + hifiberry-adc8x { bcm2712; }; diff --git a/arch/arm/boot/dts/overlays/pwm-fan-overlay.dts b/arch/arm/boot/dts/overlays/pwm-fan-overlay.dts new file mode 100644 index 0000000000000..ad3e49112dd1a --- /dev/null +++ b/arch/arm/boot/dts/overlays/pwm-fan-overlay.dts @@ -0,0 +1,150 @@ +/* + * Overlay for a GPIO connected PWM cooling fan. + * References: + * - https://forums.raspberrypi.com/viewtopic.php?t=363655 + * + * Optional parameters: + * - "fan_gpio_12" - Use GPIO12 (PWM0) to control the fan; + * - "fan_gpio_18" - Use GPIO18 (PWM0) to control the fan (default); + * - "fan_gpio_13" - Use GPIO13 (PWM1) to control the fan; + * - "fan_gpio_19" - Use GPIO19 (PWM1) to control the fan; + * - "fan_temp0" - Temperature threshold (in millicelsius) for 1st cooling level (default 50000); + * - "fan_temp0_hyst" - Temperature hysteresis (in millicelsius) for 1st cooling level (default 5000); + * - "fan_temp0_speed" - Fan PWM setting for 1st cooling level (0-255, default 75); + * - "fan_temp1" - Temperature threshold (in millicelsius) for 2nd cooling level (default 60000); + * - "fan_temp1_hyst" - Temperature hysteresis (in millicelsius) for 2nd cooling level (default 5000); + * - "fan_temp1_speed" - Fan PWM setting for 2nd cooling level (0-255, default 125); + * - "fan_temp2" - Temperature threshold (in millicelsius) for 3rd cooling level (default 67500); + * - "fan_temp2_hyst" - Temperature hysteresis (in millicelsius) for 3rd cooling level (default 5000); + * - "fan_temp2_speed" - Fan PWM setting for 3rd cooling level (0-255, default 175); + * - "fan_temp3" - Temperature threshold (in millicelsius) for 4th cooling level (default 75000); + * - "fan_temp3_hyst" - Temperature hysteresis (in millicelsius) for 4th cooling level (default 5000); + * - "fan_temp3_speed" - Fan PWM setting for 4th cooling level (0-255, default 250); + * + * Build: + * - `sudo dtc -W no-unit_address_vs_reg -@ -I dts -O dtb -o /boot/firmware/overlays/pwm-fan.dtbo pwm-fan-overlay.dts` + * + * Activate: + * - `sudo nano /boot/config.txt` add "dtoverlay=pwm-fan" + */ +/dts-v1/; +/plugin/; + +/ { + compatible = "brcm,bcm2835"; + + fragment@0 { + target = <&gpio>; + __overlay__ { + pwm_pins: pwm_pins { + brcm,pins = <18>; + brcm,function = <2>; + brcm,pull = <0>; + }; + }; + }; + + fragment@1 { + target = <&pwm>; + __overlay__ { + pinctrl-names = "default"; + pinctrl-0 = <&pwm_pins>; + status = "okay"; + }; + }; + + fragment@2 { + target-path = "/"; + __overlay__ { + fan: pwm-fan { + compatible = "pwm-fan"; + #cooling-cells = <2>; + pwms = <&pwm 0 40000 0>; + cooling-min-state = <0>; + cooling-max-state = <4>; + cooling-levels = <0 75 125 175 250>; + }; + }; + }; + + fragment@3 { + target = <&thermal_trips>; + __overlay__ { + cpu_tepid: cpu-tepid { + temperature = <50000>; + hysteresis = <5000>; + type = "active"; + }; + + cpu_warm: cpu-warm { + temperature = <60000>; + hysteresis = <5000>; + type = "active"; + }; + + cpu_hot: cpu-hot { + temperature = <67500>; + hysteresis = <5000>; + type = "active"; + }; + + cpu_vhot: cpu-vhot { + temperature = <75000>; + hysteresis = <5000>; + type = "active"; + }; + }; + }; + + fragment@4 { + target = <&cooling_maps>; + __overlay__ { + tepid { + trip = <&cpu_tepid>; + cooling-device = <&fan 1 1>; + }; + + warm { + trip = <&cpu_warm>; + cooling-device = <&fan 2 2>; + }; + + hot { + trip = <&cpu_hot>; + cooling-device = <&fan 3 3>; + }; + + vhot { + trip = <&cpu_vhot>; + cooling-device = <&fan 4 4>; + }; + }; + }; + + __overrides__ { + fan_gpio_12 = <&pwm_pins>,"brcm,pins:0=12", + <&pwm_pins>,"brcm,function:0=4", + <&fan>,"pwms:4=0"; + fan_gpio_18 = <&pwm_pins>,"brcm,pins:0=18", + <&pwm_pins>,"brcm,function:0=2", + <&fan>,"pwms:4=0"; + fan_gpio_13 = <&pwm_pins>,"brcm,pins:0=13", + <&pwm_pins>,"brcm,function:0=4", + <&fan>,"pwms:4=1"; + fan_gpio_19 = <&pwm_pins>,"brcm,pins:0=19", + <&pwm_pins>,"brcm,function:0=2", + <&fan>,"pwms:4=1"; + fan_temp0 = <&cpu_tepid>,"temperature:0"; + fan_temp0_hyst = <&cpu_tepid>,"hysteresis:0"; + fan_temp0_speed = <&fan>,"cooling-levels:4"; + fan_temp1 = <&cpu_warm>,"temperature:0"; + fan_temp1_hyst = <&cpu_warm>,"hysteresis:0"; + fan_temp1_speed = <&fan>,"cooling-levels:8"; + fan_temp2 = <&cpu_hot>,"temperature:0"; + fan_temp2_hyst = <&cpu_hot>,"hysteresis:0"; + fan_temp2_speed = <&fan>,"cooling-levels:12"; + fan_temp3 = <&cpu_vhot>,"temperature:0"; + fan_temp3_hyst = <&cpu_vhot>,"hysteresis:0"; + fan_temp3_speed = <&fan>,"cooling-levels:16"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/rt5616-overlay.dts b/arch/arm/boot/dts/overlays/rt5616-overlay.dts new file mode 100644 index 0000000000000..17769639230e0 --- /dev/null +++ b/arch/arm/boot/dts/overlays/rt5616-overlay.dts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Definitions for the Realtek RT5616 I2S audio codec +/dts-v1/; +/plugin/; + +/ { + compatible = "brcm,bcm2835"; + + fragment@0 { + target = <&i2s_clk_consumer>; + __overlay__ { + status = "okay"; + }; + }; + + fragment@1 { + target-path = "/"; + __overlay__ { + rt5616_mclk: rt5616-mclk { + compatible = "fixed-clock"; + #clock-cells = <0>; + clock-frequency = <12288000>; + }; + }; + }; + + fragment@2 { + target = <&i2c1>; + __overlay__ { + #address-cells = <1>; + #size-cells = <0>; + status = "okay"; + + rt5616: rt5616@1b { + #sound-dai-cells = <0>; + compatible = "realtek,rt5616"; + reg = <0x1b>; + clocks = <&rt5616_mclk>; + clock-names = "mclk"; + }; + }; + }; + + fragment@3 { + target = <&sound>; + rt5616_card: __overlay__ { + compatible = "simple-audio-card"; + simple-audio-card,format = "i2s"; + simple-audio-card,name = "rt5616-codec"; + simple-audio-card,mclk-fs = <256>; + status = "okay"; + + simple-audio-card,bitclock-master = <&sound_master>; + simple-audio-card,frame-master = <&sound_master>; + + simple-audio-card,widgets = + "Microphone", "Mic Jack", + "Headphone", "Headphone Jack"; + simple-audio-card,routing = + "IN1P", "Mic Jack", + "IN2P", "Mic Jack", + "Headphone Jack", "HPOL", + "Headphone Jack", "HPOR"; + + simple-audio-card,cpu { + sound-dai = <&i2s_clk_consumer>; + }; + + sound_master: simple-audio-card,codec { + sound-dai = <&rt5616>; + }; + }; + }; + + __overrides__ { + alsaname = <&rt5616_card>,"simple-audio-card,name"; + }; +}; diff --git a/arch/arm/boot/dts/overlays/sdhost-overlay.dts b/arch/arm/boot/dts/overlays/sdhost-overlay.dts index 8a539dc92c042..c39a369def429 100644 --- a/arch/arm/boot/dts/overlays/sdhost-overlay.dts +++ b/arch/arm/boot/dts/overlays/sdhost-overlay.dts @@ -54,5 +54,6 @@ pio_limit = <&frag0>,"brcm,pio-limit:0"; debug = <&frag0>,"brcm,debug?"; gpios_22_27 = <0>,"+3+4"; + preserve_mmc = <0>,"-1-2"; }; }; diff --git a/arch/arm/boot/dts/overlays/vc4-kms-dsi-waveshare-panel-overlay.dts b/arch/arm/boot/dts/overlays/vc4-kms-dsi-waveshare-panel-overlay.dts index 7af0f3763fc40..5c1cce1dddceb 100644 --- a/arch/arm/boot/dts/overlays/vc4-kms-dsi-waveshare-panel-overlay.dts +++ b/arch/arm/boot/dts/overlays/vc4-kms-dsi-waveshare-panel-overlay.dts @@ -136,6 +136,12 @@ 7_0_inchH = <&panel>, "compatible=waveshare,7.0inch-h-panel", <&touch>, "touchscreen-swapped-x-y?", <&touch>, "touchscreen-inverted-x?"; + 10_1_inchE_4lane = <&panel>, "compatible=waveshare,10.1inch-e-4lane-panel", + <&touch>, "touchscreen-swapped-x-y?", + <&touch>, "touchscreen-inverted-x?"; + 10_1_inchE_2lane = <&panel>, "compatible=waveshare,10.1inch-e-2lane-panel", + <&touch>, "touchscreen-swapped-x-y?", + <&touch>, "touchscreen-inverted-x?"; i2c1 = <&i2c_frag>, "target:0=",<&i2c1>, <0>, "-3-4+5"; disable_touch = <&touch>, "status=disabled"; diff --git a/arch/arm/boot/dts/overlays/wm8960-soundcard-overlay.dts b/arch/arm/boot/dts/overlays/wm8960-soundcard-overlay.dts index d896c59f469b9..c879d4fd89c83 100644 --- a/arch/arm/boot/dts/overlays/wm8960-soundcard-overlay.dts +++ b/arch/arm/boot/dts/overlays/wm8960-soundcard-overlay.dts @@ -75,8 +75,48 @@ }; }; + /* EDATEC media board: 24MHz MCLK oscillator */ + fragment@4 { + target = <&wm8960_mclk>; + __dormant__ { + clock-frequency = <24000000>; + }; + }; + + /* EDATEC media board: enable headphone jack detection */ + fragment@5 { + target = <&wm8960>; + __dormant__ { + wlf,hp-cfg = <2 2 3>; + }; + }; + + /* EDATEC media board: on-board mic + dual speakers sound card */ + fragment@6 { + target = <&sound>; + __dormant__ { + simple-audio-card,widgets = + "Microphone", "On Board Mic", + "Line", "Line In", + "Line", "Line Out", + "Speaker", "Speaker", + "Headphone", "Headphone Jack"; + + simple-audio-card,routing = + "Headphone Jack", "HP_L", + "Headphone Jack", "HP_R", + "Speaker", "SPK_LP", + "Speaker", "SPK_LN", + "Speaker", "SPK_RP", + "Speaker", "SPK_RN", + "LINPUT2", "Headphone Jack"; + }; + }; + __overrides__ { alsaname = <&slave_overlay>,"simple-audio-card,name"; compatible = <&wm8960>,"compatible"; + mclk = <&wm8960_mclk>,"clock-frequency:0"; + media = <0>, "=4=5=6"; }; }; diff --git a/arch/arm/configs/bcm2709_defconfig b/arch/arm/configs/bcm2709_defconfig index 67748175b48ca..69a08401f2c61 100644 --- a/arch/arm/configs/bcm2709_defconfig +++ b/arch/arm/configs/bcm2709_defconfig @@ -73,6 +73,7 @@ CONFIG_LRU_GEN=y CONFIG_LRU_GEN_ENABLED=y CONFIG_NET=y CONFIG_PACKET=y +CONFIG_TLS=m CONFIG_XFRM_USER=y CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y @@ -490,6 +491,7 @@ CONFIG_MSE102X=m CONFIG_WIZNET_W5100=m CONFIG_WIZNET_W5100_SPI=m CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m CONFIG_CAN_SLCAN=m CONFIG_CAN_MCP251X=m CONFIG_CAN_MCP251XFD=m @@ -976,6 +978,9 @@ CONFIG_VIDEO_I2C=m CONFIG_AUXDISPLAY=y CONFIG_HD44780=m CONFIG_DRM=m +CONFIG_DRM_PANIC=y +CONFIG_DRM_PANIC_BACKGROUND_COLOR=0xc41949 +CONFIG_DRM_PANIC_SCREEN="kmsg" CONFIG_DRM_LOAD_EDID_FIRMWARE=y CONFIG_DRM_UDL=m CONFIG_DRM_PANEL_ILITEK_ILI9806E=m @@ -1092,6 +1097,7 @@ CONFIG_SND_SOC_CS4265=m CONFIG_SND_SOC_ICS43432=m CONFIG_SND_SOC_MA120X0P=m CONFIG_SND_SOC_MAX98357A=m +CONFIG_SND_SOC_RT5616=m CONFIG_SND_SOC_SPDIF=m CONFIG_SND_SOC_TLV320AIC23_I2C=m CONFIG_SND_SOC_WM8804_I2C=m @@ -1301,6 +1307,9 @@ CONFIG_USB_G_ACM_MS=m CONFIG_USB_G_MULTI=m CONFIG_USB_G_HID=m CONFIG_USB_G_WEBCAM=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_FUSB302=m CONFIG_MMC=y CONFIG_MMC_BLOCK_MINORS=32 CONFIG_MMC_BCM2835_MMC=y @@ -1419,6 +1428,7 @@ CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m CONFIG_VIDEO_CODEC_BCM2835=m CONFIG_VIDEO_ISP_BCM2835=m +CONFIG_CLK_HOG=m CONFIG_COMMON_CLK_SI5351=m CONFIG_CLK_RASPBERRYPI=y CONFIG_MAILBOX=y @@ -1591,6 +1601,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DLM=m CONFIG_UNICODE=m +CONFIG_KEYS_REQUEST_CACHE=y CONFIG_KEY_DH_OPERATIONS=y CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/arm/configs/bcmrpi_defconfig b/arch/arm/configs/bcmrpi_defconfig index 9e51a93036948..cde744a826992 100644 --- a/arch/arm/configs/bcmrpi_defconfig +++ b/arch/arm/configs/bcmrpi_defconfig @@ -67,6 +67,7 @@ CONFIG_LRU_GEN=y CONFIG_LRU_GEN_ENABLED=y CONFIG_NET=y CONFIG_PACKET=y +CONFIG_TLS=m CONFIG_XFRM_USER=y CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y @@ -484,6 +485,7 @@ CONFIG_MSE102X=m CONFIG_WIZNET_W5100=m CONFIG_WIZNET_W5100_SPI=m CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m CONFIG_CAN_SLCAN=m CONFIG_CAN_MCP251X=m CONFIG_CAN_MCP251XFD=m @@ -969,6 +971,9 @@ CONFIG_VIDEO_I2C=m CONFIG_AUXDISPLAY=y CONFIG_HD44780=m CONFIG_DRM=m +CONFIG_DRM_PANIC=y +CONFIG_DRM_PANIC_BACKGROUND_COLOR=0xc41949 +CONFIG_DRM_PANIC_SCREEN="kmsg" CONFIG_DRM_LOAD_EDID_FIRMWARE=y CONFIG_DRM_UDL=m CONFIG_DRM_PANEL_ILITEK_ILI9806E=m @@ -1084,6 +1089,7 @@ CONFIG_SND_SOC_CS4265=m CONFIG_SND_SOC_ICS43432=m CONFIG_SND_SOC_MA120X0P=m CONFIG_SND_SOC_MAX98357A=m +CONFIG_SND_SOC_RT5616=m CONFIG_SND_SOC_SPDIF=m CONFIG_SND_SOC_TLV320AIC23_I2C=m CONFIG_SND_SOC_WM8804_I2C=m @@ -1293,6 +1299,9 @@ CONFIG_USB_G_ACM_MS=m CONFIG_USB_G_MULTI=m CONFIG_USB_G_HID=m CONFIG_USB_G_WEBCAM=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_FUSB302=m CONFIG_MMC=y CONFIG_MMC_BLOCK_MINORS=32 CONFIG_MMC_BCM2835_MMC=y @@ -1411,6 +1420,7 @@ CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m CONFIG_VIDEO_CODEC_BCM2835=m CONFIG_VIDEO_ISP_BCM2835=m +CONFIG_CLK_HOG=m CONFIG_COMMON_CLK_SI5351=m CONFIG_CLK_RASPBERRYPI=y CONFIG_MAILBOX=y @@ -1582,6 +1592,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DLM=m CONFIG_UNICODE=m +CONFIG_KEYS_REQUEST_CACHE=y CONFIG_KEY_DH_OPERATIONS=y CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index cd4b34c96e35e..38feb30dfb5f8 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -929,10 +929,6 @@ static void hw_breakpoint_cfi_handler(struct pt_regs *regs) break; } } -#else -static void hw_breakpoint_cfi_handler(struct pt_regs *regs) -{ -} #endif /* @@ -964,9 +960,14 @@ static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, case ARM_ENTRY_SYNC_WATCHPOINT: watchpoint_handler(addr, fsr, regs); break; +#ifdef CONFIG_CFI case ARM_ENTRY_CFI_BREAKPOINT: - hw_breakpoint_cfi_handler(regs); + if (user_mode(regs)) + ret = 1; /* Don't handle userspace BKPT */ + else + hw_breakpoint_cfi_handler(regs); break; +#endif default: ret = 1; /* Unhandled fault. */ } diff --git a/arch/arm/mach-exynos/mcpm-exynos.c b/arch/arm/mach-exynos/mcpm-exynos.c index fd0dbeb93357d..cb7d8a7b14e0d 100644 --- a/arch/arm/mach-exynos/mcpm-exynos.c +++ b/arch/arm/mach-exynos/mcpm-exynos.c @@ -215,7 +215,7 @@ static const struct of_device_id exynos_dt_mcpm_match[] = { {}, }; -static void exynos_mcpm_setup_entry_point(void) +static void exynos_mcpm_setup_entry_point(void *data) { /* * U-Boot SPL is hardcoded to jump to the start of ns_sram_base_addr @@ -228,10 +228,14 @@ static void exynos_mcpm_setup_entry_point(void) __raw_writel(__pa_symbol(mcpm_entry_point), ns_sram_base_addr + 8); } -static struct syscore_ops exynos_mcpm_syscore_ops = { +static const struct syscore_ops exynos_mcpm_syscore_ops = { .resume = exynos_mcpm_setup_entry_point, }; +static struct syscore exynos_mcpm_syscore = { + .ops = &exynos_mcpm_syscore_ops, +}; + static int __init exynos_mcpm_init(void) { struct device_node *node; @@ -300,9 +304,9 @@ static int __init exynos_mcpm_init(void) pmu_raw_writel(value, EXYNOS_COMMON_OPTION(i)); } - exynos_mcpm_setup_entry_point(); + exynos_mcpm_setup_entry_point(NULL); - register_syscore_ops(&exynos_mcpm_syscore_ops); + register_syscore(&exynos_mcpm_syscore); return ret; } diff --git a/arch/arm/mach-exynos/suspend.c b/arch/arm/mach-exynos/suspend.c index 150a1e56dcae9..22d723553f627 100644 --- a/arch/arm/mach-exynos/suspend.c +++ b/arch/arm/mach-exynos/suspend.c @@ -53,9 +53,9 @@ struct exynos_pm_data { void (*pm_prepare)(void); void (*pm_resume_prepare)(void); - void (*pm_resume)(void); - int (*pm_suspend)(void); int (*cpu_suspend)(unsigned long); + + const struct syscore_ops *syscore_ops; }; /* Used only on Exynos542x/5800 */ @@ -376,7 +376,7 @@ static void exynos5420_pm_prepare(void) } -static int exynos_pm_suspend(void) +static int exynos_pm_suspend(void *data) { exynos_pm_central_suspend(); @@ -390,7 +390,7 @@ static int exynos_pm_suspend(void) return 0; } -static int exynos5420_pm_suspend(void) +static int exynos5420_pm_suspend(void *data) { u32 this_cluster; @@ -408,7 +408,7 @@ static int exynos5420_pm_suspend(void) return 0; } -static void exynos_pm_resume(void) +static void exynos_pm_resume(void *data) { u32 cpuid = read_cpuid_part(); @@ -429,7 +429,7 @@ static void exynos_pm_resume(void) exynos_set_delayed_reset_assertion(true); } -static void exynos3250_pm_resume(void) +static void exynos3250_pm_resume(void *data) { u32 cpuid = read_cpuid_part(); @@ -473,7 +473,7 @@ static void exynos5420_prepare_pm_resume(void) } } -static void exynos5420_pm_resume(void) +static void exynos5420_pm_resume(void *data) { unsigned long tmp; @@ -596,41 +596,52 @@ static const struct platform_suspend_ops exynos_suspend_ops = { .valid = suspend_valid_only_mem, }; +static const struct syscore_ops exynos3250_syscore_ops = { + .suspend = exynos_pm_suspend, + .resume = exynos3250_pm_resume, +}; + static const struct exynos_pm_data exynos3250_pm_data = { .wkup_irq = exynos3250_wkup_irq, .wake_disable_mask = ((0xFF << 8) | (0x1F << 1)), - .pm_suspend = exynos_pm_suspend, - .pm_resume = exynos3250_pm_resume, .pm_prepare = exynos3250_pm_prepare, .cpu_suspend = exynos3250_cpu_suspend, + .syscore_ops = &exynos3250_syscore_ops, +}; + +static const struct syscore_ops exynos_syscore_ops = { + .suspend = exynos_pm_suspend, + .resume = exynos_pm_resume, }; static const struct exynos_pm_data exynos4_pm_data = { .wkup_irq = exynos4_wkup_irq, .wake_disable_mask = ((0xFF << 8) | (0x1F << 1)), - .pm_suspend = exynos_pm_suspend, - .pm_resume = exynos_pm_resume, .pm_prepare = exynos_pm_prepare, .cpu_suspend = exynos_cpu_suspend, + .syscore_ops = &exynos_syscore_ops, }; static const struct exynos_pm_data exynos5250_pm_data = { .wkup_irq = exynos5250_wkup_irq, .wake_disable_mask = ((0xFF << 8) | (0x1F << 1)), - .pm_suspend = exynos_pm_suspend, - .pm_resume = exynos_pm_resume, .pm_prepare = exynos_pm_prepare, .cpu_suspend = exynos_cpu_suspend, + .syscore_ops = &exynos_syscore_ops, +}; + +static const struct syscore_ops exynos5420_syscore_ops = { + .resume = exynos5420_pm_resume, + .suspend = exynos5420_pm_suspend, }; static const struct exynos_pm_data exynos5420_pm_data = { .wkup_irq = exynos5250_wkup_irq, .wake_disable_mask = (0x7F << 7) | (0x1F << 1), .pm_resume_prepare = exynos5420_prepare_pm_resume, - .pm_resume = exynos5420_pm_resume, - .pm_suspend = exynos5420_pm_suspend, .pm_prepare = exynos5420_pm_prepare, .cpu_suspend = exynos5420_cpu_suspend, + .syscore_ops = &exynos5420_syscore_ops, }; static const struct of_device_id exynos_pmu_of_device_ids[] __initconst = { @@ -656,7 +667,7 @@ static const struct of_device_id exynos_pmu_of_device_ids[] __initconst = { { /*sentinel*/ }, }; -static struct syscore_ops exynos_pm_syscore_ops; +static struct syscore exynos_pm_syscore; void __init exynos_pm_init(void) { @@ -684,10 +695,9 @@ void __init exynos_pm_init(void) tmp |= pm_data->wake_disable_mask; pmu_raw_writel(tmp, S5P_WAKEUP_MASK); - exynos_pm_syscore_ops.suspend = pm_data->pm_suspend; - exynos_pm_syscore_ops.resume = pm_data->pm_resume; + exynos_pm_syscore.ops = pm_data->syscore_ops; - register_syscore_ops(&exynos_pm_syscore_ops); + register_syscore(&exynos_pm_syscore); suspend_set_ops(&exynos_suspend_ops); /* diff --git a/arch/arm/mach-imx/src.c b/arch/arm/mach-imx/src.c index 59a8e8cc44693..c3c80b4c3d53b 100644 --- a/arch/arm/mach-imx/src.c +++ b/arch/arm/mach-imx/src.c @@ -171,6 +171,7 @@ void __init imx_src_init(void) if (!np) return; src_base = of_iomap(np, 0); + of_node_put(np); WARN_ON(!src_base); /* @@ -195,6 +196,7 @@ void __init imx7_src_init(void) return; src_base = of_iomap(np, 0); + of_node_put(np); if (!src_base) return; @@ -203,6 +205,7 @@ void __init imx7_src_init(void) return; gpc_base = of_iomap(np, 0); + of_node_put(np); if (!gpc_base) return; } diff --git a/arch/arm/mach-lpc32xx/common.c b/arch/arm/mach-lpc32xx/common.c index 304ea61a07160..35ed3569c5a35 100644 --- a/arch/arm/mach-lpc32xx/common.c +++ b/arch/arm/mach-lpc32xx/common.c @@ -106,7 +106,7 @@ void __init lpc32xx_map_io(void) iotable_init(lpc32xx_io_desc, ARRAY_SIZE(lpc32xx_io_desc)); } -static int __init lpc32xx_check_uid(void) +void __init lpc32xx_check_uid(void) { u32 uid[4]; @@ -119,7 +119,4 @@ static int __init lpc32xx_check_uid(void) system_serial_low = uid[0]; system_serial_high = uid[1]; } - - return 1; } -arch_initcall(lpc32xx_check_uid); diff --git a/arch/arm/mach-lpc32xx/common.h b/arch/arm/mach-lpc32xx/common.h index 32f0ad2178077..06b20bea324e1 100644 --- a/arch/arm/mach-lpc32xx/common.h +++ b/arch/arm/mach-lpc32xx/common.h @@ -16,6 +16,8 @@ * Other arch specific structures and functions */ extern void __init lpc32xx_map_io(void); +extern void __init lpc32xx_check_uid(void); +extern void __init lpc32xx_pm_init(void); extern void __init lpc32xx_serial_init(void); /* diff --git a/arch/arm/mach-lpc32xx/phy3250.c b/arch/arm/mach-lpc32xx/phy3250.c index 66701bf432488..ddc6333ca55da 100644 --- a/arch/arm/mach-lpc32xx/phy3250.c +++ b/arch/arm/mach-lpc32xx/phy3250.c @@ -71,6 +71,8 @@ static const struct of_dev_auxdata lpc32xx_auxdata_lookup[] __initconst = { static void __init lpc3250_machine_init(void) { + lpc32xx_check_uid(); + lpc32xx_pm_init(); lpc32xx_serial_init(); of_platform_default_populate(NULL, lpc32xx_auxdata_lookup, NULL); diff --git a/arch/arm/mach-lpc32xx/pm.c b/arch/arm/mach-lpc32xx/pm.c index 2572bd89a5e8d..9b5c5e1462ed3 100644 --- a/arch/arm/mach-lpc32xx/pm.c +++ b/arch/arm/mach-lpc32xx/pm.c @@ -120,7 +120,7 @@ static const struct platform_suspend_ops lpc32xx_pm_ops = { #define EMC_DYN_MEM_CTRL_OFS 0x20 #define EMC_SRMMC (1 << 3) #define EMC_CTRL_REG io_p2v(LPC32XX_EMC_BASE + EMC_DYN_MEM_CTRL_OFS) -static int __init lpc32xx_pm_init(void) +void __init lpc32xx_pm_init(void) { /* * Setup SDRAM self-refresh clock to automatically disable o @@ -129,7 +129,4 @@ static int __init lpc32xx_pm_init(void) __raw_writel(__raw_readl(EMC_CTRL_REG) | EMC_SRMMC, EMC_CTRL_REG); suspend_set_ops(&lpc32xx_pm_ops); - - return 0; } -arch_initcall(lpc32xx_pm_init); diff --git a/arch/arm/mach-npcm/platsmp.c b/arch/arm/mach-npcm/platsmp.c index 41891d3aa1247..4c1fc9983746c 100644 --- a/arch/arm/mach-npcm/platsmp.c +++ b/arch/arm/mach-npcm/platsmp.c @@ -32,6 +32,7 @@ static int npcm7xx_smp_boot_secondary(unsigned int cpu, goto out; } gcr_base = of_iomap(gcr_np, 0); + of_node_put(gcr_np); if (!gcr_base) { pr_err("could not iomap gcr"); ret = -ENOMEM; @@ -60,6 +61,7 @@ static void __init npcm7xx_smp_prepare_cpus(unsigned int max_cpus) return; } scu_base = of_iomap(scu_np, 0); + of_node_put(scu_np); if (!scu_base) { pr_err("could not iomap scu"); return; diff --git a/arch/arm/mach-pxa/generic.h b/arch/arm/mach-pxa/generic.h index c9c2c46ecead0..caad4fca8de3e 100644 --- a/arch/arm/mach-pxa/generic.h +++ b/arch/arm/mach-pxa/generic.h @@ -34,9 +34,9 @@ extern void __init pxa27x_map_io(void); extern void __init pxa3xx_init_irq(void); extern void __init pxa3xx_map_io(void); -extern struct syscore_ops pxa_irq_syscore_ops; -extern struct syscore_ops pxa2xx_mfp_syscore_ops; -extern struct syscore_ops pxa3xx_mfp_syscore_ops; +extern struct syscore pxa_irq_syscore; +extern struct syscore pxa2xx_mfp_syscore; +extern struct syscore pxa3xx_mfp_syscore; void __init pxa_set_ffuart_info(void *info); void __init pxa_set_btuart_info(void *info); diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c index 5bfce8aa4102d..99acebbbf0653 100644 --- a/arch/arm/mach-pxa/irq.c +++ b/arch/arm/mach-pxa/irq.c @@ -178,7 +178,7 @@ void __init pxa_init_irq(int irq_nr, int (*fn)(struct irq_data *, unsigned int)) static unsigned long saved_icmr[MAX_INTERNAL_IRQS/32]; static unsigned long saved_ipr[MAX_INTERNAL_IRQS]; -static int pxa_irq_suspend(void) +static int pxa_irq_suspend(void *data) { int i; @@ -197,7 +197,7 @@ static int pxa_irq_suspend(void) return 0; } -static void pxa_irq_resume(void) +static void pxa_irq_resume(void *data) { int i; @@ -219,11 +219,15 @@ static void pxa_irq_resume(void) #define pxa_irq_resume NULL #endif -struct syscore_ops pxa_irq_syscore_ops = { +static const struct syscore_ops pxa_irq_syscore_ops = { .suspend = pxa_irq_suspend, .resume = pxa_irq_resume, }; +struct syscore pxa_irq_syscore = { + .ops = &pxa_irq_syscore_ops, +}; + #ifdef CONFIG_OF static const struct of_device_id intc_ids[] __initconst = { { .compatible = "marvell,pxa-intc", }, diff --git a/arch/arm/mach-pxa/mfp-pxa2xx.c b/arch/arm/mach-pxa/mfp-pxa2xx.c index f5a3d890f682e..d1347055fbe4f 100644 --- a/arch/arm/mach-pxa/mfp-pxa2xx.c +++ b/arch/arm/mach-pxa/mfp-pxa2xx.c @@ -346,7 +346,7 @@ static unsigned long saved_gpdr[4]; static unsigned long saved_gplr[4]; static unsigned long saved_pgsr[4]; -static int pxa2xx_mfp_suspend(void) +static int pxa2xx_mfp_suspend(void *data) { int i; @@ -385,7 +385,7 @@ static int pxa2xx_mfp_suspend(void) return 0; } -static void pxa2xx_mfp_resume(void) +static void pxa2xx_mfp_resume(void *data) { int i; @@ -404,11 +404,15 @@ static void pxa2xx_mfp_resume(void) #define pxa2xx_mfp_resume NULL #endif -struct syscore_ops pxa2xx_mfp_syscore_ops = { +static const struct syscore_ops pxa2xx_mfp_syscore_ops = { .suspend = pxa2xx_mfp_suspend, .resume = pxa2xx_mfp_resume, }; +struct syscore pxa2xx_mfp_syscore = { + .ops = &pxa2xx_mfp_syscore_ops, +}; + static int __init pxa2xx_mfp_init(void) { int i; diff --git a/arch/arm/mach-pxa/mfp-pxa3xx.c b/arch/arm/mach-pxa/mfp-pxa3xx.c index d16ab7451efef..fe7498fbb62b3 100644 --- a/arch/arm/mach-pxa/mfp-pxa3xx.c +++ b/arch/arm/mach-pxa/mfp-pxa3xx.c @@ -27,13 +27,13 @@ * a pull-down mode if they're an active low chip select, and we're * just entering standby. */ -static int pxa3xx_mfp_suspend(void) +static int pxa3xx_mfp_suspend(void *data) { mfp_config_lpm(); return 0; } -static void pxa3xx_mfp_resume(void) +static void pxa3xx_mfp_resume(void *data) { mfp_config_run(); @@ -49,7 +49,11 @@ static void pxa3xx_mfp_resume(void) #define pxa3xx_mfp_resume NULL #endif -struct syscore_ops pxa3xx_mfp_syscore_ops = { +static const struct syscore_ops pxa3xx_mfp_syscore_ops = { .suspend = pxa3xx_mfp_suspend, .resume = pxa3xx_mfp_resume, }; + +struct syscore pxa3xx_mfp_syscore = { + .ops = &pxa3xx_mfp_syscore_ops, +}; diff --git a/arch/arm/mach-pxa/pxa25x.c b/arch/arm/mach-pxa/pxa25x.c index 03e34841fc004..70509a5998142 100644 --- a/arch/arm/mach-pxa/pxa25x.c +++ b/arch/arm/mach-pxa/pxa25x.c @@ -235,8 +235,8 @@ static int __init pxa25x_init(void) pxa25x_init_pm(); - register_syscore_ops(&pxa_irq_syscore_ops); - register_syscore_ops(&pxa2xx_mfp_syscore_ops); + register_syscore(&pxa_irq_syscore); + register_syscore(&pxa2xx_mfp_syscore); if (!of_have_populated_dt()) { software_node_register(&pxa2xx_gpiochip_node); diff --git a/arch/arm/mach-pxa/pxa27x.c b/arch/arm/mach-pxa/pxa27x.c index f8382477d629e..ff63619790383 100644 --- a/arch/arm/mach-pxa/pxa27x.c +++ b/arch/arm/mach-pxa/pxa27x.c @@ -337,8 +337,8 @@ static int __init pxa27x_init(void) pxa27x_init_pm(); - register_syscore_ops(&pxa_irq_syscore_ops); - register_syscore_ops(&pxa2xx_mfp_syscore_ops); + register_syscore(&pxa_irq_syscore); + register_syscore(&pxa2xx_mfp_syscore); if (!of_have_populated_dt()) { software_node_register(&pxa2xx_gpiochip_node); diff --git a/arch/arm/mach-pxa/pxa3xx.c b/arch/arm/mach-pxa/pxa3xx.c index 1d1e5713464dc..06c578ea658e3 100644 --- a/arch/arm/mach-pxa/pxa3xx.c +++ b/arch/arm/mach-pxa/pxa3xx.c @@ -424,8 +424,8 @@ static int __init pxa3xx_init(void) if (cpu_is_pxa320()) enable_irq_wake(IRQ_WAKEUP1); - register_syscore_ops(&pxa_irq_syscore_ops); - register_syscore_ops(&pxa3xx_mfp_syscore_ops); + register_syscore(&pxa_irq_syscore); + register_syscore(&pxa3xx_mfp_syscore); } return ret; diff --git a/arch/arm/mach-pxa/smemc.c b/arch/arm/mach-pxa/smemc.c index 2d2a321d82f81..fb93a8f28356e 100644 --- a/arch/arm/mach-pxa/smemc.c +++ b/arch/arm/mach-pxa/smemc.c @@ -18,7 +18,7 @@ static unsigned long msc[2]; static unsigned long sxcnfg, memclkcfg; static unsigned long csadrcfg[4]; -static int pxa3xx_smemc_suspend(void) +static int pxa3xx_smemc_suspend(void *data) { msc[0] = __raw_readl(MSC0); msc[1] = __raw_readl(MSC1); @@ -32,7 +32,7 @@ static int pxa3xx_smemc_suspend(void) return 0; } -static void pxa3xx_smemc_resume(void) +static void pxa3xx_smemc_resume(void *data) { __raw_writel(msc[0], MSC0); __raw_writel(msc[1], MSC1); @@ -46,11 +46,15 @@ static void pxa3xx_smemc_resume(void) __raw_writel(0x2, CSMSADRCFG); } -static struct syscore_ops smemc_syscore_ops = { +static const struct syscore_ops smemc_syscore_ops = { .suspend = pxa3xx_smemc_suspend, .resume = pxa3xx_smemc_resume, }; +static struct syscore smemc_syscore = { + .ops = &smemc_syscore_ops, +}; + static int __init smemc_init(void) { if (cpu_is_pxa3xx()) { @@ -64,7 +68,7 @@ static int __init smemc_init(void) */ __raw_writel(0x2, CSMSADRCFG); - register_syscore_ops(&smemc_syscore_ops); + register_syscore(&smemc_syscore); } return 0; diff --git a/arch/arm/mach-s3c/irq-pm-s3c64xx.c b/arch/arm/mach-s3c/irq-pm-s3c64xx.c index 4a1e935bada19..ab726c595001d 100644 --- a/arch/arm/mach-s3c/irq-pm-s3c64xx.c +++ b/arch/arm/mach-s3c/irq-pm-s3c64xx.c @@ -58,7 +58,7 @@ static struct irq_grp_save { static u32 irq_uart_mask[SERIAL_SAMSUNG_UARTS]; -static int s3c64xx_irq_pm_suspend(void) +static int s3c64xx_irq_pm_suspend(void *data) { struct irq_grp_save *grp = eint_grp_save; int i; @@ -79,7 +79,7 @@ static int s3c64xx_irq_pm_suspend(void) return 0; } -static void s3c64xx_irq_pm_resume(void) +static void s3c64xx_irq_pm_resume(void *data) { struct irq_grp_save *grp = eint_grp_save; int i; @@ -100,18 +100,22 @@ static void s3c64xx_irq_pm_resume(void) S3C_PMDBG("%s: IRQ configuration restored\n", __func__); } -static struct syscore_ops s3c64xx_irq_syscore_ops = { +static const struct syscore_ops s3c64xx_irq_syscore_ops = { .suspend = s3c64xx_irq_pm_suspend, .resume = s3c64xx_irq_pm_resume, }; +static struct syscore s3c64xx_irq_syscore = { + .ops = &s3c64xx_irq_syscore_ops, +}; + static __init int s3c64xx_syscore_init(void) { /* Appropriate drivers (pinctrl, uart) handle this when using DT. */ if (of_have_populated_dt() || !soc_is_s3c64xx()) return 0; - register_syscore_ops(&s3c64xx_irq_syscore_ops); + register_syscore(&s3c64xx_irq_syscore); return 0; } diff --git a/arch/arm/mach-s5pv210/pm.c b/arch/arm/mach-s5pv210/pm.c index 6fa70f787df4e..fa270750364cf 100644 --- a/arch/arm/mach-s5pv210/pm.c +++ b/arch/arm/mach-s5pv210/pm.c @@ -195,20 +195,24 @@ static const struct platform_suspend_ops s5pv210_suspend_ops = { /* * Syscore operations used to delay restore of certain registers. */ -static void s5pv210_pm_resume(void) +static void s5pv210_pm_resume(void *data) { s3c_pm_do_restore_core(s5pv210_core_save, ARRAY_SIZE(s5pv210_core_save)); } -static struct syscore_ops s5pv210_pm_syscore_ops = { +static const struct syscore_ops s5pv210_pm_syscore_ops = { .resume = s5pv210_pm_resume, }; +static struct syscore s5pv210_pm_syscore = { + .ops = &s5pv210_pm_syscore_ops, +}; + /* * Initialization entry point. */ void __init s5pv210_pm_init(void) { - register_syscore_ops(&s5pv210_pm_syscore_ops); + register_syscore(&s5pv210_pm_syscore); suspend_set_ops(&s5pv210_suspend_ops); } diff --git a/arch/arm/mach-ux500/cpu-db8500.c b/arch/arm/mach-ux500/cpu-db8500.c index b1a70f203372b..0d7530fb6ad07 100644 --- a/arch/arm/mach-ux500/cpu-db8500.c +++ b/arch/arm/mach-ux500/cpu-db8500.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include @@ -81,7 +81,7 @@ static void __init ux500_init_irq(void) struct resource r; irqchip_init(); - prcmu_early_init(); + db8500_prcmu_early_init(); np = of_find_compatible_node(NULL, NULL, "stericsson,db8500-prcmu"); of_address_to_resource(np, 0, &r); of_node_put(np); @@ -101,7 +101,7 @@ static void ux500_restart(enum reboot_mode mode, const char *cmd) local_irq_disable(); local_fiq_disable(); - prcmu_system_reset(0); + db8500_prcmu_system_reset(0); } static const struct of_device_id u8500_local_bus_nodes[] = { diff --git a/arch/arm/mach-versatile/integrator_ap.c b/arch/arm/mach-versatile/integrator_ap.c index 4bd6712e9f525..ee90d6619d0db 100644 --- a/arch/arm/mach-versatile/integrator_ap.c +++ b/arch/arm/mach-versatile/integrator_ap.c @@ -63,13 +63,13 @@ static void __init ap_map_io(void) #ifdef CONFIG_PM static unsigned long ic_irq_enable; -static int irq_suspend(void) +static int irq_suspend(void *data) { ic_irq_enable = readl(VA_IC_BASE + IRQ_ENABLE); return 0; } -static void irq_resume(void) +static void irq_resume(void *data) { /* disable all irq sources */ cm_clear_irqs(); @@ -83,14 +83,18 @@ static void irq_resume(void) #define irq_resume NULL #endif -static struct syscore_ops irq_syscore_ops = { +static const struct syscore_ops irq_syscore_ops = { .suspend = irq_suspend, .resume = irq_resume, }; +static struct syscore irq_syscore = { + .ops = &irq_syscore_ops, +}; + static int __init irq_syscore_init(void) { - register_syscore_ops(&irq_syscore_ops); + register_syscore(&irq_syscore); return 0; } diff --git a/arch/arm/mm/cache-b15-rac.c b/arch/arm/mm/cache-b15-rac.c index 6f63b90f9e1ac..e7807356dfab0 100644 --- a/arch/arm/mm/cache-b15-rac.c +++ b/arch/arm/mm/cache-b15-rac.c @@ -256,7 +256,7 @@ static int b15_rac_dead_cpu(unsigned int cpu) return 0; } -static int b15_rac_suspend(void) +static int b15_rac_suspend(void *data) { /* Suspend the read-ahead cache oeprations, forcing our cache * implementation to fallback to the regular ARMv7 calls. @@ -271,7 +271,7 @@ static int b15_rac_suspend(void) return 0; } -static void b15_rac_resume(void) +static void b15_rac_resume(void *data) { /* Coming out of a S3 suspend/resume cycle, the read-ahead cache * register RAC_CONFIG0_REG will be restored to its default value, make @@ -282,11 +282,15 @@ static void b15_rac_resume(void) clear_bit(RAC_SUSPENDED, &b15_rac_flags); } -static struct syscore_ops b15_rac_syscore_ops = { +static const struct syscore_ops b15_rac_syscore_ops = { .suspend = b15_rac_suspend, .resume = b15_rac_resume, }; +static struct syscore b15_rac_syscore = { + .ops = &b15_rac_syscore_ops, +}; + static int __init b15_rac_init(void) { struct device_node *dn, *cpu_dn; @@ -347,7 +351,7 @@ static int __init b15_rac_init(void) } if (IS_ENABLED(CONFIG_PM_SLEEP)) - register_syscore_ops(&b15_rac_syscore_ops); + register_syscore(&b15_rac_syscore); spin_lock(&rac_lock); reg = __raw_readl(b15_rac_base + RAC_CONFIG0_REG); diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index ed4330cc3f4e6..577103c37d1bf 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -190,7 +190,8 @@ __do_kernel_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr, /* * Something tried to access memory that isn't in our memory map.. - * User mode accesses just cause a SIGSEGV + * User mode accesses just cause a SIGSEGV. Ensure interrupts are enabled + * for preempt RT. */ static void __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, @@ -198,13 +199,19 @@ __do_user_fault(unsigned long addr, unsigned int fsr, unsigned int sig, { struct task_struct *tsk = current; + local_irq_enable(); + #ifdef CONFIG_DEBUG_USER if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { pr_err("8<--- cut here ---\n"); pr_err("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", tsk->comm, sig, addr, fsr); - show_pte(KERN_ERR, tsk->mm, addr); + if (likely(addr < TASK_SIZE)) { + mmap_write_lock(tsk->mm); + show_pte(KERN_ERR, tsk->mm, addr); + mmap_write_unlock(tsk->mm); + } show_regs(regs); } #endif @@ -268,6 +275,7 @@ do_kernel_address_page_fault(struct mm_struct *mm, unsigned long addr, * should not be faulting in kernel space, which includes the * vector/khelper page. Handle the branch predictor hardening * while interrupts are still disabled, then send a SIGSEGV. + * Note that __do_user_fault() will enable interrupts. */ harden_branch_predictor(); __do_user_fault(addr, fsr, SIGSEGV, SEGV_MAPERR, regs); @@ -645,10 +653,21 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); - show_pte(KERN_ALERT, current->mm, addr); + if (likely(user_mode(regs))) { + if (addr < TASK_SIZE) { + mmap_write_lock(current->mm); + show_pte(KERN_ALERT, current->mm, addr); + mmap_write_unlock(current->mm); + } + } else { + show_pte(KERN_ALERT, current->mm, addr); + } arm_notify_die("", regs, inf->sig, inf->code, (void __user *)addr, fsr, 0); @@ -675,6 +694,9 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; + if (likely(user_mode(regs))) + local_irq_enable(); + pr_alert("8<--- cut here ---\n"); pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 6c384df3ad5c7..77e122e2bd23c 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2113,6 +2113,8 @@ config ARM64_BTI_KERNEL depends on !CC_IS_GCC || GCC_VERSION >= 100100 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 depends on !CC_IS_GCC + # https://github.com/llvm/llvm-project/issues/215547 + depends on !CC_IS_CLANG || CLANG_VERSION < 210000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS) help Build the kernel with Branch Target Identification annotations diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi index 4bc6c1ef2cde4..f958bdbb0d333 100644 --- a/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi +++ b/arch/arm64/boot/dts/allwinner/sun50i-a64-pinephone.dtsi @@ -230,8 +230,8 @@ interrupts = <7 5 IRQ_TYPE_EDGE_RISING>; /* PH5 */ vdd-supply = <®_dldo1>; vddio-supply = <®_dldo1>; - mount-matrix = "0", "1", "0", - "-1", "0", "0", + mount-matrix = "0", "-1", "0", + "1", "0", "0", "0", "0", "1"; }; }; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts index 9611775b81eee..7c8a8529dbc5f 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts +++ b/arch/arm64/boot/dts/amlogic/meson-axg-s400.dts @@ -432,6 +432,10 @@ }; }; +&mipi_pcie_analog_dphy { + status = "okay"; +}; + &pdm { pinctrl-0 = <&pdm_dclk_a14_pins>, <&pdm_din0_pins>, <&pdm_din1_pins>, <&pdm_din2_pins>, <&pdm_din3_pins>; diff --git a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi index 3058b60338dbf..e9c1aae95fe7c 100644 --- a/arch/arm64/boot/dts/amlogic/meson-axg.dtsi +++ b/arch/arm64/boot/dts/amlogic/meson-axg.dtsi @@ -481,7 +481,8 @@ "nand_ale", "nand_cle", "nand_wen_clk", - "nand_ren_wr"; + "nand_ren_wr", + "nand_rb0"; function = "nand"; input-enable; bias-pull-up; diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts index c8908f6b3143b..f3b16c868e2be 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts +++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi-5-b.dts @@ -176,6 +176,7 @@ rp1_target: &pcie2 { phy1: ethernet-phy@1 { reg = <0x1>; brcm,powerdown-enable; + led-modes = <0 8>; }; }; diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi index b25e761dd23ff..efbc2c5b9aee4 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi @@ -99,13 +99,19 @@ pio: &rp1_pio { dmas = <&rp1_dma (RP1_DMA_PIO_CH0_TX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH0_RX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH1_TX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH1_RX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH2_TX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH2_RX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH3_TX | DMA_FLAG_HEAVY)>, - <&rp1_dma (RP1_DMA_PIO_CH3_RX | DMA_FLAG_HEAVY)>; + <&rp1_dma (RP1_DMA_PIO_CH0_RX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH1_TX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH1_RX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH2_TX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH2_RX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH3_TX | DMA_FLAG_HEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH3_RX | DMA_FLAG_HEAVY)>, + <&rp1_dma RP1_DMA_PIO_CH0_TX>, <&rp1_dma RP1_DMA_PIO_CH0_RX>, + <&rp1_dma RP1_DMA_PIO_CH1_TX>, <&rp1_dma RP1_DMA_PIO_CH1_RX>, + <&rp1_dma RP1_DMA_PIO_CH2_TX>, <&rp1_dma RP1_DMA_PIO_CH2_RX>, + <&rp1_dma RP1_DMA_PIO_CH3_TX>, <&rp1_dma RP1_DMA_PIO_CH3_RX>; + dma-names = "tx0", "rx0", "tx1", "rx1", "tx2", "rx2", "tx3", "rx3", + "tx0l", "rx0l", "tx1l", "rx1l", "tx2l", "rx2l", "tx3l", "rx3l"; status = "okay"; }; diff --git a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi index 0e7570e727c5a..e8e91318e8dec 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2712.dtsi @@ -587,8 +587,6 @@ , , - ; }; diff --git a/arch/arm64/boot/dts/broadcom/rp1.dtsi b/arch/arm64/boot/dts/broadcom/rp1.dtsi index 4c2a06cb85e86..7141dfcff4e70 100644 --- a/arch/arm64/boot/dts/broadcom/rp1.dtsi +++ b/arch/arm64/boot/dts/broadcom/rp1.dtsi @@ -1126,7 +1126,6 @@ snps,dis_rxdet_inp3_quirk; snps,enhanced-nak-fs-quirk; snps,parkmode-disable-ss-quirk; - snps,parkmode-disable-hs-quirk; snps,parkmode-disable-fsls-quirk; snps,tx-max-burst = /bits/ 8 <8>; snps,tx-thr-num-pkt = /bits/ 8 <2>; @@ -1143,7 +1142,6 @@ snps,dis_rxdet_inp3_quirk; snps,enhanced-nak-fs-quirk; snps,parkmode-disable-ss-quirk; - snps,parkmode-disable-hs-quirk; snps,parkmode-disable-fsls-quirk; snps,tx-max-burst = /bits/ 8 <8>; snps,tx-thr-num-pkt = /bits/ 8 <2>; diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi index c32a6947ae9c4..5ddb90be54608 100644 --- a/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi +++ b/arch/arm64/boot/dts/freescale/imx8-ss-audio.dtsi @@ -124,10 +124,10 @@ audio_subsys: bus@59000000 { compatible = "fsl,imx8qm-asrc"; reg = <0x59000000 0x10000>; interrupts = ; - clocks = <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&asrc0_lpcg IMX_LPCG_CLK_0>, - <&aud_pll_div0_lpcg IMX_LPCG_CLK_4>, - <&aud_pll_div1_lpcg IMX_LPCG_CLK_4>, + clocks = <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&asrc0_lpcg IMX_LPCG_CLK_4>, + <&aud_pll_div0_lpcg IMX_LPCG_CLK_0>, + <&aud_pll_div1_lpcg IMX_LPCG_CLK_0>, <&acm IMX_ADMA_ACM_AUD_CLK0_SEL>, <&acm IMX_ADMA_ACM_AUD_CLK1_SEL>, <&clk_dummy>, diff --git a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi index c79b1df339db1..f881912cde460 100644 --- a/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi +++ b/arch/arm64/boot/dts/freescale/imx93-kontron-osm-s.dtsi @@ -15,9 +15,9 @@ rtc1 = &bbnsm_rtc; }; - memory@40000000 { + memory@80000000 { device_type = "memory"; - reg = <0x0 0x40000000 0 0x80000000>; + reg = <0x0 0x80000000 0 0x40000000>; }; chosen { diff --git a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts index f4d73c8b1a6d3..bd00e7df87c67 100644 --- a/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts +++ b/arch/arm64/boot/dts/marvell/armada-3720-turris-mox.dts @@ -290,7 +290,8 @@ &usb3 { status = "okay"; - phys = <&comphy2 0>; + phys = <&usb2_utmi_otg_phy>, <&comphy2 0>; + phy-names = "usb2-phy", "usb3-phy"; }; &mdio { diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi index 1399342f23e1c..def60103179ff 100644 --- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi @@ -3177,6 +3177,8 @@ , , + ; interrupt-parent = <&gic>; always-on; diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi index 5b245977d6a03..345587fa1d9ad 100644 --- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi @@ -5345,7 +5345,7 @@ #size-cells = <0>; cpu0_0: cpu@0 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00000>; @@ -5364,7 +5364,7 @@ }; cpu0_1: cpu@100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00100>; @@ -5383,7 +5383,7 @@ }; cpu0_2: cpu@200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00200>; @@ -5402,7 +5402,7 @@ }; cpu0_3: cpu@300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x00300>; @@ -5421,7 +5421,7 @@ }; cpu1_0: cpu@10000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10000>; @@ -5440,7 +5440,7 @@ }; cpu1_1: cpu@10100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10100>; @@ -5459,7 +5459,7 @@ }; cpu1_2: cpu@10200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10200>; @@ -5478,7 +5478,7 @@ }; cpu1_3: cpu@10300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x10300>; @@ -5497,7 +5497,7 @@ }; cpu2_0: cpu@20000 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20000>; @@ -5516,7 +5516,7 @@ }; cpu2_1: cpu@20100 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20100>; @@ -5535,7 +5535,7 @@ }; cpu2_2: cpu@20200 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20200>; @@ -5554,7 +5554,7 @@ }; cpu2_3: cpu@20300 { - compatible = "arm,cortex-a78"; + compatible = "arm,cortex-a78ae"; device_type = "cpu"; reg = <0x20300>; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 872a69553e3c8..b16d380209a50 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -50,7 +50,7 @@ }; gpcdma: dma-controller@8400000 { - compatible = "nvidia,tegra264-gpcdma", "nvidia,tegra186-gpcdma"; + compatible = "nvidia,tegra264-gpcdma"; reg = <0x0 0x08400000 0x0 0x210000>; interrupts = , , diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile index 296688f7cb265..dffdb219f33dc 100644 --- a/arch/arm64/boot/dts/qcom/Makefile +++ b/arch/arm64/boot/dts/qcom/Makefile @@ -344,6 +344,8 @@ x1e80100-qcp-el2-dtbs := x1e80100-qcp.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1e80100-qcp.dtb x1e80100-qcp-el2.dtb x1p42100-asus-zenbook-a14-el2-dtbs := x1p42100-asus-zenbook-a14.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-asus-zenbook-a14.dtb x1p42100-asus-zenbook-a14-el2.dtb +x1p42100-asus-zenbook-a14-lcd-el2-dtbs := x1p42100-asus-zenbook-a14-lcd.dtb x1-el2.dtbo +dtb-$(CONFIG_ARCH_QCOM) += x1p42100-asus-zenbook-a14-lcd.dtb x1p42100-asus-zenbook-a14-lcd-el2.dtb x1p42100-crd-el2-dtbs := x1p42100-crd.dtb x1-el2.dtbo dtb-$(CONFIG_ARCH_QCOM) += x1p42100-crd.dtb x1p42100-crd-el2.dtb x1p42100-hp-omnibook-x14-el2-dtbs := x1p42100-hp-omnibook-x14.dtb x1-el2.dtbo diff --git a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi index 1aead50b8920b..4de7c0abb25a1 100644 --- a/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa-iot-som.dtsi @@ -3,8 +3,8 @@ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" #include #include diff --git a/arch/arm64/boot/dts/qcom/x1e80100-pmics.dtsi b/arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi similarity index 100% rename from arch/arm64/boot/dts/qcom/x1e80100-pmics.dtsi rename to arch/arm64/boot/dts/qcom/hamoa-pmics.dtsi diff --git a/arch/arm64/boot/dts/qcom/x1e80100.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi similarity index 99% rename from arch/arm64/boot/dts/qcom/x1e80100.dtsi rename to arch/arm64/boot/dts/qcom/hamoa.dtsi index ee9c12600f95a..6a456b3694fc9 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100.dtsi +++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi @@ -2826,7 +2826,7 @@ reg = <0 0x00fd3000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&rpmhcc RPMH_CXO_CLK>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>; @@ -2968,7 +2968,7 @@ reg = <0 0x00fde000 0 0x154>; #phy-cells = <0>; - clocks = <&tcsr TCSR_USB2_1_CLKREF_EN>; + clocks = <&tcsr TCSR_USB2_2_CLKREF_EN>; clock-names = "ref"; resets = <&gcc GCC_QUSB2PHY_TERT_BCR>; @@ -5522,18 +5522,18 @@ mdss_dp0_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - opp-270000000 { opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; + required-opps = <&rpmhpd_opp_low_svs>; }; opp-540000000 { opp-hz = /bits/ 64 <540000000>; + required-opps = <&rpmhpd_opp_svs>; + }; + + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; required-opps = <&rpmhpd_opp_svs_l1>; }; @@ -5574,7 +5574,7 @@ <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>, <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>; - operating-points-v2 = <&mdss_dp1_opp_table>; + operating-points-v2 = <&mdss_dp0_opp_table>; power-domains = <&rpmhpd RPMHPD_MMCX>; @@ -5606,30 +5606,6 @@ }; }; }; - - mdss_dp1_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; mdss_dp2: displayport-controller@ae9a000 { @@ -5662,7 +5638,7 @@ <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>, <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>; - operating-points-v2 = <&mdss_dp2_opp_table>; + operating-points-v2 = <&mdss_dp0_opp_table>; power-domains = <&rpmhpd RPMHPD_MMCX>; @@ -5693,30 +5669,6 @@ }; }; }; - - mdss_dp2_opp_table: opp-table { - compatible = "operating-points-v2"; - - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - - opp-270000000 { - opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; - }; - - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; - }; - - opp-810000000 { - opp-hz = /bits/ 64 <810000000>; - required-opps = <&rpmhpd_opp_nom>; - }; - }; }; mdss_dp3: displayport-controller@aea0000 { @@ -5779,19 +5731,14 @@ mdss_dp3_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; - required-opps = <&rpmhpd_opp_low_svs>; - }; - opp-270000000 { opp-hz = /bits/ 64 <270000000>; - required-opps = <&rpmhpd_opp_svs>; + required-opps = <&rpmhpd_opp_low_svs>; }; - opp-540000000 { - opp-hz = /bits/ 64 <540000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + opp-594000000 { + opp-hz = /bits/ 64 <594000000>; + required-opps = <&rpmhpd_opp_svs>; }; opp-810000000 { diff --git a/arch/arm64/boot/dts/qcom/ipq5018.dtsi b/arch/arm64/boot/dts/qcom/ipq5018.dtsi index f024b3cba33f6..40b5de02d5c5d 100644 --- a/arch/arm64/boot/dts/qcom/ipq5018.dtsi +++ b/arch/arm64/boot/dts/qcom/ipq5018.dtsi @@ -229,6 +229,9 @@ compatible = "ethernet-phy-id004d.d0c0"; reg = <7>; + clocks = <&gcc GCC_GEPHY_RX_CLK>, + <&gcc GCC_GEPHY_TX_CLK>; + clock-names = "rx", "tx"; resets = <&gcc GCC_GEPHY_MISC_ARES>; }; }; @@ -256,7 +259,7 @@ "sys"; #clock-cells = <1>; assigned-clocks = <&cmn_pll IPQ5018_CMN_PLL_CLK>; - assigned-clock-rates-u64 = /bits/ 64 <9600000000>; + assigned-clock-rates-u64 = /bits/ 64 <4800000000>; }; qfprom: qfprom@a0000 { diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi index 5c00d2e89fb8c..0861e4ead93eb 100644 --- a/arch/arm64/boot/dts/qcom/lemans.dtsi +++ b/arch/arm64/boot/dts/qcom/lemans.dtsi @@ -518,90 +518,18 @@ }; }; - aggre1_noc: interconnect-aggre1-noc { - compatible = "qcom,sa8775p-aggre1-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - aggre2_noc: interconnect-aggre2-noc { - compatible = "qcom,sa8775p-aggre2-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - clk_virt: interconnect-clk-virt { compatible = "qcom,sa8775p-clk-virt"; #interconnect-cells = <2>; qcom,bcm-voters = <&apps_bcm_voter>; }; - config_noc: interconnect-config-noc { - compatible = "qcom,sa8775p-config-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - dc_noc: interconnect-dc-noc { - compatible = "qcom,sa8775p-dc-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - gem_noc: interconnect-gem-noc { - compatible = "qcom,sa8775p-gem-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - gpdsp_anoc: interconnect-gpdsp-anoc { - compatible = "qcom,sa8775p-gpdsp-anoc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - lpass_ag_noc: interconnect-lpass-ag-noc { - compatible = "qcom,sa8775p-lpass-ag-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - mc_virt: interconnect-mc-virt { compatible = "qcom,sa8775p-mc-virt"; #interconnect-cells = <2>; qcom,bcm-voters = <&apps_bcm_voter>; }; - mmss_noc: interconnect-mmss-noc { - compatible = "qcom,sa8775p-mmss-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - nspa_noc: interconnect-nspa-noc { - compatible = "qcom,sa8775p-nspa-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - nspb_noc: interconnect-nspb-noc { - compatible = "qcom,sa8775p-nspb-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - pcie_anoc: interconnect-pcie-anoc { - compatible = "qcom,sa8775p-pcie-anoc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - - system_noc: interconnect-system-noc { - compatible = "qcom,sa8775p-system-noc"; - #interconnect-cells = <2>; - qcom,bcm-voters = <&apps_bcm_voter>; - }; - /* Will be updated by the bootloader. */ memory@80000000 { device_type = "memory"; @@ -2689,6 +2617,408 @@ reg = <0 0x010d2000 0 0x1000>; }; + config_noc: interconnect@14c0000 { + compatible = "qcom,sa8775p-config-noc"; + reg = <0x0 0x014c0000 0x0 0x13080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + system_noc: interconnect@1680000 { + compatible = "qcom,sa8775p-system-noc"; + reg = <0x0 0x01680000 0x0 0x15080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + aggre1_noc: interconnect@16c0000 { + compatible = "qcom,sa8775p-aggre1-noc"; + reg = <0x0 0x016c0000 0x0 0x18080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + clocks = <&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>, + <&gcc GCC_AGGRE_NOC_QUPV3_AXI_CLK>, + <&gcc GCC_AGGRE_USB2_PRIM_AXI_CLK>, + <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>, + <&gcc GCC_AGGRE_USB3_SEC_AXI_CLK>; + }; + + aggre2_noc: interconnect@1700000 { + compatible = "qcom,sa8775p-aggre2-noc"; + reg = <0x0 0x01700000 0x0 0x1b080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + clocks = <&gcc GCC_AGGRE_UFS_CARD_AXI_CLK>, + <&rpmhcc RPMH_IPA_CLK>; + }; + + pcie_anoc: interconnect@1760000 { + compatible = "qcom,sa8775p-pcie-anoc"; + reg = <0x0 0x01760000 0x0 0xc080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + gpdsp_anoc: interconnect@1780000 { + compatible = "qcom,sa8775p-gpdsp-anoc"; + reg = <0x0 0x01780000 0x0 0xe080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + mmss_noc: interconnect@17a0000 { + compatible = "qcom,sa8775p-mmss-noc"; + reg = <0x0 0x017a0000 0x0 0x40000>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + pcie0: pcie@1c00000 { + compatible = "qcom,pcie-sa8775p"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf20>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x4000>, + <0x0 0x40100000 0x0 0x100000>, + <0x0 0x01c03000 0x0 0x1000>; + reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; + device_type = "pci"; + + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, + <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <0>; + num-lanes = <2>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, + <0x100 &pcie_smmu 0x0001 0x1>; + + resets = <&gcc GCC_PCIE_0_BCR>, + <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc PCIE_0_GDSC>; + + phys = <&pcie0_phy>; + phy-names = "pciephy"; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55>; + + status = "disabled"; + + pcieport0: pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie0_ep: pcie-ep@1c00000 { + compatible = "qcom,sa8775p-pcie-ep"; + reg = <0x0 0x01c00000 0x0 0x3000>, + <0x0 0x40000000 0x0 0xf20>, + <0x0 0x40000f20 0x0 0xa8>, + <0x0 0x40001000 0x0 0x4000>, + <0x0 0x40200000 0x0 0x1fe00000>, + <0x0 0x01c03000 0x0 0x1000>, + <0x0 0x40005000 0x0 0x2000>; + reg-names = "parf", "dbi", "elbi", "atu", "addr_space", + "mmio", "dma"; + + clocks = <&gcc GCC_PCIE_0_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_AXI_CLK>, + <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + interrupts = , + , + ; + + interrupt-names = "global", "doorbell", "dma"; + + interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + dma-coherent; + iommus = <&pcie_smmu 0x0000 0x7f>; + resets = <&gcc GCC_PCIE_0_BCR>; + reset-names = "core"; + power-domains = <&gcc PCIE_0_GDSC>; + phys = <&pcie0_phy>; + phy-names = "pciephy"; + num-lanes = <2>; + linux,pci-domain = <0>; + + status = "disabled"; + }; + + pcie0_phy: phy@1c04000 { + compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy"; + reg = <0x0 0x1c04000 0x0 0x2000>; + + clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>, + <&gcc GCC_PCIE_0_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_0_PIPE_CLK>, + <&gcc GCC_PCIE_0_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_0_PHY_BCR>; + reset-names = "phy"; + + #clock-cells = <0>; + clock-output-names = "pcie_0_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + pcie1: pcie@1c10000 { + compatible = "qcom,pcie-sa8775p"; + reg = <0x0 0x01c10000 0x0 0x3000>, + <0x0 0x60000000 0x0 0xf20>, + <0x0 0x60000f20 0x0 0xa8>, + <0x0 0x60001000 0x0 0x4000>, + <0x0 0x60100000 0x0 0x100000>, + <0x0 0x01c13000 0x0 0x1000>; + reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; + device_type = "pci"; + + #address-cells = <3>; + #size-cells = <2>; + ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>, + <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>; + bus-range = <0x00 0xff>; + + dma-coherent; + + linux,pci-domain = <1>; + num-lanes = <4>; + + interrupts = , + , + , + , + , + , + , + , + ; + interrupt-names = "msi0", + "msi1", + "msi2", + "msi3", + "msi4", + "msi5", + "msi6", + "msi7", + "global"; + #interrupt-cells = <1>; + interrupt-map-mask = <0 0 0 0x7>; + interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, + <0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>; + + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; + assigned-clock-rates = <19200000>; + + interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, + <0x100 &pcie_smmu 0x0081 0x1>; + + resets = <&gcc GCC_PCIE_1_BCR>, + <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; + reset-names = "pci", + "link_down"; + + power-domains = <&gcc PCIE_1_GDSC>; + + phys = <&pcie1_phy>; + phy-names = "pciephy"; + + eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; + eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; + + status = "disabled"; + + pcie@0 { + device_type = "pci"; + reg = <0x0 0x0 0x0 0x0 0x0>; + bus-range = <0x01 0xff>; + + #address-cells = <3>; + #size-cells = <2>; + ranges; + }; + }; + + pcie1_ep: pcie-ep@1c10000 { + compatible = "qcom,sa8775p-pcie-ep"; + reg = <0x0 0x01c10000 0x0 0x3000>, + <0x0 0x60000000 0x0 0xf20>, + <0x0 0x60000f20 0x0 0xa8>, + <0x0 0x60001000 0x0 0x4000>, + <0x0 0x60200000 0x0 0x1fe00000>, + <0x0 0x01c13000 0x0 0x1000>, + <0x0 0x60005000 0x0 0x2000>; + reg-names = "parf", "dbi", "elbi", "atu", "addr_space", + "mmio", "dma"; + + clocks = <&gcc GCC_PCIE_1_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_AXI_CLK>, + <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; + + clock-names = "aux", + "cfg", + "bus_master", + "bus_slave", + "slave_q2a"; + + interrupts = , + , + ; + + interrupt-names = "global", "doorbell", "dma"; + + interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, + <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; + interconnect-names = "pcie-mem", "cpu-pcie"; + + dma-coherent; + iommus = <&pcie_smmu 0x80 0x7f>; + resets = <&gcc GCC_PCIE_1_BCR>; + reset-names = "core"; + power-domains = <&gcc PCIE_1_GDSC>; + phys = <&pcie1_phy>; + phy-names = "pciephy"; + num-lanes = <4>; + linux,pci-domain = <1>; + + status = "disabled"; + }; + + pcie1_phy: phy@1c14000 { + compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy"; + reg = <0x0 0x1c14000 0x0 0x4000>; + + clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>, + <&gcc GCC_PCIE_1_CFG_AHB_CLK>, + <&gcc GCC_PCIE_CLKREF_EN>, + <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>, + <&gcc GCC_PCIE_1_PIPE_CLK>, + <&gcc GCC_PCIE_1_PIPEDIV2_CLK>; + clock-names = "aux", + "cfg_ahb", + "ref", + "rchng", + "pipe", + "pipediv2"; + + assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>; + assigned-clock-rates = <100000000>; + + resets = <&gcc GCC_PCIE_1_PHY_BCR>; + reset-names = "phy"; + + #clock-cells = <0>; + clock-output-names = "pcie_1_pipe_clk"; + + #phy-cells = <0>; + + status = "disabled"; + }; + ufs_mem_hc: ufshc@1d84000 { compatible = "qcom,sa8775p-ufshc", "qcom,ufshc", "jedec,ufs-2.0"; reg = <0x0 0x01d84000 0x0 0x3000>; @@ -2773,6 +3103,25 @@ <&apps_smmu 0x481 0x00>; }; + crypto: crypto@1dfa000 { + compatible = "qcom,sa8775p-qce", "qcom,sm8150-qce", "qcom,qce"; + reg = <0x0 0x01dfa000 0x0 0x6000>; + dmas = <&cryptobam 4>, <&cryptobam 5>; + dma-names = "rx", "tx"; + iommus = <&apps_smmu 0x480 0x0>, + <&apps_smmu 0x481 0x0>; + interconnects = <&aggre2_noc MASTER_CRYPTO_CORE0 QCOM_ICC_TAG_ALWAYS + &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>; + interconnect-names = "memory"; + }; + + lpass_ag_noc: interconnect@3c40000 { + compatible = "qcom,sa8775p-lpass-ag-noc"; + reg = <0x0 0x03c40000 0x0 0x17200>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + ctcu@4001000 { compatible = "qcom,sa8775p-ctcu"; reg = <0x0 0x04001000 0x0 0x1000>; @@ -3905,6 +4254,32 @@ status = "disabled"; }; + usb_1_hsphy: phy@88e6000 { + compatible = "qcom,sa8775p-usb-hs-phy", + "qcom,usb-snps-hs-5nm-phy"; + reg = <0 0x088e6000 0 0x120>; + clocks = <&gcc GCC_USB_CLKREF_EN>; + clock-names = "ref"; + resets = <&gcc GCC_USB2_PHY_SEC_BCR>; + + #phy-cells = <0>; + + status = "disabled"; + }; + + usb_2_hsphy: phy@88e7000 { + compatible = "qcom,sa8775p-usb-hs-phy", + "qcom,usb-snps-hs-5nm-phy"; + reg = <0 0x088e7000 0 0x120>; + clocks = <&gcc GCC_USB_CLKREF_EN>; + clock-names = "ref"; + resets = <&gcc GCC_USB3_PHY_TERT_BCR>; + + #phy-cells = <0>; + + status = "disabled"; + }; + usb_0_qmpphy: phy@88e8000 { compatible = "qcom,sa8775p-qmp-usb3-uni-phy"; reg = <0 0x088e8000 0 0x2000>; @@ -3929,6 +4304,50 @@ status = "disabled"; }; + usb_1_qmpphy: phy@88ea000 { + compatible = "qcom,sa8775p-qmp-usb3-uni-phy"; + reg = <0 0x088ea000 0 0x2000>; + + clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, + <&gcc GCC_USB_CLKREF_EN>, + <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, + <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; + clock-names = "aux", "ref", "com_aux", "pipe"; + + resets = <&gcc GCC_USB3_PHY_SEC_BCR>, + <&gcc GCC_USB3PHY_PHY_SEC_BCR>; + reset-names = "phy", "phy_phy"; + + power-domains = <&gcc USB30_SEC_GDSC>; + + #clock-cells = <0>; + clock-output-names = "usb3_sec_phy_pipe_clk_src"; + + #phy-cells = <0>; + + status = "disabled"; + }; + + refgen: regulator@891c000 { + compatible = "qcom,sa8775p-refgen-regulator", + "qcom,sm8250-refgen-regulator"; + reg = <0x0 0x0891c000 0x0 0x84>; + }; + + dc_noc: interconnect@90e0000 { + compatible = "qcom,sa8775p-dc-noc"; + reg = <0x0 0x090e0000 0x0 0x5080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + + gem_noc: interconnect@9100000 { + compatible = "qcom,sa8775p-gem-noc"; + reg = <0x0 0x09100000 0x0 0xf6080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + usb_0: usb@a600000 { compatible = "qcom,sa8775p-dwc3", "qcom,snps-dwc3"; reg = <0 0x0a600000 0 0xfc100>; @@ -3977,43 +4396,6 @@ status = "disabled"; }; - usb_1_hsphy: phy@88e6000 { - compatible = "qcom,sa8775p-usb-hs-phy", - "qcom,usb-snps-hs-5nm-phy"; - reg = <0 0x088e6000 0 0x120>; - clocks = <&gcc GCC_USB_CLKREF_EN>; - clock-names = "ref"; - resets = <&gcc GCC_USB2_PHY_SEC_BCR>; - - #phy-cells = <0>; - - status = "disabled"; - }; - - usb_1_qmpphy: phy@88ea000 { - compatible = "qcom,sa8775p-qmp-usb3-uni-phy"; - reg = <0 0x088ea000 0 0x2000>; - - clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>, - <&gcc GCC_USB_CLKREF_EN>, - <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>, - <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; - clock-names = "aux", "ref", "com_aux", "pipe"; - - resets = <&gcc GCC_USB3_PHY_SEC_BCR>, - <&gcc GCC_USB3PHY_PHY_SEC_BCR>; - reset-names = "phy", "phy_phy"; - - power-domains = <&gcc USB30_SEC_GDSC>; - - #clock-cells = <0>; - clock-output-names = "usb3_sec_phy_pipe_clk_src"; - - #phy-cells = <0>; - - status = "disabled"; - }; - usb_1: usb@a800000 { compatible = "qcom,sa8775p-dwc3", "qcom,snps-dwc3"; reg = <0 0x0a800000 0 0xfc100>; @@ -4062,19 +4444,6 @@ status = "disabled"; }; - usb_2_hsphy: phy@88e7000 { - compatible = "qcom,sa8775p-usb-hs-phy", - "qcom,usb-snps-hs-5nm-phy"; - reg = <0 0x088e7000 0 0x120>; - clocks = <&gcc GCC_USB_CLKREF_EN>; - clock-names = "ref"; - resets = <&gcc GCC_USB3_PHY_TERT_BCR>; - - #phy-cells = <0>; - - status = "disabled"; - }; - usb_2: usb@a400000 { compatible = "qcom,sa8775p-dwc3", "qcom,snps-dwc3"; reg = <0 0x0a400000 0 0xfc100>; @@ -4904,6 +5273,8 @@ operating-points-v2 = <&mdss_dsi_opp_table>; power-domains = <&rpmhpd SA8775P_MMCX>; + refgen-supply = <&refgen>; + #address-cells = <1>; #size-cells = <0>; @@ -4986,6 +5357,8 @@ operating-points-v2 = <&mdss_dsi_opp_table>; power-domains = <&rpmhpd SA8775P_MMCX>; + refgen-supply = <&refgen>; + #address-cells = <1>; #size-cells = <0>; @@ -5146,8 +5519,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -5234,8 +5607,8 @@ dp1_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -6884,6 +7257,13 @@ status = "disabled"; }; + nspa_noc: interconnect@260c0000 { + compatible = "qcom,sa8775p-nspa-noc"; + reg = <0x0 0x260c0000 0x0 0x16080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + remoteproc_cdsp0: remoteproc@26300000 { compatible = "qcom,sa8775p-cdsp0-pas"; reg = <0x0 0x26300000 0x0 0x10000>; @@ -7016,6 +7396,13 @@ }; }; + nspb_noc: interconnect@2a0c0000 { + compatible = "qcom,sa8775p-nspb-noc"; + reg = <0x0 0x2a0c0000 0x0 0x16080>; + #interconnect-cells = <2>; + qcom,bcm-voters = <&apps_bcm_voter>; + }; + remoteproc_cdsp1: remoteproc@2a300000 { compatible = "qcom,sa8775p-cdsp1-pas"; reg = <0x0 0x2A300000 0x0 0x10000>; @@ -8264,350 +8651,4 @@ , ; }; - - pcie0: pcie@1c00000 { - compatible = "qcom,pcie-sa8775p"; - reg = <0x0 0x01c00000 0x0 0x3000>, - <0x0 0x40000000 0x0 0xf20>, - <0x0 0x40000f20 0x0 0xa8>, - <0x0 0x40001000 0x0 0x4000>, - <0x0 0x40100000 0x0 0x100000>, - <0x0 0x01c03000 0x0 0x1000>; - reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; - device_type = "pci"; - - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>, - <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>; - bus-range = <0x00 0xff>; - - dma-coherent; - - linux,pci-domain = <0>; - num-lanes = <2>; - - interrupts = , - , - , - , - , - , - , - , - ; - interrupt-names = "msi0", - "msi1", - "msi2", - "msi3", - "msi4", - "msi5", - "msi6", - "msi7", - "global"; - #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>; - - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>; - assigned-clock-rates = <19200000>; - - interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - iommu-map = <0x0 &pcie_smmu 0x0000 0x1>, - <0x100 &pcie_smmu 0x0001 0x1>; - - resets = <&gcc GCC_PCIE_0_BCR>, - <&gcc GCC_PCIE_0_LINK_DOWN_BCR>; - reset-names = "pci", - "link_down"; - - power-domains = <&gcc PCIE_0_GDSC>; - - phys = <&pcie0_phy>; - phy-names = "pciephy"; - - eq-presets-8gts = /bits/ 16 <0x5555 0x5555>; - eq-presets-16gts = /bits/ 8 <0x55 0x55>; - - status = "disabled"; - - pcieport0: pcie@0 { - device_type = "pci"; - reg = <0x0 0x0 0x0 0x0 0x0>; - bus-range = <0x01 0xff>; - - #address-cells = <3>; - #size-cells = <2>; - ranges; - }; - }; - - pcie0_ep: pcie-ep@1c00000 { - compatible = "qcom,sa8775p-pcie-ep"; - reg = <0x0 0x01c00000 0x0 0x3000>, - <0x0 0x40000000 0x0 0xf20>, - <0x0 0x40000f20 0x0 0xa8>, - <0x0 0x40001000 0x0 0x4000>, - <0x0 0x40200000 0x0 0x1fe00000>, - <0x0 0x01c03000 0x0 0x1000>, - <0x0 0x40005000 0x0 0x2000>; - reg-names = "parf", "dbi", "elbi", "atu", "addr_space", - "mmio", "dma"; - - clocks = <&gcc GCC_PCIE_0_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_0_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_AXI_CLK>, - <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - interrupts = , - , - ; - - interrupt-names = "global", "doorbell", "dma"; - - interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - dma-coherent; - iommus = <&pcie_smmu 0x0000 0x7f>; - resets = <&gcc GCC_PCIE_0_BCR>; - reset-names = "core"; - power-domains = <&gcc PCIE_0_GDSC>; - phys = <&pcie0_phy>; - phy-names = "pciephy"; - num-lanes = <2>; - linux,pci-domain = <0>; - - status = "disabled"; - }; - - pcie0_phy: phy@1c04000 { - compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy"; - reg = <0x0 0x1c04000 0x0 0x2000>; - - clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>, - <&gcc GCC_PCIE_0_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, - <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>, - <&gcc GCC_PCIE_0_PIPE_CLK>, - <&gcc GCC_PCIE_0_PIPEDIV2_CLK>; - clock-names = "aux", - "cfg_ahb", - "ref", - "rchng", - "pipe", - "pipediv2"; - - assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>; - assigned-clock-rates = <100000000>; - - resets = <&gcc GCC_PCIE_0_PHY_BCR>; - reset-names = "phy"; - - #clock-cells = <0>; - clock-output-names = "pcie_0_pipe_clk"; - - #phy-cells = <0>; - - status = "disabled"; - }; - - pcie1: pcie@1c10000 { - compatible = "qcom,pcie-sa8775p"; - reg = <0x0 0x01c10000 0x0 0x3000>, - <0x0 0x60000000 0x0 0xf20>, - <0x0 0x60000f20 0x0 0xa8>, - <0x0 0x60001000 0x0 0x4000>, - <0x0 0x60100000 0x0 0x100000>, - <0x0 0x01c13000 0x0 0x1000>; - reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi"; - device_type = "pci"; - - #address-cells = <3>; - #size-cells = <2>; - ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>, - <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>; - bus-range = <0x00 0xff>; - - dma-coherent; - - linux,pci-domain = <1>; - num-lanes = <4>; - - interrupts = , - , - , - , - , - , - , - , - ; - interrupt-names = "msi0", - "msi1", - "msi2", - "msi3", - "msi4", - "msi5", - "msi6", - "msi7", - "global"; - #interrupt-cells = <1>; - interrupt-map-mask = <0 0 0 0x7>; - interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>, - <0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>; - - clocks = <&gcc GCC_PCIE_1_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>; - assigned-clock-rates = <19200000>; - - interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - iommu-map = <0x0 &pcie_smmu 0x0080 0x1>, - <0x100 &pcie_smmu 0x0081 0x1>; - - resets = <&gcc GCC_PCIE_1_BCR>, - <&gcc GCC_PCIE_1_LINK_DOWN_BCR>; - reset-names = "pci", - "link_down"; - - power-domains = <&gcc PCIE_1_GDSC>; - - phys = <&pcie1_phy>; - phy-names = "pciephy"; - - eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>; - eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>; - - status = "disabled"; - - pcie@0 { - device_type = "pci"; - reg = <0x0 0x0 0x0 0x0 0x0>; - bus-range = <0x01 0xff>; - - #address-cells = <3>; - #size-cells = <2>; - ranges; - }; - }; - - pcie1_ep: pcie-ep@1c10000 { - compatible = "qcom,sa8775p-pcie-ep"; - reg = <0x0 0x01c10000 0x0 0x3000>, - <0x0 0x60000000 0x0 0xf20>, - <0x0 0x60000f20 0x0 0xa8>, - <0x0 0x60001000 0x0 0x4000>, - <0x0 0x60200000 0x0 0x1fe00000>, - <0x0 0x01c13000 0x0 0x1000>, - <0x0 0x60005000 0x0 0x2000>; - reg-names = "parf", "dbi", "elbi", "atu", "addr_space", - "mmio", "dma"; - - clocks = <&gcc GCC_PCIE_1_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_1_MSTR_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_AXI_CLK>, - <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>; - - clock-names = "aux", - "cfg", - "bus_master", - "bus_slave", - "slave_q2a"; - - interrupts = , - , - ; - - interrupt-names = "global", "doorbell", "dma"; - - interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>, - <&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>; - interconnect-names = "pcie-mem", "cpu-pcie"; - - dma-coherent; - iommus = <&pcie_smmu 0x80 0x7f>; - resets = <&gcc GCC_PCIE_1_BCR>; - reset-names = "core"; - power-domains = <&gcc PCIE_1_GDSC>; - phys = <&pcie1_phy>; - phy-names = "pciephy"; - num-lanes = <4>; - linux,pci-domain = <1>; - - status = "disabled"; - }; - - pcie1_phy: phy@1c14000 { - compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy"; - reg = <0x0 0x1c14000 0x0 0x4000>; - - clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>, - <&gcc GCC_PCIE_1_CFG_AHB_CLK>, - <&gcc GCC_PCIE_CLKREF_EN>, - <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>, - <&gcc GCC_PCIE_1_PIPE_CLK>, - <&gcc GCC_PCIE_1_PIPEDIV2_CLK>; - clock-names = "aux", - "cfg_ahb", - "ref", - "rchng", - "pipe", - "pipediv2"; - - assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>; - assigned-clock-rates = <100000000>; - - resets = <&gcc GCC_PCIE_1_PHY_BCR>; - reset-names = "phy"; - - #clock-cells = <0>; - clock-output-names = "pcie_1_pipe_clk"; - - #phy-cells = <0>; - - status = "disabled"; - }; }; diff --git a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts index 18832a3b9a1c3..57f549f06f73c 100644 --- a/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts +++ b/arch/arm64/boot/dts/qcom/msm8976-longcheer-l9360.dts @@ -455,14 +455,14 @@ bias-disable; }; - ts_int_default: ts-int-state { + ts_int_default: ts-int-default-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; bias-pull-down; }; - ts_int_sleep: ts-int-state { + ts_int_sleep: ts-int-sleep-state { pins = "gpio65"; function = "gpio"; drive-strength = <2>; diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts index bd3f39e1b98fb..9a9c674501202 100644 --- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts +++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts @@ -39,7 +39,7 @@ haptics: drv2604@5a { compatible = "ti,drv2604"; reg = <0x5a>; - enable-gpio = <&tlmm 93 0x00>; + enable-gpios = <&tlmm 93 GPIO_ACTIVE_HIGH>; mode = ; library-sel = ; pinctrl-names = "default","sleep"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi index 3650f2501886b..04d4741cdb5f0 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998-sony-xperia-yoshino.dtsi @@ -229,10 +229,6 @@ }; }; -&blsp1_i2c5_sleep { - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts index 0cac06f25a77a..2ceeec827f8e2 100644 --- a/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts +++ b/arch/arm64/boot/dts/qcom/msm8998-xiaomi-sagit.dts @@ -217,11 +217,6 @@ }; }; -&blsp1_i2c5_sleep { - /delete-property/ bias-pull-up; - bias-disable; -}; - &blsp1_uart3 { status = "okay"; diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi index 5c75fba16ce2c..515aa844a3e37 100644 --- a/arch/arm64/boot/dts/qcom/msm8998.dtsi +++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi @@ -1183,7 +1183,7 @@ pins = "gpio2", "gpio3"; function = "blsp_i2c1"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c2_default: blsp1-i2c2-default-state { @@ -1197,7 +1197,7 @@ pins = "gpio32", "gpio33"; function = "blsp_i2c2"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c3_default: blsp1-i2c3-default-state { @@ -1211,7 +1211,7 @@ pins = "gpio47", "gpio48"; function = "blsp_i2c3"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c4_default: blsp1-i2c4-default-state { @@ -1225,7 +1225,7 @@ pins = "gpio10", "gpio11"; function = "blsp_i2c4"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c5_default: blsp1-i2c5-default-state { @@ -1239,7 +1239,7 @@ pins = "gpio87", "gpio88"; function = "blsp_i2c5"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_i2c6_default: blsp1-i2c6-default-state { @@ -1253,7 +1253,7 @@ pins = "gpio43", "gpio44"; function = "blsp_i2c6"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp1_spi_b_default: blsp1-spi-b-default-state { @@ -1318,7 +1318,7 @@ pins = "gpio55", "gpio56"; function = "blsp_i2c7"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c2_default: blsp2-i2c2-default-state { @@ -1332,7 +1332,7 @@ pins = "gpio6", "gpio7"; function = "blsp_i2c8"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c3_default: blsp2-i2c3-default-state { @@ -1346,7 +1346,7 @@ pins = "gpio51", "gpio52"; function = "blsp_i2c9"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c4_default: blsp2-i2c4-default-state { @@ -1360,7 +1360,7 @@ pins = "gpio67", "gpio68"; function = "blsp_i2c10"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c5_default: blsp2-i2c5-default-state { @@ -1374,7 +1374,7 @@ pins = "gpio60", "gpio61"; function = "blsp_i2c11"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_i2c6_default: blsp2-i2c6-default-state { @@ -1388,7 +1388,7 @@ pins = "gpio83", "gpio84"; function = "blsp_i2c12"; drive-strength = <2>; - bias-pull-up; + bias-disable; }; blsp2_spi1_default: blsp2-spi1-default-state { diff --git a/arch/arm64/boot/dts/qcom/x1p42100.dtsi b/arch/arm64/boot/dts/qcom/purwa.dtsi similarity index 99% rename from arch/arm64/boot/dts/qcom/x1p42100.dtsi rename to arch/arm64/boot/dts/qcom/purwa.dtsi index 10d26958d3c66..a02b39f98f0b2 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100.dtsi +++ b/arch/arm64/boot/dts/qcom/purwa.dtsi @@ -3,8 +3,8 @@ * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. */ -/* X1P42100 is heavily based on X1E80100, with some meaningful differences */ -#include "x1e80100.dtsi" +/* X1P42100 is heavily based on hamoa, with some meaningful differences */ +#include "hamoa.dtsi" /delete-node/ &bwmon_cluster0; /delete-node/ &cluster_pd2; @@ -33,6 +33,8 @@ &gpu { compatible = "qcom,adreno-43030c00", "qcom,adreno"; + iommus = <&adreno_smmu 0 0x0>; + nvmem-cells = <&gpu_speed_bin>; nvmem-cell-names = "speed_bin"; diff --git a/arch/arm64/boot/dts/qcom/qcm2290.dtsi b/arch/arm64/boot/dts/qcom/qcm2290.dtsi index e0e400fdd2497..5a817824f924b 100644 --- a/arch/arm64/boot/dts/qcom/qcm2290.dtsi +++ b/arch/arm64/boot/dts/qcom/qcm2290.dtsi @@ -2006,6 +2006,7 @@ "gcc_disp_gpll0_div_clk_src", "dsi0_phy_pll_out_byteclk", "dsi0_phy_pll_out_dsiclk"; + power-domains = <&rpmpd QCM2290_VDDCX>; #power-domain-cells = <1>; #clock-cells = <1>; #reset-cells = <1>; diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi index 4328c1dda898c..736accfc34f90 100644 --- a/arch/arm64/boot/dts/qcom/qcs404.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi @@ -677,11 +677,11 @@ assigned-clock-rates = <19200000>, <200000000>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; @@ -716,11 +716,11 @@ assigned-clock-rates = <19200000>, <133333333>; interrupts = , - , - ; + , + ; interrupt-names = "pwr_event", - "hs_phy_irq", - "qusb2_phy"; + "qusb2_phy", + "hs_phy_irq"; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi index e6ac529e6b721..3ead078246f48 100644 --- a/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi +++ b/arch/arm64/boot/dts/qcom/qcs8550-aim300.dtsi @@ -396,8 +396,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3e_1p2>; - vdda-pll-supply = <&vreg_l3f_0p88>; + vdda-phy-supply = <&vreg_l3f_0p88>; + vdda-pll-supply = <&vreg_l3e_1p2>; }; &xo_board { diff --git a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts index 74778a5b19ba6..80428c0b3b4e6 100644 --- a/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts +++ b/arch/arm64/boot/dts/qcom/sar2130p-qar2130p.dts @@ -551,8 +551,8 @@ }; &usb_dp_qmpphy { - vdda-phy-supply = <&vreg_l3a_1p2>; - vdda-pll-supply = <&vreg_l1a_0p91>; + vdda-phy-supply = <&vreg_l1a_0p91>; + vdda-pll-supply = <&vreg_l3a_1p2>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi index 74ab321d3333c..d60fad977a284 100644 --- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi @@ -98,6 +98,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* FIXED REGULATORS - parents above children */ /* This is the top level supply and variable voltage */ diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi index 892b3d2f1bf99..dc96d3d5e943b 100644 --- a/arch/arm64/boot/dts/qcom/sc7180.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi @@ -3452,8 +3452,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi index 5c5e4f1dd2217..58ea0532c0fbb 100644 --- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi @@ -25,6 +25,11 @@ stdout-path = "serial0:115200n8"; }; + firmware { + #address-cells = <2>; + #size-cells = <2>; + }; + /* * FIXED REGULATORS * diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi index ed87bcb0d4b69..0f566dd75c964 100644 --- a/arch/arm64/boot/dts/qcom/sc7280.dtsi +++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi @@ -5196,8 +5196,8 @@ edp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -5295,8 +5295,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts index 08d0784d0cbb8..8f8f66a2297e5 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-lenovo-flex-5g.dts @@ -26,6 +26,7 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; pinctrl-0 = <&bl_pwm_default>; pinctrl-names = "default"; @@ -162,14 +163,39 @@ }; }; - vph_pwr: vph-pwr-regulator { + vled_bl_pw: regulator-vled-bl-pw { compatible = "regulator-fixed"; - regulator-name = "vph_pwr"; - regulator-min-microvolt = <3700000>; - regulator-max-microvolt = <3700000>; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; }; - vreg_s4a_1p8: pm8150-s4-regulator { + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_s4a_1p8: regulator-pm8150-s4 { compatible = "regulator-fixed"; regulator-name = "vreg_s4a_1p8"; @@ -182,6 +208,13 @@ vin-supply = <&vph_pwr>; }; + vph_pwr: regulator-vph-pwr { + compatible = "regulator-fixed"; + regulator-name = "vph_pwr"; + regulator-min-microvolt = <3700000>; + regulator-max-microvolt = <3700000>; + }; + usbprim-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; @@ -444,6 +477,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; no-hpd; backlight = <&backlight>; @@ -478,6 +512,13 @@ status = "okay"; }; +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; + &pmc8180_pwrkey { status = "okay"; }; @@ -771,6 +812,12 @@ drive-strength = <2>; }; + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; + usbprim_sbu_default: usbprim-sbu-state { oe-n-pins { pins = "gpio152"; diff --git a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts index 93de9fe918ebd..76679e311ecf9 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x-primus.dts +++ b/arch/arm64/boot/dts/qcom/sc8180x-primus.dts @@ -27,9 +27,10 @@ compatible = "pwm-backlight"; pwms = <&pmc8180c_lpg 4 1000000>; enable-gpios = <&pmc8180c_gpios 8 GPIO_ACTIVE_HIGH>; + power-supply = <&vled_bl_pw>; - pinctrl-names = "default"; pinctrl-0 = <&bl_pwm_default>; + pinctrl-names = "default"; }; chosen { @@ -165,7 +166,39 @@ }; }; - vreg_nvme_0p9: nvme-0p9-regulator { + vled_bl_pw: regulator-vled-bl-pw { + compatible = "regulator-fixed"; + + regulator-name = "VLED_BL_PW"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&pmc8180_2_gpios 1 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&bl_pwr_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_lcm_3v3: regulator-edp-3p3 { + compatible = "regulator-fixed"; + + regulator-name = "VREG_LCM_3V3"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + + gpio = <&tlmm 130 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&lcm_3v3_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; + + vreg_nvme_0p9: regulator-nvme-0p9 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_0p9"; @@ -175,7 +208,7 @@ regulator-always-on; }; - vreg_nvme_3p3: nvme-3p3-regulator { + vreg_nvme_3p3: regulator-nvme-3p3 { compatible = "regulator-fixed"; regulator-name = "vreg_nvme_3p3"; @@ -188,7 +221,7 @@ regulator-always-on; }; - vdd_kb_tp_3v3: vdd-kb-tp-3v3-regulator { + vdd_kb_tp_3v3: regulator-vdd-kb-tp-3v3 { compatible = "regulator-fixed"; regulator-name = "vdd_kb_tp_3v3"; regulator-min-microvolt = <3300000>; @@ -203,7 +236,7 @@ pinctrl-0 = <&kb_tp_3v3_en_active_state>; }; - vph_pwr: vph-pwr-regulator { + vph_pwr: regulator-vph-pwr { compatible = "regulator-fixed"; regulator-name = "vph_pwr"; regulator-min-microvolt = <3700000>; @@ -539,6 +572,7 @@ aux-bus { panel { compatible = "edp-panel"; + power-supply = <&vreg_lcm_3v3>; backlight = <&backlight>; @@ -768,6 +802,12 @@ }; /* PINCTRL */ +&pmc8180_2_gpios { + bl_pwr_en: bl-pwr-en-state { + pins = "gpio1"; + function = "normal"; + }; +}; &pmc8180c_gpios { bl_pwm_default: bl-pwm-default-state { @@ -949,4 +989,10 @@ bias-pull-up; }; }; + + lcm_3v3_en: lcm-3v3-en-state { + pins = "gpio130"; + function = "gpio"; + bias-disable; + }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi index e20a096aac7f9..c6d96022c666d 100644 --- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi @@ -3303,8 +3303,8 @@ dp0_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -3385,8 +3385,8 @@ dp1_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -3461,8 +3461,8 @@ edp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts index 490e970c54a24..5b2692bb1564e 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts @@ -182,7 +182,7 @@ regulator-min-microvolt = <3300000>; regulator-max-microvolt = <3300000>; - gpio = <&pmc8280_1_gpios 2 GPIO_ACTIVE_HIGH>; + gpio = <&pmc8280_1_gpios 1 GPIO_ACTIVE_HIGH>; enable-active-high; pinctrl-names = "default"; @@ -921,7 +921,7 @@ }; misc_3p3_reg_en: misc-3p3-reg-en-state { - pins = "gpio2"; + pins = "gpio1"; function = "normal"; }; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts index 637430719e6d7..a7559eb411018 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts @@ -1561,7 +1561,7 @@ mclk-pins { pins = "gpio17"; function = "cam_mclk"; - drive-strength = <16>; + drive-strength = <6>; bias-disable; }; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts index aeed3ef152eba..cdc2b0dcca27b 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-arcata.dts @@ -248,7 +248,7 @@ usb0-sbu-mux { compatible = "pericom,pi3usb102", "gpio-sbu-mux"; - enable-gpios = <&tlmm 101 GPIO_ACTIVE_LOW>; + enable-gpios = <&tlmm 100 GPIO_ACTIVE_LOW>; select-gpios = <&tlmm 164 GPIO_ACTIVE_HIGH>; pinctrl-0 = <&usb0_sbu_default>; @@ -1002,7 +1002,7 @@ usb0_sbu_default: usb0-sbu-state { oe-n-pins { - pins = "gpio101"; + pins = "gpio100"; function = "gpio"; bias-disable; drive-strength = <16>; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts index a40dccd70dfda..454defd259560 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts +++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts @@ -769,20 +769,11 @@ }; &pmk8280_rtc { - nvmem-cells = <&rtc_offset>; - nvmem-cell-names = "offset"; + qcom,uefi-rtc-info; status = "okay"; }; -&pmk8280_sdam_6 { - status = "okay"; - - rtc_offset: rtc-offset@bc { - reg = <0xbc 0x4>; - }; -}; - &pmk8280_vadc { channel@144 { reg = ; @@ -1009,8 +1000,8 @@ }; &usb_0_qmpphy { - vdda-phy-supply = <&vreg_l4d>; - vdda-pll-supply = <&vreg_l9d>; + vdda-phy-supply = <&vreg_l9d>; + vdda-pll-supply = <&vreg_l4d>; orientation-switch; @@ -1046,8 +1037,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l3b>; - vdda-pll-supply = <&vreg_l4b>; + vdda-phy-supply = <&vreg_l4b>; + vdda-pll-supply = <&vreg_l3b>; orientation-switch; diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi index d89938e17e093..40649fe7a5095 100644 --- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi +++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi @@ -4754,8 +4754,8 @@ mdss0_dp0_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -4836,8 +4836,8 @@ mdss0_dp1_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -4916,8 +4916,8 @@ mdss0_dp2_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -4991,8 +4991,8 @@ mdss0_dp3_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -6110,8 +6110,8 @@ mdss1_dp0_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -6190,8 +6190,8 @@ mdss1_dp1_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -6270,8 +6270,8 @@ mdss1_dp2_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; @@ -6345,8 +6345,8 @@ mdss1_dp3_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts index 2f55db0c8ce35..9ea3e5e76bf9e 100644 --- a/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts +++ b/arch/arm64/boot/dts/qcom/sdm632-motorola-ocean.dts @@ -130,7 +130,7 @@ led: led@1 { reg = <1>; - default-trigger = "backlight"; + linux,default-trigger = "backlight"; function = LED_FUNCTION_BACKLIGHT; led-sources = <0 1 2>; }; diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts index 90efbb7e3799b..bead957cf83b4 100644 --- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts +++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts @@ -347,6 +347,10 @@ status = "okay"; }; +&cluster_sleep_0 { + arm,psci-suspend-param = <0x41008244>; +}; + &gcc { protected-clocks = , , diff --git a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts index ad347ccd19755..fb5b9f10c8a77 100644 --- a/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts +++ b/arch/arm64/boot/dts/qcom/sm6115-fxtec-pro1x.dts @@ -151,7 +151,7 @@ interrupts-extended = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>; - irq-gpios = <&tlmm 80 IRQ_TYPE_LEVEL_LOW>; + irq-gpios = <&tlmm 80 GPIO_ACTIVE_LOW>; reset-gpios = <&tlmm 71 GPIO_ACTIVE_HIGH>; AVDD28-supply = <&ts_vdd_supply>; VDDIO-supply = <&ts_vddio_supply>; diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi index 19a7b9f9ea8bc..0c6787d46ddcd 100644 --- a/arch/arm64/boot/dts/qcom/sm6350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi @@ -2306,8 +2306,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts index 8cbe068645ea9..b6d2a5d0c1c88 100644 --- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts +++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts @@ -48,7 +48,7 @@ stdout-path = "serial0:115200n8"; - framebuffer0: framebuffer@a000000 { + framebuffer0: framebuffer@a0000000 { compatible = "simple-framebuffer"; reg = <0 0xa0000000 0 (2340 * 1080 * 4)>; width = <1080>; @@ -1179,8 +1179,8 @@ }; &usb_1_qmpphy { - vdda-phy-supply = <&vreg_l22a>; - vdda-pll-supply = <&vreg_l16a>; + vdda-phy-supply = <&vreg_l16a>; + vdda-pll-supply = <&vreg_l22a>; status = "okay"; }; diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi index acdba79612aa8..e1f480e3ed0b7 100644 --- a/arch/arm64/boot/dts/qcom/sm8150.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi @@ -3939,8 +3939,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi index 465fd6e954a34..a2ced6611c40d 100644 --- a/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250-xiaomi-elish-common.dtsi @@ -28,7 +28,7 @@ /* required for bootloader to select correct board */ qcom,msm-id = ; /* SM8250 v2.1 */ - qcom,board-id = <0x10008 0>; + qcom,board-id = <0x2f 0>; aliases { serial0 = &uart6; diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi index adf6b49b52c48..dfba5bc3ba7cf 100644 --- a/arch/arm64/boot/dts/qcom/sm8250.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi @@ -4321,8 +4321,12 @@ interrupts = ; power-domains = <&videocc MVS0C_GDSC>, <&videocc MVS0_GDSC>, - <&rpmhpd RPMHPD_MX>; - power-domain-names = "venus", "vcodec0", "mx"; + <&rpmhpd RPMHPD_MX>, + <&rpmhpd RPMHPD_MMCX>; + power-domain-names = "venus", + "vcodec0", + "mx", + "mmcx"; operating-points-v2 = <&venus_opp_table>; clocks = <&gcc GCC_VIDEO_AXI0_CLK>, @@ -4346,24 +4350,28 @@ venus_opp_table: opp-table { compatible = "operating-points-v2"; - opp-720000000 { - opp-hz = /bits/ 64 <720000000>; - required-opps = <&rpmhpd_opp_low_svs>; + opp-240000000 { + opp-hz = /bits/ 64 <240000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_low_svs>; }; - opp-1014000000 { - opp-hz = /bits/ 64 <1014000000>; - required-opps = <&rpmhpd_opp_svs>; + opp-338000000 { + opp-hz = /bits/ 64 <338000000>; + required-opps = <&rpmhpd_opp_svs>, + <&rpmhpd_opp_svs>; }; - opp-1098000000 { - opp-hz = /bits/ 64 <1098000000>; - required-opps = <&rpmhpd_opp_svs_l1>; + opp-366000000 { + opp-hz = /bits/ 64 <366000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_svs_l1>; }; - opp-1332000000 { - opp-hz = /bits/ 64 <1332000000>; - required-opps = <&rpmhpd_opp_nom>; + opp-444000000 { + opp-hz = /bits/ 64 <444000000>; + required-opps = <&rpmhpd_opp_svs_l1>, + <&rpmhpd_opp_nom>; }; }; }; @@ -4824,8 +4832,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi index fc4ce9d4977e8..af892b76322fd 100644 --- a/arch/arm64/boot/dts/qcom/sm8350.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi @@ -2925,8 +2925,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi index 2d49df7b69e4b..991f3cbde1c8d 100644 --- a/arch/arm64/boot/dts/qcom/sm8450.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi @@ -3483,8 +3483,8 @@ dp_opp_table: opp-table { compatible = "operating-points-v2"; - opp-160000000 { - opp-hz = /bits/ 64 <160000000>; + opp-162000000 { + opp-hz = /bits/ 64 <162000000>; required-opps = <&rpmhpd_opp_low_svs>; }; diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi index 2760c4f7e6f35..b714207e4c158 100644 --- a/arch/arm64/boot/dts/qcom/sm8750.dtsi +++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi @@ -3558,6 +3558,7 @@ phy-names = "ufsphy"; #reset-cells = <1>; + qcom,ice = <&ice>; status = "disabled"; diff --git a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi index ee3c8c5e2c50c..8e5c5575a5320 100644 --- a/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-asus-zenbook-a14.dtsi @@ -11,10 +11,9 @@ #include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { - model = "ASUS Zenbook A14"; chassis-type = "laptop"; aliases { @@ -1005,14 +1004,10 @@ status = "okay"; aux-bus { - panel { + panel: panel { compatible = "edp-panel"; - enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; power-supply = <&vreg_edp_3p3>; - pinctrl-0 = <&edp_bl_en>; - pinctrl-names = "default"; - port { edp_panel_in: endpoint { remote-endpoint = <&mdss_dp3_out>; diff --git a/arch/arm64/boot/dts/qcom/x1-crd.dtsi b/arch/arm64/boot/dts/qcom/x1-crd.dtsi index 3c9455fede5c0..5e324f35547a4 100644 --- a/arch/arm64/boot/dts/qcom/x1-crd.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-crd.dtsi @@ -9,7 +9,7 @@ #include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E80100 CRD"; diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi index 5b5a10a31a253..58a5fe65ae38b 100644 --- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi +++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi @@ -12,7 +12,7 @@ #include #include -#include "x1e80100-pmics.dtsi" +#include "hamoa-pmics.dtsi" / { chassis-type = "laptop"; @@ -589,6 +589,7 @@ regulator-min-microvolt = <1200000>; regulator-max-microvolt = <1200000>; regulator-initial-mode = ; + regulator-always-on; }; vreg_l13b_3p0: ldo13 { @@ -610,6 +611,7 @@ regulator-min-microvolt = <1800000>; regulator-max-microvolt = <1800000>; regulator-initial-mode = ; + regulator-always-on; }; }; diff --git a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts index bfc649d4b643c..701f35af7d5cd 100644 --- a/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts +++ b/arch/arm64/boot/dts/qcom/x1e001de-devkit.dts @@ -8,8 +8,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E001DE Snapdragon Devkit for Windows"; diff --git a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi index 654cbce9d6ecb..169726984d3b0 100644 --- a/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e78100-lenovo-thinkpad-t14s.dtsi @@ -12,8 +12,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Lenovo ThinkPad T14s Gen 6"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts index 0113d856b3ad4..f2960953e608f 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-asus-vivobook-s15.dts @@ -11,8 +11,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "ASUS Vivobook S 15"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts b/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts index 0d0bcc50207d7..49b12a0a7cb1e 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-asus-zenbook-a14.dts @@ -6,7 +6,7 @@ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-asus-zenbook-a14.dtsi" / { @@ -22,6 +22,14 @@ firmware-name = "qcom/x1e80100/ASUSTeK/zenbook-a14/qcdxkmsuc8380.mbn"; }; +&panel { + compatible = "samsung,atna40cu11", "samsung,atna33xc20"; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; +}; + &remoteproc_adsp { firmware-name = "qcom/x1e80100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", "qcom/x1e80100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts index dfc378e1a056a..429deffcf3e9d 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-crd.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-crd.dts @@ -5,7 +5,7 @@ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-crd.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts index cf2a7c2628881..75e10d97c3867 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-inspiron-14-plus-7441.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-dell-thena.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts index 32ad9679550ef..a8ff7ef258a13 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-latitude-7455.dts @@ -4,7 +4,7 @@ */ /dts-v1/; -#include "x1e80100.dtsi" +#include "hamoa.dtsi" #include "x1-dell-thena.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts index 58f8caaa72580..c1f49cba61fc4 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-dell-xps13-9345.dts @@ -12,8 +12,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Dell XPS 13 9345"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts b/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts index e5a839d458402..b79e59e1c413a 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-hp-omnibook-x14.dts @@ -6,8 +6,8 @@ /dts-v1/; -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" #include "x1-hp-omnibook-x14.dtsi" / { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts index e0642fe8343f6..56e4d13cca111 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-lenovo-yoga-slim7x.dts @@ -9,8 +9,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Lenovo Yoga Slim 7x"; diff --git a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi index ed468b93ba50e..3b319f65dde1e 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi +++ b/arch/arm64/boot/dts/qcom/x1e80100-microsoft-romulus.dtsi @@ -9,8 +9,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { aliases { diff --git a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts index 4a9b6d791e7f4..5a121fc449401 100644 --- a/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts +++ b/arch/arm64/boot/dts/qcom/x1e80100-qcp.dts @@ -8,8 +8,8 @@ #include #include -#include "x1e80100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "hamoa.dtsi" +#include "hamoa-pmics.dtsi" / { model = "Qualcomm Technologies, Inc. X1E80100 QCP"; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts new file mode 100644 index 0000000000000..be756069131d7 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14-lcd.dts @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2025 Aleksandrs Vinarskis + */ + +/dts-v1/; + +#include "x1p42100-asus-zenbook-a14.dtsi" + +/ { + model = "ASUS Zenbook A14 (UX3407QA, LCD)"; + compatible = "asus,zenbook-a14-ux3407qa-lcd", "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; + + backlight: backlight { + compatible = "pwm-backlight"; + pwms = <&pmk8550_pwm 0 416667>; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; + power-supply = <&vreg_edp_bl>; + + pinctrl-0 = <&edp_bl_en>, <&edp_bl_pwm>; + pinctrl-names = "default"; + }; + + vreg_edp_bl: regulator-edp-bl { + compatible = "regulator-fixed"; + + regulator-name = "VBL9"; + regulator-min-microvolt = <3600000>; + regulator-max-microvolt = <3600000>; + + gpio = <&pmc8380_3_gpios 10 GPIO_ACTIVE_HIGH>; + enable-active-high; + + pinctrl-0 = <&edp_bl_reg_en>; + pinctrl-names = "default"; + + regulator-boot-on; + }; +}; + +&panel { + backlight = <&backlight>; +}; + +&pmc8380_3_gpios { + edp_bl_reg_en: edp-bl-reg-en-state { + pins = "gpio10"; + function = "normal"; + }; +}; + +&pmk8550_gpios { + edp_bl_pwm: edp-bl-pwm-state { + pins = "gpio5"; + function = "func3"; + }; +}; + +&pmk8550_pwm { + status = "okay"; +}; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts index bd75ff898601a..68cd318d69073 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dts @@ -6,136 +6,17 @@ /dts-v1/; -#include "x1p42100.dtsi" -#include "x1-asus-zenbook-a14.dtsi" - -/delete-node/ &pmc8380_6; -/delete-node/ &pmc8380_6_thermal; +#include "x1p42100-asus-zenbook-a14.dtsi" / { model = "ASUS Zenbook A14 (UX3407QA)"; - compatible = "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; - - wcn6855-pmu { - compatible = "qcom,wcn6855-pmu"; - - vddaon-supply = <&vreg_wcn_0p95>; - vddio-supply = <&vreg_wcn_1p9>; - vddpcie1p3-supply = <&vreg_wcn_1p9>; - vddpcie1p9-supply = <&vreg_wcn_1p9>; - vddpmu-supply = <&vreg_wcn_0p95>; - vddpmucx-supply = <&vreg_wcn_0p95>; - vddpmumx-supply = <&vreg_wcn_0p95>; - vddrfa0p95-supply = <&vreg_wcn_0p95>; - vddrfa1p3-supply = <&vreg_wcn_1p9>; - vddrfa1p9-supply = <&vreg_wcn_1p9>; - - bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; - wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; - - pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>; - pinctrl-names = "default"; - - regulators { - vreg_pmu_rfa_cmn_0p8: ldo0 { - regulator-name = "vreg_pmu_rfa_cmn_0p8"; - }; - - vreg_pmu_aon_0p8: ldo1 { - regulator-name = "vreg_pmu_aon_0p8"; - }; - - vreg_pmu_wlcx_0p8: ldo2 { - regulator-name = "vreg_pmu_wlcx_0p8"; - }; - - vreg_pmu_wlmx_0p8: ldo3 { - regulator-name = "vreg_pmu_wlmx_0p8"; - }; - - vreg_pmu_btcmx_0p8: ldo4 { - regulator-name = "vreg_pmu_btcmx_0p8"; - }; - - vreg_pmu_pcie_1p8: ldo5 { - regulator-name = "vreg_pmu_pcie_1p8"; - }; - - vreg_pmu_pcie_0p9: ldo6 { - regulator-name = "vreg_pmu_pcie_0p9"; - }; - - vreg_pmu_rfa_0p8: ldo7 { - regulator-name = "vreg_pmu_rfa_0p8"; - }; - - vreg_pmu_rfa_1p2: ldo8 { - regulator-name = "vreg_pmu_rfa_1p2"; - }; - - vreg_pmu_rfa_1p7: ldo9 { - regulator-name = "vreg_pmu_rfa_1p7"; - }; - }; - }; + compatible = "asus,zenbook-a14-ux3407qa-oled", "asus,zenbook-a14-ux3407qa", "qcom,x1p42100"; }; -&gpu { - status = "okay"; -}; - -&gpu_zap_shader { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcdxkmsucpurwa.mbn"; -}; - -&pcie4_port0 { - wifi@0 { - compatible = "pci17cb,1103"; - reg = <0x10000 0x0 0x0 0x0 0x0>; - - vddaon-supply = <&vreg_pmu_aon_0p8>; - vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; - vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; - vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; - vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; - vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; - vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; - vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; - - qcom,calibration-variant = "UX3407Q"; - }; -}; - -&remoteproc_adsp { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", - "qcom/x1p42100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; - - status = "okay"; -}; - -&remoteproc_cdsp { - firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qccdsp8380.mbn", - "qcom/x1p42100/ASUSTeK/zenbook-a14/cdsp_dtbs.elf"; - - status = "okay"; -}; - -&uart14 { - status = "okay"; - - bluetooth { - compatible = "qcom,wcn6855-bt"; - - vddaon-supply = <&vreg_pmu_aon_0p8>; - vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>; - vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; - vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; - vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; - vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; - vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; - vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; +&panel { + compatible = "samsung,atna40ct06", "samsung,atna33xc20"; + enable-gpios = <&pmc8380_3_gpios 4 GPIO_ACTIVE_HIGH>; - max-speed = <3000000>; - }; + pinctrl-0 = <&edp_bl_en>; + pinctrl-names = "default"; }; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi new file mode 100644 index 0000000000000..22470a97e1e39 --- /dev/null +++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) 2025 Aleksandrs Vinarskis + */ + +/dts-v1/; + +#include "purwa.dtsi" +#include "x1-asus-zenbook-a14.dtsi" + +/delete-node/ &pmc8380_6; +/delete-node/ &pmc8380_6_thermal; + +/ { + wcn6855-pmu { + compatible = "qcom,wcn6855-pmu"; + + vddaon-supply = <&vreg_wcn_0p95>; + vddio-supply = <&vreg_wcn_1p9>; + vddpcie1p3-supply = <&vreg_wcn_1p9>; + vddpcie1p9-supply = <&vreg_wcn_1p9>; + vddpmu-supply = <&vreg_wcn_0p95>; + vddpmucx-supply = <&vreg_wcn_0p95>; + vddpmumx-supply = <&vreg_wcn_0p95>; + vddrfa0p95-supply = <&vreg_wcn_0p95>; + vddrfa1p3-supply = <&vreg_wcn_1p9>; + vddrfa1p9-supply = <&vreg_wcn_1p9>; + + bt-enable-gpios = <&tlmm 116 GPIO_ACTIVE_HIGH>; + wlan-enable-gpios = <&tlmm 117 GPIO_ACTIVE_HIGH>; + + pinctrl-0 = <&wcn_bt_en>, <&wcn_wlan_en>; + pinctrl-names = "default"; + + regulators { + vreg_pmu_rfa_cmn_0p8: ldo0 { + regulator-name = "vreg_pmu_rfa_cmn_0p8"; + }; + + vreg_pmu_aon_0p8: ldo1 { + regulator-name = "vreg_pmu_aon_0p8"; + }; + + vreg_pmu_wlcx_0p8: ldo2 { + regulator-name = "vreg_pmu_wlcx_0p8"; + }; + + vreg_pmu_wlmx_0p8: ldo3 { + regulator-name = "vreg_pmu_wlmx_0p8"; + }; + + vreg_pmu_btcmx_0p8: ldo4 { + regulator-name = "vreg_pmu_btcmx_0p8"; + }; + + vreg_pmu_pcie_1p8: ldo5 { + regulator-name = "vreg_pmu_pcie_1p8"; + }; + + vreg_pmu_pcie_0p9: ldo6 { + regulator-name = "vreg_pmu_pcie_0p9"; + }; + + vreg_pmu_rfa_0p8: ldo7 { + regulator-name = "vreg_pmu_rfa_0p8"; + }; + + vreg_pmu_rfa_1p2: ldo8 { + regulator-name = "vreg_pmu_rfa_1p2"; + }; + + vreg_pmu_rfa_1p7: ldo9 { + regulator-name = "vreg_pmu_rfa_1p7"; + }; + }; + }; +}; + +&gpu { + status = "okay"; +}; + +&gpu_zap_shader { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcdxkmsucpurwa.mbn"; +}; + +&pcie4_port0 { + wifi@0 { + compatible = "pci17cb,1103"; + reg = <0x10000 0x0 0x0 0x0 0x0>; + + vddaon-supply = <&vreg_pmu_aon_0p8>; + vddpcie0p9-supply = <&vreg_pmu_pcie_0p9>; + vddpcie1p8-supply = <&vreg_pmu_pcie_1p8>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; + + qcom,calibration-variant = "UX3407Q"; + }; +}; + +&remoteproc_adsp { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qcadsp8380.mbn", + "qcom/x1p42100/ASUSTeK/zenbook-a14/adsp_dtbs.elf"; + + status = "okay"; +}; + +&remoteproc_cdsp { + firmware-name = "qcom/x1p42100/ASUSTeK/zenbook-a14/qccdsp8380.mbn", + "qcom/x1p42100/ASUSTeK/zenbook-a14/cdsp_dtbs.elf"; + + status = "okay"; +}; + +&uart14 { + status = "okay"; + + bluetooth { + compatible = "qcom,wcn6855-bt"; + + vddaon-supply = <&vreg_pmu_aon_0p8>; + vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>; + vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>; + vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>; + vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>; + vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>; + vddwlcx-supply = <&vreg_pmu_wlcx_0p8>; + vddwlmx-supply = <&vreg_pmu_wlmx_0p8>; + + max-speed = <3000000>; + }; +}; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-crd.dts b/arch/arm64/boot/dts/qcom/x1p42100-crd.dts index cf999c2cf8d4e..7ed4116b95904 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-crd.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-crd.dts @@ -5,7 +5,7 @@ /dts-v1/; -#include "x1p42100.dtsi" +#include "purwa.dtsi" #include "x1-crd.dtsi" /delete-node/ &pmc8380_6; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts b/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts index 6696cab2de3ec..0f338e457abda 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-hp-omnibook-x14.dts @@ -2,8 +2,8 @@ /dts-v1/; -#include "x1p42100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "purwa.dtsi" +#include "hamoa-pmics.dtsi" #include "x1-hp-omnibook-x14.dtsi" /delete-node/ &pmc8380_6; /delete-node/ &pmc8380_6_thermal; diff --git a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts index 1ac46cdc43861..3186e79e862de 100644 --- a/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts +++ b/arch/arm64/boot/dts/qcom/x1p42100-lenovo-thinkbook-16.dts @@ -13,8 +13,8 @@ #include #include -#include "x1p42100.dtsi" -#include "x1e80100-pmics.dtsi" +#include "purwa.dtsi" +#include "hamoa-pmics.dtsi" /delete-node/ &pmc8380_6; /delete-node/ &pmc8380_6_thermal; diff --git a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi index add917af5de78..f14f9eca7d342 100644 --- a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi @@ -35,7 +35,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; gpio-leds { diff --git a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi index 192791993f059..02200de695d31 100644 --- a/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-pp1516.dtsi @@ -33,7 +33,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; gpio-leds { diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi index 973b4c5880e24..29794216592d8 100644 --- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi +++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi @@ -26,7 +26,7 @@ compatible = "mmc-pwrseq-emmc"; pinctrl-0 = <&emmc_reset>; pinctrl-names = "default"; - reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_HIGH>; + reset-gpios = <&gpio1 RK_PB3 GPIO_ACTIVE_LOW>; }; leds { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi index 9d07353df52c8..b44668c902b79 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-chromebook.dtsi @@ -503,8 +503,7 @@ ap_i2c_tp: &i2c5 { mvl_wifi: wifi@0,0 { compatible = "pci1b4b,2b42"; reg = <0x0000 0x0 0x0 0x0 0x0>; - interrupt-parent = <&gpio0>; - interrupts = <8 IRQ_TYPE_LEVEL_LOW>; + interrupts-extended = <&gpio0 8 IRQ_TYPE_LEVEL_LOW>; pinctrl-names = "default"; pinctrl-0 = <&wlan_host_wake_l>; wakeup-source; diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts index 5de964d369b09..c224f76c13834 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts @@ -689,12 +689,6 @@ }; }; - wifi { - wifi_host_wake_l: wifi-host-wake-l { - rockchip,pins = <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>; - }; - }; - wireless-bluetooth { bt_wake_pin: bt-wake-pin { rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>; @@ -721,19 +715,7 @@ pinctrl-names = "default"; pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>; sd-uhs-sdr104; - #address-cells = <1>; - #size-cells = <0>; status = "okay"; - - brcmf: wifi@1 { - compatible = "brcm,bcm4329-fmac"; - reg = <1>; - interrupt-parent = <&gpio4>; - interrupts = ; - interrupt-names = "host-wake"; - pinctrl-names = "default"; - pinctrl-0 = <&wifi_host_wake_l>; - }; }; &pwm0 { diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts index 8e3858cf988cb..27465cceca9a1 100644 --- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts +++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc-plus.dts @@ -132,6 +132,18 @@ <3 RK_PD7 1 &pcfg_pull_none>; }; +&i2s0_8ch_bus_bclk_off { + rockchip,pins = + <3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>, + <3 RK_PD1 1 &pcfg_pull_none>, + <3 RK_PD2 1 &pcfg_pull_none>, + <3 RK_PD3 1 &pcfg_pull_none>, + <3 RK_PD4 1 &pcfg_pull_none>, + <3 RK_PD5 1 &pcfg_pull_none>, + <3 RK_PD6 1 &pcfg_pull_none>, + <3 RK_PD7 1 &pcfg_pull_none>; +}; + &i2s1 { pinctrl-names = "default"; pinctrl-0 = <&i2s_8ch_mclk_pin>, <&i2s1_2ch_bus>; diff --git a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi index b6cf03a7ba66b..04cf285e6c2af 100644 --- a/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3566-bigtreetech-cb2.dtsi @@ -569,7 +569,7 @@ reg = <0x48>; status = "okay"; ti,x-plate-ohms = <660>; - ti,rt-thr = <3000>; + ti,max-rt = <3000>; ti,fuzzx = <32>; ti,fuzzy = <16>; }; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi index 7e74e04057cfd..a031f42c558ed 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi @@ -1648,8 +1648,8 @@ edp0: edp@fdec0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfdec0000 0x0 0x1000>; - clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP0_24M>, <&cru PCLK_EDP0>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy0>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi index a2640014ee042..b251bb129cdbf 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi +++ b/arch/arm64/boot/dts/rockchip/rk3588-extra.dtsi @@ -285,8 +285,8 @@ edp1: edp@fded0000 { compatible = "rockchip,rk3588-edp"; reg = <0x0 0xfded0000 0x0 0x1000>; - clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>; - clock-names = "dp", "pclk"; + clocks = <&cru CLK_EDP1_24M>, <&cru PCLK_EDP1>, <&cru HCLK_VO1>; + clock-names = "dp", "pclk", "hclk"; interrupts = ; phys = <&hdptxphy1>; phy-names = "dp"; diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts index 7e179862da6e5..26c431de6d963 100644 --- a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts +++ b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts @@ -23,16 +23,19 @@ compatible = "simple-audio-card"; pinctrl-names = "default"; pinctrl-0 = <&hp_detect>; + simple-audio-card,aux-devs = <&headphones_amp>; simple-audio-card,name = "rockchip,es8388"; - simple-audio-card,bitclock-master = <&masterdai>; + simple-audio-card,bitclock-master = <&cpudai>; simple-audio-card,format = "i2s"; - simple-audio-card,frame-master = <&masterdai>; + simple-audio-card,frame-master = <&cpudai>; simple-audio-card,hp-det-gpios = <&gpio1 RK_PA6 GPIO_ACTIVE_LOW>; simple-audio-card,mclk-fs = <256>; simple-audio-card,pin-switches = "Headphones"; simple-audio-card,routing = - "Headphones", "LOUT1", - "Headphones", "ROUT1", + "Headphones", "Headphone Amp OUTL", + "Headphones", "Headphone Amp OUTR", + "Headphone Amp INL", "LOUT2", + "Headphone Amp INR", "ROUT2", "LINPUT1", "Microphone Jack", "RINPUT1", "Microphone Jack", "LINPUT2", "Onboard Microphone", @@ -47,11 +50,17 @@ system-clock-frequency = <12288000>; }; - simple-audio-card,cpu { + cpudai: simple-audio-card,cpu { sound-dai = <&i2s0_8ch>; }; }; + headphones_amp: audio-amplifier-headphones { + compatible = "simple-audio-amplifier"; + enable-gpios = <&gpio1 RK_PA4 GPIO_ACTIVE_HIGH>; + sound-name-prefix = "Headphone Amp"; + }; + chosen { stdout-path = "serial2:1500000n8"; }; @@ -323,12 +332,12 @@ es8388: audio-codec@11 { compatible = "everest,es8388", "everest,es8328"; reg = <0x11>; - clocks = <&cru I2S1_8CH_MCLKOUT>; + clocks = <&cru I2S0_8CH_MCLKOUT>; AVDD-supply = <&vcc_3v3_s0>; DVDD-supply = <&vcc_1v8_s0>; HPVDD-supply = <&vcc_3v3_s0>; PVDD-supply = <&vcc_3v3_s0>; - assigned-clocks = <&cru I2S1_8CH_MCLKOUT>; + assigned-clocks = <&cru I2S0_8CH_MCLKOUT>; assigned-clock-rates = <12288000>; #sound-dai-cells = <0>; }; diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi index d872cc671094f..b7e5de3669ad6 100644 --- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi +++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi @@ -1401,7 +1401,7 @@ icssg0_mdio: mdio@32400 { compatible = "ti,davinci_mdio"; reg = <0x32400 0x100>; - clocks = <&k3_clks 62 3>; + clocks = <&k3_clks 81 0>; clock-names = "fck"; #address-cells = <1>; #size-cells = <0>; diff --git a/arch/arm64/configs/bcm2711_defconfig b/arch/arm64/configs/bcm2711_defconfig index e549f671907c4..1b60ce5525549 100644 --- a/arch/arm64/configs/bcm2711_defconfig +++ b/arch/arm64/configs/bcm2711_defconfig @@ -92,6 +92,7 @@ CONFIG_LRU_GEN_ENABLED=y CONFIG_NUMA_EMU=y CONFIG_NET=y CONFIG_PACKET=y +CONFIG_TLS=m CONFIG_XFRM_USER=m CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y @@ -544,9 +545,11 @@ CONFIG_R8169_LEDS=y CONFIG_MSE102X=m CONFIG_WIZNET_W5100=m CONFIG_WIZNET_W5100_SPI=m +CONFIG_MAXLINEAR_86110_PHY=y CONFIG_MICREL_PHY=y CONFIG_REALTEK_PHY_HWMON=y CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m CONFIG_CAN_SLCAN=m CONFIG_CAN_MCP251X=m CONFIG_CAN_MCP251XFD=m @@ -1078,6 +1081,9 @@ CONFIG_VIDEO_I2C=m CONFIG_AUXDISPLAY=y CONFIG_HD44780=m CONFIG_DRM=m +CONFIG_DRM_PANIC=y +CONFIG_DRM_PANIC_BACKGROUND_COLOR=0xc41949 +CONFIG_DRM_PANIC_SCREEN="kmsg" CONFIG_DRM_LOAD_EDID_FIRMWARE=y CONFIG_DRM_UDL=m CONFIG_DRM_PANEL_LVDS=m @@ -1206,6 +1212,7 @@ CONFIG_SND_SOC_ICS43432=m CONFIG_SND_SOC_MA120X0P=m CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_PCM3168A_I2C=m +CONFIG_SND_SOC_RT5616=m CONFIG_SND_SOC_TLV320AIC23_I2C=m CONFIG_SND_SOC_WM8804_I2C=m CONFIG_SND_SOC_WM8904=m @@ -1417,6 +1424,9 @@ CONFIG_USB_G_ACM_MS=m CONFIG_USB_G_MULTI=m CONFIG_USB_G_HID=m CONFIG_USB_G_WEBCAM=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_FUSB302=m CONFIG_MMC=y CONFIG_MMC_BLOCK_MINORS=32 CONFIG_MMC_BCM2835_MMC=y @@ -1542,6 +1552,7 @@ CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m CONFIG_VIDEO_CODEC_BCM2835=m CONFIG_VIDEO_ISP_BCM2835=m +CONFIG_CLK_HOG=m CONFIG_COMMON_CLK_RP1=y CONFIG_COMMON_CLK_RP1_SDIO=y CONFIG_COMMON_CLK_SI5351=m @@ -1725,6 +1736,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DLM=m CONFIG_UNICODE=m +CONFIG_KEYS_REQUEST_CACHE=y CONFIG_KEY_DH_OPERATIONS=y CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/arm64/configs/bcm2711_rt_defconfig b/arch/arm64/configs/bcm2711_rt_defconfig index 2424d949ac6a4..b7a94d5813b9e 100644 --- a/arch/arm64/configs/bcm2711_rt_defconfig +++ b/arch/arm64/configs/bcm2711_rt_defconfig @@ -92,6 +92,7 @@ CONFIG_LRU_GEN_ENABLED=y CONFIG_NUMA_EMU=y CONFIG_NET=y CONFIG_PACKET=y +CONFIG_TLS=m CONFIG_XFRM_USER=m CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y @@ -544,9 +545,11 @@ CONFIG_R8169_LEDS=y CONFIG_MSE102X=m CONFIG_WIZNET_W5100=m CONFIG_WIZNET_W5100_SPI=m +CONFIG_MAXLINEAR_86110_PHY=y CONFIG_MICREL_PHY=y CONFIG_REALTEK_PHY_HWMON=y CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m CONFIG_CAN_SLCAN=m CONFIG_CAN_MCP251X=m CONFIG_CAN_MCP251XFD=m @@ -1075,6 +1078,9 @@ CONFIG_VIDEO_I2C=m CONFIG_AUXDISPLAY=y CONFIG_HD44780=m CONFIG_DRM=m +CONFIG_DRM_PANIC=y +CONFIG_DRM_PANIC_BACKGROUND_COLOR=0xc41949 +CONFIG_DRM_PANIC_SCREEN="kmsg" CONFIG_DRM_LOAD_EDID_FIRMWARE=y CONFIG_DRM_UDL=m CONFIG_DRM_PANEL_LVDS=m @@ -1202,6 +1208,7 @@ CONFIG_SND_SOC_ICS43432=m CONFIG_SND_SOC_MA120X0P=m CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_PCM3168A_I2C=m +CONFIG_SND_SOC_RT5616=m CONFIG_SND_SOC_TLV320AIC23_I2C=m CONFIG_SND_SOC_WM8804_I2C=m CONFIG_SND_SOC_WM8904=m @@ -1413,6 +1420,9 @@ CONFIG_USB_G_ACM_MS=m CONFIG_USB_G_MULTI=m CONFIG_USB_G_HID=m CONFIG_USB_G_WEBCAM=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_FUSB302=m CONFIG_MMC=y CONFIG_MMC_BLOCK_MINORS=32 CONFIG_MMC_BCM2835_MMC=y @@ -1537,6 +1547,7 @@ CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m CONFIG_VIDEO_CODEC_BCM2835=m CONFIG_VIDEO_ISP_BCM2835=m +CONFIG_CLK_HOG=m CONFIG_COMMON_CLK_RP1=y CONFIG_COMMON_CLK_RP1_SDIO=y CONFIG_COMMON_CLK_SI5351=m @@ -1720,6 +1731,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DLM=m CONFIG_UNICODE=m +CONFIG_KEYS_REQUEST_CACHE=y CONFIG_KEY_DH_OPERATIONS=y CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/arm64/configs/bcm2712_defconfig b/arch/arm64/configs/bcm2712_defconfig index f79c71cfdbd98..27199c580c77b 100644 --- a/arch/arm64/configs/bcm2712_defconfig +++ b/arch/arm64/configs/bcm2712_defconfig @@ -95,6 +95,7 @@ CONFIG_LRU_GEN_ENABLED=y CONFIG_NUMA_EMU=y CONFIG_NET=y CONFIG_PACKET=y +CONFIG_TLS=m CONFIG_XFRM_USER=m CONFIG_XFRM_INTERFACE=m CONFIG_XFRM_SUB_POLICY=y @@ -547,9 +548,11 @@ CONFIG_R8169_LEDS=y CONFIG_MSE102X=m CONFIG_WIZNET_W5100=m CONFIG_WIZNET_W5100_SPI=m +CONFIG_MAXLINEAR_86110_PHY=y CONFIG_MICREL_PHY=y CONFIG_REALTEK_PHY_HWMON=y CONFIG_CAN_VCAN=m +CONFIG_CAN_VXCAN=m CONFIG_CAN_SLCAN=m CONFIG_CAN_MCP251X=m CONFIG_CAN_MCP251XFD=m @@ -1080,6 +1083,9 @@ CONFIG_VIDEO_I2C=m CONFIG_AUXDISPLAY=y CONFIG_HD44780=m CONFIG_DRM=m +CONFIG_DRM_PANIC=y +CONFIG_DRM_PANIC_BACKGROUND_COLOR=0xc41949 +CONFIG_DRM_PANIC_SCREEN="kmsg" CONFIG_DRM_LOAD_EDID_FIRMWARE=y CONFIG_DRM_UDL=m CONFIG_DRM_PANEL_LVDS=m @@ -1208,6 +1214,7 @@ CONFIG_SND_SOC_ICS43432=m CONFIG_SND_SOC_MA120X0P=m CONFIG_SND_SOC_MAX98357A=m CONFIG_SND_SOC_PCM3168A_I2C=m +CONFIG_SND_SOC_RT5616=m CONFIG_SND_SOC_TLV320AIC23_I2C=m CONFIG_SND_SOC_WM8804_I2C=m CONFIG_SND_SOC_WM8904=m @@ -1419,6 +1426,9 @@ CONFIG_USB_G_ACM_MS=m CONFIG_USB_G_MULTI=m CONFIG_USB_G_HID=m CONFIG_USB_G_WEBCAM=m +CONFIG_TYPEC=m +CONFIG_TYPEC_TCPM=m +CONFIG_TYPEC_FUSB302=m CONFIG_MMC=y CONFIG_MMC_BLOCK_MINORS=32 CONFIG_MMC_BCM2835_MMC=y @@ -1544,6 +1554,7 @@ CONFIG_SND_BCM2835=m CONFIG_VIDEO_BCM2835=m CONFIG_VIDEO_CODEC_BCM2835=m CONFIG_VIDEO_ISP_BCM2835=m +CONFIG_CLK_HOG=m CONFIG_COMMON_CLK_RP1=y CONFIG_COMMON_CLK_RP1_SDIO=y CONFIG_COMMON_CLK_SI5351=m @@ -1727,6 +1738,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_DLM=m CONFIG_UNICODE=m +CONFIG_KEYS_REQUEST_CACHE=y CONFIG_KEY_DH_OPERATIONS=y CONFIG_SECURITY=y CONFIG_SECURITY_APPARMOR=y diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h index 5fca480090434..795b351284673 100644 --- a/arch/arm64/include/asm/daifflags.h +++ b/arch/arm64/include/asm/daifflags.h @@ -19,7 +19,7 @@ /* mask/save/unmask/restore all exceptions, including interrupts. */ -static inline void local_daif_mask(void) +static __always_inline void local_daif_mask(void) { WARN_ON(system_has_prio_mask_debugging() && (read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF | @@ -38,7 +38,7 @@ static inline void local_daif_mask(void) trace_hardirqs_off(); } -static inline unsigned long local_daif_save_flags(void) +static __always_inline unsigned long local_daif_save_flags(void) { unsigned long flags; @@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void) return flags; } -static inline unsigned long local_daif_save(void) +static __always_inline unsigned long local_daif_save(void) { unsigned long flags; @@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void) return flags; } -static inline void local_daif_restore(unsigned long flags) +static __always_inline void local_daif_restore(unsigned long flags) { bool irq_disabled = flags & PSR_I_BIT; @@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags) * Called by synchronous exception handlers to restore the DAIF bits that were * modified by taking an exception. */ -static inline void local_daif_inherit(struct pt_regs *regs) +static __always_inline void local_daif_inherit(struct pt_regs *regs) { unsigned long flags = regs->pstate & DAIF_MASK; diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h index e1deed8244645..31058567eee46 100644 --- a/arch/arm64/include/asm/esr.h +++ b/arch/arm64/include/asm/esr.h @@ -130,7 +130,7 @@ * Annoyingly, the negative levels for Address size faults aren't laid out * contiguously (or in the desired order) */ -#define ESR_ELx_FSC_ADDRSZ_nL(n) ((n) == -1 ? 0x25 : 0x2C) +#define ESR_ELx_FSC_ADDRSZ_nL(n) ((n) == -1 ? 0x29 : 0x2C) #define ESR_ELx_FSC_ADDRSZ_L(n) ((n) < 0 ? ESR_ELx_FSC_ADDRSZ_nL(n) : \ (ESR_ELx_FSC_ADDRSZ + (n))) diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 635a43c4ec85b..9288021ba93b1 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -78,8 +78,12 @@ enum fixed_addresses { /* * Temporary boot-time mappings, used by early_ioremap(), * before ioremap() is functional. + * + * Reserve one extra page so a 256K mapping may start at any + * offset within a page. early_ioremap() maps the page-aligned + * physical range, so the initial offset can consume an extra page. */ -#define NR_FIX_BTMAPS (SZ_256K / PAGE_SIZE) +#define NR_FIX_BTMAPS ((SZ_256K / PAGE_SIZE) + 1) #define FIX_BTMAPS_SLOTS 7 #define TOTAL_FIX_BTMAPS (NR_FIX_BTMAPS * FIX_BTMAPS_SLOTS) diff --git a/arch/arm64/include/asm/kprobes.h b/arch/arm64/include/asm/kprobes.h index f2782560647be..35ce2c94040ef 100644 --- a/arch/arm64/include/asm/kprobes.h +++ b/arch/arm64/include/asm/kprobes.h @@ -26,6 +26,12 @@ struct prev_kprobe { struct kprobe *kp; unsigned int status; + + /* + * The original DAIF state of the outer kprobe, saved here before + * a nested kprobe overwrites kcb->saved_irqflag during reentry. + */ + unsigned long saved_irqflag; }; /* per-cpu kprobe control block */ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7bee8a357ceb0..feff0d13f052c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -397,9 +397,6 @@ struct kvm_arch { /* Masks for VNCR-backed and general EL2 sysregs */ struct kvm_sysreg_masks *sysreg_masks; - /* Count the number of VNCR_EL2 currently mapped */ - atomic_t vncr_map_count; - /* * For an untrusted host VM, 'pkvm.handle' is used to lookup * the associated pKVM instance in the hypervisor. diff --git a/arch/arm64/include/asm/kvm_nested.h b/arch/arm64/include/asm/kvm_nested.h index 6f768e361c894..b9d86614994db 100644 --- a/arch/arm64/include/asm/kvm_nested.h +++ b/arch/arm64/include/asm/kvm_nested.h @@ -257,6 +257,13 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) base = (val & GENMASK(36, 0)) << shift; + /* + * We only deal with at most 48bit VA/IPA, so 48 is where we + * sign-extend from. Should we support FEAT_L{VP}A* at some point, + * this will need to be revisited. + */ + base = (u64)sign_extend64(base, 48); + if (asid) *asid = FIELD_GET(TLBIR_ASID_MASK, val); @@ -264,6 +271,12 @@ static inline u64 decode_range_tlbi(u64 val, u64 *range, u16 *asid) num = FIELD_GET(GENMASK(43, 39), val); *range = __TLBI_RANGE_PAGES(num, scale) << shift; + /* Cap the range to the correct half of the address space */ + if (!(base & BIT(48))) + *range = min(*range, (BIT(48) - base)); + else + *range = min(*range, ~base + 1); + return base; } @@ -353,6 +366,8 @@ struct s1_walk_result { bool failed; }; +#define S1_MMU_DISABLED (-127) + static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) { wr->fst = fst; @@ -361,6 +376,11 @@ static inline void fail_s1_walk(struct s1_walk_result *wr, u8 fst, bool s1ptw) wr->failed = true; } +static inline bool s1_walk_translated(struct s1_walk_result *wr) +{ + return wr->level != S1_MMU_DISABLED; +} + int __kvm_translate_va(struct kvm_vcpu *vcpu, struct s1_walk_info *wi, struct s1_walk_result *wr, u64 va); int __kvm_find_s1_desc_level(struct kvm_vcpu *vcpu, u64 va, u64 ipa, diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 61d62bfd5a7bf..ba0517d80fa56 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -256,7 +256,7 @@ static inline void task_set_sve_vl_onexec(struct task_struct *task, #define SCTLR_USER_MASK \ (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | SCTLR_ELx_ENDA | SCTLR_ELx_ENDB | \ - SCTLR_EL1_TCF0_MASK) + SCTLR_EL1_TCF0_MASK | SCTLR_EL1_TCSO0_MASK) static inline void arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) diff --git a/arch/arm64/include/asm/ptdump.h b/arch/arm64/include/asm/ptdump.h index baff24004459e..f70e8743fca23 100644 --- a/arch/arm64/include/asm/ptdump.h +++ b/arch/arm64/include/asm/ptdump.h @@ -7,8 +7,6 @@ #include -DECLARE_STATIC_KEY_FALSE(arm64_ptdump_lock_key); - #ifdef CONFIG_PTDUMP #include @@ -52,6 +50,8 @@ struct ptdump_pg_state { const struct addr_marker *marker; const struct mm_struct *mm; unsigned long start_address; + /* exclusive end, ULONG_MAX represents an end at 1 << 64 */ + unsigned long end_address; int level; ptdesc_t current_prot; bool check_wx; diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index 2c8763876dfb7..c1fab41f671ec 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, return res.a0; } +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + /** * rsi_attestation_token_init - Initialise the operation to retrieve an * attestation token. @@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, static inline long rsi_attestation_token_init(const u8 *challenge, unsigned long size) { - struct arm_smccc_1_2_regs regs = { 0 }; + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; - /* The challenge must be at least 32bytes and at most 64bytes */ - if (!challenge || size < 32 || size > 64) + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) return -EINVAL; - regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; - memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); - if (regs.a0 == RSI_SUCCESS) - return regs.a1; + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; return -EINVAL; } diff --git a/arch/arm64/kernel/compat_alignment.c b/arch/arm64/kernel/compat_alignment.c index b68e1d328d4cb..9b58d0aa38d22 100644 --- a/arch/arm64/kernel/compat_alignment.c +++ b/arch/arm64/kernel/compat_alignment.c @@ -114,8 +114,8 @@ do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs) static int do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs) { - unsigned int rd, rn, nr_regs, regbits; - unsigned long eaddr, newaddr; + unsigned int rd, rn, regbits; + unsigned long eaddr, newaddr, nr_regs; unsigned int val; /* count the number of registers in the mask to be transferred */ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 30595bdadee9e..c00ec6c904c75 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -82,7 +82,7 @@ is_affected_midr_range(const struct arm64_cpu_capabilities *entry, int scope) for (i = 0; i < target_impl_cpu_num; i++) { if (__is_affected_midr_range(entry, target_impl_cpus[i].midr, - target_impl_cpus[i].midr)) + target_impl_cpus[i].revidr)) return true; } return false; diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba7..7bf1174277772 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); @@ -465,9 +467,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c index c0065a1d77cf3..00ebbf18b1c18 100644 --- a/arch/arm64/kernel/irq.c +++ b/arch/arm64/kernel/irq.c @@ -10,6 +10,7 @@ * Copyright (C) 2012 ARM Ltd. */ +#include #include #include #include @@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); DEFINE_PER_CPU(unsigned long *, irq_stack_ptr); - DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #ifdef CONFIG_SHADOW_CALL_STACK DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static int __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) - return; + return 0; + + for_each_possible_cpu(cpu) { + s = scs_alloc(early_cpu_to_node(cpu)); + if (!s) + return -ENOMEM; + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(early_cpu_to_node(cpu)); + return 0; } -static void __init init_irq_stacks(void) +static int __init init_irq_stacks(void) { int cpu; unsigned long *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu)); + if (!p) + return -ENOMEM; per_cpu(irq_stack_ptr, cpu) = p; } + + return 0; } #ifndef CONFIG_PREEMPT_RT @@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *)) void __init init_IRQ(void) { - init_irq_stacks(); - init_irq_scs(); + if (init_irq_stacks() || init_irq_scs()) + panic("Failed to allocate IRQ stack resources\n"); + irqchip_init(); if (system_uses_irq_prio_masking()) { diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 28df62051cc97..e40db39bcb47b 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -129,7 +129,8 @@ int machine_kexec_post_load(struct kimage *kimage) } /* Create a copy of the linear map */ - rc = trans_pgd_create_copy(&info, &trans_pgd, PAGE_OFFSET, PAGE_END); + rc = trans_pgd_create_copy(&info, &trans_pgd, + _PAGE_OFFSET(vabits_actual), PAGE_END); if (rc) return rc; kimage->arch.ttbr1 = __pa(trans_pgd); diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 43a0361a8bf04..7133da1653964 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -174,12 +174,27 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) { kcb->prev_kprobe.kp = kprobe_running(); kcb->prev_kprobe.status = kcb->kprobe_status; + + /* + * Save the outer kprobe's original DAIF flags before the nested + * kprobe calls kprobes_save_local_irqflag() and overwrites + * kcb->saved_irqflag. Without this, the outer kprobe will restore + * the wrong DAIF state and leave interrupts permanently masked. + */ + kcb->prev_kprobe.saved_irqflag = kcb->saved_irqflag; } static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) { __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); kcb->kprobe_status = kcb->prev_kprobe.status; + + /* + * Restore the outer kprobe's saved_irqflag so that when its + * single-step completes, kprobes_restore_local_irqflag() uses + * the correct original DAIF value. + */ + kcb->saved_irqflag = kcb->prev_kprobe.saved_irqflag; } static void __kprobes set_current_kprobe(struct kprobe *p) @@ -240,10 +255,16 @@ static int __kprobes reenter_kprobe(struct kprobe *p, switch (kcb->kprobe_status) { case KPROBE_HIT_SSDONE: case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SS: + /* + * A probe can be hit while another kprobe is preparing or + * executing its XOL single-step instruction. This is still a + * recoverable one-level reentry, so handle it in the same way as + * reentry from KPROBE_HIT_ACTIVE or KPROBE_HIT_SSDONE. + */ kprobes_inc_nmissed_count(p); setup_singlestep(p, regs, kcb, 1); break; - case KPROBE_HIT_SS: case KPROBE_REENTER: pr_warn("Failed to recover from reentered kprobes.\n"); dump_kprobe(p); diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index b3801f532b10b..5138002d72f15 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -1022,6 +1022,11 @@ static int __init parse_spectre_bhb_param(char *str) } early_param("nospectre_bhb", parse_spectre_bhb_param); +static bool spectre_bhb_mitigations_off(void) +{ + return __nospectre_bhb || cpu_mitigations_off(); +} + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1035,6 +1040,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { /* Do nothing */ + } else if (spectre_bhb_mitigations_off()) { + /* Mitigation disabled on the command line */ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); @@ -1200,6 +1207,6 @@ void spectre_print_disabled_mitigations(void) if (spectre_v4_mitigations_off()) pr_info("spectre-v4 %s", spectre_disabled_suffix); - if (__nospectre_bhb || cpu_mitigations_off()) + if (spectre_bhb_mitigations_off()) pr_info("spectre-bhb %s", spectre_disabled_suffix); } diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 2acc2c3dd033e..ceb047e7d49f2 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -801,7 +801,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +837,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 92b57fc87ee96..6af97292e3692 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1081,7 +1081,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 10fc6783cd608..5d2435dd76f35 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -96,14 +96,27 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, set_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER, &kvm->arch.flags); break; - case KVM_CAP_ARM_MTE: - mutex_lock(&kvm->lock); - if (system_supports_mte() && !kvm->created_vcpus) { - r = 0; - set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); + case KVM_CAP_ARM_MTE: { + struct kvm_memory_slot *memslot; + int bkt; + + guard(mutex)(&kvm->lock); + if (!system_supports_mte() || kvm->created_vcpus) + break; + + r = 0; + guard(mutex)(&kvm->slots_lock); + kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) { + if (kvm_slot_has_gmem(memslot)) { + r = -EINVAL; + break; + } } - mutex_unlock(&kvm->lock); + if (r == 0) + set_bit(KVM_ARCH_FLAG_MTE_ENABLED, &kvm->arch.flags); break; + + } case KVM_CAP_ARM_SYSTEM_SUSPEND: r = 0; set_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags); diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c index 2bdc49bef8613..6fa05138ca853 100644 --- a/arch/arm64/kvm/at.c +++ b/arch/arm64/kvm/at.c @@ -10,8 +10,6 @@ #include #include -#define S1_MMU_DISABLED (-127) - static int get_ia_size(struct s1_walk_info *wi) { return 64 - wi->txsz; diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c index 9984c492305a8..552f39d913be1 100644 --- a/arch/arm64/kvm/hyp/vhe/switch.c +++ b/arch/arm64/kvm/hyp/vhe/switch.c @@ -427,8 +427,7 @@ static bool kvm_hyp_handle_tlbi_el2(struct kvm_vcpu *vcpu, u64 *exit_code) * If we have to check for any VNCR mapping being invalidated, * go back to the slow path for further processing. */ - if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu) && - atomic_read(&vcpu->kvm->arch.vncr_map_count)) + if (vcpu_el2_e2h_is_set(vcpu) && vcpu_el2_tge_is_set(vcpu)) return false; __kvm_skip_instr(vcpu); diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 403a635483513..b4237699705b4 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2323,6 +2323,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, if (kvm_slot_has_gmem(new) && !kvm_memslot_is_gmem_only(new)) return -EINVAL; + /* guest_memfd is incompatible with MTE. */ + if (kvm_slot_has_gmem(new) && kvm_has_mte(kvm)) + return -EINVAL; + hva = new->userspace_addr; reg_end = hva + (new->npages << PAGE_SHIFT); diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index 10da38dff8b91..869209e8bd1cf 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -27,7 +27,7 @@ struct vncr_tlb { bool hpa_writable; /* -1 when not mapped on a CPU */ - int cpu; + atomic_t cpu; /* * true if the TLB is valid. Can only be changed with the @@ -48,7 +48,6 @@ void kvm_init_nested(struct kvm *kvm) { kvm->arch.nested_mmus = NULL; kvm->arch.nested_mmus_size = 0; - atomic_set(&kvm->arch.vncr_map_count, 0); } static int init_nested_s2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu) @@ -387,7 +386,7 @@ int kvm_walk_nested_s2(struct kvm_vcpu *vcpu, phys_addr_t gipa, return ret; } -static unsigned int ttl_to_size(u8 ttl) +static unsigned int __ttl_to_size(u8 ttl) { int level = ttl & 3; int gran = (ttl >> 2) & 3; @@ -443,10 +442,22 @@ static unsigned int ttl_to_size(u8 ttl) return max_size; } -static u8 pgshift_level_to_ttl(u16 shift, u8 level) +static unsigned int ttl_to_size(u8 ttl) +{ + return __ttl_to_size(ttl) ?: SZ_1G; +} + +static u8 pgshift_level_to_ttl(u16 shift, s8 level) { u8 ttl; + /* + * If we don't have a proper level, fallback to the maximum + * size. + */ + if (level < 0) + return 0; + switch(shift) { case 12: ttl = TLBI_TTL_TG_4K; @@ -556,7 +567,11 @@ unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val) ttl = get_guest_mapping_ttl(mmu, addr); } - max_size = ttl_to_size(ttl); + /* + * Don't use the default 1GB fallback, as we can adapt to the + * max mapping size we allow at S2. + */ + max_size = __ttl_to_size(ttl); if (!max_size) { /* Compute the maximum extent of the invalidation */ @@ -755,18 +770,47 @@ void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu) } } +/* + * Unmapping an L1 VNCR can happen concurrently without the mmu lock being + * effective (vcpu_put() vs TLBI handling). The atomic_xchg below ensures + * that only one CPU sets it to -1 while getting a valid CPU number back. + */ +static int unmap_l1_vncr(struct vncr_tlb *vt) +{ + int cpu = atomic_xchg_relaxed(&vt->cpu, -1); + + if (cpu != -1) + clear_fixmap(vncr_fixmap(cpu)); + + return cpu; +} + +static void this_cpu_reset_vncr_fixmap(struct kvm_vcpu *vcpu) +{ + if (!host_data_test_flag(L1_VNCR_MAPPED)) + return; + + BUG_ON(is_hyp_ctxt(vcpu)); + + /* + * Unconditionally unmap the local VNCR if we have lost the race + * against a concurrent TLBI. Otherwise we could end-up running + * another vcpu with VNCR still mapped if the TLBI thread is + * preempted between the exchange and the clear_fixmap(). + * + * Note that we do not care about the TLBI nuking the fixmap behind + * the back of an running vcpu. This will only generate a fault and + * possibly a retranslation. + */ + if (unmap_l1_vncr(vcpu->arch.vncr_tlb) == -1) + clear_fixmap(vncr_fixmap(smp_processor_id())); + host_data_clear_flag(L1_VNCR_MAPPED); +} + void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu) { /* Unconditionally drop the VNCR mapping if we have one */ - if (host_data_test_flag(L1_VNCR_MAPPED)) { - BUG_ON(vcpu->arch.vncr_tlb->cpu != smp_processor_id()); - BUG_ON(is_hyp_ctxt(vcpu)); - - clear_fixmap(vncr_fixmap(vcpu->arch.vncr_tlb->cpu)); - vcpu->arch.vncr_tlb->cpu = -1; - host_data_clear_flag(L1_VNCR_MAPPED); - atomic_dec(&vcpu->kvm->arch.vncr_map_count); - } + this_cpu_reset_vncr_fixmap(vcpu); /* * Keep a reference on the associated stage-2 MMU if the vCPU is @@ -848,8 +892,21 @@ u16 get_asid_by_regime(struct kvm_vcpu *vcpu, enum trans_regime regime) static void invalidate_vncr(struct vncr_tlb *vt) { vt->valid = false; - if (vt->cpu != -1) - clear_fixmap(vncr_fixmap(vt->cpu)); + unmap_l1_vncr(vt); +} + +static bool vncr_tlb_intersects(struct vncr_tlb *vt, u64 addr, + u64 scope_start, u64 scope_size) +{ + u64 tlb_size, tlb_start, tlb_end, scope_end; + + tlb_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, vt->wr.level)); + + tlb_start = addr & ~(tlb_size - 1); + tlb_end = tlb_start + tlb_size - 1; + scope_end = scope_start + scope_size - 1; + + return !(tlb_end < scope_start || tlb_start > scope_end); } /* @@ -874,19 +931,15 @@ static void kvm_invalidate_vncr_ipa(struct kvm *kvm, u64 start, u64 end) if (!kvm_has_feat(kvm, ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)) return; - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 ipa_start, ipa_end, ipa_size; - - ipa_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - ipa_start = vt->wr.pa & ~(ipa_size - 1); - ipa_end = ipa_start + ipa_size; - - if (ipa_end <= start || ipa_start >= end) - continue; - - invalidate_vncr(vt); - } + /* + * Note that invalidating the VNCR on the back of an MMU notifier + * doesn't require messing with the invalidation counter for a + * parallel walk. The notifier itself will have bumped the counter, + * making sure we rewalk. + */ + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) + if (vncr_tlb_intersects(vt, vt->wr.pa, start, end - start)) + invalidate_vncr(vt); } struct s1e2_tlbi_scope { @@ -911,29 +964,29 @@ static void invalidate_vncr_va(struct kvm *kvm, lockdep_assert_held_write(&kvm->mmu_lock); - kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { - u64 va_start, va_end, va_size; - - va_size = ttl_to_size(pgshift_level_to_ttl(vt->wi.pgshift, - vt->wr.level)); - va_start = vt->gva & ~(va_size - 1); - va_end = va_start + va_size; + /* + * We might be performing a parallel S1 walk, so bump up the + * invalidation counter even in the absence of an actual VNCR TLB + * invalidation, as this could indicate that the guest has gone + * through a BBM sequence. + */ + kvm->mmu_invalidate_seq++; + smp_wmb(); + kvm_for_each_vncr_tlb(i, vcpu, vt, kvm) { switch (scope->type) { case TLBI_ALL: break; case TLBI_VA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; if (vt->wr.nG && vt->wr.asid != scope->asid) continue; break; case TLBI_VAA: - if (va_end <= scope->va || - va_start >= (scope->va + scope->size)) + if (!vncr_tlb_intersects(vt, vt->gva, scope->va, scope->size)) continue; break; @@ -993,8 +1046,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VALE1OSNXS: scope->type = TLBI_VA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); scope->asid = FIELD_GET(TLBIR_ASID_MASK, val); break; @@ -1021,8 +1072,6 @@ static void compute_s1_tlbi_range(struct kvm_vcpu *vcpu, u32 inst, u64 val, case OP_TLBI_VAALE1OSNXS: scope->type = TLBI_VAA; scope->size = ttl_to_size(FIELD_GET(TLBI_TTL_MASK, val)); - if (!scope->size) - scope->size = SZ_1G; scope->va = tlbi_va_s1_to_va(val) & ~(scope->size - 1); break; case OP_TLBI_RVAE2: @@ -1222,7 +1271,8 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) * We also prepare the next walk wilst we're at it. */ scoped_guard(write_lock, &vcpu->kvm->mmu_lock) { - invalidate_vncr(vt); + this_cpu_reset_vncr_fixmap(vcpu); + vt->valid = false; vt->wi = (struct s1_walk_info) { .regime = TR_EL20, @@ -1236,15 +1286,15 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) va = read_vncr_el2(vcpu); + mmu_seq = vcpu->kvm->mmu_invalidate_seq; + smp_rmb(); + ret = __kvm_translate_va(vcpu, &vt->wi, &vt->wr, va); if (ret) return ret; write_fault = kvm_is_write_fault(vcpu); - mmu_seq = vcpu->kvm->mmu_invalidate_seq; - smp_rmb(); - gfn = vt->wr.pa >> PAGE_SHIFT; memslot = gfn_to_memslot(vcpu->kvm, gfn); if (!memslot) { @@ -1292,7 +1342,7 @@ static int kvm_translate_vncr(struct kvm_vcpu *vcpu, bool *is_gmem) vt->hpa = pfn << PAGE_SHIFT; vt->hpa_writable = writable; vt->valid = true; - vt->cpu = -1; + atomic_set(&vt->cpu, -1); kvm_make_request(KVM_REQ_MAP_L1_VNCR_EL2, vcpu); kvm_release_faultin_page(vcpu->kvm, page, false, vt->wr.pw && vt->hpa_writable); @@ -1413,14 +1463,16 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) if (!vt->valid) return; + /* We cache the MMU state in the TLB. Check that it matches. */ + if (!!(vcpu_read_sys_reg(vcpu, SCTLR_EL2) & SCTLR_ELx_M) != s1_walk_translated(&vt->wr)) + return; + if (read_vncr_el2(vcpu) != vt->gva) return; if (vt->wr.nG && get_asid_by_regime(vcpu, TR_EL20) != vt->wr.asid) return; - vt->cpu = smp_processor_id(); - if (vt->hpa_writable && vt->wr.pw && vt->wr.pr) prot = PAGE_KERNEL; else if (vt->wr.pr) @@ -1435,9 +1487,9 @@ static void kvm_map_l1_vncr(struct kvm_vcpu *vcpu) * FIXME: WO doesn't work at all, need POE support in the kernel. */ if (pgprot_val(prot) != pgprot_val(PAGE_NONE)) { - __set_fixmap(vncr_fixmap(vt->cpu), vt->hpa, prot); + atomic_set(&vt->cpu, smp_processor_id()); + __set_fixmap(vncr_fixmap(atomic_read(&vt->cpu)), vt->hpa, prot); host_data_set_flag(L1_VNCR_MAPPED); - atomic_inc(&vcpu->kvm->arch.vncr_map_count); } } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 7b7f3c932dcd5..6cb126585e71c 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -3943,6 +3943,7 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, u32 sys_encoding = sys_insn(p->Op0, p->Op1, p->CRn, p->CRm, p->Op2); u64 vttbr = vcpu_read_sys_reg(vcpu, VTTBR_EL2); u64 base, range; + int pa_bits; if (!kvm_supported_tlbi_ipas2_op(vcpu, sys_encoding)) return undef_access(vcpu, p, r); @@ -3954,6 +3955,16 @@ static bool handle_ripas2e1is(struct kvm_vcpu *vcpu, struct sys_reg_params *p, */ base = decode_range_tlbi(p->regval, &range, NULL); + /* + * Ignore TLBIs that start out of PA_bits range, and cap the + * invalidation to the [base:bit(PA_bits)] interval. + */ + pa_bits = kvm_get_pa_bits(vcpu->kvm); + if (fls64(base) > pa_bits) + return true; + + range = min(range, BIT_ULL(pa_bits) - base); + kvm_s2_mmu_iterate_by_vmid(vcpu->kvm, get_vmid(vttbr), &(union tlbi_info) { .range = { diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 30fa88e49be49..f5f3071f21a72 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -173,6 +173,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type) } kvm->arch.vgic.vgic_model = 0; + kvm->arch.vgic.in_kernel = false; goto out_unlock; } diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c index d2636d3391090..cdaf2af5e765d 100644 --- a/arch/arm64/kvm/vgic/vgic-its.c +++ b/arch/arm64/kvm/vgic/vgic-its.c @@ -2112,6 +2112,14 @@ static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev, u32 next_offset; u64 val; + /* + * MAPC with V=0 keeps the ITEs mapped but drops their collection, + * and with it the ICID. Save a zeroed entry, which the restore path + * reads back as invalid. + */ + if (!ite->collection) + return vgic_its_write_entry_lock(its, gpa, 0ULL, ite); + next_offset = compute_next_eventid_offset(&dev->itt_head, ite); val = ((u64)next_offset << KVM_ITS_ITE_NEXT_SHIFT) | ((u64)ite->irq->intid << KVM_ITS_ITE_PINTID_SHIFT) | diff --git a/arch/arm64/kvm/vgic/vgic-v3-nested.c b/arch/arm64/kvm/vgic/vgic-v3-nested.c index 7f1259b49c505..1ab2ef0e50b94 100644 --- a/arch/arm64/kvm/vgic/vgic-v3-nested.c +++ b/arch/arm64/kvm/vgic/vgic-v3-nested.c @@ -149,7 +149,7 @@ static void vgic_compute_mi_state(struct kvm_vcpu *vcpu, struct mi_state *mi_sta eisr |= BIT(i); if (!(lr & ICH_LR_STATE)) elrsr |= BIT(i); - pend |= (lr & ICH_LR_PENDING_BIT); + pend |= (lr & ICH_LR_STATE) == ICH_LR_PENDING_BIT; } mi_state->eisr = eisr; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 2f75ef14d3399..b8d9558202cb1 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -420,9 +420,13 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) bool is_pending; bool stored; + irq = vgic_get_irq(kvm, index); + if (!irq) + continue; + vcpu = irq->target_vcpu; if (!vcpu) - continue; + goto put_irq; pendbase = GICR_PENDBASER_ADDRESS(vcpu->arch.vgic_cpu.pendbaser); @@ -433,7 +437,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) if (ptr != last_ptr) { ret = kvm_read_guest_lock(kvm, ptr, &val, 1); if (ret) - goto out; + goto put_irq; last_ptr = ptr; } @@ -445,7 +449,7 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) vgic_v4_get_vlpi_state(irq, &is_pending); if (stored == is_pending) - continue; + goto put_irq; if (is_pending) val |= 1 << bit_nr; @@ -453,6 +457,8 @@ int vgic_v3_save_pending_tables(struct kvm *kvm) val &= ~(1 << bit_nr); ret = vgic_write_guest_lock(kvm, ptr, &val, 1); +put_irq: + vgic_put_irq(kvm, irq); if (ret) goto out; } diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 633e5223d944d..a74507503a4a3 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 + +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 4c62082b9a3bd..4d56cdbd0960d 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,9 @@ static void show_pte(unsigned long addr) pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n", mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K, vabits_actual, mm_to_pgd_phys(mm)); + + guard(irqsave)(); + pgdp = pgd_offset(mm, addr); pgd = READ_ONCE(*pgdp); pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd)); @@ -164,25 +168,25 @@ static void show_pte(unsigned long addr) if (pgd_none(pgd) || pgd_bad(pgd)) break; - p4dp = p4d_offset(pgdp, addr); + p4dp = p4d_offset_lockless(pgdp, pgd, addr); p4d = READ_ONCE(*p4dp); pr_cont(", p4d=%016llx", p4d_val(p4d)); if (p4d_none(p4d) || p4d_bad(p4d)) break; - pudp = pud_offset(p4dp, addr); + pudp = pud_offset_lockless(p4dp, p4d, addr); pud = READ_ONCE(*pudp); pr_cont(", pud=%016llx", pud_val(pud)); if (pud_none(pud) || pud_bad(pud)) break; - pmdp = pmd_offset(pudp, addr); + pmdp = pmd_offset_lockless(pudp, pud, addr); pmd = READ_ONCE(*pmdp); pr_cont(", pmd=%016llx", pmd_val(pmd)); if (pmd_none(pmd) || pmd_bad(pmd)) break; - ptep = pte_offset_map(pmdp, addr); + ptep = pte_offset_map(&pmd, addr); if (!ptep) break; diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c index 1d90a7e753336..fa63c96dbf79d 100644 --- a/arch/arm64/mm/hugetlbpage.c +++ b/arch/arm64/mm/hugetlbpage.c @@ -91,7 +91,7 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr, p4dp = p4d_offset(pgdp, addr); pudp = pud_offset(p4dp, addr); pmdp = pmd_offset(pudp, addr); - if ((pte_t *)pmdp == ptep) { + if ((pte_t *)PTR_ALIGN_DOWN(pmdp, sizeof(*pmdp) * CONT_PMDS) == ptep) { *pgsize = PMD_SIZE; return CONT_PMDS; } diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 971cffdb8b48c..a4503910afb8b 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -51,8 +51,6 @@ #define INVALID_PHYS_ADDR (-1ULL) -DEFINE_STATIC_KEY_FALSE(arm64_ptdump_lock_key); - u64 kimage_voffset __ro_after_init; EXPORT_SYMBOL(kimage_voffset); @@ -1878,8 +1876,7 @@ int pmd_clear_huge(pmd_t *pmdp) return 1; } -static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr, - bool acquire_mmap_lock) +int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) { pte_t *table; pmd_t pmd; @@ -1891,25 +1888,13 @@ static int __pmd_free_pte_page(pmd_t *pmdp, unsigned long addr, return 1; } - /* See comment in pud_free_pmd_page for static key logic */ table = pte_offset_kernel(pmdp, addr); pmd_clear(pmdp); __flush_tlb_kernel_pgtable(addr); - if (static_branch_unlikely(&arm64_ptdump_lock_key) && acquire_mmap_lock) { - mmap_read_lock(&init_mm); - mmap_read_unlock(&init_mm); - } - pte_free_kernel(NULL, table); return 1; } -int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr) -{ - /* If ptdump is walking the pagetables, acquire init_mm.mmap_lock */ - return __pmd_free_pte_page(pmdp, addr, /* acquire_mmap_lock = */ true); -} - int pud_free_pmd_page(pud_t *pudp, unsigned long addr) { pmd_t *table; @@ -1925,36 +1910,16 @@ int pud_free_pmd_page(pud_t *pudp, unsigned long addr) } table = pmd_offset(pudp, addr); - - /* - * Our objective is to prevent ptdump from reading a PMD table which has - * been freed. In this race, if pud_free_pmd_page observes the key on - * (which got flipped by ptdump) then the mmap lock sequence here will, - * as a result of the mmap write lock/unlock sequence in ptdump, give - * us the correct synchronization. If not, this means that ptdump has - * yet not started walking the pagetables - the sequence of barriers - * issued by __flush_tlb_kernel_pgtable() guarantees that ptdump will - * observe an empty PUD. - */ - pud_clear(pudp); - __flush_tlb_kernel_pgtable(addr); - if (static_branch_unlikely(&arm64_ptdump_lock_key)) { - mmap_read_lock(&init_mm); - mmap_read_unlock(&init_mm); - } - pmdp = table; next = addr; end = addr + PUD_SIZE; do { if (pmd_present(pmdp_get(pmdp))) - /* - * PMD has been isolated, so ptdump won't see it. No - * need to acquire init_mm.mmap_lock. - */ - __pmd_free_pte_page(pmdp, next, /* acquire_mmap_lock = */ false); + pmd_free_pte_page(pmdp, next); } while (pmdp++, next += PMD_SIZE, next != end); + pud_clear(pudp); + __flush_tlb_kernel_pgtable(addr); pmd_free(NULL, table); return 1; } diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index ab9899ca1e5f2..a1ad028bdb30d 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -278,16 +278,19 @@ void note_page_pgd(struct ptdump_state *pt_st, unsigned long addr, pgd_t pgd) void note_page_flush(struct ptdump_state *pt_st) { + struct ptdump_pg_state *st = container_of(pt_st, struct ptdump_pg_state, ptdump); + unsigned long end = st->end_address; pte_t pte_zero = {0}; - note_page(pt_st, 0, -1, pte_val(pte_zero)); -} + /* + * Address spaces that end at 1 << 64 have end_address == ULONG_MAX, + * but note_page() expects the exclusive end. In this case adjust end + * to the wraparound value 0. + */ + if (end == ULONG_MAX) + end = 0; -static void arm64_ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm) -{ - static_branch_inc(&arm64_ptdump_lock_key); - ptdump_walk_pgd(st, mm, NULL); - static_branch_dec(&arm64_ptdump_lock_key); + note_page(pt_st, end, -1, pte_val(pte_zero)); } void ptdump_walk(struct seq_file *s, struct ptdump_info *info) @@ -303,6 +306,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info) .marker = info->markers, .mm = info->mm, .pg_level = &kernel_pg_levels[0], + .end_address = end, .level = -1, .ptdump = { .note_page_pte = note_page_pte, @@ -318,7 +322,7 @@ void ptdump_walk(struct seq_file *s, struct ptdump_info *info) } }; - arm64_ptdump_walk_pgd(&st.ptdump, info->mm); + ptdump_walk_pgd(&st.ptdump, info->mm, NULL); } static void __init ptdump_initialize(void) @@ -344,6 +348,7 @@ bool ptdump_check_wx(void) { -1, NULL}, }, .pg_level = &kernel_pg_levels[0], + .end_address = ~0UL, .level = -1, .check_wx = true, .ptdump = { @@ -360,7 +365,7 @@ bool ptdump_check_wx(void) } }; - arm64_ptdump_walk_pgd(&st.ptdump, &init_mm); + ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); if (st.wx_pages || st.uxn_pages) { pr_warn("Checked W+X mappings: FAILED, %lu W+X pages found, %lu non-UXN pages found\n", diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c563bae8fce1c..7be20cb32a271 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1168,7 +1168,12 @@ static int add_exception_handler(const struct bpf_insn *insn, ex->insn = ins_offset; - if (BPF_CLASS(insn->code) != BPF_LDX) + /* + * A load-acquire is of BPF_STX class, but reads from src_reg into + * dst_reg like a BPF_LDX does, hence it must not be treated as a store + * here. + */ + if (BPF_CLASS(insn->code) != BPF_LDX && !bpf_atomic_is_load_acq(insn)) dst_reg = DONT_CLEAR; ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg); @@ -1183,7 +1188,7 @@ static int add_exception_handler(const struct bpf_insn *insn, * memory access. Pass the reg holding the unmodified 32-bit address to * ex_handler_bpf. */ - if (BPF_CLASS(insn->code) == BPF_LDX) + if (BPF_CLASS(insn->code) == BPF_LDX || bpf_atomic_is_load_acq(insn)) arena_reg = bpf2a64[insn->src_reg]; else arena_reg = bpf2a64[insn->dst_reg]; @@ -2428,9 +2433,8 @@ static void clear_garbage(struct jit_ctx *ctx, int reg, int effective_bytes) } static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, - const struct btf_func_model *m, - const struct arg_aux *a, - bool for_call_origin) + const struct btf_func_model *m, const struct arg_aux *a, + bool for_call_origin, bool is_struct_ops) { int i; int reg; @@ -2450,7 +2454,15 @@ static void save_args(struct jit_ctx *ctx, int bargs_off, int oargs_off, bargs_off += 8; } - soff = 32; /* on stack arguments start from FP + 32 */ + /* + * On-stack arguments start above the frame(s) pushed by the trampoline + * prologue. Entered through the fentry call from a traced function, the + * prologue saves both the parent (FP/x9) and the traced function + * (FP/LR) frames, so the arguments start at FP + 32. A struct_ops + * callback is called indirectly and only the FP/LR frame is saved, so + * they start at FP + 16. + */ + soff = is_struct_ops ? 16 : 32; doff = (for_call_origin ? oargs_off : bargs_off); /* save on stack arguments */ @@ -2628,7 +2640,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, emit(A64_STR64I(A64_R(10), A64_SP, nfuncargs_off), ctx); /* save args for bpf */ - save_args(ctx, bargs_off, oargs_off, m, a, false); + save_args(ctx, bargs_off, oargs_off, m, a, false, is_struct_ops); /* save callee saved registers */ emit(A64_STR64I(A64_R(19), A64_SP, regs_off), ctx); @@ -2660,7 +2672,7 @@ static int prepare_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, if (flags & BPF_TRAMP_F_CALL_ORIG) { /* save args for original func */ - save_args(ctx, bargs_off, oargs_off, m, a, true); + save_args(ctx, bargs_off, oargs_off, m, a, true, is_struct_ops); /* call original func */ emit(A64_LDR64I(A64_R(10), A64_SP, retaddr_off), ctx); emit(A64_ADR(A64_LR, AARCH64_INSN_SIZE * 2), ctx); diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index de46df0e3b115..02f16cba0cf1a 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -3672,7 +3672,9 @@ Sysreg ICC_CR0_EL1 3 1 12 0 1 Res0 63:39 Field 38 PID Field 37:32 IPPT -Res0 31:1 +Res0 31:3 +Field 2 LINK_IDLE +Field 1 LINK Field 0 EN EndSysreg diff --git a/arch/csky/kernel/entry.S b/arch/csky/kernel/entry.S index c68cdcc76d60e..3261f46f22442 100644 --- a/arch/csky/kernel/entry.S +++ b/arch/csky/kernel/entry.S @@ -93,11 +93,11 @@ csky_syscall_trace: ldw a2, (sp, LSAVE_A2) ldw a3, (sp, LSAVE_A3) #if defined(__CSKYABIV2__) - subi sp, 8 ldw r9, (sp, LSAVE_A4) + ldw r10, (sp, LSAVE_A5) + subi sp, 8 stw r9, (sp, 0x0) - ldw r9, (sp, LSAVE_A5) - stw r9, (sp, 0x4) + stw r10, (sp, 0x4) jsr syscallid /* Do system call */ addi sp, 8 #else diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig index 5b1116733d881..12d88033b0b44 100644 --- a/arch/loongarch/Kconfig +++ b/arch/loongarch/Kconfig @@ -173,7 +173,7 @@ config LOONGARCH select HAVE_RELIABLE_STACKTRACE if UNWINDER_ORC select HAVE_RETHOOK select HAVE_RSEQ - select HAVE_RUST + select HAVE_RUST if !KASAN select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA if NUMA diff --git a/arch/loongarch/include/asm/kvm_host.h b/arch/loongarch/include/asm/kvm_host.h index 377d7fb04bda1..735396787b147 100644 --- a/arch/loongarch/include/asm/kvm_host.h +++ b/arch/loongarch/include/asm/kvm_host.h @@ -122,6 +122,7 @@ struct kvm_arch { unsigned int pte_shifts[MAX_PGTABLE_LEVELS]; unsigned int root_level; spinlock_t phyid_map_lock; + spinlock_t pv_setting_lock; struct kvm_phyid_map *phyid_map; /* Enabled PV features */ unsigned long pv_features; diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h index f51fa0ecb366a..e3563a52e5b5b 100644 --- a/arch/loongarch/include/asm/pgtable.h +++ b/arch/loongarch/include/asm/pgtable.h @@ -106,6 +106,13 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]; #define KFENCE_AREA_START (VMEMMAP_END + 1) #define KFENCE_AREA_END (KFENCE_AREA_START + KFENCE_AREA_SIZE - 1) +/* Needed to limit get_free_mem_region() */ +#ifndef CONFIG_SPARSEMEM +#define DIRECT_MAP_PHYSMEM_END ((1ULL << (cpu_pabits + 1)) - 1) +#else +#define DIRECT_MAP_PHYSMEM_END min((1ULL << (cpu_pabits + 1)) - 1, (1ULL << MAX_PHYSMEM_BITS) - 1) +#endif + #define ptep_get(ptep) READ_ONCE(*(ptep)) #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c index 88ad24d1e729a..48230f2d5493e 100644 --- a/arch/loongarch/kernel/acpi.c +++ b/arch/loongarch/kernel/acpi.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -201,10 +202,12 @@ static void __init acpi_process_madt(void) } int pptt_enabled; +static int acpi_nr_packages; +static int acpi_package_ids[MAX(MAX_PACKAGES, KVM_MAX_VCPUS)]; int __init parse_acpi_topology(void) { - int cpu, topology_id; + int i, cpu, topology_id; for_each_possible_cpu(cpu) { topology_id = find_acpi_cpu_topology(cpu, 0); @@ -222,6 +225,29 @@ int __init parse_acpi_topology(void) cpu_data[cpu].core = topology_id; } + + topology_id = find_acpi_cpu_topology_package(cpu); + if (topology_id < 0) { + pr_warn("Invalid BIOS PPTT\n"); + return -ENOENT; + } + + for (i = 0; i < acpi_nr_packages; i++) + if (acpi_package_ids[i] == topology_id) + break; + + if (i == acpi_nr_packages) + acpi_package_ids[acpi_nr_packages++] = topology_id; + + cpu_data[cpu].package = topology_id; + } + + for_each_possible_cpu(cpu) { + for (i = 0; i < acpi_nr_packages; i++) + if (cpu_data[cpu].package == acpi_package_ids[i]) { + cpu_data[cpu].package = i; /* Canonicalize */ + break; + } } pptt_enabled = 1; diff --git a/arch/loongarch/kernel/kgdb.c b/arch/loongarch/kernel/kgdb.c index 7be5b4c0c9002..6728b2d67dd32 100644 --- a/arch/loongarch/kernel/kgdb.c +++ b/arch/loongarch/kernel/kgdb.c @@ -252,7 +252,8 @@ static int kgdb_loongarch_notify(struct notifier_block *self, unsigned long cmd, if (atomic_read(&kgdb_active) != -1) kgdb_nmicallback(smp_processor_id(), regs); - if (kgdb_handle_exception(args->trapnr, args->signr, cmd, regs)) + if (kgdb_handle_exception(regs->csr_era == stepped_address ? 0 : args->trapnr, + args->signr, cmd, regs)) return NOTIFY_DONE; if (atomic_read(&kgdb_setting_breakpoint)) diff --git a/arch/loongarch/kernel/kprobes.c b/arch/loongarch/kernel/kprobes.c index 1985ed30dd16f..ddfefea174727 100644 --- a/arch/loongarch/kernel/kprobes.c +++ b/arch/loongarch/kernel/kprobes.c @@ -275,6 +275,9 @@ bool kprobe_singlestep_handler(struct pt_regs *regs) struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); unsigned long addr = instruction_pointer(regs); + if (!cur) + return false; + if (cur && (kcb->kprobe_status & (KPROBE_HIT_SS | KPROBE_REENTER)) && ((unsigned long)&cur->ainsn.insn[1] == addr)) { restore_local_irqflag(kcb, regs); diff --git a/arch/loongarch/kernel/machine_kexec.c b/arch/loongarch/kernel/machine_kexec.c index d7fafda1d5417..1883cae93bc31 100644 --- a/arch/loongarch/kernel/machine_kexec.c +++ b/arch/loongarch/kernel/machine_kexec.c @@ -42,6 +42,7 @@ static unsigned long first_ind_entry; int machine_kexec_prepare(struct kimage *kimage) { int i; + char head[8]; char *bootloader = "kexec"; void *cmdline_ptr = (void *)KEXEC_CMDLINE_ADDR; @@ -59,7 +60,9 @@ int machine_kexec_prepare(struct kimage *kimage) } else { /* Find the command line */ for (i = 0; i < kimage->nr_segments; i++) { - if (!strncmp(bootloader, (char __user *)kimage->segment[i].buf, strlen(bootloader))) { + if (copy_from_user(head, kimage->segment[i].buf, strlen(bootloader))) + continue; + if (!strncmp(bootloader, head, strlen(bootloader))) { if (!copy_from_user(cmdline_ptr, kimage->segment[i].buf, COMMAND_LINE_SIZE)) kimage->arch.cmdline_ptr = (unsigned long)cmdline_ptr; break; diff --git a/arch/loongarch/kernel/rethook_trampoline.S b/arch/loongarch/kernel/rethook_trampoline.S index d4ceb2fa2a5ce..0298ebced64ae 100644 --- a/arch/loongarch/kernel/rethook_trampoline.S +++ b/arch/loongarch/kernel/rethook_trampoline.S @@ -24,7 +24,6 @@ cfi_st t6, PT_R18 cfi_st t7, PT_R19 cfi_st t8, PT_R20 - cfi_st u0, PT_R21 cfi_st fp, PT_R22 cfi_st s0, PT_R23 cfi_st s1, PT_R24 @@ -59,7 +58,6 @@ cfi_ld t6, PT_R18 cfi_ld t7, PT_R19 cfi_ld t8, PT_R20 - cfi_ld u0, PT_R21 cfi_ld fp, PT_R22 cfi_ld s0, PT_R23 cfi_ld s1, PT_R24 diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 75e3dfa060553..c0b59b813f8e9 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -610,6 +610,7 @@ void __init setup_arch(char **cmdline_p) memblock_init(); pagetable_init(); bootcmdline_init(cmdline_p); + jump_label_init(); /* Initialise the static keys for early params */ parse_early_param(); reserve_initrd_mem(); @@ -617,8 +618,6 @@ void __init setup_arch(char **cmdline_p) arch_mem_init(cmdline_p); resource_init(); - jump_label_init(); /* Initialise the static keys for paravirtualization */ - #ifdef CONFIG_SMP plat_smp_setup(); prefill_possible_map(); diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c index 5d6a60294fb2d..4e3d79c38c46e 100644 --- a/arch/loongarch/kernel/smp.c +++ b/arch/loongarch/kernel/smp.c @@ -412,10 +412,10 @@ void loongson_init_secondary(void) numa_add_cpu(cpu); #endif per_cpu(cpu_state, cpu) = CPU_ONLINE; - cpu_data[cpu].package = - cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; cpu_data[cpu].core = pptt_enabled ? cpu_data[cpu].core : cpu_logical_map(cpu) % loongson_sysconf.cores_per_package; + cpu_data[cpu].package = pptt_enabled ? cpu_data[cpu].package : + cpu_logical_map(cpu) / loongson_sysconf.cores_per_package; cpu_data[cpu].global_id = cpu_logical_map(cpu); } @@ -535,28 +535,32 @@ int hibernate_resume_nonboot_cpu_disable(void) */ #ifdef CONFIG_PM -static int loongson_ipi_suspend(void) +static int loongson_ipi_suspend(void *data) { return 0; } -static void loongson_ipi_resume(void) +static void loongson_ipi_resume(void *data) { iocsr_write32(0xffffffff, LOONGARCH_IOCSR_IPI_EN); } -static struct syscore_ops loongson_ipi_syscore_ops = { +static const struct syscore_ops loongson_ipi_syscore_ops = { .resume = loongson_ipi_resume, .suspend = loongson_ipi_suspend, }; +static struct syscore loongson_ipi_syscore = { + .ops = &loongson_ipi_syscore_ops, +}; + /* * Enable boot cpu ipi before enabling nonboot cpus * during syscore_resume. */ static int __init ipi_pm_init(void) { - register_syscore_ops(&loongson_ipi_syscore_ops); + register_syscore(&loongson_ipi_syscore); return 0; } diff --git a/arch/loongarch/kvm/exit.c b/arch/loongarch/kvm/exit.c index 7a3828de2f1fb..7074094f8e817 100644 --- a/arch/loongarch/kvm/exit.c +++ b/arch/loongarch/kvm/exit.c @@ -481,7 +481,6 @@ int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) srcu_read_unlock(&vcpu->kvm->srcu, idx); if (!ret) { kvm_complete_mmio_read(vcpu, run); - update_pc(&vcpu->arch); vcpu->mmio_needed = 0; return EMULATE_DONE; } diff --git a/arch/loongarch/kvm/main.c b/arch/loongarch/kvm/main.c index 67d234540ed4c..5af8fb2bad1e2 100644 --- a/arch/loongarch/kvm/main.c +++ b/arch/loongarch/kvm/main.c @@ -413,7 +413,11 @@ static int kvm_loongarch_init(void) if (r) return r; - return kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); + if (r) + kvm_loongarch_env_exit(); + + return r; } static void kvm_loongarch_exit(void) diff --git a/arch/loongarch/kvm/mmu.c b/arch/loongarch/kvm/mmu.c index e104897aa5328..4ca7c6ea8f088 100644 --- a/arch/loongarch/kvm/mmu.c +++ b/arch/loongarch/kvm/mmu.c @@ -383,6 +383,16 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, const struct kvm_memory_slot hva_t hva_start; size_t size, gpa_offset, hva_offset; + /* + * The generic code allocates a fresh, zeroed memslot for every change, + * so the arch flags computed below must be carried over when only the + * userspace flags change, e.g. when dirty logging is toggled. + */ + if (change == KVM_MR_FLAGS_ONLY) { + new->arch = old->arch; + return 0; + } + if ((change != KVM_MR_MOVE) && (change != KVM_MR_CREATE)) return 0; /* diff --git a/arch/loongarch/kvm/vcpu.c b/arch/loongarch/kvm/vcpu.c index 4248a96322b63..32768d8d9347e 100644 --- a/arch/loongarch/kvm/vcpu.c +++ b/arch/loongarch/kvm/vcpu.c @@ -1120,10 +1120,14 @@ static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, return -EINVAL; /* All vCPUs need set the same PV features */ + spin_lock(&kvm->arch.pv_setting_lock); if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) - && ((kvm->arch.pv_features & valid) != val)) + && ((kvm->arch.pv_features & valid) != val)) { + spin_unlock(&kvm->arch.pv_setting_lock); return -EINVAL; + } kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; + spin_unlock(&kvm->arch.pv_setting_lock); return 0; default: return -ENXIO; diff --git a/arch/loongarch/kvm/vm.c b/arch/loongarch/kvm/vm.c index e86a32fffcddd..890d27e37dde4 100644 --- a/arch/loongarch/kvm/vm.c +++ b/arch/loongarch/kvm/vm.c @@ -40,6 +40,7 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) return -ENOMEM; } spin_lock_init(&kvm->arch.phyid_map_lock); + spin_lock_init(&kvm->arch.pv_setting_lock); kvm_init_vmcs(kvm); diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index bb792da9088f6..5b5de4f91eb94 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -228,14 +228,8 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) load_offset -= sizeof(long); emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); - /* - * When push into the stack, follow the order of tcc then tcc_ptr. - * When pop from the stack, first pop tcc_ptr then followed by tcc. - */ - load_offset -= 2 * sizeof(long); - emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); - - load_offset += sizeof(long); + /* Only restore the TCC state into REG_TCC from the higher slot */ + load_offset -= sizeof(long); emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); @@ -272,17 +266,13 @@ bool bpf_jit_supports_far_kfunc_call(void) static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off, tc_ninsn = 0; + int off, jmp_offset; int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; u8 t1 = LOONGARCH_GPR_T1; u8 t2 = LOONGARCH_GPR_T2; u8 t3 = LOONGARCH_GPR_T3; - const int idx0 = ctx->idx; - -#define cur_offset (ctx->idx - idx0) -#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -292,12 +282,12 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) * if (index >= array->map.max_entries) * goto out; */ - tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; emit_zext_32(ctx, a2, true); off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JGE, a2, t1, jmp_offset) < 0) goto toofar; @@ -308,6 +298,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); emit_insn(ctx, ldd, t3, REG_TCC, 0); emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JSGE, t3, t2, jmp_offset) < 0) goto toofar; @@ -322,6 +313,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) off = offsetof(struct bpf_array, ptrs); emit_insn(ctx, ldd, t2, t2, off); /* beq $t2, $zero, jmp_offset */ + jmp_offset = ctx->image ? (ctx->offset[insn + 1] - ctx->idx) : 0; if (emit_tailcall_jmp(ctx, BPF_JEQ, t2, LOONGARCH_GPR_ZERO, jmp_offset) < 0) goto toofar; @@ -337,8 +329,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) toofar: pr_info_once("tail_call: jump too far\n"); return -1; -#undef cur_offset -#undef jmp_offset } static void emit_atomic(const struct bpf_insn *insn, struct jit_ctx *ctx) diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index c9a7e602d8a4d..83f46fd6b97cf 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -548,6 +548,10 @@ config CACHE_COPYBACK endchoice endif # HAVE_CACHE_CB +config NR_CPUS + int + default "1" + # Coldfire cores that do not have a data cache configured can do coherent DMA. config COLDFIRE_COHERENT_DMA bool diff --git a/arch/m68k/emu/nfcon.c b/arch/m68k/emu/nfcon.c index d41260672e24b..f4657d9f4f153 100644 --- a/arch/m68k/emu/nfcon.c +++ b/arch/m68k/emu/nfcon.c @@ -49,7 +49,7 @@ static void nfcon_write(struct console *con, const char *str, static struct tty_driver *nfcon_device(struct console *con, int *index) { *index = 0; - return console_is_registered(con) ? nfcon_tty_driver : NULL; + return nfcon_tty_driver; } static struct console nf_console = { diff --git a/arch/m68k/kernel/traps.c b/arch/m68k/kernel/traps.c index d2f25e8895e5a..60759ba3cbd45 100644 --- a/arch/m68k/kernel/traps.c +++ b/arch/m68k/kernel/traps.c @@ -936,8 +936,8 @@ void show_stack(struct task_struct *task, unsigned long *stack, int i; if (!stack) { - if (task) - stack = (unsigned long *)task->thread.esp0; + if (task && task != current) + stack = (unsigned long *)task->thread.ksp; else stack = (unsigned long *)&stack; } diff --git a/arch/mips/alchemy/common/dbdma.c b/arch/mips/alchemy/common/dbdma.c index 6a3c890f7bbfe..6c2c2010bbaee 100644 --- a/arch/mips/alchemy/common/dbdma.c +++ b/arch/mips/alchemy/common/dbdma.c @@ -982,7 +982,7 @@ u32 au1xxx_dbdma_put_dscr(u32 chanid, au1x_ddma_desc_t *dscr) static unsigned long alchemy_dbdma_pm_data[NUM_DBDMA_CHANS + 1][6]; -static int alchemy_dbdma_suspend(void) +static int alchemy_dbdma_suspend(void *data) { int i; void __iomem *addr; @@ -1019,7 +1019,7 @@ static int alchemy_dbdma_suspend(void) return 0; } -static void alchemy_dbdma_resume(void) +static void alchemy_dbdma_resume(void *data) { int i; void __iomem *addr; @@ -1044,11 +1044,15 @@ static void alchemy_dbdma_resume(void) } } -static struct syscore_ops alchemy_dbdma_syscore_ops = { +static const struct syscore_ops alchemy_dbdma_syscore_ops = { .suspend = alchemy_dbdma_suspend, .resume = alchemy_dbdma_resume, }; +static struct syscore alchemy_dbdma_syscore = { + .ops = &alchemy_dbdma_syscore_ops, +}; + static int __init dbdma_setup(unsigned int irq, dbdev_tab_t *idtable) { int ret; @@ -1071,7 +1075,7 @@ static int __init dbdma_setup(unsigned int irq, dbdev_tab_t *idtable) printk(KERN_ERR "Cannot grab DBDMA interrupt!\n"); else { dbdma_initialized = 1; - register_syscore_ops(&alchemy_dbdma_syscore_ops); + register_syscore(&alchemy_dbdma_syscore); } return ret; diff --git a/arch/mips/alchemy/common/irq.c b/arch/mips/alchemy/common/irq.c index da9f9220048fc..2403afcd2fb95 100644 --- a/arch/mips/alchemy/common/irq.c +++ b/arch/mips/alchemy/common/irq.c @@ -758,7 +758,7 @@ static inline void alchemy_ic_resume_one(void __iomem *base, unsigned long *d) wmb(); } -static int alchemy_ic_suspend(void) +static int alchemy_ic_suspend(void *data) { alchemy_ic_suspend_one((void __iomem *)KSEG1ADDR(AU1000_IC0_PHYS_ADDR), alchemy_gpic_pmdata); @@ -767,7 +767,7 @@ static int alchemy_ic_suspend(void) return 0; } -static void alchemy_ic_resume(void) +static void alchemy_ic_resume(void *data) { alchemy_ic_resume_one((void __iomem *)KSEG1ADDR(AU1000_IC1_PHYS_ADDR), &alchemy_gpic_pmdata[7]); @@ -775,7 +775,7 @@ static void alchemy_ic_resume(void) alchemy_gpic_pmdata); } -static int alchemy_gpic_suspend(void) +static int alchemy_gpic_suspend(void *data) { void __iomem *base = (void __iomem *)KSEG1ADDR(AU1300_GPIC_PHYS_ADDR); int i; @@ -806,7 +806,7 @@ static int alchemy_gpic_suspend(void) return 0; } -static void alchemy_gpic_resume(void) +static void alchemy_gpic_resume(void *data) { void __iomem *base = (void __iomem *)KSEG1ADDR(AU1300_GPIC_PHYS_ADDR); int i; @@ -837,16 +837,24 @@ static void alchemy_gpic_resume(void) wmb(); } -static struct syscore_ops alchemy_ic_pmops = { +static const struct syscore_ops alchemy_ic_pmops = { .suspend = alchemy_ic_suspend, .resume = alchemy_ic_resume, }; -static struct syscore_ops alchemy_gpic_pmops = { +static struct syscore alchemy_ic_pm = { + .ops = &alchemy_ic_pmops, +}; + +static const struct syscore_ops alchemy_gpic_pmops = { .suspend = alchemy_gpic_suspend, .resume = alchemy_gpic_resume, }; +static struct syscore alchemy_gpic_pm = { + .ops = &alchemy_gpic_pmops, +}; + /******************************************************************************/ /* create chained handlers for the 4 IC requests to the MIPS IRQ ctrl */ @@ -880,7 +888,7 @@ static void __init au1000_init_irq(struct alchemy_irqmap *map) ic_init((void __iomem *)KSEG1ADDR(AU1000_IC0_PHYS_ADDR)); ic_init((void __iomem *)KSEG1ADDR(AU1000_IC1_PHYS_ADDR)); - register_syscore_ops(&alchemy_ic_pmops); + register_syscore(&alchemy_ic_pm); mips_cpu_irq_init(); /* register all 64 possible IC0+IC1 irq sources as type "none". @@ -925,7 +933,7 @@ static void __init alchemy_gpic_init_irq(const struct alchemy_irqmap *dints) int i; void __iomem *bank_base; - register_syscore_ops(&alchemy_gpic_pmops); + register_syscore(&alchemy_gpic_pm); mips_cpu_irq_init(); /* disable & ack all possible interrupt sources */ diff --git a/arch/mips/alchemy/common/usb.c b/arch/mips/alchemy/common/usb.c index 5d618547ebf01..a55f32bf517cd 100644 --- a/arch/mips/alchemy/common/usb.c +++ b/arch/mips/alchemy/common/usb.c @@ -580,22 +580,26 @@ static void alchemy_usb_pm(int susp) } } -static int alchemy_usb_suspend(void) +static int alchemy_usb_suspend(void *data) { alchemy_usb_pm(1); return 0; } -static void alchemy_usb_resume(void) +static void alchemy_usb_resume(void *data) { alchemy_usb_pm(0); } -static struct syscore_ops alchemy_usb_pm_ops = { +static const struct syscore_ops alchemy_usb_pm_syscore_ops = { .suspend = alchemy_usb_suspend, .resume = alchemy_usb_resume, }; +static struct syscore alchemy_usb_pm_syscore = { + .ops = &alchemy_usb_pm_syscore_ops, +}; + static int __init alchemy_usb_init(void) { int ret = 0; @@ -620,7 +624,7 @@ static int __init alchemy_usb_init(void) } if (!ret) - register_syscore_ops(&alchemy_usb_pm_ops); + register_syscore(&alchemy_usb_pm_syscore); return ret; } diff --git a/arch/mips/kernel/cps-vec.S b/arch/mips/kernel/cps-vec.S index 2ae7034a3d5c4..70413c816eb03 100644 --- a/arch/mips/kernel/cps-vec.S +++ b/arch/mips/kernel/cps-vec.S @@ -373,8 +373,11 @@ LEAF(mips_cps_boot_vpes) .set pop PTR_LA t1, 1f + .set push + .set MIPS_ISA_LEVEL_RAW jr.hb t1 nop + .set pop 1: mfc0 t1, CP0_MVPCONTROL ori t1, t1, MVPCONTROL_VPC mtc0 t1, CP0_MVPCONTROL @@ -487,8 +490,11 @@ LEAF(mips_cps_boot_vpes) li t0, TCHALT_H mtc0 t0, CP0_TCHALT PTR_LA t0, 1f + .set push + .set MIPS_ISA_LEVEL_RAW 1: jr.hb t0 nop + .set pop 2: diff --git a/arch/mips/kernel/ptrace.c b/arch/mips/kernel/ptrace.c index 3f4c94c881241..87102a03b6eaf 100644 --- a/arch/mips/kernel/ptrace.c +++ b/arch/mips/kernel/ptrace.c @@ -1321,8 +1321,12 @@ long arch_ptrace(struct task_struct *child, long request, */ asmlinkage long syscall_trace_enter(struct pt_regs *regs) { + long syscall; + user_exit(); + syscall = current_thread_info()->syscall; + if (test_thread_flag(TIF_SYSCALL_TRACE)) { if (ptrace_report_syscall_entry(regs)) return -1; @@ -1342,7 +1346,7 @@ asmlinkage long syscall_trace_enter(struct pt_regs *regs) * Negative syscall numbers are mistaken for rejected syscalls, but * won't have had the return value set appropriately, so we do so now. */ - if (current_thread_info()->syscall < 0) + if (syscall < 0) syscall_set_return_value(current, regs, -ENOSYS, 0); return current_thread_info()->syscall; } diff --git a/arch/mips/pci/pci-alchemy.c b/arch/mips/pci/pci-alchemy.c index 58625d1b64658..6bfee0f71803e 100644 --- a/arch/mips/pci/pci-alchemy.c +++ b/arch/mips/pci/pci-alchemy.c @@ -304,7 +304,7 @@ static int alchemy_pci_def_idsel(unsigned int devsel, int assert) } /* save PCI controller register contents. */ -static int alchemy_pci_suspend(void) +static int alchemy_pci_suspend(void *data) { struct alchemy_pci_context *ctx = __alchemy_pci_ctx; if (!ctx) @@ -326,7 +326,7 @@ static int alchemy_pci_suspend(void) return 0; } -static void alchemy_pci_resume(void) +static void alchemy_pci_resume(void *data) { struct alchemy_pci_context *ctx = __alchemy_pci_ctx; if (!ctx) @@ -354,9 +354,13 @@ static void alchemy_pci_resume(void) alchemy_pci_wired_entry(ctx); /* install it */ } -static struct syscore_ops alchemy_pci_pmops = { - .suspend = alchemy_pci_suspend, - .resume = alchemy_pci_resume, +static const struct syscore_ops alchemy_pci_syscore_ops = { + .suspend = alchemy_pci_suspend, + .resume = alchemy_pci_resume, +}; + +static struct syscore alchemy_pci_syscore = { + .ops = &alchemy_pci_syscore_ops, }; static int alchemy_pci_probe(struct platform_device *pdev) @@ -478,7 +482,7 @@ static int alchemy_pci_probe(struct platform_device *pdev) __alchemy_pci_ctx = ctx; platform_set_drvdata(pdev, ctx); - register_syscore_ops(&alchemy_pci_pmops); + register_syscore(&alchemy_pci_syscore); register_pci_controller(&ctx->alchemy_pci_ctrl); dev_info(&pdev->dev, "PCI controller at %ld MHz\n", diff --git a/arch/openrisc/include/asm/processor.h b/arch/openrisc/include/asm/processor.h index 3ff893a67c13b..ae43fe79b2b5c 100644 --- a/arch/openrisc/include/asm/processor.h +++ b/arch/openrisc/include/asm/processor.h @@ -26,6 +26,8 @@ | SPR_SR_DCE | SPR_SR_SM) #define USER_SR (SPR_SR_DME | SPR_SR_IME | SPR_SR_ICE \ | SPR_SR_DCE | SPR_SR_IEE | SPR_SR_TEE) +/* SR bits user space may change via sigreturn, the rest stay kernel owned */ +#define SPR_SR_USER_MASK (SPR_SR_F | SPR_SR_CY | SPR_SR_OV) /* * User space process size. This is hardcoded into a few places, diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S index c7e90b09645e4..18e68680471eb 100644 --- a/arch/openrisc/kernel/entry.S +++ b/arch/openrisc/kernel/entry.S @@ -1223,15 +1223,50 @@ _no_syscall_trace: * */ +/* Keep this literal; hi()/lo() can't use the UL-suffixed TASK_SIZE. */ +#define OR1K_ATOMIC_ADDR_LIMIT 0x7ffffffc + ENTRY(sys_or1k_atomic) /* FIXME: This ignores r3 and always does an XCHG */ + + /* Check both user pointers before accessing them. */ + l.movhi r13,hi(OR1K_ATOMIC_ADDR_LIMIT) + l.ori r13,r13,lo(OR1K_ATOMIC_ADDR_LIMIT) + l.sfgtu r4,r13 + l.bf 9f + l.nop + l.sfgtu r5,r13 + l.bf 9f + l.nop + DISABLE_INTERRUPTS(r17,r19) - l.lwz r29,0(r4) - l.lwz r27,0(r5) - l.sw 0(r4),r27 - l.sw 0(r5),r29 +10: l.lwz r29,0(r4) +11: l.lwz r27,0(r5) +12: l.sw 0(r4),r27 +13: l.sw 0(r5),r29 ENABLE_INTERRUPTS(r17) l.jr r9 l.or r11,r0,r0 + /* + * Either pointer was outside user space, or turned out to be + * unmapped/inaccessible when we actually touched it. + */ +9: l.jr r9 + l.addi r11,r0,-EFAULT + + .section .fixup, "ax" +14: + ENABLE_INTERRUPTS(r17) + l.j 9b + l.nop + .previous + + .section __ex_table, "a" + .long 10b, 14b + .long 11b, 14b + .long 12b, 14b + .long 13b, 14b + .previous + /* ============================================================[ EOF ]=== */ diff --git a/arch/openrisc/kernel/signal.c b/arch/openrisc/kernel/signal.c index f70a13ee05936..2be5af3b98323 100644 --- a/arch/openrisc/kernel/signal.c +++ b/arch/openrisc/kernel/signal.c @@ -74,6 +74,7 @@ static long save_fp_state(struct sigcontext __user *sc) static int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc) { + unsigned long old_sr = regs->sr; int err = 0; /* Always make any pending restarted system calls return -EINTR */ @@ -89,8 +90,8 @@ static int restore_sigcontext(struct pt_regs *regs, err |= __copy_from_user(®s->sr, &sc->regs.sr, sizeof(unsigned long)); err |= restore_fp_state(sc); - /* make sure the SM-bit is cleared so user-mode cannot fool us */ - regs->sr &= ~SPR_SR_SM; + /* keep the privileged SR bits kernel owned, restore only user flags */ + regs->sr = (old_sr & ~SPR_SR_USER_MASK) | (regs->sr & SPR_SR_USER_MASK); regs->orig_gpr11 = -1; /* Avoid syscall restart checks */ diff --git a/arch/parisc/kernel/head.S b/arch/parisc/kernel/head.S index 9188c8d874370..e84d7d6e1eb94 100644 --- a/arch/parisc/kernel/head.S +++ b/arch/parisc/kernel/head.S @@ -105,6 +105,7 @@ $iodc_panic: or %r10,%r10,%r10 /* qemu idle sleep */ msg1: .ascii "Can't boot kernel which was built for PA8x00 CPUs on this machine.\r\n" msg1_end: + .align 4 $cpu_ok: #endif diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 9537a61ebae02..7b02fe19d26a9 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -1059,6 +1059,7 @@ config PPC_SECURE_BOOT depends on IMA_ARCH_POLICY imply IMA_SECURE_AND_OR_TRUSTED_BOOT select PSERIES_PLPKS if PPC_PSERIES + select HAVE_ARCH_GET_SECUREBOOT help Systems with firmware secure boot enabled need to define security policies to extend secure boot to the OS. This config allows a user diff --git a/arch/powerpc/boot/simpleboot.c b/arch/powerpc/boot/simpleboot.c index c80691d83880b..27591df41e9e8 100644 --- a/arch/powerpc/boot/simpleboot.c +++ b/arch/powerpc/boot/simpleboot.c @@ -68,7 +68,7 @@ void platform_init(unsigned long r3, unsigned long r4, unsigned long r5, /* finally, setup the timebase */ node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-akebono.c b/arch/powerpc/boot/treeboot-akebono.c index e3cc2599869cc..1b529037480fb 100644 --- a/arch/powerpc/boot/treeboot-akebono.c +++ b/arch/powerpc/boot/treeboot-akebono.c @@ -146,7 +146,7 @@ void platform_init(char *userdata) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/boot/treeboot-currituck.c b/arch/powerpc/boot/treeboot-currituck.c index d53e8a592f816..5b5363b74f9f3 100644 --- a/arch/powerpc/boot/treeboot-currituck.c +++ b/arch/powerpc/boot/treeboot-currituck.c @@ -102,7 +102,7 @@ void platform_init(void) node = fdt_node_offset_by_prop_value(_dtb_start, -1, "device_type", "cpu", sizeof("cpu")); - if (!node) + if (node < 0) fatal("Cannot find cpu node\n"); timebase = fdt_getprop(_dtb_start, node, "timebase-frequency", &size); if (timebase && (size == 4)) diff --git a/arch/powerpc/configs/85xx-hw.config b/arch/powerpc/configs/85xx-hw.config index 8aff832173977..b655c36e56dad 100644 --- a/arch/powerpc/configs/85xx-hw.config +++ b/arch/powerpc/configs/85xx-hw.config @@ -90,6 +90,7 @@ CONFIG_PPC_EPAPR_HV_BYTECHAN=y CONFIG_QE_GPIO=y CONFIG_QUICC_ENGINE=y CONFIG_RAPIDIO=y +CONFIG_RAS=y CONFIG_RTC_CLASS=y CONFIG_RTC_DRV_CMOS=y CONFIG_RTC_DRV_DS1307=y diff --git a/arch/powerpc/include/asm/papr-watchdog.h b/arch/powerpc/include/asm/papr-watchdog.h new file mode 100644 index 0000000000000..bf876fc2caae5 --- /dev/null +++ b/arch/powerpc/include/asm/papr-watchdog.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _ASM_POWERPC_PAPR_WATCHDOG_H +#define _ASM_POWERPC_PAPR_WATCHDOG_H + +/* + * H_WATCHDOG Input + * + * R4: "flags": + * + * Bits 48-55: "operation" + */ +#define PSERIES_WDTF_OP_START 0x100UL /* start timer */ +#define PSERIES_WDTF_OP_STOP 0x200UL /* stop timer */ +#define PSERIES_WDTF_OP_QUERY 0x300UL /* query timer capabilities */ + +/* + * Bits 56-63: "timeoutAction" (for "Start Watchdog" only) + */ +#define PSERIES_WDTF_ACTION_HARD_POWEROFF 0x1UL /* poweroff */ +#define PSERIES_WDTF_ACTION_HARD_RESTART 0x2UL /* restart */ +#define PSERIES_WDTF_ACTION_DUMP_RESTART 0x3UL /* dump + restart */ + +/* + * R5: "watchdogNumber": + * PAPR says use -1 (all ones) to stop all watchdogs. + */ +#define PSERIES_WDT_NUM_ALL ((unsigned long)-1) + +/* + * H_WATCHDOG Output + * + * R3: Return code + * + * H_SUCCESS The operation completed. + * + * H_BUSY The hypervisor is too busy; retry the operation. + * + * H_PARAMETER The given "flags" are somehow invalid. Either the + * "operation" or "timeoutAction" is invalid, or a + * reserved bit is set. + * + * H_P2 The given "watchdogNumber" is zero or exceeds the + * supported maximum value. + * + * H_P3 The given "timeoutInMs" is below the supported + * minimum value. + * + * H_NOOP The given "watchdogNumber" is already stopped. + * + * H_HARDWARE The operation failed for ineffable reasons. + * + * H_FUNCTION The H_WATCHDOG hypercall is not supported by this + * hypervisor. + * + * R4: + * + * - For the "Query Watchdog Capabilities" operation, a 64-bit + * structure: + */ +#define PSERIES_WDTQ_MIN_TIMEOUT(cap) (((cap) >> 48) & 0xffff) +#define PSERIES_WDTQ_MAX_NUMBER(cap) (((cap) >> 32) & 0xffff) + +#endif /* _ASM_POWERPC_PAPR_WATCHDOG_H */ diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 3fe1866354323..05550a013d8c9 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -1357,6 +1357,18 @@ #define PVR_ARCH_31 0x0f000006 #define PVR_ARCH_31_P11 0x0f000007 +/* + * Kernel-internal sentinel for invalid processor compatibility modes. + * PAPR specifies that the first byte of a valid logical PVR value is + * 0x0f. So 0xffffffff lies permanently outside the PAPR-defined range + * and is safe to repurpose. KVM stores it in vcpu->arch.arch_compat + * when userspace requests an unsupported compatibility mode (e.g., + * Power11 PVR on a Power11 host booted in Power10 compat). + * kvmppc_sanity_check() detects this and prevents the vCPU from + * running with an unsupported arch_compat. + */ +#define PVR_ARCH_INVALID 0xffffffff + /* Macros for setting and retrieving special purpose registers */ #ifndef __ASSEMBLER__ diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index 028f691585323..d64cce17a4e06 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -533,9 +533,7 @@ static void eeh_rmv_device(struct eeh_dev *edev, void *userdata) if (rmv_data) list_add(&edev->rmv_entry, &rmv_data->removed_vf_list); } else { - pci_lock_rescan_remove(); pci_stop_and_remove_bus_device(dev); - pci_unlock_rescan_remove(); } } diff --git a/arch/powerpc/kernel/ima_arch.c b/arch/powerpc/kernel/ima_arch.c index b7029beed847d..0d8892a035266 100644 --- a/arch/powerpc/kernel/ima_arch.c +++ b/arch/powerpc/kernel/ima_arch.c @@ -7,11 +7,6 @@ #include #include -bool arch_ima_get_secureboot(void) -{ - return is_ppc_secureboot_enabled(); -} - /* * The "secure_rules" are enabled only on "secureboot" enabled systems. * These rules verify the file signatures against known good values. diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index a0e8b998c9b52..cb010830c1c4d 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -217,8 +217,12 @@ static __always_inline void call_do_softirq(const void *sp) [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_MIN_SIZE), [callee] "i" (__do_softirq) : // Clobbers - "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", - "cr7", "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", "cr7", "r0", + /* r2 may be clobbered by the callee when using PCREL mode in the ELFv2 ABI. */ +#ifdef CONFIG_PPC_KERNEL_PCREL + "r2", +#endif + "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" ); } @@ -275,8 +279,12 @@ static __always_inline void call_do_irq(struct pt_regs *regs, void *sp) [sp] "b" (sp), [offset] "i" (THREAD_SIZE - STACK_FRAME_MIN_SIZE), [callee] "i" (__do_irq) : // Clobbers - "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", - "cr7", "r0", "r4", "r5", "r6", "r7", "r8", "r9", "r10", + "lr", "xer", "ctr", "memory", "cr0", "cr1", "cr5", "cr6", "cr7", "r0", + /* r2 may be clobbered by the callee when using PCREL mode in the ELFv2 ABI. */ +#ifdef CONFIG_PPC_KERNEL_PCREL + "r2", +#endif + "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12" ); } diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index eac84d687b53f..c493692adc695 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -626,19 +626,14 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size) return -ENXIO; addr = hose->io_base_virt + port; - /* WARNING: The generic code is idiotic. It gets passed a pointer - * to what can be a 1, 2 or 4 byte quantity and always reads that - * as a u32, which means that we have to correct the location of - * the data read within those 32 bits for size 1 and 2 - */ switch(size) { case 1: - out_8(addr, val >> 24); + out_8(addr, val); return 1; case 2: if (port & 1) return -EINVAL; - out_le16(addr, val >> 16); + out_le16(addr, val); return 2; case 4: if (port & 3) diff --git a/arch/powerpc/kernel/secure_boot.c b/arch/powerpc/kernel/secure_boot.c index 3a28795b4ed82..28436c1599e08 100644 --- a/arch/powerpc/kernel/secure_boot.c +++ b/arch/powerpc/kernel/secure_boot.c @@ -5,6 +5,7 @@ */ #include #include +#include #include #include @@ -44,6 +45,11 @@ bool is_ppc_secureboot_enabled(void) return enabled; } +bool arch_get_secureboot(void) +{ + return is_ppc_secureboot_enabled(); +} + bool is_ppc_trustedboot_enabled(void) { struct device_node *node; diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 68d47c53876c6..28186f9ec8da0 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -68,6 +68,7 @@ #include #include #include +#include #include "setup.h" @@ -744,6 +745,13 @@ static int ppc_panic_fadump_handler(struct notifier_block *this, */ hard_irq_disable(); + /* + * Invoke kmsg_dump (e.g., pstore) before crash_fadump() as fadump + * runs before panic()'s kmsg_dump_desc() call. + */ + if (should_fadump_crash()) + kmsg_dump_desc(KMSG_DUMP_PANIC, (char *)ptr); + /* * If firmware-assisted dump has been registered then trigger * its callback and let the firmware handles everything else. diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 0cd9c0c21af3e..b5518fba3b92e 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -289,6 +289,9 @@ void smp_muxed_ipi_set_message(int cpu, int msg) void smp_muxed_ipi_message_pass(int cpu, int msg) { + if (!smp_ops->cause_ipi) + return; + smp_muxed_ipi_set_message(cpu, msg); /* diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index b4472288e0d43..11145c40183dd 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -376,6 +376,47 @@ void vtime_task_switch(struct task_struct *prev) acct->starttime = acct0->starttime; } } + +/** + * vtime_reset - Fast forward vtime entry clocks + * + * Called from dynticks idle IRQ entry to fast-forward the clocks to current time + * so that the IRQ time is still accounted by vtime while nohz cputime is paused. + */ +void vtime_reset(void) +{ + struct cpu_accounting_data *acct = get_accounting(current); + + acct->starttime = mftb(); +#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME + acct->startspurr = read_spurr(acct->starttime); +#endif +} + +#ifdef CONFIG_NO_HZ_COMMON +/** + * vtime_dyntick_start - Inform vtime about entry to idle-dynticks + * + * Called when idle enters in dyntick mode. The idle cputime that elapsed so far + * is accumulated and the tick subsystem takes over the idle cputime accounting. + */ +void vtime_dyntick_start(void) +{ + vtime_account_idle(current); +} + +/** + * vtime_dyntick_stop - Inform vtime about exit from idle-dynticks + * + * Called when idle exits from dyntick mode. The vtime entry clocks are + * fast-forward to current time so that idle accounting restarts elapsing from + * now. + */ +void vtime_dyntick_stop(void) +{ + vtime_reset(); +} +#endif /* CONFIG_NO_HZ_COMMON */ #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ void __no_kcsan __delay(unsigned long loops) @@ -892,6 +933,7 @@ static void __init set_decrementer_max(void) static void __init init_decrementer_clockevent(void) { register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } void secondary_cpu_time_init(void) @@ -907,6 +949,7 @@ void secondary_cpu_time_init(void) /* FIME: Should make unrelated change to move snapshot_timebase * call here ! */ register_decrementer_clockevent(smp_processor_id()); + vtime_reset(); } /* diff --git a/arch/powerpc/kexec/crash.c b/arch/powerpc/kexec/crash.c index e6539f213b3d1..a520f851c3a6b 100644 --- a/arch/powerpc/kexec/crash.c +++ b/arch/powerpc/kexec/crash.c @@ -502,7 +502,7 @@ static void update_crash_elfcorehdr(struct kimage *image, struct memory_notify * ret = get_crash_memory_ranges(&cmem); if (ret) { pr_err("Failed to get crash mem range\n"); - return; + goto out; } /* diff --git a/arch/powerpc/kexec/file_load_64.c b/arch/powerpc/kexec/file_load_64.c index 8c72e12ea44e5..d990880b77dfd 100644 --- a/arch/powerpc/kexec/file_load_64.c +++ b/arch/powerpc/kexec/file_load_64.c @@ -57,7 +57,7 @@ int arch_check_excluded_range(struct kimage *image, unsigned long start, emem = image->arch.exclude_ranges; for (i = 0; i < emem->nr_ranges; i++) - if (start < emem->ranges[i].end && end > emem->ranges[i].start) + if (start <= emem->ranges[i].end && end >= emem->ranges[i].start) return 1; return 0; @@ -113,7 +113,7 @@ static int add_usable_mem(struct umem_info *um_info, u64 base, u64 end) loc_end = um_info->ranges[i].end; if (loc_base >= base && loc_end <= end) add = true; - else if (base < loc_end && end > loc_base) { + else if (base <= loc_end && end >= loc_base) { if (loc_base < base) loc_base = base; if (loc_end > end) @@ -664,7 +664,7 @@ unsigned int kexec_extra_fdt_size_ppc64(struct kimage *image, struct crash_mem * extra_size += (cpu_nodes - boot_cpu_node_count) * cpu_node_size(); /* Consider extra space for reserved memory ranges if any */ - if (rmem->nr_ranges > 0) + if (rmem && rmem->nr_ranges > 0) extra_size += sizeof(struct fdt_reserve_entry) * rmem->nr_ranges; return extra_size + kdump_extra_fdt_size_ppc64(image, cpu_nodes); diff --git a/arch/powerpc/kexec/ranges.c b/arch/powerpc/kexec/ranges.c index 426bdca4667e7..8815ab961752f 100644 --- a/arch/powerpc/kexec/ranges.c +++ b/arch/powerpc/kexec/ranges.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -105,19 +106,16 @@ static void __merge_memory_ranges(struct crash_mem *mem_rngs) struct range *ranges; int i, idx; - if (!mem_rngs) + if (!mem_rngs || mem_rngs->nr_ranges <= 1) return; idx = 0; - ranges = &(mem_rngs->ranges[0]); + ranges = mem_rngs->ranges; for (i = 1; i < mem_rngs->nr_ranges; i++) { - if (ranges[i].start <= (ranges[i-1].end + 1)) - ranges[idx].end = ranges[i].end; + if (ranges[i].start <= (ranges[idx].end + 1)) + ranges[idx].end = max(ranges[idx].end, ranges[i].end); else { idx++; - if (i == idx) - continue; - ranges[idx] = ranges[i]; } } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 7667563fb9ffb..aba6904591bc3 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -447,7 +447,19 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) guest_pcr_bit = PCR_ARCH_300; break; case PVR_ARCH_31: + guest_pcr_bit = PCR_ARCH_31; + break; case PVR_ARCH_31_P11: + /* + * Need to check this for ISA 3.1, as Power10 and + * Power11 share the same PCR. For any subsequent ISA + * versions, this will be taken care of by the guest vs + * host PCR comparison below. + */ + if (!cpu_has_feature(CPU_FTR_P11_PVR)) { + arch_compat = PVR_ARCH_INVALID; + goto out; + } guest_pcr_bit = PCR_ARCH_31; break; default: @@ -470,6 +482,7 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) return -EINVAL; } +out: spin_lock(&vc->lock); vc->arch_compat = arch_compat; kvmhv_nestedv2_mark_dirty(vcpu, KVMPPC_GSID_LOGICAL_PVR); @@ -480,7 +493,7 @@ static int kvmppc_set_arch_compat(struct kvm_vcpu *vcpu, u32 arch_compat) vc->pcr = (host_pcr_bit - guest_pcr_bit) | PCR_MASK; spin_unlock(&vc->lock); - return 0; + return kvmppc_sanity_check(vcpu); } static void kvmppc_dump_regs(struct kvm_vcpu *vcpu) @@ -6112,12 +6125,12 @@ static int kvmppc_irq_bypass_add_producer_hv(struct irq_bypass_consumer *cons, struct kvm_kernel_irqfd *irqfd = container_of(cons, struct kvm_kernel_irqfd, consumer); - irqfd->producer = prod; - ret = kvmppc_set_passthru_irq(irqfd->kvm, prod->irq, irqfd->gsi); if (ret) pr_info("kvmppc_set_passthru_irq (irq %d, gsi %d) fails: %d\n", prod->irq, irqfd->gsi, ret); + else + irqfd->producer = prod; return ret; } diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 2ba057171ebe0..cc22d41f8b2fe 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -258,6 +258,12 @@ int kvmppc_sanity_check(struct kvm_vcpu *vcpu) if (!vcpu->arch.pvr) goto out; +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) + if (vcpu->arch.vcore && + vcpu->arch.vcore->arch_compat == PVR_ARCH_INVALID) + goto out; +#endif + /* PAPR only works with book3s_64 */ if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled) goto out; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 73977dbabcf26..c5a5e7b123199 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -1313,7 +1313,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * covering out both edges. */ unsigned long addr; - unsigned long addr_pfn = start_pfn; unsigned long next; pgd_t *pgd; p4d_t *p4d; @@ -1334,7 +1333,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (pmd_leaf(READ_ONCE(*pmd))) { /* existing huge mapping. Skip the range */ - addr_pfn += (PMD_SIZE >> PAGE_SHIFT); next = pmd_addr_end(addr, end); continue; } @@ -1347,11 +1345,11 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, * page whose VMEMMAP_RESERVE_NR pages were mapped and * this request fall in those pages. */ - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } else { unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); + unsigned long addr_pfn = page_to_pfn((struct page *)addr); unsigned long pfn_offset = addr_pfn - ALIGN_DOWN(addr_pfn, nr_pages); pte_t *tail_page_pte; @@ -1375,7 +1373,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, if (!pte) return -ENOMEM; - addr_pfn += 2; next = addr + 2 * PAGE_SIZE; continue; } @@ -1391,7 +1388,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } @@ -1401,7 +1397,6 @@ int __meminit vmemmap_populate_compound_pages(unsigned long start_pfn, return -ENOMEM; vmemmap_verify(pte, node, addr, addr + PAGE_SIZE); - addr_pfn += 1; next = addr + PAGE_SIZE; continue; } diff --git a/arch/powerpc/perf/hv-gpci.c b/arch/powerpc/perf/hv-gpci.c index 241551d1282f8..0ec0bd3110652 100644 --- a/arch/powerpc/perf/hv-gpci.c +++ b/arch/powerpc/perf/hv-gpci.c @@ -210,7 +210,7 @@ static ssize_t processor_bus_topology_show(struct device *dev, struct device_att 0, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; @@ -244,12 +244,14 @@ static ssize_t processor_bus_topology_show(struct device *dev, struct device_att starting_index, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; } +out_success: + put_cpu_var(hv_gpci_reqb); return n; out: @@ -278,7 +280,7 @@ static ssize_t processor_config_show(struct device *dev, struct device_attribute 0, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; @@ -312,12 +314,14 @@ static ssize_t processor_config_show(struct device *dev, struct device_attribute starting_index, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; } +out_success: + put_cpu_var(hv_gpci_reqb); return n; out: @@ -346,7 +350,7 @@ static ssize_t affinity_domain_via_virtual_processor_show(struct device *dev, 0, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; @@ -382,12 +386,14 @@ static ssize_t affinity_domain_via_virtual_processor_show(struct device *dev, starting_index, secondary_index, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; } +out_success: + put_cpu_var(hv_gpci_reqb); return n; out: @@ -416,7 +422,7 @@ static ssize_t affinity_domain_via_domain_show(struct device *dev, struct device 0, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; @@ -448,12 +454,14 @@ static ssize_t affinity_domain_via_domain_show(struct device *dev, struct device starting_index, 0, buf, &n, arg); if (!ret) - return n; + goto out_success; if (ret != H_PARAMETER) goto out; } +out_success: + put_cpu_var(hv_gpci_reqb); return n; out: diff --git a/arch/powerpc/platforms/44x/gpio.c b/arch/powerpc/platforms/44x/gpio.c index aea0d913b59d0..4413a94cf7a6a 100644 --- a/arch/powerpc/platforms/44x/gpio.c +++ b/arch/powerpc/platforms/44x/gpio.c @@ -169,6 +169,7 @@ static int ppc4xx_gpio_probe(struct platform_device *ofdev) gc = &chip->gc; + gc->parent = dev; gc->base = -1; gc->ngpio = 32; gc->direction_input = ppc4xx_gpio_dir_in; diff --git a/arch/powerpc/platforms/85xx/common.c b/arch/powerpc/platforms/85xx/common.c index 757811155587d..c11deb2f50ed4 100644 --- a/arch/powerpc/platforms/85xx/common.c +++ b/arch/powerpc/platforms/85xx/common.c @@ -42,6 +42,8 @@ static const struct of_device_id mpc85xx_common_ids[] __initconst = { { .compatible = "fsl,qoriq-pcie-v2.3", }, { .compatible = "fsl,qoriq-pcie-v2.2", }, { .compatible = "fsl,fman", }, + /* IFC NAND and NOR controllers */ + { .compatible = "fsl,ifc", }, {}, }; diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index 2c07387201d0d..2ddb93df4817f 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -726,7 +726,7 @@ static inline void crash_register_spus(struct list_head *list) } #endif -static void spu_shutdown(void) +static void spu_shutdown(void *data) { struct spu *spu; @@ -738,10 +738,14 @@ static void spu_shutdown(void) mutex_unlock(&spu_full_list_mutex); } -static struct syscore_ops spu_syscore_ops = { +static const struct syscore_ops spu_syscore_ops = { .shutdown = spu_shutdown, }; +static struct syscore spu_syscore = { + .ops = &spu_syscore_ops, +}; + static int __init init_spu_base(void) { int i, ret = 0; @@ -774,7 +778,7 @@ static int __init init_spu_base(void) crash_register_spus(&spu_full_list); mutex_unlock(&spu_full_list_mutex); spu_add_dev_attr(&dev_attr_stat); - register_syscore_ops(&spu_syscore_ops); + register_syscore(&spu_syscore); spu_init_affinity(); diff --git a/arch/powerpc/platforms/powermac/low_i2c.c b/arch/powerpc/platforms/powermac/low_i2c.c index 02474e27df9b2..5eff89a6859e3 100644 --- a/arch/powerpc/platforms/powermac/low_i2c.c +++ b/arch/powerpc/platforms/powermac/low_i2c.c @@ -1505,7 +1505,7 @@ static int __init pmac_i2c_create_platform_devices(void) if (bus->platform_dev == NULL) return -ENOMEM; bus->platform_dev->dev.platform_data = bus; - bus->platform_dev->dev.of_node = bus->busnode; + bus->platform_dev->dev.of_node = of_node_get(bus->busnode); platform_device_add(bus->platform_dev); } diff --git a/arch/powerpc/platforms/powermac/pic.c b/arch/powerpc/platforms/powermac/pic.c index c37783a03d254..1959cc13438f5 100644 --- a/arch/powerpc/platforms/powermac/pic.c +++ b/arch/powerpc/platforms/powermac/pic.c @@ -600,7 +600,7 @@ static int pmacpic_find_viaint(void) return viaint; } -static int pmacpic_suspend(void) +static int pmacpic_suspend(void *data) { int viaint = pmacpic_find_viaint(); @@ -621,7 +621,7 @@ static int pmacpic_suspend(void) return 0; } -static void pmacpic_resume(void) +static void pmacpic_resume(void *data) { int i; @@ -634,15 +634,19 @@ static void pmacpic_resume(void) pmac_unmask_irq(irq_get_irq_data(i)); } -static struct syscore_ops pmacpic_syscore_ops = { +static const struct syscore_ops pmacpic_syscore_ops = { .suspend = pmacpic_suspend, .resume = pmacpic_resume, }; +static struct syscore pmacpic_syscore = { + .ops = &pmacpic_syscore_ops, +}; + static int __init init_pmacpic_syscore(void) { if (pmac_irq_hw[0]) - register_syscore_ops(&pmacpic_syscore_ops); + register_syscore(&pmacpic_syscore); return 0; } diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index b0c1d9d16fb52..5f10896d566a6 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -1634,15 +1634,13 @@ int64_t pnv_opal_pci_msi_eoi(struct irq_data *d) return opal_pci_msi_eoi(phb->opal_id, d->parent_data->hwirq); } -static struct irq_chip pnv_pci_msi_irq_chip; - /* * Returns true iff chip is something that we could call * pnv_opal_pci_msi_eoi for. */ bool is_pnv_opal_msi(struct irq_chip *chip) { - return chip == &pnv_pci_msi_irq_chip; + return chip && chip->name && str_has_prefix(chip->name, "PNV-"); } EXPORT_SYMBOL_GPL(is_pnv_opal_msi); @@ -1739,7 +1737,7 @@ static const struct msi_parent_ops pnv_msi_parent_ops = { .chip_flags = MSI_CHIP_FLAG_SET_EOI, .bus_select_token = DOMAIN_BUS_NEXUS, .bus_select_mask = MATCH_PCI_MSI, - .prefix = "PNV-", + .prefix = "PNV-", /* Note: is_pnv_opal_msi() uses this */ .init_dev_msi_info = pnv_init_dev_msi_info, }; diff --git a/arch/powerpc/platforms/ps3/mm.c b/arch/powerpc/platforms/ps3/mm.c index 1326de55fda65..e04952bb4360b 100644 --- a/arch/powerpc/platforms/ps3/mm.c +++ b/arch/powerpc/platforms/ps3/mm.c @@ -615,6 +615,7 @@ static int dma_ioc0_map_pages(struct ps3_dma_region *r, unsigned long phys_addr, fail_map: for (iopage--; 0 <= iopage; iopage--) { + offset = (1 << r->page_size) * iopage; lv1_put_iopte(0, c->bus_addr + offset, c->lpar_addr + offset, diff --git a/arch/powerpc/platforms/ps3/repository.c b/arch/powerpc/platforms/ps3/repository.c index b8c030eab1384..0cc755ac3e7f5 100644 --- a/arch/powerpc/platforms/ps3/repository.c +++ b/arch/powerpc/platforms/ps3/repository.c @@ -6,6 +6,8 @@ * Copyright 2006 Sony Corp. */ +#include + #include #include "platform.h" @@ -74,8 +76,9 @@ static void _dump_node(unsigned int lpar_id, u64 n1, u64 n2, u64 n3, u64 n4, static u64 make_first_field(const char *text, u64 index) { u64 n = 0; + size_t len = min(strlen(text), sizeof(n)); - memcpy((char *)&n, text, strnlen(text, sizeof(n))); + memcpy(&n, text, len); return PS3_VENDOR_ID_NONE + (n >> 32) + index; } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index eec333dd2e598..4bcb6f2220658 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -812,18 +812,11 @@ static struct device_node *pci_dma_find(struct device_node *dn, /* parse DMA window property. During normal system boot, only default * DMA window is passed in OF. But, for kdump, a dedicated adapter might - * have both default and DDW in FDT. In this scenario, DDW takes precedence - * over default window. + * have both default and DDW in FDT. In this scenario, default window + * takes precedence over DDW. For a dedicated adapter, default window will + * potentially have more unused TCEs. */ - if (ddw_win) { - struct dynamic_dma_window_prop *p; - - p = (struct dynamic_dma_window_prop *)ddw_prop; - prop->liobn = p->liobn; - prop->dma_base = p->dma_base; - prop->tce_shift = p->tce_shift; - prop->window_shift = p->window_shift; - } else if (default_win) { + if (default_win) { unsigned long offset, size, liobn; of_parse_dma_window(rdn, default_prop, &liobn, &offset, &size); @@ -832,6 +825,14 @@ static struct device_node *pci_dma_find(struct device_node *dn, prop->dma_base = cpu_to_be64(offset); prop->tce_shift = cpu_to_be32(IOMMU_PAGE_SHIFT_4K); prop->window_shift = cpu_to_be32(order_base_2(size)); + } else { + struct dynamic_dma_window_prop *p; + + p = (struct dynamic_dma_window_prop *)ddw_prop; + prop->liobn = p->liobn; + prop->dma_base = p->dma_base; + prop->tce_shift = p->tce_shift; + prop->window_shift = p->window_shift; } return rdn; diff --git a/arch/powerpc/platforms/pseries/kexec.c b/arch/powerpc/platforms/pseries/kexec.c index 431be156ca9bb..29f7c97ff1932 100644 --- a/arch/powerpc/platforms/pseries/kexec.c +++ b/arch/powerpc/platforms/pseries/kexec.c @@ -20,12 +20,15 @@ void pseries_kexec_cpu_down(int crash_shutdown, int secondary) { /* - * Don't risk a hypervisor call if we're crashing - * XXX: Why? The hypervisor is not crashing. It might be better - * to at least attempt unregister to avoid the hypervisor stepping - * on our memory. + * Ensure vpa/slb_shadow/dtl cleanup even while we are crashing. + * Why? The hypervisor is not crashing so at least attempt unregister to + * avoid the hypervisor stepping on our memory. If hypervisor or kexec + * kernel steps on the old memory allocated to these areas before the + * new kexec-kernel happens to allocate and register new areas, + * the hypervisor will see invalid content which may cause + * unexpected behavior. */ - if (firmware_has_feature(FW_FEATURE_SPLPAR) && !crash_shutdown) { + if (firmware_has_feature(FW_FEATURE_SPLPAR)) { int ret; int cpu = smp_processor_id(); int hwcpu = hard_smp_processor_id(); diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c index cc22924f159f5..cc775293d7106 100644 --- a/arch/powerpc/platforms/pseries/lparcfg.c +++ b/arch/powerpc/platforms/pseries/lparcfg.c @@ -686,7 +686,7 @@ static ssize_t lparcfg_write(struct file *file, const char __user * buf, if (!firmware_has_feature(FW_FEATURE_SPLPAR)) return -EINVAL; - if (count > sizeof(kbuf)) + if (count == 0 || count > sizeof(kbuf)) return -EINVAL; if (copy_from_user(kbuf, buf, count)) diff --git a/arch/powerpc/platforms/pseries/papr-phy-attest.c b/arch/powerpc/platforms/pseries/papr-phy-attest.c index 1907f2411567d..d661e6f961888 100644 --- a/arch/powerpc/platforms/pseries/papr-phy-attest.c +++ b/arch/powerpc/platforms/pseries/papr-phy-attest.c @@ -230,10 +230,17 @@ static long papr_phy_attest_create_handle(struct papr_phy_attest_io_block __user return -ENOMEM; if (copy_from_user(¶ms->cmd, ulc, - sizeof(struct papr_phy_attest_io_block))) + sizeof(struct papr_phy_attest_io_block))) { + kfree(params); return -EFAULT; + } params->cmd_len = be32_to_cpu(params->cmd.length); + if (params->cmd_len == 0 || params->cmd_len > sizeof(params->cmd)) { + kfree(params); + return -EINVAL; + } + seq = (struct papr_rtas_sequence) { .begin = phy_attest_sequence_begin, .end = phy_attest_sequence_end, @@ -246,6 +253,9 @@ static long papr_phy_attest_create_handle(struct papr_phy_attest_io_block __user &papr_phy_attest_handle_ops, "[papr-physical-attestation]"); + if (fd < 0) + kfree(params); + return fd; } diff --git a/arch/powerpc/platforms/pseries/pci.c b/arch/powerpc/platforms/pseries/pci.c index 6dbc73eb2ca25..82dc1ad52ac9c 100644 --- a/arch/powerpc/platforms/pseries/pci.c +++ b/arch/powerpc/platforms/pseries/pci.c @@ -132,7 +132,7 @@ static int pseries_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs) /* First integer stores max config */ max_config_vfs = of_read_number(&max_vfs[0], 1); - if (max_config_vfs < num_vfs && num_vfs > MAX_VFS_FOR_MAP_PE) { + if (max_config_vfs < num_vfs || num_vfs > MAX_VFS_FOR_MAP_PE) { dev_err(&pdev->dev, "Num VFs %x > %x Configurable VFs\n", num_vfs, (num_vfs > MAX_VFS_FOR_MAP_PE) ? diff --git a/arch/powerpc/platforms/pseries/rtas-fadump.c b/arch/powerpc/platforms/pseries/rtas-fadump.c index eceb3289383ec..3bb4ac2ab6cc3 100644 --- a/arch/powerpc/platforms/pseries/rtas-fadump.c +++ b/arch/powerpc/platforms/pseries/rtas-fadump.c @@ -179,9 +179,42 @@ static u64 rtas_fadump_get_bootmem_min(void) return RTAS_FADUMP_MIN_BOOT_MEM; } +/* + * Helper to make an ibm,configure-kernel-dump RTAS call with a bounded + * busy-wait loop. Returns the RTAS return code on completion, or + * -ETIMEDOUT if firmware keeps returning a busy status beyond + * RTAS_FADUMP_MAX_WAIT_MS milliseconds. + */ +static int rtas_fadump_call(struct fw_dump *fadump_conf, int operation, + void *fdm_ptr, unsigned int fdm_size, + const char *op_name) +{ + unsigned int wait_time, total_wait = 0; + int rc; + + do { + rc = rtas_call(fadump_conf->ibm_configure_kernel_dump, 3, 1, + NULL, operation, fdm_ptr, fdm_size); + wait_time = rtas_busy_delay_time(rc); + if (wait_time) { + pr_debug("Firmware busy during fadump %s, waiting %ums (total %ums)\n", + op_name, wait_time, total_wait); + if (total_wait >= RTAS_FADUMP_MAX_WAIT_MS) { + pr_err("Timed out waiting for firmware to complete fadump %s\n", + op_name); + return -ETIMEDOUT; + } + total_wait += wait_time; + mdelay(wait_time); + } + } while (wait_time); + + return rc; +} + static int rtas_fadump_register(struct fw_dump *fadump_conf) { - unsigned int wait_time, fdm_size; + unsigned int fdm_size; int rc, err = -EIO; /* @@ -192,16 +225,10 @@ static int rtas_fadump_register(struct fw_dump *fadump_conf) fdm_size = sizeof(struct rtas_fadump_section_header); fdm_size += be16_to_cpu(fdm.header.dump_num_sections) * sizeof(struct rtas_fadump_section); - /* TODO: Add upper time limit for the delay */ - do { - rc = rtas_call(fadump_conf->ibm_configure_kernel_dump, 3, 1, - NULL, FADUMP_REGISTER, &fdm, fdm_size); - - wait_time = rtas_busy_delay_time(rc); - if (wait_time) - mdelay(wait_time); - - } while (wait_time); + rc = rtas_fadump_call(fadump_conf, FADUMP_REGISTER, &fdm, fdm_size, + "register"); + if (rc == -ETIMEDOUT) + return -ETIMEDOUT; switch (rc) { case 0: @@ -234,19 +261,12 @@ static int rtas_fadump_register(struct fw_dump *fadump_conf) static int rtas_fadump_unregister(struct fw_dump *fadump_conf) { - unsigned int wait_time; int rc; - /* TODO: Add upper time limit for the delay */ - do { - rc = rtas_call(fadump_conf->ibm_configure_kernel_dump, 3, 1, - NULL, FADUMP_UNREGISTER, &fdm, - sizeof(struct rtas_fadump_mem_struct)); - - wait_time = rtas_busy_delay_time(rc); - if (wait_time) - mdelay(wait_time); - } while (wait_time); + rc = rtas_fadump_call(fadump_conf, FADUMP_UNREGISTER, &fdm, + sizeof(struct rtas_fadump_mem_struct), "unregister"); + if (rc == -ETIMEDOUT) + return -ETIMEDOUT; if (rc) { pr_err("Failed to un-register - unexpected error(%d).\n", rc); @@ -259,19 +279,13 @@ static int rtas_fadump_unregister(struct fw_dump *fadump_conf) static int rtas_fadump_invalidate(struct fw_dump *fadump_conf) { - unsigned int wait_time; int rc; - /* TODO: Add upper time limit for the delay */ - do { - rc = rtas_call(fadump_conf->ibm_configure_kernel_dump, 3, 1, - NULL, FADUMP_INVALIDATE, fdm_active, - sizeof(struct rtas_fadump_mem_struct)); - - wait_time = rtas_busy_delay_time(rc); - if (wait_time) - mdelay(wait_time); - } while (wait_time); + rc = rtas_fadump_call(fadump_conf, FADUMP_INVALIDATE, + (void *)fdm_active, + sizeof(struct rtas_fadump_mem_struct), "invalidate"); + if (rc == -ETIMEDOUT) + return -ETIMEDOUT; if (rc) { pr_err("Failed to invalidate - unexpected error (%d).\n", rc); diff --git a/arch/powerpc/platforms/pseries/rtas-fadump.h b/arch/powerpc/platforms/pseries/rtas-fadump.h index c109abf6befd9..65fdab7b5b8d3 100644 --- a/arch/powerpc/platforms/pseries/rtas-fadump.h +++ b/arch/powerpc/platforms/pseries/rtas-fadump.h @@ -41,6 +41,12 @@ #define MAX_SECTIONS 10 #define RTAS_FADUMP_MAX_BOOT_MEM_REGS 7 +/* + * Maximum time to wait for firmware to respond to an + * ibm,configure-kernel-dump RTAS call before giving up. + */ +#define RTAS_FADUMP_MAX_WAIT_MS 60000U + /* Kernel Dump section info */ struct rtas_fadump_section { __be32 request_flag; diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index b10a253252387..d9ee2e7735ecd 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -76,6 +76,7 @@ #include #include #include +#include #include "pseries.h" @@ -187,14 +188,37 @@ static void __init fwnmi_init(void) #endif } +static void pseries_crash_stop_watchdogs(void) +{ + long rc; + + rc = plpar_hcall_norets_notrace(H_WATCHDOG, PSERIES_WDTF_OP_STOP, + PSERIES_WDT_NUM_ALL); + if (rc != H_SUCCESS && rc != H_NOOP) + pr_warn("Could not stop watchdogs before kdump rc=%ld\n", rc); +} + /* * Affix a device for the first timer to the platform bus if * we have firmware support for the H_WATCHDOG hypercall. */ static __init int pseries_wdt_init(void) { - if (firmware_has_feature(FW_FEATURE_WATCHDOG)) - platform_device_register_simple("pseries-wdt", 0, NULL, 0); + struct platform_device *pdev; + + if (!firmware_has_feature(FW_FEATURE_WATCHDOG)) + return 0; + + pdev = platform_device_register_simple("pseries-wdt", 0, NULL, 0); + + if (IS_ERR(pdev)) { + pr_err("Failed to register pseries-wdt platform device\n"); + return PTR_ERR(pdev); + } + + if (crash_shutdown_register(pseries_crash_stop_watchdogs)) + pr_warn("Could not register watchdog crash shutdown handler\n"); + return 0; } machine_subsys_initcall(pseries, pseries_wdt_init); diff --git a/arch/powerpc/sysdev/cpm_common.c b/arch/powerpc/sysdev/cpm_common.c index 07ea605ab0e62..b5d200e3ad684 100644 --- a/arch/powerpc/sysdev/cpm_common.c +++ b/arch/powerpc/sysdev/cpm_common.c @@ -181,6 +181,18 @@ static int cpm2_gpio32_dir_in(struct gpio_chip *gc, unsigned int gpio) return 0; } +static int cpm2_gpio32_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct cpm2_gpio32_chip *cpm2_gc = gpiochip_get_data(gc); + struct cpm2_ioports __iomem *iop = cpm2_gc->regs; + u32 pin_mask = 1 << (31 - gpio); + + if (in_be32(&iop->dir) & pin_mask) + return GPIO_LINE_DIRECTION_OUT; + + return GPIO_LINE_DIRECTION_IN; +} + int cpm2_gpiochip_add32(struct device *dev) { struct device_node *np = dev->of_node; @@ -199,6 +211,7 @@ int cpm2_gpiochip_add32(struct device *dev) gc->ngpio = 32; gc->direction_input = cpm2_gpio32_dir_in; gc->direction_output = cpm2_gpio32_dir_out; + gc->get_direction = cpm2_gpio32_get_direction; gc->get = cpm2_gpio32_get; gc->set = cpm2_gpio32_set; gc->parent = dev; diff --git a/arch/powerpc/sysdev/fsl_lbc.c b/arch/powerpc/sysdev/fsl_lbc.c index 217cea150987d..7ed07232a69a4 100644 --- a/arch/powerpc/sysdev/fsl_lbc.c +++ b/arch/powerpc/sysdev/fsl_lbc.c @@ -350,7 +350,7 @@ static int fsl_lbc_ctrl_probe(struct platform_device *dev) #ifdef CONFIG_SUSPEND /* save lbc registers */ -static int fsl_lbc_syscore_suspend(void) +static int fsl_lbc_syscore_suspend(void *data) { struct fsl_lbc_ctrl *ctrl; struct fsl_lbc_regs __iomem *lbc; @@ -374,7 +374,7 @@ static int fsl_lbc_syscore_suspend(void) } /* restore lbc registers */ -static void fsl_lbc_syscore_resume(void) +static void fsl_lbc_syscore_resume(void *data) { struct fsl_lbc_ctrl *ctrl; struct fsl_lbc_regs __iomem *lbc; @@ -408,10 +408,14 @@ static const struct of_device_id fsl_lbc_match[] = { }; #ifdef CONFIG_SUSPEND -static struct syscore_ops lbc_syscore_pm_ops = { +static const struct syscore_ops lbc_syscore_pm_ops = { .suspend = fsl_lbc_syscore_suspend, .resume = fsl_lbc_syscore_resume, }; + +static struct syscore lbc_syscore_pm = { + .ops = &lbc_syscore_pm_ops, +}; #endif static struct platform_driver fsl_lbc_ctrl_driver = { @@ -425,7 +429,7 @@ static struct platform_driver fsl_lbc_ctrl_driver = { static int __init fsl_lbc_init(void) { #ifdef CONFIG_SUSPEND - register_syscore_ops(&lbc_syscore_pm_ops); + register_syscore(&lbc_syscore_pm); #endif return platform_driver_register(&fsl_lbc_ctrl_driver); } diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c index ef7707ea0db7b..4e501654cb41b 100644 --- a/arch/powerpc/sysdev/fsl_pci.c +++ b/arch/powerpc/sysdev/fsl_pci.c @@ -1258,7 +1258,7 @@ static void fsl_pci_syscore_do_suspend(struct pci_controller *hose) send_pme_turnoff_message(hose); } -static int fsl_pci_syscore_suspend(void) +static int fsl_pci_syscore_suspend(void *data) { struct pci_controller *hose, *tmp; @@ -1291,7 +1291,7 @@ static void fsl_pci_syscore_do_resume(struct pci_controller *hose) setup_pci_atmu(hose); } -static void fsl_pci_syscore_resume(void) +static void fsl_pci_syscore_resume(void *data) { struct pci_controller *hose, *tmp; @@ -1299,10 +1299,14 @@ static void fsl_pci_syscore_resume(void) fsl_pci_syscore_do_resume(hose); } -static struct syscore_ops pci_syscore_pm_ops = { +static const struct syscore_ops pci_syscore_pm_ops = { .suspend = fsl_pci_syscore_suspend, .resume = fsl_pci_syscore_resume, }; + +static struct syscore pci_syscore_pm = { + .ops = &pci_syscore_pm_ops, +}; #endif void fsl_pcibios_fixup_phb(struct pci_controller *phb) @@ -1359,7 +1363,7 @@ static struct platform_driver fsl_pci_driver = { static int __init fsl_pci_init(void) { #ifdef CONFIG_PM_SLEEP - register_syscore_ops(&pci_syscore_pm_ops); + register_syscore(&pci_syscore_pm); #endif return platform_driver_register(&fsl_pci_driver); } diff --git a/arch/powerpc/sysdev/ipic.c b/arch/powerpc/sysdev/ipic.c index 70be2105865db..290ba8427239f 100644 --- a/arch/powerpc/sysdev/ipic.c +++ b/arch/powerpc/sysdev/ipic.c @@ -817,7 +817,7 @@ static struct { u32 sercr; } ipic_saved_state; -static int ipic_suspend(void) +static int ipic_suspend(void *data) { struct ipic *ipic = primary_ipic; @@ -848,7 +848,7 @@ static int ipic_suspend(void) return 0; } -static void ipic_resume(void) +static void ipic_resume(void *data) { struct ipic *ipic = primary_ipic; @@ -870,18 +870,22 @@ static void ipic_resume(void) #define ipic_resume NULL #endif -static struct syscore_ops ipic_syscore_ops = { +static const struct syscore_ops ipic_syscore_ops = { .suspend = ipic_suspend, .resume = ipic_resume, }; +static struct syscore ipic_syscore = { + .ops = &ipic_syscore_ops, +}; + static int __init init_ipic_syscore(void) { if (!primary_ipic || !primary_ipic->regs) return -ENODEV; printk(KERN_DEBUG "Registering ipic system core operations\n"); - register_syscore_ops(&ipic_syscore_ops); + register_syscore(&ipic_syscore); return 0; } diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index ad7310bba00bc..67e51998d1aea 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -1944,7 +1944,7 @@ static void mpic_suspend_one(struct mpic *mpic) } } -static int mpic_suspend(void) +static int mpic_suspend(void *data) { struct mpic *mpic = mpics; @@ -1986,7 +1986,7 @@ static void mpic_resume_one(struct mpic *mpic) } /* end for loop */ } -static void mpic_resume(void) +static void mpic_resume(void *data) { struct mpic *mpic = mpics; @@ -1996,19 +1996,23 @@ static void mpic_resume(void) } } -static struct syscore_ops mpic_syscore_ops = { +static const struct syscore_ops mpic_syscore_ops = { .resume = mpic_resume, .suspend = mpic_suspend, }; +static struct syscore mpic_syscore = { + .ops = &mpic_syscore_ops, +}; + static int mpic_init_sys(void) { int rc; - register_syscore_ops(&mpic_syscore_ops); + register_syscore(&mpic_syscore); rc = subsys_system_register(&mpic_subsys, NULL); if (rc) { - unregister_syscore_ops(&mpic_syscore_ops); + unregister_syscore(&mpic_syscore); pr_err("mpic: Failed to register subsystem!\n"); return rc; } diff --git a/arch/powerpc/sysdev/mpic_timer.c b/arch/powerpc/sysdev/mpic_timer.c index 7166e2e0baaf6..60f5b3934b511 100644 --- a/arch/powerpc/sysdev/mpic_timer.c +++ b/arch/powerpc/sysdev/mpic_timer.c @@ -519,7 +519,7 @@ static void __init timer_group_init(struct device_node *np) kfree(priv); } -static void mpic_timer_resume(void) +static void mpic_timer_resume(void *data) { struct timer_group_priv *priv; @@ -535,10 +535,14 @@ static const struct of_device_id mpic_timer_ids[] = { {}, }; -static struct syscore_ops mpic_timer_syscore_ops = { +static const struct syscore_ops mpic_timer_syscore_ops = { .resume = mpic_timer_resume, }; +static struct syscore mpic_timer_syscore = { + .ops = &mpic_timer_syscore_ops, +}; + static int __init mpic_timer_init(void) { struct device_node *np = NULL; @@ -546,7 +550,7 @@ static int __init mpic_timer_init(void) for_each_matching_node(np, mpic_timer_ids) timer_group_init(np); - register_syscore_ops(&mpic_timer_syscore_ops); + register_syscore(&mpic_timer_syscore); if (list_empty(&timer_group_list)) return -ENODEV; diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi index ccdb454986535..b2453ae452164 100644 --- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi +++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi @@ -391,7 +391,7 @@ }; dmac: dma-controller@4330000 { - compatible = "snps,axi-dma-1.01a"; + compatible = "sophgo,cv1800b-axi-dma", "snps,axi-dma-1.01a"; reg = <0x04330000 0x1000>; interrupts = ; clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>; diff --git a/arch/riscv/boot/dts/spacemit/Makefile b/arch/riscv/boot/dts/spacemit/Makefile index 1528326448706..95889e7269d1b 100644 --- a/arch/riscv/boot/dts/spacemit/Makefile +++ b/arch/riscv/boot/dts/spacemit/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_SPACEMIT) += k1-bananapi-f3.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-milkv-jupiter.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-musepi-pro.dtb +dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-r2s.dtb dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-rv2.dtb diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts index a58192ba67b2c..b2c40bae60945 100644 --- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts +++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts @@ -46,7 +46,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -69,7 +69,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; diff --git a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts index a01f69f202e37..695af4919b94a 100644 --- a/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts +++ b/arch/riscv/boot/dts/spacemit/k1-milkv-jupiter.dts @@ -26,7 +26,7 @@ phy-handle = <&rgmii0>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac0_cfg>; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <0>; status = "okay"; @@ -49,7 +49,7 @@ phy-handle = <&rgmii1>; phy-mode = "rgmii-id"; pinctrl-names = "default"; - pinctrl-0 = <&gmac1_cfg>; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; rx-internal-delay-ps = <0>; tx-internal-delay-ps = <250>; status = "okay"; diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts new file mode 100644 index 0000000000000..42efc8c1a5818 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2024 Yangyu Chen + * Copyright (C) 2025 SpacemiT, Inc + * Copyright (C) 2025 Troy Mitchell + */ + +/dts-v1/; + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "SpacemiT MusePi Pro"; + compatible = "spacemit,musepi-pro", "spacemit,k1"; + + aliases { + ethernet0 = ð0; + serial0 = &uart0; + }; + + chosen { + stdout-path = "serial0"; + }; + + leds { + compatible = "gpio-leds"; + + led1 { + label = "sys-led"; + gpios = <&gpio K1_GPIO(96) GPIO_ACTIVE_HIGH>; + linux,default-trigger = "heartbeat"; + default-state = "on"; + }; + }; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; + pinctrl-names = "default"; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&uart0 { + pinctrl-0 = <&uart0_2_cfg>; + pinctrl-names = "default"; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts new file mode 100644 index 0000000000000..312d5bc860722 --- /dev/null +++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-r2s.dts @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (C) 2025 Michael Opdenacker + */ + +/dts-v1/; + +#include "k1.dtsi" +#include "k1-pinctrl.dtsi" + +/ { + model = "OrangePi R2S"; + compatible = "xunlong,orangepi-r2s", "spacemit,k1"; + + aliases { + serial0 = &uart0; + ethernet0 = ð0; + ethernet1 = ð1; + }; + + chosen { + stdout-path = "serial0"; + }; +}; + +&emmc { + bus-width = <8>; + mmc-hs400-1_8v; + mmc-hs400-enhanced-strobe; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +ð0 { + phy-handle = <&rgmii0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac0_cfg>, <&gmac0_clk_ref_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <0>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(110) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii0: phy@1 { + reg = <0x1>; + }; + }; +}; + +ð1 { + phy-handle = <&rgmii1>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gmac1_cfg>, <&gmac1_clk_ref_cfg>; + rx-internal-delay-ps = <0>; + tx-internal-delay-ps = <250>; + status = "okay"; + + mdio-bus { + #address-cells = <0x1>; + #size-cells = <0x0>; + + reset-gpios = <&gpio K1_GPIO(115) GPIO_ACTIVE_LOW>; + reset-delay-us = <10000>; + reset-post-delay-us = <100000>; + + rgmii1: phy@1 { + reg = <0x1>; + }; + }; +}; + +&pdma { + status = "okay"; +}; + +&uart0 { + pinctrl-names = "default"; + pinctrl-0 = <&uart0_2_cfg>; + status = "okay"; +}; diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi index aff19c86d5ff3..48c4b41311646 100644 --- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi +++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi @@ -27,8 +27,16 @@ , /* gmac0_tx_en */ , /* gmac0_mdc */ , /* gmac0_mdio */ - , /* gmac0_int_n */ - ; /* gmac0_clk_ref */ + ; /* gmac0_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac0_clk_ref_cfg: gmac0-clk-ref-cfg { + gmac0-clk-ref-pins { + pinmux = ; /* gmac0_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; @@ -51,8 +59,16 @@ , /* gmac1_tx_en */ , /* gmac1_mdc */ , /* gmac1_mdio */ - , /* gmac1_int_n */ - ; /* gmac1_clk_ref */ + ; /* gmac1_int_n */ + + bias-pull-up = <0>; + drive-strength = <21>; + }; + }; + + gmac1_clk_ref_cfg: gmac1-clk-ref-cfg { + gmac1-clk-ref-pins { + pinmux = ; /* gmac1_clk_ref */ bias-pull-up = <0>; drive-strength = <21>; diff --git a/arch/riscv/include/asm/acpi.h b/arch/riscv/include/asm/acpi.h index 6e13695120bc2..a4b9e60f747c8 100644 --- a/arch/riscv/include/asm/acpi.h +++ b/arch/riscv/include/asm/acpi.h @@ -12,6 +12,8 @@ #ifndef _ASM_ACPI_H #define _ASM_ACPI_H +#include + /* Basic configuration for ACPI */ #ifdef CONFIG_ACPI @@ -71,6 +73,23 @@ int acpi_get_riscv_isa(struct acpi_table_header *table, void acpi_get_cbo_block_size(struct acpi_table_header *table, u32 *cbom_size, u32 *cboz_size, u32 *cbop_size); + +/* + * RISC-V Functional Fixed Hardware Specification Version v1.0.1, + * Chapter 3.1.2, Table 4: Arch. Context Lost Flags + */ +#define RISCV_LPI_HART_TIMER_CTXT_LOST BIT(0) + +static inline unsigned int arch_get_idle_state_flags(u32 arch_flags) +{ + if (arch_flags & RISCV_LPI_HART_TIMER_CTXT_LOST) + return CPUIDLE_FLAG_TIMER_STOP; + + return 0; +} + +#define arch_get_idle_state_flags arch_get_idle_state_flags + #else static inline void acpi_init_rintc_map(void) { } static inline struct acpi_madt_rintc *acpi_cpu_get_madt_rintc(int cpu) diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h index 4d794573e3dbe..fd3ba183cd824 100644 --- a/arch/riscv/include/asm/kvm_host.h +++ b/arch/riscv/include/asm/kvm_host.h @@ -220,13 +220,13 @@ struct kvm_vcpu_arch { /* * VCPU interrupts * - * We have a lockless approach for tracking pending VCPU interrupts - * implemented using atomic bitops. The irqs_pending bitmap represent - * pending interrupts whereas irqs_pending_mask represent bits changed - * in irqs_pending. Our approach is modeled around multiple producer - * and single consumer problem where the consumer is the VCPU itself. + * The irqs_pending bitmap represents pending interrupts whereas + * irqs_pending_mask represents bits changed in irqs_pending. Updates + * to these bitmaps are serialized so vcpu interrupt sync/flush cannot + * drop a newly injected interrupt while syncing guest-visible HVIP. */ #define KVM_RISCV_VCPU_NR_IRQS 64 + raw_spinlock_t irqs_pending_lock; DECLARE_BITMAP(irqs_pending, KVM_RISCV_VCPU_NR_IRQS); DECLARE_BITMAP(irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); diff --git a/arch/riscv/kernel/compat_signal.c b/arch/riscv/kernel/compat_signal.c index 6ec4e34255a9a..cf3eb33a11e46 100644 --- a/arch/riscv/kernel/compat_signal.c +++ b/arch/riscv/kernel/compat_signal.c @@ -107,6 +107,8 @@ static long compat_restore_sigcontext(struct pt_regs *regs, /* sc_regs is structured the same as the start of pt_regs */ err = __copy_from_user(&cregs, &sc->sc_regs, sizeof(sc->sc_regs)); + if (unlikely(err)) + return err; cregs_to_regs(&cregs, regs); diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 2367e9755524a..17d21a708fb76 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -409,7 +409,8 @@ static const unsigned int riscv_c_exts[] = { /* * The canonical order of ISA extension names in the ISA string is defined in - * chapter 27 of the unprivileged specification. + * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA + * (Document Version 20191213). * * Ordinarily, for in-kernel data structures, this order is unimportant but * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo. diff --git a/arch/riscv/kernel/ftrace.c b/arch/riscv/kernel/ftrace.c index 8d18d6727f0fc..c201c6bac1dda 100644 --- a/arch/riscv/kernel/ftrace.c +++ b/arch/riscv/kernel/ftrace.c @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef CONFIG_DYNAMIC_FTRACE void ftrace_arch_code_modify_prepare(void) @@ -63,7 +64,9 @@ static int __ftrace_modify_call(unsigned long source, unsigned long target, bool if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE)) return -EFAULT; - if (replaced[0] != call[0]) { + /* Bypass the check if the auipc insn is a kprobe breakpoint */ + if (replaced[0] != call[0] && + !(riscv_insn_is_ebreak(replaced[0]) || riscv_insn_is_c_ebreak(replaced[0]))) { pr_err("%p: expected (%08x) but got (%08x)\n", (void *)source, call[0], replaced[0]); return -EINVAL; diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index b6af20bc300f5..017d42e778be6 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -75,28 +75,34 @@ DECLARE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); DEFINE_PER_CPU(ulong *, irq_shadow_call_stack_ptr); #endif -static void init_irq_scs(void) +static void __init init_irq_scs(void) { int cpu; + void *s; if (!scs_is_enabled()) return; - for_each_possible_cpu(cpu) - per_cpu(irq_shadow_call_stack_ptr, cpu) = - scs_alloc(cpu_to_node(cpu)); + for_each_possible_cpu(cpu) { + s = scs_alloc(cpu_to_node(cpu)); + if (!s) + panic("Failed to allocate IRQ shadow call stack resources\n"); + per_cpu(irq_shadow_call_stack_ptr, cpu) = s; + } } DEFINE_PER_CPU(ulong *, irq_stack_ptr); #ifdef CONFIG_VMAP_STACK -static void init_irq_stacks(void) +static void __init init_irq_stacks(void) { int cpu; ulong *p; for_each_possible_cpu(cpu) { p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, cpu_to_node(cpu)); + if (!p) + panic("Failed to allocate IRQ stack resources\n"); per_cpu(irq_stack_ptr, cpu) = p; } } @@ -104,7 +110,7 @@ static void init_irq_stacks(void) /* irq stack only needs to be 16 byte aligned - not IRQ_STACK_SIZE aligned. */ DEFINE_PER_CPU_ALIGNED(ulong [IRQ_STACK_SIZE/sizeof(ulong)], irq_stack); -static void init_irq_stacks(void) +static void __init init_irq_stacks(void) { int cpu; @@ -129,8 +135,8 @@ void do_softirq_own_stack(void) #endif /* CONFIG_SOFTIRQ_ON_OWN_STACK */ #else -static void init_irq_scs(void) {} -static void init_irq_stacks(void) {} +static void __init init_irq_scs(void) {} +static void __init init_irq_stacks(void) {} #endif /* CONFIG_IRQ_STACKS */ int arch_show_interrupts(struct seq_file *p, int prec) diff --git a/arch/riscv/kernel/machine_kexec_file.c b/arch/riscv/kernel/machine_kexec_file.c index dd9d92a965174..6ef5bd34d130a 100644 --- a/arch/riscv/kernel/machine_kexec_file.c +++ b/arch/riscv/kernel/machine_kexec_file.c @@ -61,7 +61,7 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) unsigned int nr_ranges; int ret; - nr_ranges = 1; /* For exclusion of crashkernel region */ + nr_ranges = 2; /* For exclusion of crashkernel region */ walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback); cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL); @@ -76,8 +76,16 @@ static int prepare_elf_headers(void **addr, unsigned long *sz) /* Exclude crashkernel region */ ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end); - if (!ret) - ret = crash_prepare_elf64_headers(cmem, true, addr, sz); + if (ret) + goto out; + + if (crashk_low_res.end) { + ret = crash_exclude_mem_range(cmem, crashk_low_res.start, crashk_low_res.end); + if (ret) + goto out; + } + + ret = crash_prepare_elf64_headers(cmem, true, addr, sz); out: kfree(cmem); diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c index b5188dc74727d..31a392993cb45 100644 --- a/arch/riscv/kernel/process.c +++ b/arch/riscv/kernel/process.c @@ -324,10 +324,8 @@ long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg) if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen)) return -EINVAL; - if (!(arg & PR_TAGGED_ADDR_ENABLE)) { + if (!(arg & PR_TAGGED_ADDR_ENABLE)) pmlen = PMLEN_0; - pmm = ENVCFG_PMM_PMLEN_0; - } if (mmap_write_lock_killable(mm)) return -EINTR; diff --git a/arch/riscv/kernel/ptrace.c b/arch/riscv/kernel/ptrace.c index 8e86305831ea4..32cd48a4cebb8 100644 --- a/arch/riscv/kernel/ptrace.c +++ b/arch/riscv/kernel/ptrace.c @@ -372,8 +372,8 @@ static int compat_riscv_gpr_set(struct task_struct *target, struct compat_user_regs_struct cregs; ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &cregs, 0, -1); - - cregs_to_regs(&cregs, task_pt_regs(target)); + if (!ret) + cregs_to_regs(&cregs, task_pt_regs(target)); return ret; } diff --git a/arch/riscv/kernel/sys_hwprobe.c b/arch/riscv/kernel/sys_hwprobe.c index 199d13f86f313..460fb297672e6 100644 --- a/arch/riscv/kernel/sys_hwprobe.c +++ b/arch/riscv/kernel/sys_hwprobe.c @@ -409,6 +409,7 @@ static int hwprobe_get_cpus(struct riscv_hwprobe __user *pairs, if (cpusetsize > cpumask_size()) cpusetsize = cpumask_size(); + cpumask_clear(&cpus); ret = copy_from_user(&cpus, cpus_user, cpusetsize); if (ret) return -EFAULT; diff --git a/arch/riscv/kernel/unaligned_access_speed.c b/arch/riscv/kernel/unaligned_access_speed.c index 70b5e69276209..f111a30d480cb 100644 --- a/arch/riscv/kernel/unaligned_access_speed.c +++ b/arch/riscv/kernel/unaligned_access_speed.c @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -375,18 +374,9 @@ static void check_vector_unaligned_access(struct work_struct *work __always_unus __free_pages(page, MISALIGNED_BUFFER_ORDER); } -/* Measure unaligned access speed on all CPUs present at boot in parallel. */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) -{ - schedule_on_each_cpu(check_vector_unaligned_access); - riscv_hwprobe_complete_async_probe(); - - return 0; -} #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ -static int __init vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused) +static void check_vector_unaligned_access(struct work_struct *work __always_unused) { - return 0; } #endif @@ -474,12 +464,7 @@ static int __init check_unaligned_access_all_cpus(void) per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; } else if (!check_vector_unaligned_access_emulated_all_cpus() && IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { - riscv_hwprobe_register_async_probe(); - if (IS_ERR(kthread_run(vec_check_unaligned_access_speed_all_cpus, - NULL, "vec_check_unaligned_access_speed_all_cpus"))) { - pr_warn("Failed to create vec_unalign_check kthread\n"); - riscv_hwprobe_complete_async_probe(); - } + schedule_on_each_cpu(check_vector_unaligned_access); } /* diff --git a/arch/riscv/kvm/aia.c b/arch/riscv/kvm/aia.c index dad3181856600..b8b5060821556 100644 --- a/arch/riscv/kvm/aia.c +++ b/arch/riscv/kvm/aia.c @@ -50,12 +50,15 @@ void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu) struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; unsigned long mask, val; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (!kvm_riscv_aia_available()) return; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask; + mask = vcpu->arch.irqs_pending_mask[1]; + if (mask) { + vcpu->arch.irqs_pending_mask[1] = 0; + val = vcpu->arch.irqs_pending[1] & mask; csr->hviph &= ~mask; csr->hviph |= val; @@ -66,6 +69,8 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; + lockdep_assert_held(&vcpu->arch.irqs_pending_lock); + if (kvm_riscv_aia_available()) csr->vsieh = ncsr_read(CSR_VSIEH); } @@ -74,13 +79,22 @@ void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu) bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { unsigned long seip; +#ifdef CONFIG_32BIT + unsigned long flags; + bool pending; +#endif if (!kvm_riscv_aia_available()) return false; #ifdef CONFIG_32BIT - if (READ_ONCE(vcpu->arch.irqs_pending[1]) & - (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask))) + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + pending = vcpu->arch.irqs_pending[1] & + (vcpu->arch.aia_context.guest_csr.vsieh & + upper_32_bits(mask)); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + + if (pending) return true; #endif @@ -198,6 +212,9 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, unsigned long val) { struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr; +#ifdef CONFIG_32BIT + unsigned long flags; +#endif if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long)) return -ENOENT; @@ -206,8 +223,12 @@ int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = val; #ifdef CONFIG_32BIT - if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0); + if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[1] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, + flags); + } #endif } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index d26c4967c20e2..6f8e9b105da7e 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -78,6 +78,7 @@ static void kvm_riscv_vcpu_context_reset(struct kvm_vcpu *vcpu, static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) { + unsigned long flags; bool loaded; /** @@ -102,8 +103,10 @@ static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu, bool kvm_sbi_reset) kvm_riscv_vcpu_aia_reset(vcpu); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_riscv_vcpu_pmu_reset(vcpu); @@ -147,6 +150,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) /* Setup VCPU hfence queue */ spin_lock_init(&vcpu->arch.hfence_lock); + raw_spin_lock_init(&vcpu->arch.irqs_pending_lock); spin_lock_init(&vcpu->arch.reset_state.lock); @@ -348,10 +352,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; unsigned long mask, val; + unsigned long flags; - if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { - mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); - val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + + mask = vcpu->arch.irqs_pending_mask[0]; + if (mask) { + vcpu->arch.irqs_pending_mask[0] = 0; + val = vcpu->arch.irqs_pending[0] & mask; csr->hvip &= ~mask; csr->hvip |= val; @@ -359,11 +367,14 @@ void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) /* Flush AIA high interrupts */ kvm_riscv_vcpu_aia_flush_interrupts(vcpu); + + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); } void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) { unsigned long hvip; + unsigned long flags; struct kvm_vcpu_arch *v = &vcpu->arch; struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; @@ -372,34 +383,41 @@ void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) /* Sync-up HVIP.VSSIP bit changes does by Guest */ hvip = ncsr_read(CSR_HVIP); + + raw_spin_lock_irqsave(&v->irqs_pending_lock, flags); + if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { if (hvip & (1UL << IRQ_VS_SOFT)) { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - set_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __set_bit(IRQ_VS_SOFT, v->irqs_pending); } else { - if (!test_and_set_bit(IRQ_VS_SOFT, - v->irqs_pending_mask)) - clear_bit(IRQ_VS_SOFT, v->irqs_pending); + if (!__test_and_set_bit(IRQ_VS_SOFT, + v->irqs_pending_mask)) + __clear_bit(IRQ_VS_SOFT, v->irqs_pending); } } /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { if (!(hvip & (1UL << IRQ_PMU_OVF)) && - !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) - clear_bit(IRQ_PMU_OVF, v->irqs_pending); + !__test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) + __clear_bit(IRQ_PMU_OVF, v->irqs_pending); } /* Sync-up AIA high interrupts */ kvm_riscv_vcpu_aia_sync_interrupts(vcpu); + raw_spin_unlock_irqrestore(&v->irqs_pending_lock, flags); + /* Sync-up timer CSRs */ kvm_riscv_vcpu_timer_sync(vcpu); } int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, and external * interrupts when irq is one of the local interrupts @@ -412,9 +430,10 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - set_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __set_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); kvm_vcpu_kick(vcpu); @@ -423,6 +442,8 @@ int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) { + unsigned long flags; + /* * We only allow VS-mode software, timer, counter overflow and external * interrupts when irq is one of the local interrupts @@ -435,26 +456,33 @@ int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) irq != IRQ_PMU_OVF) return -EINVAL; - clear_bit(irq, vcpu->arch.irqs_pending); - smp_mb__before_atomic(); - set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + __clear_bit(irq, vcpu->arch.irqs_pending); + __set_bit(irq, vcpu->arch.irqs_pending_mask); + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); return 0; } bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) { + unsigned long flags; unsigned long ie; + bool ret; + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & (unsigned long)mask; - if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) - return true; + ret = vcpu->arch.irqs_pending[0] & ie; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); /* Check AIA high interrupts */ - return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + if (!ret) + ret = kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); + + return ret; } void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index 865dae903aa0f..3e8f9a2834121 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -522,6 +522,7 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, unsigned long reg_val) { struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; + unsigned long flags; if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) return -ENOENT; @@ -533,8 +534,11 @@ static int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, ((unsigned long *)csr)[reg_num] = reg_val; - if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) - WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); + if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { + raw_spin_lock_irqsave(&vcpu->arch.irqs_pending_lock, flags); + vcpu->arch.irqs_pending_mask[0] = 0; + raw_spin_unlock_irqrestore(&vcpu->arch.irqs_pending_lock, flags); + } return 0; } diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index ee40ca01ac663..557bf160b503a 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -1738,7 +1738,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un for (; addr < end; addr = next) { next = pud_addr_end(addr, end); - pudp = pud_base + pud_index(addr); + pudp = pgtable_l4_enabled ? pud_base + pud_index(addr) : pud_base; pud = pudp_get(pudp); if (!pud_present(pud)) continue; @@ -1769,7 +1769,7 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un for (; addr < end; addr = next) { next = p4d_addr_end(addr, end); - p4dp = p4d_base + p4d_index(addr); + p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) continue; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 9e9e6dcfc4825..3e4eea261ab96 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -18,7 +18,6 @@ #define RV_MAX_REG_ARGS 8 #define RV_FENTRY_NINSNS 2 #define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4) -#define RV_KCFI_NINSNS (IS_ENABLED(CONFIG_CFI) ? 1 : 0) /* imm that allows emit_imm to emit max count insns */ #define RV_MAX_COUNT_IMM 0x7FFF7FF7FF7FF7FF @@ -272,8 +271,8 @@ static void __build_epilogue(bool is_tail_call, struct rv_jit_context *ctx) if (!is_tail_call) emit_addiw(RV_REG_A0, RV_REG_A5, 0, ctx); emit_jalr(RV_REG_ZERO, is_tail_call ? RV_REG_T3 : RV_REG_RA, - /* kcfi, fentry and TCC init insns will be skipped on tailcall */ - is_tail_call ? (RV_KCFI_NINSNS + RV_FENTRY_NINSNS + 1) * 4 : 0, + /* fentry and TCC init insns will be skipped on tailcall */ + is_tail_call ? (RV_FENTRY_NINSNS + 1) * 4 : 0, ctx); } @@ -1771,9 +1770,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, for (idx = 0; idx < fm->nr_args; idx++) { u8 reg = bpf_to_rv_reg(BPF_REG_1 + idx, ctx); + bool sign = fm->arg_flags[idx] & BTF_FMODEL_SIGNED_ARG; - if (fm->arg_size[idx] == sizeof(int)) - emit_sextw(reg, reg, ctx); + if (sign_extend(reg, reg, fm->arg_size[idx], sign, ctx)) + return -EINVAL; } } @@ -1926,7 +1926,12 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else ret = emit_atomic_rmw(rd, rs, insn, ctx); - ret = ret ?: add_exception_handler(insn, REG_DONT_CLEAR_MARKER, ctx); + /* ret can be 1 (skip-zext); extable entry still needs to be added */ + if (ret >= 0) + ret = add_exception_handler(insn, + bpf_atomic_is_load_acq(insn) ? rd : REG_DONT_CLEAR_MARKER, + ctx) ?: ret; + if (ret) return ret; break; @@ -1973,6 +1978,8 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) /* emit kcfi type preamble immediately before the first insn */ emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* bpf prog starts here as kcfi skipped during prog->bpf_func setup */ + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index e4ab5bb9c9f64..09bd4eaac63e8 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -251,6 +251,7 @@ void bpf_jit_free(struct bpf_prog *prog) */ if (jit_data) { bpf_jit_binary_pack_finalize(jit_data->ro_header, jit_data->header); + kfree(jit_data->ctx.offset); kfree(jit_data); } hdr = bpf_jit_binary_pack_hdr(prog); diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index f43a6570fd6f7..4a663929ebc31 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -74,7 +74,6 @@ config S390 # # Note: keep this list sorted alphabetically # - imply IMA_SECURE_AND_OR_TRUSTED_BOOT select ALTERNATE_USER_ADDRESS_SPACE select ARCH_32BIT_USTAT_F_TINODE select ARCH_CORRECT_STACKTRACE_ON_KRETPROBE @@ -175,6 +174,7 @@ config S390 select GENERIC_IOREMAP if PCI select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_GET_SECUREBOOT select HAVE_ARCH_JUMP_LABEL select HAVE_ARCH_JUMP_LABEL_RELATIVE select HAVE_ARCH_KASAN diff --git a/arch/s390/boot/ipl_parm.c b/arch/s390/boot/ipl_parm.c index f584d7da29cb2..45c687be4b53b 100644 --- a/arch/s390/boot/ipl_parm.c +++ b/arch/s390/boot/ipl_parm.c @@ -22,6 +22,7 @@ struct parmarea parmarea __section(".parmarea") = { }; char __bootdata(early_command_line)[COMMAND_LINE_SIZE]; +static char command_line_buf[COMMAND_LINE_SIZE]; unsigned int __bootdata_preserved(zlib_dfltcc_support) = ZLIB_DFLTCC_FULL; struct ipl_parameter_block __bootdata_preserved(ipl_block); @@ -134,31 +135,29 @@ static size_t ipl_block_get_ascii_scpdata(char *dest, size_t size, static void append_ipl_block_parm(void) { - char *parm, *delim; - size_t len, rc = 0; + size_t len, extra = 0; + char *delim; len = strlen(early_command_line); - - delim = early_command_line + len; /* '\0' character position */ - parm = early_command_line + len + 1; /* append right after '\0' */ + delim = early_command_line + len; /* '\0' character position */ switch (ipl_block.pb0_hdr.pbt) { case IPL_PBT_CCW: - rc = ipl_block_get_ascii_vmparm( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_vmparm(command_line_buf, sizeof(command_line_buf), &ipl_block); break; case IPL_PBT_FCP: case IPL_PBT_NVME: case IPL_PBT_ECKD: - rc = ipl_block_get_ascii_scpdata( - parm, COMMAND_LINE_SIZE - len - 1, &ipl_block); + extra = ipl_block_get_ascii_scpdata(command_line_buf, sizeof(command_line_buf), &ipl_block); break; } - if (rc) { - if (*parm == '=') - memmove(early_command_line, parm + 1, rc); - else + if (extra) { + if (command_line_buf[0] == '=') { + memmove(early_command_line, command_line_buf + 1, extra); + } else if (len < COMMAND_LINE_SIZE - 2) { *delim = ' '; /* replace '\0' with space */ + sized_strscpy(delim + 1, command_line_buf, COMMAND_LINE_SIZE - len - 1); + } } } @@ -244,7 +243,6 @@ static void modify_fac_list(char *str) check_cleared_facilities(); } -static char command_line_buf[COMMAND_LINE_SIZE]; void parse_boot_command_line(void) { char *param, *val; diff --git a/arch/s390/boot/physmem_info.c b/arch/s390/boot/physmem_info.c index 1f2ca5435838e..0ebb2174713f2 100644 --- a/arch/s390/boot/physmem_info.c +++ b/arch/s390/boot/physmem_info.c @@ -141,7 +141,7 @@ static int tprot(unsigned long addr) static unsigned long search_mem_end(void) { - unsigned long range = 1 << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ + unsigned long range = 1UL << (MAX_PHYSMEM_BITS - 20); /* in 1MB blocks */ unsigned long offset = 0; unsigned long pivot; diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 5d36f4020dfa3..b86edf9e84302 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -28,14 +28,14 @@ #include #include #include -#include #include +#include #include #include #include static u8 *ctrblk; -static DEFINE_MUTEX(ctrblk_lock); +static DEFINE_SEMAPHORE(ctrblk_sem, 1); static cpacf_mask_t km_functions, kmc_functions, kmctr_functions, kma_functions; @@ -235,7 +235,7 @@ static int ecb_aes_crypt(struct skcipher_request *req, unsigned long modifier) return fallback_skcipher_crypt(sctx, req, modifier); ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(sctx->fc | modifier, sctx->key, @@ -339,7 +339,7 @@ static int cbc_aes_crypt(struct skcipher_request *req, unsigned long modifier) return ret; memcpy(param.iv, walk.iv, AES_BLOCK_SIZE); memcpy(param.key, sctx->key, sctx->key_len); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_kmc(sctx->fc | modifier, ¶m, @@ -465,7 +465,7 @@ static int xts_aes_crypt(struct skcipher_request *req, unsigned long modifier) memcpy(xts_param.key + offset, xts_ctx->key, xts_ctx->key_len); memcpy(xts_param.init, pcc_param.xts, 16); - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, xts_param.key + offset, @@ -593,7 +593,7 @@ static int fullxts_aes_crypt(struct skcipher_request *req, unsigned long modifi memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE); fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */ - while ((nbytes = walk.nbytes) != 0) { + while (!ret && ((nbytes = walk.nbytes) != 0)) { /* only use complete blocks */ n = nbytes & ~(AES_BLOCK_SIZE - 1); cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset, @@ -668,48 +668,64 @@ static unsigned int __ctrblk_init(u8 *ctrptr, u8 *iv, unsigned int nbytes) return n; } +static int __ctr_aes_crypt(struct s390_aes_ctx *sctx, + struct skcipher_walk *walk, bool locked) +{ + unsigned int n, nbytes; + int ret = 0; + u8 *ctrptr; + + while (!ret && ((nbytes = walk->nbytes) >= AES_BLOCK_SIZE)) { + n = AES_BLOCK_SIZE; + if (nbytes >= 2 * AES_BLOCK_SIZE && locked) + n = __ctrblk_init(ctrblk, walk->iv, nbytes); + ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk->iv; + cpacf_kmctr(sctx->fc, sctx->key, walk->dst.virt.addr, + walk->src.virt.addr, n, ctrptr); + if (ctrptr == ctrblk) + memcpy(walk->iv, ctrptr + n - AES_BLOCK_SIZE, + AES_BLOCK_SIZE); + crypto_inc(walk->iv, AES_BLOCK_SIZE); + ret = skcipher_walk_done(walk, nbytes - n); + } + + return ret; +} + static int ctr_aes_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct s390_aes_ctx *sctx = crypto_skcipher_ctx(tfm); - u8 buf[AES_BLOCK_SIZE], *ctrptr; struct skcipher_walk walk; - unsigned int n, nbytes; - int ret, locked; + u8 buf[AES_BLOCK_SIZE]; + int ret; if (unlikely(!sctx->fc)) return fallback_skcipher_crypt(sctx, req, 0); - locked = mutex_trylock(&ctrblk_lock); - ret = skcipher_walk_virt(&walk, req, false); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - n = AES_BLOCK_SIZE; + if (ret) + return ret; - if (nbytes >= 2*AES_BLOCK_SIZE && locked) - n = __ctrblk_init(ctrblk, walk.iv, nbytes); - ctrptr = (n > AES_BLOCK_SIZE) ? ctrblk : walk.iv; - cpacf_kmctr(sctx->fc, sctx->key, walk.dst.virt.addr, - walk.src.virt.addr, n, ctrptr); - if (ctrptr == ctrblk) - memcpy(walk.iv, ctrptr + n - AES_BLOCK_SIZE, - AES_BLOCK_SIZE); - crypto_inc(walk.iv, AES_BLOCK_SIZE); - ret = skcipher_walk_done(&walk, nbytes - n); + if (down_trylock(&ctrblk_sem) == 0) { + ret = __ctr_aes_crypt(sctx, &walk, true); + up(&ctrblk_sem); + } else { + ret = __ctr_aes_crypt(sctx, &walk, false); } - if (locked) - mutex_unlock(&ctrblk_lock); + /* * final block may be < AES_BLOCK_SIZE, copy only nbytes */ - if (nbytes) { + if (!ret && walk.nbytes > 0) { memset(buf, 0, AES_BLOCK_SIZE); - memcpy(buf, walk.src.virt.addr, nbytes); + memcpy(buf, walk.src.virt.addr, walk.nbytes); cpacf_kmctr(sctx->fc, sctx->key, buf, buf, AES_BLOCK_SIZE, walk.iv); - memcpy(walk.dst.virt.addr, buf, nbytes); + memcpy(walk.dst.virt.addr, buf, walk.nbytes); crypto_inc(walk.iv, AES_BLOCK_SIZE); ret = skcipher_walk_done(&walk, 0); + memzero_explicit(buf, sizeof(buf)); } return ret; @@ -1001,10 +1017,14 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) gw_in.ptr, aad_bytes); n = aad_bytes + pc_bytes; - if (gcm_in_walk_done(&gw_in, n) != n) - return -ENOMEM; - if (gcm_out_walk_done(&gw_out, n) != n) - return -ENOMEM; + if (gcm_in_walk_done(&gw_in, n) != n) { + ret = -ENOMEM; + goto out; + } + if (gcm_out_walk_done(&gw_out, n) != n) { + ret = -ENOMEM; + goto out; + } aadlen -= aad_bytes; pclen -= pc_bytes; } while (aadlen + pclen > 0); @@ -1016,7 +1036,10 @@ static int gcm_aes_crypt(struct aead_request *req, unsigned int flags) } else scatterwalk_map_and_copy(param.t, req->dst, len, taglen, 1); +out: memzero_explicit(¶m, sizeof(param)); + memzero_explicit(gw_in.buf, sizeof(gw_in.buf)); + memzero_explicit(gw_out.buf, sizeof(gw_out.buf)); return ret; } diff --git a/arch/s390/crypto/paes_s390.c b/arch/s390/crypto/paes_s390.c index a624a43a2b540..15ac83e521615 100644 --- a/arch/s390/crypto/paes_s390.c +++ b/arch/s390/crypto/paes_s390.c @@ -213,6 +213,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, PKEY_XFLAG_NOMEMALLOC); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -558,7 +562,7 @@ static int ecb_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ecb_paes_alg = { @@ -566,6 +570,7 @@ static struct skcipher_engine_alg ecb_paes_alg = { .base.cra_name = "ecb(paes)", .base.cra_driver_name = "ecb-paes-s390", .base.cra_priority = 401, /* combo: aes + ecb + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -821,7 +826,7 @@ static int cbc_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg cbc_paes_alg = { @@ -829,6 +834,7 @@ static struct skcipher_engine_alg cbc_paes_alg = { .base.cra_name = "cbc(paes)", .base.cra_driver_name = "cbc-paes-s390", .base.cra_priority = 402, /* cbc-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1020,6 +1026,7 @@ static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, } out: + memzero_explicit(buf, sizeof(buf)); pr_debug("rc=%d\n", rc); return rc; } @@ -1126,7 +1133,7 @@ static int ctr_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg ctr_paes_alg = { @@ -1134,6 +1141,7 @@ static struct skcipher_engine_alg ctr_paes_alg = { .base.cra_name = "ctr(paes)", .base.cra_driver_name = "ctr-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = 1, .base.cra_ctxsize = sizeof(struct s390_paes_ctx), .base.cra_module = THIS_MODULE, @@ -1346,7 +1354,7 @@ static inline int __xts_2keys_prep_param(struct s390_pxts_ctx *ctx, memcpy(param->init, pcc_param.xts, 16); } - memzero_explicit(pcc_param.key, sizeof(pcc_param.key)); + memzero_explicit(&pcc_param, sizeof(pcc_param)); return rc; } @@ -1565,7 +1573,7 @@ static int xts_paes_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&ctx->via_engine_ctr); crypto_finalize_skcipher_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } static struct skcipher_engine_alg xts_paes_alg = { @@ -1573,6 +1581,7 @@ static struct skcipher_engine_alg xts_paes_alg = { .base.cra_name = "xts(paes)", .base.cra_driver_name = "xts-paes-s390", .base.cra_priority = 402, /* ecb-paes-s390 + 1 */ + .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NO_FALLBACK, .base.cra_blocksize = AES_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct s390_pxts_ctx), .base.cra_module = THIS_MODULE, diff --git a/arch/s390/crypto/phmac_s390.c b/arch/s390/crypto/phmac_s390.c index 89f3e6d8fd897..ecb1af4b82105 100644 --- a/arch/s390/crypto/phmac_s390.c +++ b/arch/s390/crypto/phmac_s390.c @@ -59,8 +59,10 @@ static inline int hwh_prepare(struct ahash_request *req, */ static inline int hwh_advance(struct hash_walk_helper *hwh, int n) { - if (n < 0) + if (n < 0) { + hwh->walkbytes = n; return crypto_hash_walk_done(&hwh->walk, n); + } hwh->walkbytes -= n; hwh->walkaddr += n; @@ -332,6 +334,10 @@ static inline int convert_key(const u8 *key, unsigned int keylen, PKEY_XFLAG_NOMEMALLOC); } + /* But finally map -EBUSY to -EIO to indicate an IO failure */ + if (rc == -EBUSY) + rc = -EIO; + out: pr_debug("rc=%d\n", rc); return rc; @@ -597,6 +603,7 @@ static int phmac_update(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -620,12 +627,15 @@ static int phmac_update(struct ahash_request *req) req_ctx->async_op = OP_UPDATE; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) { - hwh_advance(hwh, rc); + if (cleanup) { + if (hwh->walkbytes > 0) + hwh_advance(hwh, rc); memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); } @@ -640,6 +650,7 @@ static int phmac_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; + bool cleanup = true; int rc = 0; /* Try synchronous operation if no active engine usage */ @@ -658,12 +669,14 @@ static int phmac_final(struct ahash_request *req) req_ctx->async_op = OP_FINAL; atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -676,6 +689,7 @@ static int phmac_finup(struct ahash_request *req) struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); struct kmac_sha2_ctx *kmac_ctx = &req_ctx->kmac_ctx; struct hash_walk_helper *hwh = &req_ctx->hwh; + bool cleanup = true; int rc; /* prep the walk in the request context */ @@ -707,15 +721,17 @@ static int phmac_finup(struct ahash_request *req) /* req->async_op has been set to either OP_FINUP or OP_FINAL */ atomic_inc(&tfm_ctx->via_engine_ctr); rc = crypto_transfer_hash_request_to_engine(phmac_crypto_engine, req); - if (rc != -EINPROGRESS) + if (rc == -EINPROGRESS || rc == -EBUSY) + cleanup = false; + else atomic_dec(&tfm_ctx->via_engine_ctr); } - if (rc != -EINPROGRESS) + if (cleanup && hwh->walkbytes > 0) hwh_advance(hwh, rc); out: - if (rc != -EINPROGRESS) + if (cleanup) memzero_explicit(kmac_ctx, sizeof(*kmac_ctx)); pr_debug("rc=%d\n", rc); return rc; @@ -923,7 +939,7 @@ static int phmac_do_one_request(struct crypto_engine *engine, void *areq) atomic_dec(&tfm_ctx->via_engine_ctr); crypto_finalize_hash_request(engine, req, rc); local_bh_enable(); - return rc; + return 0; } #define S390_ASYNC_PHMAC_ALG(x) \ diff --git a/arch/s390/include/asm/irqflags.h b/arch/s390/include/asm/irqflags.h index bcab456dfb803..6eb6de6a99f2d 100644 --- a/arch/s390/include/asm/irqflags.h +++ b/arch/s390/include/asm/irqflags.h @@ -37,18 +37,24 @@ static __always_inline void __arch_local_irq_ssm(unsigned long flags) asm volatile("ssm %0" : : "Q" (flags) : "memory"); } -#ifdef CONFIG_KMSAN -#define arch_local_irq_attributes noinline notrace __no_sanitize_memory __maybe_unused +#if defined(CONFIG_KMSAN) && !defined(__DECOMPRESSOR) +unsigned long arch_local_save_flags(void); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable_external(void); +void arch_local_irq_enable(void); #else -#define arch_local_irq_attributes __always_inline +#define arch_local_save_flags __arch_local_save_flags +#define arch_local_irq_save __arch_local_irq_save +#define arch_local_irq_enable_external __arch_local_irq_enable_external +#define arch_local_irq_enable __arch_local_irq_enable #endif -static arch_local_irq_attributes unsigned long arch_local_save_flags(void) +static __always_inline unsigned long __arch_local_save_flags(void) { return __arch_local_irq_stnsm(0xff); } -static arch_local_irq_attributes unsigned long arch_local_irq_save(void) +static __always_inline unsigned long __arch_local_irq_save(void) { return __arch_local_irq_stnsm(0xfc); } @@ -58,12 +64,12 @@ static __always_inline void arch_local_irq_disable(void) arch_local_irq_save(); } -static arch_local_irq_attributes void arch_local_irq_enable_external(void) +static __always_inline void __arch_local_irq_enable_external(void) { __arch_local_irq_stosm(0x01); } -static arch_local_irq_attributes void arch_local_irq_enable(void) +static __always_inline void __arch_local_irq_enable(void) { __arch_local_irq_stosm(0x03); } diff --git a/arch/s390/include/asm/nmi.h b/arch/s390/include/asm/nmi.h index 6454c15318544..7919b2b9ac9cc 100644 --- a/arch/s390/include/asm/nmi.h +++ b/arch/s390/include/asm/nmi.h @@ -22,6 +22,7 @@ #define MCCK_CODE_SYSTEM_DAMAGE BIT(63) #define MCCK_CODE_EXT_DAMAGE BIT(63 - 5) #define MCCK_CODE_CP BIT(63 - 9) +#define MCCK_CODE_CK BIT(63 - 11) #define MCCK_CODE_STG_ERROR BIT(63 - 16) #define MCCK_CODE_STG_KEY_ERROR BIT(63 - 18) #define MCCK_CODE_STG_DEGRAD BIT(63 - 19) @@ -33,6 +34,8 @@ #define MCCK_CODE_FC_VALID BIT(63 - 43) #define MCCK_CODE_CPU_TIMER_VALID BIT(63 - 46) +#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE | MCCK_CODE_CK) + #ifndef __ASSEMBLER__ union mci { diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index eb06ff888314b..9ad2f87aae28e 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -74,7 +74,7 @@ obj-$(CONFIG_JUMP_LABEL) += jump_label.o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o obj-$(CONFIG_KEXEC_FILE) += kexec_elf.o obj-$(CONFIG_CERT_STORE) += cert_store.o -obj-$(CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT) += ima_arch.o +obj-$(CONFIG_KMSAN) += irqflags.o obj-$(CONFIG_PERF_EVENTS) += perf_event.o obj-$(CONFIG_PERF_EVENTS) += perf_cpum_cf.o perf_cpum_sf.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 112cf7b40dace..f5359b0100517 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -906,8 +906,8 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) mutex_unlock(&debug_mutex); } -/* Remove debugfs entries and remove from internal list. */ -static void _debug_unregister(debug_info_t *id) +/* Remove debugfs entries. */ +static void _debug_unregister_debugfs(debug_info_t *id) { int i; @@ -917,6 +917,11 @@ static void _debug_unregister(debug_info_t *id) debugfs_remove(id->debugfs_entries[i]); } debugfs_remove(id->debugfs_root_entry); +} + +/* Remove from internal list. */ +static void _debug_unregister(debug_info_t *id) +{ if (id == debug_area_first) debug_area_first = id->next; if (id == debug_area_last) @@ -942,6 +947,7 @@ void debug_unregister(debug_info_t *id) mutex_lock(&debug_mutex); _debug_unregister(id); mutex_unlock(&debug_mutex); + _debug_unregister_debugfs(id); debug_info_put(id); } diff --git a/arch/s390/kernel/diag/diag324.c b/arch/s390/kernel/diag/diag324.c index fe325c2a2d0dc..3eec0cc8fb9e0 100644 --- a/arch/s390/kernel/diag/diag324.c +++ b/arch/s390/kernel/diag/diag324.c @@ -182,8 +182,7 @@ long diag324_pibbuf(unsigned long arg) goto out; rc = copy_to_user((void __user *)address, data->pib, data->pib->len); rc |= put_user(data->sequence, &udata->sequence); - if (rc) - rc = -EFAULT; + rc = rc ? -EFAULT : data->rc; out: mutex_unlock(&pibmutex); return rc; diff --git a/arch/s390/kernel/ima_arch.c b/arch/s390/kernel/ima_arch.c deleted file mode 100644 index f3c3e6e1c5d38..0000000000000 --- a/arch/s390/kernel/ima_arch.c +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -#include -#include - -bool arch_ima_get_secureboot(void) -{ - return ipl_secure_flag; -} - -const char * const *arch_get_ima_policy(void) -{ - return NULL; -} diff --git a/arch/s390/kernel/ipl.c b/arch/s390/kernel/ipl.c index dcdc7e2748486..68fdd5616dfe1 100644 --- a/arch/s390/kernel/ipl.c +++ b/arch/s390/kernel/ipl.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1156,6 +1157,8 @@ static struct attribute_group reipl_nss_attr_group = { void set_os_info_reipl_block(void) { + if (!reipl_block_actual) + return; os_info_entry_add_data(OS_INFO_REIPL_BLOCK, reipl_block_actual, reipl_block_actual->hdr.len); } @@ -1926,7 +1929,8 @@ static struct shutdown_action __refdata dump_action = { static void dump_reipl_run(struct shutdown_trigger *trigger) { struct lowcore *abs_lc; - unsigned int csum; + unsigned long ipib = 0; + unsigned int csum = 0; /* * Set REIPL_CLEAR flag in os_info flags entry indicating @@ -1942,9 +1946,12 @@ static void dump_reipl_run(struct shutdown_trigger *trigger) reipl_type == IPL_TYPE_UNKNOWN) os_info_flags |= OS_INFO_FLAG_REIPL_CLEAR; os_info_entry_add_data(OS_INFO_FLAGS_ENTRY, &os_info_flags, sizeof(os_info_flags)); - csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + if (reipl_block_actual) { + ipib = __pa(reipl_block_actual); + csum = (__force unsigned int)cksm(reipl_block_actual, reipl_block_actual->hdr.len, 0); + } abs_lc = get_abs_lowcore(); - abs_lc->ipib = __pa(reipl_block_actual); + abs_lc->ipib = ipib; abs_lc->ipib_checksum = csum; put_abs_lowcore(abs_lc); dump_run(trigger); @@ -2377,7 +2384,7 @@ void __init setup_ipl(void) atomic_notifier_chain_register(&panic_notifier_list, &on_panic_nb); } -void s390_reset_system(void) +void __no_stack_protector s390_reset_system(void) { /* Disable prefixing */ set_prefix(0); @@ -2387,6 +2394,11 @@ void s390_reset_system(void) diag_amode31_ops.diag308_reset(); } +bool arch_get_secureboot(void) +{ + return ipl_secure_flag; +} + #ifdef CONFIG_KEXEC_FILE int ipl_report_add_component(struct ipl_report *report, struct kexec_buf *kbuf, diff --git a/arch/s390/kernel/irqflags.c b/arch/s390/kernel/irqflags.c new file mode 100644 index 0000000000000..e192f59f8918c --- /dev/null +++ b/arch/s390/kernel/irqflags.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +noinstr unsigned long arch_local_save_flags(void) +{ + return __arch_local_save_flags(); +} +EXPORT_SYMBOL(arch_local_save_flags); + +noinstr unsigned long arch_local_irq_save(void) +{ + return __arch_local_irq_save(); +} +EXPORT_SYMBOL(arch_local_irq_save); + +noinstr void arch_local_irq_enable_external(void) +{ + __arch_local_irq_enable_external(); +} +EXPORT_SYMBOL(arch_local_irq_enable_external); + +noinstr void arch_local_irq_enable(void) +{ + __arch_local_irq_enable(); +} +EXPORT_SYMBOL(arch_local_irq_enable); diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 11f33243a23f3..2f11b4f0c0b46 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -344,8 +344,7 @@ static void notrace s390_backup_mcck_info(struct pt_regs *regs) sie_page = container_of(sie_block, struct sie_page, sie_block); mcck_backup = &sie_page->mcck_info; - mcck_backup->mcic = get_lowcore()->mcck_interruption_code & - ~(MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE); + mcck_backup->mcic = get_lowcore()->mcck_interruption_code & ~MCCK_CODE_NO_GUEST; mcck_backup->ext_damage_code = get_lowcore()->external_damage_code; mcck_backup->failing_storage_address = get_lowcore()->failing_storage_address; } @@ -357,8 +356,6 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); #define ED_STP_ISLAND 6 /* External damage STP island check */ #define ED_STP_SYNC 7 /* External damage STP sync check */ -#define MCCK_CODE_NO_GUEST (MCCK_CODE_CP | MCCK_CODE_EXT_DAMAGE) - /* * machine check handler. */ diff --git a/arch/s390/kernel/perf_cpum_cf.c b/arch/s390/kernel/perf_cpum_cf.c index fbe21b1db3eed..d866562cae383 100644 --- a/arch/s390/kernel/perf_cpum_cf.c +++ b/arch/s390/kernel/perf_cpum_cf.c @@ -111,6 +111,7 @@ struct cpu_cf_ptr { static struct cpu_cf_root { /* Anchor to per CPU data */ refcount_t refcnt; /* Overall active events */ + unsigned int tskctx; /* Users tracking all CPUs (cpu == -1) */ struct cpu_cf_ptr __percpu *cfptr; } cpu_cf_root; @@ -119,13 +120,15 @@ static struct cpu_cf_root { /* Anchor to per CPU data */ * user space in task context with perf_event_open() and close() * system calls. * - * This mutex serializes functions cpum_cf_alloc_cpu() called at event - * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() - * called at event removal via call back function hw_perf_event_destroy() - * when the event is deleted. They are serialized to enforce correct - * bookkeeping of pointer and reference counts anchored by - * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the - * per CPU pointers stored in cpu_cf_root::cfptr. + * This mutex serializes the allocation and removal of the per CPU counter + * data via cpum_cf_alloc_cpu() and cpum_cf_free_cpu(). They are called with + * this mutex held at event initialization via cpumf_pmu_event_init(), at + * event removal via call back function hw_perf_event_destroy() when the + * event is deleted, and from the CPU hotplug prepare/dead callbacks. The + * mutex enforces correct bookkeeping of pointer and reference counts + * anchored by struct cpu_cf_root and protects the access to + * cpu_cf_root::refcnt, cpu_cf_root::tskctx and the per CPU pointers + * stored in cpu_cf_root::cfptr. */ static DEFINE_MUTEX(pmc_reserve_mutex); @@ -168,12 +171,14 @@ static void cpum_cf_reset_cpu(void *flags) } /* Free per CPU data when the last event is removed. */ -static void cpum_cf_free_root(void) +static void cpum_cf_free_root(unsigned int num) { - if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) + struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; + + if (!refcount_sub_and_test(num, &cpu_cf_root.refcnt)) return; - free_percpu(cpu_cf_root.cfptr); cpu_cf_root.cfptr = NULL; + free_percpu(p); irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n", @@ -187,17 +192,17 @@ static void cpum_cf_free_root(void) * CPUs possible, which might be larger than the number of CPUs currently * online. */ -static int cpum_cf_alloc_root(void) +static int cpum_cf_alloc_root(unsigned int num) { int rc = 0; - if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) + if (refcount_add_not_zero(num, &cpu_cf_root.refcnt)) return rc; /* The memory is already zeroed. */ cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); if (cpu_cf_root.cfptr) { - refcount_set(&cpu_cf_root.refcnt, 1); + refcount_set(&cpu_cf_root.refcnt, num); on_each_cpu(cpum_cf_reset_cpu, NULL, 1); irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); } else { @@ -207,20 +212,23 @@ static int cpum_cf_alloc_root(void) return rc; } -/* Free CPU counter data structure for a PMU */ -static void cpum_cf_free_cpu(int cpu) +/* + * Remove num references to the CPU counter data structure of a PMU. + * Called with pmc_reserve_mutex held. + */ +static void cpum_cf_free_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; - mutex_lock(&pmc_reserve_mutex); + lockdep_assert_held(&pmc_reserve_mutex); /* * When invoked via CPU hotplug handler, there might be no events * installed or that particular CPU might not have an * event installed. This anchor pointer can be NULL! */ if (!cpu_cf_root.cfptr) - goto out; + return; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; /* @@ -228,28 +236,29 @@ static void cpum_cf_free_cpu(int cpu) * installed on that CPU, but on different CPUs. */ if (!cpuhw) - goto out; + return; - if (refcount_dec_and_test(&cpuhw->refcnt)) { - kfree(cpuhw); + if (refcount_sub_and_test(num, &cpuhw->refcnt)) { p->cpucf = NULL; + kfree(cpuhw); } - cpum_cf_free_root(); -out: - mutex_unlock(&pmc_reserve_mutex); + cpum_cf_free_root(num); } -/* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ -static int cpum_cf_alloc_cpu(int cpu) +/* + * Add num references to the CPU counter data structure of a PMU and + * allocate it when necessary. Called with pmc_reserve_mutex held. + */ +static int cpum_cf_alloc_cpu(int cpu, unsigned int num) { struct cpu_cf_events *cpuhw; struct cpu_cf_ptr *p; int rc; - mutex_lock(&pmc_reserve_mutex); - rc = cpum_cf_alloc_root(); + lockdep_assert_held(&pmc_reserve_mutex); + rc = cpum_cf_alloc_root(num); if (rc) - goto unlock; + return rc; p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); cpuhw = p->cpucf; @@ -257,12 +266,12 @@ static int cpum_cf_alloc_cpu(int cpu) cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL); if (cpuhw) { p->cpucf = cpuhw; - refcount_set(&cpuhw->refcnt, 1); + refcount_set(&cpuhw->refcnt, num); } else { rc = -ENOMEM; } } else { - refcount_inc(&cpuhw->refcnt); + refcount_add(num, &cpuhw->refcnt); } if (rc) { /* @@ -270,10 +279,8 @@ static int cpum_cf_alloc_cpu(int cpu) * cpu_cf_event in not created, its destroy() function is not * invoked. Adjust the reference counter for the anchor. */ - cpum_cf_free_root(); + cpum_cf_free_root(num); } -unlock: - mutex_unlock(&pmc_reserve_mutex); return rc; } @@ -285,39 +292,70 @@ static int cpum_cf_alloc_cpu(int cpu) * perf_event_open() with task context and /dev/hwctr interface. * If cpu is non-zero install event on this CPU only. This setup handles * perf_event_open() with CPU context. + * Users with cpu == -1 are counted in cpu_cf_root::tskctx. The CPU hotplug + * prepare and dead callbacks use this count to install and remove the per + * CPU counter data on a new or dying CPU. */ -static int cpum_cf_alloc(int cpu) +static int cpum_cf_alloc_cpuslocked(int cpu) { cpumask_var_t mask; int rc; + lockdep_assert_cpus_held(); if (cpu == -1) { if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) return -ENOMEM; + mutex_lock(&pmc_reserve_mutex); for_each_online_cpu(cpu) { - rc = cpum_cf_alloc_cpu(cpu); + rc = cpum_cf_alloc_cpu(cpu, 1); if (rc) { for_each_cpu(cpu, mask) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); break; } cpumask_set_cpu(cpu, mask); } + if (!rc) + cpu_cf_root.tskctx++; + mutex_unlock(&pmc_reserve_mutex); free_cpumask_var(mask); } else { - rc = cpum_cf_alloc_cpu(cpu); + mutex_lock(&pmc_reserve_mutex); + rc = cpum_cf_alloc_cpu(cpu, 1); + mutex_unlock(&pmc_reserve_mutex); } return rc; } -static void cpum_cf_free(int cpu) +static int cpum_cf_alloc(int cpu) +{ + int rc; + + cpus_read_lock(); + rc = cpum_cf_alloc_cpuslocked(cpu); + cpus_read_unlock(); + return rc; +} + +static void cpum_cf_free_cpuslocked(int cpu) { + lockdep_assert_cpus_held(); + mutex_lock(&pmc_reserve_mutex); if (cpu == -1) { + cpu_cf_root.tskctx--; for_each_online_cpu(cpu) - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } else { - cpum_cf_free_cpu(cpu); + cpum_cf_free_cpu(cpu, 1); } + mutex_unlock(&pmc_reserve_mutex); +} + +static void cpum_cf_free(int cpu) +{ + cpus_read_lock(); + cpum_cf_free_cpuslocked(cpu); + cpus_read_unlock(); } #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ @@ -1091,53 +1129,67 @@ static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ static DEFINE_MUTEX(cfset_ctrset_mutex); /* - * CPU hotplug handles only /dev/hwctr device. - * For perf_event_open() the CPU hotplug handling is done on kernel common - * code: + * CPU hotplug handling: + * + * cpum_cf_prepare_cpu() and cpum_cf_dead_cpu() run while the new or dying + * CPU is offline. They create and remove the per CPU counter data for all + * users tracking every CPU (cpu == -1), that is perf_event_open() events + * with task context and /dev/hwctr device sessions. Each such user holds + * one reference to the per CPU counter data of each CPU. Therefore install + * and remove one reference per user, tracked in cpu_cf_root::tskctx. This + * guarantees the per CPU counter data exists before the new CPU executes + * its first task and is removed only after the dying CPU is gone. + * + * cpum_cf_online_cpu() and cpum_cf_offline_cpu() run while the new or + * dying CPU is online. They handle only the counter set state of open + * /dev/hwctr device sessions on that CPU. For perf_event_open() events + * nothing is done: * - CPU add: Nothing is done since a file descriptor can not be created * and returned to the user. * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and - * pmu_delete(). The event itself is removed when the file descriptor is - * closed. + * pmu_delete(). During task exit processing of grouped perf events + * triggered by CPU hotplug processing, pmu_disable() is called as part + * of perf context removal process. The event itself is removed when the + * event file descriptor is closed. */ +static int cpum_cf_prepare_cpu(unsigned int cpu) +{ + int rc = 0; + + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + rc = cpum_cf_alloc_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return rc; +} + +static int cpum_cf_dead_cpu(unsigned int cpu) +{ + mutex_lock(&pmc_reserve_mutex); + if (cpu_cf_root.tskctx) + cpum_cf_free_cpu(cpu, cpu_cf_root.tskctx); + mutex_unlock(&pmc_reserve_mutex); + return 0; +} + static int cfset_online_cpu(unsigned int cpu); static int cpum_cf_online_cpu(unsigned int cpu) { - int rc = 0; - - /* - * Ignore notification for perf_event_open(). - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { - rc = cpum_cf_alloc_cpu(cpu); - if (!rc) - cfset_online_cpu(cpu); - } + if (refcount_read(&cfset_opencnt)) + cfset_online_cpu(cpu); mutex_unlock(&cfset_ctrset_mutex); - return rc; + return 0; } static int cfset_offline_cpu(unsigned int cpu); static int cpum_cf_offline_cpu(unsigned int cpu) { - /* - * During task exit processing of grouped perf events triggered by CPU - * hotplug processing, pmu_disable() is called as part of perf context - * removal process. Therefore do not trigger event removal now for - * perf_event_open() created events. Perf common code triggers event - * destruction when the event file descriptor is closed. - * - * Handle only /dev/hwctr device sessions. - */ mutex_lock(&cfset_ctrset_mutex); - if (refcount_read(&cfset_opencnt)) { + if (refcount_read(&cfset_opencnt)) cfset_offline_cpu(cpu); - cpum_cf_free_cpu(cpu); - } mutex_unlock(&cfset_ctrset_mutex); return 0; } @@ -1184,7 +1236,7 @@ static void cpumf_measurement_alert(struct ext_code ext_code, static int cfset_init(void); static int __init cpumf_pmu_init(void) { - int rc; + int state, rc; /* Extract counter measurement facility information */ if (!cpum_cf_avail() || qctri(&cpumf_ctr_info)) @@ -1226,11 +1278,24 @@ static int __init cpumf_pmu_init(void) cfset_init(); } + rc = cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, + "perf/s390/cf:prepare", + cpum_cf_prepare_cpu, cpum_cf_dead_cpu); + if (rc < 0) + goto out3; + state = rc; + rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, "perf/s390/cf:online", cpum_cf_online_cpu, cpum_cf_offline_cpu); - return rc; + if (rc < 0) + goto out4; + return 0; +out4: + cpuhp_remove_state(state); +out3: + perf_pmu_unregister(&cpumf_pmu); out2: debug_unregister_view(cf_dbg, &debug_sprintf_view); debug_unregister(cf_dbg); @@ -1386,6 +1451,7 @@ static void cfset_all_stop(struct cfset_request *req) */ static int cfset_release(struct inode *inode, struct file *file) { + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); /* Open followed by close/exit has no private_data */ if (file->private_data) { @@ -1396,9 +1462,10 @@ static int cfset_release(struct inode *inode, struct file *file) } if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ on_each_cpu(cfset_release_cpu, NULL, 1); - cpum_cf_free(-1); + cpum_cf_free_cpuslocked(-1); } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); return 0; } @@ -1417,15 +1484,17 @@ static int cfset_open(struct inode *inode, struct file *file) return -EPERM; file->private_data = NULL; + cpus_read_lock(); mutex_lock(&cfset_ctrset_mutex); if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ - rc = cpum_cf_alloc(-1); + rc = cpum_cf_alloc_cpuslocked(-1); if (!rc) { cfset_session_init(); refcount_set(&cfset_opencnt, 1); } } mutex_unlock(&cfset_ctrset_mutex); + cpus_read_unlock(); /* nonseekable_open() never fails */ return rc ?: nonseekable_open(inode, file); diff --git a/arch/s390/kernel/time.c b/arch/s390/kernel/time.c index 63517b85f4c93..246fc5b936fa6 100644 --- a/arch/s390/kernel/time.c +++ b/arch/s390/kernel/time.c @@ -66,6 +66,7 @@ ATOMIC_NOTIFIER_HEAD(s390_epoch_delta_notifier); EXPORT_SYMBOL(s390_epoch_delta_notifier); unsigned char ptff_function_mask[16]; +EXPORT_SYMBOL(ptff_function_mask); static unsigned long lpar_offset; static unsigned long initial_leap_seconds; diff --git a/arch/s390/kernel/vtime.c b/arch/s390/kernel/vtime.c index 122d30b104401..4591f4765e6bc 100644 --- a/arch/s390/kernel/vtime.c +++ b/arch/s390/kernel/vtime.c @@ -31,7 +31,7 @@ static atomic64_t virt_timer_elapsed; DEFINE_PER_CPU(u64, mt_cycles[8]); static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; -static DEFINE_PER_CPU(u64, mt_scaling_jiffies); +static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); static inline void set_vtimer(u64 expires) { @@ -81,7 +81,7 @@ static void update_mt_scaling(void) memcpy(cycles_old, cycles_new, sizeof(u64) * (smp_cpu_mtid + 1)); } - __this_cpu_write(mt_scaling_jiffies, jiffies_64); + __this_cpu_write(mt_scaling_jiffies, jiffies); } static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) @@ -137,23 +137,16 @@ static int do_account_vtime(struct task_struct *tsk) lc->system_timer += timer; /* Update MT utilization calculation */ - if (smp_cpu_mtid && - time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies))) + if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies))) update_mt_scaling(); /* Calculate cputime delta */ - user = update_tsk_timer(&tsk->thread.user_timer, - READ_ONCE(lc->user_timer)); - guest = update_tsk_timer(&tsk->thread.guest_timer, - READ_ONCE(lc->guest_timer)); - system = update_tsk_timer(&tsk->thread.system_timer, - READ_ONCE(lc->system_timer)); - hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, - READ_ONCE(lc->hardirq_timer)); - softirq = update_tsk_timer(&tsk->thread.softirq_timer, - READ_ONCE(lc->softirq_timer)); - lc->steal_timer += - clock - user - guest - system - hardirq - softirq; + user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer); + guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer); + system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer); + hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer); + softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer); + lc->steal_timer += clock - user - guest - system - hardirq - softirq; /* Push account value */ if (user) { diff --git a/arch/s390/kvm/guestdbg.c b/arch/s390/kvm/guestdbg.c index 80879fc73c900..5644d0680b772 100644 --- a/arch/s390/kvm/guestdbg.c +++ b/arch/s390/kvm/guestdbg.c @@ -184,7 +184,7 @@ static int __import_wp_info(struct kvm_vcpu *vcpu, if (wp_info->len < 0 || wp_info->len > MAX_WP_SIZE) return -EINVAL; - wp_info->old_data = kmalloc(bp_data->len, GFP_KERNEL_ACCOUNT); + wp_info->old_data = kmalloc(wp_info->len, GFP_KERNEL_ACCOUNT); if (!wp_info->old_data) return -ENOMEM; /* try to backup the original value */ @@ -256,7 +256,7 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, ret = __import_wp_info(vcpu, &bp_data[i], &wp_info[nr_wp]); if (ret) - goto error; + goto error_wp; nr_wp++; break; case KVM_HW_BP: @@ -271,7 +271,12 @@ int kvm_s390_import_bp_data(struct kvm_vcpu *vcpu, vcpu->arch.guestdbg.hw_bp_info = bp_info; vcpu->arch.guestdbg.nr_hw_wp = nr_wp; vcpu->arch.guestdbg.hw_wp_info = wp_info; + kfree(bp_data); return 0; + +error_wp: + while (nr_wp--) + kfree(wp_info[nr_wp].old_data); error: kfree(bp_data); kfree(wp_info); diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index 7b4e1deb07dbd..1c83962baf5f6 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -2863,9 +2863,7 @@ static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, struct mcck_volatile_info *mcck_info) { - struct kvm_s390_interrupt_info inti; - struct kvm_s390_irq irq; - struct kvm_s390_mchk_info *mchk; + struct kvm_s390_irq irq = {}; union mci mci; __u64 cr14 = 0; /* upper bits are not used */ int rc; @@ -2878,20 +2876,14 @@ void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, if (mci.w) cr14 |= CR14_WARNING_SUBMASK; - mchk = mci.ck ? &inti.mchk : &irq.u.mchk; - mchk->cr14 = cr14; - mchk->mcic = mcck_info->mcic; - mchk->ext_damage_code = mcck_info->ext_damage_code; - mchk->failing_storage_address = mcck_info->failing_storage_address; - if (mci.ck) { - /* Inject the floating machine check */ - inti.type = KVM_S390_MCHK; - rc = __inject_vm(vcpu->kvm, &inti); - } else { - /* Inject the machine check to specified vcpu */ - irq.type = KVM_S390_MCHK; - rc = kvm_s390_inject_vcpu(vcpu, &irq); - } + irq.u.mchk.cr14 = cr14; + irq.u.mchk.mcic = mcck_info->mcic; + irq.u.mchk.ext_damage_code = mcck_info->ext_damage_code; + irq.u.mchk.failing_storage_address = mcck_info->failing_storage_address; + + /* Inject the machine check to specified vcpu */ + irq.type = KVM_S390_MCHK; + rc = kvm_s390_inject_vcpu(vcpu, &irq); WARN_ON_ONCE(rc); } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index a0162d03e16b9..ed08e2f3a8ec5 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3488,6 +3488,7 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); kvm_s390_clear_local_irqs(vcpu); kvm_clear_async_pf_completion_queue(vcpu); + kvm_s390_clear_bp_data(vcpu); if (!kvm_is_ucontrol(vcpu->kvm)) sca_del_vcpu(vcpu); kvm_s390_update_topology_change_report(vcpu->kvm, 1); @@ -4444,8 +4445,10 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, /* enforce guest PER */ kvm_s390_set_cpuflags(vcpu, CPUSTAT_P); - if (dbg->control & KVM_GUESTDBG_USE_HW_BP) - rc = kvm_s390_import_bp_data(vcpu, dbg); + if (dbg->control & KVM_GUESTDBG_USE_HW_BP) { + scoped_guard(srcu, &vcpu->kvm->srcu) + rc = kvm_s390_import_bp_data(vcpu, dbg); + } } else { kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P); vcpu->arch.guestdbg.last_bp = 0; @@ -4702,8 +4705,8 @@ int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clo static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, unsigned long token) { - struct kvm_s390_interrupt inti; - struct kvm_s390_irq irq; + struct kvm_s390_interrupt inti = {}; + struct kvm_s390_irq irq = {}; if (start_token) { irq.u.ext.ext_params2 = token; @@ -5352,7 +5355,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) pr_err_ratelimited("can't run stopped vcpu %d\n", vcpu->vcpu_id); rc = -EINVAL; - goto out; + goto out_sigset; } kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR); @@ -5381,9 +5384,11 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) store_regs(vcpu); kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR); + vcpu->stat.exit_userspace++; + +out_sigset: kvm_sigset_deactivate(vcpu); - vcpu->stat.exit_userspace++; out: vcpu_put(vcpu); return rc; @@ -6005,7 +6010,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd); /* Always copy over UV rc / rrc data */ - if (copy_to_user((__u8 __user *)argp, &cmd.rc, + if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc, sizeof(cmd.rc) + sizeof(cmd.rrc))) r = -EFAULT; break; diff --git a/arch/s390/kvm/pci.c b/arch/s390/kvm/pci.c index 20f738fa65569..2ac7e13640931 100644 --- a/arch/s390/kvm/pci.c +++ b/arch/s390/kvm/pci.c @@ -167,7 +167,7 @@ static int kvm_zpci_set_airq(struct zpci_dev *zdev) fib.fmt0.noi = airq_iv_end(zdev->aibv); fib.fmt0.aibv = virt_to_phys(zdev->aibv->vector); fib.fmt0.aibvo = 0; - fib.fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); + fib.fmt0.aisb = virt_to_phys(aift->sbv->vector) + (zdev->aisb / 64) * 8; fib.fmt0.aisbo = zdev->aisb & 63; fib.gd = zdev->gisa; @@ -191,40 +191,61 @@ static int kvm_zpci_clear_airq(struct zpci_dev *zdev) return cc ? -EIO : 0; } -static inline void unaccount_mem(unsigned long nr_pages) +static inline void unaccount_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { - struct user_struct *user = get_uid(current_user()); + struct user_struct *user = kzdev->user_account; + struct mm_struct *mm_account = kzdev->mm_account; - if (user) + if (user) { atomic_long_sub(nr_pages, &user->locked_vm); - if (current->mm) - atomic64_sub(nr_pages, ¤t->mm->pinned_vm); + free_uid(user); + kzdev->user_account = NULL; + } + + if (mm_account) { + atomic64_sub(nr_pages, &mm_account->pinned_vm); + mmdrop(mm_account); + kzdev->mm_account = NULL; + } } -static inline int account_mem(unsigned long nr_pages) +static inline int account_mem(struct kvm_zdev *kzdev, unsigned long nr_pages) { struct user_struct *user = get_uid(current_user()); unsigned long page_limit, cur_pages, new_pages; + int rc = 0; page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; cur_pages = atomic_long_read(&user->locked_vm); do { new_pages = cur_pages + nr_pages; - if (new_pages > page_limit) - return -ENOMEM; + if (new_pages > page_limit) { + rc = -ENOMEM; + goto out; + } } while (!atomic_long_try_cmpxchg(&user->locked_vm, &cur_pages, new_pages)); - atomic64_add(nr_pages, ¤t->mm->pinned_vm); + if (current->mm) { + mmgrab(current->mm); + atomic64_add(nr_pages, ¤t->mm->pinned_vm); + } + + kzdev->user_account = user; + kzdev->mm_account = current->mm; return 0; + +out: + free_uid(user); + return rc; } static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, bool assist) { struct page *pages[1], *aibv_page, *aisb_page = NULL; - unsigned int msi_vecs, idx; + unsigned int msi_vecs, idx, size; struct zpci_gaite *gaite; unsigned long hva, bit; struct kvm *kvm; @@ -238,6 +259,10 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, if (zdev->gisa == 0) return -EINVAL; + /* AIF already enabled for the device */ + if (zdev->kzdev->fib.fmt0.aibv != 0) + return -EINVAL; + kvm = zdev->kzdev->kvm; msi_vecs = min_t(unsigned int, fib->fmt0.noi, zdev->max_msi); @@ -247,6 +272,14 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, return gisc; /* Replace AIBV address */ + size = BITS_TO_LONGS(msi_vecs + fib->fmt0.aibvo) * sizeof(unsigned long); + npages = DIV_ROUND_UP((fib->fmt0.aibv & ~PAGE_MASK) + size, PAGE_SIZE); + /* AIBV cannot span more than 1 page */ + if (npages > 1) { + rc = -EINVAL; + goto out; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aibv)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, pages); @@ -262,6 +295,12 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, /* Pin the guest AISB if one was specified */ if (fib->fmt0.sum == 1) { + /* AISB must be dword aligned */ + if (fib->fmt0.aisb & 0x7) { + rc = -EINVAL; + goto unpin1; + } + idx = srcu_read_lock(&kvm->srcu); hva = gfn_to_hva(kvm, gpa_to_gfn((gpa_t)fib->fmt0.aisb)); npages = pin_user_pages_fast(hva, 1, FOLL_WRITE | FOLL_LONGTERM, @@ -276,20 +315,28 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, } /* Account for pinned pages, roll back on failure */ - if (account_mem(pcount)) + rc = account_mem(zdev->kzdev, pcount); + if (rc) goto unpin2; /* AISB must be allocated before we can fill in GAITE */ mutex_lock(&aift->aift_lock); bit = airq_iv_alloc_bit(aift->sbv); - if (bit == -1UL) + if (bit == -1UL) { + rc = -ENOMEM; goto unlock; + } zdev->aisb = bit; /* store the summary bit number */ zdev->aibv = airq_iv_create(msi_vecs, AIRQ_IV_DATA | AIRQ_IV_BITLOCK | AIRQ_IV_GUESTVEC, phys_to_virt(fib->fmt0.aibv)); + if (!zdev->aibv) { + rc = -ENOMEM; + goto free_aisb; + } + spin_lock_irq(&aift->gait_lock); gaite = aift->gait + zdev->aisb; @@ -312,21 +359,39 @@ static int kvm_s390_pci_aif_enable(struct zpci_dev *zdev, struct zpci_fib *fib, aift->kzdev[zdev->aisb] = zdev->kzdev; spin_unlock_irq(&aift->gait_lock); - /* Update guest FIB for re-issue */ - fib->fmt0.aisbo = zdev->aisb & 63; - fib->fmt0.aisb = virt_to_phys(aift->sbv->vector + (zdev->aisb / 64) * 8); - fib->fmt0.isc = gisc; - /* Save some guest fib values in the host for later use */ - zdev->kzdev->fib.fmt0.isc = fib->fmt0.isc; + zdev->kzdev->fib.fmt0.isc = gisc; zdev->kzdev->fib.fmt0.aibv = fib->fmt0.aibv; - mutex_unlock(&aift->aift_lock); /* Issue the clp to setup the irq now */ rc = kvm_zpci_set_airq(zdev); - return rc; + if (!rc) { + mutex_unlock(&aift->aift_lock); + return rc; + } + + /* Start cleanup */ + zdev->kzdev->fib.fmt0.isc = 0; + zdev->kzdev->fib.fmt0.aibv = 0; + spin_lock_irq(&aift->gait_lock); + gaite->count--; + gaite->aisb = 0; + gaite->gisc = 0; + gaite->aisbo = 0; + gaite->gisa = 0; + aift->kzdev[zdev->aisb] = NULL; + spin_unlock_irq(&aift->gait_lock); + + airq_iv_release(zdev->aibv); + zdev->aibv = NULL; + +free_aisb: + airq_iv_free_bit(aift->sbv, zdev->aisb); + zdev->aisb = 0; unlock: + if (pcount > 0) + unaccount_mem(zdev->kzdev, pcount); mutex_unlock(&aift->aift_lock); unpin2: if (fib->fmt0.sum == 1) @@ -397,7 +462,7 @@ static int kvm_s390_pci_aif_disable(struct zpci_dev *zdev, bool force) pcount++; } if (pcount > 0) - unaccount_mem(pcount); + unaccount_mem(kzdev, pcount); out: mutex_unlock(&aift->aift_lock); diff --git a/arch/s390/kvm/pci.h b/arch/s390/kvm/pci.h index ff0972dd5e71d..fdf8c7bf4ed08 100644 --- a/arch/s390/kvm/pci.h +++ b/arch/s390/kvm/pci.h @@ -22,6 +22,8 @@ struct kvm_zdev { struct kvm *kvm; struct zpci_fib fib; struct list_head entry; + struct user_struct *user_account; + struct mm_struct *mm_account; }; struct zpci_gaite { diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c index 347268f89f2f1..56e82253bb7df 100644 --- a/arch/s390/kvm/vsie.c +++ b/arch/s390/kvm/vsie.c @@ -184,6 +184,7 @@ static int setup_apcb10(struct kvm_vcpu *vcpu, struct kvm_s390_apcb1 *apcb_s, sizeof(struct kvm_s390_apcb0))) return -EFAULT; + memset(apcb_s, 0, sizeof(*apcb_s)); apcb_s->apm[0] = apcb_h->apm[0] & tmp.apm[0]; apcb_s->aqm[0] = apcb_h->aqm[0] & tmp.aqm[0] & 0xffff000000000000UL; apcb_s->adm[0] = apcb_h->adm[0] & tmp.adm[0] & 0xffff000000000000UL; diff --git a/arch/s390/lib/csum-partial.c b/arch/s390/lib/csum-partial.c index 458abd9bac702..9d74ceff136c5 100644 --- a/arch/s390/lib/csum-partial.c +++ b/arch/s390/lib/csum-partial.c @@ -23,7 +23,7 @@ static __always_inline __wsum csum_copy(void *dst, const void *src, int len, __w if (!cpu_has_vx()) { if (copy) memcpy(dst, src, len); - return cksm(dst, len, sum); + return cksm(src, len, sum); } kernel_fpu_begin(&vxstate, KERNEL_VXR_V16V23); fpu_vlvgf(16, (__force u32)sum, 1); diff --git a/arch/s390/pci/pci_mmio.c b/arch/s390/pci/pci_mmio.c index 51e7a28af8994..f3f79ba78410a 100644 --- a/arch/s390/pci/pci_mmio.c +++ b/arch/s390/pci/pci_mmio.c @@ -188,6 +188,7 @@ SYSCALL_DEFINE3(s390_pci_mmio_write, unsigned long, mmio_addr, goto out_unlock_mmap; } + ret = -EFAULT; io_addr = (void __iomem *)((args.pfn << PAGE_SHIFT) | (mmio_addr & ~PAGE_MASK)); diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index 68eb7cc6e5643..482eec50f404e 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -857,7 +857,7 @@ static int __init pmb_debugfs_init(void) subsys_initcall(pmb_debugfs_init); #ifdef CONFIG_PM -static void pmb_syscore_resume(void) +static void pmb_syscore_resume(void *data) { struct pmb_entry *pmbe; int i; @@ -874,13 +874,17 @@ static void pmb_syscore_resume(void) read_unlock(&pmb_rwlock); } -static struct syscore_ops pmb_syscore_ops = { +static const struct syscore_ops pmb_syscore_ops = { .resume = pmb_syscore_resume, }; +static struct syscore pmb_syscore = { + .ops = &pmb_syscore_ops, +}; + static int __init pmb_sysdev_init(void) { - register_syscore_ops(&pmb_syscore_ops); + register_syscore(&pmb_syscore); return 0; } subsys_initcall(pmb_sysdev_init); diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index a630d373e6453..454782ac54b26 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -470,6 +470,7 @@ endmenu config COMPAT bool depends on SPARC64 + depends on !LD_IS_LLD default y select HAVE_UID16 select ARCH_WANT_OLD_COMPAT_IPC diff --git a/arch/sparc/kernel/uprobes.c b/arch/sparc/kernel/uprobes.c index 305017bec164f..c8cac64e9988a 100644 --- a/arch/sparc/kernel/uprobes.c +++ b/arch/sparc/kernel/uprobes.c @@ -280,6 +280,7 @@ int arch_uprobe_exception_notify(struct notifier_block *self, case DIE_SSTEP: if (uprobe_post_sstep_notifier(args->regs)) ret = NOTIFY_STOP; + break; default: break; diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index 25d9258fa5926..02d1ad3052f1d 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1000,6 +1000,9 @@ static int vector_mmsg_rx(struct vector_private *vp, int budget) */ dev_kfree_skb_irq(skb); vp->estats.rx_encaps_errors++; + (*skbuff_vector) = NULL; + mmsg_vector++; + skbuff_vector++; continue; } if (header_check > 0) { diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 2013840d6318f..99d85503e5093 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -27,6 +27,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz vmlinux.bin.bz2 vmlinux.bin.lzma \ KBUILD_CFLAGS := -m$(BITS) -O2 $(CLANG_FLAGS) KBUILD_CFLAGS += -std=gnu11 KBUILD_CFLAGS += -fno-strict-aliasing -fPIE +KBUILD_CFLAGS += -fno-jump-tables KBUILD_CFLAGS += -Wundef KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING cflags-$(CONFIG_X86_32) := -march=i386 diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c index b25c6a9303b73..3a2bba7c25e9d 100644 --- a/arch/x86/boot/string.c +++ b/arch/x86/boot/string.c @@ -32,8 +32,15 @@ int memcmp(const void *s1, const void *s2, size_t len) { bool diff; - asm("repe cmpsb" - : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len)); + + /* + * Make sure ZF is properly set in the len==0 case because in it, + * RCX==0 and the REPE; CMPSB won't get executed. + */ + asm volatile("test %3, %3\n\t" + "repe cmpsb" + : "=@ccnz" (diff), "+D" (s1), "+S" (s2), "+c" (len) + : : "cc", "memory"); return diff; } diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c index 7b2833705d475..fcb9ec936d050 100644 --- a/arch/x86/coco/tdx/tdx.c +++ b/arch/x86/coco/tdx/tdx.c @@ -693,8 +693,8 @@ static bool handle_in(struct pt_regs *regs, int size, int port) .r13 = PORT_READ, .r14 = port, }; - u64 mask = GENMASK(BITS_PER_BYTE * size, 0); bool success; + u64 val; /* * Emulate the I/O read via hypercall. More info about ABI can be found @@ -702,18 +702,16 @@ static bool handle_in(struct pt_regs *regs, int size, int port) * "TDG.VP.VMCALL". */ success = !__tdx_hypercall(&args); + val = success ? args.r11 : 0; - /* Update part of the register affected by the emulated instruction */ - regs->ax &= ~mask; - if (success) - regs->ax |= args.r11 & mask; + insn_assign_reg(®s->ax, val, size); return success; } static bool handle_out(struct pt_regs *regs, int size, int port) { - u64 mask = GENMASK(BITS_PER_BYTE * size, 0); + u64 mask = GENMASK(BITS_PER_BYTE * size - 1, 0); /* * Emulate the I/O write via hypercall. More info about ABI can be found diff --git a/arch/x86/crypto/aria-aesni-avx2-asm_64.S b/arch/x86/crypto/aria-aesni-avx2-asm_64.S index ed53d4f46bd7c..fda8cb8a99a8a 100644 --- a/arch/x86/crypto/aria-aesni-avx2-asm_64.S +++ b/arch/x86/crypto/aria-aesni-avx2-asm_64.S @@ -982,6 +982,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_encrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_encrypt_32way) @@ -1007,6 +1008,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_decrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_decrypt_32way) @@ -1209,6 +1211,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way) @@ -1359,6 +1362,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_encrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_encrypt_32way) @@ -1384,6 +1388,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_decrypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_decrypt_32way) @@ -1428,6 +1433,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way) %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way) diff --git a/arch/x86/crypto/aria-gfni-avx512-asm_64.S b/arch/x86/crypto/aria-gfni-avx512-asm_64.S index 860887e5d02ed..ca83eb126e065 100644 --- a/arch/x86/crypto/aria-gfni-avx512-asm_64.S +++ b/arch/x86/crypto/aria-gfni-avx512-asm_64.S @@ -800,6 +800,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_encrypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_encrypt_64way) @@ -825,6 +826,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_decrypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %rax); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_decrypt_64way) @@ -966,6 +968,7 @@ SYM_TYPED_FUNC_START(aria_gfni_avx512_ctr_crypt_64way) %zmm9, %zmm8, %zmm11, %zmm10, %zmm12, %zmm13, %zmm14, %zmm15, %r10); + vzeroupper; FRAME_END RET; SYM_FUNC_END(aria_gfni_avx512_ctr_crypt_64way) diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index ed04a968cc7d0..1bb73f4c26c2c 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -937,6 +937,8 @@ SYM_CODE_START(paranoid_entry) IBRS_ENTER save_reg=%r15 UNTRAIN_RET_FROM_CALL + HANDLE_INTR_SAFERET 8(%rsp) + RET SYM_CODE_END(paranoid_entry) @@ -1039,6 +1041,11 @@ SYM_CODE_START(error_entry) movl %ecx, %eax /* zero extend */ cmpq %rax, RIP+8(%rsp) je .Lbstep_iret + + VALIDATE_UNRET_END + + HANDLE_INTR_SAFERET 8(%rsp) + cmpq $.Lgs_change, RIP+8(%rsp) jne .Lerror_entry_done_lfence @@ -1057,7 +1064,6 @@ SYM_CODE_START(error_entry) FENCE_SWAPGS_KERNEL_ENTRY CALL_DEPTH_ACCOUNT leaq 8(%rsp), %rax /* return pt_regs pointer */ - VALIDATE_UNRET_END RET .Lbstep_iret: diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S index fafbd3e68cb87..3b40519602077 100644 --- a/arch/x86/entry/entry_64_fred.S +++ b/arch/x86/entry/entry_64_fred.S @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -18,6 +19,7 @@ UNWIND_HINT_END_OF_STACK ANNOTATE_NOENDBR PUSH_AND_CLEAR_REGS + ENCODE_FRAME_POINTER movq %rsp, %rdi /* %rdi -> pt_regs */ .endm diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c index 56918cd91115c..23d834e7b565e 100644 --- a/arch/x86/events/amd/ibs.c +++ b/arch/x86/events/amd/ibs.c @@ -1719,26 +1719,30 @@ static int x86_pmu_amd_ibs_starting_cpu(unsigned int cpu) #ifdef CONFIG_PM -static int perf_ibs_suspend(void) +static int perf_ibs_suspend(void *data) { clear_APIC_ibs(); return 0; } -static void perf_ibs_resume(void) +static void perf_ibs_resume(void *data) { ibs_eilvt_setup(); setup_APIC_ibs(); } -static struct syscore_ops perf_ibs_syscore_ops = { +static const struct syscore_ops perf_ibs_syscore_ops = { .resume = perf_ibs_resume, .suspend = perf_ibs_suspend, }; +static struct syscore perf_ibs_syscore = { + .ops = &perf_ibs_syscore_ops, +}; + static void perf_ibs_pm_init(void) { - register_syscore_ops(&perf_ibs_syscore_ops); + register_syscore(&perf_ibs_syscore); } #else diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index 9a13a9f21d2f8..c960481222727 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -264,6 +264,29 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +static bool amd_uncore_group_valid(struct perf_event *event) +{ + struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counters = 0; + + if (leader->pmu == event->pmu) + counters++; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu == event->pmu && + sibling->state > PERF_EVENT_STATE_OFF) + counters++; + } + + /* + * When pmu->event_init() is called, the event is yet to be linked to + * its leader's sibling list, so it is counted separately + */ + return (counters + 1) <= pmu->num_counters; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore_pmu *pmu; @@ -281,6 +304,14 @@ static int amd_uncore_event_init(struct perf_event *event) if (!ctx) return -ENODEV; + /* + * Ensure that all events in a group can be scheduled together so that + * a failure can be reported at perf_event_open() time rather than + * silently at pmu->add() time when no free counter is found + */ + if (event->group_leader != event && !amd_uncore_group_valid(event)) + return -EINVAL; + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 7d9f6a58f3f26..4dacbf664b303 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -2621,6 +2621,27 @@ static void intel_pmu_del_event(struct perf_event *event) this_cpu_ptr(&cpu_hw_events)->n_late_setup--; } +int __intel_pmu_quiesce(void) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int pmu_enabled = cpuc->enabled; + + cpuc->enabled = 0; + if (pmu_enabled) + intel_pmu_disable_all(); + + return pmu_enabled; +} + +void __intel_pmu_resume(int pmu_enabled) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + + cpuc->enabled = pmu_enabled; + if (pmu_enabled) + intel_pmu_enable_all(0); +} + static int icl_set_topdown_event_period(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -2812,16 +2833,13 @@ static void intel_pmu_read_event(struct perf_event *event) if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) || is_pebs_counter_event_group(event)) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); - bool pmu_enabled = cpuc->enabled; + int pmu_enabled; /* Only need to call update_topdown_event() once for group read. */ if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ)) return; - cpuc->enabled = 0; - if (pmu_enabled) - intel_pmu_disable_all(); - + pmu_enabled = __intel_pmu_quiesce(); /* * If the PEBS counters snapshotting is enabled, * the topdown event is available in PEBS records. @@ -2830,10 +2848,7 @@ static void intel_pmu_read_event(struct perf_event *event) static_call(intel_pmu_update_topdown_event)(event, NULL); else intel_pmu_drain_pebs_buffer(); - - cpuc->enabled = pmu_enabled; - if (pmu_enabled) - intel_pmu_enable_all(0); + __intel_pmu_resume(pmu_enabled); return; } @@ -6975,12 +6990,6 @@ __init int intel_pmu_init(void) x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */ - if (version >= 5) { - x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated; - if (x86_pmu.intel_cap.anythread_deprecated) - pr_cont(" AnyThread deprecated, "); - } - /* * Many features on and after V6 require dynamic constraint, * e.g., Arch PEBS, ACR. @@ -7778,8 +7787,10 @@ __init int intel_pmu_init(void) &x86_pmu.intel_ctrl); /* AnyThread may be deprecated on arch perfmon v5 or later */ - if (x86_pmu.intel_cap.anythread_deprecated) + if (version >= 5 && edx.split.anythread_deprecated) { x86_pmu.format_attrs = intel_arch_formats_attr; + pr_cont("AnyThread deprecated, "); + } intel_pmu_check_event_constraints(x86_pmu.event_constraints, x86_pmu.cntr_mask64, diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 01bc59e9286cc..711e8a6f46a3a 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -961,8 +961,11 @@ int intel_pmu_drain_bts_buffer(void) void intel_pmu_drain_pebs_buffer(void) { + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct perf_sample_data data; + WARN_ON_ONCE(cpuc->enabled); + static_call(x86_pmu_drain_pebs)(NULL, &data); } @@ -1528,8 +1531,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event) static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc) { if (cpuc->n_pebs == cpuc->n_large_pebs && - cpuc->n_pebs != cpuc->n_pebs_via_pt) + cpuc->n_pebs != cpuc->n_pebs_via_pt) { + int enabled = __intel_pmu_quiesce(); intel_pmu_drain_pebs_buffer(); + __intel_pmu_resume(enabled); + } } void intel_pmu_pebs_enable(struct perf_event *event) @@ -2060,6 +2066,98 @@ static inline void __setup_pebs_counter_group(struct cpu_hw_events *cpuc, #define PEBS_LATENCY_MASK 0xffff +static inline void __setup_perf_sample_data(struct perf_event *event, + struct pt_regs *iregs, + struct perf_sample_data *data) +{ + perf_sample_data_init(data, 0, event->hw.last_period); + + /* + * We must however always use iregs for the unwinder to stay sane; the + * record BP,SP,IP can point into thin air when the record is from a + * previous PMI context or an (I)RET happened between the record and + * PMI. + */ + perf_sample_save_callchain(data, event, iregs); +} + +static inline void __setup_pebs_basic_group(struct perf_event *event, + struct pt_regs *regs, + struct perf_sample_data *data, + u64 sample_type, u64 ip, + u64 tsc, u16 retire) +{ + /* The ip in basic is EventingIP */ + set_linear_ip(regs, ip); + regs->flags |= PERF_EFLAGS_EXACT; + setup_pebs_time(event, data, tsc); + + if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) + data->weight.var3_w = retire; +} + +static inline void __setup_pebs_gpr_group(struct perf_event *event, + struct pt_regs *regs, + struct pebs_gprs *gprs, + u64 sample_type) +{ + /* + * Update flags with PEBS data. PERF_EFLAGS_EXACT must be set + * in previous basic group handling. + */ + regs->flags = gprs->flags | PERF_EFLAGS_EXACT; + + if (event->attr.precise_ip < 2) { + set_linear_ip(regs, gprs->ip); + regs->flags &= ~PERF_EFLAGS_EXACT; + } else if (regs->flags & X86_VM_MASK) { + regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK); + } + + if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) + adaptive_pebs_save_regs(regs, gprs); +} + +static inline void __setup_pebs_meminfo_group(struct perf_event *event, + struct perf_sample_data *data, + u64 sample_type, u64 latency, + u16 instr_latency, u64 address, + u64 aux, u64 tsx_tuning, u64 ax) +{ + if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) { + u64 tsx_latency = intel_get_tsx_weight(tsx_tuning); + + data->weight.var2_w = instr_latency; + + /* + * Although meminfo::latency is defined as a u64, + * only the lower 32 bits include the valid data + * in practice on Ice Lake and earlier platforms. + */ + if (sample_type & PERF_SAMPLE_WEIGHT) + data->weight.full = latency ?: tsx_latency; + else + data->weight.var1_dw = (u32)latency ?: tsx_latency; + + data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; + } + + if (sample_type & PERF_SAMPLE_DATA_SRC) { + data->data_src.val = get_data_src(event, aux); + data->sample_flags |= PERF_SAMPLE_DATA_SRC; + } + + if (sample_type & PERF_SAMPLE_ADDR_TYPE) { + data->addr = address; + data->sample_flags |= PERF_SAMPLE_ADDR; + } + + if (sample_type & PERF_SAMPLE_TRANSACTION) { + data->txn = intel_get_tsx_transaction(tsx_tuning, ax); + data->sample_flags |= PERF_SAMPLE_TRANSACTION; + } +} + /* * With adaptive PEBS the layout depends on what fields are configured. */ @@ -2069,12 +2167,14 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, struct pt_regs *regs) { struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + u64 sample_type = event->attr.sample_type; struct pebs_basic *basic = __pebs; void *next_record = basic + 1; - u64 sample_type, format_group; struct pebs_meminfo *meminfo = NULL; struct pebs_gprs *gprs = NULL; struct x86_perf_regs *perf_regs; + u64 format_group; + u16 retire; if (basic == NULL) return; @@ -2082,31 +2182,17 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, perf_regs = container_of(regs, struct x86_perf_regs, regs); perf_regs->xmm_regs = NULL; - sample_type = event->attr.sample_type; format_group = basic->format_group; - perf_sample_data_init(data, 0, event->hw.last_period); - setup_pebs_time(event, data, basic->tsc); - - /* - * We must however always use iregs for the unwinder to stay sane; the - * record BP,SP,IP can point into thin air when the record is from a - * previous PMI context or an (I)RET happened between the record and - * PMI. - */ - perf_sample_save_callchain(data, event, iregs); + __setup_perf_sample_data(event, iregs, data); *regs = *iregs; - /* The ip in basic is EventingIP */ - set_linear_ip(regs, basic->ip); - regs->flags = PERF_EFLAGS_EXACT; - if (sample_type & PERF_SAMPLE_WEIGHT_STRUCT) { - if (x86_pmu.flags & PMU_FL_RETIRE_LATENCY) - data->weight.var3_w = basic->retire_latency; - else - data->weight.var3_w = 0; - } + /* basic group */ + retire = x86_pmu.flags & PMU_FL_RETIRE_LATENCY ? + basic->retire_latency : 0; + __setup_pebs_basic_group(event, regs, data, sample_type, + basic->ip, basic->tsc, retire); /* * The record for MEMINFO is in front of GP @@ -2122,54 +2208,20 @@ static void setup_pebs_adaptive_sample_data(struct perf_event *event, gprs = next_record; next_record = gprs + 1; - if (event->attr.precise_ip < 2) { - set_linear_ip(regs, gprs->ip); - regs->flags &= ~PERF_EFLAGS_EXACT; - } - - if (sample_type & (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER)) - adaptive_pebs_save_regs(regs, gprs); + __setup_pebs_gpr_group(event, regs, gprs, sample_type); } if (format_group & PEBS_DATACFG_MEMINFO) { - if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) { - u64 latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ? - meminfo->cache_latency : meminfo->mem_latency; - - if (x86_pmu.flags & PMU_FL_INSTR_LATENCY) - data->weight.var2_w = meminfo->instr_latency; - - /* - * Although meminfo::latency is defined as a u64, - * only the lower 32 bits include the valid data - * in practice on Ice Lake and earlier platforms. - */ - if (sample_type & PERF_SAMPLE_WEIGHT) { - data->weight.full = latency ?: - intel_get_tsx_weight(meminfo->tsx_tuning); - } else { - data->weight.var1_dw = (u32)latency ?: - intel_get_tsx_weight(meminfo->tsx_tuning); - } - - data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; - } - - if (sample_type & PERF_SAMPLE_DATA_SRC) { - data->data_src.val = get_data_src(event, meminfo->aux); - data->sample_flags |= PERF_SAMPLE_DATA_SRC; - } - - if (sample_type & PERF_SAMPLE_ADDR_TYPE) { - data->addr = meminfo->address; - data->sample_flags |= PERF_SAMPLE_ADDR; - } - - if (sample_type & PERF_SAMPLE_TRANSACTION) { - data->txn = intel_get_tsx_transaction(meminfo->tsx_tuning, - gprs ? gprs->ax : 0); - data->sample_flags |= PERF_SAMPLE_TRANSACTION; - } + u64 latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ? + meminfo->cache_latency : meminfo->mem_latency; + u64 instr_latency = x86_pmu.flags & PMU_FL_INSTR_LATENCY ? + meminfo->instr_latency : 0; + u64 ax = gprs ? gprs->ax : 0; + + __setup_pebs_meminfo_group(event, data, sample_type, latency, + instr_latency, meminfo->address, + meminfo->aux, meminfo->tsx_tuning, + ax); } if (format_group & PEBS_DATACFG_XMMS) { diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index 7aa59966e7c3e..44f5556440223 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1211,7 +1211,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) { u64 from, to; int br_sel = cpuc->br_sel; - int i, j, type, to_plm; + int i, j, type, from_plm, to_plm; bool compress = false; /* if sampling all branches, then nothing to filter */ @@ -1243,8 +1243,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) type |= X86_BR_NO_TX; } - /* if type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + /* + * If type does not correspond, then discard. + * Specifically reject entries whose from address is in + * kernel space when only X86_BR_USER is requested. + */ + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) { cpuc->lbr_entries[i].from = 0; compress = true; } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index e8cf29d2b10c9..a2937f82ccd44 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -501,6 +501,29 @@ static u64 pt_config_filters(struct perf_event *event) return rtit_ctl; } +static void pt_config_enable(struct perf_event *event) +{ + struct pt *pt = this_cpu_ptr(&pt_ctx); + + /* + * Allow resume before starting so as not to overwrite a value set by a + * PMI. + */ + barrier(); + WRITE_ONCE(pt->resume_allowed, 1); + /* Configuration is complete, it is now OK to handle an NMI */ + barrier(); + WRITE_ONCE(pt->handle_nmi, 1); + barrier(); + pt_config_start(event); + barrier(); + /* + * Allow pause after starting so its pt_config_stop() doesn't race with + * pt_config_start(). + */ + WRITE_ONCE(pt->pause_allowed, 1); +} + static void pt_config(struct perf_event *event) { struct pt *pt = this_cpu_ptr(&pt_ctx); @@ -540,23 +563,7 @@ static void pt_config(struct perf_event *event) event->hw.aux_config = reg; - /* - * Allow resume before starting so as not to overwrite a value set by a - * PMI. - */ - barrier(); - WRITE_ONCE(pt->resume_allowed, 1); - /* Configuration is complete, it is now OK to handle an NMI */ - barrier(); - WRITE_ONCE(pt->handle_nmi, 1); - barrier(); - pt_config_start(event); - barrier(); - /* - * Allow pause after starting so its pt_config_stop() doesn't race with - * pt_config_start(). - */ - WRITE_ONCE(pt->pause_allowed, 1); + pt_config_enable(event); } static void pt_config_stop(struct perf_event *event) @@ -1532,12 +1539,14 @@ void intel_pt_interrupt(void) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); - if (!event->hw.state) { + event->hw.state |= PERF_HES_UPTODATE; + + if (!(event->hw.state & PERF_HES_STOPPED)) { int ret; buf = perf_aux_output_begin(&pt->handle, event); if (!buf) { - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; WRITE_ONCE(pt->resume_allowed, 0); return; } @@ -1553,6 +1562,8 @@ void intel_pt_interrupt(void) pt_config_buffer(buf); pt_config_start(event); + + event->hw.state &= ~PERF_HES_UPTODATE; } } @@ -1622,6 +1633,18 @@ static void pt_event_start(struct perf_event *event, int mode) return; } + /* + * Re-start subsequent to a call to pt_event_stop() without the + * PERF_EF_UPDATE flag. Absence of PERF_HES_UPTODATE indicates that + * perf_aux_output_begin() has already been called. This path can + * come about only in snapshot/overwrite mode - see pt_event_stop(). + */ + if (!(hwc->state & PERF_HES_UPTODATE)) { + hwc->state &= ~PERF_HES_STOPPED; + pt_config_enable(event); + return; + } + buf = perf_aux_output_begin(&pt->handle, event); if (!buf) goto fail_stop; @@ -1632,7 +1655,7 @@ static void pt_event_start(struct perf_event *event, int mode) goto fail_end_stop; } - hwc->state = 0; + hwc->state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE); pt_config_buffer(buf); pt_config(event); @@ -1642,12 +1665,13 @@ static void pt_event_start(struct perf_event *event, int mode) fail_end_stop: perf_aux_output_end(&pt->handle, 0); fail_stop: - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } static void pt_event_stop(struct perf_event *event, int mode) { struct pt *pt = this_cpu_ptr(&pt_ctx); + struct pt_buffer *buf; if (mode & PERF_EF_PAUSE) { if (READ_ONCE(pt->pause_allowed)) @@ -1673,17 +1697,24 @@ static void pt_event_stop(struct perf_event *event, int mode) pt_config_stop(event); - if (event->hw.state == PERF_HES_STOPPED) - return; - - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; - if (mode & PERF_EF_UPDATE) { - struct pt_buffer *buf = perf_get_aux(&pt->handle); + if (event->hw.state & PERF_HES_UPTODATE) + return; - if (!buf) - return; + buf = perf_get_aux(&pt->handle); + if (!buf) + return; + /* + * When not in snapshot/overwrite mode, there is a possibility that the + * buffer has run out of space. The accounting for that is handled by + * the update, so always update in that case. Snapshot/overwrite mode is + * treated differently to allow for pt_event_snapshot_aux() which can + * still get called if the AUX-sampling event is not stopped until after + * PT is stopped. + */ + if ((mode & PERF_EF_UPDATE) || !buf->snapshot) { if (WARN_ON_ONCE(pt->handle.event != event)) return; @@ -1698,6 +1729,7 @@ static void pt_event_stop(struct perf_event *event, int mode) local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); + event->hw.state |= PERF_HES_UPTODATE; } } @@ -1768,13 +1800,15 @@ static int pt_event_add(struct perf_event *event, int mode) if (pt->handle.event) goto fail; + event->hw.state |= PERF_HES_UPTODATE; + if (mode & PERF_EF_START) { pt_event_start(event, 0); ret = -EINVAL; - if (hwc->state == PERF_HES_STOPPED) + if (hwc->state & PERF_HES_STOPPED) goto fail; } else { - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED; } ret = 0; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index e228e564b15ea..a5793847ed5a9 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -67,6 +67,7 @@ int uncore_die_to_segment(int die) return bus ? pci_domain_nr(bus) : -EINVAL; } +/* Note: This API can only be used when NUMA information is available. */ int uncore_device_to_die(struct pci_dev *dev) { int node = pcibus_to_node(dev->bus); @@ -1612,8 +1613,6 @@ static int uncore_event_cpu_online(unsigned int cpu) die = topology_logical_die_id(cpu); msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; /* * Check if there is an online cpu in the package diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index 7dd3910d8de3a..8595e14199ed4 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -489,17 +489,28 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_INT); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (!box_ctl) + return; + + wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), GENERIC_PMON_BOX_CTL_FRZ); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (box_ctl) + wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_FRZ); } void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box) { - wrmsrq(intel_generic_uncore_box_ctl(box), 0); + u64 box_ctl = intel_generic_uncore_box_ctl(box); + + if (box_ctl) + wrmsrq(box_ctl, 0); } static void intel_generic_uncore_msr_enable_event(struct intel_uncore_box *box, @@ -548,6 +559,7 @@ bool intel_generic_uncore_assign_hw_event(struct perf_event *event, if (box->pci_dev) { box_ctl = UNCORE_DISCOVERY_PCI_BOX_CTRL(box_ctl); + hwc->config_base = box_ctl + uncore_pci_event_ctl(box, hwc->idx); hwc->event_base = box_ctl + uncore_pci_perf_ctr(box, hwc->idx); return true; @@ -566,27 +578,30 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); + if (!box_ctl) + return; + __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, GENERIC_PMON_BOX_CTL_FRZ); + if (box_ctl) + pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_FRZ); } void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box) { - struct pci_dev *pdev = box->pci_dev; int box_ctl = intel_pci_uncore_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, 0); + if (box_ctl) + pci_write_config_dword(box->pci_dev, box_ctl, 0); } static void intel_generic_uncore_pci_enable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index a338ee01bb242..0182785cad1fe 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -1475,13 +1475,7 @@ static int snbep_pci2phy_map_init(int devid, int nodeid_loc, int idmap_loc, bool } map->pbus_to_dieid[bus] = die_id = uncore_device_to_die(ubox_dev); - raw_spin_unlock(&pci2phy_map_lock); - - if (WARN_ON_ONCE(die_id == -1)) { - err = -EINVAL; - break; - } } } @@ -6533,7 +6527,7 @@ static void spr_update_device_location(int type_id) while ((dev = pci_get_device(PCI_VENDOR_ID_INTEL, device, dev)) != NULL) { - die = uncore_device_to_die(dev); + die = uncore_pcibus_to_dieid(dev->bus); if (die < 0) continue; @@ -6557,6 +6551,11 @@ static void spr_update_device_location(int type_id) int spr_uncore_pci_init(void) { + int ret = snbep_pci2phy_map_init(0x3250, SKX_CPUNODEID, SKX_GIDNIDMAP, true); + + if (ret) + return ret; + /* * The discovery table of UPI on some SPR variant is broken, * which impacts the detection of both UPI and M3UPI uncore PMON. diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index 2bafb8f0f9077..fc9e6640e322b 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -656,7 +656,7 @@ union perf_capabilities { u64 perf_metrics:1; u64 pebs_output_pt_available:1; u64 pebs_timing_info:1; - u64 anythread_deprecated:1; + u64 __reserved:1; u64 rdpmc_metrics_clear:1; }; u64 capabilities; @@ -1612,6 +1612,9 @@ static __always_inline void __intel_pmu_lbr_disable(void) wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl); } +extern int __intel_pmu_quiesce(void); +extern void __intel_pmu_resume(int pmu_enabled); + int intel_pmu_save_and_restart(struct perf_event *event); struct event_constraint * diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index e890fd37e9c2d..085ef4f2e73a9 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -351,7 +351,7 @@ static int __init hv_pci_init(void) return 1; } -static int hv_suspend(void) +static int hv_suspend(void *data) { union hv_x64_msr_hypercall_contents hypercall_msr; int ret; @@ -378,7 +378,7 @@ static int hv_suspend(void) return ret; } -static void hv_resume(void) +static void hv_resume(void *data) { union hv_x64_msr_hypercall_contents hypercall_msr; int ret; @@ -405,11 +405,15 @@ static void hv_resume(void) } /* Note: when the ops are called, only CPU0 is online and IRQs are disabled. */ -static struct syscore_ops hv_syscore_ops = { +static const struct syscore_ops hv_syscore_ops = { .suspend = hv_suspend, .resume = hv_resume, }; +static struct syscore hv_syscore = { + .ops = &hv_syscore_ops, +}; + static void (* __initdata old_setup_percpu_clockev)(void); static void __init hv_stimer_setup_percpu_clockev(void) @@ -569,7 +573,7 @@ void __init hyperv_init(void) x86_init.pci.arch_init = hv_pci_init; - register_syscore_ops(&hv_syscore_ops); + register_syscore(&hv_syscore); if (ms_hyperv.priv_high & HV_ACCESS_PARTITION_ID) hv_get_partition_id(); diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index db70832232d4a..3e78bb76153ce 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -17,7 +17,7 @@ #define rmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "lfence", \ X86_FEATURE_XMM2) ::: "memory", "cc") #define wmb() asm volatile(ALTERNATIVE("lock addl $0,-4(%%esp)", "sfence", \ - X86_FEATURE_XMM2) ::: "memory", "cc") + X86_FEATURE_XMM) ::: "memory", "cc") #else #define __mb() asm volatile("mfence":::"memory") #define __rmb() asm volatile("lfence":::"memory") diff --git a/arch/x86/include/asm/efi.h b/arch/x86/include/asm/efi.h index f5932705f4b07..be58b7f5c8063 100644 --- a/arch/x86/include/asm/efi.h +++ b/arch/x86/include/asm/efi.h @@ -402,9 +402,9 @@ extern int __init efi_memmap_split_count(efi_memory_desc_t *md, extern void __init efi_memmap_insert(struct efi_memory_map *old_memmap, void *buf, struct efi_mem_range *mem); -extern enum efi_secureboot_mode __x86_ima_efi_boot_mode(void); +enum efi_secureboot_mode __x86_efi_boot_mode(void); -#define arch_ima_efi_boot_mode __x86_ima_efi_boot_mode() +#define arch_efi_boot_mode __x86_efi_boot_mode() #ifdef CONFIG_EFI_RUNTIME_MAP int efi_get_runtime_map_size(void); diff --git a/arch/x86/include/asm/insn-eval.h b/arch/x86/include/asm/insn-eval.h index 54368a43abf67..b1e6c47113aaf 100644 --- a/arch/x86/include/asm/insn-eval.h +++ b/arch/x86/include/asm/insn-eval.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #define INSN_CODE_SEG_ADDR_SZ(params) ((params >> 4) & 0xf) @@ -44,4 +45,39 @@ enum insn_mmio_type { enum insn_mmio_type insn_decode_mmio(struct insn *insn, int *bytes); +/* + * Write @val into *@reg following the x86 rules for writes to + * general-purpose registers (Intel SDM Vol. 1, "General-Purpose + * Registers in 64-Bit Mode"): an 8- or 16-bit write leaves the rest of + * the register untouched, a 32-bit write zero-extends the result into + * the upper 32 bits, and a 64-bit write replaces the whole register. + * + * @bytes is the width of the write, not a property of the instruction: + * an instruction that, say, sign-extends a 32-bit immediate into a + * 64-bit register does a 64-bit write here. + * + * @reg need not be 8-byte aligned: KVM's instruction emulator offsets + * the pointer by one byte to address the high-byte registers (AH, CH, + * DH, BH). Use narrow stores for the sub-word cases so the access + * width matches @bytes and the adjacent bytes are left alone. + */ +static inline void insn_assign_reg(unsigned long *reg, u64 val, int bytes) +{ + switch (bytes) { + case 1: + *(u8 *)reg = (u8)val; + break; + case 2: + *(u16 *)reg = (u16)val; + break; + case 4: + /* A 32-bit write zero-extends into the upper 32 bits. */ + *reg = (u32)val; + break; + case 8: + *reg = val; + break; + } +} + #endif /* _ASM_X86_INSN_EVAL_H */ diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h index a7294656ad908..c9769a7b6e66c 100644 --- a/arch/x86/include/asm/linkage.h +++ b/arch/x86/include/asm/linkage.h @@ -103,7 +103,7 @@ .byte 0xb8 ASM_NL \ .long __kcfi_typeid_##name ASM_NL \ CFI_POST_PADDING \ - SYM_FUNC_END(__cfi_##name) + SYM_END(__cfi_##name, SYM_T_FUNC) /* UML needs to be able to override memcpy() and friends for KASAN. */ #ifdef CONFIG_UML diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 71e9861a16a89..e316cc994bf07 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -12,6 +12,7 @@ #include #include #include +#include /* * Call depth tracking for Intel SKL CPUs to address the RSB underflow @@ -176,6 +177,50 @@ add $(BITS_PER_LONG/8), %_ASM_SP; \ lfence; +/* + * Helper for detecting if an interrupt occurred at an unsafe location within + * Safe-RET. If Safe-RET is interrupted after the CALL or LEA the RSB may get + * poisoned by the interrupt handler. + * + * The Safe-RET sequence is: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * The two CMPs below check whether RIP points to after the CALL or after the + * LEA. + * + * The LFENCE below is to address this particular speculation case: + * + * 1. Userspace runs and poisons the BTB around the safe-RET routine + * + * 2. Userspace triggers some kind of exception + * + * 3. Kernel executes error_entry() and mis-speculates the branch into thinking + * it actually came from kernel space + * + * 4. The kernel then further mis-speculates that the exception occurred due + * to an interrupted safe-RET + * + * 5. The handle_interrupted_saferet() routine speculatively executes and + * speculatively does a safe-RET. But this is unsafe since it was never + * untrained. + * + * The LFENCE fixes this by ensuring step 5 is never reached speculatively. + * Note that this LFENCE only occurs if safe-RET was actually interrupted (so + * it's outside of the normal path). + */ +#define __HANDLE_INTR_SAFERET(name, pt_regs) \ + cmpq $(name), RIP+pt_regs; \ + jb 1f; \ + cmpq $(name)+5, RIP+pt_regs; \ + ja 1f; \ + lfence; \ + leaq pt_regs, %rdi; \ + call handle_interrupted_saferet; \ + 1: + #ifdef __ASSEMBLER__ /* @@ -293,6 +338,14 @@ #define UNTRAIN_RET_FROM_CALL \ __UNTRAIN_RET X86_FEATURE_ENTRY_IBPB, __stringify(RESET_CALL_DEPTH_FROM_CALL) +.macro HANDLE_INTR_SAFERET pt_regs +#ifdef CONFIG_MITIGATION_SRSO + ALTERNATIVE_2 "", \ + __stringify(__HANDLE_INTR_SAFERET(srso_safe_ret, \pt_regs)), X86_FEATURE_SRSO, \ + __stringify(__HANDLE_INTR_SAFERET(srso_alias_safe_ret, \pt_regs)), X86_FEATURE_SRSO_ALIAS + +#endif +.endm .macro CALL_DEPTH_ACCOUNT #ifdef CONFIG_MITIGATION_CALL_DEPTH_TRACKING @@ -625,6 +678,10 @@ static __always_inline void x86_idle_clear_cpu_buffers(void) x86_clear_cpu_buffers(); } +void srso_safe_ret(void); +void srso_alias_safe_ret(void); +void handle_interrupted_saferet(struct pt_regs *regs); + #endif /* __ASSEMBLER__ */ #endif /* _ASM_X86_NOSPEC_BRANCH_H_ */ diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h index e33df3da69804..fbbd44faffce7 100644 --- a/arch/x86/include/asm/pgtable.h +++ b/arch/x86/include/asm/pgtable.h @@ -813,7 +813,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) pmdval_t val = pmd_val(pmd), oldval = val; pmd_t pmd_result; - val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY); + val &= _HPAGE_CHG_MASK; val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK; val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK); diff --git a/arch/x86/include/asm/pkeys.h b/arch/x86/include/asm/pkeys.h index 2e6c04d8a45b4..32d327609cc41 100644 --- a/arch/x86/include/asm/pkeys.h +++ b/arch/x86/include/asm/pkeys.h @@ -89,6 +89,9 @@ int mm_pkey_alloc(struct mm_struct *mm) u16 all_pkeys_mask = ((1U << arch_max_pkey()) - 1); int ret; + if (!arch_pkeys_enabled()) + return -1; + /* * Are we out of pkeys? We must handle this specially * because ffz() behavior is undefined if there are no diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 21041898157a1..63489ec410320 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h @@ -156,6 +156,8 @@ extern unsigned int __max_threads_per_core; extern unsigned int __num_threads_per_package; extern unsigned int __num_cores_per_package; +struct cpuinfo_x86; + const char *get_topology_cpu_type_name(struct cpuinfo_x86 *c); enum x86_topology_cpu_type get_topology_cpu_type(struct cpuinfo_x86 *c); @@ -259,6 +261,7 @@ extern bool x86_topology_update; #ifdef CONFIG_SCHED_MC_PRIO #include +#include DECLARE_PER_CPU_READ_MOSTLY(int, sched_core_priority); extern bool __read_mostly sysctl_sched_itmt_enabled; diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index bd16e9f40d51a..109a285cca514 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1193,6 +1196,41 @@ static bool cfi_debug __ro_after_init; bool cfi_bhi __ro_after_init = false; #endif +#ifdef CONFIG_FINEIBT +/* + * : + * 0: f3 0f 1e fa endbr64 + * 4: 2d 78 56 34 12 sub $0x12345678, %eax + * 9: 2e 0f 85 03 00 00 00 jne,pn 13 + * 10: 0f 1f 40 d6 nopl -0x2a(%rax) + * + * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as + * UDB on x86_64 and raises #UD. + */ +asm( ".pushsection .rodata \n" + "fineibt_preamble_start: \n" + " endbr64 \n" + " subl $0x12345678, %eax \n" + "fineibt_preamble_bhi: \n" + " cs jne.d32 fineibt_preamble_start+0x13 \n" + "#fineibt_func: \n" + " nopl -42(%rax) \n" + "fineibt_preamble_end: \n" + ".popsection\n" +); + +extern u8 fineibt_preamble_start[]; +extern u8 fineibt_preamble_bhi[]; +extern u8 fineibt_preamble_end[]; + +#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) +#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) +#define fineibt_preamble_ud 0x13 +#define fineibt_preamble_hash 5 + +#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) +#endif /* CONFIG_FINEIBT */ + #ifdef CONFIG_CFI u32 cfi_get_func_hash(void *func) { @@ -1200,9 +1238,11 @@ u32 cfi_get_func_hash(void *func) func -= cfi_get_offset(); switch (cfi_mode) { +#ifdef CONFIG_FINEIBT case CFI_FINEIBT: - func += 7; + func += fineibt_preamble_hash; break; +#endif case CFI_KCFI: func += 1; break; @@ -1344,39 +1384,6 @@ early_param("cfi", cfi_parse_cmdline); * processors. Therefore, it is good to place the most frequent branch first" */ -/* - * : - * 0: f3 0f 1e fa endbr64 - * 4: 2d 78 56 34 12 sub $0x12345678, %eax - * 9: 2e 0f 85 03 00 00 00 jne,pn 13 - * 10: 0f 1f 40 d6 nopl -0x2a(%rax) - * - * Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as - * UDB on x86_64 and raises #UD. - */ -asm( ".pushsection .rodata \n" - "fineibt_preamble_start: \n" - " endbr64 \n" - " subl $0x12345678, %eax \n" - "fineibt_preamble_bhi: \n" - " cs jne.d32 fineibt_preamble_start+0x13 \n" - "#fineibt_func: \n" - " nopl -42(%rax) \n" - "fineibt_preamble_end: \n" - ".popsection\n" -); - -extern u8 fineibt_preamble_start[]; -extern u8 fineibt_preamble_bhi[]; -extern u8 fineibt_preamble_end[]; - -#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start) -#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start) -#define fineibt_preamble_ud 0x13 -#define fineibt_preamble_hash 5 - -#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE) - /* * : * 0: b8 78 56 34 12 mov $0x12345678, %eax @@ -2498,6 +2505,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len) typedef void text_poke_f(void *dst, const void *src, size_t len); +static void __poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + pages[0] = vmalloc_to_page(addr); + if (cross_page_boundary) + pages[1] = vmalloc_to_page(addr + PAGE_SIZE); +} + +static void poke_vmalloc_pages(struct page **pages, void *addr, + bool cross_page_boundary) +{ + if (in_dbg_master()) { + /* + * If called from kgdb cannot sleep, but all other CPUs stopped + * anyway so safe to proceed without locks + */ + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } else { + /* + * execmem ROX ranges are shared between modules and can be + * collapsed to huge PMD entries, and this collapse can happen + * concurrently with a racing set_memory_rox(). + * + * Prevent vmalloc_to_page() from racing by acquiring an + * init_mm read lock which pairs with the init_mm write lock in + * cpa_collapse_large_pages(). + */ + guard(mmap_read_lock)(&init_mm); + __poke_vmalloc_pages(pages, addr, cross_page_boundary); + } +} + static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len) { bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE; @@ -2515,9 +2554,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l BUG_ON(!after_bootmem); if (!core_kernel_text((unsigned long)addr)) { - pages[0] = vmalloc_to_page(addr); - if (cross_page_boundary) - pages[1] = vmalloc_to_page(addr + PAGE_SIZE); + poke_vmalloc_pages(pages, addr, cross_page_boundary); } else { pages[0] = virt_to_page(addr); WARN_ON(!PageReserved(pages[0])); diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index 3485d419c2f5e..e6e68a31634cb 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -591,7 +591,7 @@ static void gart_fixup_northbridges(void) } } -static void gart_resume(void) +static void gart_resume(void *data) { pr_info("PCI-DMA: Resuming GART IOMMU\n"); @@ -600,11 +600,15 @@ static void gart_resume(void) enable_gart_translations(); } -static struct syscore_ops gart_syscore_ops = { +static const struct syscore_ops gart_syscore_ops = { .resume = gart_resume, }; +static struct syscore gart_syscore = { + .ops = &gart_syscore_ops, +}; + /* * Private Northbridge GATT initialization in case we cannot use the * AGP driver for some reason. @@ -650,7 +654,7 @@ static __init int init_amd_gatt(struct agp_kern_info *info) agp_gatt_table = gatt; - register_syscore_ops(&gart_syscore_ops); + register_syscore(&gart_syscore); flush_gart(); diff --git a/arch/x86/kernel/amd_node.c b/arch/x86/kernel/amd_node.c index 3d0a4768d603c..306f64536969a 100644 --- a/arch/x86/kernel/amd_node.c +++ b/arch/x86/kernel/amd_node.c @@ -38,7 +38,6 @@ static struct pci_dev **amd_roots; /* Protect the PCI config register pairs used for SMN. */ static DEFINE_MUTEX(smn_mutex); -static bool smn_exclusive; #define SMN_INDEX_OFFSET 0x60 #define SMN_DATA_OFFSET 0x64 @@ -91,11 +90,16 @@ static int __amd_smn_rw(u8 i_off, u8 d_off, u16 node, u32 address, u32 *value, b if (node >= amd_num_nodes()) return err; - root = amd_roots[node]; - if (!root) + /* + * Uninitialized amd_roots indicates pci_request_config_region_exclusive() + * didn't run or failed and thus the kernel cannot rely on having + * exclusive access to SMN registers so prevent that. + */ + if (!amd_roots) return err; - if (!smn_exclusive) + root = amd_roots[node]; + if (!root) return err; guard(mutex)(&smn_mutex); @@ -247,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs); static int __init amd_smn_init(void) { u16 count, num_roots, roots_per_node, node, num_nodes; - struct pci_dev *root; + struct pci_dev *root __free(pci_dev_put) = NULL; if (!cpu_feature_enabled(X86_FEATURE_ZEN)) return 0; @@ -258,7 +262,6 @@ static int __init amd_smn_init(void) return 0; num_roots = 0; - root = NULL; while ((root = get_next_root(root))) { pci_dbg(root, "Reserving PCI config space\n"); @@ -287,17 +290,21 @@ static int __init amd_smn_init(void) return -ENOMEM; roots_per_node = num_roots / num_nodes; + if (!roots_per_node) { + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) + pr_warn(FW_BUG "Error detecting roots per node.\n"); + roots_per_node = 1; + } count = 0; node = 0; - root = NULL; while (node < num_nodes && (root = get_next_root(root))) { /* Use one root for each node and skip the rest. */ if (count++ % roots_per_node) continue; pci_dbg(root, "is root for AMD node %u\n", node); - amd_roots[node++] = root; + amd_roots[node++] = pci_dev_get(root); } if (enable_dfs) { @@ -308,8 +315,6 @@ static int __init amd_smn_init(void) debugfs_create_file("value", 0600, debugfs_dir, NULL, &smn_value_fops); } - smn_exclusive = true; - return 0; } diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index aa1b0ef5e9314..a4431f0d25805 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -2382,7 +2382,7 @@ static struct { unsigned int apic_cmci; } apic_pm_state; -static int lapic_suspend(void) +static int lapic_suspend(void *data) { unsigned long flags; int maxlvt; @@ -2430,7 +2430,7 @@ static int lapic_suspend(void) return 0; } -static void lapic_resume(void) +static void lapic_resume(void *data) { unsigned int l, h; unsigned long flags; @@ -2510,11 +2510,15 @@ static void lapic_resume(void) * are needed on every CPU up until machine_halt/restart/poweroff. */ -static struct syscore_ops lapic_syscore_ops = { +static const struct syscore_ops lapic_syscore_ops = { .resume = lapic_resume, .suspend = lapic_suspend, }; +static struct syscore lapic_syscore = { + .ops = &lapic_syscore_ops, +}; + static void apic_pm_activate(void) { apic_pm_state.active = 1; @@ -2524,7 +2528,7 @@ static int __init init_lapic_sysfs(void) { /* XXX: remove suspend/resume procs if !apic_pm_state.active? */ if (boot_cpu_has(X86_FEATURE_APIC)) - register_syscore_ops(&lapic_syscore_ops); + register_syscore(&lapic_syscore); return 0; } diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 5ba2feb2c04c6..84e200662ce6a 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -2308,7 +2308,12 @@ static void resume_ioapic_id(int ioapic_idx) } } -static void ioapic_resume(void) +static int ioapic_suspend(void *data) +{ + return save_ioapic_entries(); +} + +static void ioapic_resume(void *data) { int ioapic_idx; @@ -2318,14 +2323,18 @@ static void ioapic_resume(void) restore_ioapic_entries(); } -static struct syscore_ops ioapic_syscore_ops = { - .suspend = save_ioapic_entries, +static const struct syscore_ops ioapic_syscore_ops = { + .suspend = ioapic_suspend, .resume = ioapic_resume, }; +static struct syscore ioapic_syscore = { + .ops = &ioapic_syscore_ops, +}; + static int __init ioapic_init_ops(void) { - register_syscore_ops(&ioapic_syscore_ops); + register_syscore(&ioapic_syscore); return 0; } diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c index 138ff22a4926a..b8efe8691b2a6 100644 --- a/arch/x86/kernel/cpu/amd.c +++ b/arch/x86/kernel/cpu/amd.c @@ -513,6 +513,7 @@ static void bsp_init_amd(struct cpuinfo_x86 *c) case 0x00 ... 0x2f: case 0x40 ... 0x4f: case 0x60 ... 0x7f: + case 0xd0 ... 0xd7: setup_force_cpu_cap(X86_FEATURE_ZEN5); break; case 0x50 ... 0x5f: diff --git a/arch/x86/kernel/cpu/aperfmperf.c b/arch/x86/kernel/cpu/aperfmperf.c index a315b0627dfb1..7ffc78d5ebf21 100644 --- a/arch/x86/kernel/cpu/aperfmperf.c +++ b/arch/x86/kernel/cpu/aperfmperf.c @@ -37,7 +37,7 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct aperfmperf, cpu_samples) = { .seq = SEQCNT_ZERO(cpu_samples.seq) }; -static void init_counter_refs(void) +static void init_counter_refs(void *data) { u64 aperf, mperf; @@ -289,16 +289,20 @@ static bool __init intel_set_max_freq_ratio(void) } #ifdef CONFIG_PM_SLEEP -static struct syscore_ops freq_invariance_syscore_ops = { +static const struct syscore_ops freq_invariance_syscore_ops = { .resume = init_counter_refs, }; -static void register_freq_invariance_syscore_ops(void) +static struct syscore freq_invariance_syscore = { + .ops = &freq_invariance_syscore_ops, +}; + +static void register_freq_invariance_syscore(void) { - register_syscore_ops(&freq_invariance_syscore_ops); + register_syscore(&freq_invariance_syscore); } #else -static inline void register_freq_invariance_syscore_ops(void) {} +static inline void register_freq_invariance_syscore(void) {} #endif static void freq_invariance_enable(void) @@ -308,7 +312,7 @@ static void freq_invariance_enable(void) return; } static_branch_enable_cpuslocked(&arch_scale_freq_key); - register_freq_invariance_syscore_ops(); + register_freq_invariance_syscore(); pr_info("Estimated ratio of average max frequency by base frequency (times 1024): %llu\n", arch_max_freq_ratio); } @@ -535,7 +539,7 @@ static int __init bp_init_aperfmperf(void) if (!cpu_feature_enabled(X86_FEATURE_APERFMPERF)) return 0; - init_counter_refs(); + init_counter_refs(NULL); bp_init_freq_invariance(); return 0; } @@ -544,5 +548,5 @@ early_initcall(bp_init_aperfmperf); void ap_init_aperfmperf(void) { if (cpu_feature_enabled(X86_FEATURE_APERFMPERF)) - init_counter_refs(); + init_counter_refs(NULL); } diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index fd0b7880cf7e5..fbc465555e67c 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -3829,3 +3829,42 @@ void __warn_thunk(void) { WARN_ONCE(1, "Unpatched return thunk in use. This should not happen!\n"); } + +#ifdef CONFIG_MITIGATION_SRSO +/* + * Called during exception/interrupt entry if interrupted during the + * safe-RET sequence. The safe-RET sequence consists of 3 instructions: + * + * CALL + * LEA 8(%RSP), %RSP + * RET + * + * An interrupt after the CALL or after the LEA could potentially lead + * to branch predictor poisoning and results in the sequence not being + * able to be safely resumed. + * + * Therefore, modify the regs state as if the remaining part of the + * safe-RET sequence executed so the interrupt returns back to the + * desired return target, instead of the to the safe-RET sequence. + */ +void noinstr handle_interrupted_saferet(struct pt_regs *regs) +{ + unsigned long rip = regs->ip; + + if (rip == (unsigned long) srso_safe_ret || + rip == (unsigned long) srso_alias_safe_ret) { + /* Modify stack pointer as if LEA executed: */ + regs->sp += 8; + } + + /* + * Adjust registers as if RET executed: + * + * 1. Read the return address off the stack and into rIP: + */ + regs->ip = *(unsigned long *)(regs->sp); + + /* 2. Pop rIP off the stack: */ + regs->sp += 8; +} +#endif /* CONFIG_MITIGATION_SRSO */ diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 68053f4f2c9be..3ffeadad02447 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1210,9 +1210,6 @@ static const __initconst struct x86_cpu_id cpu_vuln_whitelist[] = { #define VULNBL_INTEL_STEPS(vfm, max_stepping, issues) \ X86_MATCH_VFM_STEPS(vfm, X86_STEP_MIN, max_stepping, issues) -#define VULNBL_INTEL_TYPE(vfm, cpu_type, issues) \ - X86_MATCH_VFM_CPU_TYPE(vfm, INTEL_CPU_TYPE_##cpu_type, issues) - #define VULNBL_AMD(family, blacklist) \ VULNBL(AMD, family, X86_MODEL_ANY, blacklist) @@ -1275,11 +1272,9 @@ static const struct x86_cpu_id cpu_vuln_blacklist[] __initconst = { VULNBL_INTEL_STEPS(INTEL_TIGERLAKE, X86_STEP_MAX, GDS | ITS | ITS_NATIVE_ONLY), VULNBL_INTEL_STEPS(INTEL_LAKEFIELD, X86_STEP_MAX, MMIO | MMIO_SBDS | RETBLEED), VULNBL_INTEL_STEPS(INTEL_ROCKETLAKE, X86_STEP_MAX, MMIO | RETBLEED | GDS | ITS | ITS_NATIVE_ONLY), - VULNBL_INTEL_TYPE(INTEL_ALDERLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_ALDERLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_ALDERLAKE_L, X86_STEP_MAX, RFDS | VMSCAPE), - VULNBL_INTEL_TYPE(INTEL_RAPTORLAKE, ATOM, RFDS | VMSCAPE), - VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, VMSCAPE), + VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_P, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_RAPTORLAKE_S, X86_STEP_MAX, RFDS | VMSCAPE), VULNBL_INTEL_STEPS(INTEL_METEORLAKE_L, X86_STEP_MAX, VMSCAPE), @@ -1347,7 +1342,21 @@ static bool __init vulnerable_to_rfds(u64 x86_arch_cap_msr) return true; /* Only consult the blacklist when there is no enumeration: */ - return cpu_matches(cpu_vuln_blacklist, RFDS); + if (!cpu_matches(cpu_vuln_blacklist, RFDS)) + return false; + + /* + * ADL and RPL are affected only if they have Atom CPUs. Hybrids have + * both Core and Atom CPUs. Mark unaffected when Atom CPUs are not + * present. + */ + if ((boot_cpu_data.x86_model == 0x97 || + boot_cpu_data.x86_model == 0xB7) && + boot_cpu_data.topo.intel_type != INTEL_CPU_TYPE_ATOM && + !boot_cpu_has(X86_FEATURE_HYBRID_CPU)) + return false; + + return true; } static bool __init vulnerable_to_its(u64 x86_arch_cap_msr) @@ -2622,3 +2631,13 @@ void __init arch_cpu_finalize_init(void) */ mem_encrypt_init(); } + +/* Control TLB flushing methods */ +static int __init tlbi_setup(char *str) +{ + if (!strcmp(str, "ipi")) + setup_clear_cpu_cap(X86_FEATURE_INVLPGB); + + return 1; +} +__setup("tlbi=", tlbi_setup); diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 92032422a8298..dca2d5845e427 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -42,15 +42,6 @@ extern const struct cpu_dev *const __x86_cpu_dev_start[], *const __x86_cpu_dev_end[]; #ifdef CONFIG_CPU_SUP_INTEL -enum tsx_ctrl_states { - TSX_CTRL_ENABLE, - TSX_CTRL_DISABLE, - TSX_CTRL_RTM_ALWAYS_ABORT, - TSX_CTRL_NOT_SUPPORTED, -}; - -extern __ro_after_init enum tsx_ctrl_states tsx_ctrl_state; - extern void __init tsx_init(void); void tsx_ap_init(void); void intel_unlock_cpuid_leafs(struct cpuinfo_x86 *c); diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c index bc7671f920a7e..2c56f8730f596 100644 --- a/arch/x86/kernel/cpu/intel_epb.c +++ b/arch/x86/kernel/cpu/intel_epb.c @@ -75,7 +75,7 @@ static u8 energ_perf_values[] = { [EPB_INDEX_POWERSAVE] = ENERGY_PERF_BIAS_POWERSAVE, }; -static int intel_epb_save(void) +static int intel_epb_save(void *data) { u64 epb; @@ -89,7 +89,7 @@ static int intel_epb_save(void) return 0; } -static void intel_epb_restore(void) +static void intel_epb_restore(void *data) { u64 val = this_cpu_read(saved_epb); u64 epb; @@ -114,11 +114,15 @@ static void intel_epb_restore(void) wrmsrq(MSR_IA32_ENERGY_PERF_BIAS, (epb & ~EPB_MASK) | val); } -static struct syscore_ops intel_epb_syscore_ops = { +static const struct syscore_ops intel_epb_syscore_ops = { .suspend = intel_epb_save, .resume = intel_epb_restore, }; +static struct syscore intel_epb_syscore = { + .ops = &intel_epb_syscore_ops, +}; + static const char * const energy_perf_strings[] = { [EPB_INDEX_PERFORMANCE] = "performance", [EPB_INDEX_BALANCE_PERFORMANCE] = "balance-performance", @@ -185,7 +189,7 @@ static int intel_epb_online(unsigned int cpu) { struct device *cpu_dev = get_cpu_device(cpu); - intel_epb_restore(); + intel_epb_restore(NULL); if (!cpuhp_tasks_frozen) sysfs_merge_group(&cpu_dev->kobj, &intel_epb_attr_group); @@ -199,7 +203,7 @@ static int intel_epb_offline(unsigned int cpu) if (!cpuhp_tasks_frozen) sysfs_unmerge_group(&cpu_dev->kobj, &intel_epb_attr_group); - intel_epb_save(); + intel_epb_save(NULL); return 0; } @@ -230,7 +234,7 @@ static __init int intel_epb_init(void) if (ret < 0) goto err_out_online; - register_syscore_ops(&intel_epb_syscore_ops); + register_syscore(&intel_epb_syscore); return 0; err_out_online: diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index c8b112c6c5492..877dd7d3bd751 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -2236,10 +2236,10 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c) mca_cfg.initialized = 1; + __mcheck_cpu_setup_timer(); __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(c); __mcheck_cpu_init_prepare_banks(); - __mcheck_cpu_setup_timer(); cr4_set_bits(X86_CR4_MCE); } @@ -2387,13 +2387,13 @@ static void vendor_disable_error_reporting(void) mce_disable_error_reporting(); } -static int mce_syscore_suspend(void) +static int mce_syscore_suspend(void *data) { vendor_disable_error_reporting(); return 0; } -static void mce_syscore_shutdown(void) +static void mce_syscore_shutdown(void *data) { vendor_disable_error_reporting(); } @@ -2403,7 +2403,7 @@ static void mce_syscore_shutdown(void) * Only one CPU is active at this time, the others get re-added later using * CPU hotplug: */ -static void mce_syscore_resume(void) +static void mce_syscore_resume(void *data) { __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info)); @@ -2411,12 +2411,16 @@ static void mce_syscore_resume(void) cr4_set_bits(X86_CR4_MCE); } -static struct syscore_ops mce_syscore_ops = { +static const struct syscore_ops mce_syscore_ops = { .suspend = mce_syscore_suspend, .shutdown = mce_syscore_shutdown, .resume = mce_syscore_resume, }; +static struct syscore mce_syscore = { + .ops = &mce_syscore_ops, +}; + /* * mce_device: Sysfs support */ @@ -2817,7 +2821,7 @@ static __init int mcheck_init_device(void) if (err < 0) goto err_out_online; - register_syscore_ops(&mce_syscore_ops); + register_syscore(&mce_syscore); return 0; diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c index 46673530bc6f0..4a33d10685587 100644 --- a/arch/x86/kernel/cpu/microcode/amd.c +++ b/arch/x86/kernel/cpu/microcode/amd.c @@ -233,11 +233,6 @@ static bool need_sha_check(u32 cur_rev) { u32 cutoff; - if (!cur_rev) { - cur_rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); - pr_info_once("No current revision, generating the lowest one: 0x%x\n", cur_rev); - } - cutoff = get_cutoff_revision(cur_rev); if (cutoff) return cur_rev <= cutoff; @@ -338,6 +333,13 @@ static u32 get_patch_level(void) } native_rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy); + if (!rev) { + if (x86_family(bsp_cpuid_1_eax) < 0x17) + return rev; + + rev = cpuid_to_ucode_rev(bsp_cpuid_1_eax); + pr_info_once("No current revision, generating the lowest one: 0x%x\n", rev); + } return rev; } diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c index 539edd6d6dc8c..81aa079fad269 100644 --- a/arch/x86/kernel/cpu/microcode/core.c +++ b/arch/x86/kernel/cpu/microcode/core.c @@ -812,8 +812,17 @@ void microcode_bsp_resume(void) reload_early_microcode(cpu); } -static struct syscore_ops mc_syscore_ops = { - .resume = microcode_bsp_resume, +static void microcode_bsp_syscore_resume(void *data) +{ + microcode_bsp_resume(); +} + +static const struct syscore_ops mc_syscore_ops = { + .resume = microcode_bsp_syscore_resume, +}; + +static struct syscore mc_syscore = { + .ops = &mc_syscore_ops, }; static int mc_cpu_online(unsigned int cpu) @@ -892,7 +901,7 @@ static int __init microcode_init(void) } } - register_syscore_ops(&mc_syscore_ops); + register_syscore(&mc_syscore); cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/microcode:online", mc_cpu_online, mc_cpu_down_prep); diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index c4febdbcfe4d8..cae0dcd9006bd 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -440,7 +440,7 @@ EXPORT_SYMBOL_GPL(hv_get_hypervisor_version); static void __init ms_hyperv_init_platform(void) { - int hv_max_functions_eax; + int hv_max_functions_eax, eax; #ifdef CONFIG_PARAVIRT pv_info.name = "Hyper-V"; @@ -475,6 +475,19 @@ static void __init ms_hyperv_init_platform(void) pr_info("Hyper-V: running on a nested hypervisor\n"); } + /* + * There is no check against the max function for HYPERV_CPUID_VIRT_STACK_* CPUID + * leaves as the hypervisor doesn't handle them. Even a nested root partition (L2 + * root) will not get them because the nested (L1) hypervisor filters them out. + * These are handled through intercept processing by the Windows Hyper-V stack + * or the paravisor. + */ + eax = cpuid_eax(HYPERV_CPUID_VIRT_STACK_PROPERTIES); + ms_hyperv.confidential_vmbus_available = + eax & HYPERV_VS_PROPERTIES_EAX_CONFIDENTIAL_VMBUS_AVAILABLE; + ms_hyperv.msi_ext_dest_id = + eax & HYPERV_VS_PROPERTIES_EAX_EXTENDED_IOAPIC_RTE; + if (ms_hyperv.features & HV_ACCESS_FREQUENCY_MSRS && ms_hyperv.misc_features & HV_FEATURE_FREQUENCY_MSRS_AVAILABLE) { x86_platform.calibrate_tsc = hv_get_tsc_khz; @@ -675,21 +688,10 @@ static bool __init ms_hyperv_x2apic_available(void) * pci-hyperv host bridge. * * Note: for a Hyper-V root partition, this will always return false. - * The hypervisor doesn't expose these HYPERV_CPUID_VIRT_STACK_* cpuids by - * default, they are implemented as intercepts by the Windows Hyper-V stack. - * Even a nested root partition (L2 root) will not get them because the - * nested (L1) hypervisor filters them out. */ static bool __init ms_hyperv_msi_ext_dest_id(void) { - u32 eax; - - eax = cpuid_eax(HYPERV_CPUID_VIRT_STACK_INTERFACE); - if (eax != HYPERV_VS_INTERFACE_EAX_SIGNATURE) - return false; - - eax = cpuid_eax(HYPERV_CPUID_VIRT_STACK_PROPERTIES); - return eax & HYPERV_VS_PROPERTIES_EAX_EXTENDED_IOAPIC_RTE; + return ms_hyperv.msi_ext_dest_id; } #ifdef CONFIG_AMD_MEM_ENCRYPT diff --git a/arch/x86/kernel/cpu/mtrr/legacy.c b/arch/x86/kernel/cpu/mtrr/legacy.c index d25882fcf181d..2415ffaaf02c0 100644 --- a/arch/x86/kernel/cpu/mtrr/legacy.c +++ b/arch/x86/kernel/cpu/mtrr/legacy.c @@ -41,7 +41,7 @@ struct mtrr_value { static struct mtrr_value *mtrr_value; -static int mtrr_save(void) +static int mtrr_save(void *data) { int i; @@ -56,7 +56,7 @@ static int mtrr_save(void) return 0; } -static void mtrr_restore(void) +static void mtrr_restore(void *data) { int i; @@ -69,11 +69,15 @@ static void mtrr_restore(void) } } -static struct syscore_ops mtrr_syscore_ops = { +static const struct syscore_ops mtrr_syscore_ops = { .suspend = mtrr_save, .resume = mtrr_restore, }; +static struct syscore mtrr_syscore = { + .ops = &mtrr_syscore_ops, +}; + void mtrr_register_syscore(void) { mtrr_value = kcalloc(num_var_ranges, sizeof(*mtrr_value), GFP_KERNEL); @@ -86,5 +90,5 @@ void mtrr_register_syscore(void) * TBD: is there any system with such CPU which supports * suspend/resume? If no, we should remove the code. */ - register_syscore_ops(&mtrr_syscore_ops); + register_syscore(&mtrr_syscore); } diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c index 79a1f2d5f5019..f9025779292a8 100644 --- a/arch/x86/kernel/cpu/resctrl/core.c +++ b/arch/x86/kernel/cpu/resctrl/core.c @@ -363,7 +363,7 @@ static void ctrl_domain_free(struct rdt_hw_ctrl_domain *hw_dom) kfree(hw_dom); } -static void mon_domain_free(struct rdt_hw_mon_domain *hw_dom) +static void l3_mon_domain_free(struct rdt_hw_l3_mon_domain *hw_dom) { int idx; @@ -396,11 +396,13 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_ctrl_domain * } /** - * arch_domain_mbm_alloc() - Allocate arch private storage for the MBM counters + * l3_mon_domain_mbm_alloc() - Allocate arch private storage for the MBM counters * @num_rmid: The size of the MBM counter array * @hw_dom: The domain that owns the allocated arrays + * + * Return: 0 for success, or -ENOMEM. */ -static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_mon_domain *hw_dom) +static int l3_mon_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_l3_mon_domain *hw_dom) { size_t tsize = sizeof(*hw_dom->arch_mbm_states[0]); enum resctrl_event_id eventid; @@ -499,9 +501,9 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) { int id = get_domain_id_from_scope(cpu, r->mon_scope); struct list_head *add_pos = NULL; - struct rdt_hw_mon_domain *hw_dom; + struct rdt_hw_l3_mon_domain *hw_dom; struct rdt_domain_hdr *hdr; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; struct cacheinfo *ci; int err; @@ -517,7 +519,7 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) if (hdr) { if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) return; - d = container_of(hdr, struct rdt_mon_domain, hdr); + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); cpumask_set_cpu(cpu, &d->hdr.cpu_mask); /* Update the mbm_assign_mode state for the CPU if supported */ @@ -536,7 +538,7 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) ci = get_cpu_cacheinfo_level(cpu, RESCTRL_L3_CACHE); if (!ci) { pr_warn_once("Can't find L3 cache for CPU:%d resource %s\n", cpu, r->name); - mon_domain_free(hw_dom); + l3_mon_domain_free(hw_dom); return; } d->ci_id = ci->id; @@ -548,8 +550,8 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) arch_mon_domain_online(r, d); - if (arch_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) { - mon_domain_free(hw_dom); + if (l3_mon_domain_mbm_alloc(r->mon.num_rmid, hw_dom)) { + l3_mon_domain_free(hw_dom); return; } @@ -559,7 +561,7 @@ static void domain_add_cpu_mon(int cpu, struct rdt_resource *r) if (err) { list_del_rcu(&d->hdr.list); synchronize_rcu(); - mon_domain_free(hw_dom); + l3_mon_domain_free(hw_dom); } } @@ -620,9 +622,9 @@ static void domain_remove_cpu_ctrl(int cpu, struct rdt_resource *r) static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) { int id = get_domain_id_from_scope(cpu, r->mon_scope); - struct rdt_hw_mon_domain *hw_dom; + struct rdt_hw_l3_mon_domain *hw_dom; struct rdt_domain_hdr *hdr; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; lockdep_assert_held(&domain_list_lock); @@ -642,7 +644,7 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) if (WARN_ON_ONCE(hdr->type != RESCTRL_MON_DOMAIN)) return; - d = container_of(hdr, struct rdt_mon_domain, hdr); + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); hw_dom = resctrl_to_arch_mon_dom(d); cpumask_clear_cpu(cpu, &d->hdr.cpu_mask); @@ -650,7 +652,7 @@ static void domain_remove_cpu_mon(int cpu, struct rdt_resource *r) resctrl_offline_mon_domain(r, d); list_del_rcu(&d->hdr.list); synchronize_rcu(); - mon_domain_free(hw_dom); + l3_mon_domain_free(hw_dom); return; } @@ -897,7 +899,7 @@ static __init bool get_rdt_mon_resources(void) if (!ret) return false; - return !rdt_get_mon_l3_config(r); + return !rdt_get_l3_mon_config(r); } static __init void __check_quirks_intel(void) diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index 6da9bd1a188b9..6c9e75440c10e 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -63,17 +63,17 @@ struct rdt_hw_ctrl_domain { }; /** - * struct rdt_hw_mon_domain - Arch private attributes of a set of CPUs that share - * a resource for a monitor function - * @d_resctrl: Properties exposed to the resctrl file system + * struct rdt_hw_l3_mon_domain - Arch private attributes of a set of CPUs sharing + * RDT_RESOURCE_L3 monitoring + * @d_resctrl: Properties exposed to the resctrl file system * @arch_mbm_states: Per-event pointer to the MBM event's saved state. * An MBM event's state is an array of struct arch_mbm_state * indexed by RMID on x86. * * Members of this structure are accessed via helpers that provide abstraction. */ -struct rdt_hw_mon_domain { - struct rdt_mon_domain d_resctrl; +struct rdt_hw_l3_mon_domain { + struct rdt_l3_mon_domain d_resctrl; struct arch_mbm_state *arch_mbm_states[QOS_NUM_L3_MBM_EVENTS]; }; @@ -82,9 +82,9 @@ static inline struct rdt_hw_ctrl_domain *resctrl_to_arch_ctrl_dom(struct rdt_ctr return container_of(r, struct rdt_hw_ctrl_domain, d_resctrl); } -static inline struct rdt_hw_mon_domain *resctrl_to_arch_mon_dom(struct rdt_mon_domain *r) +static inline struct rdt_hw_l3_mon_domain *resctrl_to_arch_mon_dom(struct rdt_l3_mon_domain *r) { - return container_of(r, struct rdt_hw_mon_domain, d_resctrl); + return container_of(r, struct rdt_hw_l3_mon_domain, d_resctrl); } /** @@ -138,7 +138,7 @@ static inline struct rdt_hw_resource *resctrl_to_arch_res(struct rdt_resource *r extern struct rdt_hw_resource rdt_resources_all[]; -void arch_mon_domain_online(struct rdt_resource *r, struct rdt_mon_domain *d); +void arch_mon_domain_online(struct rdt_resource *r, struct rdt_l3_mon_domain *d); /* CPUID.(EAX=10H, ECX=ResID=1).EAX */ union cpuid_0x10_1_eax { @@ -211,7 +211,7 @@ union l3_qos_abmc_cfg { void rdt_ctrl_update(void *arg); -int rdt_get_mon_l3_config(struct rdt_resource *r); +int rdt_get_l3_mon_config(struct rdt_resource *r); bool rdt_cpu_has(int flag); diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c index 4b2b539ddff95..9fc3adb4b232c 100644 --- a/arch/x86/kernel/cpu/resctrl/monitor.c +++ b/arch/x86/kernel/cpu/resctrl/monitor.c @@ -109,7 +109,7 @@ static inline u64 get_corrected_mbm_count(u32 rmid, unsigned long val) * * In RMID sharing mode there are fewer "logical RMID" values available * to accumulate data ("physical RMIDs" are divided evenly between SNC - * nodes that share an L3 cache). Linux creates an rdt_mon_domain for + * nodes that share an L3 cache). Linux creates an rdt_l3_mon_domain for * each SNC node. * * The value loaded into IA32_PQR_ASSOC is the "logical RMID". @@ -157,7 +157,7 @@ static int __rmid_read_phys(u32 prmid, enum resctrl_event_id eventid, u64 *val) return 0; } -static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_mon_domain *hw_dom, +static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_l3_mon_domain *hw_dom, u32 rmid, enum resctrl_event_id eventid) { @@ -171,11 +171,11 @@ static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_mon_domain *hw_do return state ? &state[rmid] : NULL; } -void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, enum resctrl_event_id eventid) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); int cpu = cpumask_any(&d->hdr.cpu_mask); struct arch_mbm_state *am; u32 prmid; @@ -194,9 +194,9 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, * Assumes that hardware counters are also reset and thus that there is * no need to record initial non-zero counts. */ -void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d) +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); enum resctrl_event_id eventid; int idx; @@ -217,10 +217,10 @@ static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width) return chunks >> shift; } -static u64 get_corrected_val(struct rdt_resource *r, struct rdt_mon_domain *d, +static u64 get_corrected_val(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 rmid, enum resctrl_event_id eventid, u64 msr_val) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); struct arch_mbm_state *am; u64 chunks; @@ -238,11 +238,11 @@ static u64 get_corrected_val(struct rdt_resource *r, struct rdt_mon_domain *d, return chunks * hw_res->mon_scale; } -int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d, +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, enum resctrl_event_id eventid, u64 *val, void *ignored) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); struct arch_mbm_state *am; u64 msr_val; u32 prmid; @@ -308,11 +308,11 @@ static int __cntr_id_read(u32 cntr_id, u64 *val) return 0; } -void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); struct arch_mbm_state *am; am = get_arch_mbm_state(hw_dom, rmid, eventid); @@ -324,7 +324,7 @@ void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, } } -int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_mon_domain *d, +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val) { @@ -354,7 +354,7 @@ int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_mon_domain *d, * must adjust RMID counter numbers based on SNC node. See * logical_rmid_to_physical_rmid() for code that does this. */ -void arch_mon_domain_online(struct rdt_resource *r, struct rdt_mon_domain *d) +void arch_mon_domain_online(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { if (snc_nodes_per_l3_cache > 1) msr_clear_bit(MSR_RMID_SNC_CONFIG, 0); @@ -423,7 +423,7 @@ static __init int snc_get_config(void) return ret; } -int __init rdt_get_mon_l3_config(struct rdt_resource *r) +int __init rdt_get_l3_mon_config(struct rdt_resource *r) { unsigned int mbm_offset = boot_cpu_data.x86_cache_mbm_width_offset; struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r); @@ -515,7 +515,7 @@ static void resctrl_abmc_set_one_amd(void *arg) */ static void _resctrl_abmc_enable(struct rdt_resource *r, bool enable) { - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; lockdep_assert_cpus_held(); @@ -554,11 +554,11 @@ static void resctrl_abmc_config_one_amd(void *info) /* * Send an IPI to the domain to assign the counter to RMID, event pair. */ -void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, enum resctrl_event_id evtid, u32 rmid, u32 closid, u32 cntr_id, bool assign) { - struct rdt_hw_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); + struct rdt_hw_l3_mon_domain *hw_dom = resctrl_to_arch_mon_dom(d); union l3_qos_abmc_cfg abmc_cfg = { 0 }; struct arch_mbm_state *am; diff --git a/arch/x86/kernel/cpu/tsx.c b/arch/x86/kernel/cpu/tsx.c index 49782724a9430..8be08ece22148 100644 --- a/arch/x86/kernel/cpu/tsx.c +++ b/arch/x86/kernel/cpu/tsx.c @@ -19,7 +19,14 @@ #undef pr_fmt #define pr_fmt(fmt) "tsx: " fmt -enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; +enum tsx_ctrl_states { + TSX_CTRL_ENABLE, + TSX_CTRL_DISABLE, + TSX_CTRL_RTM_ALWAYS_ABORT, + TSX_CTRL_NOT_SUPPORTED, +}; + +static enum tsx_ctrl_states tsx_ctrl_state __ro_after_init = TSX_CTRL_NOT_SUPPORTED; static void tsx_disable(void) { diff --git a/arch/x86/kernel/cpu/umwait.c b/arch/x86/kernel/cpu/umwait.c index 933fcd7ff2502..e4a31c5366427 100644 --- a/arch/x86/kernel/cpu/umwait.c +++ b/arch/x86/kernel/cpu/umwait.c @@ -86,15 +86,19 @@ static int umwait_cpu_offline(unsigned int cpu) * trust the firmware nor does it matter if the same value is written * again. */ -static void umwait_syscore_resume(void) +static void umwait_syscore_resume(void *data) { umwait_update_control_msr(NULL); } -static struct syscore_ops umwait_syscore_ops = { +static const struct syscore_ops umwait_syscore_ops = { .resume = umwait_syscore_resume, }; +static struct syscore umwait_syscore = { + .ops = &umwait_syscore_ops, +}; + /* sysfs interface */ /* @@ -226,7 +230,7 @@ static int __init umwait_init(void) return ret; } - register_syscore_ops(&umwait_syscore_ops); + register_syscore(&umwait_syscore); /* * Add umwait control interface. Ignore failure, so at least the diff --git a/arch/x86/kernel/i8237.c b/arch/x86/kernel/i8237.c index 2cd124ad9380c..896d46b442848 100644 --- a/arch/x86/kernel/i8237.c +++ b/arch/x86/kernel/i8237.c @@ -19,7 +19,7 @@ * in asm/dma.h. */ -static void i8237A_resume(void) +static void i8237A_resume(void *data) { unsigned long flags; int i; @@ -41,10 +41,14 @@ static void i8237A_resume(void) release_dma_lock(flags); } -static struct syscore_ops i8237_syscore_ops = { +static const struct syscore_ops i8237_syscore_ops = { .resume = i8237A_resume, }; +static struct syscore i8237_syscore = { + .ops = &i8237_syscore_ops, +}; + static int __init i8237A_init_ops(void) { /* @@ -70,7 +74,7 @@ static int __init i8237A_init_ops(void) if (x86_pnpbios_disabled() && dmi_get_bios_year() >= 2017) return -ENODEV; - register_syscore_ops(&i8237_syscore_ops); + register_syscore(&i8237_syscore); return 0; } device_initcall(i8237A_init_ops); diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c index 2bade73f49e33..f67063df67235 100644 --- a/arch/x86/kernel/i8259.c +++ b/arch/x86/kernel/i8259.c @@ -247,19 +247,19 @@ static void save_ELCR(char *trigger) trigger[1] = inb(PIC_ELCR2) & 0xDE; } -static void i8259A_resume(void) +static void i8259A_resume(void *data) { init_8259A(i8259A_auto_eoi); restore_ELCR(irq_trigger); } -static int i8259A_suspend(void) +static int i8259A_suspend(void *data) { save_ELCR(irq_trigger); return 0; } -static void i8259A_shutdown(void) +static void i8259A_shutdown(void *data) { /* Put the i8259A into a quiescent state that * the kernel initialization code can get it @@ -269,12 +269,16 @@ static void i8259A_shutdown(void) outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ } -static struct syscore_ops i8259_syscore_ops = { +static const struct syscore_ops i8259_syscore_ops = { .suspend = i8259A_suspend, .resume = i8259A_resume, .shutdown = i8259A_shutdown, }; +static struct syscore i8259_syscore = { + .ops = &i8259_syscore_ops, +}; + static void mask_8259A(void) { unsigned long flags; @@ -444,7 +448,7 @@ EXPORT_SYMBOL(legacy_pic); static int __init i8259A_init_ops(void) { if (legacy_pic == &default_legacy_pic) - register_syscore_ops(&i8259_syscore_ops); + register_syscore(&i8259_syscore); return 0; } diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c index 243a769fdd97b..85ebde361d6ae 100644 --- a/arch/x86/kernel/itmt.c +++ b/arch/x86/kernel/itmt.c @@ -110,18 +110,14 @@ int sched_set_itmt_support(void) arch_debugfs_dir, &sysctl_sched_itmt_enabled, &dfs_sched_itmt_fops); - if (IS_ERR_OR_NULL(dfs_sched_itmt)) { + if (IS_ERR(dfs_sched_itmt)) dfs_sched_itmt = NULL; - return -ENOMEM; - } dfs_sched_core_prio = debugfs_create_file("sched_core_priority", 0644, arch_debugfs_dir, NULL, &sched_core_priority_fops); - if (IS_ERR_OR_NULL(dfs_sched_core_prio)) { + if (IS_ERR(dfs_sched_core_prio)) dfs_sched_core_prio = NULL; - return -ENOMEM; - } sched_itmt_capable = true; diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index b67d7c59dca0b..1500852ba03ca 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -720,7 +720,7 @@ static int kvm_cpu_down_prepare(unsigned int cpu) #endif -static int kvm_suspend(void) +static int kvm_suspend(void *data) { u64 val = 0; @@ -734,7 +734,7 @@ static int kvm_suspend(void) return 0; } -static void kvm_resume(void) +static void kvm_resume(void *data) { kvm_cpu_online(raw_smp_processor_id()); @@ -744,11 +744,15 @@ static void kvm_resume(void) #endif } -static struct syscore_ops kvm_syscore_ops = { +static const struct syscore_ops kvm_syscore_ops = { .suspend = kvm_suspend, .resume = kvm_resume, }; +static struct syscore kvm_syscore = { + .ops = &kvm_syscore_ops, +}; + static void kvm_pv_guest_cpu_reboot(void *unused) { kvm_guest_cpu_offline(true); @@ -858,7 +862,7 @@ static void __init kvm_guest_init(void) machine_ops.crash_shutdown = kvm_crash_shutdown; #endif - register_syscore_ops(&kvm_syscore_ops); + register_syscore(&kvm_syscore); /* * Hard lockup detection is enabled by default. Disable it, as guests diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 51a849a79c985..b112178e81850 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -2,6 +2,7 @@ /* * RTC related functions */ +#include #include #include #include @@ -146,6 +147,9 @@ static __init int add_rtc_cmos(void) } } #endif + if (cmos_rtc_platform_device_present) + return 0; + if (!x86_platform.legacy.rtc) return -ENODEV; diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index c4b8950c7abee..1f823bddbf95d 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -5,8 +5,8 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror include $(srctree)/virt/kvm/Makefile.kvm -kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \ - debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o +kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o regs.o \ + mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 4e3da5b497b85..9dbf3623e79eb 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -24,6 +24,7 @@ #include "kvm_emulate.h" #include #include +#include #include #include #include @@ -436,25 +437,6 @@ static void assign_masked(ulong *dest, ulong src, ulong mask) *dest = (*dest & ~mask) | (src & mask); } -static void assign_register(unsigned long *reg, u64 val, int bytes) -{ - /* The 4-byte case *is* correct: in 64-bit mode we zero-extend. */ - switch (bytes) { - case 1: - *(u8 *)reg = (u8)val; - break; - case 2: - *(u16 *)reg = (u16)val; - break; - case 4: - *reg = (u32)val; - break; /* 64b: zero-extend */ - case 8: - *reg = val; - break; - } -} - static inline unsigned long ad_mask(struct x86_emulate_ctxt *ctxt) { return (1UL << (ctxt->ad_bytes << 3)) - 1; @@ -502,7 +484,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, int reg, int inc) { ulong *preg = reg_rmw(ctxt, reg); - assign_register(preg, *preg + inc, ctxt->ad_bytes); + insn_assign_reg(preg, *preg + inc, ctxt->ad_bytes); } static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) @@ -1756,7 +1738,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt, static void write_register_operand(struct operand *op) { - return assign_register(op->addr.reg, op->val, op->bytes); + return insn_assign_reg(op->addr.reg, op->val, op->bytes); } static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) @@ -1989,7 +1971,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt) rc = emulate_pop(ctxt, &val, ctxt->op_bytes); if (rc != X86EMUL_CONTINUE) break; - assign_register(reg_rmw(ctxt, reg), val, ctxt->op_bytes); + insn_assign_reg(reg_rmw(ctxt, reg), val, ctxt->op_bytes); --reg; } return rc; diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 4ebb3c40c6bbc..380a211f56f00 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -626,6 +626,18 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) return HRTIMER_NORESTART; } +/* + * Translate a stimer expiry given in 100ns reference ticks into an + * an absolute deadline. Saturates on overflow. + */ +static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns) +{ + if (delta_100ns >= KTIME_MAX / 100) + return KTIME_MAX; + + return ktime_add_safe(now, 100 * delta_100ns); +} + /* * stimer_start() assumptions: * a) stimer->count is not equal to 0 @@ -635,6 +647,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) { u64 time_now; ktime_t ktime_now; + ktime_t deadline; time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); ktime_now = ktime_get(); @@ -657,10 +670,8 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->exp_time); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, - 100 * (stimer->exp_time - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); return 0; } stimer->exp_time = stimer->count; @@ -679,9 +690,9 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->count); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->count - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); + return 0; } diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index f895019a55e57..e6e747507fd3e 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2065,7 +2065,7 @@ static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn) if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use) ktimer->expired_tscdeadline = ktimer->tscdeadline; - if (!from_timer_fn && apic->apicv_active) { + if (!from_timer_fn && apic->apicv_active && vcpu->wants_to_run) { WARN_ON(kvm_get_running_vcpu() != vcpu); kvm_apic_inject_pending_timer_irqs(apic); return; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index eadeca9786f6f..e6985e00cba0a 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1218,18 +1218,9 @@ struct rmap_iterator { int pos; /* index of the sptep */ }; -/* - * Iteration must be started by this function. This should also be used after - * removing/dropping sptes from the rmap link because in such cases the - * information in the iterator may not be valid. - * - * Returns sptep if found, NULL otherwise. - */ -static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, - struct rmap_iterator *iter) +static u64 *__rmap_get_first(unsigned long rmap_val, + struct rmap_iterator *iter) { - unsigned long rmap_val = kvm_rmap_get(rmap_head); - if (!rmap_val) return NULL; @@ -1243,6 +1234,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, return iter->desc->sptes[iter->pos]; } +/* + * Iteration must be started by this function. This should also be used after + * removing/dropping sptes from the rmap link because in such cases the + * information in the iterator may not be valid. + * + * Returns sptep if found, NULL otherwise. + */ +static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, + struct rmap_iterator *iter) +{ + return __rmap_get_first(kvm_rmap_get(rmap_head), iter); +} + /* * Must be used with a valid iterator: e.g. after rmap_get_first(). * @@ -1277,8 +1281,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter) __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \ -#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \ - __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ +#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \ + for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \ + _sptep_; _sptep_ = rmap_get_next(_iter_)) \ if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep))) static void drop_spte(struct kvm *kvm, u64 *sptep) @@ -1716,7 +1721,7 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) { + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { if (!is_accessed_spte(spte)) continue; @@ -2415,6 +2420,9 @@ static union kvm_mmu_page_role kvm_mmu_child_role(u64 *sptep, bool direct, role.direct = direct; role.passthrough = 0; + WARN_ON_ONCE(role.invalid); + role.invalid = 0; + /* * If the guest has 4-byte PTEs then that means it's using 32-bit, * 2-level, non-PAE paging. KVM shadows such guests with PAE paging @@ -4806,16 +4814,17 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault if (r != RET_PF_CONTINUE) return r; - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = direct_map(vcpu, fault); out_unlock: @@ -6692,20 +6701,11 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm) kvm_mmu_commit_zap_page(kvm, &invalid_list); } -/* - * Fast invalidate all shadow pages and use lock-break technique - * to zap obsolete pages. - * - * It's required when memslot is being deleted or VM is being - * destroyed, in these cases, we should ensure that KVM MMU does - * not use any resource of the being-deleted slot or all slots - * after calling the function. - */ -static void kvm_mmu_zap_all_fast(struct kvm *kvm) +static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm) { lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_held_write(&kvm->mmu_lock); - write_lock(&kvm->mmu_lock); trace_kvm_mmu_zap_all_fast(kvm); /* @@ -6742,8 +6742,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS); kvm_zap_obsolete_pages(kvm); +} - write_unlock(&kvm->mmu_lock); +static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_not_held(&kvm->mmu_lock); /* * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before @@ -6757,6 +6761,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_tdp_mmu_zap_invalidated_roots(kvm, true); } +/* + * Fast invalidate all shadow pages and use lock-break technique + * to zap obsolete pages. + * + * It's required when memslot is being deleted or VM is being + * destroyed, in these cases, we should ensure that KVM MMU does + * not use any resource of the being-deleted slot or all slots + * after calling the function. + */ +static void kvm_mmu_zap_all_fast(struct kvm *kvm) +{ + write_lock(&kvm->mmu_lock); + __kvm_mmu_zap_all_fast_front_half(kvm); + write_unlock(&kvm->mmu_lock); + + __kvm_mmu_zap_all_fast_back_half(kvm); +} + int kvm_mmu_init_vm(struct kvm *kvm) { int r, i; @@ -7331,8 +7353,8 @@ static void kvm_mmu_zap_memslot_pages_and_flush(struct kvm *kvm, kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static void kvm_mmu_zap_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) { struct kvm_gfn_range range = { .slot = slot, @@ -7341,27 +7363,23 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm, .may_block = true, .attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED, }; + bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); bool flush; write_lock(&kvm->mmu_lock); - flush = kvm_unmap_gfn_range(kvm, &range); - kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); -} -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} + if (zap_all) { + __kvm_mmu_zap_all_fast_front_half(kvm); + } else { + flush = kvm_unmap_gfn_range(kvm, &range); + kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); + } -void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - if (kvm_memslot_flush_zap_all(kvm)) - kvm_mmu_zap_all_fast(kvm); - else - kvm_mmu_zap_memslot(kvm, slot); + write_unlock(&kvm->mmu_lock); + + if (zap_all) + __kvm_mmu_zap_all_fast_back_half(kvm); } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) @@ -7392,7 +7410,9 @@ void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) static void mmu_destroy_caches(void) { kmem_cache_destroy(pte_list_desc_cache); + pte_list_desc_cache = NULL; kmem_cache_destroy(mmu_page_header_cache); + mmu_page_header_cache = NULL; } static void kvm_wake_nx_recovery_thread(struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 1b17b12393a8c..7e8195a311bb0 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -130,13 +130,23 @@ void __kvm_write_track_remove_gfn(struct kvm *kvm, kvm_mmu_gfn_allow_lpage(slot, gfn); } -/* - * check if the corresponding access on the specified guest page is tracked. - */ +static bool __kvm_gfn_is_write_tracked(const struct kvm_memory_slot *slot, + gfn_t gfn) +{ + int index; + + if (!slot) + return false; + + index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); + return !!READ_ONCE(slot->arch.gfn_write_track[index]); +} + +/* check if write access is tracked on the specified guest page. */ bool kvm_gfn_is_write_tracked(struct kvm *kvm, const struct kvm_memory_slot *slot, gfn_t gfn) { - int index; + const struct kvm_memory_slot *other_slot; if (!slot) return false; @@ -144,8 +154,18 @@ bool kvm_gfn_is_write_tracked(struct kvm *kvm, if (!kvm_page_track_write_tracking_enabled(kvm)) return false; - index = gfn_to_index(gfn, slot->base_gfn, PG_LEVEL_4K); - return !!READ_ONCE(slot->arch.gfn_write_track[index]); + BUILD_BUG_ON(KVM_MAX_NR_ADDRESS_SPACES > 2); + + if (__kvm_gfn_is_write_tracked(slot, gfn)) + return true; + + if (kvm_arch_nr_memslot_as_ids(kvm) > 1) { + other_slot = __gfn_to_memslot(__kvm_memslots(kvm, slot->as_id ^ 1), gfn); + if (__kvm_gfn_is_write_tracked(other_slot, gfn)) + return true; + } + + return false; } #ifdef CONFIG_KVM_EXTERNAL_WRITE_TRACKING diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index ed762bb4b007b..af220c9c8ab8d 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -827,15 +827,17 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault } #endif - r = RET_PF_RETRY; write_lock(&vcpu->kvm->mmu_lock); - if (is_page_fault_stale(vcpu, fault)) - goto out_unlock; - r = make_mmu_pages_available(vcpu); if (r) goto out_unlock; + + if (is_page_fault_stale(vcpu, fault)) { + r = RET_PF_RETRY; + goto out_unlock; + } + r = FNAME(fetch)(vcpu, fault, &walker); out_unlock: diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h index 364c5da6c4991..f898d8d0d93c2 100644 --- a/arch/x86/kvm/mmu/tdp_iter.h +++ b/arch/x86/kvm/mmu/tdp_iter.h @@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) return READ_ONCE(*rcu_dereference(sptep)); } +/* + * WARNING! mmu_lock must be held for write when using the "write atomic" or + * "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE + * value, i.e. clobber an update from a different CPU. The only exception is + * when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care + * about the exact old SPTE value (KVM will react to the actual old value). + */ static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) { KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index c5734ca5c17dd..7bd2be5a393a9 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1380,19 +1380,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter) { u64 new_spte; - if (spte_ad_enabled(iter->old_spte)) { - iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep, - shadow_accessed_mask); + if (spte_ad_enabled(iter->old_spte)) new_spte = iter->old_spte & ~shadow_accessed_mask; - } else { + else new_spte = mark_spte_for_access_track(iter->old_spte); - /* - * It is safe for the following cmpxchg to fail. Leave the - * Accessed bit set, as the spte is most likely young anyway. - */ - if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) - return; - } + + /* + * Don't bother retrying if another CPU modified the SPTE, the SPTE is + * either being zapped or is likely still in-use, i.e. is still young. + */ + if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) + return; trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, iter->old_spte, new_spte); diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c new file mode 100644 index 0000000000000..7b997c4e6329f --- /dev/null +++ b/arch/x86/kvm/regs.c @@ -0,0 +1,787 @@ +// SPDX-License-Identifier: GPL-2.0-only +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include + +#include "lapic.h" +#include "mmu.h" +#include "x86.h" + +static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { + /* + * We are here if userspace calls get_regs() in the middle of + * instruction emulation. Registers state needs to be copied + * back from emulation context to vcpu. Userspace shouldn't do + * that usually, but some bad designed PV devices (vmware + * backdoor interface) need this to work + */ + emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + } + regs->rax = kvm_rax_read(vcpu); + regs->rbx = kvm_rbx_read(vcpu); + regs->rcx = kvm_rcx_read(vcpu); + regs->rdx = kvm_rdx_read(vcpu); + regs->rsi = kvm_rsi_read(vcpu); + regs->rdi = kvm_rdi_read(vcpu); + regs->rsp = kvm_rsp_read(vcpu); + regs->rbp = kvm_rbp_read(vcpu); +#ifdef CONFIG_X86_64 + regs->r8 = kvm_r8_read(vcpu); + regs->r9 = kvm_r9_read(vcpu); + regs->r10 = kvm_r10_read(vcpu); + regs->r11 = kvm_r11_read(vcpu); + regs->r12 = kvm_r12_read(vcpu); + regs->r13 = kvm_r13_read(vcpu); + regs->r14 = kvm_r14_read(vcpu); + regs->r15 = kvm_r15_read(vcpu); +#endif + + regs->rip = kvm_rip_read(vcpu); + regs->rflags = kvm_get_rflags(vcpu); +} + +int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + vcpu->arch.emulate_regs_need_sync_from_vcpu = true; + vcpu->arch.emulate_regs_need_sync_to_vcpu = false; + + kvm_rax_write(vcpu, regs->rax); + kvm_rbx_write(vcpu, regs->rbx); + kvm_rcx_write(vcpu, regs->rcx); + kvm_rdx_write(vcpu, regs->rdx); + kvm_rsi_write(vcpu, regs->rsi); + kvm_rdi_write(vcpu, regs->rdi); + kvm_rsp_write(vcpu, regs->rsp); + kvm_rbp_write(vcpu, regs->rbp); +#ifdef CONFIG_X86_64 + kvm_r8_write(vcpu, regs->r8); + kvm_r9_write(vcpu, regs->r9); + kvm_r10_write(vcpu, regs->r10); + kvm_r11_write(vcpu, regs->r11); + kvm_r12_write(vcpu, regs->r12); + kvm_r13_write(vcpu, regs->r13); + kvm_r14_write(vcpu, regs->r14); + kvm_r15_write(vcpu, regs->r15); +#endif + + kvm_rip_write(vcpu, regs->rip); + kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); + + vcpu->arch.exception.pending = false; + vcpu->arch.exception_vmexit.pending = false; + + kvm_make_request(KVM_REQ_EVENT, vcpu); +} + +int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __set_regs(vcpu, regs); + vcpu_put(vcpu); + return 0; +} + +static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) +{ + return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); +} + +/* + * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. + */ +int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + struct kvm_mmu *mmu = vcpu->arch.walk_mmu; + gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; + gpa_t real_gpa; + int i; + int ret; + u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; + + /* + * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated + * to an L1 GPA. + */ + real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), + PFERR_USER_MASK | PFERR_WRITE_MASK, NULL); + if (real_gpa == INVALID_GPA) + return 0; + + /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ + ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, + cr3 & GENMASK(11, 5), sizeof(pdpte)); + if (ret < 0) + return 0; + + for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { + if ((pdpte[i] & PT_PRESENT_MASK) && + (pdpte[i] & pdptr_rsvd_bits(vcpu))) { + return 0; + } + } + + /* + * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled. + * Shadow page roots need to be reconstructed instead. + */ + if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs))) + kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); + + memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); + kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); + vcpu->arch.pdptrs_from_userspace = false; + + return 1; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); + +static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ +#ifdef CONFIG_X86_64 + if (cr0 & 0xffffffff00000000UL) + return false; +#endif + + if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) + return false; + + if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) + return false; + + return kvm_x86_call(is_valid_cr0)(vcpu, cr0); +} + +int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) +{ + unsigned long old_cr0 = kvm_read_cr0(vcpu); + + if (!kvm_is_valid_cr0(vcpu, cr0)) + return 1; + + cr0 |= X86_CR0_ET; + + /* Write to CR0 reserved bits are ignored, even on Intel. */ + cr0 &= ~CR0_RESERVED_BITS; + +#ifdef CONFIG_X86_64 + if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && + (cr0 & X86_CR0_PG)) { + int cs_db, cs_l; + + if (!is_pae(vcpu)) + return 1; + kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); + if (cs_l) + return 1; + } +#endif + if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && + is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && + !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if (!(cr0 & X86_CR0_PG) && + (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) + return 1; + + if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) + return 1; + + kvm_x86_call(set_cr0)(vcpu, cr0); + + kvm_post_set_cr0(vcpu, old_cr0, cr0); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); + +void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) +{ + (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); + +int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) +{ + bool skip_tlb_flush = false; + unsigned long pcid = 0; +#ifdef CONFIG_X86_64 + if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { + skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; + cr3 &= ~X86_CR3_PCID_NOFLUSH; + pcid = cr3 & X86_CR3_PCID_MASK; + } +#endif + + /* PDPTRs are always reloaded for PAE paging. */ + if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) + goto handle_tlb_flush; + + /* + * Do not condition the GPA check on long mode, this helper is used to + * stuff CR3, e.g. for RSM emulation, and there is no guarantee that + * the current vCPU mode is accurate. + */ + if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) + return 1; + + if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) + return 1; + + if (cr3 != kvm_read_cr3(vcpu)) + kvm_mmu_new_pgd(vcpu, cr3); + + vcpu->arch.cr3 = cr3; + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); + /* Do not call post_set_cr3, we do not get here for confidential guests. */ + +handle_tlb_flush: + /* + * A load of CR3 that flushes the TLB flushes only the current PCID, + * even if PCID is disabled, in which case PCID=0 is flushed. It's a + * moot point in the end because _disabling_ PCID will flush all PCIDs, + * and it's impossible to use a non-zero PCID when PCID is disabled, + * i.e. only PCID=0 can be relevant. + */ + if (!skip_tlb_flush) + kvm_invalidate_pcid(vcpu, pcid); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); + +static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + return __kvm_is_valid_cr4(vcpu, cr4) && + kvm_x86_call(is_valid_cr4)(vcpu, cr4); +} + +void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) +{ + if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) + kvm_mmu_reset_context(vcpu); + + /* + * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB + * according to the SDM; however, stale prev_roots could be reused + * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we + * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST + * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, + * so fall through. + */ + if (!tdp_enabled && + (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) + kvm_mmu_unload(vcpu); + + /* + * The TLB has to be flushed for all PCIDs if any of the following + * (architecturally required) changes happen: + * - CR4.PCIDE is changed from 1 to 0 + * - CR4.PGE is toggled + * + * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. + */ + if (((cr4 ^ old_cr4) & X86_CR4_PGE) || + (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + + /* + * The TLB has to be flushed for the current PCID if any of the + * following (architecturally required) changes happen: + * - CR4.SMEP is changed from 0 to 1 + * - CR4.PAE is toggled + */ + else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || + ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); + +int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) +{ + unsigned long old_cr4 = kvm_read_cr4(vcpu); + + if (!kvm_is_valid_cr4(vcpu, cr4)) + return 1; + + if (is_long_mode(vcpu)) { + if (!(cr4 & X86_CR4_PAE)) + return 1; + if ((cr4 ^ old_cr4) & X86_CR4_LA57) + return 1; + } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) + && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) + && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) + return 1; + + if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { + /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ + if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) + return 1; + } + + if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) + return 1; + + kvm_x86_call(set_cr4)(vcpu, cr4); + + kvm_post_set_cr4(vcpu, old_cr4, cr4); + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); + +int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) +{ + if (cr8 & CR8_RESERVED_BITS) + return 1; + if (lapic_in_kernel(vcpu)) + kvm_lapic_set_tpr(vcpu, cr8); + else + vcpu->arch.cr8 = cr8; + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); + +unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) +{ + if (lapic_in_kernel(vcpu)) + return kvm_lapic_get_cr8(vcpu); + else + return vcpu->arch.cr8; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); + +static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + struct desc_ptr dt; + + if (vcpu->arch.guest_state_protected) + goto skip_protected_regs; + + kvm_handle_exception_payload_quirk(vcpu); + + kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_x86_call(get_idt)(vcpu, &dt); + sregs->idt.limit = dt.size; + sregs->idt.base = dt.address; + kvm_x86_call(get_gdt)(vcpu, &dt); + sregs->gdt.limit = dt.size; + sregs->gdt.base = dt.address; + + sregs->cr2 = vcpu->arch.cr2; + sregs->cr3 = kvm_read_cr3(vcpu); + +skip_protected_regs: + sregs->cr0 = kvm_read_cr0(vcpu); + sregs->cr4 = kvm_read_cr4(vcpu); + sregs->cr8 = kvm_get_cr8(vcpu); + sregs->efer = vcpu->arch.efer; + sregs->apic_base = vcpu->arch.apic_base; +} + +static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + __get_sregs_common(vcpu, sregs); + + if (vcpu->arch.guest_state_protected) + return; + + if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) + set_bit(vcpu->arch.interrupt.nr, + (unsigned long *)sregs->interrupt_bitmap); +} + +int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + __get_sregs(vcpu, sregs); + vcpu_put(vcpu); + return 0; +} + +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int i; + + __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); + + if (vcpu->arch.guest_state_protected) + return; + + if (is_pae_paging(vcpu)) { + kvm_vcpu_srcu_read_lock(vcpu); + for (i = 0 ; i < 4 ; i++) + sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); + sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + kvm_vcpu_srcu_read_unlock(vcpu); + } +} + +static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { + /* + * When EFER.LME and CR0.PG are set, the processor is in + * 64-bit mode (though maybe in a 32-bit code segment). + * CR4.PAE and EFER.LMA must be set. + */ + if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) + return false; + if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) + return false; + } else { + /* + * Not in 64-bit mode: EFER.LMA is clear and the code + * segment cannot be 64-bit. + */ + if (sregs->efer & EFER_LMA || sregs->cs.l) + return false; + } + + return kvm_is_valid_cr4(vcpu, sregs->cr4) && + kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_valid_efer(vcpu, sregs->efer); +} + +static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, + int *mmu_reset_needed, bool update_pdptrs) +{ + int idx; + struct desc_ptr dt; + + if (!kvm_is_valid_sregs(vcpu, sregs)) + return -EINVAL; + + if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) + return -EINVAL; + + if (vcpu->arch.guest_state_protected) + return 0; + + dt.size = sregs->idt.limit; + dt.address = sregs->idt.base; + kvm_x86_call(set_idt)(vcpu, &dt); + dt.size = sregs->gdt.limit; + dt.address = sregs->gdt.base; + kvm_x86_call(set_gdt)(vcpu, &dt); + + vcpu->arch.cr2 = sregs->cr2; + *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; + vcpu->arch.cr3 = sregs->cr3; + kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); + kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); + + *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; + kvm_x86_call(set_efer)(vcpu, sregs->efer); + + *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; + kvm_x86_call(set_cr0)(vcpu, sregs->cr0); + + *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; + kvm_x86_call(set_cr4)(vcpu, sregs->cr4); + + if (update_pdptrs) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + if (is_pae_paging(vcpu)) { + load_pdptrs(vcpu, kvm_read_cr3(vcpu)); + *mmu_reset_needed = 1; + } + srcu_read_unlock(&vcpu->kvm->srcu, idx); + } + + kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); + kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); + kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); + kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); + kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); + kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); + + kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); + kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); + + kvm_set_cr8(vcpu, sregs->cr8); + + /* Older userspace won't unhalt the vcpu on reset. */ + if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && + sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && + !is_protmode(vcpu)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); + + return 0; +} + +static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + int pending_vec, max_bits; + int mmu_reset_needed = 0; + int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); + + if (ret) + return ret; + + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + + max_bits = KVM_NR_INTERRUPTS; + pending_vec = find_first_bit( + (const unsigned long *)sregs->interrupt_bitmap, max_bits); + + if (pending_vec < max_bits) { + kvm_queue_interrupt(vcpu, pending_vec, false); + pr_debug("Set back pending irq %d\n", pending_vec); + kvm_make_request(KVM_REQ_EVENT, vcpu); + } + return 0; +} + +int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, + struct kvm_sregs *sregs) +{ + int ret; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + vcpu_load(vcpu); + ret = __set_sregs(vcpu, sregs); + vcpu_put(vcpu); + return ret; +} + +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2) +{ + int mmu_reset_needed = 0; + bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; + bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && + !(sregs2->efer & EFER_LMA); + int i, ret; + + if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) + return -EINVAL; + + if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) + return -EINVAL; + + ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, + &mmu_reset_needed, !valid_pdptrs); + if (ret) + return ret; + + if (valid_pdptrs) { + for (i = 0; i < 4 ; i++) + kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); + + kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + mmu_reset_needed = 1; + vcpu->arch.pdptrs_from_userspace = true; + } + if (mmu_reset_needed) { + kvm_mmu_reset_context(vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); + } + return 0; +} + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu) +{ + BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) + __get_regs(vcpu, &vcpu->run->s.regs.regs); + + if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) + __get_sregs(vcpu, &vcpu->run->s.regs.sregs); +} + +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu) +{ + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { + __set_regs(vcpu, &vcpu->run->s.regs.regs); + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; + } + + if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { + struct kvm_sregs sregs = vcpu->run->s.regs.sregs; + + if (__set_sregs(vcpu, &sregs)) + return -EINVAL; + + vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; + } + + return 0; +} + +void kvm_update_dr0123(struct kvm_vcpu *vcpu) +{ + int i; + + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { + for (i = 0; i < KVM_NR_DB_REGS; i++) + vcpu->arch.eff_db[i] = vcpu->arch.db[i]; + } +} + +void kvm_update_dr7(struct kvm_vcpu *vcpu) +{ + unsigned long dr7; + + if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) + dr7 = vcpu->arch.guest_debug_dr7; + else + dr7 = vcpu->arch.dr7; + kvm_x86_call(set_dr7)(vcpu, dr7); + vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; + if (dr7 & DR7_BP_EN_MASK) + vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); + +static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) +{ + u64 fixed = DR6_FIXED_1; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) + fixed |= DR6_RTM; + + if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) + fixed |= DR6_BUS_LOCK; + return fixed; +} + +int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + vcpu->arch.db[array_index_nospec(dr, size)] = val; + if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) + vcpu->arch.eff_db[dr] = val; + break; + case 4: + case 6: + if (!kvm_dr6_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); + break; + case 5: + default: /* 7 */ + if (!kvm_dr7_valid(val)) + return 1; /* #GP */ + vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; + kvm_update_dr7(vcpu); + break; + } + + return 0; +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); + +unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) +{ + size_t size = ARRAY_SIZE(vcpu->arch.db); + + switch (dr) { + case 0 ... 3: + return vcpu->arch.db[array_index_nospec(dr, size)]; + case 4: + case 6: + return vcpu->arch.dr6; + case 5: + default: /* 7 */ + return vcpu->arch.dr7; + } +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); + +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + kvm_handle_exception_payload_quirk(vcpu); + + memset(dbgregs, 0, sizeof(*dbgregs)); + + BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + dbgregs->db[i] = vcpu->arch.db[i]; + + dbgregs->dr6 = vcpu->arch.dr6; + dbgregs->dr7 = vcpu->arch.dr7; + return 0; +} + +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs) +{ + unsigned int i; + + if (vcpu->kvm->arch.has_protected_state && + vcpu->arch.guest_state_protected) + return -EINVAL; + + if (dbgregs->flags) + return -EINVAL; + + if (!kvm_dr6_valid(dbgregs->dr6)) + return -EINVAL; + if (!kvm_dr7_valid(dbgregs->dr7)) + return -EINVAL; + + for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) + vcpu->arch.db[i] = dbgregs->db[i]; + + kvm_update_dr0123(vcpu); + vcpu->arch.dr6 = dbgregs->dr6; + vcpu->arch.dr7 = dbgregs->dr7; + kvm_update_dr7(vcpu); + + return 0; +} diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index a700f3795c176..62eb7b871a84d 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -220,14 +220,6 @@ static void avic_deactivate_vmcb(struct vcpu_svm *svm) if (!sev_es_guest(svm->vcpu.kvm)) svm_set_intercept(svm, INTERCEPT_CR8_WRITE); - /* - * If running nested and the guest uses its own MSR bitmap, there - * is no need to update L0's msr bitmap - */ - if (is_guest_mode(&svm->vcpu) && - vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT)) - return; - /* Enabling MSR intercept for x2APIC registers */ avic_set_x2apic_msr_interception(svm, true); } diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 36b8488aba6b0..c32c38520a34e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -85,6 +85,8 @@ module_param_named(ciphertext_hiding_asids, nr_ciphertext_hiding_asids, uint, 04 static u8 sev_enc_bit; static DECLARE_RWSEM(sev_deactivate_lock); static DEFINE_MUTEX(sev_bitmap_lock); +/* Protects kvm_sev_info's enc_context_owner, mirror_vms and mirror_entry. */ +static DEFINE_MUTEX(sev_mirror_lock); unsigned int max_sev_asid; static unsigned int min_sev_asid; static unsigned int max_sev_es_asid; @@ -1975,7 +1977,6 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) dst->asid = src->asid; dst->handle = src->handle; dst->pages_locked = src->pages_locked; - dst->enc_context_owner = src->enc_context_owner; dst->es_active = src->es_active; dst->vmsa_features = src->vmsa_features; @@ -1983,11 +1984,12 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) src->active = false; src->handle = 0; src->pages_locked = 0; - src->enc_context_owner = NULL; src->es_active = false; list_cut_before(&dst->regions_list, &src->regions_list, &src->regions_list); + mutex_lock(&sev_mirror_lock); + /* * If this VM has mirrors, "transfer" each mirror's refcount of the * source to the destination (this KVM). The caller holds a reference @@ -2004,12 +2006,15 @@ static void sev_migrate_from(struct kvm *dst_kvm, struct kvm *src_kvm) * If this VM is a mirror, remove the old mirror from the owners list * and add the new mirror to the list. */ - if (is_mirroring_enc_context(dst_kvm)) { - struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(dst->enc_context_owner); + if (is_mirroring_enc_context(src_kvm)) { + struct kvm_sev_info *owner_sev_info = to_kvm_sev_info(src->enc_context_owner); + dst->enc_context_owner = src->enc_context_owner; + src->enc_context_owner = NULL; list_del(&src->mirror_entry); list_add_tail(&dst->mirror_entry, &owner_sev_info->mirror_vms); } + mutex_unlock(&sev_mirror_lock); kvm_for_each_vcpu(i, dst_vcpu, dst_kvm) { dst_svm = to_svm(dst_vcpu); @@ -2722,8 +2727,12 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; + /* + * Do NOT specify FOLL_WRITE, as KVM isn't using the pinned pages to + * write memory, and FOLL_LONGTERM itself triggers CoW unshare. + */ region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, - FOLL_WRITE | FOLL_LONGTERM); + FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); goto e_free; @@ -2851,11 +2860,14 @@ int sev_vm_copy_enc_context_from(struct kvm *kvm, unsigned int source_fd) * disappear until we're done with it */ source_sev = to_kvm_sev_info(source_kvm); - kvm_get_kvm(source_kvm); - list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); /* Set enc_context_owner and copy its encryption context over */ + mutex_lock(&sev_mirror_lock); + kvm_get_kvm(source_kvm); + list_add_tail(&mirror_sev->mirror_entry, &source_sev->mirror_vms); mirror_sev->enc_context_owner = source_kvm; + mutex_unlock(&sev_mirror_lock); + mirror_sev->active = true; mirror_sev->asid = source_sev->asid; mirror_sev->fd = source_sev->fd; @@ -2921,11 +2933,19 @@ void sev_vm_destroy(struct kvm *kvm) * Note, mirror VMs don't support registering encrypted regions. */ if (is_mirroring_enc_context(kvm)) { - struct kvm *owner_kvm = sev->enc_context_owner; + struct kvm *owner_kvm; - mutex_lock(&owner_kvm->lock); + mutex_lock(&sev_mirror_lock); + owner_kvm = sev->enc_context_owner; list_del(&sev->mirror_entry); - mutex_unlock(&owner_kvm->lock); + sev->enc_context_owner = NULL; + + /* + * The reference to owner_kvm cannot move after sev_mirror_lock is + * released. Release it before kvm_put_kvm() so that owner_kvm is + * never destroyed inside sev_mirror_lock. + */ + mutex_unlock(&sev_mirror_lock); kvm_put_kvm(owner_kvm); return; } @@ -3993,30 +4013,19 @@ static int snp_begin_psc(struct vcpu_svm *svm) BUG(); } -/* - * Invoked as part of svm_vcpu_reset() processing of an init event. - */ -static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + gfn_t gfn = gpa_to_gfn(gpa); struct page *page; kvm_pfn_t pfn; - gfn_t gfn; - guard(mutex)(&svm->sev_es.snp_vmsa_mutex); - - if (!svm->sev_es.snp_ap_waiting_for_reset) - return; - - svm->sev_es.snp_ap_waiting_for_reset = false; - - /* Mark the vCPU as offline and not runnable */ - vcpu->arch.pv.pv_unhalted = false; - kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); + lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - /* Clear use of the VMSA */ + /* Clear use of the VMSA. */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4024,12 +4033,9 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa)) + if (!VALID_PAGE(gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa); - svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; - slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) return; @@ -4053,11 +4059,9 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_has_guest_vmsa = true; /* Use the new VMSA */ + svm->sev_es.snp_guest_vmsa_gpa = gpa; svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - /* Mark the vCPU as runnable */ - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - /* * gmem pages aren't currently migratable, but if this ever changes * then care should be taken to ensure svm->sev_es.vmsa is pinned @@ -4066,6 +4070,40 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_release_page_clean(page); } +/* + * Invoked as part of svm_vcpu_reset() processing of an init event. + */ +static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + gpa_t gpa; + + guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + + if (!svm->sev_es.snp_ap_waiting_for_reset) + return; + + svm->sev_es.snp_ap_waiting_for_reset = false; + + /* Mark the vCPU as offline and not runnable */ + vcpu->arch.pv.pv_unhalted = false; + kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); + + gpa = svm->sev_es.snp_pending_vmsa_gpa; + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + + sev_snp_reload_vmsa(vcpu, gpa); + + /* + * Mark the vCPU as runnable for CREATE requests, indicated by a valid + * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will + * fail instead of blocking indefinitely and hanging the vCPU, e.g. if + * the backing guest_memfd page is unavailable. + */ + if (VALID_PAGE(gpa)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); @@ -4119,10 +4157,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm) return -EINVAL; } - target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2; + target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2; break; case SVM_VMGEXIT_AP_DESTROY: - target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; break; default: vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n", @@ -4711,6 +4749,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu) return -ENOMEM; svm->sev_es.vmsa = page_address(vmsa_page); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm); diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index bb034896bf6df..a24a6871b6936 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -523,7 +523,12 @@ static int svm_enable_virtualization_cpu(void) return -EBUSY; sd = per_cpu_ptr(&svm_data, me); - sd->asid_generation = 1; + /* + * Bump the current asid_generation value to ensure any vCPU that + * previously ran on this CPU sees a stale generation and is forced + * to acquire a new ASID, preventing a latent ASID collision. + */ + sd->asid_generation++; sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; sd->next_asid = sd->max_asid + 1; sd->min_asid = max_sev_asid + 1; @@ -5257,6 +5262,10 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, + + .gmem_prepare = sev_gmem_prepare, + .gmem_invalidate = sev_gmem_invalidate, + .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, @@ -5268,10 +5277,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, .alloc_apic_backing_page = svm_alloc_apic_backing_page, - - .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, - .gmem_max_mapping_level = sev_gmem_max_mapping_level, }; /* diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 990b4caf8b6df..85f64d9ebe59a 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -105,6 +105,7 @@ struct kvm_sev_info { u64 ap_jump_table; /* SEV-ES AP Jump Table address */ u64 vmsa_features; u16 ghcb_version; /* Highest guest GHCB protocol version allowed */ + /* The three fields below are protected by sev_mirror_lock */ struct kvm *enc_context_owner; /* Owner of copied encryption context */ struct list_head mirror_vms; /* List of VMs mirroring */ struct list_head mirror_entry; /* Use as a list entry of mirrors */ @@ -247,7 +248,8 @@ struct vcpu_sev_es_state { u64 ghcb_registered_gpa; struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */ - gpa_t snp_vmsa_gpa; + gpa_t snp_pending_vmsa_gpa; + gpa_t snp_guest_vmsa_gpa; bool snp_ap_waiting_for_reset; bool snp_has_guest_vmsa; }; @@ -904,16 +906,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; } static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; } #define max_sev_asid 0 static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {} -static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) -{ - return 0; -} -static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {} -static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) -{ - return 0; -} - static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu) { return NULL; diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a5a525ae8e783..bc5aa7456aac9 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -333,6 +333,7 @@ static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu) static void free_nested(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); + struct vmcs *shadow_vmcs; if (WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01)) vmx_switch_vmcs(vcpu, &vmx->vmcs01); @@ -350,9 +351,15 @@ static void free_nested(struct kvm_vcpu *vcpu) vmx->nested.current_vmptr = INVALID_GPA; if (enable_shadow_vmcs) { vmx_disable_shadow_vmcs(vmx); - vmcs_clear(vmx->vmcs01.shadow_vmcs); - free_vmcs(vmx->vmcs01.shadow_vmcs); + + /* + * Keep the pointer visible until after VMCLEAR, so migration + * can clear an active shadow VMCS on the old CPU. + */ + shadow_vmcs = vmx->vmcs01.shadow_vmcs; + vmcs_clear(shadow_vmcs); vmx->vmcs01.shadow_vmcs = NULL; + free_vmcs(shadow_vmcs); } kfree(vmx->nested.cached_vmcs12); vmx->nested.cached_vmcs12 = NULL; @@ -1262,6 +1269,9 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * is the VPID incorporated into the MMU context. I.e. KVM must assume * that the new vpid12 has never been used and thus represents a new * guest ASID that cannot have entries in the TLB. + * + * Note, last_vpid is initialized as 0, so the first nested VM-Enter + * after VMXON will always flush the TLB to avoid using stale entries. */ if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { vmx->nested.last_vpid = vmcs12->virtual_processor_id; @@ -3791,6 +3801,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, vmentry_fail_vmexit_guest_mode: if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) vcpu->arch.tsc_offset -= vmcs12->tsc_offset; + + /* + * Handle any TLB flush requests that were queued for L2 if KVM made it + * far enough along to switch to L2 context. Note, loading host state + * will generate any flushes for L1 required by VM-Exit. + */ + kvm_service_local_tlb_flush_requests(vcpu); + leave_guest_mode(vcpu); vmentry_fail_vmexit: @@ -5125,8 +5143,9 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ WARN_ON_ONCE(vmx->nested.nested_run_pending); -#ifdef CONFIG_KVM_HYPERV + /* Note, "checking" the request also clears the request. */ if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { +#ifdef CONFIG_KVM_HYPERV /* * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map * Enlightened VMCS after migration and we still need to @@ -5134,8 +5153,8 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, * the first L2 run. */ (void)nested_get_evmcs_page(vcpu); - } #endif + } /* Service pending TLB flush requests for L2 before switching to L1. */ kvm_service_local_tlb_flush_requests(vcpu); @@ -5507,6 +5526,13 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) vmx->nested.vpid02 = allocate_vpid(); + /* + * Clear last_vpid to ensure that the VPID is flushed on the first + * nested VM-Enter. Otherwise, stale TLB entries from a previous life of + * the VPID (e.g. different vCPU or even different VM) could be used. + */ + vmx->nested.last_vpid = 0; + vmx->nested.vmcs02_initialized = false; vmx->nested.vmxon = true; @@ -6134,8 +6160,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 vpid; u64 gla; } operand; - u16 vpid02; int r, gpr_index; + int cpu; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_VPID) || @@ -6169,42 +6195,34 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) return kvm_handle_memory_failure(vcpu, r, &e); if (operand.vpid >> 16) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + /* LAM doesn't apply to addresses that are inputs to TLB invalidation. */ + if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR && + is_noncanonical_invlpg_address(operand.gla, vcpu)) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); /* * Always flush the effective vpid02, i.e. never flush the current VPID * and never explicitly flush vpid01. INVVPID targets a VPID, not a * VMCS, and so whether or not the current vmcs12 has VPID enabled is * irrelevant (and there may not be a loaded vmcs12). + * + * If vmcs02 was last loaded on a different pCPU, then defer the flush + * by invalidating the nested VPID tracking to ensure that KVM performs + * the invalidation on the correct pCPU. */ - vpid02 = nested_get_vpid02(vcpu); - switch (type) { - case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: - /* - * LAM doesn't apply to addresses that are inputs to TLB - * invalidation. - */ - if (!operand.vpid || - is_noncanonical_invlpg_address(operand.gla, vcpu)) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_vcpu_addr(vpid02, operand.gla); - break; - case VMX_VPID_EXTENT_SINGLE_CONTEXT: - case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: - if (!operand.vpid) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_context(vpid02); - break; - case VMX_VPID_EXTENT_ALL_CONTEXT: - vpid_sync_context(vpid02); - break; - default: - WARN_ON_ONCE(1); - return kvm_skip_emulated_instruction(vcpu); - } + cpu = get_cpu(); + if (cpu != vmx->nested.vmcs02.cpu) + vmx->nested.last_vpid = 0; + else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) + vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla); + else + vpid_sync_context(nested_get_vpid02(vcpu)); + put_cpu(); /* * Sync the shadow page tables if EPT is disabled, L1 is invalidating diff --git a/arch/x86/kvm/vmx/vmx_ops.h b/arch/x86/kvm/vmx/vmx_ops.h index 96677576c836d..e131c7b555dcf 100644 --- a/arch/x86/kvm/vmx/vmx_ops.h +++ b/arch/x86/kvm/vmx/vmx_ops.h @@ -101,7 +101,7 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) : [output] "=r" (value) : [field] "r" (field) - : "cc" + : "cc", "memory" : do_fail, do_exception); return value; @@ -146,7 +146,7 @@ static __always_inline unsigned long __vmcs_readl(unsigned long field) : ASM_CALL_CONSTRAINT, [output] "=&r" (value) : [field] "r" (field) - : "cc"); + : "cc", "memory"); return value; #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ @@ -194,7 +194,7 @@ do { \ ".byte 0x2e\n\t" /* branch not taken hint */ \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1 : "cc" : error, fault); \ + : : op1 : "cc", "memory" : error, fault); \ return; \ error: \ instrumentation_begin(); \ @@ -211,7 +211,7 @@ do { \ ".byte 0x2e\n\t" /* branch not taken hint */ \ "jna %l[error]\n\t" \ _ASM_EXTABLE(1b, %l[fault]) \ - : : op1, op2 : "cc" : error, fault); \ + : : op1, op2 : "cc", "memory" : error, fault);\ return; \ error: \ instrumentation_begin(); \ diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index f2708419f7870..f08a34d3ff3d9 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -106,16 +106,8 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host); #define emul_to_vcpu(ctxt) \ ((struct kvm_vcpu *)(ctxt)->vcpu) -/* EFER defaults: - * - enable syscall per default because its emulated by KVM - * - enable LME and LMA per default on 64 bit KVM - */ -#ifdef CONFIG_X86_64 -static -u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); -#else +/* Enable syscall by default because its emulated by KVM */ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); -#endif #define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) @@ -132,9 +124,6 @@ static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); static int kvm_vcpu_do_singlestep(struct kvm_vcpu *vcpu); -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); - static DEFINE_MUTEX(vendor_module_lock); static void kvm_load_guest_fpu(struct kvm_vcpu *vcpu); static void kvm_put_guest_fpu(struct kvm_vcpu *vcpu); @@ -1044,77 +1033,6 @@ static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu) return (vcpu->arch.apf.msr_en_val & mask) == mask; } -static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); -} - -/* - * Load the pae pdptrs. Return 1 if they are all valid, 0 otherwise. - */ -int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - gfn_t pdpt_gfn = cr3 >> PAGE_SHIFT; - gpa_t real_gpa; - int i; - int ret; - u64 pdpte[ARRAY_SIZE(mmu->pdptrs)]; - - /* - * If the MMU is nested, CR3 holds an L2 GPA and needs to be translated - * to an L1 GPA. - */ - real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn), - PFERR_USER_MASK | PFERR_WRITE_MASK, NULL); - if (real_gpa == INVALID_GPA) - return 0; - - /* Note the offset, PDPTRs are 32 byte aligned when using PAE paging. */ - ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(real_gpa), pdpte, - cr3 & GENMASK(11, 5), sizeof(pdpte)); - if (ret < 0) - return 0; - - for (i = 0; i < ARRAY_SIZE(pdpte); ++i) { - if ((pdpte[i] & PT_PRESENT_MASK) && - (pdpte[i] & pdptr_rsvd_bits(vcpu))) { - return 0; - } - } - - /* - * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled. - * Shadow page roots need to be reconstructed instead. - */ - if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs))) - kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT); - - memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); - kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); - kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - vcpu->arch.pdptrs_from_userspace = false; - - return 1; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(load_pdptrs); - -static bool kvm_is_valid_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ -#ifdef CONFIG_X86_64 - if (cr0 & 0xffffffff00000000UL) - return false; -#endif - - if ((cr0 & X86_CR0_NW) && !(cr0 & X86_CR0_CD)) - return false; - - if ((cr0 & X86_CR0_PG) && !(cr0 & X86_CR0_PE)) - return false; - - return kvm_x86_call(is_valid_cr0)(vcpu, cr0); -} - void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) { /* @@ -1157,56 +1075,6 @@ void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned lon } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr0); -int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0) -{ - unsigned long old_cr0 = kvm_read_cr0(vcpu); - - if (!kvm_is_valid_cr0(vcpu, cr0)) - return 1; - - cr0 |= X86_CR0_ET; - - /* Write to CR0 reserved bits are ignored, even on Intel. */ - cr0 &= ~CR0_RESERVED_BITS; - -#ifdef CONFIG_X86_64 - if ((vcpu->arch.efer & EFER_LME) && !is_paging(vcpu) && - (cr0 & X86_CR0_PG)) { - int cs_db, cs_l; - - if (!is_pae(vcpu)) - return 1; - kvm_x86_call(get_cs_db_l_bits)(vcpu, &cs_db, &cs_l); - if (cs_l) - return 1; - } -#endif - if (!(vcpu->arch.efer & EFER_LME) && (cr0 & X86_CR0_PG) && - is_pae(vcpu) && ((cr0 ^ old_cr0) & X86_CR0_PDPTR_BITS) && - !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if (!(cr0 & X86_CR0_PG) && - (is_64_bit_mode(vcpu) || kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE))) - return 1; - - if (!(cr0 & X86_CR0_WP) && kvm_is_cr4_bit_set(vcpu, X86_CR4_CET)) - return 1; - - kvm_x86_call(set_cr0)(vcpu, cr0); - - kvm_post_set_cr0(vcpu, old_cr0, cr0); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr0); - -void kvm_lmsw(struct kvm_vcpu *vcpu, unsigned long msw) -{ - (void)kvm_set_cr0(vcpu, kvm_read_cr0_bits(vcpu, ~0x0eul) | (msw & 0x0f)); -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_lmsw); - void kvm_load_guest_xsave_state(struct kvm_vcpu *vcpu) { if (vcpu->arch.guest_state_protected) @@ -1322,89 +1190,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_emulate_xsetbv); -static bool kvm_is_valid_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - return __kvm_is_valid_cr4(vcpu, cr4) && - kvm_x86_call(is_valid_cr4)(vcpu, cr4); -} - -void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) -{ - if ((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) - kvm_mmu_reset_context(vcpu); - - /* - * If CR4.PCIDE is changed 0 -> 1, there is no need to flush the TLB - * according to the SDM; however, stale prev_roots could be reused - * incorrectly in the future after a MOV to CR3 with NOFLUSH=1, so we - * free them all. This is *not* a superset of KVM_REQ_TLB_FLUSH_GUEST - * or KVM_REQ_TLB_FLUSH_CURRENT, because the hardware TLB is not flushed, - * so fall through. - */ - if (!tdp_enabled && - (cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) - kvm_mmu_unload(vcpu); - - /* - * The TLB has to be flushed for all PCIDs if any of the following - * (architecturally required) changes happen: - * - CR4.PCIDE is changed from 1 to 0 - * - CR4.PGE is toggled - * - * This is a superset of KVM_REQ_TLB_FLUSH_CURRENT. - */ - if (((cr4 ^ old_cr4) & X86_CR4_PGE) || - (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - - /* - * The TLB has to be flushed for the current PCID if any of the - * following (architecturally required) changes happen: - * - CR4.SMEP is changed from 0 to 1 - * - CR4.PAE is toggled - */ - else if (((cr4 ^ old_cr4) & X86_CR4_PAE) || - ((cr4 & X86_CR4_SMEP) && !(old_cr4 & X86_CR4_SMEP))) - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_post_set_cr4); - -int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) -{ - unsigned long old_cr4 = kvm_read_cr4(vcpu); - - if (!kvm_is_valid_cr4(vcpu, cr4)) - return 1; - - if (is_long_mode(vcpu)) { - if (!(cr4 & X86_CR4_PAE)) - return 1; - if ((cr4 ^ old_cr4) & X86_CR4_LA57) - return 1; - } else if (is_paging(vcpu) && (cr4 & X86_CR4_PAE) - && ((cr4 ^ old_cr4) & X86_CR4_PDPTR_BITS) - && !load_pdptrs(vcpu, kvm_read_cr3(vcpu))) - return 1; - - if ((cr4 & X86_CR4_PCIDE) && !(old_cr4 & X86_CR4_PCIDE)) { - /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ - if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) - return 1; - } - - if ((cr4 & X86_CR4_CET) && !kvm_is_cr0_bit_set(vcpu, X86_CR0_WP)) - return 1; - - kvm_x86_call(set_cr4)(vcpu, cr4); - - kvm_post_set_cr4(vcpu, old_cr4, cr4); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr4); - -static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) { struct kvm_mmu *mmu = vcpu->arch.mmu; unsigned long roots_to_free = 0; @@ -1447,159 +1233,6 @@ static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) kvm_mmu_free_roots(vcpu->kvm, mmu, roots_to_free); } -int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) -{ - bool skip_tlb_flush = false; - unsigned long pcid = 0; -#ifdef CONFIG_X86_64 - if (kvm_is_cr4_bit_set(vcpu, X86_CR4_PCIDE)) { - skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; - cr3 &= ~X86_CR3_PCID_NOFLUSH; - pcid = cr3 & X86_CR3_PCID_MASK; - } -#endif - - /* PDPTRs are always reloaded for PAE paging. */ - if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) - goto handle_tlb_flush; - - /* - * Do not condition the GPA check on long mode, this helper is used to - * stuff CR3, e.g. for RSM emulation, and there is no guarantee that - * the current vCPU mode is accurate. - */ - if (!kvm_vcpu_is_legal_cr3(vcpu, cr3)) - return 1; - - if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, cr3)) - return 1; - - if (cr3 != kvm_read_cr3(vcpu)) - kvm_mmu_new_pgd(vcpu, cr3); - - vcpu->arch.cr3 = cr3; - kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); - /* Do not call post_set_cr3, we do not get here for confidential guests. */ - -handle_tlb_flush: - /* - * A load of CR3 that flushes the TLB flushes only the current PCID, - * even if PCID is disabled, in which case PCID=0 is flushed. It's a - * moot point in the end because _disabling_ PCID will flush all PCIDs, - * and it's impossible to use a non-zero PCID when PCID is disabled, - * i.e. only PCID=0 can be relevant. - */ - if (!skip_tlb_flush) - kvm_invalidate_pcid(vcpu, pcid); - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr3); - -int kvm_set_cr8(struct kvm_vcpu *vcpu, unsigned long cr8) -{ - if (cr8 & CR8_RESERVED_BITS) - return 1; - if (lapic_in_kernel(vcpu)) - kvm_lapic_set_tpr(vcpu, cr8); - else - vcpu->arch.cr8 = cr8; - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_cr8); - -unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) -{ - if (lapic_in_kernel(vcpu)) - return kvm_lapic_get_cr8(vcpu); - else - return vcpu->arch.cr8; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_cr8); - -static void kvm_update_dr0123(struct kvm_vcpu *vcpu) -{ - int i; - - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { - for (i = 0; i < KVM_NR_DB_REGS; i++) - vcpu->arch.eff_db[i] = vcpu->arch.db[i]; - } -} - -void kvm_update_dr7(struct kvm_vcpu *vcpu) -{ - unsigned long dr7; - - if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) - dr7 = vcpu->arch.guest_debug_dr7; - else - dr7 = vcpu->arch.dr7; - kvm_x86_call(set_dr7)(vcpu, dr7); - vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED; - if (dr7 & DR7_BP_EN_MASK) - vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_update_dr7); - -static u64 kvm_dr6_fixed(struct kvm_vcpu *vcpu) -{ - u64 fixed = DR6_FIXED_1; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_RTM)) - fixed |= DR6_RTM; - - if (!guest_cpu_cap_has(vcpu, X86_FEATURE_BUS_LOCK_DETECT)) - fixed |= DR6_BUS_LOCK; - return fixed; -} - -int kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - vcpu->arch.db[array_index_nospec(dr, size)] = val; - if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) - vcpu->arch.eff_db[dr] = val; - break; - case 4: - case 6: - if (!kvm_dr6_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr6 = (val & DR6_VOLATILE) | kvm_dr6_fixed(vcpu); - break; - case 5: - default: /* 7 */ - if (!kvm_dr7_valid(val)) - return 1; /* #GP */ - vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; - kvm_update_dr7(vcpu); - break; - } - - return 0; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_set_dr); - -unsigned long kvm_get_dr(struct kvm_vcpu *vcpu, int dr) -{ - size_t size = ARRAY_SIZE(vcpu->arch.db); - - switch (dr) { - case 0 ... 3: - return vcpu->arch.db[array_index_nospec(dr, size)]; - case 4: - case 6: - return vcpu->arch.dr6; - case 5: - default: /* 7 */ - return vcpu->arch.dr7; - } -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_get_dr); - int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu) { u32 pmc = kvm_rcx_read(vcpu); @@ -5570,7 +5203,7 @@ static struct kvm_queued_exception *kvm_get_exception_to_save(struct kvm_vcpu *v return &vcpu->arch.exception; } -static void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu) { struct kvm_queued_exception *ex = kvm_get_exception_to_save(vcpu); @@ -5774,56 +5407,6 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, return 0; } -static int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - kvm_handle_exception_payload_quirk(vcpu); - - memset(dbgregs, 0, sizeof(*dbgregs)); - - BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.db) != ARRAY_SIZE(dbgregs->db)); - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - dbgregs->db[i] = vcpu->arch.db[i]; - - dbgregs->dr6 = vcpu->arch.dr6; - dbgregs->dr7 = vcpu->arch.dr7; - return 0; -} - -static int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, - struct kvm_debugregs *dbgregs) -{ - unsigned int i; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - if (dbgregs->flags) - return -EINVAL; - - if (!kvm_dr6_valid(dbgregs->dr6)) - return -EINVAL; - if (!kvm_dr7_valid(dbgregs->dr7)) - return -EINVAL; - - for (i = 0; i < ARRAY_SIZE(vcpu->arch.db); i++) - vcpu->arch.db[i] = dbgregs->db[i]; - - kvm_update_dr0123(vcpu); - vcpu->arch.dr6 = dbgregs->dr6; - vcpu->arch.dr7 = dbgregs->dr7; - kvm_update_dr7(vcpu); - - return 0; -} - static int kvm_vcpu_ioctl_x86_get_xsave2(struct kvm_vcpu *vcpu, u8 *state, unsigned int size) @@ -6661,7 +6244,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, r = -ENOMEM; if (!u.sregs2) goto out; - __get_sregs2(vcpu, u.sregs2); + kvm_vcpu_ioctl_x86_get_sregs2(vcpu, u.sregs2); r = -EFAULT; if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) goto out; @@ -6680,7 +6263,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u.sregs2 = NULL; goto out; } - r = __set_sregs2(vcpu, u.sregs2); + r = kvm_vcpu_ioctl_x86_set_sregs2(vcpu, u.sregs2); break; } case KVM_HAS_DEVICE_ATTR: @@ -6774,7 +6357,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break; fallthrough; case KVM_CAP_DISABLE_QUIRKS: - kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + mutex_lock(&kvm->lock); + WRITE_ONCE(kvm->arch.disabled_quirks, + kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); + mutex_unlock(&kvm->lock); r = 0; break; case KVM_CAP_SPLIT_IRQCHIP: { @@ -10154,6 +9740,9 @@ int kvm_x86_vendor_init(struct kvm_x86_init_ops *ops) if (r != 0) goto out_mmu_exit; + if (kvm_cpu_cap_has(X86_FEATURE_LM)) + kvm_enable_efer_bits(EFER_LME | EFER_LMA); + enable_device_posted_irqs &= enable_apicv && irq_remapping_cap(IRQ_POSTING_CAP); @@ -12017,179 +11606,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) return r; } -static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->arch.emulate_regs_need_sync_to_vcpu) { - /* - * We are here if userspace calls get_regs() in the middle of - * instruction emulation. Registers state needs to be copied - * back from emulation context to vcpu. Userspace shouldn't do - * that usually, but some bad designed PV devices (vmware - * backdoor interface) need this to work - */ - emulator_writeback_register_cache(vcpu->arch.emulate_ctxt); - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - } - regs->rax = kvm_rax_read(vcpu); - regs->rbx = kvm_rbx_read(vcpu); - regs->rcx = kvm_rcx_read(vcpu); - regs->rdx = kvm_rdx_read(vcpu); - regs->rsi = kvm_rsi_read(vcpu); - regs->rdi = kvm_rdi_read(vcpu); - regs->rsp = kvm_rsp_read(vcpu); - regs->rbp = kvm_rbp_read(vcpu); -#ifdef CONFIG_X86_64 - regs->r8 = kvm_r8_read(vcpu); - regs->r9 = kvm_r9_read(vcpu); - regs->r10 = kvm_r10_read(vcpu); - regs->r11 = kvm_r11_read(vcpu); - regs->r12 = kvm_r12_read(vcpu); - regs->r13 = kvm_r13_read(vcpu); - regs->r14 = kvm_r14_read(vcpu); - regs->r15 = kvm_r15_read(vcpu); -#endif - - regs->rip = kvm_rip_read(vcpu); - regs->rflags = kvm_get_rflags(vcpu); -} - -int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - vcpu->arch.emulate_regs_need_sync_from_vcpu = true; - vcpu->arch.emulate_regs_need_sync_to_vcpu = false; - - kvm_rax_write(vcpu, regs->rax); - kvm_rbx_write(vcpu, regs->rbx); - kvm_rcx_write(vcpu, regs->rcx); - kvm_rdx_write(vcpu, regs->rdx); - kvm_rsi_write(vcpu, regs->rsi); - kvm_rdi_write(vcpu, regs->rdi); - kvm_rsp_write(vcpu, regs->rsp); - kvm_rbp_write(vcpu, regs->rbp); -#ifdef CONFIG_X86_64 - kvm_r8_write(vcpu, regs->r8); - kvm_r9_write(vcpu, regs->r9); - kvm_r10_write(vcpu, regs->r10); - kvm_r11_write(vcpu, regs->r11); - kvm_r12_write(vcpu, regs->r12); - kvm_r13_write(vcpu, regs->r13); - kvm_r14_write(vcpu, regs->r14); - kvm_r15_write(vcpu, regs->r15); -#endif - - kvm_rip_write(vcpu, regs->rip); - kvm_set_rflags(vcpu, regs->rflags | X86_EFLAGS_FIXED); - - vcpu->arch.exception.pending = false; - vcpu->arch.exception_vmexit.pending = false; - - kvm_make_request(KVM_REQ_EVENT, vcpu); -} - -int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __set_regs(vcpu, regs); - vcpu_put(vcpu); - return 0; -} - -static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - struct desc_ptr dt; - - if (vcpu->arch.guest_state_protected) - goto skip_protected_regs; - - kvm_handle_exception_payload_quirk(vcpu); - - kvm_get_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_get_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_get_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_get_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_get_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_get_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_get_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_get_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_x86_call(get_idt)(vcpu, &dt); - sregs->idt.limit = dt.size; - sregs->idt.base = dt.address; - kvm_x86_call(get_gdt)(vcpu, &dt); - sregs->gdt.limit = dt.size; - sregs->gdt.base = dt.address; - - sregs->cr2 = vcpu->arch.cr2; - sregs->cr3 = kvm_read_cr3(vcpu); - -skip_protected_regs: - sregs->cr0 = kvm_read_cr0(vcpu); - sregs->cr4 = kvm_read_cr4(vcpu); - sregs->cr8 = kvm_get_cr8(vcpu); - sregs->efer = vcpu->arch.efer; - sregs->apic_base = vcpu->arch.apic_base; -} - -static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - __get_sregs_common(vcpu, sregs); - - if (vcpu->arch.guest_state_protected) - return; - - if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) - set_bit(vcpu->arch.interrupt.nr, - (unsigned long *)sregs->interrupt_bitmap); -} - -static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int i; - - __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); - - if (vcpu->arch.guest_state_protected) - return; - - if (is_pae_paging(vcpu)) { - kvm_vcpu_srcu_read_lock(vcpu); - for (i = 0 ; i < 4 ; i++) - sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); - sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; - kvm_vcpu_srcu_read_unlock(vcpu); - } -} - -int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - __get_sregs(vcpu, sregs); - vcpu_put(vcpu); - return 0; -} - int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { @@ -12309,173 +11725,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_task_switch); -static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - if ((sregs->efer & EFER_LME) && (sregs->cr0 & X86_CR0_PG)) { - /* - * When EFER.LME and CR0.PG are set, the processor is in - * 64-bit mode (though maybe in a 32-bit code segment). - * CR4.PAE and EFER.LMA must be set. - */ - if (!(sregs->cr4 & X86_CR4_PAE) || !(sregs->efer & EFER_LMA)) - return false; - if (!kvm_vcpu_is_legal_cr3(vcpu, sregs->cr3)) - return false; - } else { - /* - * Not in 64-bit mode: EFER.LMA is clear and the code - * segment cannot be 64-bit. - */ - if (sregs->efer & EFER_LMA || sregs->cs.l) - return false; - } - - return kvm_is_valid_cr4(vcpu, sregs->cr4) && - kvm_is_valid_cr0(vcpu, sregs->cr0); -} - -static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, - int *mmu_reset_needed, bool update_pdptrs) -{ - int idx; - struct desc_ptr dt; - - if (!kvm_is_valid_sregs(vcpu, sregs)) - return -EINVAL; - - if (kvm_apic_set_base(vcpu, sregs->apic_base, true)) - return -EINVAL; - - if (vcpu->arch.guest_state_protected) - return 0; - - dt.size = sregs->idt.limit; - dt.address = sregs->idt.base; - kvm_x86_call(set_idt)(vcpu, &dt); - dt.size = sregs->gdt.limit; - dt.address = sregs->gdt.base; - kvm_x86_call(set_gdt)(vcpu, &dt); - - vcpu->arch.cr2 = sregs->cr2; - *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; - vcpu->arch.cr3 = sregs->cr3; - kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3); - kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3); - - *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; - kvm_x86_call(set_efer)(vcpu, sregs->efer); - - *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; - kvm_x86_call(set_cr0)(vcpu, sregs->cr0); - - *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; - kvm_x86_call(set_cr4)(vcpu, sregs->cr4); - - if (update_pdptrs) { - idx = srcu_read_lock(&vcpu->kvm->srcu); - if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, kvm_read_cr3(vcpu)); - *mmu_reset_needed = 1; - } - srcu_read_unlock(&vcpu->kvm->srcu, idx); - } - - kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); - kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); - kvm_set_segment(vcpu, &sregs->es, VCPU_SREG_ES); - kvm_set_segment(vcpu, &sregs->fs, VCPU_SREG_FS); - kvm_set_segment(vcpu, &sregs->gs, VCPU_SREG_GS); - kvm_set_segment(vcpu, &sregs->ss, VCPU_SREG_SS); - - kvm_set_segment(vcpu, &sregs->tr, VCPU_SREG_TR); - kvm_set_segment(vcpu, &sregs->ldt, VCPU_SREG_LDTR); - - kvm_set_cr8(vcpu, sregs->cr8); - - /* Older userspace won't unhalt the vcpu on reset. */ - if (kvm_vcpu_is_bsp(vcpu) && kvm_rip_read(vcpu) == 0xfff0 && - sregs->cs.selector == 0xf000 && sregs->cs.base == 0xffff0000 && - !is_protmode(vcpu)) - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - - return 0; -} - -static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) -{ - int pending_vec, max_bits; - int mmu_reset_needed = 0; - int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); - - if (ret) - return ret; - - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - - max_bits = KVM_NR_INTERRUPTS; - pending_vec = find_first_bit( - (const unsigned long *)sregs->interrupt_bitmap, max_bits); - - if (pending_vec < max_bits) { - kvm_queue_interrupt(vcpu, pending_vec, false); - pr_debug("Set back pending irq %d\n", pending_vec); - kvm_make_request(KVM_REQ_EVENT, vcpu); - } - return 0; -} - -static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) -{ - int mmu_reset_needed = 0; - bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; - bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && - !(sregs2->efer & EFER_LMA); - int i, ret; - - if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) - return -EINVAL; - - if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) - return -EINVAL; - - ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, - &mmu_reset_needed, !valid_pdptrs); - if (ret) - return ret; - - if (valid_pdptrs) { - for (i = 0; i < 4 ; i++) - kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); - - kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); - mmu_reset_needed = 1; - vcpu->arch.pdptrs_from_userspace = true; - } - if (mmu_reset_needed) { - kvm_mmu_reset_context(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); - } - return 0; -} - -int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, - struct kvm_sregs *sregs) -{ - int ret; - - if (vcpu->kvm->arch.has_protected_state && - vcpu->arch.guest_state_protected) - return -EINVAL; - - vcpu_load(vcpu); - ret = __set_sregs(vcpu, sregs); - vcpu_put(vcpu); - return ret; -} - static void kvm_arch_vcpu_guestdbg_update_apicv_inhibit(struct kvm *kvm) { bool set = false; @@ -12631,13 +11880,7 @@ int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) static void store_regs(struct kvm_vcpu *vcpu) { - BUILD_BUG_ON(sizeof(struct kvm_sync_regs) > SYNC_REGS_SIZE_BYTES); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_REGS) - __get_regs(vcpu, &vcpu->run->s.regs.regs); - - if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_SREGS) - __get_sregs(vcpu, &vcpu->run->s.regs.sregs); + kvm_run_sync_regs_to_user(vcpu); if (vcpu->run->kvm_valid_regs & KVM_SYNC_X86_EVENTS) kvm_vcpu_ioctl_x86_get_vcpu_events( @@ -12646,19 +11889,8 @@ static void store_regs(struct kvm_vcpu *vcpu) static int sync_regs(struct kvm_vcpu *vcpu) { - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_REGS) { - __set_regs(vcpu, &vcpu->run->s.regs.regs); - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_REGS; - } - - if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_SREGS) { - struct kvm_sregs sregs = vcpu->run->s.regs.sregs; - - if (__set_sregs(vcpu, &sregs)) - return -EINVAL; - - vcpu->run->kvm_dirty_regs &= ~KVM_SYNC_X86_SREGS; - } + if (kvm_run_sync_regs_from_user(vcpu)) + return -EINVAL; if (vcpu->run->kvm_dirty_regs & KVM_SYNC_X86_EVENTS) { struct kvm_vcpu_events events = vcpu->run->s.regs.events; @@ -13320,12 +12552,18 @@ void kvm_arch_pre_destroy_vm(struct kvm *kvm) * iterating over vCPUs in a different task while vCPUs are being freed * is unsafe, i.e. will lead to use-after-free. The PIT also needs to * be stopped before IRQ routing is freed. + * + * Do NOT free the in-kernel PIC or I/O APIC here (but as above, make + * sure to flush any background work), as KVM expects interrupt routing + * structures to be valid until vCPUs are destroyed. */ cancel_delayed_work_sync(&kvm->arch.kvmclock_sync_work); cancel_delayed_work_sync(&kvm->arch.kvmclock_update_work); #ifdef CONFIG_KVM_IOAPIC kvm_free_pit(kvm); + if (kvm->arch.vioapic) + cancel_delayed_work_sync(&kvm->arch.vioapic->eoi_inject); #endif kvm_mmu_pre_destroy_vm(kvm); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index c8a561c17e9ad..f4cd73a074903 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -54,6 +54,20 @@ struct kvm_host_values { u64 arch_capabilities; }; +void kvm_vcpu_ioctl_x86_get_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); +int kvm_vcpu_ioctl_x86_set_sregs2(struct kvm_vcpu *vcpu, + struct kvm_sregs2 *sregs2); + +void kvm_run_sync_regs_to_user(struct kvm_vcpu *vcpu); +int kvm_run_sync_regs_from_user(struct kvm_vcpu *vcpu); + +void kvm_update_dr0123(struct kvm_vcpu *vcpu); +int kvm_vcpu_ioctl_x86_get_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); +int kvm_vcpu_ioctl_x86_set_debugregs(struct kvm_vcpu *vcpu, + struct kvm_debugregs *dbgregs); + void kvm_spurious_fault(void); #define SIZE_OF_MEMSLOTS_HASHTABLE \ @@ -417,7 +431,7 @@ static inline void kvm_register_write(struct kvm_vcpu *vcpu, static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) { - return !(kvm->arch.disabled_quirks & quirk); + return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk); } void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip); @@ -439,6 +453,7 @@ int handle_ud(struct kvm_vcpu *vcpu); void kvm_deliver_exception_payload(struct kvm_vcpu *vcpu, struct kvm_queued_exception *ex); +void kvm_handle_exception_payload_quirk(struct kvm_vcpu *vcpu); int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata); @@ -629,6 +644,7 @@ void kvm_load_host_xsave_state(struct kvm_vcpu *vcpu); int kvm_spec_ctrl_test_value(u64 value); int kvm_handle_memory_failure(struct kvm_vcpu *vcpu, int r, struct x86_exception *e); +void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid); int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva); bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type); diff --git a/arch/x86/lib/retpoline.S b/arch/x86/lib/retpoline.S index 8f1fed0c3b83f..f9ca1d8eabca0 100644 --- a/arch/x86/lib/retpoline.S +++ b/arch/x86/lib/retpoline.S @@ -207,10 +207,24 @@ __EXPORT_THUNK(srso_alias_untrain_ret) .pushsection .text..__x86.rethunk_safe SYM_CODE_START_NOALIGN(srso_alias_safe_ret) + + /* + * Tell objtool that those are not function pointers referenced by + * __HANDLE_INTR_SAFERET(). Below too. + */ + ANNOTATE_NOENDBR + + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP UNWIND_HINT_FUNC + + ANNOTATE_NOENDBR ANNOTATE_UNRET_SAFE ret + /* End of Safe-RET sequence */ int3 SYM_FUNC_END(srso_alias_safe_ret) @@ -245,8 +259,14 @@ SYM_CODE_START_LOCAL_NOALIGN(srso_untrain_ret) * the stack. */ SYM_INNER_LABEL(srso_safe_ret, SYM_L_GLOBAL) + /* + * Safe-RET sequence. If you need to change it, adjust + * handle_interrupted_saferet() too. + */ lea 8(%_ASM_SP), %_ASM_SP ret + /* End of Safe-RET sequence */ + int3 int3 /* end of movabs */ diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c index fffb6ef1997d2..62e7c4ee06f9d 100644 --- a/arch/x86/mm/pat/set_memory.c +++ b/arch/x86/mm/pat/set_memory.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -49,7 +50,8 @@ struct cpa_data { unsigned int flags; unsigned int force_split : 1, force_static_prot : 1, - force_flush_all : 1; + force_flush_all : 1, + init_mm_read_locked : 1; struct page **pages; }; @@ -62,18 +64,18 @@ enum cpa_warn { static const int cpa_warn_level = CPA_PROTECT; /* - * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings) - * using cpa_lock. So that we don't allow any other cpu, with stale large tlb - * entries change the page attribute in parallel to some other cpu - * splitting a large page entry along with changing the attribute. + * Serialize cpa() using cpa_lock so that we don't allow any other cpu, with + * stale large tlb entries, to change the page attribute in parallel to some + * other cpu splitting a large page entry along with changing the attribute. */ static DEFINE_SPINLOCK(cpa_lock); -#define CPA_FLUSHTLB 1 -#define CPA_ARRAY 2 -#define CPA_PAGES_ARRAY 4 -#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */ -#define CPA_COLLAPSE 16 /* try to collapse large pages */ +#define CPA_FLUSHTLB 0x01 +#define CPA_ARRAY 0x02 +#define CPA_PAGES_ARRAY 0x04 +#define CPA_NO_CHECK_ALIAS 0x08 /* Do not search for aliases */ +#define CPA_COLLAPSE 0x10 /* try to collapse large pages */ +#define CPA_DEBUG_PAGEALLOC 0x20 static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) { @@ -401,7 +403,7 @@ static void cpa_flush_all(unsigned long cache) static int collapse_large_pages(unsigned long addr, struct list_head *pgtables); -static void cpa_collapse_large_pages(struct cpa_data *cpa) +static void __cpa_collapse_large_pages(struct cpa_data *cpa) { unsigned long start, addr, end; struct ptdesc *ptdesc, *tmp; @@ -409,6 +411,8 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) int collapsed = 0; int i; + spin_lock(&cpa_lock); + if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { for (i = 0; i < cpa->numpages; i++) collapsed += collapse_large_pages(__cpa_addr(cpa, i), @@ -422,15 +426,39 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa) collapsed += collapse_large_pages(addr, &pgtables); } - if (!collapsed) + if (!collapsed) { + spin_unlock(&cpa_lock); return; + } flush_tlb_all(); list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) { list_del(&ptdesc->pt_list); - pagetable_free(ptdesc); + /* + * Only early alloc'd direct map should not be flagged PG_table + * here and those shouldn't be collapsed. However be abundantly + * cautious and handle the !PG_table case too. + */ + if (PageTable((ptdesc_page(ptdesc)))) + pagetable_dtor_free(ptdesc); + else + pagetable_free(ptdesc); } + + spin_unlock(&cpa_lock); +} + +static void cpa_collapse_large_pages(struct cpa_data *cpa) +{ + /* + * Take the mmap write lock on init_mm to: + * - Avoid a use-after-free if raced by ptdump (which takes its own + * write lock on init_mm). + * - Serialise concurrent CPA walkers. + */ + scoped_guard(mmap_write_lock, &init_mm) + __cpa_collapse_large_pages(cpa); } static void cpa_flush(struct cpa_data *cpa, int cache) @@ -1119,10 +1147,10 @@ static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, static int __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, - struct page *base) + pte_t *pbase) { unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1; - pte_t *pbase = (pte_t *)page_address(base); + struct page *base = virt_to_page(pbase); unsigned int i, level; pgprot_t ref_prot; bool nx, rw; @@ -1226,18 +1254,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, static int split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address) { - struct page *base; + pte_t *pte; - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); - base = alloc_pages(GFP_KERNEL, 0); - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - if (!base) + spin_unlock(&cpa_lock); + if (cpa->init_mm_read_locked) + mmap_read_unlock(&init_mm); + pte = pte_alloc_one_kernel(&init_mm); + if (cpa->init_mm_read_locked) + mmap_read_lock(&init_mm); + spin_lock(&cpa_lock); + if (!pte) return -ENOMEM; - if (__split_large_page(cpa, kpte, address, base)) - __free_page(base); + if (__split_large_page(cpa, kpte, address, pte)) + pte_free_kernel(&init_mm, pte); return 0; } @@ -1985,6 +2015,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) { unsigned long numpages = cpa->numpages; unsigned long rempages = numpages; + bool lock = true; int ret = 0; /* @@ -1994,6 +2025,29 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) !cpa->force_split) return ret; + /* + * DEBUG_PAGEALLOC is special; it is called from any context the + * page-allocator is, which violates the normal cpa_lock locking + * rules. + * + * However, since it is part of the page-allocator, things are still + * properly serialized by the page-allocator locking and the fact that + * when a page is owned by the page-allocator, it isn't owned by + * anybody else. That is, you *SHOULD NOT* be calling cpa() on memory + * that isn't allocated. + * + * Additionally, DEBUG_PAGEALLOC ensures (per probe_page_size_mask()) + * that the kernel mapping is 4k pages, therefore there are no large + * pages to split/collapse. + * + * Furthermore, the page-allocator strictly manages pages that + * *exist*, avoiding pgd_lock. + * + * Therefore, it is safe to not take cpa_lock. + */ + if (debug_pagealloc_enabled() && (cpa->flags & CPA_DEBUG_PAGEALLOC)) + lock = false; + while (rempages) { /* * Store the remaining nr of pages for the large page @@ -2004,11 +2058,12 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) cpa->numpages = 1; - if (!debug_pagealloc_enabled()) - spin_lock(&cpa_lock); - ret = __change_page_attr(cpa, primary); - if (!debug_pagealloc_enabled()) - spin_unlock(&cpa_lock); + if (lock) { + guard(spinlock)(&cpa_lock); + ret = __change_page_attr(cpa, primary); + } else { + ret = __change_page_attr(cpa, primary); + } if (ret) goto out; @@ -2090,7 +2145,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, cpa.curpage = 0; cpa.force_split = force_split; - ret = __change_page_attr_set_clr(&cpa, 1); + /* Avoid race with concurrent CPA collapse. */ + cpa.init_mm_read_locked = true; + scoped_guard(mmap_read_lock, &init_mm) + ret = __change_page_attr_set_clr(&cpa, 1); + cpa.init_mm_read_locked = false; /* * Check whether we really changed something: @@ -2600,7 +2659,7 @@ int set_pages_rw(struct page *page, int numpages) return set_memory_rw(addr, numpages); } -static int __set_pages_p(struct page *page, int numpages) +static int __set_pages_p(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2608,7 +2667,7 @@ static int __set_pages_p(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW), .mask_clr = __pgprot(0), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting present flag. otherwise, @@ -2619,7 +2678,7 @@ static int __set_pages_p(struct page *page, int numpages) return __change_page_attr_set_clr(&cpa, 1); } -static int __set_pages_np(struct page *page, int numpages) +static int __set_pages_np(struct page *page, int numpages, unsigned int cpa_flags) { unsigned long tempaddr = (unsigned long) page_address(page); struct cpa_data cpa = { .vaddr = &tempaddr, @@ -2627,7 +2686,7 @@ static int __set_pages_np(struct page *page, int numpages) .numpages = numpages, .mask_set = __pgprot(0), .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW | _PAGE_DIRTY), - .flags = CPA_NO_CHECK_ALIAS }; + .flags = CPA_NO_CHECK_ALIAS | cpa_flags }; /* * No alias checking needed for setting not present flag. otherwise, @@ -2640,20 +2699,20 @@ static int __set_pages_np(struct page *page, int numpages) int set_direct_map_invalid_noflush(struct page *page) { - return __set_pages_np(page, 1); + return __set_pages_np(page, 1, 0); } int set_direct_map_default_noflush(struct page *page) { - return __set_pages_p(page, 1); + return __set_pages_p(page, 1, 0); } int set_direct_map_valid_noflush(struct page *page, unsigned nr, bool valid) { if (valid) - return __set_pages_p(page, nr); + return __set_pages_p(page, nr, 0); - return __set_pages_np(page, nr); + return __set_pages_np(page, nr, 0); } #ifdef CONFIG_DEBUG_PAGEALLOC @@ -2672,15 +2731,23 @@ void __kernel_map_pages(struct page *page, int numpages, int enable) * and hence no memory allocations during large page split. */ if (enable) - __set_pages_p(page, numpages); + __set_pages_p(page, numpages, CPA_DEBUG_PAGEALLOC); else - __set_pages_np(page, numpages); + __set_pages_np(page, numpages, CPA_DEBUG_PAGEALLOC); /* - * We should perform an IPI and flush all tlbs, - * but that can deadlock->flush only current cpu. - * Preemption needs to be disabled around __flush_tlb_all() due to - * CR3 reload in __native_flush_tlb(). + * We should perform an IPI and flush all tlbs, but that can + * deadlock, settle for a local flush. + * + * Not doing a global TLB flush means that remote CPUs will retain + * stale TLB entries. In case of P->NP (on free) this means the remote + * CPUs will not take the faults, making the debug scheme less + * reliable. On the NP->P (on alloc) this means the remote CPUs can + * take a spurious fault. However spurious_kernel_fault() will observe + * *_present() and fix it up. + * + * Preemption needs to be disabled around __flush_tlb_all() due to CR3 + * reload in __native_flush_tlb(). */ preempt_disable(); __flush_tlb_all(); diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index c3798cab3b7db..39cf6974e11e1 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1756,7 +1756,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image EMIT_mov(dst_reg, src_reg); #ifdef CONFIG_SMP /* add , gs:[] */ - EMIT2(0x65, add_1mod(0x48, dst_reg)); + EMIT2(0x65, add_2mod(0x48, 0, dst_reg)); EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25); EMIT((u32)(unsigned long)&this_cpu_off, 4); #endif @@ -2175,8 +2175,13 @@ st: if (is_imm8(insn->off)) * BPF_PROBE_ATOMIC) before being used for the memory access. Pass * the reg holding the unmodified 32-bit address to * ex_handler_bpf(). + * + * A load-acquire is of BPF_STX class, but reads from src_reg + * into dst_reg like a BPF_LDX does, hence it must not be + * treated as a store here. */ - if (BPF_CLASS(insn->code) == BPF_LDX) { + if (BPF_CLASS(insn->code) == BPF_LDX || + bpf_atomic_is_load_acq(insn)) { arena_reg = reg2pt_regs[src_reg]; fixup_reg = reg2pt_regs[dst_reg]; } else { @@ -3187,11 +3192,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im WARN_ON_ONCE((flags & BPF_TRAMP_F_INDIRECT) && (flags & ~(BPF_TRAMP_F_INDIRECT | BPF_TRAMP_F_RET_FENTRY_RET))); - /* extra registers for struct arguments */ - for (i = 0; i < m->nr_args; i++) { - if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) - nr_regs += (m->arg_size[i] + 7) / 8 - 1; - } + for (i = 0; i < m->nr_args; i++) + nr_regs += (m->arg_size[i] + 7) / 8 - 1; /* x86-64 supports up to MAX_BPF_FUNC_ARGS arguments. 1-6 * are passed through regs, the remains are through stack. diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 791c52c8393f4..5b00b13f72d4a 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -921,7 +921,7 @@ umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n) return attr->mode; } -enum efi_secureboot_mode __x86_ima_efi_boot_mode(void) +enum efi_secureboot_mode __x86_efi_boot_mode(void) { return boot_params.secure_boot; } diff --git a/block/bio.c b/block/bio.c index b184f0a3fcb2b..63994b8cd8e1f 100644 --- a/block/bio.c +++ b/block/bio.c @@ -253,6 +253,7 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, bio->bi_write_hint = 0; bio->bi_write_stream = 0; bio->bi_status = 0; + bio->bi_bvec_gap_bit = 0; bio->bi_iter.bi_sector = 0; bio->bi_iter.bi_size = 0; bio->bi_iter.bi_idx = 0; @@ -833,6 +834,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; + bio->bi_bvec_gap_bit = bio_src->bi_bvec_gap_bit; bio->bi_iter = bio_src->bi_iter; if (bio->bi_bdev) { @@ -1716,6 +1718,14 @@ struct bio *bio_split(struct bio *bio, int sectors, bio_advance(bio, split->bi_iter.bi_size); + /* + * The gap bit is set when splitting to limits and only applies to the + * front bio that was split off. The remaining bio will calcualte its + * gap value when it is subsequently split to limits, so it is safe to + * re-initialize the value back to 0. + */ + bio->bi_bvec_gap_bit = 0; + if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION); diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index a2347b5795874..171efbbd3c8a8 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -432,15 +432,15 @@ static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk, blkg->pd[i]->online = true; } } + blkg->online = true; } - blkg->online = true; spin_unlock(&blkcg->lock); if (!ret) return blkg; /* @blkg failed fully initialized, use the usual release path */ - blkg_put(blkg); + percpu_ref_kill(&blkg->refcnt); return ERR_PTR(ret); err_put_css: @@ -1241,13 +1241,10 @@ static int blkcg_print_stat(struct seq_file *sf, void *v) else css_rstat_flush(&blkcg->css); - rcu_read_lock(); - hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) { - spin_lock_irq(&blkg->q->queue_lock); + guard(spinlock_irq)(&blkcg->lock); + hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) blkcg_print_one_stat(blkg, sf); - spin_unlock_irq(&blkg->q->queue_lock); - } - rcu_read_unlock(); + return 0; } @@ -1612,6 +1609,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); + + mutex_lock(&q->blkcg_mutex); retry: spin_lock_irq(&q->queue_lock); @@ -1621,6 +1620,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (blkg->pd[pol->plid]) continue; + if (hlist_unhashed(&blkg->blkcg_node)) + continue; /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ if (blkg == pinned_blkg) { @@ -1674,6 +1675,7 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) spin_unlock_irq(&q->queue_lock); out: + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); if (pinned_blkg) diff --git a/block/blk-core.c b/block/blk-core.c index 14ae73eebe0d7..c2300b201d4ff 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -628,7 +628,7 @@ static void __submit_bio(struct bio *bio) /* If plug is not used, add new plug here to cache nsecs time. */ struct blk_plug plug; - if (unlikely(!blk_crypto_bio_prep(&bio))) + if (unlikely(!blk_crypto_bio_prep(bio))) return; blk_start_plug(&plug); @@ -639,12 +639,10 @@ static void __submit_bio(struct bio *bio) struct gendisk *disk = bio->bi_bdev->bd_disk; if ((bio->bi_opf & REQ_POLLED) && - !(disk->queue->limits.features & BLK_FEAT_POLL)) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - } else { + !(disk->queue->limits.features & BLK_FEAT_POLL)) + bio_endio_status(bio, BLK_STS_NOTSUPP); + else disk->fops->submit_bio(bio); - } blk_queue_exit(disk->queue); } @@ -882,8 +880,7 @@ void submit_bio_noacct(struct bio *bio) not_supported: status = BLK_STS_NOTSUPP; end_io: - bio->bi_status = status; - bio_endio(bio); + bio_endio_status(bio, status); } EXPORT_SYMBOL(submit_bio_noacct); diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 86b27f96051ae..f523b0cf143af 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -75,13 +75,13 @@ static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; static struct blk_crypto_fallback_keyslot { enum blk_crypto_mode_num crypto_mode; - struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; + struct crypto_sync_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; } *blk_crypto_keyslots; static struct blk_crypto_profile *blk_crypto_fallback_profile; static struct workqueue_struct *blk_crypto_wq; static mempool_t *blk_crypto_bounce_page_pool; -static struct bio_set crypto_bio_split; +static struct bio_set enc_bio_set; /* * This is the key we set when evicting a keyslot. This *should* be the all 0's @@ -98,7 +98,7 @@ static void blk_crypto_fallback_evict_keyslot(unsigned int slot) WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, blk_crypto_modes[crypto_mode].keysize); WARN_ON(err); slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; @@ -119,7 +119,7 @@ blk_crypto_fallback_keyslot_program(struct blk_crypto_profile *profile, blk_crypto_fallback_evict_keyslot(slot); slotp->crypto_mode = crypto_mode; - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, + err = crypto_sync_skcipher_setkey(slotp->tfms[crypto_mode], key->bytes, key->size); if (err) { blk_crypto_fallback_evict_keyslot(slot); @@ -150,88 +150,39 @@ static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) mempool_free(enc_bio->bi_io_vec[i].bv_page, blk_crypto_bounce_page_pool); - src_bio->bi_status = enc_bio->bi_status; + if (enc_bio->bi_status) + cmpxchg(&src_bio->bi_status, 0, enc_bio->bi_status); - bio_uninit(enc_bio); - kfree(enc_bio); + bio_put(enc_bio); bio_endio(src_bio); } -static struct bio *blk_crypto_fallback_clone_bio(struct bio *bio_src) +static struct bio *blk_crypto_alloc_enc_bio(struct bio *bio_src, + unsigned int nr_segs) { - unsigned int nr_segs = bio_segments(bio_src); - struct bvec_iter iter; - struct bio_vec bv; struct bio *bio; - bio = bio_kmalloc(nr_segs, GFP_NOIO); - if (!bio) - return NULL; - bio_init_inline(bio, bio_src->bi_bdev, nr_segs, bio_src->bi_opf); + bio = bio_alloc_bioset(bio_src->bi_bdev, nr_segs, bio_src->bi_opf, + GFP_NOIO, &enc_bio_set); if (bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); + bio->bi_private = bio_src; + bio->bi_end_io = blk_crypto_fallback_encrypt_endio; bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; - bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; - - bio_for_each_segment(bv, bio_src, iter) - bio->bi_io_vec[bio->bi_vcnt++] = bv; - bio_clone_blkg_association(bio, bio_src); - return bio; } -static bool -blk_crypto_fallback_alloc_cipher_req(struct blk_crypto_keyslot *slot, - struct skcipher_request **ciph_req_ret, - struct crypto_wait *wait) +static struct crypto_sync_skcipher * +blk_crypto_fallback_tfm(struct blk_crypto_keyslot *slot) { - struct skcipher_request *ciph_req; - const struct blk_crypto_fallback_keyslot *slotp; - int keyslot_idx = blk_crypto_keyslot_index(slot); - - slotp = &blk_crypto_keyslots[keyslot_idx]; - ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], - GFP_NOIO); - if (!ciph_req) - return false; - - skcipher_request_set_callback(ciph_req, - CRYPTO_TFM_REQ_MAY_BACKLOG | - CRYPTO_TFM_REQ_MAY_SLEEP, - crypto_req_done, wait); - *ciph_req_ret = ciph_req; + const struct blk_crypto_fallback_keyslot *slotp = + &blk_crypto_keyslots[blk_crypto_keyslot_index(slot)]; - return true; -} - -static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) -{ - struct bio *bio = *bio_ptr; - unsigned int i = 0; - unsigned int num_sectors = 0; - struct bio_vec bv; - struct bvec_iter iter; - - bio_for_each_segment(bv, bio, iter) { - num_sectors += bv.bv_len >> SECTOR_SHIFT; - if (++i == BIO_MAX_VECS) - break; - } - - if (num_sectors < bio_sectors(bio)) { - bio = bio_submit_split_bioset(bio, num_sectors, - &crypto_bio_split); - if (!bio) - return false; - - *bio_ptr = bio; - } - - return true; + return slotp->tfms[slotp->crypto_mode]; } union blk_crypto_iv { @@ -248,59 +199,22 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], iv->dun[i] = cpu_to_le64(dun[i]); } -/* - * The crypto API fallback's encryption routine. - * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, - * and replace *bio_ptr with the bounce bio. May split input bio if it's too - * large. Returns true on success. Returns false and sets bio->bi_status on - * error. - */ -static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) +static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio, + struct crypto_sync_skcipher *tfm) { - struct bio *src_bio, *enc_bio; - struct bio_crypt_ctx *bc; - struct blk_crypto_keyslot *slot; - int data_unit_size; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; + int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; struct scatterlist src, dst; union blk_crypto_iv iv; - unsigned int i, j; - bool ret = false; - blk_status_t blk_st; - - /* Split the bio if it's too big for single page bvec */ - if (!blk_crypto_fallback_split_bio_if_needed(bio_ptr)) - return false; - - src_bio = *bio_ptr; - bc = src_bio->bi_crypt_context; - data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; - - /* Allocate bounce bio for encryption */ - enc_bio = blk_crypto_fallback_clone_bio(src_bio); - if (!enc_bio) { - src_bio->bi_status = BLK_STS_RESOURCE; - return false; - } - - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - src_bio->bi_status = blk_st; - goto out_put_enc_bio; - } + unsigned int nr_enc_pages, enc_idx; + struct bio *enc_bio; + unsigned int i; - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_release_keyslot; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&src, 1); @@ -309,101 +223,111 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, iv.bytes); - /* Encrypt each page in the bounce bio */ - for (i = 0; i < enc_bio->bi_vcnt; i++) { - struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; - struct page *plaintext_page = enc_bvec->bv_page; - struct page *ciphertext_page = - mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); - - enc_bvec->bv_page = ciphertext_page; - - if (!ciphertext_page) { - src_bio->bi_status = BLK_STS_RESOURCE; - goto out_free_bounce_pages; - } - - sg_set_page(&src, plaintext_page, data_unit_size, - enc_bvec->bv_offset); - sg_set_page(&dst, ciphertext_page, data_unit_size, - enc_bvec->bv_offset); - - /* Encrypt each data unit in this page */ - for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + /* + * Encrypt each page in the source bio. Because the source bio could + * have bio_vecs that span more than a single page, but the encrypted + * bios are limited to a single page per bio_vec, this can generate + * more than a single encrypted bio per source bio. + */ +new_bio: + nr_enc_pages = min(bio_segments(src_bio), BIO_MAX_VECS); + enc_bio = blk_crypto_alloc_enc_bio(src_bio, nr_enc_pages); + enc_idx = 0; + for (;;) { + struct bio_vec src_bv = + bio_iter_iovec(src_bio, src_bio->bi_iter); + struct page *enc_page; + + enc_page = mempool_alloc(blk_crypto_bounce_page_pool, + GFP_NOIO); + __bio_add_page(enc_bio, enc_page, src_bv.bv_len, + src_bv.bv_offset); + + sg_set_page(&src, src_bv.bv_page, data_unit_size, + src_bv.bv_offset); + sg_set_page(&dst, enc_page, data_unit_size, src_bv.bv_offset); + + /* + * Increment the index now that the encrypted page is added to + * the bio. This is important for the error unwind path. + */ + enc_idx++; + + /* + * Encrypt each data unit in this page. + */ + for (i = 0; i < src_bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_encrypt(ciph_req), - &wait)) { - i++; - src_bio->bi_status = BLK_STS_IOERR; - goto out_free_bounce_pages; + if (crypto_skcipher_encrypt(ciph_req)) { + bio_io_error(enc_bio); + return; } bio_crypt_dun_increment(curr_dun, 1); src.offset += data_unit_size; dst.offset += data_unit_size; } - } - enc_bio->bi_private = src_bio; - enc_bio->bi_end_io = blk_crypto_fallback_encrypt_endio; - *bio_ptr = enc_bio; - ret = true; + bio_advance_iter_single(src_bio, &src_bio->bi_iter, + src_bv.bv_len); + if (!src_bio->bi_iter.bi_size) + break; - enc_bio = NULL; - goto out_free_ciph_req; + if (enc_idx == nr_enc_pages) { + /* + * For each additional encrypted bio submitted, + * increment the source bio's remaining count. Each + * encrypted bio's completion handler calls bio_endio on + * the source bio, so this keeps the source bio from + * completing until the last encrypted bio does. + */ + bio_inc_remaining(src_bio); + submit_bio(enc_bio); + goto new_bio; + } + } -out_free_bounce_pages: - while (i > 0) - mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_bounce_page_pool); -out_free_ciph_req: - skcipher_request_free(ciph_req); -out_release_keyslot: - blk_crypto_put_keyslot(slot); -out_put_enc_bio: - if (enc_bio) - bio_uninit(enc_bio); - kfree(enc_bio); - return ret; + submit_bio(enc_bio); } /* - * The crypto API fallback's main decryption routine. - * Decrypts input bio in place, and calls bio_endio on the bio. + * The crypto API fallback's encryption routine. + * + * Allocate one or more bios for encryption, encrypt the input bio using the + * crypto API, and submit the encrypted bios. Sets bio->bi_status and + * completes the source bio on error */ -static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +static void blk_crypto_fallback_encrypt_bio(struct bio *src_bio) { - struct bio_fallback_crypt_ctx *f_ctx = - container_of(work, struct bio_fallback_crypt_ctx, work); - struct bio *bio = f_ctx->bio; - struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct bio_crypt_ctx *bc = src_bio->bi_crypt_context; struct blk_crypto_keyslot *slot; - struct skcipher_request *ciph_req = NULL; - DECLARE_CRYPTO_WAIT(wait); + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status != BLK_STS_OK) { + bio_endio_status(src_bio, status); + return; + } + __blk_crypto_fallback_encrypt_bio(src_bio, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); +} + +static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio, + struct bio_crypt_ctx *bc, struct bvec_iter iter, + struct crypto_sync_skcipher *tfm) +{ + SYNC_SKCIPHER_REQUEST_ON_STACK(ciph_req, tfm); u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; union blk_crypto_iv iv; struct scatterlist sg; struct bio_vec bv; - struct bvec_iter iter; const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size; unsigned int i; - blk_status_t blk_st; - - /* - * Get a blk-crypto-fallback keyslot that contains a crypto_skcipher for - * this bio's algorithm and key. - */ - blk_st = blk_crypto_get_keyslot(blk_crypto_fallback_profile, - bc->bc_key, &slot); - if (blk_st != BLK_STS_OK) { - bio->bi_status = blk_st; - goto out_no_keyslot; - } - /* and then allocate an skcipher_request for it */ - if (!blk_crypto_fallback_alloc_cipher_req(slot, &ciph_req, &wait)) { - bio->bi_status = BLK_STS_RESOURCE; - goto out; - } + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP, + NULL, NULL); memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); sg_init_table(&sg, 1); @@ -411,7 +335,7 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) iv.bytes); /* Decrypt each segment in the bio */ - __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + __bio_for_each_segment(bv, bio, iter, iter) { struct page *page = bv.bv_page; sg_set_page(&sg, page, data_unit_size, bv.bv_offset); @@ -419,22 +343,41 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) /* Decrypt each data unit in the segment */ for (i = 0; i < bv.bv_len; i += data_unit_size) { blk_crypto_dun_to_iv(curr_dun, &iv); - if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), - &wait)) { - bio->bi_status = BLK_STS_IOERR; - goto out; - } + if (crypto_skcipher_decrypt(ciph_req)) + return BLK_STS_IOERR; bio_crypt_dun_increment(curr_dun, 1); sg.offset += data_unit_size; } } -out: - skcipher_request_free(ciph_req); - blk_crypto_put_keyslot(slot); -out_no_keyslot: + return BLK_STS_OK; +} + +/* + * The crypto API fallback's main decryption routine. + * + * Decrypts input bio in place, and calls bio_endio on the bio. + */ +static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) +{ + struct bio_fallback_crypt_ctx *f_ctx = + container_of(work, struct bio_fallback_crypt_ctx, work); + struct bio *bio = f_ctx->bio; + struct bio_crypt_ctx *bc = &f_ctx->crypt_ctx; + struct blk_crypto_keyslot *slot; + blk_status_t status; + + status = blk_crypto_get_keyslot(blk_crypto_fallback_profile, + bc->bc_key, &slot); + if (status == BLK_STS_OK) { + status = __blk_crypto_fallback_decrypt_bio(bio, bc, + f_ctx->crypt_iter, + blk_crypto_fallback_tfm(slot)); + blk_crypto_put_keyslot(slot); + } mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); - bio_endio(bio); + + bio_endio_status(bio, status); } /** @@ -466,44 +409,43 @@ static void blk_crypto_fallback_decrypt_endio(struct bio *bio) /** * blk_crypto_fallback_bio_prep - Prepare a bio to use fallback en/decryption + * @bio: bio to prepare * - * @bio_ptr: pointer to the bio to prepare + * If bio is doing a WRITE operation, allocate one or more bios to contain the + * encrypted payload and submit them. * - * If bio is doing a WRITE operation, this splits the bio into two parts if it's - * too big (see blk_crypto_fallback_split_bio_if_needed()). It then allocates a - * bounce bio for the first part, encrypts it, and updates bio_ptr to point to - * the bounce bio. - * - * For a READ operation, we mark the bio for decryption by using bi_private and + * For a READ operation, mark the bio for decryption by using bi_private and * bi_end_io. * - * In either case, this function will make the bio look like a regular bio (i.e. - * as if no encryption context was ever specified) for the purposes of the rest - * of the stack except for blk-integrity (blk-integrity and blk-crypto are not - * currently supported together). + * In either case, this function will make the submitted bio(s) look like + * regular bios (i.e. as if no encryption context was ever specified) for the + * purposes of the rest of the stack except for blk-integrity (blk-integrity and + * blk-crypto are not currently supported together). * - * Return: true on success. Sets bio->bi_status and returns false on error. + * Return: true if @bio should be submitted to the driver by the caller, else + * false. Sets bio->bi_status, calls bio_endio and returns false on error. */ -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) +bool blk_crypto_fallback_bio_prep(struct bio *bio) { - struct bio *bio = *bio_ptr; struct bio_crypt_ctx *bc = bio->bi_crypt_context; struct bio_fallback_crypt_ctx *f_ctx; if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) { /* User didn't call blk_crypto_start_using_key() first */ - bio->bi_status = BLK_STS_IOERR; + bio_io_error(bio); return false; } if (!__blk_crypto_cfg_supported(blk_crypto_fallback_profile, &bc->bc_key->crypto_cfg)) { - bio->bi_status = BLK_STS_NOTSUPP; + bio_endio_status(bio, BLK_STS_NOTSUPP); return false; } - if (bio_data_dir(bio) == WRITE) - return blk_crypto_fallback_encrypt_bio(bio_ptr); + if (bio_data_dir(bio) == WRITE) { + blk_crypto_fallback_encrypt_bio(bio); + return false; + } /* * bio READ case: Set up a f_ctx in the bio's bi_private and set the @@ -537,7 +479,7 @@ static int blk_crypto_fallback_init(void) get_random_bytes(blank_key, sizeof(blank_key)); - err = bioset_init(&crypto_bio_split, 64, 0, 0); + err = bioset_init(&enc_bio_set, 64, 0, BIOSET_NEED_BVECS); if (err) goto out; @@ -607,7 +549,7 @@ static int blk_crypto_fallback_init(void) fail_free_profile: kfree(blk_crypto_fallback_profile); fail_free_bioset: - bioset_exit(&crypto_bio_split); + bioset_exit(&enc_bio_set); out: return err; } @@ -641,7 +583,8 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); + slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(cipher_str, + 0, 0); if (IS_ERR(slotp->tfms[mode_num])) { err = PTR_ERR(slotp->tfms[mode_num]); if (err == -ENOENT) { @@ -653,7 +596,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) goto out_free_tfms; } - crypto_skcipher_set_flags(slotp->tfms[mode_num], + crypto_sync_skcipher_set_flags(slotp->tfms[mode_num], CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); } @@ -667,7 +610,7 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) out_free_tfms: for (i = 0; i < blk_crypto_num_keyslots; i++) { slotp = &blk_crypto_keyslots[i]; - crypto_free_skcipher(slotp->tfms[mode_num]); + crypto_free_sync_skcipher(slotp->tfms[mode_num]); slotp->tfms[mode_num] = NULL; } out: diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h index ccf6dff6ff6be..d650231203411 100644 --- a/block/blk-crypto-internal.h +++ b/block/blk-crypto-internal.h @@ -165,11 +165,11 @@ static inline void bio_crypt_do_front_merge(struct request *rq, #endif } -bool __blk_crypto_bio_prep(struct bio **bio_ptr); -static inline bool blk_crypto_bio_prep(struct bio **bio_ptr) +bool __blk_crypto_bio_prep(struct bio *bio); +static inline bool blk_crypto_bio_prep(struct bio *bio) { - if (bio_has_crypt_ctx(*bio_ptr)) - return __blk_crypto_bio_prep(bio_ptr); + if (bio_has_crypt_ctx(bio)) + return __blk_crypto_bio_prep(bio); return true; } @@ -215,12 +215,12 @@ static inline int blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, return 0; } +bool blk_crypto_fallback_bio_prep(struct bio *bio); + #ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num); -bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr); - int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); #else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ @@ -232,13 +232,6 @@ blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) return -ENOPKG; } -static inline bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) -{ - pr_warn_once("crypto API fallback disabled; failing request.\n"); - (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; - return false; -} - static inline int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) { diff --git a/block/blk-crypto.c b/block/blk-crypto.c index 3e7bf1974cbd8..9792e7273110b 100644 --- a/block/blk-crypto.c +++ b/block/blk-crypto.c @@ -260,54 +260,54 @@ void __blk_crypto_free_request(struct request *rq) /** * __blk_crypto_bio_prep - Prepare bio for inline encryption - * - * @bio_ptr: pointer to original bio pointer + * @bio: bio to prepare * * If the bio crypt context provided for the bio is supported by the underlying * device's inline encryption hardware, do nothing. * * Otherwise, try to perform en/decryption for this bio by falling back to the - * kernel crypto API. When the crypto API fallback is used for encryption, - * blk-crypto may choose to split the bio into 2 - the first one that will - * continue to be processed and the second one that will be resubmitted via - * submit_bio_noacct. A bounce bio will be allocated to encrypt the contents - * of the aforementioned "first one", and *bio_ptr will be updated to this - * bounce bio. + * kernel crypto API. For encryption this means submitting newly allocated + * bios for the encrypted payload while keeping back the source bio until they + * complete, while for reads the decryption happens in-place by a hooked in + * completion handler. * * Caller must ensure bio has bio_crypt_ctx. * - * Return: true on success; false on error (and bio->bi_status will be set - * appropriately, and bio_endio() will have been called so bio - * submission should abort). + * Return: true if @bio should be submitted to the driver by the caller, else + * false. Sets bio->bi_status, calls bio_endio and returns false on error. */ -bool __blk_crypto_bio_prep(struct bio **bio_ptr) +bool __blk_crypto_bio_prep(struct bio *bio) { - struct bio *bio = *bio_ptr; const struct blk_crypto_key *bc_key = bio->bi_crypt_context->bc_key; + struct block_device *bdev = bio->bi_bdev; /* Error if bio has no data. */ if (WARN_ON_ONCE(!bio_has_data(bio))) { - bio->bi_status = BLK_STS_IOERR; - goto fail; + bio_io_error(bio); + return false; } if (!bio_crypt_check_alignment(bio)) { bio->bi_status = BLK_STS_INVAL; - goto fail; + bio_endio(bio); + return false; } /* - * Success if device supports the encryption context, or if we succeeded - * in falling back to the crypto API. + * If the device does not natively support the encryption context, try to use + * the fallback if available. */ - if (blk_crypto_config_supported_natively(bio->bi_bdev, - &bc_key->crypto_cfg)) - return true; - if (blk_crypto_fallback_bio_prep(bio_ptr)) - return true; -fail: - bio_endio(*bio_ptr); - return false; + if (!blk_crypto_config_supported_natively(bdev, &bc_key->crypto_cfg)) { + if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)) { + pr_warn_once("%pg: crypto API fallback disabled; failing request.\n", + bdev); + bio_endio_status(bio, BLK_STS_NOTSUPP); + return false; + } + return blk_crypto_fallback_bio_prep(bio); + } + + return true; } int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio, diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 5bfd70311359c..ecf4969125050 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -1603,7 +1603,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p u64 rq_wait_ns = 0; int cpu, rw; - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu); u64 this_rq_wait_ns; diff --git a/block/blk-map.c b/block/blk-map.c index 60faf036fb6e4..17a1dc2886786 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -459,6 +459,8 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) if (rq->bio) { if (!ll_back_merge_fn(rq, bio, nr_segs)) return -EINVAL; + rq->phys_gap_bit = bio_seg_gap(rq->q, rq->biotail, bio, + rq->phys_gap_bit); rq->biotail->bi_next = bio; rq->biotail = bio; rq->__data_len += bio->bi_iter.bi_size; @@ -469,6 +471,7 @@ int blk_rq_append_bio(struct request *rq, struct bio *bio) rq->nr_phys_segments = nr_segs; rq->bio = rq->biotail = bio; rq->__data_len = bio->bi_iter.bi_size; + rq->phys_gap_bit = bio->bi_bvec_gap_bit; return 0; } EXPORT_SYMBOL(blk_rq_append_bio); diff --git a/block/blk-merge.c b/block/blk-merge.c index 03b61923cf109..5d895c7898984 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -122,8 +122,7 @@ struct bio *bio_submit_split_bioset(struct bio *bio, unsigned int split_sectors, struct bio *split = bio_split(bio, split_sectors, GFP_NOIO, bs); if (IS_ERR(split)) { - bio->bi_status = errno_to_blk_status(PTR_ERR(split)); - bio_endio(bio); + bio_endio_status(bio, errno_to_blk_status(PTR_ERR(split))); return NULL; } @@ -143,8 +142,7 @@ EXPORT_SYMBOL_GPL(bio_submit_split_bioset); static struct bio *bio_submit_split(struct bio *bio, int split_sectors) { if (unlikely(split_sectors < 0)) { - bio->bi_status = errno_to_blk_status(split_sectors); - bio_endio(bio); + bio_endio_status(bio, errno_to_blk_status(split_sectors)); return NULL; } @@ -315,6 +313,12 @@ static unsigned int bio_split_alignment(struct bio *bio, return lim->logical_block_size; } +static inline unsigned int bvec_seg_gap(struct bio_vec *bvprv, + struct bio_vec *bv) +{ + return bv->bv_offset | (bvprv->bv_offset + bvprv->bv_len); +} + /** * bio_split_io_at - check if and where to split a bio * @bio: [in] bio to be split @@ -332,8 +336,8 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, unsigned *segs, unsigned max_bytes, unsigned len_align_mask) { struct bio_vec bv, bvprv, *bvprvp = NULL; + unsigned nsegs = 0, bytes = 0, gaps = 0; struct bvec_iter iter; - unsigned nsegs = 0, bytes = 0; bio_for_each_bvec(bv, bio, iter) { if (bv.bv_offset & lim->dma_alignment || @@ -344,8 +348,11 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. */ - if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) - goto split; + if (bvprvp) { + if (bvec_gap_to_prev(lim, bvprvp, bv.bv_offset)) + goto split; + gaps |= bvec_seg_gap(bvprvp, &bv); + } if (nsegs < lim->max_segments && bytes + bv.bv_len <= max_bytes && @@ -363,6 +370,7 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, } *segs = nsegs; + bio->bi_bvec_gap_bit = ffs(gaps); return 0; split: if (bio->bi_opf & REQ_ATOMIC) @@ -398,6 +406,7 @@ int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, * big IO can be trival, disable iopoll when split needed. */ bio_clear_polled(bio); + bio->bi_bvec_gap_bit = ffs(gaps); return bytes >> SECTOR_SHIFT; } EXPORT_SYMBOL_GPL(bio_split_io_at); @@ -734,6 +743,24 @@ static bool blk_atomic_write_mergeable_rqs(struct request *rq, return (rq->cmd_flags & REQ_ATOMIC) == (next->cmd_flags & REQ_ATOMIC); } +u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next, + u8 gaps_bit) +{ + struct bio_vec pb, nb; + + if (!bio_has_data(prev)) + return 0; + + gaps_bit = min_not_zero(gaps_bit, prev->bi_bvec_gap_bit); + gaps_bit = min_not_zero(gaps_bit, next->bi_bvec_gap_bit); + + bio_get_last_bvec(prev, &pb); + bio_get_first_bvec(next, &nb); + if (!biovec_phys_mergeable(q, &pb, &nb)) + gaps_bit = min_not_zero(gaps_bit, ffs(bvec_seg_gap(&pb, &nb))); + return gaps_bit; +} + /* * For non-mq, this has to be called with the request spinlock acquired. * For mq with scheduling, the appropriate queue wide lock should be held. @@ -798,6 +825,9 @@ static struct request *attempt_merge(struct request_queue *q, if (next->start_time_ns < req->start_time_ns) req->start_time_ns = next->start_time_ns; + req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, next->bio, + min_not_zero(next->phys_gap_bit, + req->phys_gap_bit)); req->biotail->bi_next = next->bio; req->biotail = next->biotail; @@ -921,6 +951,8 @@ enum bio_merge_status bio_attempt_back_merge(struct request *req, if (req->rq_flags & RQF_ZONE_WRITE_PLUGGING) blk_zone_write_plug_bio_merged(bio); + req->phys_gap_bit = bio_seg_gap(req->q, req->biotail, bio, + req->phys_gap_bit); req->biotail->bi_next = bio; req->biotail = bio; req->__data_len += bio->bi_iter.bi_size; @@ -955,6 +987,8 @@ static enum bio_merge_status bio_attempt_front_merge(struct request *req, blk_update_mixed_merge(req, bio, true); + req->phys_gap_bit = bio_seg_gap(req->q, bio, req->bio, + req->phys_gap_bit); bio->bi_next = req->bio; req->bio = bio; diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index 449950029872a..b7c26599f6b05 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -79,8 +79,7 @@ static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter, static inline bool blk_can_dma_map_iova(struct request *req, struct device *dma_dev) { - return !((queue_virt_boundary(req->q) + 1) & - dma_get_merge_boundary(dma_dev)); + return !(req_phys_gap_mask(req) & dma_get_merge_boundary(dma_dev)); } static bool blk_dma_map_bus(struct blk_dma_iter *iter, struct phys_vec *vec) @@ -194,6 +193,7 @@ static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev, if (blk_can_dma_map_iova(req, dma_dev) && dma_iova_try_alloc(dma_dev, state, vec.paddr, total_len)) return blk_rq_dma_map_iova(req, dma_dev, state, iter, &vec); + memset(state, 0, sizeof(*state)); return blk_dma_map_direct(req, dma_dev, iter, &vec); } diff --git a/block/blk-mq.c b/block/blk-mq.c index ab05c5c9e6ae2..e050ea631cdac 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -377,6 +377,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq) INIT_LIST_HEAD(&rq->queuelist); rq->q = q; rq->__sector = (sector_t) -1; + rq->phys_gap_bit = 0; INIT_HLIST_NODE(&rq->hash); RB_CLEAR_NODE(&rq->rb_node); rq->tag = BLK_MQ_NO_TAG; @@ -669,6 +670,7 @@ struct request *blk_mq_alloc_request(struct request_queue *q, blk_opf_t opf, goto out_queue_exit; } rq->__data_len = 0; + rq->phys_gap_bit = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; return rq; @@ -749,6 +751,7 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q, rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag); blk_mq_rq_time_init(rq, alloc_time_ns); rq->__data_len = 0; + rq->phys_gap_bit = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; return rq; @@ -2675,6 +2678,8 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio, rq->bio = rq->biotail = bio; rq->__sector = bio->bi_iter.bi_sector; rq->__data_len = bio->bi_iter.bi_size; + rq->phys_gap_bit = bio->bi_bvec_gap_bit; + rq->nr_phys_segments = nr_segs; if (bio_integrity(bio)) rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q, @@ -3057,7 +3062,7 @@ static struct request *blk_mq_get_new_requests(struct request_queue *q, /* * Check if there is a suitable cached request and return it. */ -static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, +static struct request *blk_mq_get_cached_request(struct blk_plug *plug, struct request_queue *q, blk_opf_t opf) { enum hctx_type type = blk_mq_get_hctx_type(opf); @@ -3073,27 +3078,10 @@ static struct request *blk_mq_peek_cached_request(struct blk_plug *plug, return NULL; if (op_is_flush(rq->cmd_flags) != op_is_flush(opf)) return NULL; + rq_list_pop(&plug->cached_rqs); return rq; } -static void blk_mq_use_cached_rq(struct request *rq, struct blk_plug *plug, - struct bio *bio) -{ - if (rq_list_pop(&plug->cached_rqs) != rq) - WARN_ON_ONCE(1); - - /* - * If any qos ->throttle() end up blocking, we will have flushed the - * plug and hence killed the cached_rq list as well. Pop this entry - * before we throttle. - */ - rq_qos_throttle(rq->q, bio); - - blk_mq_rq_time_init(rq, blk_time_get_ns()); - rq->cmd_flags = bio->bi_opf; - INIT_LIST_HEAD(&rq->queuelist); -} - static bool bio_unaligned(const struct bio *bio, struct request_queue *q) { unsigned int bs_mask = queue_logical_block_size(q) - 1; @@ -3131,7 +3119,7 @@ void blk_mq_submit_bio(struct bio *bio) /* * If the plug has a cached request for this queue, try to use it. */ - rq = blk_mq_peek_cached_request(plug, q, bio->bi_opf); + rq = blk_mq_get_cached_request(plug, q, bio->bi_opf); /* * A BIO that was released from a zone write plug has already been @@ -3166,8 +3154,7 @@ void blk_mq_submit_bio(struct bio *bio) } if ((bio->bi_opf & REQ_POLLED) && !blk_mq_can_poll(q)) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_NOTSUPP); goto queue_exit; } @@ -3189,7 +3176,10 @@ void blk_mq_submit_bio(struct bio *bio) new_request: if (rq) { - blk_mq_use_cached_rq(rq, plug, bio); + rq_qos_throttle(rq->q, bio); + blk_mq_rq_time_init(rq, blk_time_get_ns()); + rq->cmd_flags = bio->bi_opf; + INIT_LIST_HEAD(&rq->queuelist); } else { rq = blk_mq_get_new_requests(q, plug, bio); if (unlikely(!rq)) { @@ -3207,8 +3197,7 @@ void blk_mq_submit_bio(struct bio *bio) ret = blk_crypto_rq_get_keyslot(rq); if (ret != BLK_STS_OK) { - bio->bi_status = ret; - bio_endio(bio); + bio_endio_status(bio, ret); blk_mq_free_request(rq); return; } @@ -3235,12 +3224,10 @@ void blk_mq_submit_bio(struct bio *bio) return; queue_exit: - /* - * Don't drop the queue reference if we were trying to use a cached - * request and thus didn't acquire one. - */ if (!rq) blk_queue_exit(q); + else + rq_list_add_head(&plug->cached_rqs, rq); } #ifdef CONFIG_BLK_MQ_STACKING @@ -3400,6 +3387,7 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src, } rq->nr_phys_segments = rq_src->nr_phys_segments; rq->nr_integrity_segments = rq_src->nr_integrity_segments; + rq->phys_gap_bit = rq_src->phys_gap_bit; if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0) goto free_and_out; diff --git a/block/blk-stat.c b/block/blk-stat.c index 682a8ddb11734..5c5289b085504 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -83,7 +83,7 @@ static void blk_stat_timer_fn(struct timer_list *t) for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cb->stat[bucket]); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct blk_rq_stat *cpu_stat; cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); diff --git a/block/fops.c b/block/fops.c index 5e3db9fead77c..bbc89773e07f8 100644 --- a/block/fops.c +++ b/block/fops.c @@ -221,8 +221,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ret = blkdev_iov_iter_get_pages(bio, iter, bdev); if (unlikely(ret)) { - bio->bi_status = BLK_STS_IOERR; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_IOERR); break; } if (iocb->ki_flags & IOCB_NOWAIT) { @@ -242,8 +241,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, } if (iocb->ki_flags & IOCB_HAS_METADATA) { ret = bio_integrity_map_iter(bio, iocb->private); - if (unlikely(ret)) - goto fail; + if (unlikely(ret)) { + bio_endio_status(bio, errno_to_blk_status(ret)); + break; + } } if (is_read) { diff --git a/block/genhd.c b/block/genhd.c index d23cb33ea1276..bbcfbd1a363c5 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -447,6 +447,13 @@ static int __add_disk(struct device *parent, struct gendisk *disk, bdev_set_flag(disk->part0, BD_HAS_SUBMIT_BIO); } + /* + * We do not support partitions with zoned block devices, so do not try + * to scan the partitions table. + */ + if (blk_queue_is_zoned(disk->queue)) + disk->flags |= GENHD_FL_NO_PART; + /* * If the driver provides an explicit major number it also must provide * the number of minors numbers supported, and those will be used to @@ -681,6 +688,7 @@ static bool __blk_mark_disk_dead(struct gendisk *disk) */ void blk_mark_disk_dead(struct gendisk *disk) { + blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue); __blk_mark_disk_dead(disk); blk_report_disk_dead(disk, true); } @@ -1281,14 +1289,18 @@ static void disk_release(struct device *dev) /* * To undo the all initialization from blk_mq_init_allocated_queue in * case of a probe failure where add_disk is never called we have to - * call blk_mq_exit_queue here. We can't do this for the more common - * teardown case (yet) as the tagset can be gone by the time the disk - * is released once it was added. + * call blk_mq_exit_queue here, after stopping the timer and work items + * that I/O issued before add_disk may have left pending. We can't do + * this for the more common teardown case (yet) as the tagset can be + * gone by the time the disk is released once it was added. */ if (queue_is_mq(disk->queue) && test_bit(GD_OWNS_QUEUE, &disk->state) && - !test_bit(GD_ADDED, &disk->state)) + !test_bit(GD_ADDED, &disk->state)) { + blk_sync_queue(disk->queue); + blk_mq_cancel_work_sync(disk->queue); blk_mq_exit_queue(disk->queue); + } blkcg_exit_disk(disk); diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index e3eaeea62e24d..927df06e6b940 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -279,7 +279,7 @@ static void kyber_timer_fn(struct timer_list *t) bool bad = false; /* Sum all of the per-cpu latency histograms. */ - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct kyber_cpu_latency *cpu_latency; cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu); diff --git a/crypto/Makefile b/crypto/Makefile index e430e6e99b6a2..ef963cc03a038 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -183,6 +183,11 @@ obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o obj-$(CONFIG_CRYPTO_ECC) += ecc.o obj-$(CONFIG_CRYPTO_ESSIV) += essiv.o +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124949 +ifeq ($(CONFIG_ARM)$(CONFIG_KASAN_STACK)$(CONFIG_CC_IS_GCC),yyy) +CFLAGS_ecc.o += $(call cc-option,-Wframe-larger-than=1536) +endif + ecdh_generic-y += ecdh.o ecdh_generic-y += ecdh_helper.o obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o diff --git a/crypto/acompress.c b/crypto/acompress.c index 25af7697d6bb9..aa427b6235a50 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -567,12 +567,22 @@ EXPORT_SYMBOL_GPL(acomp_walk_virt); struct acomp_req *acomp_request_clone(struct acomp_req *req, size_t total, gfp_t gfp) { + struct crypto_tfm *tfm = req->base.tfm; struct acomp_req *nreq; + size_t len; - nreq = container_of(crypto_request_clone(&req->base, total, gfp), - struct acomp_req, base); - if (nreq == req) + len = sizeof(*req) + + crypto_acomp_reqsize(crypto_acomp_reqtfm(req)); + len = ALIGN(len, CRYPTO_MINALIGN); + + nreq = kzalloc(len, gfp); + if (!nreq) { + req->base.tfm = tfm->fb; return req; + } + + memcpy(nreq, req, sizeof(*req)); + nreq->base.flags &= ~CRYPTO_TFM_REQ_ON_STACK; if (req->src == &req->chain.ssg) nreq->src = &nreq->chain.ssg; diff --git a/crypto/ccm.c b/crypto/ccm.c index 2ae929ffdef8e..916441e4f2b86 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -747,7 +747,7 @@ static int crypto_rfc4309_create(struct crypto_template *tmpl, inst->alg.ivsize = 8; inst->alg.chunksize = crypto_aead_alg_chunksize(alg); - inst->alg.maxauthsize = 16; + inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx); diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c index 03395b89cc61e..b377f4f7558da 100644 --- a/crypto/krb5/krb5_api.c +++ b/crypto/krb5/krb5_api.c @@ -263,10 +263,10 @@ struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *kr goto err; } - kfree(keys.data); + kfree_sensitive(keys.data); return ci; err: - kfree(keys.data); + kfree_sensitive(keys.data); return ERR_PTR(ret); } EXPORT_SYMBOL(crypto_krb5_prepare_encryption); @@ -333,10 +333,10 @@ struct crypto_shash *crypto_krb5_prepare_checksum(const struct krb5_enctype *krb goto err; } - kfree(keys.data); + kfree_sensitive(keys.data); return ci; err: - kfree(keys.data); + kfree_sensitive(keys.data); return ERR_PTR(ret); } EXPORT_SYMBOL(crypto_krb5_prepare_checksum); diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c index c2e2c38b5aa8d..091761d6ca0bf 100644 --- a/crypto/lskcipher.c +++ b/crypto/lskcipher.c @@ -95,7 +95,6 @@ static int crypto_lskcipher_crypt_unaligned( while (len >= bs) { unsigned chunk = min((unsigned)PAGE_SIZE, len); - int err; if (chunk > cs) chunk &= ~(cs - 1); diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c index 612ad6302622a..9ec3d90c2d200 100644 --- a/drivers/accel/amdxdna/aie2_ctx.c +++ b/drivers/accel/amdxdna/aie2_ctx.c @@ -794,6 +794,16 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) found = false; down_write(&xdna->notifier_lock); list_for_each_entry(mapp, &abo->mem.umap_list, node) { + /* + * Skip entries that have already been unmapped. + * + * If userspace unmaps the address and later submits I/O using + * it, the IOMMU will reject the access and report a fault. + * Ignore such entries here. + */ + if (mapp->unmapped) + continue; + if (mapp->invalid && kref_get_unless_zero(&mapp->refcnt)) { found = true; break; @@ -801,6 +811,12 @@ static int aie2_populate_range(struct amdxdna_gem_obj *abo) } if (!found) { + /* + * This also covers the case where all mappings have been + * removed. There are no invalid mappings left to process. + * Any subsequent I/O using the unmapped address will be + * rejected by the IOMMU. + */ abo->mem.map_invalid = false; up_write(&xdna->notifier_lock); return 0; diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c index cfee89681ff3c..a49c059e6ed49 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.c +++ b/drivers/accel/amdxdna/amdxdna_ctx.c @@ -393,6 +393,7 @@ void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job) amdxdna_arg_bos_put(job); amdxdna_gem_put_obj(job->cmd_bo); dma_fence_put(job->fence); + mmdrop(job->mm); } int amdxdna_cmd_submit(struct amdxdna_client *client, @@ -405,6 +406,10 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, int ret, idx; XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt); + + if (!xdna->dev_info->ops->cmd_submit) + return -EOPNOTSUPP; + job = kzalloc(struct_size(job, bos, arg_bo_cnt), GFP_KERNEL); if (!job) return -ENOMEM; @@ -443,6 +448,7 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, job->hwctx = hwctx; job->mm = current->mm; + mmgrab(job->mm); job->fence = amdxdna_fence_create(hwctx); if (!job->fence) { @@ -475,6 +481,8 @@ int amdxdna_cmd_submit(struct amdxdna_client *client, cmd_put: amdxdna_gem_put_obj(job->cmd_bo); free_job: + if (job->mm) + mmdrop(job->mm); kfree(job); return ret; } diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h index 8c1d181df6e79..383f3f15fac8c 100644 --- a/drivers/accel/amdxdna/amdxdna_ctx.h +++ b/drivers/accel/amdxdna/amdxdna_ctx.h @@ -51,7 +51,7 @@ struct amdxdna_cmd_chain { u32 submit_index; u32 error_index; u32 reserved[3]; - u64 data[] __counted_by(command_count); + u64 data[]; }; /* Exec buffer command header format */ diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c index a719dc6aab339..ca747457cec79 100644 --- a/drivers/accel/amdxdna/amdxdna_gem.c +++ b/drivers/accel/amdxdna/amdxdna_gem.c @@ -949,6 +949,9 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, goto put_obj; } + if (!args->size) + goto unpin; + if (is_import_bo(abo)) drm_clflush_sg(abo->base.sgt); else if (abo->mem.kva) @@ -958,6 +961,7 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, else drm_WARN(&xdna->ddev, 1, "Can not get flush memory"); +unpin: amdxdna_gem_unpin(abo); XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n", diff --git a/drivers/accel/ivpu/ivpu_fw_log.c b/drivers/accel/ivpu/ivpu_fw_log.c index 716467aa31567..9eafc42120b6a 100644 --- a/drivers/accel/ivpu/ivpu_fw_log.c +++ b/drivers/accel/ivpu/ivpu_fw_log.c @@ -26,10 +26,17 @@ MODULE_PARM_DESC(fw_log_level, " error=" __stringify(IVPU_FW_LOG_ERROR) " fatal=" __stringify(IVPU_FW_LOG_FATAL)); +struct ivpu_fw_log_desc { + struct vpu_tracing_buffer_header *log; + u32 header_size; + u32 size; +}; + static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *offset, - struct vpu_tracing_buffer_header **out_log) + struct ivpu_fw_log_desc *desc) { struct vpu_tracing_buffer_header *log; + u32 header_size, size; if ((*offset + sizeof(*log)) > ivpu_bo_size(bo)) return -EINVAL; @@ -39,26 +46,32 @@ static int fw_log_from_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, u32 *off if (log->vpu_canary_start != VPU_TRACING_BUFFER_CANARY) return -EINVAL; - if (log->header_size < sizeof(*log) || log->header_size > 1024) { - ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", log->header_size); + header_size = READ_ONCE(log->header_size); + size = READ_ONCE(log->size); + + if (header_size < sizeof(*log) || header_size > 1024) { + ivpu_dbg(vdev, FW_BOOT, "Invalid header size 0x%x\n", header_size); return -EINVAL; } - if (log->size < log->header_size) { - ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + if ((char *)log + size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", size); return -EINVAL; } - if ((char *)log + log->size > (char *)ivpu_bo_vaddr(bo) + ivpu_bo_size(bo)) { - ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x\n", log->size); + if (size < header_size) { + ivpu_dbg(vdev, FW_BOOT, "Invalid log size 0x%x < header size 0x%x\n", + size, header_size); return -EINVAL; } - *out_log = log; - *offset += log->size; + desc->log = log; + desc->header_size = header_size; + desc->size = size; + *offset += size; ivpu_dbg(vdev, FW_BOOT, - "FW log name \"%s\", write offset 0x%x size 0x%x, wrap count %d, hdr version %d size %d format %d, alignment %d", - log->name, log->write_index, log->size, log->wrap_count, log->header_version, - log->header_size, log->format, log->alignment); + "FW log name \"%.*s\", write offset 0x%x size 0x%x, wrap count %d, hdr version %d size %d format %d, alignment %d", + (int)ARRAY_SIZE(log->name), log->name, log->write_index, size, log->wrap_count, + log->header_version, header_size, log->format, log->alignment); return 0; } @@ -94,11 +107,12 @@ static void fw_log_print_lines(char *buffer, u32 size, struct drm_printer *p) drm_printf(p, "%s", line); } -static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const char *prefix, +static void fw_log_print_buffer(struct ivpu_fw_log_desc *desc, const char *prefix, bool only_new_msgs, struct drm_printer *p) { - char *log_data = (void *)log + log->header_size; - u32 data_size = log->size - log->header_size; + struct vpu_tracing_buffer_header *log = desc->log; + char *log_data = (void *)log + desc->header_size; + u32 data_size = desc->size - desc->header_size; u32 log_start = only_new_msgs ? READ_ONCE(log->read_index) : 0; u32 log_end = READ_ONCE(log->write_index); @@ -109,7 +123,8 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha if (log->wrap_count == log->read_wrap_count) { if (log_end <= log_start) { - drm_printf(p, "==== %s \"%s\" log empty ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log empty ====\n", prefix, + (int)ARRAY_SIZE(log->name), log->name); return; } } else if (log->wrap_count == log->read_wrap_count + 1) { @@ -119,7 +134,8 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha log_start = log_end; } - drm_printf(p, "==== %s \"%s\" log start ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log start ====\n", prefix, (int)ARRAY_SIZE(log->name), + log->name); if (log_end > log_start) { fw_log_print_lines(log_data + log_start, log_end - log_start, p); } else { @@ -127,18 +143,19 @@ static void fw_log_print_buffer(struct vpu_tracing_buffer_header *log, const cha fw_log_print_lines(log_data, log_end, p); } drm_printf(p, "\n\x1b[0m"); /* add new line and clear formatting */ - drm_printf(p, "==== %s \"%s\" log end ====\n", prefix, log->name); + drm_printf(p, "==== %s \"%.*s\" log end ====\n", prefix, (int)ARRAY_SIZE(log->name), + log->name); } static void fw_log_print_all_in_bo(struct ivpu_device *vdev, const char *name, struct ivpu_bo *bo, bool only_new_msgs, struct drm_printer *p) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next = 0; - while (fw_log_from_bo(vdev, bo, &next, &log) == 0) - fw_log_print_buffer(log, name, only_new_msgs, p); + while (fw_log_from_bo(vdev, bo, &next, &desc) == 0) + fw_log_print_buffer(&desc, name, only_new_msgs, p); } void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_printer *p) @@ -149,36 +166,36 @@ void ivpu_fw_log_print(struct ivpu_device *vdev, bool only_new_msgs, struct drm_ void ivpu_fw_log_mark_read(struct ivpu_device *vdev) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next; next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { - log->read_index = READ_ONCE(log->write_index); - log->read_wrap_count = READ_ONCE(log->wrap_count); + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &desc) == 0) { + desc.log->read_index = READ_ONCE(desc.log->write_index); + desc.log->read_wrap_count = READ_ONCE(desc.log->wrap_count); } next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { - log->read_index = READ_ONCE(log->write_index); - log->read_wrap_count = READ_ONCE(log->wrap_count); + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &desc) == 0) { + desc.log->read_index = READ_ONCE(desc.log->write_index); + desc.log->read_wrap_count = READ_ONCE(desc.log->wrap_count); } } void ivpu_fw_log_reset(struct ivpu_device *vdev) { - struct vpu_tracing_buffer_header *log; + struct ivpu_fw_log_desc desc; u32 next; next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &log) == 0) { - log->read_index = 0; - log->read_wrap_count = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_crit, &next, &desc) == 0) { + desc.log->read_index = 0; + desc.log->read_wrap_count = 0; } next = 0; - while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &log) == 0) { - log->read_index = 0; - log->read_wrap_count = 0; + while (fw_log_from_bo(vdev, vdev->fw->mem_log_verb, &next, &desc) == 0) { + desc.log->read_index = 0; + desc.log->read_wrap_count = 0; } } diff --git a/drivers/accel/ivpu/ivpu_gem.h b/drivers/accel/ivpu/ivpu_gem.h index ade0d127453ff..57c9c2db02069 100644 --- a/drivers/accel/ivpu/ivpu_gem.h +++ b/drivers/accel/ivpu/ivpu_gem.h @@ -74,15 +74,23 @@ static inline bool ivpu_bo_is_snooped(struct ivpu_bo *bo) return ivpu_bo_cache_mode(bo) == DRM_IVPU_BO_CACHED; } -static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u32 vpu_addr) +static inline void *ivpu_to_cpu_addr(struct ivpu_bo *bo, u64 vpu_addr, u64 size) { + u64 bo_size = ivpu_bo_size(bo); + u64 offset; + if (vpu_addr < bo->vpu_addr) return NULL; - if (vpu_addr >= (bo->vpu_addr + ivpu_bo_size(bo))) + if (size > bo_size) + return NULL; + + offset = vpu_addr - bo->vpu_addr; + + if (offset > bo_size - size) return NULL; - return ivpu_bo_vaddr(bo) + (vpu_addr - bo->vpu_addr); + return ivpu_bo_vaddr(bo) + offset; } static inline u32 cpu_to_vpu_addr(struct ivpu_bo *bo, void *cpu_addr) diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c index aa33f562d29c1..e9786204feca4 100644 --- a/drivers/accel/ivpu/ivpu_hw_btrs.c +++ b/drivers/accel/ivpu/ivpu_hw_btrs.c @@ -845,7 +845,7 @@ static void diagnose_failure_mtl(struct ivpu_device *vdev) static void diagnose_failure_lnl(struct ivpu_device *vdev) { - u32 reg = REGB_RD32(VPU_HW_BTRS_MTL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; + u32 reg = REGB_RD32(VPU_HW_BTRS_LNL_INTERRUPT_STAT) & BTRS_LNL_IRQ_MASK; if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, reg)) { ivpu_err(vdev, "ATS_ERR_LOG1 0x%08x ATS_ERR_LOG2 0x%08x\n", diff --git a/drivers/accel/ivpu/ivpu_ipc.c b/drivers/accel/ivpu/ivpu_ipc.c index 7fea203eef329..0c8b721d3061a 100644 --- a/drivers/accel/ivpu/ivpu_ipc.c +++ b/drivers/accel/ivpu/ivpu_ipc.c @@ -79,7 +79,7 @@ ivpu_ipc_tx_prepare(struct ivpu_device *vdev, struct ivpu_ipc_consumer *cons, return -ENOMEM; } - tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr); + tx_buf = ivpu_to_cpu_addr(ipc->mem_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); if (drm_WARN_ON(&vdev->drm, !tx_buf)) { gen_pool_free(ipc->mm_tx, tx_buf_vpu_addr, sizeof(*tx_buf)); return -EIO; @@ -416,7 +416,7 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev) return; } - ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr); + ipc_hdr = ivpu_to_cpu_addr(ipc->mem_rx, vpu_addr, sizeof(*ipc_hdr)); if (!ipc_hdr) { ivpu_warn_ratelimited(vdev, "IPC msg 0x%x out of range\n", vpu_addr); continue; @@ -425,7 +425,8 @@ void ivpu_ipc_irq_handler(struct ivpu_device *vdev) jsm_msg = NULL; if (ipc_hdr->channel != IVPU_IPC_CHAN_BOOT_MSG) { - jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr); + jsm_msg = ivpu_to_cpu_addr(ipc->mem_rx, ipc_hdr->data_addr, + sizeof(*jsm_msg)); if (!jsm_msg) { ivpu_warn_ratelimited(vdev, "JSM msg 0x%x out of range\n", ipc_hdr->data_addr); diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c index 8eae30fe14f98..b10ace7853b72 100644 --- a/drivers/accel/qaic/qaic_control.c +++ b/drivers/accel/qaic/qaic_control.c @@ -782,7 +782,7 @@ static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg, break; } trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len); - if (trans_hdr->len < sizeof(trans_hdr) || + if (trans_hdr->len < sizeof(*trans_hdr) || size_add(user_len, trans_hdr->len) > user_msg->len) { ret = -EINVAL; break; @@ -958,11 +958,13 @@ static int decode_status(struct qaic_device *qdev, void *trans, struct manage_ms static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, struct wire_msg *msg, struct ioctl_resources *resources, - struct qaic_user *usr) + struct qaic_user *usr, bool orphaned_deactivate) { + u32 msg_hdr_count = le32_to_cpu(msg->hdr.count); u32 msg_hdr_len = le32_to_cpu(msg->hdr.len); struct wire_trans_hdr *trans_hdr; u32 msg_len = 0; + int trans_type; int ret; int i; @@ -970,10 +972,12 @@ static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH) return -EINVAL; - user_msg->len = 0; - user_msg->count = le32_to_cpu(msg->hdr.count); + if (user_msg) { + user_msg->len = 0; + user_msg->count = msg_hdr_count; + } - for (i = 0; i < user_msg->count; ++i) { + for (i = 0; i < msg_hdr_count; ++i) { u32 hdr_len; if (msg_len > msg_hdr_len - sizeof(*trans_hdr)) @@ -985,7 +989,20 @@ static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg, size_add(msg_len, hdr_len) > msg_hdr_len) return -EINVAL; - switch (le32_to_cpu(trans_hdr->type)) { + trans_type = le32_to_cpu(trans_hdr->type); + /* + * orphaned_deactivate is the case where a deactivate response + * is received from the device after the user owning the DBC, + * and the message requesting deactivation, has gone away. + * In this case, only process QAIC_TRANS_DEACTIVATE_FROM_DEV + * transaction and skip the others. + */ + if (orphaned_deactivate && trans_type != QAIC_TRANS_DEACTIVATE_FROM_DEV) { + msg_len += hdr_len; + continue; + } + + switch (trans_type) { case QAIC_TRANS_PASSTHROUGH_FROM_DEV: ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len); break; @@ -1277,7 +1294,7 @@ static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct m goto dma_cont_failed; } - ret = decode_message(qdev, user_msg, rsp, &resources, usr); + ret = decode_message(qdev, user_msg, rsp, &resources, usr, false); dma_cont_failed: free_dbc_buf(qdev, &resources); @@ -1442,22 +1459,7 @@ static void resp_worker(struct work_struct *work) * response to the QAIC_TRANS_TERMINATE_TO_DEV transaction, * otherwise, the user can issue an soc_reset to the device. */ - u32 msg_count = le32_to_cpu(msg->hdr.count); - u32 msg_len = le32_to_cpu(msg->hdr.len); - u32 len = 0; - int j; - - for (j = 0; j < msg_count && len < msg_len; ++j) { - struct wire_trans_hdr *trans_hdr; - - trans_hdr = (struct wire_trans_hdr *)(msg->data + len); - if (le32_to_cpu(trans_hdr->type) == QAIC_TRANS_DEACTIVATE_FROM_DEV) { - if (decode_deactivate(qdev, trans_hdr, &len, NULL)) - len += le32_to_cpu(trans_hdr->len); - } else { - len += le32_to_cpu(trans_hdr->len); - } - } + decode_message(qdev, NULL, msg, NULL, NULL, true); /* request must have timed out, drop packet */ kfree(msg); } diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c index acd606160dc9b..b374d7ee54b68 100644 --- a/drivers/accel/rocket/rocket_job.c +++ b/drivers/accel/rocket/rocket_job.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,7 @@ rocket_copy_tasks(struct drm_device *dev, fail: kvfree(rjob->tasks); + rjob->tasks = NULL; return ret; } @@ -188,14 +190,19 @@ static int rocket_job_push(struct rocket_job *job) struct rocket_device *rdev = job->rdev; struct drm_gem_object **bos; struct ww_acquire_ctx acquire_ctx; + u32 bo_count; int ret = 0; - bos = kvmalloc_array(job->in_bo_count + job->out_bo_count, sizeof(void *), - GFP_KERNEL); + if (check_add_overflow(job->in_bo_count, job->out_bo_count, &bo_count)) + return -EINVAL; + + bos = kvmalloc_array(bo_count, sizeof(*bos), GFP_KERNEL); + if (!bos) + return -ENOMEM; memcpy(bos, job->in_bos, job->in_bo_count * sizeof(void *)); memcpy(&bos[job->in_bo_count], job->out_bos, job->out_bo_count * sizeof(void *)); - ret = drm_gem_lock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx); + ret = drm_gem_lock_reservations(bos, bo_count, &acquire_ctx); if (ret) goto err; @@ -220,7 +227,7 @@ static int rocket_job_push(struct rocket_job *job) rocket_attach_object_fences(job->out_bos, job->out_bo_count, job->inference_done_fence); err_unlock: - drm_gem_unlock_reservations(bos, job->in_bo_count + job->out_bo_count, &acquire_ctx); + drm_gem_unlock_reservations(bos, bo_count, &acquire_ctx); err: kvfree(bos); @@ -310,13 +317,13 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) dma_fence_put(job->done_fence); job->done_fence = dma_fence_get(fence); - ret = pm_runtime_get_sync(core->dev); + ret = pm_runtime_resume_and_get(core->dev); if (ret < 0) - return fence; + goto err_put_fences; ret = iommu_attach_group(job->domain->domain, core->iommu_group); if (ret < 0) - return fence; + goto err_put_pm; scoped_guard(mutex, &core->job_lock) { core->in_flight_job = job; @@ -324,6 +331,14 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job) } return fence; + +err_put_pm: + pm_runtime_put(core->dev); +err_put_fences: + dma_fence_put(job->done_fence); + job->done_fence = NULL; + dma_fence_put(fence); + return ERR_PTR(ret); } static void rocket_job_handle_irq(struct rocket_core *core) @@ -550,6 +565,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file kref_init(&rjob->refcount); rjob->rdev = rdev; + rjob->domain = rocket_iommu_domain_get(file_priv); ret = drm_sched_job_init(&rjob->base, &file_priv->sched_entity, @@ -575,8 +591,6 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file rjob->out_bo_count = job->out_bo_handle_count; - rjob->domain = rocket_iommu_domain_get(file_priv); - ret = rocket_job_push(rjob); if (ret) goto out_cleanup_job; diff --git a/drivers/accessibility/speakup/keyhelp.c b/drivers/accessibility/speakup/keyhelp.c index 822ceac830683..e632c53d6246e 100644 --- a/drivers/accessibility/speakup/keyhelp.c +++ b/drivers/accessibility/speakup/keyhelp.c @@ -8,6 +8,7 @@ */ #include +#include #include "spk_priv.h" #include "speakup.h" @@ -111,7 +112,7 @@ static void say_key(int key) spk_msg_get(MSG_KEYNAMES_START + (key - 1))); } -static int help_init(void) +static void help_init(void) { char start = SPACE; int i; @@ -120,13 +121,19 @@ static int help_init(void) state_tbl = spk_our_keys[0] + SHIFT_TBL_SIZE + 2; for (i = 0; i < num_funcs; i++) { char *cur_funcname = spk_msg_get(MSG_FUNCNAMES_START + i); + char first_letter; - if (start == *cur_funcname) + first_letter = tolower(*cur_funcname); + + /* Accept only 'a'..'z' to index letter_offsets[] safely */ + if (first_letter < 'a' || first_letter > 'z') + continue; + + if (start == first_letter) continue; - start = *cur_funcname; + start = first_letter; letter_offsets[(start & 31) - 1] = i; } - return 0; } int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) @@ -144,7 +151,7 @@ int spk_handle_help(struct vc_data *vc, u_char type, u_char ch, u_short key) synth_printf("%s\n", spk_msg_get(MSG_LEAVING_HELP)); return 1; } - ch |= 32; /* lower case */ + ch = tolower(ch); if (ch < 'a' || ch > 'z') return -1; if (letter_offsets[ch - 'a'] == -1) { diff --git a/drivers/accessibility/speakup/kobjects.c b/drivers/accessibility/speakup/kobjects.c index 0dfdb6608e022..943ef71b1329b 100644 --- a/drivers/accessibility/speakup/kobjects.c +++ b/drivers/accessibility/speakup/kobjects.c @@ -92,9 +92,9 @@ static void report_char_chartab_status(int reset, int received, int used, if (reset) { pr_info("%s reset to defaults\n", object_type[do_characters]); } else if (received) { - len = snprintf(buf, sizeof(buf), - " updated %d of %d %s\n", - used, received, object_type[do_characters]); + len = scnprintf(buf, sizeof(buf), + " updated %d of %d %s\n", + used, received, object_type[do_characters]); if (rejected) snprintf(buf + (len - 1), sizeof(buf) - (len - 1), " with %d reject%s\n", diff --git a/drivers/accessibility/speakup/main.c b/drivers/accessibility/speakup/main.c index e68cf1d83787f..afd1bf9e3a840 100644 --- a/drivers/accessibility/speakup/main.c +++ b/drivers/accessibility/speakup/main.c @@ -2444,6 +2444,7 @@ static int __init speakup_init(void) mutex_lock(&spk_mutex); synth_release(); mutex_unlock(&spk_mutex); + spk_ttyio_unregister_ldisc(); speakup_kobj_exit(); error_kobjects: diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c index 49539f7528c64..cd0fcfaeafc75 100644 --- a/drivers/acpi/acpi_apd.c +++ b/drivers/acpi/acpi_apd.c @@ -181,6 +181,12 @@ static const struct apd_device_desc hip08_spi_desc = { .setup = acpi_apd_setup, .fixed_clk_rate = 250000000, }; + +static const struct apd_device_desc leca_spi_desc = { + .setup = acpi_apd_setup, + .fixed_clk_rate = 400000000, +}; + #endif /* CONFIG_ARM64 */ #endif @@ -251,6 +257,7 @@ static const struct acpi_device_id acpi_apd_device_ids[] = { { "HISI02A2", APD_ADDR(hip08_i2c_desc) }, { "HISI02A3", APD_ADDR(hip08_lite_i2c_desc) }, { "HISI0173", APD_ADDR(hip08_spi_desc) }, + { "LECA0002", APD_ADDR(leca_spi_desc) }, { "NXP0001", APD_ADDR(nxp_i2c_desc) }, #endif { } diff --git a/drivers/acpi/acpi_pnp.c b/drivers/acpi/acpi_pnp.c index 4ad88187dc7a0..7eb9561b95893 100644 --- a/drivers/acpi/acpi_pnp.c +++ b/drivers/acpi/acpi_pnp.c @@ -385,7 +385,7 @@ static int is_cmos_rtc_device(struct acpi_device *adev) { "PNP0B02" }, {""}, }; - return !acpi_match_device_ids(adev, ids); + return !cmos_rtc_platform_device_present && !acpi_match_device_ids(adev, ids); } bool acpi_is_pnp_device(struct acpi_device *adev) diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index fcebfa7d507f5..0d62a75a4b920 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,9 @@ MODULE_AUTHOR("Rafael J. Wysocki"); /* Special value for disabled timer or expired timer wake policy. */ #define ACPI_TAD_WAKE_DISABLED (~(u32)0) +/* ACPI TAD RTC */ +#define ACPI_TAD_TZ_UNSPEC 2047 + struct acpi_tad_driver_data { u32 capabilities; }; @@ -67,6 +71,18 @@ struct acpi_tad_rt { u8 padding[3]; /* must be 0 */ } __packed; +static bool acpi_tad_rt_is_invalid(struct acpi_tad_rt *rt) +{ + return rt->year < 1900 || rt->year > 9999 || + rt->month < 1 || rt->month > 12 || + rt->hour > 23 || rt->minute > 59 || rt->second > 59 || + rt->tz < -1440 || + (rt->tz > 1440 && rt->tz != ACPI_TAD_TZ_UNSPEC) || + rt->daylight > 3; +} + +static DEFINE_MUTEX(acpi_tad_aml_lock); + static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) { acpi_handle handle = ACPI_HANDLE(dev); @@ -80,18 +96,16 @@ static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) unsigned long long retval; acpi_status status; - if (rt->year < 1900 || rt->year > 9999 || - rt->month < 1 || rt->month > 12 || - rt->hour > 23 || rt->minute > 59 || rt->second > 59 || - rt->tz < -1440 || (rt->tz > 1440 && rt->tz != 2047) || - rt->daylight > 3) - return -ERANGE; + if (acpi_tad_rt_is_invalid(rt)) + return -EINVAL; args[0].buffer.pointer = (u8 *)rt; args[0].buffer.length = sizeof(*rt); pm_runtime_get_sync(dev); + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_SRT", &arg_list, &retval); pm_runtime_put_sync(dev); @@ -102,43 +116,113 @@ static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) return 0; } -static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) +static int acpi_tad_evaluate_grt(struct device *dev, struct acpi_tad_rt *rt) { acpi_handle handle = ACPI_HANDLE(dev); struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER }; - union acpi_object *out_obj; - struct acpi_tad_rt *data; acpi_status status; int ret = -EIO; - pm_runtime_get_sync(dev); + guard(mutex)(&acpi_tad_aml_lock); status = acpi_evaluate_object(handle, "_GRT", NULL, &output); + if (ACPI_SUCCESS(status)) { + union acpi_object *out_obj; + + out_obj = output.pointer; + if (out_obj->type == ACPI_TYPE_BUFFER && + out_obj->buffer.length == sizeof(*rt)) { + struct acpi_tad_rt *data; + + data = (struct acpi_tad_rt *)(out_obj->buffer.pointer); + if (data->valid) { + memcpy(rt, data, sizeof(*rt)); + ret = 0; + } + } + } + ACPI_FREE(output.pointer); + return ret; +} - pm_runtime_put_sync(dev); +static int __acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) +{ + int ret; - if (ACPI_FAILURE(status)) - goto out_free; + ret = acpi_tad_evaluate_grt(dev, rt); + if (ret) + return ret; - out_obj = output.pointer; - if (out_obj->type != ACPI_TYPE_BUFFER) - goto out_free; + if (acpi_tad_rt_is_invalid(rt)) + return -ENODATA; - if (out_obj->buffer.length != sizeof(*rt)) - goto out_free; + return 0; +} - data = (struct acpi_tad_rt *)(out_obj->buffer.pointer); - if (!data->valid) - goto out_free; +static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) +{ + int ret; - memcpy(rt, data, sizeof(*rt)); - ret = 0; + pm_runtime_get_sync(dev); + + ret = __acpi_tad_get_real_time(dev, rt); + + pm_runtime_put_sync(dev); -out_free: - ACPI_FREE(output.pointer); return ret; } +static int __acpi_tad_wake_set(struct device *dev, char *method, u32 timer_id, + u32 value) +{ + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object args[] = { + { .type = ACPI_TYPE_INTEGER, }, + { .type = ACPI_TYPE_INTEGER, }, + }; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + unsigned long long retval; + acpi_status status; + + args[0].integer.value = timer_id; + args[1].integer.value = value; + + guard(mutex)(&acpi_tad_aml_lock); + + status = acpi_evaluate_integer(handle, method, &arg_list, &retval); + if (ACPI_FAILURE(status) || retval) + return -EIO; + + return 0; +} + +static int __acpi_tad_wake_read(struct device *dev, char *method, u32 timer_id, + unsigned long long *retval) +{ + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object args[] = { + { .type = ACPI_TYPE_INTEGER, }, + }; + struct acpi_object_list arg_list = { + .pointer = args, + .count = ARRAY_SIZE(args), + }; + acpi_status status; + + args[0].integer.value = timer_id; + + guard(mutex)(&acpi_tad_aml_lock); + + status = acpi_evaluate_integer(handle, method, &arg_list, retval); + if (ACPI_FAILURE(status)) + return -EIO; + + return 0; +} + static char *acpi_tad_rt_next_field(char *s, int *val) { char *p; @@ -251,31 +335,15 @@ static const struct attribute_group acpi_tad_time_attr_group = { static int acpi_tad_wake_set(struct device *dev, char *method, u32 timer_id, u32 value) { - acpi_handle handle = ACPI_HANDLE(dev); - union acpi_object args[] = { - { .type = ACPI_TYPE_INTEGER, }, - { .type = ACPI_TYPE_INTEGER, }, - }; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; - unsigned long long retval; - acpi_status status; - - args[0].integer.value = timer_id; - args[1].integer.value = value; + int ret; pm_runtime_get_sync(dev); - status = acpi_evaluate_integer(handle, method, &arg_list, &retval); + ret = __acpi_tad_wake_set(dev, method, timer_id, value); pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status) || retval) - return -EIO; - - return 0; + return ret; } static int acpi_tad_wake_write(struct device *dev, const char *buf, char *method, @@ -301,27 +369,17 @@ static int acpi_tad_wake_write(struct device *dev, const char *buf, char *method static ssize_t acpi_tad_wake_read(struct device *dev, char *buf, char *method, u32 timer_id, const char *specval) { - acpi_handle handle = ACPI_HANDLE(dev); - union acpi_object args[] = { - { .type = ACPI_TYPE_INTEGER, }, - }; - struct acpi_object_list arg_list = { - .pointer = args, - .count = ARRAY_SIZE(args), - }; unsigned long long retval; - acpi_status status; - - args[0].integer.value = timer_id; + int ret; pm_runtime_get_sync(dev); - status = acpi_evaluate_integer(handle, method, &arg_list, &retval); + ret = __acpi_tad_wake_read(dev, method, timer_id, &retval); pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status)) - return -EIO; + if (ret) + return ret; if ((u32)retval == ACPI_TAD_WAKE_DISABLED) return sprintf(buf, "%s\n", specval); @@ -372,6 +430,8 @@ static int acpi_tad_clear_status(struct device *dev, u32 timer_id) pm_runtime_get_sync(dev); + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_CWS", &arg_list, &retval); pm_runtime_put_sync(dev); @@ -413,6 +473,8 @@ static ssize_t acpi_tad_status_read(struct device *dev, char *buf, u32 timer_id) pm_runtime_get_sync(dev); + guard(mutex)(&acpi_tad_aml_lock); + status = acpi_evaluate_integer(handle, "_GWS", &arg_list, &retval); pm_runtime_put_sync(dev); diff --git a/drivers/acpi/acpica/dsmethod.c b/drivers/acpi/acpica/dsmethod.c index 45ec32e81903a..08bfe83030838 100644 --- a/drivers/acpi/acpica/dsmethod.c +++ b/drivers/acpi/acpica/dsmethod.c @@ -705,6 +705,8 @@ void acpi_ds_terminate_control_method(union acpi_operand_object *method_desc, struct acpi_walk_state *walk_state) { + u32 i; + struct acpi_namespace_node *ref_node; ACPI_FUNCTION_TRACE_PTR(ds_terminate_control_method, walk_state); @@ -715,6 +717,47 @@ acpi_ds_terminate_control_method(union acpi_operand_object *method_desc, } if (walk_state) { + /* + * Check if the return value is a ref_of reference to a method local + * or argument. If so, clear the reference to avoid use-after-free + * when the walk state is deleted. + */ + if (walk_state->return_desc && + (walk_state->return_desc->common.type == + ACPI_TYPE_LOCAL_REFERENCE) + && (walk_state->return_desc->reference.class == + ACPI_REFCLASS_REFOF)) { + ref_node = walk_state->return_desc->reference.object; + if (ref_node) { + + /* Check against method locals */ + for (i = 0; i < ACPI_METHOD_NUM_LOCALS; i++) { + if (ref_node == + &walk_state->local_variables[i]) { + acpi_ut_remove_reference + (walk_state->return_desc); + walk_state->return_desc = NULL; + break; + } + } + + /* Check against method arguments if not already cleared */ + if (walk_state->return_desc) { + for (i = 0; i < ACPI_METHOD_NUM_ARGS; + i++) { + if (ref_node == + &walk_state->arguments[i]) { + acpi_ut_remove_reference + (walk_state-> + return_desc); + walk_state-> + return_desc = NULL; + break; + } + } + } + } + } /* Delete all arguments and locals */ diff --git a/drivers/acpi/acpica/evhandler.c b/drivers/acpi/acpica/evhandler.c index 5a35dae945e20..f16c1148e602e 100644 --- a/drivers/acpi/acpica/evhandler.c +++ b/drivers/acpi/acpica/evhandler.c @@ -130,6 +130,14 @@ acpi_ev_has_default_handler(struct acpi_namespace_node *node, /* Walk the linked list of handlers for this object */ while (handler_obj) { + + /* Validate handler object type before accessing fields */ + + if (handler_obj->common.type != + ACPI_TYPE_LOCAL_ADDRESS_HANDLER) { + break; + } + if (handler_obj->address_space.space_id == space_id) { if (handler_obj->address_space.handler_flags & ACPI_ADDR_HANDLER_DEFAULT_INSTALLED) { @@ -292,6 +300,9 @@ union acpi_operand_object *acpi_ev_find_region_handler(acpi_adr_space_type /* Walk the handler list for this device */ while (handler_obj) { + if (handler_obj->common.type != ACPI_TYPE_LOCAL_ADDRESS_HANDLER) { + break; + } /* Same space_id indicates a handler is installed */ diff --git a/drivers/acpi/acpica/exconfig.c b/drivers/acpi/acpica/exconfig.c index 4d7dd0fc6b07b..894695db0cf94 100644 --- a/drivers/acpi/acpica/exconfig.c +++ b/drivers/acpi/acpica/exconfig.c @@ -90,6 +90,8 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state, union acpi_operand_object *return_obj; union acpi_operand_object *ddb_handle; u32 table_index; + char oem_id[ACPI_OEM_ID_SIZE + 1]; + char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; ACPI_FUNCTION_TRACE(ex_load_table_op); @@ -102,12 +104,32 @@ acpi_ex_load_table_op(struct acpi_walk_state *walk_state, *return_desc = return_obj; + /* + * Validate OEM ID and OEM Table ID string lengths. + * acpi_tb_find_table expects strings that can safely read + * ACPI_OEM_ID_SIZE and ACPI_OEM_TABLE_ID_SIZE bytes. + */ + if ((operand[1]->string.length > ACPI_OEM_ID_SIZE) || + (operand[2]->string.length > ACPI_OEM_TABLE_ID_SIZE)) { + return_ACPI_STATUS(AE_AML_STRING_LIMIT); + } + + /* + * Copy OEM strings to local buffers with guaranteed null-termination. + * This prevents heap-buffer-overflow when acpi_tb_find_table reads + * ACPI_OEM_ID_SIZE/ACPI_OEM_TABLE_ID_SIZE bytes. + */ + memcpy(oem_id, operand[1]->string.pointer, operand[1]->string.length); + oem_id[operand[1]->string.length] = 0; + memcpy(oem_table_id, operand[2]->string.pointer, + operand[2]->string.length); + oem_table_id[operand[2]->string.length] = 0; + /* Find the ACPI table in the RSDT/XSDT */ acpi_ex_exit_interpreter(); status = acpi_tb_find_table(operand[0]->string.pointer, - operand[1]->string.pointer, - operand[2]->string.pointer, &table_index); + oem_id, oem_table_id, &table_index); acpi_ex_enter_interpreter(); if (ACPI_FAILURE(status)) { if (status != AE_NOT_FOUND) { diff --git a/drivers/acpi/acpica/exoparg3.c b/drivers/acpi/acpica/exoparg3.c index c8c8c4e49563e..df770f36b4d08 100644 --- a/drivers/acpi/acpica/exoparg3.c +++ b/drivers/acpi/acpica/exoparg3.c @@ -152,7 +152,7 @@ acpi_status acpi_ex_opcode_3A_1T_1R(struct acpi_walk_state *walk_state) /* Truncate request if larger than the actual String/Buffer */ - else if ((index + length) > operand[0]->string.length) { + else if ((index + length) > operand[0]->string.length || (index + length) < index) { /* Check for overflow */ length = (acpi_size)operand[0]->string.length - (acpi_size)index; diff --git a/drivers/acpi/acpica/nsnames.c b/drivers/acpi/acpica/nsnames.c index 22aeeeb56cffd..19802da865c54 100644 --- a/drivers/acpi/acpica/nsnames.c +++ b/drivers/acpi/acpica/nsnames.c @@ -222,6 +222,12 @@ acpi_ns_build_normalized_path(struct acpi_namespace_node *node, goto build_trailing_null; } + /* Validate the Node to avoid use-after-free vulnerabilities */ + + if (ACPI_GET_DESCRIPTOR_TYPE(node) != ACPI_DESC_TYPE_NAMED) { + goto build_trailing_null; + } + next_node = node; while (next_node && next_node != acpi_gbl_root_node) { if (next_node != node) { diff --git a/drivers/acpi/acpica/nsprepkg.c b/drivers/acpi/acpica/nsprepkg.c index ca137ce5674fb..c32770570120a 100644 --- a/drivers/acpi/acpica/nsprepkg.c +++ b/drivers/acpi/acpica/nsprepkg.c @@ -631,6 +631,13 @@ acpi_ns_custom_package(struct acpi_evaluate_info *info, /* Get version number, must be Integer */ + if (!(*elements)) { + ACPI_WARN_PREDEFINED((AE_INFO, info->full_pathname, + info->node_flags, + "Return Package has a NULL version element")); + return_ACPI_STATUS(AE_AML_OPERAND_TYPE); + } + if ((*elements)->common.type != ACPI_TYPE_INTEGER) { ACPI_WARN_PREDEFINED((AE_INFO, info->full_pathname, info->node_flags, diff --git a/drivers/acpi/acpica/nsxfname.c b/drivers/acpi/acpica/nsxfname.c index 1db831545ec8c..821fb4930e9d8 100644 --- a/drivers/acpi/acpica/nsxfname.c +++ b/drivers/acpi/acpica/nsxfname.c @@ -512,6 +512,10 @@ acpi_status acpi_install_method(u8 *buffer) parser_state.aml += acpi_ps_get_opcode_size(opcode); parser_state.pkg_end = acpi_ps_get_next_package_end(&parser_state); + if ((parser_state.pkg_end > parser_state.aml_end) || + (parser_state.pkg_end < parser_state.aml)) { + return (AE_AML_PACKAGE_LIMIT); + } path = acpi_ps_get_next_namestring(&parser_state); method_flags = *parser_state.aml++; diff --git a/drivers/acpi/acpica/psargs.c b/drivers/acpi/acpica/psargs.c index 6f6ae38ec044c..a7293332197aa 100644 --- a/drivers/acpi/acpica/psargs.c +++ b/drivers/acpi/acpica/psargs.c @@ -48,6 +48,7 @@ acpi_ps_get_next_package_length(struct acpi_parse_state *parser_state) u32 package_length = 0; u32 byte_count; u8 byte_zero_mask = 0x3F; /* Default [0:5] */ + u32 remaining; ACPI_FUNCTION_TRACE(ps_get_next_package_length); @@ -55,7 +56,23 @@ acpi_ps_get_next_package_length(struct acpi_parse_state *parser_state) * Byte 0 bits [6:7] contain the number of additional bytes * used to encode the package length, either 0,1,2, or 3 */ + + /* Check if we have at least one byte to read */ + remaining = (u32)ACPI_PTR_DIFF(parser_state->aml_end, aml); + if (remaining == 0) { + return_UINT32(0); + } + byte_count = (aml[0] >> 6); + + /* Validate byte_count and ensure we have enough bytes to read */ + if (byte_count >= remaining) { + + /* Clamp to available bytes and advance to end */ + parser_state->aml = parser_state->aml_end; + return_UINT32(0); + } + parser_state->aml += ((acpi_size)byte_count + 1); /* Get bytes 3, 2, 1 as needed */ @@ -131,10 +148,16 @@ char *acpi_ps_get_next_namestring(struct acpi_parse_state *parser_state) /* Point past any namestring prefix characters (backslash or carat) */ - while (ACPI_IS_ROOT_PREFIX(*end) || ACPI_IS_PARENT_PREFIX(*end)) { + while (end < parser_state->aml_end && + (ACPI_IS_ROOT_PREFIX(*end) || ACPI_IS_PARENT_PREFIX(*end))) { end++; } + if (end >= parser_state->aml_end) { + parser_state->aml = parser_state->aml_end; + return_PTR(NULL); + } + /* Decode the path prefix character */ switch (*end) { @@ -159,6 +182,11 @@ char *acpi_ps_get_next_namestring(struct acpi_parse_state *parser_state) /* Multiple name segments, 4 chars each, count in next byte */ + if ((end + 1) >= parser_state->aml_end) { + parser_state->aml = parser_state->aml_end; + return_PTR(NULL); + } + end += 2 + (*(end + 1) * ACPI_NAMESEG_SIZE); break; @@ -170,6 +198,11 @@ char *acpi_ps_get_next_namestring(struct acpi_parse_state *parser_state) break; } + if (end > parser_state->aml_end) { + parser_state->aml = parser_state->aml_end; + return_PTR(NULL); + } + parser_state->aml = end; return_PTR((char *)start); } @@ -367,6 +400,8 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, u32 length; u16 opcode; u8 *aml = parser_state->aml; + u32 remaining = (u32)ACPI_PTR_DIFF(parser_state->aml_end, aml); + u64 partial_value; ACPI_FUNCTION_TRACE_U32(ps_get_next_simple_arg, arg_type); @@ -376,8 +411,13 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, /* Get 1 byte from the AML stream */ opcode = AML_BYTE_OP; - arg->common.value.integer = (u64) *aml; - length = 1; + if (remaining >= 1) { + arg->common.value.integer = (u64)*aml; + length = 1; + } else { + arg->common.value.integer = 0; + length = 0; + } break; case ARGP_WORDDATA: @@ -385,8 +425,19 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, /* Get 2 bytes from the AML stream */ opcode = AML_WORD_OP; - ACPI_MOVE_16_TO_64(&arg->common.value.integer, aml); - length = 2; + if (remaining >= 2) { + ACPI_MOVE_16_TO_64(&arg->common.value.integer, aml); + length = 2; + } else { + arg->common.value.integer = 0; + length = 0; + if (remaining > 0) { + partial_value = 0; + memcpy(&partial_value, aml, remaining); + arg->common.value.integer = partial_value; + length = remaining; + } + } break; case ARGP_DWORDDATA: @@ -394,8 +445,19 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, /* Get 4 bytes from the AML stream */ opcode = AML_DWORD_OP; - ACPI_MOVE_32_TO_64(&arg->common.value.integer, aml); - length = 4; + if (remaining >= 4) { + ACPI_MOVE_32_TO_64(&arg->common.value.integer, aml); + length = 4; + } else { + arg->common.value.integer = 0; + length = 0; + if (remaining > 0) { + partial_value = 0; + memcpy(&partial_value, aml, remaining); + arg->common.value.integer = partial_value; + length = remaining; + } + } break; case ARGP_QWORDDATA: @@ -403,8 +465,19 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, /* Get 8 bytes from the AML stream */ opcode = AML_QWORD_OP; - ACPI_MOVE_64_TO_64(&arg->common.value.integer, aml); - length = 8; + if (remaining >= 8) { + ACPI_MOVE_64_TO_64(&arg->common.value.integer, aml); + length = 8; + } else { + arg->common.value.integer = 0; + length = 0; + if (remaining > 0) { + partial_value = 0; + memcpy(&partial_value, aml, remaining); + arg->common.value.integer = partial_value; + length = remaining; + } + } break; case ARGP_CHARLIST: @@ -417,10 +490,28 @@ acpi_ps_get_next_simple_arg(struct acpi_parse_state *parser_state, /* Find the null terminator */ length = 0; - while (aml[length]) { + while ((length < remaining) && aml[length]) { length++; } - length++; + if (length < remaining) { + + /* Account for the terminating null */ + length++; + } else { + /* + * No terminator found - add null at buffer boundary + * and report a warning + */ + ACPI_WARNING((AE_INFO, + "Invalid AML string: no null terminator, truncating at offset %u", + (u32)(aml - parser_state->aml))); + + /* Add null terminator at the boundary */ + if (remaining > 0) { + aml[remaining - 1] = 0; + length = remaining; + } + } break; case ARGP_NAME: @@ -474,6 +565,10 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state ASL_CV_CAPTURE_COMMENTS_ONLY(parser_state); aml = parser_state->aml; + if (aml >= parser_state->aml_end) { + return_PTR(NULL); + } + /* Determine field type */ switch (ACPI_GET8(parser_state->aml)) { @@ -522,6 +617,11 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state /* Get the 4-character name */ + if ((parser_state->aml + ACPI_NAMESEG_SIZE) > + parser_state->aml_end) { + acpi_ps_free_op(field); + return_PTR(NULL); + } ACPI_MOVE_32_TO_32(&name, parser_state->aml); acpi_ps_set_name(field, name); parser_state->aml += ACPI_NAMESEG_SIZE; @@ -567,6 +667,10 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state /* Get the two bytes (Type/Attribute) */ + if ((parser_state->aml + 2) > parser_state->aml_end) { + acpi_ps_free_op(field); + return_PTR(NULL); + } access_type = ACPI_GET8(parser_state->aml); parser_state->aml++; access_attribute = ACPI_GET8(parser_state->aml); @@ -578,6 +682,10 @@ static union acpi_parse_object *acpi_ps_get_next_field(struct acpi_parse_state /* This opcode has a third byte, access_length */ if (opcode == AML_INT_EXTACCESSFIELD_OP) { + if (parser_state->aml >= parser_state->aml_end) { + acpi_ps_free_op(field); + return_PTR(NULL); + } access_length = ACPI_GET8(parser_state->aml); parser_state->aml++; @@ -775,6 +883,10 @@ acpi_ps_get_next_arg(struct acpi_walk_state *walk_state, parser_state->pkg_end = acpi_ps_get_next_package_end(parser_state); + if ((parser_state->pkg_end > parser_state->aml_end) + || (parser_state->pkg_end < parser_state->aml)) { + return_ACPI_STATUS(AE_AML_PACKAGE_LIMIT); + } break; case ARGP_FIELDLIST: diff --git a/drivers/acpi/acpica/psloop.c b/drivers/acpi/acpica/psloop.c index c989cadf271ce..7c3caf0ccab62 100644 --- a/drivers/acpi/acpica/psloop.c +++ b/drivers/acpi/acpica/psloop.c @@ -361,6 +361,13 @@ acpi_status acpi_ps_parse_loop(struct acpi_walk_state *walk_state) walk_state->parser_state.aml = acpi_ps_get_next_package_end (&walk_state->parser_state); + if ((walk_state->parser_state.aml > + walk_state->parser_state.aml_end) + || (walk_state->parser_state.aml < + walk_state->aml)) { + return_ACPI_STATUS + (AE_AML_PACKAGE_LIMIT); + } walk_state->aml = walk_state->parser_state.aml; } @@ -421,11 +428,22 @@ acpi_status acpi_ps_parse_loop(struct acpi_walk_state *walk_state) parser_state->aml = acpi_ps_get_next_package_end (parser_state); + if ((parser_state->aml > + parser_state->aml_end) + || (parser_state->aml < + walk_state->control_state-> + control.aml_predicate_start)) { + return_ACPI_STATUS + (AE_AML_PACKAGE_LIMIT); + } walk_state->aml = parser_state->aml; ACPI_ERROR((AE_INFO, "Skipping While/If block")); - if (*walk_state->aml == AML_ELSE_OP) { + if ((walk_state->aml < + parser_state->aml_end) + && (*walk_state->aml == + AML_ELSE_OP)) { ACPI_ERROR((AE_INFO, "Skipping Else block")); walk_state->parser_state.aml = @@ -433,6 +451,16 @@ acpi_status acpi_ps_parse_loop(struct acpi_walk_state *walk_state) walk_state->parser_state.aml = acpi_ps_get_next_package_end (parser_state); + if ((walk_state->parser_state. + aml > + walk_state->parser_state. + aml_end) + || (walk_state-> + parser_state.aml < + walk_state->aml)) { + return_ACPI_STATUS + (AE_AML_PACKAGE_LIMIT); + } walk_state->aml = parser_state->aml; } diff --git a/drivers/acpi/acpica/psparse.c b/drivers/acpi/acpica/psparse.c index 55a416e56fd83..41587ee0a9c78 100644 --- a/drivers/acpi/acpica/psparse.c +++ b/drivers/acpi/acpica/psparse.c @@ -70,6 +70,9 @@ u16 acpi_ps_peek_opcode(struct acpi_parse_state * parser_state) u16 opcode; aml = parser_state->aml; + if (aml >= parser_state->aml_end) { + return (0xFFFF); + } opcode = (u16) ACPI_GET8(aml); if (opcode == AML_EXTENDED_PREFIX) { @@ -77,6 +80,9 @@ u16 acpi_ps_peek_opcode(struct acpi_parse_state * parser_state) /* Extended opcode, get the second opcode byte */ aml++; + if (aml >= parser_state->aml_end) { + return (0xFFFF); + } opcode = (u16) ((opcode << 8) | ACPI_GET8(aml)); } @@ -300,6 +306,7 @@ acpi_ps_next_parse_state(struct acpi_walk_state *walk_state, { struct acpi_parse_state *parser_state = &walk_state->parser_state; acpi_status status = AE_CTRL_PENDING; + u8 *aml; ACPI_FUNCTION_TRACE_PTR(ps_next_parse_state, op); @@ -344,7 +351,14 @@ acpi_ps_next_parse_state(struct acpi_walk_state *walk_state, * Predicate of an IF was true, and we are at the matching ELSE. * Just close out this package */ + aml = parser_state->aml; + parser_state->aml = acpi_ps_get_next_package_end(parser_state); + if ((parser_state->aml > parser_state->aml_end) || + (parser_state->aml < aml)) { + status = AE_AML_PACKAGE_LIMIT; + break; + } status = AE_CTRL_PENDING; break; diff --git a/drivers/acpi/acpica/utcopy.c b/drivers/acpi/acpica/utcopy.c index 80458e70ac2bf..9ecf5c3f49ba6 100644 --- a/drivers/acpi/acpica/utcopy.c +++ b/drivers/acpi/acpica/utcopy.c @@ -731,7 +731,15 @@ acpi_ut_copy_simple_object(union acpi_operand_object *source_desc, break; } - acpi_ut_add_reference(source_desc->reference.object); + /* + * Local/Arg/Debug references do not have a valid Object pointer + * that can be referenced + */ + if ((source_desc->reference.class != ACPI_REFCLASS_LOCAL) && + (source_desc->reference.class != ACPI_REFCLASS_ARG) && + (source_desc->reference.class != ACPI_REFCLASS_DEBUG)) { + acpi_ut_add_reference(source_desc->reference.object); + } break; case ACPI_TYPE_REGION: diff --git a/drivers/acpi/acpica/utresrc.c b/drivers/acpi/acpica/utresrc.c index e1cc3d3487508..86ebd9fb869af 100644 --- a/drivers/acpi/acpica/utresrc.c +++ b/drivers/acpi/acpica/utresrc.c @@ -165,6 +165,28 @@ acpi_ut_walk_aml_resources(struct acpi_walk_state *walk_state, /* Walk the byte list, abort on any invalid descriptor type or length */ while (aml < end_aml) { + /* + * Validate that the remaining buffer space can hold enough + * bytes to safely access fields during validation. + * For large resource descriptors (bit 7 set), we need enough + * bytes to access the Type field in serial_bus resources. + * Small resource descriptors only need sizeof(struct aml_resource_end_tag). + */ + if ((acpi_size)(end_aml - aml) < + sizeof(struct aml_resource_end_tag)) { + return_ACPI_STATUS(AE_AML_BUFFER_LENGTH); + } + + /* + * For large resource descriptors, ensure enough space for + * the header plus serial_bus Type field access. + */ + if ((ACPI_GET8(aml) & ACPI_RESOURCE_NAME_LARGE) && + ((acpi_size)(end_aml - aml) < + ACPI_OFFSET(struct aml_resource_common_serialbus, + type) + 1)) { + return_ACPI_STATUS(AE_AML_BUFFER_LENGTH); + } /* Validate the Resource Type and Resource Length */ @@ -182,6 +204,14 @@ acpi_ut_walk_aml_resources(struct acpi_walk_state *walk_state, length = acpi_ut_get_descriptor_length(aml); + /* + * Validate that the descriptor length doesn't exceed the + * remaining buffer size to prevent reading beyond the end. + */ + if (length > (acpi_size)(end_aml - aml)) { + return_ACPI_STATUS(AE_AML_BUFFER_LENGTH); + } + /* Invoke the user function */ if (user_function) { diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c index bf65e34615314..15ab797641cfc 100644 --- a/drivers/acpi/apei/erst.c +++ b/drivers/acpi/apei/erst.c @@ -108,7 +108,7 @@ static inline u64 erst_get_timeout(void) if (erst_erange.attr & ERST_RANGE_SLOW) { timeout = ((erst_erange.timings & ERST_EXEC_TIMING_MAX_MASK) >> - ERST_EXEC_TIMING_MAX_SHIFT) * NSEC_PER_MSEC; + ERST_EXEC_TIMING_MAX_SHIFT) * NSEC_PER_USEC; if (timeout < FIRMWARE_TIMEOUT) timeout = FIRMWARE_TIMEOUT; } diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index fc96a5e234f06..5b52455d86e02 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -581,7 +581,7 @@ static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, return false; p = (char *)(err + 1); - length -= sizeof(err); + length -= sizeof(*err); for (i = 0; i < err->err_info_num; i++) { struct cper_arm_err_info *err_info; @@ -736,7 +736,7 @@ static DEFINE_KFIFO(cxl_cper_prot_err_fifo, struct cxl_cper_prot_err_work_data, CXL_CPER_PROT_ERR_FIFO_DEPTH); /* Synchronize schedule_work() with cxl_cper_prot_err_work changes */ -static DEFINE_SPINLOCK(cxl_cper_prot_err_work_lock); +static DEFINE_RAW_SPINLOCK(cxl_cper_prot_err_work_lock); struct work_struct *cxl_cper_prot_err_work; static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, @@ -749,7 +749,7 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, if (cxl_cper_sec_prot_err_valid(prot_err)) return; - guard(spinlock_irqsave)(&cxl_cper_prot_err_work_lock); + guard(raw_spinlock_irqsave)(&cxl_cper_prot_err_work_lock); if (!cxl_cper_prot_err_work) return; @@ -787,10 +787,11 @@ static void cxl_cper_post_prot_err(struct cxl_cper_sec_prot_err *prot_err, int cxl_cper_register_prot_err_work(struct work_struct *work) { - if (cxl_cper_prot_err_work) - return -EINVAL; + guard(raw_spinlock_irqsave)(&cxl_cper_prot_err_work_lock); - guard(spinlock)(&cxl_cper_prot_err_work_lock); + if (WARN_ONCE(cxl_cper_prot_err_work, + "CPER-CXL kfifo consumer already registered\n")) + return -EINVAL; cxl_cper_prot_err_work = work; return 0; } @@ -798,11 +799,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_register_prot_err_work, "CXL"); int cxl_cper_unregister_prot_err_work(struct work_struct *work) { - if (cxl_cper_prot_err_work != work) - return -EINVAL; + scoped_guard(raw_spinlock_irqsave, &cxl_cper_prot_err_work_lock) { + if (WARN_ONCE(cxl_cper_prot_err_work != work, + "CPER-CXL kfifo consumer mismatch on unregister\n")) + return -EINVAL; + cxl_cper_prot_err_work = NULL; + } + + cancel_work_sync(work); + + /* Discard stale entries so they are not replayed on next module load */ + kfifo_reset(&cxl_cper_prot_err_fifo); - guard(spinlock)(&cxl_cper_prot_err_work_lock); - cxl_cper_prot_err_work = NULL; return 0; } EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_prot_err_work, "CXL"); @@ -818,7 +826,7 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_prot_err_kfifo_get, "CXL"); DEFINE_KFIFO(cxl_cper_fifo, struct cxl_cper_work_data, CXL_CPER_FIFO_DEPTH); /* Synchronize schedule_work() with cxl_cper_work changes */ -static DEFINE_SPINLOCK(cxl_cper_work_lock); +static DEFINE_RAW_SPINLOCK(cxl_cper_work_lock); struct work_struct *cxl_cper_work; static void cxl_cper_post_event(enum cxl_event_type event_type, @@ -838,7 +846,7 @@ static void cxl_cper_post_event(enum cxl_event_type event_type, return; } - guard(spinlock_irqsave)(&cxl_cper_work_lock); + guard(raw_spinlock_irqsave)(&cxl_cper_work_lock); if (!cxl_cper_work) return; @@ -856,10 +864,11 @@ static void cxl_cper_post_event(enum cxl_event_type event_type, int cxl_cper_register_work(struct work_struct *work) { - if (cxl_cper_work) + guard(raw_spinlock_irqsave)(&cxl_cper_work_lock); + if (WARN_ONCE(cxl_cper_work, + "CXL CPER kfifo consumer already registered\n")) return -EINVAL; - guard(spinlock)(&cxl_cper_work_lock); cxl_cper_work = work; return 0; } @@ -867,11 +876,18 @@ EXPORT_SYMBOL_NS_GPL(cxl_cper_register_work, "CXL"); int cxl_cper_unregister_work(struct work_struct *work) { - if (cxl_cper_work != work) - return -EINVAL; + scoped_guard(raw_spinlock_irqsave, &cxl_cper_work_lock) { + if (WARN_ONCE(cxl_cper_work != work, + "CXL CPER kfifo consumer mismatch on unregister\n")) + return -EINVAL; + cxl_cper_work = NULL; + } + + cancel_work_sync(work); + + /* Discard stale entries so they are not replayed on next module load */ + kfifo_reset(&cxl_cper_fifo); - guard(spinlock)(&cxl_cper_work_lock); - cxl_cper_work = NULL; return 0; } EXPORT_SYMBOL_NS_GPL(cxl_cper_unregister_work, "CXL"); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 8196c17b5a970..296c2981b1a73 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -151,27 +151,28 @@ static int acpi_battery_technology(struct acpi_battery *battery) static int acpi_battery_get_state(struct acpi_battery *battery); -static int acpi_battery_is_charged(struct acpi_battery *battery) +static bool acpi_battery_is_full(struct acpi_battery *battery) { - /* charging, discharging, critical low or charge limited */ - if (battery->state != 0) - return 0; - /* battery not reporting charge */ if (battery->capacity_now == ACPI_BATTERY_VALUE_UNKNOWN || battery->capacity_now == 0) - return 0; + return false; /* good batteries update full_charge as the batteries degrade */ if (battery->full_charge_capacity == battery->capacity_now) - return 1; + return true; /* fallback to using design values for broken batteries */ - if (battery->design_capacity <= battery->capacity_now) - return 1; + return battery->design_capacity <= battery->capacity_now; +} - /* we don't do any sort of metric based on percentages */ - return 0; +static int acpi_battery_is_charged(struct acpi_battery *battery) +{ + /* charging, discharging, critical low or charge limited */ + if (battery->state != 0) + return 0; + + return acpi_battery_is_full(battery); } static bool acpi_battery_is_degraded(struct acpi_battery *battery) @@ -212,13 +213,14 @@ static int acpi_battery_get_property(struct power_supply *psy, if (battery->state & ACPI_BATTERY_STATE_DISCHARGING) val->intval = acpi_battery_handle_discharging(battery); else if (battery->state & ACPI_BATTERY_STATE_CHARGING) - /* Validate the status by checking the current. */ - if (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && - battery->rate_now == 0) { - /* On charge but no current (0W/0mA). */ - val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; - } else { + /* Check the rate and capacity to validate the status. */ + if (!acpi_battery_is_full(battery) || + (battery->rate_now != ACPI_BATTERY_VALUE_UNKNOWN && + battery->rate_now > 0)) { val->intval = POWER_SUPPLY_STATUS_CHARGING; + } else { + /* Full and zero rate. */ + val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; } else if (battery->state & ACPI_BATTERY_STATE_CHARGE_LIMITING) val->intval = POWER_SUPPLY_STATUS_NOT_CHARGING; diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 60d2548f0eea4..531777557b47c 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -768,7 +768,6 @@ int acpi_cppc_processor_probe(struct acpi_processor *pr) } if (cpc_rev > CPPC_V3_REV) { num_ent = CPPC_V3_NUM_ENT; - cpc_rev = CPPC_V3_REV; } cpc_ptr->num_entries = num_ent; @@ -1289,10 +1288,20 @@ static int cppc_set_reg_val(int cpu, enum cppc_regs reg_idx, u64 val) * @cpunum: CPU from which to get desired performance. * @desired_perf: Return address. * - * Return: 0 for success, -EIO otherwise. + * Return: 0 for success, -EOPNOTSUPP for _CPC revision 4 or later, and a + * negative errno otherwise. */ int cppc_get_desired_perf(int cpunum, u64 *desired_perf) { + struct cpc_desc *cpc_desc = per_cpu(cpc_desc_ptr, cpunum); + + if (!cpc_desc) + return -ENODEV; + + /* _CPC revision 4 no longer specifies Desired Performance as readable. */ + if (cpc_desc->version > CPPC_V3_REV) + return -EOPNOTSUPP; + return cppc_get_reg_val(cpunum, DESIRED_PERF, desired_perf); } EXPORT_SYMBOL_GPL(cppc_get_desired_perf); diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 1f4fc78a124fa..16213dfb3aabf 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -1512,6 +1512,24 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) IRQF_SHARED | IRQF_ONESHOT, "ACPI EC", ec) >= 0; } +static int ec_prepare_gpio_irq(struct acpi_ec *ec, struct acpi_device *device) +{ + int irq; + + if (!device || ec->gpe >= 0 || ec->irq >= 0) + return 0; + + /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ + irq = acpi_dev_gpio_irq_get(device, 0); + if (irq == -EPROBE_DEFER) + return irq; + + if (irq >= 0) + ec->irq = irq; + + return 0; +} + /** * ec_install_handlers - Install service callbacks and register query methods. * @ec: Target EC. @@ -1526,7 +1544,6 @@ static bool install_gpio_irq_event_handler(struct acpi_ec *ec) * Return: * -ENODEV if the address space handler cannot be installed, which means * "unable to handle transactions", - * -EPROBE_DEFER if GPIO IRQ acquisition needs to be deferred, * or 0 (success) otherwise. */ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, @@ -1559,19 +1576,6 @@ static int ec_install_handlers(struct acpi_ec *ec, struct acpi_device *device, if (!device) return 0; - if (ec->gpe < 0) { - /* ACPI reduced hardware platforms use a GpioInt from _CRS. */ - int irq = acpi_dev_gpio_irq_get(device, 0); - /* - * Bail out right away for deferred probing or complete the - * initialization regardless of any other errors. - */ - if (irq == -EPROBE_DEFER) - return -EPROBE_DEFER; - else if (irq >= 0) - ec->irq = irq; - } - if (!test_bit(EC_FLAGS_QUERY_METHODS_INSTALLED, &ec->flags)) { /* Find and register all query methods */ acpi_walk_namespace(ACPI_TYPE_METHOD, ec->handle, 1, @@ -1649,6 +1653,14 @@ static int acpi_ec_setup(struct acpi_ec *ec, struct acpi_device *device, bool ca { int ret; + /* + * GPIO IRQ lookup can defer. Do it before publishing the EC + * OpRegion to AML to avoid a spurious _REG(disconnect). + */ + ret = ec_prepare_gpio_irq(ec, device); + if (ret) + return ret; + /* First EC capable of handling transactions */ if (!first_ec) first_ec = ec; diff --git a/drivers/acpi/pci_link.c b/drivers/acpi/pci_link.c index e4560b33b8ad1..bed7dc85612e3 100644 --- a/drivers/acpi/pci_link.c +++ b/drivers/acpi/pci_link.c @@ -761,7 +761,7 @@ static int acpi_pci_link_resume(struct acpi_pci_link *link) return 0; } -static void irqrouter_resume(void) +static void irqrouter_resume(void *data) { struct acpi_pci_link *link; @@ -888,10 +888,14 @@ static int __init acpi_irq_balance_set(char *str) __setup("acpi_irq_balance", acpi_irq_balance_set); -static struct syscore_ops irqrouter_syscore_ops = { +static const struct syscore_ops irqrouter_syscore_ops = { .resume = irqrouter_resume, }; +static struct syscore irqrouter_syscore = { + .ops = &irqrouter_syscore_ops, +}; + void __init acpi_pci_link_init(void) { if (acpi_noirq) @@ -904,6 +908,6 @@ void __init acpi_pci_link_init(void) else acpi_irq_balance = 0; } - register_syscore_ops(&irqrouter_syscore_ops); + register_syscore(&irqrouter_syscore); acpi_scan_add_handler(&pci_link_handler); } diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c index 74ade41603145..34b53dc892b21 100644 --- a/drivers/acpi/pci_root.c +++ b/drivers/acpi/pci_root.c @@ -730,7 +730,6 @@ static int acpi_pci_root_add(struct acpi_device *device, dev_err(&device->dev, "Bus %04x:%02x not present in PCI namespace\n", root->segment, (unsigned int)root->secondary.start); - device->driver_data = NULL; result = -ENODEV; goto remove_dmar; } @@ -760,12 +759,17 @@ static int acpi_pci_root_add(struct acpi_device *device, pci_lock_rescan_remove(); pci_bus_add_devices(root->bus); pci_unlock_rescan_remove(); + + /* Clear _DEP dependencies to allow consumers to enumerate */ + acpi_dev_clear_dependencies(device); + return 1; remove_dmar: if (hotadd) dmar_device_remove(handle); end: + device->driver_data = NULL; kfree(root); return result; } @@ -789,6 +793,7 @@ static void acpi_pci_root_remove(struct acpi_device *device) pci_unlock_rescan_remove(); + device->driver_data = NULL; kfree(root); } diff --git a/drivers/acpi/pfr_update.c b/drivers/acpi/pfr_update.c index 6283105bb0e8b..9afd2c52fdbd8 100644 --- a/drivers/acpi/pfr_update.c +++ b/drivers/acpi/pfr_update.c @@ -120,7 +120,7 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, struct pfru_device *pfru_dev) { acpi_handle handle = ACPI_HANDLE(pfru_dev->parent_dev); - union acpi_object *out_obj; + union acpi_object *out_obj, *elem; int ret = -EINVAL; out_obj = acpi_evaluate_dsm_typed(handle, &pfru_guid, @@ -150,7 +150,9 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, goto free_acpi_buffer; } - cap_hdr->status = out_obj->package.elements[CAP_STATUS_IDX].integer.value; + elem = out_obj->package.elements; + + cap_hdr->status = elem[CAP_STATUS_IDX].integer.value; if (cap_hdr->status != DSM_SUCCEED) { ret = -EBUSY; dev_dbg(pfru_dev->parent_dev, "Query cap Error Status:%d\n", @@ -158,29 +160,30 @@ static int query_capability(struct pfru_update_cap_info *cap_hdr, goto free_acpi_buffer; } - cap_hdr->update_cap = out_obj->package.elements[CAP_UPDATE_IDX].integer.value; + if (elem[CAP_CODE_TYPE_IDX].buffer.length > sizeof(cap_hdr->code_type) || + elem[CAP_DRV_TYPE_IDX].buffer.length > sizeof(cap_hdr->drv_type) || + elem[CAP_PLAT_ID_IDX].buffer.length > sizeof(cap_hdr->platform_id) || + elem[CAP_OEM_ID_IDX].buffer.length > sizeof(cap_hdr->oem_id)) + goto free_acpi_buffer; + + cap_hdr->update_cap = elem[CAP_UPDATE_IDX].integer.value; memcpy(&cap_hdr->code_type, - out_obj->package.elements[CAP_CODE_TYPE_IDX].buffer.pointer, - out_obj->package.elements[CAP_CODE_TYPE_IDX].buffer.length); - cap_hdr->fw_version = - out_obj->package.elements[CAP_FW_VER_IDX].integer.value; - cap_hdr->code_rt_version = - out_obj->package.elements[CAP_CODE_RT_VER_IDX].integer.value; + elem[CAP_CODE_TYPE_IDX].buffer.pointer, + elem[CAP_CODE_TYPE_IDX].buffer.length); + cap_hdr->fw_version = elem[CAP_FW_VER_IDX].integer.value; + cap_hdr->code_rt_version = elem[CAP_CODE_RT_VER_IDX].integer.value; memcpy(&cap_hdr->drv_type, - out_obj->package.elements[CAP_DRV_TYPE_IDX].buffer.pointer, - out_obj->package.elements[CAP_DRV_TYPE_IDX].buffer.length); - cap_hdr->drv_rt_version = - out_obj->package.elements[CAP_DRV_RT_VER_IDX].integer.value; - cap_hdr->drv_svn = - out_obj->package.elements[CAP_DRV_SVN_IDX].integer.value; + elem[CAP_DRV_TYPE_IDX].buffer.pointer, + elem[CAP_DRV_TYPE_IDX].buffer.length); + cap_hdr->drv_rt_version = elem[CAP_DRV_RT_VER_IDX].integer.value; + cap_hdr->drv_svn = elem[CAP_DRV_SVN_IDX].integer.value; memcpy(&cap_hdr->platform_id, - out_obj->package.elements[CAP_PLAT_ID_IDX].buffer.pointer, - out_obj->package.elements[CAP_PLAT_ID_IDX].buffer.length); + elem[CAP_PLAT_ID_IDX].buffer.pointer, + elem[CAP_PLAT_ID_IDX].buffer.length); memcpy(&cap_hdr->oem_id, - out_obj->package.elements[CAP_OEM_ID_IDX].buffer.pointer, - out_obj->package.elements[CAP_OEM_ID_IDX].buffer.length); - cap_hdr->oem_info_len = - out_obj->package.elements[CAP_OEM_INFO_IDX].buffer.length; + elem[CAP_OEM_ID_IDX].buffer.pointer, + elem[CAP_OEM_ID_IDX].buffer.length); + cap_hdr->oem_info_len = elem[CAP_OEM_INFO_IDX].buffer.length; ret = 0; diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index a4498357bd165..3bf076c150fa1 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -336,11 +336,26 @@ int acpi_get_cpuid(acpi_handle handle, int type, u32 acpi_id) EXPORT_SYMBOL_GPL(acpi_get_cpuid); #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC -static int get_ioapic_id(struct acpi_subtable_header *entry, u32 gsi_base, +static bool madt_entry_is_valid(struct acpi_subtable_header *entry, + unsigned long end) +{ + unsigned long start = (unsigned long)entry; + + if (start >= end || end - start < sizeof(*entry)) + return false; + + return entry->length >= sizeof(*entry) && entry->length <= end - start; +} + +static int get_ioapic_id(struct acpi_subtable_header *entry, + const unsigned long end, u32 gsi_base, u64 *phys_addr, int *ioapic_id) { struct acpi_madt_io_apic *ioapic = (struct acpi_madt_io_apic *)entry; + if (!madt_entry_is_valid(entry, end) || BAD_MADT_ENTRY(ioapic, end)) + return 0; + if (ioapic->global_irq_base != gsi_base) return 0; @@ -361,17 +376,19 @@ static int parse_madt_ioapic_entry(u32 gsi_base, u64 *phys_addr) return apic_id; entry = (unsigned long)madt; + if (madt->header.length < sizeof(*madt)) + return apic_id; madt_end = entry + madt->header.length; /* Parse all entries looking for a match. */ entry += sizeof(struct acpi_table_madt); - while (entry + sizeof(struct acpi_subtable_header) < madt_end) { + while (madt_entry_is_valid((struct acpi_subtable_header *)entry, + madt_end)) { hdr = (struct acpi_subtable_header *)entry; if (hdr->type == ACPI_MADT_TYPE_IO_APIC && - get_ioapic_id(hdr, gsi_base, phys_addr, &apic_id)) + get_ioapic_id(hdr, madt_end, gsi_base, phys_addr, &apic_id)) break; - else - entry += hdr->length; + entry += hdr->length; } return apic_id; @@ -398,7 +415,9 @@ static int parse_mat_ioapic_entry(acpi_handle handle, u32 gsi_base, header = (struct acpi_subtable_header *)obj->buffer.pointer; if (header->type == ACPI_MADT_TYPE_IO_APIC) - get_ioapic_id(header, gsi_base, phys_addr, &apic_id); + get_ioapic_id(header, + (unsigned long)header + obj->buffer.length, + gsi_base, phys_addr, &apic_id); exit: kfree(buffer.pointer); diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 65e779be64ffc..06589bf488f74 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -259,9 +259,11 @@ static int __init acpi_processor_driver_init(void) acpi_processor_ignore_ppc_init(); } + acpi_processor_register_idle_driver(); + result = driver_register(&acpi_processor_driver); if (result < 0) - return result; + goto unregister_idle_drv; result = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "acpi/cpu-drv:online", @@ -283,8 +285,19 @@ static int __init acpi_processor_driver_init(void) acpi_idle_rescan_dead_smt_siblings(); return 0; + err: driver_unregister(&acpi_processor_driver); + +unregister_idle_drv: + acpi_processor_unregister_idle_driver(); + + if (acpi_processor_cpufreq_init) { + cpufreq_unregister_notifier(&acpi_processor_notifier_block, + CPUFREQ_POLICY_NOTIFIER); + acpi_processor_cpufreq_init = false; + } + return result; } @@ -302,6 +315,7 @@ static void __exit acpi_processor_driver_exit(void) cpuhp_remove_state_nocalls(hp_online); cpuhp_remove_state_nocalls(CPUHP_ACPI_CPUDRV_DEAD); driver_unregister(&acpi_processor_driver); + acpi_processor_unregister_idle_driver(); } module_init(acpi_processor_driver_init); diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 2468c1d971e74..5c6f736ba3b52 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -831,19 +831,13 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr) return 0; } -static inline void acpi_processor_cstate_first_run_checks(void) +static inline void acpi_processor_update_max_cstate(void) { - static int first_run; - - if (first_run) - return; dmi_check_system(processor_power_dmi_table); max_cstate = acpi_processor_cstate_check(max_cstate); if (max_cstate < ACPI_C_STATES_MAX) pr_notice("processor limited to max C-state %d\n", max_cstate); - first_run++; - if (nocst) return; @@ -852,7 +846,7 @@ static inline void acpi_processor_cstate_first_run_checks(void) #else static inline int disabled_by_idle_boot_param(void) { return 0; } -static inline void acpi_processor_cstate_first_run_checks(void) { } +static inline void acpi_processor_update_max_cstate(void) { } static int acpi_processor_get_cstate_info(struct acpi_processor *pr) { return -ENODEV; @@ -945,6 +939,13 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj->type == ACPI_TYPE_BUFFER) { struct acpi_power_register *reg; + if (obj->buffer.length < sizeof(*reg)) { + acpi_handle_debug(handle, + "Invalid register data for _LPI state %d\n", + state_idx); + continue; + } + reg = (struct acpi_power_register *)obj->buffer.pointer; if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO && reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) @@ -961,13 +962,6 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, continue; } - /* elements[7,8] skipped for now i.e. Residency/Usage counter*/ - - obj = pkg_elem + 9; - if (obj->type == ACPI_TYPE_STRING) - strscpy(lpi_state->desc, obj->string.pointer, - ACPI_CX_DESC_LEN); - lpi_state->index = state_idx; if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) { pr_debug("No min. residency found, assuming 10 us\n"); @@ -990,6 +984,20 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle, if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state)) lpi_state->enable_parent_state = 0; + + /* Skip elements [7-8] i.e. Residency/Usage counters. */ + + /* + * Avoid out-of-bounds access if the size of the package is less + * than expected. + */ + if (element->package.count < 10) + continue; + + obj = pkg_elem + 9; + if (obj->type == ACPI_TYPE_STRING) + strscpy(lpi_state->desc, obj->string.pointer, + ACPI_CX_DESC_LEN); } acpi_handle_debug(handle, "Found %d power states\n", state_idx); @@ -1357,7 +1365,59 @@ int acpi_processor_power_state_has_changed(struct acpi_processor *pr) return 0; } -static int acpi_processor_registered; +void acpi_processor_register_idle_driver(void) +{ + struct acpi_processor *pr; + int ret = -ENODEV; + int cpu; + + /* + * If a cpuidle driver is already registered, there is no need to + * evaluate _CST or attempt to register the ACPI idle driver. + */ + if (cpuidle_get_driver()) { + pr_debug("cpuidle driver %pS already registered.\n", cpuidle_get_driver()); + return; + } + + acpi_processor_update_max_cstate(); + + /* + * ACPI idle driver is used by all possible CPUs. + * Use the processor power info of one in them to set up idle states. + * Note that the existing idle handler will be used on platforms that + * only support C1. + */ + for_each_possible_cpu(cpu) { + pr = per_cpu(processors, cpu); + if (!pr) + continue; + + ret = acpi_processor_get_power_info(pr); + if (!ret) { + pr->flags.power_setup_done = 1; + acpi_processor_setup_cpuidle_states(pr); + break; + } + } + + if (ret) { + pr_debug("No ACPI power information from any CPUs.\n"); + return; + } + + ret = cpuidle_register_driver(&acpi_idle_driver); + if (ret) { + pr_debug("register %s failed.\n", acpi_idle_driver.name); + return; + } + pr_debug("%s registered with cpuidle.\n", acpi_idle_driver.name); +} + +void acpi_processor_unregister_idle_driver(void) +{ + cpuidle_unregister_driver(&acpi_idle_driver); +} int acpi_processor_power_init(struct acpi_processor *pr) { @@ -1367,27 +1427,10 @@ int acpi_processor_power_init(struct acpi_processor *pr) if (disabled_by_idle_boot_param()) return 0; - acpi_processor_cstate_first_run_checks(); - if (!acpi_processor_get_power_info(pr)) pr->flags.power_setup_done = 1; - /* - * Install the idle handler if processor power management is supported. - * Note that we use previously set idle handler will be used on - * platforms that only support C1. - */ if (pr->flags.power) { - /* Register acpi_idle_driver if not already registered */ - if (!acpi_processor_registered) { - acpi_processor_setup_cpuidle_states(pr); - retval = cpuidle_register_driver(&acpi_idle_driver); - if (retval) - return retval; - pr_debug("%s registered with cpuidle\n", - acpi_idle_driver.name); - } - dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) return -ENOMEM; @@ -1400,14 +1443,11 @@ int acpi_processor_power_init(struct acpi_processor *pr) */ retval = cpuidle_register_device(dev); if (retval) { - if (acpi_processor_registered == 0) - cpuidle_unregister_driver(&acpi_idle_driver); per_cpu(acpi_cpuidle_device, pr->id) = NULL; kfree(dev); return retval; } - acpi_processor_registered++; } return 0; } @@ -1421,10 +1461,6 @@ int acpi_processor_power_exit(struct acpi_processor *pr) if (pr->flags.power) { cpuidle_unregister_device(dev); - acpi_processor_registered--; - if (acpi_processor_registered == 0) - cpuidle_unregister_driver(&acpi_idle_driver); - kfree(dev); } diff --git a/drivers/acpi/riscv/irq.c b/drivers/acpi/riscv/irq.c index d9a2154d6c6ab..f41ab67040374 100644 --- a/drivers/acpi/riscv/irq.c +++ b/drivers/acpi/riscv/irq.c @@ -299,6 +299,7 @@ static acpi_status riscv_acpi_irq_get_parent(struct acpi_resource *ares, void *c return AE_OK; ctx->handle = riscv_acpi_get_gsi_handle(eirq->interrupts[ctx->index]); + ctx->rc = 0; return AE_CTRL_TERMINATE; } @@ -314,10 +315,22 @@ static int riscv_acpi_irq_get_dep(acpi_handle handle, unsigned int index, acpi_h acpi_walk_resources(handle, METHOD_NAME__CRS, riscv_acpi_irq_get_parent, &ctx); *gsi_handle = ctx.handle; - if (*gsi_handle) - return 1; - return 0; + return ctx.rc; +} + +static bool acpi_prt_entry_valid(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return entry && entry->length > 0; +} + +static void *acpi_prt_next_entry(void *prt_entry) +{ + struct acpi_pci_routing_table *entry = prt_entry; + + return prt_entry + entry->length; } static u32 riscv_acpi_add_prt_dep(acpi_handle handle) @@ -338,9 +351,11 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) } entry = buffer.pointer; - while (entry && (entry->length > 0)) { + for (; acpi_prt_entry_valid(entry); entry = acpi_prt_next_entry(entry)) { if (entry->source[0]) { - acpi_get_handle(handle, entry->source, &link_handle); + status = acpi_get_handle(handle, entry->source, &link_handle); + if (ACPI_FAILURE(status)) + continue; dep_devices.count = 1; dep_devices.handles = kcalloc(1, sizeof(*dep_devices.handles), GFP_KERNEL); if (!dep_devices.handles) { @@ -362,9 +377,6 @@ static u32 riscv_acpi_add_prt_dep(acpi_handle handle) dep_devices.handles[0] = gsi_handle; count += acpi_scan_add_dep(handle, &dep_devices); } - - entry = (struct acpi_pci_routing_table *) - ((unsigned long)entry + entry->length); } kfree(buffer.pointer); @@ -379,8 +391,11 @@ static u32 riscv_acpi_add_irq_dep(acpi_handle handle) int i; for (i = 0; - riscv_acpi_irq_get_dep(handle, i, &gsi_handle); + !riscv_acpi_irq_get_dep(handle, i, &gsi_handle); i++) { + if (!gsi_handle) + continue; + dep_devices.count = 1; dep_devices.handles = kcalloc(1, sizeof(*dep_devices.handles), GFP_KERNEL); if (!dep_devices.handles) { diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index 16704c2a730c0..32fa1c1ded601 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -522,12 +522,10 @@ static void acpi_device_release(struct device *dev) kfree(acpi_dev); } -static void acpi_device_del(struct acpi_device *device) +static void acpi_device_cleanup(struct acpi_device *device) { struct acpi_device_bus_id *acpi_device_bus_id; - mutex_lock(&acpi_device_lock); - list_for_each_entry(acpi_device_bus_id, &acpi_bus_id_list, node) if (!strcmp(acpi_device_bus_id->bus_id, acpi_device_hid(device))) { @@ -542,6 +540,13 @@ static void acpi_device_del(struct acpi_device *device) } list_del(&device->wakeup_list); +} + +static void acpi_device_del(struct acpi_device *device) +{ + mutex_lock(&acpi_device_lock); + + acpi_device_cleanup(device); mutex_unlock(&acpi_device_lock); @@ -802,7 +807,7 @@ int acpi_device_add(struct acpi_device *device) err: mutex_lock(&acpi_device_lock); - list_del(&device->wakeup_list); + acpi_device_cleanup(device); err_unlock: mutex_unlock(&acpi_device_lock); @@ -865,6 +870,7 @@ static const char * const acpi_honor_dep_ids[] = { "RSCV0005", /* RISC-V SBI MPXY MBOX */ "RSCV0006", /* RISC-V RPMI SYSMSI */ "PNP0C0F", /* PCI Link Device */ + "ACPI0016", /* CXL/PCIe host bridge: CXL root (ACPI0017) depends on PCI root attach */ NULL }; diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index 0b7fa4a8c379c..1b9579bb97c27 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -892,13 +892,13 @@ bool acpi_s2idle_wakeup(void) #ifdef CONFIG_PM_SLEEP static u32 saved_bm_rld; -static int acpi_save_bm_rld(void) +static int acpi_save_bm_rld(void *data) { acpi_read_bit_register(ACPI_BITREG_BUS_MASTER_RLD, &saved_bm_rld); return 0; } -static void acpi_restore_bm_rld(void) +static void acpi_restore_bm_rld(void *data) { u32 resumed_bm_rld = 0; @@ -909,14 +909,18 @@ static void acpi_restore_bm_rld(void) acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_RLD, saved_bm_rld); } -static struct syscore_ops acpi_sleep_syscore_ops = { +static const struct syscore_ops acpi_sleep_syscore_ops = { .suspend = acpi_save_bm_rld, .resume = acpi_restore_bm_rld, }; +static struct syscore acpi_sleep_syscore = { + .ops = &acpi_sleep_syscore_ops, +}; + static void acpi_sleep_syscore_init(void) { - register_syscore_ops(&acpi_sleep_syscore_ops); + register_syscore(&acpi_sleep_syscore); } #else static inline void acpi_sleep_syscore_init(void) {} diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c index 2c120ade8f51a..2a562cb425081 100644 --- a/drivers/acpi/video_detect.c +++ b/drivers/acpi/video_detect.c @@ -137,8 +137,10 @@ static int video_detect_portege_r100(const struct dmi_system_id *d) struct pci_dev *dev; /* Search for Trident CyberBlade XP4m32 to confirm Portégé R100 */ dev = pci_get_device(PCI_VENDOR_ID_TRIDENT, 0x2100, NULL); - if (dev) + if (dev) { acpi_backlight_dmi = acpi_backlight_vendor; + pci_dev_put(dev); + } return 0; } diff --git a/drivers/acpi/x86/cmos_rtc.c b/drivers/acpi/x86/cmos_rtc.c index 45db7e51cbe60..a6df5b991c960 100644 --- a/drivers/acpi/x86/cmos_rtc.c +++ b/drivers/acpi/x86/cmos_rtc.c @@ -18,12 +18,11 @@ #include "../internal.h" static const struct acpi_device_id acpi_cmos_rtc_ids[] = { - { "PNP0B00" }, - { "PNP0B01" }, - { "PNP0B02" }, - {} + ACPI_CMOS_RTC_IDS }; +bool cmos_rtc_platform_device_present; + static bool cmos_rtc_space_handler_present __read_mostly; static acpi_status acpi_cmos_rtc_space_handler(u32 function, @@ -103,6 +102,12 @@ static int acpi_cmos_rtc_attach(struct acpi_device *adev, if (ret < 0) return ret; + if (IS_ERR_OR_NULL(acpi_create_platform_device(adev, NULL))) { + pr_err("Failed to create CMOS-RTC platform device\n"); + return 0; + } else { + cmos_rtc_platform_device_present = true; + } return 1; } diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index d33af8a043676..7e2e8427c5fa2 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -1547,6 +1547,10 @@ impl Process { cmd: u32, reader: &mut UserSliceReader, ) -> Result { + if cmd == uapi::BINDER_FREEZE { + return ioctl_freeze(reader); + } + let thread = this.get_current_thread()?; match cmd { uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?), @@ -1558,7 +1562,6 @@ impl Process { uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => { this.set_oneway_spam_detection_enabled(reader.read()?) } - uapi::BINDER_FREEZE => ioctl_freeze(reader)?, _ => return Err(EINVAL), } Ok(()) @@ -1573,15 +1576,16 @@ impl Process { cmd: u32, data: UserSlice, ) -> Result { - let thread = this.get_current_thread()?; let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0; match cmd { - uapi::BINDER_WRITE_READ => thread.write_read(data, blocking)?, + uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?, uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?, uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?, uapi::BINDER_VERSION => this.version(data)?, uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?, - uapi::BINDER_GET_EXTENDED_ERROR => thread.get_extended_error(data)?, + uapi::BINDER_GET_EXTENDED_ERROR => { + this.get_current_thread()?.get_extended_error(data)? + } _ => return Err(EINVAL), } Ok(()) diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index ad8bdf762ddc6..93b550e6b60e7 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -1268,14 +1268,14 @@ impl Thread { inner.extended_error = ExtendedError::new(info.debug_id as u32, err.reply, source.to_errno()); } - } - pr_warn!( - "{}:{} transaction to {} failed: {err:?}", - info.from_pid, - info.from_tid, - info.to_pid - ); + pr_warn!( + "{}:{} transaction to {} failed: {err:?}", + info.from_pid, + info.from_tid, + info.to_pid + ); + } } } diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c index eb2d896daa661..3fdcfcc2f7beb 100644 --- a/drivers/ata/ahci.c +++ b/drivers/ata/ahci.c @@ -1686,6 +1686,51 @@ static irqreturn_t ahci_thunderx_irq_handler(int irq, void *dev_instance) } #endif +/* + * The Marvell 88SE6111/6121/6145 ("Thor") family stops reporting interrupts + * for a port when HOST_IRQ_STAT is cleared while PxIS still holds bits: PxIS + * keeps its content, HOST_IRQ_STAT reads back as 0, the port is never looked + * at again and the command in flight only ends in a timeout. On a 88SE6121 + * this makes every SATA-2 or SATA-3 disk fail to IDENTIFY, while SATA-1 disks + * happen to win the race often enough to work. + * + * Clearing the host status before servicing the ports avoids it. Marvell's + * own driver for these chips does the same and says so ("clear global before + * channel"), and ahci_xgene handles its broken edge latch the same way. The + * price is at most one spurious interrupt per valid one, which is why this is + * not the generic behaviour - see AHCI 1.1 section 10.6.2. + * + * Link: https://bugzilla.kernel.org/show_bug.cgi?id=216094 + */ +static irqreturn_t ahci_mv_irq_handler(int irq, void *dev_instance) +{ + struct ata_host *host = dev_instance; + struct ahci_host_priv *hpriv = host->private_data; + void __iomem *mmio = hpriv->mmio; + unsigned int rc; + u32 irq_stat, irq_masked; + + irq_stat = readl(mmio + HOST_IRQ_STAT); + if (!irq_stat) + return IRQ_NONE; + + irq_masked = irq_stat & hpriv->port_map; + + spin_lock(&host->lock); + + /* + * Use the unmasked value to clear the interrupt, as a spurious pending + * event on a dummy port might cause a screaming IRQ. + */ + writel(irq_stat, mmio + HOST_IRQ_STAT); + + rc = ahci_handle_port_intr(host, irq_masked); + + spin_unlock(&host->lock); + + return IRQ_RETVAL(rc); +} + static void ahci_remap_check(struct pci_dev *pdev, int bar, struct ahci_host_priv *hpriv) { @@ -1888,6 +1933,24 @@ static ssize_t remapped_nvme_show(struct device *dev, static DEVICE_ATTR_RO(remapped_nvme); +static int ahci_validate_bar_size(struct pci_dev *pdev, int bar, + struct ahci_host_priv *hpriv) +{ + u32 cap = readl(hpriv->mmio + HOST_CAP); + unsigned int max_ports = ahci_nr_ports(cap); + u32 last_port_end = 0x100 + (max_ports * 0x80); + resource_size_t bar_size = pci_resource_len(pdev, bar); + + if (last_port_end > bar_size) { + dev_warn(&pdev->dev, + "BAR%d too small for %u ports (last port ends at %#x, BAR %pa)\n", + bar, max_ports, last_port_end, &bar_size); + return -ENODEV; + } + + return 0; +} + static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { unsigned int board_id = ent->driver_data; @@ -1971,6 +2034,10 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; hpriv->flags |= (unsigned long)pi.private_data; + /* the Marvell "Thor" family needs HOST_IRQ_STAT cleared first */ + if (board_id == board_ahci_mv) + hpriv->irq_handler = ahci_mv_irq_handler; + /* MCP65 revision A1 and A2 can't do MSI */ if (board_id == board_ahci_mcp65 && (pdev->revision == 0xa1 || pdev->revision == 0xa2)) @@ -1988,6 +2055,10 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (!hpriv->mmio) return -ENOMEM; + rc = ahci_validate_bar_size(pdev, ahci_pci_bar, hpriv); + if (rc) + return rc; + /* detect remapped nvme devices */ ahci_remap_check(pdev, ahci_pci_bar, hpriv); diff --git a/drivers/ata/ahci_ceva.c b/drivers/ata/ahci_ceva.c index 2d6a08c23d6ad..2961e53288f42 100644 --- a/drivers/ata/ahci_ceva.c +++ b/drivers/ata/ahci_ceva.c @@ -211,7 +211,7 @@ static int ceva_ahci_platform_enable_resources(struct ahci_host_priv *hpriv) rc = phy_init(hpriv->phys[i]); if (rc) - goto disable_rsts; + goto exit_phys; } /* De-assert the controller reset */ @@ -230,14 +230,24 @@ static int ceva_ahci_platform_enable_resources(struct ahci_host_priv *hpriv) return 0; -disable_rsts: - ahci_platform_deassert_rsts(hpriv); - disable_phys: while (--i >= 0) { + if (ahci_ignore_port(hpriv, i)) + continue; + phy_power_off(hpriv->phys[i]); phy_exit(hpriv->phys[i]); } + ahci_platform_assert_rsts(hpriv); + goto disable_clks; + +exit_phys: + while (--i >= 0) { + if (ahci_ignore_port(hpriv, i)) + continue; + + phy_exit(hpriv->phys[i]); + } disable_clks: ahci_platform_disable_clks(hpriv); diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index 22ce6accbc5bb..ef7f93aca36d1 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -2833,6 +2833,24 @@ static void ata_dev_config_cpr(struct ata_device *dev) if (!nr_cpr) goto out; + /* + * The device reports the number of CPR descriptors independently of the + * log size, and that count is also used to emit VPD page B9h into the + * fixed-size rbuf. Reject a count larger than what that buffer can hold + * (ATA_DEV_MAX_CPR) or larger than the log the device actually returned. + */ + if (nr_cpr > ATA_DEV_MAX_CPR) { + ata_dev_warn(dev, + "Too many concurrent positioning ranges\n"); + goto out; + } + + if (buf_len < 64 + (size_t)nr_cpr * 32) { + ata_dev_warn(dev, + "Invalid number of concurrent positioning ranges\n"); + goto out; + } + cpr_log = kzalloc(struct_size(cpr_log, cpr, nr_cpr), GFP_KERNEL); if (!cpr_log) goto out; @@ -4354,6 +4372,16 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { { "WDC WD3000JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM }, { "WDC WD3200JD-*", NULL, ATA_QUIRK_WD_BROKEN_LPM }, + /* + * WD drives with LPM issues (irrespective of supported SATA speeds). + * (Unlike ATA_QUIRK_WD_BROKEN_LPM, which is only applied if the drive + * exposes SATA Gen1 speed support, and SATA Gen1 speed support only.) + */ + { "WDC WD100EFGX-68CPLN0", NULL, ATA_QUIRK_NOLPM }, + { "WDC WD102KFBX-68M95N0", NULL, ATA_QUIRK_NOLPM }, + { "WDC WD141KFGX-68FH9N0", NULL, ATA_QUIRK_NOLPM }, + { "WD Green 2.5 480GB", NULL, ATA_QUIRK_NOLPM }, + /* * This sata dom device goes on a walkabout when the ATA_LOG_DIRECTORY * log page is accessed. Ensure we never ask for this log page with diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c index 5e8a63206108e..f21efb0f6419f 100644 --- a/drivers/ata/libata-eh.c +++ b/drivers/ata/libata-eh.c @@ -106,6 +106,12 @@ static const unsigned int ata_eh_flush_timeouts[] = { UINT_MAX, }; +static const unsigned int ata_eh_standby_timeouts[] = { + 15000, /* Some drives may be slow to standby */ + /* but don't hold up a suspend too long waiting for them */ + UINT_MAX, +}; + static const unsigned int ata_eh_other_timeouts[] = { 5000, /* same rationale as identify timeout */ 10000, /* ditto */ @@ -147,6 +153,8 @@ ata_eh_cmd_timeout_table[ATA_EH_CMD_TIMEOUT_TABLE_SIZE] = { .timeouts = ata_eh_other_timeouts, }, { .commands = CMDS(ATA_CMD_FLUSH, ATA_CMD_FLUSH_EXT), .timeouts = ata_eh_flush_timeouts }, + { .commands = CMDS(ATA_CMD_STANDBYNOW1), + .timeouts = ata_eh_standby_timeouts }, { .commands = CMDS(ATA_CMD_VERIFY), .timeouts = ata_eh_reset_timeouts }, }; @@ -640,29 +648,12 @@ void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, set_host_byte(scmd, DID_OK); ata_qc_for_each_raw(ap, qc, i) { - if (qc->scsicmd != scmd) - continue; - if ((qc->flags & ATA_QCFLAG_ACTIVE) || - qc == qc->dev->link->deferred_qc) + if (qc->scsicmd == scmd && + qc->flags & ATA_QCFLAG_ACTIVE) break; } - if (i < ATA_MAX_QUEUE && qc == qc->dev->link->deferred_qc) { - /* - * This is a deferred command that timed out while - * waiting for the command queue to drain. Since the qc - * is not active yet (deferred_qc is still set, so the - * deferred qc work has not issued the command yet), - * simply signal the timeout by finishing the SCSI - * command and clear the deferred qc to prevent the - * deferred qc work from issuing this qc. - */ - WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE); - qc->dev->link->deferred_qc = NULL; - cancel_work(&qc->dev->link->deferred_qc_work); - set_host_byte(scmd, DID_TIME_OUT); - scsi_eh_finish_cmd(scmd, &ap->eh_done_q); - } else if (i < ATA_MAX_QUEUE) { + if (i < ATA_MAX_QUEUE) { /* the scmd has an associated qc */ if (!(qc->flags & ATA_QCFLAG_EH)) { /* which hasn't failed yet, timeout */ @@ -936,10 +927,10 @@ static void ata_eh_set_pending(struct ata_port *ap, bool fastdrain) ap->pflags |= ATA_PFLAG_EH_PENDING; /* - * If we have a deferred qc, requeue it so that it is retried once EH - * completes. + * If we have deferred QCs, requeue them so that the SCSI EH task can + * run. */ - ata_scsi_requeue_deferred_qc(ap); + ata_scsi_requeue_deferred_qc(ap, NULL); if (!fastdrain) return; diff --git a/drivers/ata/libata-pmp.c b/drivers/ata/libata-pmp.c index 48ac09d9031e6..299b1823e0d09 100644 --- a/drivers/ata/libata-pmp.c +++ b/drivers/ata/libata-pmp.c @@ -457,8 +457,13 @@ static void sata_pmp_quirks(struct ata_port *ap) * otherwise. Don't try hard to recover it. */ ap->pmp_link[ap->nr_pmp_links - 1].flags |= ATA_LFLAG_NO_RETRY; - } else if (vendor == 0x197b && (devid == 0x2352 || devid == 0x0325)) { + } else if (vendor == 0x197b && + (devid == 0x0562 || devid == 0x2352 || devid == 0x0325)) { /* + * 0x0562: JMicron JMS562, as used in QNAP QDA-A2AR RAID1 + * adapters. The exported device may stay not ready + * while the array is rebuilding, and SRST/classify can + * time out before the RAID volume is exported. * 0x2352: found in Thermaltake BlackX Duet, jmicron JMB350? * 0x0325: jmicron JMB394. */ diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c index 04e1e774645e2..9b374e8f99384 100644 --- a/drivers/ata/libata-sata.c +++ b/drivers/ata/libata-sata.c @@ -913,7 +913,7 @@ static bool ata_scsi_lpm_supported(struct ata_port *ap) return false; ata_for_each_link(link, ap, EDGE) { - ata_for_each_dev(dev, &ap->link, ENABLED) { + ata_for_each_dev(dev, link, ENABLED) { if (dev->quirks & ATA_QUIRK_NOLPM) return false; } diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c index 46ef1ef6f8128..8df501fa8defd 100644 --- a/drivers/ata/libata-scsi.c +++ b/drivers/ata/libata-scsi.c @@ -37,8 +37,6 @@ #include "libata.h" #include "libata-transport.h" -#define ATA_SCSI_RBUF_SIZE 2048 - static DEFINE_SPINLOCK(ata_scsi_rbuf_lock); static u8 ata_scsi_rbuf[ATA_SCSI_RBUF_SIZE]; @@ -1682,30 +1680,81 @@ void ata_scsi_deferred_qc_work(struct work_struct *work) spin_unlock_irqrestore(ap->lock, flags); } -void ata_scsi_requeue_deferred_qc(struct ata_port *ap) +enum scsi_timeout_action ata_scsi_requeue_deferred_qc(struct ata_port *ap, + struct scsi_cmnd *timedout_scmd) { + enum scsi_timeout_action action = SCSI_EH_NOT_HANDLED; + struct ata_queued_cmd *qc; struct ata_link *link; + u32 host_byte; lockdep_assert_held(ap->lock); /* - * If we have a deferred qc when a reset occurs or NCQ commands fail, - * do not try to be smart about what to do with this deferred command - * and simply requeue it by completing it with DID_REQUEUE. + * If we have deferred QCs when a reset, a timeout or an NCQ command + * fails, do not try to be smart about what to do with the deferred + * commands and simply terminate them and let the SCSI layer decide + * what to do. */ ata_for_each_link(link, ap, PMP_FIRST) { - struct ata_queued_cmd *qc = link->deferred_qc; - struct scsi_cmnd *scmd; - - if (qc) { - scmd = qc->scsicmd; - link->deferred_qc = NULL; - cancel_work(&link->deferred_qc_work); - ata_qc_free(qc); - scmd->result = (DID_REQUEUE << 16); - scsi_done(scmd); + qc = link->deferred_qc; + if (!qc) + continue; + + /* + * Clear the deferred QC so that the deferred work does not try + * to issue it. + */ + link->deferred_qc = NULL; + cancel_work(&link->deferred_qc_work); + + /* + * We are going to complete some scsi command, either with + * DID_TIME_OUT if the command timed out while waiting for being + * issued, or with DID_REQUEUE if another command timed out or + * we had a failed command. However, the block layer may re-issue + * these commands immediately, keeping the scsi host busy and + * thus preventing the SCSI EH task from running. + * So schedule EH on the port to prevent accepting new commands + * until everything is sorted out with the error or timeout that + * got us here in the first place. Note that we set EH pending + * on the port before calling ata_port_schedule_eh() so that we + * do not reenter this function from ata_eh_set_pending() with + * timedout_scmd being NULL and erroneously retry deferred QCs + * that have timed out on other links. + */ + if (!ata_port_eh_scheduled(ap)) { + ap->pflags |= ATA_PFLAG_EH_PENDING; + ata_port_schedule_eh(ap); } + + /* + * If we are being called from scsi_timeout(), then we have a + * non-NULL timedout_scmd. If the timed out command is for a + * deferred QC, terminate that deferred QC with DID_TIME_OUT and + * requeue all other deferred QCs. In this case we need to + * return SCSI_EH_DONE, because the timed out command was + * handled. + * If the timed out command is not for a deferred QC, we need to + * requeue all deferred QCs, and return SCSI_EH_NOT_HANDLED so + * that the timed out command gets added to the EH work queue + * with scsi_eh_scmd_add(), for later handling with libata EH + * ata_scsi_cmd_error_handler(). + * If timedout_scmd is NULL, we simply need to requeue all + * deferred QCs and the return value does not matter as we were + * not called from scsi_timeout(). + */ + if (timedout_scmd && qc->scsicmd == timedout_scmd) { + host_byte = DID_TIME_OUT; + action = SCSI_EH_DONE; + } else { + host_byte = DID_REQUEUE; + } + qc->scsicmd->result = host_byte << 16; + ata_qc_done(qc); } + + return action; } static void ata_scsi_schedule_deferred_qc(struct ata_link *link) @@ -1724,13 +1773,41 @@ static void ata_scsi_schedule_deferred_qc(struct ata_link *link) return; if (ata_port_eh_scheduled(ap)) { - ata_scsi_requeue_deferred_qc(ap); + ata_scsi_requeue_deferred_qc(ap, NULL); return; } if (!ap->ops->qc_defer(qc)) queue_work(system_highpri_wq, &link->deferred_qc_work); } +static enum scsi_timeout_action +ata_scsi_retry_deferred_qc(struct ata_port *ap, struct scsi_cmnd *scmd) +{ + enum scsi_timeout_action action; + unsigned long flags; + + spin_lock_irqsave(ap->lock, flags); + action = ata_scsi_requeue_deferred_qc(ap, scmd); + spin_unlock_irqrestore(ap->lock, flags); + + return action; +} + +enum scsi_timeout_action ata_scsi_eh_timed_out(struct scsi_cmnd *scmd) +{ + struct ata_port *ap = ata_shost_to_port(scmd->device->host); + + /* + * ata_scsi_cmd_error_handler() takes care of commands that timed out + * while executing. However, if we have deferred QCs while a timeout + * triggers, we must requeue these commands for retry so that we do not + * unnecessarily delay starting the SCSI EH task until these deferred + * commands also time out. + */ + return ata_scsi_retry_deferred_qc(ap, scmd); +} +EXPORT_SYMBOL_GPL(ata_scsi_eh_timed_out); + static void ata_scsi_qc_complete(struct ata_queued_cmd *qc) { struct ata_link *link = qc->dev->link; @@ -2935,7 +3012,13 @@ static void atapi_qc_complete(struct ata_queued_cmd *qc) if (qc->cdb[0] == ALLOW_MEDIUM_REMOVAL && qc->dev->sdev) qc->dev->sdev->locked = 0; - qc->scsicmd->result = SAM_STAT_CHECK_CONDITION; + if (!cmd->result) + cmd->result = SAM_STAT_CHECK_CONDITION; + ata_qc_done(qc); + return; + } + + if (cmd->result) { ata_qc_done(qc); return; } @@ -3451,17 +3534,13 @@ static unsigned int ata_scsi_pass_thru(struct ata_queued_cmd *qc) static size_t ata_format_dsm_trim_descr(struct scsi_cmnd *cmd, u32 trmax, u64 sector, u32 count) { - struct scsi_device *sdp = cmd->device; - size_t len = sdp->sector_size; + size_t len = ATA_SECT_SIZE; size_t r; __le64 *buf; u32 i = 0; unsigned long flags; - WARN_ON(len > ATA_SCSI_RBUF_SIZE); - - if (len > ATA_SCSI_RBUF_SIZE) - len = ATA_SCSI_RBUF_SIZE; + BUILD_BUG_ON(ATA_SECT_SIZE > ATA_SCSI_RBUF_SIZE); spin_lock_irqsave(&ata_scsi_rbuf_lock, flags); buf = ((void *)ata_scsi_rbuf); @@ -3496,13 +3575,11 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) { struct ata_taskfile *tf = &qc->tf; struct scsi_cmnd *scmd = qc->scsicmd; - struct scsi_device *sdp = scmd->device; - size_t len = sdp->sector_size; struct ata_device *dev = qc->dev; const u8 *cdb = scmd->cmnd; u64 block; u32 n_block; - const u32 trmax = len >> 3; + const u32 trmax = ATA_MAX_TRIM_RNUM; u32 size; u16 fp; u8 bp = 0xff; @@ -3546,13 +3623,13 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) goto invalid_param_len; /* - * size must match sector size in bytes - * For DATA SET MANAGEMENT TRIM in ACS-2 nsect (aka count) - * is defined as number of 512 byte blocks to be transferred. + * The TRIM descriptor is a single 512-byte page, which is the maximum + * WRITE SAME length advertised in the Block Limits VPD page. For DATA + * SET MANAGEMENT TRIM the COUNT field (aka nsect) is the number of + * 512-byte blocks to be transferred. */ - size = ata_format_dsm_trim_descr(scmd, trmax, block, n_block); - if (size != len) + if (size != ATA_SECT_SIZE) goto invalid_param_len; if (ata_ncq_enabled(dev) && ata_fpdma_dsm_supported(dev)) { @@ -3578,6 +3655,12 @@ static unsigned int ata_scsi_write_same_xlat(struct ata_queued_cmd *qc) ATA_TFLAG_WRITE; ata_qc_set_pc_nbytes(qc); + /* + * The DSM TRIM payload is a single 512-byte page, which may be smaller + * than the WRITE SAME data-out buffer (one logical block); only + * transfer that page so the length matches the COUNT field. + */ + qc->nbytes = size; return 0; diff --git a/drivers/ata/libata.h b/drivers/ata/libata.h index 0002994ddfc9b..9f96ca2acfd90 100644 --- a/drivers/ata/libata.h +++ b/drivers/ata/libata.h @@ -144,6 +144,15 @@ static inline void ata_acpi_bind_dev(struct ata_device *dev) {} #endif /* libata-scsi.c */ +#define ATA_SCSI_RBUF_SIZE 2048 + +/* + * Maximum number of concurrent positioning ranges (CPR) supported. The ACS + * specifications allow up to 255, but we limit this to the number of CPR + * descriptors that fit in the rbuf buffer used to emit VPD page B9h. + */ +#define ATA_DEV_MAX_CPR min(255, ((ATA_SCSI_RBUF_SIZE - 64) / 32)) + extern struct ata_device *ata_scsi_find_dev(struct ata_port *ap, const struct scsi_device *scsidev); extern int ata_scsi_add_hosts(struct ata_host *host, @@ -163,7 +172,8 @@ int ata_scsi_dev_config(struct scsi_device *sdev, struct queue_limits *lim, struct ata_device *dev); int __ata_scsi_queuecmd(struct scsi_cmnd *scmd, struct ata_device *dev); void ata_scsi_deferred_qc_work(struct work_struct *work); -void ata_scsi_requeue_deferred_qc(struct ata_port *ap); +enum scsi_timeout_action ata_scsi_requeue_deferred_qc(struct ata_port *ap, + struct scsi_cmnd *scmd); /* libata-eh.c */ extern unsigned int ata_internal_cmd_timeout(struct ata_device *dev, u8 cmd); diff --git a/drivers/ata/pata_sl82c105.c b/drivers/ata/pata_sl82c105.c index 2d24c6b3e9d9f..25bce3fe2a101 100644 --- a/drivers/ata/pata_sl82c105.c +++ b/drivers/ata/pata_sl82c105.c @@ -264,6 +264,7 @@ static struct ata_port_operations sl82c105_port_ops = { static int sl82c105_bridge_revision(struct pci_dev *pdev) { struct pci_dev *bridge; + u8 revision; /* * The bridge should be part of the same device, but function 0. @@ -285,8 +286,9 @@ static int sl82c105_bridge_revision(struct pci_dev *pdev) /* * We need to find function 0's revision, not function 1 */ + revision = bridge->revision; pci_dev_put(bridge); - return bridge->revision; + return revision; } static void sl82c105_fixup(struct pci_dev *pdev) diff --git a/drivers/ata/sata_dwc_460ex.c b/drivers/ata/sata_dwc_460ex.c index 7a4f59202156e..36cc28b898650 100644 --- a/drivers/ata/sata_dwc_460ex.c +++ b/drivers/ata/sata_dwc_460ex.c @@ -19,7 +19,6 @@ #include #include #include -#include #include #include #include @@ -226,7 +225,6 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev, struct sata_dwc_device *hsdev) { struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; hsdev->dma = devm_kzalloc(dev, sizeof(*hsdev->dma), GFP_KERNEL); if (!hsdev->dma) @@ -236,11 +234,9 @@ static int sata_dwc_dma_init_old(struct platform_device *pdev, hsdev->dma->id = pdev->id; /* Get SATA DMA interrupt number */ - hsdev->dma->irq = irq_of_parse_and_map(np, 1); - if (!hsdev->dma->irq) { - dev_err(dev, "no SATA DMA irq\n"); - return -ENODEV; - } + hsdev->dma->irq = platform_get_irq(pdev, 1); + if (hsdev->dma->irq < 0) + return hsdev->dma->irq; /* Get physical SATA DMA register base address */ hsdev->dma->regs = devm_platform_ioremap_resource(pdev, 1); @@ -398,8 +394,7 @@ static void clear_serror(struct ata_port *ap) static void clear_interrupt_bit(struct sata_dwc_device *hsdev, u32 bit) { - sata_dwc_writel(&hsdev->sata_dwc_regs->intpr, - sata_dwc_readl(&hsdev->sata_dwc_regs->intpr)); + sata_dwc_writel(&hsdev->sata_dwc_regs->intpr, bit); } static u32 qcmd_tag_to_mask(u8 tag) @@ -612,14 +607,9 @@ static irqreturn_t sata_dwc_isr(int irq, void *dev_instance) status = ap->ops->sff_check_status(ap); dev_dbg(ap->dev, "%s ATA status register=0x%x\n", __func__, status); - tag = 0; while (tag_mask) { - while (!(tag_mask & 0x00000001)) { - tag++; - tag_mask <<= 1; - } - - tag_mask &= (~0x00000001); + tag = __ffs(tag_mask); + tag_mask &= ~(1U << tag); qc = ata_qc_from_tag(ap, tag); if (unlikely(!qc)) { dev_err(ap->dev, "failed to get qc"); @@ -1125,7 +1115,6 @@ static const struct ata_port_info sata_dwc_port_info[] = { static int sata_dwc_probe(struct platform_device *ofdev) { struct device *dev = &ofdev->dev; - struct device_node *np = dev->of_node; struct sata_dwc_device *hsdev; u32 idr, versionr; char *ver = (char *)&versionr; @@ -1168,18 +1157,13 @@ static int sata_dwc_probe(struct platform_device *ofdev) /* Save dev for later use in dev_xxx() routines */ hsdev->dev = dev; - /* Enable SATA Interrupts */ - sata_dwc_enable_interrupts(hsdev); - /* Get SATA interrupt number */ - irq = irq_of_parse_and_map(np, 0); - if (!irq) { - dev_err(dev, "no SATA DMA irq\n"); - return -ENODEV; - } + irq = platform_get_irq(ofdev, 0); + if (irq < 0) + return irq; #ifdef CONFIG_SATA_DWC_OLD_DMA - if (!of_property_present(np, "dmas")) { + if (!of_property_present(dev->of_node, "dmas")) { err = sata_dwc_dma_init_old(ofdev, hsdev); if (err) return err; @@ -1203,6 +1187,8 @@ static int sata_dwc_probe(struct platform_device *ofdev) if (err) dev_err(dev, "failed to activate host"); + /* Enable SATA Interrupts */ + sata_dwc_enable_interrupts(hsdev); return 0; error_out: diff --git a/drivers/ata/sata_mv.c b/drivers/ata/sata_mv.c index ffb396f61731f..1d8ca95fb7328 100644 --- a/drivers/ata/sata_mv.c +++ b/drivers/ata/sata_mv.c @@ -4026,7 +4026,7 @@ static int mv_platform_probe(struct platform_device *pdev) /* * Simple resource validation .. */ - if (unlikely(pdev->num_resources != 1)) { + if (unlikely(pdev->num_resources != 1 && pdev->num_resources != 2)) { dev_err(&pdev->dev, "invalid number of resources\n"); return -EINVAL; } diff --git a/drivers/auxdisplay/charlcd.c b/drivers/auxdisplay/charlcd.c index 09020bb8ad15f..ae2692ec39639 100644 --- a/drivers/auxdisplay/charlcd.c +++ b/drivers/auxdisplay/charlcd.c @@ -595,6 +595,16 @@ static int charlcd_init(struct charlcd *lcd) return 0; } +static void charlcd_deinit(struct charlcd *lcd) +{ + struct charlcd_priv *priv = charlcd_to_priv(lcd); + + if (lcd->ops->backlight) { + cancel_delayed_work_sync(&priv->bl_work); + lcd->ops->backlight(lcd, CHARLCD_OFF); + } +} + struct charlcd *charlcd_alloc(unsigned int drvdata_size) { struct charlcd_priv *priv; @@ -654,8 +664,10 @@ int charlcd_register(struct charlcd *lcd) return ret; ret = misc_register(&charlcd_dev); - if (ret) + if (ret) { + charlcd_deinit(lcd); return ret; + } the_charlcd = lcd; register_reboot_notifier(&panel_notifier); @@ -665,16 +677,11 @@ EXPORT_SYMBOL_GPL(charlcd_register); int charlcd_unregister(struct charlcd *lcd) { - struct charlcd_priv *priv = charlcd_to_priv(lcd); - unregister_reboot_notifier(&panel_notifier); charlcd_puts(lcd, "\x0cLCD driver unloaded.\x1b[Lc\x1b[Lb\x1b[L-"); misc_deregister(&charlcd_dev); the_charlcd = NULL; - if (lcd->ops->backlight) { - cancel_delayed_work_sync(&priv->bl_work); - priv->lcd.ops->backlight(&priv->lcd, CHARLCD_OFF); - } + charlcd_deinit(lcd); return 0; } diff --git a/drivers/base/arch_numa.c b/drivers/base/arch_numa.c index c99f2ab105e5b..a36c939b8e905 100644 --- a/drivers/base/arch_numa.c +++ b/drivers/base/arch_numa.c @@ -105,6 +105,18 @@ static void __init setup_node_to_cpumask_map(void) if (nr_node_ids == MAX_NUMNODES) setup_nr_node_ids(); + /* + * This check should never be true but it makes it clear to compilers + * that node_to_cpumask_map is bound by nr_node_ids, avoiding false + * positive fortify warnings when accessing node_to_cpumask_map in the + * for loop below. + */ + if (unlikely(nr_node_ids > MAX_NUMNODES)) { + pr_err("nr_node_ids (%u) is larger than MAX_NUMNODES (%u)\n", + nr_node_ids, MAX_NUMNODES); + return; + } + /* allocate and clear the mapping */ for (node = 0; node < nr_node_ids; node++) { alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]); diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index 613410705a47e..338804742dce3 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -382,9 +382,14 @@ static int cache_setup_properties(unsigned int cpu) else if (!acpi_disabled) ret = cache_setup_acpi(cpu); - // Assume there is no cache information available in DT/ACPI from now. - if (ret && use_arch_cache_info()) + /* + * No DT/ACPI cache nodes; fall back to arch-derived topology (e.g. + * arm64 CLIDR_EL1) and clear the error to avoid a spurious warning. + */ + if (ret && use_arch_cache_info()) { use_arch_info = true; + ret = 0; + } return ret; } diff --git a/drivers/base/core.c b/drivers/base/core.c index 5034d9b103642..5ba783e5055ae 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -1084,7 +1084,7 @@ static void device_links_missing_supplier(struct device *dev) static bool dev_is_best_effort(struct device *dev) { - return (fw_devlink_best_effort && dev->can_match) || + return (fw_devlink_best_effort && dev_can_match(dev)) || (dev->fwnode && fwnode_test_flag(dev->fwnode, FWNODE_FLAG_BEST_EFFORT)); } @@ -1152,7 +1152,7 @@ int device_links_check_suppliers(struct device *dev) if (dev_is_best_effort(dev) && device_link_test(link, DL_FLAG_INFERRED) && - !link->supplier->can_match) { + !dev_can_match(link->supplier)) { ret = -EAGAIN; continue; } @@ -1435,7 +1435,7 @@ void device_links_driver_bound(struct device *dev) } else if (dev_is_best_effort(dev) && device_link_test(link, DL_FLAG_INFERRED) && link->status != DL_STATE_CONSUMER_PROBE && - !link->supplier->can_match) { + !dev_can_match(link->supplier)) { /* * When dev_is_best_effort() is true, we ignore device * links to suppliers that don't have a driver. If the @@ -1500,7 +1500,8 @@ static void __device_links_no_driver(struct device *dev) if (link->supplier->links.status == DL_DEV_DRIVER_BOUND) { WRITE_ONCE(link->status, DL_STATE_AVAILABLE); } else { - WARN_ON(!device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); + WARN_ON(link->supplier->links.status != DL_DEV_UNBINDING && + !device_link_test(link, DL_FLAG_SYNC_STATE_ONLY)); WRITE_ONCE(link->status, DL_STATE_DORMANT); } } @@ -1823,7 +1824,7 @@ static int fw_devlink_no_driver(struct device *dev, void *data) { struct device_link *link = to_devlink(dev); - if (!link->supplier->can_match) + if (!dev_can_match(link->supplier)) fw_devlink_relax_link(link); return 0; @@ -3775,7 +3776,7 @@ int device_add(struct device *dev) * match with any driver, don't block its consumers from probing in * case the consumer device is able to operate without this supplier. */ - if (dev->fwnode && fw_devlink_drv_reg_done && !dev->can_match) + if (dev->fwnode && fw_devlink_drv_reg_done && !dev_can_match(dev)) fw_devlink_unblock_consumers(dev); if (parent) diff --git a/drivers/base/dd.c b/drivers/base/dd.c index dabdfc088f3f6..d019d0f98ad47 100644 --- a/drivers/base/dd.c +++ b/drivers/base/dd.c @@ -132,7 +132,7 @@ static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func); void driver_deferred_probe_add(struct device *dev) { - if (!dev->can_match) + if (!dev_can_match(dev)) return; mutex_lock(&deferred_probe_mutex); @@ -858,14 +858,14 @@ static int __driver_probe_device(const struct device_driver *drv, struct device return dev_err_probe(dev, -EPROBE_DEFER, "Device not ready to probe\n"); /* - * Set can_match = true after calling dev_ready_to_probe(), so + * Call dev_set_can_match() after calling dev_ready_to_probe(), so * driver_deferred_probe_add() won't actually add the device to the * deferred probe list when dev_ready_to_probe() returns false. * * When dev_ready_to_probe() returns false, it means that device_add() * will do another probe() attempt for us. */ - dev->can_match = true; + dev_set_can_match(dev); dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n", drv->bus->name, __func__, drv->name); @@ -1011,7 +1011,7 @@ static int __device_attach_driver(struct device_driver *drv, void *_data) return 0; } else if (ret == -EPROBE_DEFER) { dev_dbg(dev, "Device match requests probe deferral\n"); - dev->can_match = true; + dev_set_can_match(dev); driver_deferred_probe_add(dev); /* * Device can't match with a driver right now, so don't attempt @@ -1255,7 +1255,7 @@ static int __driver_attach(struct device *dev, void *data) return 0; } else if (ret == -EPROBE_DEFER) { dev_dbg(dev, "Device match requests probe deferral\n"); - dev->can_match = true; + dev_set_can_match(dev); driver_deferred_probe_add(dev); /* * Driver could not match with device, but may match with diff --git a/drivers/base/firmware_loader/fallback.c b/drivers/base/firmware_loader/fallback.c index 3ef0b312ae719..00d6b6e594616 100644 --- a/drivers/base/firmware_loader/fallback.c +++ b/drivers/base/firmware_loader/fallback.c @@ -95,6 +95,16 @@ static int fw_load_sysfs_fallback(struct fw_sysfs *fw_sysfs, long timeout) retval = -EINTR; goto out; } + + /* + * device_add() exposes the loading interface before pending_list is + * linked into pending_fw_head, so fw_state_done() may run first. + */ + if (fw_state_is_done(fw_priv)) { + mutex_unlock(&fw_lock); + goto out; + } + list_add(&fw_priv->pending_list, &pending_fw_head); mutex_unlock(&fw_lock); diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 0f3e1779d1e87..214cdd9096669 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -1646,16 +1646,20 @@ static int fw_pm_notify(struct notifier_block *notify_block, } /* stop caching firmware once syscore_suspend is reached */ -static int fw_suspend(void) +static int fw_suspend(void *data) { fw_cache.state = FW_LOADER_NO_CACHE; return 0; } -static struct syscore_ops fw_syscore_ops = { +static const struct syscore_ops fw_syscore_ops = { .suspend = fw_suspend, }; +static struct syscore fw_syscore = { + .ops = &fw_syscore_ops, +}; + static int __init register_fw_pm_ops(void) { int ret; @@ -1671,14 +1675,14 @@ static int __init register_fw_pm_ops(void) if (ret) return ret; - register_syscore_ops(&fw_syscore_ops); + register_syscore(&fw_syscore); return ret; } static inline void unregister_fw_pm_ops(void) { - unregister_syscore_ops(&fw_syscore_ops); + unregister_syscore(&fw_syscore); unregister_pm_notifier(&fw_cache.pm_notify); } #else diff --git a/drivers/base/memory.c b/drivers/base/memory.c index fdbec49f5f5b2..8adee0e8c753d 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -198,7 +198,7 @@ static ssize_t state_show(struct device *dev, struct device_attribute *attr, break; default: WARN_ON(1); - return sysfs_emit(buf, "ERROR-UNKNOWN-%ld\n", mem->state); + return sysfs_emit(buf, "ERROR-UNKNOWN-%d\n", mem->state); } return sysfs_emit(buf, "%s\n", output); @@ -473,7 +473,7 @@ static ssize_t phys_device_show(struct device *dev, static int print_allowed_zone(char *buf, int len, int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages, - int online_type, struct zone *default_zone) + enum mmop online_type, struct zone *default_zone) { struct zone *zone; diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 13c2c48e587f8..12e12e16047f9 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -2161,6 +2161,7 @@ static int device_prepare(struct device *dev, pm_message_t state) if (ret < 0) { suspend_report_result(dev, callback, ret); + pm_runtime_unblock(dev); pm_runtime_put(dev); return ret; } diff --git a/drivers/base/property.c b/drivers/base/property.c index b2dc2bd7e5aca..bdaaf833aa260 100644 --- a/drivers/base/property.c +++ b/drivers/base/property.c @@ -797,18 +797,31 @@ struct fwnode_handle * fwnode_get_next_child_node(const struct fwnode_handle *fwnode, struct fwnode_handle *child) { + const struct fwnode_handle *parent; + struct fwnode_handle *child_parent __free(fwnode_handle) = NULL; struct fwnode_handle *next; - if (IS_ERR_OR_NULL(fwnode)) + /* + * If this function is in a loop and the previous iteration returned + * an child from fwnode->secondary, then we need to use the secondary + * as parent rather than @fwnode. + */ + if (child) { + child_parent = fwnode_get_parent(child); + parent = child_parent; + } else { + parent = fwnode; + } + if (IS_ERR_OR_NULL(parent)) return NULL; /* Try to find a child in primary fwnode */ - next = fwnode_call_ptr_op(fwnode, get_next_child_node, child); + next = fwnode_call_ptr_op(parent, get_next_child_node, child); if (next) return next; /* When no more children in primary, continue with secondary */ - return fwnode_call_ptr_op(fwnode->secondary, get_next_child_node, child); + return fwnode_get_next_child_node(parent->secondary, NULL); } EXPORT_SYMBOL_GPL(fwnode_get_next_child_node); diff --git a/drivers/base/regmap/regmap-sdw-mbq.c b/drivers/base/regmap/regmap-sdw-mbq.c index 86644bbd07100..fe1a8c442034a 100644 --- a/drivers/base/regmap/regmap-sdw-mbq.c +++ b/drivers/base/regmap/regmap-sdw-mbq.c @@ -54,10 +54,10 @@ static int regmap_sdw_mbq_poll_busy(struct sdw_slave *slave, unsigned int reg, reg = SDW_SDCA_CTL(SDW_SDCA_CTL_FUNC(reg), 0, SDCA_CTL_ENTITY_0_FUNCTION_STATUS, 0); - if (ctx->readable_reg(dev, reg)) { + if (!ctx->readable_reg || ctx->readable_reg(dev, reg)) { ret = read_poll_timeout(sdw_read_no_pm, val, val < 0 || !(val & SDCA_CTL_ENTITY_0_FUNCTION_BUSY), - ctx->cfg.timeout_us, ctx->cfg.retry_us, + ctx->cfg.retry_us, ctx->cfg.timeout_us, false, slave, reg); if (val < 0) return val; diff --git a/drivers/base/soc.c b/drivers/base/soc.c index 282c38aece0de..c43940940aa34 100644 --- a/drivers/base/soc.c +++ b/drivers/base/soc.c @@ -194,6 +194,7 @@ EXPORT_SYMBOL_GPL(soc_device_unregister); static int __init soc_bus_register(void) { + struct soc_device *soc_dev; int ret; ret = bus_register(&soc_bus_type); @@ -201,10 +202,20 @@ static int __init soc_bus_register(void) return ret; soc_bus_registered = true; - if (early_soc_dev_attr) - return PTR_ERR(soc_device_register(early_soc_dev_attr)); + if (early_soc_dev_attr) { + soc_dev = soc_device_register(early_soc_dev_attr); + if (IS_ERR(soc_dev)) { + ret = PTR_ERR(soc_dev); + goto err_unregister_bus; + } + } return 0; + +err_unregister_bus: + soc_bus_registered = false; + bus_unregister(&soc_bus_type); + return ret; } core_initcall(soc_bus_register); diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c index be1e9e61a7bf4..1e08492935eeb 100644 --- a/drivers/base/swnode.c +++ b/drivers/base/swnode.c @@ -529,7 +529,7 @@ software_node_get_reference_args(const struct fwnode_handle *fwnode, if (prop->is_inline) return -EINVAL; - if ((index + 1) * sizeof(*ref) > prop->length) + if (index >= prop->length / sizeof(*ref)) return -ENOENT; ref_array = prop->pointer; diff --git a/drivers/base/syscore.c b/drivers/base/syscore.c index 13db1f78d2ce9..483adb796654f 100644 --- a/drivers/base/syscore.c +++ b/drivers/base/syscore.c @@ -11,32 +11,32 @@ #include #include -static LIST_HEAD(syscore_ops_list); -static DEFINE_MUTEX(syscore_ops_lock); +static LIST_HEAD(syscore_list); +static DEFINE_MUTEX(syscore_lock); /** - * register_syscore_ops - Register a set of system core operations. - * @ops: System core operations to register. + * register_syscore - Register a set of system core operations. + * @syscore: System core operations to register. */ -void register_syscore_ops(struct syscore_ops *ops) +void register_syscore(struct syscore *syscore) { - mutex_lock(&syscore_ops_lock); - list_add_tail(&ops->node, &syscore_ops_list); - mutex_unlock(&syscore_ops_lock); + mutex_lock(&syscore_lock); + list_add_tail(&syscore->node, &syscore_list); + mutex_unlock(&syscore_lock); } -EXPORT_SYMBOL_GPL(register_syscore_ops); +EXPORT_SYMBOL_GPL(register_syscore); /** - * unregister_syscore_ops - Unregister a set of system core operations. - * @ops: System core operations to unregister. + * unregister_syscore - Unregister a set of system core operations. + * @syscore: System core operations to unregister. */ -void unregister_syscore_ops(struct syscore_ops *ops) +void unregister_syscore(struct syscore *syscore) { - mutex_lock(&syscore_ops_lock); - list_del(&ops->node); - mutex_unlock(&syscore_ops_lock); + mutex_lock(&syscore_lock); + list_del(&syscore->node); + mutex_unlock(&syscore_lock); } -EXPORT_SYMBOL_GPL(unregister_syscore_ops); +EXPORT_SYMBOL_GPL(unregister_syscore); #ifdef CONFIG_PM_SLEEP /** @@ -46,7 +46,7 @@ EXPORT_SYMBOL_GPL(unregister_syscore_ops); */ int syscore_suspend(void) { - struct syscore_ops *ops; + struct syscore *syscore; int ret = 0; trace_suspend_resume(TPS("syscore_suspend"), 0, true); @@ -59,25 +59,27 @@ int syscore_suspend(void) WARN_ONCE(!irqs_disabled(), "Interrupts enabled before system core suspend.\n"); - list_for_each_entry_reverse(ops, &syscore_ops_list, node) - if (ops->suspend) { - pm_pr_dbg("Calling %pS\n", ops->suspend); - ret = ops->suspend(); + list_for_each_entry_reverse(syscore, &syscore_list, node) + if (syscore->ops->suspend) { + pm_pr_dbg("Calling %pS\n", syscore->ops->suspend); + ret = syscore->ops->suspend(syscore->data); if (ret) goto err_out; WARN_ONCE(!irqs_disabled(), - "Interrupts enabled after %pS\n", ops->suspend); + "Interrupts enabled after %pS\n", + syscore->ops->suspend); } trace_suspend_resume(TPS("syscore_suspend"), 0, false); return 0; err_out: - pr_err("PM: System core suspend callback %pS failed.\n", ops->suspend); + pr_err("PM: System core suspend callback %pS failed.\n", + syscore->ops->suspend); - list_for_each_entry_continue(ops, &syscore_ops_list, node) - if (ops->resume) - ops->resume(); + list_for_each_entry_continue(syscore, &syscore_list, node) + if (syscore->ops->resume) + syscore->ops->resume(syscore->data); return ret; } @@ -90,18 +92,19 @@ EXPORT_SYMBOL_GPL(syscore_suspend); */ void syscore_resume(void) { - struct syscore_ops *ops; + struct syscore *syscore; trace_suspend_resume(TPS("syscore_resume"), 0, true); WARN_ONCE(!irqs_disabled(), "Interrupts enabled before system core resume.\n"); - list_for_each_entry(ops, &syscore_ops_list, node) - if (ops->resume) { - pm_pr_dbg("Calling %pS\n", ops->resume); - ops->resume(); + list_for_each_entry(syscore, &syscore_list, node) + if (syscore->ops->resume) { + pm_pr_dbg("Calling %pS\n", syscore->ops->resume); + syscore->ops->resume(syscore->data); WARN_ONCE(!irqs_disabled(), - "Interrupts enabled after %pS\n", ops->resume); + "Interrupts enabled after %pS\n", + syscore->ops->resume); } trace_suspend_resume(TPS("syscore_resume"), 0, false); } @@ -113,16 +116,17 @@ EXPORT_SYMBOL_GPL(syscore_resume); */ void syscore_shutdown(void) { - struct syscore_ops *ops; + struct syscore *syscore; - mutex_lock(&syscore_ops_lock); + mutex_lock(&syscore_lock); - list_for_each_entry_reverse(ops, &syscore_ops_list, node) - if (ops->shutdown) { + list_for_each_entry_reverse(syscore, &syscore_list, node) + if (syscore->ops->shutdown) { if (initcall_debug) - pr_info("PM: Calling %pS\n", ops->shutdown); - ops->shutdown(); + pr_info("PM: Calling %pS\n", + syscore->ops->shutdown); + syscore->ops->shutdown(syscore->data); } - mutex_unlock(&syscore_ops_lock); + mutex_unlock(&syscore_lock); } diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 567192e371a8f..a7bf990ce9c32 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3048,6 +3048,8 @@ static int mtip_block_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3086,6 +3088,8 @@ static int mtip_block_compat_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3711,6 +3715,7 @@ static int mtip_pci_probe(struct pci_dev *pdev, dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); if (!dd) return -ENOMEM; + mutex_init(&dd->ioctl_mutex); /* Attach the private data to this PCI device. */ pci_set_drvdata(pdev, dd); @@ -3874,6 +3879,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) } set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + mutex_lock(&dd->ioctl_mutex); if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); @@ -3902,6 +3908,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) /* De-initialize the protocol layer. */ mtip_hw_exit(dd); + mutex_unlock(&dd->ioctl_mutex); if (dd->isr_workq) { destroy_workqueue(dd->isr_workq); diff --git a/drivers/block/mtip32xx/mtip32xx.h b/drivers/block/mtip32xx/mtip32xx.h index f7328f19ac5c2..0963c07b58457 100644 --- a/drivers/block/mtip32xx/mtip32xx.h +++ b/drivers/block/mtip32xx/mtip32xx.h @@ -12,6 +12,7 @@ #define __MTIP32XX_H__ #include +#include #include #include #include @@ -432,6 +433,7 @@ struct driver_data { struct request_queue *queue; /* Our request queue. */ struct blk_mq_tag_set tags; /* blk_mq tags */ + struct mutex ioctl_mutex; struct mtip_port *port; /* Pointer to the port data structure. */ diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 50938881219bc..35e3a4eb21d80 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -66,7 +66,7 @@ struct nullb_page { #define NULLB_PAGE_FREE (MAP_SZ - 2) static LIST_HEAD(nullb_list); -static struct mutex lock; +static DEFINE_MUTEX(lock); static int null_major; static DEFINE_IDA(nullb_indexes); static struct blk_mq_tag_set tag_set; @@ -340,7 +340,15 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, return count; } -/* The following macro should only be used with TYPE = {uint, ulong, bool}. */ +/* + * The following macro should only be used with TYPE = {uint, ulong, bool}. + * + * The device configuration is modified under the global lock to serialize + * attribute changes against null_add_dev() and null_del_dev(): without this, + * an attribute could be changed while null_add_dev() is running, that is, + * before NULLB_DEV_FL_CONFIGURED is set, which would let null_add_dev() + * observe inconsistent values for the device configuration. + */ #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ @@ -360,6 +368,7 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ if (ret < 0) \ return ret; \ + guard(mutex)(&lock); \ if (apply_fn) \ ret = apply_fn(dev, new_value); \ else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ @@ -379,9 +388,20 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, struct blk_mq_tag_set *set; int ret, nr_hw_queues; + lockdep_assert_held(&lock); + if (!dev->nullb) return 0; + /* + * A shared tag_set is mapped via the module-wide queue counts, so a + * per-device resize is meaningless. On shrink it would also leave + * mq_map[] pointing at NULLed hctx slots, causing a NULL deref in + * blk_mq_map_swqueue(). Reject it. + */ + if (dev->shared_tags) + return -EINVAL; + /* * Make sure at least one submit queue exists. */ @@ -421,25 +441,13 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, static int nullb_apply_submit_queues(struct nullb_device *dev, unsigned int submit_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); } static int nullb_apply_poll_queues(struct nullb_device *dev, unsigned int poll_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); @@ -579,8 +587,13 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; - return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_unlock(&lock); + + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -588,8 +601,13 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; + + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + mutex_unlock(&lock); - return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -836,7 +854,6 @@ static void null_free_dev(struct nullb_device *dev) if (!dev) return; - null_free_zoned_dev(dev); badblocks_exit(&dev->badblocks); kfree(dev); } @@ -1777,6 +1794,7 @@ static void null_del_dev(struct nullb *nullb) } put_disk(nullb->disk); + null_free_zoned_dev(dev); if (nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); @@ -2163,17 +2181,9 @@ static int __init null_init(void) config_group_init(&nullb_subsys.su_group); mutex_init(&nullb_subsys.su_mutex); - ret = configfs_register_subsystem(&nullb_subsys); - if (ret) - return ret; - - mutex_init(&lock); - null_major = register_blkdev(0, "nullb"); - if (null_major < 0) { - ret = null_major; - goto err_conf; - } + if (null_major < 0) + return null_major; for (i = 0; i < nr_devices; i++) { ret = null_create_dev(); @@ -2181,6 +2191,10 @@ static int __init null_init(void) goto err_dev; } + ret = configfs_register_subsystem(&nullb_subsys); + if (ret) + goto err_dev; + pr_info("module loaded\n"); return 0; @@ -2190,8 +2204,8 @@ static int __init null_init(void) null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); -err_conf: - configfs_unregister_subsystem(&nullb_subsys); + if (tag_set.ops) + blk_mq_free_tag_set(&tag_set); return ret; } @@ -2212,8 +2226,6 @@ static void __exit null_exit(void) if (tag_set.ops) blk_mq_free_tag_set(&tag_set); - - mutex_destroy(&lock); } module_init(null_init); diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 4e5728f459899..be3c3a68411b0 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -18,6 +18,8 @@ static inline sector_t mb_to_sects(unsigned long mb) static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect) { + if (WARN_ON_ONCE(!dev->zone_size_sects)) + return 0; return sect >> ilog2(dev->zone_size_sects); } @@ -56,8 +58,8 @@ int null_init_zoned_dev(struct nullb_device *dev, sector_t sector = 0; unsigned int i; - if (!is_power_of_2(dev->zone_size)) { - pr_err("zone_size must be power-of-two\n"); + if (!dev->zone_size || !is_power_of_2(dev->zone_size)) { + pr_err("zone_size must be non-zero power-of-two\n"); return -EINVAL; } if (dev->zone_size > dev->size) { @@ -88,6 +90,10 @@ int null_init_zoned_dev(struct nullb_device *dev, zone_capacity_sects = mb_to_sects(dev->zone_capacity); dev_capacity_sects = mb_to_sects(dev->size); dev->zone_size_sects = mb_to_sects(dev->zone_size); + if (!dev->zone_size_sects) { + pr_err("zone_size too large or too small, leads to zero sectors\n"); + return -EINVAL; + } dev->nr_zones = round_up(dev_capacity_sects, dev->zone_size_sects) >> ilog2(dev->zone_size_sects); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index 97cd8cf7bad30..6c802f029f57d 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -1957,9 +1957,14 @@ static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req, bool has_current_state; void *p; - if (osd_req->r_result) + if (osd_req->r_result < 0) return osd_req->r_result; + /* + * Writes aren't allowed to return a data payload. + */ + WARN_ON_ONCE(osd_req->r_result > 0); + /* * Nothing to do for a snapshot object map. */ diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index db1fe9772a4d7..2eb96136632ac 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -525,6 +525,23 @@ static int __send_request(struct request *req) err = __vdc_tx_trigger(port); if (err < 0) { printk(KERN_ERR PFX "vdc_tx_trigger() failure, err=%d\n", err); + /* + * If the port was reset (-ENOTCONN), the dring and the + * LDC channel including all of its mappings are already + * torn down and reallocated - there is nothing to undo + * and @desc must not be touched. + * + * For any other failure the descriptor was never handed + * to the peer: unmap the cookies and free the descriptor + * again, so that a later retry of the request does not + * leak LDC map table entries. + */ + if (err != -ENOTCONN) { + ldc_unmap(port->vio.lp, desc->cookies, + desc->ncookies); + desc->hdr.state = VIO_DESC_FREE; + rqe->req = NULL; + } } else { port->req_id++; dr->prod = vio_dring_next(dr, dr->prod); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index c339222513b03..a11805b29b630 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -26,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -230,7 +230,6 @@ struct ublk_device { struct ublk_params params; - struct completion completion; u32 nr_io_ready; bool unprivileged_daemons; struct mutex cancel_mutex; @@ -604,7 +603,7 @@ static int ublk_validate_params(const struct ublk_device *ub) if (p->max_sectors < PAGE_SECTORS) return -EINVAL; - if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) + if (ublk_dev_is_zoned(ub) && !is_power_of_2(p->chunk_sectors)) return -EINVAL; } else return -EINVAL; @@ -1051,7 +1050,10 @@ static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1072,7 +1074,10 @@ static int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1191,8 +1196,14 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, * * Re-read simply for this unlikely case. */ - if (unlikely(unmapped_bytes < io->res)) + if (unlikely(unmapped_bytes < io->res)) { + if (unlikely(!unmapped_bytes)) { + res = BLK_STS_IOERR; + goto exit; + } + io->res = unmapped_bytes; + } /* * Run bio->bi_end_io() with softirqs disabled. If the final fput @@ -1863,6 +1874,12 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) if (vma->vm_flags & VM_WRITE) return -EPERM; + /* + * The per-queue command buffer is kernel-written ABI; prevent + * the daemon from upgrading to writable via mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end) return -EINVAL; @@ -2150,9 +2167,13 @@ static void ublk_mark_io_ready(struct ublk_device *ub) ub->nr_io_ready++; if (ublk_dev_ready(ub)) { - /* now we are ready for handling ublk io request */ + /* + * now we are ready for handling ublk io request, clear + * device-level canceling flag and wake ublk_dev_ready() + * waiters + */ ublk_reset_io_flags(ub); - complete_all(&ub->completion); + wake_up_var(&ub->nr_io_ready); } } @@ -2829,7 +2850,6 @@ static int ublk_init_queues(struct ublk_device *ub) goto fail; } - init_completion(&ub->completion); return 0; fail: @@ -2966,6 +2986,26 @@ static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid) return ub->ublksrv_tgid == ublksrv_pid; } +/* + * Wait until all queues have fetched their I/O commands, and return with + * ub->mutex held and readiness guaranteed: then every queue's ->canceling + * is cleared. Ready may regress between wakeup and mutex_lock() (daemon + * death), so re-check it under the mutex and wait again. + */ +static int ublk_wait_dev_ready_and_lock(struct ublk_device *ub) +{ + while (true) { + if (wait_var_event_interruptible(&ub->nr_io_ready, + ublk_dev_ready(ub))) + return -EINTR; + + mutex_lock(&ub->mutex); + if (ublk_dev_ready(ub)) + return 0; + mutex_unlock(&ub->mutex); + } +} + static int ublk_ctrl_start_dev(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { @@ -3031,13 +3071,13 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, lim.max_segments = ub->params.seg.max_segments; } - if (wait_for_completion_interruptible(&ub->completion) != 0) + if (ublk_wait_dev_ready_and_lock(ub)) return -EINTR; - if (!ublk_validate_user_pid(ub, ublksrv_pid)) - return -EINVAL; - - mutex_lock(&ub->mutex); + if (!ublk_validate_user_pid(ub, ublksrv_pid)) { + ret = -EINVAL; + goto out_unlock; + } if (ub->dev_info.state == UBLK_S_DEV_LIVE || test_bit(UB_STATE_USED, &ub->state)) { ret = -EEXIST; @@ -3265,6 +3305,15 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) /* update device id */ ub->dev_info.dev_id = ub->ub_number; + /* + * ->state and ->ublksrv_pid are owned by the driver and only read back + * by userspace, but they come from the copied-in dev_info, so reset + * them. Otherwise a device added with ->state != DEAD looks live while + * ->ub_disk is still NULL. + */ + ub->dev_info.state = UBLK_S_DEV_DEAD; + ub->dev_info.ublksrv_pid = -1; + /* * 64bit flags will be copied back to userspace as feature * negotiation result, so have to clear flags which driver @@ -3549,7 +3598,6 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub, goto out_unlock; } pr_devel("%s: start recovery for dev id %d.\n", __func__, header->dev_id); - init_completion(&ub->completion); ret = 0; out_unlock: mutex_unlock(&ub->mutex); @@ -3565,16 +3613,17 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub, pr_devel("%s: Waiting for all FETCH_REQs, dev id %d...\n", __func__, header->dev_id); - if (wait_for_completion_interruptible(&ub->completion)) + if (ublk_wait_dev_ready_and_lock(ub)) return -EINTR; pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__, header->dev_id); - if (!ublk_validate_user_pid(ub, ublksrv_pid)) - return -EINVAL; + if (!ublk_validate_user_pid(ub, ublksrv_pid)) { + ret = -EINVAL; + goto out_unlock; + } - mutex_lock(&ub->mutex); if (ublk_nosrv_should_stop_dev(ub)) goto out_unlock; diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 26364e43aeb34..45088c163286e 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -319,7 +319,8 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) zone->cond == BLK_ZONE_COND_FULL) goto unlock; - if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) { + if (vfs_truncate(&zone->file->f_path, + zlo->zone_capacity << SECTOR_SHIFT)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index b75016e0e654b..fd438dcdfad60 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -24,8 +24,16 @@ static int deflate_setup_params(struct zcomp_params *params) { if (params->level == ZCOMP_PARAM_NOT_SET) params->level = Z_DEFAULT_COMPRESSION; - if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; + } else { + s32 wb = params->deflate.winbits; + + if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { + pr_err("invalid deflate winbits: %d\n", wb); + return -EINVAL; + } + } return 0; } diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index f51dd33784291..5a08e8bebd6df 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -501,7 +501,8 @@ static ssize_t idle_store(struct device *dev, #ifdef CONFIG_ZRAM_WRITEBACK static ssize_t writeback_limit_enable_store(struct device *dev, - struct device_attribute *attr, const char *buf, size_t len) + struct device_attribute *attr, + const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); u64 val; @@ -510,33 +511,31 @@ static ssize_t writeback_limit_enable_store(struct device *dev, if (kstrtoull(buf, 10, &val)) return ret; - down_read(&zram->init_lock); - spin_lock(&zram->wb_limit_lock); + down_write(&zram->init_lock); zram->wb_limit_enable = val; - spin_unlock(&zram->wb_limit_lock); - up_read(&zram->init_lock); + up_write(&zram->init_lock); ret = len; return ret; } static ssize_t writeback_limit_enable_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, + char *buf) { bool val; struct zram *zram = dev_to_zram(dev); down_read(&zram->init_lock); - spin_lock(&zram->wb_limit_lock); val = zram->wb_limit_enable; - spin_unlock(&zram->wb_limit_lock); up_read(&zram->init_lock); return sysfs_emit(buf, "%d\n", val); } static ssize_t writeback_limit_store(struct device *dev, - struct device_attribute *attr, const char *buf, size_t len) + struct device_attribute *attr, + const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); u64 val; @@ -545,26 +544,32 @@ static ssize_t writeback_limit_store(struct device *dev, if (kstrtoull(buf, 10, &val)) return ret; - down_read(&zram->init_lock); - spin_lock(&zram->wb_limit_lock); + /* + * When the page size is greater than 4KB, if bd_wb_limit is set to + * a value that is not page - size aligned, it will cause value + * wrapping. For example, when the page size is set to 16KB and + * bd_wb_limit is set to 3, a single write - back operation will + * cause bd_wb_limit to become -1. Even more terrifying is that + * bd_wb_limit is an unsigned number. + */ + val = rounddown(val, PAGE_SIZE / 4096); + + down_write(&zram->init_lock); zram->bd_wb_limit = val; - spin_unlock(&zram->wb_limit_lock); - up_read(&zram->init_lock); + up_write(&zram->init_lock); ret = len; return ret; } static ssize_t writeback_limit_show(struct device *dev, - struct device_attribute *attr, char *buf) + struct device_attribute *attr, char *buf) { u64 val; struct zram *zram = dev_to_zram(dev); down_read(&zram->init_lock); - spin_lock(&zram->wb_limit_lock); val = zram->bd_wb_limit; - spin_unlock(&zram->wb_limit_lock); up_read(&zram->init_lock); return sysfs_emit(buf, "%llu\n", val); @@ -744,18 +749,17 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl) int ret = 0, err; u32 index; + lockdep_assert_held_read(&zram->init_lock); + page = alloc_page(GFP_KERNEL); if (!page) return -ENOMEM; while ((pps = select_pp_slot(ctl))) { - spin_lock(&zram->wb_limit_lock); if (zram->wb_limit_enable && !zram->bd_wb_limit) { - spin_unlock(&zram->wb_limit_lock); ret = -EIO; break; } - spin_unlock(&zram->wb_limit_lock); if (!blk_idx) { blk_idx = alloc_block_bdev(zram); @@ -821,10 +825,8 @@ static int zram_writeback_slots(struct zram *zram, struct zram_pp_ctl *ctl) zram_set_handle(zram, index, blk_idx); blk_idx = 0; atomic64_inc(&zram->stats.pages_stored); - spin_lock(&zram->wb_limit_lock); if (zram->wb_limit_enable && zram->bd_wb_limit > 0) zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12); - spin_unlock(&zram->wb_limit_lock); next: zram_slot_unlock(zram, index); release_pp_slot(zram, pps); @@ -1135,8 +1137,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, char *kbuf; ssize_t index, written = 0; struct zram *zram = file->private_data; - unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; - struct timespec64 ts; + unsigned long nr_pages; kbuf = kvmalloc(count, GFP_KERNEL); if (!kbuf) @@ -1149,8 +1150,11 @@ static ssize_t read_block_state(struct file *file, char __user *buf, return -EINVAL; } + nr_pages = zram->disksize >> PAGE_SHIFT; + for (index = *ppos; index < nr_pages; index++) { int copied; + struct timespec64 ts; zram_slot_lock(zram, index); if (!zram_allocated(zram, index)) @@ -1740,14 +1744,14 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, ret = zram_read_from_zspool(zram, page, index); zram_slot_unlock(zram, index); } else { + unsigned long blk_idx = zram_get_handle(zram, index); + /* * The slot should be unlocked before reading from the backing * device. */ zram_slot_unlock(zram, index); - - ret = read_from_bdev(zram, page, zram_get_handle(zram, index), - parent); + ret = read_from_bdev(zram, page, blk_idx, parent); } /* Should NEVER happen. Return bio error if it does. */ @@ -2460,11 +2464,12 @@ static void zram_destroy_comps(struct zram *zram) } zram_comp_params_reset(zram); + comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static void zram_reset_device(struct zram *zram) { - down_write(&zram->init_lock); + guard(rwsem_write)(&zram->init_lock); zram->limit_pages = 0; @@ -2478,9 +2483,6 @@ static void zram_reset_device(struct zram *zram) memset(&zram->stats, 0, sizeof(zram->stats)); atomic_set(&zram->pp_in_progress, 0); reset_bdev(zram); - - comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); - up_write(&zram->init_lock); } static ssize_t disksize_store(struct device *dev, @@ -2692,9 +2694,6 @@ static int zram_add(void) device_id = ret; init_rwsem(&zram->init_lock); -#ifdef CONFIG_ZRAM_WRITEBACK - spin_lock_init(&zram->wb_limit_lock); -#endif /* gendisk structure */ zram->disk = blk_alloc_disk(&lim, NUMA_NO_NODE); diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 6cee93f9c0d06..8f64317e3b6c5 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -127,7 +127,6 @@ struct zram { bool claim; /* Protected by disk->open_mutex */ #ifdef CONFIG_ZRAM_WRITEBACK struct file *backing_dev; - spinlock_t wb_limit_lock; bool wb_limit_enable; u64 bd_wb_limit; struct block_device *bdev; diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 5e0a05edcbfd1..0e5c2d3893717 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -554,12 +554,44 @@ int btintel_version_info_tlv(struct hci_dev *hdev, } EXPORT_SYMBOL_GPL(btintel_version_info_tlv); +static u8 btintel_version_tlv_min_len(u8 type) +{ + switch (type) { + case INTEL_TLV_CNVI_TOP: + case INTEL_TLV_CNVR_TOP: + case INTEL_TLV_CNVI_BT: + case INTEL_TLV_CNVR_BT: + case INTEL_TLV_BUILD_NUM: + case INTEL_TLV_GIT_SHA1: + return sizeof(u32); + case INTEL_TLV_DEV_REV_ID: + case INTEL_TLV_TIME_STAMP: + return sizeof(u16); + case INTEL_TLV_IMAGE_TYPE: + case INTEL_TLV_BUILD_TYPE: + case INTEL_TLV_SECURE_BOOT: + case INTEL_TLV_OTP_LOCK: + case INTEL_TLV_API_LOCK: + case INTEL_TLV_DEBUG_LOCK: + case INTEL_TLV_LIMITED_CCE: + case INTEL_TLV_SBE_TYPE: + return sizeof(u8); + case INTEL_TLV_MIN_FW: + return 3; + case INTEL_TLV_OTP_BDADDR: + return sizeof(bdaddr_t); + default: + return 0; + } +} + int btintel_parse_version_tlv(struct hci_dev *hdev, struct intel_version_tlv *version, struct sk_buff *skb) { /* Consume Command Complete Status field */ - skb_pull(skb, 1); + if (!skb_pull(skb, 1)) + return -EINVAL; /* Event parameters contain multiple TLVs. Read each of them * and only keep the required data. Also, it use existing legacy @@ -579,6 +611,9 @@ int btintel_parse_version_tlv(struct hci_dev *hdev, if (skb->len < tlv->len + sizeof(*tlv)) return -EINVAL; + if (tlv->len < btintel_version_tlv_min_len(tlv->type)) + return -EINVAL; + switch (tlv->type) { case INTEL_TLV_CNVI_TOP: version->cnvi_top = get_unaligned_le32(tlv->val); @@ -650,7 +685,7 @@ int btintel_parse_version_tlv(struct hci_dev *hdev, break; case INTEL_TLV_FW_ID: snprintf(version->fw_id, sizeof(version->fw_id), - "%s", tlv->val); + "%.*s", tlv->len, tlv->val); break; default: /* Ignore rest of information */ @@ -3694,6 +3729,9 @@ static int btintel_diagnostics(struct hci_dev *hdev, struct sk_buff *skb) { struct intel_tlv *tlv = (void *)&skb->data[5]; + if (skb->len < 5 + sizeof(*tlv) + sizeof(tlv->val[0])) + goto recv_frame; + /* The first event is always an event type TLV */ if (tlv->type != INTEL_TLV_TYPE_ID) goto recv_frame; @@ -3754,7 +3792,7 @@ int btintel_recv_event(struct hci_dev *hdev, struct sk_buff *skb) /* Handle all diagnostics events separately. May still call * hci_recv_frame. */ - if (len >= sizeof(diagnostics_hdr) && + if (len + 1 >= sizeof(diagnostics_hdr) && memcmp(&skb->data[2], diagnostics_hdr, sizeof(diagnostics_hdr)) == 0) { return btintel_diagnostics(hdev, skb); diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c index d0aa1666d6ac6..56ba643d5ff7e 100644 --- a/drivers/bluetooth/btintel_pcie.c +++ b/drivers/bluetooth/btintel_pcie.c @@ -740,6 +740,11 @@ static void btintel_pcie_dump_traces(struct hci_dev *hdev) bt_dev_err(hdev, "Failed to dump traces: (%d)", ret); } +static bool btintel_pcie_is_blazariw(struct pci_dev *pdev) +{ + return pdev->device == 0x4D76; +} + /* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0. @@ -759,6 +764,14 @@ static int btintel_pcie_enable_bt(struct btintel_pcie_data *data) btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG, (u64)data->ci_p_addr >> 32); + /* On BlazarIW, the D0 entry to MAC init does not complete in + * time. Wait 50 ms (worst case as per HW analysis) for the + * shared hardware reset flow to complete before proceeding with + * MAC init. + */ + if (btintel_pcie_is_blazariw(data->pdev)) + msleep(50); + /* Reset the cached value of boot stage. it is updated by the MSI-X * gp0 interrupt handler. */ @@ -1005,7 +1018,7 @@ static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data) urbd0 = &txq->urbd0s[cr_tia]; - if (urbd0->tfd_index > txq->count) + if (urbd0->tfd_index >= txq->count) return; cr_tia = (cr_tia + 1) % txq->count; @@ -1352,7 +1365,9 @@ static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status rfh_hdr = buf; len = rfh_hdr->packet_len; - if (len <= 0) { + if (len == 0 || len > BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)) { + bt_dev_err(data->hdev, "Invalid packet_len %d (max %zu)", len, + BTINTEL_PCIE_BUFFER_SIZE - sizeof(*rfh_hdr)); ret = -EINVAL; goto resubmit; } @@ -1449,6 +1464,9 @@ static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id) if (unlikely(!(intr_fh | intr_hw))) { /* Ignore interrupt, inta == 0 */ + bt_warn_ratelimited("Bluetooth: btintel_pcie: Received spurious interrupt\n"); + btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST, + BIT(entry->entry)); return IRQ_NONE; } diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c index a908e7b46ecaf..d1817dd6ab650 100644 --- a/drivers/bluetooth/btmtk.c +++ b/drivers/bluetooth/btmtk.c @@ -112,7 +112,11 @@ static void btmtk_coredump_notify(struct hci_dev *hdev, int state) void btmtk_fw_get_filename(char *buf, size_t size, u32 dev_id, u32 fw_ver, u32 fw_flavor) { - if (dev_id == 0x7925) + if (dev_id == 0x6639) + snprintf(buf, size, + "mediatek/mt7927/BT_RAM_CODE_MT%04x_2_%x_hdr.bin", + dev_id & 0xffff, (fw_ver & 0xff) + 1); + else if (dev_id == 0x7925) snprintf(buf, size, "mediatek/mt%04x/BT_RAM_CODE_MT%04x_1_%x_hdr.bin", dev_id & 0xffff, dev_id & 0xffff, (fw_ver & 0xff) + 1); @@ -128,7 +132,8 @@ void btmtk_fw_get_filename(char *buf, size_t size, u32 dev_id, u32 fw_ver, EXPORT_SYMBOL_GPL(btmtk_fw_get_filename); int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, - wmt_cmd_sync_func_t wmt_cmd_sync) + wmt_cmd_sync_func_t wmt_cmd_sync, + u32 dev_id) { struct btmtk_hci_wmt_params wmt_params; struct btmtk_patch_header *hdr; @@ -166,6 +171,14 @@ int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, section_offset = le32_to_cpu(sectionmap->secoffset); dl_size = le32_to_cpu(sectionmap->bin_info_spec.dlsize); + /* MT6639: only download sections where dlmode byte0 == 0x01, + * matching the Windows driver behavior which skips WiFi/other + * sections that would cause the chip to hang. + */ + if (dev_id == 0x6639 && dl_size > 0 && + (le32_to_cpu(sectionmap->bin_info_spec.dlmodecrctype) & 0xff) != 0x01) + continue; + if (dl_size > 0) { retry = 20; while (retry > 0) { @@ -442,6 +455,22 @@ int btmtk_process_coredump(struct hci_dev *hdev, struct sk_buff *skb) EXPORT_SYMBOL_GPL(btmtk_process_coredump); #if IS_ENABLED(CONFIG_BT_HCIBTUSB_MTK) +/* Known MT6639 (MT7927) Bluetooth USB devices. + * Used to scope the zero-CHIPID workaround to real MT6639 hardware, + * since some boards return 0x0000 from the MMIO chip ID register. + */ +static const struct { + u16 vendor; + u16 product; +} btmtk_mt6639_devs[] = { + { 0x0489, 0xe13a }, /* ASUS ROG Crosshair X870E Hero */ + { 0x0489, 0xe0fa }, /* Lenovo Legion Pro 7 16ARX9 */ + { 0x0489, 0xe10f }, /* Gigabyte Z790 AORUS MASTER X */ + { 0x0489, 0xe110 }, /* MSI X870E Ace Max */ + { 0x0489, 0xe116 }, /* TP-Link Archer TBE550E */ + { 0x13d3, 0x3588 }, /* ASUS ROG STRIX X870E-E */ +}; + static void btmtk_usb_wmt_recv(struct urb *urb) { struct hci_dev *hdev = urb->context; @@ -763,56 +792,44 @@ static int btmtk_usb_uhw_reg_write(struct hci_dev *hdev, u32 reg, u32 val) static int btmtk_usb_uhw_reg_read(struct hci_dev *hdev, u32 reg, u32 *val) { struct btmtk_data *data = hci_get_priv(hdev); - int pipe, err; - void *buf; - - buf = kzalloc(4, GFP_KERNEL); - if (!buf) - return -ENOMEM; + u8 buf[sizeof(u32)]; + int err; - pipe = usb_rcvctrlpipe(data->udev, 0); - err = usb_control_msg(data->udev, pipe, 0x01, - 0xDE, - reg >> 16, reg & 0xffff, - buf, 4, USB_CTRL_GET_TIMEOUT); - if (err < 0) { + *val = 0; + err = usb_control_msg_recv(data->udev, 0, 0x01, + 0xDE, + reg >> 16, reg & 0xffff, + buf, sizeof(buf), USB_CTRL_GET_TIMEOUT, + GFP_KERNEL); + if (err) { bt_dev_err(hdev, "Failed to read uhw reg(%d)", err); - goto err_free_buf; + return err; } *val = get_unaligned_le32(buf); bt_dev_dbg(hdev, "reg=%x, value=0x%08x", reg, *val); -err_free_buf: - kfree(buf); - - return err; + return 0; } static int btmtk_usb_reg_read(struct hci_dev *hdev, u32 reg, u32 *val) { struct btmtk_data *data = hci_get_priv(hdev); - int pipe, err, size = sizeof(u32); - void *buf; - - buf = kzalloc(size, GFP_KERNEL); - if (!buf) - return -ENOMEM; + u8 buf[sizeof(u32)]; + int err; - pipe = usb_rcvctrlpipe(data->udev, 0); - err = usb_control_msg(data->udev, pipe, 0x63, - USB_TYPE_VENDOR | USB_DIR_IN, - reg >> 16, reg & 0xffff, - buf, size, USB_CTRL_GET_TIMEOUT); + *val = 0; + err = usb_control_msg_recv(data->udev, 0, 0x63, + USB_TYPE_VENDOR | USB_DIR_IN, + reg >> 16, reg & 0xffff, + buf, sizeof(buf), USB_CTRL_GET_TIMEOUT, + GFP_KERNEL); if (err < 0) - goto err_free_buf; + return err; *val = get_unaligned_le32(buf); -err_free_buf: - kfree(buf); - - return err; + return 0; } static int btmtk_usb_id_get(struct hci_dev *hdev, u32 reg, u32 *id) @@ -831,12 +848,13 @@ static u32 btmtk_usb_reset_done(struct hci_dev *hdev) int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) { + int reset_err = 0; u32 val; int err; if (dev_id == 0x7922) { err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_SUBSYS_RST, &val); - if (err < 0) + if (err) return err; val |= 0x00002020; err = btmtk_usb_uhw_reg_write(hdev, MTK_BT_SUBSYS_RST, val); @@ -846,23 +864,23 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_SUBSYS_RST, &val); - if (err < 0) + if (err) return err; val |= BIT(0); err = btmtk_usb_uhw_reg_write(hdev, MTK_BT_SUBSYS_RST, val); if (err < 0) return err; msleep(100); - } else if (dev_id == 0x7925) { + } else if (dev_id == 0x7925 || dev_id == 0x6639) { err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_RESET_REG_CONNV3, &val); - if (err < 0) + if (err) return err; val |= (1 << 5); err = btmtk_usb_uhw_reg_write(hdev, MTK_BT_RESET_REG_CONNV3, val); if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_RESET_REG_CONNV3, &val); - if (err < 0) + if (err) return err; val &= 0xFFFF00FF; val |= (1 << 13); @@ -873,7 +891,7 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_RESET_REG_CONNV3, &val); - if (err < 0) + if (err) return err; val |= (1 << 0); err = btmtk_usb_uhw_reg_write(hdev, MTK_BT_RESET_REG_CONNV3, val); @@ -883,13 +901,13 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_UDMA_INT_STA_BT, &val); - if (err < 0) + if (err) return err; err = btmtk_usb_uhw_reg_write(hdev, MTK_UDMA_INT_STA_BT1, 0x000000FF); if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_UDMA_INT_STA_BT1, &val); - if (err < 0) + if (err) return err; msleep(100); } else { @@ -899,7 +917,7 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_WDT_STATUS, &val); - if (err < 0) + if (err) return err; /* Reset the bluetooth chip via USB interface. */ err = btmtk_usb_uhw_reg_write(hdev, MTK_BT_SUBSYS_RST, 1); @@ -909,13 +927,13 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_UDMA_INT_STA_BT, &val); - if (err < 0) + if (err) return err; err = btmtk_usb_uhw_reg_write(hdev, MTK_UDMA_INT_STA_BT1, 0x000000FF); if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_UDMA_INT_STA_BT1, &val); - if (err < 0) + if (err) return err; /* MT7921 need to delay 20ms between toggle reset bit */ msleep(20); @@ -923,14 +941,16 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) if (err < 0) return err; err = btmtk_usb_uhw_reg_read(hdev, MTK_BT_SUBSYS_RST, &val); - if (err < 0) + if (err) return err; } err = readx_poll_timeout(btmtk_usb_reset_done, hdev, val, val & MTK_BT_RST_DONE, 20000, 1000000); - if (err < 0) + if (err < 0) { bt_dev_err(hdev, "Reset timeout"); + reset_err = err; + } if (dev_id == 0x7922) { err = btmtk_usb_uhw_reg_write(hdev, MTK_UDMA_INT_STA_BT, 0x000000FF); @@ -939,10 +959,12 @@ int btmtk_usb_subsys_reset(struct hci_dev *hdev, u32 dev_id) } err = btmtk_usb_id_get(hdev, 0x70010200, &val); - if (err < 0 || !val) + if (err || (!val && dev_id != 0x6639)) { bt_dev_err(hdev, "Can't get device id, subsys reset fail."); + return err ? err : -ENODEV; + } - return err; + return reset_err; } EXPORT_SYMBOL_GPL(btmtk_usb_subsys_reset); @@ -1301,30 +1323,48 @@ int btmtk_usb_setup(struct hci_dev *hdev) calltime = ktime_get(); err = btmtk_usb_id_get(hdev, 0x80000008, &dev_id); - if (err < 0) { + if (err) { bt_dev_err(hdev, "Failed to get device id (%d)", err); return err; } if (!dev_id || dev_id != 0x7663) { err = btmtk_usb_id_get(hdev, 0x70010200, &dev_id); - if (err < 0) { + if (err) { bt_dev_err(hdev, "Failed to get device id (%d)", err); return err; } err = btmtk_usb_id_get(hdev, 0x80021004, &fw_version); - if (err < 0) { + if (err) { bt_dev_err(hdev, "Failed to get fw version (%d)", err); return err; } err = btmtk_usb_id_get(hdev, 0x70010020, &fw_flavor); - if (err < 0) { + if (err) { bt_dev_err(hdev, "Failed to get fw flavor (%d)", err); return err; } fw_flavor = (fw_flavor & 0x00000080) >> 7; } + if (!dev_id) { + u16 vid = le16_to_cpu(btmtk_data->udev->descriptor.idVendor); + u16 pid = le16_to_cpu(btmtk_data->udev->descriptor.idProduct); + int i; + + for (i = 0; i < ARRAY_SIZE(btmtk_mt6639_devs); i++) { + if (vid == btmtk_mt6639_devs[i].vendor && + pid == btmtk_mt6639_devs[i].product) { + dev_id = 0x6639; + break; + } + } + + if (dev_id) + bt_dev_info(hdev, "MT6639: CHIPID=0x0000 with VID=%04x PID=%04x, using 0x6639", + vid, pid); + } + btmtk_data->dev_id = dev_id; err = btmtk_register_coredump(hdev, btmtk_data->drv_name, fw_version); @@ -1340,12 +1380,25 @@ int btmtk_usb_setup(struct hci_dev *hdev) break; case 0x7922: case 0x7925: + /* + * A remote wakeup could cause the device completely unresponsive, and + * recovering from such a state needs a power cycle. + * + * Since the remote wakeup capability is super broken, just disable it + * to get rid of the troubles. The device can still be autosuspended + * when the bluetooth interface is closed. + */ + device_set_wakeup_capable(&btmtk_data->udev->dev, false); + fallthrough; case 0x7961: + case 0x7902: + case 0x6639: btmtk_fw_get_filename(fw_bin_name, sizeof(fw_bin_name), dev_id, fw_version, fw_flavor); err = btmtk_setup_firmware_79xx(hdev, fw_bin_name, - btmtk_usb_hci_wmt_sync); + btmtk_usb_hci_wmt_sync, + dev_id); if (err < 0) { bt_dev_err(hdev, "Failed to set up firmware (%d)", err); return err; @@ -1511,4 +1564,6 @@ MODULE_FIRMWARE(FIRMWARE_MT7663); MODULE_FIRMWARE(FIRMWARE_MT7668); MODULE_FIRMWARE(FIRMWARE_MT7922); MODULE_FIRMWARE(FIRMWARE_MT7961); +MODULE_FIRMWARE(FIRMWARE_MT7920); MODULE_FIRMWARE(FIRMWARE_MT7925); +MODULE_FIRMWARE(FIRMWARE_MT7927); diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h index 5df7c32966247..92e2deb29b586 100644 --- a/drivers/bluetooth/btmtk.h +++ b/drivers/bluetooth/btmtk.h @@ -5,8 +5,11 @@ #define FIRMWARE_MT7663 "mediatek/mt7663pr2h.bin" #define FIRMWARE_MT7668 "mediatek/mt7668pr2h.bin" #define FIRMWARE_MT7922 "mediatek/BT_RAM_CODE_MT7922_1_1_hdr.bin" +#define FIRMWARE_MT7902 "mediatek/BT_RAM_CODE_MT7902_1_1_hdr.bin" #define FIRMWARE_MT7961 "mediatek/BT_RAM_CODE_MT7961_1_2_hdr.bin" +#define FIRMWARE_MT7920 "mediatek/BT_RAM_CODE_MT7961_1a_2_hdr.bin" #define FIRMWARE_MT7925 "mediatek/mt7925/BT_RAM_CODE_MT7925_1_1_hdr.bin" +#define FIRMWARE_MT7927 "mediatek/mt7927/BT_RAM_CODE_MT6639_2_1_hdr.bin" #define HCI_EV_WMT 0xe4 #define HCI_WMT_MAX_EVENT_SIZE 64 @@ -187,7 +190,8 @@ typedef int (*wmt_cmd_sync_func_t)(struct hci_dev *, int btmtk_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr); int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, - wmt_cmd_sync_func_t wmt_cmd_sync); + wmt_cmd_sync_func_t wmt_cmd_sync, + u32 dev_id); int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname, wmt_cmd_sync_func_t wmt_cmd_sync); @@ -226,7 +230,8 @@ static inline int btmtk_set_bdaddr(struct hci_dev *hdev, static inline int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, - wmt_cmd_sync_func_t wmt_cmd_sync) + wmt_cmd_sync_func_t wmt_cmd_sync, + u32 dev_id) { return -EOPNOTSUPP; } diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index d2583bc742632..f7dcabc11b859 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -42,24 +42,35 @@ struct btmtksdio_data { const char *fwname; u16 chipid; bool lp_mbox_supported; + bool pm_runtime_supported; }; static const struct btmtksdio_data mt7663_data = { .fwname = FIRMWARE_MT7663, .chipid = 0x7663, .lp_mbox_supported = false, + .pm_runtime_supported = true, }; static const struct btmtksdio_data mt7668_data = { .fwname = FIRMWARE_MT7668, .chipid = 0x7668, .lp_mbox_supported = false, + .pm_runtime_supported = true, }; static const struct btmtksdio_data mt7921_data = { .fwname = FIRMWARE_MT7961, .chipid = 0x7921, .lp_mbox_supported = true, + .pm_runtime_supported = true, +}; + +static const struct btmtksdio_data mt7902_data = { + .fwname = FIRMWARE_MT7902, + .chipid = 0x7902, + .lp_mbox_supported = false, + .pm_runtime_supported = false, }; static const struct sdio_device_id btmtksdio_table[] = { @@ -69,6 +80,8 @@ static const struct sdio_device_id btmtksdio_table[] = { .driver_data = (kernel_ulong_t)&mt7668_data }, {SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7961), .driver_data = (kernel_ulong_t)&mt7921_data }, + {SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7902), + .driver_data = (kernel_ulong_t)&mt7902_data }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(sdio, btmtksdio_table); @@ -259,12 +272,24 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, struct sk_buff *skb) { struct mtkbtsdio_hdr *sdio_hdr; + unsigned int len, pad_len; int err; - /* Make sure that there are enough rooms for SDIO header */ - if (unlikely(skb_headroom(skb) < sizeof(*sdio_hdr))) { - err = pskb_expand_head(skb, sizeof(*sdio_hdr), 0, - GFP_ATOMIC); + /* Make sure that the data buffer is not shared with anyone else and + * that there is enough room for the SDIO header + */ + err = skb_cow_head(skb, sizeof(*sdio_hdr)); + if (err < 0) + return err; + + /* The transfer is rounded up to the SDIO block size, so the buffer + * has to provide tailroom for the padding as well + */ + len = skb->len + sizeof(*sdio_hdr); + pad_len = round_up(len, MTK_SDIO_BLOCK_SIZE) - len; + + if (unlikely(skb_tailroom(skb) < pad_len)) { + err = pskb_expand_head(skb, 0, pad_len, GFP_ATOMIC); if (err < 0) return err; } @@ -277,19 +302,22 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, sdio_hdr->reserved = cpu_to_le16(0); sdio_hdr->bt_type = hci_skb_pkt_type(skb); + /* Zero the padding so that no uninitialised memory is sent out */ + skb_put_zero(skb, pad_len); + clear_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state); - err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, - round_up(skb->len, MTK_SDIO_BLOCK_SIZE)); + err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, skb->len); if (err < 0) - goto err_skb_pull; + goto err_skb_restore; - bdev->hdev->stat.byte_tx += skb->len; + bdev->hdev->stat.byte_tx += len; kfree_skb(skb); return 0; -err_skb_pull: +err_skb_restore: + skb_trim(skb, len); skb_pull(skb, sizeof(*sdio_hdr)); return err; @@ -871,7 +899,7 @@ static int mt79xx_setup(struct hci_dev *hdev, const char *fwname) u8 param = 0x1; int err; - err = btmtk_setup_firmware_79xx(hdev, fwname, mtk_hci_wmt_sync); + err = btmtk_setup_firmware_79xx(hdev, fwname, mtk_hci_wmt_sync, 0); if (err < 0) { bt_dev_err(hdev, "Failed to setup 79xx firmware (%d)", err); return err; @@ -1091,6 +1119,7 @@ static int btmtksdio_setup(struct hci_dev *hdev) set_bit(BTMTKSDIO_HW_TX_READY, &bdev->tx_state); switch (bdev->data->chipid) { + case 0x7902: case 0x7921: if (test_bit(BTMTKSDIO_HW_RESET_ACTIVE, &bdev->tx_state)) { err = btmtksdio_mtk_reg_read(hdev, MT7921_DLSTATUS, @@ -1168,22 +1197,24 @@ static int btmtksdio_setup(struct hci_dev *hdev) delta = ktime_sub(rettime, calltime); duration = (unsigned long long)ktime_to_ns(delta) >> 10; - pm_runtime_set_autosuspend_delay(bdev->dev, - MTKBTSDIO_AUTOSUSPEND_DELAY); - pm_runtime_use_autosuspend(bdev->dev); + if (bdev->data->pm_runtime_supported) { + pm_runtime_set_autosuspend_delay(bdev->dev, + MTKBTSDIO_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(bdev->dev); - err = pm_runtime_set_active(bdev->dev); - if (err < 0) - return err; + err = pm_runtime_set_active(bdev->dev); + if (err < 0) + return err; - /* Default forbid runtime auto suspend, that can be allowed by - * enable_autosuspend flag or the PM runtime entry under sysfs. - */ - pm_runtime_forbid(bdev->dev); - pm_runtime_enable(bdev->dev); + /* Default forbid runtime auto suspend, that can be allowed by + * enable_autosuspend flag or the PM runtime entry under sysfs. + */ + pm_runtime_forbid(bdev->dev); + pm_runtime_enable(bdev->dev); - if (enable_autosuspend) - pm_runtime_allow(bdev->dev); + if (enable_autosuspend) + pm_runtime_allow(bdev->dev); + } bt_dev_info(hdev, "Device setup in %llu usecs", duration); @@ -1465,6 +1496,9 @@ static void btmtksdio_remove(struct sdio_func *func) if (test_bit(BTMTKSDIO_FUNC_ENABLED, &bdev->tx_state)) btmtksdio_close(hdev); + if (bdev->data->pm_runtime_supported) + pm_runtime_dont_use_autosuspend(bdev->dev); + /* Be consistent the state in btmtksdio_probe */ pm_runtime_get_noresume(bdev->dev); diff --git a/drivers/bluetooth/btnxpuart.c b/drivers/bluetooth/btnxpuart.c index 970da4f84e60f..509d837e353e6 100644 --- a/drivers/bluetooth/btnxpuart.c +++ b/drivers/bluetooth/btnxpuart.c @@ -1368,12 +1368,21 @@ static int nxp_process_fw_dump(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_acl_hdr *acl_hdr = (struct hci_acl_hdr *)skb_pull_data(skb, sizeof(*acl_hdr)); - struct nxp_fw_dump_hdr *fw_dump_hdr = (struct nxp_fw_dump_hdr *)skb->data; + struct nxp_fw_dump_hdr *fw_dump_hdr; struct btnxpuart_dev *nxpdev = hci_get_drvdata(hdev); - __u16 seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); - __u16 buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + __u16 seq_num; + __u16 buf_len; int err; + fw_dump_hdr = skb_pull_data(skb, sizeof(*fw_dump_hdr)); + if (!fw_dump_hdr) { + bt_dev_warn(hdev, "FW dump: invalid or corrupt fw dump chunk"); + goto free_skb; + } + + seq_num = __le16_to_cpu(fw_dump_hdr->seq_num); + buf_len = __le16_to_cpu(fw_dump_hdr->buf_len); + if (seq_num == 0x0001) { if (test_and_set_bit(BTNXPUART_FW_DUMP_IN_PROGRESS, &nxpdev->tx_state)) { bt_dev_err(hdev, "FW dump already in progress"); diff --git a/drivers/bluetooth/btqca.c b/drivers/bluetooth/btqca.c index 86a48d009d1ba..f404eefbfc85b 100644 --- a/drivers/bluetooth/btqca.c +++ b/drivers/bluetooth/btqca.c @@ -413,7 +413,7 @@ static int qca_tlv_check_data(struct hci_dev *hdev, idx = 0; data = tlv->data; - while (idx < length - sizeof(struct tlv_type_nvm)) { + while (idx + sizeof(struct tlv_type_nvm) <= length) { tlv_nvm = (struct tlv_type_nvm *)(data + idx); tag_id = le16_to_cpu(tlv_nvm->tag_id); @@ -1014,8 +1014,7 @@ int qca_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) baswap(&bdaddr_swapped, bdaddr); skb = __hci_cmd_sync_ev(hdev, EDL_WRITE_BD_ADDR_OPCODE, 6, - &bdaddr_swapped, HCI_EV_VENDOR, - HCI_INIT_TIMEOUT); + &bdaddr_swapped, 0, HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { err = PTR_ERR(skb); bt_dev_err(hdev, "QCA Change address cmd failed (%d)", err); diff --git a/drivers/bluetooth/btqcomsmd.c b/drivers/bluetooth/btqcomsmd.c index d2e13fcb6babf..d669ea4eb3eb0 100644 --- a/drivers/bluetooth/btqcomsmd.c +++ b/drivers/bluetooth/btqcomsmd.c @@ -188,7 +188,10 @@ static int btqcomsmd_probe(struct platform_device *pdev) return 0; hci_free_dev: + rpmsg_destroy_ept(btq->cmd_channel); + rpmsg_destroy_ept(btq->acl_channel); hci_free_dev(hdev); + return ret; destroy_cmd_channel: rpmsg_destroy_ept(btq->cmd_channel); destroy_acl_channel: @@ -202,10 +205,11 @@ static void btqcomsmd_remove(struct platform_device *pdev) struct btqcomsmd *btq = platform_get_drvdata(pdev); hci_unregister_dev(btq->hdev); - hci_free_dev(btq->hdev); rpmsg_destroy_ept(btq->cmd_channel); rpmsg_destroy_ept(btq->acl_channel); + + hci_free_dev(btq->hdev); } static const struct of_device_id btqcomsmd_of_match[] = { diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c index a62eae7276d64..564ed584c22f8 100644 --- a/drivers/bluetooth/btrtl.c +++ b/drivers/bluetooth/btrtl.c @@ -72,6 +72,7 @@ enum btrtl_chip_id { CHIP_ID_8851B = 36, CHIP_ID_8922A = 44, CHIP_ID_8852BT = 47, + CHIP_ID_8761C = 51, }; struct id_table { @@ -230,6 +231,14 @@ static const struct id_table ic_id_table[] = { .cfg_name = "rtl_bt/rtl8761bu_config", .hw_info = "rtl8761bu" }, + /* 8761CU */ + { IC_INFO(RTL_ROM_LMP_8761A, 0x0e, 0, HCI_USB), + .config_needed = false, + .has_rom_version = true, + .fw_name = "rtl_bt/rtl8761cu_fw", + .cfg_name = "rtl_bt/rtl8761cu_config", + .hw_info = "rtl8761cu" }, + /* 8822C with UART interface */ { IC_INFO(RTL_ROM_LMP_8822B, 0xc, 0x8, HCI_UART), .config_needed = true, @@ -344,7 +353,8 @@ static const struct id_table *btrtl_match_ic(u16 lmp_subver, u16 hci_rev, (ic_id_table[i].hci_rev != hci_rev)) continue; if ((ic_id_table[i].match_flags & IC_MATCH_FL_HCIVER) && - (ic_id_table[i].hci_ver != hci_ver)) + (ic_id_table[i].hci_ver != hci_ver) && + (ic_id_table[i].hci_ver != 0)) continue; if ((ic_id_table[i].match_flags & IC_MATCH_FL_HCIBUS) && (ic_id_table[i].hci_bus != hci_bus)) @@ -581,7 +591,7 @@ static int rtlbt_parse_firmware_v2(struct hci_dev *hdev, * headers. */ if (!key_id) - break; + continue; rc = btrtl_parse_section(hdev, btrtl_dev, opcode, ptr, section_len); break; @@ -590,8 +600,7 @@ static int rtlbt_parse_firmware_v2(struct hci_dev *hdev, ptr, section_len); break; default: - rc = 0; - break; + continue; } if (rc < 0) { rtl_dev_err(hdev, "RTL: Parse section (%u) err %d", @@ -662,6 +671,7 @@ static int rtlbt_parse_firmware(struct hci_dev *hdev, { RTL_ROM_LMP_8851B, 36 }, /* 8851B */ { RTL_ROM_LMP_8922A, 44 }, /* 8922A */ { RTL_ROM_LMP_8852A, 47 }, /* 8852BT */ + { RTL_ROM_LMP_8761A, 51 }, /* 8761C */ }; if (btrtl_dev->fw_len <= 8) @@ -1306,6 +1316,7 @@ void btrtl_set_quirks(struct hci_dev *hdev, struct btrtl_device_info *btrtl_dev) case CHIP_ID_8851B: case CHIP_ID_8922A: case CHIP_ID_8852BT: + case CHIP_ID_8761C: hci_set_quirk(hdev, HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED); /* RTL8852C needs to transmit mSBC data continuously without @@ -1525,6 +1536,8 @@ MODULE_FIRMWARE("rtl_bt/rtl8761b_fw.bin"); MODULE_FIRMWARE("rtl_bt/rtl8761b_config.bin"); MODULE_FIRMWARE("rtl_bt/rtl8761bu_fw.bin"); MODULE_FIRMWARE("rtl_bt/rtl8761bu_config.bin"); +MODULE_FIRMWARE("rtl_bt/rtl8761cu_fw.bin"); +MODULE_FIRMWARE("rtl_bt/rtl8761cu_config.bin"); MODULE_FIRMWARE("rtl_bt/rtl8821a_fw.bin"); MODULE_FIRMWARE("rtl_bt/rtl8821a_config.bin"); MODULE_FIRMWARE("rtl_bt/rtl8821c_fw.bin"); diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index b9e5adecfed60..81b25c0231c4b 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -67,6 +67,7 @@ static struct usb_driver btusb_driver; #define BTUSB_INTEL_NO_WBS_SUPPORT BIT(26) #define BTUSB_ACTIONS_SEMI BIT(27) #define BTUSB_BARROT BIT(28) +#define BTUSB_BROKEN_EXT_SCAN BIT(29) static const struct usb_device_id btusb_table[] = { /* Generic Bluetooth USB device */ @@ -484,6 +485,7 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x8087, 0x0037), .driver_info = BTUSB_INTEL_COMBINED }, { USB_DEVICE(0x8087, 0x0038), .driver_info = BTUSB_INTEL_COMBINED }, { USB_DEVICE(0x8087, 0x0039), .driver_info = BTUSB_INTEL_COMBINED }, + { USB_DEVICE(0x8087, 0x0040), .driver_info = BTUSB_INTEL_COMBINED }, /* Lizard Peak 2 */ { USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR }, { USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL_COMBINED | BTUSB_INTEL_NO_WBS_SUPPORT | @@ -601,6 +603,10 @@ static const struct usb_device_id quirks_table[] = { /* Realtek 8922AE Bluetooth devices */ { USB_DEVICE(0x0bda, 0x8922), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0bda, 0xd922), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0bda, 0xd923), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3617), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3616), .driver_info = BTUSB_REALTEK | @@ -608,6 +614,10 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x0489, 0xe130), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + /* Realtek 8761BU Bluetooth devices */ + { USB_DEVICE(0x0bda, 0xa728), .driver_info = BTUSB_REALTEK | + BTUSB_BROKEN_EXT_SCAN }, + /* Realtek Bluetooth devices */ { USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01), .driver_info = BTUSB_REALTEK }, @@ -707,6 +717,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0e8d, 0x223c), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3584), .driver_info = BTUSB_MEDIATEK | @@ -745,6 +757,8 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0489, 0xe151), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0e8d, 0x8c38), .driver_info = BTUSB_MEDIATEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3602), .driver_info = BTUSB_MEDIATEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x13d3, 0x3603), .driver_info = BTUSB_MEDIATEK | @@ -786,8 +800,12 @@ static const struct usb_device_id quirks_table[] = { { USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK }, /* Additional Realtek 8761BUV Bluetooth devices */ + { USB_DEVICE(0x2c4e, 0x0115), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x2357, 0x0607), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK | @@ -802,6 +820,14 @@ static const struct usb_device_id quirks_table[] = { BTUSB_WIDEBAND_SPEECH }, { USB_DEVICE(0x2b89, 0x6275), .driver_info = BTUSB_REALTEK | BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x37ad, 0x0600), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + + /* Additional Realtek 8761CU Bluetooth devices */ + { USB_DEVICE(0x0b05, 0x1bef), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, + { USB_DEVICE(0x0b05, 0x1d70), .driver_info = BTUSB_REALTEK | + BTUSB_WIDEBAND_SPEECH }, /* Additional Realtek 8821AE Bluetooth devices */ { USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK }, @@ -910,6 +936,7 @@ struct qca_dump_info { #define BTUSB_USE_ALT3_FOR_WBS 15 #define BTUSB_ALT6_CONTINUOUS_TX 16 #define BTUSB_HW_SSR_ACTIVE 17 +#define BTUSB_RESET 19 struct btusb_data { struct hci_dev *hdev; @@ -963,6 +990,7 @@ struct btusb_data { bool usb_alt6_packet_flow; int isoc_altsetting; int suspend_count; + const struct usb_device_id *match_id; int (*recv_event)(struct hci_dev *hdev, struct sk_buff *skb); int (*recv_acl)(struct hci_dev *hdev, struct sk_buff *skb); @@ -985,13 +1013,15 @@ static void btusb_reset(struct hci_dev *hdev) int err; data = hci_get_drvdata(hdev); - /* This is not an unbalanced PM reference since the device will reset */ err = usb_autopm_get_interface(data->intf); if (err) { bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err); return; } + if (test_and_set_bit(BTUSB_RESET, &data->flags)) + usb_autopm_put_interface_no_suspend(data->intf); + bt_dev_err(hdev, "Resetting usb device."); usb_queue_reset_device(data->intf); } @@ -1995,18 +2025,24 @@ static int btusb_close(struct hci_dev *hdev) BT_DBG("%s", hdev->name); - cancel_delayed_work(&data->rx_work); cancel_work_sync(&data->work); cancel_work_sync(&data->waker); - skb_queue_purge(&data->acl_q); - clear_bit(BTUSB_ISOC_RUNNING, &data->flags); clear_bit(BTUSB_BULK_RUNNING, &data->flags); clear_bit(BTUSB_INTR_RUNNING, &data->flags); clear_bit(BTUSB_DIAG_RUNNING, &data->flags); btusb_stop_traffic(data); + + /* rx_work must only be canceled once the URBs that can rearm it are + * gone, and it must be canceled synchronously since btusb_disconnect() + * frees the btusb_data it dereferences right after hci_unregister_dev(). + */ + cancel_delayed_work_sync(&data->rx_work); + + skb_queue_purge(&data->acl_q); + btusb_free_frags(data); err = usb_autopm_get_interface(data->intf); @@ -2032,7 +2068,7 @@ static int btusb_flush(struct hci_dev *hdev) BT_DBG("%s", hdev->name); - cancel_delayed_work(&data->rx_work); + cancel_delayed_work_sync(&data->rx_work); skb_queue_purge(&data->acl_q); @@ -2716,7 +2752,9 @@ static int btusb_setup_realtek(struct hci_dev *hdev) static int btusb_recv_event_realtek(struct hci_dev *hdev, struct sk_buff *skb) { - if (skb->data[0] == HCI_VENDOR_PKT && skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) { + if (skb->len >= HCI_EVENT_HDR_SIZE + 1 && + skb->data[0] == HCI_VENDOR_PKT && + skb->data[2] == RTK_SUB_EVENT_CODE_COREDUMP) { struct rtk_dev_coredump_hdr hdr = { .code = RTK_DEVCOREDUMP_CODE_MEMDUMP, }; @@ -2823,8 +2861,11 @@ static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data) } err = usb_autopm_get_interface(data->intf); - if (err < 0) + if (err < 0) { + bt_dev_err(hdev, "Failed usb_autopm_get_interface: %d", err); + clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags); return err; + } /* Release MediaTek ISO data interface */ btusb_mtk_release_iso_intf(hdev); @@ -2834,6 +2875,11 @@ static int btusb_mtk_reset(struct hci_dev *hdev, void *rst_data) err = btmtk_usb_subsys_reset(hdev, btmtk_data->dev_id); + if (test_and_set_bit(BTUSB_RESET, &data->flags)) { + bt_dev_err(hdev, "last usb reset failed? Resetting again"); + usb_autopm_put_interface_no_suspend(data->intf); + } + usb_queue_reset_device(data->intf); clear_bit(BTMTK_HW_RESET_ACTIVE, &btmtk_data->flags); @@ -2991,14 +3037,15 @@ static int btusb_set_bdaddr_ath3012(struct hci_dev *hdev, static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev, const bdaddr_t *bdaddr) { + bdaddr_t bdaddr_swapped; struct sk_buff *skb; - u8 buf[6]; long ret; - memcpy(buf, bdaddr, sizeof(bdaddr_t)); + baswap(&bdaddr_swapped, bdaddr); - skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(buf), buf, - HCI_EV_CMD_COMPLETE, HCI_INIT_TIMEOUT); + skb = __hci_cmd_sync_ev(hdev, 0xfc14, sizeof(bdaddr_swapped), + &bdaddr_swapped, HCI_EV_CMD_COMPLETE, + HCI_INIT_TIMEOUT); if (IS_ERR(skb)) { ret = PTR_ERR(skb); bt_dev_err(hdev, "Change address command failed (%ld)", ret); @@ -3343,28 +3390,16 @@ static const char *qca_get_fw_subdirectory(const struct qca_version *ver) static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request, void *data, u16 size) { - int pipe, err; - u8 *buf; - - buf = kmalloc(size, GFP_KERNEL); - if (!buf) - return -ENOMEM; + int err; /* Found some of USB hosts have IOT issues with ours so that we should * not wait until HCI layer is ready. */ - pipe = usb_rcvctrlpipe(udev, 0); - err = usb_control_msg(udev, pipe, request, USB_TYPE_VENDOR | USB_DIR_IN, - 0, 0, buf, size, USB_CTRL_GET_TIMEOUT); - if (err < 0) { + err = usb_control_msg_recv(udev, 0, request, USB_TYPE_VENDOR | USB_DIR_IN, + 0, 0, data, size, USB_CTRL_GET_TIMEOUT, + GFP_KERNEL); + if (err) dev_err(&udev->dev, "Failed to access otp area (%d)", err); - goto done; - } - - memcpy(data, buf, size); - -done: - kfree(buf); return err; } @@ -3571,7 +3606,7 @@ static bool btusb_qca_need_patch(struct usb_device *udev) struct qca_version ver; if (btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver, - sizeof(ver)) < 0) + sizeof(ver))) return false; /* only low ROM versions need patches */ return !(le32_to_cpu(ver.rom_version) & ~0xffffU); @@ -3589,7 +3624,7 @@ static int btusb_setup_qca(struct hci_dev *hdev) err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver, sizeof(ver)); - if (err < 0) + if (err) return err; ver_rom = le32_to_cpu(ver.rom_version); @@ -3612,7 +3647,7 @@ static int btusb_setup_qca(struct hci_dev *hdev) err = btusb_qca_send_vendor_req(udev, QCA_CHECK_STATUS, &status, sizeof(status)); - if (err < 0) + if (err) return err; if (!(status & QCA_PATCH_UPDATED)) { @@ -3623,11 +3658,13 @@ static int btusb_setup_qca(struct hci_dev *hdev) err = btusb_qca_send_vendor_req(udev, QCA_GET_TARGET_VERSION, &ver, sizeof(ver)); - if (err < 0) + if (err) return err; - btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); - btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + if (btdata->match_id->driver_info & BTUSB_QCA_WCN6855) { + btdata->qca_dump.fw_version = le32_to_cpu(ver.patch_version); + btdata->qca_dump.controller_id = le32_to_cpu(ver.rom_version); + } if (!(status & QCA_SYSCFG_UPDATED)) { err = btusb_setup_qca_load_nvm(hdev, &ver, info); @@ -3654,31 +3691,14 @@ static inline int __set_diag_interface(struct hci_dev *hdev) { struct btusb_data *data = hci_get_drvdata(hdev); struct usb_interface *intf = data->diag; - int i; + int ret; if (!data->diag) return -ENODEV; - data->diag_tx_ep = NULL; - data->diag_rx_ep = NULL; - - for (i = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) { - struct usb_endpoint_descriptor *ep_desc; - - ep_desc = &intf->cur_altsetting->endpoint[i].desc; - - if (!data->diag_tx_ep && usb_endpoint_is_bulk_out(ep_desc)) { - data->diag_tx_ep = ep_desc; - continue; - } - - if (!data->diag_rx_ep && usb_endpoint_is_bulk_in(ep_desc)) { - data->diag_rx_ep = ep_desc; - continue; - } - } - - if (!data->diag_tx_ep || !data->diag_rx_ep) { + ret = usb_find_common_endpoints(intf->cur_altsetting, &data->diag_rx_ep, + &data->diag_tx_ep, NULL, NULL); + if (ret) { bt_dev_err(hdev, "invalid diagnostic descriptors"); return -ENODEV; } @@ -4009,12 +4029,11 @@ static struct hci_drv btusb_hci_drv = { static int btusb_probe(struct usb_interface *intf, const struct usb_device_id *id) { - struct usb_endpoint_descriptor *ep_desc; struct gpio_desc *reset_gpio; struct btusb_data *data; struct hci_dev *hdev; unsigned ifnum_base; - int i, err, priv_size; + int err, priv_size; BT_DBG("intf %p id %p", intf, id); @@ -4051,26 +4070,10 @@ static int btusb_probe(struct usb_interface *intf, if (!data) return -ENOMEM; - for (i = 0; i < intf->cur_altsetting->desc.bNumEndpoints; i++) { - ep_desc = &intf->cur_altsetting->endpoint[i].desc; - - if (!data->intr_ep && usb_endpoint_is_int_in(ep_desc)) { - data->intr_ep = ep_desc; - continue; - } - - if (!data->bulk_tx_ep && usb_endpoint_is_bulk_out(ep_desc)) { - data->bulk_tx_ep = ep_desc; - continue; - } - - if (!data->bulk_rx_ep && usb_endpoint_is_bulk_in(ep_desc)) { - data->bulk_rx_ep = ep_desc; - continue; - } - } - - if (!data->intr_ep || !data->bulk_tx_ep || !data->bulk_rx_ep) { + data->match_id = id; + err = usb_find_common_endpoints(intf->cur_altsetting, &data->bulk_rx_ep, + &data->bulk_tx_ep, &data->intr_ep, NULL); + if (err) { kfree(data); return -ENODEV; } @@ -4327,6 +4330,9 @@ static int btusb_probe(struct usb_interface *intf, if (id->driver_info & BTUSB_INVALID_LE_STATES) hci_set_quirk(hdev, HCI_QUIRK_BROKEN_LE_STATES); + if (id->driver_info & BTUSB_BROKEN_EXT_SCAN) + hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN); + if (id->driver_info & BTUSB_DIGIANSWER) { data->cmdreq_type = USB_TYPE_VENDOR; hci_set_quirk(hdev, HCI_QUIRK_RESET_ON_CLOSE); @@ -4450,6 +4456,9 @@ static void btusb_disconnect(struct usb_interface *intf) if (data->reset_gpio) gpiod_put(data->reset_gpio); + if (test_and_clear_bit(BTUSB_RESET, &data->flags)) + usb_autopm_put_interface_no_suspend(data->intf); + if (intf == data->intf) { if (data->isoc) usb_driver_release_interface(&btusb_driver, data->isoc); diff --git a/drivers/bluetooth/hci_aml.c b/drivers/bluetooth/hci_aml.c index b1f32c5a8a3f4..0f227a0915b8c 100644 --- a/drivers/bluetooth/hci_aml.c +++ b/drivers/bluetooth/hci_aml.c @@ -247,7 +247,7 @@ static int aml_download_firmware(struct hci_dev *hdev, const char *fw_name) struct hci_uart *hu = hci_get_drvdata(hdev); struct aml_serdev *amldev = serdev_device_get_drvdata(hu->serdev); const struct firmware *firmware = NULL; - struct aml_fw_len *fw_len = NULL; + const struct aml_fw_len *fw_len = NULL; u8 *iccm_start = NULL, *dccm_start = NULL; u32 iccm_len, dccm_len; u32 value = 0; @@ -281,7 +281,21 @@ static int aml_download_firmware(struct hci_dev *hdev, const char *fw_name) goto exit; } - fw_len = (struct aml_fw_len *)firmware->data; + if (firmware->size < sizeof(*fw_len)) { + bt_dev_err(hdev, "Firmware is too small for its header"); + ret = -EINVAL; + goto exit; + } + + fw_len = (const struct aml_fw_len *)firmware->data; + if (fw_len->iccm_len < amldev->aml_dev_data->iccm_offset || + fw_len->iccm_len > firmware->size - sizeof(*fw_len) || + fw_len->dccm_len > firmware->size - sizeof(*fw_len) - + fw_len->iccm_len) { + bt_dev_err(hdev, "Invalid firmware segment lengths"); + ret = -EINVAL; + goto exit; + } /* Download ICCM */ iccm_start = (u8 *)(firmware->data) + sizeof(struct aml_fw_len) diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index fff845ed44e35..0a54a7cf2bdef 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -548,6 +548,7 @@ static int bcm_close(struct hci_uart *hu) if (IS_ENABLED(CONFIG_PM) && bdev->irq_acquired) { devm_free_irq(bdev->dev, bdev->irq, bdev); device_init_wakeup(bdev->dev, false); + pm_runtime_dont_use_autosuspend(bdev->dev); pm_runtime_disable(bdev->dev); } diff --git a/drivers/bluetooth/hci_bcm4377.c b/drivers/bluetooth/hci_bcm4377.c index 45e6d84224ee3..1d41401f3dba7 100644 --- a/drivers/bluetooth/hci_bcm4377.c +++ b/drivers/bluetooth/hci_bcm4377.c @@ -2486,6 +2486,7 @@ static const struct bcm4377_hw bcm4377_hw_variants[] = { .has_bar0_core2_window2 = true, .broken_mws_transport_config = true, .broken_le_coded = true, + .broken_le_ext_adv_report_phy = true, .send_calibration = bcm4378_send_calibration, .send_ptb = bcm4378_send_ptb, }, diff --git a/drivers/bluetooth/hci_h5.c b/drivers/bluetooth/hci_h5.c index c6bf30801b49d..c3a472a1f5b1b 100644 --- a/drivers/bluetooth/hci_h5.c +++ b/drivers/bluetooth/hci_h5.c @@ -988,8 +988,10 @@ static void h5_btrtl_open(struct h5 *h5) static void h5_btrtl_close(struct h5 *h5) { - if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags)) + if (!test_bit(H5_WAKEUP_DISABLE, &h5->flags)) { + pm_runtime_dont_use_autosuspend(&h5->hu->serdev->dev); pm_runtime_disable(&h5->hu->serdev->dev); + } gpiod_set_value_cansleep(h5->device_wake_gpio, 0); gpiod_set_value_cansleep(h5->enable_gpio, 0); diff --git a/drivers/bluetooth/hci_intel.c b/drivers/bluetooth/hci_intel.c index 1d6e09508f1f2..4d52b1d3d8fe4 100644 --- a/drivers/bluetooth/hci_intel.c +++ b/drivers/bluetooth/hci_intel.c @@ -348,6 +348,7 @@ static int intel_set_power(struct hci_uart *hu, bool powered) devm_free_irq(&idev->pdev->dev, idev->irq, idev); device_wakeup_disable(&idev->pdev->dev); + pm_runtime_dont_use_autosuspend(&idev->pdev->dev); pm_runtime_disable(&idev->pdev->dev); } } diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c index c27413a371c2d..55e0dceee54a9 100644 --- a/drivers/bluetooth/hci_ldisc.c +++ b/drivers/bluetooth/hci_ldisc.c @@ -451,7 +451,7 @@ static int hci_uart_setup(struct hci_dev *hdev) if (IS_ERR(skb)) { BT_ERR("%s: Reading local version information failed (%ld)", hdev->name, PTR_ERR(skb)); - return 0; + return PTR_ERR(skb); } if (skb->len != sizeof(*ver)) { diff --git a/drivers/bluetooth/hci_mrvl.c b/drivers/bluetooth/hci_mrvl.c index 8767522ec4c68..c0db366502749 100644 --- a/drivers/bluetooth/hci_mrvl.c +++ b/drivers/bluetooth/hci_mrvl.c @@ -307,9 +307,8 @@ static int mrvl_load_firmware(struct hci_dev *hdev, const char *name) err = wait_on_bit_timeout(&mrvl->flags, STATE_FW_REQ_PENDING, TASK_INTERRUPTIBLE, msecs_to_jiffies(2000)); - if (err == 1) { + if (err == -EINTR) { bt_dev_err(hdev, "Firmware load interrupted"); - err = -EINTR; break; } else if (err) { bt_dev_err(hdev, "Firmware request timeout"); diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c index c649a3f702c04..b510b03dae870 100644 --- a/drivers/bluetooth/hci_qca.c +++ b/drivers/bluetooth/hci_qca.c @@ -1083,6 +1083,10 @@ static void qca_controller_memdump(struct work_struct *work) if (!(qca_memdump->ram_dump_size)) { bt_dev_err(hu->hdev, "Rx invalid memdump size"); kfree(qca_memdump); + qca->qca_memdump = NULL; + qca->memdump_state = QCA_MEMDUMP_COLLECTED; + clear_and_wake_up_bit(QCA_MEMDUMP_COLLECTION, &qca->flags); + clear_bit(QCA_IBS_DISABLED, &qca->flags); kfree_skb(skb); mutex_unlock(&qca->hci_memdump_lock); return; diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c index 593d9cefbbf92..d2eaf2f12aa27 100644 --- a/drivers/bluetooth/hci_serdev.c +++ b/drivers/bluetooth/hci_serdev.c @@ -221,7 +221,7 @@ static int hci_uart_setup(struct hci_dev *hdev) if (IS_ERR(skb)) { bt_dev_err(hdev, "Reading local version info failed (%ld)", PTR_ERR(skb)); - return 0; + return PTR_ERR(skb); } if (skb->len != sizeof(*ver)) diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 9341c5eed5b53..152432cc2260c 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -120,9 +120,13 @@ static int virtbt_setup_zephyr(struct hci_dev *hdev) if (IS_ERR(skb)) return PTR_ERR(skb); - bt_dev_info(hdev, "%s", (char *)(skb->data + 1)); + /* Bounded print: the backend controls skb->len. */ + if (skb->len > 1) { + int len = skb->len - 1; - hci_set_fw_info(hdev, "%s", skb->data + 1); + bt_dev_info(hdev, "%.*s", len, (char *)(skb->data + 1)); + hci_set_fw_info(hdev, "%.*s", len, skb->data + 1); + } kfree_skb(skb); return 0; diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c index 996379ace3764..21eb0a3c190fc 100644 --- a/drivers/bus/fsl-mc/fsl-mc-bus.c +++ b/drivers/bus/fsl-mc/fsl-mc-bus.c @@ -66,6 +66,13 @@ struct fsl_mc_addr_translation_range { #define GCR1_P1_STOP BIT(31) #define GCR1_P2_STOP BIT(30) +#define FSL_MC_GSR 0x8 +#define FSL_MC_GSR_BOOT_DONE BIT(0) +#define FSL_MC_GSR_MCS_MASK GENMASK(7, 0) +#define FSL_MC_GSR_MCS_ERR_MASK GENMASK(7, 1) +#define FSL_MC_GSR_BC_MASK GENMASK(15, 8) +#define FSL_MC_GSR_BC_SHIFT 8 + #define FSL_MC_FAPR 0x28 #define MC_FAPR_PL BIT(18) #define MC_FAPR_BMT BIT(17) @@ -1007,6 +1014,41 @@ static int get_mc_addr_translation_ranges(struct device *dev, return 0; } +static u32 fsl_mc_read_gsr(struct fsl_mc *mc) +{ + return readl(mc->fsl_mc_regs + FSL_MC_GSR); +} + +static int fsl_mc_firmware_check(struct platform_device *pdev) +{ + struct fsl_mc *mc = platform_get_drvdata(pdev); + u32 gsr, boot_done, boot_code, mcs; + + gsr = fsl_mc_read_gsr(mc); + boot_code = (gsr & FSL_MC_GSR_BC_MASK) >> FSL_MC_GSR_BC_SHIFT; + if (boot_code == 0xDD) { + dev_err(&pdev->dev, + "fsl-mc: DPL processing was not started, DPAA2 will not work!\n"); + return -EOPNOTSUPP; + } + + boot_done = gsr & FSL_MC_GSR_BOOT_DONE; + if (!boot_done) { + dev_dbg(&pdev->dev, + "fsl-mc: DPL processing in progress, defer probe\n"); + return -EPROBE_DEFER; + } + + mcs = gsr & FSL_MC_GSR_MCS_MASK; + if (mcs & FSL_MC_GSR_MCS_ERR_MASK) { + dev_err(&pdev->dev, + "fsl-mc: MC boot completed with error 0x%x\n", mcs); + return -EINVAL; + } + + return 0; +} + /* * fsl_mc_bus_probe - callback invoked when the root MC bus is being * added @@ -1071,6 +1113,10 @@ static int fsl_mc_bus_probe(struct platform_device *pdev) mc->fsl_mc_regs + FSL_MC_GCR1); } + error = fsl_mc_firmware_check(pdev); + if (error) + return error; + /* * Get physical address of MC portal for the root DPRC: */ diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c index 2f58ad0f14b65..fd1d7c84fb1c5 100644 --- a/drivers/bus/mhi/ep/main.c +++ b/drivers/bus/mhi/ep/main.c @@ -1340,14 +1340,19 @@ static int mhi_ep_create_device(struct mhi_ep_cntrl *mhi_cntrl, u32 ch_id) ret = dev_set_name(&mhi_dev->dev, "%s_%s", dev_name(&mhi_cntrl->mhi_dev->dev), mhi_dev->name); - if (ret) { - put_device(&mhi_dev->dev); - return ret; - } + if (ret) + goto err_put_channels; ret = device_add(&mhi_dev->dev); if (ret) - put_device(&mhi_dev->dev); + goto err_put_channels; + + return 0; + +err_put_channels: + put_device(&mhi_dev->dev); /* DL channel reference */ + put_device(&mhi_dev->dev); /* UL channel reference */ + put_device(&mhi_dev->dev); /* device_initialize() reference */ return ret; } diff --git a/drivers/bus/mhi/host/init.c b/drivers/bus/mhi/host/init.c index 099be8dd19007..504a5d3f78309 100644 --- a/drivers/bus/mhi/host/init.c +++ b/drivers/bus/mhi/host/init.c @@ -1042,7 +1042,7 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, if (mhi_cntrl->edl_trigger) { ret = sysfs_create_file(&mhi_dev->dev.kobj, &dev_attr_trigger_edl.attr); if (ret) - goto err_release_dev; + goto err_del_dev; } mhi_cntrl->mhi_dev = mhi_dev; @@ -1051,6 +1051,8 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, return 0; +err_del_dev: + device_del(&mhi_dev->dev); err_release_dev: put_device(&mhi_dev->dev); error_setup_irq: diff --git a/drivers/bus/mhi/host/main.c b/drivers/bus/mhi/host/main.c index 8615512743199..eeb277dee7486 100644 --- a/drivers/bus/mhi/host/main.c +++ b/drivers/bus/mhi/host/main.c @@ -170,6 +170,9 @@ EXPORT_SYMBOL_GPL(mhi_get_mhi_state); void mhi_soc_reset(struct mhi_controller *mhi_cntrl) { + int __maybe_unused ret; + u32 tmp; + if (mhi_cntrl->reset) { mhi_cntrl->reset(mhi_cntrl); return; @@ -178,6 +181,9 @@ void mhi_soc_reset(struct mhi_controller *mhi_cntrl) /* Generic MHI SoC reset */ mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, MHI_SOC_RESET_REQ); + /* Flush the posted write to the device (ignore return value) */ + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + &tmp); } EXPORT_SYMBOL_GPL(mhi_soc_reset); diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c index 646f3bcdc636a..a5b47728f9aba 100644 --- a/drivers/bus/mhi/host/pci_generic.c +++ b/drivers/bus/mhi/host/pci_generic.c @@ -1139,7 +1139,8 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl, */ mhi_cntrl->nr_irqs = 1 + mhi_cntrl_config->num_events; - nr_vectors = pci_alloc_irq_vectors(pdev, 1, mhi_cntrl->nr_irqs, PCI_IRQ_MSIX | PCI_IRQ_MSI); + nr_vectors = pci_alloc_irq_vectors(pdev, 1, roundup_pow_of_two(mhi_cntrl->nr_irqs), + PCI_IRQ_MSIX | PCI_IRQ_MSI); if (nr_vectors < 0) { dev_err(&pdev->dev, "Error allocating MSI vectors %d\n", nr_vectors); diff --git a/drivers/bus/mvebu-mbus.c b/drivers/bus/mvebu-mbus.c index 00cb792bda18b..dd94145c9b222 100644 --- a/drivers/bus/mvebu-mbus.c +++ b/drivers/bus/mvebu-mbus.c @@ -1006,7 +1006,7 @@ static __init int mvebu_mbus_debugfs_init(void) } fs_initcall(mvebu_mbus_debugfs_init); -static int mvebu_mbus_suspend(void) +static int mvebu_mbus_suspend(void *data) { struct mvebu_mbus_state *s = &mbus_state; int win; @@ -1040,7 +1040,7 @@ static int mvebu_mbus_suspend(void) return 0; } -static void mvebu_mbus_resume(void) +static void mvebu_mbus_resume(void *data) { struct mvebu_mbus_state *s = &mbus_state; int win; @@ -1069,9 +1069,13 @@ static void mvebu_mbus_resume(void) } } -static struct syscore_ops mvebu_mbus_syscore_ops = { - .suspend = mvebu_mbus_suspend, - .resume = mvebu_mbus_resume, +static const struct syscore_ops mvebu_mbus_syscore_ops = { + .suspend = mvebu_mbus_suspend, + .resume = mvebu_mbus_resume, +}; + +static struct syscore mvebu_mbus_syscore = { + .ops = &mvebu_mbus_syscore_ops, }; static int __init mvebu_mbus_common_init(struct mvebu_mbus_state *mbus, @@ -1118,7 +1122,7 @@ static int __init mvebu_mbus_common_init(struct mvebu_mbus_state *mbus, writel(UNIT_SYNC_BARRIER_ALL, mbus->mbuswins_base + UNIT_SYNC_BARRIER_OFF); - register_syscore_ops(&mvebu_mbus_syscore_ops); + register_syscore(&mvebu_mbus_syscore); return 0; } diff --git a/drivers/bus/qcom-ebi2.c b/drivers/bus/qcom-ebi2.c index c1fef1b4bd89b..8d2eb955dc921 100644 --- a/drivers/bus/qcom-ebi2.c +++ b/drivers/bus/qcom-ebi2.c @@ -292,7 +292,6 @@ static void qcom_ebi2_setup_chipselect(struct device_node *np, static int qcom_ebi2_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; - struct device_node *child; struct device *dev = &pdev->dev; struct resource *res; void __iomem *ebi2_base; @@ -303,41 +302,23 @@ static int qcom_ebi2_probe(struct platform_device *pdev) u32 val; int ret; - ebi2xclk = devm_clk_get(dev, "ebi2x"); + ebi2xclk = devm_clk_get_enabled(dev, "ebi2x"); if (IS_ERR(ebi2xclk)) return PTR_ERR(ebi2xclk); - ret = clk_prepare_enable(ebi2xclk); - if (ret) { - dev_err(dev, "could not enable EBI2X clk (%d)\n", ret); - return ret; - } - - ebi2clk = devm_clk_get(dev, "ebi2"); - if (IS_ERR(ebi2clk)) { - ret = PTR_ERR(ebi2clk); - goto err_disable_2x_clk; - } - - ret = clk_prepare_enable(ebi2clk); - if (ret) { - dev_err(dev, "could not enable EBI2 clk\n"); - goto err_disable_2x_clk; - } + ebi2clk = devm_clk_get_enabled(dev, "ebi2"); + if (IS_ERR(ebi2clk)) + return PTR_ERR(ebi2clk); res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ebi2_base = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_base)) { - ret = PTR_ERR(ebi2_base); - goto err_disable_clk; - } + if (IS_ERR(ebi2_base)) + return PTR_ERR(ebi2_base); res = platform_get_resource(pdev, IORESOURCE_MEM, 1); ebi2_xmem = devm_ioremap_resource(dev, res); - if (IS_ERR(ebi2_xmem)) { - ret = PTR_ERR(ebi2_xmem); - goto err_disable_clk; - } + if (IS_ERR(ebi2_xmem)) + return PTR_ERR(ebi2_xmem); /* Allegedly this turns the power save mode off */ writel(0UL, ebi2_xmem + EBI2_XMEM_CFG); @@ -348,15 +329,13 @@ static int qcom_ebi2_probe(struct platform_device *pdev) writel(val, ebi2_base); /* Walk over the child nodes and see what chipselects we use */ - for_each_available_child_of_node(np, child) { + for_each_available_child_of_node_scoped(np, child) { u32 csindex; /* Figure out the chipselect */ ret = of_property_read_u32(child, "reg", &csindex); - if (ret) { - of_node_put(child); + if (ret) return ret; - } if (csindex > 5) { dev_err(dev, @@ -376,15 +355,9 @@ static int qcom_ebi2_probe(struct platform_device *pdev) } if (have_children) - return of_platform_default_populate(np, NULL, dev); - return 0; - -err_disable_clk: - clk_disable_unprepare(ebi2clk); -err_disable_2x_clk: - clk_disable_unprepare(ebi2xclk); + return devm_of_platform_populate(dev); - return ret; + return 0; } static const struct of_device_id qcom_ebi2_of_match[] = { diff --git a/drivers/bus/ti-sysc.c b/drivers/bus/ti-sysc.c index 610354ce7f8f0..5c096eb3f4279 100644 --- a/drivers/bus/ti-sysc.c +++ b/drivers/bus/ti-sysc.c @@ -682,6 +682,7 @@ static struct device_node *stdout_path; static void sysc_init_stdout_path(struct sysc *ddata) { + struct device_node *chosen; struct device_node *np = NULL; const char *uart; @@ -691,15 +692,18 @@ static void sysc_init_stdout_path(struct sysc *ddata) if (stdout_path) return; - np = of_find_node_by_path("/chosen"); - if (!np) + chosen = of_find_node_by_path("/chosen"); + if (!chosen) goto err; - uart = of_get_property(np, "stdout-path", NULL); - if (!uart) + uart = of_get_property(chosen, "stdout-path", NULL); + if (!uart) { + of_node_put(chosen); goto err; + } np = of_find_node_by_path(uart); + of_node_put(chosen); if (!np) goto err; diff --git a/drivers/cdrom/cdrom.c b/drivers/cdrom/cdrom.c index e30414f0d4560..ce2de543bac32 100644 --- a/drivers/cdrom/cdrom.c +++ b/drivers/cdrom/cdrom.c @@ -3187,6 +3187,7 @@ static noinline int mmc_ioctl_cdrom_volume(struct cdrom_device_info *cdi, /* set volume */ cgc->buffer = buffer + offset - 8; + cgc->buflen -= offset - 8; memset(cgc->buffer, 0, 8); return cdrom_mode_select(cdi, cgc); } diff --git a/drivers/cdx/cdx.c b/drivers/cdx/cdx.c index 9d6d24f49938b..a066e0105c779 100644 --- a/drivers/cdx/cdx.c +++ b/drivers/cdx/cdx.c @@ -745,7 +745,6 @@ static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num) sysfs_bin_attr_init(res_attr); - cdx_dev->res_attr[num] = res_attr; sprintf(res_attr_name, "resource%d", num); res_attr->mmap = cdx_mmap_resource; @@ -754,8 +753,12 @@ static int cdx_create_res_attr(struct cdx_device *cdx_dev, int num) res_attr->size = cdx_resource_len(cdx_dev, num); res_attr->private = (void *)(unsigned long)num; ret = sysfs_create_bin_file(&cdx_dev->dev.kobj, res_attr); - if (ret) + if (ret) { kfree(res_attr); + return ret; + } + + cdx_dev->res_attr[num] = res_attr; return ret; } diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 036de7294bbda..054ffbc9e4220 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -604,11 +604,13 @@ int hwrng_register(struct hwrng *rng) */ err = set_current_rng(rng); if (err) - goto out_unlock; + goto out_list_del; } } mutex_unlock(&rng_mutex); return 0; +out_list_del: + list_del_init(&rng->list); out_unlock: mutex_unlock(&rng_mutex); out: diff --git a/drivers/char/hw_random/imx-rngc.c b/drivers/char/hw_random/imx-rngc.c index 241664a9b5d9a..447424f7ded1c 100644 --- a/drivers/char/hw_random/imx-rngc.c +++ b/drivers/char/hw_random/imx-rngc.c @@ -314,8 +314,10 @@ static int __init imx_rngc_probe(struct platform_device *pdev) devm_pm_runtime_enable(&pdev->dev); ret = devm_hwrng_register(&pdev->dev, &rngc->rng); - if (ret) + if (ret) { + clk_disable_unprepare(rngc->clk); return dev_err_probe(&pdev->dev, ret, "hwrng registration failed\n"); + } dev_info(&pdev->dev, "Freescale RNG%c registered (HW revision %d.%02d)\n", diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 9e408144a10c1..4494f1e4ab4db 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -242,7 +242,14 @@ static int ks_sa_rng_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "Failed to enable SA power-domain\n"); } - return devm_hwrng_register(&pdev->dev, &ks_sa_rng->rng); + ret = devm_hwrng_register(dev, &ks_sa_rng->rng); + if (ret) { + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; + } + + return 0; } static void ks_sa_rng_remove(struct platform_device *pdev) diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 9a8c00586ab09..f5bfe54c01dc7 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -368,11 +368,6 @@ static int stm32_rng_init(struct hwrng *rng) return 0; } -static void stm32_rng_remove(struct platform_device *ofdev) -{ - pm_runtime_disable(&ofdev->dev); -} - static int __maybe_unused stm32_rng_runtime_suspend(struct device *dev) { struct stm32_rng_private *priv = dev_get_drvdata(dev); @@ -590,7 +585,9 @@ static int stm32_rng_probe(struct platform_device *ofdev) pm_runtime_set_autosuspend_delay(dev, 100); pm_runtime_use_autosuspend(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; return devm_hwrng_register(dev, &priv->rng); } @@ -602,7 +599,6 @@ static struct platform_driver stm32_rng_driver = { .of_match_table = stm32_rng_match, }, .probe = stm32_rng_probe, - .remove = stm32_rng_remove, }; module_platform_driver(stm32_rng_driver); diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c index ee2bdc7ed0daa..bc0564d878671 100644 --- a/drivers/char/ipmi/ipmb_dev_int.c +++ b/drivers/char/ipmi/ipmb_dev_int.c @@ -141,13 +141,14 @@ static ssize_t ipmb_write(struct file *file, const char __user *buf, u8 msg[MAX_MSG_LEN]; ssize_t ret; - if (count > sizeof(msg)) + if (!count || count > sizeof(msg)) return -EINVAL; if (copy_from_user(&msg, buf, count)) return -EFAULT; - if (count < msg[0]) + if (msg[IPMB_MSG_LEN_IDX] < IPMB_REQUEST_LEN_MIN || + count < (size_t)msg[IPMB_MSG_LEN_IDX] + 1) return -EINVAL; rq_sa = GET_7BIT_ADDR(msg[RQ_SA_8BIT_IDX]); diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c index 51f76d755f987..0247079db185e 100644 --- a/drivers/char/ipmi/ipmi_msghandler.c +++ b/drivers/char/ipmi/ipmi_msghandler.c @@ -1393,6 +1393,7 @@ static void _ipmi_destroy_user(struct ipmi_user *user) } } mutex_unlock(&intf->cmd_rcvrs_mutex); + synchronize_rcu(); while (rcvrs) { rcvr = rcvrs; rcvrs = rcvr->next; @@ -3740,6 +3741,7 @@ int ipmi_add_smi(struct module *owner, sysfs_attr_init(&intf->maintenance_mode_devattr.attr); rv = device_create_file(intf->si_dev, &intf->maintenance_mode_devattr); if (rv) { + device_remove_file(intf->si_dev, &intf->nr_msgs_devattr); device_remove_file(intf->si_dev, &intf->nr_users_devattr); goto out_err_bmc_reg; } @@ -3757,12 +3759,14 @@ int ipmi_add_smi(struct module *owner, out_err_bmc_reg: ipmi_bmc_unregister(intf); out_err_started: + intf->in_shutdown = true; if (intf->handlers->shutdown) intf->handlers->shutdown(intf->send_info); out_err: list_del(&intf->link); mutex_unlock(&ipmi_interfaces_mutex); mutex_unlock(&smi_watchers_mutex); + cancel_work_sync(&intf->smi_work); kref_put(&intf->refcount, intf_free); return rv; diff --git a/drivers/char/ipmi/ipmi_si_platform.c b/drivers/char/ipmi/ipmi_si_platform.c index fb6e359ae4946..704b06c919f03 100644 --- a/drivers/char/ipmi/ipmi_si_platform.c +++ b/drivers/char/ipmi/ipmi_si_platform.c @@ -276,7 +276,10 @@ static int of_ipmi_probe(struct platform_device *pdev) io.regspacing = regspacing ? be32_to_cpup(regspacing) : DEFAULT_REGSPACING; io.regshift = regshift ? be32_to_cpup(regshift) : 0; - io.irq = irq_of_parse_and_map(pdev->dev.of_node, 0); + io.irq = platform_get_irq_optional(pdev, 0); + if (io.irq < 0) + io.irq = 0; + io.dev = &pdev->dev; dev_dbg(&pdev->dev, "addr 0x%lx regsize %d spacing %d irq %d\n", diff --git a/drivers/char/nvram.c b/drivers/char/nvram.c index 9eff426a92865..e89cc1f1c89ea 100644 --- a/drivers/char/nvram.c +++ b/drivers/char/nvram.c @@ -53,7 +53,6 @@ #include #endif -static DEFINE_MUTEX(nvram_mutex); static DEFINE_SPINLOCK(nvram_state_lock); static int nvram_open_cnt; /* #times opened */ static int nvram_open_mode; /* special open modes */ @@ -310,11 +309,8 @@ static long nvram_misc_ioctl(struct file *file, unsigned int cmd, break; #ifdef CONFIG_PPC32 case IOC_NVRAM_SYNC: - if (ppc_md.nvram_sync != NULL) { - mutex_lock(&nvram_mutex); + if (ppc_md.nvram_sync) ppc_md.nvram_sync(); - mutex_unlock(&nvram_mutex); - } ret = 0; break; #endif @@ -324,11 +320,8 @@ static long nvram_misc_ioctl(struct file *file, unsigned int cmd, if (!capable(CAP_SYS_ADMIN)) return -EACCES; - if (arch_nvram_ops.initialize != NULL) { - mutex_lock(&nvram_mutex); + if (arch_nvram_ops.initialize) ret = arch_nvram_ops.initialize(); - mutex_unlock(&nvram_mutex); - } break; case NVRAM_SETCKS: /* just set checksum, contents unchanged (maybe useful after @@ -336,11 +329,8 @@ static long nvram_misc_ioctl(struct file *file, unsigned int cmd, if (!capable(CAP_SYS_ADMIN)) return -EACCES; - if (arch_nvram_ops.set_checksum != NULL) { - mutex_lock(&nvram_mutex); + if (arch_nvram_ops.set_checksum) ret = arch_nvram_ops.set_checksum(); - mutex_unlock(&nvram_mutex); - } break; #endif /* CONFIG_X86 || CONFIG_M68K */ } diff --git a/drivers/char/ppdev.c b/drivers/char/ppdev.c index d1dfbd8d4d426..3dec6516a5eb1 100644 --- a/drivers/char/ppdev.c +++ b/drivers/char/ppdev.c @@ -340,15 +340,17 @@ static enum ieee1284_phase init_phase(int mode) return IEEE1284_PH_FWD_IDLE; } -static int pp_set_timeout(struct pardevice *pdev, long tv_sec, int tv_usec) +static int pp_set_timeout(struct pardevice *pdev, s64 tv_sec, s64 tv_usec) { + struct timespec64 ts; long to_jiffies; - if ((tv_sec < 0) || (tv_usec < 0)) + if (tv_sec < 0 || tv_usec < 0 || tv_usec >= USEC_PER_SEC) return -EINVAL; - to_jiffies = usecs_to_jiffies(tv_usec); - to_jiffies += tv_sec * HZ; + ts.tv_sec = tv_sec; + ts.tv_nsec = tv_usec * NSEC_PER_USEC; + to_jiffies = timespec64_to_jiffies(&ts); if (to_jiffies <= 0) return -EINVAL; diff --git a/drivers/char/tpm/st33zp24/st33zp24.c b/drivers/char/tpm/st33zp24/st33zp24.c index e2b7451ea7ccd..0e2deff94c367 100644 --- a/drivers/char/tpm/st33zp24/st33zp24.c +++ b/drivers/char/tpm/st33zp24/st33zp24.c @@ -93,7 +93,9 @@ static u8 st33zp24_status(struct tpm_chip *chip) struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1); + if (tpm_dev->ops->recv(tpm_dev->phy_id, TPM_STS, &data, 1) != 1) + return 0; + return data; } @@ -104,10 +106,10 @@ static bool check_locality(struct tpm_chip *chip) { struct st33zp24_dev *tpm_dev = dev_get_drvdata(&chip->dev); u8 data; - u8 status; + int status; status = tpm_dev->ops->recv(tpm_dev->phy_id, TPM_ACCESS, &data, 1); - if (status && (data & + if (status == 1 && (data & (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) == (TPM_ACCESS_ACTIVE_LOCALITY | TPM_ACCESS_VALID)) return true; diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c index d44903b299294..aa3673002f3aa 100644 --- a/drivers/char/tpm/tpm_i2c_nuvoton.c +++ b/drivers/char/tpm/tpm_i2c_nuvoton.c @@ -182,8 +182,10 @@ static int i2c_nuvoton_wait_for_stat(struct tpm_chip *chip, u8 mask, u8 value, timeout); if (rc > 0) return 0; - /* At this point we know that the SINT pin is asserted, so we - * do not need to do i2c_nuvoton_check_status */ + + disable_irq(priv->irq); + if (rc < 0) + return rc; } else { unsigned long ten_msec, stop; bool status_valid; diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c index c355f6d392748..76f934b5d84be 100644 --- a/drivers/char/virtio_console.c +++ b/drivers/char/virtio_console.c @@ -1893,13 +1893,28 @@ static const struct file_operations portdev_fops = { static void remove_vqs(struct ports_device *portdev) { struct virtqueue *vq; + bool multiport = use_multiport(portdev); virtio_device_for_each_vq(portdev->vdev, vq) { struct port_buffer *buf; + unsigned int len; - flush_bufs(vq, true); - while ((buf = virtqueue_detach_unused_buf(vq))) - free_buf(buf, true); + /* + * c_ovq cookies are &portdev->cpkt, not port_buffer. + * Detach them but do not free_buf(). + */ + if (multiport && vq == portdev->c_ovq) { + spin_lock(&portdev->c_ovq_lock); + while (virtqueue_get_buf(vq, &len)) + ; + while (virtqueue_detach_unused_buf(vq)) + ; + spin_unlock(&portdev->c_ovq_lock); + } else { + flush_bufs(vq, true); + while ((buf = virtqueue_detach_unused_buf(vq))) + free_buf(buf, true); + } cond_resched(); } portdev->vdev->config->del_vqs(portdev->vdev); diff --git a/drivers/char/xilinx_hwicap/xilinx_hwicap.c b/drivers/char/xilinx_hwicap/xilinx_hwicap.c index 34a345dc5e724..9bb5fa642fd88 100644 --- a/drivers/char/xilinx_hwicap/xilinx_hwicap.c +++ b/drivers/char/xilinx_hwicap/xilinx_hwicap.c @@ -760,7 +760,7 @@ static int __init hwicap_module_init(void) HWICAP_DEVICES, DRIVER_NAME); if (retval < 0) - return retval; + goto failed_class; retval = platform_driver_register(&hwicap_platform_driver); if (retval) @@ -771,6 +771,9 @@ static int __init hwicap_module_init(void) failed: unregister_chrdev_region(devt, HWICAP_DEVICES); + failed_class: + class_unregister(&icap_class); + return retval; } diff --git a/drivers/clk/Kconfig b/drivers/clk/Kconfig index 60b454677d474..5d1805d05486c 100644 --- a/drivers/clk/Kconfig +++ b/drivers/clk/Kconfig @@ -33,6 +33,17 @@ menuconfig COMMON_CLK if COMMON_CLK +config CLK_HOG + tristate "Clock hog driver" + depends on OF + help + A trivial driver, modelled on the gpio-hog concept, for a device + tree node whose only job is to claim, configure (via the standard + assigned-clocks properties) and enable one or more clocks, keeping + them running for as long as the node is bound. Useful for holding + a clock output (such as a GPCLK routed to a GPIO pin) enabled when + there is no other consumer driver to do it. + config COMMON_CLK_WM831X tristate "Clock driver for WM831x/2x PMICs" depends on MFD_WM831X diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile index 29b2c68387532..5cd6c4b484402 100644 --- a/drivers/clk/Makefile +++ b/drivers/clk/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_COMMON_CLK) += clk-gpio.o ifeq ($(CONFIG_OF), y) obj-$(CONFIG_COMMON_CLK) += clk-conf.o endif +obj-$(CONFIG_CLK_HOG) += clk-hog.o # KUnit specific helpers ifeq ($(CONFIG_COMMON_CLK), y) diff --git a/drivers/clk/at91/pmc.c b/drivers/clk/at91/pmc.c index 4ef4f7b9d0bd1..073b2d506f787 100644 --- a/drivers/clk/at91/pmc.c +++ b/drivers/clk/at91/pmc.c @@ -115,7 +115,7 @@ struct pmc_data *pmc_data_allocate(unsigned int ncore, unsigned int nsystem, /* Address in SECURAM that say if we suspend to backup mode. */ static void __iomem *at91_pmc_backup_suspend; -static int at91_pmc_suspend(void) +static int at91_pmc_suspend(void *data) { unsigned int backup; @@ -129,7 +129,7 @@ static int at91_pmc_suspend(void) return clk_save_context(); } -static void at91_pmc_resume(void) +static void at91_pmc_resume(void *data) { unsigned int backup; @@ -143,11 +143,15 @@ static void at91_pmc_resume(void) clk_restore_context(); } -static struct syscore_ops pmc_syscore_ops = { +static const struct syscore_ops pmc_syscore_ops = { .suspend = at91_pmc_suspend, .resume = at91_pmc_resume, }; +static struct syscore pmc_syscore = { + .ops = &pmc_syscore_ops, +}; + static const struct of_device_id pmc_dt_ids[] = { { .compatible = "atmel,sama5d2-pmc" }, { .compatible = "microchip,sama7g5-pmc", }, @@ -185,7 +189,7 @@ static int __init pmc_register_ops(void) return -ENOMEM; } - register_syscore_ops(&pmc_syscore_ops); + register_syscore(&pmc_syscore); return 0; } diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c index fa5ccef73e60d..26f76a6db8202 100644 --- a/drivers/clk/clk-axi-clkgen.c +++ b/drivers/clk/clk-axi-clkgen.c @@ -521,7 +521,7 @@ static int axi_clkgen_setup_limits(struct axi_clkgen *axi_clkgen, axi_clkgen->limits.fvco_max = 1200000; axi_clkgen->limits.fpfd_max = 450000; break; - case ADI_AXI_FPGA_SPEED_2 ... ADI_AXI_FPGA_SPEED_2LV: + case ADI_AXI_FPGA_SPEED_2 ... ADI_AXI_FPGA_SPEED_2MP: axi_clkgen->limits.fvco_max = 1440000; axi_clkgen->limits.fpfd_max = 500000; if (family == ADI_AXI_FPGA_FAMILY_KINTEX || family == ADI_AXI_FPGA_FAMILY_ARTIX) { @@ -546,6 +546,9 @@ static int axi_clkgen_setup_limits(struct axi_clkgen *axi_clkgen, if (tech == ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS) { axi_clkgen->limits.fvco_max = 1600000; axi_clkgen->limits.fvco_min = 800000; + } else if (tech == ADI_AXI_FPGA_TECH_VERSAL) { + axi_clkgen->limits.fvco_max = 4320000; + axi_clkgen->limits.fvco_min = 2160000; } return 0; diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c index 5368d92d9b399..09fd5e8fb72c4 100644 --- a/drivers/clk/clk-devres.c +++ b/drivers/clk/clk-devres.c @@ -99,6 +99,11 @@ struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id) } EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled); +static void devm_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, const char *id, unsigned long rate) @@ -106,8 +111,7 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, struct clk *clk; int ret; - clk = __devm_clk_get(dev, id, clk_get_optional, NULL, - clk_disable_unprepare); + clk = devm_clk_get_optional(dev, id); if (IS_ERR(clk)) return ERR_CAST(clk); @@ -119,6 +123,10 @@ struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, if (ret) goto out_put_clk; + ret = devm_add_action_or_reset(dev, devm_clk_disable_unprepare, clk); + if (ret) + goto out_put_clk; + return clk; out_put_clk: diff --git a/drivers/clk/clk-hog.c b/drivers/clk/clk-hog.c new file mode 100644 index 0000000000000..c36fff9dbccea --- /dev/null +++ b/drivers/clk/clk-hog.c @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * clk-hog.c - a trivial clock "hog" driver + * + * Modelled on the gpio-hog concept in gpiolib-of.c: a device tree node + * whose sole purpose is to claim, configure (via the standard + * assigned-clocks/assigned-clock-rates/assigned-clock-parents + * properties) and enable one or more clocks, and keep them running for + * as long as the node is bound - e.g. to hold a GPCLK output enabled + * with no other consumer driver involved. + */ + +#include +#include +#include +#include + +struct clk_hog { + struct clk_bulk_data *clks; + int num_clks; +}; + +static int clk_hog_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct clk_hog *hog; + int ret; + + hog = devm_kzalloc(dev, sizeof(*hog), GFP_KERNEL); + if (!hog) + return -ENOMEM; + + ret = devm_clk_bulk_get_all(dev, &hog->clks); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to get clocks\n"); + + if (!ret) + return dev_err_probe(dev, -EINVAL, "no clocks specified\n"); + + hog->num_clks = ret; + + ret = clk_bulk_prepare_enable(hog->num_clks, hog->clks); + if (ret) + return dev_err_probe(dev, ret, "failed to enable clocks\n"); + + platform_set_drvdata(pdev, hog); + + return 0; +} + +static void clk_hog_remove(struct platform_device *pdev) +{ + struct clk_hog *hog = platform_get_drvdata(pdev); + + clk_bulk_disable_unprepare(hog->num_clks, hog->clks); +} + +static const struct of_device_id clk_hog_of_match[] = { + { .compatible = "clk-hog" }, + { } +}; +MODULE_DEVICE_TABLE(of, clk_hog_of_match); + +static struct platform_driver clk_hog_driver = { + .probe = clk_hog_probe, + .remove = clk_hog_remove, + .driver = { + .name = "clk-hog", + .of_match_table = clk_hog_of_match, + }, +}; +module_platform_driver(clk_hog_driver); + +MODULE_DESCRIPTION("Clock hog driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/clk/clk-moxart.c b/drivers/clk/clk-moxart.c index 3786a0153ad17..40663ef3ef0ae 100644 --- a/drivers/clk/clk-moxart.c +++ b/drivers/clk/clk-moxart.c @@ -17,7 +17,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *ref_clk; unsigned int mul; const char *name = node->name; const char *parent_name; @@ -34,12 +33,6 @@ static void __init moxart_of_pll_clk_init(struct device_node *node) mul = readl(base + 0x30) >> 3 & 0x3f; iounmap(base); - ref_clk = of_clk_get(node, 0); - if (IS_ERR(ref_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, mul, 1); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); @@ -56,7 +49,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) { void __iomem *base; struct clk_hw *hw; - struct clk *pll_clk; unsigned int div, val; unsigned int div_idx[] = { 2, 3, 4, 6, 8}; const char *name = node->name; @@ -78,12 +70,6 @@ static void __init moxart_of_apb_clk_init(struct device_node *node) val = 0; div = div_idx[val] * 2; - pll_clk = of_clk_get(node, 0); - if (IS_ERR(pll_clk)) { - pr_err("%pOF: of_clk_get failed\n", node); - return; - } - hw = clk_hw_register_fixed_factor(NULL, name, parent_name, 0, 1, div); if (IS_ERR(hw)) { pr_err("%pOF: failed to register clock\n", node); diff --git a/drivers/clk/clk-palmas.c b/drivers/clk/clk-palmas.c index 39049f62dbbb3..86a51edac8272 100644 --- a/drivers/clk/clk-palmas.c +++ b/drivers/clk/clk-palmas.c @@ -194,6 +194,13 @@ static void palmas_clks_get_clk_data(struct platform_device *pdev, cinfo->ext_control_pin = prop; } +static void palmas_clks_unprepare_ext_control(void *data) +{ + struct palmas_clock_info *cinfo = data; + + clk_unprepare(cinfo->hw.clk); +} + static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) { int ret; @@ -214,13 +221,18 @@ static int palmas_clks_init_configure(struct palmas_clock_info *cinfo) return ret; } + ret = devm_add_action_or_reset(cinfo->dev, + palmas_clks_unprepare_ext_control, + cinfo); + if (ret) + return ret; + ret = palmas_ext_control_req_config(cinfo->palmas, cinfo->clk_desc->sleep_reqstr_id, cinfo->ext_control_pin, true); if (ret < 0) { dev_err(cinfo->dev, "Ext config for %s failed, %d\n", cinfo->clk_desc->clk_name, ret); - clk_unprepare(cinfo->hw.clk); return ret; } } diff --git a/drivers/clk/clk-versaclock7.c b/drivers/clk/clk-versaclock7.c index adcc603e32593..e3a36dcd98b80 100644 --- a/drivers/clk/clk-versaclock7.c +++ b/drivers/clk/clk-versaclock7.c @@ -1197,7 +1197,7 @@ static int vc7_probe(struct i2c_client *client) if (ret) { dev_err_probe(&client->dev, ret, "unable to register output %d\n", i); - return ret; + goto err_clk; } switch (bank_src_map.type) { diff --git a/drivers/clk/hisilicon/reset.c b/drivers/clk/hisilicon/reset.c index 93cee17db8b16..c3b7daac93132 100644 --- a/drivers/clk/hisilicon/reset.c +++ b/drivers/clk/hisilicon/reset.c @@ -91,7 +91,7 @@ struct hisi_reset_controller *hisi_reset_init(struct platform_device *pdev) { struct hisi_reset_controller *rstc; - rstc = devm_kmalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); + rstc = devm_kzalloc(&pdev->dev, sizeof(*rstc), GFP_KERNEL); if (!rstc) return NULL; diff --git a/drivers/clk/imx/clk-scu.c b/drivers/clk/imx/clk-scu.c index c03f7821824d1..b2b0e5f05b238 100644 --- a/drivers/clk/imx/clk-scu.c +++ b/drivers/clk/imx/clk-scu.c @@ -465,7 +465,6 @@ struct clk_hw *__imx_clk_scu(struct device *dev, const char *name, clk->clk_type = clk_type; init.name = name; - init.ops = &clk_scu_ops; if (rsrc_id == IMX_SC_R_A35 || rsrc_id == IMX_SC_R_A53 || rsrc_id == IMX_SC_R_A72) init.ops = &clk_scu_cpu_ops; else if (rsrc_id == IMX_SC_R_PI_0_PLL) diff --git a/drivers/clk/imx/clk-vf610.c b/drivers/clk/imx/clk-vf610.c index 9e11f1c7c397e..41eb38552a9c7 100644 --- a/drivers/clk/imx/clk-vf610.c +++ b/drivers/clk/imx/clk-vf610.c @@ -139,7 +139,7 @@ static struct clk * __init vf610_get_fixed_clock( return clk; }; -static int vf610_clk_suspend(void) +static int vf610_clk_suspend(void *data) { int i; @@ -156,7 +156,7 @@ static int vf610_clk_suspend(void) return 0; } -static void vf610_clk_resume(void) +static void vf610_clk_resume(void *data) { int i; @@ -171,11 +171,15 @@ static void vf610_clk_resume(void) writel_relaxed(ccgr[i], CCM_CCGRx(i)); } -static struct syscore_ops vf610_clk_syscore_ops = { +static const struct syscore_ops vf610_clk_syscore_ops = { .suspend = vf610_clk_suspend, .resume = vf610_clk_resume, }; +static struct syscore vf610_clk_syscore = { + .ops = &vf610_clk_syscore_ops, +}; + static void __init vf610_clocks_init(struct device_node *ccm_node) { struct device_node *np; @@ -462,7 +466,7 @@ static void __init vf610_clocks_init(struct device_node *ccm_node) for (i = 0; i < ARRAY_SIZE(clks_init_on); i++) clk_prepare_enable(clk[clks_init_on[i]]); - register_syscore_ops(&vf610_clk_syscore_ops); + register_syscore(&vf610_clk_syscore); /* Add the clocks to provider list */ clk_data.clks = clk; diff --git a/drivers/clk/ingenic/jz4725b-cgu.c b/drivers/clk/ingenic/jz4725b-cgu.c index 590e9c85cb25b..94cee44c854f9 100644 --- a/drivers/clk/ingenic/jz4725b-cgu.c +++ b/drivers/clk/ingenic/jz4725b-cgu.c @@ -268,6 +268,6 @@ static void __init jz4725b_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } CLK_OF_DECLARE_DRIVER(jz4725b_cgu, "ingenic,jz4725b-cgu", jz4725b_cgu_init); diff --git a/drivers/clk/ingenic/jz4740-cgu.c b/drivers/clk/ingenic/jz4740-cgu.c index 3e0a30574ebb3..2def3aedc8dd2 100644 --- a/drivers/clk/ingenic/jz4740-cgu.c +++ b/drivers/clk/ingenic/jz4740-cgu.c @@ -266,6 +266,6 @@ static void __init jz4740_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } CLK_OF_DECLARE_DRIVER(jz4740_cgu, "ingenic,jz4740-cgu", jz4740_cgu_init); diff --git a/drivers/clk/ingenic/jz4755-cgu.c b/drivers/clk/ingenic/jz4755-cgu.c index f2c2d848dab7a..17cf5dcaece99 100644 --- a/drivers/clk/ingenic/jz4755-cgu.c +++ b/drivers/clk/ingenic/jz4755-cgu.c @@ -337,7 +337,7 @@ static void __init jz4755_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } /* * CGU has some children devices, this is useful for probing children devices diff --git a/drivers/clk/ingenic/jz4760-cgu.c b/drivers/clk/ingenic/jz4760-cgu.c index e407f00bd5942..372fe4b079927 100644 --- a/drivers/clk/ingenic/jz4760-cgu.c +++ b/drivers/clk/ingenic/jz4760-cgu.c @@ -436,7 +436,7 @@ static void __init jz4760_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } /* We only probe via devicetree, no need for a platform driver */ diff --git a/drivers/clk/ingenic/jz4770-cgu.c b/drivers/clk/ingenic/jz4770-cgu.c index 6ae1740367f9e..58f1d3bad6774 100644 --- a/drivers/clk/ingenic/jz4770-cgu.c +++ b/drivers/clk/ingenic/jz4770-cgu.c @@ -456,7 +456,7 @@ static void __init jz4770_cgu_init(struct device_node *np) if (retval) pr_err("%s: failed to register CGU Clocks\n", __func__); - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } /* We only probe via devicetree, no need for a platform driver */ diff --git a/drivers/clk/ingenic/jz4780-cgu.c b/drivers/clk/ingenic/jz4780-cgu.c index 07e2f3c5c4549..1e88aef7ac0f3 100644 --- a/drivers/clk/ingenic/jz4780-cgu.c +++ b/drivers/clk/ingenic/jz4780-cgu.c @@ -803,6 +803,6 @@ static void __init jz4780_cgu_init(struct device_node *np) return; } - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } CLK_OF_DECLARE_DRIVER(jz4780_cgu, "ingenic,jz4780-cgu", jz4780_cgu_init); diff --git a/drivers/clk/ingenic/pm.c b/drivers/clk/ingenic/pm.c index 341752b640d2b..206d5cf2872f6 100644 --- a/drivers/clk/ingenic/pm.c +++ b/drivers/clk/ingenic/pm.c @@ -15,7 +15,7 @@ static void __iomem * __maybe_unused ingenic_cgu_base; -static int __maybe_unused ingenic_cgu_pm_suspend(void) +static int __maybe_unused ingenic_cgu_pm_suspend(void *data) { u32 val = readl(ingenic_cgu_base + CGU_REG_LCR); @@ -24,22 +24,26 @@ static int __maybe_unused ingenic_cgu_pm_suspend(void) return 0; } -static void __maybe_unused ingenic_cgu_pm_resume(void) +static void __maybe_unused ingenic_cgu_pm_resume(void *data) { u32 val = readl(ingenic_cgu_base + CGU_REG_LCR); writel(val & ~LCR_LOW_POWER_MODE, ingenic_cgu_base + CGU_REG_LCR); } -static struct syscore_ops __maybe_unused ingenic_cgu_pm_ops = { +static const struct syscore_ops __maybe_unused ingenic_cgu_pm_ops = { .suspend = ingenic_cgu_pm_suspend, .resume = ingenic_cgu_pm_resume, }; -void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu) +static struct syscore __maybe_unused ingenic_cgu_pm = { + .ops = &ingenic_cgu_pm_ops, +}; + +void ingenic_cgu_register_syscore(struct ingenic_cgu *cgu) { if (IS_ENABLED(CONFIG_PM_SLEEP)) { ingenic_cgu_base = cgu->base; - register_syscore_ops(&ingenic_cgu_pm_ops); + register_syscore(&ingenic_cgu_pm); } } diff --git a/drivers/clk/ingenic/pm.h b/drivers/clk/ingenic/pm.h index fa7540407b6b7..0dcb57dc64cb4 100644 --- a/drivers/clk/ingenic/pm.h +++ b/drivers/clk/ingenic/pm.h @@ -7,6 +7,6 @@ struct ingenic_cgu; -void ingenic_cgu_register_syscore_ops(struct ingenic_cgu *cgu); +void ingenic_cgu_register_syscore(struct ingenic_cgu *cgu); #endif /* DRIVERS_CLK_INGENIC_PM_H */ diff --git a/drivers/clk/ingenic/tcu.c b/drivers/clk/ingenic/tcu.c index 7d04ef40b7cf9..bc6a51da20725 100644 --- a/drivers/clk/ingenic/tcu.c +++ b/drivers/clk/ingenic/tcu.c @@ -455,7 +455,7 @@ static int __init ingenic_tcu_probe(struct device_node *np) return ret; } -static int __maybe_unused tcu_pm_suspend(void) +static int __maybe_unused tcu_pm_suspend(void *data) { struct ingenic_tcu *tcu = ingenic_tcu; @@ -465,7 +465,7 @@ static int __maybe_unused tcu_pm_suspend(void) return 0; } -static void __maybe_unused tcu_pm_resume(void) +static void __maybe_unused tcu_pm_resume(void *data) { struct ingenic_tcu *tcu = ingenic_tcu; @@ -473,11 +473,15 @@ static void __maybe_unused tcu_pm_resume(void) clk_enable(tcu->clk); } -static struct syscore_ops __maybe_unused tcu_pm_ops = { +static const struct syscore_ops __maybe_unused tcu_pm_ops = { .suspend = tcu_pm_suspend, .resume = tcu_pm_resume, }; +static struct syscore __maybe_unused tcu_pm = { + .ops = &tcu_pm_ops, +}; + static void __init ingenic_tcu_init(struct device_node *np) { int ret = ingenic_tcu_probe(np); @@ -486,7 +490,7 @@ static void __init ingenic_tcu_init(struct device_node *np) pr_crit("Failed to initialize TCU clocks: %d\n", ret); if (IS_ENABLED(CONFIG_PM_SLEEP)) - register_syscore_ops(&tcu_pm_ops); + register_syscore(&tcu_pm); } CLK_OF_DECLARE_DRIVER(jz4740_cgu, "ingenic,jz4740-tcu", ingenic_tcu_init); diff --git a/drivers/clk/ingenic/x1000-cgu.c b/drivers/clk/ingenic/x1000-cgu.c index d80886caf3933..d89bdfb7c219a 100644 --- a/drivers/clk/ingenic/x1000-cgu.c +++ b/drivers/clk/ingenic/x1000-cgu.c @@ -556,7 +556,7 @@ static void __init x1000_cgu_init(struct device_node *np) return; } - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } /* * CGU has some children devices, this is useful for probing children devices diff --git a/drivers/clk/ingenic/x1830-cgu.c b/drivers/clk/ingenic/x1830-cgu.c index 0fd46e50a513f..acf856e5009e0 100644 --- a/drivers/clk/ingenic/x1830-cgu.c +++ b/drivers/clk/ingenic/x1830-cgu.c @@ -463,7 +463,7 @@ static void __init x1830_cgu_init(struct device_node *np) return; } - ingenic_cgu_register_syscore_ops(cgu); + ingenic_cgu_register_syscore(cgu); } /* * CGU has some children devices, this is useful for probing children devices diff --git a/drivers/clk/keystone/sci-clk.c b/drivers/clk/keystone/sci-clk.c index a4b42811de55d..066823458a75c 100644 --- a/drivers/clk/keystone/sci-clk.c +++ b/drivers/clk/keystone/sci-clk.c @@ -333,6 +333,14 @@ static int _sci_clk_build(struct sci_clk_provider *provider, init.ops = &sci_clk_ops; init.num_parents = sci_clk->num_parents; + + /* + * A clock rate query to the SCI firmware will return 0 if either the + * clock itself is disabled or the attached device/consumer is disabled. + * This makes it inherently unsuitable for the caching of the clk + * framework. + */ + init.flags = CLK_GET_RATE_NOCACHE; sci_clk->hw.init = &init; ret = devm_clk_hw_register(provider->dev, &sci_clk->hw); diff --git a/drivers/clk/mediatek/Kconfig b/drivers/clk/mediatek/Kconfig index 0e8dd82aa84ed..e0b797687b407 100644 --- a/drivers/clk/mediatek/Kconfig +++ b/drivers/clk/mediatek/Kconfig @@ -1006,6 +1006,7 @@ config COMMON_CLK_MT8196 tristate "Clock driver for MediaTek MT8196" depends on ARM64 || COMPILE_TEST select COMMON_CLK_MEDIATEK + select REGMAP_MMIO default ARCH_MEDIATEK help This driver supports MediaTek MT8196 basic clocks. diff --git a/drivers/clk/mediatek/clk-mt2701.c b/drivers/clk/mediatek/clk-mt2701.c index 1e88ad8b93f44..d9f40fda73d1a 100644 --- a/drivers/clk/mediatek/clk-mt2701.c +++ b/drivers/clk/mediatek/clk-mt2701.c @@ -978,7 +978,7 @@ static int mtk_apmixedsys_init(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, apmixed_plls, ARRAY_SIZE(apmixed_plls), + mtk_clk_register_plls(&pdev->dev, apmixed_plls, ARRAY_SIZE(apmixed_plls), clk_data); mtk_clk_register_factors(apmixed_fixed_divs, ARRAY_SIZE(apmixed_fixed_divs), clk_data); diff --git a/drivers/clk/mediatek/clk-mt2712-apmixedsys.c b/drivers/clk/mediatek/clk-mt2712-apmixedsys.c index a60622d251ff3..54b18e9f83f8f 100644 --- a/drivers/clk/mediatek/clk-mt2712-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt2712-apmixedsys.c @@ -119,7 +119,7 @@ static int clk_mt2712_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + r = mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); if (r) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c index e0949911e8f7d..b6eb6a581c31e 100644 --- a/drivers/clk/mediatek/clk-mt6735-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt6735-apmixedsys.c @@ -93,8 +93,8 @@ static int clk_mt6735_apmixed_probe(struct platform_device *pdev) return -ENOMEM; platform_set_drvdata(pdev, clk_data); - ret = mtk_clk_register_plls(pdev->dev.of_node, apmixedsys_plls, - ARRAY_SIZE(apmixedsys_plls), clk_data); + ret = mtk_clk_register_plls(&pdev->dev, apmixedsys_plls, + ARRAY_SIZE(apmixedsys_plls), clk_data); if (ret) { dev_err(&pdev->dev, "Failed to register PLLs: %d\n", ret); return ret; @@ -102,9 +102,12 @@ static int clk_mt6735_apmixed_probe(struct platform_device *pdev) ret = devm_of_clk_add_hw_provider(&pdev->dev, of_clk_hw_onecell_get, clk_data); - if (ret) + if (ret) { dev_err(&pdev->dev, "Failed to register clock provider: %d\n", ret); + mtk_clk_unregister_plls(apmixedsys_plls, ARRAY_SIZE(apmixedsys_plls), + clk_data); + } return ret; } diff --git a/drivers/clk/mediatek/clk-mt6765.c b/drivers/clk/mediatek/clk-mt6765.c index d53731e7933f4..60f6f9fa7dcf2 100644 --- a/drivers/clk/mediatek/clk-mt6765.c +++ b/drivers/clk/mediatek/clk-mt6765.c @@ -740,7 +740,7 @@ static int clk_mt6765_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); mtk_clk_register_gates(&pdev->dev, node, apmixed_clks, ARRAY_SIZE(apmixed_clks), clk_data); diff --git a/drivers/clk/mediatek/clk-mt6779.c b/drivers/clk/mediatek/clk-mt6779.c index 86732f5acf934..4b9dcb910b03f 100644 --- a/drivers/clk/mediatek/clk-mt6779.c +++ b/drivers/clk/mediatek/clk-mt6779.c @@ -1220,7 +1220,7 @@ static int clk_mt6779_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); mtk_clk_register_gates(&pdev->dev, node, apmixed_clks, ARRAY_SIZE(apmixed_clks), clk_data); diff --git a/drivers/clk/mediatek/clk-mt6795-apmixedsys.c b/drivers/clk/mediatek/clk-mt6795-apmixedsys.c index 91665d7f125ef..123d5d7fea855 100644 --- a/drivers/clk/mediatek/clk-mt6795-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt6795-apmixedsys.c @@ -152,7 +152,7 @@ static int clk_mt6795_apmixed_probe(struct platform_device *pdev) return -ENOMEM; fhctl_parse_dt(fhctl_node, pllfhs, ARRAY_SIZE(pllfhs)); - ret = mtk_clk_register_pllfhs(node, plls, ARRAY_SIZE(plls), + ret = mtk_clk_register_pllfhs(dev, plls, ARRAY_SIZE(plls), pllfhs, ARRAY_SIZE(pllfhs), clk_data); if (ret) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt6797.c b/drivers/clk/mediatek/clk-mt6797.c index fb59e71af58e3..ebf850ac57f54 100644 --- a/drivers/clk/mediatek/clk-mt6797.c +++ b/drivers/clk/mediatek/clk-mt6797.c @@ -655,7 +655,7 @@ static int mtk_apmixedsys_init(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); return of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); } diff --git a/drivers/clk/mediatek/clk-mt7622-apmixedsys.c b/drivers/clk/mediatek/clk-mt7622-apmixedsys.c index 2350592d9a934..8a29eaab0cfcb 100644 --- a/drivers/clk/mediatek/clk-mt7622-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt7622-apmixedsys.c @@ -96,7 +96,7 @@ static int clk_mt7622_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(dev, plls, ARRAY_SIZE(plls), clk_data); if (ret) return ret; diff --git a/drivers/clk/mediatek/clk-mt7629.c b/drivers/clk/mediatek/clk-mt7629.c index baf94e7bea373..e154771b1b8bb 100644 --- a/drivers/clk/mediatek/clk-mt7629.c +++ b/drivers/clk/mediatek/clk-mt7629.c @@ -634,7 +634,7 @@ static int mtk_apmixedsys_init(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); mtk_clk_register_gates(&pdev->dev, node, apmixed_clks, diff --git a/drivers/clk/mediatek/clk-mt7981-apmixed.c b/drivers/clk/mediatek/clk-mt7981-apmixed.c index e8211eb4e09e1..6606b54fb3769 100644 --- a/drivers/clk/mediatek/clk-mt7981-apmixed.c +++ b/drivers/clk/mediatek/clk-mt7981-apmixed.c @@ -76,7 +76,7 @@ static int clk_mt7981_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); if (r) { diff --git a/drivers/clk/mediatek/clk-mt7986-apmixed.c b/drivers/clk/mediatek/clk-mt7986-apmixed.c index 93751abe6be89..1c79418d08a77 100644 --- a/drivers/clk/mediatek/clk-mt7986-apmixed.c +++ b/drivers/clk/mediatek/clk-mt7986-apmixed.c @@ -74,7 +74,7 @@ static int clk_mt7986_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); if (r) { diff --git a/drivers/clk/mediatek/clk-mt7988-apmixed.c b/drivers/clk/mediatek/clk-mt7988-apmixed.c index 63d33a78cb488..416a4b88d100b 100644 --- a/drivers/clk/mediatek/clk-mt7988-apmixed.c +++ b/drivers/clk/mediatek/clk-mt7988-apmixed.c @@ -86,7 +86,7 @@ static int clk_mt7988_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + r = mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); if (r) goto free_apmixed_data; diff --git a/drivers/clk/mediatek/clk-mt8135-apmixedsys.c b/drivers/clk/mediatek/clk-mt8135-apmixedsys.c index bdadc35c64cbd..19e4ee489ec39 100644 --- a/drivers/clk/mediatek/clk-mt8135-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8135-apmixedsys.c @@ -57,7 +57,8 @@ static int clk_mt8135_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), + clk_data); if (ret) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt8135.c b/drivers/clk/mediatek/clk-mt8135.c index 084e48a554c26..1d20e15608f77 100644 --- a/drivers/clk/mediatek/clk-mt8135.c +++ b/drivers/clk/mediatek/clk-mt8135.c @@ -409,6 +409,9 @@ static const struct mtk_gate_regs infra_cg_regs = { GATE_MTK_FLAGS(_id, _name, _parent, &infra_cg_regs, _shift, \ &mtk_clk_gate_ops_setclr, CLK_IS_CRITICAL) +#define GATE_ICG_INV(_id, _name, _parent, _shift) \ + GATE_MTK(_id, _name, _parent, &infra_cg_regs, _shift, &mtk_clk_gate_ops_setclr_inv) + static const struct mtk_gate infra_clks[] = { GATE_DUMMY(CLK_DUMMY, "infra_dummy"), GATE_ICG(CLK_INFRA_PMIC_WRAP, "pmic_wrap_ck", "axi_sel", 23), @@ -419,7 +422,7 @@ static const struct mtk_gate infra_clks[] = { GATE_ICG(CLK_INFRA_CPUM, "cpum_ck", "cpum_tck_in", 15), GATE_ICG_AO(CLK_INFRA_M4U, "m4u_ck", "mem_sel", 8), GATE_ICG(CLK_INFRA_MFGAXI, "mfgaxi_ck", "axi_sel", 7), - GATE_ICG(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), + GATE_ICG_INV(CLK_INFRA_DEVAPC, "devapc_ck", "axi_sel", 6), GATE_ICG(CLK_INFRA_AUDIO, "audio_ck", "aud_intbus_sel", 5), GATE_ICG(CLK_INFRA_MFG_BUS, "mfg_bus_ck", "axi_sel", 2), GATE_ICG(CLK_INFRA_SMI, "smi_ck", "smi_sel", 1), diff --git a/drivers/clk/mediatek/clk-mt8167-apmixedsys.c b/drivers/clk/mediatek/clk-mt8167-apmixedsys.c index adf576786696e..fb6c21bbeef81 100644 --- a/drivers/clk/mediatek/clk-mt8167-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8167-apmixedsys.c @@ -105,7 +105,7 @@ static int clk_mt8167_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(dev, plls, ARRAY_SIZE(plls), clk_data); if (ret) return ret; diff --git a/drivers/clk/mediatek/clk-mt8173-apmixedsys.c b/drivers/clk/mediatek/clk-mt8173-apmixedsys.c index 95385bb67d551..d7d416172ab35 100644 --- a/drivers/clk/mediatek/clk-mt8173-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8173-apmixedsys.c @@ -140,13 +140,13 @@ MODULE_DEVICE_TABLE(of, of_match_clk_mt8173_apmixed); static int clk_mt8173_apmixed_probe(struct platform_device *pdev) { const u8 *fhctl_node = "mediatek,mt8173-fhctl"; - struct device_node *node = pdev->dev.of_node; struct clk_hw_onecell_data *clk_data; + struct device *dev = &pdev->dev; void __iomem *base; struct clk_hw *hw; int r; - base = of_iomap(node, 0); + base = of_iomap(dev->of_node, 0); if (!base) return -ENOMEM; @@ -157,25 +157,25 @@ static int clk_mt8173_apmixed_probe(struct platform_device *pdev) } fhctl_parse_dt(fhctl_node, pllfhs, ARRAY_SIZE(pllfhs)); - r = mtk_clk_register_pllfhs(node, plls, ARRAY_SIZE(plls), - pllfhs, ARRAY_SIZE(pllfhs), clk_data); + r = mtk_clk_register_pllfhs(dev, plls, ARRAY_SIZE(plls), pllfhs, + ARRAY_SIZE(pllfhs), clk_data); if (r) goto free_clk_data; hw = mtk_clk_register_ref2usb_tx("ref2usb_tx", "clk26m", base + REGOFF_REF2USB); if (IS_ERR(hw)) { r = PTR_ERR(hw); - dev_err(&pdev->dev, "Failed to register ref2usb_tx: %d\n", r); + dev_err(dev, "Failed to register ref2usb_tx: %d\n", r); goto unregister_plls; } clk_data->hws[CLK_APMIXED_REF2USB_TX] = hw; - hw = devm_clk_hw_register_divider(&pdev->dev, "hdmi_ref", "tvdpll_594m", 0, + hw = devm_clk_hw_register_divider(dev, "hdmi_ref", "tvdpll_594m", 0, base + REGOFF_HDMI_REF, 16, 3, CLK_DIVIDER_POWER_OF_TWO, NULL); clk_data->hws[CLK_APMIXED_HDMI_REF] = hw; - r = of_clk_add_hw_provider(node, of_clk_hw_onecell_get, clk_data); + r = of_clk_add_hw_provider(dev->of_node, of_clk_hw_onecell_get, clk_data); if (r) goto unregister_ref2usb; diff --git a/drivers/clk/mediatek/clk-mt8183-apmixedsys.c b/drivers/clk/mediatek/clk-mt8183-apmixedsys.c index 551adbfd7ac93..6242d4f5376e7 100644 --- a/drivers/clk/mediatek/clk-mt8183-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8183-apmixedsys.c @@ -155,7 +155,7 @@ static int clk_mt8183_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(dev, plls, ARRAY_SIZE(plls), clk_data); if (ret) return ret; diff --git a/drivers/clk/mediatek/clk-mt8186-apmixedsys.c b/drivers/clk/mediatek/clk-mt8186-apmixedsys.c index 4b2b16578232d..d35dd2632e43a 100644 --- a/drivers/clk/mediatek/clk-mt8186-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8186-apmixedsys.c @@ -151,7 +151,7 @@ static int clk_mt8186_apmixed_probe(struct platform_device *pdev) fhctl_parse_dt(fhctl_node, pllfhs, ARRAY_SIZE(pllfhs)); - r = mtk_clk_register_pllfhs(node, plls, ARRAY_SIZE(plls), + r = mtk_clk_register_pllfhs(&pdev->dev, plls, ARRAY_SIZE(plls), pllfhs, ARRAY_SIZE(pllfhs), clk_data); if (r) goto free_apmixed_data; diff --git a/drivers/clk/mediatek/clk-mt8188-apmixedsys.c b/drivers/clk/mediatek/clk-mt8188-apmixedsys.c index 21d7a9a2ab1af..a1de596bff994 100644 --- a/drivers/clk/mediatek/clk-mt8188-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8188-apmixedsys.c @@ -106,7 +106,7 @@ static int clk_mt8188_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + r = mtk_clk_register_plls(&pdev->dev, plls, ARRAY_SIZE(plls), clk_data); if (r) goto free_apmixed_data; diff --git a/drivers/clk/mediatek/clk-mt8192-apmixedsys.c b/drivers/clk/mediatek/clk-mt8192-apmixedsys.c index 0b66a27e4d5ac..b0563a285bd66 100644 --- a/drivers/clk/mediatek/clk-mt8192-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8192-apmixedsys.c @@ -162,7 +162,7 @@ static int clk_mt8192_apmixed_probe(struct platform_device *pdev) fhctl_parse_dt(fhctl_node, pllfhs, ARRAY_SIZE(pllfhs)); - r = mtk_clk_register_pllfhs(node, plls, ARRAY_SIZE(plls), + r = mtk_clk_register_pllfhs(&pdev->dev, plls, ARRAY_SIZE(plls), pllfhs, ARRAY_SIZE(pllfhs), clk_data); if (r) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt8195-apmixedsys.c b/drivers/clk/mediatek/clk-mt8195-apmixedsys.c index 282a3137dc894..44917ab034c56 100644 --- a/drivers/clk/mediatek/clk-mt8195-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8195-apmixedsys.c @@ -181,7 +181,7 @@ static int clk_mt8195_apmixed_probe(struct platform_device *pdev) fhctl_parse_dt(fhctl_node, pllfhs, ARRAY_SIZE(pllfhs)); - r = mtk_clk_register_pllfhs(node, plls, ARRAY_SIZE(plls), + r = mtk_clk_register_pllfhs(&pdev->dev, plls, ARRAY_SIZE(plls), pllfhs, ARRAY_SIZE(pllfhs), clk_data); if (r) goto free_apmixed_data; diff --git a/drivers/clk/mediatek/clk-mt8195-apusys_pll.c b/drivers/clk/mediatek/clk-mt8195-apusys_pll.c index 8b45a3fad02f1..a2d98ed58e348 100644 --- a/drivers/clk/mediatek/clk-mt8195-apusys_pll.c +++ b/drivers/clk/mediatek/clk-mt8195-apusys_pll.c @@ -66,7 +66,8 @@ static int clk_mt8195_apusys_pll_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, apusys_plls, ARRAY_SIZE(apusys_plls), clk_data); + r = mtk_clk_register_plls(&pdev->dev, apusys_plls, + ARRAY_SIZE(apusys_plls), clk_data); if (r) goto free_apusys_pll_data; diff --git a/drivers/clk/mediatek/clk-mt8196-apmixedsys.c b/drivers/clk/mediatek/clk-mt8196-apmixedsys.c index 617f5449b88b8..c4ebb0170b82b 100644 --- a/drivers/clk/mediatek/clk-mt8196-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8196-apmixedsys.c @@ -152,7 +152,8 @@ static int clk_mt8196_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, mcd->clks, mcd->num_clks, clk_data); + r = mtk_clk_register_plls(&pdev->dev, mcd->clks, mcd->num_clks, + clk_data); if (r) goto free_apmixed_data; diff --git a/drivers/clk/mediatek/clk-mt8196-mcu.c b/drivers/clk/mediatek/clk-mt8196-mcu.c index 5cbcc411ae734..13642fc673c26 100644 --- a/drivers/clk/mediatek/clk-mt8196-mcu.c +++ b/drivers/clk/mediatek/clk-mt8196-mcu.c @@ -122,7 +122,7 @@ static int clk_mt8196_mcu_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, plls, num_plls, clk_data); + r = mtk_clk_register_plls(&pdev->dev, plls, num_plls, clk_data); if (r) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt8196-mfg.c b/drivers/clk/mediatek/clk-mt8196-mfg.c index f40795b47ff1f..a317183f1681b 100644 --- a/drivers/clk/mediatek/clk-mt8196-mfg.c +++ b/drivers/clk/mediatek/clk-mt8196-mfg.c @@ -106,7 +106,7 @@ static int clk_mt8196_mfg_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - r = mtk_clk_register_plls(node, plls, num_plls, clk_data); + r = mtk_clk_register_plls(&pdev->dev, plls, num_plls, clk_data); if (r) goto free_clk_data; diff --git a/drivers/clk/mediatek/clk-mt8196-vlpckgen.c b/drivers/clk/mediatek/clk-mt8196-vlpckgen.c index d59a8a9d98550..7dcc164627c57 100644 --- a/drivers/clk/mediatek/clk-mt8196-vlpckgen.c +++ b/drivers/clk/mediatek/clk-mt8196-vlpckgen.c @@ -664,7 +664,7 @@ static int clk_mt8196_vlp_probe(struct platform_device *pdev) if (r) goto unregister_factors; - r = mtk_clk_register_plls(node, vlp_plls, ARRAY_SIZE(vlp_plls), + r = mtk_clk_register_plls(dev, vlp_plls, ARRAY_SIZE(vlp_plls), clk_data); if (r) goto unregister_muxes; diff --git a/drivers/clk/mediatek/clk-mt8365-apmixedsys.c b/drivers/clk/mediatek/clk-mt8365-apmixedsys.c index f41b991a0178a..e331aa28a4bd5 100644 --- a/drivers/clk/mediatek/clk-mt8365-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8365-apmixedsys.c @@ -133,7 +133,7 @@ static int clk_mt8365_apmixed_probe(struct platform_device *pdev) return PTR_ERR(hw); clk_data->hws[CLK_APMIXED_USB20_EN] = hw; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(dev, plls, ARRAY_SIZE(plls), clk_data); if (ret) return ret; diff --git a/drivers/clk/mediatek/clk-mt8516-apmixedsys.c b/drivers/clk/mediatek/clk-mt8516-apmixedsys.c index edd9174d2f2ff..2a6206cae2f08 100644 --- a/drivers/clk/mediatek/clk-mt8516-apmixedsys.c +++ b/drivers/clk/mediatek/clk-mt8516-apmixedsys.c @@ -87,7 +87,7 @@ static int clk_mt8516_apmixed_probe(struct platform_device *pdev) if (!clk_data) return -ENOMEM; - ret = mtk_clk_register_plls(node, plls, ARRAY_SIZE(plls), clk_data); + ret = mtk_clk_register_plls(dev, plls, ARRAY_SIZE(plls), clk_data); if (ret) return ret; diff --git a/drivers/clk/mediatek/clk-pll.c b/drivers/clk/mediatek/clk-pll.c index de3eb02670554..0f3759fcd9d02 100644 --- a/drivers/clk/mediatek/clk-pll.c +++ b/drivers/clk/mediatek/clk-pll.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "clk-pll.h" @@ -368,7 +369,7 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, init.parent_names = &parent_name; init.num_parents = 1; - ret = clk_hw_register(NULL, &pll->hw); + ret = clk_hw_register(pll->dev, &pll->hw); if (ret) return ERR_PTR(ret); @@ -376,7 +377,8 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, return &pll->hw; } -struct clk_hw *mtk_clk_register_pll(const struct mtk_pll_data *data, +struct clk_hw *mtk_clk_register_pll(struct device *dev, + const struct mtk_pll_data *data, void __iomem *base) { struct mtk_clk_pll *pll; @@ -387,6 +389,8 @@ struct clk_hw *mtk_clk_register_pll(const struct mtk_pll_data *data, if (!pll) return ERR_PTR(-ENOMEM); + pll->dev = dev; + hw = mtk_clk_register_pll_ops(pll, data, base, pll_ops); if (IS_ERR(hw)) kfree(pll); @@ -407,7 +411,7 @@ void mtk_clk_unregister_pll(struct clk_hw *hw) kfree(pll); } -int mtk_clk_register_plls(struct device_node *node, +int mtk_clk_register_plls(struct device *dev, const struct mtk_pll_data *plls, int num_plls, struct clk_hw_onecell_data *clk_data) { @@ -415,7 +419,7 @@ int mtk_clk_register_plls(struct device_node *node, int i; struct clk_hw *hw; - base = of_iomap(node, 0); + base = of_iomap(dev->of_node, 0); if (!base) { pr_err("%s(): ioremap failed\n", __func__); return -EINVAL; @@ -426,11 +430,11 @@ int mtk_clk_register_plls(struct device_node *node, if (!IS_ERR_OR_NULL(clk_data->hws[pll->id])) { pr_warn("%pOF: Trying to register duplicate clock ID: %d\n", - node, pll->id); + dev->of_node, pll->id); continue; } - hw = mtk_clk_register_pll(pll, base); + hw = mtk_clk_register_pll(dev, pll, base); if (IS_ERR(hw)) { pr_err("Failed to register clk %s: %pe\n", pll->name, diff --git a/drivers/clk/mediatek/clk-pll.h b/drivers/clk/mediatek/clk-pll.h index de5a8fb7cbcfe..f49dc2732ffee 100644 --- a/drivers/clk/mediatek/clk-pll.h +++ b/drivers/clk/mediatek/clk-pll.h @@ -10,9 +10,7 @@ #include #include -struct clk_ops; -struct clk_hw_onecell_data; -struct device_node; +struct device; struct mtk_pll_div_table { u32 div; @@ -64,6 +62,7 @@ struct mtk_pll_data { */ struct mtk_clk_pll { + struct device *dev; struct clk_hw hw; void __iomem *base_addr; void __iomem *pd_addr; @@ -79,9 +78,9 @@ struct mtk_clk_pll { const struct mtk_pll_data *data; }; -int mtk_clk_register_plls(struct device_node *node, - const struct mtk_pll_data *plls, int num_plls, - struct clk_hw_onecell_data *clk_data); +int mtk_clk_register_plls(struct device *dev, const struct mtk_pll_data *plls, + int num_plls, struct clk_hw_onecell_data *clk_data); + void mtk_clk_unregister_plls(const struct mtk_pll_data *plls, int num_plls, struct clk_hw_onecell_data *clk_data); @@ -111,7 +110,8 @@ struct clk_hw *mtk_clk_register_pll_ops(struct mtk_clk_pll *pll, const struct mtk_pll_data *data, void __iomem *base, const struct clk_ops *pll_ops); -struct clk_hw *mtk_clk_register_pll(const struct mtk_pll_data *data, +struct clk_hw *mtk_clk_register_pll(struct device *dev, + const struct mtk_pll_data *data, void __iomem *base); void mtk_clk_unregister_pll(struct clk_hw *hw); diff --git a/drivers/clk/mediatek/clk-pllfh.c b/drivers/clk/mediatek/clk-pllfh.c index 83630ee07ee97..d8b7eb6d18e1c 100644 --- a/drivers/clk/mediatek/clk-pllfh.c +++ b/drivers/clk/mediatek/clk-pllfh.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "clk-mtk.h" #include "clk-pllfh.h" @@ -149,7 +150,7 @@ static bool fhctl_is_supported_and_enabled(const struct mtk_pllfh_data *pllfh) } static struct clk_hw * -mtk_clk_register_pllfh(const struct mtk_pll_data *pll_data, +mtk_clk_register_pllfh(struct device *dev, const struct mtk_pll_data *pll_data, struct mtk_pllfh_data *pllfh_data, void __iomem *base) { struct clk_hw *hw; @@ -166,6 +167,8 @@ mtk_clk_register_pllfh(const struct mtk_pll_data *pll_data, goto out; } + fh->clk_pll.dev = dev; + hw = mtk_clk_register_pll_ops(&fh->clk_pll, pll_data, base, &mtk_pllfh_ops); @@ -194,16 +197,60 @@ static void mtk_clk_unregister_pllfh(struct clk_hw *hw) kfree(fh); } -int mtk_clk_register_pllfhs(struct device_node *node, +static void mtk_clk_cleanup_pllfhs(void __iomem *iomem_base, + const struct mtk_pll_data *plls, int num_plls, + void __iomem *iomem_fhctl_base, + struct mtk_pllfh_data *pllfhs, int num_fhs, + struct clk_hw_onecell_data *clk_data) +{ + void __iomem *base = iomem_base; + void __iomem *fhctl_base = iomem_fhctl_base; + int i; + + for (i = num_plls - 1; i >= 0; i--) { + const struct mtk_pll_data *pll = &plls[i]; + struct mtk_pllfh_data *pllfh; + bool use_fhctl; + + if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) + continue; + + pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); + use_fhctl = fhctl_is_supported_and_enabled(pllfh); + + if (!base) + base = mtk_clk_pll_get_base(clk_data->hws[pll->id], + pll); + + if (use_fhctl) { + if (!fhctl_base) + fhctl_base = pllfh->state.base; + mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); + } else { + mtk_clk_unregister_pll(clk_data->hws[pll->id]); + } + + clk_data->hws[pll->id] = ERR_PTR(-ENOENT); + } + + if (fhctl_base) + iounmap(fhctl_base); + + if (base) + iounmap(base); +} + + +int mtk_clk_register_pllfhs(struct device *dev, const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base; + void __iomem *base, *fhctl_base = NULL; int i; struct clk_hw *hw; - base = of_iomap(node, 0); + base = of_iomap(dev->of_node, 0); if (!base) { pr_err("%s(): ioremap failed\n", __func__); return -EINVAL; @@ -218,9 +265,9 @@ int mtk_clk_register_pllfhs(struct device_node *node, use_fhctl = fhctl_is_supported_and_enabled(pllfh); if (use_fhctl) - hw = mtk_clk_register_pllfh(pll, pllfh, base); + hw = mtk_clk_register_pllfh(dev, pll, pllfh, base); else - hw = mtk_clk_register_pll(pll, base); + hw = mtk_clk_register_pll(dev, pll, base); if (IS_ERR(hw)) { pr_err("Failed to register %s clk %s: %ld\n", @@ -235,24 +282,8 @@ int mtk_clk_register_pllfhs(struct device_node *node, return 0; err: - while (--i >= 0) { - const struct mtk_pll_data *pll = &plls[i]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - else - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - iounmap(base); - + mtk_clk_cleanup_pllfhs(base, plls, i, fhctl_base, pllfhs, num_fhs, + clk_data); return PTR_ERR(hw); } EXPORT_SYMBOL_GPL(mtk_clk_register_pllfhs); @@ -261,38 +292,10 @@ void mtk_clk_unregister_pllfhs(const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_fhs, struct clk_hw_onecell_data *clk_data) { - void __iomem *base = NULL, *fhctl_base = NULL; - int i; - if (!clk_data) return; - for (i = num_plls; i > 0; i--) { - const struct mtk_pll_data *pll = &plls[i - 1]; - struct mtk_pllfh_data *pllfh; - bool use_fhctl; - - if (IS_ERR_OR_NULL(clk_data->hws[pll->id])) - continue; - - pllfh = get_pllfh_by_id(pllfhs, num_fhs, pll->id); - use_fhctl = fhctl_is_supported_and_enabled(pllfh); - - if (use_fhctl) { - fhctl_base = pllfh->state.base; - mtk_clk_unregister_pllfh(clk_data->hws[pll->id]); - } else { - base = mtk_clk_pll_get_base(clk_data->hws[pll->id], - pll); - mtk_clk_unregister_pll(clk_data->hws[pll->id]); - } - - clk_data->hws[pll->id] = ERR_PTR(-ENOENT); - } - - if (fhctl_base) - iounmap(fhctl_base); - - iounmap(base); + mtk_clk_cleanup_pllfhs(NULL, plls, num_plls, NULL, pllfhs, + num_fhs, clk_data); } EXPORT_SYMBOL_GPL(mtk_clk_unregister_pllfhs); diff --git a/drivers/clk/mediatek/clk-pllfh.h b/drivers/clk/mediatek/clk-pllfh.h index 5f419c2ec01f9..a4f337acad713 100644 --- a/drivers/clk/mediatek/clk-pllfh.h +++ b/drivers/clk/mediatek/clk-pllfh.h @@ -68,7 +68,7 @@ struct fh_operation { int (*ssc_enable)(struct mtk_fh *fh, u32 rate); }; -int mtk_clk_register_pllfhs(struct device_node *node, +int mtk_clk_register_pllfhs(struct device *dev, const struct mtk_pll_data *plls, int num_plls, struct mtk_pllfh_data *pllfhs, int num_pllfhs, struct clk_hw_onecell_data *clk_data); diff --git a/drivers/clk/meson/gxbb.c b/drivers/clk/meson/gxbb.c index ec9a3414875ac..b709ab48b1e7f 100644 --- a/drivers/clk/meson/gxbb.c +++ b/drivers/clk/meson/gxbb.c @@ -1394,7 +1394,7 @@ static struct clk_regmap gxbb_32k_clk_sel = { .name = "32k_clk_sel", .ops = &clk_regmap_mux_ops, .parent_data = gxbb_32k_clk_parents, - .num_parents = 4, + .num_parents = ARRAY_SIZE(gxbb_32k_clk_parents), .flags = CLK_SET_RATE_PARENT, }, }; diff --git a/drivers/clk/mvebu/common.c b/drivers/clk/mvebu/common.c index 785dbede48354..5adbbd91a6db3 100644 --- a/drivers/clk/mvebu/common.c +++ b/drivers/clk/mvebu/common.c @@ -215,22 +215,26 @@ static struct clk *clk_gating_get_src( return ERR_PTR(-ENODEV); } -static int mvebu_clk_gating_suspend(void) +static int mvebu_clk_gating_suspend(void *data) { ctrl->saved_reg = readl(ctrl->base); return 0; } -static void mvebu_clk_gating_resume(void) +static void mvebu_clk_gating_resume(void *data) { writel(ctrl->saved_reg, ctrl->base); } -static struct syscore_ops clk_gate_syscore_ops = { +static const struct syscore_ops clk_gate_syscore_ops = { .suspend = mvebu_clk_gating_suspend, .resume = mvebu_clk_gating_resume, }; +static struct syscore clk_gate_syscore = { + .ops = &clk_gate_syscore_ops, +}; + void __init mvebu_clk_gating_setup(struct device_node *np, const struct clk_gating_soc_desc *desc) { @@ -284,7 +288,7 @@ void __init mvebu_clk_gating_setup(struct device_node *np, of_clk_add_provider(np, clk_gating_get_src, ctrl); - register_syscore_ops(&clk_gate_syscore_ops); + register_syscore(&clk_gate_syscore); return; gates_out: diff --git a/drivers/clk/nuvoton/clk-ma35d1-pll.c b/drivers/clk/nuvoton/clk-ma35d1-pll.c index 4620acfe47e85..c7c0dc91a012c 100644 --- a/drivers/clk/nuvoton/clk-ma35d1-pll.c +++ b/drivers/clk/nuvoton/clk-ma35d1-pll.c @@ -48,7 +48,7 @@ #define PLL_CTL1_PD BIT(0) #define PLL_CTL1_BP BIT(1) #define PLL_CTL1_OUTDIV GENMASK(6, 4) -#define PLL_CTL1_FRAC GENMASK(31, 24) +#define PLL_CTL1_FRAC GENMASK(31, 8) #define PLL_CTL2_SLOPE GENMASK(23, 0) #define INDIV_MIN 1 @@ -92,7 +92,7 @@ static unsigned long ma35d1_calc_smic_pll_freq(u32 pll0_ctl0, p = FIELD_GET(SPLL0_CTL0_OUTDIV, pll0_ctl0); outdiv = 1 << p; pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * outdiv); + pll_freq = div_u64(pll_freq, m * outdiv); return pll_freq; } @@ -110,12 +110,12 @@ static unsigned long ma35d1_calc_pll_freq(u8 mode, u32 *reg_ctl, unsigned long p if (mode == PLL_MODE_INT) { pll_freq = (u64)parent_rate * n; - div_u64(pll_freq, m * p); + pll_freq = div_u64(pll_freq, m * p); } else { x = FIELD_GET(PLL_CTL1_FRAC, reg_ctl[1]); - /* 2 decimal places floating to integer (ex. 1.23 to 123) */ - n = n * 100 + ((x * 100) / FIELD_MAX(PLL_CTL1_FRAC)); - pll_freq = div_u64(parent_rate * n, 100 * m * p); + /* convert 24-bit fraction to 3 decimal digits, rounding to closest */ + n = n * 1000 + DIV_ROUND_CLOSEST_ULL((u64)x * 1000, 1ULL << 24); + pll_freq = div_u64((u64)parent_rate * n, 1000 * m * p); } return pll_freq; } @@ -255,32 +255,32 @@ static int ma35d1_clk_pll_determine_rate(struct clk_hw *hw, if (req->best_parent_rate < PLL_FREF_MIN_FREQ || req->best_parent_rate > PLL_FREF_MAX_FREQ) return -EINVAL; - ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, - reg_ctl, &pll_freq); - if (ret < 0) - return ret; - switch (pll->id) { case CAPLL: + case DDRPLL: + /* Read-only PLLs: return current rate */ reg_ctl[0] = readl_relaxed(pll->ctl0_base); - pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + if (pll->id == CAPLL) { + pll_freq = ma35d1_calc_smic_pll_freq(reg_ctl[0], req->best_parent_rate); + } else { + reg_ctl[1] = readl_relaxed(pll->ctl1_base); + pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + } req->rate = pll_freq; - return 0; - case DDRPLL: case APLL: case EPLL: case VPLL: - reg_ctl[0] = readl_relaxed(pll->ctl0_base); - reg_ctl[1] = readl_relaxed(pll->ctl1_base); - pll_freq = ma35d1_calc_pll_freq(pll->mode, reg_ctl, req->best_parent_rate); + /* Configurable PLLs: find closest achievable rate */ + ret = ma35d1_pll_find_closest(pll, req->rate, req->best_parent_rate, + reg_ctl, &pll_freq); + if (ret < 0) + return ret; req->rate = pll_freq; - return 0; } req->rate = 0; - return 0; } diff --git a/drivers/clk/qcom/camcc-sc8280xp.c b/drivers/clk/qcom/camcc-sc8280xp.c index 18f5a3eb313e1..1de238a85ab9f 100644 --- a/drivers/clk/qcom/camcc-sc8280xp.c +++ b/drivers/clk/qcom/camcc-sc8280xp.c @@ -1754,24 +1754,6 @@ static struct clk_branch camcc_csiphy3_clk = { }, }; -static struct clk_branch camcc_gdsc_clk = { - .halt_reg = 0xc1e4, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0xc1e4, - .enable_mask = BIT(0), - .hw.init = &(struct clk_init_data){ - .name = "camcc_gdsc_clk", - .parent_hws = (const struct clk_hw*[]){ - &camcc_xo_clk_src.clkr.hw, - }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch camcc_icp_ahb_clk = { .halt_reg = 0xc0d8, .halt_check = BRANCH_HALT, @@ -2840,7 +2822,6 @@ static struct clk_regmap *camcc_sc8280xp_clocks[] = { [CAMCC_CSIPHY2_CLK] = &camcc_csiphy2_clk.clkr, [CAMCC_CSIPHY3_CLK] = &camcc_csiphy3_clk.clkr, [CAMCC_FAST_AHB_CLK_SRC] = &camcc_fast_ahb_clk_src.clkr, - [CAMCC_GDSC_CLK] = &camcc_gdsc_clk.clkr, [CAMCC_ICP_AHB_CLK] = &camcc_icp_ahb_clk.clkr, [CAMCC_ICP_CLK] = &camcc_icp_clk.clkr, [CAMCC_ICP_CLK_SRC] = &camcc_icp_clk_src.clkr, diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c index 63c38cb47bc45..82f18e15cd77b 100644 --- a/drivers/clk/qcom/clk-rpmh.c +++ b/drivers/clk/qcom/clk-rpmh.c @@ -66,8 +66,6 @@ struct clk_rpmh { struct clk_rpmh_desc { struct clk_hw **clks; size_t num_clks; - /* RPMh clock clkaN are optional for this platform */ - bool clka_optional; }; static DEFINE_MUTEX(rpmh_clk_lock); @@ -680,7 +678,6 @@ static struct clk_hw *sm8550_rpmh_clocks[] = { static const struct clk_rpmh_desc clk_rpmh_sm8550 = { .clks = sm8550_rpmh_clocks, .num_clks = ARRAY_SIZE(sm8550_rpmh_clocks), - .clka_optional = true, }; static struct clk_hw *sm8650_rpmh_clocks[] = { @@ -712,7 +709,6 @@ static struct clk_hw *sm8650_rpmh_clocks[] = { static const struct clk_rpmh_desc clk_rpmh_sm8650 = { .clks = sm8650_rpmh_clocks, .num_clks = ARRAY_SIZE(sm8650_rpmh_clocks), - .clka_optional = true, }; static struct clk_hw *sc7280_rpmh_clocks[] = { @@ -881,7 +877,6 @@ static struct clk_hw *sm8750_rpmh_clocks[] = { static const struct clk_rpmh_desc clk_rpmh_sm8750 = { .clks = sm8750_rpmh_clocks, .num_clks = ARRAY_SIZE(sm8750_rpmh_clocks), - .clka_optional = true, }; static struct clk_hw *glymur_rpmh_clocks[] = { @@ -943,8 +938,7 @@ static int clk_rpmh_probe(struct platform_device *pdev) if (!res_addr) { hw_clks[i] = NULL; - if (desc->clka_optional && - !strncmp(rpmh_clk->res_name, "clka", sizeof("clka") - 1)) + if (rpmh_clk->res_addr == CLK_RPMH_VRM_EN_OFFSET) continue; dev_err(&pdev->dev, "missing RPMh resource address for %s\n", diff --git a/drivers/clk/qcom/common.c b/drivers/clk/qcom/common.c index eec369d2173b5..0e8f380873af0 100644 --- a/drivers/clk/qcom/common.c +++ b/drivers/clk/qcom/common.c @@ -169,7 +169,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (!node) { fixed = devm_kzalloc(dev, sizeof(*fixed), GFP_KERNEL); if (!fixed) - return -EINVAL; + return -ENOMEM; fixed->fixed_rate = rate; fixed->hw.init = &init_data; @@ -186,7 +186,7 @@ static int _qcom_cc_register_board_clk(struct device *dev, const char *path, if (add_factor) { factor = devm_kzalloc(dev, sizeof(*factor), GFP_KERNEL); if (!factor) - return -EINVAL; + return -ENOMEM; factor->mult = factor->div = 1; factor->hw.init = &init_data; diff --git a/drivers/clk/qcom/gcc-glymur.c b/drivers/clk/qcom/gcc-glymur.c index eff3248d483ad..7adbe59915f0f 100644 --- a/drivers/clk/qcom/gcc-glymur.c +++ b/drivers/clk/qcom/gcc-glymur.c @@ -3671,21 +3671,6 @@ static struct clk_branch gcc_disp_hf_axi_clk = { }, }; -static struct clk_branch gcc_eva_ahb_clk = { - .halt_reg = 0x9b004, - .halt_check = BRANCH_HALT_VOTED, - .hwcg_reg = 0x9b004, - .hwcg_bit = 1, - .clkr = { - .enable_reg = 0x9b004, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_ahb_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_eva_axi0_clk = { .halt_reg = 0x9b008, .halt_check = BRANCH_HALT_SKIP, @@ -3716,19 +3701,6 @@ static struct clk_branch gcc_eva_axi0c_clk = { }, }; -static struct clk_branch gcc_eva_xo_clk = { - .halt_reg = 0x9b024, - .halt_check = BRANCH_HALT, - .clkr = { - .enable_reg = 0x9b024, - .enable_mask = BIT(0), - .hw.init = &(const struct clk_init_data) { - .name = "gcc_eva_xo_clk", - .ops = &clk_branch2_ops, - }, - }, -}; - static struct clk_branch gcc_gp1_clk = { .halt_reg = 0x64000, .halt_check = BRANCH_HALT, @@ -7995,10 +7967,8 @@ static struct clk_regmap *gcc_glymur_clocks[] = { [GCC_CFG_NOC_USB_ANOC_AHB_CLK] = &gcc_cfg_noc_usb_anoc_ahb_clk.clkr, [GCC_CFG_NOC_USB_ANOC_SOUTH_AHB_CLK] = &gcc_cfg_noc_usb_anoc_south_ahb_clk.clkr, [GCC_DISP_HF_AXI_CLK] = &gcc_disp_hf_axi_clk.clkr, - [GCC_EVA_AHB_CLK] = &gcc_eva_ahb_clk.clkr, [GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr, [GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr, - [GCC_EVA_XO_CLK] = &gcc_eva_xo_clk.clkr, [GCC_GP1_CLK] = &gcc_gp1_clk.clkr, [GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr, [GCC_GP2_CLK] = &gcc_gp2_clk.clkr, @@ -8547,6 +8517,8 @@ static u32 gcc_glymur_critical_cbcrs[] = { 0x71004, /* GCC_GPU_CFG_AHB_CLK */ 0x32004, /* GCC_VIDEO_AHB_CLK */ 0x32058, /* GCC_VIDEO_XO_CLK */ + 0x9b004, /* GCC_EVA_AHB_CLK */ + 0x9b024, /* GCC_EVA_XO_CLK */ }; static const struct regmap_config gcc_glymur_regmap_config = { @@ -8579,6 +8551,7 @@ static const struct qcom_cc_desc gcc_glymur_desc = { .num_resets = ARRAY_SIZE(gcc_glymur_resets), .gdscs = gcc_glymur_gdscs, .num_gdscs = ARRAY_SIZE(gcc_glymur_gdscs), + .use_rpm = true, .driver_data = &gcc_glymur_driver_data, }; diff --git a/drivers/clk/qcom/gcc-mdm9607.c b/drivers/clk/qcom/gcc-mdm9607.c index 07f1b78d737a1..4b749c99586b7 100644 --- a/drivers/clk/qcom/gcc-mdm9607.c +++ b/drivers/clk/qcom/gcc-mdm9607.c @@ -26,7 +26,6 @@ enum { P_XO, - P_BIMC, P_GPLL0, P_GPLL1, P_GPLL2, @@ -121,7 +120,7 @@ static struct clk_alpha_pll gpll2_early = { .regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_DEFAULT], .clkr = { .enable_reg = 0x45000, - .enable_mask = BIT(3), /* Yeah, apparently it's not 2 */ + .enable_mask = BIT(3), /* BIT(2) is used for BIMC PLL */ .hw.init = &(struct clk_init_data) { .name = "gpll2_early", @@ -158,20 +157,6 @@ static const struct clk_parent_data gcc_xo_gpll0_gpll2[] = { { .hw = &gpll2.clkr.hw }, }; -static const struct parent_map gcc_xo_gpll0_gpll1_gpll2_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_GPLL1, 2 }, - { P_GPLL2, 3 }, -}; - -static const struct clk_parent_data gcc_xo_gpll0_gpll1_gpll2[] = { - { .fw_name = "xo" }, - { .hw = &gpll0.clkr.hw }, - { .hw = &gpll1_vote.hw }, - { .hw = &gpll2.clkr.hw }, -}; - static const struct freq_tbl ftbl_apss_ahb_clk[] = { F(19200000, P_XO, 1, 0, 0), F(50000000, P_GPLL0, 16, 0, 0), @@ -192,80 +177,6 @@ static struct clk_rcg2 apss_ahb_clk_src = { }, }; -static struct clk_pll bimc_pll = { - .l_reg = 0x23004, - .m_reg = 0x23008, - .n_reg = 0x2300c, - .config_reg = 0x23010, - .mode_reg = 0x23000, - .status_reg = 0x2301c, - .status_bit = 17, - .clkr.hw.init = &(struct clk_init_data){ - .name = "bimc_pll", - .parent_data = &(const struct clk_parent_data){ - .fw_name = "xo", - }, - .num_parents = 1, - .ops = &clk_pll_ops, - }, -}; - -static struct clk_regmap bimc_pll_vote = { - .enable_reg = 0x45000, - .enable_mask = BIT(3), - .hw.init = &(struct clk_init_data){ - .name = "bimc_pll_vote", - .parent_hws = (const struct clk_hw *[]){ &bimc_pll.clkr.hw }, - .num_parents = 1, - .ops = &clk_pll_vote_ops, - }, -}; - -static const struct parent_map gcc_xo_gpll0_bimc_map[] = { - { P_XO, 0 }, - { P_GPLL0, 1 }, - { P_BIMC, 2 }, -}; - -static const struct clk_parent_data gcc_xo_gpll0_bimc[] = { - { .fw_name = "xo" }, - { .hw = &gpll0.clkr.hw }, - { .hw = &bimc_pll_vote.hw }, -}; - -static const struct freq_tbl ftbl_pcnoc_bfdcd_clk_src[] = { - F(19200000, P_XO, 1, 0, 0), - F(50000000, P_GPLL0, 16, 0, 0), - F(100000000, P_GPLL0, 8, 0, 0), - { } -}; - -static struct clk_rcg2 pcnoc_bfdcd_clk_src = { - .cmd_rcgr = 0x27000, - .freq_tbl = ftbl_pcnoc_bfdcd_clk_src, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "pcnoc_bfdcd_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), - .ops = &clk_rcg2_ops, - .flags = CLK_IS_CRITICAL, - }, -}; - -static struct clk_rcg2 system_noc_bfdcd_clk_src = { - .cmd_rcgr = 0x26004, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "system_noc_bfdcd_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = ARRAY_SIZE(gcc_xo_gpll0_bimc), - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_blsp1_qup1_6_i2c_apps_clk[] = { F(19200000, P_XO, 1, 0, 0), F(50000000, P_GPLL0, 16, 0, 0), @@ -674,26 +585,6 @@ static struct clk_rcg2 sdcc2_apps_clk_src = { }, }; -static const struct freq_tbl ftbl_gcc_apss_tcu_clk[] = { - F(155000000, P_GPLL2, 6, 0, 0), - F(310000000, P_GPLL2, 3, 0, 0), - F(400000000, P_GPLL0, 2, 0, 0), - { } -}; - -static struct clk_rcg2 apss_tcu_clk_src = { - .cmd_rcgr = 0x1207c, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_gpll1_gpll2_map, - .freq_tbl = ftbl_gcc_apss_tcu_clk, - .clkr.hw.init = &(struct clk_init_data){ - .name = "apss_tcu_clk_src", - .parent_data = gcc_xo_gpll0_gpll1_gpll2, - .num_parents = 4, - .ops = &clk_rcg2_ops, - }, -}; - static const struct freq_tbl ftbl_gcc_usb_hs_system_clk[] = { F(19200000, P_XO, 1, 0, 0), F(57140000, P_GPLL0, 14, 0, 0), @@ -781,8 +672,6 @@ static struct clk_branch gcc_blsp1_ahb_clk = { .enable_mask = BIT(10), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -790,9 +679,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x1004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x1004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .parent_data = &(const struct clk_parent_data){ @@ -1083,8 +973,6 @@ static struct clk_branch gcc_boot_rom_ahb_clk = { .enable_mask = BIT(7), .hw.init = &(struct clk_init_data){ .name = "gcc_boot_rom_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1098,9 +986,6 @@ static struct clk_branch gcc_crypto_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1114,9 +999,6 @@ static struct clk_branch gcc_crypto_axi_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_crypto_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1190,9 +1072,6 @@ static struct clk_branch gcc_mss_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_cfg_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1220,9 +1099,6 @@ static struct clk_branch gcc_pdm_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_pdm_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1236,9 +1112,6 @@ static struct clk_branch gcc_prng_ahb_clk = { .enable_mask = BIT(8), .hw.init = &(struct clk_init_data){ .name = "gcc_prng_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1251,9 +1124,6 @@ static struct clk_branch gcc_sdcc1_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc1_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1281,9 +1151,6 @@ static struct clk_branch gcc_sdcc2_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_sdcc2_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1304,19 +1171,6 @@ static struct clk_branch gcc_sdcc2_apps_clk = { }, }; -static struct clk_rcg2 bimc_ddr_clk_src = { - .cmd_rcgr = 0x32004, - .hid_width = 5, - .parent_map = gcc_xo_gpll0_bimc_map, - .clkr.hw.init = &(struct clk_init_data){ - .name = "bimc_ddr_clk_src", - .parent_data = gcc_xo_gpll0_bimc, - .num_parents = 3, - .ops = &clk_rcg2_ops, - .flags = CLK_GET_RATE_NOCACHE, - }, -}; - static struct clk_branch gcc_mss_q6_bimc_axi_clk = { .halt_reg = 0x49004, .clkr = { @@ -1324,9 +1178,6 @@ static struct clk_branch gcc_mss_q6_bimc_axi_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_mss_q6_bimc_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &bimc_ddr_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1340,8 +1191,6 @@ static struct clk_branch gcc_apss_tcu_clk = { .enable_mask = BIT(1), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_tcu_clk", - .parent_hws = (const struct clk_hw *[]){ &bimc_ddr_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1355,9 +1204,6 @@ static struct clk_branch gcc_smmu_cfg_clk = { .enable_mask = BIT(12), .hw.init = &(struct clk_init_data){ .name = "gcc_smmu_cfg_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1405,9 +1251,6 @@ static struct clk_branch gcc_usb_hs_phy_cfg_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_phy_cfg_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1420,9 +1263,6 @@ static struct clk_branch gcc_usb_hs_ahb_clk = { .enable_mask = BIT(0), .hw.init = &(struct clk_init_data){ .name = "gcc_usb_hs_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, - .flags = CLK_SET_RATE_PARENT, .ops = &clk_branch2_ops, }, }, @@ -1451,23 +1291,19 @@ static struct clk_branch gcc_apss_ahb_clk = { .enable_mask = BIT(14), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_ahb_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, }; static struct clk_branch gcc_apss_axi_clk = { - .halt_reg = 0x4601c, + .halt_reg = 0x46020, .halt_check = BRANCH_HALT_VOTED, .clkr = { .enable_reg = 0x45004, .enable_mask = BIT(13), .hw.init = &(struct clk_init_data){ .name = "gcc_apss_axi_clk", - .parent_hws = (const struct clk_hw *[]){ &pcnoc_bfdcd_clk_src.clkr.hw }, - .num_parents = 1, .ops = &clk_branch2_ops, }, }, @@ -1480,11 +1316,6 @@ static struct clk_regmap *gcc_mdm9607_clocks[] = { [GPLL1_VOTE] = &gpll1_vote, [GPLL2] = &gpll2.clkr, [GPLL2_EARLY] = &gpll2_early.clkr, - [BIMC_PLL] = &bimc_pll.clkr, - [BIMC_PLL_VOTE] = &bimc_pll_vote, - [BIMC_DDR_CLK_SRC] = &bimc_ddr_clk_src.clkr, - [PCNOC_BFDCD_CLK_SRC] = &pcnoc_bfdcd_clk_src.clkr, - [SYSTEM_NOC_BFDCD_CLK_SRC] = &system_noc_bfdcd_clk_src.clkr, [APSS_AHB_CLK_SRC] = &apss_ahb_clk_src.clkr, [BLSP1_QUP1_I2C_APPS_CLK_SRC] = &blsp1_qup1_i2c_apps_clk_src.clkr, [BLSP1_QUP1_SPI_APPS_CLK_SRC] = &blsp1_qup1_spi_apps_clk_src.clkr, @@ -1511,7 +1342,6 @@ static struct clk_regmap *gcc_mdm9607_clocks[] = { [PDM2_CLK_SRC] = &pdm2_clk_src.clkr, [SDCC1_APPS_CLK_SRC] = &sdcc1_apps_clk_src.clkr, [SDCC2_APPS_CLK_SRC] = &sdcc2_apps_clk_src.clkr, - [APSS_TCU_CLK_SRC] = &apss_tcu_clk_src.clkr, [USB_HS_SYSTEM_CLK_SRC] = &usb_hs_system_clk_src.clkr, [GCC_BLSP1_AHB_CLK] = &gcc_blsp1_ahb_clk.clkr, [GCC_BLSP1_SLEEP_CLK] = &gcc_blsp1_sleep_clk.clkr, @@ -1567,7 +1397,7 @@ static const struct qcom_reset_map gcc_mdm9607_resets[] = { [USB_HS_HSIC_BCR] = { 0x3d05c }, [GCC_MSS_RESTART] = { 0x3e000 }, [USB_HS_BCR] = { 0x41000 }, - [USB2_HS_PHY_ONLY_BCR] = { 0x41034 }, + [USB2_HS_PHY_ONLY_BCR] = { .reg = 0x41034, .udelay = 15 }, [QUSB2_PHY_BCR] = { 0x4103c }, }; diff --git a/drivers/clk/qcom/gcc-msm8916.c b/drivers/clk/qcom/gcc-msm8916.c index 9c7c6b23ce327..e91406e6f5273 100644 --- a/drivers/clk/qcom/gcc-msm8916.c +++ b/drivers/clk/qcom/gcc-msm8916.c @@ -1589,9 +1589,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x01004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x01004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .parent_data = &(const struct clk_parent_data){ diff --git a/drivers/clk/qcom/gcc-msm8939.c b/drivers/clk/qcom/gcc-msm8939.c index 45193b3d714ba..ffd7f14fcbaf8 100644 --- a/drivers/clk/qcom/gcc-msm8939.c +++ b/drivers/clk/qcom/gcc-msm8939.c @@ -1929,9 +1929,10 @@ static struct clk_branch gcc_blsp1_ahb_clk = { static struct clk_branch gcc_blsp1_sleep_clk = { .halt_reg = 0x01004, + .halt_check = BRANCH_HALT_VOTED, .clkr = { - .enable_reg = 0x01004, - .enable_mask = BIT(0), + .enable_reg = 0x45004, + .enable_mask = BIT(9), .hw.init = &(struct clk_init_data){ .name = "gcc_blsp1_sleep_clk", .ops = &clk_branch2_ops, diff --git a/drivers/clk/qcom/gcc-qcm2290.c b/drivers/clk/qcom/gcc-qcm2290.c index 6684cab63ae11..77cff0e9af93b 100644 --- a/drivers/clk/qcom/gcc-qcm2290.c +++ b/drivers/clk/qcom/gcc-qcm2290.c @@ -116,7 +116,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll10 = { @@ -148,7 +148,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll11 = { @@ -309,7 +309,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055B, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, }; static struct clk_alpha_pll gpll8 = { @@ -1082,7 +1082,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s0_clk_src_init = { .name = "gcc_qupv3_wrap0_s0_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s0_clk_src = { @@ -1098,7 +1098,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s1_clk_src_init = { .name = "gcc_qupv3_wrap0_s1_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s1_clk_src = { @@ -1114,7 +1114,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s2_clk_src_init = { .name = "gcc_qupv3_wrap0_s2_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s2_clk_src = { @@ -1130,7 +1130,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s3_clk_src_init = { .name = "gcc_qupv3_wrap0_s3_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s3_clk_src = { @@ -1146,7 +1146,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s4_clk_src_init = { .name = "gcc_qupv3_wrap0_s4_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s4_clk_src = { @@ -1162,7 +1162,7 @@ static struct clk_init_data gcc_qupv3_wrap0_s5_clk_src_init = { .name = "gcc_qupv3_wrap0_s5_clk_src", .parent_data = gcc_parents_1, .num_parents = ARRAY_SIZE(gcc_parents_1), - .ops = &clk_rcg2_shared_ops, + .ops = &clk_rcg2_shared_no_init_park_ops, }; static struct clk_rcg2 gcc_qupv3_wrap0_s5_clk_src = { diff --git a/drivers/clk/qcom/gcc-qcs8300.c b/drivers/clk/qcom/gcc-qcs8300.c index 80831c7dea3bc..fe0632687f9f0 100644 --- a/drivers/clk/qcom/gcc-qcs8300.c +++ b/drivers/clk/qcom/gcc-qcs8300.c @@ -3268,7 +3268,7 @@ static struct gdsc gcc_pcie_0_gdsc = { .pd = { .name = "gcc_pcie_0_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3282,7 +3282,7 @@ static struct gdsc gcc_pcie_1_gdsc = { .pd = { .name = "gcc_pcie_1_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = VOTABLE | RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3306,7 +3306,7 @@ static struct gdsc gcc_usb20_prim_gdsc = { .pd = { .name = "gcc_usb20_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; @@ -3318,7 +3318,7 @@ static struct gdsc gcc_usb30_prim_gdsc = { .pd = { .name = "gcc_usb30_prim_gdsc", }, - .pwrsts = PWRSTS_OFF_ON, + .pwrsts = PWRSTS_RET_ON, .flags = RETAIN_FF_ENABLE | POLL_CFG_GDSCR, }; diff --git a/drivers/clk/qcom/gcc-sm6115.c b/drivers/clk/qcom/gcc-sm6115.c index 4c3804701e243..4bd2ebc158946 100644 --- a/drivers/clk/qcom/gcc-sm6115.c +++ b/drivers/clk/qcom/gcc-sm6115.c @@ -120,7 +120,7 @@ static const struct alpha_pll_config gpll10_config = { .vco_mask = GENMASK(21, 20), .main_output_mask = BIT(0), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -173,7 +173,7 @@ static const struct alpha_pll_config gpll11_config = { .vco_val = 0x2 << 20, .vco_mask = GENMASK(21, 20), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; @@ -367,7 +367,7 @@ static const struct alpha_pll_config gpll8_config = { .post_div_val = 0x1 << 8, .post_div_mask = GENMASK(11, 8), .config_ctl_val = 0x4001055b, - .test_ctl_hi1_val = 0x1, + .test_ctl_hi_val = 0x1, .test_ctl_hi_mask = 0x1, }; diff --git a/drivers/clk/qcom/gdsc.c b/drivers/clk/qcom/gdsc.c index 95aa071202455..71826ccbd9bd1 100644 --- a/drivers/clk/qcom/gdsc.c +++ b/drivers/clk/qcom/gdsc.c @@ -103,14 +103,21 @@ static int gdsc_hwctrl(struct gdsc *sc, bool en) static int gdsc_poll_status(struct gdsc *sc, enum gdsc_status status) { ktime_t start; + int ret; start = ktime_get(); do { - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; } while (ktime_us_delta(ktime_get(), start) < STATUS_POLL_TIMEOUT_US); - if (gdsc_check_status(sc, status)) + ret = gdsc_check_status(sc, status); + if (ret < 0) + return ret; + if (ret) return 0; return -ETIMEDOUT; @@ -474,7 +481,9 @@ static int gdsc_init(struct gdsc *sc) } else if (sc->flags & ALWAYS_ON) { /* If ALWAYS_ON GDSCs are not ON, turn them ON */ - gdsc_enable(&sc->pd); + ret = gdsc_enable(&sc->pd); + if (ret) + return ret; on = true; } @@ -636,10 +645,18 @@ int gdsc_register(struct gdsc_desc *desc, void gdsc_unregister(struct gdsc_desc *desc) { struct device *dev = desc->dev; + struct gdsc **scs = desc->scs; size_t num = desc->num; + int i; - gdsc_pm_subdomain_remove(desc, num); of_genpd_del_provider(dev->of_node); + gdsc_pm_subdomain_remove(desc, num); + + for (i = 0; i < num; i++) { + if (!scs[i]) + continue; + pm_genpd_remove(&scs[i]->pd); + } } /* diff --git a/drivers/clk/qcom/gpucc-qcm2290.c b/drivers/clk/qcom/gpucc-qcm2290.c index dc369dff882e6..3ccab8f3b4e0f 100644 --- a/drivers/clk/qcom/gpucc-qcm2290.c +++ b/drivers/clk/qcom/gpucc-qcm2290.c @@ -144,7 +144,7 @@ static struct clk_rcg2 gpu_cc_gx_gfx3d_clk_src = { .parent_data = gpu_cc_parent_data_1, .num_parents = ARRAY_SIZE(gpu_cc_parent_data_1), .flags = CLK_SET_RATE_PARENT, - .ops = &clk_rcg2_ops, + .ops = &clk_rcg2_shared_ops, }, }; diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c index a0a68ec0490f7..87ede1ee64d7a 100644 --- a/drivers/clk/renesas/renesas-cpg-mssr.c +++ b/drivers/clk/renesas/renesas-cpg-mssr.c @@ -349,6 +349,9 @@ struct clk *cpg_mssr_clk_src_twocell_get(struct of_phandle_args *clkspec, struct clk *clk; int range_check; + if (clkspec->args_count != 2) + return ERR_PTR(-EINVAL); + switch (clkspec->args[0]) { case CPG_CORE: type = "core"; diff --git a/drivers/clk/rockchip/clk-pll.c b/drivers/clk/rockchip/clk-pll.c index 86dba3826a77e..0dada00b375fa 100644 --- a/drivers/clk/rockchip/clk-pll.c +++ b/drivers/clk/rockchip/clk-pll.c @@ -900,6 +900,13 @@ static void rockchip_rk3588_pll_get_params(struct rockchip_clk_pll *pll, rate->k = ((pllcon >> RK3588_PLLCON2_K_SHIFT) & RK3588_PLLCON2_K_MASK); } +/* + * 2250 MHz <= Fvco <= 4500 MHz + * For Fvco > 3 GHz: period jitter +-1% frac PLL, +-0.75% int PLL + * For Fvco < 3 GHz: period jitter +-2% frac PLL, +-1.50% int PLL + * Fvco = ((m + k / 65536) * Fin) / p + * Fout = ((m + k / 65536) * Fin) / (p * 2^s) + */ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned long prate) { struct rockchip_clk_pll *pll = to_rockchip_clk_pll(hw); @@ -915,7 +922,7 @@ static unsigned long rockchip_rk3588_pll_recalc_rate(struct clk_hw *hw, unsigned /* fractional mode */ u64 frac_rate64 = prate * cur.k; - postdiv = cur.p * 65535; + postdiv = cur.p * 65536; do_div(frac_rate64, postdiv); rate64 += frac_rate64; } diff --git a/drivers/clk/rockchip/clk-rk3288.c b/drivers/clk/rockchip/clk-rk3288.c index 0a1e017df7c66..9cf3e1e43b781 100644 --- a/drivers/clk/rockchip/clk-rk3288.c +++ b/drivers/clk/rockchip/clk-rk3288.c @@ -871,7 +871,7 @@ static const int rk3288_saved_cru_reg_ids[] = { static u32 rk3288_saved_cru_regs[ARRAY_SIZE(rk3288_saved_cru_reg_ids)]; -static int rk3288_clk_suspend(void) +static int rk3288_clk_suspend(void *data) { int i, reg_id; @@ -906,7 +906,7 @@ static int rk3288_clk_suspend(void) return 0; } -static void rk3288_clk_resume(void) +static void rk3288_clk_resume(void *data) { int i, reg_id; @@ -923,11 +923,15 @@ static void rk3288_clk_shutdown(void) writel_relaxed(0xf3030000, rk3288_cru_base + RK3288_MODE_CON); } -static struct syscore_ops rk3288_clk_syscore_ops = { +static const struct syscore_ops rk3288_clk_syscore_ops = { .suspend = rk3288_clk_suspend, .resume = rk3288_clk_resume, }; +static struct syscore rk3288_clk_syscore = { + .ops = &rk3288_clk_syscore_ops, +}; + static void __init rk3288_common_init(struct device_node *np, enum rk3288_variant soc) { @@ -976,7 +980,7 @@ static void __init rk3288_common_init(struct device_node *np, rockchip_register_restart_notifier(ctx, RK3288_GLB_SRST_FST, rk3288_clk_shutdown); - register_syscore_ops(&rk3288_clk_syscore_ops); + register_syscore(&rk3288_clk_syscore); rockchip_clk_of_add_provider(np, ctx); } diff --git a/drivers/clk/rockchip/clk-rk3576.c b/drivers/clk/rockchip/clk-rk3576.c index 9bc0ef51ef682..95693e4381430 100644 --- a/drivers/clk/rockchip/clk-rk3576.c +++ b/drivers/clk/rockchip/clk-rk3576.c @@ -315,6 +315,7 @@ PNAME(mux_100m_24m_lclk0_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_0" } PNAME(mux_100m_24m_lclk1_p) = { "clk_cpll_div10", "xin24m", "lclk_asrc_src_1" }; PNAME(mux_150m_100m_50m_24m_p) = { "clk_gpll_div8", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; PNAME(mux_200m_100m_50m_24m_p) = { "clk_gpll_div6", "clk_cpll_div10", "clk_cpll_div20", "xin24m" }; +PNAME(mux_200m_150m_100m_24m_p) = { "clk_gpll_div6", "clk_gpll_div8", "clk_cpll_div10", "xin24m" }; PNAME(mux_400m_200m_100m_24m_p) = { "clk_gpll_div3", "clk_gpll_div6", "clk_cpll_div10", "xin24m" }; PNAME(mux_500m_250m_100m_24m_p) = { "clk_cpll_div2", "clk_cpll_div4", "clk_cpll_div10", "xin24m" }; PNAME(mux_600m_400m_300m_24m_p) = { "clk_gpll_div2", "clk_gpll_div3", "clk_gpll_div4", "xin24m" }; @@ -706,19 +707,19 @@ static struct rockchip_clk_branch rk3576_clk_branches[] __initdata = { RK3576_CLKGATE_CON(16), 0, GFLAGS), GATE(PCLK_SPI4, "pclk_spi4", "pclk_bus_root", 0, RK3576_CLKGATE_CON(16), 1, GFLAGS), - COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI0, "clk_spi0", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(70), 13, 2, MFLAGS, RK3576_CLKGATE_CON(16), 2, GFLAGS), - COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI1, "clk_spi1", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 0, 2, MFLAGS, RK3576_CLKGATE_CON(16), 3, GFLAGS), - COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI2, "clk_spi2", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 2, 2, MFLAGS, RK3576_CLKGATE_CON(16), 4, GFLAGS), - COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI3, "clk_spi3", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 4, 2, MFLAGS, RK3576_CLKGATE_CON(16), 5, GFLAGS), - COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_100m_50m_24m_p, 0, + COMPOSITE_NODIV(CLK_SPI4, "clk_spi4", mux_200m_150m_100m_24m_p, 0, RK3576_CLKSEL_CON(71), 6, 2, MFLAGS, RK3576_CLKGATE_CON(16), 6, GFLAGS), GATE(PCLK_WDT0, "pclk_wdt0", "pclk_bus_root", 0, diff --git a/drivers/clk/rockchip/clk-rk3588.c b/drivers/clk/rockchip/clk-rk3588.c index 1694223f4f842..bea2deed0e234 100644 --- a/drivers/clk/rockchip/clk-rk3588.c +++ b/drivers/clk/rockchip/clk-rk3588.c @@ -2094,7 +2094,7 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = { COMPOSITE(DCLK_VOP1_SRC, "dclk_vop1_src", gpll_cpll_v0pll_aupll_p, 0, RK3588_CLKSEL_CON(111), 14, 2, MFLAGS, 9, 5, DFLAGS, RK3588_CLKGATE_CON(52), 11, GFLAGS), - COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, CLK_SET_RATE_PARENT | CLK_SET_RATE_NO_REPARENT, + COMPOSITE(DCLK_VOP2_SRC, "dclk_vop2_src", gpll_cpll_v0pll_aupll_p, CLK_SET_RATE_NO_REPARENT, RK3588_CLKSEL_CON(112), 5, 2, MFLAGS, 0, 5, DFLAGS, RK3588_CLKGATE_CON(52), 12, GFLAGS), COMPOSITE_NODIV(DCLK_VOP0, "dclk_vop0", dclk_vop0_p, diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c index 56f27697c76b1..413faf14eded9 100644 --- a/drivers/clk/samsung/clk-exynos850.c +++ b/drivers/clk/samsung/clk-exynos850.c @@ -684,10 +684,11 @@ static const struct samsung_gate_clock apm_gate_clks[] __initconst = { CLK_CON_GAT_GOUT_APM_APBIF_RTC_PCLK, 21, 0, 0), GATE(CLK_GOUT_TOP_RTC_PCLK, "gout_top_rtc_pclk", "dout_apm_bus", CLK_CON_GAT_GOUT_APM_APBIF_TOP_RTC_PCLK, 21, 0, 0), + /* TODO: Should be dealt with or enabled in PMIC ACPM driver */ GATE(CLK_GOUT_I3C_PCLK, "gout_i3c_pclk", "dout_apm_bus", - CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_I3C_SCLK, "gout_i3c_sclk", "mout_apm_i3c", - CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, 21, 0, 0), + CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, 21, CLK_IS_CRITICAL, 0), GATE(CLK_GOUT_SPEEDY_PCLK, "gout_speedy_pclk", "dout_apm_bus", CLK_CON_GAT_GOUT_APM_SPEEDY_APM_PCLK, 21, 0, 0), /* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */ diff --git a/drivers/clk/samsung/clk-exynos990.c b/drivers/clk/samsung/clk-exynos990.c index 6277dd557fab6..4385c3b76dd68 100644 --- a/drivers/clk/samsung/clk-exynos990.c +++ b/drivers/clk/samsung/clk-exynos990.c @@ -1546,54 +1546,44 @@ static const unsigned long peric0_clk_regs[] __initconst = { /* Parent clock list for CMU_PERIC0 muxes */ PNAME(mout_peric0_bus_user_p) = { "oscclk", "dout_cmu_peric0_bus" }; -PNAME(mout_peric0_uart_dbg_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi00_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi01_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi02_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi03_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi04_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi05_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi13_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi14_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi15_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; -PNAME(mout_peric0_usi_i2c_user_p) = { "oscclk", "dout_cmu_peric0_ip" }; +PNAME(mout_peric0_nonbususer_p) = { "oscclk", "dout_cmu_peric0_ip" }; static const struct samsung_mux_clock peric0_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC0_BUS_USER, "mout_peric0_bus_user", mout_peric0_bus_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_BUS_USER, 4, 1), - MUX(CLK_MOUT_PERIC0_UART_DBG, "mout_peric0_uart_dbg", - mout_peric0_uart_dbg_p, PLL_CON0_MUX_CLKCMU_PERIC0_UART_DBG, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI00_USI_USER, "mout_peric0_usi00_usi_user", - mout_peric0_usi00_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI00_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI01_USI_USER, "mout_peric0_usi01_usi_user", - mout_peric0_usi01_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI01_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI02_USI_USER, "mout_peric0_usi02_usi_user", - mout_peric0_usi02_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI02_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI03_USI_USER, "mout_peric0_usi03_usi_user", - mout_peric0_usi03_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI03_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI04_USI_USER, "mout_peric0_usi04_usi_user", - mout_peric0_usi04_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI04_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI05_USI_USER, "mout_peric0_usi05_usi_user", - mout_peric0_usi05_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI05_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI13_USI_USER, "mout_peric0_usi13_usi_user", - mout_peric0_usi13_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI13_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI14_USI_USER, "mout_peric0_usi14_usi_user", - mout_peric0_usi14_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC0_USI15_USI_USER, "mout_peric0_usi15_usi_user", - mout_peric0_usi15_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI15_USI_USER, - 4, 1), + nMUX(CLK_MOUT_PERIC0_UART_DBG, "mout_peric0_uart_dbg", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_UART_DBG, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI00_USI_USER, "mout_peric0_usi00_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI00_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI01_USI_USER, "mout_peric0_usi01_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI01_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI02_USI_USER, "mout_peric0_usi02_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI02_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI03_USI_USER, "mout_peric0_usi03_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI03_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI04_USI_USER, "mout_peric0_usi04_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI04_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI05_USI_USER, "mout_peric0_usi05_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI05_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI13_USI_USER, "mout_peric0_usi13_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI13_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI14_USI_USER, "mout_peric0_usi14_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI14_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC0_USI15_USI_USER, "mout_peric0_usi15_usi_user", + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI15_USI_USER, + 4, 1), MUX(CLK_MOUT_PERIC0_USI_I2C_USER, "mout_peric0_usi_i2c_user", - mout_peric0_usi_i2c_user_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI_I2C_USER, + mout_peric0_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC0_USI_I2C_USER, 4, 1), }; @@ -1602,42 +1592,42 @@ static const struct samsung_div_clock peric0_div_clks[] __initconst = { "mout_peric0_uart_dbg", CLK_CON_DIV_DIV_CLK_PERIC0_UART_DBG, 0, 4), - DIV(CLK_DOUT_PERIC0_USI00_USI, "dout_peric0_usi00_usi", - "mout_peric0_usi00_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI00_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI01_USI, "dout_peric0_usi01_usi", - "mout_peric0_usi01_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI01_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI02_USI, "dout_peric0_usi02_usi", - "mout_peric0_usi02_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI02_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI03_USI, "dout_peric0_usi03_usi", - "mout_peric0_usi03_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI03_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI04_USI, "dout_peric0_usi04_usi", - "mout_peric0_usi04_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI04_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI05_USI, "dout_peric0_usi05_usi", - "mout_peric0_usi05_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI05_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI13_USI, "dout_peric0_usi13_usi", - "mout_peric0_usi13_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI13_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI14_USI, "dout_peric0_usi14_usi", - "mout_peric0_usi14_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, - 0, 4), - DIV(CLK_DOUT_PERIC0_USI15_USI, "dout_peric0_usi15_usi", - "mout_peric0_usi15_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC0_USI15_USI, - 0, 4), + DIV_F(CLK_DOUT_PERIC0_USI00_USI, "dout_peric0_usi00_usi", + "mout_peric0_usi00_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI00_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI01_USI, "dout_peric0_usi01_usi", + "mout_peric0_usi01_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI01_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI02_USI, "dout_peric0_usi02_usi", + "mout_peric0_usi02_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI02_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI03_USI, "dout_peric0_usi03_usi", + "mout_peric0_usi03_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI03_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI04_USI, "dout_peric0_usi04_usi", + "mout_peric0_usi04_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI04_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI05_USI, "dout_peric0_usi05_usi", + "mout_peric0_usi05_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI05_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI13_USI, "dout_peric0_usi13_usi", + "mout_peric0_usi13_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI13_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI14_USI, "dout_peric0_usi14_usi", + "mout_peric0_usi14_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI14_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC0_USI15_USI, "dout_peric0_usi15_usi", + "mout_peric0_usi15_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC0_USI15_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), DIV(CLK_DOUT_PERIC0_USI_I2C, "dout_peric0_usi_i2c", "mout_peric0_usi_i2c_user", CLK_CON_DIV_DIV_CLK_PERIC0_USI_I2C, @@ -2107,58 +2097,47 @@ static const unsigned long peric1_clk_regs[] __initconst = { /* Parent clock list for CMU_PERIC1 muxes */ PNAME(mout_peric1_bus_user_p) = { "oscclk", "dout_cmu_peric1_bus" }; -PNAME(mout_peric1_uart_bt_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi06_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi07_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi08_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi09_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi10_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi11_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi12_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi18_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi16_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi17_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; -PNAME(mout_peric1_usi_i2c_user_p) = { "oscclk", "dout_cmu_peric1_ip" }; +PNAME(mout_peric1_nonbususer_p) = { "oscclk", "dout_cmu_peric1_ip" }; static const struct samsung_mux_clock peric1_mux_clks[] __initconst = { MUX(CLK_MOUT_PERIC1_BUS_USER, "mout_peric1_bus_user", mout_peric1_bus_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_BUS_USER, 4, 1), - MUX(CLK_MOUT_PERIC1_UART_BT_USER, "mout_peric1_uart_bt_user", - mout_peric1_uart_bt_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_UART_BT_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI06_USI_USER, "mout_peric1_usi06_usi_user", - mout_peric1_usi06_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI06_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI07_USI_USER, "mout_peric1_usi07_usi_user", - mout_peric1_usi07_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI07_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI08_USI_USER, "mout_peric1_usi08_usi_user", - mout_peric1_usi08_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI08_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI09_USI_USER, "mout_peric1_usi09_usi_user", - mout_peric1_usi09_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI09_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI10_USI_USER, "mout_peric1_usi10_usi_user", - mout_peric1_usi10_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI11_USI_USER, "mout_peric1_usi11_usi_user", - mout_peric1_usi11_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI12_USI_USER, "mout_peric1_usi12_usi_user", - mout_peric1_usi12_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI18_USI_USER, "mout_peric1_usi18_usi_user", - mout_peric1_usi18_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI18_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI16_USI_USER, "mout_peric1_usi16_usi_user", - mout_peric1_usi16_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI16_USI_USER, - 4, 1), - MUX(CLK_MOUT_PERIC1_USI17_USI_USER, "mout_peric1_usi17_usi_user", - mout_peric1_usi17_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI17_USI_USER, - 4, 1), + nMUX(CLK_MOUT_PERIC1_UART_BT_USER, "mout_peric1_uart_bt_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_UART_BT_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI06_USI_USER, "mout_peric1_usi06_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI06_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI07_USI_USER, "mout_peric1_usi07_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI07_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI08_USI_USER, "mout_peric1_usi08_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI08_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI09_USI_USER, "mout_peric1_usi09_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI09_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI10_USI_USER, "mout_peric1_usi10_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI10_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI11_USI_USER, "mout_peric1_usi11_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI11_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI12_USI_USER, "mout_peric1_usi12_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI12_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI18_USI_USER, "mout_peric1_usi18_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI18_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI16_USI_USER, "mout_peric1_usi16_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI16_USI_USER, + 4, 1), + nMUX(CLK_MOUT_PERIC1_USI17_USI_USER, "mout_peric1_usi17_usi_user", + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI17_USI_USER, + 4, 1), MUX(CLK_MOUT_PERIC1_USI_I2C_USER, "mout_peric1_usi_i2c_user", - mout_peric1_usi_i2c_user_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI_I2C_USER, + mout_peric1_nonbususer_p, PLL_CON0_MUX_CLKCMU_PERIC1_USI_I2C_USER, 4, 1), }; @@ -2167,46 +2146,46 @@ static const struct samsung_div_clock peric1_div_clks[] __initconst = { "mout_peric1_uart_bt_user", CLK_CON_DIV_DIV_CLK_PERIC1_UART_BT, 0, 4), - DIV(CLK_DOUT_PERIC1_USI06_USI, "dout_peric1_usi06_usi", - "mout_peric1_usi06_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI06_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI07_USI, "dout_peric1_usi07_usi", - "mout_peric1_usi07_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI07_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI08_USI, "dout_peric1_usi08_usi", - "mout_peric1_usi08_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI08_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI18_USI, "dout_peric1_usi18_usi", - "mout_peric1_usi18_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI18_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI12_USI, "dout_peric1_usi12_usi", - "mout_peric1_usi12_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI09_USI, "dout_peric1_usi09_usi", - "mout_peric1_usi09_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI09_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI10_USI, "dout_peric1_usi10_usi", - "mout_peric1_usi10_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI11_USI, "dout_peric1_usi11_usi", - "mout_peric1_usi11_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI16_USI, "dout_peric1_usi16_usi", - "mout_peric1_usi16_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI16_USI, - 0, 4), - DIV(CLK_DOUT_PERIC1_USI17_USI, "dout_peric1_usi17_usi", - "mout_peric1_usi17_usi_user", - CLK_CON_DIV_DIV_CLK_PERIC1_USI17_USI, - 0, 4), + DIV_F(CLK_DOUT_PERIC1_USI06_USI, "dout_peric1_usi06_usi", + "mout_peric1_usi06_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI06_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI07_USI, "dout_peric1_usi07_usi", + "mout_peric1_usi07_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI07_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI08_USI, "dout_peric1_usi08_usi", + "mout_peric1_usi08_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI08_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI18_USI, "dout_peric1_usi18_usi", + "mout_peric1_usi18_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI18_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI12_USI, "dout_peric1_usi12_usi", + "mout_peric1_usi12_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI12_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI09_USI, "dout_peric1_usi09_usi", + "mout_peric1_usi09_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI09_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI10_USI, "dout_peric1_usi10_usi", + "mout_peric1_usi10_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI10_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI11_USI, "dout_peric1_usi11_usi", + "mout_peric1_usi11_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI11_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI16_USI, "dout_peric1_usi16_usi", + "mout_peric1_usi16_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI16_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), + DIV_F(CLK_DOUT_PERIC1_USI17_USI, "dout_peric1_usi17_usi", + "mout_peric1_usi17_usi_user", + CLK_CON_DIV_DIV_CLK_PERIC1_USI17_USI, 0, 4, + CLK_SET_RATE_PARENT, 0), DIV(CLK_DOUT_PERIC1_USI_I2C, "dout_peric1_usi_i2c", "mout_peric1_usi_i2c_user", CLK_CON_DIV_DIV_CLK_PERIC1_USI_I2C, diff --git a/drivers/clk/samsung/clk-s5pv210-audss.c b/drivers/clk/samsung/clk-s5pv210-audss.c index b1fd8fac3a4c0..c9fcb23de183e 100644 --- a/drivers/clk/samsung/clk-s5pv210-audss.c +++ b/drivers/clk/samsung/clk-s5pv210-audss.c @@ -36,7 +36,7 @@ static unsigned long reg_save[][2] = { {ASS_CLK_GATE, 0}, }; -static int s5pv210_audss_clk_suspend(void) +static int s5pv210_audss_clk_suspend(void *data) { int i; @@ -46,7 +46,7 @@ static int s5pv210_audss_clk_suspend(void) return 0; } -static void s5pv210_audss_clk_resume(void) +static void s5pv210_audss_clk_resume(void *data) { int i; @@ -54,10 +54,14 @@ static void s5pv210_audss_clk_resume(void) writel(reg_save[i][1], reg_base + reg_save[i][0]); } -static struct syscore_ops s5pv210_audss_clk_syscore_ops = { +static const struct syscore_ops s5pv210_audss_clk_syscore_ops = { .suspend = s5pv210_audss_clk_suspend, .resume = s5pv210_audss_clk_resume, }; + +static struct syscore s5pv210_audss_clk_syscore = { + .ops = &s5pv210_audss_clk_syscore_ops, +}; #endif /* CONFIG_PM_SLEEP */ /* register s5pv210_audss clocks */ @@ -175,7 +179,7 @@ static int s5pv210_audss_clk_probe(struct platform_device *pdev) } #ifdef CONFIG_PM_SLEEP - register_syscore_ops(&s5pv210_audss_clk_syscore_ops); + register_syscore(&s5pv210_audss_clk_syscore); #endif return 0; diff --git a/drivers/clk/samsung/clk.c b/drivers/clk/samsung/clk.c index dbc9925ca8f46..c149ca6c22172 100644 --- a/drivers/clk/samsung/clk.c +++ b/drivers/clk/samsung/clk.c @@ -271,7 +271,7 @@ void __init samsung_clk_of_register_fixed_ext(struct samsung_clk_provider *ctx, } #ifdef CONFIG_PM_SLEEP -static int samsung_clk_suspend(void) +static int samsung_clk_suspend(void *data) { struct samsung_clock_reg_cache *reg_cache; @@ -284,7 +284,7 @@ static int samsung_clk_suspend(void) return 0; } -static void samsung_clk_resume(void) +static void samsung_clk_resume(void *data) { struct samsung_clock_reg_cache *reg_cache; @@ -293,11 +293,15 @@ static void samsung_clk_resume(void) reg_cache->rd_num); } -static struct syscore_ops samsung_clk_syscore_ops = { +static const struct syscore_ops samsung_clk_syscore_ops = { .suspend = samsung_clk_suspend, .resume = samsung_clk_resume, }; +static struct syscore samsung_clk_syscore = { + .ops = &samsung_clk_syscore_ops, +}; + void samsung_clk_extended_sleep_init(void __iomem *reg_base, const unsigned long *rdump, unsigned long nr_rdump, @@ -316,7 +320,7 @@ void samsung_clk_extended_sleep_init(void __iomem *reg_base, panic("could not allocate register dump storage.\n"); if (list_empty(&clock_reg_cache_list)) - register_syscore_ops(&samsung_clk_syscore_ops); + register_syscore(&samsung_clk_syscore); reg_cache->reg_base = reg_base; reg_cache->rd_num = nr_rdump; diff --git a/drivers/clk/socfpga/clk-agilex.c b/drivers/clk/socfpga/clk-agilex.c index 8dd94f64756b9..2bdea1997b5ef 100644 --- a/drivers/clk/socfpga/clk-agilex.c +++ b/drivers/clk/socfpga/clk-agilex.c @@ -259,6 +259,8 @@ static const struct stratix10_perip_cnt_clock agilex_main_perip_cnt_clks[] = { 0, 0x3C, 0, 0, 0}, { AGILEX_NOC_FREE_CLK, "noc_free_clk", NULL, noc_free_mux, ARRAY_SIZE(noc_free_mux), 0, 0x40, 0, 0, 0}, + { AGILEX_L3_MAIN_FREE_CLK, "l3_main_free_clk", "noc_free_clk", NULL, + 1, 0, 0, 1, 0, 0}, { AGILEX_L4_SYS_FREE_CLK, "l4_sys_free_clk", NULL, noc_mux, ARRAY_SIZE(noc_mux), 0, 0, 4, 0x30, 1}, { AGILEX_EMAC_A_FREE_CLK, "emaca_free_clk", NULL, emaca_free_mux, ARRAY_SIZE(emaca_free_mux), diff --git a/drivers/clk/stm32/clk-stm32mp21.c b/drivers/clk/stm32/clk-stm32mp21.c index c8a37b716bd55..bdb17419908c8 100644 --- a/drivers/clk/stm32/clk-stm32mp21.c +++ b/drivers/clk/stm32/clk-stm32mp21.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/stm32/clk-stm32mp25.c b/drivers/clk/stm32/clk-stm32mp25.c index 52f0e8a129262..eb0bc918ecee0 100644 --- a/drivers/clk/stm32/clk-stm32mp25.c +++ b/drivers/clk/stm32/clk-stm32mp25.c @@ -4,6 +4,7 @@ * Author: Gabriel Fernandez for STMicroelectronics. */ +#include #include #include #include diff --git a/drivers/clk/tegra/clk-tegra124-emc.c b/drivers/clk/tegra/clk-tegra124-emc.c index 5f1af6dfe7154..674aef2785394 100644 --- a/drivers/clk/tegra/clk-tegra124-emc.c +++ b/drivers/clk/tegra/clk-tegra124-emc.c @@ -539,6 +539,7 @@ struct clk *tegra124_clk_register_emc(void __iomem *base, struct device_node *np clk = clk_register(NULL, &tegra->hw); if (IS_ERR(clk)) { + of_node_put(tegra->emc_node); kfree(tegra); return clk; } diff --git a/drivers/clk/tegra/clk-tegra210.c b/drivers/clk/tegra/clk-tegra210.c index 412902f573b54..504d0ea997a51 100644 --- a/drivers/clk/tegra/clk-tegra210.c +++ b/drivers/clk/tegra/clk-tegra210.c @@ -3444,7 +3444,7 @@ static void tegra210_disable_cpu_clock(u32 cpu) static u32 spare_reg_ctx, misc_clk_enb_ctx, clk_msk_arm_ctx; static u32 cpu_softrst_ctx[3]; -static int tegra210_clk_suspend(void) +static int tegra210_clk_suspend(void *data) { unsigned int i; @@ -3465,7 +3465,7 @@ static int tegra210_clk_suspend(void) return 0; } -static void tegra210_clk_resume(void) +static void tegra210_clk_resume(void *data) { unsigned int i; @@ -3523,13 +3523,17 @@ static void tegra210_cpu_clock_resume(void) } #endif -static struct syscore_ops tegra_clk_syscore_ops = { +static const struct syscore_ops tegra_clk_syscore_ops = { #ifdef CONFIG_PM_SLEEP .suspend = tegra210_clk_suspend, .resume = tegra210_clk_resume, #endif }; +static struct syscore tegra_clk_syscore = { + .ops = &tegra_clk_syscore_ops, +}; + static struct tegra_cpu_car_ops tegra210_cpu_car_ops = { .wait_for_reset = tegra210_wait_cpu_in_reset, .disable_clock = tegra210_disable_cpu_clock, @@ -3813,6 +3817,6 @@ static void __init tegra210_clock_init(struct device_node *np) tegra_cpu_car_ops = &tegra210_cpu_car_ops; - register_syscore_ops(&tegra_clk_syscore_ops); + register_syscore(&tegra_clk_syscore); } CLK_OF_DECLARE(tegra210, "nvidia,tegra210-car", tegra210_clock_init); diff --git a/drivers/clk/ti/adpll.c b/drivers/clk/ti/adpll.c index e305fcbac6475..8885d28face50 100644 --- a/drivers/clk/ti/adpll.c +++ b/drivers/clk/ti/adpll.c @@ -483,7 +483,7 @@ static const struct clk_ops ti_adpll_ops = { static int ti_adpll_init_dco(struct ti_adpll_data *d) { - struct clk_init_data init; + struct clk_init_data init = {}; struct clk *clock; const char *postfix; int width, err; @@ -576,7 +576,7 @@ static int ti_adpll_init_clkout(struct ti_adpll_data *d, struct clk *clk1) { struct ti_adpll_clkout_data *co; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_ops *ops; const char *parent_names[2]; const char *child_name; diff --git a/drivers/clk/ti/divider.c b/drivers/clk/ti/divider.c index 6f58a0f2e74ae..ea744c343eac8 100644 --- a/drivers/clk/ti/divider.c +++ b/drivers/clk/ti/divider.c @@ -311,7 +311,7 @@ static struct clk *_register_divider(struct device_node *node, u32 flags, struct clk_omap_divider *div) { - struct clk_init_data init; + struct clk_init_data init = {}; const char *parent_name; const char *name; diff --git a/drivers/clk/ti/mux.c b/drivers/clk/ti/mux.c index d6d247ff2be59..ef2a952a0afe5 100644 --- a/drivers/clk/ti/mux.c +++ b/drivers/clk/ti/mux.c @@ -119,14 +119,14 @@ const struct clk_ops ti_clk_mux_ops = { }; static struct clk *_register_mux(struct device_node *node, const char *name, - const char * const *parent_names, + const struct clk_parent_data *parent_data, u8 num_parents, unsigned long flags, struct clk_omap_reg *reg, u8 shift, u32 mask, s8 latch, u8 clk_mux_flags, u32 *table) { + struct clk_init_data init = {}; struct clk_omap_mux *mux; struct clk *clk; - struct clk_init_data init; /* allocate the mux */ mux = kzalloc(sizeof(*mux), GFP_KERNEL); @@ -136,7 +136,7 @@ static struct clk *_register_mux(struct device_node *node, const char *name, init.name = name; init.ops = &ti_clk_mux_ops; init.flags = flags; - init.parent_names = parent_names; + init.parent_data = parent_data; init.num_parents = num_parents; /* struct clk_mux assignments */ @@ -167,24 +167,26 @@ static void of_mux_clk_setup(struct device_node *node) struct clk *clk; struct clk_omap_reg reg; unsigned int num_parents; - const char **parent_names; + struct clk_parent_data *parent_data; const char *name; u8 clk_mux_flags = 0; u32 mask = 0; u32 shift = 0; s32 latch = -EINVAL; u32 flags = CLK_SET_RATE_NO_REPARENT; + int i; num_parents = of_clk_get_parent_count(node); if (num_parents < 2) { pr_err("mux-clock %pOFn must have parents\n", node); return; } - parent_names = kcalloc(num_parents, sizeof(char *), GFP_KERNEL); - if (!parent_names) - goto cleanup; + parent_data = kcalloc(num_parents, sizeof(*parent_data), GFP_KERNEL); + if (!parent_data) + return; - of_clk_parent_fill(node, parent_names, num_parents); + for (i = 0; i < num_parents; i++) + parent_data[i].index = i; if (ti_clk_get_reg_addr(node, 0, ®)) goto cleanup; @@ -207,7 +209,7 @@ static void of_mux_clk_setup(struct device_node *node) mask = (1 << fls(mask)) - 1; name = ti_dt_clk_name(node); - clk = _register_mux(node, name, parent_names, num_parents, + clk = _register_mux(node, name, parent_data, num_parents, flags, ®, shift, mask, latch, clk_mux_flags, NULL); @@ -215,7 +217,7 @@ static void of_mux_clk_setup(struct device_node *node) of_clk_add_provider(node, of_clk_src_simple_get, clk); cleanup: - kfree(parent_names); + kfree(parent_data); } CLK_OF_DECLARE(mux_clk, "ti,mux-clock", of_mux_clk_setup); diff --git a/drivers/clk/ux500/clk-prcmu.c b/drivers/clk/ux500/clk-prcmu.c index f775e18acd468..4e811c73ae51a 100644 --- a/drivers/clk/ux500/clk-prcmu.c +++ b/drivers/clk/ux500/clk-prcmu.c @@ -7,7 +7,7 @@ */ #include -#include +#include #include #include #include @@ -35,13 +35,13 @@ static int clk_prcmu_prepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - return prcmu_request_clock(clk->cg_sel, true); + return db8500_prcmu_request_clock(clk->cg_sel, true); } static void clk_prcmu_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) + if (db8500_prcmu_request_clock(clk->cg_sel, false)) pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); } @@ -86,7 +86,7 @@ static int clk_prcmu_opp_prepare(struct clk_hw *hw) clk->opp_requested = 1; } - err = prcmu_request_clock(clk->cg_sel, true); + err = db8500_prcmu_request_clock(clk->cg_sel, true); if (err) { prcmu_qos_remove_requirement(PRCMU_QOS_APE_OPP, (char *)clk_hw_get_name(hw)); @@ -101,7 +101,7 @@ static void clk_prcmu_opp_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) { + if (db8500_prcmu_request_clock(clk->cg_sel, false)) { pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); return; @@ -120,7 +120,7 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw) struct clk_prcmu *clk = to_clk_prcmu(hw); if (!clk->opp_requested) { - err = prcmu_request_ape_opp_100_voltage(true); + err = db8500_prcmu_request_ape_opp_100_voltage(true); if (err) { pr_err("clk_prcmu: %s fail req APE OPP VOLT for %s.\n", __func__, clk_hw_get_name(hw)); @@ -129,9 +129,9 @@ static int clk_prcmu_opp_volt_prepare(struct clk_hw *hw) clk->opp_requested = 1; } - err = prcmu_request_clock(clk->cg_sel, true); + err = db8500_prcmu_request_clock(clk->cg_sel, true); if (err) { - prcmu_request_ape_opp_100_voltage(false); + db8500_prcmu_request_ape_opp_100_voltage(false); clk->opp_requested = 0; return err; } @@ -143,14 +143,14 @@ static void clk_prcmu_opp_volt_unprepare(struct clk_hw *hw) { struct clk_prcmu *clk = to_clk_prcmu(hw); - if (prcmu_request_clock(clk->cg_sel, false)) { + if (db8500_prcmu_request_clock(clk->cg_sel, false)) { pr_err("clk_prcmu: %s failed to disable %s.\n", __func__, clk_hw_get_name(hw)); return; } if (clk->opp_requested) { - prcmu_request_ape_opp_100_voltage(false); + db8500_prcmu_request_ape_opp_100_voltage(false); clk->opp_requested = 0; } } diff --git a/drivers/clk/ux500/u8500_of_clk.c b/drivers/clk/ux500/u8500_of_clk.c index 8e2f6c65db2a7..512b0610d83b5 100644 --- a/drivers/clk/ux500/u8500_of_clk.c +++ b/drivers/clk/ux500/u8500_of_clk.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include "clk.h" #include "prcc.h" diff --git a/drivers/clk/visconti/clkc.c b/drivers/clk/visconti/clkc.c index d0b193b5d0b35..4018d1298880d 100644 --- a/drivers/clk/visconti/clkc.c +++ b/drivers/clk/visconti/clkc.c @@ -81,9 +81,9 @@ static struct clk_hw *visconti_clk_register_gate(struct device *dev, u8 rs_idx, spinlock_t *lock) { + struct clk_init_data init = {}; struct visconti_clk_gate *gate; struct clk_parent_data *pdata; - struct clk_init_data init; struct clk_hw *hw; int ret; diff --git a/drivers/clk/x86/clk-pmc-atom.c b/drivers/clk/x86/clk-pmc-atom.c index 99291ba65da73..08c83e0abc41d 100644 --- a/drivers/clk/x86/clk-pmc-atom.c +++ b/drivers/clk/x86/clk-pmc-atom.c @@ -160,6 +160,9 @@ static struct clk_plt *plt_clk_register(struct platform_device *pdev, int id, return ERR_PTR(-ENOMEM); init.name = kasprintf(GFP_KERNEL, "%s_%d", PLT_CLK_NAME_BASE, id); + if (!init.name) + return ERR_PTR(-ENOMEM); + init.ops = &plt_clk_ops; init.flags = 0; init.parent_names = parent_names; diff --git a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c index 4a0136349f71a..dbef983eb4252 100644 --- a/drivers/clk/xilinx/clk-xlnx-clock-wizard.c +++ b/drivers/clk/xilinx/clk-xlnx-clock-wizard.c @@ -663,8 +663,8 @@ static int clk_wzrd_determine_rate_all(struct clk_hw *hw, d = divider->d; o = divider->o; - req->rate = div_u64(req->best_parent_rate * (m * 1000 + divider->m_frac), - d * (o * 1000 + divider->o_frac)); + req->rate = mult_frac(req->best_parent_rate, m * 1000 + divider->m_frac, + d * (o * 1000 + divider->o_frac)); return 0; } diff --git a/drivers/clocksource/clps711x-timer.c b/drivers/clocksource/clps711x-timer.c index bbceb0289d457..95f8a50ba1ebb 100644 --- a/drivers/clocksource/clps711x-timer.c +++ b/drivers/clocksource/clps711x-timer.c @@ -94,7 +94,7 @@ static int __init clps711x_timer_init(struct device_node *np) switch (of_alias_get_id(np, "timer")) { case CLPS711X_CLKSRC_CLOCKSOURCE: clps711x_clksrc_init(clock, base); - break; + return 0; case CLPS711X_CLKSRC_CLOCKEVENT: ret = _clps711x_clkevt_init(clock, base, irq); break; diff --git a/drivers/clocksource/timer-armada-370-xp.c b/drivers/clocksource/timer-armada-370-xp.c index 54284c1c06514..fd441ca0b0adf 100644 --- a/drivers/clocksource/timer-armada-370-xp.c +++ b/drivers/clocksource/timer-armada-370-xp.c @@ -207,14 +207,14 @@ static int armada_370_xp_timer_dying_cpu(unsigned int cpu) static u32 timer0_ctrl_reg, timer0_local_ctrl_reg; -static int armada_370_xp_timer_suspend(void) +static int armada_370_xp_timer_suspend(void *data) { timer0_ctrl_reg = readl(timer_base + TIMER_CTRL_OFF); timer0_local_ctrl_reg = readl(local_base + TIMER_CTRL_OFF); return 0; } -static void armada_370_xp_timer_resume(void) +static void armada_370_xp_timer_resume(void *data) { writel(0xffffffff, timer_base + TIMER0_VAL_OFF); writel(0xffffffff, timer_base + TIMER0_RELOAD_OFF); @@ -222,11 +222,15 @@ static void armada_370_xp_timer_resume(void) writel(timer0_local_ctrl_reg, local_base + TIMER_CTRL_OFF); } -static struct syscore_ops armada_370_xp_timer_syscore_ops = { +static const struct syscore_ops armada_370_xp_timer_syscore_ops = { .suspend = armada_370_xp_timer_suspend, .resume = armada_370_xp_timer_resume, }; +static struct syscore armada_370_xp_timer_syscore = { + .ops = &armada_370_xp_timer_syscore_ops, +}; + static unsigned long armada_370_delay_timer_read(void) { return ~readl(timer_base + TIMER0_VAL_OFF); @@ -324,7 +328,7 @@ static int __init armada_370_xp_timer_common_init(struct device_node *np) return res; } - register_syscore_ops(&armada_370_xp_timer_syscore_ops); + register_syscore(&armada_370_xp_timer_syscore); return 0; } @@ -345,7 +349,11 @@ static int __init armada_xp_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk); - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_xp, "marvell,armada-xp-timer", armada_xp_timer_init); @@ -383,7 +391,11 @@ static int __init armada_375_timer_init(struct device_node *np) timer25Mhz = false; } - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_375, "marvell,armada-375-timer", armada_375_timer_init); @@ -406,7 +418,11 @@ static int __init armada_370_timer_init(struct device_node *np) timer_clk = clk_get_rate(clk) / TIMER_DIVIDER; timer25Mhz = false; - return armada_370_xp_timer_common_init(np); + ret = armada_370_xp_timer_common_init(np); + if (ret) + clk_disable_unprepare(clk); + + return ret; } TIMER_OF_DECLARE(armada_370, "marvell,armada-370-timer", armada_370_timer_init); diff --git a/drivers/clocksource/timer-nxp-pit.c b/drivers/clocksource/timer-nxp-pit.c index d1740f18f7180..2cf3b4ca6e1fd 100644 --- a/drivers/clocksource/timer-nxp-pit.c +++ b/drivers/clocksource/timer-nxp-pit.c @@ -328,8 +328,10 @@ static int pit_timer_init(struct device_node *np) if (pit_instances == max_pit_instances) { ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "PIT timer:starting", pit_clockevent_starting_cpu, NULL); - if (ret < 0) + if (ret < 0) { + pit_clockevent_per_cpu_exit(pit, pit_instances); goto out_pit_clocksource_unregister; + } } return 0; diff --git a/drivers/clocksource/timer-sun4i.c b/drivers/clocksource/timer-sun4i.c index 7bdcc60ad43cb..c2d04ab7cf2d3 100644 --- a/drivers/clocksource/timer-sun4i.c +++ b/drivers/clocksource/timer-sun4i.c @@ -208,7 +208,7 @@ static int __init sun4i_timer_init(struct device_node *node) sun4i_timer_clear_interrupt(timer_of_base(&to)); clockevents_config_and_register(&to.clkevt, timer_of_rate(&to), - TIMER_SYNC_TICKS, 0xffffffff); + TIMER_SYNC_TICKS + 1, 0xffffffff); /* Enable timer0 interrupt */ val = readl(timer_of_base(&to) + TIMER_IRQ_EN_REG); diff --git a/drivers/comedi/drivers/comedi_parport.c b/drivers/comedi/drivers/comedi_parport.c index 098738a688fe6..db9f58792ab91 100644 --- a/drivers/comedi/drivers/comedi_parport.c +++ b/drivers/comedi/drivers/comedi_parport.c @@ -211,6 +211,13 @@ static irqreturn_t parport_interrupt(int irq, void *d) unsigned int ctrl; unsigned short val = 0; + /* + * Check device is fully attached. Device interrupts should have + * been disabled, but do this in case of bad hardware. + */ + if (!dev->attached) + return IRQ_NONE; + ctrl = inb(dev->iobase + PARPORT_CTRL_REG); if (!(ctrl & PARPORT_CTRL_IRQ_ENA)) return IRQ_NONE; @@ -231,6 +238,9 @@ static int parport_attach(struct comedi_device *dev, if (ret) return ret; + outb(0, dev->iobase + PARPORT_DATA_REG); + outb(0, dev->iobase + PARPORT_CTRL_REG); + if (it->options[1]) { ret = request_irq(it->options[1], parport_interrupt, 0, dev->board_name, dev); @@ -286,9 +296,6 @@ static int parport_attach(struct comedi_device *dev, s->cancel = parport_intr_cancel; } - outb(0, dev->iobase + PARPORT_DATA_REG); - outb(0, dev->iobase + PARPORT_CTRL_REG); - return 0; } diff --git a/drivers/counter/microchip-tcb-capture.c b/drivers/counter/microchip-tcb-capture.c index 19d457ae4c3bb..e53a8390756b7 100644 --- a/drivers/counter/microchip-tcb-capture.c +++ b/drivers/counter/microchip-tcb-capture.c @@ -483,7 +483,7 @@ static int mchp_tc_probe(struct platform_device *pdev) char clk_name[7]; struct regmap *regmap; struct clk *clk[3]; - int channel; + u32 channel; int ret, i; counter = devm_counter_alloc(&pdev->dev, sizeof(*priv)); @@ -517,7 +517,7 @@ static int mchp_tc_probe(struct platform_device *pdev) priv->channel[i] = channel; - snprintf(clk_name, sizeof(clk_name), "t%d_clk", channel); + snprintf(clk_name, sizeof(clk_name), "t%u_clk", channel); clk[i] = of_clk_get_by_name(np->parent, clk_name); if (IS_ERR(clk[i])) { diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86 index 2c5c228408bf2..027e6ea2e0384 100644 --- a/drivers/cpufreq/Kconfig.x86 +++ b/drivers/cpufreq/Kconfig.x86 @@ -40,6 +40,8 @@ config X86_AMD_PSTATE select ACPI_PROCESSOR select ACPI_CPPC_LIB if X86_64 select CPU_FREQ_GOV_SCHEDUTIL if SMP + select ACPI_PLATFORM_PROFILE + select POWER_SUPPLY help This driver adds a CPUFreq driver which utilizes a fine grain processor performance frequency control range instead of legacy @@ -68,6 +70,18 @@ config X86_AMD_PSTATE_DEFAULT_MODE For details, take a look at: . +config X86_AMD_PSTATE_DYNAMIC_EPP + bool "AMD Processor P-State dynamic EPP support" + depends on X86_AMD_PSTATE + default n + help + Allow the kernel to dynamically change the energy performance + value from events like ACPI platform profile and AC adapter plug + events. + + This feature can also be changed at runtime, this configuration + option only sets the kernel default value behavior. + config X86_AMD_PSTATE_UT tristate "selftest for AMD Processor P-State driver" depends on X86 && ACPI_PROCESSOR diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c index 447b9aa5ce40b..93b80bf992a6a 100644 --- a/drivers/cpufreq/amd-pstate-ut.c +++ b/drivers/cpufreq/amd-pstate-ut.c @@ -273,6 +273,11 @@ static int amd_pstate_ut_check_driver(u32 index) static int __init amd_pstate_ut_init(void) { u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases); + enum amd_pstate_mode mode = amd_pstate_get_status(); + + /* don't test if no running amd-pstate driver */ + if (mode == AMD_PSTATE_UNDEFINED || mode == AMD_PSTATE_DISABLE) + return -EOPNOTSUPP; for (i = 0; i < arr_size; i++) { int ret = amd_pstate_ut_cases[i].func(i); diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index ce6d6b3ff58a3..b1a200531628f 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -86,6 +87,11 @@ static struct cpufreq_driver amd_pstate_driver; static struct cpufreq_driver amd_pstate_epp_driver; static int cppc_state = AMD_PSTATE_UNDEFINED; static bool amd_pstate_prefcore = true; +#ifdef CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP +static bool dynamic_epp = CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP; +#else +static bool dynamic_epp; +#endif static struct quirk_entry *quirks; /* @@ -103,6 +109,7 @@ static struct quirk_entry *quirks; * 2 balance_performance * 3 balance_power * 4 power + * 5 custom (for raw EPP values) */ enum energy_perf_value_index { EPP_INDEX_DEFAULT = 0, @@ -110,6 +117,8 @@ enum energy_perf_value_index { EPP_INDEX_BALANCE_PERFORMANCE, EPP_INDEX_BALANCE_POWERSAVE, EPP_INDEX_POWERSAVE, + EPP_INDEX_CUSTOM, + EPP_INDEX_MAX, }; static const char * const energy_perf_strings[] = { @@ -118,8 +127,9 @@ static const char * const energy_perf_strings[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", [EPP_INDEX_POWERSAVE] = "power", - NULL + [EPP_INDEX_CUSTOM] = "custom", }; +static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); static unsigned int epp_values[] = { [EPP_INDEX_DEFAULT] = 0, @@ -127,7 +137,8 @@ static unsigned int epp_values[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, - }; +}; +static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX - 1); typedef int (*cppc_mode_transition_fn)(int); @@ -183,7 +194,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) { int i; - for (i=0; i < AMD_PSTATE_MAX; i++) { + for (i = 0; i < AMD_PSTATE_MAX; i++) { if (!strncmp(str, amd_pstate_mode_string[i], size)) return i; } @@ -192,7 +203,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) static DEFINE_MUTEX(amd_pstate_driver_lock); -static u8 msr_get_epp(struct amd_cpudata *cpudata) +static int msr_get_epp(struct amd_cpudata *cpudata) { u64 value; int ret; @@ -208,12 +219,12 @@ static u8 msr_get_epp(struct amd_cpudata *cpudata) DEFINE_STATIC_CALL(amd_pstate_get_epp, msr_get_epp); -static inline s16 amd_pstate_get_epp(struct amd_cpudata *cpudata) +static inline int amd_pstate_get_epp(struct amd_cpudata *cpudata) { return static_call(amd_pstate_get_epp)(cpudata); } -static u8 shmem_get_epp(struct amd_cpudata *cpudata) +static int shmem_get_epp(struct amd_cpudata *cpudata) { u64 epp; int ret; @@ -452,9 +463,6 @@ static int shmem_init_perf(struct amd_cpudata *cpudata) WRITE_ONCE(cpudata->perf, perf); WRITE_ONCE(cpudata->prefcore_ranking, cppc_perf.highest_perf); - if (cppc_state == AMD_PSTATE_ACTIVE) - return 0; - ret = cppc_get_auto_sel(cpudata->cpu, &auto_sel); if (ret) { pr_warn("failed to get auto_sel, ret: %d\n", ret); @@ -633,8 +641,31 @@ static void amd_pstate_update_min_max_limit(struct cpufreq_policy *policy) WRITE_ONCE(cpudata->max_limit_freq, policy->max); if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { - perf.min_limit_perf = min(perf.nominal_perf, perf.max_limit_perf); - WRITE_ONCE(cpudata->min_limit_freq, min(cpudata->nominal_freq, cpudata->max_limit_freq)); + u8 min_limit_perf = perf.bios_min_perf ?: perf.nominal_perf; + u32 min_limit_freq; + + /* + * For performance policy, set MinPerf to nominal_perf / bios_min_perf + * rather than highest_perf or lowest_nonlinear_perf. + * + * Per commit 0c411b39e4f4c, using highest_perf was observed + * to cause frequency throttling on power-limited platforms, leading to + * performance regressions. Using lowest_nonlinear_perf would limit + * performance too much for HPC workloads requiring high frequency + * operation and minimal wakeup latency from idle states. + * + * nominal_perf therefore provides a balanced default by avoiding + * throttling while still maintaining enough performance for HPC + * workloads when bios_min_perf is not available. + * + * When bios_min_perf is available, users have profiled their workloads + * to understand the best idling frequency. Use that instead. + */ + min_limit_perf = min(min_limit_perf, perf.max_limit_perf); + min_limit_freq = perf_to_freq(perf, cpudata->nominal_freq, min_limit_perf); + perf.min_limit_perf = min_limit_perf; + + WRITE_ONCE(cpudata->min_limit_freq, min(min_limit_freq, cpudata->max_limit_freq)); } else { perf.min_limit_perf = freq_to_perf(perf, cpudata->nominal_freq, policy->min); WRITE_ONCE(cpudata->min_limit_freq, policy->min); @@ -1063,6 +1094,174 @@ static void amd_pstate_cpu_exit(struct cpufreq_policy *policy) kfree(cpudata); } +static int amd_pstate_get_balanced_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (power_supply_is_system_supplied()) + return cpudata->epp_default_ac; + else + return cpudata->epp_default_dc; +} + +static int amd_pstate_power_supply_notifier(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct amd_cpudata *cpudata = container_of(nb, struct amd_cpudata, power_nb); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + u8 epp; + int ret; + + if (event != PSY_EVENT_PROP_CHANGED) + return NOTIFY_OK; + + /* dynamic actions are only applied while platform profile is in balanced */ + if (cpudata->current_profile != PLATFORM_PROFILE_BALANCED) + return 0; + + if (!policy) + return NOTIFY_OK; + + epp = amd_pstate_get_balanced_epp(policy); + + ret = amd_pstate_set_epp(policy, epp); + if (ret) + pr_warn("Failed to set CPU %d EPP %u: %d\n", cpudata->cpu, epp, ret); + + return NOTIFY_OK; +} + +static int amd_pstate_profile_probe(void *drvdata, unsigned long *choices) +{ + set_bit(PLATFORM_PROFILE_LOW_POWER, choices); + set_bit(PLATFORM_PROFILE_BALANCED, choices); + set_bit(PLATFORM_PROFILE_PERFORMANCE, choices); + + return 0; +} + +static int amd_pstate_profile_get(struct device *dev, + enum platform_profile_option *profile) +{ + struct amd_cpudata *cpudata = dev_get_drvdata(dev); + + *profile = cpudata->current_profile; + + return 0; +} + +static int amd_pstate_profile_set(struct device *dev, + enum platform_profile_option profile) +{ + struct amd_cpudata *cpudata = dev_get_drvdata(dev); + struct cpufreq_policy *policy __free(put_cpufreq_policy) = cpufreq_cpu_get(cpudata->cpu); + int ret; + + if (!policy) + return -ENODEV; + + switch (profile) { + case PLATFORM_PROFILE_LOW_POWER: + ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_POWERSAVE); + if (ret) + return ret; + break; + case PLATFORM_PROFILE_BALANCED: + ret = amd_pstate_set_epp(policy, + amd_pstate_get_balanced_epp(policy)); + if (ret) + return ret; + break; + case PLATFORM_PROFILE_PERFORMANCE: + ret = amd_pstate_set_epp(policy, AMD_CPPC_EPP_PERFORMANCE); + if (ret) + return ret; + break; + default: + pr_err("Unknown Platform Profile %d\n", profile); + return -EOPNOTSUPP; + } + + cpudata->current_profile = profile; + + return 0; +} + +static const struct platform_profile_ops amd_pstate_profile_ops = { + .probe = amd_pstate_profile_probe, + .profile_set = amd_pstate_profile_set, + .profile_get = amd_pstate_profile_get, +}; + +static void amd_pstate_clear_dynamic_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + + if (cpudata->power_nb.notifier_call) + power_supply_unreg_notifier(&cpudata->power_nb); + if (cpudata->ppdev) { + platform_profile_remove(cpudata->ppdev); + cpudata->ppdev = NULL; + } + kfree(cpudata->profile_name); + cpudata->dynamic_epp = false; +} + +static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy) +{ + struct amd_cpudata *cpudata = policy->driver_data; + int ret; + u8 epp; + + switch (cpudata->current_profile) { + case PLATFORM_PROFILE_PERFORMANCE: + epp = AMD_CPPC_EPP_PERFORMANCE; + break; + case PLATFORM_PROFILE_LOW_POWER: + epp = AMD_CPPC_EPP_POWERSAVE; + break; + case PLATFORM_PROFILE_BALANCED: + epp = amd_pstate_get_balanced_epp(policy); + break; + default: + pr_err("Unknown Platform Profile %d\n", cpudata->current_profile); + return -EOPNOTSUPP; + } + ret = amd_pstate_set_epp(policy, epp); + if (ret) + return ret; + + cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu); + if (!cpudata->profile_name) + return -ENOMEM; + + cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu), + cpudata->profile_name, + policy->driver_data, + &amd_pstate_profile_ops); + if (IS_ERR(cpudata->ppdev)) { + ret = PTR_ERR(cpudata->ppdev); + goto cleanup; + } + + /* only enable notifier if things will actually change */ + if (cpudata->epp_default_ac != cpudata->epp_default_dc) { + cpudata->power_nb.notifier_call = amd_pstate_power_supply_notifier; + ret = power_supply_reg_notifier(&cpudata->power_nb); + if (ret) + goto cleanup; + } + + cpudata->dynamic_epp = true; + + return 0; + +cleanup: + amd_pstate_clear_dynamic_epp(policy); + + return ret; +} + /* Sysfs attributes */ /* @@ -1135,61 +1334,74 @@ static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { - int i = 0; - int offset = 0; + int offset = 0, i; struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) return sysfs_emit_at(buf, offset, "%s\n", energy_perf_strings[EPP_INDEX_PERFORMANCE]); - while (energy_perf_strings[i] != NULL) - offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); + for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) + offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); offset += sysfs_emit_at(buf, offset, "\n"); return offset; } -static ssize_t store_energy_performance_preference( - struct cpufreq_policy *policy, const char *buf, size_t count) +static ssize_t store_energy_performance_preference(struct cpufreq_policy *policy, + const char *buf, size_t count) { struct amd_cpudata *cpudata = policy->driver_data; - char str_preference[21]; ssize_t ret; + bool raw_epp = false; u8 epp; - ret = sscanf(buf, "%20s", str_preference); - if (ret != 1) - return -EINVAL; - - ret = match_string(energy_perf_strings, -1, str_preference); - if (ret < 0) - return -EINVAL; + if (cpudata->dynamic_epp) { + pr_debug("EPP cannot be set when dynamic EPP is enabled\n"); + return -EBUSY; + } - if (!ret) - epp = cpudata->epp_default; - else - epp = epp_values[ret]; + /* + * if the value matches a number, use that, otherwise see if + * matches an index in the energy_perf_strings array + */ + ret = kstrtou8(buf, 0, &epp); + raw_epp = !ret; + if (ret) { + ret = sysfs_match_string(energy_perf_strings, buf); + if (ret < 0 || ret == EPP_INDEX_CUSTOM) + return -EINVAL; + if (ret) + epp = epp_values[ret]; + else + epp = cpudata->epp_default_dc; + } - if (epp > 0 && policy->policy == CPUFREQ_POLICY_PERFORMANCE) { + if (epp > 0 && cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) { pr_debug("EPP cannot be set under performance policy\n"); return -EBUSY; } ret = amd_pstate_set_epp(policy, epp); + if (ret) + return ret; - return ret ? ret : count; + cpudata->raw_epp = raw_epp; + + return count; } -static ssize_t show_energy_performance_preference( - struct cpufreq_policy *policy, char *buf) +static ssize_t show_energy_performance_preference(struct cpufreq_policy *policy, char *buf) { struct amd_cpudata *cpudata = policy->driver_data; u8 preference, epp; epp = FIELD_GET(AMD_CPPC_EPP_PERF_MASK, cpudata->cppc_req_cached); + if (cpudata->raw_epp) + return sysfs_emit(buf, "%u\n", epp); + switch (epp) { case AMD_CPPC_EPP_PERFORMANCE: preference = EPP_INDEX_PERFORMANCE; @@ -1210,12 +1422,87 @@ static ssize_t show_energy_performance_preference( return sysfs_emit(buf, "%s\n", energy_perf_strings[preference]); } +cpufreq_freq_attr_ro(amd_pstate_max_freq); +cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); + +cpufreq_freq_attr_ro(amd_pstate_highest_perf); +cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); +cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); +cpufreq_freq_attr_rw(energy_performance_preference); +cpufreq_freq_attr_ro(energy_performance_available_preferences); + +struct freq_attr_visibility { + struct freq_attr *attr; + bool (*visibility_fn)(void); +}; + +/* For attributes which are always visible */ +static bool always_visible(void) +{ + return true; +} + +/* Determines whether prefcore related attributes should be visible */ +static bool prefcore_visibility(void) +{ + return amd_pstate_prefcore; +} + +/* Determines whether energy performance preference should be visible */ +static bool epp_visibility(void) +{ + return cppc_state == AMD_PSTATE_ACTIVE; +} + +static struct freq_attr_visibility amd_pstate_attr_visibility[] = { + {&amd_pstate_max_freq, always_visible}, + {&amd_pstate_lowest_nonlinear_freq, always_visible}, + {&amd_pstate_highest_perf, always_visible}, + {&amd_pstate_prefcore_ranking, prefcore_visibility}, + {&amd_pstate_hw_prefcore, prefcore_visibility}, + {&energy_performance_preference, epp_visibility}, + {&energy_performance_available_preferences, epp_visibility}, +}; + +static struct freq_attr **get_freq_attrs(void) +{ + bool attr_visible[ARRAY_SIZE(amd_pstate_attr_visibility)]; + struct freq_attr **attrs; + int i, j, count; + + for (i = 0, count = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { + struct freq_attr_visibility *v = &amd_pstate_attr_visibility[i]; + + attr_visible[i] = v->visibility_fn(); + if (attr_visible[i]) + count++; + } + + /* amd_pstate_{max_freq, lowest_nonlinear_freq, highest_perf} should always be visible */ + BUG_ON(!count); + + attrs = kcalloc(count + 1, sizeof(struct freq_attr *), GFP_KERNEL); + if (!attrs) + return ERR_PTR(-ENOMEM); + + for (i = 0, j = 0; i < ARRAY_SIZE(amd_pstate_attr_visibility); i++) { + if (!attr_visible[i]) + continue; + + attrs[j++] = amd_pstate_attr_visibility[i].attr; + } + + return attrs; +} + static void amd_pstate_driver_cleanup(void) { if (amd_pstate_prefcore) sched_clear_itmt_support(); cppc_state = AMD_PSTATE_DISABLE; + kfree(current_pstate_driver->attr); + current_pstate_driver->attr = NULL; current_pstate_driver = NULL; } @@ -1240,6 +1527,7 @@ static int amd_pstate_set_driver(int mode_idx) static int amd_pstate_register_driver(int mode) { + struct freq_attr **attr = NULL; int ret; ret = amd_pstate_set_driver(mode); @@ -1248,6 +1536,22 @@ static int amd_pstate_register_driver(int mode) cppc_state = mode; + /* + * Note: It is important to compute the attrs _after_ + * re-initializing the cppc_state. Some attributes become + * visible only when cppc_state is AMD_PSTATE_ACTIVE. + */ + attr = get_freq_attrs(); + if (IS_ERR(attr)) { + ret = (int) PTR_ERR(attr); + pr_err("Couldn't compute freq_attrs for current mode %s [%d]\n", + amd_pstate_get_mode_string(cppc_state), ret); + amd_pstate_driver_cleanup(); + return ret; + } + + current_pstate_driver->attr = attr; + /* at least one CPU supports CPB */ current_pstate_driver->boost_enabled = cpu_feature_enabled(X86_FEATURE_CPB); @@ -1291,6 +1595,8 @@ static int amd_pstate_change_driver_mode(int mode) { int ret; + lockdep_assert_held(&amd_pstate_driver_lock); + ret = amd_pstate_unregister_driver(0); if (ret) return ret; @@ -1390,40 +1696,50 @@ static ssize_t prefcore_show(struct device *dev, return sysfs_emit(buf, "%s\n", str_enabled_disabled(amd_pstate_prefcore)); } -cpufreq_freq_attr_ro(amd_pstate_max_freq); -cpufreq_freq_attr_ro(amd_pstate_lowest_nonlinear_freq); +static ssize_t dynamic_epp_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%s\n", str_enabled_disabled(dynamic_epp)); +} -cpufreq_freq_attr_ro(amd_pstate_highest_perf); -cpufreq_freq_attr_ro(amd_pstate_prefcore_ranking); -cpufreq_freq_attr_ro(amd_pstate_hw_prefcore); -cpufreq_freq_attr_rw(energy_performance_preference); -cpufreq_freq_attr_ro(energy_performance_available_preferences); -static DEVICE_ATTR_RW(status); -static DEVICE_ATTR_RO(prefcore); +static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b, + const char *buf, size_t count) +{ + bool enabled; + int ret; -static struct freq_attr *amd_pstate_attr[] = { - &amd_pstate_max_freq, - &amd_pstate_lowest_nonlinear_freq, - &amd_pstate_highest_perf, - &amd_pstate_prefcore_ranking, - &amd_pstate_hw_prefcore, - NULL, -}; + ret = kstrtobool(buf, &enabled); + if (ret) + return ret; -static struct freq_attr *amd_pstate_epp_attr[] = { - &amd_pstate_max_freq, - &amd_pstate_lowest_nonlinear_freq, - &amd_pstate_highest_perf, - &amd_pstate_prefcore_ranking, - &amd_pstate_hw_prefcore, - &energy_performance_preference, - &energy_performance_available_preferences, - NULL, -}; + guard(mutex)(&amd_pstate_driver_lock); + + if (cppc_state != AMD_PSTATE_ACTIVE) { + pr_debug("dynamic_epp can only be toggled in active mode\n"); + return -EINVAL; + } + + /* Nothing to do */ + if (dynamic_epp == enabled) + return count; + + /* reinitialize with desired dynamic EPP value */ + dynamic_epp = enabled; + ret = amd_pstate_change_driver_mode(cppc_state); + if (ret) + dynamic_epp = false; + + return ret ? ret : count; +} + +static DEVICE_ATTR_RW(status); +static DEVICE_ATTR_RO(prefcore); +static DEVICE_ATTR_RW(dynamic_epp); static struct attribute *pstate_global_attributes[] = { &dev_attr_status.attr, &dev_attr_prefcore.attr, + &dev_attr_dynamic_epp.attr, NULL }; @@ -1457,6 +1773,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) struct amd_cpudata *cpudata; union perf_cached perf; struct device *dev; + int default_epp; int ret; /* @@ -1508,6 +1825,13 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) policy->boost_supported = READ_ONCE(cpudata->boost_supported); + /* Fetch the firmware programmed default EPP value */ + default_epp = amd_pstate_get_epp(cpudata); + if (default_epp < 0) { + ret = default_epp; + goto free_cpudata1; + } + /* * Set the policy to provide a valid fallback value in case * the default cpufreq governor is neither powersave nor performance. @@ -1515,13 +1839,19 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy) if (amd_pstate_acpi_pm_profile_server() || amd_pstate_acpi_pm_profile_undefined()) { policy->policy = CPUFREQ_POLICY_PERFORMANCE; - cpudata->epp_default = amd_pstate_get_epp(cpudata); + cpudata->epp_default_ac = cpudata->epp_default_dc = default_epp; + cpudata->current_profile = PLATFORM_PROFILE_PERFORMANCE; } else { policy->policy = CPUFREQ_POLICY_POWERSAVE; - cpudata->epp_default = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + cpudata->epp_default_ac = AMD_CPPC_EPP_PERFORMANCE; + cpudata->epp_default_dc = AMD_CPPC_EPP_BALANCE_PERFORMANCE; + cpudata->current_profile = PLATFORM_PROFILE_BALANCED; } - ret = amd_pstate_set_epp(policy, cpudata->epp_default); + if (dynamic_epp) + ret = amd_pstate_set_dynamic_epp(policy); + else + ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc); if (ret) goto free_cpudata1; @@ -1542,6 +1872,9 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy) if (cpudata) { union perf_cached perf = READ_ONCE(cpudata->perf); + if (cpudata->dynamic_epp) + amd_pstate_clear_dynamic_epp(policy); + /* Reset CPPC_REQ MSR to the BIOS value */ amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false); @@ -1687,7 +2020,6 @@ static struct cpufreq_driver amd_pstate_driver = { .set_boost = amd_pstate_set_boost, .update_limits = amd_pstate_update_limits, .name = "amd-pstate", - .attr = amd_pstate_attr, }; static struct cpufreq_driver amd_pstate_epp_driver = { @@ -1703,7 +2035,6 @@ static struct cpufreq_driver amd_pstate_epp_driver = { .update_limits = amd_pstate_update_limits, .set_boost = amd_pstate_set_boost, .name = "amd-pstate-epp", - .attr = amd_pstate_epp_attr, }; /* @@ -1849,7 +2180,7 @@ static int __init amd_pstate_init(void) return ret; global_attr_free: - cpufreq_unregister_driver(current_pstate_driver); + amd_pstate_unregister_driver(0); return ret; } device_initcall(amd_pstate_init); diff --git a/drivers/cpufreq/amd-pstate.h b/drivers/cpufreq/amd-pstate.h index 75136d2250c1a..325d935718f65 100644 --- a/drivers/cpufreq/amd-pstate.h +++ b/drivers/cpufreq/amd-pstate.h @@ -9,6 +9,7 @@ #define _LINUX_AMD_PSTATE_H #include +#include /********************************************************************* * AMD P-state INTERFACE * @@ -77,6 +78,11 @@ struct amd_aperf_mperf { * Only when hw_prefcore and early prefcore param are true, * AMD P-State driver supports preferred core featue. * @policy: Cpufreq policy value + * @suspended: If CPU core if offlined + * @epp_default_ac: Default EPP value for AC power source + * @epp_default_dc: Default EPP value for DC power source + * @dynamic_epp: Whether dynamic EPP is enabled + * @power_nb: Notifier block for power events * * The amd_cpudata is key private data for each CPU thread in AMD P-State, and * represents all the attributes and goals that AMD P-State requests at runtime. @@ -105,7 +111,16 @@ struct amd_cpudata { /* EPP feature related attributes*/ u32 policy; bool suspended; - u8 epp_default; + u8 epp_default_ac; + u8 epp_default_dc; + bool dynamic_epp; + bool raw_epp; + struct notifier_block power_nb; + + /* platform profile */ + enum platform_profile_option current_profile; + struct device *ppdev; + char *profile_name; }; /* diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c index b1d29b7af2326..9474988a2a15f 100644 --- a/drivers/cpufreq/apple-soc-cpufreq.c +++ b/drivers/cpufreq/apple-soc-cpufreq.c @@ -251,21 +251,19 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) return -ENODEV; } - ret = dev_pm_opp_of_add_table(cpu_dev); - if (ret < 0) { - dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); - return ret; - } + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; ret = apple_soc_cpufreq_find_cluster(policy, ®_base, &info); if (ret) { dev_err(cpu_dev, "%s: failed to get cluster info: %d\n", __func__, ret); - return ret; + goto out_free_priv; } - ret = dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); - if (ret) { - dev_err(cpu_dev, "%s: failed to mark OPPs as shared: %d\n", __func__, ret); + ret = dev_pm_opp_of_cpumask_add_table(policy->cpus); + if (ret < 0) { + dev_err(cpu_dev, "%s: failed to add OPP table: %d\n", __func__, ret); goto out_iounmap; } @@ -273,19 +271,13 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) if (ret <= 0) { dev_dbg(cpu_dev, "OPP table is not ready, deferring probe\n"); ret = -EPROBE_DEFER; - goto out_free_opp; - } - - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) { - ret = -ENOMEM; - goto out_free_opp; + goto out_free_table; } ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { dev_err(cpu_dev, "failed to init cpufreq table: %d\n", ret); - goto out_free_priv; + goto out_free_table; } /* Get OPP levels (p-state indexes) and stash them in driver_data */ @@ -320,12 +312,12 @@ static int apple_soc_cpufreq_init(struct cpufreq_policy *policy) out_free_cpufreq_table: dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); -out_free_priv: - kfree(priv); -out_free_opp: - dev_pm_opp_remove_all_dynamic(cpu_dev); +out_free_table: + dev_pm_opp_of_cpumask_remove_table(policy->cpus); out_iounmap: iounmap(reg_base); +out_free_priv: + kfree(priv); return ret; } @@ -334,7 +326,7 @@ static void apple_soc_cpufreq_exit(struct cpufreq_policy *policy) struct apple_cpu_priv *priv = policy->driver_data; dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); - dev_pm_opp_remove_all_dynamic(priv->cpu_dev); + dev_pm_opp_of_cpumask_remove_table(policy->cpus); iounmap(priv->reg_base); kfree(priv); } diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index e23d9abea1359..0330bc25fbe1a 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -766,7 +766,7 @@ static unsigned int cppc_cpufreq_get_rate(unsigned int cpu) * value first as some platforms may update the actual delivered perf * there; if failed, resort to the cached desired perf. */ - if (cppc_get_desired_perf(cpu, &delivered_perf)) + if (cppc_get_desired_perf(cpu, &delivered_perf) || !delivered_perf) delivered_perf = cpu_data->perf_ctrls.desired_perf; return cppc_perf_to_khz(&cpu_data->perf_caps, delivered_perf); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index a70df77f9b7d1..ec577a20567b1 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1261,7 +1261,7 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) if (!policy) return NULL; - if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL)) + if (!zalloc_cpumask_var(&policy->cpus, GFP_KERNEL)) goto err_free_policy; if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL)) @@ -1270,6 +1270,8 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL)) goto err_free_rcpumask; + init_rwsem(&policy->rwsem); + init_completion(&policy->kobj_unregister); ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq, cpufreq_global_kobject, "policy%u", cpu); @@ -1284,8 +1286,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu) goto err_free_real_cpus; } - init_rwsem(&policy->rwsem); - freq_constraints_init(&policy->constraints); policy->nb_min.notifier_call = cpufreq_notifier_min; @@ -2584,6 +2584,9 @@ static void cpufreq_update_pressure(struct cpufreq_policy *policy) cpu = cpumask_first(policy->related_cpus); max_freq = arch_scale_freq_ref(cpu); + if (!max_freq) + max_freq = policy->cpuinfo.max_freq; + capped_freq = policy->max; /* diff --git a/drivers/cpufreq/imx6q-cpufreq.c b/drivers/cpufreq/imx6q-cpufreq.c index e93697d3edfd9..731f5721ff1ed 100644 --- a/drivers/cpufreq/imx6q-cpufreq.c +++ b/drivers/cpufreq/imx6q-cpufreq.c @@ -55,7 +55,6 @@ static unsigned int max_freq; static unsigned int transition_latency; static u32 *imx6_soc_volt; -static u32 soc_opp_count; static int imx6q_set_target(struct cpufreq_policy *policy, unsigned int index) { @@ -330,6 +329,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) const struct property *prop; const __be32 *val; u32 nr, i, j; + u32 soc_opp_count = 0; cpu_dev = get_cpu_device(0); if (!cpu_dev) { @@ -400,7 +400,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) } /* Make imx6_soc_volt array's size same as arm opp number */ - imx6_soc_volt = devm_kcalloc(cpu_dev, num, sizeof(*imx6_soc_volt), + imx6_soc_volt = devm_kcalloc(&pdev->dev, num, sizeof(*imx6_soc_volt), GFP_KERNEL); if (imx6_soc_volt == NULL) { ret = -ENOMEM; @@ -485,6 +485,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) return 0; free_freq_table: + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); out_free_opp: dev_pm_opp_of_remove_table(cpu_dev); @@ -506,6 +507,7 @@ static int imx6q_cpufreq_probe(struct platform_device *pdev) static void imx6q_cpufreq_remove(struct platform_device *pdev) { cpufreq_unregister_driver(&imx6q_cpufreq_driver); + imx6_soc_volt = NULL; dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table); dev_pm_opp_of_remove_table(cpu_dev); regulator_put(arm_reg); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index e6d6a30cf6c90..1de12358c0b98 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -2389,8 +2389,6 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) if (pstate_funcs.get_vid) pstate_funcs.get_vid(cpu); - - intel_pstate_set_min_pstate(cpu); } /* @@ -3098,6 +3096,7 @@ static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) static int intel_pstate_cpu_init(struct cpufreq_policy *policy) { int ret = __intel_pstate_cpu_init(policy); + struct cpudata *cpu; if (ret) return ret; @@ -3108,11 +3107,11 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) */ policy->policy = CPUFREQ_POLICY_POWERSAVE; - if (hwp_active) { - struct cpudata *cpu = all_cpu_data[policy->cpu]; - + cpu = all_cpu_data[policy->cpu]; + if (hwp_active) cpu->epp_cached = intel_pstate_get_epp(cpu, 0); - } + else + intel_pstate_set_min_pstate(cpu); return 0; } @@ -3336,8 +3335,6 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) return ret; policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; - /* This reflects the intel_pstate_get_cpu_pstates() setting. */ - policy->cur = policy->cpuinfo.min_freq; req = kcalloc(2, sizeof(*req), GFP_KERNEL); if (!req) { @@ -3358,9 +3355,15 @@ static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) WRITE_ONCE(cpu->hwp_req_cached, value); cpu->epp_cached = intel_pstate_get_epp(cpu, value); + + intel_cpufreq_hwp_update(cpu, cpu->pstate.min_pstate, + cpu->pstate.max_pstate, + cpu->pstate.min_pstate, false); } else { policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY; + intel_pstate_set_min_pstate(cpu); } + policy->cur = policy->cpuinfo.min_freq; freq = DIV_ROUND_UP(cpu->pstate.turbo_freq * global.min_perf_pct, 100); diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c index f7512b4e923e7..0012c3e3a9151 100644 --- a/drivers/cpufreq/powernow-k8.c +++ b/drivers/cpufreq/powernow-k8.c @@ -1083,6 +1083,7 @@ static int powernowk8_cpu_init(struct cpufreq_policy *pol) err_out_exit_acpi: powernow_k8_cpu_exit_acpi(data); + kfree(data->powernow_table); err_out: kfree(data); diff --git a/drivers/cpufreq/spear-cpufreq.c b/drivers/cpufreq/spear-cpufreq.c index 2a1550e1aa21f..6f8cd59baa36f 100644 --- a/drivers/cpufreq/spear-cpufreq.c +++ b/drivers/cpufreq/spear-cpufreq.c @@ -79,9 +79,9 @@ static int spear1340_set_cpu_rate(struct clk *sys_pclk, unsigned long newfreq) int ret = 0; sys_clk = clk_get_parent(spear_cpufreq.clk); - if (IS_ERR(sys_clk)) { + if (!sys_clk) { pr_err("failed to get cpu's parent (sys) clock\n"); - return PTR_ERR(sys_clk); + return -EINVAL; } /* Set the rate of the source clock before changing the parent */ diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index b19bc60cc627b..49156e67db275 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -177,26 +177,30 @@ static void psci_idle_syscore_switch(bool suspend) } } -static int psci_idle_syscore_suspend(void) +static int psci_idle_syscore_suspend(void *data) { psci_idle_syscore_switch(true); return 0; } -static void psci_idle_syscore_resume(void) +static void psci_idle_syscore_resume(void *data) { psci_idle_syscore_switch(false); } -static struct syscore_ops psci_idle_syscore_ops = { +static const struct syscore_ops psci_idle_syscore_ops = { .suspend = psci_idle_syscore_suspend, .resume = psci_idle_syscore_resume, }; +static struct syscore psci_idle_syscore = { + .ops = &psci_idle_syscore_ops, +}; + static void psci_idle_init_syscore(void) { if (psci_cpuidle_use_syscore) - register_syscore_ops(&psci_idle_syscore_ops); + register_syscore(&psci_idle_syscore); } static void psci_idle_init_cpuhp(void) @@ -424,14 +428,14 @@ static int psci_idle_init_cpu(struct device *dev, int cpu) * to register cpuidle driver then rollback to cancel all CPUs * registration. */ -static int psci_cpuidle_probe(struct faux_device *fdev) +static int psci_cpuidle_probe(struct platform_device *pdev) { int cpu, ret; struct cpuidle_driver *drv; struct cpuidle_device *dev; for_each_present_cpu(cpu) { - ret = psci_idle_init_cpu(&fdev->dev, cpu); + ret = psci_idle_init_cpu(&pdev->dev, cpu); if (ret) goto out_fail; } @@ -451,36 +455,26 @@ static int psci_cpuidle_probe(struct faux_device *fdev) return ret; } -static struct faux_device_ops psci_cpuidle_ops = { +static struct platform_driver psci_cpuidle_driver = { .probe = psci_cpuidle_probe, + .driver = { + .name = "psci-cpuidle", + }, }; -static bool __init dt_idle_state_present(void) -{ - struct device_node *cpu_node __free(device_node) = - of_cpu_device_node_get(cpumask_first(cpu_possible_mask)); - if (!cpu_node) - return false; - - struct device_node *state_node __free(device_node) = - of_get_cpu_state_node(cpu_node, 0); - if (!state_node) - return false; - - return !!of_match_node(psci_idle_state_match, state_node); -} - static int __init psci_idle_init(void) { - struct faux_device *fdev; + struct platform_device *pdev; + int ret; - if (!dt_idle_state_present()) - return 0; + ret = platform_driver_register(&psci_cpuidle_driver); + if (ret) + return ret; - fdev = faux_device_create("psci-cpuidle", NULL, &psci_cpuidle_ops); - if (!fdev) { - pr_err("Failed to create psci-cpuidle device\n"); - return -ENODEV; + pdev = platform_device_register_simple("psci-cpuidle", -1, NULL, 0); + if (IS_ERR(pdev)) { + platform_driver_unregister(&psci_cpuidle_driver); + return PTR_ERR(pdev); } return 0; diff --git a/drivers/cpuidle/cpuidle-ux500.c b/drivers/cpuidle/cpuidle-ux500.c index f7d778580e9be..6d6c52c0bcc2d 100644 --- a/drivers/cpuidle/cpuidle-ux500.c +++ b/drivers/cpuidle/cpuidle-ux500.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include @@ -66,7 +66,7 @@ static inline int ux500_enter_idle(struct cpuidle_device *dev, /* Go to the retention state, the prcmu will wait for the * cpu to go WFI and this is what happens after exiting this * 'master' critical section */ - if (prcmu_set_power_state(PRCMU_AP_IDLE, true, true)) + if (db8500_prcmu_set_power_state(PRCMU_AP_IDLE, true, true)) goto out; /* When we switch to retention, the prcmu is in charge @@ -109,7 +109,7 @@ static struct cpuidle_driver ux500_idle_driver = { static int dbx500_cpuidle_probe(struct platform_device *pdev) { /* Configure wake up reasons */ - prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) | + db8500_prcmu_enable_wakeups(PRCMU_WAKEUP(ARM) | PRCMU_WAKEUP(RTC) | PRCMU_WAKEUP(ABB)); return cpuidle_register(&ux500_idle_driver, NULL); diff --git a/drivers/cpuidle/dt_idle_genpd.c b/drivers/cpuidle/dt_idle_genpd.c index 203e9b754aeac..82ff79177358c 100644 --- a/drivers/cpuidle/dt_idle_genpd.c +++ b/drivers/cpuidle/dt_idle_genpd.c @@ -99,7 +99,7 @@ struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, if (!pd) goto out; - pd->name = kasprintf(GFP_KERNEL, "%pOF", np); + pd->name = kstrdup(kbasename(of_node_full_name(np)), GFP_KERNEL); if (!pd->name) goto free_pd; @@ -112,7 +112,6 @@ struct generic_pm_domain *dt_idle_pd_alloc(struct device_node *np, goto free_name; pd->free_states = pd_free_states; - pd->name = kbasename(pd->name); pd->states = states; pd->state_count = state_count; diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 8c4a2e4ca8fa7..b8c379d808632 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -659,7 +659,6 @@ config CRYPTO_DEV_QCOM_RNG tristate "Qualcomm Random Number Generator Driver" depends on ARCH_QCOM || COMPILE_TEST depends on HW_RANDOM - select CRYPTO_RNG help This driver provides support for the Random Number Generator hardware found on Qualcomm SoCs. diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig index 06ea0e9fe6f22..d86ae005fbe2e 100644 --- a/drivers/crypto/allwinner/Kconfig +++ b/drivers/crypto/allwinner/Kconfig @@ -70,14 +70,6 @@ config CRYPTO_DEV_SUN8I_CE_HASH help Say y to enable support for hash algorithms. -config CRYPTO_DEV_SUN8I_CE_PRNG - bool "Support for Allwinner Crypto Engine PRNG" - depends on CRYPTO_DEV_SUN8I_CE - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Crypto Engine. - config CRYPTO_DEV_SUN8I_CE_TRNG bool "Support for Allwinner Crypto Engine TRNG" depends on CRYPTO_DEV_SUN8I_CE @@ -113,14 +105,6 @@ config CRYPTO_DEV_SUN8I_SS_DEBUG This will create /sys/kernel/debug/sun8i-ss/stats for displaying the number of requests per flow and per algorithm. -config CRYPTO_DEV_SUN8I_SS_PRNG - bool "Support for Allwinner Security System PRNG" - depends on CRYPTO_DEV_SUN8I_SS - select CRYPTO_RNG - help - Select this option if you want to provide kernel-side support for - the Pseudo-Random Number Generator found in the Security System. - config CRYPTO_DEV_SUN8I_SS_HASH bool "Enable support for hash on sun8i-ss" depends on CRYPTO_DEV_SUN8I_SS diff --git a/drivers/crypto/allwinner/sun8i-ce/Makefile b/drivers/crypto/allwinner/sun8i-ce/Makefile index 0842eb2d9408d..ea708b427e2e4 100644 --- a/drivers/crypto/allwinner/sun8i-ce/Makefile +++ b/drivers/crypto/allwinner/sun8i-ce/Makefile @@ -1,5 +1,4 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_CE) += sun8i-ce.o sun8i-ce-y += sun8i-ce-core.o sun8i-ce-cipher.o sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_HASH) += sun8i-ce-hash.o -sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG) += sun8i-ce-prng.o sun8i-ce-$(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) += sun8i-ce-trng.o diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index c16bb6ce6ee37..ef6f601c6259c 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -12,7 +12,6 @@ #include #include -#include #include #include #include @@ -49,7 +48,6 @@ static const struct ce_variant ce_h3_variant = { { "mod", 50000000, 0 }, }, .esr = ESR_H3, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -66,7 +64,6 @@ static const struct ce_variant ce_h5_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_H5, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -80,7 +77,6 @@ static const struct ce_variant ce_h6_variant = { }, .cipher_t_dlen_in_bytes = true, .hash_t_dlen_in_bits = true, - .prng_t_dlen_in_bytes = true, .trng_t_dlen_in_bytes = true, .ce_clks = { { "bus", 0, 200000000 }, @@ -88,7 +84,6 @@ static const struct ce_variant ce_h6_variant = { { "ram", 0, 400000000 }, }, .esr = ESR_H6, - .prng = CE_ALG_PRNG_V2, .trng = CE_ALG_TRNG_V2, }; @@ -102,7 +97,6 @@ static const struct ce_variant ce_h616_variant = { }, .cipher_t_dlen_in_bytes = true, .hash_t_dlen_in_bits = true, - .prng_t_dlen_in_bytes = true, .trng_t_dlen_in_bytes = true, .needs_word_addresses = true, .ce_clks = { @@ -112,7 +106,6 @@ static const struct ce_variant ce_h616_variant = { { "trng", 0, 0 }, }, .esr = ESR_H6, - .prng = CE_ALG_PRNG_V2, .trng = CE_ALG_TRNG_V2, }; @@ -129,7 +122,6 @@ static const struct ce_variant ce_a64_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_A64, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -148,7 +140,6 @@ static const struct ce_variant ce_d1_variant = { { "trng", 0, 0 }, }, .esr = ESR_D1, - .prng = CE_ALG_PRNG, .trng = CE_ALG_TRNG, }; @@ -165,7 +156,6 @@ static const struct ce_variant ce_r40_variant = { { "mod", 300000000, 0 }, }, .esr = ESR_R40, - .prng = CE_ALG_PRNG, .trng = CE_ID_NOTSUPP, }; @@ -614,25 +604,6 @@ static struct sun8i_ce_alg_template ce_algs[] = { }, }, #endif -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun8i-ce-prng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct sun8i_ce_rng_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_init = sun8i_ce_prng_init, - .cra_exit = sun8i_ce_prng_exit, - }, - .generate = sun8i_ce_prng_generate, - .seed = sun8i_ce_prng_seed, - .seedsize = PRNG_SEED_SIZE, - } -}, -#endif }; static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) @@ -692,12 +663,6 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ce_algs[i].stat_fb_maxsg); break; - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n", - ce_algs[i].alg.rng.base.cra_driver_name, - ce_algs[i].alg.rng.base.cra_name, - ce_algs[i].stat_req, ce_algs[i].stat_bytes); - break; } } #if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \ @@ -925,23 +890,6 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce) return err; } break; - case CRYPTO_ALG_TYPE_RNG: - if (ce->variant->prng == CE_ID_NOTSUPP) { - dev_info(ce->dev, - "DEBUG: Algo of %s not supported\n", - ce_algs[i].alg.rng.base.cra_name); - ce_algs[i].ce = NULL; - break; - } - dev_info(ce->dev, "Register %s\n", - ce_algs[i].alg.rng.base.cra_name); - err = crypto_register_rng(&ce_algs[i].alg.rng); - if (err) { - dev_err(ce->dev, "Fail to register %s\n", - ce_algs[i].alg.rng.base.cra_name); - ce_algs[i].ce = NULL; - } - break; default: ce_algs[i].ce = NULL; dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); @@ -968,11 +916,6 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce) ce_algs[i].alg.hash.base.halg.base.cra_name); crypto_engine_unregister_ahash(&ce_algs[i].alg.hash); break; - case CRYPTO_ALG_TYPE_RNG: - dev_info(ce->dev, "Unregister %d %s\n", i, - ce_algs[i].alg.rng.base.cra_name); - crypto_unregister_rng(&ce_algs[i].alg.rng); - break; } } } diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c deleted file mode 100644 index d0a1ac66738bf..0000000000000 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-prng.c +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sun8i-ce-prng.c - hardware cryptographic offloader for - * Allwinner H3/A64/H5/H2+/H6/R40 SoC - * - * Copyright (C) 2015-2020 Corentin Labbe - * - * This file handle the PRNG - * - * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst - */ -#include "sun8i-ce.h" -#include -#include -#include - -int sun8i_ce_prng_init(struct crypto_tfm *tfm) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - memset(ctx, 0, sizeof(struct sun8i_ce_rng_tfm_ctx)); - return 0; -} - -void sun8i_ce_prng_exit(struct crypto_tfm *tfm) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - kfree_sensitive(ctx->seed); - ctx->seed = NULL; - ctx->slen = 0; -} - -int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - - if (ctx->seed && ctx->slen != slen) { - kfree_sensitive(ctx->seed); - ctx->slen = 0; - ctx->seed = NULL; - } - if (!ctx->seed) - ctx->seed = kmalloc(slen, GFP_KERNEL | GFP_DMA); - if (!ctx->seed) - return -ENOMEM; - - memcpy(ctx->seed, seed, slen); - ctx->slen = slen; - - return 0; -} - -int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun8i_ce_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - struct rng_alg *alg = crypto_rng_alg(tfm); - struct sun8i_ce_alg_template *algt; - struct sun8i_ce_dev *ce; - dma_addr_t dma_iv, dma_dst; - int err = 0; - int flow = 3; - unsigned int todo; - struct sun8i_ce_flow *chan; - struct ce_task *cet; - u32 common, sym; - void *d; - - algt = container_of(alg, struct sun8i_ce_alg_template, alg.rng); - ce = algt->ce; - - if (ctx->slen == 0) { - dev_err(ce->dev, "not seeded\n"); - return -EINVAL; - } - - /* we want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE */ - todo = dlen + ctx->slen + PRNG_DATA_SIZE * 2; - todo -= todo % PRNG_DATA_SIZE; - - d = kzalloc(todo, GFP_KERNEL | GFP_DMA); - if (!d) { - err = -ENOMEM; - goto err_mem; - } - - dev_dbg(ce->dev, "%s PRNG slen=%u dlen=%u todo=%u multi=%u\n", __func__, - slen, dlen, todo, todo / PRNG_DATA_SIZE); - -#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG - algt->stat_req++; - algt->stat_bytes += todo; -#endif - - dma_iv = dma_map_single(ce->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); - if (dma_mapping_error(ce->dev, dma_iv)) { - dev_err(ce->dev, "Cannot DMA MAP IV\n"); - err = -EFAULT; - goto err_iv; - } - - dma_dst = dma_map_single(ce->dev, d, todo, DMA_FROM_DEVICE); - if (dma_mapping_error(ce->dev, dma_dst)) { - dev_err(ce->dev, "Cannot DMA MAP DST\n"); - err = -EFAULT; - goto err_dst; - } - - err = pm_runtime_resume_and_get(ce->dev); - if (err < 0) - goto err_pm; - - mutex_lock(&ce->rnglock); - chan = &ce->chanlist[flow]; - - cet = &chan->tl[0]; - memset(cet, 0, sizeof(struct ce_task)); - - cet->t_id = cpu_to_le32(flow); - common = ce->variant->prng | CE_COMM_INT; - cet->t_common_ctl = cpu_to_le32(common); - - /* recent CE (H6) need length in bytes, in word otherwise */ - if (ce->variant->prng_t_dlen_in_bytes) - cet->t_dlen = cpu_to_le32(todo); - else - cet->t_dlen = cpu_to_le32(todo / 4); - - sym = PRNG_LD; - cet->t_sym_ctl = cpu_to_le32(sym); - cet->t_asym_ctl = 0; - - cet->t_key = desc_addr_val_le32(ce, dma_iv); - cet->t_iv = desc_addr_val_le32(ce, dma_iv); - - cet->t_dst[0].addr = desc_addr_val_le32(ce, dma_dst); - cet->t_dst[0].len = cpu_to_le32(todo / 4); - - err = sun8i_ce_run_task(ce, 3, "PRNG"); - mutex_unlock(&ce->rnglock); - - pm_runtime_put(ce->dev); - -err_pm: - dma_unmap_single(ce->dev, dma_dst, todo, DMA_FROM_DEVICE); -err_dst: - dma_unmap_single(ce->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); - - if (!err) { - memcpy(dst, d, dlen); - memcpy(ctx->seed, d + dlen, ctx->slen); - } -err_iv: - kfree_sensitive(d); -err_mem: - return err; -} diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h index 71f5a0cd3d458..468d99bf5bf66 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -58,9 +57,7 @@ #define CE_ALG_SHA384 20 #define CE_ALG_SHA512 21 #define CE_ALG_TRNG 48 -#define CE_ALG_PRNG 49 #define CE_ALG_TRNG_V2 0x1c -#define CE_ALG_PRNG_V2 0x1d /* Used in ce_variant */ #define CE_ID_NOTSUPP 0xFF @@ -96,10 +93,6 @@ #define ESR_H6 4 #define ESR_D1 5 -#define PRNG_DATA_SIZE (160 / 8) -#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) -#define PRNG_LD BIT(17) - #define CE_DIE_ID_SHIFT 16 #define CE_DIE_ID_MASK 0x07 @@ -136,13 +129,10 @@ struct ce_clock { * bytes or words * @hash_t_dlen_in_bytes: Does the request size for hash is in * bits or words - * @prng_t_dlen_in_bytes: Does the request size for PRNG is in - * bytes or words * @trng_t_dlen_in_bytes: Does the request size for TRNG is in * bytes or words * @ce_clks: list of clocks needed by this variant * @esr: The type of error register - * @prng: The CE_ALG_XXX value for the PRNG * @trng: The CE_ALG_XXX value for the TRNG */ struct ce_variant { @@ -151,12 +141,10 @@ struct ce_variant { u32 op_mode[CE_ID_OP_MAX]; bool cipher_t_dlen_in_bytes; bool hash_t_dlen_in_bits; - bool prng_t_dlen_in_bytes; bool trng_t_dlen_in_bytes; bool needs_word_addresses; struct ce_clock ce_clks[CE_MAX_CLOCKS]; int esr; - unsigned char prng; unsigned char trng; }; @@ -327,16 +315,6 @@ struct sun8i_ce_hash_reqctx { struct ahash_request fallback_req; // keep at the end }; -/* - * struct sun8i_ce_prng_ctx - context for PRNG TFM - * @seed: The seed to use - * @slen: The size of the seed - */ -struct sun8i_ce_rng_tfm_ctx { - void *seed; - unsigned int slen; -}; - /* * struct sun8i_ce_alg_template - crypto_alg template * @type: the CRYPTO_ALG_TYPE for this template @@ -357,7 +335,6 @@ struct sun8i_ce_alg_template { union { struct skcipher_engine_alg skcipher; struct ahash_engine_alg hash; - struct rng_alg rng; } alg; unsigned long stat_req; unsigned long stat_fb; @@ -398,11 +375,5 @@ int sun8i_ce_hash_finup(struct ahash_request *areq); int sun8i_ce_hash_digest(struct ahash_request *areq); int sun8i_ce_hash_run(struct crypto_engine *engine, void *breq); -int sun8i_ce_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun8i_ce_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); -void sun8i_ce_prng_exit(struct crypto_tfm *tfm); -int sun8i_ce_prng_init(struct crypto_tfm *tfm); - int sun8i_ce_hwrng_register(struct sun8i_ce_dev *ce); void sun8i_ce_hwrng_unregister(struct sun8i_ce_dev *ce); diff --git a/drivers/crypto/allwinner/sun8i-ss/Makefile b/drivers/crypto/allwinner/sun8i-ss/Makefile index aabfd893c8174..2d6458a42e583 100644 --- a/drivers/crypto/allwinner/sun8i-ss/Makefile +++ b/drivers/crypto/allwinner/sun8i-ss/Makefile @@ -1,4 +1,3 @@ obj-$(CONFIG_CRYPTO_DEV_SUN8I_SS) += sun8i-ss.o sun8i-ss-y += sun8i-ss-core.o sun8i-ss-cipher.o -sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG) += sun8i-ss-prng.o sun8i-ss-$(CONFIG_CRYPTO_DEV_SUN8I_SS_HASH) += sun8i-ss-hash.o diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index f45685707e0d3..119cc15509d1a 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -11,7 +11,6 @@ */ #include -#include #include #include #include @@ -283,25 +282,6 @@ static struct sun8i_ss_alg_template ss_algs[] = { .do_one_request = sun8i_ss_handle_cipher_request, }, }, -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG -{ - .type = CRYPTO_ALG_TYPE_RNG, - .alg.rng = { - .base = { - .cra_name = "stdrng", - .cra_driver_name = "sun8i-ss-prng", - .cra_priority = 300, - .cra_ctxsize = sizeof(struct sun8i_ss_rng_tfm_ctx), - .cra_module = THIS_MODULE, - .cra_init = sun8i_ss_prng_init, - .cra_exit = sun8i_ss_prng_exit, - }, - .generate = sun8i_ss_prng_generate, - .seed = sun8i_ss_prng_seed, - .seedsize = PRNG_SEED_SIZE, - } -}, -#endif #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH { .type = CRYPTO_ALG_TYPE_AHASH, .ss_algo_id = SS_ID_HASH_MD5, @@ -501,12 +481,6 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v) seq_printf(seq, "\tFallback due to SG numbers: %lu\n", ss_algs[i].stat_fb_sgnum); break; - case CRYPTO_ALG_TYPE_RNG: - seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n", - ss_algs[i].alg.rng.base.cra_driver_name, - ss_algs[i].alg.rng.base.cra_name, - ss_algs[i].stat_req, ss_algs[i].stat_bytes); - break; case CRYPTO_ALG_TYPE_AHASH: seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", ss_algs[i].alg.hash.base.halg.base.cra_driver_name, @@ -707,14 +681,6 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss) return err; } break; - case CRYPTO_ALG_TYPE_RNG: - err = crypto_register_rng(&ss_algs[i].alg.rng); - if (err) { - dev_err(ss->dev, "Fail to register %s\n", - ss_algs[i].alg.rng.base.cra_name); - ss_algs[i].ss = NULL; - } - break; case CRYPTO_ALG_TYPE_AHASH: id = ss_algs[i].ss_algo_id; ss_method = ss->variant->alg_hash[id]; @@ -756,11 +722,6 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss) ss_algs[i].alg.skcipher.base.base.cra_name); crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher); break; - case CRYPTO_ALG_TYPE_RNG: - dev_info(ss->dev, "Unregister %d %s\n", i, - ss_algs[i].alg.rng.base.cra_name); - crypto_unregister_rng(&ss_algs[i].alg.rng); - break; case CRYPTO_ALG_TYPE_AHASH: dev_info(ss->dev, "Unregister %d %s\n", i, ss_algs[i].alg.hash.base.halg.base.cra_name); diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c deleted file mode 100644 index a923cfc6553f4..0000000000000 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-prng.c +++ /dev/null @@ -1,177 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * sun8i-ss-prng.c - hardware cryptographic offloader for - * Allwinner A80/A83T SoC - * - * Copyright (C) 2015-2020 Corentin Labbe - * - * This file handle the PRNG found in the SS - * - * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst - */ -#include "sun8i-ss.h" -#include -#include -#include -#include -#include - -int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - - if (ctx->seed && ctx->slen != slen) { - kfree_sensitive(ctx->seed); - ctx->slen = 0; - ctx->seed = NULL; - } - if (!ctx->seed) - ctx->seed = kmalloc(slen, GFP_KERNEL); - if (!ctx->seed) - return -ENOMEM; - - memcpy(ctx->seed, seed, slen); - ctx->slen = slen; - - return 0; -} - -int sun8i_ss_prng_init(struct crypto_tfm *tfm) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - memset(ctx, 0, sizeof(struct sun8i_ss_rng_tfm_ctx)); - return 0; -} - -void sun8i_ss_prng_exit(struct crypto_tfm *tfm) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_tfm_ctx(tfm); - - kfree_sensitive(ctx->seed); - ctx->seed = NULL; - ctx->slen = 0; -} - -int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen) -{ - struct sun8i_ss_rng_tfm_ctx *ctx = crypto_rng_ctx(tfm); - struct rng_alg *alg = crypto_rng_alg(tfm); - struct sun8i_ss_alg_template *algt; - unsigned int todo_with_padding; - struct sun8i_ss_dev *ss; - dma_addr_t dma_iv, dma_dst; - unsigned int todo; - int err = 0; - int flow; - void *d; - u32 v; - - algt = container_of(alg, struct sun8i_ss_alg_template, alg.rng); - ss = algt->ss; - - if (ctx->slen == 0) { - dev_err(ss->dev, "The PRNG is not seeded\n"); - return -EINVAL; - } - - /* The SS does not give an updated seed, so we need to get a new one. - * So we will ask for an extra PRNG_SEED_SIZE data. - * We want dlen + seedsize rounded up to a multiple of PRNG_DATA_SIZE - */ - todo = dlen + PRNG_SEED_SIZE + PRNG_DATA_SIZE; - todo -= todo % PRNG_DATA_SIZE; - - todo_with_padding = ALIGN(todo, dma_get_cache_alignment()); - if (todo_with_padding < todo || todo < dlen) - return -EOVERFLOW; - - d = kzalloc(todo_with_padding, GFP_KERNEL); - if (!d) - return -ENOMEM; - - flow = sun8i_ss_get_engine_number(ss); - -#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG - algt->stat_req++; - algt->stat_bytes += todo; -#endif - - v = SS_ALG_PRNG | SS_PRNG_CONTINUE | SS_START; - if (flow) - v |= SS_FLOW1; - else - v |= SS_FLOW0; - - dma_iv = dma_map_single(ss->dev, ctx->seed, ctx->slen, DMA_TO_DEVICE); - if (dma_mapping_error(ss->dev, dma_iv)) { - dev_err(ss->dev, "Cannot DMA MAP IV\n"); - err = -EFAULT; - goto err_free; - } - - dma_dst = dma_map_single(ss->dev, d, todo, DMA_FROM_DEVICE); - if (dma_mapping_error(ss->dev, dma_dst)) { - dev_err(ss->dev, "Cannot DMA MAP DST\n"); - err = -EFAULT; - goto err_iv; - } - - err = pm_runtime_resume_and_get(ss->dev); - if (err < 0) - goto err_pm; - err = 0; - - mutex_lock(&ss->mlock); - writel(dma_iv, ss->base + SS_IV_ADR_REG); - /* the PRNG act badly (failing rngtest) without SS_KEY_ADR_REG set */ - writel(dma_iv, ss->base + SS_KEY_ADR_REG); - writel(dma_dst, ss->base + SS_DST_ADR_REG); - writel(todo / 4, ss->base + SS_LEN_ADR_REG); - - reinit_completion(&ss->flows[flow].complete); - ss->flows[flow].status = 0; - /* Be sure all data is written before enabling the task */ - wmb(); - - writel(v, ss->base + SS_CTL_REG); - - wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, - msecs_to_jiffies(todo)); - if (ss->flows[flow].status == 0) { - dev_err(ss->dev, "DMA timeout for PRNG (size=%u)\n", todo); - err = -EFAULT; - } - /* Since cipher and hash use the linux/cryptoengine and that we have - * a cryptoengine per flow, we are sure that they will issue only one - * request per flow. - * Since the cryptoengine wait for completion before submitting a new - * one, the mlock could be left just after the final writel. - * But cryptoengine cannot handle crypto_rng, so we need to be sure - * nothing will use our flow. - * The easiest way is to grab mlock until the hardware end our requests. - * We could have used a per flow lock, but this would increase - * complexity. - * The drawback is that no request could be handled for the other flow. - */ - mutex_unlock(&ss->mlock); - - pm_runtime_put(ss->dev); - -err_pm: - dma_unmap_single(ss->dev, dma_dst, todo, DMA_FROM_DEVICE); -err_iv: - dma_unmap_single(ss->dev, dma_iv, ctx->slen, DMA_TO_DEVICE); - - if (!err) { - memcpy(dst, d, dlen); - /* Update seed */ - memcpy(ctx->seed, d + dlen, ctx->slen); - } -err_free: - kfree_sensitive(d); - - return err; -} diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h index ae66eb45fb246..9ec6c2ad56992 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -27,7 +26,6 @@ #define SS_ALG_DES (1 << 2) #define SS_ALG_3DES (2 << 2) #define SS_ALG_MD5 (3 << 2) -#define SS_ALG_PRNG (4 << 2) #define SS_ALG_SHA1 (6 << 2) #define SS_ALG_SHA224 (7 << 2) #define SS_ALG_SHA256 (8 << 2) @@ -68,8 +66,6 @@ #define SS_FLOW0 BIT(30) #define SS_FLOW1 BIT(31) -#define SS_PRNG_CONTINUE BIT(18) - #define MAX_SG 8 #define MAXFLOW 2 @@ -79,9 +75,6 @@ #define SS_DIE_ID_SHIFT 20 #define SS_DIE_ID_MASK 0x07 -#define PRNG_DATA_SIZE (160 / 8) -#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8) - #define MAX_PAD_SIZE 4096 /* @@ -213,16 +206,6 @@ struct sun8i_cipher_tfm_ctx { struct crypto_skcipher *fallback_tfm; }; -/* - * struct sun8i_ss_prng_ctx - context for PRNG TFM - * @seed: The seed to use - * @slen: The size of the seed - */ -struct sun8i_ss_rng_tfm_ctx { - void *seed; - unsigned int slen; -}; - /* * struct sun8i_ss_hash_tfm_ctx - context for an ahash TFM * @fallback_tfm: pointer to the fallback TFM @@ -272,7 +255,6 @@ struct sun8i_ss_alg_template { struct sun8i_ss_dev *ss; union { struct skcipher_engine_alg skcipher; - struct rng_alg rng; struct ahash_engine_alg hash; } alg; unsigned long stat_req; @@ -298,11 +280,6 @@ int sun8i_ss_skencrypt(struct skcipher_request *areq); int sun8i_ss_get_engine_number(struct sun8i_ss_dev *ss); int sun8i_ss_run_task(struct sun8i_ss_dev *ss, struct sun8i_cipher_req_ctx *rctx, const char *name); -int sun8i_ss_prng_generate(struct crypto_rng *tfm, const u8 *src, - unsigned int slen, u8 *dst, unsigned int dlen); -int sun8i_ss_prng_seed(struct crypto_rng *tfm, const u8 *seed, unsigned int slen); -int sun8i_ss_prng_init(struct crypto_tfm *tfm); -void sun8i_ss_prng_exit(struct crypto_tfm *tfm); int sun8i_ss_hash_init_tfm(struct crypto_ahash *tfm); void sun8i_ss_hash_exit_tfm(struct crypto_ahash *tfm); diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 9da5a03880807..3f384eb694f58 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -27,15 +27,14 @@ static struct atmel_ecc_driver_data driver_data; /** * struct atmel_ecdh_ctx - transformation context - * @client : pointer to i2c client device - * @fallback : used for unsupported curves or when user wants to use its own - * private key. - * @public_key : generated when calling set_secret(). It's the responsibility - * of the user to not call set_secret() while - * generate_public_key() or compute_shared_secret() are in flight. - * @curve_id : elliptic curve id - * @do_fallback: true when the device doesn't support the curve or when the user - * wants to use its own private key. + * @client: I2C client device + * @fallback: ECDH fallback used for caller-provided private keys + * @public_key: cached public key for the device-generated private key + * @curve_id: elliptic curve id + * @do_fallback: true when ECDH operations should use @fallback + * + * The caller must not invoke set_secret() while generate_public_key() + * or compute_shared_secret() are in flight. */ struct atmel_ecdh_ctx { struct i2c_client *client; @@ -55,8 +54,8 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, if (status) goto free_work_data; - /* might want less than we've got */ - n_sz = min_t(size_t, ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); + /* copy only as much as requested, capped at 32 bytes */ + n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len); /* copy the shared secret */ copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, n_sz), @@ -64,15 +63,15 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, if (copied != n_sz) status = -EINVAL; - /* fall through */ free_work_data: kfree_sensitive(work_data); kpp_request_complete(req, status); } /* - * A random private key is generated and stored in the device. The device - * returns the pair public key. + * If no private key is provided, generate one in the device and cache + * the corresponding public key. The generated private key never leaves + * the device. */ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) @@ -83,10 +82,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, struct ecdh params; int ret = -ENOMEM; - /* free the old public key, if any */ kfree(ctx->public_key); - /* make sure you don't free the old public key twice */ ctx->public_key = NULL; + ctx->do_fallback = false; if (crypto_ecdh_decode_key(buf, len, ¶ms) < 0) { dev_err(&ctx->client->dev, "crypto_ecdh_decode_key failed\n"); @@ -94,33 +92,25 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, } if (params.key_size) { - /* fallback to ecdh software implementation */ - ctx->do_fallback = true; - return crypto_kpp_set_secret(ctx->fallback, buf, len); + ret = crypto_kpp_set_secret(ctx->fallback, buf, len); + ctx->do_fallback = !ret; + return ret; } cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); if (!cmd) return -ENOMEM; - /* - * The device only supports NIST P256 ECC keys. The public key size will - * always be the same. Use a macro for the key size to avoid unnecessary - * computations. - */ public_key = kmalloc(ATMEL_ECC_PUBKEY_SIZE, GFP_KERNEL); if (!public_key) goto free_cmd; - ctx->do_fallback = false; - atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2); ret = atmel_i2c_send_receive(ctx->client, cmd); if (ret) goto free_public_key; - /* save the public key */ memcpy(public_key, &cmd->data[RSP_DATA_IDX], ATMEL_ECC_PUBKEY_SIZE); ctx->public_key = public_key; @@ -149,8 +139,8 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req) if (!ctx->public_key) return -EINVAL; - /* might want less than we've got */ - nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len); + /* copy only as much as requested, capped at 64 bytes */ + nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len); /* public key was saved at private key generation */ copied = sg_copy_from_buffer(req->dst, @@ -175,7 +165,10 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) return crypto_kpp_compute_shared_secret(req); } - /* must have exactly two points to be on the curve */ + if (!ctx->public_key) + return -EINVAL; + + /* A P-256 public key must contain two 32-byte coordinates */ if (req->src_len != ATMEL_ECC_PUBKEY_SIZE) return -EINVAL; @@ -372,6 +365,8 @@ static void atmel_ecc_remove(struct i2c_client *client) static const struct of_device_id atmel_ecc_dt_ids[] = { { .compatible = "atmel,atecc508a", + }, { + .compatible = "atmel,atecc608b", }, { /* sentinel */ } @@ -381,6 +376,7 @@ MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids); static const struct i2c_device_id atmel_ecc_id[] = { { "atecc508a" }, + { "atecc608b" }, { } }; MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index 8a3520c9a0ff5..3e52e604ded39 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -31,10 +31,14 @@ static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; struct hwrng *rng = areq; - if (status) + if (status) { dev_warn_ratelimited(&i2c_priv->client->dev, "i2c transaction failed (%d)\n", status); + kfree(work_data); + atomic_dec(&i2c_priv->tfm_count); + return; + } rng->priv = (unsigned long)work_data; atomic_dec(&i2c_priv->tfm_count); @@ -205,11 +209,10 @@ static void atmel_sha204a_remove(struct i2c_client *client) { struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); devm_hwrng_unregister(&client->dev, &i2c_priv->hwrng); atmel_i2c_flush_queue(); - sysfs_remove_group(&client->dev.kobj, &atmel_sha204a_groups); - kfree((void *)i2c_priv->hwrng.priv); } diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 643e507f9c020..d380f6741a2c5 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -463,14 +463,13 @@ static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd) IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); fast = in && out; - if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) + if (dd->in_sg->length != dd->out_sg->length) fast = 0; } if (fast) { - count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg)); - count = min_t(size_t, count, sg_dma_len(dd->out_sg)); + count = min_t(size_t, dd->total, dd->in_sg->length); err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); if (!err) { diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 4eaad21fd8481..8820efe5ef0af 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -1514,7 +1514,7 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) if (sev_init_ex_buffer) return 0; - page = alloc_pages(GFP_KERNEL, get_order(NV_LENGTH)); + page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(NV_LENGTH)); if (!page) { dev_err(sev->dev, "SEV: INIT_EX NV memory allocation failed\n"); return -ENOMEM; @@ -1524,20 +1524,31 @@ static int __sev_platform_init_handle_init_ex_path(struct sev_device *sev) rc = sev_read_init_ex_file(); if (rc) - return rc; + goto err_free; /* If SEV-SNP is initialized, transition to firmware page. */ if (sev->snp_initialized) { unsigned long npages; npages = 1UL << get_order(NV_LENGTH); - if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, false)) { + if (rmp_mark_pages_firmware(__pa(sev_init_ex_buffer), npages, true)) { dev_err(sev->dev, "SEV: INIT_EX NV memory page state change failed.\n"); - return -ENOMEM; + rc = -ENOMEM; + /* + * Pages can be in an inconsistent state, don't release them back to the + * system. + */ + goto err_reset; } } return 0; + +err_free: + __free_pages(page, get_order(NV_LENGTH)); +err_reset: + sev_init_ex_buffer = NULL; + return rc; } static int __sev_platform_init_locked(int *error) diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c index 583010b2d0071..02ec4282333b6 100644 --- a/drivers/crypto/gemini/sl3516-ce-cipher.c +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -56,10 +56,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; @@ -72,10 +68,6 @@ static bool sl3516_ce_need_fallback(struct skcipher_request *areq) ce->fallback_mod16++; return true; } - if ((sg_dma_len(sg) % 16) != 0) { - ce->fallback_mod16++; - return true; - } if (!IS_ALIGNED(sg->offset, 16)) { ce->fallback_align16++; return true; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 2d0b248c59ebd..66d76134f1dfa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -96,7 +96,6 @@ #define IV_FLAGS_OFFSET 0x6 #define IV_CM_OFFSET 0x3 #define IV_LAST_BYTE1 1 -#define IV_LAST_BYTE2 2 #define IV_LAST_BYTE_MASK 0xFF #define IV_CTR_INIT 0x1 #define IV_BYTE_OFFSET 0x8 @@ -1697,7 +1696,7 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) struct sec_cipher_req *c_req = &req->c_req; u32 data_size = aead_req->cryptlen; u8 flage = 0; - u8 cm, cl; + u8 cm, cl, i; /* the specification has been checked in aead_iv_demension_check() */ cl = c_req->c_ivin[0] + 1; @@ -1721,15 +1720,16 @@ static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) * the last 32bit is counter's initial number, * but the nonce uses the first 16bit * the tail 16bit fill with the cipher length + * When CL is 3, the tail 24bit fill with the cipher length. */ if (!c_req->encrypt) data_size = aead_req->cryptlen - authsize; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = - data_size & IV_LAST_BYTE_MASK; - data_size >>= IV_BYTE_OFFSET; - a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = + for (i = 1; i <= cl; i++) { + a_req->a_ivin[ctx->c_ctx.ivsize - i] = data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + } } static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 78218f3a3cd06..6eabcbe461455 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -2,6 +2,7 @@ /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */ #include +#include #include #include #include @@ -1001,6 +1002,19 @@ static int deflate_generic_decompress(struct acomp_req *req) return ret; } +static int deflate_generic_compress(struct acomp_req *req) +{ + ACOMP_FBREQ_ON_STACK(fbreq, req); + int ret; + + ret = crypto_acomp_compress(fbreq); + req->dlen = fbreq->dlen; + + update_total_sw_comp_calls(); + + return ret; +} + static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq, struct acomp_req *req, dma_addr_t *src_addr, dma_addr_t *dst_addr); @@ -1049,13 +1063,17 @@ static void iaa_desc_complete(struct idxd_desc *idxd_desc, pr_warn("%s: falling back to deflate-generic decompress, " "analytics error code %x\n", __func__, idxd_desc->iax_completion->error_code); + dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), + DMA_FROM_DEVICE); + dma_unmap_sg(dev, ctx->req->src, 1, DMA_TO_DEVICE); + ret = deflate_generic_decompress(ctx->req); if (ret) { dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", __func__, ret); err = -EIO; - goto err; } + goto out; } else { err = -EIO; goto err; @@ -1438,19 +1456,9 @@ static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req, ret = check_completion(dev, idxd_desc->iax_completion, false, false); if (ret) { dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret); - if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) { - pr_warn("%s: falling back to deflate-generic decompress, " - "analytics error code %x\n", __func__, - idxd_desc->iax_completion->error_code); - ret = deflate_generic_decompress(req); - if (ret) { - dev_dbg(dev, "%s: deflate-generic failed ret=%d\n", - __func__, ret); - goto err; - } - } else { - goto err; - } + if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) + ret = -EAGAIN; + goto err; } else { req->dlen = idxd_desc->iax_completion->output_size; } @@ -1477,7 +1485,7 @@ static int iaa_comp_acompress(struct acomp_req *req) struct iaa_compression_ctx *compression_ctx; struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct idxd_wq *wq; struct device *dev; @@ -1489,11 +1497,15 @@ static int iaa_comp_acompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not compressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not compressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_compress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1512,30 +1524,25 @@ static int iaa_comp_acompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_compress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_compress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr, @@ -1555,8 +1562,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE); - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_TO_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_FROM_DEVICE); goto out; } @@ -1564,9 +1571,8 @@ static int iaa_comp_acompress(struct acomp_req *req) if (ret) dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); out: iaa_wq_put(wq); @@ -1577,7 +1583,7 @@ static int iaa_comp_adecompress(struct acomp_req *req) { struct crypto_tfm *tfm = req->base.tfm; dma_addr_t src_addr, dst_addr; - int nr_sgs, cpu, ret = 0; + int cpu, ret = 0; struct iaa_wq *iaa_wq; struct device *dev; struct idxd_wq *wq; @@ -1587,11 +1593,15 @@ static int iaa_comp_adecompress(struct acomp_req *req) return -ENODEV; } - if (!req->src || !req->slen) { - pr_debug("invalid src, not decompressing\n"); + if (!req->src || !req->slen || !req->dst) { + pr_debug("invalid req, not decompressing\n"); return -EINVAL; } + /* Fall back to software if src or dst has multiple sg entries */ + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return deflate_generic_decompress(req); + cpu = get_cpu(); wq = wq_table_next_wq(cpu); put_cpu(); @@ -1610,30 +1620,25 @@ static int iaa_comp_adecompress(struct acomp_req *req) dev = &wq->idxd->pdev->dev; - nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map src sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto out; + if (!dma_map_sg(dev, req->src, 1, DMA_TO_DEVICE)) { + dev_dbg(dev, "couldn't map src sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } src_addr = sg_dma_address(req->src); - dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p," - " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs, + dev_dbg(dev, "map src %llx req->src %p slen %d sg_len %d\n", src_addr, req->src, req->slen, sg_dma_len(req->src)); - nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); - if (nr_sgs <= 0 || nr_sgs > 1) { - dev_dbg(dev, "couldn't map dst sg for iaa device %d," - " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id, - iaa_wq->wq->id, ret); - ret = -EIO; - goto err_map_dst; + if (!dma_map_sg(dev, req->dst, 1, DMA_FROM_DEVICE)) { + dev_dbg(dev, "couldn't map dst sg for iaa device %d, wq %d\n", + iaa_wq->iaa_device->idxd->id, iaa_wq->wq->id); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); + iaa_wq_put(wq); + return deflate_generic_decompress(req); } dst_addr = sg_dma_address(req->dst); - dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p," - " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs, + dev_dbg(dev, "map dst %llx req->dst %p dlen %d sg_len %d\n", dst_addr, req->dst, req->dlen, sg_dma_len(req->dst)); ret = iaa_decompress(tfm, req, wq, src_addr, req->slen, @@ -1641,15 +1646,16 @@ static int iaa_comp_adecompress(struct acomp_req *req) if (ret == -EINPROGRESS) return ret; - if (ret != 0) + if (ret != 0 && ret != -EAGAIN) dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret); - dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE); -err_map_dst: - dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE); -out: + dma_unmap_sg(dev, req->dst, 1, DMA_FROM_DEVICE); + dma_unmap_sg(dev, req->src, 1, DMA_TO_DEVICE); iaa_wq_put(wq); + if (ret == -EAGAIN) + ret = deflate_generic_decompress(req); + return ret; } diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.c b/drivers/crypto/intel/iaa/iaa_crypto_stats.c index f5cc3d29ca19e..2f2ed88c88120 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.c @@ -19,6 +19,7 @@ static atomic64_t total_comp_calls; static atomic64_t total_decomp_calls; +static atomic64_t total_sw_comp_calls; static atomic64_t total_sw_decomp_calls; static atomic64_t total_comp_bytes_out; static atomic64_t total_decomp_bytes_in; @@ -43,6 +44,11 @@ void update_total_decomp_calls(void) atomic64_inc(&total_decomp_calls); } +void update_total_sw_comp_calls(void) +{ + atomic64_inc(&total_sw_comp_calls); +} + void update_total_sw_decomp_calls(void) { atomic64_inc(&total_sw_decomp_calls); @@ -104,6 +110,7 @@ static void reset_iaa_crypto_stats(void) { atomic64_set(&total_comp_calls, 0); atomic64_set(&total_decomp_calls, 0); + atomic64_set(&total_sw_comp_calls, 0); atomic64_set(&total_sw_decomp_calls, 0); atomic64_set(&total_comp_bytes_out, 0); atomic64_set(&total_decomp_bytes_in, 0); @@ -174,6 +181,8 @@ static int global_stats_show(struct seq_file *m, void *v) atomic64_read(&total_comp_calls)); seq_printf(m, " total_decomp_calls: %llu\n", atomic64_read(&total_decomp_calls)); + seq_printf(m, " total_sw_comp_calls: %llu\n", + atomic64_read(&total_sw_comp_calls)); seq_printf(m, " total_sw_decomp_calls: %llu\n", atomic64_read(&total_sw_decomp_calls)); seq_printf(m, " total_comp_bytes_out: %llu\n", diff --git a/drivers/crypto/intel/iaa/iaa_crypto_stats.h b/drivers/crypto/intel/iaa/iaa_crypto_stats.h index 3787a5f507eb2..6e0c6f9939bfa 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_stats.h +++ b/drivers/crypto/intel/iaa/iaa_crypto_stats.h @@ -11,6 +11,7 @@ void iaa_crypto_debugfs_cleanup(void); void update_total_comp_calls(void); void update_total_comp_bytes_out(int n); void update_total_decomp_calls(void); +void update_total_sw_comp_calls(void); void update_total_sw_decomp_calls(void); void update_total_decomp_bytes_in(int n); void update_completion_einval_errs(void); @@ -29,6 +30,7 @@ static inline void iaa_crypto_debugfs_cleanup(void) {} static inline void update_total_comp_calls(void) {} static inline void update_total_comp_bytes_out(int n) {} static inline void update_total_decomp_calls(void) {} +static inline void update_total_sw_comp_calls(void) {} static inline void update_total_sw_decomp_calls(void) {} static inline void update_total_decomp_bytes_in(int n) {} static inline void update_completion_einval_errs(void) {} diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c index fcc0cf4df637d..5b90cf0fb0e41 100644 --- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c +++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c @@ -884,8 +884,9 @@ static struct buffer_desc *chainup_buffers(struct device *dev, ptr = sg_virt(sg); next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys); if (!next_buf) { - buf = NULL; - break; + buf->next = NULL; + buf->phys_next = 0; + return NULL; } sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir); buf->next = next_buf; @@ -983,7 +984,7 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) unsigned int nbytes = req->cryptlen; enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; struct ablk_ctx *req_ctx = skcipher_request_ctx(req); - struct buffer_desc src_hook; + struct buffer_desc *buf, src_hook; struct device *dev = &pdev->dev; unsigned int offset; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? @@ -1025,22 +1026,24 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) /* This was never tested by Intel * for more than one dst buffer, I think. */ req_ctx->dst = NULL; - if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook, - flags, DMA_FROM_DEVICE)) - goto free_buf_dest; - src_direction = DMA_TO_DEVICE; + buf = chainup_buffers(dev, req->dst, nbytes, &dst_hook, + flags, DMA_FROM_DEVICE); req_ctx->dst = dst_hook.next; crypt->dst_buf = dst_hook.phys_next; + if (!buf) + goto free_buf_dest; + src_direction = DMA_TO_DEVICE; } else { req_ctx->dst = NULL; } req_ctx->src = NULL; - if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags, - src_direction)) - goto free_buf_src; - + buf = chainup_buffers(dev, req->src, nbytes, &src_hook, flags, + src_direction); req_ctx->src = src_hook.next; crypt->src_buf = src_hook.phys_next; + if (!buf) + goto free_buf_src; + crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK; qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); BUG_ON(qmgr_stat_overflow(send_qid)); diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c index 8a8f6c81e010c..419f88af1031b 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -1541,7 +1541,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) /* * If any algorithm fails to register, all preceding algorithms that - * were successfully registered will be automatically unregistered. + * were registered in the same call are automatically unregistered. */ ret = crypto_engine_register_aeads(algs_aead, ARRAY_SIZE(algs_aead)); if (ret) @@ -1549,7 +1549,7 @@ static int register_aes_algs(struct ocs_aes_dev *aes_dev) ret = crypto_engine_register_skciphers(algs, ARRAY_SIZE(algs)); if (ret) - crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs)); + crypto_engine_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead)); return ret; } @@ -1561,6 +1561,7 @@ static const struct of_device_id kmb_ocs_aes_of_match[] = { }, {} }; +MODULE_DEVICE_TABLE(of, kmb_ocs_aes_of_match); static void kmb_ocs_aes_remove(struct platform_device *pdev) { @@ -1602,6 +1603,8 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) if (IS_ERR(aes_dev->base_reg)) return PTR_ERR(aes_dev->base_reg); + init_completion(&aes_dev->irq_completion); + /* Get and request IRQ */ aes_dev->irq = platform_get_irq(pdev, 0); if (aes_dev->irq < 0) @@ -1619,8 +1622,6 @@ static int kmb_ocs_aes_probe(struct platform_device *pdev) list_add_tail(&aes_dev->list, &ocs_aes.dev_list); spin_unlock(&ocs_aes.lock); - init_completion(&aes_dev->irq_completion); - /* Initialize crypto engine */ aes_dev->engine = crypto_engine_alloc_init(dev, true); if (!aes_dev->engine) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index f26cadc19d6bb..ea826333c45f4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -190,6 +190,8 @@ static void adf_device_reset_worker(struct work_struct *work) queue_work(device_sriov_wq, &sriov_data.sriov_work); if (wait_for_completion_timeout(&sriov_data.compl, wait_jiffies)) adf_pf2vf_notify_restarted(accel_dev); + else + cancel_work_sync(&sriov_data.sriov_work); adf_dev_restarted_notify(accel_dev); clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg.c b/drivers/crypto/intel/qat/qat_common/adf_cfg.c index 97273da40b438..7180dd2b504a1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include #include "adf_accel_devices.h" @@ -284,16 +285,13 @@ int adf_cfg_add_key_value_param(struct adf_accel_dev *accel_dev, return -ENOMEM; INIT_LIST_HEAD(&key_val->list); - strscpy(key_val->key, key, sizeof(key_val->key)); + strscpy(key_val->key, key); if (type == ADF_DEC) { snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, "%ld", (*((long *)val))); } else if (type == ADF_STR) { - strscpy(key_val->val, (char *)val, sizeof(key_val->val)); - } else if (type == ADF_HEX) { - snprintf(key_val->val, ADF_CFG_MAX_VAL_LEN_IN_BYTES, - "0x%lx", (unsigned long)val); + strscpy(key_val->val, (char *)val); } else { dev_err(&GET_DEV(accel_dev), "Unknown type given.\n"); kfree(key_val); @@ -350,7 +348,7 @@ int adf_cfg_section_add(struct adf_accel_dev *accel_dev, const char *name) if (!sec) return -ENOMEM; - strscpy(sec->name, name, sizeof(sec->name)); + strscpy(sec->name, name); INIT_LIST_HEAD(&sec->param_head); down_write(&cfg->lock); list_add_tail(&sec->list, &cfg->sec_list); diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h index d63f4dcccbb5d..5922d018f5b98 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_common.h @@ -28,7 +28,6 @@ enum adf_cfg_service_type { enum adf_cfg_val_type { ADF_DEC, - ADF_HEX, ADF_STR }; diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 7d00bcb41ce70..11cba347d12d1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -60,7 +60,7 @@ static int adf_service_string_to_mask(struct adf_accel_dev *accel_dev, const cha if (len > ADF_CFG_MAX_VAL_LEN_IN_BYTES - 1) return -EINVAL; - strscpy(services, buf, ADF_CFG_MAX_VAL_LEN_IN_BYTES); + strscpy(services, buf); substr = services; while ((token = strsep(&substr, ADF_SERVICES_DELIMITER))) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c index 41cc763a74aa2..df4600ab72d6b 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_mstate_mgr.c @@ -2,6 +2,7 @@ /* Copyright(c) 2024 Intel Corporation */ #include +#include #include #include "adf_mstate_mgr.h" @@ -158,7 +159,7 @@ static struct adf_mstate_sect_h *adf_mstate_sect_add_header(struct adf_mstate_mg return NULL; } - strscpy(sect->id, id, sizeof(sect->id)); + strscpy(sect->id, id); sect->size = 0; sect->sub_sects = 0; mgr->state += sizeof(*sect); diff --git a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c index 6c22bc9b28e47..ad45ca7e8eb86 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c +++ b/drivers/crypto/intel/qat/qat_common/adf_transport_debug.c @@ -2,6 +2,7 @@ /* Copyright(c) 2014 - 2020 Intel Corporation */ #include #include +#include #include #include "adf_accel_devices.h" #include "adf_transport_internal.h" @@ -103,7 +104,7 @@ int adf_ring_debugfs_add(struct adf_etr_ring_data *ring, const char *name) if (!ring_debug) return -ENOMEM; - strscpy(ring_debug->ring_name, name, sizeof(ring_debug->ring_name)); + strscpy(ring_debug->ring_name, name); snprintf(entry_name, sizeof(entry_name), "ring_%02d", ring->ring_number); diff --git a/drivers/crypto/intel/qat/qat_common/qat_algs.c b/drivers/crypto/intel/qat/qat_common/qat_algs.c index 7f638a62e3ade..91663805d9e60 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_algs.c @@ -405,6 +405,7 @@ static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen, memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE, AES_BLOCK_SIZE); } + memzero_explicit(&aes_expanded, sizeof(aes_expanded)); } static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx, diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 53a4db5507ec2..69db4178a419c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -2,6 +2,7 @@ /* Copyright(c) 2022 Intel Corporation */ #include #include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_transport.h" @@ -123,7 +124,7 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) int i; INIT_LIST_HEAD(&accel_dev->compression_list); - strscpy(key, ADF_NUM_DC, sizeof(key)); + strscpy(key, ADF_NUM_DC); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); if (ret) return ret; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index 133ebc9982362..595b2fd84667e 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -353,7 +353,7 @@ static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) for_each_sg(req->src, src, sg_nents(req->src), i) { src_buf = sg_virt(src); - len = sg_dma_len(src); + len = src->length; tlen += len; limit_hit = tlen > req->cryptlen; diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 149ebd77710bf..16d5c617d5ee8 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -938,6 +938,20 @@ static int omap_des_get_pdev(struct omap_des_dev *dd, return 0; } +static void omap_des_unregister_algs(const struct omap_des_pdata *pdata) +{ + struct omap_des_algs_info *alg_info; + int i; + + for (i = pdata->algs_info_size - 1; i >= 0; i--) { + alg_info = &pdata->algs_info[i]; + + crypto_engine_unregister_skciphers(alg_info->algs_list, + alg_info->registered); + alg_info->registered = 0; + } +} + static int omap_des_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1043,11 +1057,7 @@ static int omap_des_probe(struct platform_device *pdev) return 0; err_algs: - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); - + omap_des_unregister_algs(dd->pdata); err_engine: if (dd->engine) crypto_engine_exit(dd->engine); @@ -1067,16 +1077,12 @@ static int omap_des_probe(struct platform_device *pdev) static void omap_des_remove(struct platform_device *pdev) { struct omap_des_dev *dd = platform_get_drvdata(pdev); - int i, j; spin_lock_bh(&list_lock); list_del(&dd->list); spin_unlock_bh(&list_lock); - for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) - crypto_engine_unregister_skcipher( - &dd->pdata->algs_info[i].algs_list[j]); + omap_des_unregister_algs(dd->pdata); cancel_work_sync(&dd->done_task); omap_des_dma_cleanup(dd); diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c index 97b56e92ea33f..d1f5b17b5313d 100644 --- a/drivers/crypto/qce/aead.c +++ b/drivers/crypto/qce/aead.c @@ -8,8 +8,6 @@ #include #include #include -#include -#include #include #include #include "aead.h" @@ -203,7 +201,7 @@ qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) /* Get the msg */ msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); - rctx->adata = kzalloc((ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) * + rctx->adata = kzalloc(ALIGN(assoclen + MAX_CCM_ADATA_HEADER_LEN, 16) * sizeof(unsigned char), GFP_ATOMIC); if (!rctx->adata) return -ENOMEM; @@ -597,7 +595,6 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_authenc_keys authenc_keys; unsigned long flags = to_aead_tmpl(tfm)->alg_flags; - u32 _key[6]; int err; err = crypto_authenc_extractkeys(&authenc_keys, key, keylen); @@ -608,26 +605,7 @@ static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) return -EINVAL; - if (IS_DES(flags)) { - err = verify_aead_des_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); - if (err) - return err; - } else if (IS_3DES(flags)) { - err = verify_aead_des3_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); - if (err) - return err; - /* - * The crypto engine does not support any two keys - * being the same for triple des algorithms. The - * verify_skcipher_des3_key does not check for all the - * below conditions. Schedule fallback in this case. - */ - memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE); - if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || - !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || - !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) - ctx->need_fallback = true; - } else if (IS_AES(flags)) { + if (IS_AES(flags)) { /* No random key sizes */ if (authenc_keys.enckeylen != AES_KEYSIZE_128 && authenc_keys.enckeylen != AES_KEYSIZE_192 && @@ -698,38 +676,6 @@ struct qce_aead_def { }; static const struct qce_aead_def aead_def[] = { - { - .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, - .name = "authenc(hmac(sha1),cbc(des))", - .drv_name = "authenc-hmac-sha1-cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, - .name = "authenc(hmac(sha1),cbc(des3_ede))", - .drv_name = "authenc-hmac-sha1-cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA1_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, - .name = "authenc(hmac(sha256),cbc(des))", - .drv_name = "authenc-hmac-sha256-cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, - .name = "authenc(hmac(sha256),cbc(des3_ede))", - .drv_name = "authenc-hmac-sha256-cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .maxauthsize = SHA256_DIGEST_SIZE, - }, { .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, .name = "authenc(hmac(sha256),cbc(aes))", diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index 04253a8d33409..0754a1bdd9d64 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include "cipher.h" @@ -115,18 +114,16 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size) cfg |= AUTH_KEY_SZ_AES256 << AUTH_KEY_SIZE_SHIFT; } - if (IS_SHA1(flags) || IS_SHA1_HMAC(flags)) - cfg |= AUTH_SIZE_SHA1 << AUTH_SIZE_SHIFT; - else if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) + if (IS_SHA256(flags) || IS_SHA256_HMAC(flags)) cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT; else if (IS_CMAC(flags)) cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT; else if (IS_CCM(flags)) cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT; - if (IS_SHA1(flags) || IS_SHA256(flags)) + if (IS_SHA256(flags)) cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT; - else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) + else if (IS_SHA256_HMAC(flags)) cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT; else if (IS_CCM(flags)) cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT; @@ -191,7 +188,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) else qce_cpu_to_be32p_array(auth, rctx->digest, digestsize); - iv_words = (IS_SHA1(rctx->flags) || IS_SHA1_HMAC(rctx->flags)) ? 5 : 8; + iv_words = 8; qce_write_array(qce, REG_AUTH_IV0, (u32 *)auth, iv_words); if (rctx->first_blk) @@ -243,19 +240,8 @@ static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) if (IS_AES(flags)) cfg |= ENCR_ALG_AES << ENCR_ALG_SHIFT; - else if (IS_DES(flags) || IS_3DES(flags)) - cfg |= ENCR_ALG_DES << ENCR_ALG_SHIFT; - - if (IS_DES(flags)) - cfg |= ENCR_KEY_SZ_DES << ENCR_KEY_SZ_SHIFT; - - if (IS_3DES(flags)) - cfg |= ENCR_KEY_SZ_3DES << ENCR_KEY_SZ_SHIFT; switch (flags & QCE_MODE_MASK) { - case QCE_MODE_ECB: - cfg |= ENCR_MODE_ECB << ENCR_MODE_SHIFT; - break; case QCE_MODE_CBC: cfg |= ENCR_MODE_CBC << ENCR_MODE_SHIFT; break; @@ -340,13 +326,7 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) encr_cfg = qce_encr_cfg(flags, keylen); - if (IS_DES(flags)) { - enciv_words = 2; - enckey_words = 2; - } else if (IS_3DES(flags)) { - enciv_words = 2; - enckey_words = 6; - } else if (IS_AES(flags)) { + if (IS_AES(flags)) { if (IS_XTS(flags)) qce_xtskey(qce, ctx->enc_key, ctx->enc_keylen, rctx->cryptlen); @@ -357,14 +337,12 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) qce_write_array(qce, REG_ENCR_KEY0, (u32 *)enckey, enckey_words); - if (!IS_ECB(flags)) { - if (IS_XTS(flags)) - qce_xts_swapiv(enciv, rctx->iv, ivsize); - else - qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); + if (IS_XTS(flags)) + qce_xts_swapiv(enciv, rctx->iv, ivsize); + else + qce_cpu_to_be32p_array(enciv, rctx->iv, ivsize); - qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); - } + qce_write_array(qce, REG_CNTR0_IV0, (u32 *)enciv, enciv_words); if (IS_ENCRYPT(flags)) encr_cfg |= BIT(ENCODE_SHIFT); @@ -393,10 +371,6 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) #endif #ifdef CONFIG_CRYPTO_DEV_QCE_AEAD -static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 -}; - static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 @@ -473,13 +447,8 @@ static int qce_setup_regs_aead(struct crypto_async_request *async_req) /* Write initial authentication IV only for HMAC algorithms */ if (IS_SHA_HMAC(rctx->flags)) { /* Write default authentication iv */ - if (IS_SHA1_HMAC(rctx->flags)) { - auth_ivsize = SHA1_DIGEST_SIZE; - memcpy(authiv, std_iv_sha1, auth_ivsize); - } else if (IS_SHA256_HMAC(rctx->flags)) { - auth_ivsize = SHA256_DIGEST_SIZE; - memcpy(authiv, std_iv_sha256, auth_ivsize); - } + auth_ivsize = SHA256_DIGEST_SIZE; + memcpy(authiv, std_iv_sha256, auth_ivsize); authiv_words = auth_ivsize / sizeof(u32); qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words); } else if (IS_CCM(rctx->flags)) { diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h index 02e63ad9f2455..9cd2e6ed8bbb0 100644 --- a/drivers/crypto/qce/common.h +++ b/drivers/crypto/qce/common.h @@ -22,7 +22,7 @@ /* IV length in bytes */ #define QCE_AES_IV_LENGTH AES_BLOCK_SIZE -/* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ +/* max of AES_BLOCK_SIZE */ #define QCE_MAX_IV_SIZE AES_BLOCK_SIZE /* maximum nonce bytes */ @@ -33,14 +33,10 @@ #define QCE_MAX_ALIGN_SIZE 64 /* cipher algorithms */ -#define QCE_ALG_DES BIT(0) -#define QCE_ALG_3DES BIT(1) #define QCE_ALG_AES BIT(2) /* hash and hmac algorithms */ -#define QCE_HASH_SHA1 BIT(3) #define QCE_HASH_SHA256 BIT(4) -#define QCE_HASH_SHA1_HMAC BIT(5) #define QCE_HASH_SHA256_HMAC BIT(6) #define QCE_HASH_AES_CMAC BIT(7) @@ -58,21 +54,15 @@ #define QCE_ENCRYPT BIT(30) #define QCE_DECRYPT BIT(31) -#define IS_DES(flags) (flags & QCE_ALG_DES) -#define IS_3DES(flags) (flags & QCE_ALG_3DES) #define IS_AES(flags) (flags & QCE_ALG_AES) -#define IS_SHA1(flags) (flags & QCE_HASH_SHA1) #define IS_SHA256(flags) (flags & QCE_HASH_SHA256) -#define IS_SHA1_HMAC(flags) (flags & QCE_HASH_SHA1_HMAC) #define IS_SHA256_HMAC(flags) (flags & QCE_HASH_SHA256_HMAC) #define IS_CMAC(flags) (flags & QCE_HASH_AES_CMAC) -#define IS_SHA(flags) (IS_SHA1(flags) || IS_SHA256(flags)) -#define IS_SHA_HMAC(flags) \ - (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) +#define IS_SHA(flags) IS_SHA256(flags) +#define IS_SHA_HMAC(flags) IS_SHA256_HMAC(flags) #define IS_CBC(mode) (mode & QCE_MODE_CBC) -#define IS_ECB(mode) (mode & QCE_MODE_ECB) #define IS_CTR(mode) (mode & QCE_MODE_CTR) #define IS_XTS(mode) (mode & QCE_MODE_XTS) #define IS_CCM(mode) (mode & QCE_MODE_CCM) diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index e95e84486d9ae..f5bccb3453740 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -60,7 +60,7 @@ static int devm_qce_register_algs(struct qce_device *qce) ret = ops->register_algs(qce); if (ret) { for (j = i - 1; j >= 0; j--) - ops->unregister_algs(qce); + qce_ops[j]->unregister_algs(qce); return ret; } } diff --git a/drivers/crypto/qce/regs-v5.h b/drivers/crypto/qce/regs-v5.h index d59ed27989062..431a7db1a4e72 100644 --- a/drivers/crypto/qce/regs-v5.h +++ b/drivers/crypto/qce/regs-v5.h @@ -203,7 +203,6 @@ #define AUTH_SIZE_SHIFT 9 #define AUTH_SIZE_MASK GENMASK(13, 9) -#define AUTH_SIZE_SHA1 0 #define AUTH_SIZE_SHA256 1 #define AUTH_SIZE_ENUM_1_BYTES 0 #define AUTH_SIZE_ENUM_2_BYTES 1 @@ -284,15 +283,12 @@ #define ENCR_KEY_SZ_SHIFT 3 #define ENCR_KEY_SZ_MASK GENMASK(5, 3) -#define ENCR_KEY_SZ_DES 0 -#define ENCR_KEY_SZ_3DES 1 #define ENCR_KEY_SZ_AES128 0 #define ENCR_KEY_SZ_AES256 2 #define ENCR_ALG_SHIFT 0 #define ENCR_ALG_MASK GENMASK(2, 0) #define ENCR_ALG_NONE 0 -#define ENCR_ALG_DES 1 #define ENCR_ALG_AES 2 #define ENCR_ALG_KASUMI 4 #define ENCR_ALG_SNOW_3G 5 diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 71b748183cfa8..fc2b891983628 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -24,10 +24,6 @@ struct qce_sha_saved_state { static LIST_HEAD(ahash_algs); -static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 -}; - static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 @@ -348,9 +344,7 @@ static int qce_ahash_hmac_setkey(struct crypto_ahash *tfm, const u8 *key, return 0; } - if (digestsize == SHA1_DIGEST_SIZE) - alg_name = "sha1-qce"; - else if (digestsize == SHA256_DIGEST_SIZE) + if (digestsize == SHA256_DIGEST_SIZE) alg_name = "sha256-qce"; else return -EINVAL; @@ -411,15 +405,6 @@ struct qce_ahash_def { }; static const struct qce_ahash_def ahash_def[] = { - { - .flags = QCE_HASH_SHA1, - .name = "sha1", - .drv_name = "sha1-qce", - .digestsize = SHA1_DIGEST_SIZE, - .blocksize = SHA1_BLOCK_SIZE, - .statesize = sizeof(struct qce_sha_saved_state), - .std_iv = std_iv_sha1, - }, { .flags = QCE_HASH_SHA256, .name = "sha256", @@ -429,15 +414,6 @@ static const struct qce_ahash_def ahash_def[] = { .statesize = sizeof(struct qce_sha_saved_state), .std_iv = std_iv_sha256, }, - { - .flags = QCE_HASH_SHA1_HMAC, - .name = "hmac(sha1)", - .drv_name = "hmac-sha1-qce", - .digestsize = SHA1_DIGEST_SIZE, - .blocksize = SHA1_BLOCK_SIZE, - .statesize = sizeof(struct qce_sha_saved_state), - .std_iv = std_iv_sha1, - }, { .flags = QCE_HASH_SHA256_HMAC, .name = "hmac(sha256)", @@ -475,9 +451,7 @@ static int qce_ahash_register_one(const struct qce_ahash_def *def, alg->halg.digestsize = def->digestsize; alg->halg.statesize = def->statesize; - if (IS_SHA1(def->flags)) - tmpl->hash_zero = sha1_zero_message_hash; - else if (IS_SHA256(def->flags)) + if (IS_SHA256(def->flags)) tmpl->hash_zero = sha256_zero_message_hash; base = &alg->halg.base; diff --git a/drivers/crypto/qce/sha.h b/drivers/crypto/qce/sha.h index a22695361f165..cb822fc334dc1 100644 --- a/drivers/crypto/qce/sha.h +++ b/drivers/crypto/qce/sha.h @@ -7,7 +7,6 @@ #define _SHA_H_ #include -#include #include #include "common.h" diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index ffb334eb5b346..61f6450bfd0aa 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include "cipher.h" @@ -208,51 +207,6 @@ static int qce_skcipher_setkey(struct crypto_skcipher *ablk, const u8 *key, return ret; } -static int qce_des_setkey(struct crypto_skcipher *ablk, const u8 *key, - unsigned int keylen) -{ - struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk); - int err; - - err = verify_skcipher_des_key(ablk, key); - if (err) - return err; - - ctx->enc_keylen = keylen; - memcpy(ctx->enc_key, key, keylen); - return 0; -} - -static int qce_des3_setkey(struct crypto_skcipher *ablk, const u8 *key, - unsigned int keylen) -{ - struct qce_cipher_ctx *ctx = crypto_skcipher_ctx(ablk); - u32 _key[6]; - int err; - - err = verify_skcipher_des3_key(ablk, key); - if (err) - return err; - - /* - * The crypto engine does not support any two keys - * being the same for triple des algorithms. The - * verify_skcipher_des3_key does not check for all the - * below conditions. Return -ENOKEY in case any two keys - * are the same. Revisit to see if a fallback cipher - * is needed to handle this condition. - */ - memcpy(_key, key, DES3_EDE_KEY_SIZE); - if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || - !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || - !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) - return -ENOKEY; - - ctx->enc_keylen = keylen; - memcpy(ctx->enc_key, key, keylen); - return 0; -} - static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); @@ -275,7 +229,7 @@ static int qce_skcipher_crypt(struct skcipher_request *req, int encrypt) * ECB and CBC algorithms require message lengths to be * multiples of block size. */ - if (IS_ECB(rctx->flags) || IS_CBC(rctx->flags)) + if (IS_CBC(rctx->flags)) if (!IS_ALIGNED(req->cryptlen, blocksize)) return -EINVAL; @@ -358,15 +312,6 @@ struct qce_skcipher_def { }; static const struct qce_skcipher_def skcipher_def[] = { - { - .flags = QCE_ALG_AES | QCE_MODE_ECB, - .name = "ecb(aes)", - .drv_name = "ecb-aes-qce", - .blocksize = AES_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - }, { .flags = QCE_ALG_AES | QCE_MODE_CBC, .name = "cbc(aes)", @@ -395,42 +340,6 @@ static const struct qce_skcipher_def skcipher_def[] = { .min_keysize = AES_MIN_KEY_SIZE * 2, .max_keysize = AES_MAX_KEY_SIZE * 2, }, - { - .flags = QCE_ALG_DES | QCE_MODE_ECB, - .name = "ecb(des)", - .drv_name = "ecb-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - }, - { - .flags = QCE_ALG_DES | QCE_MODE_CBC, - .name = "cbc(des)", - .drv_name = "cbc-des-qce", - .blocksize = DES_BLOCK_SIZE, - .ivsize = DES_BLOCK_SIZE, - .min_keysize = DES_KEY_SIZE, - .max_keysize = DES_KEY_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_ECB, - .name = "ecb(des3_ede)", - .drv_name = "ecb-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = 0, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - }, - { - .flags = QCE_ALG_3DES | QCE_MODE_CBC, - .name = "cbc(des3_ede)", - .drv_name = "cbc-3des-qce", - .blocksize = DES3_EDE_BLOCK_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, - .min_keysize = DES3_EDE_KEY_SIZE, - .max_keysize = DES3_EDE_KEY_SIZE, - }, }; static int qce_skcipher_register_one(const struct qce_skcipher_def *def, @@ -455,9 +364,7 @@ static int qce_skcipher_register_one(const struct qce_skcipher_def *def, alg->ivsize = def->ivsize; alg->min_keysize = def->min_keysize; alg->max_keysize = def->max_keysize; - alg->setkey = IS_3DES(def->flags) ? qce_des3_setkey : - IS_DES(def->flags) ? qce_des_setkey : - qce_skcipher_setkey; + alg->setkey = qce_skcipher_setkey; alg->encrypt = qce_skcipher_encrypt; alg->decrypt = qce_skcipher_decrypt; diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 0685ba122e8ac..233aa226d994d 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -3,10 +3,8 @@ // // Based on msm-rng.c and downstream driver -#include #include #include -#include #include #include #include @@ -32,24 +30,15 @@ #define QCOM_TRNG_QUALITY 1024 struct qcom_rng { - struct mutex lock; void __iomem *base; struct clk *clk; struct hwrng hwrng; - struct qcom_rng_match_data *match_data; -}; - -struct qcom_rng_ctx { - struct qcom_rng *rng; }; struct qcom_rng_match_data { - bool skip_init; bool hwrng_support; }; -static struct qcom_rng *qcom_rng_dev; - static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) { unsigned int currsize = 0; @@ -65,8 +54,6 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) return ret; val = readl_relaxed(rng->base + PRNG_DATA_OUT); - if (!val) - return -EINVAL; if ((max - currsize) >= WORD_SZ) { memcpy(data, &val, WORD_SZ); @@ -82,35 +69,11 @@ static int qcom_rng_read(struct qcom_rng *rng, u8 *data, unsigned int max) return currsize; } -static int qcom_rng_generate(struct crypto_rng *tfm, - const u8 *src, unsigned int slen, - u8 *dstn, unsigned int dlen) +static int qcom_hwrng_init(struct hwrng *hwrng) { - struct qcom_rng_ctx *ctx = crypto_rng_ctx(tfm); - struct qcom_rng *rng = ctx->rng; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - mutex_lock(&rng->lock); - - ret = qcom_rng_read(rng, dstn, dlen); - - mutex_unlock(&rng->lock); - clk_disable_unprepare(rng->clk); - - if (ret >= 0) - ret = 0; - - return ret; -} + struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng); -static int qcom_rng_seed(struct crypto_rng *tfm, const u8 *seed, - unsigned int slen) -{ - return 0; + return clk_prepare_enable(qrng->clk); } static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wait) @@ -120,74 +83,33 @@ static int qcom_hwrng_read(struct hwrng *hwrng, void *data, size_t max, bool wai return qcom_rng_read(qrng, data, max); } -static int qcom_rng_enable(struct qcom_rng *rng) +static void qcom_hwrng_cleanup(struct hwrng *hwrng) { - u32 val; - int ret; - - ret = clk_prepare_enable(rng->clk); - if (ret) - return ret; - - /* Enable PRNG only if it is not already enabled */ - val = readl_relaxed(rng->base + PRNG_CONFIG); - if (val & PRNG_CONFIG_HW_ENABLE) - goto already_enabled; - - val = readl_relaxed(rng->base + PRNG_LFSR_CFG); - val &= ~PRNG_LFSR_CFG_MASK; - val |= PRNG_LFSR_CFG_CLOCKS; - writel(val, rng->base + PRNG_LFSR_CFG); - - val = readl_relaxed(rng->base + PRNG_CONFIG); - val |= PRNG_CONFIG_HW_ENABLE; - writel(val, rng->base + PRNG_CONFIG); - -already_enabled: - clk_disable_unprepare(rng->clk); - - return 0; -} - -static int qcom_rng_init(struct crypto_tfm *tfm) -{ - struct qcom_rng_ctx *ctx = crypto_tfm_ctx(tfm); - - ctx->rng = qcom_rng_dev; - - if (!ctx->rng->match_data->skip_init) - return qcom_rng_enable(ctx->rng); + struct qcom_rng *qrng = container_of(hwrng, struct qcom_rng, hwrng); - return 0; + clk_disable_unprepare(qrng->clk); } -static struct rng_alg qcom_rng_alg = { - .generate = qcom_rng_generate, - .seed = qcom_rng_seed, - .seedsize = 0, - .base = { - .cra_name = "stdrng", - .cra_driver_name = "qcom-rng", - .cra_flags = CRYPTO_ALG_TYPE_RNG, - .cra_priority = 300, - .cra_ctxsize = sizeof(struct qcom_rng_ctx), - .cra_module = THIS_MODULE, - .cra_init = qcom_rng_init, - } -}; - static int qcom_rng_probe(struct platform_device *pdev) { + const struct qcom_rng_match_data *match_data; struct qcom_rng *rng; int ret; + match_data = device_get_match_data(&pdev->dev); + if (match_data == NULL || !match_data->hwrng_support) { + dev_info(&pdev->dev, "TRNG support not detected\n"); + /* + * In this case the driver does nothing except the dev_info(), + * but bind the device anyway to avoid effects on GCC state. + */ + return 0; + } + rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); if (!rng) return -ENOMEM; - platform_set_drvdata(pdev, rng); - mutex_init(&rng->lock); - rng->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rng->base)) return PTR_ERR(rng->base); @@ -196,53 +118,26 @@ static int qcom_rng_probe(struct platform_device *pdev) if (IS_ERR(rng->clk)) return PTR_ERR(rng->clk); - rng->match_data = (struct qcom_rng_match_data *)device_get_match_data(&pdev->dev); - - qcom_rng_dev = rng; - ret = crypto_register_rng(&qcom_rng_alg); - if (ret) { - dev_err(&pdev->dev, "Register crypto rng failed: %d\n", ret); - qcom_rng_dev = NULL; - return ret; - } - - if (rng->match_data->hwrng_support) { - rng->hwrng.name = "qcom_hwrng"; - rng->hwrng.read = qcom_hwrng_read; - rng->hwrng.quality = QCOM_TRNG_QUALITY; - ret = devm_hwrng_register(&pdev->dev, &rng->hwrng); - if (ret) { - dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret); - qcom_rng_dev = NULL; - goto fail; - } - } - - return ret; -fail: - crypto_unregister_rng(&qcom_rng_alg); + rng->hwrng.name = "qcom_hwrng"; + rng->hwrng.init = qcom_hwrng_init; + rng->hwrng.read = qcom_hwrng_read; + rng->hwrng.cleanup = qcom_hwrng_cleanup; + rng->hwrng.quality = QCOM_TRNG_QUALITY; + ret = devm_hwrng_register(&pdev->dev, &rng->hwrng); + if (ret) + dev_err(&pdev->dev, "Register hwrng failed: %d\n", ret); return ret; } -static void qcom_rng_remove(struct platform_device *pdev) -{ - crypto_unregister_rng(&qcom_rng_alg); - - qcom_rng_dev = NULL; -} - static struct qcom_rng_match_data qcom_prng_match_data = { - .skip_init = false, .hwrng_support = false, }; static struct qcom_rng_match_data qcom_prng_ee_match_data = { - .skip_init = true, .hwrng_support = false, }; static struct qcom_rng_match_data qcom_trng_match_data = { - .skip_init = true, .hwrng_support = true, }; @@ -262,7 +157,6 @@ MODULE_DEVICE_TABLE(of, qcom_rng_of_match); static struct platform_driver qcom_rng_driver = { .probe = qcom_rng_probe, - .remove = qcom_rng_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = of_match_ptr(qcom_rng_of_match), diff --git a/drivers/crypto/rockchip/rk3288_crypto_ahash.c b/drivers/crypto/rockchip/rk3288_crypto_ahash.c index b9f5a8b42e661..d3482619aa2f1 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_ahash.c +++ b/drivers/crypto/rockchip/rk3288_crypto_ahash.c @@ -324,7 +324,12 @@ static int rk_hash_run(struct crypto_engine *engine, void *breq) * efficiency, and make it response quickly when dma * complete. */ - readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, v == 0, 10, 1000); + err = readl_poll_timeout(rkc->reg + RK_CRYPTO_HASH_STS, v, + v == 0, 10, 1000); + if (err) { + dev_err(rkc->dev, "HASH idle timeout\n"); + goto theend; + } for (i = 0; i < crypto_ahash_digestsize(tfm) / 4; i++) { v = readl(rkc->reg + RK_CRYPTO_HASH_DOUT_0 + i * 4); diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 52fe4baeff934..73034aeb63862 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -2395,7 +2395,10 @@ static int sa_ul_probe(struct platform_device *pdev) return ret; } - sa_init_mem(dev_data); + ret = sa_init_mem(dev_data); + if (ret) + goto disable_pm; + ret = sa_dma_init(dev_data); if (ret) goto destroy_dma_pool; @@ -2430,6 +2433,7 @@ static int sa_ul_probe(struct platform_device *pdev) destroy_dma_pool: dma_pool_destroy(dev_data->sc_pool); +disable_pm: pm_runtime_put_sync(dev); pm_runtime_disable(dev); diff --git a/drivers/crypto/starfive/jh7110-aes.c b/drivers/crypto/starfive/jh7110-aes.c index 01195664cc7cd..8e02275a70798 100644 --- a/drivers/crypto/starfive/jh7110-aes.c +++ b/drivers/crypto/starfive/jh7110-aes.c @@ -677,7 +677,7 @@ static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq if (cryp->total_in) sg_zero_buffer(rctx->in_sg, sg_nents(rctx->in_sg), - sg_dma_len(rctx->in_sg) - cryp->total_in, + rctx->in_sg->length - cryp->total_in, cryp->total_in); ctx->rctx = rctx; diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index 9094c03e991f6..0fd1d70358996 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -45,7 +45,6 @@ struct tegra_aes_reqctx { struct tegra_aead_ctx { struct tegra_se *se; - unsigned int authsize; u32 alg; u32 key_id; u32 keylen; @@ -1290,7 +1289,7 @@ static int tegra_gcm_do_one_req(struct crypto_engine *engine, void *areq) if (rctx->encrypt) rctx->cryptlen = req->cryptlen; else - rctx->cryptlen = req->cryptlen - ctx->authsize; + rctx->cryptlen = req->cryptlen - rctx->authsize; memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE); rctx->iv[3] = (1 << 24); @@ -1394,8 +1393,6 @@ static int tegra_aead_cra_init(struct crypto_aead *tfm) static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); - switch (authsize) { case 4: case 6: @@ -1404,28 +1401,15 @@ static int tegra_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize case 12: case 14: case 16: - break; + return 0; default: return -EINVAL; } - - ctx->authsize = authsize; - - return 0; } static int tegra_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { - struct tegra_aead_ctx *ctx = crypto_aead_ctx(tfm); - int ret; - - ret = crypto_gcm_check_authsize(authsize); - if (ret) - return ret; - - ctx->authsize = authsize; - - return 0; + return crypto_gcm_check_authsize(authsize); } static void tegra_aead_cra_exit(struct crypto_aead *tfm) diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index 2755f19ad05e9..77b923a6b7c18 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -52,7 +52,7 @@ tegra_se_cmdbuf_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_dire return ERR_PTR(-ENOMEM); kref_init(&map->ref); - map->bo = host1x_bo_get(bo); + map->bo = bo; map->direction = direction; map->dev = dev; @@ -93,7 +93,6 @@ static void tegra_se_cmdbuf_unpin(struct host1x_bo_mapping *map) dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); sg_free_table(map->sgt); kfree(map->sgt); - host1x_bo_put(map->bo); kfree(map); } diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index 2e44915c9f239..090280af6b0cd 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -88,7 +88,8 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * } /* actual length may be less than dst buffer */ - akcipher_req->dst_len = len - sizeof(vc_req->status); + akcipher_req->dst_len = min_t(unsigned int, len - sizeof(vc_req->status), + akcipher_req->dst_len); sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), vc_akcipher_req->dst_buf, akcipher_req->dst_len); virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c index 3b193471cc94c..7f9762ea042bb 100644 --- a/drivers/crypto/xilinx/xilinx-trng.c +++ b/drivers/crypto/xilinx/xilinx-trng.c @@ -91,8 +91,8 @@ static void xtrng_softreset(struct xilinx_rng *rng) xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSRST_MASK, 0); } -/* Return no. of bytes read */ -static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) +/* Return no. of bytes read or a negative error before any data is read. */ +static int xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks32, bool wait) { int read = 0, ret; int timeout = 1; @@ -107,8 +107,11 @@ static size_t xtrng_readblock32(void __iomem *rng_base, __be32 *buf, int blocks3 ret = readl_poll_timeout(rng_base + TRNG_STATUS_OFFSET, val, (val & TRNG_STATUS_QCNT_MASK) == TRNG_STATUS_QCNT_16_BYTES, !!wait, timeout); - if (ret) + if (ret) { + if (!read) + return ret; break; + } for (idx = 0; idx < TRNG_READ_4_WORD; idx++) { *(buf + read) = cpu_to_be32(ioread32(rng_base + TRNG_CORE_OUTPUT_OFFSET)); @@ -123,27 +126,40 @@ static int xtrng_collect_random_data(struct xilinx_rng *rng, u8 *rand_gen_buf, { u8 randbuf[TRNG_SEC_STRENGTH_BYTES]; int byteleft, blocks, count = 0; + int full_blocks_bytes; int ret; byteleft = no_of_random_bytes & (TRNG_SEC_STRENGTH_BYTES - 1); blocks = no_of_random_bytes >> TRNG_SEC_STRENGTH_SHIFT; + full_blocks_bytes = blocks * TRNG_SEC_STRENGTH_BYTES; xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGSTART_MASK, TRNG_CTRL_PRNGSTART_MASK); if (blocks) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)rand_gen_buf, blocks, wait); - if (!ret) - return 0; + if (ret <= 0) { + count = ret; + goto out_stop; + } count += ret; + if (ret < full_blocks_bytes) + goto out_stop; } if (byteleft) { ret = xtrng_readblock32(rng->rng_base, (__be32 *)randbuf, 1, wait); + if (ret < 0) { + if (!count) + count = ret; + goto out_stop; + } if (!ret) - return count; - memcpy(rand_gen_buf + (blocks * TRNG_SEC_STRENGTH_BYTES), randbuf, byteleft); - count += byteleft; + goto out_stop; + ret = min(ret, no_of_random_bytes - count); + memcpy(rand_gen_buf + count, randbuf, ret); + count += ret; } +out_stop: xtrng_readwrite32(rng->rng_base + TRNG_CTRL_OFFSET, TRNG_CTRL_PRNGMODE_MASK | TRNG_CTRL_PRNGSTART_MASK, 0U); diff --git a/drivers/cxl/Kconfig b/drivers/cxl/Kconfig index 599e126a18eb7..4bdec46b2a6bb 100644 --- a/drivers/cxl/Kconfig +++ b/drivers/cxl/Kconfig @@ -234,4 +234,8 @@ config CXL_MCE def_bool y depends on X86_MCE && MEMORY_FAILURE +config CXL_RAS + def_bool y + depends on ACPI_APEI_GHES && PCIEAER && CXL_PCI + endif diff --git a/drivers/cxl/core/Makefile b/drivers/cxl/core/Makefile index 5ad8fef210b5c..b37f38d502d8c 100644 --- a/drivers/cxl/core/Makefile +++ b/drivers/cxl/core/Makefile @@ -14,9 +14,10 @@ cxl_core-y += pci.o cxl_core-y += hdm.o cxl_core-y += pmu.o cxl_core-y += cdat.o -cxl_core-y += ras.o cxl_core-$(CONFIG_TRACING) += trace.o cxl_core-$(CONFIG_CXL_REGION) += region.o cxl_core-$(CONFIG_CXL_MCE) += mce.o cxl_core-$(CONFIG_CXL_FEATURES) += features.o cxl_core-$(CONFIG_CXL_EDAC_MEM_FEATURES) += edac.o +cxl_core-$(CONFIG_CXL_RAS) += ras.o +cxl_core-$(CONFIG_CXL_RAS) += ras_rch.o diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 1fb66132b7777..724361195057e 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -144,8 +144,34 @@ int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c); int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port, struct access_coordinate *c); +#ifdef CONFIG_CXL_RAS int cxl_ras_init(void); void cxl_ras_exit(void); +bool cxl_handle_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base); +void cxl_handle_cor_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base); +void cxl_dport_map_rch_aer(struct cxl_dport *dport); +void cxl_disable_rch_root_ints(struct cxl_dport *dport); +void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds); +#else +static inline int cxl_ras_init(void) +{ + return 0; +} + +static inline void cxl_ras_exit(void) +{ +} + +static inline bool cxl_handle_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base) +{ + return false; +} +static inline void cxl_handle_cor_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base) { } +static inline void cxl_dport_map_rch_aer(struct cxl_dport *dport) { } +static inline void cxl_disable_rch_root_ints(struct cxl_dport *dport) { } +static inline void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } +#endif /* CONFIG_CXL_RAS */ + int cxl_gpf_port_setup(struct cxl_dport *dport); struct cxl_hdm; diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index c3cb2060bbeac..6e25223d3409b 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -225,7 +225,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, void *feat_out, size_t feat_out_size, u16 offset, u16 *return_code) { - size_t data_to_rd_size, size_out; + size_t data_to_rd_size; struct cxl_mbox_get_feat_in pi; struct cxl_mbox_cmd mbox_cmd; size_t data_rcvd_size = 0; @@ -237,7 +237,6 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, if (!feat_out || !feat_out_size) return 0; - size_out = min(feat_out_size, cxl_mbox->payload_size); uuid_copy(&pi.uuid, feat_uuid); pi.selection = selection; do { @@ -250,7 +249,7 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, .opcode = CXL_MBOX_OP_GET_FEATURE, .size_in = sizeof(pi), .payload_in = &pi, - .size_out = size_out, + .size_out = data_to_rd_size, .payload_out = feat_out + data_rcvd_size, .min_out = data_to_rd_size, }; @@ -471,6 +470,10 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, if (!count) return ERR_PTR(-EINVAL); + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload) || + count > out_size - offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -516,6 +519,9 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs, flags = le32_to_cpu(feat_in->flags); out_size = *out_len; + if (out_size < offsetof(struct fwctl_rpc_cxl_out, payload)) + return ERR_PTR(-EINVAL); + struct fwctl_rpc_cxl_out *rpc_out __free(kvfree) = kvzalloc(out_size, GFP_KERNEL); if (!rpc_out) @@ -649,7 +655,13 @@ static void *cxlctl_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope, struct cxl_memdev *cxlmd = fwctl_to_memdev(fwctl_dev); struct cxl_features_state *cxlfs = to_cxlfs(cxlmd->cxlds); const struct fwctl_rpc_cxl *rpc_in = in; - u16 opcode = rpc_in->opcode; + u16 opcode; + + if (in_len < sizeof(rpc_in->hdr) || + rpc_in->op_size > in_len - sizeof(rpc_in->hdr)) + return ERR_PTR(-EINVAL); + + opcode = rpc_in->opcode; if (!cxlctl_validate_hw_command(cxlfs, rpc_in, scope, opcode)) return ERR_PTR(-EINVAL); diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index e7a6452bf5445..e1c23aae2a2ed 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -11,7 +11,6 @@ #include "core.h" #include "trace.h" -#include "mce.h" static bool cxl_raw_allow_all; @@ -380,11 +379,7 @@ static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox_cmd, } } - /* Prepare to handle a full payload for variable sized output */ - if (out_size == CXL_VARIABLE_PAYLOAD) - mbox_cmd->size_out = cxl_mbox->payload_size; - else - mbox_cmd->size_out = out_size; + mbox_cmd->size_out = min_t(size_t, out_size, cxl_mbox->payload_size); if (mbox_cmd->size_out) { mbox_cmd->payload_out = kvzalloc(mbox_cmd->size_out, GFP_KERNEL); @@ -1454,6 +1449,11 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, if (rc) break; + if (!le16_to_cpu(po->count)) { + dev_dbg(&cxlmd->dev, "Poison empty payload!\n"); + break; + } + for (int i = 0; i < le16_to_cpu(po->count); i++) trace_cxl_poison(cxlmd, cxlr, &po->record[i], po->flags, po->overflow_ts, @@ -1524,7 +1524,6 @@ EXPORT_SYMBOL_NS_GPL(cxl_mailbox_init, "CXL"); struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) { struct cxl_memdev_state *mds; - int rc; mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL); if (!mds) { @@ -1539,12 +1538,6 @@ struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev) mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE; mds->cxlds.type = CXL_DEVTYPE_CLASSMEM; - rc = devm_cxl_register_mce_notifier(dev, &mds->mce_notifier); - if (rc == -EOPNOTSUPP) - dev_warn(dev, "CXL MCE unsupported\n"); - else if (rc) - return ERR_PTR(rc); - return mds; } EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, "CXL"); diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c index ff8d078c6ca1e..65fed913b2217 100644 --- a/drivers/cxl/core/mce.c +++ b/drivers/cxl/core/mce.c @@ -4,16 +4,16 @@ #include #include #include -#include +#include +#include "core.h" #include "mce.h" static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, void *data) { - struct cxl_memdev_state *mds = container_of(nb, struct cxl_memdev_state, - mce_notifier); - struct cxl_memdev *cxlmd = mds->cxlds.cxlmd; - struct cxl_port *endpoint = cxlmd->endpoint; + struct cxl_region *cxlr = container_of(nb, struct cxl_region, + mce_notifier); + struct cxl_region_params *p = &cxlr->params; struct mce *mce = data; u64 spa, spa_alias; unsigned long pfn; @@ -21,26 +21,25 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, if (!mce || !mce_usable_address(mce)) return NOTIFY_DONE; - if (!endpoint) - return NOTIFY_DONE; - spa = mce->addr & MCI_ADDR_PHYSADDR; - pfn = spa >> PAGE_SHIFT; - if (!pfn_valid(pfn)) + if (!cxl_resource_contains_addr(p->res, spa)) return NOTIFY_DONE; - spa_alias = cxl_port_get_spa_cache_alias(endpoint, spa); - if (spa_alias == ~0ULL) - return NOTIFY_DONE; + if (spa >= p->res->start + p->cache_size) + spa_alias = spa - p->cache_size; + else + spa_alias = spa + p->cache_size; pfn = spa_alias >> PAGE_SHIFT; + if (!pfn_valid(pfn)) + return NOTIFY_DONE; /* * Take down the aliased memory page. The original memory page flagged * by the MCE will be taken cared of by the standard MCE handler. */ - dev_emerg(mds->cxlds.dev, "Offlining aliased SPA address0: %#llx\n", + dev_emerg(&cxlr->dev, "Offlining aliased SPA address0: %#llx\n", spa_alias); if (!memory_failure(pfn, 0)) set_mce_nospec(pfn); diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index 4dff7f44d908e..3bb5835aa548e 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -870,7 +870,7 @@ static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data, if (!size) return FW_UPLOAD_ERR_INVALID_SIZE; - mds->fw.oneshot = struct_size(transfer, data, size) < + mds->fw.oneshot = struct_size(transfer, data, size) <= cxl_mbox->payload_size; if (cxl_mem_get_fw_info(mds)) diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 18825e1505d6a..68859eba30db3 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -711,324 +711,6 @@ void read_cdat_data(struct cxl_port *port) } EXPORT_SYMBOL_NS_GPL(read_cdat_data, "CXL"); -static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds, - void __iomem *ras_base) -{ - void __iomem *addr; - u32 status; - - if (!ras_base) - return; - - addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; - status = readl(addr); - if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { - writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); - trace_cxl_aer_correctable_error(cxlds->cxlmd, status); - } -} - -static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds) -{ - return __cxl_handle_cor_ras(cxlds, cxlds->regs.ras); -} - -/* CXL spec rev3.0 8.2.4.16.1 */ -static void header_log_copy(void __iomem *ras_base, u32 *log) -{ - void __iomem *addr; - u32 *log_addr; - int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); - - addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; - log_addr = log; - - for (i = 0; i < log_u32_size; i++) { - *log_addr = readl(addr); - log_addr++; - addr += sizeof(u32); - } -} - -/* - * Log the state of the RAS status registers and prepare them to log the - * next error status. Return 1 if reset needed. - */ -static bool __cxl_handle_ras(struct cxl_dev_state *cxlds, - void __iomem *ras_base) -{ - u32 hl[CXL_HEADERLOG_SIZE_U32]; - void __iomem *addr; - u32 status; - u32 fe; - - if (!ras_base) - return false; - - addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; - status = readl(addr); - if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) - return false; - - /* If multiple errors, log header points to first error from ctrl reg */ - if (hweight32(status) > 1) { - void __iomem *rcc_addr = - ras_base + CXL_RAS_CAP_CONTROL_OFFSET; - - fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, - readl(rcc_addr))); - } else { - fe = status; - } - - header_log_copy(ras_base, hl); - trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); - writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); - - return true; -} - -static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) -{ - return __cxl_handle_ras(cxlds, cxlds->regs.ras); -} - -#ifdef CONFIG_PCIEAER_CXL - -static void cxl_dport_map_rch_aer(struct cxl_dport *dport) -{ - resource_size_t aer_phys; - struct device *host; - u16 aer_cap; - - aer_cap = cxl_rcrb_to_aer(dport->dport_dev, dport->rcrb.base); - if (aer_cap) { - host = dport->reg_map.host; - aer_phys = aer_cap + dport->rcrb.base; - dport->regs.dport_aer = devm_cxl_iomap_block(host, aer_phys, - sizeof(struct aer_capability_regs)); - } -} - -static void cxl_dport_map_ras(struct cxl_dport *dport) -{ - struct cxl_register_map *map = &dport->reg_map; - struct device *dev = dport->dport_dev; - - if (!map->component_map.ras.valid) - dev_dbg(dev, "RAS registers not found\n"); - else if (cxl_map_component_regs(map, &dport->regs.component, - BIT(CXL_CM_CAP_CAP_ID_RAS))) - dev_dbg(dev, "Failed to map RAS capability.\n"); -} - -static void cxl_disable_rch_root_ints(struct cxl_dport *dport) -{ - void __iomem *aer_base = dport->regs.dport_aer; - u32 aer_cmd_mask, aer_cmd; - - if (!aer_base) - return; - - /* - * Disable RCH root port command interrupts. - * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors - * - * This sequence may not be necessary. CXL spec states disabling - * the root cmd register's interrupts is required. But, PCI spec - * shows these are disabled by default on reset. - */ - aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | - PCI_ERR_ROOT_CMD_NONFATAL_EN | - PCI_ERR_ROOT_CMD_FATAL_EN); - aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); - aer_cmd &= ~aer_cmd_mask; - writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); -} - -/** - * cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport - * @dport: the cxl_dport that needs to be initialized - * @host: host device for devm operations - */ -void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host) -{ - dport->reg_map.host = host; - cxl_dport_map_ras(dport); - - if (dport->rch) { - struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev); - - if (!host_bridge->native_aer) - return; - - cxl_dport_map_rch_aer(dport); - cxl_disable_rch_root_ints(dport); - } -} -EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, "CXL"); - -static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds, - struct cxl_dport *dport) -{ - return __cxl_handle_cor_ras(cxlds, dport->regs.ras); -} - -static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds, - struct cxl_dport *dport) -{ - return __cxl_handle_ras(cxlds, dport->regs.ras); -} - -/* - * Copy the AER capability registers using 32 bit read accesses. - * This is necessary because RCRB AER capability is MMIO mapped. Clear the - * status after copying. - * - * @aer_base: base address of AER capability block in RCRB - * @aer_regs: destination for copying AER capability - */ -static bool cxl_rch_get_aer_info(void __iomem *aer_base, - struct aer_capability_regs *aer_regs) -{ - int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32); - u32 *aer_regs_buf = (u32 *)aer_regs; - int n; - - if (!aer_base) - return false; - - /* Use readl() to guarantee 32-bit accesses */ - for (n = 0; n < read_cnt; n++) - aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); - - writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS); - writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS); - - return true; -} - -/* Get AER severity. Return false if there is no error. */ -static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, - int *severity) -{ - if (aer_regs->uncor_status & ~aer_regs->uncor_mask) { - if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV) - *severity = AER_FATAL; - else - *severity = AER_NONFATAL; - return true; - } - - if (aer_regs->cor_status & ~aer_regs->cor_mask) { - *severity = AER_CORRECTABLE; - return true; - } - - return false; -} - -static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) -{ - struct pci_dev *pdev = to_pci_dev(cxlds->dev); - struct aer_capability_regs aer_regs; - struct cxl_dport *dport; - int severity; - - struct cxl_port *port __free(put_cxl_port) = - cxl_pci_find_port(pdev, &dport); - if (!port) - return; - - if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) - return; - - if (!cxl_rch_get_aer_severity(&aer_regs, &severity)) - return; - - pci_print_aer(pdev, severity, &aer_regs); - - if (severity == AER_CORRECTABLE) - cxl_handle_rdport_cor_ras(cxlds, dport); - else - cxl_handle_rdport_ras(cxlds, dport); -} - -#else -static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } -#endif - -void cxl_cor_error_detected(struct pci_dev *pdev) -{ - struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); - struct device *dev = &cxlds->cxlmd->dev; - - scoped_guard(device, dev) { - if (!dev->driver) { - dev_warn(&pdev->dev, - "%s: memdev disabled, abort error handling\n", - dev_name(dev)); - return; - } - - if (cxlds->rcd) - cxl_handle_rdport_errors(cxlds); - - cxl_handle_endpoint_cor_ras(cxlds); - } -} -EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL"); - -pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, - pci_channel_state_t state) -{ - struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); - struct cxl_memdev *cxlmd = cxlds->cxlmd; - struct device *dev = &cxlmd->dev; - bool ue; - - scoped_guard(device, dev) { - if (!dev->driver) { - dev_warn(&pdev->dev, - "%s: memdev disabled, abort error handling\n", - dev_name(dev)); - return PCI_ERS_RESULT_DISCONNECT; - } - - if (cxlds->rcd) - cxl_handle_rdport_errors(cxlds); - /* - * A frozen channel indicates an impending reset which is fatal to - * CXL.mem operation, and will likely crash the system. On the off - * chance the situation is recoverable dump the status of the RAS - * capability registers and bounce the active state of the memdev. - */ - ue = cxl_handle_endpoint_ras(cxlds); - } - - - switch (state) { - case pci_channel_io_normal: - if (ue) { - device_release_driver(dev); - return PCI_ERS_RESULT_NEED_RESET; - } - return PCI_ERS_RESULT_CAN_RECOVER; - case pci_channel_io_frozen: - dev_warn(&pdev->dev, - "%s: frozen state error detected, disable CXL.mem\n", - dev_name(dev)); - device_release_driver(dev); - return PCI_ERS_RESULT_NEED_RESET; - case pci_channel_io_perm_failure: - dev_warn(&pdev->dev, - "failure state error detected, request disconnect\n"); - return PCI_ERS_RESULT_DISCONNECT; - } - return PCI_ERS_RESULT_NEED_RESET; -} -EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL"); - static int cxl_flit_size(struct pci_dev *pdev) { if (cxl_pci_flit_256(pdev)) diff --git a/drivers/cxl/core/pmem.c b/drivers/cxl/core/pmem.c index e3a8b8d813333..a62d84818f934 100644 --- a/drivers/cxl/core/pmem.c +++ b/drivers/cxl/core/pmem.c @@ -219,12 +219,14 @@ static struct cxl_nvdimm *cxl_nvdimm_alloc(struct cxl_nvdimm_bridge *cxl_nvb, dev->bus = &cxl_bus_type; dev->type = &cxl_nvdimm_type; /* - * A "%llx" string is 17-bytes vs dimm_id that is max - * NVDIMM_KEY_DESC_LEN + * dev_id is the nvdimm dimm_id used for security key lookup. + * It must match id_show(), which emits the CXL serial as an + * unsigned decimal. A u64 decimal string is at most 20 digits + * plus NUL. */ - BUILD_BUG_ON(sizeof(cxl_nvd->dev_id) < 17 || + BUILD_BUG_ON(sizeof(cxl_nvd->dev_id) < 21 || sizeof(cxl_nvd->dev_id) > NVDIMM_KEY_DESC_LEN); - sprintf(cxl_nvd->dev_id, "%llx", cxlmd->cxlds->serial); + sprintf(cxl_nvd->dev_id, "%llu", cxlmd->cxlds->serial); return cxl_nvd; } diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index c53a13d4f1662..2b9e166ea4e0e 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -1732,8 +1732,8 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, parent_dport, uport_dev, dport_dev); if (IS_ERR(dport)) { - /* Port already exists, restart iteration */ - if (PTR_ERR(dport) == -EAGAIN) + /* Port or dport already exists, restart iteration */ + if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY) return 0; return PTR_ERR(dport); } diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index 2731ba3a07993..f2d99865c6df5 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -5,8 +5,13 @@ #include #include #include +#include #include "trace.h" +/* Check that UCE header definition is maintained to keep ABI intact */ +static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128, + "rasdaemon ABI requires exactly 128 u32s"); + static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { @@ -18,6 +23,7 @@ static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -27,8 +33,8 @@ static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, else fe = status; - trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, - ras_cap.header_log); + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl); } static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, @@ -43,6 +49,7 @@ static void cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, struct cxl_ras_capability_regs ras_cap) { + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; u32 fe; @@ -52,8 +59,15 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, else fe = status; - trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, - ras_cap.header_log); + /* + * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware + * dwords. Copy them into the front of a zero-filled + * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace + * event memcpy sees a full 512-byte source and the userspace ABI + * (rasdaemon) is preserved. + */ + memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); + trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); } static int match_memdev_by_parent(struct device *dev, const void *uport) @@ -122,5 +136,179 @@ int cxl_ras_init(void) void cxl_ras_exit(void) { cxl_cper_unregister_prot_err_work(&cxl_cper_prot_err_work); - cancel_work_sync(&cxl_cper_prot_err_work); } + +static void cxl_dport_map_ras(struct cxl_dport *dport) +{ + struct cxl_register_map *map = &dport->reg_map; + struct device *dev = dport->dport_dev; + + if (!map->component_map.ras.valid) + dev_dbg(dev, "RAS registers not found\n"); + else if (cxl_map_component_regs(map, &dport->regs.component, + BIT(CXL_CM_CAP_CAP_ID_RAS))) + dev_dbg(dev, "Failed to map RAS capability.\n"); +} + +/** + * cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport + * @dport: the cxl_dport that needs to be initialized + * @host: host device for devm operations + */ +void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host) +{ + dport->reg_map.host = host; + cxl_dport_map_ras(dport); + + if (dport->rch) { + struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev); + + if (!host_bridge->native_aer) + return; + + cxl_dport_map_rch_aer(dport); + cxl_disable_rch_root_ints(dport); + } +} +EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, "CXL"); + +void cxl_handle_cor_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base) +{ + void __iomem *addr; + u32 status; + + if (!ras_base) + return; + + addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; + status = readl(addr); + if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { + writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); + trace_cxl_aer_correctable_error(cxlds->cxlmd, status); + } +} + +/* CXL spec rev3.0 8.2.4.16.1 */ +static void header_log_copy(void __iomem *ras_base, u32 *log) +{ + void __iomem *addr; + u32 *log_addr; + int i; + + addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; + log_addr = log; + + for (i = 0; i < CXL_HEADERLOG_SIZE_U32; i++) { + *log_addr = readl(addr); + log_addr++; + addr += sizeof(u32); + } +} + +/* + * Log the state of the RAS status registers and prepare them to log the + * next error status. Return 1 if reset needed. + */ +bool cxl_handle_ras(struct cxl_dev_state *cxlds, void __iomem *ras_base) +{ + u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; + void __iomem *addr; + u32 status; + u32 fe; + + if (!ras_base) + return false; + + addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; + status = readl(addr); + if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) + return false; + + /* If multiple errors, log header points to first error from ctrl reg */ + if (hweight32(status) > 1) { + void __iomem *rcc_addr = + ras_base + CXL_RAS_CAP_CONTROL_OFFSET; + + fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, + readl(rcc_addr))); + } else { + fe = status; + } + + header_log_copy(ras_base, hl); + trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); + writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); + + return true; +} + +void cxl_cor_error_detected(struct pci_dev *pdev) +{ + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + struct device *dev = &cxlds->cxlmd->dev; + + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return; + } + + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + + cxl_handle_cor_ras(cxlds, cxlds->regs.ras); + } +} +EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL"); + +pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); + struct cxl_memdev *cxlmd = cxlds->cxlmd; + struct device *dev = &cxlmd->dev; + bool ue; + + scoped_guard(device, dev) { + if (!dev->driver) { + dev_warn(&pdev->dev, + "%s: memdev disabled, abort error handling\n", + dev_name(dev)); + return PCI_ERS_RESULT_DISCONNECT; + } + + if (cxlds->rcd) + cxl_handle_rdport_errors(cxlds); + /* + * A frozen channel indicates an impending reset which is fatal to + * CXL.mem operation, and will likely crash the system. On the off + * chance the situation is recoverable dump the status of the RAS + * capability registers and bounce the active state of the memdev. + */ + ue = cxl_handle_ras(cxlds, cxlds->regs.ras); + } + + + switch (state) { + case pci_channel_io_normal: + if (ue) { + device_release_driver(dev); + return PCI_ERS_RESULT_NEED_RESET; + } + return PCI_ERS_RESULT_CAN_RECOVER; + case pci_channel_io_frozen: + dev_warn(&pdev->dev, + "%s: frozen state error detected, disable CXL.mem\n", + dev_name(dev)); + device_release_driver(dev); + return PCI_ERS_RESULT_NEED_RESET; + case pci_channel_io_perm_failure: + dev_warn(&pdev->dev, + "failure state error detected, request disconnect\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + return PCI_ERS_RESULT_NEED_RESET; +} +EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL"); diff --git a/drivers/cxl/core/ras_rch.c b/drivers/cxl/core/ras_rch.c new file mode 100644 index 0000000000000..07f350c5bdfc0 --- /dev/null +++ b/drivers/cxl/core/ras_rch.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 AMD Corporation. All rights reserved. */ + +#include +#include +#include "cxl.h" +#include "core.h" +#include "cxlmem.h" + +void cxl_dport_map_rch_aer(struct cxl_dport *dport) +{ + resource_size_t aer_phys; + struct device *host; + u16 aer_cap; + + aer_cap = cxl_rcrb_to_aer(dport->dport_dev, dport->rcrb.base); + if (aer_cap) { + host = dport->reg_map.host; + aer_phys = aer_cap + dport->rcrb.base; + dport->regs.dport_aer = + devm_cxl_iomap_block(host, aer_phys, + sizeof(struct aer_capability_regs)); + } +} + +void cxl_disable_rch_root_ints(struct cxl_dport *dport) +{ + void __iomem *aer_base = dport->regs.dport_aer; + u32 aer_cmd_mask, aer_cmd; + + if (!aer_base) + return; + + /* + * Disable RCH root port command interrupts. + * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors + * + * This sequence may not be necessary. CXL spec states disabling + * the root cmd register's interrupts is required. But, PCI spec + * shows these are disabled by default on reset. + */ + aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | + PCI_ERR_ROOT_CMD_NONFATAL_EN | + PCI_ERR_ROOT_CMD_FATAL_EN); + aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); + aer_cmd &= ~aer_cmd_mask; + writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); +} + +/* + * Copy the AER capability registers using 32 bit read accesses. + * This is necessary because RCRB AER capability is MMIO mapped. Clear the + * status after copying. + * + * @aer_base: base address of AER capability block in RCRB + * @aer_regs: destination for copying AER capability + */ +static bool cxl_rch_get_aer_info(void __iomem *aer_base, + struct aer_capability_regs *aer_regs) +{ + /* + * Bound the copy to the physically-defined AER registers (header + * through the 16-byte Header Log). struct aer_capability_regs is a + * software layout whose embedded struct pcie_tlp_log is larger than + * the on-wire AER capability; copying sizeof(*aer_regs) would + * over-read the RCRB-mapped MMIO block. + */ + int read_cnt = (PCI_ERR_HEADER_LOG + 16) / sizeof(u32); + u32 *aer_regs_buf = (u32 *)aer_regs; + int n; + + if (!aer_base) + return false; + + /* + * Zero the destination so the software-only tail fields + * (e.g. header_log.header_len) are deterministic rather than + * left as uninitialized stack, which could drive a bogus loop + * length in pcie_print_tlp_log(). + */ + memset(aer_regs, 0, sizeof(*aer_regs)); + + /* Use readl() to guarantee 32-bit accesses */ + for (n = 0; n < read_cnt; n++) + aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); + + writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS); + writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS); + + return true; +} + +/* Get AER severity. Return false if there is no error. */ +static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, + int *severity) +{ + u32 uncor_status = aer_regs->uncor_status & ~aer_regs->uncor_mask; + + if (uncor_status) { + *severity = (uncor_status & aer_regs->uncor_severity) ? + AER_FATAL : AER_NONFATAL; + return true; + } + + if (aer_regs->cor_status & ~aer_regs->cor_mask) { + *severity = AER_CORRECTABLE; + return true; + } + + return false; +} + +void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) +{ + struct pci_dev *pdev = to_pci_dev(cxlds->dev); + struct aer_capability_regs aer_regs; + struct cxl_dport *dport; + int severity; + + struct cxl_port *port __free(put_cxl_port) = + cxl_pci_find_port(pdev, &dport); + if (!port) + return; + + if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) + return; + + if (!cxl_rch_get_aer_severity(&aer_regs, &severity)) + return; + + pci_print_aer(pdev, severity, &aer_regs); + if (severity == AER_CORRECTABLE) + cxl_handle_cor_ras(cxlds, dport->regs.ras); + else + cxl_handle_ras(cxlds, dport->regs.ras); +} diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 41b64d871c5a1..e840ac002f9b1 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -15,6 +15,7 @@ #include #include #include "core.h" +#include "mce.h" /** * DOC: cxl core region @@ -1814,14 +1815,13 @@ static int find_pos_and_ways(struct cxl_port *port, struct range *range, break; } } - put_device(dev); - if (rc) dev_err(port->uport_dev, "failed to find %s:%s in target list of %s\n", dev_name(&port->dev), - dev_name(port->parent_dport->dport_dev), - dev_name(&cxlsd->cxld.dev)); + dev_name(port->parent_dport->dport_dev), dev_name(dev)); + + put_device(dev); return rc; } @@ -3521,6 +3521,9 @@ static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, int rc, part = READ_ONCE(cxled->part); struct cxl_region *cxlr; + if (part < 0) + return ERR_PTR(-EBUSY); + do { cxlr = __create_region(cxlrd, cxlds->part[part].mode, atomic_read(&cxlrd->region_id)); @@ -3605,34 +3608,6 @@ int cxl_add_to_region(struct cxl_endpoint_decoder *cxled) } EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa) -{ - struct cxl_region_ref *iter; - unsigned long index; - - if (!endpoint) - return ~0ULL; - - guard(rwsem_write)(&cxl_rwsem.region); - - xa_for_each(&endpoint->regions, index, iter) { - struct cxl_region_params *p = &iter->region->params; - - if (cxl_resource_contains_addr(p->res, spa)) { - if (!p->cache_size) - return ~0ULL; - - if (spa >= p->res->start + p->cache_size) - return spa - p->cache_size; - - return spa + p->cache_size; - } - } - - return ~0ULL; -} -EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL"); - static int is_system_ram(struct resource *res, void *arg) { struct cxl_region *cxlr = arg; @@ -3751,6 +3726,61 @@ static int cxl_region_debugfs_poison_clear(void *data, u64 offset) DEFINE_DEBUGFS_ATTRIBUTE(cxl_poison_clear_fops, NULL, cxl_region_debugfs_poison_clear, "%llx\n"); +static int cxl_region_setup_poison(struct cxl_region *cxlr) +{ + struct device *dev = &cxlr->dev; + struct cxl_region_params *p = &cxlr->params; + struct dentry *dentry; + + /* Create poison attributes if all memdevs support the capabilities */ + for (int i = 0; i < p->nr_targets; i++) { + struct cxl_endpoint_decoder *cxled = p->targets[i]; + struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); + + if (!cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_INJECT) || + !cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_CLEAR)) + return 0; + } + + dentry = cxl_debugfs_create_dir(dev_name(dev)); + debugfs_create_file("inject_poison", 0200, dentry, cxlr, + &cxl_poison_inject_fops); + debugfs_create_file("clear_poison", 0200, dentry, cxlr, + &cxl_poison_clear_fops); + + return devm_add_action_or_reset(dev, remove_debugfs, dentry); +} + +static int region_contains_resource(struct device *dev, const void *data) +{ + const struct resource *res = data; + struct cxl_region *cxlr; + struct cxl_region_params *p; + + if (!is_cxl_region(dev)) + return 0; + + cxlr = to_cxl_region(dev); + p = &cxlr->params; + + if (p->state != CXL_CONFIG_COMMIT) + return 0; + + if (!p->res) + return 0; + + return resource_contains(p->res, res) ? 1 : 0; +} + +bool cxl_region_contains_resource(const struct resource *res) +{ + guard(rwsem_read)(&cxl_rwsem.region); + struct device *dev __free(put_device) = bus_find_device( + &cxl_bus_type, NULL, res, region_contains_resource); + return !!dev; +} +EXPORT_SYMBOL_FOR_MODULES(cxl_region_contains_resource, "dax_hmem"); + static int cxl_region_can_probe(struct cxl_region *cxlr) { struct cxl_region_params *p = &cxlr->params; @@ -3780,7 +3810,6 @@ static int cxl_region_probe(struct device *dev) { struct cxl_region *cxlr = to_cxl_region(dev); struct cxl_region_params *p = &cxlr->params; - bool poison_supported = true; int rc; rc = cxl_region_can_probe(cxlr); @@ -3804,31 +3833,24 @@ static int cxl_region_probe(struct device *dev) if (rc) return rc; - /* Create poison attributes if all memdevs support the capabilities */ - for (int i = 0; i < p->nr_targets; i++) { - struct cxl_endpoint_decoder *cxled = p->targets[i]; - struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); - - if (!cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_INJECT) || - !cxl_memdev_has_poison_cmd(cxlmd, CXL_POISON_ENABLED_CLEAR)) { - poison_supported = false; - break; - } - } - - if (poison_supported) { - struct dentry *dentry; - - dentry = cxl_debugfs_create_dir(dev_name(dev)); - debugfs_create_file("inject_poison", 0200, dentry, cxlr, - &cxl_poison_inject_fops); - debugfs_create_file("clear_poison", 0200, dentry, cxlr, - &cxl_poison_clear_fops); - rc = devm_add_action_or_reset(dev, remove_debugfs, dentry); - if (rc) + /* + * Regions fronted by an extended linear cache need the MCE notifier to + * offline the aliased page on a memory error. + */ + if (p->cache_size) { + rc = devm_cxl_register_mce_notifier(&cxlr->dev, + &cxlr->mce_notifier); + if (rc == -EOPNOTSUPP) + dev_warn(&cxlr->dev, + "CONFIG_CXL_MCE disabled, MCE notifier not registered\n"); + else if (rc) return rc; } + rc = cxl_region_setup_poison(cxlr); + if (rc) + return rc; + switch (cxlr->mode) { case CXL_PARTMODE_PMEM: rc = devm_cxl_region_edac_register(cxlr); diff --git a/drivers/cxl/core/trace.h b/drivers/cxl/core/trace.h index a972e4ef19368..d37876096dd7c 100644 --- a/drivers/cxl/core/trace.h +++ b/drivers/cxl/core/trace.h @@ -56,7 +56,7 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __string(host, dev_name(dev->parent)) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(device); @@ -64,10 +64,14 @@ TRACE_EVENT(cxl_port_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("device=%s host=%s status: '%s' first_error: '%s'", __get_str(device), __get_str(host), @@ -85,7 +89,7 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __field(u64, serial) __field(u32, status) __field(u32, first_error) - __array(u32, header_log, CXL_HEADERLOG_SIZE_U32) + __array(u32, header_log, CXL_HEADERLOG_TRACE_SIZE_U32) ), TP_fast_assign( __assign_str(memdev); @@ -94,10 +98,14 @@ TRACE_EVENT(cxl_aer_uncorrectable_error, __entry->status = status; __entry->first_error = fe; /* - * Embed the 512B headerlog data for user app retrieval and - * parsing, but no need to print this in the trace buffer. + * Embed headerlog data for user app retrieval and parsing, + * but no need to print in the trace buffer. Only + * CXL_HEADERLOG_SIZE_U32 (16) dwords are hardware data; + * the remaining entries preserve the 512-byte ABI layout + * rasdaemon depends on and are zero-filled by the caller. */ - memcpy(__entry->header_log, hl, CXL_HEADERLOG_SIZE); + memcpy(__entry->header_log, hl, + CXL_HEADERLOG_TRACE_SIZE_U32 * sizeof(u32)); ), TP_printk("memdev=%s host=%s serial=%lld: status: '%s' first_error: '%s'", __get_str(memdev), __get_str(host), __entry->serial, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index 3a794278cc7fe..091c52111d949 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -148,8 +148,18 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw) #define CXL_RAS_CAP_CONTROL_FE_MASK GENMASK(5, 0) #define CXL_RAS_HEADER_LOG_OFFSET 0x18 #define CXL_RAS_CAPABILITY_LENGTH 0x58 -#define CXL_HEADERLOG_SIZE SZ_512 -#define CXL_HEADERLOG_SIZE_U32 SZ_512 / sizeof(u32) +#define CXL_HEADERLOG_SIZE SZ_64 +#define CXL_HEADERLOG_SIZE_U32 (CXL_HEADERLOG_SIZE / sizeof(u32)) + +/* + * The RAS UCE trace event header array was originally sized at SZ_512/sizeof(u32) + * = 128 u32s due to a bug. Userspace tools (rasdaemon) have grown a dependency + * on that 512-byte layout. Keep the trace array at 128 u32s to preserve the + * ABI; only CXL_HEADERLOG_SIZE_U32 (16) dwords are valid hardware data, the + * remainder are zero-filled. + */ +#define CXL_HEADERLOG_TRACE_SIZE SZ_512 +#define CXL_HEADERLOG_TRACE_SIZE_U32 (CXL_HEADERLOG_TRACE_SIZE / sizeof(u32)) /* CXL 2.0 8.2.8.1 Device Capabilities Array Register */ #define CXLDEV_CAP_ARRAY_OFFSET 0x0 @@ -530,6 +540,7 @@ enum cxl_partition_mode { * @coord: QoS access coordinates for the region * @node_notifier: notifier for setting the access coordinates to node * @adist_notifier: notifier for calculating the abstract distance of node + * @mce_notifier: notifier for MCE */ struct cxl_region { struct device dev; @@ -543,6 +554,7 @@ struct cxl_region { struct access_coordinate coord[ACCESS_COORDINATE_MAX]; struct notifier_block node_notifier; struct notifier_block adist_notifier; + struct notifier_block mce_notifier; }; struct cxl_nvdimm_bridge { @@ -553,7 +565,8 @@ struct cxl_nvdimm_bridge { struct nvdimm_bus_descriptor nd_desc; }; -#define CXL_DEV_ID_LEN 19 +/* Holds a u64 serial as a decimal string: up to 20 digits + NUL */ +#define CXL_DEV_ID_LEN 21 enum { CXL_NVD_F_INVALIDATED = 0, @@ -781,14 +794,6 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port, struct device *dport_dev, int port_id, resource_size_t rcrb); -#ifdef CONFIG_PCIEAER_CXL -void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport); -void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host); -#else -static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport, - struct device *host) { } -#endif - struct cxl_decoder *to_cxl_decoder(struct device *dev); struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev); struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev); @@ -883,7 +888,7 @@ bool is_cxl_pmem_region(struct device *dev); struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev); int cxl_add_to_region(struct cxl_endpoint_decoder *cxled); struct cxl_dax_region *to_cxl_dax_region(struct device *dev); -u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa); +bool cxl_region_contains_resource(const struct resource *res); #else static inline bool is_cxl_pmem_region(struct device *dev) { @@ -901,10 +906,9 @@ static inline struct cxl_dax_region *to_cxl_dax_region(struct device *dev) { return NULL; } -static inline u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, - u64 spa) +static inline bool cxl_region_contains_resource(const struct resource *res) { - return 0; + return false; } #endif diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index c12ab4fc95123..f7e3a33991b8d 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -484,7 +484,6 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) * @poison: poison driver state info * @security: security driver state info * @fw: firmware upload / activation state - * @mce_notifier: MCE notifier * * See CXL 3.0 8.2.9.8.2 Capacity Configuration and Label Storage for * details on capacity parameters. @@ -504,7 +503,6 @@ struct cxl_memdev_state { struct cxl_poison_state poison; struct cxl_security_state security; struct cxl_fw_state fw; - struct notifier_block mce_notifier; }; static inline struct cxl_memdev_state * diff --git a/drivers/cxl/cxlpci.h b/drivers/cxl/cxlpci.h index 7ae621e618e79..970e84cf49e9a 100644 --- a/drivers/cxl/cxlpci.h +++ b/drivers/cxl/cxlpci.h @@ -130,7 +130,23 @@ static inline bool cxl_pci_flit_256(struct pci_dev *pdev) int devm_cxl_port_enumerate_dports(struct cxl_port *port); struct cxl_dev_state; void read_cdat_data(struct cxl_port *port); + +#ifdef CONFIG_CXL_RAS void cxl_cor_error_detected(struct pci_dev *pdev); pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, pci_channel_state_t state); +void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host); +#else +static inline void cxl_cor_error_detected(struct pci_dev *pdev) { } + +static inline pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + return PCI_ERS_RESULT_NONE; +} + +static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport, + struct device *host) { } +#endif + #endif /* __CXL_PCI_H__ */ diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index c78d7d88744bf..59c6d503437be 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -807,12 +807,6 @@ static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) { int rc; - /* - * Fail the init if there's no mailbox. For a type3 this is out of spec. - */ - if (!cxlds->reg_map.device_map.mbox.valid) - return -ENODEV; - rc = cxl_mailbox_init(&cxlds->cxl_mbox, cxlds->dev); if (rc) return rc; @@ -952,10 +946,13 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) */ rc = cxl_pci_setup_regs(pdev, CXL_REGLOC_RBI_COMPONENT, &cxlds->reg_map); - if (rc) + if (rc) { + if (rc == -EPROBE_DEFER) + return rc; dev_warn(&pdev->dev, "No component registers (%d)\n", rc); - else if (!cxlds->reg_map.component_map.ras.valid) + } else if (!cxlds->reg_map.component_map.ras.valid) { dev_dbg(&pdev->dev, "RAS registers not found\n"); + } rc = cxl_map_component_regs(&cxlds->reg_map, &cxlds->regs.component, BIT(CXL_CM_CAP_CAP_ID_RAS)); @@ -1198,7 +1195,6 @@ static int __init cxl_pci_driver_init(void) static void __exit cxl_pci_driver_exit(void) { cxl_cper_unregister_work(&cxl_cper_work); - cancel_work_sync(&cxl_cper_work); pci_unregister_driver(&cxl_pci_driver); } diff --git a/drivers/cxl/pmem.c b/drivers/cxl/pmem.c index 3432fd83b1e2a..255ecfd67ca1e 100644 --- a/drivers/cxl/pmem.c +++ b/drivers/cxl/pmem.c @@ -52,7 +52,7 @@ static ssize_t id_show(struct device *dev, struct device_attribute *attr, char * struct cxl_nvdimm *cxl_nvd = nvdimm_provider_data(nvdimm); struct cxl_dev_state *cxlds = cxl_nvd->cxlmd->cxlds; - return sysfs_emit(buf, "%lld\n", cxlds->serial); + return sysfs_emit(buf, "%llu\n", cxlds->serial); } static DEVICE_ATTR_RO(id); diff --git a/drivers/dax/Makefile b/drivers/dax/Makefile index 5ed5c39857c8b..70e996bf15261 100644 --- a/drivers/dax/Makefile +++ b/drivers/dax/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +obj-y += hmem/ obj-$(CONFIG_DAX) += dax.o obj-$(CONFIG_DEV_DAX) += device_dax.o obj-$(CONFIG_DEV_DAX_KMEM) += kmem.o @@ -10,5 +11,3 @@ dax-y += bus.o device_dax-y := device.o dax_pmem-y := pmem.o dax_cxl-y := cxl.o - -obj-y += hmem/ diff --git a/drivers/dax/cxl.c b/drivers/dax/cxl.c index 13cd94d32ff7a..a2136adfa186e 100644 --- a/drivers/dax/cxl.c +++ b/drivers/dax/cxl.c @@ -38,10 +38,35 @@ static struct cxl_driver cxl_dax_region_driver = { .id = CXL_DEVICE_DAX_REGION, .drv = { .suppress_bind_attrs = true, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, }, }; -module_cxl_driver(cxl_dax_region_driver); +static void cxl_dax_region_driver_register(struct work_struct *work) +{ + cxl_driver_register(&cxl_dax_region_driver); +} + +static DECLARE_WORK(cxl_dax_region_driver_work, cxl_dax_region_driver_register); + +static int __init cxl_dax_region_init(void) +{ + /* + * Need to resolve a race with dax_hmem wanting to drive regions + * instead of CXL + */ + queue_work(system_long_wq, &cxl_dax_region_driver_work); + return 0; +} +module_init(cxl_dax_region_init); + +static void __exit cxl_dax_region_exit(void) +{ + flush_work(&cxl_dax_region_driver_work); + cxl_driver_unregister(&cxl_dax_region_driver); +} +module_exit(cxl_dax_region_exit); + MODULE_ALIAS_CXL(CXL_DEVICE_DAX_REGION); MODULE_DESCRIPTION("CXL DAX: direct access to CXL regions"); MODULE_LICENSE("GPL"); diff --git a/drivers/dax/super.c b/drivers/dax/super.c index d7714d8afb0fa..efa5f61860190 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -232,6 +232,7 @@ EXPORT_SYMBOL_GPL(dax_recovery_write); int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, int mf_flags) { + const struct dax_holder_operations *ops; int rc, id; id = dax_read_lock(); @@ -240,12 +241,19 @@ int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, goto out; } - if (!dax_dev->holder_ops) { + /* + * Read holder_ops once: a concurrent fs_put_dax() can clear it without + * synchronizing against readers. Without the single fetch the compiler + * could reload between the NULL check and the call and dereference a + * NULL ops. + */ + ops = READ_ONCE(dax_dev->holder_ops); + if (!ops) { rc = -EOPNOTSUPP; goto out; } - rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags); + rc = ops->notify_failure(dax_dev, off, len, mf_flags); out: dax_read_unlock(id); return rc; diff --git a/drivers/dibs/dibs_loopback.c b/drivers/dibs/dibs_loopback.c index 8e165bdedb33b..578db7a48812e 100644 --- a/drivers/dibs/dibs_loopback.c +++ b/drivers/dibs/dibs_loopback.c @@ -118,14 +118,9 @@ static int dibs_lo_register_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb, return rc; } -static void __dibs_lo_unregister_dmb(struct dibs_lo_dev *ldev, - struct dibs_lo_dmb_node *dmb_node) +static void dibs_lo_free_dmb(struct dibs_lo_dev *ldev, + struct dibs_lo_dmb_node *dmb_node) { - /* remove dmb from hash table */ - write_lock_bh(&ldev->dmb_ht_lock); - hash_del(&dmb_node->list); - write_unlock_bh(&ldev->dmb_ht_lock); - clear_bit(dmb_node->sba_idx, ldev->sba_idx_mask); folio_put(virt_to_folio(dmb_node->cpu_addr)); kfree(dmb_node); @@ -139,27 +134,33 @@ static int dibs_lo_unregister_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev; unsigned long flags; + bool last; ldev = dibs->drv_priv; /* find dmb from hash table */ - read_lock_bh(&ldev->dmb_ht_lock); + write_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, dmb->dmb_tok) { if (tmp_node->token == dmb->dmb_tok) { dmb_node = tmp_node; break; } } - read_unlock_bh(&ldev->dmb_ht_lock); - if (!dmb_node) + if (!dmb_node) { + write_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; + } + last = refcount_dec_and_test(&dmb_node->refcnt); + if (last) + hash_del(&dmb_node->list); + write_unlock_bh(&ldev->dmb_ht_lock); - if (refcount_dec_and_test(&dmb_node->refcnt)) { + if (last) { spin_lock_irqsave(&dibs->lock, flags); dibs->dmb_clientid_arr[dmb_node->sba_idx] = NO_DIBS_CLIENT; spin_unlock_irqrestore(&dibs->lock, flags); - __dibs_lo_unregister_dmb(ldev, dmb_node); + dibs_lo_free_dmb(ldev, dmb_node); } return 0; } @@ -188,14 +189,9 @@ static int dibs_lo_attach_dmb(struct dibs_dev *dibs, struct dibs_dmb *dmb) read_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } + refcount_inc(&dmb_node->refcnt); read_unlock_bh(&ldev->dmb_ht_lock); - if (!refcount_inc_not_zero(&dmb_node->refcnt)) - /* the dmb is being unregistered, but has - * not been removed from the hash table. - */ - return -EINVAL; - /* provide dmb information */ dmb->idx = dmb_node->sba_idx; dmb->dmb_tok = dmb_node->token; @@ -209,11 +205,12 @@ static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token) { struct dibs_lo_dmb_node *dmb_node = NULL, *tmp_node; struct dibs_lo_dev *ldev; + bool last; ldev = dibs->drv_priv; /* find dmb_node according to dmb->dmb_tok */ - read_lock_bh(&ldev->dmb_ht_lock); + write_lock_bh(&ldev->dmb_ht_lock); hash_for_each_possible(ldev->dmb_ht, tmp_node, list, token) { if (tmp_node->token == token) { dmb_node = tmp_node; @@ -221,13 +218,17 @@ static int dibs_lo_detach_dmb(struct dibs_dev *dibs, u64 token) } } if (!dmb_node) { - read_unlock_bh(&ldev->dmb_ht_lock); + write_unlock_bh(&ldev->dmb_ht_lock); return -EINVAL; } - read_unlock_bh(&ldev->dmb_ht_lock); + last = refcount_dec_and_test(&dmb_node->refcnt); + if (last) + hash_del(&dmb_node->list); + write_unlock_bh(&ldev->dmb_ht_lock); + + if (last) + dibs_lo_free_dmb(ldev, dmb_node); - if (refcount_dec_and_test(&dmb_node->refcnt)) - __dibs_lo_unregister_dmb(ldev, dmb_node); return 0; } @@ -334,7 +335,6 @@ static int dibs_lo_dev_probe(void) return 0; err_reg: - kfree(dibs->dmb_clientid_arr); /* pairs with dibs_dev_alloc() */ put_device(&dibs->dev); kfree(ldev); diff --git a/drivers/dibs/dibs_main.c b/drivers/dibs/dibs_main.c index 0374f8350ff7f..e05041841f139 100644 --- a/drivers/dibs/dibs_main.c +++ b/drivers/dibs/dibs_main.c @@ -20,7 +20,9 @@ MODULE_DESCRIPTION("Direct Internal Buffer Sharing class"); MODULE_LICENSE("GPL"); -static struct class *dibs_class; +static const struct class dibs_class = { + .name = "dibs", +}; /* use an array rather a list for fast mapping: */ static struct dibs_client *clients[MAX_DIBS_CLIENTS]; @@ -127,6 +129,7 @@ static void dibs_dev_release(struct device *dev) dibs = container_of(dev, struct dibs_dev, dev); + kfree(dibs->dmb_clientid_arr); kfree(dibs); } @@ -137,8 +140,9 @@ struct dibs_dev *dibs_dev_alloc(void) dibs = kzalloc(sizeof(*dibs), GFP_KERNEL); if (!dibs) return dibs; + spin_lock_init(&dibs->lock); dibs->dev.release = dibs_dev_release; - dibs->dev.class = dibs_class; + dibs->dev.class = &dibs_class; device_initialize(&dibs->dev); return dibs; @@ -185,7 +189,6 @@ int dibs_dev_add(struct dibs_dev *dibs) int i, ret; max_dmbs = dibs->ops->max_dmbs(); - spin_lock_init(&dibs->lock); dibs->dmb_clientid_arr = kzalloc(max_dmbs, GFP_KERNEL); if (!dibs->dmb_clientid_arr) return -ENOMEM; @@ -193,12 +196,13 @@ int dibs_dev_add(struct dibs_dev *dibs) ret = device_add(&dibs->dev); if (ret) - goto free_client_arr; + return ret; ret = sysfs_create_group(&dibs->dev.kobj, &dibs_dev_attr_group); if (ret) { dev_err(&dibs->dev, "sysfs_create_group failed for dibs_dev\n"); - goto err_device_del; + device_del(&dibs->dev); + return ret; } mutex_lock(&dibs_dev_list.mutex); mutex_lock(&clients_lock); @@ -213,13 +217,6 @@ int dibs_dev_add(struct dibs_dev *dibs) mutex_unlock(&dibs_dev_list.mutex); return 0; - -err_device_del: - device_del(&dibs->dev); -free_client_arr: - kfree(dibs->dmb_clientid_arr); - return ret; - } EXPORT_SYMBOL_GPL(dibs_dev_add); @@ -246,7 +243,6 @@ void dibs_dev_del(struct dibs_dev *dibs) mutex_unlock(&dibs_dev_list.mutex); device_del(&dibs->dev); - kfree(dibs->dmb_clientid_arr); } EXPORT_SYMBOL_GPL(dibs_dev_del); @@ -254,24 +250,27 @@ static int __init dibs_init(void) { int rc; - memset(clients, 0, sizeof(clients)); - max_client = 0; - - dibs_class = class_create("dibs"); - if (IS_ERR(dibs_class)) - return PTR_ERR(dibs_class); + rc = class_register(&dibs_class); + if (rc) + goto err; rc = dibs_loopback_init(); if (rc) - pr_err("%s fails with %d\n", __func__, rc); + goto err_unregister; + + return rc; +err_unregister: + class_unregister(&dibs_class); +err: + pr_err("%s fails with %d\n", __func__, rc); return rc; } static void __exit dibs_exit(void) { dibs_loopback_exit(); - class_destroy(dibs_class); + class_unregister(&dibs_class); } module_init(dibs_init); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index 8ab49924f8b71..0c628e4146e26 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -49,33 +49,6 @@ static dev_t dma_heap_devt; static struct class *dma_heap_class; static DEFINE_XARRAY_ALLOC(dma_heap_minors); -static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, - u32 fd_flags, - u64 heap_flags) -{ - struct dma_buf *dmabuf; - int fd; - - /* - * Allocations from all heaps have to begin - * and end on page boundaries. - */ - len = PAGE_ALIGN(len); - if (!len) - return -EINVAL; - - dmabuf = heap->ops->allocate(heap, len, fd_flags, heap_flags); - if (IS_ERR(dmabuf)) - return PTR_ERR(dmabuf); - - fd = dma_buf_fd(dmabuf, fd_flags); - if (fd < 0) { - dma_buf_put(dmabuf); - /* just return, as put will call release and that will free */ - } - return fd; -} - static int dma_heap_open(struct inode *inode, struct file *file) { struct dma_heap *heap; @@ -93,30 +66,42 @@ static int dma_heap_open(struct inode *inode, struct file *file) return 0; } -static long dma_heap_ioctl_allocate(struct file *file, void *data) +static struct dma_buf *dma_heap_ioctl_allocate(struct file *file, void *data) { struct dma_heap_allocation_data *heap_allocation = data; struct dma_heap *heap = file->private_data; + struct dma_buf *dmabuf; int fd; + size_t len; if (heap_allocation->fd) - return -EINVAL; + return ERR_PTR(-EINVAL); if (heap_allocation->fd_flags & ~DMA_HEAP_VALID_FD_FLAGS) - return -EINVAL; + return ERR_PTR(-EINVAL); if (heap_allocation->heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS) - return -EINVAL; + return ERR_PTR(-EINVAL); + + len = PAGE_ALIGN(heap_allocation->len); + if (!len) + return ERR_PTR(-EINVAL); + + dmabuf = heap->ops->allocate(heap, len, heap_allocation->fd_flags, + heap_allocation->heap_flags); - fd = dma_heap_buffer_alloc(heap, heap_allocation->len, - heap_allocation->fd_flags, - heap_allocation->heap_flags); - if (fd < 0) - return fd; + if (IS_ERR(dmabuf)) + return dmabuf; + + fd = get_unused_fd_flags(heap_allocation->fd_flags); + if (fd < 0) { + dma_buf_put(dmabuf); + return ERR_PTR(fd); + } heap_allocation->fd = fd; - return 0; + return dmabuf; } static unsigned int dma_heap_ioctl_cmds[] = { @@ -132,6 +117,8 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd, unsigned int in_size, out_size, drv_size, ksize; int nr = _IOC_NR(ucmd); int ret = 0; + int fd; + struct dma_buf *dmabuf; if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds)) return -EINVAL; @@ -168,15 +155,28 @@ static long dma_heap_ioctl(struct file *file, unsigned int ucmd, switch (kcmd) { case DMA_HEAP_IOCTL_ALLOC: - ret = dma_heap_ioctl_allocate(file, kdata); + dmabuf = dma_heap_ioctl_allocate(file, kdata); + + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + break; + } + + fd = ((struct dma_heap_allocation_data *)kdata)->fd; + if (copy_to_user((void __user *)arg, kdata, out_size) != 0) { + put_unused_fd(fd); + dma_buf_put(dmabuf); + ret = -EFAULT; + } else { + dma_buf_fd_install(dmabuf, fd); + } + break; default: ret = -ENOTTY; goto err; } - if (copy_to_user((void __user *)arg, kdata, out_size) != 0) - ret = -EFAULT; err: if (kdata != stack_kdata) kfree(kdata); diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index be47d1ff6c508..30b0f0c72e928 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -249,21 +249,22 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, { struct udmabuf *ubuf = buf->priv; struct device *dev = ubuf->device->this_device; - int ret = 0; if (!ubuf->sg) { ubuf->sg = get_sg_table(dev, buf, direction); if (IS_ERR(ubuf->sg)) { + int ret; + ret = PTR_ERR(ubuf->sg); ubuf->sg = NULL; + return ret; } else { ubuf->sg_dir = direction; } - } else { - dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); } - return ret; + dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); + return 0; } static int end_cpu_udmabuf(struct dma_buf *buf, diff --git a/drivers/dma/altera-msgdma.c b/drivers/dma/altera-msgdma.c index a203fdd84950e..e1811a9be0bbe 100644 --- a/drivers/dma/altera-msgdma.c +++ b/drivers/dma/altera-msgdma.c @@ -498,6 +498,11 @@ static void msgdma_copy_one(struct msgdma_device *mdev, { void __iomem *hw_desc = mdev->desc; + /* Ensure control is the last field — required for correct FIFO flush ordering */ + static_assert(offsetof(struct msgdma_extended_desc, control) == + sizeof(struct msgdma_extended_desc) - sizeof(u32), + "control must be the last field in msgdma_extended_desc"); + /* * Check if the DESC FIFO it not full. If its full, we need to wait * for at least one entry to become free again @@ -506,17 +511,18 @@ static void msgdma_copy_one(struct msgdma_device *mdev, MSGDMA_CSR_STAT_DESC_BUF_FULL) mdelay(1); + /* Ensure control is the last field — required for correct FIFO flush ordering */ + static_assert(offsetof(struct msgdma_extended_desc, control) == + sizeof(struct msgdma_extended_desc) - sizeof(u32), + "control must be the last field in msgdma_extended_desc"); + /* - * The descriptor needs to get copied into the descriptor FIFO - * of the DMA controller. The descriptor will get flushed to the - * FIFO, once the last word (control word) is written. Since we - * are not 100% sure that memcpy() writes all word in the "correct" - * order (address from low to high) on all architectures, we make - * sure this control word is written last by single coding it and - * adding some write-barriers here. + * Copy the descriptor into the descriptor FIFO of the DMA controller, + * excluding the control word. The FIFO is flushed and the descriptor + * becomes valid once the control word is written last. */ - memcpy((void __force *)hw_desc, &desc->hw_desc, - sizeof(desc->hw_desc) - sizeof(u32)); + memcpy_toio(hw_desc, &desc->hw_desc, + offsetof(struct msgdma_extended_desc, control)); /* Write control word last to flush this descriptor into the FIFO */ mdev->idle = false; diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index e74a2ee746164..9a4ba81a9a9d0 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -555,7 +555,8 @@ static void axi_chan_block_xfer_start(struct axi_dma_chan *chan, write_chan_llp(chan, first->hw_desc[0].llp | lms); - irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_ALL_ERR; + irq_mask = DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_BLOCK_TRF | + DWAXIDMAC_IRQ_ALL_ERR; axi_chan_irq_sig_set(chan, irq_mask); /* Generate 'suspend' status but don't generate interrupt */ @@ -605,10 +606,15 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); int ret; + ret = pm_runtime_resume_and_get(chan->chip->dev); + if (ret < 0) + return ret; + /* ASSERT: channel is idle */ if (axi_chan_is_hw_enable(chan)) { dev_err(chan2dev(chan), "%s is non-idle!\n", axi_chan_name(chan)); + pm_runtime_put(chan->chip->dev); return -EBUSY; } @@ -619,22 +625,11 @@ static int dma_chan_alloc_chan_resources(struct dma_chan *dchan) 64, 0); if (!chan->desc_pool) { dev_err(chan2dev(chan), "No memory for descriptors\n"); + pm_runtime_put(chan->chip->dev); return -ENOMEM; } dev_vdbg(dchan2dev(dchan), "%s: allocating\n", axi_chan_name(chan)); - /* - * Callers configure and use the channel's registers as soon as this - * returns, so the chip's clocks must already be running - a plain - * pm_runtime_get() only schedules the resume asynchronously. - */ - ret = pm_runtime_resume_and_get(chan->chip->dev); - if (ret < 0) { - dma_pool_destroy(chan->desc_pool); - chan->desc_pool = NULL; - return ret; - } - return 0; } @@ -707,6 +702,15 @@ static void set_desc_last(struct axi_dma_hw_desc *desc) desc->lli->ctl_hi = cpu_to_le32(val); } +static void set_desc_blktrf(struct axi_dma_hw_desc *desc) +{ + u32 val; + + val = le32_to_cpu(desc->lli->ctl_hi); + val |= CH_CTL_H_LLI_BLKTRF; + desc->lli->ctl_hi = cpu_to_le32(val); +} + static void write_desc_sar(struct axi_dma_hw_desc *desc, dma_addr_t adr) { desc->lli->sar = cpu_to_le64(adr); @@ -889,19 +893,20 @@ dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, size_t axi_block_len; u32 total_segments; u32 segment_len; - unsigned int i; + unsigned int i, p; int status; u64 llp = 0; u8 lms = 0; /* Select AXI0 master for LLI fetching */ num_periods = buf_len / period_len; + if (num_periods * period_len != buf_len) + return NULL; axi_block_len = calculate_block_len(chan, dma_addr, buf_len, direction); if (axi_block_len == 0) return NULL; num_segments = DIV_ROUND_UP(period_len, axi_block_len); - segment_len = DIV_ROUND_UP(period_len, num_segments); total_segments = num_periods * num_segments; @@ -915,26 +920,34 @@ dw_axi_dma_chan_prep_cyclic(struct dma_chan *dchan, dma_addr_t dma_addr, desc->length = 0; desc->period_len = period_len; - for (i = 0; i < total_segments; i++) { - hw_desc = &desc->hw_desc[i]; + for (p = 0; p < num_periods; p++) { + u32 len = period_len; - status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr, - segment_len); - if (status < 0) - goto err_desc_get; + for (i = 0; i < num_segments; i++) { + hw_desc = &desc->hw_desc[p * num_segments + i]; - desc->length += hw_desc->len; + segment_len = min(len, axi_block_len); + status = dw_axi_dma_set_hw_desc(chan, hw_desc, src_addr, + segment_len); + if (status < 0) + goto err_desc_get; + + desc->length += hw_desc->len; + + src_addr += segment_len; + len -= segment_len; + } + set_desc_blktrf(hw_desc); /* Set end-of-link to the linked descriptor, so that cyclic * callback function can be triggered during interrupt. */ - set_desc_last(hw_desc); - - src_addr += segment_len; } desc->nr_hw_descs = total_segments; llp = desc->hw_desc[0].llp; + if (!total_segments) + goto err_desc_get; /* Managed transfer list */ do { @@ -963,7 +976,7 @@ dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); struct axi_dma_hw_desc *hw_desc = NULL; struct axi_dma_desc *desc = NULL; - u32 num_segments, segment_len; + u32 segment_len; unsigned int loop = 0; struct scatterlist *sg; size_t axi_block_len; @@ -998,13 +1011,11 @@ dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, for_each_sg(sgl, sg, sg_len, i) { mem = sg_dma_address(sg); len = sg_dma_len(sg); - num_segments = DIV_ROUND_UP(sg_dma_len(sg), axi_block_len); - if (!num_segments) + if (!len) continue; - segment_len = DIV_ROUND_UP(sg_dma_len(sg), num_segments); - - do { + while (len) { + segment_len = min(len, axi_block_len); hw_desc = &desc->hw_desc[loop++]; status = dw_axi_dma_set_hw_desc(chan, hw_desc, mem, segment_len); if (status < 0) @@ -1013,7 +1024,7 @@ dw_axi_dma_chan_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, desc->length += hw_desc->len; len -= segment_len; mem += segment_len; - } while (len >= segment_len); + } } desc->nr_hw_descs = loop; @@ -1216,18 +1227,14 @@ static noinline void axi_chan_handle_err(struct axi_dma_chan *chan, u32 status) spin_unlock_irqrestore(&chan->vc.lock, flags); } -static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) +static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan, bool block_trf) { - struct axi_dma_hw_desc *hw_desc; struct axi_dma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; - int count; - u64 llp; - int i; spin_lock_irqsave(&chan->vc.lock, flags); - if (unlikely(axi_chan_is_hw_enable(chan))) { + if (!block_trf && unlikely(axi_chan_is_hw_enable(chan))) { dev_err(chan2dev(chan), "BUG: %s caught DWAXIDMAC_IRQ_DMA_TRF, but channel not idle!\n", axi_chan_name(chan)); axi_chan_disable(chan); @@ -1244,22 +1251,52 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) if (chan->cyclic) { desc = vd_to_axi_desc(vd); if (desc) { - count = desc->nr_hw_descs; - llp = lo_hi_readq(chan->chan_regs + CH_LLP); - for (i = 0; i < count; i++) { - hw_desc = &desc->hw_desc[i]; - if (hw_desc->llp == llp) { - axi_chan_irq_clear(chan, hw_desc->lli->status_lo); - hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; - desc->completed_blocks = i; - - if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) - vchan_cyclic_callback(vd); - break; - } - } + u32 num_periods = desc->length / desc->period_len; + u32 perlen = desc->nr_hw_descs / num_periods; + + if (block_trf) + axi_chan_irq_clear(chan, DWAXIDMAC_IRQ_BLOCK_TRF); + else + axi_chan_irq_clear(chan, DWAXIDMAC_IRQ_DMA_TRF); - axi_chan_enable(chan); + /* + * The DMAC clears LLI_VALID in memory as it retires + * each descriptor, and writes the period data into the + * destination buffer. Order both against the reads + * below and against the consumer woken by the callback. + */ + dma_rmb(); + + for (int i = 0; i < num_periods; i++) { + u32 idx = i * perlen; + u32 last = idx + perlen - 1; + int j; + + if (le32_to_cpu(desc->hw_desc[last].lli->ctl_hi) & + CH_CTL_H_LLI_VALID) + continue; + + desc->hw_desc[last].lli->status_lo &= ~cpu_to_le32(4); + + /* + * Re-arm back-to-front and publish the period's + * first descriptor last. The engine enters the + * period at hw_desc[idx] and walks forward, so + * validating that one before its successors + * lets it run into a descriptor that is still + * retired. + */ + for (j = last; j > idx; j--) + desc->hw_desc[j].lli->ctl_hi |= + cpu_to_le32(CH_CTL_H_LLI_VALID); + dma_wmb(); + desc->hw_desc[idx].lli->ctl_hi |= + cpu_to_le32(CH_CTL_H_LLI_VALID); + /* Publish the re-armed chain to the DMAC. */ + dma_wmb(); + + vchan_cyclic_callback(vd); + } } } else { /* Remove the completed descriptor from issued list before completing */ @@ -1296,8 +1333,8 @@ static irqreturn_t dw_axi_dma_interrupt(int irq, void *dev_id) if (status & DWAXIDMAC_IRQ_ALL_ERR) axi_chan_handle_err(chan, status); - else if (status & DWAXIDMAC_IRQ_DMA_TRF) - axi_chan_block_xfer_complete(chan); + else if (status & (DWAXIDMAC_IRQ_DMA_TRF | DWAXIDMAC_IRQ_BLOCK_TRF)) + axi_chan_block_xfer_complete(chan, status & DWAXIDMAC_IRQ_BLOCK_TRF); } /* Re-enable interrupts */ @@ -1881,6 +1918,8 @@ static void dw_remove(struct platform_device *pdev) } static const struct dev_pm_ops dw_axi_dma_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) SET_RUNTIME_PM_OPS(axi_dma_runtime_suspend, axi_dma_runtime_resume, NULL) }; diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index ab868c16d6b5a..4ffbcf5aa2f97 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -252,6 +252,7 @@ enum { DWAXIDMAC_ARWLEN_MAX = DWAXIDMAC_ARWLEN_256 }; +#define CH_CTL_H_LLI_BLKTRF BIT(26) #define CH_CTL_H_LLI_LAST BIT(30) #define CH_CTL_H_LLI_VALID BIT(31) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 8e5f7defa6b67..cb34385d605e3 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -201,6 +202,35 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) return 1; } +static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd) +{ + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); +} + +static void dw_edma_terminate_vdesc_list(struct list_head *head) +{ + struct virt_dma_desc *vd, *_vd; + + list_for_each_entry_safe(vd, _vd, head, node) + dw_edma_terminate_vdesc(vd); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) +{ + /* + * This order must not be reversed. Cookies are assigned when + * descriptors are submitted, so desc_issued contains older cookies + * than desc_submitted. Completing desc_submitted first could move + * chan->vc.chan.completed_cookie backwards when desc_issued is + * terminated afterwards. + */ + dw_edma_terminate_vdesc_list(&chan->vc.desc_issued); + dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -235,6 +265,8 @@ static int dw_edma_device_pause(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) err = -EPERM; else if (chan->status != EDMA_ST_BUSY) @@ -252,6 +284,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { err = -EPERM; } else if (chan->status != EDMA_ST_PAUSE) { @@ -260,7 +294,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) err = -EPERM; } else { chan->status = EDMA_ST_BUSY; - dw_edma_start_transfer(chan); + if (!dw_edma_start_transfer(chan)) + chan->status = EDMA_ST_IDLE; } return err; @@ -271,25 +306,29 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { - /* Do nothing */ + dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { - chan->configured = false; + dw_edma_terminate_all_descs(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { chan->request = EDMA_REQ_STOP; } + if (chan->status == EDMA_ST_IDLE) + chan->request = EDMA_REQ_NONE; return err; } @@ -299,11 +338,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; - if (!chan->configured) - return; - spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + if (chan->configured && vchan_issue_pending(&chan->vc) && + chan->request == EDMA_REQ_NONE && chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); @@ -610,10 +647,16 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); + if (chan->status == EDMA_ST_PAUSE) { + spin_unlock_irqrestore(&chan->vc.lock, flags); + return; + } + vd = vchan_next_desc(&chan->vc); if (vd) { switch (chan->request) { case EDMA_REQ_NONE: + case EDMA_REQ_PAUSE: desc = vd2dw_edma_desc(vd); if (!desc->chunks_alloc) { dw_hdma_set_callback_result(vd, @@ -622,23 +665,23 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) vchan_cookie_complete(vd); } + if (chan->request == EDMA_REQ_PAUSE) { + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_PAUSE; + break; + } + /* Continue transferring if there are remaining chunks or issued requests. */ chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; break; case EDMA_REQ_STOP: - list_del(&vd->node); - vchan_cookie_complete(vd); + dw_edma_terminate_all_descs(chan); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; break; - case EDMA_REQ_PAUSE: - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_PAUSE; - break; - default: break; } @@ -653,14 +696,16 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); - if (vd) { + if (vd && chan->request == EDMA_REQ_STOP) { + dw_edma_terminate_all_descs(chan); + } else if (vd) { dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); } - spin_unlock_irqrestore(&chan->vc.lock, flags); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + spin_unlock_irqrestore(&chan->vc.lock, flags); } static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) @@ -701,21 +746,49 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) return 0; } -static void dw_edma_free_chan_resources(struct dma_chan *dchan) +static void dw_edma_wait_termination(struct dma_chan *dchan) { + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long timeout = jiffies + msecs_to_jiffies(5000); - int ret; + bool stopping; + /* + * A STOP may be deferred to a later interrupt while the channel is still + * running. Wait until that handler completes the termination. + */ while (time_before(jiffies, timeout)) { - ret = dw_edma_device_terminate_all(dchan); - if (!ret) - break; + scoped_guard(spinlock_irqsave, &chan->vc.lock) + stopping = chan->request == EDMA_REQ_STOP; - if (time_after_eq(jiffies, timeout)) + if (!stopping) return; - cpu_relax(); + fsleep(1000); } + + dev_warn(chan->dw->chip->dev, + "timeout waiting for channel termination\n"); +} + +static void dw_edma_device_synchronize(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_wait_termination(dchan); + vchan_synchronize(&chan->vc); +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_device_terminate_all(dchan); + dw_edma_device_synchronize(dchan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) + chan->configured = false; + + vchan_free_chan_resources(&chan->vc); } static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) @@ -774,7 +847,6 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) else irq->rd_mask |= BIT(chan->id); - irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", @@ -813,6 +885,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_synchronize = dw_edma_device_synchronize; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; @@ -862,6 +935,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ irq = chip->ops->irq_vector(dev, 0); + dw->irq[0].dw = dw; err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -887,6 +961,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { irq = chip->ops->irq_vector(dev, i); + dw->irq[i].dw = dw; err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 3371e0a76d3ca..dc6ffb6941ade 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -161,13 +161,16 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; - struct dw_edma_pcie_data *vsec_data __free(kfree) = NULL; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; - vsec_data = kmalloc(sizeof(*vsec_data), GFP_KERNEL); + if (!pdata) + return -ENODEV; + + struct dw_edma_pcie_data *vsec_data __free(kfree) = + kmalloc(sizeof(*vsec_data), GFP_KERNEL); if (!vsec_data) return -ENOMEM; diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index ce8f7254bab21..d2e5ddbb7b667 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -79,7 +79,7 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) u32 tmp; tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, - GET_CH_32(dw, chan->id, chan->dir, ch_stat)); + GET_CH_32(dw, chan->dir, chan->id, ch_stat)); if (tmp == 1) return DMA_IN_PROGRESS; diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index fbc12521da266..ca91878bf3ed5 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -414,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", dev_name(&pdev->dev)); + if (!errirq_name) + return -ENOMEM; ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, 0, errirq_name, fsl_edma); diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h index d3541301a2470..45d964a3726d6 100644 --- a/drivers/dma/fsl-edma-trace.h +++ b/drivers/dma/fsl-edma-trace.h @@ -19,14 +19,16 @@ DECLARE_EVENT_CLASS(edma_log_io, __field(struct fsl_edma_engine *, edma) __field(void __iomem *, addr) __field(u32, value) + __field(void __iomem *, membase) ), TP_fast_assign( __entry->edma = edma; __entry->addr = addr; __entry->value = value; + __entry->membase = edma->membase; ), TP_printk("offset %08x: value %08x", - (u32)(__entry->addr - __entry->edma->membase), __entry->value) + (u32)(__entry->addr - __entry->membase), __entry->value) ); DEFINE_EVENT(edma_log_io, edma_readl, diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 25a4134be36b7..ede094e0d0905 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), GFP_KERNEL); if (!hdma_dev) - return -EINVAL; + return -ENOMEM; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c index 4105688cf3f06..37addcdc10bc2 100644 --- a/drivers/dma/idxd/cdev.c +++ b/drivers/dma/idxd/cdev.c @@ -288,6 +288,7 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) fdev->parent = cdev_dev(idxd_cdev); fdev->bus = &dsa_bus_type; fdev->type = &idxd_cdev_file_type; + idxd_wq_get(wq); rc = dev_set_name(fdev, "file%d", ctx->id); if (rc < 0) { @@ -301,13 +302,14 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp) goto failed_dev_add; } - idxd_wq_get(wq); mutex_unlock(&wq->wq_lock); return 0; failed_dev_add: failed_dev_name: + mutex_unlock(&wq->wq_lock); put_device(fdev); + return rc; failed_ida: failed_set_pasid: if (device_user_pasid_enabled(idxd)) diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index 96c18c815f1df..cdba081637975 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -530,7 +530,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) for (i = 0; i < mtkd->dma_requests; i++) { c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); if (!c) { - rc = -ENODEV; + rc = -ENOMEM; goto err_no_dma; } diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 2cf060174795f..781cea16196f5 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1391,6 +1391,7 @@ static void bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_force_suspend(&pdev->dev); of_dma_controller_free(pdev->dev.of_node); diff --git a/drivers/dma/sh/rz-dmac.c b/drivers/dma/sh/rz-dmac.c index 818d1ef6f0bf9..4db84b83177a6 100644 --- a/drivers/dma/sh/rz-dmac.c +++ b/drivers/dma/sh/rz-dmac.c @@ -811,27 +811,6 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, channel->index = index; channel->mid_rid = -EINVAL; - /* Request the channel interrupt. */ - scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); - channel->irq = platform_get_irq_byname(pdev, pdev_irqname); - if (channel->irq < 0) - return channel->irq; - - irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", - dev_name(dmac->dev), index); - if (!irqname) - return -ENOMEM; - - ret = devm_request_threaded_irq(dmac->dev, channel->irq, - rz_dmac_irq_handler, - rz_dmac_irq_handler_thread, 0, - irqname, channel); - if (ret) { - dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", - channel->irq, ret); - return ret; - } - /* Set io base address for each channel */ if (index < 8) { channel->ch_base = dmac->base + CHANNEL_0_7_OFFSET + @@ -844,9 +823,9 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, } /* Allocate descriptors */ - lmdesc = dma_alloc_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - &channel->lmdesc.base_dma, GFP_KERNEL); + lmdesc = dmam_alloc_coherent(&pdev->dev, + sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, + &channel->lmdesc.base_dma, GFP_KERNEL); if (!lmdesc) { dev_err(&pdev->dev, "Can't allocate memory (lmdesc)\n"); return -ENOMEM; @@ -862,7 +841,26 @@ static int rz_dmac_chan_probe(struct rz_dmac *dmac, INIT_LIST_HEAD(&channel->ld_free); INIT_LIST_HEAD(&channel->ld_active); - return 0; + /* Request the channel interrupt. */ + scnprintf(pdev_irqname, sizeof(pdev_irqname), "ch%u", index); + channel->irq = platform_get_irq_byname(pdev, pdev_irqname); + if (channel->irq < 0) + return channel->irq; + + irqname = devm_kasprintf(dmac->dev, GFP_KERNEL, "%s:%u", + dev_name(dmac->dev), index); + if (!irqname) + return -ENOMEM; + + ret = devm_request_threaded_irq(dmac->dev, channel->irq, + rz_dmac_irq_handler, + rz_dmac_irq_handler_thread, 0, + irqname, channel); + if (ret) + dev_err(dmac->dev, "failed to request IRQ %u (%d)\n", + channel->irq, ret); + + return ret; } static void rz_dmac_put_device(void *_dev) @@ -932,7 +930,6 @@ static int rz_dmac_probe(struct platform_device *pdev) const char *irqname = "error"; struct dma_device *engine; struct rz_dmac *dmac; - int channel_num; int ret; int irq; u8 i; @@ -964,19 +961,6 @@ static int rz_dmac_probe(struct platform_device *pdev) return PTR_ERR(dmac->ext_base); } - /* Register interrupt handler for error */ - irq = platform_get_irq_byname(pdev, irqname); - if (irq < 0) - return irq; - - ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, - irqname, NULL); - if (ret) { - dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", - irq, ret); - return ret; - } - /* Initialize the channels. */ INIT_LIST_HEAD(&dmac->engine.channels); @@ -1002,6 +986,21 @@ static int rz_dmac_probe(struct platform_device *pdev) goto err; } + /* Register interrupt handler for error */ + irq = platform_get_irq_byname(pdev, irqname); + if (irq < 0) { + ret = irq; + goto err; + } + + ret = devm_request_irq(&pdev->dev, irq, rz_dmac_irq_handler, 0, + irqname, NULL); + if (ret) { + dev_err(&pdev->dev, "failed to request IRQ %u (%d)\n", + irq, ret); + goto err; + } + /* Register the DMAC as a DMA provider for DT. */ ret = of_dma_controller_register(pdev->dev.of_node, rz_dmac_of_xlate, NULL); @@ -1040,16 +1039,6 @@ static int rz_dmac_probe(struct platform_device *pdev) dma_register_err: of_dma_controller_free(pdev->dev.of_node); err: - channel_num = i ? i - 1 : 0; - for (i = 0; i < channel_num; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } - reset_control_assert(dmac->rstc); err_pm_runtime_put: pm_runtime_put(&pdev->dev); @@ -1062,18 +1051,9 @@ static int rz_dmac_probe(struct platform_device *pdev) static void rz_dmac_remove(struct platform_device *pdev) { struct rz_dmac *dmac = platform_get_drvdata(pdev); - unsigned int i; dma_async_device_unregister(&dmac->engine); of_dma_controller_free(pdev->dev.of_node); - for (i = 0; i < dmac->n_channels; i++) { - struct rz_dmac_chan *channel = &dmac->channels[i]; - - dma_free_coherent(&pdev->dev, - sizeof(struct rz_lmdesc) * DMAC_NR_LMDESC, - channel->lmdesc.base, - channel->lmdesc.base_dma); - } reset_control_assert(dmac->rstc); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); diff --git a/drivers/dma/sun6i-dma.c b/drivers/dma/sun6i-dma.c index f9d876deb1f05..ae362a2f7c40f 100644 --- a/drivers/dma/sun6i-dma.c +++ b/drivers/dma/sun6i-dma.c @@ -935,16 +935,13 @@ static int sun6i_dma_terminate_all(struct dma_chan *chan) spin_lock_irqsave(&vchan->vc.lock, flags); - if (vchan->cyclic) { - vchan->cyclic = false; - if (pchan && pchan->desc) { - struct virt_dma_desc *vd = &pchan->desc->vd; - struct virt_dma_chan *vc = &vchan->vc; + if (pchan && pchan->desc && pchan->desc != pchan->done) { + struct virt_dma_desc *vd = &pchan->desc->vd; - list_add_tail(&vd->node, &vc->desc_completed); - } + vchan_terminate_vdesc(vd); } + vchan->cyclic = false; vchan_get_all_descriptors(&vchan->vc, &head); if (pchan) { diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 7dec5e6babe14..30e3db94ddf07 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1872,8 +1872,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) if (status & XILINX_MCDMA_IRQ_IOC_MASK) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -1929,8 +1931,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index f7e584de4335e..23e07d72a5225 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1173,9 +1173,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - if (pm_runtime_active(zdev->dev)) - zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); + if (!pm_runtime_status_suspended(zdev->dev)) + zynqmp_dma_runtime_suspend(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c index de8b065280d15..eb23ee401aaef 100644 --- a/drivers/dpll/dpll_netlink.c +++ b/drivers/dpll/dpll_netlink.c @@ -486,6 +486,9 @@ dpll_msg_add_pin_ref_sync(struct sk_buff *msg, struct dpll_pin *pin, if (!dpll_pin_available(ref_sync_pin)) continue; ref_sync_pin_priv = dpll_pin_on_dpll_priv(dpll, ref_sync_pin); + /* Pin may have been unregistered from this dpll already */ + if (!ref_sync_pin_priv) + continue; if (WARN_ON(!ops->ref_sync_get)) return -EOPNOTSUPP; ret = ops->ref_sync_get(pin, pin_priv, ref_sync_pin, diff --git a/drivers/edac/altera_edac.c b/drivers/edac/altera_edac.c index 4edd2088c2db6..a6e8e76c8abd5 100644 --- a/drivers/edac/altera_edac.c +++ b/drivers/edac/altera_edac.c @@ -1507,6 +1507,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) int edac_idx, rc; struct device_node *np; const struct edac_device_prv_data *prv = &a10_sdmmceccb_data; + bool is_s10 = device->edac->is_s10; rc = altr_check_ecc_deps(device); if (rc) @@ -1533,7 +1534,7 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) altdev = dci->pvt_info; *altdev = *device; - if (!devres_open_group(&altdev->ddev, altr_portb_setup, GFP_KERNEL)) + if (!devres_open_group(device->edac->dev, altr_portb_setup, GFP_KERNEL)) return -ENOMEM; /* Update PortB specific values */ @@ -1548,21 +1549,20 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) /* * Update the PortB IRQs - A10 has 4, S10 has 2, Index accordingly - * - * FIXME: Instead of ifdefs with different architectures the driver - * should properly use compatibles. */ -#ifdef CONFIG_64BIT - altdev->sb_irq = irq_of_parse_and_map(np, 1); -#else - altdev->sb_irq = irq_of_parse_and_map(np, 2); -#endif + + /* Using compatibles to determine the IRQ Index */ + if (is_s10) + altdev->sb_irq = irq_of_parse_and_map(np, 1); + else + altdev->sb_irq = irq_of_parse_and_map(np, 2); + if (!altdev->sb_irq) { edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB SBIRQ alloc\n"); rc = -ENODEV; goto err_release_group_1; } - rc = devm_request_irq(&altdev->ddev, altdev->sb_irq, + rc = devm_request_irq(device->edac->dev, altdev->sb_irq, prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, ecc_name, altdev); if (rc) { @@ -1570,29 +1570,28 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) goto err_release_group_1; } -#ifdef CONFIG_64BIT - /* Use IRQ to determine SError origin instead of assigning IRQ */ - rc = of_property_read_u32_index(np, "interrupts", 1, &altdev->db_irq); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, - "Error PortB DBIRQ alloc\n"); - goto err_release_group_1; - } -#else - altdev->db_irq = irq_of_parse_and_map(np, 3); - if (!altdev->db_irq) { - edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n"); - rc = -ENODEV; - goto err_release_group_1; - } - rc = devm_request_irq(&altdev->ddev, altdev->db_irq, - prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, - ecc_name, altdev); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); - goto err_release_group_1; + if (is_s10) { + /* Use IRQ to determine SError origin instead of assigning IRQ */ + rc = of_property_read_u32_index(np, "interrupts", 1, &altdev->db_irq); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n"); + goto err_release_group_1; + } + } else { + altdev->db_irq = irq_of_parse_and_map(np, 3); + if (!altdev->db_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error PortB DBIRQ alloc\n"); + rc = -ENODEV; + goto err_release_group_1; + } + rc = devm_request_irq(device->edac->dev, altdev->db_irq, + prv->ecc_irq_handler, IRQF_TRIGGER_HIGH, + ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "PortB DBERR IRQ error\n"); + goto err_release_group_1; + } } -#endif rc = edac_device_add_device(dci); if (rc) { @@ -1605,13 +1604,13 @@ static int altr_portb_setup(struct altr_edac_device_dev *device) list_add(&altdev->next, &altdev->edac->a10_ecc_devices); - devres_remove_group(&altdev->ddev, altr_portb_setup); + devres_remove_group(device->edac->dev, altr_portb_setup); return 0; err_release_group_1: edac_device_free_ctl_info(dci); - devres_release_group(&altdev->ddev, altr_portb_setup); + devres_release_group(device->edac->dev, altr_portb_setup); edac_printk(KERN_ERR, EDAC_DEVICE, "%s:Error setting up EDAC device: %d\n", ecc_name, rc); return rc; @@ -1974,29 +1973,29 @@ static int altr_edac_a10_device_add(struct altr_arria10_edac *edac, goto err_release_group1; } -#ifdef CONFIG_64BIT - /* Use IRQ to determine SError origin instead of assigning IRQ */ - rc = of_property_read_u32_index(np, "interrupts", 0, &altdev->db_irq); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, - "Unable to parse DB IRQ index\n"); - goto err_release_group1; - } -#else - altdev->db_irq = irq_of_parse_and_map(np, 1); - if (!altdev->db_irq) { - edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n"); - rc = -ENODEV; - goto err_release_group1; - } - rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, - IRQF_TRIGGER_HIGH, - ecc_name, altdev); - if (rc) { - edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); - goto err_release_group1; + if (edac->is_s10) { + /* Use IRQ to determine SError origin instead of assigning IRQ */ + rc = of_property_read_u32_index(np, "interrupts", 0, &altdev->db_irq); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, + "Unable to parse DB IRQ index\n"); + goto err_release_group1; + } + } else { + altdev->db_irq = irq_of_parse_and_map(np, 1); + if (!altdev->db_irq) { + edac_printk(KERN_ERR, EDAC_DEVICE, "Error allocating DBIRQ\n"); + rc = -ENODEV; + goto err_release_group1; + } + rc = devm_request_irq(edac->dev, altdev->db_irq, prv->ecc_irq_handler, + IRQF_TRIGGER_HIGH, + ecc_name, altdev); + if (rc) { + edac_printk(KERN_ERR, EDAC_DEVICE, "No DBERR IRQ resource\n"); + goto err_release_group1; + } } -#endif rc = edac_device_add_device(dci); if (rc) { @@ -2122,6 +2121,8 @@ static int altr_edac_a10_probe(struct platform_device *pdev) platform_set_drvdata(pdev, edac); INIT_LIST_HEAD(&edac->a10_ecc_devices); + edac->is_s10 = !!device_get_match_data(&pdev->dev); + edac->ecc_mgr_map = altr_sysmgr_regmap_lookup_by_phandle(pdev->dev.of_node, "altr,sysmgr-syscon"); @@ -2207,7 +2208,7 @@ static int altr_edac_a10_probe(struct platform_device *pdev) static const struct of_device_id altr_edac_a10_of_match[] = { { .compatible = "altr,socfpga-a10-ecc-manager" }, - { .compatible = "altr,socfpga-s10-ecc-manager" }, + { .compatible = "altr,socfpga-s10-ecc-manager", .data = (void *)1 }, {}, }; MODULE_DEVICE_TABLE(of, altr_edac_a10_of_match); diff --git a/drivers/edac/altera_edac.h b/drivers/edac/altera_edac.h index 7248d24c4908d..5b26ef0cfe600 100644 --- a/drivers/edac/altera_edac.h +++ b/drivers/edac/altera_edac.h @@ -395,6 +395,7 @@ struct altr_arria10_edac { struct irq_chip irq_chip; struct list_head a10_ecc_devices; struct notifier_block panic_notifier; + bool is_s10; }; #endif /* #ifndef _ALTERA_EDAC_H */ diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2f6ab783bf209..8bec21dbc7575 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -4201,6 +4201,8 @@ static int __init amd64_edac_init(void) goto err_pci; } + request_module_nowait("amd_atl"); + /* register stuff with EDAC MCE */ if (boot_cpu_data.x86 >= 0x17) { amd_register_ecc_decoder(decode_umc_error); diff --git a/drivers/edac/edac_device_sysfs.c b/drivers/edac/edac_device_sysfs.c index fcebc4ffea26d..71940f43a0dd8 100644 --- a/drivers/edac/edac_device_sysfs.c +++ b/drivers/edac/edac_device_sysfs.c @@ -90,14 +90,21 @@ static ssize_t edac_device_ctl_poll_msec_store(struct edac_device_ctl_info *ctl_info, const char *data, size_t count) { - unsigned long value; + unsigned int value; + int ret; /* get the value and enforce that it is non-zero, must be at least * one millisecond for the delay period, between scans * Then cancel last outstanding delay for the work request * and set a new one. */ - value = simple_strtoul(data, NULL, 0); + ret = kstrtouint(data, 0, &value); + if (ret < 0) + return ret; + + if (value < 1) + return -EINVAL; + edac_device_reset_delay_period(ctl_info, value); return count; diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 42eadece2d71b..c25081f61a51f 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -704,14 +704,22 @@ static enum mem_type get_memory_type(u32 mad_inter) static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) { - u64 hash_addr = addr & mask, hash = 0; - u64 intlv = (addr >> intlv_bit) & 1; + u64 hash_addr, hash = 0; int i; + /* + * In hash mode, the @intlv_bit is the lowest selected bit of @addr + * to be XORed. While @mask may or may not include this @intlv_bit, + * we enforce that @mask includes @intlv_bit to ensure @intlv_bit is + * XORed exactly once. + */ + mask |= 1 << intlv_bit; + hash_addr = addr & mask; + for (i = 6; i < 20; i++) hash ^= (hash_addr >> i) & 1; - return (int)hash ^ intlv; + return (int)hash; } static u64 decode_channel_addr(u64 addr, int intlv_bit) @@ -730,19 +738,18 @@ static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, { int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; - if (addr > 2 * s_size) { + if (addr >= 2 * s_size) { *sub_addr = addr - s_size; *idx = l_map; return; } - if (CHANNEL_HASH_MODE(hash)) { - *sub_addr = decode_channel_addr(addr, intlv_bit); + *sub_addr = decode_channel_addr(addr, intlv_bit); + + if (CHANNEL_HASH_MODE(hash)) *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); - } else { - *sub_addr = decode_channel_addr(addr, 6); - *idx = GET_BITFIELD(addr, 6, 6); - } + else + *idx = GET_BITFIELD(addr, intlv_bit, intlv_bit); } static int igen6_decode(struct decoded_addr *res) diff --git a/drivers/firewire/.kunitconfig b/drivers/firewire/.kunitconfig index 21b7e9eef63d1..7406acb004782 100644 --- a/drivers/firewire/.kunitconfig +++ b/drivers/firewire/.kunitconfig @@ -6,3 +6,4 @@ CONFIG_FIREWIRE_KUNIT_DEVICE_ATTRIBUTE_TEST=y CONFIG_FIREWIRE_KUNIT_PACKET_SERDES_TEST=y CONFIG_FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST=y CONFIG_FIREWIRE_KUNIT_OHCI_SERDES_TEST=y +CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST=y diff --git a/drivers/firewire/Kconfig b/drivers/firewire/Kconfig index a5f5e250223a1..b5abe00accc9d 100644 --- a/drivers/firewire/Kconfig +++ b/drivers/firewire/Kconfig @@ -81,6 +81,21 @@ config FIREWIRE_KUNIT_SELF_ID_SEQUENCE_HELPER_TEST For more information on KUnit and unit tests in general, refer to the KUnit documentation in Documentation/dev-tools/kunit/. +config FIREWIRE_KUNIT_NODE_TREE_TEST + tristate "KUnit tests for node tree" if !KUNIT_ALL_TESTS + depends on FIREWIRE && KUNIT + default KUNIT_ALL_TESTS + help + This builds the KUnit tests for node tree. + + KUnit tests run during boot and output the results to the debug + log in TAP format (https://testanything.org/). Only useful for + kernel devs running KUnit test harness and are not for inclusion + into a production build. + + For more information on KUnit and unit tests in general, refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + config FIREWIRE_OHCI tristate "OHCI-1394 controllers" depends on PCI && FIREWIRE diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c index ed3ae8cdb0cdb..14eb8e3f84c84 100644 --- a/drivers/firewire/core-topology.c +++ b/drivers/firewire/core-topology.c @@ -88,6 +88,17 @@ static inline struct fw_node *fw_node(struct list_head *l) return list_entry(l, struct fw_node, link); } +typedef void (*fw_node_callback_t)(struct fw_card *card, struct fw_node *node, + struct fw_node *parent); + +static void for_each_fw_node(struct fw_card *card, struct fw_node *root, + fw_node_callback_t callback); + +static void free_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + kfree(node); +} + /* * This function builds the tree representation of the topology given * by the self IDs from the latest bus reset. During the construction @@ -119,8 +130,8 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self while (enumerator.quadlet_count > 0) { unsigned int child_port_count = 0; + unsigned int parent_port_count = 0; unsigned int total_port_count = 0; - unsigned int parent_count = 0; unsigned int quadlet_count; const u32 *self_id_sequence; unsigned int port_capacity; @@ -134,7 +145,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self if (PTR_ERR(self_id_sequence) != -ENODATA) { fw_err(card, "inconsistent extended self IDs: %ld\n", PTR_ERR(self_id_sequence)); - return NULL; + goto error; } break; } @@ -148,26 +159,38 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self switch (port_status) { case PHY_PACKET_SELF_ID_PORT_STATUS_CHILD: ++child_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_PARENT: + ++parent_port_count; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NCONN: ++total_port_count; - fallthrough; + break; case PHY_PACKET_SELF_ID_PORT_STATUS_NONE: default: break; } } + total_port_count += child_port_count + parent_port_count; + + // Check that the node reports exactly one parent port, except for the root, which + // of course should have no parents. + if ((enumerator.quadlet_count == 0 && parent_port_count != 0) || + (enumerator.quadlet_count > 0 && parent_port_count != 1)) { + fw_err(card, "parent port inconsistency for node %d: parent_count=%d\n", + phy_id, parent_port_count); + goto error; + } if (phy_id != phy_packet_self_id_get_phy_id(self_id_sequence[0])) { fw_err(card, "PHY ID mismatch in self ID: %d != %d\n", phy_id, phy_packet_self_id_get_phy_id(self_id_sequence[0])); - return NULL; + goto error; } if (child_port_count > stack_depth) { fw_err(card, "topology stack underflow\n"); - return NULL; + goto error; } /* @@ -185,7 +208,7 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self node = fw_node_create(self_id_sequence[0], total_port_count, card->color); if (node == NULL) { fw_err(card, "out of memory while building topology\n"); - return NULL; + goto error; } if (phy_id == (card->node_id & 0x3f)) @@ -203,7 +226,6 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self // we temporarily abuse node->color for remembering the entry in // the node->ports array where the parent node should be. Later, // when we handle the parent node, we fix up the reference. - ++parent_count; node->color = port_index; break; @@ -221,21 +243,12 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self } } - // Check that the node reports exactly one parent port, except for the root, which - // of course should have no parents. - if ((enumerator.quadlet_count == 0 && parent_count != 0) || - (enumerator.quadlet_count > 0 && parent_count != 1)) { - fw_err(card, "parent port inconsistency for node %d: " - "parent_count=%d\n", phy_id, parent_count); - return NULL; - } - /* Pop the child nodes off the stack and push the new node. */ __list_del(h->prev, &stack); list_add_tail(&node->link, &stack); stack_depth += 1 - child_port_count; - if (node->phy_speed == SCODE_BETA && parent_count + child_port_count > 1) + if (node->phy_speed == SCODE_BETA && parent_port_count + child_port_count > 1) beta_repeaters_present = true; // If PHYs report different gap counts, set an invalid count which will force a gap @@ -254,12 +267,13 @@ static struct fw_node *build_tree(struct fw_card *card, const u32 *sid, int self card->beta_repeaters_present = beta_repeaters_present; return local_node; +error: + ++card->color; + list_for_each_entry_safe(node, child, &stack, link) + for_each_fw_node(card, node, free_fw_node); + return NULL; } -typedef void (*fw_node_callback_t)(struct fw_card * card, - struct fw_node * node, - struct fw_node * parent); - static void for_each_fw_node(struct fw_card *card, struct fw_node *root, fw_node_callback_t callback) { @@ -505,3 +519,7 @@ void fw_core_handle_bus_reset(struct fw_card *card, int node_id, int generation, } } EXPORT_SYMBOL(fw_core_handle_bus_reset); + +#ifdef CONFIG_FIREWIRE_KUNIT_NODE_TREE_TEST +#include "node-tree-test.c" +#endif diff --git a/drivers/firewire/net.c b/drivers/firewire/net.c index e829454089550..354c81409946a 100644 --- a/drivers/firewire/net.c +++ b/drivers/firewire/net.c @@ -298,31 +298,34 @@ static struct fwnet_fragment_info *fwnet_frag_new( if (fi->offset + fi->len == offset) { /* The new fragment can be tacked on to the end */ /* Did the new fragment plug a hole? */ - fi2 = list_entry(fi->fi_link.next, - struct fwnet_fragment_info, fi_link); - if (fi->offset + fi->len == fi2->offset) { - /* glue fragments together */ - fi->len += len + fi2->len; - list_del(&fi2->fi_link); - kfree(fi2); - } else { - fi->len += len; + if (!list_is_last(&fi->fi_link, &pd->fi_list)) { + fi2 = list_next_entry(fi, fi_link); + if (offset + len == fi2->offset) { + /* glue fragments together */ + fi->len += len + fi2->len; + list_del(&fi2->fi_link); + kfree(fi2); + + return fi; + } } + fi->len += len; return fi; } if (offset + len == fi->offset) { /* The new fragment can be tacked on to the beginning */ /* Did the new fragment plug a hole? */ - fi2 = list_entry(fi->fi_link.prev, - struct fwnet_fragment_info, fi_link); - if (fi2->offset + fi2->len == fi->offset) { - /* glue fragments together */ - fi2->len += fi->len + len; - list_del(&fi->fi_link); - kfree(fi); - - return fi2; + if (!list_is_first(&fi->fi_link, &pd->fi_list)) { + fi2 = list_prev_entry(fi, fi_link); + if (fi2->offset + fi2->len == offset) { + /* glue fragments together */ + fi2->len += fi->len + len; + list_del(&fi->fi_link); + kfree(fi); + + return fi2; + } } fi->offset = offset; fi->len += len; diff --git a/drivers/firewire/node-tree-test.c b/drivers/firewire/node-tree-test.c new file mode 100644 index 0000000000000..5e6bb58537e94 --- /dev/null +++ b/drivers/firewire/node-tree-test.c @@ -0,0 +1,607 @@ +// SPDX-License-Identifier: GPL-2.0-only +// +// node-tree-test.c - An application of Kunit to test node tree. +// +// Copyright (c) 2026 Takashi Sakamoto +// +// This file can not be built independently since it is intentionally included in core-topology.c. + +#include +#include +#include + +struct private_data { + struct fw_card *card; + unsigned int release_count; +}; + +static int node_tree_test_init(struct kunit *test) +{ + struct private_data *data; + + data = kunit_kzalloc(test, sizeof(*data), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data); + + data->card = kunit_kzalloc(test, sizeof(struct fw_card), GFP_KERNEL); + KUNIT_ASSERT_NOT_NULL(test, data->card); + + data->card->device = kunit_device_register(test, "dummy-device"); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->card->device); + + test->priv = data; + + return 0; +} + +static void node_tree_test_exit(struct kunit *test) +{ + struct private_data *data = test->priv; + + kunit_device_unregister(test, data->card->device); + kunit_kfree(test, data->card); + kunit_kfree(test, data); +} + +static void release_fw_node(struct fw_card *card, struct fw_node *node, struct fw_node *parent) +{ + struct private_data *data = kunit_get_current_test()->priv; + + fw_node_put(node); + ++data->release_count; +} + +static void node_tree_test_two_nodes(struct kunit *test) +{ + // root + // ++============++ + // || phy 1 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // +-----+ + // | + // ++===|==x==x==++ + // || P0 P1 P2 || + // || phy 0 || + // ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100005e, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_two_nodes_1394a(struct kunit *test) +{ + // root + // ++===============++ + // || phy 0 || + // || P0 P1 P2 P3 || + // ++===|==|==|==|==++ + // | + // +--+ + // | + // ++===|==|==|==|==|==++ + // || P0 P1 P2 P3 P4 || + // || phy 1 || + // ++==================++ + // + // NOTE: Just for Self-ID Packets Zero and One. + static const u32 self_id_sequence[] = { + 0x80000065, 0x80814000, + 0x8100005d, 0x81810000, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x01; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 4); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 5); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NULL(test, node->ports[2]); + KUNIT_EXPECT_NULL(test, node->ports[3]); + KUNIT_EXPECT_NULL(test, node->ports[4]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 2); +} + +static void node_tree_test_three_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // +--+ +-----------------+ + // | | + // ++===|==|==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000060, + 0x81000058, + 0x820000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_three_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 2 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +-----------+ + // | | | + // ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || + // ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x8100006c, + 0x82000070, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x02; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 3); +} + +static void node_tree_test_four_nodes_case0(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | + // | +-----------+ +--------------+ + // | | | | | + // ++===|==|==|==++ ++===|==|==x==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000080, + 0x810000b0, + 0x8200006c, + 0x83000074, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case1(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==x==++ + // | + // | +--------------------------------+ + // | | +-----------+ | + // ++===|==|==|==++ ++===|==x==x==++ ++===|==|==|==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 2 || || phy 1 || || phy 0 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d0, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + struct fw_node *parent = node; + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case2(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | + // | +-----------------------------+ + // | +--------------+ | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==x==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 1 || || phy 0 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000094, + 0x810000b0, + 0x82000080, + 0x830000dc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 1); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + + parent = node; + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NULL(test, node->ports[2]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_four_nodes_case3(struct kunit *test) +{ + // root + // ++============++ + // || phy 3 || + // || P0 P1 P2 || + // ++===|==|==|==++ + // | | +--------------------------------+ + // | +--------------------+ | + // | | | + // ++===|==|==x==++ ++===|==|==|==++ ++===|==|==x==++ + // || P0 P1 P2 || || P0 P1 P2 || || P0 P1 P2 || + // || phy 0 || || phy 1 || || phy 2 || + // ++============++ ++============++ ++============++ + // + static const u32 self_id_sequence[] = { + 0x80000090, + 0x81000058, + 0x82000060, + 0x830000fc, + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NOT_NULL(test, card->local_node); + KUNIT_EXPECT_PTR_EQ(test, card->local_node, card->root_node); + + struct fw_node *node = card->root_node; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x03); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NOT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[1]); + KUNIT_EXPECT_NOT_NULL(test, node->ports[2]); + + struct fw_node *parent = node; + node = parent->ports[2]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x02); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[1], parent); + + node = parent->ports[1]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x01); + KUNIT_EXPECT_EQ(test, node->port_count, 3); + KUNIT_EXPECT_NULL(test, node->ports[0]); + KUNIT_EXPECT_NULL(test, node->ports[1]); + KUNIT_EXPECT_PTR_EQ(test, node->ports[2], parent); + + node = parent->ports[0]; + KUNIT_EXPECT_EQ(test, node->node_id, LOCAL_BUS | 0x00); + KUNIT_EXPECT_EQ(test, node->port_count, 2); + KUNIT_EXPECT_PTR_EQ(test, node->ports[0], parent); + KUNIT_EXPECT_NULL(test, node->ports[1]); + + ++card->color; + for_each_fw_node(card, card->root_node, release_fw_node); + KUNIT_EXPECT_EQ(test, data->release_count, 4); +} + +static void node_tree_test_invalid_extended_self_id_sequence(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // content of self ID packet for the phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000d1, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_phy_id(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // phy ID for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x8f0000d0, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_child_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of child ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000fc, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static void node_tree_test_invalid_parent_port_count(struct kunit *test) +{ + // Use the same node tree as node_tree_test_four_nodes_case1, except for the invalid + // count of parent ports for phy 3. + static const u32 self_id_sequence[] = { + 0x80000094, + 0x81000080, + 0x820000bc, + 0x830000e8, // Invalid. + }; + struct private_data *data = test->priv; + struct fw_card *card = data->card; + + card->node_id = LOCAL_BUS | 0x03; + + card->local_node = build_tree(card, self_id_sequence, ARRAY_SIZE(self_id_sequence), 123); + KUNIT_EXPECT_NULL(test, card->local_node); +} + +static struct kunit_case node_tree_test_cases[] = { + KUNIT_CASE(node_tree_test_two_nodes), + KUNIT_CASE(node_tree_test_two_nodes_1394a), + KUNIT_CASE(node_tree_test_three_nodes_case0), + KUNIT_CASE(node_tree_test_three_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case0), + KUNIT_CASE(node_tree_test_four_nodes_case1), + KUNIT_CASE(node_tree_test_four_nodes_case2), + KUNIT_CASE(node_tree_test_four_nodes_case3), + KUNIT_CASE(node_tree_test_invalid_extended_self_id_sequence), + KUNIT_CASE(node_tree_test_invalid_phy_id), + KUNIT_CASE(node_tree_test_invalid_child_port_count), + KUNIT_CASE(node_tree_test_invalid_parent_port_count), + {} +}; + +static struct kunit_suite node_tree_test_suite = { + .name = "firewire-node-tree", + .init = node_tree_test_init, + .exit = node_tree_test_exit, + .test_cases = node_tree_test_cases, +}; +kunit_test_suite(node_tree_test_suite); diff --git a/drivers/firewire/ohci.c b/drivers/firewire/ohci.c index 030aed5453a17..a72b0dad6c3f1 100644 --- a/drivers/firewire/ohci.c +++ b/drivers/firewire/ohci.c @@ -538,19 +538,25 @@ static void ar_context_link_page(struct ar_context *ctx, unsigned int index) static void ar_context_release(struct ar_context *ctx) { - struct device *dev = ctx->ohci->card.device; - unsigned int i; + struct device *dev; if (!ctx->buffer) return; - vunmap(ctx->buffer); + dev = ctx->ohci->card.device; - for (i = 0; i < AR_BUFFERS; i++) { - if (ctx->pages[i]) - dma_free_pages(dev, PAGE_SIZE, ctx->pages[i], - ar_buffer_bus(ctx, i), DMA_FROM_DEVICE); + for (int i = 0; i < AR_BUFFERS; ++i) { + dma_addr_t dma_addr = page_private(ctx->pages[i]); + + dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); + set_page_private(ctx->pages[i], 0); } + + vunmap(ctx->buffer); + ctx->buffer = NULL; + + release_pages(ctx->pages, AR_BUFFERS); + memset(ctx->pages, 0, sizeof(ctx->pages)); } static void ar_context_abort(struct ar_context *ctx, const char *error_msg) @@ -845,31 +851,57 @@ static int ar_context_init(struct ar_context *ctx, struct fw_ohci *ohci, { struct device *dev = ohci->card.device; unsigned int i; - dma_addr_t dma_addr; - struct page *pages[AR_BUFFERS + AR_WRAPAROUND_PAGES]; + struct page *pages[AR_BUFFERS + AR_WRAPAROUND_PAGES] = { NULL }; + void *vaddr; struct descriptor *d; ctx->regs = regs; ctx->ohci = ohci; INIT_WORK(&ctx->work, ohci_ar_context_work); - for (i = 0; i < AR_BUFFERS; i++) { - ctx->pages[i] = dma_alloc_pages(dev, PAGE_SIZE, &dma_addr, - DMA_FROM_DEVICE, GFP_KERNEL); - if (!ctx->pages[i]) - goto out_of_memory; - set_page_private(ctx->pages[i], dma_addr); - dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, - DMA_FROM_DEVICE); + // Retrieve noncontiguous pages. The descriptors for 1394 OHCI AR DMA contexts have a set + // of address and length per each. The reason to use pages is to construct contiguous + // address range in kernel virtual address space. + unsigned long nr_populated = alloc_pages_bulk(GFP_KERNEL | GFP_DMA32, AR_BUFFERS, pages); + + if (nr_populated != AR_BUFFERS) { + release_pages(pages, nr_populated); + return -ENOMEM; } - for (i = 0; i < AR_BUFFERS; i++) - pages[i] = ctx->pages[i]; + // Map the pages into contiguous kernel virtual addresses so that the packet data + // across the pages can be referred as being contiguous, especially across the last + // and first pages. for (i = 0; i < AR_WRAPAROUND_PAGES; i++) - pages[AR_BUFFERS + i] = ctx->pages[i]; - ctx->buffer = vmap(pages, ARRAY_SIZE(pages), VM_MAP, PAGE_KERNEL); - if (!ctx->buffer) - goto out_of_memory; + pages[AR_BUFFERS + i] = pages[i]; + vaddr = vmap(pages, ARRAY_SIZE(pages), VM_MAP, PAGE_KERNEL); + if (!vaddr) { + release_pages(pages, nr_populated); + return -ENOMEM; + } + + // Retrieve DMA mapping addresses for the pages. They are not contiguous. Maintain the cache + // coherency for the pages by hand. + for (i = 0; i < AR_BUFFERS; i++) { + // The dma_map_phys() with a physical address per page is available here, instead. + dma_addr_t dma_addr = dma_map_page(dev, pages[i], 0, PAGE_SIZE, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, dma_addr)) + break; + set_page_private(pages[i], dma_addr); + dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); + } + if (i < AR_BUFFERS) { + while (i-- > 0) { + dma_addr_t dma_addr = page_private(pages[i]); + dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE); + } + vunmap(vaddr); + release_pages(pages, nr_populated); + return -ENOMEM; + } + + ctx->buffer = vaddr; + memcpy(ctx->pages, pages, sizeof(ctx->pages)); ctx->descriptors = ohci->misc_buffer + descriptors_offset; ctx->descriptors_bus = ohci->misc_buffer_bus + descriptors_offset; @@ -886,11 +918,6 @@ static int ar_context_init(struct ar_context *ctx, struct fw_ohci *ohci, } return 0; - -out_of_memory: - ar_context_release(ctx); - - return -ENOMEM; } static void ar_context_run(struct ar_context *ctx) diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c index 3fbc75caf0c50..1d0bd586da4d6 100644 --- a/drivers/firmware/arm_ffa/driver.c +++ b/drivers/firmware/arm_ffa/driver.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -55,7 +56,9 @@ (FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r))) #define RXTX_MAP_MIN_BUFSZ_MASK GENMASK(1, 0) -#define RXTX_MAP_MIN_BUFSZ(x) ((x) & RXTX_MAP_MIN_BUFSZ_MASK) +#define RXTX_MAP_MAX_BUFSZ_MASK GENMASK(31, 16) +#define RXTX_MAP_MIN_BUFSZ(x) (FIELD_GET(RXTX_MAP_MIN_BUFSZ_MASK, (x))) +#define RXTX_MAP_MAX_BUFSZ(x) (FIELD_GET(RXTX_MAP_MAX_BUFSZ_MASK, (x))) #define FFA_MAX_NOTIFICATIONS 64 @@ -704,30 +707,39 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, struct ffa_composite_mem_region *composite; struct ffa_mem_region_addr_range *constituents; struct ffa_mem_region_attributes *ep_mem_access; - u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg); + u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg), ep_offset; + u32 emad_end, emad_size = ffa_emad_size_get(drv_info->version); mem_region->tag = args->tag; mem_region->flags = args->flags; mem_region->sender_id = drv_info->vm_id; mem_region->attributes = ffa_memory_attributes_get(func_id); + + ffa_mem_region_additional_setup(drv_info->version, mem_region); composite_offset = ffa_mem_desc_offset(buffer, args->nattrs, drv_info->version); + if (composite_offset + sizeof(*composite) > max_fragsize) + return -ENXIO; for (idx = 0; idx < args->nattrs; idx++) { - ep_mem_access = buffer + - ffa_mem_desc_offset(buffer, idx, drv_info->version); + ep_offset = ffa_mem_desc_offset(buffer, idx, drv_info->version); + if (check_add_overflow(ep_offset, emad_size, &emad_end)) + return -ENXIO; + + if (emad_end > max_fragsize) + return -ENXIO; + + ep_mem_access = buffer + ep_offset; + memset(ep_mem_access, 0, emad_size); ep_mem_access->receiver = args->attrs[idx].receiver; ep_mem_access->attrs = args->attrs[idx].attrs; ep_mem_access->composite_off = composite_offset; - ep_mem_access->flag = 0; - ep_mem_access->reserved = 0; ffa_emad_impdef_value_init(drv_info->version, ep_mem_access->impdef_val, args->attrs[idx].impdef_val); } mem_region->handle = 0; mem_region->ep_count = args->nattrs; - ffa_mem_region_additional_setup(drv_info->version, mem_region); composite = buffer + composite_offset; composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg); @@ -760,7 +772,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, constituents = buffer; } - if ((void *)constituents - buffer > max_fragsize) { + if ((void *)constituents + sizeof(*constituents) - buffer > max_fragsize) { pr_err("Memory Region Fragment > Tx Buffer size\n"); return -EFAULT; } @@ -769,7 +781,7 @@ ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE; constituents->reserved = 0; constituents++; - frag_len += sizeof(struct ffa_mem_region_addr_range); + frag_len += sizeof(*constituents); } while ((args->sg = sg_next(args->sg))); return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len, @@ -1130,7 +1142,7 @@ static int ffa_partition_info_get(const char *uuid_str, uuid_t uuid; struct ffa_partition_info *pbuf; - if (uuid_parse(uuid_str, &uuid)) { + if (!uuid_str || uuid_parse(uuid_str, &uuid)) { pr_err("invalid uuid (%s)\n", uuid_str); return -ENODEV; } @@ -2088,7 +2100,7 @@ static int __init ffa_init(void) { int ret; u32 buf_sz; - size_t rxtx_bufsz = SZ_4K; + size_t rxtx_min_bufsz = SZ_4K, rxtx_max_bufsz = 0, rxtx_bufsz; ret = ffa_transport_init(&invoke_ffa_fn); if (ret) @@ -2111,15 +2123,18 @@ static int __init ffa_init(void) ret = ffa_features(FFA_FN_NATIVE(RXTX_MAP), 0, &buf_sz, NULL); if (!ret) { if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 1) - rxtx_bufsz = SZ_64K; + rxtx_min_bufsz = SZ_64K; else if (RXTX_MAP_MIN_BUFSZ(buf_sz) == 2) - rxtx_bufsz = SZ_16K; + rxtx_min_bufsz = SZ_16K; else - rxtx_bufsz = SZ_4K; + rxtx_min_bufsz = SZ_4K; + + rxtx_max_bufsz = RXTX_MAP_MAX_BUFSZ(buf_sz) * SZ_4K; + if (rxtx_max_bufsz != 0 && rxtx_max_bufsz < rxtx_min_bufsz) + rxtx_max_bufsz = rxtx_min_bufsz; } - rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz); - drv_info->rxtx_bufsz = rxtx_bufsz; + rxtx_bufsz = min_not_zero(PAGE_ALIGN(rxtx_min_bufsz), rxtx_max_bufsz); drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL); if (!drv_info->rx_buffer) { ret = -ENOMEM; @@ -2135,10 +2150,17 @@ static int __init ffa_init(void) ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer), virt_to_phys(drv_info->rx_buffer), rxtx_bufsz / FFA_PAGE_SIZE); + if (ret == -EINVAL && !rxtx_max_bufsz && rxtx_min_bufsz < rxtx_bufsz) { + rxtx_bufsz = rxtx_min_bufsz; + ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer), + virt_to_phys(drv_info->rx_buffer), + rxtx_bufsz / FFA_PAGE_SIZE); + } if (ret) { pr_err("failed to register FFA RxTx buffers\n"); goto free_pages; } + drv_info->rxtx_bufsz = rxtx_bufsz; mutex_init(&drv_info->rx_lock); mutex_init(&drv_info->tx_lock); diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c index 86b376c50a13f..25aa52746bd10 100644 --- a/drivers/firmware/arm_scmi/base.c +++ b/drivers/firmware/arm_scmi/base.c @@ -325,6 +325,8 @@ static void *scmi_base_fill_custom_report(const struct scmi_protocol_handle *ph, void *report, u32 *src_id) { int i; + u32 error_status; + size_t expected_sz; const struct scmi_base_error_notify_payld *p = payld; struct scmi_base_error_report *r = report; @@ -338,10 +340,19 @@ static void *scmi_base_fill_custom_report(const struct scmi_protocol_handle *ph, if (evt_id != SCMI_EVENT_BASE_ERROR_EVENT || sizeof(*p) < payld_sz) return NULL; + expected_sz = offsetof(typeof(*p), msg_reports); + if (payld_sz < expected_sz) + return NULL; + r->timestamp = timestamp; r->agent_id = le32_to_cpu(p->agent_id); - r->fatal = IS_FATAL_ERROR(le32_to_cpu(p->error_status)); - r->cmd_count = ERROR_CMD_COUNT(le32_to_cpu(p->error_status)); + error_status = le32_to_cpu(p->error_status); + r->fatal = IS_FATAL_ERROR(error_status); + r->cmd_count = ERROR_CMD_COUNT(error_status); + expected_sz += r->cmd_count * sizeof(p->msg_reports[0]); + if (payld_sz < expected_sz) + return NULL; + for (i = 0; i < r->cmd_count; i++) r->reports[i] = le64_to_cpu(p->msg_reports[i]); *src_id = 0; diff --git a/drivers/firmware/arm_scmi/bus.c b/drivers/firmware/arm_scmi/bus.c index c7698cfaa4e83..290047b46d51d 100644 --- a/drivers/firmware/arm_scmi/bus.c +++ b/drivers/firmware/arm_scmi/bus.c @@ -136,17 +136,6 @@ static int scmi_protocol_device_request(const struct scmi_device_id *id_table) return ret; } -static int scmi_protocol_table_register(const struct scmi_device_id *id_table) -{ - int ret = 0; - const struct scmi_device_id *entry; - - for (entry = id_table; entry->name && ret == 0; entry++) - ret = scmi_protocol_device_request(entry); - - return ret; -} - /** * scmi_protocol_device_unrequest - Helper to unrequest a device * @@ -159,6 +148,7 @@ static int scmi_protocol_table_register(const struct scmi_device_id *id_table) */ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table) { + struct scmi_requested_dev *rdev, *victim = NULL; struct list_head *phead; pr_debug("Unrequesting SCMI device (%s) for protocol %x\n", @@ -167,29 +157,48 @@ static void scmi_protocol_device_unrequest(const struct scmi_device_id *id_table mutex_lock(&scmi_requested_devices_mtx); phead = idr_find(&scmi_requested_devices, id_table->protocol_id); if (phead) { - struct scmi_requested_dev *victim, *tmp; - - list_for_each_entry_safe(victim, tmp, phead, node) { - if (!strcmp(victim->id_table->name, id_table->name)) { - list_del(&victim->node); - - mutex_unlock(&scmi_requested_devices_mtx); - blocking_notifier_call_chain(&scmi_requested_devices_nh, - SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, - (void *)victim->id_table); - kfree(victim); - mutex_lock(&scmi_requested_devices_mtx); + list_for_each_entry(rdev, phead, node) { + if (!strcmp(rdev->id_table->name, id_table->name)) { + victim = rdev; + list_del(&rdev->node); break; } } - if (list_empty(phead)) { + if (victim && list_empty(phead)) { idr_remove(&scmi_requested_devices, id_table->protocol_id); kfree(phead); } } mutex_unlock(&scmi_requested_devices_mtx); + + if (victim) { + blocking_notifier_call_chain(&scmi_requested_devices_nh, + SCMI_BUS_NOTIFY_DEVICE_UNREQUEST, + (void *)victim->id_table); + kfree(victim); + } +} + +static int scmi_protocol_table_register(const struct scmi_device_id *id_table) +{ + const struct scmi_device_id *entry; + int ret; + + for (entry = id_table; entry->name; entry++) { + ret = scmi_protocol_device_request(entry); + if (ret) + goto err_unrequest; + } + + return 0; + +err_unrequest: + while (entry != id_table) + scmi_protocol_device_unrequest(--entry); + + return ret; } static void @@ -201,21 +210,33 @@ scmi_protocol_table_unregister(const struct scmi_device_id *id_table) scmi_protocol_device_unrequest(entry); } -static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, - const struct scmi_device_id *id_table) +static bool scmi_device_is_transport(const struct scmi_device *scmi_dev) +{ + return !strncmp(scmi_dev->name, SCMI_TRANSPORT_DEVNAME_PREFIX, + strlen(SCMI_TRANSPORT_DEVNAME_PREFIX)); +} + +static int __scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table, + bool skip_transport) { if (!id_table || !id_table->name) return 0; - /* Always skip transport devices from matching */ for (; id_table->protocol_id && id_table->name; id_table++) if (id_table->protocol_id == scmi_dev->protocol_id && - strncmp(scmi_dev->name, "__scmi_transport_device", 23) && + !(skip_transport && scmi_device_is_transport(scmi_dev)) && !strcmp(id_table->name, scmi_dev->name)) return 1; return 0; } +static int scmi_dev_match_by_id_table(struct scmi_device *scmi_dev, + const struct scmi_device_id *id_table) +{ + return __scmi_dev_match_by_id_table(scmi_dev, id_table, true); +} + static int scmi_dev_match_id(struct scmi_device *scmi_dev, const struct scmi_driver *scmi_drv) { @@ -235,7 +256,7 @@ static int scmi_match_by_id_table(struct device *dev, const void *data) struct scmi_device *scmi_dev = to_scmi_dev(dev); const struct scmi_device_id *id_table = data; - return scmi_dev_match_by_id_table(scmi_dev, id_table); + return __scmi_dev_match_by_id_table(scmi_dev, id_table, false); } static struct scmi_device *scmi_child_dev_find(struct device *parent, @@ -377,10 +398,14 @@ int scmi_driver_register(struct scmi_driver *driver, struct module *owner, driver->driver.mod_name = mod_name; retval = driver_register(&driver->driver); - if (!retval) - pr_debug("Registered new scmi driver %s\n", driver->name); + if (retval) { + scmi_protocol_table_unregister(driver->id_table); + return retval; + } - return retval; + pr_debug("Registered new scmi driver %s\n", driver->name); + + return 0; } EXPORT_SYMBOL_GPL(scmi_driver_register); diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h index 7c35c95fddbaf..ef803e1d25e15 100644 --- a/drivers/firmware/arm_scmi/common.h +++ b/drivers/firmware/arm_scmi/common.h @@ -33,6 +33,8 @@ #define SCMI_SHMEM_MAX_PAYLOAD_SIZE 104 +#define SCMI_TRANSPORT_DEVNAME_PREFIX "__scmi_transport_device" + enum scmi_error_codes { SCMI_SUCCESS = 0, /* Success */ SCMI_ERR_SUPPORT = -1, /* Not supported */ diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c index 5caa9191a8d1a..5a9d6df50679c 100644 --- a/drivers/firmware/arm_scmi/driver.c +++ b/drivers/firmware/arm_scmi/driver.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -2540,21 +2541,31 @@ static int scmi_handle_put(const struct scmi_handle *handle) return 0; } -static void scmi_device_link_add(struct device *consumer, +static bool scmi_device_link_add(struct device *consumer, struct device *supplier) { struct device_link *link; link = device_link_add(consumer, supplier, DL_FLAG_AUTOREMOVE_CONSUMER); - WARN_ON(!link); + return !WARN_ON(!link); +} + +static void scmi_clear_handle(struct scmi_device *scmi_dev) +{ + if (!scmi_dev->handle) + return; + + scmi_handle_put(scmi_dev->handle); + scmi_dev->handle = NULL; } static void scmi_set_handle(struct scmi_device *scmi_dev) { scmi_dev->handle = scmi_handle_get(&scmi_dev->dev); - if (scmi_dev->handle) - scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev); + if (scmi_dev->handle && + !scmi_device_link_add(&scmi_dev->dev, scmi_dev->handle->dev)) + scmi_clear_handle(scmi_dev); } static int __scmi_xfer_info_init(struct scmi_info *sinfo, @@ -2663,6 +2674,9 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, idx = tx ? 0 : 1; idr = tx ? &info->tx_idr : &info->rx_idr; + if (idr_find(idr, prot_id)) + return -EEXIST; + if (!info->desc->ops->chan_available(of_node, idx)) { cinfo = idr_find(idr, SCMI_PROTOCOL_BASE); if (unlikely(!cinfo)) /* Possible only if platform has no Rx */ @@ -2679,7 +2693,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, cinfo->max_msg_size = info->desc->max_msg_size; /* Create a unique name for this transport device */ - snprintf(name, 32, "__scmi_transport_device_%s_%02X", + snprintf(name, sizeof(name), SCMI_TRANSPORT_DEVNAME_PREFIX "_%s_%02X", idx ? "rx" : "tx", prot_id); /* Create a uniquely named, dedicated transport device for this chan */ tdev = scmi_device_create(of_node, info->dev, prot_id, name); @@ -2693,6 +2707,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, cinfo->id = prot_id; cinfo->dev = &tdev->dev; + cinfo->handle = &info->handle; ret = info->desc->ops->chan_setup(cinfo, info->dev, tx); if (ret) { of_node_put(of_node); @@ -2718,6 +2733,7 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, "unable to allocate SCMI idr slot err %d\n", ret); /* Destroy channel and device only if created by this call. */ if (tdev) { + info->desc->ops->chan_free(prot_id, cinfo, idr); of_node_put(of_node); scmi_device_destroy(info->dev, prot_id, name); devm_kfree(info->dev, cinfo); @@ -2725,7 +2741,6 @@ static int scmi_chan_setup(struct scmi_info *info, struct device_node *of_node, return ret; } - cinfo->handle = &info->handle; return 0; } @@ -2783,9 +2798,11 @@ static int scmi_channels_setup(struct scmi_info *info) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(info->dev, "Out of range protocol %d\n", prot_id); + continue; + } ret = scmi_txrx_setup(info, child, prot_id); if (ret) @@ -2795,7 +2812,7 @@ static int scmi_channels_setup(struct scmi_info *info) return 0; } -static int scmi_chan_destroy(int id, void *p, void *idr) +static int scmi_chan_destroy(int id, void *p, void *data) { struct scmi_chan_info *cinfo = p; @@ -2804,12 +2821,10 @@ static int scmi_chan_destroy(int id, void *p, void *idr) struct scmi_device *sdev = to_scmi_dev(cinfo->dev); of_node_put(cinfo->dev->of_node); - scmi_device_destroy(info->dev, id, sdev->name); + scmi_device_destroy(info->dev, cinfo->id, sdev->name); cinfo->dev = NULL; } - idr_remove(idr, id); - return 0; } @@ -2836,6 +2851,7 @@ static int scmi_bus_notifier(struct notifier_block *nb, { struct scmi_info *info = bus_nb_to_scmi_info(nb); struct scmi_device *sdev = to_scmi_dev(data); + const char *status; /* Skip devices of different SCMI instances */ if (sdev->dev.parent != info->dev) @@ -2845,18 +2861,22 @@ static int scmi_bus_notifier(struct notifier_block *nb, case BUS_NOTIFY_BIND_DRIVER: /* setup handle now as the transport is ready */ scmi_set_handle(sdev); + status = "about to be BOUND."; + break; + case BUS_NOTIFY_DRIVER_NOT_BOUND: + scmi_clear_handle(sdev); + status = "NOT BOUND."; break; case BUS_NOTIFY_UNBOUND_DRIVER: - scmi_handle_put(sdev->handle); - sdev->handle = NULL; + scmi_clear_handle(sdev); + status = "UNBOUND."; break; default: return NOTIFY_DONE; } dev_dbg(info->dev, "Device %s (%s) is now %s\n", dev_name(&sdev->dev), - sdev->name, action == BUS_NOTIFY_BIND_DRIVER ? - "about to be BOUND." : "UNBOUND."); + sdev->name, status); return NOTIFY_OK; } @@ -2868,7 +2888,9 @@ static int scmi_device_request_notifier(struct notifier_block *nb, struct scmi_device_id *id_table = data; struct scmi_info *info = req_nb_to_scmi_info(nb); + rcu_read_lock(); np = idr_find(&info->active_protocols, id_table->protocol_id); + rcu_read_unlock(); if (!np) return NOTIFY_DONE; @@ -3171,7 +3193,7 @@ static int scmi_probe(struct platform_device *pdev) ret = scmi_channels_setup(info); if (ret) { err_str = "failed to setup channels\n"; - goto clear_ida; + goto clear_txrx_setup; } ret = bus_register_notifier(&scmi_bus_type, &info->bus_nb); @@ -3233,7 +3255,7 @@ static int scmi_probe(struct platform_device *pdev) dev_err(dev, "%s", err_str); return 0; } - goto notification_exit; + goto raw_mode_cleanup; } mutex_lock(&scmi_list_mutex); @@ -3248,8 +3270,10 @@ static int scmi_probe(struct platform_device *pdev) if (of_property_read_u32(child, "reg", &prot_id)) continue; - if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) + if (!FIELD_FIT(MSG_PROTOCOL_ID_MASK, prot_id)) { dev_err(dev, "Out of range protocol %d\n", prot_id); + continue; + } if (!scmi_is_protocol_implemented(handle, prot_id)) { dev_err(dev, "SCMI protocol %d not implemented\n", @@ -3275,18 +3299,18 @@ static int scmi_probe(struct platform_device *pdev) return 0; -notification_exit: +raw_mode_cleanup: if (IS_ENABLED(CONFIG_ARM_SCMI_RAW_MODE_SUPPORT)) scmi_raw_mode_cleanup(info->raw); - scmi_notification_exit(&info->handle); clear_dev_req_notifier: blocking_notifier_chain_unregister(&scmi_requested_devices_nh, &info->dev_req_nb); clear_bus_notifier: bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); clear_txrx_setup: + scmi_notification_quiesce(&info->handle); scmi_cleanup_txrx_channels(info); -clear_ida: + scmi_notification_exit(&info->handle); ida_free(&scmi_id, info->id); out_err: @@ -3309,6 +3333,12 @@ static void scmi_remove(struct platform_device *pdev) list_del(&info->node); mutex_unlock(&scmi_list_mutex); + blocking_notifier_chain_unregister(&scmi_requested_devices_nh, + &info->dev_req_nb); + + /* Stop transport callbacks before tearing down notifications. */ + scmi_notification_quiesce(&info->handle); + scmi_cleanup_txrx_channels(info); scmi_notification_exit(&info->handle); mutex_lock(&info->protocols_mtx); @@ -3319,13 +3349,8 @@ static void scmi_remove(struct platform_device *pdev) of_node_put(child); idr_destroy(&info->active_protocols); - blocking_notifier_chain_unregister(&scmi_requested_devices_nh, - &info->dev_req_nb); bus_unregister_notifier(&scmi_bus_type, &info->bus_nb); - /* Safe to free channels since no more users */ - scmi_cleanup_txrx_channels(info); - ida_free(&scmi_id, info->id); } diff --git a/drivers/firmware/arm_scmi/notify.c b/drivers/firmware/arm_scmi/notify.c index 2047edbdc5f6b..672c91197a68a 100644 --- a/drivers/firmware/arm_scmi/notify.c +++ b/drivers/firmware/arm_scmi/notify.c @@ -600,9 +600,9 @@ int scmi_notify(const struct scmi_handle *handle, u8 proto_id, u8 evt_id, return -EINVAL; } if (kfifo_avail(&r_evt->proto->equeue.kfifo) < sizeof(eh) + len) { - dev_warn(handle->dev, - "queue full, dropping proto_id:%d evt_id:%d ts:%lld\n", - proto_id, evt_id, ktime_to_ns(ts)); + dev_warn_ratelimited(handle->dev, + "queue full, dropping proto_id:%d evt_id:%d ts:%lld\n", + proto_id, evt_id, ktime_to_ns(ts)); return -ENOMEM; } @@ -1706,6 +1706,25 @@ int scmi_notification_init(struct scmi_handle *handle) return -ENOMEM; } +/** + * scmi_notification_quiesce() - Stop notification late initialization + * @handle: The handle identifying the platform instance to quiesce + * + * Prevent new late-init work from being queued and wait for any already queued + * or running late-init work to complete before transport channels are torn + * down. + */ +void scmi_notification_quiesce(struct scmi_handle *handle) +{ + struct scmi_notify_instance *ni; + + ni = scmi_notification_instance_data_get(handle); + if (!ni) + return; + + disable_work_sync(&ni->init_work); +} + /** * scmi_notification_exit() - Shutdown and clean Notification core * @handle: The handle identifying the platform instance to shutdown @@ -1717,6 +1736,8 @@ void scmi_notification_exit(struct scmi_handle *handle) ni = scmi_notification_instance_data_get(handle); if (!ni) return; + + scmi_notification_quiesce(handle); scmi_notification_instance_data_set(handle, NULL); /* Destroy while letting pending work complete */ diff --git a/drivers/firmware/arm_scmi/notify.h b/drivers/firmware/arm_scmi/notify.h index 76758a736cf47..f18f98c5ab3ba 100644 --- a/drivers/firmware/arm_scmi/notify.h +++ b/drivers/firmware/arm_scmi/notify.h @@ -82,6 +82,7 @@ struct scmi_protocol_events { }; int scmi_notification_init(struct scmi_handle *handle); +void scmi_notification_quiesce(struct scmi_handle *handle); void scmi_notification_exit(struct scmi_handle *handle); int scmi_register_protocol_events(const struct scmi_handle *handle, u8 proto_id, const struct scmi_protocol_handle *ph, diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c index 1be0f89fc2c4b..99233742714c9 100644 --- a/drivers/firmware/arm_scmi/sensors.c +++ b/drivers/firmware/arm_scmi/sensors.c @@ -1074,12 +1074,15 @@ scmi_sensor_fill_custom_report(const struct scmi_protocol_handle *ph, case SCMI_EVENT_SENSOR_UPDATE: { int i; + size_t expected_sz; struct scmi_sensor_info *s; const struct scmi_sensor_update_notify_payld *p = payld; struct scmi_sensor_update_report *r = report; struct sensors_info *sinfo = ph->get_priv(ph); - /* payld_sz is variable for this event */ + if (payld_sz < sizeof(*p)) + break; + r->sensor_id = le32_to_cpu(p->sensor_id); if (r->sensor_id >= sinfo->num_sensors) break; @@ -1093,6 +1096,11 @@ scmi_sensor_fill_custom_report(const struct scmi_protocol_handle *ph, * readings defined for this sensor or 1 for scalar sensors. */ r->readings_count = s->num_axis ?: 1; + expected_sz = sizeof(*p) + r->readings_count * + sizeof(p->readings[0]); + if (payld_sz < expected_sz) + break; + for (i = 0; i < r->readings_count; i++) scmi_parse_sensor_readings(&r->readings[i], &p->readings[i]); diff --git a/drivers/firmware/arm_scmi/transports/mailbox.c b/drivers/firmware/arm_scmi/transports/mailbox.c index ae0f67e6cc45f..308736c3ead9c 100644 --- a/drivers/firmware/arm_scmi/transports/mailbox.c +++ b/drivers/firmware/arm_scmi/transports/mailbox.c @@ -211,13 +211,18 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, cl->tx_block = false; cl->knows_txdone = tx; + cinfo->transport_info = smbox; + smbox->cinfo = cinfo; + mutex_init(&smbox->chan_lock); + smbox->chan = mbox_request_channel(cl, tx ? 0 : p2a_chan); if (IS_ERR(smbox->chan)) { ret = PTR_ERR(smbox->chan); + smbox->chan = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI %s mailbox\n", desc); - return ret; + goto err_clear_cinfo; } /* Additional unidirectional channel for TX if needed */ @@ -225,9 +230,10 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_receiver = mbox_request_channel(cl, a2p_rx_chan); if (IS_ERR(smbox->chan_receiver)) { ret = PTR_ERR(smbox->chan_receiver); + smbox->chan_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI Tx Receiver mailbox\n"); - return ret; + goto err_free_chan; } } @@ -235,17 +241,23 @@ static int mailbox_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, smbox->chan_platform_receiver = mbox_request_channel(cl, p2a_rx_chan); if (IS_ERR(smbox->chan_platform_receiver)) { ret = PTR_ERR(smbox->chan_platform_receiver); + smbox->chan_platform_receiver = NULL; if (ret != -EPROBE_DEFER) dev_err(cdev, "failed to request SCMI P2A Receiver mailbox\n"); - return ret; + goto err_free_chan; } } - cinfo->transport_info = smbox; - smbox->cinfo = cinfo; - mutex_init(&smbox->chan_lock); - return 0; + +err_free_chan: + mbox_free_channel(smbox->chan); +err_clear_cinfo: + cinfo->transport_info = NULL; + smbox->cinfo = NULL; + devm_iounmap(dev, smbox->shmem); + devm_kfree(dev, smbox); + return ret; } static int mailbox_chan_free(int id, void *p, void *data) diff --git a/drivers/firmware/arm_scmi/transports/smc.c b/drivers/firmware/arm_scmi/transports/smc.c index 21abb571e4f2f..1fce3ccdeb7fc 100644 --- a/drivers/firmware/arm_scmi/transports/smc.c +++ b/drivers/firmware/arm_scmi/transports/smc.c @@ -172,6 +172,13 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, scmi_info->param_page = SHMEM_PAGE(res.start); scmi_info->param_offset = SHMEM_OFFSET(res.start); } + + scmi_info->func_id = func_id; + scmi_info->cap_id = cap_id; + scmi_info->cinfo = cinfo; + smc_channel_lock_init(scmi_info); + cinfo->transport_info = scmi_info; + /* * If there is an interrupt named "a2p", then the service and * completion of a message is signaled by an interrupt rather than by @@ -183,18 +190,14 @@ static int smc_chan_setup(struct scmi_chan_info *cinfo, struct device *dev, IRQF_NO_SUSPEND, dev_name(dev), scmi_info); if (ret) { dev_err(dev, "failed to setup SCMI smc irq\n"); + cinfo->transport_info = NULL; + scmi_info->cinfo = NULL; return ret; } } else { cinfo->no_completion_irq = true; } - scmi_info->func_id = func_id; - scmi_info->cap_id = cap_id; - scmi_info->cinfo = cinfo; - smc_channel_lock_init(scmi_info); - cinfo->transport_info = scmi_info; - return 0; } diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c index 882db32e51be9..2f1122635098b 100644 --- a/drivers/firmware/google/coreboot_table.c +++ b/drivers/firmware/google/coreboot_table.c @@ -101,16 +101,20 @@ void coreboot_driver_unregister(struct coreboot_driver *driver) } EXPORT_SYMBOL(coreboot_driver_unregister); -static int coreboot_table_populate(struct device *dev, void *ptr) +static int coreboot_table_populate(struct device *dev, void *ptr, resource_size_t len) { int i, ret; void *ptr_entry; struct coreboot_device *device; struct coreboot_table_entry *entry; struct coreboot_table_header *header = ptr; + void *ptr_end; + ptr_end = ptr + len; ptr_entry = ptr + header->header_bytes; for (i = 0; i < header->table_entries; i++) { + if (ptr_entry + sizeof(*entry) > ptr_end) + return -EINVAL; entry = ptr_entry; if (entry->size < sizeof(*entry)) { @@ -118,6 +122,9 @@ static int coreboot_table_populate(struct device *dev, void *ptr) return -EINVAL; } + if (ptr_entry + entry->size > ptr_end) + return -EINVAL; + device = kzalloc(sizeof(device->dev) + entry->size, GFP_KERNEL); if (!device) return -ENOMEM; @@ -152,6 +159,7 @@ static int coreboot_table_populate(struct device *dev, void *ptr) static int coreboot_table_probe(struct platform_device *pdev) { resource_size_t len; + resource_size_t table_span; struct coreboot_table_header *header; struct resource *res; struct device *dev = &pdev->dev; @@ -163,7 +171,7 @@ static int coreboot_table_probe(struct platform_device *pdev) return -EINVAL; len = resource_size(res); - if (!res->start || !len) + if (!res->start || len < sizeof(*header)) return -EINVAL; /* Check just the header first to make sure things are sane */ @@ -171,19 +179,27 @@ static int coreboot_table_probe(struct platform_device *pdev) if (!header) return -ENOMEM; - len = header->header_bytes + header->table_bytes; ret = strncmp(header->signature, "LBIO", sizeof(header->signature)); + + if (!ret && + (header->header_bytes < sizeof(*header) || + check_add_overflow((resource_size_t)header->header_bytes, + (resource_size_t)header->table_bytes, + &table_span) || + table_span > len)) + ret = -EINVAL; + memunmap(header); if (ret) { dev_warn(dev, "coreboot table missing or corrupt!\n"); return -ENODEV; } - ptr = memremap(res->start, len, MEMREMAP_WB); + ptr = memremap(res->start, table_span, MEMREMAP_WB); if (!ptr) return -ENOMEM; - ret = coreboot_table_populate(dev, ptr); + ret = coreboot_table_populate(dev, ptr, table_span); memunmap(ptr); diff --git a/drivers/firmware/qcom/Makefile b/drivers/firmware/qcom/Makefile index 0be40a1abc13c..b679d3fc2c267 100644 --- a/drivers/firmware/qcom/Makefile +++ b/drivers/firmware/qcom/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_QCOM_SCM) += qcom-scm.o qcom-scm-objs += qcom_scm.o qcom_scm-smc.o qcom_scm-legacy.o +CFLAGS_qcom_scm-smc.o := -I$(src) obj-$(CONFIG_QCOM_TZMEM) += qcom_tzmem.o obj-$(CONFIG_QCOM_QSEECOM) += qcom_qseecom.o obj-$(CONFIG_QCOM_QSEECOM_UEFISECAPP) += qcom_qseecom_uefisecapp.o diff --git a/drivers/firmware/qcom/qcom_scm-smc.c b/drivers/firmware/qcom/qcom_scm-smc.c index 574930729ddd7..127365ab11fc2 100644 --- a/drivers/firmware/qcom/qcom_scm-smc.c +++ b/drivers/firmware/qcom/qcom_scm-smc.c @@ -24,6 +24,9 @@ struct arm_smccc_args { unsigned long args[8]; }; +#define CREATE_TRACE_POINTS +#include "qcom_scm_trace.h" + static DEFINE_MUTEX(qcom_scm_lock); #define QCOM_SCM_EBUSY_WAIT_MS 30 @@ -44,6 +47,7 @@ static void __scm_smc_do_quirk(const struct arm_smccc_args *smc, quirk.state.a6 = 0; do { + trace_scm_smc_request(a0, smc); arm_smccc_smc_quirk(a0, smc->args[1], smc->args[2], smc->args[3], smc->args[4], smc->args[5], quirk.state.a6, smc->args[7], res, &quirk); @@ -83,6 +87,7 @@ int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending) if (ret) return ret; + trace_scm_waitq_get_wq_ctx(get_wq_res.a1, get_wq_res.a2, get_wq_res.a3); *wq_ctx = get_wq_res.a1; *flags = get_wq_res.a2; *more_pending = get_wq_res.a3; @@ -105,10 +110,12 @@ static int __scm_smc_do_quirk_handle_waitq(struct device *dev, struct arm_smccc_ wq_ctx = res->a1; smc_call_ctx = res->a2; - ret = qcom_scm_wait_for_wq_completion(wq_ctx); + trace_scm_waitq_sleep(wq_ctx, smc_call_ctx); + ret = qcom_scm_wait_for_wq_completion(dev, wq_ctx); if (ret) return ret; + trace_scm_waitq_resume(smc_call_ctx); fill_wq_resume_args(&resume, smc_call_ctx); smc = &resume; } @@ -201,6 +208,9 @@ int __scm_smc_call(struct device *dev, const struct qcom_scm_desc *desc, } ret = __scm_smc_do(dev, &smc, &smc_res, atomic); + + trace_scm_smc_done(ret, smc.args[0], &smc_res); + if (ret) return ret; diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c index e777b7cb9b127..226265b311fe9 100644 --- a/drivers/firmware/qcom/qcom_scm.c +++ b/drivers/firmware/qcom/qcom_scm.c @@ -30,18 +30,25 @@ #include #include +#include + #include "qcom_scm.h" #include "qcom_tzmem.h" static u32 download_mode; +#define GIC_SPI_BASE 32 +#define GIC_MAX_SPI 1019 // SPIs in GICv3 spec range from 32..1019 +#define GIC_ESPI_BASE 4096 +#define GIC_MAX_ESPI 5119 // ESPIs in GICv3 spec range from 4096..5119 + struct qcom_scm { struct device *dev; struct clk *core_clk; struct clk *iface_clk; struct clk *bus_clk; struct icc_path *path; - struct completion waitq_comp; + struct completion *waitq_comps; struct reset_controller_dev reset; /* control access to the interconnect path */ @@ -51,6 +58,7 @@ struct qcom_scm { u64 dload_mode_addr; struct qcom_tzmem_pool *mempool; + unsigned int wq_cnt; }; struct qcom_scm_current_perm_info { @@ -130,6 +138,8 @@ static const u8 qcom_scm_cpu_warm_bits[QCOM_SCM_BOOT_MAX_CPUS] = { #define QCOM_DLOAD_MINIDUMP 2 #define QCOM_DLOAD_BOTHDUMP 3 +#define QCOM_SCM_DEFAULT_WAITQ_COUNT 1 + static const char * const qcom_scm_convention_names[] = { [SMC_CONVENTION_UNKNOWN] = "unknown", [SMC_CONVENTION_ARM_32] = "smc arm 32", @@ -558,11 +568,45 @@ static void qcom_scm_set_download_mode(u32 dload_mode) dev_err(__scm->dev, "failed to set download mode: %d\n", ret); } +/** + * devm_qcom_scm_pas_context_alloc() - Allocate peripheral authentication service + * context for a given peripheral + * + * PAS context is device-resource managed, so the caller does not need + * to worry about freeing the context memory. + * + * @dev: PAS firmware device + * @pas_id: peripheral authentication service id + * @mem_phys: Subsystem reserve memory start address + * @mem_size: Subsystem reserve memory size + * + * Returns: The new PAS context, or ERR_PTR() on failure. + */ +struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev, + u32 pas_id, + phys_addr_t mem_phys, + size_t mem_size) +{ + struct qcom_scm_pas_context *ctx; + + ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); + if (!ctx) + return ERR_PTR(-ENOMEM); + + ctx->dev = dev; + ctx->pas_id = pas_id; + ctx->mem_phys = mem_phys; + ctx->mem_size = mem_size; + + return ctx; +} +EXPORT_SYMBOL_GPL(devm_qcom_scm_pas_context_alloc); + /** * qcom_scm_pas_init_image() - Initialize peripheral authentication service * state machine for a given peripheral, using the * metadata - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * @metadata: pointer to memory containing ELF header, program header table * and optional blob of data used for authenticating the metadata * and the rest of the firmware @@ -575,7 +619,7 @@ static void qcom_scm_set_download_mode(u32 dload_mode) * track the metadata allocation, this needs to be released by invoking * qcom_scm_pas_metadata_release() by the caller. */ -int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, +int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size, struct qcom_scm_pas_metadata *ctx) { dma_addr_t mdata_phys; @@ -585,7 +629,7 @@ int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_INIT_IMAGE, .arginfo = QCOM_SCM_ARGS(2, QCOM_SCM_VAL, QCOM_SCM_RW), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -658,20 +702,20 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_metadata_release); /** * qcom_scm_pas_mem_setup() - Prepare the memory related to a given peripheral * for firmware loading - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * @addr: start address of memory area to prepare * @size: size of the memory area to prepare * * Returns 0 on success. */ -int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size) +int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_MEM_SETUP, .arginfo = QCOM_SCM_ARGS(3), - .args[0] = peripheral, + .args[0] = pas_id, .args[1] = addr, .args[2] = size, .owner = ARM_SMCCC_OWNER_SIP, @@ -699,18 +743,18 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_mem_setup); /** * qcom_scm_pas_auth_and_reset() - Authenticate the given peripheral firmware * and reset the remote processor - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Return 0 on success. */ -int qcom_scm_pas_auth_and_reset(u32 peripheral) +int qcom_scm_pas_auth_and_reset(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_AUTH_AND_RESET, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -735,18 +779,18 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_auth_and_reset); /** * qcom_scm_pas_shutdown() - Shut down the remote processor - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Returns 0 on success. */ -int qcom_scm_pas_shutdown(u32 peripheral) +int qcom_scm_pas_shutdown(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_SHUTDOWN, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -772,18 +816,18 @@ EXPORT_SYMBOL_GPL(qcom_scm_pas_shutdown); /** * qcom_scm_pas_supported() - Check if the peripheral authentication service is * available for the given peripherial - * @peripheral: peripheral id + * @pas_id: peripheral authentication service id * * Returns true if PAS is supported for this peripheral, otherwise false. */ -bool qcom_scm_pas_supported(u32 peripheral) +bool qcom_scm_pas_supported(u32 pas_id) { int ret; struct qcom_scm_desc desc = { .svc = QCOM_SCM_SVC_PIL, .cmd = QCOM_SCM_PIL_PAS_IS_SUPPORTED, .arginfo = QCOM_SCM_ARGS(1), - .args[0] = peripheral, + .args[0] = pas_id, .owner = ARM_SMCCC_OWNER_SIP, }; struct qcom_scm_res res; @@ -2223,42 +2267,104 @@ bool qcom_scm_is_available(void) } EXPORT_SYMBOL_GPL(qcom_scm_is_available); -static int qcom_scm_assert_valid_wq_ctx(u32 wq_ctx) +static int qcom_scm_fill_irq_fwspec_params(struct irq_fwspec *fwspec, u32 hwirq) { - /* FW currently only supports a single wq_ctx (zero). - * TODO: Update this logic to include dynamic allocation and lookup of - * completion structs when FW supports more wq_ctx values. - */ - if (wq_ctx != 0) { - dev_err(__scm->dev, "Firmware unexpectedly passed non-zero wq_ctx\n"); - return -EINVAL; + if (hwirq >= GIC_SPI_BASE && hwirq <= GIC_MAX_SPI) { + fwspec->param[0] = GIC_SPI; + fwspec->param[1] = hwirq - GIC_SPI_BASE; + } else if (hwirq >= GIC_ESPI_BASE && hwirq <= GIC_MAX_ESPI) { + fwspec->param[0] = GIC_ESPI; + fwspec->param[1] = hwirq - GIC_ESPI_BASE; + } else { + WARN(1, "Unexpected hwirq: %d\n", hwirq); + return -ENXIO; } + fwspec->param[2] = IRQ_TYPE_EDGE_RISING; + fwspec->param_count = 3; + return 0; } -int qcom_scm_wait_for_wq_completion(u32 wq_ctx) +static int qcom_scm_query_waitq_count(struct qcom_scm *scm) { + struct qcom_scm_desc desc = { + .svc = QCOM_SCM_SVC_WAITQ, + .cmd = QCOM_SCM_WAITQ_GET_INFO, + .owner = ARM_SMCCC_OWNER_SIP + }; + struct qcom_scm_res res; int ret; - ret = qcom_scm_assert_valid_wq_ctx(wq_ctx); + ret = qcom_scm_call_atomic(scm->dev, &desc, &res); if (ret) return ret; - wait_for_completion(&__scm->waitq_comp); - - return 0; + return res.result[0] & GENMASK(7, 0); } -static int qcom_scm_waitq_wakeup(unsigned int wq_ctx) +static int qcom_scm_get_waitq_irq(struct qcom_scm *scm) { + struct qcom_scm_desc desc = { + .svc = QCOM_SCM_SVC_WAITQ, + .cmd = QCOM_SCM_WAITQ_GET_INFO, + .owner = ARM_SMCCC_OWNER_SIP + }; + struct device_node *parent_irq_node; + struct irq_fwspec fwspec; + struct qcom_scm_res res; + u32 hwirq; int ret; - ret = qcom_scm_assert_valid_wq_ctx(wq_ctx); + ret = qcom_scm_call_atomic(scm->dev, &desc, &res); + if (ret) + return ret; + + hwirq = res.result[1] & GENMASK(15, 0); + ret = qcom_scm_fill_irq_fwspec_params(&fwspec, hwirq); if (ret) return ret; - complete(&__scm->waitq_comp); + parent_irq_node = of_irq_find_parent(scm->dev->of_node); + if (!parent_irq_node) + return -ENODEV; + + fwspec.fwnode = of_fwnode_handle(parent_irq_node); + + return irq_create_fwspec_mapping(&fwspec); +} + +static struct completion *qcom_scm_get_completion(struct qcom_scm *scm, u32 wq_ctx) +{ + if (WARN_ON_ONCE(wq_ctx >= scm->wq_cnt)) + return ERR_PTR(-EINVAL); + + return &scm->waitq_comps[wq_ctx]; +} + +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx) +{ + struct qcom_scm *scm = dev_get_drvdata(dev); + struct completion *wq; + + wq = qcom_scm_get_completion(scm, wq_ctx); + if (IS_ERR(wq)) + return PTR_ERR(wq); + + wait_for_completion(wq); + + return 0; +} + +static int qcom_scm_waitq_wakeup(struct qcom_scm *scm, unsigned int wq_ctx) +{ + struct completion *wq; + + wq = qcom_scm_get_completion(scm, wq_ctx); + if (IS_ERR(wq)) + return PTR_ERR(wq); + + complete(wq); return 0; } @@ -2281,7 +2387,7 @@ static irqreturn_t qcom_scm_irq_handler(int irq, void *data) goto out; } - ret = qcom_scm_waitq_wakeup(wq_ctx); + ret = qcom_scm_waitq_wakeup(scm, wq_ctx); if (ret) goto out; } while (more_pending); @@ -2334,17 +2440,18 @@ static int qcom_scm_probe(struct platform_device *pdev) struct qcom_tzmem_pool_config pool_config; struct qcom_scm *scm; int irq, ret; + int i; scm = devm_kzalloc(&pdev->dev, sizeof(*scm), GFP_KERNEL); if (!scm) return -ENOMEM; scm->dev = &pdev->dev; + platform_set_drvdata(pdev, scm); ret = qcom_scm_find_dload_address(&pdev->dev, &scm->dload_mode_addr); if (ret < 0) return ret; - init_completion(&scm->waitq_comp); mutex_init(&scm->scm_bw_lock); scm->path = devm_of_icc_get(&pdev->dev, NULL); @@ -2382,9 +2489,11 @@ static int qcom_scm_probe(struct platform_device *pdev) "Failed to setup the reserved memory region for TZ mem\n"); ret = qcom_tzmem_enable(scm->dev); - if (ret) - return dev_err_probe(scm->dev, ret, - "Failed to enable the TrustZone memory allocator\n"); + if (ret) { + ret = dev_err_probe(scm->dev, ret, + "Failed to enable the TrustZone memory allocator\n"); + goto err_rmem; + } memset(&pool_config, 0, sizeof(pool_config)); pool_config.initial_size = 0; @@ -2392,11 +2501,26 @@ static int qcom_scm_probe(struct platform_device *pdev) pool_config.max_size = SZ_256K; scm->mempool = devm_qcom_tzmem_pool_new(scm->dev, &pool_config); - if (IS_ERR(scm->mempool)) - return dev_err_probe(scm->dev, PTR_ERR(scm->mempool), - "Failed to create the SCM memory pool\n"); + if (IS_ERR(scm->mempool)) { + ret = dev_err_probe(scm->dev, PTR_ERR(scm->mempool), + "Failed to create the SCM memory pool\n"); + goto err_rmem; + } + + ret = qcom_scm_query_waitq_count(scm); + scm->wq_cnt = ret < 0 ? QCOM_SCM_DEFAULT_WAITQ_COUNT : ret; + scm->waitq_comps = devm_kcalloc(&pdev->dev, scm->wq_cnt, sizeof(*scm->waitq_comps), + GFP_KERNEL); + if (!scm->waitq_comps) + return -ENOMEM; + + for (i = 0; i < scm->wq_cnt; i++) + init_completion(&scm->waitq_comps[i]); + + irq = qcom_scm_get_waitq_irq(scm); + if (irq < 0) + irq = platform_get_irq_optional(pdev, 0); - irq = platform_get_irq_optional(pdev, 0); if (irq < 0) { if (irq != -ENXIO) return irq; @@ -2449,6 +2573,10 @@ static int qcom_scm_probe(struct platform_device *pdev) qcom_scm_qtee_init(scm); return 0; + +err_rmem: + of_reserved_mem_device_release(scm->dev); + return ret; } static void qcom_scm_shutdown(struct platform_device *pdev) diff --git a/drivers/firmware/qcom/qcom_scm.h b/drivers/firmware/qcom/qcom_scm.h index a56c8212cc0c4..2111188f9fad0 100644 --- a/drivers/firmware/qcom/qcom_scm.h +++ b/drivers/firmware/qcom/qcom_scm.h @@ -66,7 +66,7 @@ struct qcom_scm_res { u64 result[MAX_QCOM_SCM_RETS]; }; -int qcom_scm_wait_for_wq_completion(u32 wq_ctx); +int qcom_scm_wait_for_wq_completion(struct device *dev, u32 wq_ctx); int scm_get_wq_ctx(u32 *wq_ctx, u32 *flags, u32 *more_pending); #define SCM_SMC_FNID(s, c) ((((s) & 0xFF) << 8) | ((c) & 0xFF)) @@ -152,6 +152,7 @@ int qcom_scm_shm_bridge_enable(struct device *scm_dev); #define QCOM_SCM_SVC_WAITQ 0x24 #define QCOM_SCM_WAITQ_RESUME 0x02 #define QCOM_SCM_WAITQ_GET_WQ_CTX 0x03 +#define QCOM_SCM_WAITQ_GET_INFO 0x04 #define QCOM_SCM_SVC_GPU 0x28 #define QCOM_SCM_SVC_GPU_INIT_REGS 0x01 diff --git a/drivers/firmware/qcom/qcom_scm_trace.h b/drivers/firmware/qcom/qcom_scm_trace.h new file mode 100644 index 0000000000000..6c911124fc56b --- /dev/null +++ b/drivers/firmware/qcom/qcom_scm_trace.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM qcom_scm + +#if !defined(_TRACE_SCM_SMC_INTERFACE_H) || defined(TRACE_HEADER_MULTI_READ) + +#define _TRACE_SCM_SMC_INTERFACE_H + +#include + +TRACE_EVENT(scm_smc_request, + + TP_PROTO(unsigned long a0, const struct arm_smccc_args *smc), + + TP_ARGS(a0, smc), + + TP_STRUCT__entry( + __field(u64, smc_id) + __field(u8, svc_id) + __field(u8, cmd_id) + __field(u8, args_cnt) + __dynamic_array(unsigned long, args, + min_t(u8, (smc->args[1] & 0xF), (u8)6)) + ), + + TP_fast_assign( + __entry->smc_id = a0; + __entry->svc_id = (smc->args[0] >> 8) & 0xFF; + __entry->cmd_id = smc->args[0] & 0xFF; + u8 n = min_t(u8, (smc->args[1] & 0xF), (u8)6); + + __entry->args_cnt = n; + + unsigned long *dst = __get_dynamic_array(args); + + for (int i = 0; i < n; i++) + dst[i] = smc->args[2 + i]; + ), + + TP_printk("smc_id:0x%08llx svc_id:0x%02x cmd_id:0x%02x args_cnt:%u args:%s", + __entry->smc_id, __entry->svc_id, __entry->cmd_id, __entry->args_cnt, + __print_dynamic_array(args, sizeof(unsigned long))) +); + +TRACE_EVENT(scm_waitq_sleep, + + TP_PROTO(u32 wq_ctx, u32 smc_ctx), + + TP_ARGS(wq_ctx, smc_ctx), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("wq_ctx:%u, smc_call_ctx:%u", __entry->wq_ctx, __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_resume, + + TP_PROTO(u32 smc_ctx), + + TP_ARGS(smc_ctx), + + TP_STRUCT__entry( + __field(u32, smc_call_ctx) + ), + + TP_fast_assign( + __entry->smc_call_ctx = smc_ctx; + ), + + TP_printk("smc_call_ctx:%u", __entry->smc_call_ctx) +); + +TRACE_EVENT(scm_waitq_get_wq_ctx, + + TP_PROTO(u32 wq_ctx, u32 flags, u32 pending), + + TP_ARGS(wq_ctx, flags, pending), + + TP_STRUCT__entry( + __field(u32, wq_ctx) + __field(u32, flags) + __field(u32, more_pending) + ), + + TP_fast_assign( + __entry->wq_ctx = wq_ctx; + __entry->flags = flags; + __entry->more_pending = pending; + ), + + TP_printk("wq_ctx:%u, flags:%u, more_pending:%u", + __entry->wq_ctx, __entry->flags, __entry->more_pending) +); + +TRACE_EVENT(scm_smc_done, + + TP_PROTO(int ret, u64 smc_id, struct arm_smccc_res *smc_res), + + TP_ARGS(ret, smc_id, smc_res), + + TP_STRUCT__entry( + __field(int, ret) + __field(u64, smc_id) + __field(unsigned long, res) + __field(unsigned long, res0) + __field(unsigned long, res1) + __field(unsigned long, res2) + ), + + TP_fast_assign( + __entry->ret = ret; + __entry->smc_id = smc_id; + __entry->res = smc_res->a0; + __entry->res0 = smc_res->a1; + __entry->res1 = smc_res->a2; + __entry->res2 = smc_res->a3; + ), + + TP_printk("smc_id:0x%08llx, ret:%d res_to_callee:0x%lx res0:0x%lx res1:0x%lx res2:0x%lx", + __entry->smc_id, __entry->ret, __entry->res, + __entry->res0, __entry->res1, __entry->res2) +); + +#endif /* _TRACE_SCM_SMC_INTERFACE_H */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE qcom_scm_trace + +#include + diff --git a/drivers/firmware/qcom/qcom_tzmem.c b/drivers/firmware/qcom/qcom_tzmem.c index 9f232e53115ea..b867b9d8122a6 100644 --- a/drivers/firmware/qcom/qcom_tzmem.c +++ b/drivers/firmware/qcom/qcom_tzmem.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -508,14 +509,18 @@ phys_addr_t qcom_tzmem_to_phys(void *vaddr) } EXPORT_SYMBOL_GPL(qcom_tzmem_to_phys); +static void qcom_tzmem_do_init(int *result) +{ + *result = qcom_tzmem_init(); +} + int qcom_tzmem_enable(struct device *dev) { - if (qcom_tzmem_dev) - return -EBUSY; + static int result; qcom_tzmem_dev = dev; - - return qcom_tzmem_init(); + DO_ONCE(qcom_tzmem_do_init, &result); + return result; } EXPORT_SYMBOL_GPL(qcom_tzmem_enable); diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index 1d24e82f3d09d..d2bce209d2179 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -340,6 +340,7 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, case COMMAND_FCS_SEND_CERTIFICATE: case COMMAND_FCS_DATA_ENCRYPTION: case COMMAND_FCS_DATA_DECRYPTION: + case COMMAND_FCS_GET_PROVISION_DATA: cb_data->status = BIT(SVC_STATUS_OK); break; case COMMAND_RECONFIG_DATA_SUBMIT: @@ -366,7 +367,6 @@ static void svc_thread_recv_status_ok(struct stratix10_svc_data *p_data, cb_data->kaddr2 = &res.a2; break; case COMMAND_FCS_RANDOM_NUMBER_GEN: - case COMMAND_FCS_GET_PROVISION_DATA: case COMMAND_POLL_SERVICE_STATUS: cb_data->status = BIT(SVC_STATUS_OK); cb_data->kaddr1 = &res.a1; @@ -533,7 +533,7 @@ static int svc_normal_to_secure_thread(void *data) break; case COMMAND_FCS_GET_PROVISION_DATA: a0 = INTEL_SIP_SMC_FCS_GET_PROVISION_DATA; - a1 = (unsigned long)pdata->paddr; + a1 = 0; a2 = 0; break; @@ -1080,14 +1080,16 @@ void *stratix10_svc_allocate_memory(struct stratix10_svc_chan *chan, struct gen_pool *genpool = chan->ctrl->genpool; size_t s = roundup(size, 1 << genpool->min_alloc_order); - pmem = devm_kzalloc(chan->ctrl->dev, sizeof(*pmem), GFP_KERNEL); + pmem = kzalloc_obj(*pmem); if (!pmem) return ERR_PTR(-ENOMEM); guard(mutex)(&svc_mem_lock); va = gen_pool_alloc(genpool, s); - if (!va) + if (!va) { + kfree(pmem); return ERR_PTR(-ENOMEM); + } memset((void *)va, 0, s); pa = gen_pool_virt_to_phys(genpool, va); @@ -1113,6 +1115,7 @@ EXPORT_SYMBOL_GPL(stratix10_svc_allocate_memory); void stratix10_svc_free_memory(struct stratix10_svc_chan *chan, void *kaddr) { struct stratix10_svc_data_mem *pmem; + guard(mutex)(&svc_mem_lock); list_for_each_entry(pmem, &svc_data_mem, node) @@ -1121,10 +1124,9 @@ void stratix10_svc_free_memory(struct stratix10_svc_chan *chan, void *kaddr) (unsigned long)kaddr, pmem->size); pmem->vaddr = NULL; list_del(&pmem->node); + kfree(pmem); return; } - - list_del(&svc_data_mem); } EXPORT_SYMBOL_GPL(stratix10_svc_free_memory); diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c index 02da3e48bc8f3..dc9c5d32108d5 100644 --- a/drivers/firmware/xilinx/zynqmp.c +++ b/drivers/firmware/xilinx/zynqmp.c @@ -461,6 +461,7 @@ int zynqmp_pm_invoke_fn(u32 pm_api_id, u32 *ret_payload, u32 num_args, ...) return do_fw_call(ret_payload, 8, smc_arg[0], smc_arg[1], smc_arg[2], smc_arg[3], smc_arg[4], smc_arg[5], smc_arg[6], smc_arg[7]); } +EXPORT_SYMBOL_GPL(zynqmp_pm_invoke_fn); static u32 pm_api_version; static u32 pm_tz_version; diff --git a/drivers/fpga/altera-cvp.c b/drivers/fpga/altera-cvp.c index 5af0bd33890c0..e2a2307a41ee1 100644 --- a/drivers/fpga/altera-cvp.c +++ b/drivers/fpga/altera-cvp.c @@ -16,6 +16,7 @@ #include #include #include +#include #define CVP_BAR 0 /* BAR used for data transfer in memory mode */ #define CVP_DUMMY_WR 244 /* dummy writes to clear CvP state machine */ @@ -264,7 +265,7 @@ static int altera_cvp_v2_wait_for_credit(struct fpga_manager *mgr, static int altera_cvp_send_block(struct altera_cvp_conf *conf, const u32 *data, size_t len) { - u32 mask, words = len / sizeof(u32); + u32 words = len / sizeof(u32); int i, remainder; for (i = 0; i < words; i++) @@ -273,9 +274,10 @@ static int altera_cvp_send_block(struct altera_cvp_conf *conf, /* write up to 3 trailing bytes, if any */ remainder = len % sizeof(u32); if (remainder) { - mask = BIT(remainder * 8) - 1; - if (mask) - conf->write_data(conf, *data & mask); + u32 word = 0; + + memcpy(&word, data, remainder); + conf->write_data(conf, word); } return 0; diff --git a/drivers/fpga/dfl-fme-perf.c b/drivers/fpga/dfl-fme-perf.c index 7422d2bc6f374..db56d52411efc 100644 --- a/drivers/fpga/dfl-fme-perf.c +++ b/drivers/fpga/dfl-fme-perf.c @@ -925,6 +925,8 @@ static int fme_perf_pmu_register(struct platform_device *pdev, PERF_PMU_CAP_NO_EXCLUDE; name = devm_kasprintf(priv->dev, GFP_KERNEL, "dfl_fme%d", pdev->id); + if (!name) + return -ENOMEM; ret = perf_pmu_register(pmu, name, -1); if (ret) diff --git a/drivers/fpga/stratix10-soc.c b/drivers/fpga/stratix10-soc.c index 0a295ccf1644a..b8ec2e6f615fa 100644 --- a/drivers/fpga/stratix10-soc.c +++ b/drivers/fpga/stratix10-soc.c @@ -195,20 +195,18 @@ static int s10_ops_write_init(struct fpga_manager *mgr, ret = s10_svc_send_msg(priv, COMMAND_RECONFIG, &ctype, sizeof(ctype)); if (ret < 0) - goto init_done; + goto init_error; - ret = wait_for_completion_timeout( - &priv->status_return_completion, S10_RECONFIG_TIMEOUT); - if (!ret) { + if (!wait_for_completion_timeout(&priv->status_return_completion, + S10_RECONFIG_TIMEOUT)) { dev_err(dev, "timeout waiting for RECONFIG_REQUEST\n"); ret = -ETIMEDOUT; - goto init_done; + goto init_error; } - ret = 0; if (!test_and_clear_bit(SVC_STATUS_OK, &priv->status)) { ret = -ETIMEDOUT; - goto init_done; + goto init_error; } /* Allocate buffers from the service layer's pool. */ @@ -217,14 +215,16 @@ static int s10_ops_write_init(struct fpga_manager *mgr, if (IS_ERR(kbuf)) { s10_free_buffers(mgr); ret = PTR_ERR(kbuf); - goto init_done; + goto init_error; } priv->svc_bufs[i].buf = kbuf; priv->svc_bufs[i].lock = 0; } -init_done: + return 0; + +init_error: stratix10_svc_done(priv->chan); return ret; } @@ -342,6 +342,9 @@ static int s10_ops_write(struct fpga_manager *mgr, const char *buf, if (!s10_free_buffers(mgr)) dev_err(dev, "%s not all buffers were freed\n", __func__); + if (ret < 0) + stratix10_svc_done(priv->chan); + return ret; } diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c index 2ad9f6ac66362..85bd994d15d48 100644 --- a/drivers/gpio/gpio-ath79.c +++ b/drivers/gpio/gpio-ath79.c @@ -11,6 +11,7 @@ #include #include #include +#include /* For WLAN GPIOs */ #include #include #include @@ -214,6 +215,56 @@ static const struct of_device_id ath79_gpio_of_match[] = { }; MODULE_DEVICE_TABLE(of, ath79_gpio_of_match); +#if IS_ENABLED(CONFIG_ATH9K_AHB) +/* + * This registers all of the ath79k GPIOs as descriptors to be picked + * directly from the ATH79K wifi driver if the two are jitted together + * in the same SoC. + */ +#define ATH79K_WIFI_DESCS 32 +static int ath79_gpio_register_wifi_descriptors(struct device *dev, + const char *label) +{ + struct gpiod_lookup_table *lookup; + int i; + + /* Create a gpiod lookup using gpiochip-local offsets + 1 for NULL */ + lookup = devm_kzalloc(dev, + struct_size(lookup, table, ATH79K_WIFI_DESCS + 1), + GFP_KERNEL); + if (!lookup) + return -ENOMEM; + + /* + * Ugly system-wide lookup for the NULL device: we know this + * is already NULL but explicitly assign it here for people to + * know what is going on. (Yes this is an ugly legacy hack, live + * with it.) + */ + lookup->dev_id = NULL; + + for (i = 0; i < ATH79K_WIFI_DESCS; i++) { + lookup->table[i] = + /* + * Set the HW offset on the chip and the lookup + * index to the same value, so looking up index 0 + * will get HW offset 0, index 1 HW offset 1 etc. + */ + GPIO_LOOKUP_IDX(label, i, "ath9k", i, GPIO_ACTIVE_HIGH); + } + + gpiod_add_lookup_table(lookup); + + return 0; +} +#else +static int ath79_gpio_register_wifi_descriptors(struct device *dev, + const char *label) +{ + return 0; +} +#endif + static int ath79_gpio_probe(struct platform_device *pdev) { struct gpio_generic_chip_config config; @@ -276,7 +327,11 @@ static int ath79_gpio_probe(struct platform_device *pdev) girq->handler = handle_simple_irq; } - return devm_gpiochip_add_data(dev, &ctrl->chip.gc, ctrl); + err = devm_gpiochip_add_data(dev, &ctrl->chip.gc, ctrl); + if (err) + return err; + + return ath79_gpio_register_wifi_descriptors(dev, ctrl->chip.gc.label); } static struct platform_driver ath79_gpio_driver = { diff --git a/drivers/gpio/gpio-dwapb.c b/drivers/gpio/gpio-dwapb.c index 0259c65973323..6ece05f3afe2d 100644 --- a/drivers/gpio/gpio-dwapb.c +++ b/drivers/gpio/gpio-dwapb.c @@ -201,6 +201,22 @@ static void dwapb_toggle_trigger(struct dwapb_gpio *gpio, unsigned int offs) dwapb_write(gpio, GPIO_INT_POLARITY, pol); } +static int dwapb_irq_init_hw(struct gpio_chip *gc) +{ + struct dwapb_gpio *gpio = to_dwapb_gpio(gc); + + /* + * GPIO interrupts may retain stale state across warm reboots when + * peripherals stay powered. Force a known-safe state before the GPIO + * irqchip and irq domain are set up. + */ + dwapb_write(gpio, GPIO_INTEN, 0); + dwapb_write(gpio, GPIO_INTMASK, 0xffffffff); + dwapb_write(gpio, GPIO_PORTA_EOI, 0xffffffff); + + return 0; +} + static u32 dwapb_do_irq(struct dwapb_gpio *gpio) { struct gpio_generic_chip *gen_gc = &gpio->ports[0].chip; @@ -472,6 +488,7 @@ static void dwapb_configure_irqs(struct dwapb_gpio *gpio, girq = &gc->irq; girq->handler = handle_bad_irq; girq->default_type = IRQ_TYPE_NONE; + girq->init_hw = dwapb_irq_init_hw; port->pirq = pirq; diff --git a/drivers/gpio/gpio-ml-ioh.c b/drivers/gpio/gpio-ml-ioh.c index f6af81bf2b13c..031ddf29539cd 100644 --- a/drivers/gpio/gpio-ml-ioh.c +++ b/drivers/gpio/gpio-ml-ioh.c @@ -73,7 +73,7 @@ struct ioh_gpio_reg_data { * @gpio_use_sel: Save GPIO_USE_SEL1~4 register for PM * @ch: Indicate GPIO channel * @irq_base: Save base of IRQ number for interrupt - * @spinlock: Used for register access protection + * @spinlock: Shared register access lock */ struct ioh_gpio { void __iomem *base; @@ -84,7 +84,12 @@ struct ioh_gpio { u32 gpio_use_sel; int ch; int irq_base; - spinlock_t spinlock; + raw_spinlock_t *spinlock; +}; + +struct ioh_gpio_device { + raw_spinlock_t spinlock; + struct ioh_gpio chip[8]; }; static const int num_ports[] = {6, 12, 16, 16, 15, 16, 16, 12}; @@ -95,7 +100,7 @@ static int ioh_gpio_set(struct gpio_chip *gpio, unsigned int nr, int val) struct ioh_gpio *chip = gpiochip_get_data(gpio); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); reg_val = ioread32(&chip->reg->regs[chip->ch].po); if (val) reg_val |= BIT(nr); @@ -103,7 +108,7 @@ static int ioh_gpio_set(struct gpio_chip *gpio, unsigned int nr, int val) reg_val &= ~BIT(nr); iowrite32(reg_val, &chip->reg->regs[chip->ch].po); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); return 0; } @@ -123,7 +128,7 @@ static int ioh_gpio_direction_output(struct gpio_chip *gpio, unsigned nr, u32 reg_val; unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); pm = ioread32(&chip->reg->regs[chip->ch].pm); pm &= BIT(num_ports[chip->ch]) - 1; pm |= BIT(nr); @@ -136,7 +141,7 @@ static int ioh_gpio_direction_output(struct gpio_chip *gpio, unsigned nr, reg_val &= ~BIT(nr); iowrite32(reg_val, &chip->reg->regs[chip->ch].po); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); return 0; } @@ -147,12 +152,12 @@ static int ioh_gpio_direction_input(struct gpio_chip *gpio, unsigned nr) u32 pm; unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); pm = ioread32(&chip->reg->regs[chip->ch].pm); pm &= BIT(num_ports[chip->ch]) - 1; pm &= ~BIT(nr); iowrite32(pm, &chip->reg->regs[chip->ch].pm); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); return 0; } @@ -256,7 +261,7 @@ static int ioh_irq_type(struct irq_data *d, unsigned int type) dev_dbg(chip->dev, "%s:irq=%d type=%d ch=%d pos=%d type=%d\n", __func__, irq, type, ch, im_pos, type); - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); switch (type) { case IRQ_TYPE_EDGE_RISING: @@ -296,7 +301,7 @@ static int ioh_irq_type(struct irq_data *d, unsigned int type) ien = ioread32(&chip->reg->regs[chip->ch].ien); iowrite32(ien | BIT(ch), &chip->reg->regs[chip->ch].ien); end: - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); return 0; } @@ -326,11 +331,11 @@ static void ioh_irq_disable(struct irq_data *d) unsigned long flags; u32 ien; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); ien = ioread32(&chip->reg->regs[chip->ch].ien); ien &= ~BIT(d->irq - chip->irq_base); iowrite32(ien, &chip->reg->regs[chip->ch].ien); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); } static void ioh_irq_enable(struct irq_data *d) @@ -340,11 +345,11 @@ static void ioh_irq_enable(struct irq_data *d) unsigned long flags; u32 ien; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(chip->spinlock, flags); ien = ioread32(&chip->reg->regs[chip->ch].ien); ien |= BIT(d->irq - chip->irq_base); iowrite32(ien, &chip->reg->regs[chip->ch].ien); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(chip->spinlock, flags); } static irqreturn_t ioh_gpio_handler(int irq, void *dev_id) @@ -407,8 +412,8 @@ static int ioh_gpio_probe(struct pci_dev *pdev, int ret; int i, j; struct ioh_gpio *chip; + struct ioh_gpio_device *priv; void __iomem *base; - void *chip_save; int irq_base; ret = pcim_enable_device(pdev); @@ -429,18 +434,18 @@ static int ioh_gpio_probe(struct pci_dev *pdev, return -ENOMEM; } - chip_save = devm_kcalloc(dev, 8, sizeof(*chip), GFP_KERNEL); - if (chip_save == NULL) { + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) return -ENOMEM; - } - chip = chip_save; + raw_spin_lock_init(&priv->spinlock); + chip = priv->chip; for (i = 0; i < 8; i++, chip++) { chip->dev = dev; chip->base = base; chip->reg = chip->base; chip->ch = i; - spin_lock_init(&chip->spinlock); + chip->spinlock = &priv->spinlock; ioh_gpio_setup(chip, num_ports[i]); ret = devm_gpiochip_add_data(dev, &chip->gpio, chip); if (ret) { @@ -449,7 +454,7 @@ static int ioh_gpio_probe(struct pci_dev *pdev, } } - chip = chip_save; + chip = priv->chip; for (j = 0; j < 8; j++, chip++) { irq_base = devm_irq_alloc_descs(dev, -1, IOH_IRQ_BASE, num_ports[j], NUMA_NO_NODE); @@ -466,7 +471,7 @@ static int ioh_gpio_probe(struct pci_dev *pdev, return ret; } - chip = chip_save; + chip = priv->chip; ret = devm_request_irq(dev, pdev->irq, ioh_gpio_handler, IRQF_SHARED, KBUILD_MODNAME, chip); if (ret != 0) { @@ -474,33 +479,33 @@ static int ioh_gpio_probe(struct pci_dev *pdev, return ret; } - pci_set_drvdata(pdev, chip); + pci_set_drvdata(pdev, priv); return 0; } static int __maybe_unused ioh_gpio_suspend(struct device *dev) { - struct ioh_gpio *chip = dev_get_drvdata(dev); + struct ioh_gpio_device *priv = dev_get_drvdata(dev); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); - ioh_gpio_save_reg_conf(chip); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_lock_irqsave(&priv->spinlock, flags); + ioh_gpio_save_reg_conf(priv->chip); + raw_spin_unlock_irqrestore(&priv->spinlock, flags); return 0; } static int __maybe_unused ioh_gpio_resume(struct device *dev) { - struct ioh_gpio *chip = dev_get_drvdata(dev); + struct ioh_gpio_device *priv = dev_get_drvdata(dev); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); - iowrite32(0x01, &chip->reg->srst); - iowrite32(0x00, &chip->reg->srst); - ioh_gpio_restore_reg_conf(chip); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_lock_irqsave(&priv->spinlock, flags); + iowrite32(0x01, &priv->chip->reg->srst); + iowrite32(0x00, &priv->chip->reg->srst); + ioh_gpio_restore_reg_conf(priv->chip); + raw_spin_unlock_irqrestore(&priv->spinlock, flags); return 0; } diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c index dbdf0f41b6bb9..12f11a6c96653 100644 --- a/drivers/gpio/gpio-mxc.c +++ b/drivers/gpio/gpio-mxc.c @@ -675,7 +675,7 @@ static const struct dev_pm_ops mxc_gpio_dev_pm_ops = { RUNTIME_PM_OPS(mxc_gpio_runtime_suspend, mxc_gpio_runtime_resume, NULL) }; -static int mxc_gpio_syscore_suspend(void) +static int mxc_gpio_syscore_suspend(void *data) { struct mxc_gpio_port *port; int ret; @@ -692,7 +692,7 @@ static int mxc_gpio_syscore_suspend(void) return 0; } -static void mxc_gpio_syscore_resume(void) +static void mxc_gpio_syscore_resume(void *data) { struct mxc_gpio_port *port; int ret; @@ -709,11 +709,15 @@ static void mxc_gpio_syscore_resume(void) } } -static struct syscore_ops mxc_gpio_syscore_ops = { +static const struct syscore_ops mxc_gpio_syscore_ops = { .suspend = mxc_gpio_syscore_suspend, .resume = mxc_gpio_syscore_resume, }; +static struct syscore mxc_gpio_syscore = { + .ops = &mxc_gpio_syscore_ops, +}; + static struct platform_driver mxc_gpio_driver = { .driver = { .name = "gpio-mxc", @@ -726,7 +730,7 @@ static struct platform_driver mxc_gpio_driver = { static int __init gpio_mxc_init(void) { - register_syscore_ops(&mxc_gpio_syscore_ops); + register_syscore(&mxc_gpio_syscore); return platform_driver_register(&mxc_gpio_driver); } diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c index ac52ba9573f0d..beb33c4096d7a 100644 --- a/drivers/gpio/gpio-pca953x.c +++ b/drivers/gpio/gpio-pca953x.c @@ -1370,9 +1370,20 @@ static int pca953x_restore_context(struct pca953x_chip *chip) regcache_mark_dirty(chip->regmap); ret = pca953x_regcache_sync(chip); if (ret) - return ret; + goto err; + + ret = regcache_sync(chip->regmap); + if (ret) + goto err; + + return 0; - return regcache_sync(chip->regmap); +err: + if (chip->client->irq > 0) + disable_irq(chip->client->irq); + regcache_cache_only(chip->regmap, true); + + return ret; } static void pca953x_save_context(struct pca953x_chip *chip) diff --git a/drivers/gpio/gpio-pch.c b/drivers/gpio/gpio-pch.c index 9925687e05fb4..1a7e422675295 100644 --- a/drivers/gpio/gpio-pch.c +++ b/drivers/gpio/gpio-pch.c @@ -96,7 +96,7 @@ struct pch_gpio { struct pch_gpio_reg_data pch_gpio_reg; int irq_base; enum pch_type_t ioh; - spinlock_t spinlock; + raw_spinlock_t spinlock; }; static int pch_gpio_set(struct gpio_chip *gpio, unsigned int nr, int val) @@ -105,7 +105,7 @@ static int pch_gpio_set(struct gpio_chip *gpio, unsigned int nr, int val) struct pch_gpio *chip = gpiochip_get_data(gpio); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); reg_val = ioread32(&chip->reg->po); if (val) reg_val |= BIT(nr); @@ -113,7 +113,7 @@ static int pch_gpio_set(struct gpio_chip *gpio, unsigned int nr, int val) reg_val &= ~BIT(nr); iowrite32(reg_val, &chip->reg->po); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } @@ -133,7 +133,7 @@ static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned int nr, u32 reg_val; unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); reg_val = ioread32(&chip->reg->po); if (val) @@ -147,7 +147,7 @@ static int pch_gpio_direction_output(struct gpio_chip *gpio, unsigned int nr, pm |= BIT(nr); iowrite32(pm, &chip->reg->pm); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } @@ -158,12 +158,12 @@ static int pch_gpio_direction_input(struct gpio_chip *gpio, unsigned int nr) u32 pm; unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); pm = ioread32(&chip->reg->pm); pm &= BIT(gpio_pins[chip->ioh]) - 1; pm &= ~BIT(nr); iowrite32(pm, &chip->reg->pm); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } @@ -265,7 +265,7 @@ static int pch_irq_type(struct irq_data *d, unsigned int type) return 0; } - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); /* Set interrupt mode */ im = ioread32(im_reg) & ~(PCH_IM_MASK << (im_pos * 4)); @@ -277,7 +277,7 @@ static int pch_irq_type(struct irq_data *d, unsigned int type) else if (type & IRQ_TYPE_EDGE_BOTH) irq_set_handler_locked(d, handle_edge_irq); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } @@ -374,7 +374,7 @@ static int pch_gpio_probe(struct pci_dev *pdev, chip->ioh = id->driver_data; chip->reg = chip->base; pci_set_drvdata(pdev, chip); - spin_lock_init(&chip->spinlock); + raw_spin_lock_init(&chip->spinlock); pch_gpio_setup(chip); ret = devm_gpiochip_add_data(dev, &chip->gpio, chip); @@ -407,9 +407,9 @@ static int __maybe_unused pch_gpio_suspend(struct device *dev) struct pch_gpio *chip = dev_get_drvdata(dev); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); pch_gpio_save_reg_conf(chip); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } @@ -419,11 +419,11 @@ static int __maybe_unused pch_gpio_resume(struct device *dev) struct pch_gpio *chip = dev_get_drvdata(dev); unsigned long flags; - spin_lock_irqsave(&chip->spinlock, flags); + raw_spin_lock_irqsave(&chip->spinlock, flags); iowrite32(0x01, &chip->reg->reset); iowrite32(0x00, &chip->reg->reset); pch_gpio_restore_reg_conf(chip); - spin_unlock_irqrestore(&chip->spinlock, flags); + raw_spin_unlock_irqrestore(&chip->spinlock, flags); return 0; } diff --git a/drivers/gpio/gpio-pisosr.c b/drivers/gpio/gpio-pisosr.c index 7ec6a46ed6007..2732ea8c16b7b 100644 --- a/drivers/gpio/gpio-pisosr.c +++ b/drivers/gpio/gpio-pisosr.c @@ -112,6 +112,7 @@ static int pisosr_gpio_probe(struct spi_device *spi) { struct device *dev = &spi->dev; struct pisosr_gpio *gpio; + u32 ngpios; int ret; gpio = devm_kzalloc(dev, sizeof(*gpio), GFP_KERNEL); @@ -120,7 +121,8 @@ static int pisosr_gpio_probe(struct spi_device *spi) gpio->chip = template_chip; gpio->chip.parent = dev; - of_property_read_u16(dev->of_node, "ngpios", &gpio->chip.ngpio); + if (!of_property_read_u32(dev->of_node, "ngpios", &ngpios)) + gpio->chip.ngpio = ngpios; gpio->spi = spi; diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c index fa22f3faa163c..664cf1eef494c 100644 --- a/drivers/gpio/gpio-pxa.c +++ b/drivers/gpio/gpio-pxa.c @@ -747,7 +747,7 @@ static int __init pxa_gpio_dt_init(void) device_initcall(pxa_gpio_dt_init); #ifdef CONFIG_PM -static int pxa_gpio_suspend(void) +static int pxa_gpio_suspend(void *data) { struct pxa_gpio_chip *pchip = pxa_gpio_chip; struct pxa_gpio_bank *c; @@ -768,7 +768,7 @@ static int pxa_gpio_suspend(void) return 0; } -static void pxa_gpio_resume(void) +static void pxa_gpio_resume(void *data) { struct pxa_gpio_chip *pchip = pxa_gpio_chip; struct pxa_gpio_bank *c; @@ -792,14 +792,18 @@ static void pxa_gpio_resume(void) #define pxa_gpio_resume NULL #endif -static struct syscore_ops pxa_gpio_syscore_ops = { +static const struct syscore_ops pxa_gpio_syscore_ops = { .suspend = pxa_gpio_suspend, .resume = pxa_gpio_resume, }; +static struct syscore pxa_gpio_syscore = { + .ops = &pxa_gpio_syscore_ops, +}; + static int __init pxa_gpio_sysinit(void) { - register_syscore_ops(&pxa_gpio_syscore_ops); + register_syscore(&pxa_gpio_syscore); return 0; } postcore_initcall(pxa_gpio_sysinit); diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c index 7f6a62f5d1eeb..1938ffa2f4f33 100644 --- a/drivers/gpio/gpio-sa1100.c +++ b/drivers/gpio/gpio-sa1100.c @@ -256,7 +256,7 @@ static void sa1100_gpio_handler(struct irq_desc *desc) } while (mask); } -static int sa1100_gpio_suspend(void) +static int sa1100_gpio_suspend(void *data) { struct sa1100_gpio_chip *sgc = &sa1100_gpio_chip; @@ -275,19 +275,23 @@ static int sa1100_gpio_suspend(void) return 0; } -static void sa1100_gpio_resume(void) +static void sa1100_gpio_resume(void *data) { sa1100_update_edge_regs(&sa1100_gpio_chip); } -static struct syscore_ops sa1100_gpio_syscore_ops = { +static const struct syscore_ops sa1100_gpio_syscore_ops = { .suspend = sa1100_gpio_suspend, .resume = sa1100_gpio_resume, }; +static struct syscore sa1100_gpio_syscore = { + .ops = &sa1100_gpio_syscore_ops, +}; + static int __init sa1100_gpio_init_devicefs(void) { - register_syscore_ops(&sa1100_gpio_syscore_ops); + register_syscore(&sa1100_gpio_syscore); return 0; } diff --git a/drivers/gpio/gpio-sloppy-logic-analyzer.c b/drivers/gpio/gpio-sloppy-logic-analyzer.c index 969dddd3d6faf..3b6825b0f2f97 100644 --- a/drivers/gpio/gpio-sloppy-logic-analyzer.c +++ b/drivers/gpio/gpio-sloppy-logic-analyzer.c @@ -161,6 +161,13 @@ static int fops_buf_size_get(void *data, u64 *val) return 0; } +static void fops_buf_release(void *data) +{ + struct gpio_la_poll_priv *priv = data; + + vfree(priv->blob.data); +} + static int fops_buf_size_set(void *data, u64 val) { struct gpio_la_poll_priv *priv = data; @@ -239,6 +246,9 @@ static int gpio_la_poll_probe(struct platform_device *pdev) return ret; fops_buf_size_set(priv, GPIO_LA_DEFAULT_BUF_SIZE); + ret = devm_add_action_or_reset(dev, fops_buf_release, priv); + if (ret) + return ret; priv->descs = devm_gpiod_get_array(dev, "probe", GPIOD_IN); if (IS_ERR(priv->descs)) @@ -291,7 +301,7 @@ static int gpio_la_poll_probe(struct platform_device *pdev) debugfs_create_ulong("delay_ns_acquisition", 0400, priv->debug_dir, &priv->acq_delay); debugfs_create_file_unsafe("buf_size", 0600, priv->debug_dir, priv, &fops_buf_size); debugfs_create_file_unsafe("capture", 0200, priv->debug_dir, priv, &fops_capture); - debugfs_create_file_unsafe("trigger", 0200, priv->debug_dir, priv, &fops_trigger); + debugfs_create_file("trigger", 0200, priv->debug_dir, priv, &fops_trigger); return 0; } diff --git a/drivers/gpio/gpio-usbio.c b/drivers/gpio/gpio-usbio.c index 34d42c743d5bc..489c8ac6299e1 100644 --- a/drivers/gpio/gpio-usbio.c +++ b/drivers/gpio/gpio-usbio.c @@ -31,6 +31,7 @@ static const struct acpi_device_id usbio_gpio_acpi_hids[] = { { "INTC10B5" }, /* LNL */ { "INTC10D1" }, /* MTL-CVF */ { "INTC10E2" }, /* PTL */ + { "INTC1116" }, /* NVL */ { } }; diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c index e4ed169e31999..d8ad5357002d9 100644 --- a/drivers/gpio/gpiolib.c +++ b/drivers/gpio/gpiolib.c @@ -53,7 +53,12 @@ #define dont_test_bit(b,d) (0) -#define REJECT_HOTPLUG_TAX 1 +/* + * The SRCU bypass below skips srcu_read_lock() while the core retains its + * lockdep_assert_held(&gc->gpiodev->srcu) checks, so every gpiod_*() call + * splats under lockdep. Keep the optimisation for production builds only. + */ +#define REJECT_HOTPLUG_TAX (!IS_ENABLED(CONFIG_LOCKDEP)) #if REJECT_HOTPLUG_TAX diff --git a/drivers/gpu/Kconfig b/drivers/gpu/Kconfig new file mode 100644 index 0000000000000..ebb2ad4b7ea0b --- /dev/null +++ b/drivers/gpu/Kconfig @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 + +config GPU_BUDDY + bool + help + A page based buddy allocator for GPU memory. + +config GPU_BUDDY_KUNIT_TEST + tristate "KUnit tests for GPU buddy allocator" if !KUNIT_ALL_TESTS + depends on GPU_BUDDY && KUNIT + default KUNIT_ALL_TESTS + help + KUnit tests for the GPU buddy allocator. diff --git a/drivers/gpu/Makefile b/drivers/gpu/Makefile index 36a54d4566303..b4e5e338efa23 100644 --- a/drivers/gpu/Makefile +++ b/drivers/gpu/Makefile @@ -2,7 +2,9 @@ # drm/tegra depends on host1x, so if both drivers are built-in care must be # taken to initialize them in the correct order. Link order is the only way # to ensure this currently. -obj-y += host1x/ drm/ vga/ +# Similarly, buddy must come first since it is used by other drivers. +obj-$(CONFIG_GPU_BUDDY) += buddy.o +obj-y += host1x/ drm/ vga/ tests/ obj-$(CONFIG_IMX_IPUV3_CORE) += ipu-v3/ obj-$(CONFIG_TRACE_GPU_MEM) += trace/ obj-$(CONFIG_NOVA_CORE) += nova-core/ diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c new file mode 100644 index 0000000000000..bf5920bd22580 --- /dev/null +++ b/drivers/gpu/buddy.c @@ -0,0 +1,1347 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include + +#include +#include +#include +#include + +#include + +static struct kmem_cache *slab_blocks; + +static struct gpu_buddy_block *gpu_block_alloc(struct gpu_buddy *mm, + struct gpu_buddy_block *parent, + unsigned int order, + u64 offset) +{ + struct gpu_buddy_block *block; + + BUG_ON(order > GPU_BUDDY_MAX_ORDER); + + block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL); + if (!block) + return NULL; + + block->header = offset; + block->header |= order; + block->parent = parent; + + RB_CLEAR_NODE(&block->rb); + + BUG_ON(block->header & GPU_BUDDY_HEADER_UNUSED); + return block; +} + +static void gpu_block_free(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + kmem_cache_free(slab_blocks, block); +} + +static enum gpu_buddy_free_tree +get_block_tree(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_is_clear(block) ? + GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; +} + +static struct gpu_buddy_block * +rbtree_get_free_block(const struct rb_node *node) +{ + return node ? rb_entry(node, struct gpu_buddy_block, rb) : NULL; +} + +static struct gpu_buddy_block * +rbtree_last_free_block(struct rb_root *root) +{ + return rbtree_get_free_block(rb_last(root)); +} + +static bool rbtree_is_empty(struct rb_root *root) +{ + return RB_EMPTY_ROOT(root); +} + +static bool gpu_buddy_block_offset_less(const struct gpu_buddy_block *block, + const struct gpu_buddy_block *node) +{ + return gpu_buddy_block_offset(block) < gpu_buddy_block_offset(node); +} + +static bool rbtree_block_offset_less(struct rb_node *block, + const struct rb_node *node) +{ + return gpu_buddy_block_offset_less(rbtree_get_free_block(block), + rbtree_get_free_block(node)); +} + +static void rbtree_insert(struct gpu_buddy *mm, + struct gpu_buddy_block *block, + enum gpu_buddy_free_tree tree) +{ + rb_add(&block->rb, + &mm->free_trees[tree][gpu_buddy_block_order(block)], + rbtree_block_offset_less); +} + +static void rbtree_remove(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + unsigned int order = gpu_buddy_block_order(block); + enum gpu_buddy_free_tree tree; + struct rb_root *root; + + tree = get_block_tree(block); + root = &mm->free_trees[tree][order]; + + rb_erase(&block->rb, root); + RB_CLEAR_NODE(&block->rb); +} + +static void clear_reset(struct gpu_buddy_block *block) +{ + block->header &= ~GPU_BUDDY_HEADER_CLEAR; +} + +static void mark_cleared(struct gpu_buddy_block *block) +{ + block->header |= GPU_BUDDY_HEADER_CLEAR; +} + +static void mark_allocated(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + block->header &= ~GPU_BUDDY_HEADER_STATE; + block->header |= GPU_BUDDY_ALLOCATED; + + rbtree_remove(mm, block); +} + +static void mark_free(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + enum gpu_buddy_free_tree tree; + + block->header &= ~GPU_BUDDY_HEADER_STATE; + block->header |= GPU_BUDDY_FREE; + + tree = get_block_tree(block); + rbtree_insert(mm, block, tree); +} + +static void mark_split(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + block->header &= ~GPU_BUDDY_HEADER_STATE; + block->header |= GPU_BUDDY_SPLIT; + + rbtree_remove(mm, block); +} + +static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= e2 && e1 >= s2; +} + +static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) +{ + return s1 <= s2 && e1 >= e2; +} + +static struct gpu_buddy_block * +__get_buddy(struct gpu_buddy_block *block) +{ + struct gpu_buddy_block *parent; + + parent = block->parent; + if (!parent) + return NULL; + + if (parent->left == block) + return parent->right; + + return parent->left; +} + +static unsigned int __gpu_buddy_free(struct gpu_buddy *mm, + struct gpu_buddy_block *block, + bool force_merge) +{ + struct gpu_buddy_block *parent; + unsigned int order; + + while ((parent = block->parent)) { + struct gpu_buddy_block *buddy; + + buddy = __get_buddy(block); + + if (!gpu_buddy_block_is_free(buddy)) + break; + + if (!force_merge) { + /* + * Check the block and its buddy clear state and exit + * the loop if they both have the dissimilar state. + */ + if (gpu_buddy_block_is_clear(block) != + gpu_buddy_block_is_clear(buddy)) + break; + + if (gpu_buddy_block_is_clear(block)) + mark_cleared(parent); + } + + rbtree_remove(mm, buddy); + if (force_merge && gpu_buddy_block_is_clear(buddy)) + mm->clear_avail -= gpu_buddy_block_size(mm, buddy); + + gpu_block_free(mm, block); + gpu_block_free(mm, buddy); + + block = parent; + } + + order = gpu_buddy_block_order(block); + mark_free(mm, block); + + return order; +} + +static int __force_merge(struct gpu_buddy *mm, + u64 start, + u64 end, + unsigned int min_order) +{ + unsigned int tree, order; + int i; + + if (!min_order) + return -ENOMEM; + + if (min_order > mm->max_order) + return -EINVAL; + + for_each_free_tree(tree) { + for (i = min_order - 1; i >= 0; i--) { + struct rb_node *iter = rb_last(&mm->free_trees[tree][i]); + + while (iter) { + struct gpu_buddy_block *block, *buddy; + u64 block_start, block_end; + + block = rbtree_get_free_block(iter); + iter = rb_prev(iter); + + if (!block || !block->parent) + continue; + + block_start = gpu_buddy_block_offset(block); + block_end = block_start + gpu_buddy_block_size(mm, block) - 1; + + if (!contains(start, end, block_start, block_end)) + continue; + + buddy = __get_buddy(block); + if (!gpu_buddy_block_is_free(buddy)) + continue; + + WARN_ON(gpu_buddy_block_is_clear(block) == + gpu_buddy_block_is_clear(buddy)); + + /* + * Advance to the next node when the current node is the buddy, + * as freeing the block will also remove its buddy from the tree. + */ + if (iter == &buddy->rb) + iter = rb_prev(iter); + + rbtree_remove(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail -= gpu_buddy_block_size(mm, block); + + order = __gpu_buddy_free(mm, block, true); + if (order >= min_order) + return 0; + } + } + } + + return -ENOMEM; +} + +/** + * gpu_buddy_init - init memory manager + * + * @mm: GPU buddy manager to initialize + * @size: size in bytes to manage + * @chunk_size: minimum page size in bytes for our allocations + * + * Initializes the memory manager and its resources. + * + * Returns: + * 0 on success, error code on failure. + */ +int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size) +{ + unsigned int i, j, root_count = 0; + u64 offset = 0; + + if (size < chunk_size) + return -EINVAL; + + if (chunk_size < SZ_4K) + return -EINVAL; + + if (!is_power_of_2(chunk_size)) + return -EINVAL; + + size = round_down(size, chunk_size); + + mm->size = size; + mm->avail = size; + mm->clear_avail = 0; + mm->chunk_size = chunk_size; + mm->max_order = ilog2(size) - ilog2(chunk_size); + + BUG_ON(mm->max_order > GPU_BUDDY_MAX_ORDER); + + mm->free_trees = kmalloc_array(GPU_BUDDY_MAX_FREE_TREES, + sizeof(*mm->free_trees), + GFP_KERNEL); + if (!mm->free_trees) + return -ENOMEM; + + for_each_free_tree(i) { + mm->free_trees[i] = kmalloc_array(mm->max_order + 1, + sizeof(struct rb_root), + GFP_KERNEL); + if (!mm->free_trees[i]) + goto out_free_tree; + + for (j = 0; j <= mm->max_order; ++j) + mm->free_trees[i][j] = RB_ROOT; + } + + mm->n_roots = hweight64(size); + + mm->roots = kmalloc_array(mm->n_roots, + sizeof(struct gpu_buddy_block *), + GFP_KERNEL); + if (!mm->roots) + goto out_free_tree; + + /* + * Split into power-of-two blocks, in case we are given a size that is + * not itself a power-of-two. + */ + do { + struct gpu_buddy_block *root; + unsigned int order; + u64 root_size; + + order = ilog2(size) - ilog2(chunk_size); + root_size = chunk_size << order; + + root = gpu_block_alloc(mm, NULL, order, offset); + if (!root) + goto out_free_roots; + + mark_free(mm, root); + + BUG_ON(root_count > mm->max_order); + BUG_ON(gpu_buddy_block_size(mm, root) < chunk_size); + + mm->roots[root_count] = root; + + offset += root_size; + size -= root_size; + root_count++; + } while (size); + + return 0; + +out_free_roots: + while (root_count--) + gpu_block_free(mm, mm->roots[root_count]); + kfree(mm->roots); +out_free_tree: + while (i--) + kfree(mm->free_trees[i]); + kfree(mm->free_trees); + return -ENOMEM; +} +EXPORT_SYMBOL(gpu_buddy_init); + +/** + * gpu_buddy_fini - tear down the memory manager + * + * @mm: GPU buddy manager to free + * + * Cleanup memory manager resources and the freetree + */ +void gpu_buddy_fini(struct gpu_buddy *mm) +{ + u64 root_size, size, start; + unsigned int order; + int i; + + size = mm->size; + + for (i = 0; i < mm->n_roots; ++i) { + order = ilog2(size) - ilog2(mm->chunk_size); + start = gpu_buddy_block_offset(mm->roots[i]); + __force_merge(mm, start, start + size, order); + + if (WARN_ON(!gpu_buddy_block_is_free(mm->roots[i]))) + kunit_fail_current_test("buddy_fini() root"); + + gpu_block_free(mm, mm->roots[i]); + + root_size = mm->chunk_size << order; + size -= root_size; + } + + WARN_ON(mm->avail != mm->size); + + for_each_free_tree(i) + kfree(mm->free_trees[i]); + kfree(mm->free_trees); + kfree(mm->roots); +} +EXPORT_SYMBOL(gpu_buddy_fini); + +static int split_block(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + unsigned int block_order = gpu_buddy_block_order(block) - 1; + u64 offset = gpu_buddy_block_offset(block); + + BUG_ON(!gpu_buddy_block_is_free(block)); + BUG_ON(!gpu_buddy_block_order(block)); + + block->left = gpu_block_alloc(mm, block, block_order, offset); + if (!block->left) + return -ENOMEM; + + block->right = gpu_block_alloc(mm, block, block_order, + offset + (mm->chunk_size << block_order)); + if (!block->right) { + gpu_block_free(mm, block->left); + return -ENOMEM; + } + + mark_split(mm, block); + + if (gpu_buddy_block_is_clear(block)) { + mark_cleared(block->left); + mark_cleared(block->right); + clear_reset(block); + } + + mark_free(mm, block->left); + mark_free(mm, block->right); + + return 0; +} + +/** + * gpu_get_buddy - get buddy address + * + * @block: GPU buddy block + * + * Returns the corresponding buddy block for @block, or NULL + * if this is a root block and can't be merged further. + * Requires some kind of locking to protect against + * any concurrent allocate and free operations. + */ +struct gpu_buddy_block * +gpu_get_buddy(struct gpu_buddy_block *block) +{ + return __get_buddy(block); +} +EXPORT_SYMBOL(gpu_get_buddy); + +/** + * gpu_buddy_reset_clear - reset blocks clear state + * + * @mm: GPU buddy manager + * @is_clear: blocks clear state + * + * Reset the clear state based on @is_clear value for each block + * in the freetree. + */ +void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear) +{ + enum gpu_buddy_free_tree src_tree, dst_tree; + u64 root_size, size, start; + unsigned int order; + int i; + + size = mm->size; + for (i = 0; i < mm->n_roots; ++i) { + order = ilog2(size) - ilog2(mm->chunk_size); + start = gpu_buddy_block_offset(mm->roots[i]); + __force_merge(mm, start, start + size, order); + + root_size = mm->chunk_size << order; + size -= root_size; + } + + src_tree = is_clear ? GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE; + dst_tree = is_clear ? GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; + + for (i = 0; i <= mm->max_order; ++i) { + struct rb_root *root = &mm->free_trees[src_tree][i]; + struct gpu_buddy_block *block, *tmp; + + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + rbtree_remove(mm, block); + if (is_clear) { + mark_cleared(block); + mm->clear_avail += gpu_buddy_block_size(mm, block); + } else { + clear_reset(block); + mm->clear_avail -= gpu_buddy_block_size(mm, block); + } + + rbtree_insert(mm, block, dst_tree); + } + } +} +EXPORT_SYMBOL(gpu_buddy_reset_clear); + +/** + * gpu_buddy_free_block - free a block + * + * @mm: GPU buddy manager + * @block: block to be freed + */ +void gpu_buddy_free_block(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + BUG_ON(!gpu_buddy_block_is_allocated(block)); + mm->avail += gpu_buddy_block_size(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail += gpu_buddy_block_size(mm, block); + + __gpu_buddy_free(mm, block, false); +} +EXPORT_SYMBOL(gpu_buddy_free_block); + +static void __gpu_buddy_free_list(struct gpu_buddy *mm, + struct list_head *objects, + bool mark_clear, + bool mark_dirty) +{ + struct gpu_buddy_block *block, *on; + + WARN_ON(mark_dirty && mark_clear); + + list_for_each_entry_safe(block, on, objects, link) { + if (mark_clear) + mark_cleared(block); + else if (mark_dirty) + clear_reset(block); + gpu_buddy_free_block(mm, block); + cond_resched(); + } + INIT_LIST_HEAD(objects); +} + +static void gpu_buddy_free_list_internal(struct gpu_buddy *mm, + struct list_head *objects) +{ + /* + * Don't touch the clear/dirty bit, since allocation is still internal + * at this point. For example we might have just failed part of the + * allocation. + */ + __gpu_buddy_free_list(mm, objects, false, false); +} + +/** + * gpu_buddy_free_list - free blocks + * + * @mm: GPU buddy manager + * @objects: input list head to free blocks + * @flags: optional flags like GPU_BUDDY_CLEARED + */ +void gpu_buddy_free_list(struct gpu_buddy *mm, + struct list_head *objects, + unsigned int flags) +{ + bool mark_clear = flags & GPU_BUDDY_CLEARED; + + __gpu_buddy_free_list(mm, objects, mark_clear, !mark_clear); +} +EXPORT_SYMBOL(gpu_buddy_free_list); + +static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags) +{ + bool needs_clear = flags & GPU_BUDDY_CLEAR_ALLOCATION; + + return needs_clear != gpu_buddy_block_is_clear(block); +} + +static struct gpu_buddy_block * +__alloc_range_bias(struct gpu_buddy *mm, + u64 start, u64 end, + unsigned int order, + unsigned long flags, + bool fallback) +{ + u64 req_size = mm->chunk_size << order; + struct gpu_buddy_block *block; + struct gpu_buddy_block *buddy; + LIST_HEAD(dfs); + int err; + int i; + + end = end - 1; + + for (i = 0; i < mm->n_roots; ++i) + list_add_tail(&mm->roots[i]->tmp_link, &dfs); + + do { + u64 block_start; + u64 block_end; + + block = list_first_entry_or_null(&dfs, + struct gpu_buddy_block, + tmp_link); + if (!block) + break; + + list_del(&block->tmp_link); + + if (gpu_buddy_block_order(block) < order) + continue; + + block_start = gpu_buddy_block_offset(block); + block_end = block_start + gpu_buddy_block_size(mm, block) - 1; + + if (!overlaps(start, end, block_start, block_end)) + continue; + + if (gpu_buddy_block_is_allocated(block)) + continue; + + if (block_start < start || block_end > end) { + u64 adjusted_start = max(block_start, start); + u64 adjusted_end = min(block_end, end); + + if (round_down(adjusted_end + 1, req_size) <= + round_up(adjusted_start, req_size)) + continue; + } + + if (!fallback && block_incompatible(block, flags)) + continue; + + if (contains(start, end, block_start, block_end) && + order == gpu_buddy_block_order(block)) { + /* + * Find the free block within the range. + */ + if (gpu_buddy_block_is_free(block)) + return block; + + continue; + } + + if (!gpu_buddy_block_is_split(block)) { + err = split_block(mm, block); + if (unlikely(err)) + goto err_undo; + } + + list_add(&block->right->tmp_link, &dfs); + list_add(&block->left->tmp_link, &dfs); + } while (1); + + return ERR_PTR(-ENOSPC); + +err_undo: + /* + * We really don't want to leave around a bunch of split blocks, since + * bigger is better, so make sure we merge everything back before we + * free the allocated blocks. + */ + buddy = __get_buddy(block); + if (buddy && + (gpu_buddy_block_is_free(block) && + gpu_buddy_block_is_free(buddy))) + __gpu_buddy_free(mm, block, false); + return ERR_PTR(err); +} + +static struct gpu_buddy_block * +__gpu_buddy_alloc_range_bias(struct gpu_buddy *mm, + u64 start, u64 end, + unsigned int order, + unsigned long flags) +{ + struct gpu_buddy_block *block; + bool fallback = false; + + block = __alloc_range_bias(mm, start, end, order, + flags, fallback); + if (IS_ERR(block)) + return __alloc_range_bias(mm, start, end, order, + flags, !fallback); + + return block; +} + +static struct gpu_buddy_block * +get_maxblock(struct gpu_buddy *mm, + unsigned int order, + enum gpu_buddy_free_tree tree) +{ + struct gpu_buddy_block *max_block = NULL, *block = NULL; + struct rb_root *root; + unsigned int i; + + for (i = order; i <= mm->max_order; ++i) { + root = &mm->free_trees[tree][i]; + block = rbtree_last_free_block(root); + if (!block) + continue; + + if (!max_block) { + max_block = block; + continue; + } + + if (gpu_buddy_block_offset(block) > + gpu_buddy_block_offset(max_block)) { + max_block = block; + } + } + + return max_block; +} + +static struct gpu_buddy_block * +alloc_from_freetree(struct gpu_buddy *mm, + unsigned int order, + unsigned long flags) +{ + struct gpu_buddy_block *block = NULL; + struct rb_root *root; + enum gpu_buddy_free_tree tree; + unsigned int tmp; + int err; + + tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ? + GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; + + if (flags & GPU_BUDDY_TOPDOWN_ALLOCATION) { + block = get_maxblock(mm, order, tree); + if (block) + /* Store the obtained block order */ + tmp = gpu_buddy_block_order(block); + } else { + for (tmp = order; tmp <= mm->max_order; ++tmp) { + /* Get RB tree root for this order and tree */ + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); + if (block) + break; + } + } + + if (!block) { + /* Try allocating from the other tree */ + tree = (tree == GPU_BUDDY_CLEAR_TREE) ? + GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE; + + for (tmp = order; tmp <= mm->max_order; ++tmp) { + root = &mm->free_trees[tree][tmp]; + block = rbtree_last_free_block(root); + if (block) + break; + } + + if (!block) + return ERR_PTR(-ENOSPC); + } + + BUG_ON(!gpu_buddy_block_is_free(block)); + + while (tmp != order) { + err = split_block(mm, block); + if (unlikely(err)) + goto err_undo; + + block = block->right; + tmp--; + } + return block; + +err_undo: + if (tmp != order) + __gpu_buddy_free(mm, block, false); + return ERR_PTR(err); +} + +static int __alloc_range(struct gpu_buddy *mm, + struct list_head *dfs, + u64 start, u64 size, + struct list_head *blocks, + u64 *total_allocated_on_err) +{ + struct gpu_buddy_block *block; + struct gpu_buddy_block *buddy; + u64 total_allocated = 0; + LIST_HEAD(allocated); + u64 end; + int err; + + end = start + size - 1; + + do { + u64 block_start; + u64 block_end; + + block = list_first_entry_or_null(dfs, + struct gpu_buddy_block, + tmp_link); + if (!block) + break; + + list_del(&block->tmp_link); + + block_start = gpu_buddy_block_offset(block); + block_end = block_start + gpu_buddy_block_size(mm, block) - 1; + + if (!overlaps(start, end, block_start, block_end)) + continue; + + if (gpu_buddy_block_is_allocated(block)) { + err = -ENOSPC; + goto err_free; + } + + if (contains(start, end, block_start, block_end)) { + if (gpu_buddy_block_is_free(block)) { + mark_allocated(mm, block); + total_allocated += gpu_buddy_block_size(mm, block); + mm->avail -= gpu_buddy_block_size(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail -= gpu_buddy_block_size(mm, block); + list_add_tail(&block->link, &allocated); + continue; + } else if (!mm->clear_avail) { + err = -ENOSPC; + goto err_free; + } + } + + if (!gpu_buddy_block_is_split(block)) { + err = split_block(mm, block); + if (unlikely(err)) + goto err_undo; + } + + list_add(&block->right->tmp_link, dfs); + list_add(&block->left->tmp_link, dfs); + } while (1); + + if (total_allocated < size) { + err = -ENOSPC; + goto err_free; + } + + list_splice_tail(&allocated, blocks); + + return 0; + +err_undo: + /* + * We really don't want to leave around a bunch of split blocks, since + * bigger is better, so make sure we merge everything back before we + * free the allocated blocks. + */ + buddy = __get_buddy(block); + if (buddy && + (gpu_buddy_block_is_free(block) && + gpu_buddy_block_is_free(buddy))) + __gpu_buddy_free(mm, block, false); + +err_free: + if (err == -ENOSPC && total_allocated_on_err) { + list_splice_tail(&allocated, blocks); + *total_allocated_on_err = total_allocated; + } else { + gpu_buddy_free_list_internal(mm, &allocated); + } + + return err; +} + +static int __gpu_buddy_alloc_range(struct gpu_buddy *mm, + u64 start, + u64 size, + u64 *total_allocated_on_err, + struct list_head *blocks) +{ + LIST_HEAD(dfs); + int i; + + for (i = 0; i < mm->n_roots; ++i) + list_add_tail(&mm->roots[i]->tmp_link, &dfs); + + return __alloc_range(mm, &dfs, start, size, + blocks, total_allocated_on_err); +} + +static int __alloc_contig_aligned_retry(struct gpu_buddy *mm, + u64 unaligned_offset, + u64 size, + u64 min_block_size, + struct list_head *blocks) +{ + u64 aligned_offset = round_down(unaligned_offset, min_block_size); + + return __gpu_buddy_alloc_range(mm, aligned_offset, size, NULL, blocks); +} + +static int __alloc_contig_try_harder(struct gpu_buddy *mm, + u64 size, + u64 min_block_size, + struct list_head *blocks) +{ + u64 rhs_offset, lhs_offset, filled; + struct gpu_buddy_block *block; + unsigned int tree, order; + u64 modify_size; + int err; + + modify_size = rounddown_pow_of_two(size); + order = ilog2(modify_size) - ilog2(mm->chunk_size); + if (order == 0) + return -ENOSPC; + + for_each_free_tree(tree) { + struct rb_root *root; + struct rb_node *iter; + + root = &mm->free_trees[tree][order]; + if (rbtree_is_empty(root)) + continue; + + iter = rb_last(root); + while (iter) { + block = rbtree_get_free_block(iter); + + rhs_offset = gpu_buddy_block_offset(block); + + /* Allocate blocks traversing RHS */ + err = __gpu_buddy_alloc_range(mm, rhs_offset, size, + &filled, blocks); + if (err && err != -ENOSPC) + return err; + if (!err && IS_ALIGNED(rhs_offset, min_block_size)) + return 0; + if (!err) { + /* Allocate the unaligned RHS offset using round_down */ + gpu_buddy_free_list_internal(mm, blocks); + err = __alloc_contig_aligned_retry(mm, rhs_offset, + size, + min_block_size, + blocks); + if (!err) + return 0; + if (err != -ENOSPC) { + gpu_buddy_free_list_internal(mm, blocks); + return err; + } + goto next; + } + + if (size - filled > rhs_offset) + goto next; + + lhs_offset = rhs_offset - (size - filled); + + /* Allocate the unaligned LHS offset using round_down */ + gpu_buddy_free_list_internal(mm, blocks); + err = __alloc_contig_aligned_retry(mm, lhs_offset, size, + min_block_size, blocks); + if (!err) + return 0; + if (err != -ENOSPC) { + gpu_buddy_free_list_internal(mm, blocks); + return err; + } +next: + gpu_buddy_free_list_internal(mm, blocks); + iter = rb_prev(iter); + } + } + + return -ENOSPC; +} + +/** + * gpu_buddy_block_trim - free unused pages + * + * @mm: GPU buddy manager + * @start: start address to begin the trimming. + * @new_size: original size requested + * @blocks: Input and output list of allocated blocks. + * MUST contain single block as input to be trimmed. + * On success will contain the newly allocated blocks + * making up the @new_size. Blocks always appear in + * ascending order + * + * For contiguous allocation, we round up the size to the nearest + * power of two value, drivers consume *actual* size, so remaining + * portions are unused and can be optionally freed with this function + * + * Returns: + * 0 on success, error code on failure. + */ +int gpu_buddy_block_trim(struct gpu_buddy *mm, + u64 *start, + u64 new_size, + struct list_head *blocks) +{ + struct gpu_buddy_block *parent; + struct gpu_buddy_block *block; + u64 block_start, block_end; + LIST_HEAD(dfs); + u64 new_start; + int err; + + if (!list_is_singular(blocks)) + return -EINVAL; + + block = list_first_entry(blocks, + struct gpu_buddy_block, + link); + + block_start = gpu_buddy_block_offset(block); + block_end = block_start + gpu_buddy_block_size(mm, block); + + if (WARN_ON(!gpu_buddy_block_is_allocated(block))) + return -EINVAL; + + if (new_size > gpu_buddy_block_size(mm, block)) + return -EINVAL; + + if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size)) + return -EINVAL; + + if (new_size == gpu_buddy_block_size(mm, block)) + return 0; + + new_start = block_start; + if (start) { + new_start = *start; + + if (new_start < block_start) + return -EINVAL; + + if (!IS_ALIGNED(new_start, mm->chunk_size)) + return -EINVAL; + + if (range_overflows(new_start, new_size, block_end)) + return -EINVAL; + } + + list_del(&block->link); + mark_free(mm, block); + mm->avail += gpu_buddy_block_size(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail += gpu_buddy_block_size(mm, block); + + /* Prevent recursively freeing this node */ + parent = block->parent; + block->parent = NULL; + + list_add(&block->tmp_link, &dfs); + err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL); + if (err) { + mark_allocated(mm, block); + mm->avail -= gpu_buddy_block_size(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail -= gpu_buddy_block_size(mm, block); + list_add(&block->link, blocks); + } + + block->parent = parent; + return err; +} +EXPORT_SYMBOL(gpu_buddy_block_trim); + +static struct gpu_buddy_block * +__gpu_buddy_alloc_blocks(struct gpu_buddy *mm, + u64 start, u64 end, + unsigned int order, + unsigned long flags) +{ + if (flags & GPU_BUDDY_RANGE_ALLOCATION) + /* Allocate traversing within the range */ + return __gpu_buddy_alloc_range_bias(mm, start, end, + order, flags); + else + /* Allocate from freetree */ + return alloc_from_freetree(mm, order, flags); +} + +/** + * gpu_buddy_alloc_blocks - allocate power-of-two blocks + * + * @mm: GPU buddy manager to allocate from + * @start: start of the allowed range for this block + * @end: end of the allowed range for this block + * @size: size of the allocation in bytes + * @min_block_size: alignment of the allocation + * @blocks: output list head to add allocated blocks + * @flags: GPU_BUDDY_*_ALLOCATION flags + * + * alloc_range_bias() called on range limitations, which traverses + * the tree and returns the desired block. + * + * alloc_from_freetree() called when *no* range restrictions + * are enforced, which picks the block from the freetree. + * + * Returns: + * 0 on success, error code on failure. + */ +int gpu_buddy_alloc_blocks(struct gpu_buddy *mm, + u64 start, u64 end, u64 size, + u64 min_block_size, + struct list_head *blocks, + unsigned long flags) +{ + struct gpu_buddy_block *block = NULL; + u64 original_size, original_min_size; + unsigned int min_order, order; + LIST_HEAD(allocated); + unsigned long pages; + int err; + + if (size < mm->chunk_size) + return -EINVAL; + + if (min_block_size < mm->chunk_size) + return -EINVAL; + + if (!is_power_of_2(min_block_size)) + return -EINVAL; + + if (!IS_ALIGNED(start | end | size, mm->chunk_size)) + return -EINVAL; + + if (end > mm->size) + return -EINVAL; + + if (range_overflows(start, size, mm->size)) + return -EINVAL; + + /* Actual range allocation */ + if (start + size == end) { + if (!IS_ALIGNED(start | end, min_block_size)) + return -EINVAL; + + return __gpu_buddy_alloc_range(mm, start, size, NULL, blocks); + } + + original_size = size; + original_min_size = min_block_size; + + /* Roundup the size to power of 2 */ + if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) { + size = roundup_pow_of_two(size); + min_block_size = size; + /* Align size value to min_block_size */ + } else if (!IS_ALIGNED(size, min_block_size)) { + size = round_up(size, min_block_size); + } + + pages = size >> ilog2(mm->chunk_size); + order = fls(pages) - 1; + min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); + + if (order > mm->max_order || size > mm->size) { + if ((flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) && + !(flags & GPU_BUDDY_RANGE_ALLOCATION)) + return __alloc_contig_try_harder(mm, original_size, + original_min_size, blocks); + + return -EINVAL; + } + + do { + order = min(order, (unsigned int)fls(pages) - 1); + BUG_ON(order > mm->max_order); + BUG_ON(order < min_order); + + do { + block = __gpu_buddy_alloc_blocks(mm, start, + end, + order, + flags); + if (!IS_ERR(block)) + break; + + if (order-- == min_order) { + /* Try allocation through force merge method */ + if (mm->clear_avail && + !__force_merge(mm, start, end, min_order)) { + block = __gpu_buddy_alloc_blocks(mm, start, + end, + min_order, + flags); + if (!IS_ERR(block)) { + order = min_order; + break; + } + } + + /* + * Try contiguous block allocation through + * try harder method. + */ + if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION && + !(flags & GPU_BUDDY_RANGE_ALLOCATION)) + return __alloc_contig_try_harder(mm, + original_size, + original_min_size, + blocks); + err = -ENOSPC; + goto err_free; + } + } while (1); + + mark_allocated(mm, block); + mm->avail -= gpu_buddy_block_size(mm, block); + if (gpu_buddy_block_is_clear(block)) + mm->clear_avail -= gpu_buddy_block_size(mm, block); + kmemleak_update_trace(block); + list_add_tail(&block->link, &allocated); + + pages -= BIT(order); + + if (!pages) + break; + } while (1); + + /* Trim the allocated block to the required size */ + if (!(flags & GPU_BUDDY_TRIM_DISABLE) && + original_size != size) { + struct list_head *trim_list; + LIST_HEAD(temp); + u64 trim_size; + + trim_list = &allocated; + trim_size = original_size; + + if (!list_is_singular(&allocated)) { + block = list_last_entry(&allocated, typeof(*block), link); + list_move(&block->link, &temp); + trim_list = &temp; + trim_size = gpu_buddy_block_size(mm, block) - + (size - original_size); + } + + gpu_buddy_block_trim(mm, + NULL, + trim_size, + trim_list); + + if (!list_empty(&temp)) + list_splice_tail(trim_list, &allocated); + } + + list_splice_tail(&allocated, blocks); + return 0; + +err_free: + gpu_buddy_free_list_internal(mm, &allocated); + return err; +} +EXPORT_SYMBOL(gpu_buddy_alloc_blocks); + +/** + * gpu_buddy_block_print - print block information + * + * @mm: GPU buddy manager + * @block: GPU buddy block + */ +void gpu_buddy_block_print(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + u64 start = gpu_buddy_block_offset(block); + u64 size = gpu_buddy_block_size(mm, block); + + pr_info("%#018llx-%#018llx: %llu\n", start, start + size, size); +} +EXPORT_SYMBOL(gpu_buddy_block_print); + +/** + * gpu_buddy_print - print allocator state + * + * @mm: GPU buddy manager + * @p: GPU printer to use + */ +void gpu_buddy_print(struct gpu_buddy *mm) +{ + int order; + + pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n", + mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20); + + for (order = mm->max_order; order >= 0; order--) { + struct gpu_buddy_block *block, *tmp; + struct rb_root *root; + u64 count = 0, free; + unsigned int tree; + + for_each_free_tree(tree) { + root = &mm->free_trees[tree][order]; + + rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { + BUG_ON(!gpu_buddy_block_is_free(block)); + count++; + } + } + + free = count * (mm->chunk_size << order); + if (free < SZ_1M) + pr_info("order-%2d free: %8llu KiB, blocks: %llu\n", + order, free >> 10, count); + else + pr_info("order-%2d free: %8llu MiB, blocks: %llu\n", + order, free >> 20, count); + } +} +EXPORT_SYMBOL(gpu_buddy_print); + +static void gpu_buddy_module_exit(void) +{ + kmem_cache_destroy(slab_blocks); +} + +static int __init gpu_buddy_module_init(void) +{ + slab_blocks = KMEM_CACHE(gpu_buddy_block, 0); + if (!slab_blocks) + return -ENOMEM; + + return 0; +} + +module_init(gpu_buddy_module_init); +module_exit(gpu_buddy_module_exit); + +MODULE_DESCRIPTION("GPU Buddy Allocator"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 5b1dee660a07d..5ebdd4b40cb77 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -220,6 +220,7 @@ config DRM_GPUSVM config DRM_BUDDY tristate depends on DRM + select GPU_BUDDY help A page based buddy allocator @@ -418,10 +419,6 @@ config DRM_HYPERV config DRM_PANEL_BACKLIGHT_QUIRKS tristate -config DRM_LIB_RANDOM - bool - default n - config DRM_PRIVACY_SCREEN bool default n diff --git a/drivers/gpu/drm/Kconfig.debug b/drivers/gpu/drm/Kconfig.debug index 05dc43c0b8c5a..3b7886865335a 100644 --- a/drivers/gpu/drm/Kconfig.debug +++ b/drivers/gpu/drm/Kconfig.debug @@ -69,7 +69,6 @@ config DRM_KUNIT_TEST select DRM_EXPORT_FOR_TESTS if m select DRM_GEM_SHMEM_HELPER select DRM_KUNIT_TEST_HELPERS - select DRM_LIB_RANDOM select DRM_SYSFB_HELPER select PRIME_NUMBERS default KUNIT_ALL_TESTS diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index dfe5a88c3b01d..83da18b3ce1e5 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -77,7 +77,6 @@ drm-$(CONFIG_DRM_CLIENT) += \ drm_client.o \ drm_client_event.o \ drm_client_modeset.o -drm-$(CONFIG_DRM_LIB_RANDOM) += lib/drm_random.o drm-$(CONFIG_COMPAT) += drm_ioc32.o drm-$(CONFIG_DRM_PANEL) += drm_panel.o drm-$(CONFIG_OF) += drm_of.o @@ -239,7 +238,7 @@ obj-$(CONFIG_DRM_POWERVR) += imagination/ # Ensure drm headers are self-contained and pass kernel-doc hdrtest-files := \ $(shell cd $(src) && find . -maxdepth 1 -name 'drm_*.h') \ - $(shell cd $(src) && find display lib -name '*.h') + $(shell cd $(src) && find display -name '*.h') always-$(CONFIG_DRM_HEADER_TEST) += \ $(patsubst %.h,%.hdrtest, $(hdrtest-files)) diff --git a/drivers/gpu/drm/adp/Kconfig b/drivers/gpu/drm/adp/Kconfig index 9fcc27eb200db..acfa21ee06d22 100644 --- a/drivers/gpu/drm/adp/Kconfig +++ b/drivers/gpu/drm/adp/Kconfig @@ -6,7 +6,6 @@ config DRM_ADP select DRM_KMS_HELPER select DRM_BRIDGE_CONNECTOR select DRM_DISPLAY_HELPER - select DRM_KMS_DMA_HELPER select DRM_GEM_DMA_HELPER select DRM_PANEL_BRIDGE select VIDEOMODE_HELPERS diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c index 4926996f94da0..fe39b150ee533 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c @@ -505,6 +505,7 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) u32 val = 0; u32 count = 0; struct amdgpu_device *adev = ip_block->adev; + int ret = 0; /* return early if no ACP */ if (!adev->acp.acp_genpd) { @@ -526,7 +527,8 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } @@ -543,20 +545,23 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block) break; if (--count == 0) { dev_err(&adev->pdev->dev, "Failed to reset ACP\n"); - return -ETIMEDOUT; + ret = -ETIMEDOUT; + goto out; } udelay(100); } - +out: device_for_each_child(adev->acp.parent, NULL, acp_genpd_remove_device); mfd_remove_devices(adev->acp.parent); kfree(adev->acp.acp_res); + pm_genpd_remove(&adev->acp.acp_genpd->gpd); kfree(adev->acp.acp_genpd); + adev->acp.acp_genpd = NULL; kfree(adev->acp.acp_cell); - return 0; + return ret; } static int acp_suspend(struct amdgpu_ip_block *ip_block) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index 1ec26be82f30e..5e8d0d6b55ab6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -528,7 +528,7 @@ uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev) int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id) { @@ -563,9 +563,24 @@ int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, *dmabuf_adev = adev; if (bo_size) *bo_size = amdgpu_bo_size(bo); - if (metadata_buffer) - r = amdgpu_bo_get_metadata(bo, metadata_buffer, buffer_size, - metadata_size, &metadata_flags); + if (metadata_buffer) { + /* first get metadata_size by buffer = NULL */ + r = amdgpu_bo_get_metadata(bo, NULL, 0, + metadata_size, NULL); + + /* user buf_size is bigger than bo metadata_size + * allocate a buf at kernel space and copy */ + if (*metadata_size <= buffer_size) { + *metadata_buffer = kzalloc(*metadata_size, GFP_KERNEL); + + if (!*metadata_buffer) + return -ENOMEM; + + r = amdgpu_bo_get_metadata(bo, *metadata_buffer, *metadata_size, + NULL, &metadata_flags); + } else + r = -EINVAL; + } if (flags) { *flags = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? KFD_IOC_ALLOC_MEM_FLAGS_VRAM diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 9e120c934cc17..c59b5d9cd36b6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -255,7 +255,7 @@ uint64_t amdgpu_amdkfd_get_gpu_clock_counter(struct amdgpu_device *adev); uint32_t amdgpu_amdkfd_get_max_engine_clock_in_mhz(struct amdgpu_device *adev); int amdgpu_amdkfd_get_dmabuf_info(struct amdgpu_device *adev, int dma_buf_fd, struct amdgpu_device **dmabuf_adev, - uint64_t *bo_size, void *metadata_buffer, + uint64_t *bo_size, void **metadata_buffer, size_t buffer_size, uint32_t *metadata_size, uint32_t *flags, int8_t *xcp_id); int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct amdgpu_device *adev, bool is_min); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 285c35545ebbf..d56728b257f49 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1777,6 +1777,12 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( size >>= 1; aligned_size = PAGE_ALIGN(size); + /* reject AQL queue with size < 2 */ + if (!aligned_size) { + ret = -EINVAL; + goto err_alignment_size; + } + (*mem)->alloc_flags = flags; amdgpu_sync_create(&(*mem)->sync); @@ -1868,6 +1874,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); err_reserve_limit: amdgpu_sync_free(&(*mem)->sync); +err_alignment_size: mutex_destroy(&(*mem)->lock); if (gobj) drm_gem_object_put(gobj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c index 763f2b8dcf13a..b8f7e3a18d324 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c @@ -36,6 +36,21 @@ #include "atombios_encoders.h" #include "bif/bif_4_1_d.h" +/* VBIOS-reported table size is unchecked against the image; cap iterations and + * adev->i2c_bus[] indexing to AMDGPU_MAX_I2C_BUS. + */ +static int amdgpu_atombios_gpio_i2c_num_entries(uint16_t size) +{ + u32 bytes; + + if (size < sizeof(ATOM_COMMON_TABLE_HEADER)) + return 0; + + bytes = size - sizeof(ATOM_COMMON_TABLE_HEADER); + return (int)min_t(u32, bytes / sizeof(ATOM_GPIO_I2C_ASSIGMENT), + AMDGPU_MAX_I2C_BUS); +} + static struct amdgpu_i2c_bus_rec amdgpu_atombios_get_bus_rec_for_i2c_gpio(ATOM_GPIO_I2C_ASSIGMENT *gpio) { struct amdgpu_i2c_bus_rec i2c; @@ -96,8 +111,7 @@ struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device * if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -127,8 +141,7 @@ void amdgpu_atombios_i2c_init(struct amdgpu_device *adev) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { @@ -158,8 +171,7 @@ void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id) if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) { i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset); - num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) / - sizeof(ATOM_GPIO_I2C_ASSIGMENT); + num_indices = amdgpu_atombios_gpio_i2c_num_entries(size); gpio = &i2c_info->asGPIO_Info[0]; for (i = 0; i < num_indices; i++) { diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c index 00e96419fcdad..dec312b0f3b0a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c @@ -33,6 +33,7 @@ #include #include #include +#include /* * BIOS. */ @@ -355,19 +356,59 @@ static bool amdgpu_read_disabled_bios(struct amdgpu_device *adev) } #ifdef CONFIG_ACPI +/** + * amdgpu_acpi_vfct_match() - Check if a VFCT entry matches the device + * @adev: AMDGPU device + * @vhdr: VFCT image header to check + * + * VFCT entries contain the PCI bus number as recorded during BIOS POST. + * On systems where the kernel renumbers PCI buses (e.g. pci=realloc or + * resource conflicts), the runtime bus number may differ from the POST + * value. Match by device identity (vendor + device + function) and use + * the bus number as a preference: exact bus match is preferred, but when + * the bus numbers disagree we accept the entry if the device identity + * matches. + * + * Returns: 0 on match, -ENODEV on no match + */ +static int amdgpu_acpi_vfct_match(struct amdgpu_device *adev, + VFCT_IMAGE_HEADER *vhdr) +{ + /* Vendor and device IDs must always match */ + if (vhdr->VendorID != adev->pdev->vendor || + vhdr->DeviceID != adev->pdev->device) + return -ENODEV; + + if (vhdr->PCIDevice != PCI_SLOT(adev->pdev->devfn) || + vhdr->PCIFunction != PCI_FUNC(adev->pdev->devfn)) + return -ENODEV; + + /* Exact bus number match - preferred */ + if (vhdr->PCIBus == adev->pdev->bus->number) + return 0; + + /* Bus mismatch but device identity matches (PCI renumbering case) */ + dev_notice(adev->dev, + "VFCT bus number mismatch: table %u != runtime %u, matching by device identity (vendor 0x%04x device 0x%04x)\n", + vhdr->PCIBus, adev->pdev->bus->number, + adev->pdev->vendor, adev->pdev->device); + return 0; +} + static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) { struct acpi_table_header *hdr; acpi_size tbl_size; UEFI_ACPI_VFCT *vfct; unsigned int offset; + bool r = false; if (!ACPI_SUCCESS(acpi_get_table("VFCT", 1, &hdr))) return false; tbl_size = hdr->length; if (tbl_size < sizeof(UEFI_ACPI_VFCT)) { dev_info(adev->dev, "ACPI VFCT table present but broken (too short #1),skipping\n"); - return false; + goto out; } vfct = (UEFI_ACPI_VFCT *)hdr; @@ -380,36 +421,36 @@ static bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) offset += sizeof(VFCT_IMAGE_HEADER); if (offset > tbl_size) { dev_info(adev->dev, "ACPI VFCT image header truncated,skipping\n"); - return false; + goto out; } offset += vhdr->ImageLength; if (offset > tbl_size) { dev_info(adev->dev, "ACPI VFCT image truncated,skipping\n"); - return false; + goto out; } if (vhdr->ImageLength && - vhdr->PCIBus == adev->pdev->bus->number && - vhdr->PCIDevice == PCI_SLOT(adev->pdev->devfn) && - vhdr->PCIFunction == PCI_FUNC(adev->pdev->devfn) && - vhdr->VendorID == adev->pdev->vendor && - vhdr->DeviceID == adev->pdev->device) { + !amdgpu_acpi_vfct_match(adev, vhdr)) { adev->bios = kmemdup(&vbios->VbiosContent, vhdr->ImageLength, GFP_KERNEL); if (!check_atom_bios(adev, vhdr->ImageLength)) { amdgpu_bios_release(adev); - return false; + goto out; } adev->bios_size = vhdr->ImageLength; - return true; + r = true; + goto out; } } dev_info(adev->dev, "ACPI VFCT table present but broken (too short #2),skipping\n"); - return false; + +out: + acpi_put_table(hdr); + return r; } #else static inline bool amdgpu_acpi_vfct_bios(struct amdgpu_device *adev) @@ -451,7 +492,8 @@ static bool amdgpu_prefer_rom_resource(struct amdgpu_device *adev) { struct resource *res = &adev->pdev->resource[PCI_ROM_RESOURCE]; - return (res->flags & IORESOURCE_ROM_SHADOW); + return (res->flags & IORESOURCE_ROM_SHADOW) || + adev->pdev == vga_default_device(); } static bool amdgpu_get_bios_dgpu(struct amdgpu_device *adev) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c index 425a3e5643608..2694facb06c73 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c @@ -465,7 +465,7 @@ static u32 amdgpu_cper_ring_get_ent_sz(struct amdgpu_ring *ring, u64 pos) void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) { - u64 pos, wptr_old, rptr; + u64 pos, wptr_old, rptr, next_rptr; int rec_cnt_dw = count >> 2; u32 chunk, ent_sz; u8 *s = (u8 *)src; @@ -506,9 +506,19 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) do { ent_sz = amdgpu_cper_ring_get_ent_sz(ring, pos); - - rptr += (ent_sz >> 2); - rptr &= ring->ptr_mask; + next_rptr = rptr; + if (ent_sz >= sizeof(u32)) + next_rptr = (rptr + (ent_sz >> 2)) & ring->ptr_mask; + + if (next_rptr == rptr) { + /* Corrupt entry size, reset the ring to avoid an infinite loop. */ + rptr = ring->wptr; + *ring->rptr_cpu_addr = rptr; + ring->count_dw = (ring->ring_size - 4) >> 2; + goto out_unlock; + } + + rptr = next_rptr; *ring->rptr_cpu_addr = rptr; pos = rptr; @@ -517,6 +527,8 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count) if (ring->count_dw >= rec_cnt_dw) ring->count_dw -= rec_cnt_dw; + +out_unlock: mutex_unlock(&ring->adev->cper.ring_lock); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c index 8669e2fd6eff6..38ce4d7c86f00 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c @@ -42,6 +42,26 @@ #include "amdgpu_gem.h" #include "amdgpu_ras.h" +/* + * Maximum IB length (dwords) for rings whose emit_ib packet format + * documents a 20-bit size field. + */ +#define AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW 0xFFFFF +#define AMDGPU_MM_IB_PACKET_SIZE_MAX_DW 0x7FFFF0 + +static u32 amdgpu_cs_ib_packet_size_max_dw(enum amdgpu_ring_type type) +{ + switch (type) { + case AMDGPU_RING_TYPE_GFX: + case AMDGPU_RING_TYPE_COMPUTE: + case AMDGPU_RING_TYPE_SDMA: + case AMDGPU_RING_TYPE_VPE: + return AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW; + default: + return AMDGPU_MM_IB_PACKET_SIZE_MAX_DW; + } +} + static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, struct amdgpu_device *adev, struct drm_file *filp, @@ -241,6 +261,10 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, if (size < sizeof(struct drm_amdgpu_cs_chunk_fence)) goto free_partial_kdata; + /* Only a single user fence is allowed to simplify handling. */ + if (p->uf_bo) + goto free_partial_kdata; + ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata, &uf_offset); if (ret) @@ -260,13 +284,17 @@ static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, goto free_partial_kdata; break; + case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: + if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow)) + goto free_partial_kdata; + break; + case AMDGPU_CHUNK_ID_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_IN: case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: - case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: break; default: @@ -354,7 +382,6 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p, job = p->jobs[r]; ring = amdgpu_job_ring(job); - ib = &job->ibs[job->num_ibs++]; /* submissions to kernel queues are disabled */ if (ring->no_user_submission) @@ -383,6 +410,12 @@ static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p, return -EINVAL; } + if (chunk_ib->ib_bytes / 4 > + amdgpu_cs_ib_packet_size_max_dw(ring->funcs->type)) + return -EINVAL; + + ib = &job->ibs[job->num_ibs++]; + if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; @@ -860,7 +893,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct drm_gem_object *obj; - unsigned long index; unsigned int i; int r; @@ -965,7 +997,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, goto out_free_user_pages; } - drm_exec_for_each_locked_object(&p->exec, index, obj) { + drm_exec_for_each_locked_object(&p->exec, obj) { r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj)); if (unlikely(r)) goto out_free_user_pages; @@ -1183,19 +1215,6 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); } - if (adev->debug_vm) { - /* Invalidate all BOs to test for userspace bugs */ - amdgpu_bo_list_for_each_entry(e, p->bo_list) { - struct amdgpu_bo *bo = e->bo; - - /* ignore duplicates */ - if (!bo) - continue; - - amdgpu_vm_bo_invalidate(bo, false); - } - } - return 0; } @@ -1205,7 +1224,6 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) struct drm_gpu_scheduler *sched; struct drm_gem_object *obj; struct dma_fence *fence; - unsigned long index; unsigned int i; int r; @@ -1216,7 +1234,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) return r; } - drm_exec_for_each_locked_object(&p->exec, index, obj) { + drm_exec_for_each_locked_object(&p->exec, obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); struct dma_resv *resv = bo->tbo.base.resv; @@ -1285,7 +1303,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, struct amdgpu_vm *vm = &fpriv->vm; struct amdgpu_bo_list_entry *e; struct drm_gem_object *gobj; - unsigned long index; unsigned int i; uint64_t seq; int r; @@ -1336,7 +1353,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, } p->fence = dma_fence_get(&leader->base.s_fence->finished); - drm_exec_for_each_locked_object(&p->exec, index, gobj) { + drm_exec_for_each_locked_object(&p->exec, gobj) { ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo); @@ -1385,6 +1402,8 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, /* Cleanup the parser structure */ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser) { + struct amdgpu_device *adev = parser->adev; + struct amdgpu_bo_list_entry *e; unsigned int i; amdgpu_sync_free(&parser->sync); @@ -1400,8 +1419,21 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser) if (parser->ctx) amdgpu_ctx_put(parser->ctx); - if (parser->bo_list) + if (parser->bo_list) { + if (adev->debug_vm) { + /* Invalidate all BOs to test for userspace bugs */ + amdgpu_bo_list_for_each_entry(e, parser->bo_list) { + struct amdgpu_bo *bo = e->bo; + + /* ignore duplicates */ + if (!bo) + continue; + + amdgpu_vm_bo_invalidate(bo, false); + } + } amdgpu_bo_list_put(parser->bo_list); + } for (i = 0; i < parser->nchunks; i++) kvfree(parser->chunks[i].kdata); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c index afedea02188dd..32f3d7c49e1f4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c @@ -321,7 +321,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, struct drm_file *filp, struct amdgpu_ctx *ctx) { struct amdgpu_fpriv *fpriv = filp->driver_priv; - u32 current_stable_pstate; int r; r = amdgpu_ctx_priority_permit(filp, priority); @@ -339,37 +338,22 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, ctx->generation = amdgpu_vm_generation(mgr->adev, &fpriv->vm); ctx->init_priority = priority; ctx->override_priority = AMDGPU_CTX_PRIORITY_UNSET; - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r) - return r; - - if (mgr->adev->pm.stable_pstate_ctx) - ctx->stable_pstate = mgr->adev->pm.stable_pstate_ctx->stable_pstate; - else - ctx->stable_pstate = current_stable_pstate; + ctx->stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; ctx->ctx_mgr = &(fpriv->ctx_mgr); return 0; } -static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, - u32 stable_pstate) +static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) { struct amdgpu_device *adev = ctx->mgr->adev; enum amd_dpm_forced_level level; + struct amdgpu_ctx *current_ctx; u32 current_stable_pstate; - int r; + int r = 0; - mutex_lock(&adev->pm.stable_pstate_ctx_lock); - if (adev->pm.stable_pstate_ctx && adev->pm.stable_pstate_ctx != ctx) { - r = -EBUSY; - goto done; - } - - r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); - if (r || (stable_pstate == current_stable_pstate)) - goto done; + lockdep_assert_held(&adev->pm.stable_pstate_ctx_lock); switch (stable_pstate) { case AMDGPU_CTX_STABLE_PSTATE_NONE: @@ -388,17 +372,41 @@ static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, level = AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; break; default: - r = -EINVAL; - goto done; + return -EINVAL; } + current_ctx = adev->pm.stable_pstate_ctx; + if (current_ctx && current_ctx != ctx) + return -EBUSY; + + r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); + if (r || current_stable_pstate == stable_pstate) + return r; + r = amdgpu_dpm_force_performance_level(adev, level); + if (r) + return r; - if (level == AMD_DPM_FORCED_LEVEL_AUTO) - adev->pm.stable_pstate_ctx = NULL; - else + if (!current_ctx) { adev->pm.stable_pstate_ctx = ctx; -done: + /* + * Serialized by context taking ownership for the first time + * while holding adev->pm.stable_pstate_ctx_lock). + */ + WRITE_ONCE(ctx->stable_pstate, current_stable_pstate); + } + + return 0; +} + +static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, + u32 stable_pstate) +{ + struct amdgpu_device *adev = ctx->mgr->adev; + int r; + + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + r = __amdgpu_ctx_set_stable_pstate(ctx, stable_pstate); mutex_unlock(&adev->pm.stable_pstate_ctx_lock); return r; @@ -424,7 +432,12 @@ static void amdgpu_ctx_fini(struct kref *ref) } if (drm_dev_enter(adev_to_drm(adev), &idx)) { - amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + mutex_lock(&adev->pm.stable_pstate_ctx_lock); + if (adev->pm.stable_pstate_ctx == ctx) { + __amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate); + adev->pm.stable_pstate_ctx = NULL; + } + mutex_unlock(&adev->pm.stable_pstate_ctx_lock); drm_dev_exit(idx); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c index a70651050acff..d6f58128bc741 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c @@ -1960,7 +1960,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) /* swap out the old fences */ amdgpu_ib_preempt_fences_swap(ring, fences); - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); /* resubmit unfinished jobs */ amdgpu_ib_preempt_job_recovery(&ring->sched); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index d0cb300cd382b..dda5d62c8e7b0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -468,6 +468,13 @@ void amdgpu_device_detect_runtime_pm_mode(struct amdgpu_device *adev) int bamaco_support; adev->pm.rpm_mode = AMDGPU_RUNPM_NONE; + if (pci_is_thunderbolt_attached(adev->pdev) || + dev_is_removable(&adev->pdev->dev)) { + dev_info(adev->dev, + "Runtime PM disabled for externally attached device\n"); + return; + } + bamaco_support = amdgpu_device_supports_baco(adev); switch (amdgpu_runtime_pm) { @@ -1878,6 +1885,15 @@ static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device if (c->x86_vendor == X86_VENDOR_INTEL) return false; + + /* + * AMD Ryzen Pinnacle Ridge (Zen+, family 0x17 model 0x08) CPUs don't + * support PCIe dynamic speed switching. + * https://gitlab.freedesktop.org/drm/amd/-/work_items/5436 + */ + if (c->x86_vendor == X86_VENDOR_AMD && c->x86 == 0x17 && + c->x86_model == 0x08) + return false; #endif return true; } @@ -1888,7 +1904,8 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) * It's unclear if this is a platform-specific or GPU-specific issue. * Disable ASPM on SI for the time being. */ - if (adev->family == AMDGPU_FAMILY_SI) + if (adev->family == AMDGPU_FAMILY_SI || + (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK) && adev->family == AMDGPU_FAMILY_VI)) return true; #if IS_ENABLED(CONFIG_X86) @@ -1918,6 +1935,31 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) #endif } +/* + * Some dGPUs expose their display endpoint below an internal PCIe switch. + * Use the switch upstream port to query the host-facing link. + */ +static struct pci_dev *amdgpu_device_get_aspm_pdev(struct amdgpu_device *adev) +{ + struct pci_dev *swds, *swus; + + swds = pci_upstream_bridge(adev->pdev); + if (!swds || + (swds->vendor != PCI_VENDOR_ID_ATI && + swds->vendor != PCI_VENDOR_ID_AMD) || + pci_pcie_type(swds) != PCI_EXP_TYPE_DOWNSTREAM) + return adev->pdev; + + swus = pci_upstream_bridge(swds); + if (!swus || + (swus->vendor != PCI_VENDOR_ID_ATI && + swus->vendor != PCI_VENDOR_ID_AMD) || + pci_pcie_type(swus) != PCI_EXP_TYPE_UPSTREAM) + return adev->pdev; + + return swus; +} + /** * amdgpu_device_should_use_aspm - check if the device should program ASPM * @@ -1930,6 +1972,9 @@ static bool amdgpu_device_aspm_support_quirk(struct amdgpu_device *adev) */ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) { + struct pci_dev *aspm_pdev, *parent; + bool enabled; + switch (amdgpu_aspm) { case -1: break; @@ -1944,7 +1989,27 @@ bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) return false; if (amdgpu_device_aspm_support_quirk(adev)) return false; - return pcie_aspm_enabled(adev->pdev); + + /* + * pcie_aspm_enabled() checks the link between its argument and + * the immediate upstream bridge. Use SWUS for dGPUs with an + * internal switch so that this is the host-facing link. + */ + aspm_pdev = amdgpu_device_get_aspm_pdev(adev); + parent = pci_upstream_bridge(aspm_pdev); + if (!parent) { + dev_dbg(adev->dev, "ASPM: no upstream PCIe link for %s\n", + pci_name(aspm_pdev)); + return false; + } + + enabled = pcie_aspm_enabled(aspm_pdev); + /* Report the exact link used for the automatic ASPM decision. */ + dev_dbg(adev->dev, "ASPM: link %s <-> %s is %s\n", + pci_name(parent), pci_name(aspm_pdev), + enabled ? "enabled" : "disabled"); + + return enabled; } /* if we get transitioned to only one device, take VGA back */ @@ -4405,6 +4470,14 @@ static void amdgpu_device_set_mcbp(struct amdgpu_device *adev) dev_info(adev->dev, "MCBP is enabled\n"); } +static bool +amdgpu_device_should_register_switcheroo(struct amdgpu_device *adev, bool px) +{ + return !pci_is_thunderbolt_attached(adev->pdev) && + (px || (!dev_is_removable(&adev->pdev->dev) && + apple_gmux_detect(NULL, NULL))); +} + /** * amdgpu_device_init - initialize the driver * @@ -4882,8 +4955,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, px = amdgpu_device_supports_px(adev); - if (px || (!dev_is_removable(&adev->pdev->dev) && - apple_gmux_detect(NULL, NULL))) + if (amdgpu_device_should_register_switcheroo(adev, px)) vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, px); @@ -5059,8 +5131,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev) px = amdgpu_device_supports_px(adev); - if (px || (!dev_is_removable(&adev->pdev->dev) && - apple_gmux_detect(NULL, NULL))) + if (amdgpu_device_should_register_switcheroo(adev, px)) vga_switcheroo_unregister_client(adev->pdev); if (px) @@ -5819,6 +5890,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, { int i, r = 0; struct amdgpu_job *job = NULL; + struct dma_fence *fence = NULL; struct amdgpu_device *tmp_adev = reset_context->reset_req_dev; bool need_full_reset = test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); @@ -5831,6 +5903,9 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, amdgpu_fence_driver_isr_toggle(adev, true); + if (job) + fence = &job->hw_fence.base; + /* block all schedulers and reset given job's ring */ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { struct amdgpu_ring *ring = adev->rings[i]; @@ -5844,7 +5919,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, amdgpu_fence_driver_clear_job_fences(ring); /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, fence); } amdgpu_fence_driver_isr_toggle(adev, false); @@ -7492,8 +7567,8 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, struct amdgpu_ring *ring, struct amdgpu_job *job) { - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; struct drm_sched_fence *f = job->base.s_fence; + struct amdgpu_isolation *isolation; struct dma_fence *dep; void *owner; int r; @@ -7506,6 +7581,9 @@ struct dma_fence *amdgpu_device_enforce_isolation(struct amdgpu_device *adev, ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE) return NULL; + isolation = &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; + /* * All submissions where enforce isolation is false are handled as if * they come from a single client. Use ~0l as the owner to distinct it diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index 7248947835f10..fb753613e00d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -2539,6 +2539,7 @@ amdgpu_pci_remove(struct pci_dev *pdev) if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } amdgpu_driver_unload_kms(dev); @@ -2891,6 +2892,19 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev) return 0; } +static void amdgpu_restore_umd_profile_pstate_after_runpm(struct amdgpu_device *adev) +{ + enum amd_dpm_forced_level level; + uint32_t profile_mode_mask = AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD | + AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK | + AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK | + AMD_DPM_FORCED_LEVEL_PROFILE_PEAK; + + level = amdgpu_dpm_get_performance_level(adev); + if (level & profile_mode_mask) + amdgpu_asic_update_umd_stable_pstate(adev, true); +} + static int amdgpu_pmops_runtime_resume(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); @@ -2935,6 +2949,8 @@ static int amdgpu_pmops_runtime_resume(struct device *dev) if (adev->pm.rpm_mode == AMDGPU_RUNPM_PX) drm_dev->switch_power_state = DRM_SWITCH_POWER_ON; + + amdgpu_restore_umd_profile_pstate_after_runpm(adev); adev->in_runpm = false; return 0; } @@ -3202,6 +3218,14 @@ static void __exit amdgpu_exit(void) amdgpu_userq_fence_slab_fini(); mmu_notifier_synchronize(); amdgpu_xcp_drv_release(); + + /* + * Flush outstanding call_rcu() callbacks before the + * module text is freed. Otherwise a grace period elapsing after + * unload invokes a callback in already-freed module memory and + * faults in rcu_do_batch(). + */ + rcu_barrier(); } module_init(amdgpu_init); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c index 23d7d0b0d6252..83fad3a371847 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_eviction_fence.c @@ -49,7 +49,6 @@ amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, { struct amdgpu_eviction_fence *old_ef, *new_ef; struct drm_gem_object *obj; - unsigned long index; int ret; if (evf_mgr->ev_fence && @@ -77,7 +76,7 @@ amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, spin_unlock(&evf_mgr->ev_fence_lock); /* Attach the new fence */ - drm_exec_for_each_locked_object(exec, index, obj) { + drm_exec_for_each_locked_object(exec, obj) { struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); if (!bo) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 89a639044d520..c45267373fd86 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -582,7 +582,7 @@ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev) r = -ENODEV; /* no need to trigger GPU reset as we are unloading */ if (r) - amdgpu_fence_driver_force_completion(ring); + amdgpu_fence_driver_force_completion(ring, NULL); if (!drm_dev_is_unplugged(adev_to_drm(adev)) && ring->fence_drv.irq_src && @@ -727,16 +727,34 @@ void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error) * amdgpu_fence_driver_force_completion - force signal latest fence of ring * * @ring: fence of the ring to signal + * @timedout_fence: fence of the timedout job * */ -void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring) +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring, + struct dma_fence *timedout_fence) { - amdgpu_fence_driver_set_error(ring, -ECANCELED); + struct amdgpu_fence_driver *drv = &ring->fence_drv; + unsigned long flags; + + spin_lock_irqsave(&drv->lock, flags); + for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) { + struct dma_fence *fence; + + fence = rcu_dereference_protected(drv->fences[i], + lockdep_is_held(&drv->lock)); + if (fence && !dma_fence_is_signaled_locked(fence)) { + if (fence == timedout_fence) + dma_fence_set_error(fence, -ETIME); + else + dma_fence_set_error(fence, -ECANCELED); + } + } + spin_unlock_irqrestore(&drv->lock, flags); + amdgpu_fence_write(ring, ring->fence_drv.sync_seq); amdgpu_fence_process(ring); } - /* * Kernel queue reset handling * diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c index b0082aa7f3c61..2875627dce8e9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fru_eeprom.c @@ -114,6 +114,43 @@ static bool is_fru_eeprom_supported(struct amdgpu_device *adev, u32 *fru_addr) } } +/* + * IPMI FRU Product Info Area fields are TLV: one type/length byte + * (low 6 bits = data length) followed by that many data bytes. These + * helpers walk the cursor and copy a single field while bounding all + * accesses to the actual buffer length read from the EEPROM. + */ +#define FRU_FIELD_LEN(p, a) ((p)[a] & 0x3F) + +/* Advance cursor past the current TLV. Returns false if no more data. */ +static bool fru_pia_advance(u32 *addr, const unsigned char *pia, int len) +{ + if (*addr >= (u32)len) + return false; + *addr += 1 + FRU_FIELD_LEN(pia, *addr); + return true; +} + +/* + * Copy the current TLV's data into dst (NUL-terminated). Returns false if + * the TLV header or data would read past the end of pia. + */ +static bool fru_pia_copy_field(char *dst, size_t dst_size, + const unsigned char *pia, u32 addr, int len) +{ + size_t fl; + + if (addr + 1 >= (u32)len) + return false; + + fl = min3((size_t)FRU_FIELD_LEN(pia, addr), + dst_size - 1, + (size_t)(len - addr - 1)); + memcpy(dst, pia + addr + 1, fl); + dst[fl] = '\0'; + return true; +} + int amdgpu_fru_get_product_info(struct amdgpu_device *adev) { struct amdgpu_fru_info *fru_info; @@ -222,52 +259,46 @@ int amdgpu_fru_get_product_info(struct amdgpu_device *adev) * Read Manufacturer Name field whose length is [3]. */ addr = 3; - if (addr + 1 >= len) + if (!fru_pia_copy_field(fru_info->manufacturer_name, + sizeof(fru_info->manufacturer_name), + pia, addr, len)) goto Out; - memcpy(fru_info->manufacturer_name, pia + addr + 1, - min_t(size_t, sizeof(fru_info->manufacturer_name), - pia[addr] & 0x3F)); - fru_info->manufacturer_name[sizeof(fru_info->manufacturer_name) - 1] = - '\0'; /* Read Product Name field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->product_name, + sizeof(fru_info->product_name), + pia, addr, len)) goto Out; - memcpy(fru_info->product_name, pia + addr + 1, - min_t(size_t, sizeof(fru_info->product_name), pia[addr] & 0x3F)); - fru_info->product_name[sizeof(fru_info->product_name) - 1] = '\0'; /* Go to the Product Part/Model Number field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->product_number, + sizeof(fru_info->product_number), + pia, addr, len)) goto Out; - memcpy(fru_info->product_number, pia + addr + 1, - min_t(size_t, sizeof(fru_info->product_number), - pia[addr] & 0x3F)); - fru_info->product_number[sizeof(fru_info->product_number) - 1] = '\0'; - /* Go to the Product Version field. */ - addr += 1 + (pia[addr] & 0x3F); + /* Skip the Product Version field. */ + if (!fru_pia_advance(&addr, pia, len)) + goto Out; - /* Go to the Product Serial Number field. */ - addr += 1 + (pia[addr] & 0x3F); - if (addr + 1 >= len) + /* Read the Product Serial Number field. */ + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->serial, + sizeof(fru_info->serial), + pia, addr, len)) goto Out; - memcpy(fru_info->serial, pia + addr + 1, - min_t(size_t, sizeof(fru_info->serial), pia[addr] & 0x3F)); - fru_info->serial[sizeof(fru_info->serial) - 1] = '\0'; - /* Asset Tag field */ - addr += 1 + (pia[addr] & 0x3F); + /* Skip the Asset Tag field. */ + if (!fru_pia_advance(&addr, pia, len)) + goto Out; /* FRU File Id field. This could be 'null'. */ - addr += 1 + (pia[addr] & 0x3F); - if ((addr + 1 >= len) || !(pia[addr] & 0x3F)) + if (!fru_pia_advance(&addr, pia, len) || + !fru_pia_copy_field(fru_info->fru_id, + sizeof(fru_info->fru_id), + pia, addr, len)) goto Out; - memcpy(fru_info->fru_id, pia + addr + 1, - min_t(size_t, sizeof(fru_info->fru_id), pia[addr] & 0x3F)); - fru_info->fru_id[sizeof(fru_info->fru_id) - 1] = '\0'; Out: kfree(pia); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c index cd52ddd27c58b..81c6003b3e794 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c @@ -385,6 +385,25 @@ const struct drm_gem_object_funcs amdgpu_gem_object_funcs = { .vm_ops = &amdgpu_gem_vm_ops, }; +static bool amdgpu_gem_are_domains_valid(u32 domains) +{ + u32 normal = AMDGPU_GEM_DOMAIN_CPU | + AMDGPU_GEM_DOMAIN_GTT | + AMDGPU_GEM_DOMAIN_VRAM; + /* Treat all non CPU/GTT/VRAM domains as special domains. */ + u32 special = AMDGPU_GEM_DOMAIN_MASK & ~normal; + u32 normal_mask = domains & normal; + u32 special_mask = domains & special; + + if (!special_mask) + return true; + + if (normal_mask) + return false; + + return !(special_mask & (special_mask - 1)); +} + /* * GEM ioctls. */ @@ -409,6 +428,8 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, /* reject invalid gem domains */ if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK) return -EINVAL; + if (!amdgpu_gem_are_domains_valid(args->in.domains)) + return -EINVAL; if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) { DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n"); @@ -523,6 +544,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, bo = gem_to_amdgpu_bo(gobj); bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo); r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags); if (r) goto release_object; @@ -817,7 +839,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, struct dma_fence_chain *timeline_chain = NULL; struct dma_fence *fence; struct drm_exec exec; - uint64_t vm_size; + uint64_t vm_size, tmp; int r = 0; /* Validate virtual address range against reserved regions. */ @@ -841,7 +863,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_TOP; - if (args->va_address + args->map_size > vm_size) { + if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) { dev_dbg(dev->dev, "va_address 0x%llx is in top reserved area 0x%llx\n", args->va_address + args->map_size, vm_size); @@ -1075,6 +1097,11 @@ int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, * If that number is larger than the size of the array, the ioctl must * be retried. */ + if (!bo_va) { + r = -ENOENT; + goto out_exec; + } + if (args->num_entries > INT_MAX / sizeof(*vm_entries)) { r = -EINVAL; goto out_exec; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c index ebe2b4c68b0f3..46c0b986db51d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c @@ -934,10 +934,7 @@ int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *r if (r) return r; - if (amdgpu_sriov_vf(adev)) - return r; - - if (adev->gfx.cp_ecc_error_irq.funcs) { + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) { r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); if (r) goto late_fini; @@ -952,6 +949,21 @@ int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *r return r; } +void amdgpu_gfx_ras_suspend(struct amdgpu_device *adev, + struct ras_common_if *ras_block) +{ + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) + amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); +} + +void amdgpu_gfx_ras_fini(struct amdgpu_device *adev, + struct ras_common_if *ras_block) +{ + if (!amdgpu_sriov_vf(adev) && adev->gfx.cp_ecc_error_irq.funcs) + amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); + amdgpu_ras_block_late_fini(adev, ras_block); +} + int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) { int err = 0; @@ -980,6 +992,12 @@ int amdgpu_gfx_ras_sw_init(struct amdgpu_device *adev) if (!ras->ras_block.ras_late_init) ras->ras_block.ras_late_init = amdgpu_gfx_ras_late_init; + if (!ras->ras_block.ras_suspend) + ras->ras_block.ras_suspend = amdgpu_gfx_ras_suspend; + + if (!ras->ras_block.ras_fini) + ras->ras_block.ras_fini = amdgpu_gfx_ras_fini; + /* If not defined special ras_cb function, use default ras_cb */ if (!ras->ras_block.ras_cb) ras->ras_block.ras_cb = amdgpu_gfx_process_ras_data_cb; @@ -1456,12 +1474,13 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) struct amdgpu_device *adev = ring->adev; struct drm_gpu_scheduler *sched = &ring->sched; struct drm_sched_entity entity; + unsigned int ib_size_dw = 16; static atomic_t counter; struct dma_fence *f; struct amdgpu_job *job; struct amdgpu_ib *ib; void *owner; - int i, r; + int r; /* Initialize the scheduler entity */ r = drm_sched_entity_init(&entity, DRM_SCHED_PRIORITY_NORMAL, @@ -1479,7 +1498,7 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) owner = (void *)(unsigned long)atomic_inc_return(&counter); r = amdgpu_job_alloc_with_ib(ring->adev, &entity, owner, - 64, 0, &job, + ib_size_dw * sizeof(uint32_t), 0, &job, AMDGPU_KERNEL_JOB_ID_CLEANER_SHADER); if (r) goto err; @@ -1489,9 +1508,8 @@ static int amdgpu_gfx_run_cleaner_shader_job(struct amdgpu_ring *ring) job->run_cleaner_shader = true; ib = &job->ibs[0]; - for (i = 0; i <= ring->funcs->align_mask; ++i) - ib->ptr[i] = ring->funcs->nop; - ib->length_dw = ring->funcs->align_mask + 1; + memset32(ib->ptr, ring->funcs->nop, ib_size_dw); + ib->length_dw = ib_size_dw; f = amdgpu_job_submit(job); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h index fb5f7a0ee029f..8949037b62a43 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h @@ -603,7 +603,8 @@ void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable); void amdgpu_gfx_off_ctrl_immediate(struct amdgpu_device *adev, bool enable); int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value); int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block); -void amdgpu_gfx_ras_fini(struct amdgpu_device *adev); +void amdgpu_gfx_ras_suspend(struct amdgpu_device *adev, struct ras_common_if *ras_block); +void amdgpu_gfx_ras_fini(struct amdgpu_device *adev, struct ras_common_if *ras_block); int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value); int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *residency); int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c index 0760e70402ec1..52ed0fa000d75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c @@ -217,7 +217,20 @@ static bool amdgpu_gtt_mgr_intersects(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (place->fpfn >= (node->start + num_pages) || + (place->lpfn && place->lpfn <= node->start)) + return false; + + return true; } /** @@ -235,7 +248,20 @@ static bool amdgpu_gtt_mgr_compatible(struct ttm_resource_manager *man, const struct ttm_place *place, size_t size) { - return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res); + const struct drm_mm_node *const node = &to_ttm_range_mgr_node(res)->mm_nodes[0]; + const u32 num_pages = PFN_UP(size); + + if (!place->lpfn) + return true; + + if (!amdgpu_gtt_mgr_has_gart_addr(res)) + return false; + + if (node->start < place->fpfn || + (place->lpfn && (node->start + num_pages) > place->lpfn)) + return false; + + return true; } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c index ccda3b58a4558..f60f26470af03 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c @@ -69,7 +69,6 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, { struct amdgpu_bo *bo = container_of(mni, struct amdgpu_bo, notifier); struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct amdgpu_bo *vm_root = bo->vm_bo->vm->root.bo; long r; if (!mmu_notifier_range_blockable(range)) @@ -80,7 +79,7 @@ static bool amdgpu_hmm_invalidate_gfx(struct mmu_interval_notifier *mni, mmu_interval_set_seq(mni, cur_seq); amdgpu_vm_bo_invalidate(bo, false); - r = dma_resv_wait_timeout(vm_root->tbo.base.resv, + r = dma_resv_wait_timeout(bo->parent->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, false, MAX_SCHEDULE_TIMEOUT); mutex_unlock(&adev->notifier_lock); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c index 9581643dbb726..623a2ae3a10cd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c @@ -208,16 +208,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, dma_fence_put(tmp); } - if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync) - ring->funcs->emit_mem_sync(ring); - - if (ring->funcs->emit_wave_limit && - ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) - ring->funcs->emit_wave_limit(ring, true); - - if (ring->funcs->insert_start) - ring->funcs->insert_start(ring); - if (job) { r = amdgpu_vm_flush(ring, job, need_pipe_sync); if (r) { @@ -228,6 +218,16 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, amdgpu_ring_ib_begin(ring); + if (ring->funcs->insert_start) + ring->funcs->insert_start(ring); + + if ((ib->flags & AMDGPU_IB_FLAG_EMIT_MEM_SYNC) && ring->funcs->emit_mem_sync) + ring->funcs->emit_mem_sync(ring); + + if (ring->funcs->emit_wave_limit && + ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) + ring->funcs->emit_wave_limit(ring, true); + if (ring->funcs->emit_gfx_shadow && adev->gfx.cp_gfx_shadow) amdgpu_ring_emit_gfx_shadow(ring, shadow_va, csa_va, gds_va, init_shadow, vmid); @@ -312,6 +312,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, ring->hw_prio == AMDGPU_GFX_PIPE_PRIO_HIGH) ring->funcs->emit_wave_limit(ring, false); + amdgpu_ring_ib_end(ring); /* Save the wptr associated with this fence. * This must be last for resets to work properly * as we need to save the wptr associated with this @@ -320,7 +321,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned int num_ibs, */ amdgpu_fence_save_wptr(*f); - amdgpu_ring_ib_end(ring); amdgpu_ring_commit(ring); return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c index 8d7f97eed5a90..ccc378233bc5a 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c @@ -512,7 +512,7 @@ void amdgpu_irq_delegate(struct amdgpu_device *adev, unsigned int num_dw) { amdgpu_ih_ring_write(adev, &adev->irq.ih_soft, entry->iv_entry, num_dw); - schedule_work(&adev->irq.ih_soft_work); + queue_work(system_unbound_wq, &adev->irq.ih_soft_work); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c index e08f58de4b17f..cdab2c098cfab 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c @@ -284,10 +284,12 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev, goto error_free; } - r = amdgpu_bo_pin(*bo_ptr, domain); - if (r) { - dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); - goto error_unreserve; + if (free) { + r = amdgpu_bo_pin(*bo_ptr, domain); + if (r) { + dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); + goto error_unreserve; + } } r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); @@ -310,7 +312,8 @@ int amdgpu_bo_create_reserved(struct amdgpu_device *adev, return 0; error_unpin: - amdgpu_bo_unpin(*bo_ptr); + if (free) + amdgpu_bo_unpin(*bo_ptr); error_unreserve: amdgpu_bo_unreserve(*bo_ptr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 5f7aa840b2151..9ca867965e7ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -372,6 +372,12 @@ static bool psp_get_runtime_db_entry(struct amdgpu_device *adev, bool ret = false; int i; + /* + * Runtime DB is for dGPUs only. + */ + if (adev->flags & AMD_IS_APU) + return false; + if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14)) @@ -832,7 +838,11 @@ static int psp_load_toc(struct psp_context *psp, struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp); /* Copy toc to psp firmware private buffer */ - psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + ret = psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes); @@ -1159,8 +1169,11 @@ static int psp_rl_load(struct amdgpu_device *adev) cmd = acquire_psp_cmd_buf(psp); - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes); + ret = psp_copy_fw(psp, psp->rl.start_addr, psp->rl.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW; cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr); @@ -1383,8 +1396,12 @@ int psp_ta_load(struct psp_context *psp, struct ta_context *context) cmd = acquire_psp_cmd_buf(psp); - psp_copy_fw(psp, context->bin_desc.start_addr, - context->bin_desc.size_bytes); + ret = psp_copy_fw(psp, context->bin_desc.start_addr, + context->bin_desc.size_bytes); + if (ret) { + release_psp_cmd_buf(psp); + return ret; + } if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) && context->mem_context.shared_bo) @@ -4154,17 +4171,24 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev, return count; } -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size) { int idx; if (!drm_dev_enter(adev_to_drm(psp->adev), &idx)) - return; + return -ENODEV; + + if (!bin_size || bin_size > PSP_1_MEG) { + dev_err(psp->adev->dev, "PSP firmware is invalid\n"); + drm_dev_exit(idx); + return -EINVAL; + } memset(psp->fw_pri_buf, 0, PSP_1_MEG); memcpy(psp->fw_pri_buf, start_addr, bin_size); drm_dev_exit(idx); + return 0; } /** @@ -4272,7 +4296,7 @@ static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj, */ static const struct bin_attribute psp_vbflash_bin_attr = { .attr = {.name = "psp_vbflash", .mode = 0660}, - .size = 0, + .size = AMD_VBIOS_FILE_MAX_SIZE_B, .write = amdgpu_psp_vbflash_write, .read = amdgpu_psp_vbflash_read, }; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h index 237b624aa51ca..c3a5940e311aa 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h @@ -605,7 +605,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp, int psp_update_fw_reservation(struct psp_context *psp); int psp_load_fw_list(struct psp_context *psp, struct amdgpu_firmware_info **ucode_list, int ucode_count); -void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); +int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size); int psp_spatial_partition(struct psp_context *psp, int mode); int psp_memory_partition(struct psp_context *psp, int mode); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c index e9ed1a3b135cc..16ae44e131ad4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c @@ -92,6 +92,9 @@ struct amdgpu_ras_block_list { struct list_head node; struct amdgpu_ras_block_object *ras_obj; + + /* set by ras_late_init, cleared by ras_suspend/ras_fini */ + bool active; }; const char *get_ras_block_str(struct ras_common_if *ras_block) @@ -2884,6 +2887,25 @@ static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, return -EINVAL; } +static bool __check_record_in_range(struct amdgpu_device *adev, + struct eeprom_table_record *bps, int count) +{ + int i; + + for (i = 0; i < count; i++) { + if (bps[i].retired_page >= + (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, + "Recorded address out of range: 0x%llx, 0x%llx, 0x%x, 0x%x\n", + bps[i].address, bps[i].retired_page, + bps[i].mem_channel, bps[i].mcumc_id); + return false; + } + } + + return true; +} + static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct eeprom_table_record *bps, int count) { @@ -2891,6 +2913,9 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev, struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct ras_err_handler_data *data = con->eh_data; + if (!__check_record_in_range(adev, bps, count)) + return 0; + for (j = 0; j < count; j++) { if (!data->space_left && amdgpu_ras_realloc_eh_data_space(adev, data, 256)) { @@ -4370,10 +4395,23 @@ void amdgpu_ras_resume(struct amdgpu_device *adev) void amdgpu_ras_suspend(struct amdgpu_device *adev) { struct amdgpu_ras *con = amdgpu_ras_get_context(adev); + struct amdgpu_ras_block_list *node; + struct amdgpu_ras_block_object *obj; if (!adev->ras_enabled || !con) return; + /* run per-block ras_suspend before tearing down the RAS context */ + list_for_each_entry(node, &adev->ras_list, node) { + if (!node->active) + continue; + + obj = node->ras_obj; + if (obj && obj->ras_suspend) + obj->ras_suspend(adev, &obj->ras_comm); + node->active = false; + } + amdgpu_ras_disable_all_features(adev, 0); /* Make sure all ras objects are disabled. */ if (AMDGPU_RAS_GET_FEATURES(con->features)) @@ -4427,8 +4465,15 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev) obj->ras_comm.name, r); return r; } - } else - amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); + } else { + r = amdgpu_ras_block_late_init_default(adev, &obj->ras_comm); + if (r) { + dev_err(adev->dev, "%s failed to execute ras_block_late_init_default! ret:%d\n", + obj->ras_comm.name, r); + return r; + } + } + node->active = true; } return 0; @@ -4465,11 +4510,12 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) { if (ras_node->ras_obj) { obj = ras_node->ras_obj; - if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) && - obj->ras_fini) + /* fall back to default cleanup if ras_suspend already ran */ + if (ras_node->active && obj->ras_fini) obj->ras_fini(adev, &obj->ras_comm); else amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm); + ras_node->active = false; } /* Clear ras blocks from ras_list and free ras block list node */ @@ -5370,6 +5416,11 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn) uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT; int ret = 0; + if (pfn >= (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) { + dev_warn(adev->dev, "Ignoring out-of-range bad page 0x%llx", start); + return 0; + } + if (amdgpu_ras_check_critical_address(adev, start)) return 0; @@ -5416,7 +5467,7 @@ int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, struct amdgpu_ras *con = amdgpu_ras_get_context(adev); struct amdgpu_vram_mgr_resource *vres; struct ras_critical_region *region; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; int ret = 0; if (!bo || !bo->tbo.resource) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h index 6cf0dfd38be8b..8160c4d598543 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h @@ -731,6 +731,7 @@ struct amdgpu_ras_block_object { int (*ras_block_match)(struct amdgpu_ras_block_object *block_obj, enum amdgpu_ras_block block, uint32_t sub_block_index); int (*ras_late_init)(struct amdgpu_device *adev, struct ras_common_if *ras_block); + void (*ras_suspend)(struct amdgpu_device *adev, struct ras_common_if *ras_block); void (*ras_fini)(struct amdgpu_device *adev, struct ras_common_if *ras_block); ras_ih_cb ras_cb; const struct amdgpu_ras_block_hw_ops *hw_ops; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c index dafa46a9656ca..51382d604b1f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c @@ -142,12 +142,15 @@ #define RAS_RI_TO_AI(_C, _I) (((_I) + (_C)->ras_fri) % \ (_C)->ras_max_record_count) -#define RAS_NUM_RECS(_tbl_hdr) (((_tbl_hdr)->tbl_size - \ - RAS_TABLE_HEADER_SIZE) / RAS_TABLE_RECORD_SIZE) +#define RAS_NUM_RECS(_tbl_hdr) \ + (((_tbl_hdr)->tbl_size < RAS_TABLE_HEADER_SIZE) ? 0u : \ + (((_tbl_hdr)->tbl_size - RAS_TABLE_HEADER_SIZE) / RAS_TABLE_RECORD_SIZE)) -#define RAS_NUM_RECS_V2_1(_tbl_hdr) (((_tbl_hdr)->tbl_size - \ - RAS_TABLE_HEADER_SIZE - \ - RAS_TABLE_V2_1_INFO_SIZE) / RAS_TABLE_RECORD_SIZE) +#define RAS_NUM_RECS_V2_1(_tbl_hdr) \ + (((_tbl_hdr)->tbl_size < RAS_TABLE_HEADER_SIZE + \ + RAS_TABLE_V2_1_INFO_SIZE) ? 0u : \ + (((_tbl_hdr)->tbl_size - RAS_TABLE_HEADER_SIZE - \ + RAS_TABLE_V2_1_INFO_SIZE) / RAS_TABLE_RECORD_SIZE)) #define to_amdgpu_device(x) ((container_of(x, struct amdgpu_ras, eeprom_control))->adev) @@ -1415,11 +1418,24 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) switch (hdr->version) { case RAS_TABLE_VER_V2_1: case RAS_TABLE_VER_V3: + if (hdr->tbl_size < RAS_TABLE_HEADER_SIZE + RAS_TABLE_V2_1_INFO_SIZE) { + dev_err(adev->dev, + "RAS header invalid, tbl_size %u smaller than minimum %u, resetting table\n", + hdr->tbl_size, + RAS_TABLE_HEADER_SIZE + RAS_TABLE_V2_1_INFO_SIZE); + return amdgpu_ras_eeprom_reset_table(control); + } control->ras_num_recs = RAS_NUM_RECS_V2_1(hdr); control->ras_record_offset = RAS_RECORD_START_V2_1; control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1; break; case RAS_TABLE_VER_V1: + if (hdr->tbl_size < RAS_TABLE_HEADER_SIZE) { + dev_err(adev->dev, + "RAS header invalid, tbl_size %u smaller than minimum %u, resetting table\n", + hdr->tbl_size, RAS_TABLE_HEADER_SIZE); + return amdgpu_ras_eeprom_reset_table(control); + } control->ras_num_recs = RAS_NUM_RECS(hdr); control->ras_record_offset = RAS_RECORD_START; control->ras_max_record_count = RAS_MAX_RECORD_COUNT; @@ -1439,6 +1455,14 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) } control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset); + if (hdr->first_rec_offset < control->ras_record_offset || + control->ras_fri >= control->ras_max_record_count) { + dev_err(adev->dev, + "RAS header invalid, ras_fri: %u, first_rec_offset:0x%x", + control->ras_fri, hdr->first_rec_offset); + return -EINVAL; + } + control->ras_num_mca_recs = 0; control->ras_num_pa_recs = 0; return 0; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h index be2e56ce1355e..8908d9e08a304 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_res_cursor.h @@ -55,7 +55,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res, uint64_t start, uint64_t size, struct amdgpu_res_cursor *cur) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct list_head *head, *next; struct drm_mm_node *node; @@ -71,7 +71,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res, head = &to_amdgpu_vram_mgr_resource(res)->blocks; block = list_first_entry_or_null(head, - struct drm_buddy_block, + struct gpu_buddy_block, link); if (!block) goto fallback; @@ -81,7 +81,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res, next = block->link.next; if (next != head) - block = list_entry(next, struct drm_buddy_block, link); + block = list_entry(next, struct gpu_buddy_block, link); } cur->start = amdgpu_vram_mgr_block_start(block) + start; @@ -125,7 +125,7 @@ static inline void amdgpu_res_first(struct ttm_resource *res, */ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct drm_mm_node *node; struct list_head *next; @@ -146,7 +146,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) block = cur->node; next = block->link.next; - block = list_entry(next, struct drm_buddy_block, link); + block = list_entry(next, struct gpu_buddy_block, link); cur->node = block; cur->start = amdgpu_vram_mgr_block_start(block); @@ -175,7 +175,7 @@ static inline void amdgpu_res_next(struct amdgpu_res_cursor *cur, uint64_t size) */ static inline bool amdgpu_res_cleared(struct amdgpu_res_cursor *cur) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; switch (cur->mem_type) { case TTM_PL_VRAM: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c index 304564ec2f59a..431cc39ea0178 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c @@ -498,8 +498,9 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, size_t size, loff_t *pos) { struct amdgpu_ring *ring = file_inode(f)->i_private; - uint32_t value, result, early[3]; + u32 value, result, early[3] = { 0 }; uint64_t p; + u32 avail_dw, start_dw, read_dw; loff_t i; int r; @@ -508,10 +509,10 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, result = 0; - if (*pos < 12) { - if (ring->funcs->type == AMDGPU_RING_TYPE_CPER) - mutex_lock(&ring->adev->cper.ring_lock); + if (ring->funcs->type == AMDGPU_RING_TYPE_CPER) + mutex_lock(&ring->adev->cper.ring_lock); + if (*pos < 12) { early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask; early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask; early[2] = ring->wptr & ring->buf_mask; @@ -543,13 +544,24 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, *pos += 4; } } else { + early[0] = amdgpu_ring_get_rptr(ring) & ring->buf_mask; + early[1] = amdgpu_ring_get_wptr(ring) & ring->buf_mask; + p = early[0]; if (early[0] <= early[1]) - size = (early[1] - early[0]); + avail_dw = early[1] - early[0]; else - size = ring->ring_size - (early[0] - early[1]); + avail_dw = ring->buf_mask + 1 - (early[0] - early[1]); - while (size) { + start_dw = (*pos > 12) ? ((*pos - 12) >> 2) : 0; + if (start_dw >= avail_dw) + goto out; + + p = (p + start_dw) & ring->ptr_mask; + avail_dw -= start_dw; + read_dw = min_t(u32, avail_dw, size >> 2); + + while (read_dw) { if (p == early[1]) goto out; @@ -562,9 +574,10 @@ static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf, buf += 4; result += 4; - size--; + read_dw--; p++; p &= ring->ptr_mask; + *pos += 4; } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h index 0098b1eadfc87..2024aa6d231d8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h @@ -154,7 +154,8 @@ extern const struct drm_sched_backend_ops amdgpu_sched_ops; void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring); void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error); -void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring); +void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring, + struct dma_fence *timedout_fence); void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *af); void amdgpu_fence_save_wptr(struct dma_fence *fence); @@ -516,6 +517,17 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, ring->count_dw -= count_dw; } +static inline unsigned int amdgpu_ring_get_dw_distance(struct amdgpu_ring *ring, + u64 start_wptr, u64 end_wptr) +{ + unsigned int start = start_wptr & ring->buf_mask; + unsigned int end = end_wptr & ring->buf_mask; + + if (end < start) + end += ring->ring_size >> 2; + return end - start; +} + /** * amdgpu_ring_patch_cond_exec - patch dw count of conditional execute * @ring: amdgpu_ring structure @@ -526,18 +538,13 @@ static inline void amdgpu_ring_write_multiple(struct amdgpu_ring *ring, static inline void amdgpu_ring_patch_cond_exec(struct amdgpu_ring *ring, unsigned int offset) { - unsigned cur; - if (!ring->funcs->init_cond_exec) return; WARN_ON(offset > ring->buf_mask); WARN_ON(ring->ring[offset] != 0); - cur = (ring->wptr - 1) & ring->buf_mask; - if (cur < offset) - cur += ring->ring_size >> 2; - ring->ring[offset] = cur - offset; + ring->ring[offset] = amdgpu_ring_get_dw_distance(ring, offset, ring->wptr - 1); } int amdgpu_ring_test_helper(struct amdgpu_ring *ring); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c index 321310ba2c08e..fcd81242059e6 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c @@ -600,10 +600,10 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, * to be submitted to the queues after the reset is complete. */ if (!ret) { - amdgpu_fence_driver_force_completion(gfx_ring); + amdgpu_fence_driver_force_completion(gfx_ring, NULL); drm_sched_wqueue_start(&gfx_ring->sched); if (adev->sdma.has_page_queue) { - amdgpu_fence_driver_force_completion(page_ring); + amdgpu_fence_driver_force_completion(page_ring, NULL); drm_sched_wqueue_start(&page_ring->sched); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c index a0b479d5fff19..f4be192235889 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c @@ -175,11 +175,14 @@ int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *va, { unsigned long bit_pos; - bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem); - if (bit_pos >= adev->seq64.num_sem) - return -ENOSPC; + for (;;) { + bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem); + if (bit_pos >= adev->seq64.num_sem) + return -ENOSPC; - __set_bit(bit_pos, adev->seq64.used); + if (!test_and_set_bit(bit_pos, adev->seq64.used)) + break; + } *va = bit_pos * sizeof(u64) + amdgpu_seq64_get_va_base(adev); @@ -205,7 +208,7 @@ void amdgpu_seq64_free(struct amdgpu_device *adev, u64 va) bit_pos = (va - amdgpu_seq64_get_va_base(adev)) / sizeof(u64); if (bit_pos < adev->seq64.num_sem) - __clear_bit(bit_pos, adev->seq64.used); + clear_bit(bit_pos, adev->seq64.used); } /** diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index d629c5f73bf59..c48c266cc7a75 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -505,6 +505,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, if (new_mem->mem_type == TTM_PL_TT || new_mem->mem_type == AMDGPU_PL_PREEMPT) { + if (old_mem && (old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT)) { + r = ttm_bo_wait_ctx(bo, ctx); + if (r) + return r; + + amdgpu_ttm_backend_unbind(bo->bdev, bo->ttm); + } + r = amdgpu_ttm_backend_bind(bo->bdev, bo->ttm, new_mem); if (r) return r; @@ -539,6 +548,15 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict, ttm_bo_assign_mem(bo, new_mem); return 0; } + if ((old_mem->mem_type == TTM_PL_TT || + old_mem->mem_type == AMDGPU_PL_PREEMPT) && + (new_mem->mem_type == TTM_PL_TT || + new_mem->mem_type == AMDGPU_PL_PREEMPT)) { + amdgpu_bo_move_notify(bo, evict, new_mem); + ttm_resource_free(bo, &bo->resource); + ttm_bo_assign_mem(bo, new_mem); + return 0; + } if (old_mem->mem_type == AMDGPU_PL_GDS || old_mem->mem_type == AMDGPU_PL_GWS || @@ -1966,7 +1984,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) if (adev->gmc.xgmi.connected_to_cpu) adev->mman.aper_base_kaddr = ioremap_cache(adev->gmc.aper_base, adev->gmc.visible_vram_size); - else if (adev->gmc.is_app_apu) DRM_DEBUG_DRIVER( "No need to ioremap when real vram size is 0\n"); @@ -2055,6 +2072,18 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) gtt_size = configured_size; } + /* Cap GTT so that it does not exceed total physical RAM. */ + if (adev->flags & AMD_IS_APU) { + u64 phys_ram = (u64)totalram_pages() << PAGE_SHIFT; + + if (gtt_size > phys_ram) { + gtt_size = phys_ram; + dev_info(adev->dev, + "Capping GTT to %uM to not exceed available system memory\n", + (unsigned int)(gtt_size / (1024 * 1024))); + } + } + /* Initialize GTT memory pool */ r = amdgpu_gtt_mgr_init(adev, gtt_size); if (r) { @@ -2134,8 +2163,6 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) */ void amdgpu_ttm_fini(struct amdgpu_device *adev) { - int idx; - if (!adev->mman.initialized) return; @@ -2162,13 +2189,9 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev) amdgpu_ttm_fw_reserve_vram_fini(adev); amdgpu_ttm_drv_reserve_vram_fini(adev); - if (drm_dev_enter(adev_to_drm(adev), &idx)) { - - if (adev->mman.aper_base_kaddr) - iounmap(adev->mman.aper_base_kaddr); + if (adev->mman.aper_base_kaddr) { + iounmap(adev->mman.aper_base_kaddr); adev->mman.aper_base_kaddr = NULL; - - drm_dev_exit(idx); } if (!adev->gmc.is_app_apu) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c index 7e3175f82a20d..0f4281c9aea2f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c @@ -270,16 +270,20 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, goto free_obj; } + r = amdgpu_bo_pin(userq_obj->obj, AMDGPU_GEM_DOMAIN_GTT); + if (r) + goto unresv; + r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo); if (r) { drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r); - goto unresv; + goto unpin_bo; } r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr); if (r) { drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r); - goto unresv; + goto unpin_bo; } userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj); @@ -287,11 +291,13 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr, memset(userq_obj->cpu_ptr, 0, size); return 0; +unpin_bo: + amdgpu_bo_unpin(userq_obj->obj); unresv: amdgpu_bo_unreserve(userq_obj->obj); - free_obj: amdgpu_bo_unref(&userq_obj->obj); + return r; } @@ -299,6 +305,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_userq_obj *userq_obj) { amdgpu_bo_kunmap(userq_obj->obj); + amdgpu_bo_unpin(userq_obj->obj); amdgpu_bo_unref(&userq_obj->obj); } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 5c38f0d30c87a..5f8c036773552 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -135,7 +135,7 @@ MODULE_FIRMWARE(FIRMWARE_VEGA12); MODULE_FIRMWARE(FIRMWARE_VEGA20); static void amdgpu_uvd_idle_work_handler(struct work_struct *work); -static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo); +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *abo); static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, uint32_t size, @@ -158,7 +158,7 @@ static int amdgpu_uvd_create_msg_bo_helper(struct amdgpu_device *adev, amdgpu_bo_kunmap(bo); amdgpu_bo_unpin(bo); amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM); - amdgpu_uvd_force_into_uvd_segment(bo); + amdgpu_uvd_force_into_vcpu_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (r) goto err; @@ -188,6 +188,7 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) const struct common_firmware_header *hdr; unsigned int family_id; int i, j, r; + u32 vcpu_bo_domain; INIT_DELAYED_WORK(&adev->uvd.idle_work, amdgpu_uvd_idle_work_handler); @@ -319,12 +320,20 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); + /* UVD 5.0 and newer HW can use 64 bit addressing. */ + adev->uvd.address_64_bit = + !amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0); + + vcpu_bo_domain = AMDGPU_GEM_DOMAIN_VRAM; + if (adev->uvd.address_64_bit) + vcpu_bo_domain |= AMDGPU_GEM_DOMAIN_GTT; + for (j = 0; j < adev->uvd.num_uvd_inst; j++) { if (adev->uvd.harvest_config & (1 << j)) continue; + r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, - AMDGPU_GEM_DOMAIN_VRAM | - AMDGPU_GEM_DOMAIN_GTT, + vcpu_bo_domain, &adev->uvd.inst[j].vcpu_bo, &adev->uvd.inst[j].gpu_addr, &adev->uvd.inst[j].cpu_addr); @@ -339,10 +348,6 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) adev->uvd.filp[i] = NULL; } - /* from uvd v5.0 HW addressing capacity increased to 64 bits */ - if (!amdgpu_device_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0)) - adev->uvd.address_64_bit = true; - r = amdgpu_uvd_create_msg_bo_helper(adev, 128 << 10, &adev->uvd.ib_bo); if (r) return r; @@ -511,7 +516,8 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) } memset_io(ptr, 0, size); /* to restore uvd fence seq */ - amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring); + if (adev->uvd.inst[i].ring.fence_drv.initialized) + amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL); } } return 0; @@ -544,6 +550,24 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp) } } +static void amdgpu_uvd_force_into_vcpu_segment(struct amdgpu_bo *bo) +{ + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); + struct amdgpu_bo *vcpu_bo = adev->uvd.inst[0].vcpu_bo; + struct amdgpu_res_cursor vcpu_cur; + + amdgpu_res_first(vcpu_bo->tbo.resource, 0, + amdgpu_bo_size(vcpu_bo), &vcpu_cur); + + bo->placement.num_placement = 1; + bo->placement.placement = &bo->placements[0]; + bo->placements[0].fpfn = ALIGN_DOWN(vcpu_cur.start, SZ_256M) >> PAGE_SHIFT; + bo->placements[0].lpfn = bo->placements[0].fpfn + (SZ_256M >> PAGE_SHIFT); + bo->placements[0].mem_type = vcpu_bo->tbo.resource->mem_type; + if (bo->placements[0].mem_type == TTM_PL_VRAM) + bo->placements[0].flags |= TTM_PL_FLAG_CONTIGUOUS; +} + static void amdgpu_uvd_force_into_uvd_segment(struct amdgpu_bo *abo) { int i; @@ -594,13 +618,10 @@ static int amdgpu_uvd_cs_pass1(struct amdgpu_uvd_cs_ctx *ctx) if (!ctx->parser->adev->uvd.address_64_bit) { /* check if it's a message or feedback command */ cmd = amdgpu_ib_get_value(ctx->ib, ctx->idx) >> 1; - if (cmd == 0x0 || cmd == 0x3) { - /* yes, force it into VRAM */ - uint32_t domain = AMDGPU_GEM_DOMAIN_VRAM; - - amdgpu_bo_placement_from_domain(bo, domain); - } - amdgpu_uvd_force_into_uvd_segment(bo); + if (cmd == 0x0 || cmd == 0x3) + amdgpu_uvd_force_into_vcpu_segment(bo); + else + amdgpu_uvd_force_into_uvd_segment(bo); r = ttm_bo_validate(&bo->tbo, &bo->placement, &tctx); } @@ -625,50 +646,30 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, unsigned int height = msg[7]; unsigned int dpb_size = msg[9]; unsigned int pitch = msg[28]; - unsigned int level = msg[57]; unsigned int width_in_mb = width / 16; unsigned int height_in_mb = ALIGN(height / 16, 2); - unsigned int fs_in_mb = width_in_mb * height_in_mb; unsigned int image_size, tmp, min_dpb_size, num_dpb_buffer; unsigned int min_ctx_size = ~0; + /* Reject invalid dimensions */ + if (width < 16 || height < 16 || width > 4096 || height > 4096) { + dev_WARN_ONCE(adev->dev, 1, + "Invalid UVD decoding dimensions (%dx%d)!\n", + width, height); + return -EINVAL; + } + image_size = width * height; image_size += image_size / 2; image_size = ALIGN(image_size, 1024); switch (stream_type) { case 0: /* H264 */ - switch (level) { - case 30: - num_dpb_buffer = 8100 / fs_in_mb; - break; - case 31: - num_dpb_buffer = 18000 / fs_in_mb; - break; - case 32: - num_dpb_buffer = 20480 / fs_in_mb; - break; - case 41: - num_dpb_buffer = 32768 / fs_in_mb; - break; - case 42: - num_dpb_buffer = 34816 / fs_in_mb; - break; - case 50: - num_dpb_buffer = 110400 / fs_in_mb; - break; - case 51: - num_dpb_buffer = 184320 / fs_in_mb; - break; - default: - num_dpb_buffer = 184320 / fs_in_mb; - break; - } - num_dpb_buffer++; + num_dpb_buffer = ((msg[61] >> 16) & 0xff) + 1; if (num_dpb_buffer > 17) - num_dpb_buffer = 17; + return -EINVAL; /* reference picture buffer */ min_dpb_size = image_size * num_dpb_buffer; @@ -718,35 +719,9 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, break; case 7: /* H264 Perf */ - switch (level) { - case 30: - num_dpb_buffer = 8100 / fs_in_mb; - break; - case 31: - num_dpb_buffer = 18000 / fs_in_mb; - break; - case 32: - num_dpb_buffer = 20480 / fs_in_mb; - break; - case 41: - num_dpb_buffer = 32768 / fs_in_mb; - break; - case 42: - num_dpb_buffer = 34816 / fs_in_mb; - break; - case 50: - num_dpb_buffer = 110400 / fs_in_mb; - break; - case 51: - num_dpb_buffer = 184320 / fs_in_mb; - break; - default: - num_dpb_buffer = 184320 / fs_in_mb; - break; - } - num_dpb_buffer++; + num_dpb_buffer = ((msg[61] >> 16) & 0xff) + 1; if (num_dpb_buffer > 17) - num_dpb_buffer = 17; + return -EINVAL; /* reference picture buffer */ min_dpb_size = image_size * num_dpb_buffer; @@ -774,6 +749,9 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, image_size = ALIGN(image_size, 256); num_dpb_buffer = (le32_to_cpu(msg[59]) & 0xff) + 2; + if (num_dpb_buffer > 17) + return -EINVAL; + min_dpb_size = image_size * num_dpb_buffer; min_ctx_size = ((width + 255) / 16) * ((height + 255) / 16) * 16 * num_dpb_buffer + 52 * 1024; @@ -784,7 +762,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, return -EINVAL; } - if (width > pitch) { + if (width > pitch || pitch > 4096) { DRM_ERROR("Invalid UVD decoding target pitch!\n"); return -EINVAL; } @@ -796,7 +774,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg, } buf_sizes[0x1] = dpb_size; - buf_sizes[0x2] = image_size; + buf_sizes[0x2] = (pitch * height) * 3 / 2; buf_sizes[0x4] = min_ctx_size; /* store image width to adjust nb memory pstate */ adev->uvd.decode_image_width = width; @@ -943,15 +921,16 @@ static int amdgpu_uvd_cs_pass2(struct amdgpu_uvd_cs_ctx *ctx) ctx->buf_sizes[cmd]); return -EINVAL; } + } else if (cmd == 0x204 || cmd == 0x206) { + unsigned int min_size = ctx->buf_sizes[cmd == 0x204 ? 5 : 4]; - } else if (cmd == 0x206) { - if ((end - start) < ctx->buf_sizes[4]) { + if ((end - start) < min_size) { DRM_ERROR("buffer (%d) to small (%d / %d)!\n", cmd, (unsigned int)(end - start), - ctx->buf_sizes[4]); + min_size); return -EINVAL; } - } else if ((cmd != 0x100) && (cmd != 0x204)) { + } else if ((cmd != 0x100)) { DRM_ERROR("invalid UVD command %X!\n", cmd); return -EINVAL; } @@ -1081,11 +1060,12 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, { struct amdgpu_uvd_cs_ctx ctx = {}; unsigned int buf_sizes[] = { - [0x00000000] = 2048, + [0x00000000] = 3556, [0x00000001] = 0xFFFFFFFF, [0x00000002] = 0xFFFFFFFF, [0x00000003] = 2048, [0x00000004] = 0xFFFFFFFF, + [0x00000005] = 992, }; int r; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index 3b9a9010ed354..3dc2b7f91e288 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -853,9 +853,20 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, goto out; } - *size = amdgpu_ib_get_value(ib, idx + 8) * - amdgpu_ib_get_value(ib, idx + 10) * - 8 * 3 / 2; + uint32_t width, height; + width = amdgpu_ib_get_value(ib, idx + 8); + height = amdgpu_ib_get_value(ib, idx + 10); + + if (width == 0 || height == 0 || + width > 4096 || height > 2304) { + DRM_ERROR("invalid VCE image size: %ux%u\n", + width, height); + r = -EINVAL; + goto out; + } + + *size = width * height * 8 * 3 / 2; + break; case 0x04000001: /* config extension */ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 5e0786ea911b3..2679f32715c0b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -1480,15 +1480,16 @@ int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block, /** * amdgpu_vcn_reset_engine - Reset a specific VCN engine - * @adev: Pointer to the AMDGPU device - * @instance_id: VCN engine instance to reset + * @ring: Pointer to the VCN ring + * @timedout_fence: fence that timed out * * Returns: 0 on success, or a negative error code on failure. */ -static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev, - uint32_t instance_id) +static int amdgpu_vcn_reset_engine(struct amdgpu_ring *ring, + struct amdgpu_fence *timedout_fence) { - struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[instance_id]; + struct amdgpu_device *adev = ring->adev; + struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; int r, i; mutex_lock(&vinst->engine_reset_mutex); @@ -1512,9 +1513,13 @@ static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev, if (r) goto unlock; } - amdgpu_fence_driver_force_completion(&vinst->ring_dec); + amdgpu_fence_driver_force_completion(&vinst->ring_dec, + (&vinst->ring_dec == ring) ? + &timedout_fence->base : NULL); for (i = 0; i < vinst->num_enc_rings; i++) - amdgpu_fence_driver_force_completion(&vinst->ring_enc[i]); + amdgpu_fence_driver_force_completion(&vinst->ring_enc[i], + (&vinst->ring_enc[i] == ring) ? + &timedout_fence->base : NULL); /* Restart the scheduler's work queue for the dec and enc rings * if they were stopped by this function. This allows new tasks @@ -1550,7 +1555,7 @@ int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring, if (adev->vcn.inst[ring->me].using_unified_queue) return -EINVAL; - return amdgpu_vcn_reset_engine(adev, ring->me); + return amdgpu_vcn_reset_engine(ring, timedout_fence); } int amdgpu_vcn_reg_dump_init(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index ffbae605caaa5..6aae94ca7dcb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -768,7 +768,9 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync) { struct amdgpu_device *adev = ring->adev; - struct amdgpu_isolation *isolation = &adev->isolation[ring->xcp_id]; + struct amdgpu_isolation *isolation = + &adev->isolation[ring->xcp_id == AMDGPU_XCP_NO_PARTITION ? + 0 : ring->xcp_id]; unsigned vmhub = ring->vm_hub; struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; @@ -812,6 +814,17 @@ int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, return 0; amdgpu_ring_ib_begin(ring); + + /* There is no matching insert_end for this on purpose for the vm flush. + * The IB portion of the submission has both. Having multiple + * insert_start sequences is ok, but you can only have one insert_end + * per submission based on the way VCN FW works. For JPEG + * you can as many insert_start and insert_end sequences as you like as + * long as the rest of the packets come between start and end sequences. + */ + if (ring->funcs->insert_start) + ring->funcs->insert_start(ring); + if (ring->funcs->init_cond_exec) patch = amdgpu_ring_init_cond_exec(ring, ring->cond_exe_gpu_addr); @@ -2052,7 +2065,7 @@ int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, after->start = eaddr + 1; after->last = tmp->last; after->offset = tmp->offset; - after->offset += (after->start - tmp->start) << PAGE_SHIFT; + after->offset += (after->start - tmp->start) << AMDGPU_GPU_PAGE_SHIFT; after->flags = tmp->flags; after->bo_va = tmp->bo_va; list_add(&after->list, &tmp->bo_va->invalids); @@ -2453,19 +2466,6 @@ static void amdgpu_vm_destroy_task_info(struct kref *kref) kfree(ti); } -static inline struct amdgpu_vm * -amdgpu_vm_get_vm_from_pasid(struct amdgpu_device *adev, u32 pasid) -{ - struct amdgpu_vm *vm; - unsigned long flags; - - xa_lock_irqsave(&adev->vm_manager.pasids, flags); - vm = xa_load(&adev->vm_manager.pasids, pasid); - xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); - - return vm; -} - /** * amdgpu_vm_put_task_info - reference down the vm task_info ptr * @@ -2512,8 +2512,16 @@ amdgpu_vm_get_task_info_vm(struct amdgpu_vm *vm) struct amdgpu_task_info * amdgpu_vm_get_task_info_pasid(struct amdgpu_device *adev, u32 pasid) { - return amdgpu_vm_get_task_info_vm( - amdgpu_vm_get_vm_from_pasid(adev, pasid)); + struct amdgpu_task_info *ti; + struct amdgpu_vm *vm; + unsigned long flags; + + xa_lock_irqsave(&adev->vm_manager.pasids, flags); + vm = xa_load(&adev->vm_manager.pasids, pasid); + ti = amdgpu_vm_get_task_info_vm(vm); + xa_unlock_irqrestore(&adev->vm_manager.pasids, flags); + + return ti; } static int amdgpu_vm_create_task_info(struct amdgpu_vm *vm) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 9d934c07fa6b8..e51e058443ece 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "amdgpu.h" #include "amdgpu_vm.h" @@ -52,15 +53,15 @@ to_amdgpu_device(struct amdgpu_vram_mgr *mgr) return container_of(mgr, struct amdgpu_device, mman.vram_mgr); } -static inline struct drm_buddy_block * +static inline struct gpu_buddy_block * amdgpu_vram_mgr_first_block(struct list_head *list) { - return list_first_entry_or_null(list, struct drm_buddy_block, link); + return list_first_entry_or_null(list, struct gpu_buddy_block, link); } static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; u64 start, size; block = amdgpu_vram_mgr_first_block(head); @@ -71,7 +72,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) start = amdgpu_vram_mgr_block_start(block); size = amdgpu_vram_mgr_block_size(block); - block = list_entry(block->link.next, struct drm_buddy_block, link); + block = list_entry(block->link.next, struct gpu_buddy_block, link); if (start + size != amdgpu_vram_mgr_block_start(block)) return false; } @@ -81,7 +82,7 @@ static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; u64 size = 0; list_for_each_entry(block, head, link) @@ -254,7 +255,7 @@ const struct attribute_group amdgpu_vram_mgr_attr_group = { * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM */ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, - struct drm_buddy_block *block) + struct gpu_buddy_block *block) { u64 start = amdgpu_vram_mgr_block_start(block); u64 end = start + amdgpu_vram_mgr_block_size(block); @@ -279,7 +280,7 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); struct ttm_resource *res = bo->tbo.resource; struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); - struct drm_buddy_block *block; + struct gpu_buddy_block *block; u64 usage = 0; if (amdgpu_gmc_vram_full_visible(&adev->gmc)) @@ -299,15 +300,15 @@ static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; struct amdgpu_vram_reservation *rsv, *temp; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; uint64_t vis_usage; list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { - if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, + if (gpu_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, rsv->size, mm->chunk_size, &rsv->allocated, - DRM_BUDDY_RANGE_ALLOCATION)) + GPU_BUDDY_RANGE_ALLOCATION)) continue; block = amdgpu_vram_mgr_first_block(&rsv->allocated); @@ -403,7 +404,7 @@ int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr, uint64_t address, struct amdgpu_vram_block_info *info) { struct amdgpu_vram_mgr_resource *vres; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; u64 start, size; int ret = -ENOENT; @@ -450,8 +451,8 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, struct amdgpu_vram_mgr_resource *vres; u64 size, remaining_size, lpfn, fpfn; unsigned int adjust_dcc_size = 0; - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; unsigned long pages_per_block; int r; @@ -493,17 +494,17 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, INIT_LIST_HEAD(&vres->blocks); if (place->flags & TTM_PL_FLAG_TOPDOWN) - vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) - vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION; + vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED) - vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION; + vres->flags |= GPU_BUDDY_CLEAR_ALLOCATION; if (fpfn || lpfn != mgr->mm.size) /* Allocate blocks in desired range */ - vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; + vres->flags |= GPU_BUDDY_RANGE_ALLOCATION; if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC && adev->gmc.gmc_funcs->get_dcc_alignment) @@ -516,7 +517,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size); remaining_size = (u64)dcc_size; - vres->flags |= DRM_BUDDY_TRIM_DISABLE; + vres->flags |= GPU_BUDDY_TRIM_DISABLE; } mutex_lock(&mgr->lock); @@ -536,7 +537,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, BUG_ON(min_block_size < mm->chunk_size); - r = drm_buddy_alloc_blocks(mm, fpfn, + r = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, size, min_block_size, @@ -545,7 +546,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul && !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) { - vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION; + vres->flags &= ~GPU_BUDDY_CONTIGUOUS_ALLOCATION; pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT), tbo->page_alignment); @@ -566,7 +567,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, list_add_tail(&vres->vres_node, &mgr->allocated_vres_list); if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) { - struct drm_buddy_block *dcc_block; + struct gpu_buddy_block *dcc_block; unsigned long dcc_start; u64 trim_start; @@ -576,7 +577,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block), adjust_dcc_size); trim_start = (u64)dcc_start; - drm_buddy_block_trim(mm, &trim_start, + gpu_buddy_block_trim(mm, &trim_start, (u64)vres->base.size, &vres->blocks); } @@ -614,7 +615,7 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, return 0; error_free_blocks: - drm_buddy_free_list(mm, &vres->blocks, 0); + gpu_buddy_free_list(mm, &vres->blocks, 0); mutex_unlock(&mgr->lock); error_fini: ttm_resource_fini(man, &vres->base); @@ -637,8 +638,8 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); struct amdgpu_device *adev = to_amdgpu_device(mgr); - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; uint64_t vis_usage = 0; mutex_lock(&mgr->lock); @@ -649,7 +650,7 @@ static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, list_for_each_entry(block, &vres->blocks, link) vis_usage += amdgpu_vram_mgr_vis_size(adev, block); - drm_buddy_free_list(mm, &vres->blocks, vres->flags); + gpu_buddy_free_list(mm, &vres->blocks, vres->flags); amdgpu_vram_mgr_do_reserve(man); mutex_unlock(&mgr->lock); @@ -688,7 +689,7 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, if (!*sgt) return -ENOMEM; - /* Determine the number of DRM_BUDDY blocks to export */ + /* Determine the number of GPU_BUDDY blocks to export */ amdgpu_res_first(res, offset, length, &cursor); while (cursor.remaining) { num_entries++; @@ -704,10 +705,10 @@ int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, sg->length = 0; /* - * Walk down DRM_BUDDY blocks to populate scatterlist nodes - * @note: Use iterator api to get first the DRM_BUDDY block + * Walk down GPU_BUDDY blocks to populate scatterlist nodes + * @note: Use iterator api to get first the GPU_BUDDY block * and the number of bytes from it. Access the following - * DRM_BUDDY block(s) if more buffer needs to exported + * GPU_BUDDY block(s) if more buffer needs to exported */ amdgpu_res_first(res, offset, length, &cursor); for_each_sgtable_sg((*sgt), sg, i) { @@ -792,10 +793,10 @@ uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) void amdgpu_vram_mgr_clear_reset_blocks(struct amdgpu_device *adev) { struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; mutex_lock(&mgr->lock); - drm_buddy_reset_clear(mm, false); + gpu_buddy_reset_clear(mm, false); mutex_unlock(&mgr->lock); } @@ -815,7 +816,7 @@ static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, size_t size) { struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); - struct drm_buddy_block *block; + struct gpu_buddy_block *block; /* Check each drm buddy block individually */ list_for_each_entry(block, &mgr->blocks, link) { @@ -848,7 +849,7 @@ static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, size_t size) { struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); - struct drm_buddy_block *block; + struct gpu_buddy_block *block; /* Check each drm buddy block individually */ list_for_each_entry(block, &mgr->blocks, link) { @@ -877,7 +878,7 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; struct amdgpu_vram_reservation *rsv; drm_printf(printer, " vis usage:%llu\n", @@ -917,9 +918,6 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) struct ttm_resource_manager *man = &mgr->manager; int err; - man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); - if (IS_ERR(man->cg)) - return PTR_ERR(man->cg); ttm_resource_manager_init(man, &adev->mman.bdev, adev->gmc.real_vram_size); @@ -930,10 +928,14 @@ int amdgpu_vram_mgr_init(struct amdgpu_device *adev) mgr->default_page_size = PAGE_SIZE; man->func = &amdgpu_vram_mgr_func; - err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); + err = gpu_buddy_init(&mgr->mm, man->size, PAGE_SIZE); if (err) return err; + man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); + ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); ttm_resource_manager_set_used(man, true); return 0; @@ -965,11 +967,11 @@ void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) kfree(rsv); list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { - drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0); + gpu_buddy_free_list(&mgr->mm, &rsv->allocated, 0); kfree(rsv); } if (!adev->gmc.is_app_apu) - drm_buddy_fini(&mgr->mm); + gpu_buddy_fini(&mgr->mm); mutex_unlock(&mgr->lock); ttm_resource_manager_cleanup(man); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h index 5f5fd9a911c26..429a21a2e9b23 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.h @@ -24,11 +24,11 @@ #ifndef __AMDGPU_VRAM_MGR_H__ #define __AMDGPU_VRAM_MGR_H__ -#include +#include struct amdgpu_vram_mgr { struct ttm_resource_manager manager; - struct drm_buddy mm; + struct gpu_buddy mm; /* protects access to buffer objects */ struct mutex lock; struct list_head reservations_pending; @@ -57,19 +57,19 @@ struct amdgpu_vram_mgr_resource { struct amdgpu_vres_task task; }; -static inline u64 amdgpu_vram_mgr_block_start(struct drm_buddy_block *block) +static inline u64 amdgpu_vram_mgr_block_start(struct gpu_buddy_block *block) { - return drm_buddy_block_offset(block); + return gpu_buddy_block_offset(block); } -static inline u64 amdgpu_vram_mgr_block_size(struct drm_buddy_block *block) +static inline u64 amdgpu_vram_mgr_block_size(struct gpu_buddy_block *block) { - return (u64)PAGE_SIZE << drm_buddy_block_order(block); + return (u64)PAGE_SIZE << gpu_buddy_block_order(block); } -static inline bool amdgpu_vram_mgr_is_cleared(struct drm_buddy_block *block) +static inline bool amdgpu_vram_mgr_is_cleared(struct gpu_buddy_block *block) { - return drm_buddy_block_is_clear(block); + return gpu_buddy_block_is_clear(block); } static inline struct amdgpu_vram_mgr_resource * @@ -82,8 +82,8 @@ static inline void amdgpu_vram_mgr_set_cleared(struct ttm_resource *res) { struct amdgpu_vram_mgr_resource *ares = to_amdgpu_vram_mgr_resource(res); - WARN_ON(ares->flags & DRM_BUDDY_CLEARED); - ares->flags |= DRM_BUDDY_CLEARED; + WARN_ON(ares->flags & GPU_BUDDY_CLEARED); + ares->flags |= GPU_BUDDY_CLEARED; } int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr, diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c index 7a063e44d4298..639649ef6dd03 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.c +++ b/drivers/gpu/drm/amd/amdgpu/atom.c @@ -59,6 +59,9 @@ #define ATOM_CMD_TIMEOUT_SEC 20 +/* Limit ATOM command table recursion (calltable) to avoid kernel stack overflow. */ +#define ATOM_EXECUTE_MAX_DEPTH 32 + typedef struct { struct atom_context *ctx; uint32_t *ps, *ws; @@ -1229,6 +1232,13 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, if (!base) return -EINVAL; + if (ctx->execute_depth >= ATOM_EXECUTE_MAX_DEPTH) { + DRM_ERROR("atombios command table nesting exceeded limit (%u)\n", + ATOM_EXECUTE_MAX_DEPTH); + return -ELOOP; + } + ctx->execute_depth++; + len = CU16(base + ATOM_CT_SIZE_PTR); ws = CU8(base + ATOM_CT_WS_PTR); ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK; @@ -1285,6 +1295,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index, free: if (ws) kfree(ectx.ws); + ctx->execute_depth--; return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h index 825ff28731f52..bb3d9eb7eb6b5 100644 --- a/drivers/gpu/drm/amd/amdgpu/atom.h +++ b/drivers/gpu/drm/amd/amdgpu/atom.h @@ -153,6 +153,9 @@ struct atom_context { uint8_t vbios_ver_str[STRLEN_NORMAL]; uint8_t date[STRLEN_NORMAL]; uint8_t build_num[STRLEN_NORMAL]; + + /* Nesting depth for ATOM_OP_CALLTABLE */ + unsigned int execute_depth; }; extern int amdgpu_atom_debug; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c index 234753a10361f..b8cc213b347cc 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c @@ -4022,7 +4022,7 @@ static void gfx_v10_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -5349,6 +5349,15 @@ static void gfx_v10_0_constants_init(struct amdgpu_device *adev) gfx_v10_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = gfx_v10_0_init_pa_sc_tile_steering_override(adev); + /* Program DB_RING_CONTROL for multiple GFX pipes + * Default power up value is 1. + * Possible values: + * 0 - split occlusion counters between gfx pipes + * 1 - all occlusion counters to pipe 0 + * 2 - all occlusion counters to pipe 1 + */ + WREG32_FIELD15(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, + (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ @@ -8665,7 +8674,7 @@ static void gfx_v10_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8700,7 +8709,7 @@ static void gfx_v10_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -8733,9 +8742,9 @@ static void gfx_v10_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -8783,9 +8792,6 @@ static void gfx_v10_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c index cf23b5da6dbbb..a78e8f98f4d00 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c @@ -537,7 +537,7 @@ static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -1616,10 +1616,10 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(11, 0, 2): case IP_VERSION(11, 0, 3): if (!adev->gfx.disable_uq && - adev->gfx.me_fw_version >= 2420 && - adev->gfx.pfp_fw_version >= 2580 && - adev->gfx.mec_fw_version >= 2650 && - adev->mes.fw_version[0] >= 120) { + adev->gfx.me_fw_version >= 3090 && + adev->gfx.pfp_fw_version >= 3190 && + adev->gfx.mec_fw_version >= 3450 && + adev->mes.fw_version[0] >= 147) { adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; } @@ -5901,7 +5901,7 @@ static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5936,7 +5936,7 @@ static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5969,9 +5969,9 @@ static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -6025,9 +6025,6 @@ static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c index 50e39b9d9df6f..5358ee33914e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c @@ -439,7 +439,7 @@ static void gfx_v12_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -1421,10 +1421,10 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block) case IP_VERSION(12, 0, 0): case IP_VERSION(12, 0, 1): if (!adev->gfx.disable_uq && - adev->gfx.me_fw_version >= 2780 && - adev->gfx.pfp_fw_version >= 2840 && - adev->gfx.mec_fw_version >= 3050 && - adev->mes.fw_version[0] >= 123) { + adev->gfx.me_fw_version >= 3090 && + adev->gfx.pfp_fw_version >= 3190 && + adev->gfx.mec_fw_version >= 3450 && + adev->mes.fw_version[0] >= 147) { adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; } @@ -1810,6 +1810,11 @@ static void gfx_v12_0_constants_init(struct amdgpu_device *adev) gfx_v12_0_get_tcc_info(adev); adev->gfx.config.pa_sc_tile_steering_override = 0; + /* Set whether texture coordinate truncation is conformant. */ + tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); + adev->gfx.config.ta_cntl2_truncate_coord_mode = + REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); + /* XXX SH_MEM regs */ /* where to put LDS, scratch, GPUVM in FSA64 space */ mutex_lock(&adev->srbm_mutex); @@ -4424,7 +4429,7 @@ static void gfx_v12_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, control |= ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4443,7 +4448,7 @@ static void gfx_v12_0_ring_emit_ib_compute(struct amdgpu_ring *ring, u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -4474,9 +4479,9 @@ static void gfx_v12_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -4524,9 +4529,6 @@ static void gfx_v12_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c index 066cdf6863e11..ef399fce3ef3e 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c @@ -1881,11 +1881,13 @@ static int gfx_v6_0_ring_test_ring(struct amdgpu_ring *ring) return r; } -static void gfx_v6_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) +static void gfx_v6_0_ring_emit_event_write(struct amdgpu_ring *ring, + uint32_t event_type, + uint32_t event_index) { amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); - amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | - EVENT_INDEX(0)); + amdgpu_ring_write(ring, EVENT_TYPE(event_type) | + EVENT_INDEX(event_index)); } static void gfx_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, @@ -1924,12 +1926,6 @@ static void gfx_v6_0_ring_emit_ib(struct amdgpu_ring *ring, unsigned vmid = AMDGPU_JOB_GET_VMID(job); u32 header, control = 0; - /* insert SWITCH_BUFFER packet before first IB in the ring frame */ - if (flags & AMDGPU_HAVE_CTX_SWITCH) { - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } - if (ib->flags & AMDGPU_IB_FLAG_CE) header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); else @@ -2351,7 +2347,7 @@ static int gfx_v6_0_cp_resume(struct amdgpu_device *adev) static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; uint32_t seq = ring->fence_drv.sync_seq; uint64_t addr = ring->fence_drv.gpu_addr; @@ -2364,20 +2360,12 @@ static void gfx_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) amdgpu_ring_write(ring, seq); amdgpu_ring_write(ring, 0xffffffff); amdgpu_ring_write(ring, 4); /* poll interval */ - - if (usepfp) { - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - } } static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); @@ -2395,19 +2383,13 @@ static void gfx_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring, /* sync PFP to ME, otherwise we might get invalid PFP reads */ amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); amdgpu_ring_write(ring, 0x0); - - /* synce CE with ME to prevent CE fetch CEIB before context switch done */ - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); - amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); - amdgpu_ring_write(ring, 0); } } static void gfx_v6_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, uint32_t val) { - int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); + int usepfp = 1; amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) | @@ -2996,12 +2978,30 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev) return clock; } +static void gfx_v6_0_ring_emit_sb(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); + amdgpu_ring_write(ring, 0); +} + static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) { - if (flags & AMDGPU_HAVE_CTX_SWITCH) - gfx_v6_0_ring_emit_vgt_flush(ring); + u32 dw2 = 0x80000000; /* set load_enable otherwise this package is just NOPs */ + + if (flags & AMDGPU_HAVE_CTX_SWITCH) { + gfx_v6_0_ring_emit_event_write(ring, VS_PARTIAL_FLUSH, 4); + gfx_v6_0_ring_emit_event_write(ring, VGT_FLUSH, 0); + + /* set load_global_config (load_global_uconfig doesn't exist on GFX6) */ + dw2 |= 0x1; + /* set load_cs_sh_regs */ + dw2 |= 0x01000000; + /* set load_per_context_state & load_gfx_sh_regs */ + dw2 |= 0x10002; + } + amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); - amdgpu_ring_write(ring, 0x80000000); + amdgpu_ring_write(ring, dw2); amdgpu_ring_write(ring, 0); } @@ -3527,11 +3527,12 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ - 7 + 4 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 6 + /* gfx_v6_0_ring_emit_vm_flush */ - 3 + 2 + /* gfx_v6_ring_emit_cntxcntl including vgt flush */ + 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ + 3 + 2 + 2 + /* gfx_v6_ring_emit_cntxcntl including VGT flush */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3539,6 +3540,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_gfx = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_cntxcntl = gfx_v6_ring_emit_cntxcntl, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, @@ -3554,10 +3556,11 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .emit_frame_size = 5 + 5 + /* hdp flush / invalidate */ 7 + /* gfx_v6_0_ring_emit_pipeline_sync */ - SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v6_0_ring_emit_vm_flush */ + SI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + 2 + /* gfx_v6_0_ring_emit_vm_flush */ 14 + 14 + 14 + /* gfx_v6_0_ring_emit_fence x3 for user fence, vm fence */ + 3 * 2 + /* gfx_v6_0_ring_emit_sb x3 (from amdgpu_vm_flush, amdgpu_ib_schedule) */ 5, /* SURFACE_SYNC */ - .emit_ib_size = 6, /* gfx_v6_0_ring_emit_ib */ + .emit_ib_size = 4, /* gfx_v6_0_ring_emit_ib */ .emit_ib = gfx_v6_0_ring_emit_ib, .emit_fence = gfx_v6_0_ring_emit_fence, .emit_pipeline_sync = gfx_v6_0_ring_emit_pipeline_sync, @@ -3565,6 +3568,7 @@ static const struct amdgpu_ring_funcs gfx_v6_0_ring_funcs_compute = { .test_ring = gfx_v6_0_ring_test_ring, .test_ib = gfx_v6_0_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .emit_switch_buffer = gfx_v6_0_ring_emit_sb, .emit_wreg = gfx_v6_0_ring_emit_wreg, .emit_mem_sync = gfx_v6_0_emit_mem_sync, }; diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c index 8a81713d97aac..690128ad3094a 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c @@ -6226,9 +6226,6 @@ static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, u64 seq, unsigned int flags) { - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c index f589457f60187..9d7214bcaadb9 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c @@ -1183,7 +1183,7 @@ static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -4045,12 +4045,45 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) return r; } +static void gfx_v9_0_deactivate_kcq_hqd(struct amdgpu_device *adev) +{ + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); + for (int i = 0; i < adev->gfx.num_compute_rings; i++) { + u32 tmp; + struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; + + mutex_lock(&adev->srbm_mutex); + soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + /* disable the queue if it's active */ + if (tmp & CP_HQD_ACTIVE__ACTIVE_MASK) { + int j; + + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); + for (j = 0; j < adev->usec_timeout; j++) { + tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); + if (!(tmp & CP_HQD_ACTIVE__ACTIVE_MASK)) + break; + udelay(1); + } + if (j == AMDGPU_MAX_USEC_TIMEOUT) { + DRM_DEBUG("comp_%u_%u_%u dequeue request failed.\n", + ring->me, ring->pipe, ring->queue); + /* Manual disable if dequeue request times out */ + WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0); + } + WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0); + } + soc15_grbm_select(adev, 0, 0, 0, 0, 0); + mutex_unlock(&adev->srbm_mutex); + } + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); +} + static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) - amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); @@ -4071,6 +4104,10 @@ static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) return 0; } + if ((adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev) && + amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_MODE2) + gfx_v9_0_deactivate_kcq_hqd(adev); + /* Use deinitialize sequence from CAIL when unbinding device from driver, * otherwise KIQ is hanging when binding back */ @@ -5434,7 +5471,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, header); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5530,7 +5567,7 @@ static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -5571,9 +5608,9 @@ static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c index a4ebb6c5af557..314916bd2368f 100644 --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c @@ -405,7 +405,7 @@ static void gfx_v9_4_3_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, WAIT_REG_MEM_ENGINE(eng_sel))); if (mem_space) - BUG_ON(addr0 & 0x3); /* Dword align */ + WARN_ON(addr0 & 0x3); /* Dword align */ amdgpu_ring_write(ring, addr0); amdgpu_ring_write(ring, addr1); amdgpu_ring_write(ring, ref); @@ -2871,7 +2871,7 @@ static void gfx_v9_4_3_ring_emit_ib_compute(struct amdgpu_ring *ring, } amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); - BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ + WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ amdgpu_ring_write(ring, #ifdef __BIG_ENDIAN (2 << 0) | @@ -2905,9 +2905,9 @@ static void gfx_v9_4_3_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, * aligned if only send 32bit data low (discard data high) */ if (write64bit) - BUG_ON(addr & 0x7); + WARN_ON(addr & 0x7); else - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -2967,9 +2967,6 @@ static void gfx_v9_4_3_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, { struct amdgpu_device *adev = ring->adev; - /* we only allocate 32bit for each seq wb address */ - BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); - /* write fence seq to the "addr" */ amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c index 99e6dabc36ee8..4521465506682 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c @@ -1010,7 +1010,7 @@ void jpeg_v4_0_3_dec_ring_nop(struct amdgpu_ring *ring, uint32_t count) static bool jpeg_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c index a43582b9c876c..3f61d5367d946 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_5.c @@ -772,15 +772,28 @@ static int jpeg_v4_0_5_ring_reset(struct amdgpu_ring *ring, unsigned int vmid, struct amdgpu_fence *timedout_fence) { + struct amdgpu_device *adev = ring->adev; + u32 pg_flags = adev->pg_flags; int r; amdgpu_ring_reset_helper_begin(ring, timedout_fence); - r = jpeg_v4_0_5_stop(ring->adev); - if (r) - return r; - r = jpeg_v4_0_5_start(ring->adev); + + /* + * The DPG stop path only clears the JPEG_PG_MODE bit and never resets a + * hung JRBC, so the post-reset ring test times out and the driver falls + * back to a full MODE1 reset. Temporarily force the static power-gating + * path so the stop/start sequence actually power-cycles the JPEG block + * (JMI soft reset + static power off/on), matching the working jpeg_v4_0 + * reset. + */ + adev->pg_flags &= ~AMD_PG_SUPPORT_JPEG_DPG; + r = jpeg_v4_0_5_stop(adev); + if (!r) + r = jpeg_v4_0_5_start(adev); + adev->pg_flags = pg_flags; if (r) return r; + return amdgpu_ring_reset_helper_end(ring, timedout_fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c index 72a4b2d0676fa..c696a0dea32ce 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_0.c @@ -648,15 +648,28 @@ static int jpeg_v5_0_0_ring_reset(struct amdgpu_ring *ring, unsigned int vmid, struct amdgpu_fence *timedout_fence) { + struct amdgpu_device *adev = ring->adev; + u32 pg_flags = adev->pg_flags; int r; amdgpu_ring_reset_helper_begin(ring, timedout_fence); - r = jpeg_v5_0_0_stop(ring->adev); - if (r) - return r; - r = jpeg_v5_0_0_start(ring->adev); + + /* + * The DPG stop path only clears the JPEG_PG_MODE bit and never resets a + * hung JRBC, so the post-reset ring test times out and the driver falls + * back to a full MODE1 reset. Temporarily force the static power-gating + * path so the stop/start sequence actually power-cycles the JPEG block + * (JMI soft reset + ONO1 power off/on), matching the working jpeg_v4_0 + * reset. + */ + adev->pg_flags &= ~AMD_PG_SUPPORT_JPEG_DPG; + r = jpeg_v5_0_0_stop(adev); + if (!r) + r = jpeg_v5_0_0_start(adev); + adev->pg_flags = pg_flags; if (r) return r; + return amdgpu_ring_reset_helper_end(ring, timedout_fence); } diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c index 1fb2386352ccb..de9ca229479e8 100644 --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v5_0_1.c @@ -657,7 +657,7 @@ static void jpeg_v5_0_1_dec_ring_set_wptr(struct amdgpu_ring *ring) static bool jpeg_v5_0_1_is_idle(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; - bool ret = false; + bool ret = true; int i, j; for (i = 0; i < adev->jpeg.num_jpeg_inst; ++i) { diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c index 9b4025c39e440..13c7f09fb9511 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c @@ -315,7 +315,6 @@ static u32 nbif_v6_3_1_get_rom_offset(struct amdgpu_device *adev) static void nbif_v6_3_1_program_ltr(struct amdgpu_device *adev) { uint32_t def, data; - u16 devctl2; def = RREG32_SOC15(NBIO, 0, regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL); data = 0x35EB; @@ -329,15 +328,8 @@ static void nbif_v6_3_1_program_ltr(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP2, data); - pcie_capability_read_word(adev->pdev, PCI_EXP_DEVCTL2, &devctl2); - - if (adev->pdev->ltr_path == (devctl2 & PCI_EXP_DEVCTL2_LTR_EN)) - return; - - if (adev->pdev->ltr_path) - pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_LTR_EN); - else - pcie_capability_clear_word(adev->pdev, PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_LTR_EN); + pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); } #endif @@ -345,7 +337,7 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) { #ifdef CONFIG_PCIEASPM uint32_t def, data; - u16 devctl2, ltr; + u16 ltr; def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL); data &= ~PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK; @@ -375,11 +367,8 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP5, data); - pcie_capability_read_word(adev->pdev, PCI_EXP_DEVCTL2, &devctl2); - data = def = devctl2; - data &= ~PCI_EXP_DEVCTL2_LTR_EN; - if (def != data) - pcie_capability_set_word(adev->pdev, PCI_EXP_DEVCTL2, (u16)data); + pcie_capability_clear_word(adev->pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_LTR_EN); ltr = pci_find_ext_capability(adev->pdev, PCI_EXT_CAP_ID_LTR); @@ -387,15 +376,13 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) pci_write_config_dword(adev->pdev, ltr + PCI_LTR_MAX_SNOOP_LAT, 0x10011001); } -#if 0 - /* regPSWUSP0_PCIE_LC_CNTL2 should be replace by PCIE_LC_CNTL2 or someone else ? */ - def = data = RREG32_SOC15(NBIO, 0, regPSWUSP0_PCIE_LC_CNTL2); - data |= PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | - PSWUSP0_PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; - data &= ~PSWUSP0_PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; + def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL2); + data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | + PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK; + data &= ~PCIE_LC_CNTL2__LC_RCV_L0_TO_RCV_L0S_DIS_MASK; if (def != data) - WREG32_SOC15(NBIO, 0, regPSWUSP0_PCIE_LC_CNTL2, data); -#endif + WREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL2, data); + def = data = RREG32_SOC15(PCIE, 0, regPCIE_LC_CNTL4); data |= PCIE_LC_CNTL4__LC_L1_POWERDOWN_MASK; if (def != data) @@ -406,7 +393,12 @@ static void nbif_v6_3_1_program_aspm(struct amdgpu_device *adev) if (def != data) WREG32_SOC15(PCIE, 0, regPCIE_LC_RXRECOVER_RXSTANDBY_CNTL, data); - nbif_v6_3_1_program_ltr(adev); + /* + * Do not enable endpoint LTR unless the Root Complex and every + * upstream switch support it. + */ + if (adev->pdev->ltr_path) + nbif_v6_3_1_program_ltr(adev); def = data = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_BIF_STRAP3); data |= 0x5DE0 << RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT; diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c index 27d883fda5fa9..6f131f4b81134 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c @@ -217,7 +217,9 @@ static int psp_v11_0_bootloader_load_component(struct psp_context *psp, return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -263,7 +265,9 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c index 4c6450d62299a..80ba57cce3916 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c @@ -87,7 +87,9 @@ static int psp_v12_0_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -123,7 +125,9 @@ static int psp_v12_0_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c index af4a7d7c4abd8..8100930e47eb1 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c @@ -265,10 +265,9 @@ static int psp_v13_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -347,10 +346,9 @@ static int psp_v13_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c index 5f39a2edcc956..3d5e26b3fa00a 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c @@ -105,10 +105,9 @@ static int psp_v13_0_4_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, @@ -168,10 +167,9 @@ static int psp_v13_0_4_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c index 38dfc5c19f2a7..040a61aefa866 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c @@ -140,10 +140,9 @@ static int psp_v14_0_bootloader_load_component(struct psp_context *psp, if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy PSP KDB binary to memory */ - memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes); + ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes); + if (ret) + return ret; /* Provide the PSP KDB to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, @@ -214,10 +213,9 @@ static int psp_v14_0_bootloader_load_sos(struct psp_context *psp) if (ret) return ret; - memset(psp->fw_pri_buf, 0, PSP_1_MEG); - - /* Copy Secure OS binary to PSP memory */ - memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c index 833830bc3e2e3..409f097f4c524 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c @@ -96,7 +96,9 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp) return ret; /* Copy PSP System Driver binary to memory */ - psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes); + if (ret) + return ret; /* Provide the sys driver to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, @@ -135,7 +137,9 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp) return ret; /* Copy Secure OS binary to PSP memory */ - psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes); + if (ret) + return ret; /* Provide the PSP secure OS to bootloader */ WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36, diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c index a1443990d5c60..c8b4fda590168 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c @@ -458,7 +458,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 /* write the fence */ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -468,7 +468,7 @@ static void sdma_v4_4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 addr += 4; amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c index 45e2933214a80..e1d9cea98a2d6 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c @@ -528,7 +528,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -539,7 +539,7 @@ static void sdma_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c index 5b982cc91af39..48c672c1b74d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c @@ -378,7 +378,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -389,7 +389,7 @@ static void sdma_v5_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c index 6809c6d4be5b1..ef411bd02237f 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c @@ -360,7 +360,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -371,7 +371,7 @@ static void sdma_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index 2b81344dcd668..d1318fb8060f9 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -364,7 +364,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* Ucached(UC) */ /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, lower_32_bits(seq)); @@ -375,7 +375,7 @@ static void sdma_v7_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 se amdgpu_ring_write(ring, SDMA_PKT_COPY_LINEAR_HEADER_OP(SDMA_OP_FENCE) | SDMA_PKT_FENCE_HEADER_MTYPE(0x3)); /* zero in first two bits */ - BUG_ON(addr & 0x3); + WARN_ON(addr & 0x3); amdgpu_ring_write(ring, lower_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(addr)); amdgpu_ring_write(ring, upper_32_bits(seq)); diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c index 972b449ab89fa..999d9da6756db 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc24.c +++ b/drivers/gpu/drm/amd/amdgpu/soc24.c @@ -526,8 +526,36 @@ static int soc24_common_suspend(struct amdgpu_ip_block *ip_block) return soc24_common_hw_fini(ip_block); } +static bool soc24_need_reset_on_resume(struct amdgpu_device *adev) +{ + u32 sol_reg1, sol_reg2; + + /* Will reset for the following suspend abort cases. + * 1) Only reset dGPU side. + * 2) S3 suspend got aborted and TOS is active. + * As for dGPU suspend abort cases the SOL value + * will be kept as zero at this resume point. + */ + if (!(adev->flags & AMD_IS_APU) && adev->in_s3) { + sol_reg1 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + msleep(100); + sol_reg2 = RREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_81); + + return (sol_reg1 != sol_reg2); + } + + return false; +} + static int soc24_common_resume(struct amdgpu_ip_block *ip_block) { + struct amdgpu_device *adev = ip_block->adev; + + if (soc24_need_reset_on_resume(adev)) { + dev_info(adev->dev, "S3 suspend aborted, resetting..."); + soc24_asic_reset(adev); + } + return soc24_common_hw_init(ip_block); } diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index 708123899c416..b54359efac06f 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -870,6 +870,23 @@ static void vce_v3_0_ring_emit_ib(struct amdgpu_ring *ring, amdgpu_ring_write(ring, ib->length_dw); } +static void vce_v3_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, + u64 seq, unsigned flags) +{ + WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT); + + amdgpu_ring_write(ring, VCE_CMD_FENCE); + amdgpu_ring_write(ring, addr); + amdgpu_ring_write(ring, upper_32_bits(addr)); + amdgpu_ring_write(ring, seq); + amdgpu_ring_write(ring, VCE_CMD_TRAP); +} + +static void vce_v3_0_ring_insert_end(struct amdgpu_ring *ring) +{ + amdgpu_ring_write(ring, VCE_CMD_END); +} + static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring, unsigned int vmid, uint64_t pd_addr) { @@ -879,7 +896,6 @@ static void vce_v3_0_emit_vm_flush(struct amdgpu_ring *ring, amdgpu_ring_write(ring, VCE_CMD_FLUSH_TLB); amdgpu_ring_write(ring, vmid); - amdgpu_ring_write(ring, VCE_CMD_END); } static void vce_v3_0_emit_pipeline_sync(struct amdgpu_ring *ring) @@ -948,17 +964,19 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_vm_funcs = { .set_wptr = vce_v3_0_ring_set_wptr, .patch_cs_in_place = amdgpu_vce_ring_parse_cs_vm, .emit_frame_size = - 6 + /* vce_v3_0_emit_vm_flush */ + 5 + /* vce_v3_0_emit_vm_flush */ 4 + /* vce_v3_0_emit_pipeline_sync */ - 6 + 6, /* amdgpu_vce_ring_emit_fence x2 vm fence */ + 5 + 5 + /* vce_v3_0_ring_emit_fence x2 vm fence */ + 1, /* vce_v3_0_ring_insert_end */ .emit_ib_size = 5, /* vce_v3_0_ring_emit_ib */ .emit_ib = vce_v3_0_ring_emit_ib, .emit_vm_flush = vce_v3_0_emit_vm_flush, .emit_pipeline_sync = vce_v3_0_emit_pipeline_sync, - .emit_fence = amdgpu_vce_ring_emit_fence, + .emit_fence = vce_v3_0_ring_emit_fence, .test_ring = amdgpu_vce_ring_test_ring, .test_ib = amdgpu_vce_ring_test_ib, .insert_nop = amdgpu_ring_insert_nop, + .insert_end = vce_v3_0_ring_insert_end, .pad_ib = amdgpu_ring_generic_pad_ib, .begin_use = amdgpu_vce_ring_begin_use, .end_use = amdgpu_vce_ring_end_use, diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c index f773f7ddfd13c..c042ae868c970 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c @@ -1963,9 +1963,13 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, len_dw = msg[1] / 4; num_buffers = msg[2]; - /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ - if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { - DRM_ERROR("VCN message has too many buffers!\n"); + /* Verify that all indices fit within the claimed length. + * There are 6 dwords in the header before the first buffer. + * Each buffer has 4 dwords. Any trailing dwords after the + * last buffer are ignored. + */ + if (len_dw < 6 || num_buffers > (len_dw - 6) / 4) { + DRM_ERROR("Invalid VCN message!\n"); r = -EINVAL; goto out; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c index 21e40eb2078d1..633c563ed5136 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c @@ -1879,9 +1879,13 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, len_dw = msg[1] / 4; num_buffers = msg[2]; - /* Verify that all indices fit within the claimed length. Each index is 4 DWORDs */ - if (num_buffers > len_dw || 6 + num_buffers * 4 > len_dw) { - DRM_ERROR("VCN message has too many buffers!\n"); + /* Verify that all indices fit within the claimed length. + * There are 6 dwords in the header before the first buffer. + * Each buffer has 4 dwords. Any trailing dwords after the + * last buffer are ignored. + */ + if (len_dw < 6 || num_buffers > (len_dw - 6) / 4) { + DRM_ERROR("Invalid VCN message!\n"); r = -EINVAL; goto out; } @@ -1926,14 +1930,17 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job, #define RENCODE_IB_PARAM_SESSION_INIT 0x00000003 /* return the offset in ib if id is found, -1 otherwise */ -static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start) +static int vcn_v4_0_enc_find_ib_param(struct amdgpu_ib *ib, uint32_t id, int start, uint32_t *length) { int i; uint32_t len; for (i = start; (len = amdgpu_ib_get_value(ib, i)) >= 8; i += len / 4) { - if (amdgpu_ib_get_value(ib, i + 1) == id) + if (amdgpu_ib_get_value(ib, i + 1) == id) { + if (length) + *length = len; return i; + } } return -1; } @@ -1943,14 +1950,14 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, struct amdgpu_ib *ib) { struct amdgpu_ring *ring = amdgpu_job_ring(job); - uint32_t val; + uint32_t val, len; int idx = 0, sidx; /* The first instance can decode anything */ if (!ring->me) return 0; - while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx)) >= 0) { + while ((idx = vcn_v4_0_enc_find_ib_param(ib, RADEON_VCN_ENGINE_INFO, idx, &len)) >= 0) { val = amdgpu_ib_get_value(ib, idx + 2); /* RADEON_VCN_ENGINE_TYPE */ if (val == RADEON_VCN_ENGINE_TYPE_DECODE) { uint32_t valid_buf_flag = amdgpu_ib_get_value(ib, idx + 6); @@ -1963,12 +1970,12 @@ static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p, amdgpu_ib_get_value(ib, idx + 8); return vcn_v4_0_dec_msg(p, job, msg_buffer_addr); } else if (val == RADEON_VCN_ENGINE_TYPE_ENCODE) { - sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx); + sidx = vcn_v4_0_enc_find_ib_param(ib, RENCODE_IB_PARAM_SESSION_INIT, idx, NULL); if (sidx >= 0 && amdgpu_ib_get_value(ib, sidx + 2) == RENCODE_ENCODE_STANDARD_AV1) return vcn_v4_0_limit_sched(p, job); } - idx += amdgpu_ib_get_value(ib, idx) / 4; + idx += len / 4; } return 0; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index c2100fd78b522..04c2d412fa7da 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1162,7 +1162,8 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, if (flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM) size >>= 1; - atomic64_add(PAGE_ALIGN(size), &pdd->vram_usage); + size = PAGE_ALIGN(size); + atomic64_add(size, &pdd->vram_usage); } mutex_unlock(&p->mutex); @@ -1524,16 +1525,10 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep, if (!dev) return -EINVAL; - if (args->metadata_ptr) { - metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL); - if (!metadata_buffer) - return -ENOMEM; - } - /* Get dmabuf info from KGD */ r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd, &dmabuf_adev, &args->size, - metadata_buffer, args->metadata_size, + &metadata_buffer, args->metadata_size, &args->metadata_size, &flags, &xcp_id); if (r) goto exit; @@ -1545,7 +1540,7 @@ static int kfd_ioctl_get_dmabuf_info(struct file *filep, args->flags = flags; /* Copy metadata buffer to user mode */ - if (metadata_buffer) { + if (metadata_buffer && args->metadata_ptr) { r = copy_to_user((void __user *)args->metadata_ptr, metadata_buffer, args->metadata_size); if (r != 0) @@ -1790,13 +1785,13 @@ static int criu_checkpoint_devices(struct kfd_process *p, struct kfd_criu_device_bucket *device_buckets = NULL; int ret = 0, i; - device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL); + device_buckets = kvcalloc(num_devices, sizeof(*device_buckets), GFP_KERNEL); if (!device_buckets) { ret = -ENOMEM; goto exit; } - device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL); + device_priv = kvcalloc(num_devices, sizeof(*device_priv), GFP_KERNEL); if (!device_priv) { ret = -ENOMEM; goto exit; @@ -1916,17 +1911,17 @@ static int criu_checkpoint_bos(struct kfd_process *p, int ret = 0, pdd_index, bo_index = 0, id; void *mem; - bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL); + bo_buckets = kvcalloc(num_bos, sizeof(*bo_buckets), GFP_KERNEL); if (!bo_buckets) return -ENOMEM; - bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL); + bo_privs = kvcalloc(num_bos, sizeof(*bo_privs), GFP_KERNEL); if (!bo_privs) { ret = -ENOMEM; goto exit; } - files = kvzalloc(num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; @@ -2463,7 +2458,7 @@ static int criu_restore_bos(struct kfd_process *p, if (!bo_buckets) return -ENOMEM; - files = kvzalloc(args->num_bos * sizeof(struct file *), GFP_KERNEL); + files = kvcalloc(args->num_bos, sizeof(struct file *), GFP_KERNEL); if (!files) { ret = -ENOMEM; goto exit; @@ -2971,10 +2966,14 @@ static int kfd_ioctl_set_debug_trap(struct file *filep, struct kfd_process *p, v goto out; } - /* Check if target is still PTRACED. */ + /* + * Verify debugger has permission to debug target process. + * For cross-process debugging, require active ptrace relationship. + * This applies to ALL operations to prevent unauthorized interference. + */ rcu_read_lock(); - if (target != p && args->op != KFD_IOC_DBG_TRAP_DISABLE - && ptrace_parent(target->lead_thread) != current) { + if (target != p && ptrace_parent(target->lead_thread) != current + && target->debugger_process != p) { pr_err("PID %i is not PTRACED and cannot be debugged\n", args->pid); r = -EPERM; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index 986cb297de8f8..78e1dfdb27147 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -816,6 +816,12 @@ int kfd_dbg_trap_enable(struct kfd_process *target, uint32_t fd, if (copy_to_user(runtime_info, (void *)&target->runtime_info, copy_size)) { kfd_dbg_trap_deactivate(target, false, 0); + fput(target->dbg_ev_file); + target->dbg_ev_file = NULL; + if (target->debugger_process) + atomic_dec(&target->debugger_process->debugged_process_count); + target->debug_trap_enabled = false; + kfd_unref_process(target); r = -EFAULT; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 18d03f33c6202..af8e6f53249da 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -472,6 +472,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd, } else { /* For CP queues on SOC15 */ if (restore_id) { + if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) + return -EINVAL; + /* make sure that ID is free */ if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap)) return -EINVAL; @@ -653,6 +656,11 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for this GPU\n"); + retval = -EOPNOTSUPP; + goto deallocate_vmid; + } if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) { retval = allocate_hqd(dqm, q); if (retval) @@ -1233,6 +1241,14 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, } } + /* + * Heavy-weight TLB flush after MES removes queues to ensure + * in-flight memory accesses complete before memory is freed/migrated. + * HWS does this automatically, MES does not. + */ + if (dqm->dev->kfd->shared_resources.enable_mes) + kfd_flush_tlb(pdd); + if (!dqm->dev->kfd->shared_resources.enable_mes) { pdd->last_evict_timestamp = get_jiffies_64(); retval = execute_queues_cpsch(dqm, @@ -1586,6 +1602,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); @@ -1612,6 +1631,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, return -ENOMEM; } if (restore_sdma_id) { + if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm)) + return -EINVAL; + /* Re-use existing sdma_id */ if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) { dev_err(dev, "SDMA queue already in use\n"); @@ -1994,6 +2016,11 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, mqd_mgr = dqm->mqd_mgrs[get_mqd_type_from_queue_type( q->properties.type)]; + if (qd && !mqd_mgr->restore_mqd) { + pr_debug("restore_mqd not implemented for this GPU\n"); + retval = -EOPNOTSUPP; + goto out_deallocate_doorbell; + } if (q->properties.type == KFD_QUEUE_TYPE_SDMA || q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) @@ -2490,6 +2517,9 @@ static int wait_on_destroy_queue(struct device_queue_manager *dqm, if (pdd->qpd.is_debug) return ret; + if (q->properties.is_being_destroyed) + return -EBUSY; + q->properties.is_being_destroyed = true; if (pdd->process->debug_trap_enabled && q->properties.is_suspended) { @@ -2502,6 +2532,9 @@ static int wait_on_destroy_queue(struct device_queue_manager *dqm, dqm_lock(dqm); } + if (ret) + q->properties.is_being_destroyed = false; + return ret; } @@ -2595,7 +2628,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, return retval; failed_try_destroy_debugged_queue: - + q->properties.is_being_destroyed = false; dqm_unlock(dqm); return retval; } @@ -3457,8 +3490,11 @@ int suspend_queues(struct kfd_process *p, if (!per_device_suspended) { dqm_unlock(dqm); mutex_unlock(&p->event_mutex); - if (total_suspended) + if (total_suspended) { amdgpu_amdkfd_debug_mem_fence(dqm->dev->adev); + /* Heavy-weight TLB flush after MES suspends queues */ + kfd_flush_tlb(pdd); + } continue; } @@ -3628,6 +3664,12 @@ bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, dqm_unlock(dqm); return r; } + +size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, enum kfd_queue_type type) +{ + return dqm->mqd_mgrs[get_mqd_type_from_queue_type(type)]->mqd_size; +} + #if defined(CONFIG_DEBUG_FS) static void seq_reg_dump(struct seq_file *m, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 74a61b5b2f0b4..46c6849df13f4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -327,6 +327,8 @@ int debug_refresh_runlist(struct device_queue_manager *dqm); bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, struct qcm_process_device *qpd, int doorbell_off, u32 *queue_format); +size_t mqd_size_from_queue_type(struct device_queue_manager *dqm, + enum kfd_queue_type type); static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c index 05c74887fd6fd..fdcf7f2d1b5b4 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c @@ -153,14 +153,16 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, u32 inx; mutex_lock(&kfd->doorbell_mutex); + inx = find_first_zero_bit(kfd->doorbell_bitmap, PAGE_SIZE / sizeof(u32)); + if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) { + mutex_unlock(&kfd->doorbell_mutex); + return NULL; + } __set_bit(inx, kfd->doorbell_bitmap); mutex_unlock(&kfd->doorbell_mutex); - if (inx >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS) - return NULL; - *doorbell_off = amdgpu_doorbell_index_on_bar(kfd->adev, kfd->doorbells, inx, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c index 63039035b1941..b033c3a9ce545 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c @@ -107,6 +107,9 @@ static int allocate_event_notification_slot(struct kfd_process *p, } if (restore_id) { + if (*restore_id >= KFD_SIGNAL_EVENT_LIMIT) + return -EINVAL; + id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, GFP_KERNEL); } else { @@ -518,6 +521,9 @@ int kfd_criu_restore_event(struct file *devkfd, ret = create_other_event(p, ev, &ev_priv->event_id); break; + default: + ret = -EINVAL; + break; } mutex_unlock(&p->event_mutex); @@ -539,15 +545,27 @@ int kfd_criu_checkpoint_events(struct kfd_process *p, int ret = 0; struct kfd_event *ev; uint32_t ev_id; + uint32_t num_events; - uint32_t num_events = kfd_get_num_events(p); + /* Serialize the count and the walk below against concurrent event + * create/destroy. Those paths take only p->event_mutex, not the + * p->mutex held by the CRIU checkpoint caller, so without this the + * event_idr can grow between kfd_get_num_events() and the loop and the + * walk writes past the ev_privs allocation. + */ + mutex_lock(&p->event_mutex); - if (!num_events) + num_events = kfd_get_num_events(p); + if (!num_events) { + mutex_unlock(&p->event_mutex); return 0; + } ev_privs = kvzalloc(num_events * sizeof(*ev_privs), GFP_KERNEL); - if (!ev_privs) + if (!ev_privs) { + mutex_unlock(&p->event_mutex); return -ENOMEM; + } idr_for_each_entry(&p->event_idr, ev, ev_id) { @@ -588,6 +606,8 @@ int kfd_criu_checkpoint_events(struct kfd_process *p, i++; } + mutex_unlock(&p->event_mutex); + ret = copy_to_user(user_priv_data + *priv_data_offset, ev_privs, num_events * sizeof(*ev_privs)); if (ret) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c index ea8377071c390..c47c9bda51f64 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c @@ -132,7 +132,7 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys, u64 gart_s, gart_d; struct dma_fence *next; u64 size; - int r; + int r = 0; mutex_lock(&adev->mman.gtt_window_lock); @@ -252,15 +252,6 @@ svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr) return page; } -static void svm_migrate_put_sys_page(unsigned long addr) -{ - struct page *page; - - page = pfn_to_page(addr >> PAGE_SHIFT); - unlock_page(page); - put_page(page); -} - static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate) { unsigned long mpages = 0; @@ -287,7 +278,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, dma_addr_t *src; u64 *dst; u64 i, j; - int r; + int r = 0; pr_debug("svms 0x%p [0x%lx 0x%lx 0x%llx]\n", prange->svms, prange->start, prange->last, ttm_res_offset); @@ -313,6 +304,7 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, DMA_BIDIRECTIONAL); r = dma_mapping_error(dev, src[i]); if (r) { + src[i] = 0; dev_err(dev, "%s: fail %d dma_map_page\n", __func__, r); goto out_free_vram_pages; @@ -337,7 +329,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, pr_debug_ratelimited("dma mapping src to 0x%llx, pfn 0x%lx\n", src[i] >> PAGE_SHIFT, page_to_pfn(spage)); - if (j >= (cursor.size >> PAGE_SHIFT) - 1 && i < npages - 1) { + /* accumulated j + 1 pages reach end of current drm_buddy_block */ + if (j + 1 >= (cursor.size >> PAGE_SHIFT)) { r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j + 1, FROM_RAM_TO_VRAM, @@ -351,7 +344,8 @@ svm_migrate_copy_to_vram(struct kfd_node *node, struct svm_range *prange, } } - r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j, FROM_RAM_TO_VRAM, mfence); out_free_vram_pages: @@ -584,9 +578,10 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dma_addr_t *scratch, u64 npages) { struct device *dev = adev->dev; - u64 *src; + struct page *dpage = NULL; dma_addr_t *dst; - struct page *dpage; + u64 *src; + u64 i = 0, j; u64 addr; int r = 0; @@ -640,6 +635,7 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, r = dma_mapping_error(dev, dst[i]); if (r) { dev_err(adev->dev, "%s: fail %d dma_map_page\n", __func__, r); + dst[i] = 0; goto out_oom; } @@ -647,17 +643,39 @@ svm_migrate_copy_to_ram(struct amdgpu_device *adev, struct svm_range *prange, dst[i] >> PAGE_SHIFT, page_to_pfn(dpage)); migrate->dst[i] = migrate_pfn(page_to_pfn(dpage)); + + dpage = NULL; j++; } - r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, - FROM_VRAM_TO_RAM, mfence); - + if (j > 0) + r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j, + FROM_VRAM_TO_RAM, mfence); out_oom: if (r) { pr_debug("failed %d copy to ram\n", r); + + /* first release current dpage when dma_map_page fail */ + if (dpage) { + unlock_page(dpage); + put_page(dpage); + } + + /* release previous allocated sys pages and unmap dma address */ while (i--) { - svm_migrate_put_sys_page(dst[i]); + + if (dst[i]) { + dma_unmap_page(dev, dst[i], PAGE_SIZE, + DMA_BIDIRECTIONAL); + dst[i] = 0; + } + + dpage = migrate_pfn_to_page(migrate->dst[i]); + if (!dpage) + continue; + + unlock_page(dpage); + put_page(dpage); migrate->dst[i] = 0; } } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index f2dee320fada4..e3a33a92b1799 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -341,11 +341,15 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, { struct v9_mqd *m; struct kfd_context_save_area_header header; + u32 cntl_stack_size; + u32 cntl_stack_offset; /* Control stack is located one page after MQD. */ void *mqd_ctl_stack = (void *)((uintptr_t)mqd + PAGE_SIZE); m = get_mqd(mqd); + cntl_stack_size = min_t(u32, m->cp_hqd_cntl_stack_size, q->ctl_stack_size); + cntl_stack_offset = min_t(u32, m->cp_hqd_cntl_stack_offset, cntl_stack_size); *ctl_stack_used_size = m->cp_hqd_cntl_stack_size - m->cp_hqd_cntl_stack_offset; @@ -361,9 +365,10 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd, if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state))) return -EFAULT; - if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset, - mqd_ctl_stack + m->cp_hqd_cntl_stack_offset, - *ctl_stack_used_size)) + *ctl_stack_used_size = cntl_stack_size - cntl_stack_offset; + + if (copy_to_user(ctl_stack + cntl_stack_offset, mqd_ctl_stack + cntl_stack_offset, + *ctl_stack_used_size)) return -EFAULT; return 0; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 24e85440473a1..d7e9150402428 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -436,7 +436,8 @@ enum kfd_queue_type { KFD_QUEUE_TYPE_HIQ, KFD_QUEUE_TYPE_DIQ, KFD_QUEUE_TYPE_SDMA_XGMI, - KFD_QUEUE_TYPE_SDMA_BY_ENG_ID + KFD_QUEUE_TYPE_SDMA_BY_ENG_ID, + KFD_QUEUE_TYPE_MAX, }; enum kfd_queue_format { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 7bf712032c52c..912c4ebd30405 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -375,7 +375,7 @@ int pqm_create_queue(struct process_queue_manager *pqm, false); if (retval) { dev_err(dev->adev->dev, "failed to allocate process context bo\n"); - return retval; + goto err_allocate_pqn; } memset(pdd->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); } @@ -974,8 +974,8 @@ static void set_queue_properties_from_criu(struct queue_properties *qp, qp->priority = q_data->priority; qp->queue_address = q_data->q_address; qp->queue_size = q_data->q_size; - qp->read_ptr = (uint32_t *) q_data->read_ptr_addr; - qp->write_ptr = (uint32_t *) q_data->write_ptr_addr; + qp->read_ptr = (void __user *)q_data->read_ptr_addr; + qp->write_ptr = (void __user *)q_data->write_ptr_addr; qp->eop_ring_buffer_address = q_data->eop_ring_buffer_address; qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size; qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address; @@ -1015,6 +1015,23 @@ int kfd_criu_restore_queue(struct kfd_process *p, goto exit; } + pdd = kfd_process_device_data_by_id(p, q_data->gpu_id); + if (!pdd) { + pr_err("Failed to get pdd\n"); + ret = -EINVAL; + goto exit; + } + + if (q_data->type >= KFD_QUEUE_TYPE_MAX) { + ret = -EINVAL; + goto exit; + } + + if (q_data->mqd_size != mqd_size_from_queue_type(pdd->dev->dqm, q_data->type)) { + ret = -EINVAL; + goto exit; + } + *priv_data_offset += sizeof(*q_data); q_extra_data_size = (uint64_t)q_data->ctl_stack_size + q_data->mqd_size; @@ -1037,13 +1054,6 @@ int kfd_criu_restore_queue(struct kfd_process *p, *priv_data_offset += q_extra_data_size; - pdd = kfd_process_device_data_by_id(p, q_data->gpu_id); - if (!pdd) { - pr_err("Failed to get pdd\n"); - ret = -EINVAL; - goto exit; - } - /* * data stored in this order: * mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]... @@ -1054,10 +1064,18 @@ int kfd_criu_restore_queue(struct kfd_process *p, memset(&qp, 0, sizeof(qp)); set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask)); + ret = kfd_queue_acquire_buffers(pdd, &qp); + if (ret) { + pr_debug("failed to acquire user queue buffers for CRIU\n"); + goto exit; + } + print_queue_properties(&qp); ret = pqm_create_queue(&p->pqm, pdd->dev, &qp, &queue_id, q_data, mqd, ctl_stack, NULL); if (ret) { + kfd_queue_unref_bo_vas(pdd, &qp); + kfd_queue_release_buffers(pdd, &qp); pr_err("Failed to create new queue err:%d\n", ret); goto exit; } diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c index b97f4a51db6e3..6f06516da32a8 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_queue.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_queue.c @@ -23,6 +23,7 @@ */ #include +#include #include "kfd_priv.h" #include "kfd_topology.h" #include "kfd_svm.h" @@ -235,7 +236,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope struct kfd_topology_device *topo_dev; u64 expected_queue_size; struct amdgpu_vm *vm; - u32 total_cwsr_size; + u64 total_cwsr_size; int err; topo_dev = kfd_topology_device_by_id(pdd->dev->id); @@ -284,7 +285,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope } err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address, &properties->eop_buf_bo, - ALIGN(properties->eop_ring_buffer_size, PAGE_SIZE)); + ALIGN((u64)properties->eop_ring_buffer_size, PAGE_SIZE)); if (err) goto out_err_unreserve; } @@ -305,8 +306,14 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope goto out_err_unreserve; } - total_cwsr_size = (properties->ctx_save_restore_area_size + - topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask); + total_cwsr_size = (u64)properties->ctx_save_restore_area_size + + topo_dev->node_props.debug_memory_size; + if (check_mul_overflow(total_cwsr_size, + NUM_XCC(pdd->dev->xcc_mask), + &total_cwsr_size)) { + err = -EINVAL; + goto out_err_unreserve; + } total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address, @@ -341,7 +348,7 @@ int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_prope int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties) { struct kfd_topology_device *topo_dev; - u32 total_cwsr_size; + u64 total_cwsr_size; kfd_queue_buffer_put(&properties->wptr_bo); kfd_queue_buffer_put(&properties->rptr_bo); @@ -352,8 +359,12 @@ int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_prope topo_dev = kfd_topology_device_by_id(pdd->dev->id); if (!topo_dev) return -EINVAL; - total_cwsr_size = (properties->ctx_save_restore_area_size + - topo_dev->node_props.debug_memory_size) * NUM_XCC(pdd->dev->xcc_mask); + total_cwsr_size = (u64)properties->ctx_save_restore_area_size + + topo_dev->node_props.debug_memory_size; + if (check_mul_overflow(total_cwsr_size, + NUM_XCC(pdd->dev->xcc_mask), + &total_cwsr_size)) + return -EINVAL; total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE); kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c index d2bc169e84b0b..fe4b936923854 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_smi_events.c @@ -254,8 +254,10 @@ void kfd_smi_event_update_vmfault(struct kfd_node *dev, uint16_t pasid) if (task_info) { /* Report VM faults from user applications, not retry from kernel */ if (task_info->task.pid) - kfd_smi_event_add(0, dev, KFD_SMI_EVENT_VMFAULT, KFD_EVENT_FMT_VMFAULT( - task_info->task.pid, task_info->task.comm)); + kfd_smi_event_add(task_info->tgid, dev, + KFD_SMI_EVENT_VMFAULT, + KFD_EVENT_FMT_VMFAULT(task_info->task.pid, + task_info->task.comm)); amdgpu_vm_put_task_info(task_info); } } @@ -356,7 +358,7 @@ void kfd_smi_event_process(struct kfd_process_device *pdd, bool start) task_info = amdgpu_vm_get_task_info_vm(avm); if (task_info) { - kfd_smi_event_add(0, pdd->dev, + kfd_smi_event_add(task_info->tgid, pdd->dev, start ? KFD_SMI_EVENT_PROCESS_START : KFD_SMI_EVENT_PROCESS_END, KFD_EVENT_FMT_PROCESS(task_info->task.pid, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index a2ca1de6b43cf..89686abcaf92d 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -575,89 +575,25 @@ static void schedule_dc_vmin_vmax(struct amdgpu_device *adev, queue_work(system_wq, &offload_work->work); } -static void dm_vupdate_high_irq(void *interrupt_params) -{ - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; - struct amdgpu_crtc *acrtc; - struct drm_device *drm_dev; - struct drm_vblank_crtc *vblank; - ktime_t frame_duration_ns, previous_timestamp; - unsigned long flags; - int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); - - if (acrtc) { - vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); - drm_dev = acrtc->base.dev; - vblank = drm_crtc_vblank_crtc(&acrtc->base); - previous_timestamp = atomic64_read(&irq_params->previous_timestamp); - frame_duration_ns = vblank->time - previous_timestamp; - - if (frame_duration_ns > 0) { - trace_amdgpu_refresh_rate_track(acrtc->base.index, - frame_duration_ns, - ktime_divns(NSEC_PER_SEC, frame_duration_ns)); - atomic64_set(&irq_params->previous_timestamp, vblank->time); - } - - drm_dbg_vbl(drm_dev, - "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, - vrr_active); - - /* Core vblank handling is done here after end of front-porch in - * vrr mode, as vblank timestamping will give valid results - * while now done after front-porch. This will also deliver - * page-flip completion events that have been queued to us - * if a pageflip happened inside front-porch. - */ - if (vrr_active && acrtc->dm_irq_params.stream) { - bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; - bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; - bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state - == VRR_STATE_ACTIVE_VARIABLE; - - amdgpu_dm_crtc_handle_vblank(acrtc); - - /* BTR processing for pre-DCE12 ASICs */ - if (adev->family < AMDGPU_FAMILY_AI) { - spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); - mod_freesync_handle_v_update( - adev->dm.freesync_module, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params); - - if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { - schedule_dc_vmin_vmax(adev, - acrtc->dm_irq_params.stream, - &acrtc->dm_irq_params.vrr_params.adjust); - } - spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); - } - } - } -} - /** - * dm_crtc_high_irq() - Handles CRTC interrupt - * @interrupt_params: used for determining the CRTC instance + * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling + * @adev: amdgpu device + * @acrtc: the CRTC to service * - * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK - * event handler. + * Performs writeback completion, vblank event handling, CRC processing, VRR BTR + * updates and pageflip completion delivery. + * + * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from + * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank + * from dm_crtc_high_irq(). */ -static void dm_crtc_high_irq(void *interrupt_params) +static void dm_crtc_high_irq_handler(struct amdgpu_device *adev, + struct amdgpu_crtc *acrtc) { - struct common_irq_params *irq_params = interrupt_params; - struct amdgpu_device *adev = irq_params->adev; struct drm_writeback_job *job; - struct amdgpu_crtc *acrtc; unsigned long flags; int vrr_active; - - acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); - if (!acrtc) - return; + bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0; if (acrtc->wb_conn) { spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags); @@ -694,12 +630,17 @@ static void dm_crtc_high_irq(void *interrupt_params) vrr_active, acrtc->dm_irq_params.active_planes); /** - * Core vblank handling at start of front-porch is only possible - * in non-vrr mode, as only there vblank timestamping will give - * valid results while done in front-porch. Otherwise defer it - * to dm_vupdate_high_irq after end of front-porch. + * Core vblank handling. + * + * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch + * point, which is the correct place to timestamp both VRR and non-VRR + * vblanks. + * + * On DCE this handler runs at the start of front-porch, where only + * non-VRR timestamping is valid; VRR vblank is deferred to + * dm_vupdate_high_irq() after end of front-porch. */ - if (!vrr_active) + if (is_dcn || !vrr_active) amdgpu_dm_crtc_handle_vblank(acrtc); /** @@ -732,18 +673,33 @@ static void dm_crtc_high_irq(void *interrupt_params) } /* - * If there aren't any active_planes then DCH HUBP may be clock-gated. - * In that case, pageflip completion interrupts won't fire and pageflip - * completion events won't get delivered. Prevent this by sending - * pending pageflip events from here if a flip is still pending. + * Deliver pageflip completion events (DCN only). * - * If any planes are enabled, use dm_pflip_high_irq() instead, to - * avoid race conditions between flip programming and completion, - * which could cause too early flip completion events. + * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch + * point. Deliver any pending pageflip completion event from here, + * once HW has consumed the new address (the OTG no longer reports a + * pending flip). + * + * Also handle the case here where there aren't any active planes and + * DCN HUBP may be clock-gated, so the flip-pending status may be + * undefined. */ - if (adev->family >= AMDGPU_FAMILY_RV && - acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && - acrtc->dm_irq_params.active_planes == 0) { + if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc->event) { + + if (!dc_get_flip_pending_on_otg(adev->dm.dc, acrtc->otg_inst)) { + drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); + acrtc->event = NULL; + drm_crtc_vblank_put(&acrtc->base); + acrtc->pflip_status = AMDGPU_FLIP_NONE; + } + /* + * If the flip is still pending, leave it armed and + * retry on the next vupdate. + */ + } else if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc->dm_irq_params.active_planes == 0) { + if (acrtc->event) { drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); acrtc->event = NULL; @@ -755,6 +711,104 @@ static void dm_crtc_high_irq(void *interrupt_params) spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); } +static void dm_vupdate_high_irq(void *interrupt_params) +{ + struct common_irq_params *irq_params = interrupt_params; + struct amdgpu_device *adev = irq_params->adev; + struct amdgpu_crtc *acrtc; + struct drm_device *drm_dev; + struct drm_vblank_crtc *vblank; + ktime_t frame_duration_ns, previous_timestamp; + unsigned long flags; + int vrr_active; + + acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); + if (!acrtc) + return; + + vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); + drm_dev = acrtc->base.dev; + vblank = drm_crtc_vblank_crtc(&acrtc->base); + previous_timestamp = atomic64_read(&irq_params->previous_timestamp); + frame_duration_ns = vblank->time - previous_timestamp; + + if (frame_duration_ns > 0) { + trace_amdgpu_refresh_rate_track(acrtc->base.index, + frame_duration_ns, + ktime_divns(NSEC_PER_SEC, frame_duration_ns)); + atomic64_set(&irq_params->previous_timestamp, vblank->time); + } + + drm_dbg_vbl(drm_dev, + "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, + vrr_active); + + /* + * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver + * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are + * not used. Run the full handler here. + */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { + dm_crtc_high_irq_handler(adev, acrtc); + return; + } + + /* DCE only below. */ + + /* Core vblank handling is done here after end of front-porch in + * vrr mode, as vblank timestamping will give valid results + * while now done after front-porch. This will also deliver + * page-flip completion events that have been queued to us + * if a pageflip happened inside front-porch. + */ + if (vrr_active && acrtc->dm_irq_params.stream) { + bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; + bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; + bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state + == VRR_STATE_ACTIVE_VARIABLE; + + amdgpu_dm_crtc_handle_vblank(acrtc); + + /* BTR processing for pre-DCE12 ASICs */ + if (adev->family < AMDGPU_FAMILY_AI) { + spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); + mod_freesync_handle_v_update( + adev->dm.freesync_module, + acrtc->dm_irq_params.stream, + &acrtc->dm_irq_params.vrr_params); + + if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { + schedule_dc_vmin_vmax(adev, + acrtc->dm_irq_params.stream, + &acrtc->dm_irq_params.vrr_params.adjust); + } + spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); + } + } +} + +/** + * dm_crtc_high_irq() - Handles CRTC interrupt + * @interrupt_params: used for determining the CRTC instance + * + * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler. + * + * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is + * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq(). + */ +static void dm_crtc_high_irq(void *interrupt_params) +{ + struct common_irq_params *irq_params = interrupt_params; + struct amdgpu_device *adev = irq_params->adev; + struct amdgpu_crtc *acrtc; + + acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); + if (!acrtc) + return; + + dm_crtc_high_irq_handler(adev, acrtc); +} + #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) /** * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for @@ -2986,17 +3040,26 @@ static int dm_hw_init(struct amdgpu_ip_block *ip_block) struct amdgpu_device *adev = ip_block->adev; int r; + adev->dm.i2c_devres_group = devres_open_group(adev->dev, NULL, GFP_KERNEL); + if (!adev->dm.i2c_devres_group) + return -ENOMEM; + /* Create DAL display manager */ r = amdgpu_dm_init(adev); if (r) - return r; + goto err_release_i2c; amdgpu_dm_hpd_init(adev); r = dm_oem_i2c_hw_init(adev); if (r) drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n"); + devres_close_group(adev->dev, adev->dm.i2c_devres_group); return 0; + +err_release_i2c: + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + return r; } /** @@ -3011,6 +3074,9 @@ static int dm_hw_fini(struct amdgpu_ip_block *ip_block) { struct amdgpu_device *adev = ip_block->adev; + if (adev->dm.i2c_devres_group) + devres_release_group(adev->dev, adev->dm.i2c_devres_group); + amdgpu_dm_hpd_fini(adev); amdgpu_dm_irq_fini(adev); @@ -3031,7 +3097,8 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, acrtc = get_crtc_by_otg_inst( adev, state->stream_status[i].primary_otg_inst); - if (acrtc && state->stream_status[i].plane_count != 0) { + if (acrtc && state->stream_status[i].plane_count != 0 && + amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst; rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; if (rc) @@ -3059,6 +3126,13 @@ static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, */ if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); + + } else if (acrtc && state->stream_status[i].plane_count != 0) { + /* DCN only needs to toggle VUPDATE_NO_LOCK */ + rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, enable); + if (rc) + drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n", + enable ? "en" : "dis"); } } @@ -3199,6 +3273,9 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block) res = amdgpu_dm_commit_zero_streams(dm->dc); if (res != DC_OK) { drm_err(adev_to_drm(adev), "Failed to commit zero streams: %d\n", res); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return -EINVAL; } @@ -3443,6 +3520,9 @@ static int dm_resume(struct amdgpu_ip_block *ip_block) r = dm_dmub_hw_init(adev); if (r) { drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r); + dc_state_release(dm->cached_dc_state); + dm->cached_dc_state = NULL; + mutex_unlock(&dm->dc_lock); return r; } @@ -3680,6 +3760,8 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps; caps->aux_support = false; + panel_backlight_quirk = drm_get_panel_backlight_quirk(aconnector->drm_edid); + if (caps->ext_caps->bits.oled == 1 /* * || @@ -3692,6 +3774,9 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) caps->aux_support = false; else if (amdgpu_backlight == 1) caps->aux_support = true; + else if (!IS_ERR_OR_NULL(panel_backlight_quirk) && + panel_backlight_quirk->force_pwm) + caps->aux_support = false; if (caps->aux_support) aconnector->dc_link->backlight_control_type = BACKLIGHT_CONTROL_AMD_AUX; @@ -3707,8 +3792,6 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector) else caps->aux_min_input_signal = 1; - panel_backlight_quirk = - drm_get_panel_backlight_quirk(aconnector->drm_edid); if (!IS_ERR_OR_NULL(panel_backlight_quirk)) { if (panel_backlight_quirk->min_brightness) { caps->min_input_signal = @@ -4558,38 +4641,6 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) * for acknowledging and handling. */ - /* Use VSTARTUP interrupt */ - for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP; - i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq); - - if (r) { - drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 || - int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) { - drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_crtc_high_irq, c_irq_params)) - return -ENOMEM; - } - /* Use otg vertical line interrupt */ #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) { @@ -4661,37 +4712,6 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) return -ENOMEM; } - /* Use GRPH_PFLIP interrupt */ - for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT; - i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1; - i++) { - r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq); - if (r) { - drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n"); - return r; - } - - int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT; - int_params.irq_source = - dc_interrupt_to_irq_source(dc, i, 0); - - if (int_params.irq_source == DC_IRQ_SOURCE_INVALID || - int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST || - int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) { - drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n"); - return -EINVAL; - } - - c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST]; - - c_irq_params->adev = adev; - c_irq_params->irq_src = int_params.irq_source; - - if (!amdgpu_dm_irq_register_interrupt(adev, &int_params, - dm_pflip_high_irq, c_irq_params)) - return -ENOMEM; - } - /* HPD */ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT, &adev->hpd_irq); @@ -4950,10 +4970,10 @@ static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, return 1; } -/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */ -static inline u32 scale_input_to_fw(int min, int max, u64 input) +/* Rescale userspace [0..max] to the firmware curve's [0..255]. */ +static inline u32 scale_input_to_fw(int max, u64 input) { - return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min); + return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max); } /* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */ @@ -4966,7 +4986,7 @@ static void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *cap unsigned int min, unsigned int max, uint32_t *user_brightness) { - u32 brightness = scale_input_to_fw(min, max, *user_brightness); + u32 brightness = scale_input_to_fw(max, *user_brightness); u8 lower_signal, upper_signal, upper_lum, lower_lum, lum; int left, right; @@ -5220,11 +5240,11 @@ amdgpu_dm_register_backlight_device(struct amdgpu_dm_connector *aconnector) caps = &dm->backlight_caps[aconnector->bl_idx]; if (get_brightness_range(caps, &min, &max)) { if (power_supply_is_system_supplied() > 0) - props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->ac_level, 100); + props.brightness = DIV_ROUND_CLOSEST(max * caps->ac_level, 100); else - props.brightness = DIV_ROUND_CLOSEST((max - min) * caps->dc_level, 100); + props.brightness = DIV_ROUND_CLOSEST(max * caps->dc_level, 100); /* min is zero, so max needs to be adjusted */ - props.max_brightness = max - min; + props.max_brightness = max; drm_dbg(drm, "Backlight caps: min: %d, max: %d, ac %d, dc %d\n", min, max, caps->ac_level, caps->dc_level); } else @@ -6240,8 +6260,8 @@ static void fill_dc_dirty_rects(struct drm_plane *plane, { struct dm_crtc_state *dm_crtc_state = to_dm_crtc_state(crtc_state); struct rect *dirty_rects = flip_addrs->dirty_rects; - u32 num_clips; - struct drm_mode_rect *clips; + u32 num_clips = 0; + struct drm_mode_rect *clips = NULL; bool bb_changed; bool fb_changed; u32 i = 0; @@ -6257,8 +6277,10 @@ static void fill_dc_dirty_rects(struct drm_plane *plane, if (new_plane_state->rotation != DRM_MODE_ROTATE_0) goto ffu; - num_clips = drm_plane_get_damage_clips_count(new_plane_state); - clips = drm_plane_get_damage_clips(new_plane_state); + if (!new_plane_state->ignore_damage_clips) { + num_clips = drm_plane_get_damage_clips_count(new_plane_state); + clips = drm_plane_get_damage_clips(new_plane_state); + } if (num_clips && (!amdgpu_damage_clips || (amdgpu_damage_clips < 0 && is_psr_su))) @@ -6489,10 +6511,14 @@ get_output_color_space(const struct dc_crtc_timing *dc_crtc_timing, break; case DRM_MODE_COLORIMETRY_BT2020_RGB: case DRM_MODE_COLORIMETRY_BT2020_YCC: - if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) - color_space = COLOR_SPACE_2020_RGB_FULLRANGE; - else + if (dc_crtc_timing->pixel_encoding == PIXEL_ENCODING_RGB) { + if (connector_state->hdmi.broadcast_rgb == DRM_HDMI_BROADCAST_RGB_LIMITED) + color_space = COLOR_SPACE_2020_RGB_LIMITEDRANGE; + else + color_space = COLOR_SPACE_2020_RGB_FULLRANGE; + } else { color_space = COLOR_SPACE_2020_YCBCR_LIMITED; + } break; case DRM_MODE_COLORIMETRY_DEFAULT: // ITU601 default: @@ -7161,7 +7187,7 @@ create_stream_for_sink(struct drm_connector *connector, int preferred_refresh = 0; enum color_transfer_func tf = TRANSFER_FUNC_UNKNOWN; #if defined(CONFIG_DRM_AMD_DC_FP) - struct dsc_dec_dpcd_caps dsc_caps; + struct dsc_dec_dpcd_caps dsc_caps = {0}; #endif struct dc_link *link = NULL; struct dc_sink *sink = NULL; @@ -7323,29 +7349,68 @@ create_stream_for_sink(struct drm_connector *connector, return stream; } +/* + * Apple Studio Display exposes two SST DP links for a 2x1 tiled panel. + * The primary tile advertises the full 5120x2880 mode (with DSC on the + * bandwidth-sufficient link) while the secondary carries a per-tile + * 2560x2880 timing on a insufficient bandwidth link. Hide the secondary + * connector from userspace so compositors configure a single 5K stream + * on the primary link only. + */ +static bool amdgpu_dm_hide_secondary_tile_from_userspace(struct drm_connector *connector) +{ + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + + if (!aconnector->dc_sink) + return false; + + if (!aconnector->dc_sink->edid_caps.panel_patch.disable_second_tile) + return false; + + drm_edid_connector_update(connector, aconnector->drm_edid); + + if (!connector->has_tile) + return false; + + if (!connector->tile_h_loc && !connector->tile_v_loc) + return false; + + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] hiding secondary Apple Studio Display tile from userspace\n", + connector->base.id, connector->name); + + return true; +} + +/** + * amdgpu_dm_connector_detect() - Detect whether a DRM connector is connected to a display + * + * A connector is considered connected when it has a sink that is not NULL. + * For connectors that support HPD (hotplug detection), the connection is + * handled in the HPD interrupt. + * + * Notes: + * 1. This interface is NOT called in context of HPD irq. + * 2. This interface *is called* in context of user-mode ioctl. Which + * makes it a bad place for *any* MST-related activity. + */ static enum drm_connector_status amdgpu_dm_connector_detect(struct drm_connector *connector, bool force) { - bool connected; struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); - /* - * Notes: - * 1. This interface is NOT called in context of HPD irq. - * 2. This interface *is called* in context of user-mode ioctl. Which - * makes it a bad place for *any* MST-related activity. - */ + update_subconnector_property(aconnector); - if (aconnector->base.force == DRM_FORCE_UNSPECIFIED && - !aconnector->fake_enable) - connected = (aconnector->dc_sink != NULL); - else - connected = (aconnector->base.force == DRM_FORCE_ON || - aconnector->base.force == DRM_FORCE_ON_DIGITAL); + if (aconnector->base.force == DRM_FORCE_ON || + aconnector->base.force == DRM_FORCE_ON_DIGITAL) + return connector_status_connected; + else if (aconnector->base.force == DRM_FORCE_OFF) + return connector_status_disconnected; - update_subconnector_property(aconnector); + if (amdgpu_dm_hide_secondary_tile_from_userspace(connector)) + return connector_status_disconnected; - return (connected ? connector_status_connected : + return (aconnector->dc_sink ? connector_status_connected : connector_status_disconnected); } @@ -8497,6 +8562,47 @@ static void amdgpu_dm_connector_add_common_modes(struct drm_encoder *encoder, } } +/* + * The Apple Studio Display primary tile advertises both the full 5120x2880 + * mode and the per-tile 2560x2880 timing. As the secondary tile is hidden from + * userspace (see amdgpu_dm_hide_secondary_tile_from_userspace()), drop the + * per-tile timing from the primary connector so compositors only pick the full + * 5K mode. + */ +static void amdgpu_dm_prune_primary_tile_modes(struct drm_connector *connector) +{ + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + struct drm_display_mode *mode, *t; + + if (!aconnector->dc_sink) + return; + + if (!aconnector->dc_sink->edid_caps.panel_patch.disable_second_tile) + return; + + if (!connector->has_tile) + return; + + /* Only prune the per-tile timing from the primary tile. */ + if (connector->tile_h_loc || connector->tile_v_loc) + return; + + list_for_each_entry_safe(mode, t, &connector->probed_modes, head) { + if (mode->hdisplay != connector->tile_h_size || + mode->vdisplay != connector->tile_v_size) + continue; + + drm_dbg_kms(connector->dev, + "[CONNECTOR:%d:%s] pruning per-tile %dx%d timing from primary Apple Studio Display tile\n", + connector->base.id, connector->name, + mode->hdisplay, mode->vdisplay); + + list_del(&mode->head); + drm_mode_destroy(connector->dev, mode); + aconnector->num_modes--; + } +} + static void amdgpu_set_panel_orientation(struct drm_connector *connector) { struct drm_encoder *encoder; @@ -8539,6 +8645,8 @@ static void amdgpu_dm_connector_ddc_get_modes(struct drm_connector *connector, amdgpu_dm_connector->num_modes = drm_edid_connector_add_modes(connector); + amdgpu_dm_prune_primary_tile_modes(connector); + /* sorting the probed modes before calling function * amdgpu_dm_get_native_mode() since EDID can have * more than one preferred mode. The modes that are @@ -9116,14 +9224,22 @@ static void manage_dm_interrupts(struct amdgpu_device *adev, drm_crtc_vblank_on_config(&acrtc->base, &config); - /* Allow RX6xxx, RX7700, RX7800 GPUs to call amdgpu_irq_get.*/ + /* + * Since pflip_high_irq is no longer registered for DCN, grab an + * extra reference to vupdate irq instead to workaround this + * issue: + * https://gitlab.freedesktop.org/drm/amd/-/work_items/3936 + * + * The callbacks to drm_vblank_on/off should really take care of + * this though. + */ switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { case IP_VERSION(3, 0, 0): case IP_VERSION(3, 0, 2): case IP_VERSION(3, 0, 3): case IP_VERSION(3, 2, 0): - if (amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type)) - drm_err(dev, "DM_IRQ: Cannot get pageflip irq!\n"); + if (amdgpu_irq_get(adev, &adev->vupdate_irq, irq_type)) + drm_err(dev, "DM_IRQ: Cannot get vupdate irq!\n"); #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) if (amdgpu_irq_get(adev, &adev->vline0_irq, irq_type)) drm_err(dev, "DM_IRQ: Cannot get vline0 irq!\n"); @@ -9141,8 +9257,8 @@ static void manage_dm_interrupts(struct amdgpu_device *adev, if (amdgpu_irq_put(adev, &adev->vline0_irq, irq_type)) drm_err(dev, "DM_IRQ: Cannot put vline0 irq!\n"); #endif - if (amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type)) - drm_err(dev, "DM_IRQ: Cannot put pageflip irq!\n"); + if (amdgpu_irq_put(adev, &adev->vupdate_irq, irq_type)) + drm_err(dev, "DM_IRQ: Cannot put vupdate irq!\n"); } drm_crtc_vblank_off(&acrtc->base); @@ -9155,6 +9271,10 @@ static void dm_update_pflip_irq_state(struct amdgpu_device *adev, int irq_type = amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); + /* GRPH_PFLIP is not used on DCN; nothing to reapply. */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) + return; + /** * This reads the current state for the IRQ and force reapplies * the setting to hardware. @@ -9310,25 +9430,6 @@ static void remove_stream(struct amdgpu_device *adev, acrtc->enabled = false; } -static void prepare_flip_isr(struct amdgpu_crtc *acrtc) -{ - - assert_spin_locked(&acrtc->base.dev->event_lock); - WARN_ON(acrtc->event); - - acrtc->event = acrtc->base.state->event; - - /* Set the flip status */ - acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED; - - /* Mark this event as consumed */ - acrtc->base.state->event = NULL; - - drm_dbg_state(acrtc->base.dev, - "crtc:%d, pflip_stat:AMDGPU_FLIP_SUBMITTED\n", - acrtc->crtc_id); -} - static void update_freesync_state_on_stream( struct amdgpu_display_manager *dm, struct dm_crtc_state *new_crtc_state, @@ -9486,9 +9587,13 @@ static void update_stream_irq_parameters( static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, struct dm_crtc_state *new_state) { + struct amdgpu_device *adev = drm_to_adev(new_state->base.crtc->dev); bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state); bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state); + /* Only DCE gates vupdate on VRR, keep it enabled for DCN */ + bool vrr_gates_vupdate = amdgpu_ip_version(adev, DCE_HWIP, 0) == 0; + if (!old_vrr_active && new_vrr_active) { /* Transition VRR inactive -> active: * While VRR is active, we must not disable vblank irq, as a @@ -9498,7 +9603,8 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, * We also need vupdate irq for the actual core vblank handling * at end of vblank. */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); + if (vrr_gates_vupdate) + WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0); WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0); drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n", __func__, new_state->base.crtc->base.id); @@ -9506,7 +9612,8 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, /* Transition VRR active -> inactive: * Allow vblank irq disable again for fixed refresh rate. */ - WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); + if (vrr_gates_vupdate) + WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0); drm_crtc_vblank_put(new_state->base.crtc); drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n", __func__, new_state->base.crtc->base.id); @@ -9659,6 +9766,58 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach, } } +static void dm_arm_vblank_event(struct amdgpu_crtc *acrtc, + struct dm_crtc_state *acrtc_state, + bool pflip_update, + bool cursor_update) +{ + assert_spin_locked(&acrtc->base.dev->event_lock); + + if (!acrtc->base.state->event || acrtc_state->active_planes == 0) + return; + + if (pflip_update) { + WARN_ON(acrtc->pflip_status != AMDGPU_FLIP_NONE); + WARN_ON(acrtc->event); + + acrtc->pflip_status = AMDGPU_FLIP_SUBMITTED; + acrtc->event = acrtc->base.state->event; + acrtc->base.state->event = NULL; + + drm_dbg_state(acrtc->base.dev, + "crtc:%d, pflip_stat:AMDGPU_FLIP_SUBMITTED\n", + acrtc->crtc_id); + } else if (cursor_update) { + acrtc->event = acrtc->base.state->event; + acrtc->base.state->event = NULL; + } +} + +/** + * dm_arm_vblank_event_pre_programming - Prepare for programming + * @acrtc: The amdgpu CRTC to prepare + * @acrtc_state: The new CRTC state + * @pflip_update: Whether a page flip is being programmed + * @cursor_update: Whether a cursor update is being programmed + * + * Grab a reference on the vblank counter if a page flip or cursor update is to + * be programmed. Do this before programming so the HW is not in any + * idle-optimized state (such as PSR). + */ +static void dm_arm_vblank_event_pre_programming(struct amdgpu_crtc *acrtc, + struct dm_crtc_state *acrtc_state, + bool pflip_update, + bool cursor_update) +{ + assert_spin_locked(&acrtc->base.dev->event_lock); + + if (!acrtc->base.state->event || acrtc_state->active_planes == 0) + return; + + if (pflip_update || cursor_update) + drm_crtc_vblank_get(&acrtc->base); +} + static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, struct drm_device *dev, struct amdgpu_display_manager *dm, @@ -9681,6 +9840,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state); bool cursor_update = false; bool pflip_present = false; + bool immediate_flip = false; + bool flip_latched_during_prog = false; bool dirty_rects_changed = false; bool updated_planes_and_streams = false; struct { @@ -9846,6 +10007,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, acrtc_state->update_type == UPDATE_TYPE_FAST && get_mem_type(old_plane_state->fb) == get_mem_type(fb); + immediate_flip |= bundle->flip_addrs[planes_count].flip_immediate; + timestamp_ns = ktime_get_ns(); bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000); bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count]; @@ -9914,39 +10077,27 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, usleep_range(1000, 1100); } - /** - * Prepare the flip event for the pageflip interrupt to handle. - * - * This only works in the case where we've already turned on the - * appropriate hardware blocks (eg. HUBP) so in the transition case - * from 0 -> n planes we have to skip a hardware generated event - * and rely on sending it from software. - */ - if (acrtc_attach->base.state->event && - acrtc_state->active_planes > 0) { - drm_crtc_vblank_get(pcrtc); - - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - - WARN_ON(acrtc_attach->pflip_status != AMDGPU_FLIP_NONE); - prepare_flip_isr(acrtc_attach); - - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); - } - if (acrtc_state->stream) { if (acrtc_state->freesync_vrr_info_changed) bundle->stream_update.vrr_infopacket = &acrtc_state->stream->vrr_infopacket; } - } else if (cursor_update && acrtc_state->active_planes > 0) { - spin_lock_irqsave(&pcrtc->dev->event_lock, flags); - if (acrtc_attach->base.state->event) { - drm_crtc_vblank_get(pcrtc); - acrtc_attach->event = acrtc_attach->base.state->event; - acrtc_attach->base.state->event = NULL; + } + + scoped_guard(spinlock_irqsave, &pcrtc->dev->event_lock) { + dm_arm_vblank_event_pre_programming(acrtc_attach, acrtc_state, + pflip_present, + cursor_update); + /* + * DCE depends on a combination of GRPH_FLIP, VLINE0, and + * VUPDATE for event delivery. Only GRPH_FLIP handler can send + * pflip events, and it only fires if HW latched to the flip. + * Maintain legacy behavior by arming event before programming. + */ + if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) == 0) { + dm_arm_vblank_event(acrtc_attach, acrtc_state, + pflip_present, cursor_update); } - spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); } /* Update the planes if changed or disable if we don't have any. */ @@ -10047,6 +10198,115 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state, acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE) amdgpu_dm_commit_cursors(state); + /* + * DCN specific vblank handling + * ============================ + * + * With the event_lock held, arm the vblank event, and determine whether + * deliver it immediately, or in VUPDATE_NO_LOCK IRQ (i.e. HW latch + * point) handler. Do this *after* programming so that the IRQ handler + * will not deliver the event before HW laches onto the programmed + * values: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * arm_vblank_event() + * vupdate() + * vupdate_handler() + * cook_timestamp() + * # prev flip already latched, + * # so flip_latched == true. + * if event_armed && flip_latched: + * send_vblank_event() + * # sent before latch, **BAD!** + * hw_program() + * vupdate() + * **latch** + * + * There's a consequence of arming after: it's possible for HW to latch + * between start of HW programming and acrtc->event/pflip_status arming. + * When this happens, the IRQ handler will send the event on the next + * immediate latch point, even though HW has already latched. This is + * handled by optimistically checking for HW latch after programming, + * and if latched, send the event immediately: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * hw_program() + * vupdate() + * **latch** + * vupdate_handler() + * cook_timestamp() + * # event_armed == false + * # **no event sent!** + * arm_vblank_event() + * if flip_latched: + * **send_vblank_event()** + * disarm_vblank_event() + * + * The IRQ handler is expected to cook the timestamp, but we need to + * cook the timestamp before optimistic sending as well. That's because + * the following sequence is possible: + * + * Commit thread IRQ handler HW + * ----------------------------------------------------------------- + * hw_program() + * arm_vblank_event() + * vupdate() + * **latch** + * if flip_latched: + * # Need cook before send! + * **cook_timestamp()** + * send_vblank_event() + * disarm_vblank_event() + * vupdate_handler() + * cook_timestamp() + * # event_armed == false + * # no event sent! + * + * Cooking twice is OK, since DRM scanout accurate timestamps report A) + * the previous vactive start if currently in vactive, or B) the next + * vactive start if currently in vblank (see &get_vblank_counter). 'A)' + * is what we want for the optimistic send, and for 'B)', we'll cook a + * timestamp no later than the next IRQ handler run. + * + * The more correct fix is to wrap programming and arming with the + * event_lock and thus serializing it with the IRQ handler. However, + * there are various sleep-waits within + * update_planes_and_stream_adapter() that makes spin locking illegal. + * And on full updates, it can take 1-2 frame-times to return (see + * commit_planes_for_stream). + * + * On DCE, GRPH_PFLIP IRQ is used and takes care of this. + */ + if (amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) { + spin_lock_irqsave(&pcrtc->dev->event_lock, flags); + + if (updated_planes_and_streams) { + flip_latched_during_prog = + !dc_get_flip_pending_on_otg(dm->dc, acrtc_attach->otg_inst); + } + + dm_arm_vblank_event(acrtc_attach, acrtc_state, + pflip_present, cursor_update); + + /* + * Deliver the event immediately on immediate flip, or on a + * update that has already latched. + */ + if ((immediate_flip || flip_latched_during_prog) && + acrtc_attach->pflip_status == AMDGPU_FLIP_SUBMITTED && + acrtc_attach->event) { + drm_crtc_accurate_vblank_count(&acrtc_attach->base); + drm_crtc_send_vblank_event(&acrtc_attach->base, + acrtc_attach->event); + acrtc_attach->event = NULL; + drm_crtc_vblank_put(&acrtc_attach->base); + acrtc_attach->pflip_status = AMDGPU_FLIP_NONE; + } + spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags); + } + cleanup: kfree(bundle); } @@ -11381,6 +11641,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm, /* Release extra reference */ if (new_stream) dc_stream_release(new_stream); + new_stream = NULL; /* * We want to do dc stream updates that do not require a diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h index cd362d22a2774..caf6f7387a20a 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h @@ -664,6 +664,13 @@ struct amdgpu_display_manager { */ void *bb_from_dmub; + /** + * @i2c_devres_group: + * + * Devres group for DM i2c adapter lifetime management. + */ + void *i2c_devres_group; + /** * @oem_i2c: * diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index a4ac6d442278e..6b05cf597d712 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -360,6 +360,12 @@ static bool __is_lut_linear(const struct drm_color_lut *lut, uint32_t size) uint32_t expected; int delta; + /* A LUT with fewer than two entries can't be interpolated and would + * divide by zero below (size - 1); it can't be treated as linear. + */ + if (size < 2) + return false; + for (i = 0; i < size; i++) { /* All color values should equal */ if ((lut[i].red != lut[i].green) || (lut[i].green != lut[i].blue)) @@ -1143,6 +1149,13 @@ __set_dm_plane_degamma(struct drm_plane_state *plane_state, degamma_lut = __extract_blob_lut(dm_plane_state->degamma_lut, °amma_size); + if (degamma_lut && degamma_size != MAX_COLOR_LUT_ENTRIES) { + drm_dbg(plane_state->state->dev, + "Invalid Plane Degamma LUT size. Should be %u but got %u.\n", + MAX_COLOR_LUT_ENTRIES, degamma_size); + return -EINVAL; + } + has_degamma_lut = degamma_lut && !__is_lut_linear(degamma_lut, degamma_size); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index e20aa74380665..596a97092e0dd 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -584,8 +584,13 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name) */ ret = wait_for_completion_interruptible_timeout( &commit->hw_done, 10 * HZ); - if (ret) + if (ret < 0) + goto cleanup; + + if (ret == 0) { + ret = -ETIMEDOUT; goto cleanup; + } } enable = amdgpu_dm_is_valid_crc_source(source); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 2e7ee77c010e1..7880683031524 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -306,7 +306,7 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) irq_type = amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id); - if (enable) { + if (enable && acrtc_state->stream) { struct dc *dc = adev->dm.dc; struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc); struct psr_settings *psr = &acrtc_state->stream->link->psr_settings; @@ -327,9 +327,25 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) drm_crtc_vblank_restore(crtc); } - if (dc_supports_vrr(dm->dc->ctx->dce_version)) { + /* + * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver + * vblank and pageflip completion events, so enable it whenever vblank + * is enabled. On DCE, vupdate is only needed in VRR mode. + */ + if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) { if (enable) { - /* vblank irq on -> Only need vupdate irq in vrr mode */ + rc = amdgpu_irq_get(adev, &adev->vupdate_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Get vupdate_irq ret=%d\n", rc); + } else { + rc = amdgpu_irq_put(adev, &adev->vupdate_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Put vupdate_irq ret=%d\n", rc); + } + } else if (dc_supports_vrr(dm->dc->ctx->dce_version)) { + if (enable) { + /* vblank irq on -> Only need vupdate irq in vrr mode + * Not ref-counted since we need explicit enable/disable + * for DCE VRR handling + */ if (amdgpu_dm_crtc_vrr_active(acrtc_state)) rc = amdgpu_dm_crtc_set_vupdate_irq(crtc, true); } else { @@ -338,39 +354,46 @@ static inline int amdgpu_dm_crtc_set_vblank(struct drm_crtc *crtc, bool enable) } } - if (rc) - return rc; - - /* crtc vblank or vstartup interrupt */ - if (enable) { - rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc); - } else { - rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc); - } - if (rc) return rc; /* - * hubp surface flip interrupt - * - * We have no guarantee that the frontend index maps to the same - * backend index - some even map to more than one. - * - * TODO: Use a different interrupt or check DC itself for the mapping. + * VLINE0 (crtc_irq) and GRPH_PFLIP (pageflip_irq) are only used on + * DCE. On DCN, vblank and pageflip completion are delivered from + * VUPDATE_NO_LOCK (enabled above), so don't touch them here. */ - if (enable) { - rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc); - } else { - rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type); - drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc); - } + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == 0) { + /* crtc vblank or vstartup interrupt */ + if (enable) { + rc = amdgpu_irq_get(adev, &adev->crtc_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Get crtc_irq ret=%d\n", rc); + } else { + rc = amdgpu_irq_put(adev, &adev->crtc_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Put crtc_irq ret=%d\n", rc); + } - if (rc) - return rc; + if (rc) + return rc; + + /* + * hubp surface flip interrupt + * + * We have no guarantee that the frontend index maps to the same + * backend index - some even map to more than one. + * + * TODO: Use a different interrupt or check DC itself for the mapping. + */ + if (enable) { + rc = amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Get pageflip_irq ret=%d\n", rc); + } else { + rc = amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type); + drm_dbg_vbl(crtc->dev, "Put pageflip_irq ret=%d\n", rc); + } + + if (rc) + return rc; + } #if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) /* crtc vline0 interrupt, only available on DCN+ */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c index 8d452dbd4eef7..9b5252fd64467 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c @@ -194,7 +194,6 @@ static ssize_t dp_link_settings_read(struct file *f, char __user *buf, char *rd_buf_ptr = NULL; const uint32_t rd_buf_size = 100; uint32_t result = 0; - uint8_t str_len = 0; int r; if (*pos & 3 || size & 3) @@ -206,29 +205,26 @@ static ssize_t dp_link_settings_read(struct file *f, char __user *buf, rd_buf_ptr = rd_buf; - str_len = strlen("Current: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Current: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Current: %d 0x%x %d ", link->cur_link_settings.lane_count, link->cur_link_settings.link_rate, link->cur_link_settings.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Verified: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Verified: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Verified: %d 0x%x %d ", link->verified_link_cap.lane_count, link->verified_link_cap.link_rate, link->verified_link_cap.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Reported: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Reported: %d 0x%x %d ", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Reported: %d 0x%x %d ", link->reported_link_cap.lane_count, link->reported_link_cap.link_rate, link->reported_link_cap.link_spread); - rd_buf_ptr += str_len; - str_len = strlen("Preferred: %d 0x%x %d "); - snprintf(rd_buf_ptr, str_len, "Preferred: %d 0x%x %d\n", + rd_buf_ptr += scnprintf(rd_buf_ptr, rd_buf_size - (rd_buf_ptr - rd_buf), + "Preferred: %d 0x%x %d\n", link->preferred_link_setting.lane_count, link->preferred_link_setting.link_rate, link->preferred_link_setting.link_spread); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index d0f770dd0a956..ddda474d58788 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -56,8 +56,11 @@ static u32 edid_extract_panel_id(struct edid *edid) (u32)EDID_PRODUCT_ID(edid); } -static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct dc_edid_caps *edid_caps) +static void apply_edid_quirks(struct dc_link *link, struct edid *edid, + struct dc_edid_caps *edid_caps) { + struct amdgpu_dm_connector *aconnector = link->priv; + struct drm_device *dev = aconnector->base.dev; uint32_t panel_id = edid_extract_panel_id(edid); switch (panel_id) { @@ -87,6 +90,22 @@ static void apply_edid_quirks(struct drm_device *dev, struct edid *edid, struct drm_dbg_driver(dev, "Disabling VSC on monitor with panel id %X\n", panel_id); edid_caps->panel_patch.disable_colorimetry = true; break; + /* Workaround for monitors that get corrupted by the PHY SSC reduction */ + case drm_edid_encode_panel_id('D', 'E', 'L', 0x4147): + drm_dbg_driver(dev, "Skip PHY SSC reduction on panel id %X\n", panel_id); + link->wa_flags.skip_phy_ssc_reduction = true; + break; + /* + * Workaround for Apple Studio Display which exposes a 2x1 tiled panel + * over two SST DP links. Hide the secondary tile from userspace so + * compositors drive a single 5K stream on the primary link only. + */ + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE3A): + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE42): + case drm_edid_encode_panel_id('A', 'P', 'P', 0xAE46): + drm_dbg_driver(dev, "Hiding secondary tile on panel id %X\n", panel_id); + edid_caps->panel_patch.disable_second_tile = true; + break; default: return; } @@ -108,7 +127,6 @@ enum dc_edid_status dm_helpers_parse_edid_caps( { struct amdgpu_dm_connector *aconnector = link->priv; struct drm_connector *connector = &aconnector->base; - struct drm_device *dev = connector->dev; struct edid *edid_buf = edid ? (struct edid *) edid->raw_edid : NULL; struct cea_sad *sads; int sad_count = -1; @@ -138,7 +156,10 @@ enum dc_edid_status dm_helpers_parse_edid_caps( edid_caps->edid_hdmi = connector->display_info.is_hdmi; - apply_edid_quirks(dev, edid_buf, edid_caps); + if (edid_caps->edid_hdmi) + edid_caps->qs_bit = connector->display_info.rgb_quant_range_selectable; + + apply_edid_quirks(link, edid_buf, edid_caps); sad_count = drm_edid_to_sad((struct edid *) edid->raw_edid, &sads); if (sad_count <= 0) diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c index 7da7b41bd0925..b98d946a8d2ee 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c @@ -276,13 +276,20 @@ static void dcn32_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr, struct dtbclk_dto_params dto_params = {0}; /* use mask to program DTO once per tg */ - if (pipe_ctx->stream_res.tg && + if (pipe_ctx->stream && pipe_ctx->stream_res.tg && !(tg_mask & (1 << pipe_ctx->stream_res.tg->inst))) { tg_mask |= (1 << pipe_ctx->stream_res.tg->inst); dto_params.otg_inst = pipe_ctx->stream_res.tg->inst; dto_params.ref_dtbclk_khz = ref_dtbclk_khz; + if (dccg->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) + dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10; + + if (dc_is_hdmi_signal(pipe_ctx->stream->signal) || + dc_is_dvi_signal(pipe_ctx->stream->signal)) + dto_params.is_hdmi = true; + dccg->funcs->set_dtbclk_dto(clk_mgr->dccg, &dto_params); //dccg->funcs->set_audio_dtbclk_dto(clk_mgr->dccg, &dto_params); } diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c index 817a0253d10e5..d1785a5c7f85f 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c @@ -264,13 +264,20 @@ static void dcn35_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr, struct dtbclk_dto_params dto_params = {0}; /* use mask to program DTO once per tg */ - if (pipe_ctx->stream_res.tg && + if (pipe_ctx->stream && pipe_ctx->stream_res.tg && !(tg_mask & (1 << pipe_ctx->stream_res.tg->inst))) { tg_mask |= (1 << pipe_ctx->stream_res.tg->inst); dto_params.otg_inst = pipe_ctx->stream_res.tg->inst; dto_params.ref_dtbclk_khz = ref_dtbclk_khz; + if (dccg->ctx->dc->link_srv->dp_is_128b_132b_signal(pipe_ctx)) + dto_params.pixclk_khz = pipe_ctx->stream->timing.pix_clk_100hz / 10; + + if (dc_is_hdmi_signal(pipe_ctx->stream->signal) || + dc_is_dvi_signal(pipe_ctx->stream->signal)) + dto_params.is_hdmi = true; + dccg->funcs->set_dtbclk_dto(clk_mgr->dccg, &dto_params); //dccg->funcs->set_audio_dtbclk_dto(clk_mgr->dccg, &dto_params); } diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 0347174173f34..31094d5c0f588 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -3602,6 +3602,7 @@ static void commit_planes_do_stream_update(struct dc *dc, dc->hwss.setup_periodic_interrupt(dc, pipe_ctx); if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) || + stream_update->output_color_space || stream_update->vrr_infopacket || stream_update->vsc_infopacket || stream_update->vsp_infopacket || @@ -3690,27 +3691,34 @@ static void commit_planes_do_stream_update(struct dc *dc, resource_build_test_pattern_params(&context->res_ctx, pipe_ctx); } + // DPMS should not use partially updated pipe context + struct pipe_ctx *dpms_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[j]; + if (stream_update->dpms_off) { if (*stream_update->dpms_off) { - dc->link_srv->set_dpms_off(pipe_ctx); + dc->link_srv->set_dpms_off(dpms_pipe_ctx); /* for dpms, keep acquired resources*/ - if (pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) - pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio); + if (dpms_pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) { + struct audio *audio = dpms_pipe_ctx->stream_res.audio; + + audio->funcs->az_disable(audio); + } dc->optimized_required = true; } else { if (get_seamless_boot_stream_count(context) == 0) dc->hwss.prepare_bandwidth(dc, dc->current_state); - dc->link_srv->set_dpms_on(dc->current_state, pipe_ctx); + dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx); } - } else if (pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && stream_update->output_color_space - && !stream->dpms_off && dc_is_dp_signal(pipe_ctx->stream->signal)) { + } else if (dpms_pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && + stream_update->output_color_space && + !stream->dpms_off && dc_is_dp_signal(dpms_pipe_ctx->stream->signal)) { /* * Workaround for firmware issue in some receivers where they don't pick up * correct output color space unless DP link is disabled/re-enabled */ - dc->link_srv->set_dpms_on(dc->current_state, pipe_ctx); + dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx); } if (stream_update->abm_level && pipe_ctx->stream_res.abm) { @@ -5427,6 +5435,51 @@ void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src) dal_irq_service_ack(dc->res_pool->irqs, src); } +/* + * dc_get_flip_pending_on_otg() - Check if a GRPH_FLIP is still pending on OTG + * + * @dc: display core context @otg_inst: OTG instance to query + * + * Reads the HUBP flip-pending status for the pipe(s) bound to @otg_inst, + * returning true if any of them has not yet latched its programmed surface + * address. + * + * Unlike dc_plane_get_status(), this does not take or mutate a dc_plane_state, + * so it is safe to call from interrupt context without racing a concurrent + * commit that may be updating plane state. + * + * Return: true if a flip is still pending on the OTG, false otherwise. + */ +bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst) +{ + bool flip_pending = false; + int i; + + if (!dc || !dc->current_state) + return false; + + dc_exit_ips_for_hw_access(dc); + + for (i = 0; i < dc->res_pool->pipe_count; i++) { + struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i]; + struct hubp *hubp = pipe_ctx->plane_res.hubp; + + if (!pipe_ctx->plane_state || !pipe_ctx->stream_res.tg) + continue; + + if (pipe_ctx->stream_res.tg->inst != otg_inst) + continue; + + if (hubp && hubp->funcs->hubp_is_flip_pending && + hubp->funcs->hubp_is_flip_pending(hubp)) { + flip_pending = true; + break; + } + } + + return flip_pending; +} + void dc_power_down_on_boot(struct dc *dc) { if (dc->ctx->dce_environment != DCE_ENV_VIRTUAL_HW && diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c index d82b1cb467f4b..f56d1ae879ee3 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c @@ -51,7 +51,8 @@ enum dc_color_space_type { COLOR_SPACE_RGB_LIMITED_TYPE, COLOR_SPACE_YCBCR601_TYPE, COLOR_SPACE_YCBCR709_TYPE, - COLOR_SPACE_YCBCR2020_TYPE, + COLOR_SPACE_YCBCR2020_LIMITED_TYPE, + COLOR_SPACE_YCBCR2020_FULL_TYPE, COLOR_SPACE_YCBCR601_LIMITED_TYPE, COLOR_SPACE_YCBCR709_LIMITED_TYPE, COLOR_SPACE_YCBCR709_BLACK_TYPE, @@ -103,9 +104,15 @@ static const struct out_csc_color_matrix_type output_csc_matrix[] = { { 0xE00, 0xF349, 0xFEB7, 0x1000, 0x6CE, 0x16E3, 0x24F, 0x200, 0xFCCB, 0xF535, 0xE00, 0x1000} }, - { COLOR_SPACE_YCBCR2020_TYPE, + /* Corrected. Not included in the TODO above. */ + { COLOR_SPACE_YCBCR2020_LIMITED_TYPE, + { 0x0E04, 0xF31D, 0xFEDF, 0x1004, + 0x0733, 0x1294, 0x01A0, 0x0201, + 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, + /* Corrected. Not included in the TODO above. */ + { COLOR_SPACE_YCBCR2020_FULL_TYPE, { 0x1000, 0xF149, 0xFEB7, 0x1004, - 0x0868, 0x15B2, 0x01E6, 0x201, + 0x0868, 0x15B2, 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} }, { COLOR_SPACE_YCBCR709_BLACK_TYPE, { 0x0000, 0x0000, 0x0000, 0x1000, @@ -172,14 +179,14 @@ static bool is_ycbcr709_type( return ret; } -static bool is_ycbcr2020_type( - enum dc_color_space color_space) +static bool is_ycbcr2020_limited_type(enum dc_color_space color_space) { - bool ret = false; + return color_space == COLOR_SPACE_2020_YCBCR_LIMITED; +} - if (color_space == COLOR_SPACE_2020_YCBCR_LIMITED || color_space == COLOR_SPACE_2020_YCBCR_FULL) - ret = true; - return ret; +static bool is_ycbcr2020_full_type(enum dc_color_space color_space) +{ + return color_space == COLOR_SPACE_2020_YCBCR_FULL; } static bool is_ycbcr709_limited_type( @@ -208,8 +215,10 @@ static enum dc_color_space_type get_color_space_type(enum dc_color_space color_s type = COLOR_SPACE_YCBCR601_LIMITED_TYPE; else if (is_ycbcr709_limited_type(color_space)) type = COLOR_SPACE_YCBCR709_LIMITED_TYPE; - else if (is_ycbcr2020_type(color_space)) - type = COLOR_SPACE_YCBCR2020_TYPE; + else if (is_ycbcr2020_limited_type(color_space)) + type = COLOR_SPACE_YCBCR2020_LIMITED_TYPE; + else if (is_ycbcr2020_full_type(color_space)) + type = COLOR_SPACE_YCBCR2020_FULL_TYPE; else if (color_space == COLOR_SPACE_YCBCR709) type = COLOR_SPACE_YCBCR709_BLACK_TYPE; else if (color_space == COLOR_SPACE_YCBCR709_BLACK) diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h index 98f0b6b3c2136..390f948dad393 100644 --- a/drivers/gpu/drm/amd/display/dc/dc.h +++ b/drivers/gpu/drm/amd/display/dc/dc.h @@ -2611,6 +2611,7 @@ enum dc_irq_source dc_interrupt_to_irq_source( uint32_t ext_id); bool dc_interrupt_set(struct dc *dc, enum dc_irq_source src, bool enable); void dc_interrupt_ack(struct dc *dc, enum dc_irq_source src); +bool dc_get_flip_pending_on_otg(struct dc *dc, int otg_inst); enum dc_irq_source dc_get_hpd_irq_source_at_index( struct dc *dc, uint32_t link_index); diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c index 6518d5639d66a..04bc014906f80 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c @@ -949,7 +949,10 @@ void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv) { uint32_t i; - if (!dc_dmub_srv || !dc_dmub_srv->dmub) { + if (!dc_dmub_srv) + return; + + if (!dc_dmub_srv->dmub) { DC_LOG_ERROR("%s: invalid parameters.", __func__); return; } @@ -1146,7 +1149,10 @@ void dc_dmub_srv_enable_dpia_trace(const struct dc *dc) { struct dc_dmub_srv *dc_dmub_srv = dc->ctx->dmub_srv; - if (!dc_dmub_srv || !dc_dmub_srv->dmub) { + if (!dc_dmub_srv) + return; + + if (!dc_dmub_srv->dmub) { DC_LOG_ERROR("%s: invalid parameters.", __func__); return; } diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h index b5aa03a3e39cf..ff06a1dae5c61 100644 --- a/drivers/gpu/drm/amd/display/dc/dc_types.h +++ b/drivers/gpu/drm/amd/display/dc/dc_types.h @@ -178,6 +178,7 @@ struct dc_panel_patch { unsigned int skip_audio_sab_check; unsigned int mst_start_top_delay; unsigned int remove_sink_ext_caps; + bool disable_second_tile; unsigned int disable_colorimetry; uint8_t blankstream_before_otg_off; bool oled_optimize_display_on; diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c index b4f5b4a6331a1..20c52b0038d10 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c @@ -45,9 +45,7 @@ clk_src->base.ctx #define DC_LOGGER \ - calc_pll_cs->ctx->logger -#define DC_LOGGER_INIT() \ - struct calc_pll_clock_source *calc_pll_cs = &clk_src->calc_pll + CTX->logger #undef FN #define FN(reg_name, field_name) \ @@ -291,6 +289,7 @@ static bool calc_pll_dividers_in_range( } static uint32_t calculate_pixel_clock_pll_dividers( + struct dce110_clk_src *clk_src, struct calc_pll_clock_source *calc_pll_cs, struct pll_settings *pll_settings) { @@ -479,7 +478,7 @@ static uint32_t dce110_get_pix_clk_dividers_helper ( { uint32_t field = 0; uint32_t pll_calc_error = MAX_PLL_CALC_ERROR; - DC_LOGGER_INIT(); + /* Check if reference clock is external (not pcie/xtalin) * HW Dce80 spec: * 00 - PCIE_REFCLK, 01 - XTALIN, 02 - GENERICA, 03 - GENERICB @@ -522,12 +521,14 @@ static uint32_t dce110_get_pix_clk_dividers_helper ( /*Calculate Dividers by HDMI object, no SS case or SS case */ pll_calc_error = calculate_pixel_clock_pll_dividers( + clk_src, &clk_src->calc_pll_hdmi, pll_settings); else /*Calculate Dividers by default object, no SS case or SS case */ pll_calc_error = calculate_pixel_clock_pll_dividers( + clk_src, &clk_src->calc_pll, pll_settings); @@ -572,7 +573,6 @@ static uint32_t dce110_get_pix_clk_dividers( { struct dce110_clk_src *clk_src = TO_DCE110_CLK_SRC(cs); uint32_t pll_calc_error = MAX_PLL_CALC_ERROR; - DC_LOGGER_INIT(); if (pix_clk_params == NULL || pll_settings == NULL || pix_clk_params->requested_pix_clk_100hz == 0) { @@ -604,7 +604,6 @@ static uint32_t dce112_get_pix_clk_dividers( struct pll_settings *pll_settings) { struct dce110_clk_src *clk_src = TO_DCE110_CLK_SRC(cs); - DC_LOGGER_INIT(); if (pix_clk_params == NULL || pll_settings == NULL || pix_clk_params->requested_pix_clk_100hz == 0) { @@ -1370,8 +1369,6 @@ static uint32_t dcn3_get_pix_clk_dividers( unsigned long long actual_pix_clk_100Hz = pix_clk_params ? pix_clk_params->requested_pix_clk_100hz : 0; struct dce110_clk_src *clk_src = TO_DCE110_CLK_SRC(cs); - DC_LOGGER_INIT(); - if (pix_clk_params == NULL || pll_settings == NULL || pix_clk_params->requested_pix_clk_100hz == 0) { DC_LOG_ERROR( @@ -1441,7 +1438,6 @@ static const struct clock_source_funcs dce110_clk_src_funcs = { .get_pixel_clk_frequency_100hz = get_pixel_clk_frequency_100hz }; - static void get_ss_info_from_atombios( struct dce110_clk_src *clk_src, enum as_signal_type as_signal, @@ -1454,7 +1450,7 @@ static void get_ss_info_from_atombios( struct spread_spectrum_info *ss_info_cur; struct spread_spectrum_data *ss_data_cur; uint32_t i; - DC_LOGGER_INIT(); + if (ss_entries_num == NULL) { DC_LOG_SYNC( "Invalid entry !!!\n"); @@ -1589,6 +1585,7 @@ static void ss_info_from_atombios_create( } static bool calc_pll_max_vco_construct( + struct dce110_clk_src *clk_src, struct calc_pll_clock_source *calc_pll_cs, struct calc_pll_clock_source_init_data *init_data) { @@ -1740,6 +1737,7 @@ bool dce110_clk_src_construct( ss_info_from_atombios_create(clk_src); if (!calc_pll_max_vco_construct( + clk_src, &clk_src->calc_pll, &calc_pll_cs_init_data)) { ASSERT_CRITICAL(false); @@ -1754,7 +1752,7 @@ bool dce110_clk_src_construct( if (!calc_pll_max_vco_construct( - &clk_src->calc_pll_hdmi, &calc_pll_cs_init_data_hdmi)) { + clk_src, &clk_src->calc_pll_hdmi, &calc_pll_cs_init_data_hdmi)) { ASSERT_CRITICAL(false); goto unexpected_failure; } diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c index 23b35393bf422..351cc84e146bf 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c @@ -115,10 +115,11 @@ static const struct out_csc_color_matrix global_color_matrix[] = { { 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} }, { COLOR_SPACE_2020_RGB_LIMITEDRANGE, { 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} }, -{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, - 0x15B2, 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} }, +/* COLOR_SPACE_2020_YCBCR_* values corrected. Not included in the TODO above. */ +{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x0E04, 0xF31D, 0xFEDF, 0x1004, 0x0733, + 0x1294, 0x01A0, 0x201, 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, { COLOR_SPACE_2020_YCBCR_FULL, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, 0x15B2, - 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} } + 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} } }; static bool setup_scaling_configuration( diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c index f7b2be02333cb..870ae2e3c1b16 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_opp_csc_v.c @@ -93,10 +93,11 @@ static const struct out_csc_color_matrix global_color_matrix[] = { { 0x2000, 0, 0, 0, 0, 0x2000, 0, 0, 0, 0, 0x2000, 0} }, { COLOR_SPACE_2020_RGB_LIMITEDRANGE, { 0x1B67, 0, 0, 0x201, 0, 0x1B67, 0, 0x201, 0, 0, 0x1B67, 0x201} }, -{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, - 0x15B2, 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} }, +/* COLOR_SPACE_2020_YCBCR_* values corrected. Not included in the TODO above. */ +{ COLOR_SPACE_2020_YCBCR_LIMITED, { 0x0E04, 0xF31D, 0xFEDF, 0x1004, 0x0733, + 0x1294, 0x01A0, 0x201, 0xFC16, 0xF5E6, 0x0E04, 0x1004} }, { COLOR_SPACE_2020_YCBCR_FULL, { 0x1000, 0xF149, 0xFEB7, 0x1004, 0x0868, 0x15B2, - 0x01E6, 0x201, 0xFB88, 0xF478, 0x1000, 0x1004} } + 0x01E6, 0, 0xFB88, 0xF478, 0x1000, 0x1004} } }; enum csc_color_mode { diff --git a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c index 3b866e876bf4d..9fea55fddded4 100644 --- a/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c +++ b/drivers/gpu/drm/amd/display/dc/dml2/dml2_translation_helper.c @@ -299,6 +299,7 @@ void dml2_init_socbb_params(struct dml2_context *dml2, const struct dc *in_dc, s out->smn_latency_us = 2; out->dispclk_dppclk_vco_speed_mhz = 3600; out->pct_ideal_dram_bw_after_urgent_pixel_only = 65.0; + out->gpuvm_min_page_size_kbytes = 4; break; case dml_project_dcn401: diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c index b62bbadb0d440..029f59a58c780 100644 --- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c +++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp.c @@ -389,9 +389,11 @@ void dscl401_spl_calc_lb_num_partitions( lb_memory_size_a = 1290; } else if (lb_config == LB_MEMORY_CONFIG_3) { if (scl_data->viewport.width == scl_data->h_active && - scl_data->viewport.height == scl_data->v_active) { + scl_data->viewport.height == scl_data->v_active && + scl_data->taps.h_taps == 1 && scl_data->taps.v_taps == 1) { /* 420 mode: luma using all 3 mem from Y, plus 3rd mem from Cr and Cb */ /* use increased LB size for calculation only if Scaler not enabled */ + /* Scaler is forced on when sharpening is on. Add check for taps = 1 */ lb_memory_size = 970 + 1290 + 1170 + 1170 + 1170; lb_memory_size_c = 970 + 1290; lb_memory_size_a = 970 + 1290 + 1170; @@ -403,8 +405,10 @@ void dscl401_spl_calc_lb_num_partitions( } } else { if (scl_data->viewport.width == scl_data->h_active && - scl_data->viewport.height == scl_data->v_active) { + scl_data->viewport.height == scl_data->v_active && + scl_data->taps.h_taps == 1 && scl_data->taps.v_taps == 1) { /* use increased LB size for calculation only if Scaler not enabled */ + /* Scaler is forced on when sharpening is on. Add check for taps = 1 */ lb_memory_size = 970 + 1290 + 1170; lb_memory_size_c = 970 + 1290 + 1170; lb_memory_size_a = 970 + 1290 + 1170; diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c index 9b026600b90e8..99625cba3df66 100644 --- a/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c +++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn10/dcn10_hubp.c @@ -769,8 +769,7 @@ bool hubp1_is_flip_pending(struct hubp *hubp) if (flip_pending) return true; - if (hubp && - earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) + if (earliest_inuse_address.grph.addr.quad_part != hubp->request_address.grph.addr.quad_part) return true; return false; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index 8f86177de48dc..8eec1feafc4ad 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -1266,8 +1266,30 @@ void dce110_blank_stream(struct pipe_ctx *pipe_ctx) void dce110_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) { - if (pipe_ctx != NULL && pipe_ctx->stream_res.stream_enc != NULL) + if (pipe_ctx == NULL || pipe_ctx->stream_res.stream_enc == NULL) + return; + + if (dc_is_hdmi_signal(pipe_ctx->stream->signal)) { pipe_ctx->stream_res.stream_enc->funcs->set_avmute(pipe_ctx->stream_res.stream_enc, enable); + + /* Wait for three frames to make sure AV mute is sent out. + * Some HDMI sinks need additional GCP packets to properly + * process the mute state, especially after link re-establishment + * with HDMI 2.0 scrambling enabled. + */ + if (enable && pipe_ctx->stream_res.tg && + pipe_ctx->stream_res.tg->funcs->is_tg_enabled && + pipe_ctx->stream_res.tg->funcs->wait_for_state && + pipe_ctx->stream_res.tg->funcs->is_tg_enabled(pipe_ctx->stream_res.tg)) { + int i; + + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + for (i = 0; i < 3; i++) { + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + } + } + } } enum audio_dto_source translate_to_dto_source(enum controller_id crtc_id) @@ -1731,20 +1753,22 @@ static void power_down_encoders(struct dc *dc) for (i = 0; i < dc->link_count; i++) { struct dc_link *link = dc->links[i]; - struct link_encoder *link_enc = link->link_enc; + struct link_encoder *link_enc = link_enc_cfg_get_link_enc(link); enum signal_type signal = link->connector_signal; dc->link_srv->blank_dp_stream(link, false); if (signal != SIGNAL_TYPE_EDP) signal = SIGNAL_TYPE_NONE; - if (link->ep_type == DISPLAY_ENDPOINT_PHY) + if (link->ep_type == DISPLAY_ENDPOINT_PHY && link_enc) link_enc->funcs->disable_output(link_enc, signal); if (link->fec_state == dc_link_fec_enabled) { - link_enc->funcs->fec_set_enable(link_enc, false); - link_enc->funcs->fec_set_ready(link_enc, false); - link->fec_state = dc_link_fec_not_ready; + if (link_enc && link_enc->funcs->fec_set_enable && link_enc->funcs->fec_set_ready) { + link_enc->funcs->fec_set_enable(link_enc, false); + link_enc->funcs->fec_set_ready(link_enc, false); + link->fec_state = dc_link_fec_not_ready; + } } link->link_status.link_active = false; diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c index 731645a2ab9aa..20b408b688248 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c @@ -837,13 +837,19 @@ void dcn30_set_avmute(struct pipe_ctx *pipe_ctx, bool enable) pipe_ctx->stream_res.stream_enc, enable); - /* Wait for two frame to make sure AV mute is sent out */ + /* Wait for three frames to make sure AV mute is sent out. + * Some HDMI sinks need additional GCP packets to properly + * process the mute state, especially after link re-establishment + * with HDMI 2.0 scrambling enabled. + */ if (enable && pipe_ctx->stream_res.tg->funcs->is_tg_enabled(pipe_ctx->stream_res.tg)) { + int i; + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); - pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); - pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); - pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); - pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + for (i = 0; i < 3; i++) { + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VBLANK); + pipe_ctx->stream_res.tg->funcs->wait_for_state(pipe_ctx->stream_res.tg, CRTC_STATE_VACTIVE); + } } } } diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c index 2676ae9f6fe83..5f70793b8372e 100644 --- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c +++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c @@ -143,6 +143,8 @@ static void dp_retrain_link_dp_test(struct dc_link *link, // Set DPMS on with stream update // Cache all streams on current link since dc_update_planes_and_stream might kill current_state for (i = 0; i < MAX_PIPES; i++) { + if (state->streams[i] && state->streams[i]->is_phantom) + continue; if (state->streams[i] && state->streams[i]->link && state->streams[i]->link == link) streams_on_link[num_streams_on_link++] = state->streams[i]; } diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c index 31b404e9b9ae0..37ca3bb734c38 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c @@ -979,8 +979,11 @@ static bool detect_link_and_local_sink(struct dc_link *link, link->link_enc->features.flags.bits.DP_IS_USB_C == 1) { /* if alt mode times out, return false */ - if (!wait_for_entering_dp_alt_mode(link)) + if (!wait_for_entering_dp_alt_mode(link)) { + if (prev_sink) + dc_sink_release(prev_sink); return false; + } } if (!detect_dp(link, &sink_caps, reason)) { diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c index 8a3c18ae97a7b..8daaa3014f999 100644 --- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c +++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c @@ -103,6 +103,11 @@ static int get_estimated_bw(struct dc_link *link) { uint8_t bw_estimated_bw = 0; + if (link->dpia_bw_alloc_config.bw_granularity == 0) { + DC_LOG_ERROR("%s: BW granularity is zero!\n", __func__); + return 0; + } + core_link_read_dpcd( link, ESTIMATED_BW, diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c index ef146f253577a..91fd73f4a96cf 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c @@ -957,6 +957,11 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( for (i = 0; i < pool->stream_enc_count; i++) { if (!res_ctx->is_stream_enc_acquired[i] && pool->stream_enc[i]) { + /* DP/MST needs a digital encoder; skip analog/no-DP encoders */ + if (dc_is_dp_signal(stream->signal) && + (!pool->stream_enc[i]->funcs || + !pool->stream_enc[i]->funcs->dp_set_stream_attribute)) + continue; /* Store first available for MST second display * in daisy chain use case */ @@ -980,7 +985,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link( * required for non DP connectors. */ - if (j >= 0 && link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT) + if (j >= 0 && dc_is_dp_signal(stream->signal)) return pool->stream_enc[j]; return NULL; diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c index ae858a43c35f4..b3d55cac35694 100644 --- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c +++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c @@ -447,8 +447,6 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream, * * @stream: contains data we may need to construct VSIF (i.e. timing_3d_format, etc.) * @info_packet: output structure where to store VSIF - * @ALLMEnabled: indicates whether ALLM HF-VSIF should be generated - * @ALLMValue: ALLM bit value to advertise in HF-VSIF */ void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream, struct dc_info_packet *info_packet) diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c index 8253d2977408d..d1c4d7f4accc3 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c @@ -348,7 +348,8 @@ int amdgpu_dpm_switch_power_profile(struct amdgpu_device *adev, const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; int ret = 0; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) || + amdgpu_in_reset(adev)) return 0; if (pp_funcs && pp_funcs->switch_power_profile) { @@ -367,7 +368,8 @@ int amdgpu_dpm_pause_power_profile(struct amdgpu_device *adev, const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; int ret = 0; - if (amdgpu_sriov_vf(adev)) + if (amdgpu_sriov_vf(adev) || + amdgpu_in_reset(adev)) return 0; if (pp_funcs && pp_funcs->pause_power_profile) { @@ -1183,17 +1185,28 @@ int amdgpu_dpm_dispatch_task(struct amdgpu_device *adev, return ret; } -int amdgpu_dpm_get_pp_table(struct amdgpu_device *adev, char **table) +int amdgpu_dpm_get_pp_table(struct amdgpu_device *adev, char *table, + size_t size) { const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; + char *pptable = NULL; int ret = 0; - if (!pp_funcs->get_pp_table) - return 0; + if ((!table && size) || (table && !size)) + return -EINVAL; + + if (amdgpu_sriov_vf(adev) || !pp_funcs->get_pp_table || adev->scpm_enabled) + return -EOPNOTSUPP; mutex_lock(&adev->pm.mutex); ret = pp_funcs->get_pp_table(adev->powerplay.pp_handle, - table); + &pptable); + if (ret > 0 && !pptable) { + ret = -EINVAL; + } else if (ret > 0 && table) { + ret = min_t(size_t, ret, size); + memcpy(table, pptable, ret); + } mutex_unlock(&adev->pm.mutex); return ret; @@ -1442,17 +1455,23 @@ int amdgpu_dpm_set_power_profile_mode(struct amdgpu_device *adev, return ret; } -int amdgpu_dpm_get_gpu_metrics(struct amdgpu_device *adev, void **table) +ssize_t amdgpu_dpm_get_gpu_metrics(struct amdgpu_device *adev, void *buf, + size_t size) { const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; - int ret = 0; + void *table; + ssize_t ret; if (!pp_funcs->get_gpu_metrics) return 0; mutex_lock(&adev->pm.mutex); ret = pp_funcs->get_gpu_metrics(adev->powerplay.pp_handle, - table); + &table); + if (ret > 0) { + ret = min_t(ssize_t, ret, size); + memcpy(buf, table, ret); + } mutex_unlock(&adev->pm.mutex); return ret; @@ -1715,7 +1734,10 @@ int amdgpu_dpm_set_pp_table(struct amdgpu_device *adev, const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; int ret = 0; - if (!pp_funcs->set_pp_table) + if (!buf || !size) + return -EINVAL; + + if (amdgpu_sriov_vf(adev) || !pp_funcs->set_pp_table || adev->scpm_enabled) return -EOPNOTSUPP; mutex_lock(&adev->pm.mutex); diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c index 0d72118a12b4c..6cdb8dda97e0f 100644 --- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c +++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c @@ -40,6 +40,8 @@ #define DEVICE_ATTR_IS(_name) (attr_id == device_attr_id__##_name) +#define power_2_mwatt(power) (((power) >> 8) * 1000 + ((power) & 0xff)) + struct od_attribute { struct kobj_attribute attribute; struct list_head entry; @@ -554,25 +556,19 @@ static ssize_t amdgpu_get_pp_table(struct device *dev, { struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = drm_to_adev(ddev); - char *table = NULL; int size, ret; ret = amdgpu_pm_get_access_if_active(adev); if (ret) return ret; - size = amdgpu_dpm_get_pp_table(adev, &table); + size = amdgpu_dpm_get_pp_table(adev, buf, PAGE_SIZE - 1); amdgpu_pm_put_access(adev); if (size <= 0) return size; - if (size >= PAGE_SIZE) - size = PAGE_SIZE - 1; - - memcpy(buf, table, size); - return size; } @@ -1752,7 +1748,6 @@ static ssize_t amdgpu_get_gpu_metrics(struct device *dev, { struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = drm_to_adev(ddev); - void *gpu_metrics; ssize_t size = 0; int ret; @@ -1760,15 +1755,10 @@ static ssize_t amdgpu_get_gpu_metrics(struct device *dev, if (ret) return ret; - size = amdgpu_dpm_get_gpu_metrics(adev, &gpu_metrics); + size = amdgpu_dpm_get_gpu_metrics(adev, buf, PAGE_SIZE - 1); if (size <= 0) goto out; - if (size >= PAGE_SIZE) - size = PAGE_SIZE - 1; - - memcpy(buf, gpu_metrics, size); - out: amdgpu_pm_put_access(adev); @@ -1863,12 +1853,12 @@ static ssize_t amdgpu_set_smartshift_bias(struct device *dev, { struct drm_device *ddev = dev_get_drvdata(dev); struct amdgpu_device *adev = drm_to_adev(ddev); - int r = 0; + int r; int bias = 0; r = kstrtoint(buf, 10, &bias); if (r) - goto out; + return r; r = amdgpu_pm_get_access(adev); if (r < 0) @@ -1880,14 +1870,12 @@ static ssize_t amdgpu_set_smartshift_bias(struct device *dev, bias = AMDGPU_SMARTSHIFT_MIN_BIAS; amdgpu_smartshift_bias = bias; - r = count; /* TODO: update bias level with SMU message */ -out: amdgpu_pm_put_access(adev); - return r; + return count; } static int ss_power_attr_update(struct amdgpu_device *adev, struct amdgpu_device_attr *attr, @@ -2506,7 +2494,7 @@ static struct amdgpu_device_attr amdgpu_device_attrs[] = { AMDGPU_DEVICE_ATTR_RO(pp_num_states, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RO(pp_cur_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), AMDGPU_DEVICE_ATTR_RW(pp_force_state, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), - AMDGPU_DEVICE_ATTR_RW(pp_table, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF), + AMDGPU_DEVICE_ATTR_RW(pp_table, ATTR_FLAG_BASIC), AMDGPU_DEVICE_ATTR_RW(pp_dpm_sclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF, .attr_update = pp_dpm_clk_default_attr_update), AMDGPU_DEVICE_ATTR_RW(pp_dpm_mclk, ATTR_FLAG_BASIC|ATTR_FLAG_ONEVF, @@ -2617,6 +2605,11 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ gc_ver != IP_VERSION(9, 4, 3)) || gc_ver < IP_VERSION(9, 0, 0)) *states = ATTR_STATE_UNSUPPORTED; + + if (adev->scpm_enabled) { + dev_attr->attr.mode &= ~S_IWUGO; + dev_attr->store = NULL; + } } else if (DEVICE_ATTR_IS(gpu_metrics)) { if (gc_ver < IP_VERSION(9, 1, 0)) *states = ATTR_STATE_UNSUPPORTED; @@ -2638,6 +2631,14 @@ static int default_attr_update(struct amdgpu_device *adev, struct amdgpu_device_ if (amdgpu_dpm_get_apu_thermal_limit(adev, &limit) == -EOPNOTSUPP) *states = ATTR_STATE_UNSUPPORTED; + } else if (DEVICE_ATTR_IS(pp_table)) { + int ret; + + ret = amdgpu_dpm_get_pp_table(adev, NULL, 0); + if (ret <= 0) + *states = ATTR_STATE_UNSUPPORTED; + else + *states = ATTR_STATE_SUPPORTED; } switch (gc_ver) { @@ -3261,7 +3262,6 @@ static int amdgpu_hwmon_get_power(struct device *dev, enum amd_pp_sensors sensor) { struct amdgpu_device *adev = dev_get_drvdata(dev); - unsigned int uw; u32 query = 0; int r; @@ -3270,9 +3270,7 @@ static int amdgpu_hwmon_get_power(struct device *dev, return r; /* convert to microwatts */ - uw = (query >> 8) * 1000000 + (query & 0xff) * 1000; - - return uw; + return power_2_mwatt(query) * 1000; } static ssize_t amdgpu_hwmon_show_power_avg(struct device *dev, @@ -4804,7 +4802,7 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a { uint32_t mp1_ver = amdgpu_ip_version(adev, MP1_HWIP, 0); uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); - uint32_t value; + uint32_t value, mwatt, centiwatt; uint64_t value64 = 0; uint32_t query = 0; int size; @@ -4829,17 +4827,21 @@ static int amdgpu_debugfs_pm_info_pp(struct seq_file *m, struct amdgpu_device *a seq_printf(m, "\t%u mV (VDDNB)\n", value); size = sizeof(uint32_t); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_AVG_POWER, (void *)&query, &size)) { + mwatt = power_2_mwatt(query); + centiwatt = DIV_ROUND_CLOSEST(mwatt, 10); if (adev->flags & AMD_IS_APU) - seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (average SoC including CPU)\n", centiwatt / 100, centiwatt % 100); else - seq_printf(m, "\t%u.%02u W (average SoC)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (average SoC)\n", centiwatt / 100, centiwatt % 100); } size = sizeof(uint32_t); if (!amdgpu_dpm_read_sensor(adev, AMDGPU_PP_SENSOR_GPU_INPUT_POWER, (void *)&query, &size)) { + mwatt = power_2_mwatt(query); + centiwatt = DIV_ROUND_CLOSEST(mwatt, 10); if (adev->flags & AMD_IS_APU) - seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (current SoC including CPU)\n", centiwatt / 100, centiwatt % 100); else - seq_printf(m, "\t%u.%02u W (current SoC)\n", query >> 8, query & 0xff); + seq_printf(m, "\t%u.%02u W (current SoC)\n", centiwatt / 100, centiwatt % 100); } size = sizeof(value); seq_printf(m, "\n"); diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h index af48aead12f70..13a834575ed77 100644 --- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h +++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h @@ -487,7 +487,8 @@ int amdgpu_dpm_get_pp_num_states(struct amdgpu_device *adev, int amdgpu_dpm_dispatch_task(struct amdgpu_device *adev, enum amd_pp_task task_id, enum amd_pm_state_type *user_state); -int amdgpu_dpm_get_pp_table(struct amdgpu_device *adev, char **table); +int amdgpu_dpm_get_pp_table(struct amdgpu_device *adev, char *table, + size_t size); int amdgpu_dpm_set_fine_grain_clk_vol(struct amdgpu_device *adev, uint32_t type, long *input, @@ -517,7 +518,8 @@ int amdgpu_dpm_get_power_profile_mode(struct amdgpu_device *adev, char *buf); int amdgpu_dpm_set_power_profile_mode(struct amdgpu_device *adev, long *input, uint32_t size); -int amdgpu_dpm_get_gpu_metrics(struct amdgpu_device *adev, void **table); +ssize_t amdgpu_dpm_get_gpu_metrics(struct amdgpu_device *adev, void *buf, + size_t size); ssize_t amdgpu_dpm_get_xcp_metrics(struct amdgpu_device *adev, int xcp_id, void *table); ssize_t amdgpu_dpm_get_temp_metrics(struct amdgpu_device *adev, diff --git a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c index 0385b24e7e4a2..39928eb71d669 100644 --- a/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/legacy-dpm/si_dpm.c @@ -7238,6 +7238,7 @@ static void si_parse_pplib_clock_info(struct amdgpu_device *adev, struct evergreen_power_info *eg_pi = evergreen_get_pi(adev); struct si_power_info *si_pi = si_get_pi(adev); struct si_ps *ps = si_get_ps(rps); + struct amdgpu_clock_and_voltage_limits *limits; u16 leakage_voltage; struct rv7xx_pl *pl = &ps->performance_levels[index]; int ret; @@ -7297,12 +7298,30 @@ static void si_parse_pplib_clock_info(struct amdgpu_device *adev, si_pi->mvdd_bootup_value = mvdd; } + /* + * Update maximum allowed clock limits. + * VBIOS can contain conflicting values between: + * - the maximum allowed clocks and voltages on AC or DC + * - the clocks and voltages in power states on AC or DC + */ if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == - ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) { - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc; - adev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci; + ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) + limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_ac; + else if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == + ATOM_PPLIB_CLASSIFICATION_UI_BATTERY) + limits = &adev->pm.dpm.dyn_state.max_clock_voltage_on_dc; + else + limits = NULL; + + if (limits) { + if (pl->sclk > limits->sclk) + limits->sclk = pl->sclk; + if (pl->mclk > limits->mclk) + limits->mclk = pl->mclk; + if (pl->vddc > limits->vddc) + limits->vddc = pl->vddc; + if (pl->vddci > limits->vddci) + limits->vddci = pl->vddci; } } @@ -7671,7 +7690,7 @@ static int si_dpm_process_interrupt(struct amdgpu_device *adev, break; } - if (queue_thermal) + if (queue_thermal && amdgpu_dpm) schedule_work(&adev->pm.dpm.thermal.work); return 0; diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c index 554492dfa3c00..5e197c463ed96 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c @@ -634,9 +634,12 @@ static int pp_dpm_get_pp_table(void *handle, char **table) { struct pp_hwmgr *hwmgr = handle; - if (!hwmgr || !hwmgr->pm_en || !hwmgr->soft_pp_table) + if (!hwmgr || !hwmgr->pm_en || !table) return -EINVAL; + if (!hwmgr->soft_pp_table) + return -EOPNOTSUPP; + *table = (char *)hwmgr->soft_pp_table; return hwmgr->soft_pp_table_size; } @@ -665,6 +668,9 @@ static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size) if (!hwmgr || !hwmgr->pm_en) return -EINVAL; + if (size > hwmgr->soft_pp_table_size) + return -EINVAL; + if (!hwmgr->hardcode_pp_table) { hwmgr->hardcode_pp_table = kmemdup(hwmgr->soft_pp_table, hwmgr->soft_pp_table_size, diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c index 1d6e30269d567..4d553be56396f 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/hwmgr.c @@ -106,11 +106,8 @@ int hwmgr_early_init(struct pp_hwmgr *hwmgr) hwmgr->od_enabled = false; switch (hwmgr->chip_id) { case CHIP_BONAIRE: - /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM - * R7 260X cards with old MC ucode: MCLK DPM is unstable - */ - if (adev->pdev->subsystem_vendor == 0x106B || - adev->pdev->device == 0x6658) { + /* R9 M380 in iMac 2015: SMU hangs when enabling MCLK DPM */ + if (adev->pdev->subsystem_vendor == 0x106B) { dev_info(adev->dev, "disabling MCLK DPM on quirky ASIC"); adev->pm.pp_feature &= ~PP_MCLK_DPM_MASK; hwmgr->feature_mask &= ~PP_MCLK_DPM_MASK; diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c index 6529a91a613b6..ba953654a0206 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/smu7_hwmgr.c @@ -2216,12 +2216,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( if (data->vdd_gfx_control == SMU7_VOLTAGE_CONTROL_BY_SVID2) { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddgfx_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddgfx index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddgfx_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddgfx = table_info->vddgfx_lookup_table->entries[voltage_id].us_vdd; } } else { for (entry_id = 0; entry_id < sclk_table->count; ++entry_id) { voltage_id = sclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: sclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } sclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -2229,12 +2241,24 @@ static int smu7_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c index 9ace863792d48..b3fed2a478aab 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c +++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_hwmgr.c @@ -685,10 +685,18 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( case 3: vdt = table_info->vdd_dep_on_pixclk; break; case 4: vdt = table_info->vdd_dep_on_dispclk; break; case 5: vdt = table_info->vdd_dep_on_phyclk; break; + default: + continue; } for (entry_id = 0; entry_id < vdt->count; entry_id++) { voltage_id = vdt->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: clk_dep[%u][%u] vddc index %u out of bounds (%u)\n", + i, entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } vdt->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } @@ -696,23 +704,48 @@ static int vega10_patch_voltage_dependency_tables_with_lookup_table( for (entry_id = 0; entry_id < mm_table->count; ++entry_id) { voltage_id = mm_table->entries[entry_id].vddcInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mm[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mm_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; } for (entry_id = 0; entry_id < mclk_table->count; ++entry_id) { voltage_id = mclk_table->entries[entry_id].vddInd; + if (voltage_id >= table_info->vddc_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddc index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddc_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddc = table_info->vddc_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].vddciInd; + if (voltage_id >= table_info->vddci_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddci index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddci_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].vddci = table_info->vddci_lookup_table->entries[voltage_id].us_vdd; + voltage_id = mclk_table->entries[entry_id].mvddInd; + if (voltage_id >= table_info->vddmem_lookup_table->count) { + pr_err("amdgpu: mclk[%u] vddmem index %u out of bounds (%u)\n", + entry_id, voltage_id, + table_info->vddmem_lookup_table->count); + return -EINVAL; + } mclk_table->entries[entry_id].mvdd = table_info->vddmem_lookup_table->entries[voltage_id].us_vdd; } - return 0; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index cfa4ff5d21b27..8473eaac1e7fc 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -620,7 +620,7 @@ static int smu_sys_get_pp_table(void *handle, return -EOPNOTSUPP; if (!smu_table->power_play_table && !smu_table->hardcode_pptable) - return -EINVAL; + return -EOPNOTSUPP; if (smu_table->hardcode_pptable) *table = smu_table->hardcode_pptable; @@ -2234,7 +2234,6 @@ static int smu_resume(struct amdgpu_ip_block *ip_block) int ret; struct amdgpu_device *adev = ip_block->adev; struct smu_context *smu = adev->powerplay.pp_handle; - struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); if (amdgpu_sriov_multi_vf_mode(adev)) return 0; @@ -2266,18 +2265,6 @@ static int smu_resume(struct amdgpu_ip_block *ip_block) adev->pm.dpm_enabled = true; - if (smu->current_power_limit) { - ret = smu_set_power_limit(smu, smu->current_power_limit); - if (ret && ret != -EOPNOTSUPP) - return ret; - } - - if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL && smu->od_enabled) { - ret = smu_od_edit_dpm_table(smu, PP_OD_COMMIT_DPM_TABLE, NULL, 0); - if (ret) - return ret; - } - dev_info(adev->dev, "SMU is resumed successfully!\n"); return 0; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c index 96a2b5ab87d85..f5c20f2320621 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c @@ -2481,11 +2481,14 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu, uint32_t pp_limit = smu->adev->pm.ac_power ? skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0; + uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + uint32_t od_percent_upper = 0, od_percent_lower = 0; int ret; if (current_power_limit) { - ret = smu_v13_0_get_current_power_limit(smu, &power_limit); + ret = smu_v13_0_get_current_power_limit(smu, current_power_limit); if (ret) *current_power_limit = pp_limit; } @@ -2508,12 +2511,12 @@ static int smu_v13_0_0_get_power_limit(struct smu_context *smu, od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = pp_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = pp_limit * (100 - od_percent_lower); + *min_power_limit = min_limit * (100 - od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c index 43965b1135fe7..0d065e4073655 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c @@ -768,6 +768,10 @@ int smu_v13_0_6_get_metrics_table(struct smu_context *smu, void *metrics_table, amdgpu_asic_invalidate_hdp(smu->adev, NULL); memcpy(smu_table->metrics_table, table->cpu_addr, table_size); + if (!memchr_inv(smu_table->metrics_table, 0xff, + min(16, table_size))) + return -EHWPOISON; + smu_table->metrics_time = jiffies; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c index 0843fa0f5e2ae..427f3569f1d39 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_7_ppt.c @@ -2442,15 +2442,16 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu, uint32_t pp_limit = smu->adev->pm.ac_power ? skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; - uint32_t power_limit = 0, od_percent_upper = 0, od_percent_lower = 0; + uint32_t msg_limit = skutable->MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + uint32_t od_percent_upper = 0, od_percent_lower = 0; int ret; if (current_power_limit) { - ret = smu_v13_0_get_current_power_limit(smu, &power_limit); + ret = smu_v13_0_get_current_power_limit(smu, current_power_limit); if (ret) - power_limit = pp_limit; - - *current_power_limit = power_limit; + *current_power_limit = pp_limit; } if (default_power_limit) @@ -2471,12 +2472,12 @@ static int smu_v13_0_7_get_power_limit(struct smu_context *smu, od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = pp_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = pp_limit * (100 - od_percent_lower); + *min_power_limit = min_limit * (100 - od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c index 969be5ad06819..9e42cc8dd133e 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu14/smu_v14_0_2_ppt.c @@ -1701,19 +1701,23 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu, table_context->power_play_table; PPTable_t *pptable = table_context->driver_pptable; CustomSkuTable_t *skutable = &pptable->CustomSkuTable; - int16_t od_percent_upper = 0, od_percent_lower = 0; + uint32_t pp_limit = smu->adev->pm.ac_power ? + skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : + skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; uint32_t msg_limit = pptable->SkuTable.MsgLimits.Power[PPT_THROTTLER_PPT0][POWER_SOURCE_AC]; - uint32_t power_limit; + uint32_t min_limit = min_t(uint32_t, pp_limit, msg_limit); + uint32_t max_limit = max_t(uint32_t, pp_limit, msg_limit); + int16_t od_percent_upper = 0, od_percent_lower = 0; + int ret; - if (smu_v14_0_get_current_power_limit(smu, &power_limit)) - power_limit = smu->adev->pm.ac_power ? - skutable->SocketPowerLimitAc[PPT_THROTTLER_PPT0] : - skutable->SocketPowerLimitDc[PPT_THROTTLER_PPT0]; + if (current_power_limit) { + ret = smu_v14_0_get_current_power_limit(smu, current_power_limit); + if (ret) + *current_power_limit = pp_limit; + } - if (current_power_limit) - *current_power_limit = power_limit; if (default_power_limit) - *default_power_limit = power_limit; + *default_power_limit = pp_limit; if (powerplay_table) { if (smu->od_enabled && @@ -1727,15 +1731,15 @@ static int smu_v14_0_2_get_power_limit(struct smu_context *smu, } dev_dbg(smu->adev->dev, "od percent upper:%d, od percent lower:%d (default power: %d)\n", - od_percent_upper, od_percent_lower, power_limit); + od_percent_upper, od_percent_lower, pp_limit); if (max_power_limit) { - *max_power_limit = msg_limit * (100 + od_percent_upper); + *max_power_limit = max_limit * (100 + od_percent_upper); *max_power_limit /= 100; } if (min_power_limit) { - *min_power_limit = power_limit * (100 + od_percent_lower); + *min_power_limit = min_limit * (100 + od_percent_lower); *min_power_limit /= 100; } diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_dev.c b/drivers/gpu/drm/arm/display/komeda/komeda_dev.c index 5ba62e637a616..9aad1d1d28ec0 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_dev.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_dev.c @@ -313,7 +313,11 @@ void komeda_dev_destroy(struct komeda_dev *mdev) int komeda_dev_resume(struct komeda_dev *mdev) { - clk_prepare_enable(mdev->aclk); + int err; + + err = clk_prepare_enable(mdev->aclk); + if (err) + return err; mdev->funcs->enable_irq(mdev); diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c index 358c1512b0879..fc1816c634087 100644 --- a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c +++ b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c @@ -74,8 +74,11 @@ static int komeda_platform_probe(struct platform_device *pdev) } pm_runtime_enable(dev); - if (!pm_runtime_enabled(dev)) - komeda_dev_resume(mdrv->mdev); + if (!pm_runtime_enabled(dev)) { + err = komeda_dev_resume(mdrv->mdev); + if (err) + goto err_destroy_mdev; + } mdrv->kms = komeda_kms_attach(mdrv->mdev); if (IS_ERR(mdrv->kms)) { @@ -93,7 +96,7 @@ static int komeda_platform_probe(struct platform_device *pdev) pm_runtime_disable(dev); else komeda_dev_suspend(mdrv->mdev); - +err_destroy_mdev: komeda_dev_destroy(mdrv->mdev); free_mdrv: @@ -139,11 +142,12 @@ static int __maybe_unused komeda_pm_suspend(struct device *dev) static int __maybe_unused komeda_pm_resume(struct device *dev) { struct komeda_drv *mdrv = dev_get_drvdata(dev); + int err = 0; if (!pm_runtime_status_suspended(dev)) - komeda_dev_resume(mdrv->mdev); + err = komeda_dev_resume(mdrv->mdev); - return drm_mode_config_helper_resume(&mdrv->kms->base); + return err ? err : drm_mode_config_helper_resume(&mdrv->kms->base); } static const struct dev_pm_ops komeda_pm_ops = { diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c index bc5f5e9798c32..2bf4a647e4b28 100644 --- a/drivers/gpu/drm/arm/malidp_drv.c +++ b/drivers/gpu/drm/arm/malidp_drv.c @@ -669,6 +669,11 @@ static int malidp_runtime_pm_suspend(struct device *dev) struct drm_device *drm = dev_get_drvdata(dev); struct malidp_drm *malidp = drm_to_malidp(drm); struct malidp_hw_device *hwdev = malidp->dev; + struct clk_bulk_data clks[] = { + { .clk = hwdev->pclk }, + { .clk = hwdev->aclk }, + { .clk = hwdev->mclk }, + }; /* we can only suspend if the hardware is in config mode */ WARN_ON(!hwdev->hw->in_config_mode(hwdev)); @@ -676,9 +681,7 @@ static int malidp_runtime_pm_suspend(struct device *dev) malidp_se_irq_fini(hwdev); malidp_de_irq_fini(hwdev); hwdev->pm_suspended = true; - clk_disable_unprepare(hwdev->mclk); - clk_disable_unprepare(hwdev->aclk); - clk_disable_unprepare(hwdev->pclk); + clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); return 0; } @@ -688,10 +691,17 @@ static int malidp_runtime_pm_resume(struct device *dev) struct drm_device *drm = dev_get_drvdata(dev); struct malidp_drm *malidp = drm_to_malidp(drm); struct malidp_hw_device *hwdev = malidp->dev; + struct clk_bulk_data clks[] = { + { .clk = hwdev->pclk }, + { .clk = hwdev->aclk }, + { .clk = hwdev->mclk }, + }; + int err; + + err = clk_bulk_prepare_enable(ARRAY_SIZE(clks), clks); + if (err) + return err; - clk_prepare_enable(hwdev->pclk); - clk_prepare_enable(hwdev->aclk); - clk_prepare_enable(hwdev->mclk); hwdev->pm_suspended = false; malidp_de_irq_hw_init(hwdev); malidp_se_irq_hw_init(hwdev); diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c index cb53cc91bafb0..4a8c432e311ee 100644 --- a/drivers/gpu/drm/armada/armada_fbdev.c +++ b/drivers/gpu/drm/armada/armada_fbdev.c @@ -45,10 +45,10 @@ int armada_fbdev_driver_fbdev_probe(struct drm_fb_helper *fbh, struct drm_fb_helper_surface_size *sizes) { struct drm_device *dev = fbh->dev; + struct fb_info *info = fbh->info; struct drm_mode_fb_cmd2 mode; struct armada_framebuffer *dfb; struct armada_gem_object *obj; - struct fb_info *info; int size, ret; void *ptr; @@ -92,12 +92,6 @@ int armada_fbdev_driver_fbdev_probe(struct drm_fb_helper *fbh, if (IS_ERR(dfb)) return PTR_ERR(dfb); - info = drm_fb_helper_alloc_info(fbh); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_fballoc; - } - info->fbops = &armada_fb_ops; info->fix.smem_start = obj->phys_addr; info->fix.smem_len = obj->obj.size; @@ -113,8 +107,4 @@ int armada_fbdev_driver_fbdev_probe(struct drm_fb_helper *fbh, (unsigned long long)obj->phys_addr); return 0; - - err_fballoc: - dfb->fb.funcs->destroy(&dfb->fb); - return ret; } diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c index 09b289f0fcbf5..16b7e56142006 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c @@ -1015,7 +1015,7 @@ cdns_dsi_bridge_atomic_reset(struct drm_bridge *bridge) dsi_state = kzalloc(sizeof(*dsi_state), GFP_KERNEL); if (!dsi_state) - return NULL; + return ERR_PTR(-ENOMEM); memset(dsi_state, 0, sizeof(*dsi_state)); dsi_state->base.bridge = bridge; @@ -1230,7 +1230,7 @@ static const struct mipi_dsi_host_ops cdns_dsi_ops = { .transfer = cdns_dsi_transfer, }; -static int __maybe_unused cdns_dsi_resume(struct device *dev) +static int cdns_dsi_resume(struct device *dev) { struct cdns_dsi *dsi = dev_get_drvdata(dev); @@ -1241,7 +1241,7 @@ static int __maybe_unused cdns_dsi_resume(struct device *dev) return 0; } -static int __maybe_unused cdns_dsi_suspend(struct device *dev) +static int cdns_dsi_suspend(struct device *dev) { struct cdns_dsi *dsi = dev_get_drvdata(dev); @@ -1251,8 +1251,9 @@ static int __maybe_unused cdns_dsi_suspend(struct device *dev) return 0; } -static UNIVERSAL_DEV_PM_OPS(cdns_dsi_pm_ops, cdns_dsi_suspend, cdns_dsi_resume, - NULL); +static const struct dev_pm_ops cdns_dsi_pm_ops = { + RUNTIME_PM_OPS(cdns_dsi_suspend, cdns_dsi_resume, NULL) +}; static int cdns_dsi_drm_probe(struct platform_device *pdev) { @@ -1399,7 +1400,7 @@ static struct platform_driver cdns_dsi_platform_driver = { .driver = { .name = "cdns-dsi", .of_match_table = cdns_dsi_of_match, - .pm = &cdns_dsi_pm_ops, + .pm = pm_ptr(&cdns_dsi_pm_ops), }, }; module_platform_driver(cdns_dsi_platform_driver); diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c index 7ee19b7cc92fc..f81db7da711d2 100644 --- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c +++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c @@ -2088,7 +2088,7 @@ cdns_mhdp_bridge_atomic_reset(struct drm_bridge *bridge) cdns_mhdp_state = kzalloc(sizeof(*cdns_mhdp_state), GFP_KERNEL); if (!cdns_mhdp_state) - return NULL; + return ERR_PTR(-ENOMEM); __drm_atomic_helper_bridge_reset(bridge, &cdns_mhdp_state->base); diff --git a/drivers/gpu/drm/bridge/display-connector.c b/drivers/gpu/drm/bridge/display-connector.c index e9f16dbc95353..ba1d41816f3ec 100644 --- a/drivers/gpu/drm/bridge/display-connector.c +++ b/drivers/gpu/drm/bridge/display-connector.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -25,6 +26,8 @@ struct display_connector { struct regulator *supply; struct gpio_desc *ddc_en; + + struct work_struct hpd_work; }; static inline struct display_connector * @@ -40,6 +43,13 @@ static int display_connector_attach(struct drm_bridge *bridge, return flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR ? 0 : -EINVAL; } +static void display_connector_destroy(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + i2c_put_adapter(conn->bridge.ddc); +} + static enum drm_connector_status display_connector_detect(struct drm_bridge *bridge) { struct display_connector *conn = to_display_connector(bridge); @@ -87,6 +97,34 @@ display_connector_bridge_detect(struct drm_bridge *bridge, struct drm_connector return display_connector_detect(bridge); } +static void display_connector_hpd_enable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + enable_irq(conn->hpd_irq); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + schedule_work(&conn->hpd_work); +} + +static void display_connector_hpd_disable(struct drm_bridge *bridge) +{ + struct display_connector *conn = to_display_connector(bridge); + + if (conn->bridge.type == DRM_MODE_CONNECTOR_DisplayPort) + cancel_work_sync(&conn->hpd_work); + + disable_irq(conn->hpd_irq); +} + +static void display_connector_hpd_work(struct work_struct *work) +{ + struct display_connector *conn = container_of(work, struct display_connector, hpd_work); + struct drm_bridge *bridge = &conn->bridge; + + drm_bridge_hpd_notify(bridge, display_connector_detect(bridge)); +} + static const struct drm_edid *display_connector_edid_read(struct drm_bridge *bridge, struct drm_connector *connector) { @@ -177,7 +215,10 @@ static u32 *display_connector_get_input_bus_fmts(struct drm_bridge *bridge, static const struct drm_bridge_funcs display_connector_bridge_funcs = { .attach = display_connector_attach, + .destroy = display_connector_destroy, .detect = display_connector_bridge_detect, + .hpd_enable = display_connector_hpd_enable, + .hpd_disable = display_connector_hpd_disable, .edid_read = display_connector_edid_read, .atomic_get_output_bus_fmts = display_connector_get_output_bus_fmts, .atomic_get_input_bus_fmts = display_connector_get_input_bus_fmts, @@ -307,6 +348,7 @@ static int display_connector_probe(struct platform_device *pdev) NULL, display_connector_hpd_irq, IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING | + IRQF_NO_AUTOEN | IRQF_ONESHOT, "HPD", conn); if (ret) { @@ -378,6 +420,8 @@ static int display_connector_probe(struct platform_device *pdev) conn->bridge.ops |= DRM_BRIDGE_OP_DETECT; if (conn->hpd_irq >= 0) conn->bridge.ops |= DRM_BRIDGE_OP_HPD; + if (conn->hpd_irq >= 0 && type == DRM_MODE_CONNECTOR_DisplayPort) + INIT_WORK(&conn->hpd_work, display_connector_hpd_work); dev_dbg(&pdev->dev, "Found %s display connector '%s' %s DDC bus and %s HPD GPIO (ops 0x%x)\n", @@ -403,9 +447,6 @@ static void display_connector_remove(struct platform_device *pdev) regulator_disable(conn->supply); drm_bridge_remove(&conn->bridge); - - if (!IS_ERR(conn->bridge.ddc)) - i2c_put_adapter(conn->bridge.ddc); } static const struct of_device_id display_connector_match[] = { diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c index 825777a5758f6..db8e5889dc7f1 100644 --- a/drivers/gpu/drm/bridge/parade-ps8640.c +++ b/drivers/gpu/drm/bridge/parade-ps8640.c @@ -257,8 +257,14 @@ static ssize_t ps8640_aux_transfer_msg(struct drm_dp_aux *aux, addr_len[PAGE0_SWAUX_LENGTH - base] = (len == 0) ? SWAUX_NO_PAYLOAD : ((len - 1) & SWAUX_LENGTH_MASK); - regmap_bulk_write(map, PAGE0_SWAUX_ADDR_7_0, addr_len, - ARRAY_SIZE(addr_len)); + ret = regmap_bulk_write(map, PAGE0_SWAUX_ADDR_7_0, addr_len, + ARRAY_SIZE(addr_len)); + if (ret) { + DRM_DEV_ERROR(dev, + "failed to write AUX address %#x, len %zu: %d\n", + msg->address, len, ret); + return ret; + } if (len && (request == DP_AUX_NATIVE_WRITE || request == DP_AUX_I2C_WRITE)) { @@ -274,13 +280,22 @@ static ssize_t ps8640_aux_transfer_msg(struct drm_dp_aux *aux, } } - regmap_write(map, PAGE0_SWAUX_CTRL, SWAUX_SEND); + ret = regmap_write(map, PAGE0_SWAUX_CTRL, SWAUX_SEND); + if (ret) { + DRM_DEV_ERROR(dev, "failed to start AUX transfer: %d\n", ret); + return ret; + } /* Zero delay loop because i2c transactions are slow already */ - regmap_read_poll_timeout(map, PAGE0_SWAUX_CTRL, data, - !(data & SWAUX_SEND), 0, 50 * 1000); + ret = regmap_read_poll_timeout(map, PAGE0_SWAUX_CTRL, data, + !(data & SWAUX_SEND), 0, 50 * 1000); + if (ret) { + DRM_DEV_ERROR(dev, "failed to complete AUX transfer: %d\n", + ret); + return ret; + } - regmap_read(map, PAGE0_SWAUX_STATUS, &data); + ret = regmap_read(map, PAGE0_SWAUX_STATUS, &data); if (ret) { DRM_DEV_ERROR(dev, "failed to read PAGE0_SWAUX_STATUS: %d\n", ret); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c index e82960163018a..901bdec4a76ca 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c @@ -352,12 +352,6 @@ enum { DW_DP_YCBCR420_16BIT, }; -enum { - DW_DP_MP_SINGLE_PIXEL, - DW_DP_MP_DUAL_PIXEL, - DW_DP_MP_QUAD_PIXEL, -}; - enum { DW_DP_SDP_VERTICAL_INTERVAL = BIT(0), DW_DP_SDP_HORIZONTAL_INTERVAL = BIT(1), @@ -1984,7 +1978,7 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, return ERR_CAST(dp); dp->dev = dev; - dp->pixel_mode = DW_DP_MP_QUAD_PIXEL; + dp->pixel_mode = plat_data->pixel_mode; dp->plat_data.max_link_rate = plat_data->max_link_rate; bridge = &dp->bridge; @@ -2098,6 +2092,12 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, } EXPORT_SYMBOL_GPL(dw_dp_bind); +void dw_dp_unbind(struct dw_dp *dp) +{ + drm_dp_aux_unregister(&dp->aux); +} +EXPORT_SYMBOL_GPL(dw_dp_unbind); + MODULE_AUTHOR("Andy Yan "); MODULE_DESCRIPTION("DW DP Core Library"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c index 206b099a35e9a..748ed62e48a67 100644 --- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c +++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c @@ -3385,7 +3385,8 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev, break; default: dev_err(dev, "reg-io-width must be 1 or 4\n"); - return ERR_PTR(-EINVAL); + ret = -EINVAL; + goto err_res; } iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c index 4097fef4b86b5..26ba5a6c1b742 100644 --- a/drivers/gpu/drm/bridge/tc358767.c +++ b/drivers/gpu/drm/bridge/tc358767.c @@ -527,7 +527,7 @@ static ssize_t tc_aux_transfer(struct drm_dp_aux *aux, * address-only transfer */ if (size) - size = FIELD_GET(AUX_BYTES, auxstatus); + size = min_t(size_t, size, FIELD_GET(AUX_BYTES, auxstatus)); msg->reply = FIELD_GET(AUX_STATUS, auxstatus); switch (request) { diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c index fbdc44e162293..a44946aeb6a21 100644 --- a/drivers/gpu/drm/bridge/tc358768.c +++ b/drivers/gpu/drm/bridge/tc358768.c @@ -448,6 +448,8 @@ static int tc358768_dsi_host_attach(struct mipi_dsi_host *host, DRM_MODE_CONNECTOR_DSI); if (IS_ERR(bridge)) return PTR_ERR(bridge); + + bridge->pre_enable_prev_first = true; } priv->output.dev = dev; @@ -1150,10 +1152,13 @@ tc358768_atomic_get_input_bus_fmts(struct drm_bridge *bridge, return input_fmts; } -static bool tc358768_mode_fixup(struct drm_bridge *bridge, - const struct drm_display_mode *mode, - struct drm_display_mode *adjusted_mode) +static int tc358768_bridge_atomic_check(struct drm_bridge *bridge, + struct drm_bridge_state *bridge_state, + struct drm_crtc_state *crtc_state, + struct drm_connector_state *conn_state) { + struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode; + /* Default to positive sync */ if (!(adjusted_mode->flags & @@ -1164,13 +1169,15 @@ static bool tc358768_mode_fixup(struct drm_bridge *bridge, (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC))) adjusted_mode->flags |= DRM_MODE_FLAG_PVSYNC; - return true; + bridge_state->input_bus_cfg.flags = bridge->timings->input_bus_flags; + + return 0; } static const struct drm_bridge_funcs tc358768_bridge_funcs = { .attach = tc358768_bridge_attach, .mode_valid = tc358768_bridge_mode_valid, - .mode_fixup = tc358768_mode_fixup, + .atomic_check = tc358768_bridge_atomic_check, .atomic_pre_enable = tc358768_bridge_atomic_pre_enable, .atomic_enable = tc358768_bridge_atomic_enable, .atomic_disable = tc358768_bridge_atomic_disable, diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c index 43344c2e15b77..11513b0a0ed89 100644 --- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c +++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c @@ -399,7 +399,7 @@ static int sn65dsi83_reset_pipe(struct sn65dsi83 *sn65dsi83) drm_modeset_drop_locks(&ctx); drm_modeset_acquire_fini(&ctx); - return 0; + return err; } static void sn65dsi83_reset_work(struct work_struct *ws) @@ -413,6 +413,7 @@ static void sn65dsi83_reset_work(struct work_struct *ws) dev_err(ctx->dev, "reset pipe failed %pe\n", ERR_PTR(ret)); return; } + if (ctx->irq) enable_irq(ctx->irq); } diff --git a/drivers/gpu/drm/clients/drm_log.c b/drivers/gpu/drm/clients/drm_log.c index d239f1e3c4563..1e137786b36a1 100644 --- a/drivers/gpu/drm/clients/drm_log.c +++ b/drivers/gpu/drm/clients/drm_log.c @@ -160,6 +160,9 @@ static void drm_log_draw_kmsg_record(struct drm_log_scanout *scanout, { u32 prefix_len = 0; + if (!len) + return; + if (len > TS_PREFIX_LEN && s[0] == '[' && s[6] == '.' && s[TS_PREFIX_LEN] == ']') prefix_len = TS_PREFIX_LEN + 1; @@ -215,6 +218,12 @@ static int drm_log_setup_modeset(struct drm_client_dev *client, scanout->scaled_font_w = scanout->font->width * scale; scanout->rows = height / scanout->scaled_font_h; scanout->columns = width / scanout->scaled_font_w; + if (!scanout->rows || !scanout->columns) { + drm_client_buffer_delete(scanout->buffer); + scanout->buffer = NULL; + mode_set->fb = NULL; + return -EINVAL; + } scanout->front_color = drm_draw_color_from_xrgb8888(0xffffff, format); scanout->prefix_color = drm_draw_color_from_xrgb8888(0x4e9a06, format); return 0; @@ -272,7 +281,7 @@ static void drm_log_init_client(struct drm_log *dlog) err_failed_commit: for (i = 0; i < n_modeset; i++) - drm_client_framebuffer_delete(dlog->scanout[i].buffer); + drm_client_buffer_delete(dlog->scanout[i].buffer); err_nomodeset: kfree(dlog->scanout); @@ -286,7 +295,7 @@ static void drm_log_free_scanout(struct drm_client_dev *client) if (dlog->n_scanout) { for (i = 0; i < dlog->n_scanout; i++) - drm_client_framebuffer_delete(dlog->scanout[i].buffer); + drm_client_buffer_delete(dlog->scanout[i].buffer); dlog->n_scanout = 0; kfree(dlog->scanout); dlog->scanout = NULL; diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c index 4aaeae4fa03c3..77336bc3e62c2 100644 --- a/drivers/gpu/drm/display/drm_dp_helper.c +++ b/drivers/gpu/drm/display/drm_dp_helper.c @@ -2537,6 +2537,8 @@ static const struct dpcd_quirk dpcd_quirk_list[] = { { OUI(0x00, 0x00, 0x00), DEVICE_ID('C', 'H', '7', '5', '1', '1'), false, BIT(DP_DPCD_QUIRK_NO_SINK_COUNT) }, /* Synaptics DP1.4 MST hubs can support DSC without virtual DPCD */ { OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) }, + /* Realtek DP1.4 MST hubs can support DSC without virtual DPCD */ + { OUI(0x00, 0xe0, 0x4c), DEVICE_ID('D', 'p', '1', '.', '4', 0), true, BIT(DP_DPCD_QUIRK_DSC_WITHOUT_VIRTUAL_DPCD) }, /* Synaptics DP1.4 MST hubs require DSC for some modes on which it applies HBLANK expansion. */ { OUI(0x90, 0xCC, 0x24), DEVICE_ID_ANY, true, BIT(DP_DPCD_QUIRK_HBLANK_EXPANSION_REQUIRES_DSC) }, /* MediaTek panels (at least in U3224KBA) require DSC for modes with a short HBLANK on UHBR links. */ @@ -3457,6 +3459,18 @@ int drm_dp_get_pcon_max_frl_bw(const u8 dpcd[DP_RECEIVER_CAP_SIZE], int bw; u8 buf; + if (!drm_dp_is_branch(dpcd)) + return 0; + + if (dpcd[DP_DPCD_REV] < 0x11) + return 0; + + if ((dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DETAILED_CAP_INFO_AVAILABLE) == 0) + return 0; + + if ((port_cap[0] & DP_DS_PORT_TYPE_MASK) != DP_DS_PORT_TYPE_HDMI) + return 0; + buf = port_cap[2]; bw = buf & DP_PCON_MAX_FRL_BW; diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c index be749dcad3b58..383cfd99f955e 100644 --- a/drivers/gpu/drm/display/drm_dp_mst_topology.c +++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c @@ -789,6 +789,12 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg, { u8 crc4; + /* curchunk_len must be >= 1 (min 1 CRC byte) and fit in chunk[] */ + if (!msg->curchunk_len || + msg->curchunk_len > ARRAY_SIZE(msg->chunk) || + msg->curchunk_idx + replybuflen > ARRAY_SIZE(msg->chunk)) + return false; + memcpy(&msg->chunk[msg->curchunk_idx], replybuf, replybuflen); msg->curchunk_idx += replybuflen; @@ -799,6 +805,9 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg, print_hex_dump(KERN_DEBUG, "wrong crc", DUMP_PREFIX_NONE, 16, 1, msg->chunk, msg->curchunk_len, false); + /* Guard against accumulated msg[] overflow */ + if (msg->curlen + msg->curchunk_len - 1 > ARRAY_SIZE(msg->msg)) + return false; /* copy chunk into bigger msg */ memcpy(&msg->msg[msg->curlen], msg->chunk, msg->curchunk_len - 1); msg->curlen += msg->curchunk_len - 1; @@ -871,7 +880,7 @@ static bool drm_dp_sideband_parse_remote_dpcd_read(struct drm_dp_sideband_msg_rx goto fail_len; repmsg->u.remote_dpcd_read_ack.num_bytes = raw->msg[idx]; idx++; - if (idx > raw->curlen) + if (idx + repmsg->u.remote_dpcd_read_ack.num_bytes > raw->curlen) goto fail_len; memcpy(repmsg->u.remote_dpcd_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_dpcd_read_ack.num_bytes); @@ -907,7 +916,9 @@ static bool drm_dp_sideband_parse_remote_i2c_read_ack(struct drm_dp_sideband_msg goto fail_len; repmsg->u.remote_i2c_read_ack.num_bytes = raw->msg[idx]; idx++; - /* TODO check */ + if (idx + repmsg->u.remote_i2c_read_ack.num_bytes > raw->curlen) + goto fail_len; + memcpy(repmsg->u.remote_i2c_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_i2c_read_ack.num_bytes); return true; fail_len: @@ -923,16 +934,13 @@ static bool drm_dp_sideband_parse_enum_path_resources_ack(struct drm_dp_sideband repmsg->u.path_resources.port_number = (raw->msg[idx] >> 4) & 0xf; repmsg->u.path_resources.fec_capable = raw->msg[idx] & 0x1; idx++; - if (idx > raw->curlen) + if (idx + 2 > raw->curlen) goto fail_len; repmsg->u.path_resources.full_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]); idx += 2; - if (idx > raw->curlen) + if (idx + 2 > raw->curlen) goto fail_len; repmsg->u.path_resources.avail_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]); - idx += 2; - if (idx > raw->curlen) - goto fail_len; return true; fail_len: DRM_DEBUG_KMS("enum resource parse length fail %d %d\n", idx, raw->curlen); @@ -950,12 +958,9 @@ static bool drm_dp_sideband_parse_allocate_payload_ack(struct drm_dp_sideband_ms goto fail_len; repmsg->u.allocate_payload.vcpi = raw->msg[idx]; idx++; - if (idx > raw->curlen) + if (idx + 2 > raw->curlen) goto fail_len; repmsg->u.allocate_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx+1]); - idx += 2; - if (idx > raw->curlen) - goto fail_len; return true; fail_len: DRM_DEBUG_KMS("allocate payload parse length fail %d %d\n", idx, raw->curlen); @@ -969,12 +974,9 @@ static bool drm_dp_sideband_parse_query_payload_ack(struct drm_dp_sideband_msg_r repmsg->u.query_payload.port_number = (raw->msg[idx] >> 4) & 0xf; idx++; - if (idx > raw->curlen) + if (idx + 2 > raw->curlen) goto fail_len; repmsg->u.query_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx + 1]); - idx += 2; - if (idx > raw->curlen) - goto fail_len; return true; fail_len: DRM_DEBUG_KMS("query payload parse length fail %d %d\n", idx, raw->curlen); @@ -3738,8 +3740,10 @@ void drm_dp_mst_topology_queue_probe(struct drm_dp_mst_topology_mgr *mgr) { mutex_lock(&mgr->lock); - if (drm_WARN_ON(mgr->dev, !mgr->mst_state || !mgr->mst_primary)) + if (!mgr->mst_state || !mgr->mst_primary) { + drm_dbg_kms(mgr->dev, "queue_probe skipped: topology torn down\n"); goto out_unlock; + } drm_dp_mst_topology_mgr_invalidate_mstb(mgr->mst_primary); drm_dp_mst_queue_probe_work(mgr); diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c index bbfc0dfafe48b..bc24515c3ef89 100644 --- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c +++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c @@ -1077,7 +1077,8 @@ drm_atomic_helper_connector_hdmi_update_audio_infoframe(struct drm_connector *co mutex_lock(&connector->hdmi.infoframes.lock); - memcpy(&infoframe->data, frame, sizeof(infoframe->data)); + BUILD_BUG_ON(sizeof(*frame) > sizeof(infoframe->data)); + memcpy(&infoframe->data, frame, sizeof(*frame)); infoframe->set = true; ret = write_infoframe(connector, infoframe); diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index adc75d821d1d3..b59d397d6fac7 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1379,6 +1379,8 @@ static void complete_signaling(struct drm_device *dev, * to prevent a double free in drm_atomic_state_clear. */ if (event && (event->base.fence || event->base.file_priv)) { + if (crtc_state->commit && crtc_state->commit->abort_completion) + drm_crtc_commit_put(crtc_state->commit); drm_event_cancel_free(dev, &event->base); crtc_state->event = NULL; } diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c index 640d93070bb7c..841f3de5f307a 100644 --- a/drivers/gpu/drm/drm_buddy.c +++ b/drivers/gpu/drm/drm_buddy.c @@ -10,1250 +10,9 @@ #include #include +#include #include - -enum drm_buddy_free_tree { - DRM_BUDDY_CLEAR_TREE = 0, - DRM_BUDDY_DIRTY_TREE, - DRM_BUDDY_MAX_FREE_TREES, -}; - -static struct kmem_cache *slab_blocks; - -#define for_each_free_tree(tree) \ - for ((tree) = 0; (tree) < DRM_BUDDY_MAX_FREE_TREES; (tree)++) - -static struct drm_buddy_block *drm_block_alloc(struct drm_buddy *mm, - struct drm_buddy_block *parent, - unsigned int order, - u64 offset) -{ - struct drm_buddy_block *block; - - BUG_ON(order > DRM_BUDDY_MAX_ORDER); - - block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL); - if (!block) - return NULL; - - block->header = offset; - block->header |= order; - block->parent = parent; - - RB_CLEAR_NODE(&block->rb); - - BUG_ON(block->header & DRM_BUDDY_HEADER_UNUSED); - return block; -} - -static void drm_block_free(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - kmem_cache_free(slab_blocks, block); -} - -static enum drm_buddy_free_tree -get_block_tree(struct drm_buddy_block *block) -{ - return drm_buddy_block_is_clear(block) ? - DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; -} - -static struct drm_buddy_block * -rbtree_get_free_block(const struct rb_node *node) -{ - return node ? rb_entry(node, struct drm_buddy_block, rb) : NULL; -} - -static struct drm_buddy_block * -rbtree_last_free_block(struct rb_root *root) -{ - return rbtree_get_free_block(rb_last(root)); -} - -static bool rbtree_is_empty(struct rb_root *root) -{ - return RB_EMPTY_ROOT(root); -} - -static bool drm_buddy_block_offset_less(const struct drm_buddy_block *block, - const struct drm_buddy_block *node) -{ - return drm_buddy_block_offset(block) < drm_buddy_block_offset(node); -} - -static bool rbtree_block_offset_less(struct rb_node *block, - const struct rb_node *node) -{ - return drm_buddy_block_offset_less(rbtree_get_free_block(block), - rbtree_get_free_block(node)); -} - -static void rbtree_insert(struct drm_buddy *mm, - struct drm_buddy_block *block, - enum drm_buddy_free_tree tree) -{ - rb_add(&block->rb, - &mm->free_trees[tree][drm_buddy_block_order(block)], - rbtree_block_offset_less); -} - -static void rbtree_remove(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - unsigned int order = drm_buddy_block_order(block); - enum drm_buddy_free_tree tree; - struct rb_root *root; - - tree = get_block_tree(block); - root = &mm->free_trees[tree][order]; - - rb_erase(&block->rb, root); - RB_CLEAR_NODE(&block->rb); -} - -static void clear_reset(struct drm_buddy_block *block) -{ - block->header &= ~DRM_BUDDY_HEADER_CLEAR; -} - -static void mark_cleared(struct drm_buddy_block *block) -{ - block->header |= DRM_BUDDY_HEADER_CLEAR; -} - -static void mark_allocated(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - block->header &= ~DRM_BUDDY_HEADER_STATE; - block->header |= DRM_BUDDY_ALLOCATED; - - rbtree_remove(mm, block); -} - -static void mark_free(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - enum drm_buddy_free_tree tree; - - block->header &= ~DRM_BUDDY_HEADER_STATE; - block->header |= DRM_BUDDY_FREE; - - tree = get_block_tree(block); - rbtree_insert(mm, block, tree); -} - -static void mark_split(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - block->header &= ~DRM_BUDDY_HEADER_STATE; - block->header |= DRM_BUDDY_SPLIT; - - rbtree_remove(mm, block); -} - -static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= e2 && e1 >= s2; -} - -static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) -{ - return s1 <= s2 && e1 >= e2; -} - -static struct drm_buddy_block * -__get_buddy(struct drm_buddy_block *block) -{ - struct drm_buddy_block *parent; - - parent = block->parent; - if (!parent) - return NULL; - - if (parent->left == block) - return parent->right; - - return parent->left; -} - -static unsigned int __drm_buddy_free(struct drm_buddy *mm, - struct drm_buddy_block *block, - bool force_merge) -{ - struct drm_buddy_block *parent; - unsigned int order; - - while ((parent = block->parent)) { - struct drm_buddy_block *buddy; - - buddy = __get_buddy(block); - - if (!drm_buddy_block_is_free(buddy)) - break; - - if (!force_merge) { - /* - * Check the block and its buddy clear state and exit - * the loop if they both have the dissimilar state. - */ - if (drm_buddy_block_is_clear(block) != - drm_buddy_block_is_clear(buddy)) - break; - - if (drm_buddy_block_is_clear(block)) - mark_cleared(parent); - } - - rbtree_remove(mm, buddy); - if (force_merge && drm_buddy_block_is_clear(buddy)) - mm->clear_avail -= drm_buddy_block_size(mm, buddy); - - drm_block_free(mm, block); - drm_block_free(mm, buddy); - - block = parent; - } - - order = drm_buddy_block_order(block); - mark_free(mm, block); - - return order; -} - -static int __force_merge(struct drm_buddy *mm, - u64 start, - u64 end, - unsigned int min_order) -{ - unsigned int tree, order; - int i; - - if (!min_order) - return -ENOMEM; - - if (min_order > mm->max_order) - return -EINVAL; - - for_each_free_tree(tree) { - for (i = min_order - 1; i >= 0; i--) { - struct rb_node *iter = rb_last(&mm->free_trees[tree][i]); - - while (iter) { - struct drm_buddy_block *block, *buddy; - u64 block_start, block_end; - - block = rbtree_get_free_block(iter); - iter = rb_prev(iter); - - if (!block || !block->parent) - continue; - - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block) - 1; - - if (!contains(start, end, block_start, block_end)) - continue; - - buddy = __get_buddy(block); - if (!drm_buddy_block_is_free(buddy)) - continue; - - WARN_ON(drm_buddy_block_is_clear(block) == - drm_buddy_block_is_clear(buddy)); - - /* - * Advance to the next node when the current node is the buddy, - * as freeing the block will also remove its buddy from the tree. - */ - if (iter == &buddy->rb) - iter = rb_prev(iter); - - rbtree_remove(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); - - order = __drm_buddy_free(mm, block, true); - if (order >= min_order) - return 0; - } - } - } - - return -ENOMEM; -} - -/** - * drm_buddy_init - init memory manager - * - * @mm: DRM buddy manager to initialize - * @size: size in bytes to manage - * @chunk_size: minimum page size in bytes for our allocations - * - * Initializes the memory manager and its resources. - * - * Returns: - * 0 on success, error code on failure. - */ -int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size) -{ - unsigned int i, j, root_count = 0; - u64 offset = 0; - - if (size < chunk_size) - return -EINVAL; - - if (chunk_size < SZ_4K) - return -EINVAL; - - if (!is_power_of_2(chunk_size)) - return -EINVAL; - - size = round_down(size, chunk_size); - - mm->size = size; - mm->avail = size; - mm->clear_avail = 0; - mm->chunk_size = chunk_size; - mm->max_order = ilog2(size) - ilog2(chunk_size); - - BUG_ON(mm->max_order > DRM_BUDDY_MAX_ORDER); - - mm->free_trees = kmalloc_array(DRM_BUDDY_MAX_FREE_TREES, - sizeof(*mm->free_trees), - GFP_KERNEL); - if (!mm->free_trees) - return -ENOMEM; - - for_each_free_tree(i) { - mm->free_trees[i] = kmalloc_array(mm->max_order + 1, - sizeof(struct rb_root), - GFP_KERNEL); - if (!mm->free_trees[i]) - goto out_free_tree; - - for (j = 0; j <= mm->max_order; ++j) - mm->free_trees[i][j] = RB_ROOT; - } - - mm->n_roots = hweight64(size); - - mm->roots = kmalloc_array(mm->n_roots, - sizeof(struct drm_buddy_block *), - GFP_KERNEL); - if (!mm->roots) - goto out_free_tree; - - /* - * Split into power-of-two blocks, in case we are given a size that is - * not itself a power-of-two. - */ - do { - struct drm_buddy_block *root; - unsigned int order; - u64 root_size; - - order = ilog2(size) - ilog2(chunk_size); - root_size = chunk_size << order; - - root = drm_block_alloc(mm, NULL, order, offset); - if (!root) - goto out_free_roots; - - mark_free(mm, root); - - BUG_ON(root_count > mm->max_order); - BUG_ON(drm_buddy_block_size(mm, root) < chunk_size); - - mm->roots[root_count] = root; - - offset += root_size; - size -= root_size; - root_count++; - } while (size); - - return 0; - -out_free_roots: - while (root_count--) - drm_block_free(mm, mm->roots[root_count]); - kfree(mm->roots); -out_free_tree: - while (i--) - kfree(mm->free_trees[i]); - kfree(mm->free_trees); - return -ENOMEM; -} -EXPORT_SYMBOL(drm_buddy_init); - -/** - * drm_buddy_fini - tear down the memory manager - * - * @mm: DRM buddy manager to free - * - * Cleanup memory manager resources and the freetree - */ -void drm_buddy_fini(struct drm_buddy *mm) -{ - u64 root_size, size, start; - unsigned int order; - int i; - - size = mm->size; - - for (i = 0; i < mm->n_roots; ++i) { - order = ilog2(size) - ilog2(mm->chunk_size); - start = drm_buddy_block_offset(mm->roots[i]); - __force_merge(mm, start, start + size, order); - - if (WARN_ON(!drm_buddy_block_is_free(mm->roots[i]))) - kunit_fail_current_test("buddy_fini() root"); - - drm_block_free(mm, mm->roots[i]); - - root_size = mm->chunk_size << order; - size -= root_size; - } - - WARN_ON(mm->avail != mm->size); - - for_each_free_tree(i) - kfree(mm->free_trees[i]); - kfree(mm->free_trees); - kfree(mm->roots); -} -EXPORT_SYMBOL(drm_buddy_fini); - -static int split_block(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - unsigned int block_order = drm_buddy_block_order(block) - 1; - u64 offset = drm_buddy_block_offset(block); - - BUG_ON(!drm_buddy_block_is_free(block)); - BUG_ON(!drm_buddy_block_order(block)); - - block->left = drm_block_alloc(mm, block, block_order, offset); - if (!block->left) - return -ENOMEM; - - block->right = drm_block_alloc(mm, block, block_order, - offset + (mm->chunk_size << block_order)); - if (!block->right) { - drm_block_free(mm, block->left); - return -ENOMEM; - } - - mark_split(mm, block); - - if (drm_buddy_block_is_clear(block)) { - mark_cleared(block->left); - mark_cleared(block->right); - clear_reset(block); - } - - mark_free(mm, block->left); - mark_free(mm, block->right); - - return 0; -} - -/** - * drm_get_buddy - get buddy address - * - * @block: DRM buddy block - * - * Returns the corresponding buddy block for @block, or NULL - * if this is a root block and can't be merged further. - * Requires some kind of locking to protect against - * any concurrent allocate and free operations. - */ -struct drm_buddy_block * -drm_get_buddy(struct drm_buddy_block *block) -{ - return __get_buddy(block); -} -EXPORT_SYMBOL(drm_get_buddy); - -/** - * drm_buddy_reset_clear - reset blocks clear state - * - * @mm: DRM buddy manager - * @is_clear: blocks clear state - * - * Reset the clear state based on @is_clear value for each block - * in the freetree. - */ -void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear) -{ - enum drm_buddy_free_tree src_tree, dst_tree; - u64 root_size, size, start; - unsigned int order; - int i; - - size = mm->size; - for (i = 0; i < mm->n_roots; ++i) { - order = ilog2(size) - ilog2(mm->chunk_size); - start = drm_buddy_block_offset(mm->roots[i]); - __force_merge(mm, start, start + size, order); - - root_size = mm->chunk_size << order; - size -= root_size; - } - - src_tree = is_clear ? DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; - dst_tree = is_clear ? DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; - - for (i = 0; i <= mm->max_order; ++i) { - struct rb_root *root = &mm->free_trees[src_tree][i]; - struct drm_buddy_block *block, *tmp; - - rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { - rbtree_remove(mm, block); - if (is_clear) { - mark_cleared(block); - mm->clear_avail += drm_buddy_block_size(mm, block); - } else { - clear_reset(block); - mm->clear_avail -= drm_buddy_block_size(mm, block); - } - - rbtree_insert(mm, block, dst_tree); - } - } -} -EXPORT_SYMBOL(drm_buddy_reset_clear); - -/** - * drm_buddy_free_block - free a block - * - * @mm: DRM buddy manager - * @block: block to be freed - */ -void drm_buddy_free_block(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - BUG_ON(!drm_buddy_block_is_allocated(block)); - mm->avail += drm_buddy_block_size(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail += drm_buddy_block_size(mm, block); - - __drm_buddy_free(mm, block, false); -} -EXPORT_SYMBOL(drm_buddy_free_block); - -static void __drm_buddy_free_list(struct drm_buddy *mm, - struct list_head *objects, - bool mark_clear, - bool mark_dirty) -{ - struct drm_buddy_block *block, *on; - - WARN_ON(mark_dirty && mark_clear); - - list_for_each_entry_safe(block, on, objects, link) { - if (mark_clear) - mark_cleared(block); - else if (mark_dirty) - clear_reset(block); - drm_buddy_free_block(mm, block); - cond_resched(); - } - INIT_LIST_HEAD(objects); -} - -static void drm_buddy_free_list_internal(struct drm_buddy *mm, - struct list_head *objects) -{ - /* - * Don't touch the clear/dirty bit, since allocation is still internal - * at this point. For example we might have just failed part of the - * allocation. - */ - __drm_buddy_free_list(mm, objects, false, false); -} - -/** - * drm_buddy_free_list - free blocks - * - * @mm: DRM buddy manager - * @objects: input list head to free blocks - * @flags: optional flags like DRM_BUDDY_CLEARED - */ -void drm_buddy_free_list(struct drm_buddy *mm, - struct list_head *objects, - unsigned int flags) -{ - bool mark_clear = flags & DRM_BUDDY_CLEARED; - - __drm_buddy_free_list(mm, objects, mark_clear, !mark_clear); -} -EXPORT_SYMBOL(drm_buddy_free_list); - -static bool block_incompatible(struct drm_buddy_block *block, unsigned int flags) -{ - bool needs_clear = flags & DRM_BUDDY_CLEAR_ALLOCATION; - - return needs_clear != drm_buddy_block_is_clear(block); -} - -static struct drm_buddy_block * -__alloc_range_bias(struct drm_buddy *mm, - u64 start, u64 end, - unsigned int order, - unsigned long flags, - bool fallback) -{ - u64 req_size = mm->chunk_size << order; - struct drm_buddy_block *block; - struct drm_buddy_block *buddy; - LIST_HEAD(dfs); - int err; - int i; - - end = end - 1; - - for (i = 0; i < mm->n_roots; ++i) - list_add_tail(&mm->roots[i]->tmp_link, &dfs); - - do { - u64 block_start; - u64 block_end; - - block = list_first_entry_or_null(&dfs, - struct drm_buddy_block, - tmp_link); - if (!block) - break; - - list_del(&block->tmp_link); - - if (drm_buddy_block_order(block) < order) - continue; - - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block) - 1; - - if (!overlaps(start, end, block_start, block_end)) - continue; - - if (drm_buddy_block_is_allocated(block)) - continue; - - if (block_start < start || block_end > end) { - u64 adjusted_start = max(block_start, start); - u64 adjusted_end = min(block_end, end); - - if (round_down(adjusted_end + 1, req_size) <= - round_up(adjusted_start, req_size)) - continue; - } - - if (!fallback && block_incompatible(block, flags)) - continue; - - if (contains(start, end, block_start, block_end) && - order == drm_buddy_block_order(block)) { - /* - * Find the free block within the range. - */ - if (drm_buddy_block_is_free(block)) - return block; - - continue; - } - - if (!drm_buddy_block_is_split(block)) { - err = split_block(mm, block); - if (unlikely(err)) - goto err_undo; - } - - list_add(&block->right->tmp_link, &dfs); - list_add(&block->left->tmp_link, &dfs); - } while (1); - - return ERR_PTR(-ENOSPC); - -err_undo: - /* - * We really don't want to leave around a bunch of split blocks, since - * bigger is better, so make sure we merge everything back before we - * free the allocated blocks. - */ - buddy = __get_buddy(block); - if (buddy && - (drm_buddy_block_is_free(block) && - drm_buddy_block_is_free(buddy))) - __drm_buddy_free(mm, block, false); - return ERR_PTR(err); -} - -static struct drm_buddy_block * -__drm_buddy_alloc_range_bias(struct drm_buddy *mm, - u64 start, u64 end, - unsigned int order, - unsigned long flags) -{ - struct drm_buddy_block *block; - bool fallback = false; - - block = __alloc_range_bias(mm, start, end, order, - flags, fallback); - if (IS_ERR(block)) - return __alloc_range_bias(mm, start, end, order, - flags, !fallback); - - return block; -} - -static struct drm_buddy_block * -get_maxblock(struct drm_buddy *mm, - unsigned int order, - enum drm_buddy_free_tree tree) -{ - struct drm_buddy_block *max_block = NULL, *block = NULL; - struct rb_root *root; - unsigned int i; - - for (i = order; i <= mm->max_order; ++i) { - root = &mm->free_trees[tree][i]; - block = rbtree_last_free_block(root); - if (!block) - continue; - - if (!max_block) { - max_block = block; - continue; - } - - if (drm_buddy_block_offset(block) > - drm_buddy_block_offset(max_block)) { - max_block = block; - } - } - - return max_block; -} - -static struct drm_buddy_block * -alloc_from_freetree(struct drm_buddy *mm, - unsigned int order, - unsigned long flags) -{ - struct drm_buddy_block *block = NULL; - struct rb_root *root; - enum drm_buddy_free_tree tree; - unsigned int tmp; - int err; - - tree = (flags & DRM_BUDDY_CLEAR_ALLOCATION) ? - DRM_BUDDY_CLEAR_TREE : DRM_BUDDY_DIRTY_TREE; - - if (flags & DRM_BUDDY_TOPDOWN_ALLOCATION) { - block = get_maxblock(mm, order, tree); - if (block) - /* Store the obtained block order */ - tmp = drm_buddy_block_order(block); - } else { - for (tmp = order; tmp <= mm->max_order; ++tmp) { - /* Get RB tree root for this order and tree */ - root = &mm->free_trees[tree][tmp]; - block = rbtree_last_free_block(root); - if (block) - break; - } - } - - if (!block) { - /* Try allocating from the other tree */ - tree = (tree == DRM_BUDDY_CLEAR_TREE) ? - DRM_BUDDY_DIRTY_TREE : DRM_BUDDY_CLEAR_TREE; - - for (tmp = order; tmp <= mm->max_order; ++tmp) { - root = &mm->free_trees[tree][tmp]; - block = rbtree_last_free_block(root); - if (block) - break; - } - - if (!block) - return ERR_PTR(-ENOSPC); - } - - BUG_ON(!drm_buddy_block_is_free(block)); - - while (tmp != order) { - err = split_block(mm, block); - if (unlikely(err)) - goto err_undo; - - block = block->right; - tmp--; - } - return block; - -err_undo: - if (tmp != order) - __drm_buddy_free(mm, block, false); - return ERR_PTR(err); -} - -static int __alloc_range(struct drm_buddy *mm, - struct list_head *dfs, - u64 start, u64 size, - struct list_head *blocks, - u64 *total_allocated_on_err) -{ - struct drm_buddy_block *block; - struct drm_buddy_block *buddy; - u64 total_allocated = 0; - LIST_HEAD(allocated); - u64 end; - int err; - - end = start + size - 1; - - do { - u64 block_start; - u64 block_end; - - block = list_first_entry_or_null(dfs, - struct drm_buddy_block, - tmp_link); - if (!block) - break; - - list_del(&block->tmp_link); - - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block) - 1; - - if (!overlaps(start, end, block_start, block_end)) - continue; - - if (drm_buddy_block_is_allocated(block)) { - err = -ENOSPC; - goto err_free; - } - - if (contains(start, end, block_start, block_end)) { - if (drm_buddy_block_is_free(block)) { - mark_allocated(mm, block); - total_allocated += drm_buddy_block_size(mm, block); - mm->avail -= drm_buddy_block_size(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); - list_add_tail(&block->link, &allocated); - continue; - } else if (!mm->clear_avail) { - err = -ENOSPC; - goto err_free; - } - } - - if (!drm_buddy_block_is_split(block)) { - err = split_block(mm, block); - if (unlikely(err)) - goto err_undo; - } - - list_add(&block->right->tmp_link, dfs); - list_add(&block->left->tmp_link, dfs); - } while (1); - - if (total_allocated < size) { - err = -ENOSPC; - goto err_free; - } - - list_splice_tail(&allocated, blocks); - - return 0; - -err_undo: - /* - * We really don't want to leave around a bunch of split blocks, since - * bigger is better, so make sure we merge everything back before we - * free the allocated blocks. - */ - buddy = __get_buddy(block); - if (buddy && - (drm_buddy_block_is_free(block) && - drm_buddy_block_is_free(buddy))) - __drm_buddy_free(mm, block, false); - -err_free: - if (err == -ENOSPC && total_allocated_on_err) { - list_splice_tail(&allocated, blocks); - *total_allocated_on_err = total_allocated; - } else { - drm_buddy_free_list_internal(mm, &allocated); - } - - return err; -} - -static int __drm_buddy_alloc_range(struct drm_buddy *mm, - u64 start, - u64 size, - u64 *total_allocated_on_err, - struct list_head *blocks) -{ - LIST_HEAD(dfs); - int i; - - for (i = 0; i < mm->n_roots; ++i) - list_add_tail(&mm->roots[i]->tmp_link, &dfs); - - return __alloc_range(mm, &dfs, start, size, - blocks, total_allocated_on_err); -} - -static int __alloc_contig_try_harder(struct drm_buddy *mm, - u64 size, - u64 min_block_size, - struct list_head *blocks) -{ - u64 rhs_offset, lhs_offset, lhs_size, filled; - struct drm_buddy_block *block; - unsigned int tree, order; - LIST_HEAD(blocks_lhs); - unsigned long pages; - u64 modify_size; - int err; - - modify_size = rounddown_pow_of_two(size); - pages = modify_size >> ilog2(mm->chunk_size); - order = fls(pages) - 1; - if (order == 0) - return -ENOSPC; - - for_each_free_tree(tree) { - struct rb_root *root; - struct rb_node *iter; - - root = &mm->free_trees[tree][order]; - if (rbtree_is_empty(root)) - continue; - - iter = rb_last(root); - while (iter) { - block = rbtree_get_free_block(iter); - - /* Allocate blocks traversing RHS */ - rhs_offset = drm_buddy_block_offset(block); - err = __drm_buddy_alloc_range(mm, rhs_offset, size, - &filled, blocks); - if (!err || err != -ENOSPC) - return err; - - lhs_size = max((size - filled), min_block_size); - if (!IS_ALIGNED(lhs_size, min_block_size)) - lhs_size = round_up(lhs_size, min_block_size); - - /* Allocate blocks traversing LHS */ - lhs_offset = drm_buddy_block_offset(block) - lhs_size; - err = __drm_buddy_alloc_range(mm, lhs_offset, lhs_size, - NULL, &blocks_lhs); - if (!err) { - list_splice(&blocks_lhs, blocks); - return 0; - } else if (err != -ENOSPC) { - drm_buddy_free_list_internal(mm, blocks); - return err; - } - /* Free blocks for the next iteration */ - drm_buddy_free_list_internal(mm, blocks); - - iter = rb_prev(iter); - } - } - - return -ENOSPC; -} - -/** - * drm_buddy_block_trim - free unused pages - * - * @mm: DRM buddy manager - * @start: start address to begin the trimming. - * @new_size: original size requested - * @blocks: Input and output list of allocated blocks. - * MUST contain single block as input to be trimmed. - * On success will contain the newly allocated blocks - * making up the @new_size. Blocks always appear in - * ascending order - * - * For contiguous allocation, we round up the size to the nearest - * power of two value, drivers consume *actual* size, so remaining - * portions are unused and can be optionally freed with this function - * - * Returns: - * 0 on success, error code on failure. - */ -int drm_buddy_block_trim(struct drm_buddy *mm, - u64 *start, - u64 new_size, - struct list_head *blocks) -{ - struct drm_buddy_block *parent; - struct drm_buddy_block *block; - u64 block_start, block_end; - LIST_HEAD(dfs); - u64 new_start; - int err; - - if (!list_is_singular(blocks)) - return -EINVAL; - - block = list_first_entry(blocks, - struct drm_buddy_block, - link); - - block_start = drm_buddy_block_offset(block); - block_end = block_start + drm_buddy_block_size(mm, block); - - if (WARN_ON(!drm_buddy_block_is_allocated(block))) - return -EINVAL; - - if (new_size > drm_buddy_block_size(mm, block)) - return -EINVAL; - - if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size)) - return -EINVAL; - - if (new_size == drm_buddy_block_size(mm, block)) - return 0; - - new_start = block_start; - if (start) { - new_start = *start; - - if (new_start < block_start) - return -EINVAL; - - if (!IS_ALIGNED(new_start, mm->chunk_size)) - return -EINVAL; - - if (range_overflows(new_start, new_size, block_end)) - return -EINVAL; - } - - list_del(&block->link); - mark_free(mm, block); - mm->avail += drm_buddy_block_size(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail += drm_buddy_block_size(mm, block); - - /* Prevent recursively freeing this node */ - parent = block->parent; - block->parent = NULL; - - list_add(&block->tmp_link, &dfs); - err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL); - if (err) { - mark_allocated(mm, block); - mm->avail -= drm_buddy_block_size(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); - list_add(&block->link, blocks); - } - - block->parent = parent; - return err; -} -EXPORT_SYMBOL(drm_buddy_block_trim); - -static struct drm_buddy_block * -__drm_buddy_alloc_blocks(struct drm_buddy *mm, - u64 start, u64 end, - unsigned int order, - unsigned long flags) -{ - if (flags & DRM_BUDDY_RANGE_ALLOCATION) - /* Allocate traversing within the range */ - return __drm_buddy_alloc_range_bias(mm, start, end, - order, flags); - else - /* Allocate from freetree */ - return alloc_from_freetree(mm, order, flags); -} - -/** - * drm_buddy_alloc_blocks - allocate power-of-two blocks - * - * @mm: DRM buddy manager to allocate from - * @start: start of the allowed range for this block - * @end: end of the allowed range for this block - * @size: size of the allocation in bytes - * @min_block_size: alignment of the allocation - * @blocks: output list head to add allocated blocks - * @flags: DRM_BUDDY_*_ALLOCATION flags - * - * alloc_range_bias() called on range limitations, which traverses - * the tree and returns the desired block. - * - * alloc_from_freetree() called when *no* range restrictions - * are enforced, which picks the block from the freetree. - * - * Returns: - * 0 on success, error code on failure. - */ -int drm_buddy_alloc_blocks(struct drm_buddy *mm, - u64 start, u64 end, u64 size, - u64 min_block_size, - struct list_head *blocks, - unsigned long flags) -{ - struct drm_buddy_block *block = NULL; - u64 original_size, original_min_size; - unsigned int min_order, order; - LIST_HEAD(allocated); - unsigned long pages; - int err; - - if (size < mm->chunk_size) - return -EINVAL; - - if (min_block_size < mm->chunk_size) - return -EINVAL; - - if (!is_power_of_2(min_block_size)) - return -EINVAL; - - if (!IS_ALIGNED(start | end | size, mm->chunk_size)) - return -EINVAL; - - if (end > mm->size) - return -EINVAL; - - if (range_overflows(start, size, mm->size)) - return -EINVAL; - - /* Actual range allocation */ - if (start + size == end) { - if (!IS_ALIGNED(start | end, min_block_size)) - return -EINVAL; - - return __drm_buddy_alloc_range(mm, start, size, NULL, blocks); - } - - original_size = size; - original_min_size = min_block_size; - - /* Roundup the size to power of 2 */ - if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) { - size = roundup_pow_of_two(size); - min_block_size = size; - /* Align size value to min_block_size */ - } else if (!IS_ALIGNED(size, min_block_size)) { - size = round_up(size, min_block_size); - } - - pages = size >> ilog2(mm->chunk_size); - order = fls(pages) - 1; - min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); - - if (order > mm->max_order || size > mm->size) { - if ((flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION) && - !(flags & DRM_BUDDY_RANGE_ALLOCATION)) - return __alloc_contig_try_harder(mm, original_size, - original_min_size, blocks); - - return -EINVAL; - } - - do { - order = min(order, (unsigned int)fls(pages) - 1); - BUG_ON(order > mm->max_order); - BUG_ON(order < min_order); - - do { - block = __drm_buddy_alloc_blocks(mm, start, - end, - order, - flags); - if (!IS_ERR(block)) - break; - - if (order-- == min_order) { - /* Try allocation through force merge method */ - if (mm->clear_avail && - !__force_merge(mm, start, end, min_order)) { - block = __drm_buddy_alloc_blocks(mm, start, - end, - min_order, - flags); - if (!IS_ERR(block)) { - order = min_order; - break; - } - } - - /* - * Try contiguous block allocation through - * try harder method. - */ - if (flags & DRM_BUDDY_CONTIGUOUS_ALLOCATION && - !(flags & DRM_BUDDY_RANGE_ALLOCATION)) - return __alloc_contig_try_harder(mm, - original_size, - original_min_size, - blocks); - err = -ENOSPC; - goto err_free; - } - } while (1); - - mark_allocated(mm, block); - mm->avail -= drm_buddy_block_size(mm, block); - if (drm_buddy_block_is_clear(block)) - mm->clear_avail -= drm_buddy_block_size(mm, block); - kmemleak_update_trace(block); - list_add_tail(&block->link, &allocated); - - pages -= BIT(order); - - if (!pages) - break; - } while (1); - - /* Trim the allocated block to the required size */ - if (!(flags & DRM_BUDDY_TRIM_DISABLE) && - original_size != size) { - struct list_head *trim_list; - LIST_HEAD(temp); - u64 trim_size; - - trim_list = &allocated; - trim_size = original_size; - - if (!list_is_singular(&allocated)) { - block = list_last_entry(&allocated, typeof(*block), link); - list_move(&block->link, &temp); - trim_list = &temp; - trim_size = drm_buddy_block_size(mm, block) - - (size - original_size); - } - - drm_buddy_block_trim(mm, - NULL, - trim_size, - trim_list); - - if (!list_empty(&temp)) - list_splice_tail(trim_list, &allocated); - } - - list_splice_tail(&allocated, blocks); - return 0; - -err_free: - drm_buddy_free_list_internal(mm, &allocated); - return err; -} -EXPORT_SYMBOL(drm_buddy_alloc_blocks); +#include /** * drm_buddy_block_print - print block information @@ -1262,12 +21,12 @@ EXPORT_SYMBOL(drm_buddy_alloc_blocks); * @block: DRM buddy block * @p: DRM printer to use */ -void drm_buddy_block_print(struct drm_buddy *mm, - struct drm_buddy_block *block, +void drm_buddy_block_print(struct gpu_buddy *mm, + struct gpu_buddy_block *block, struct drm_printer *p) { - u64 start = drm_buddy_block_offset(block); - u64 size = drm_buddy_block_size(mm, block); + u64 start = gpu_buddy_block_offset(block); + u64 size = gpu_buddy_block_size(mm, block); drm_printf(p, "%#018llx-%#018llx: %llu\n", start, start + size, size); } @@ -1279,7 +38,7 @@ EXPORT_SYMBOL(drm_buddy_block_print); * @mm: DRM buddy manager * @p: DRM printer to use */ -void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) +void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p) { int order; @@ -1287,7 +46,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20); for (order = mm->max_order; order >= 0; order--) { - struct drm_buddy_block *block, *tmp; + struct gpu_buddy_block *block, *tmp; struct rb_root *root; u64 count = 0, free; unsigned int tree; @@ -1296,7 +55,7 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) root = &mm->free_trees[tree][order]; rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { - BUG_ON(!drm_buddy_block_is_free(block)); + BUG_ON(!gpu_buddy_block_is_free(block)); count++; } } @@ -1314,22 +73,5 @@ void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p) } EXPORT_SYMBOL(drm_buddy_print); -static void drm_buddy_module_exit(void) -{ - kmem_cache_destroy(slab_blocks); -} - -static int __init drm_buddy_module_init(void) -{ - slab_blocks = KMEM_CACHE(drm_buddy_block, 0); - if (!slab_blocks) - return -ENOMEM; - - return 0; -} - -module_init(drm_buddy_module_init); -module_exit(drm_buddy_module_exit); - -MODULE_DESCRIPTION("DRM Buddy Allocator"); +MODULE_DESCRIPTION("DRM-specific GPU Buddy Allocator Print Helpers"); MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c index 3fa38d4ac70b6..6a6ba50e69289 100644 --- a/drivers/gpu/drm/drm_client.c +++ b/drivers/gpu/drm/drm_client.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -172,25 +173,51 @@ void drm_client_release(struct drm_client_dev *client) } EXPORT_SYMBOL(drm_client_release); -static void drm_client_buffer_delete(struct drm_client_buffer *buffer) +/** + * drm_client_buffer_delete - Delete a client buffer + * @buffer: DRM client buffer + */ +void drm_client_buffer_delete(struct drm_client_buffer *buffer) { - if (buffer->gem) { - drm_gem_vunmap(buffer->gem, &buffer->map); - drm_gem_object_put(buffer->gem); - } + struct drm_gem_object *gem; + int ret; + + if (!buffer) + return; + + gem = buffer->fb->obj[0]; + drm_gem_vunmap(gem, &buffer->map); + + ret = drm_mode_rmfb(buffer->client->dev, buffer->fb->base.id, buffer->client->file); + if (ret) + drm_err(buffer->client->dev, + "Error removing FB:%u (%d)\n", buffer->fb->base.id, ret); + + drm_gem_object_put(buffer->gem); kfree(buffer); } +EXPORT_SYMBOL(drm_client_buffer_delete); static struct drm_client_buffer * drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height, - u32 format, u32 *handle) + u32 format, u32 handle, u32 pitch) { - const struct drm_format_info *info = drm_format_info(format); - struct drm_mode_create_dumb dumb_args = { }; + struct drm_mode_fb_cmd2 fb_req = { + .width = width, + .height = height, + .pixel_format = format, + .handles = { + handle, + }, + .pitches = { + pitch, + }, + }; struct drm_device *dev = client->dev; struct drm_client_buffer *buffer; struct drm_gem_object *obj; + struct drm_framebuffer *fb; int ret; buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); @@ -199,28 +226,38 @@ drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height, buffer->client = client; - dumb_args.width = width; - dumb_args.height = height; - dumb_args.bpp = drm_format_info_bpp(info, 0); - ret = drm_mode_create_dumb(dev, &dumb_args, client->file); - if (ret) - goto err_delete; - - obj = drm_gem_object_lookup(client->file, dumb_args.handle); + obj = drm_gem_object_lookup(client->file, handle); if (!obj) { ret = -ENOENT; goto err_delete; } - buffer->pitch = dumb_args.pitch; + ret = drm_mode_addfb2(dev, &fb_req, client->file); + if (ret) + goto err_drm_gem_object_put; + + fb = drm_framebuffer_lookup(dev, client->file, fb_req.fb_id); + if (drm_WARN_ON(dev, !fb)) { + ret = -ENOENT; + goto err_drm_mode_rmfb; + } + + /* drop the reference we picked up in framebuffer lookup */ + drm_framebuffer_put(fb); + + strscpy(fb->comm, client->name, TASK_COMM_LEN); + buffer->gem = obj; - *handle = dumb_args.handle; + buffer->fb = fb; return buffer; +err_drm_mode_rmfb: + drm_mode_rmfb(dev, fb_req.fb_id, client->file); +err_drm_gem_object_put: + drm_gem_object_put(obj); err_delete: - drm_client_buffer_delete(buffer); - + kfree(buffer); return ERR_PTR(ret); } @@ -247,7 +284,7 @@ drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height, int drm_client_buffer_vmap_local(struct drm_client_buffer *buffer, struct iosys_map *map_copy) { - struct drm_gem_object *gem = buffer->gem; + struct drm_gem_object *gem = buffer->fb->obj[0]; struct iosys_map *map = &buffer->map; int ret; @@ -276,7 +313,7 @@ EXPORT_SYMBOL(drm_client_buffer_vmap_local); */ void drm_client_buffer_vunmap_local(struct drm_client_buffer *buffer) { - struct drm_gem_object *gem = buffer->gem; + struct drm_gem_object *gem = buffer->fb->obj[0]; struct iosys_map *map = &buffer->map; drm_gem_vunmap_locked(gem, map); @@ -307,9 +344,10 @@ EXPORT_SYMBOL(drm_client_buffer_vunmap_local); int drm_client_buffer_vmap(struct drm_client_buffer *buffer, struct iosys_map *map_copy) { + struct drm_gem_object *gem = buffer->fb->obj[0]; int ret; - ret = drm_gem_vmap(buffer->gem, &buffer->map); + ret = drm_gem_vmap(gem, &buffer->map); if (ret) return ret; *map_copy = buffer->map; @@ -328,54 +366,11 @@ EXPORT_SYMBOL(drm_client_buffer_vmap); */ void drm_client_buffer_vunmap(struct drm_client_buffer *buffer) { - drm_gem_vunmap(buffer->gem, &buffer->map); -} -EXPORT_SYMBOL(drm_client_buffer_vunmap); - -static void drm_client_buffer_rmfb(struct drm_client_buffer *buffer) -{ - int ret; + struct drm_gem_object *gem = buffer->fb->obj[0]; - if (!buffer->fb) - return; - - ret = drm_mode_rmfb(buffer->client->dev, buffer->fb->base.id, buffer->client->file); - if (ret) - drm_err(buffer->client->dev, - "Error removing FB:%u (%d)\n", buffer->fb->base.id, ret); - - buffer->fb = NULL; -} - -static int drm_client_buffer_addfb(struct drm_client_buffer *buffer, - u32 width, u32 height, u32 format, - u32 handle) -{ - struct drm_client_dev *client = buffer->client; - struct drm_mode_fb_cmd2 fb_req = { }; - int ret; - - fb_req.width = width; - fb_req.height = height; - fb_req.pixel_format = format; - fb_req.handles[0] = handle; - fb_req.pitches[0] = buffer->pitch; - - ret = drm_mode_addfb2(client->dev, &fb_req, client->file); - if (ret) - return ret; - - buffer->fb = drm_framebuffer_lookup(client->dev, buffer->client->file, fb_req.fb_id); - if (WARN_ON(!buffer->fb)) - return -ENOENT; - - /* drop the reference we picked up in framebuffer lookup */ - drm_framebuffer_put(buffer->fb); - - strscpy(buffer->fb->comm, client->name, TASK_COMM_LEN); - - return 0; + drm_gem_vunmap(gem, &buffer->map); } +EXPORT_SYMBOL(drm_client_buffer_vunmap); /** * drm_client_framebuffer_create - Create a client framebuffer @@ -386,7 +381,7 @@ static int drm_client_buffer_addfb(struct drm_client_buffer *buffer, * * This function creates a &drm_client_buffer which consists of a * &drm_framebuffer backed by a dumb buffer. - * Call drm_client_framebuffer_delete() to free the buffer. + * Call drm_client_buffer_delete() to free the buffer. * * Returns: * Pointer to a client buffer or an error pointer on failure. @@ -394,16 +389,25 @@ static int drm_client_buffer_addfb(struct drm_client_buffer *buffer, struct drm_client_buffer * drm_client_framebuffer_create(struct drm_client_dev *client, u32 width, u32 height, u32 format) { + const struct drm_format_info *info = drm_format_info(format); + struct drm_device *dev = client->dev; + struct drm_mode_create_dumb dumb_args = { }; struct drm_client_buffer *buffer; - u32 handle; int ret; - buffer = drm_client_buffer_create(client, width, height, format, - &handle); - if (IS_ERR(buffer)) - return buffer; + dumb_args.width = width; + dumb_args.height = height; + dumb_args.bpp = drm_format_info_bpp(info, 0); + ret = drm_mode_create_dumb(dev, &dumb_args, client->file); + if (ret) + return ERR_PTR(ret); - ret = drm_client_buffer_addfb(buffer, width, height, format, handle); + buffer = drm_client_buffer_create(client, width, height, format, + dumb_args.handle, dumb_args.pitch); + if (IS_ERR(buffer)) { + ret = PTR_ERR(buffer); + goto err_drm_mode_destroy_dumb; + } /* * The handle is only needed for creating the framebuffer, destroy it @@ -411,30 +415,15 @@ drm_client_framebuffer_create(struct drm_client_dev *client, u32 width, u32 heig * object as DMA-buf. The framebuffer and our buffer structure are still * holding references to the GEM object to prevent its destruction. */ - drm_mode_destroy_dumb(client->dev, handle, client->file); - - if (ret) { - drm_client_buffer_delete(buffer); - return ERR_PTR(ret); - } + drm_mode_destroy_dumb(client->dev, dumb_args.handle, client->file); return buffer; -} -EXPORT_SYMBOL(drm_client_framebuffer_create); - -/** - * drm_client_framebuffer_delete - Delete a client framebuffer - * @buffer: DRM client buffer (can be NULL) - */ -void drm_client_framebuffer_delete(struct drm_client_buffer *buffer) -{ - if (!buffer) - return; - drm_client_buffer_rmfb(buffer); - drm_client_buffer_delete(buffer); +err_drm_mode_destroy_dumb: + drm_mode_destroy_dumb(client->dev, dumb_args.handle, client->file); + return ERR_PTR(ret); } -EXPORT_SYMBOL(drm_client_framebuffer_delete); +EXPORT_SYMBOL(drm_client_framebuffer_create); /** * drm_client_framebuffer_flush - Manually flush client framebuffer diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index 9a5f20596126a..79eeaf66d0924 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -3613,7 +3613,7 @@ EXPORT_SYMBOL(drm_mode_put_tile_group); /** * drm_mode_get_tile_group - get a reference to an existing tile group * @dev: DRM device - * @topology: 8-bytes unique per monitor. + * @topology_id: 9-byte unique ID per monitor. * * Use the unique bytes to get a reference to an existing tile group. * @@ -3621,14 +3621,14 @@ EXPORT_SYMBOL(drm_mode_put_tile_group); * tile group or NULL if not found. */ struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev, - const char topology[8]) + const char topology_id[9]) { struct drm_tile_group *tg; int id; mutex_lock(&dev->mode_config.idr_mutex); idr_for_each_entry(&dev->mode_config.tile_idr, tg, id) { - if (!memcmp(tg->group_data, topology, 8)) { + if (!memcmp(tg->group_data, topology_id, sizeof(tg->group_data))) { if (!kref_get_unless_zero(&tg->refcount)) tg = NULL; mutex_unlock(&dev->mode_config.idr_mutex); @@ -3643,7 +3643,7 @@ EXPORT_SYMBOL(drm_mode_get_tile_group); /** * drm_mode_create_tile_group - create a tile group from a displayid description * @dev: DRM device - * @topology: 8-bytes unique per monitor. + * @topology_id: 9-byte unique ID per monitor. * * Create a tile group for the unique monitor, and get a unique * identifier for the tile group. @@ -3652,7 +3652,7 @@ EXPORT_SYMBOL(drm_mode_get_tile_group); * new tile group or NULL. */ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev, - const char topology[8]) + const char topology_id[9]) { struct drm_tile_group *tg; int ret; @@ -3662,7 +3662,7 @@ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev, return NULL; kref_init(&tg->refcount); - memcpy(tg->group_data, topology, 8); + memcpy(tg->group_data, topology_id, sizeof(tg->group_data)); tg->dev = dev; mutex_lock(&dev->mode_config.idr_mutex); diff --git a/drivers/gpu/drm/drm_displayid_internal.h b/drivers/gpu/drm/drm_displayid_internal.h index 5b1b32f735166..4590d6a3d8215 100644 --- a/drivers/gpu/drm/drm_displayid_internal.h +++ b/drivers/gpu/drm/drm_displayid_internal.h @@ -109,7 +109,7 @@ struct displayid_tiled_block { u8 topo[3]; u8 tile_size[4]; u8 tile_pixel_bezel[5]; - u8 topology_id[8]; + u8 topology_id[9]; } __packed; struct displayid_detailed_timings_1 { diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c index 8e3cb08241c81..281231f10d853 100644 --- a/drivers/gpu/drm/drm_drv.c +++ b/drivers/gpu/drm/drm_drv.c @@ -472,6 +472,22 @@ void drm_dev_exit(int idx) } EXPORT_SYMBOL(drm_dev_exit); +/* + * Mark the device as unplugged and wait for any in-flight drm_dev_enter() + * critical sections to complete. + */ +static void drm_dev_synchronize_unplug(struct drm_device *dev) +{ + /* + * After synchronizing any critical read section is guaranteed to see + * the new value of ->unplugged, and any critical section which might + * still have seen the old value of ->unplugged is guaranteed to have + * finished. + */ + dev->unplugged = true; + synchronize_srcu(&drm_unplug_srcu); +} + /** * drm_dev_unplug - unplug a DRM device * @dev: DRM device @@ -484,15 +500,7 @@ EXPORT_SYMBOL(drm_dev_exit); */ void drm_dev_unplug(struct drm_device *dev) { - /* - * After synchronizing any critical read section is guaranteed to see - * the new value of ->unplugged, and any critical section which might - * still have seen the old value of ->unplugged is guaranteed to have - * finished. - */ - dev->unplugged = true; - synchronize_srcu(&drm_unplug_srcu); - + drm_dev_synchronize_unplug(dev); drm_dev_unregister(dev); /* Clear all CPU mappings pointing to this device */ @@ -1087,6 +1095,7 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) goto err_minors; dev->registered = true; + dev->unplugged = false; if (driver->load) { ret = driver->load(dev, flags); @@ -1113,6 +1122,13 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags) if (dev->driver->unload) dev->driver->unload(dev); err_minors: + /* + * If a minor was registered before the failure, userspace could have + * opened it and entered a drm_dev_enter() critical section. Ensure all + * such sections complete before we clean up. + */ + drm_dev_synchronize_unplug(dev); + remove_compat_control_link(dev); drm_minor_unregister(dev, DRM_MINOR_ACCEL); drm_minor_unregister(dev, DRM_MINOR_PRIMARY); diff --git a/drivers/gpu/drm/drm_dumb_buffers.c b/drivers/gpu/drm/drm_dumb_buffers.c index 70032bba1c97e..7b4ba9fab299a 100644 --- a/drivers/gpu/drm/drm_dumb_buffers.c +++ b/drivers/gpu/drm/drm_dumb_buffers.c @@ -29,7 +29,6 @@ #include #include "drm_crtc_internal.h" -#include "drm_internal.h" /** * DOC: overview diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c index 8d06014001828..5c97f8ba84981 100644 --- a/drivers/gpu/drm/drm_exec.c +++ b/drivers/gpu/drm/drm_exec.c @@ -24,7 +24,6 @@ * * struct drm_gem_object *obj; * struct drm_exec exec; - * unsigned long index; * int ret; * * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT); @@ -40,7 +39,7 @@ * goto error; * } * - * drm_exec_for_each_locked_object(&exec, index, obj) { + * drm_exec_for_each_locked_object(&exec, obj) { * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ); * ... * } @@ -56,9 +55,8 @@ static void drm_exec_unlock_all(struct drm_exec *exec) { struct drm_gem_object *obj; - unsigned long index; - drm_exec_for_each_locked_object_reverse(exec, index, obj) { + drm_exec_for_each_locked_object_reverse(exec, obj) { dma_resv_unlock(obj->resv); drm_gem_object_put(obj); } @@ -327,6 +325,19 @@ int drm_exec_prepare_array(struct drm_exec *exec, { int ret; + /* + * Make sure to lock a contended object even when no objects are + * given, otherwise drm_exec_retry_on_contention() would loop + * forever on patterns like: + * + * ret = drm_exec_prepare_array(exec, objs, num_objects, ...); + * drm_exec_retry_on_contention(exec); + * + * with num_objects == 0. + */ + if (!num_objects) + return drm_exec_lock_contended(exec); + for (unsigned int i = 0; i < num_objects; ++i) { ret = drm_exec_prepare_obj(exec, objects[i], num_fences); if (unlikely(ret)) diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 2f4bae05a1953..15dbf3f1d0b03 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -489,20 +489,7 @@ int drm_fb_helper_init(struct drm_device *dev, } EXPORT_SYMBOL(drm_fb_helper_init); -/** - * drm_fb_helper_alloc_info - allocate fb_info and some of its members - * @fb_helper: driver-allocated fbdev helper - * - * A helper to alloc fb_info and the member cmap. Called by the driver - * within the struct &drm_driver.fbdev_probe callback function. Drivers do - * not need to release the allocated fb_info structure themselves, this is - * automatically done when calling drm_fb_helper_fini(). - * - * RETURNS: - * fb_info pointer if things went okay, pointer containing error code - * otherwise - */ -struct fb_info *drm_fb_helper_alloc_info(struct drm_fb_helper *fb_helper) +static struct fb_info *drm_fb_helper_alloc_info(struct drm_fb_helper *fb_helper) { struct device *dev = fb_helper->dev->dev; struct fb_info *info; @@ -529,17 +516,8 @@ struct fb_info *drm_fb_helper_alloc_info(struct drm_fb_helper *fb_helper) framebuffer_release(info); return ERR_PTR(ret); } -EXPORT_SYMBOL(drm_fb_helper_alloc_info); -/** - * drm_fb_helper_release_info - release fb_info and its members - * @fb_helper: driver-allocated fbdev helper - * - * A helper to release fb_info and the member cmap. Drivers do not - * need to release the allocated fb_info structure themselves, this is - * automatically done when calling drm_fb_helper_fini(). - */ -void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper) +static void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper) { struct fb_info *info = fb_helper->info; @@ -552,7 +530,6 @@ void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper) fb_dealloc_cmap(&info->cmap); framebuffer_release(info); } -EXPORT_SYMBOL(drm_fb_helper_release_info); /** * drm_fb_helper_unregister_info - unregister fb_info framebuffer device @@ -1813,6 +1790,13 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper) height = dev->mode_config.max_height; drm_client_modeset_probe(&fb_helper->client, width, height); + + info = drm_fb_helper_alloc_info(fb_helper); + if (IS_ERR(info)) { + mutex_unlock(&fb_helper->lock); + return PTR_ERR(info); + } + ret = drm_fb_helper_single_fb_probe(fb_helper); if (ret < 0) { if (ret == -EAGAIN) { @@ -1821,13 +1805,12 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper) } mutex_unlock(&fb_helper->lock); - return ret; + goto err_drm_fb_helper_release_info; } drm_setup_crtcs_fb(fb_helper); fb_helper->deferred_setup = false; - info = fb_helper->info; info->var.pixclock = 0; /* Need to drop locks to avoid recursive deadlock in @@ -1859,6 +1842,10 @@ __drm_fb_helper_initial_config_and_unlock(struct drm_fb_helper *fb_helper) mutex_unlock(&kernel_fb_helper_lock); return 0; + +err_drm_fb_helper_release_info: + drm_fb_helper_release_info(fb_helper); + return ret; } /** diff --git a/drivers/gpu/drm/drm_fbdev_dma.c b/drivers/gpu/drm/drm_fbdev_dma.c index 8bd626ef16c71..feecd44e91eb8 100644 --- a/drivers/gpu/drm/drm_fbdev_dma.c +++ b/drivers/gpu/drm/drm_fbdev_dma.c @@ -55,7 +55,7 @@ static void drm_fbdev_dma_fb_destroy(struct fb_info *info) drm_fb_helper_fini(fb_helper); drm_client_buffer_vunmap(fb_helper->buffer); - drm_client_framebuffer_delete(fb_helper->buffer); + drm_client_buffer_delete(fb_helper->buffer); drm_client_release(&fb_helper->client); drm_fb_helper_unprepare(fb_helper); kfree(fb_helper); @@ -90,7 +90,7 @@ static void drm_fbdev_dma_shadowed_fb_destroy(struct fb_info *info) vfree(shadow); drm_client_buffer_vunmap(fb_helper->buffer); - drm_client_framebuffer_delete(fb_helper->buffer); + drm_client_buffer_delete(fb_helper->buffer); drm_client_release(&fb_helper->client); drm_fb_helper_unprepare(fb_helper); kfree(fb_helper); @@ -272,9 +272,9 @@ int drm_fbdev_dma_driver_fbdev_probe(struct drm_fb_helper *fb_helper, { struct drm_client_dev *client = &fb_helper->client; struct drm_device *dev = fb_helper->dev; + struct fb_info *info = fb_helper->info; struct drm_client_buffer *buffer; struct drm_framebuffer *fb; - struct fb_info *info; u32 format; struct iosys_map map; int ret; @@ -304,12 +304,6 @@ int drm_fbdev_dma_driver_fbdev_probe(struct drm_fb_helper *fb_helper, fb_helper->buffer = buffer; fb_helper->fb = fb; - info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_drm_client_buffer_vunmap; - } - drm_fb_helper_fill_info(info, fb_helper, sizes); if (fb->funcs->dirty) @@ -317,18 +311,16 @@ int drm_fbdev_dma_driver_fbdev_probe(struct drm_fb_helper *fb_helper, else ret = drm_fbdev_dma_driver_fbdev_probe_tail(fb_helper, sizes); if (ret) - goto err_drm_fb_helper_release_info; + goto err_drm_client_buffer_vunmap; return 0; -err_drm_fb_helper_release_info: - drm_fb_helper_release_info(fb_helper); err_drm_client_buffer_vunmap: fb_helper->fb = NULL; fb_helper->buffer = NULL; drm_client_buffer_vunmap(buffer); err_drm_client_buffer_delete: - drm_client_framebuffer_delete(buffer); + drm_client_buffer_delete(buffer); return ret; } EXPORT_SYMBOL(drm_fbdev_dma_driver_fbdev_probe); diff --git a/drivers/gpu/drm/drm_fbdev_shmem.c b/drivers/gpu/drm/drm_fbdev_shmem.c index 1e827bf8b815d..d435caed70029 100644 --- a/drivers/gpu/drm/drm_fbdev_shmem.c +++ b/drivers/gpu/drm/drm_fbdev_shmem.c @@ -63,7 +63,7 @@ static void drm_fbdev_shmem_fb_destroy(struct fb_info *info) drm_fb_helper_fini(fb_helper); drm_client_buffer_vunmap(fb_helper->buffer); - drm_client_framebuffer_delete(fb_helper->buffer); + drm_client_buffer_delete(fb_helper->buffer); drm_client_release(&fb_helper->client); drm_fb_helper_unprepare(fb_helper); kfree(fb_helper); @@ -136,10 +136,10 @@ int drm_fbdev_shmem_driver_fbdev_probe(struct drm_fb_helper *fb_helper, { struct drm_client_dev *client = &fb_helper->client; struct drm_device *dev = fb_helper->dev; + struct fb_info *info = fb_helper->info; struct drm_client_buffer *buffer; struct drm_gem_shmem_object *shmem; struct drm_framebuffer *fb; - struct fb_info *info; u32 format; struct iosys_map map; int ret; @@ -169,12 +169,6 @@ int drm_fbdev_shmem_driver_fbdev_probe(struct drm_fb_helper *fb_helper, fb_helper->buffer = buffer; fb_helper->fb = fb; - info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_drm_client_buffer_vunmap; - } - drm_fb_helper_fill_info(info, fb_helper, sizes); info->fbops = &drm_fbdev_shmem_fb_ops; @@ -195,18 +189,16 @@ int drm_fbdev_shmem_driver_fbdev_probe(struct drm_fb_helper *fb_helper, info->fbdefio = &fb_helper->fbdefio; ret = fb_deferred_io_init(info); if (ret) - goto err_drm_fb_helper_release_info; + goto err_drm_client_buffer_vunmap; return 0; -err_drm_fb_helper_release_info: - drm_fb_helper_release_info(fb_helper); err_drm_client_buffer_vunmap: fb_helper->fb = NULL; fb_helper->buffer = NULL; drm_client_buffer_vunmap(buffer); err_drm_client_buffer_delete: - drm_client_framebuffer_delete(buffer); + drm_client_buffer_delete(buffer); return ret; } EXPORT_SYMBOL(drm_fbdev_shmem_driver_fbdev_probe); diff --git a/drivers/gpu/drm/drm_fbdev_ttm.c b/drivers/gpu/drm/drm_fbdev_ttm.c index 85feb55bba113..c2aa65b2384f7 100644 --- a/drivers/gpu/drm/drm_fbdev_ttm.c +++ b/drivers/gpu/drm/drm_fbdev_ttm.c @@ -50,7 +50,7 @@ static void drm_fbdev_ttm_fb_destroy(struct fb_info *info) fb_deferred_io_cleanup(info); drm_fb_helper_fini(fb_helper); vfree(shadow); - drm_client_framebuffer_delete(fb_helper->buffer); + drm_client_buffer_delete(fb_helper->buffer); drm_client_release(&fb_helper->client); drm_fb_helper_unprepare(fb_helper); @@ -176,8 +176,8 @@ int drm_fbdev_ttm_driver_fbdev_probe(struct drm_fb_helper *fb_helper, { struct drm_client_dev *client = &fb_helper->client; struct drm_device *dev = fb_helper->dev; + struct fb_info *info = fb_helper->info; struct drm_client_buffer *buffer; - struct fb_info *info; size_t screen_size; void *screen_buffer; u32 format; @@ -202,13 +202,7 @@ int drm_fbdev_ttm_driver_fbdev_probe(struct drm_fb_helper *fb_helper, screen_buffer = vzalloc(screen_size); if (!screen_buffer) { ret = -ENOMEM; - goto err_drm_client_framebuffer_delete; - } - - info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_vfree; + goto err_drm_client_buffer_delete; } drm_fb_helper_fill_info(info, fb_helper, sizes); @@ -227,18 +221,16 @@ int drm_fbdev_ttm_driver_fbdev_probe(struct drm_fb_helper *fb_helper, info->fbdefio = &fb_helper->fbdefio; ret = fb_deferred_io_init(info); if (ret) - goto err_drm_fb_helper_release_info; + goto err_vfree; return 0; -err_drm_fb_helper_release_info: - drm_fb_helper_release_info(fb_helper); err_vfree: vfree(screen_buffer); -err_drm_client_framebuffer_delete: +err_drm_client_buffer_delete: fb_helper->fb = NULL; fb_helper->buffer = NULL; - drm_client_framebuffer_delete(buffer); + drm_client_buffer_delete(buffer); return ret; } EXPORT_SYMBOL(drm_fbdev_ttm_driver_fbdev_probe); diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c index 5c4a113b08941..224ed96bd0d4a 100644 --- a/drivers/gpu/drm/drm_gem.c +++ b/drivers/gpu/drm/drm_gem.c @@ -1601,6 +1601,16 @@ drm_gem_lru_scan(struct drm_gem_lru *lru, */ WARN_ON(obj->lru == &still_in_lru); WARN_ON(obj->lru == lru); + } else if (obj->lru == &still_in_lru) { + /* + * If the object wasn't moved and wasn't shrunk either, + * it's still remaining as reclaimable. Note that + * obj->lru is supposed to be checked with the LRU lock + * held for an accurate result, but we don't care about + * accuracy here. Worst thing that could happen is an + * extra scan. + */ + *remaining += obj->size >> PAGE_SHIFT; } dma_resv_unlock(obj->resv); diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c index 4cde5bf4aedd8..e5c7befe31081 100644 --- a/drivers/gpu/drm/drm_gpusvm.c +++ b/drivers/gpu/drm/drm_gpusvm.c @@ -1390,6 +1390,14 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, err = -EAGAIN; goto err_unmap; } + + /* + * Set the dpagemap as soon as the first + * device page is mapped so the err_unmap path + * can device_unmap() the device mappings that + * have already been created. + */ + svm_pages->dpagemap = dpagemap; } svm_pages->dma_addr[j] = dpagemap->ops->device_map(dpagemap, @@ -1432,10 +1440,8 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, flags.has_dma_mapping = true; } - if (pagemap) { + if (pagemap) flags.has_devmem_pages = true; - svm_pages->dpagemap = dpagemap; - } /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); @@ -1566,8 +1572,10 @@ int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, return -EFAULT; pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); - if (!pfns) + if (!pfns) { + mmput(mm); return -ENOMEM; + } hmm_range.hmm_pfns = pfns; while (!time_after(jiffies, timeout)) { diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c index 5c8bf010cd965..61c3df0711a35 100644 --- a/drivers/gpu/drm/drm_gpuvm.c +++ b/drivers/gpu/drm/drm_gpuvm.c @@ -1521,9 +1521,8 @@ drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, enum dma_resv_usage extobj_usage) { struct drm_gem_object *obj; - unsigned long index; - drm_exec_for_each_locked_object(exec, index, obj) { + drm_exec_for_each_locked_object(exec, obj) { dma_resv_assert_held(obj->resv); dma_resv_add_fence(obj->resv, fence, drm_gpuvm_is_extobj(gpuvm, obj) ? diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c index 445cf0e6589fa..4cd5182f82051 100644 --- a/drivers/gpu/drm/drm_pagemap.c +++ b/drivers/gpu/drm/drm_pagemap.c @@ -606,12 +606,12 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) goto err_finalize; err_finalize: + drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, + DMA_FROM_DEVICE); if (err) drm_pagemap_migration_unlock_put_pages(npages, dst); migrate_device_pages(src, dst, npages); migrate_device_finalize(src, dst, npages); - drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages, - DMA_FROM_DEVICE); err_free: kvfree(buf); err_out: @@ -742,13 +742,13 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, goto err_finalize; err_finalize: + if (dev) + drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages, + DMA_FROM_DEVICE); if (err) drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); migrate_vma_pages(&migrate); migrate_vma_finalize(&migrate); - if (dev) - drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages, - DMA_FROM_DEVICE); err_free: kvfree(buf); err_out: diff --git a/drivers/gpu/drm/drm_panel_backlight_quirks.c b/drivers/gpu/drm/drm_panel_backlight_quirks.c index 537dc6dd05343..2d0238382ebd5 100644 --- a/drivers/gpu/drm/drm_panel_backlight_quirks.c +++ b/drivers/gpu/drm/drm_panel_backlight_quirks.c @@ -21,6 +21,15 @@ struct drm_get_panel_backlight_quirk { }; static const struct drm_get_panel_backlight_quirk drm_panel_min_backlight_quirks[] = { + /* Lenovo Legion 5 15ARH05, AUX backlight non-functional, force PWM */ + { + .dmi_match.field = DMI_SYS_VENDOR, + .dmi_match.value = "LENOVO", + .dmi_match_other.field = DMI_PRODUCT_VERSION, + .dmi_match_other.value = "Lenovo Legion 5 15ARH05", + .ident.panel_id = drm_edid_encode_panel_id('B', 'O', 'E', 0x08df), + .quirk = { .force_pwm = true, }, + }, /* 13 inch matte panel */ { .dmi_match.field = DMI_BOARD_VENDOR, diff --git a/drivers/gpu/drm/drm_panic_qr.rs b/drivers/gpu/drm/drm_panic_qr.rs index ac27e86c601c8..4d7eb75a3afc1 100644 --- a/drivers/gpu/drm/drm_panic_qr.rs +++ b/drivers/gpu/drm/drm_panic_qr.rs @@ -407,8 +407,8 @@ impl DecFifo { for i in (0..self.len).rev() { self.decimals[i + len] = self.decimals[i]; } - for i in 0..len { - self.decimals[i] = (chunk % 10) as u8; + for decimal in &mut self.decimals[..len] { + *decimal = (chunk % 10) as u8; chunk = div10(chunk); } self.len += len; diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index 451ec96202266..25d19e6037b48 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -649,9 +649,15 @@ void drm_calc_timestamping_constants(struct drm_crtc *crtc, framedur_ns = div_u64((u64) frame_size * 1000000, dotclock); /* - * Fields of interlaced scanout modes are only half a frame duration. + * Fields of interlaced scanout modes are only half a frame + * duration. Halving is only correct while the crtc timings + * still describe a whole frame: a driver that applied + * CRTC_INTERLACE_HALVE_V has already reduced crtc_vtotal to a + * single field, so frame_size above is a field already and + * halving it again reports half the real vblank interval. */ - if (mode->flags & DRM_MODE_FLAG_INTERLACE) + if ((mode->flags & DRM_MODE_FLAG_INTERLACE) && + mode->crtc_vtotal >= mode->vtotal) framedur_ns /= 2; } else { drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n", diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c index 93de25b77e686..b0bd0697910e1 100644 --- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c +++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c @@ -59,18 +59,11 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper, struct drm_fb_helper_surface_size *sizes, struct exynos_drm_gem *exynos_gem) { - struct fb_info *fbi; + struct fb_info *fbi = helper->info; struct drm_framebuffer *fb = helper->fb; unsigned int size = fb->width * fb->height * fb->format->cpp[0]; unsigned long offset; - fbi = drm_fb_helper_alloc_info(helper); - if (IS_ERR(fbi)) { - DRM_DEV_ERROR(to_dma_dev(helper->dev), - "failed to allocate fb info.\n"); - return PTR_ERR(fbi); - } - fbi->fbops = &exynos_drm_fb_ops; drm_fb_helper_fill_info(fbi, helper, sizes); diff --git a/drivers/gpu/drm/gma500/fbdev.c b/drivers/gpu/drm/gma500/fbdev.c index a6af21514cff7..dacc118b12429 100644 --- a/drivers/gpu/drm/gma500/fbdev.c +++ b/drivers/gpu/drm/gma500/fbdev.c @@ -111,7 +111,7 @@ int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, struct drm_device *dev = fb_helper->dev; struct drm_psb_private *dev_priv = to_drm_psb_private(dev); struct pci_dev *pdev = to_pci_dev(dev->dev); - struct fb_info *info; + struct fb_info *info = fb_helper->info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd2 mode_cmd = { }; int size; @@ -170,12 +170,6 @@ int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, fb_helper->funcs = &psb_fbdev_fb_helper_funcs; fb_helper->fb = fb; - info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_drm_framebuffer_unregister_private; - } - info->fbops = &psb_fbdev_fb_ops; /* Accessed stolen memory directly */ @@ -199,10 +193,6 @@ int psb_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, return 0; -err_drm_framebuffer_unregister_private: - drm_framebuffer_unregister_private(fb); - drm_framebuffer_cleanup(fb); - kfree(fb); err_drm_gem_object_put: drm_gem_object_put(obj); return ret; diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi_i2c.c b/drivers/gpu/drm/gma500/oaktrail_hdmi_i2c.c index 6daa6669ed237..4f1d095fc5a81 100644 --- a/drivers/gpu/drm/gma500/oaktrail_hdmi_i2c.c +++ b/drivers/gpu/drm/gma500/oaktrail_hdmi_i2c.c @@ -95,6 +95,7 @@ static int xfer_read(struct i2c_adapter *adap, struct i2c_msg *pmsg) struct oaktrail_hdmi_dev *hdmi_dev = i2c_get_adapdata(adap); struct hdmi_i2c_dev *i2c_dev = hdmi_dev->i2c_dev; u32 temp; + int ret; i2c_dev->status = I2C_STAT_INIT; i2c_dev->msg = pmsg; @@ -106,9 +107,14 @@ static int xfer_read(struct i2c_adapter *adap, struct i2c_msg *pmsg) HDMI_WRITE(HDMI_HI2CHCR, temp); HDMI_READ(HDMI_HI2CHCR); - while (i2c_dev->status != I2C_TRANSACTION_DONE) - wait_for_completion_interruptible_timeout(&i2c_dev->complete, + while (i2c_dev->status != I2C_TRANSACTION_DONE) { + ret = wait_for_completion_interruptible_timeout(&i2c_dev->complete, 10 * HZ); + if (ret < 0) + return ret; + if (!ret) + return -ETIMEDOUT; + } return 0; } @@ -127,7 +133,7 @@ static int oaktrail_hdmi_i2c_access(struct i2c_adapter *adap, { struct oaktrail_hdmi_dev *hdmi_dev = i2c_get_adapdata(adap); struct hdmi_i2c_dev *i2c_dev = hdmi_dev->i2c_dev; - int i; + int i, ret = 0; mutex_lock(&i2c_dev->i2c_lock); @@ -139,9 +145,11 @@ static int oaktrail_hdmi_i2c_access(struct i2c_adapter *adap, for (i = 0; i < num; i++) { if (pmsg->len && pmsg->buf) { if (pmsg->flags & I2C_M_RD) - xfer_read(adap, pmsg); + ret = xfer_read(adap, pmsg); else - xfer_write(adap, pmsg); + ret = xfer_write(adap, pmsg); + if (ret) + break; } pmsg++; /* next message */ } @@ -151,6 +159,9 @@ static int oaktrail_hdmi_i2c_access(struct i2c_adapter *adap, mutex_unlock(&i2c_dev->i2c_lock); + if (ret) + return ret; + return i; } diff --git a/drivers/gpu/drm/gud/gud_connector.c b/drivers/gpu/drm/gud/gud_connector.c index 4a15695fa933c..753b6dcd7ce50 100644 --- a/drivers/gpu/drm/gud/gud_connector.c +++ b/drivers/gpu/drm/gud/gud_connector.c @@ -396,8 +396,16 @@ static int gud_connector_add_tv_mode(struct gud_device *gdrm, struct drm_connect } num_modes = ret / GUD_CONNECTOR_TV_MODE_NAME_LEN; - for (i = 0; i < num_modes; i++) - modes[i] = &buf[i * GUD_CONNECTOR_TV_MODE_NAME_LEN]; + for (i = 0; i < num_modes; i++) { + char *mode = &buf[i * GUD_CONNECTOR_TV_MODE_NAME_LEN]; + + if (!memchr(mode, '\0', GUD_CONNECTOR_TV_MODE_NAME_LEN)) { + ret = -EIO; + goto free; + } + + modes[i] = mode; + } ret = drm_mode_create_tv_properties_legacy(connector->dev, num_modes, modes); free: diff --git a/drivers/gpu/drm/gud/gud_drv.c b/drivers/gpu/drm/gud/gud_drv.c index a2000991ecbe4..81c0440204c7c 100644 --- a/drivers/gpu/drm/gud/gud_drv.c +++ b/drivers/gpu/drm/gud/gud_drv.c @@ -289,6 +289,8 @@ static int gud_plane_add_properties(struct gud_device *gdrm) * but mask out any additions on future devices. */ val &= GUD_ROTATION_MASK; + if (!(val & GUD_ROTATION_0)) + continue; ret = drm_plane_create_rotation_property(&gdrm->plane, DRM_MODE_ROTATE_0, val); break; @@ -684,6 +686,7 @@ static int gud_resume(struct usb_interface *intf) static const struct usb_device_id gud_id_table[] = { { USB_DEVICE_INTERFACE_CLASS(0x1d50, 0x614d, USB_CLASS_VENDOR_SPEC) }, { USB_DEVICE_INTERFACE_CLASS(0x16d0, 0x10a9, USB_CLASS_VENDOR_SPEC) }, + { USB_DEVICE_INTERFACE_CLASS(0x1209, 0x4fb3, USB_CLASS_VENDOR_SPEC) }, { } }; diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c index db7fce4e8cc34..165e7e26e6895 100644 --- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c +++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c @@ -72,46 +72,28 @@ static int hibmc_get_best_clock_idx(const struct drm_display_mode *mode) static int hibmc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state) { - struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, - plane); - struct drm_framebuffer *fb = new_plane_state->fb; - struct drm_crtc *crtc = new_plane_state->crtc; - struct drm_crtc_state *crtc_state; - u32 src_w = new_plane_state->src_w >> 16; - u32 src_h = new_plane_state->src_h >> 16; - - if (!crtc || !fb) - return 0; + struct drm_plane_state *new_plane_state = + drm_atomic_get_new_plane_state(state, plane); + struct drm_crtc_state *new_crtc_state = NULL; + int ret; - crtc_state = drm_atomic_get_crtc_state(state, crtc); - if (IS_ERR(crtc_state)) - return PTR_ERR(crtc_state); + if (new_plane_state->crtc) + new_crtc_state = drm_atomic_get_new_crtc_state(state, new_plane_state->crtc); - if (src_w != new_plane_state->crtc_w || src_h != new_plane_state->crtc_h) { - drm_dbg_atomic(plane->dev, "scale not support\n"); - return -EINVAL; - } - - if (new_plane_state->crtc_x < 0 || new_plane_state->crtc_y < 0) { - drm_dbg_atomic(plane->dev, "crtc_x/y of drm_plane state is invalid\n"); - return -EINVAL; - } - - if (!crtc_state->enable) + ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state, + DRM_PLANE_NO_SCALING, + DRM_PLANE_NO_SCALING, + false, true); + if (ret) + return ret; + else if (!new_plane_state->visible) return 0; - if (new_plane_state->crtc_x + new_plane_state->crtc_w > - crtc_state->adjusted_mode.hdisplay || - new_plane_state->crtc_y + new_plane_state->crtc_h > - crtc_state->adjusted_mode.vdisplay) { - drm_dbg_atomic(plane->dev, "visible portion of plane is invalid\n"); - return -EINVAL; - } - if (new_plane_state->fb->pitches[0] % 128 != 0) { drm_dbg_atomic(plane->dev, "wrong stride with 128-byte aligned\n"); return -EINVAL; } + return 0; } @@ -153,10 +135,8 @@ static void hibmc_plane_atomic_update(struct drm_plane *plane, } static const u32 channel_formats1[] = { - DRM_FORMAT_RGB565, DRM_FORMAT_BGR565, DRM_FORMAT_RGB888, - DRM_FORMAT_BGR888, DRM_FORMAT_XRGB8888, DRM_FORMAT_XBGR8888, - DRM_FORMAT_RGBA8888, DRM_FORMAT_BGRA8888, DRM_FORMAT_ARGB8888, - DRM_FORMAT_ABGR8888 + DRM_FORMAT_XRGB8888, + DRM_FORMAT_RGB565, }; static const struct drm_plane_funcs hibmc_plane_funcs = { diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c index 3596dce84c286..a2b6f40b4b0c0 100644 --- a/drivers/gpu/drm/i915/display/intel_bios.c +++ b/drivers/gpu/drm/i915/display/intel_bios.c @@ -624,6 +624,21 @@ get_lfp_data_tail(const struct bdb_lfp_data *data, return NULL; } +static bool is_panel_type_valid(int panel_type) +{ + return panel_type >= 0 && panel_type < 16; +} + +static bool is_panel_type_pnp(int panel_type) +{ + return panel_type == 0xff; +} + +static bool is_panel_type_valid_or_pnp(int panel_type) +{ + return is_panel_type_valid(panel_type) || is_panel_type_pnp(panel_type); +} + static int opregion_get_panel_type(struct intel_display *display, const struct intel_bios_encoder_data *devdata, const struct drm_edid *drm_edid, bool use_fallback) @@ -641,15 +656,21 @@ static int vbt_get_panel_type(struct intel_display *display, if (!lfp_options) return -1; - if (lfp_options->panel_type > 0xf && - lfp_options->panel_type != 0xff) { + if (!is_panel_type_valid_or_pnp(lfp_options->panel_type)) { drm_dbg_kms(display->drm, "Invalid VBT panel type 0x%x\n", lfp_options->panel_type); return -1; } - if (devdata && devdata->child.handle == DEVICE_HANDLE_LFP2) + if (devdata && devdata->child.handle == DEVICE_HANDLE_LFP2) { + if (!is_panel_type_valid_or_pnp(lfp_options->panel_type2)) { + drm_dbg_kms(display->drm, "Invalid VBT panel type 2 0x%x\n", + lfp_options->panel_type2); + return -1; + } + return lfp_options->panel_type2; + } drm_WARN_ON(display->drm, devdata && devdata->child.handle != DEVICE_HANDLE_LFP1); @@ -763,13 +784,12 @@ static int get_panel_type(struct intel_display *display, panel_types[i].name, panel_types[i].panel_type); } - if (panel_types[PANEL_TYPE_OPREGION].panel_type >= 0) + if (is_panel_type_valid(panel_types[PANEL_TYPE_OPREGION].panel_type)) i = PANEL_TYPE_OPREGION; - else if (panel_types[PANEL_TYPE_VBT].panel_type == 0xff && - panel_types[PANEL_TYPE_PNPID].panel_type >= 0) + else if (is_panel_type_pnp(panel_types[PANEL_TYPE_VBT].panel_type) && + is_panel_type_valid(panel_types[PANEL_TYPE_PNPID].panel_type)) i = PANEL_TYPE_PNPID; - else if (panel_types[PANEL_TYPE_VBT].panel_type != 0xff && - panel_types[PANEL_TYPE_VBT].panel_type >= 0) + else if (is_panel_type_valid(panel_types[PANEL_TYPE_VBT].panel_type)) i = PANEL_TYPE_VBT; else i = PANEL_TYPE_FALLBACK; diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c index c09aa759f4d4f..3de3e9167d0d6 100644 --- a/drivers/gpu/drm/i915/display/intel_ddi.c +++ b/drivers/gpu/drm/i915/display/intel_ddi.c @@ -3502,6 +3502,8 @@ static void intel_ddi_enable_hdmi(struct intel_atomic_state *state, } intel_ddi_buf_enable(encoder, buf_ctl); + + intel_hdmi_poll_for_scrambling_enable(crtc_state, connector); } static void intel_ddi_enable(struct intel_atomic_state *state, diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index c7886b3647706..84fca860c93ee 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -3859,7 +3859,14 @@ static int intel_dp_hdmi_sink_max_frl(struct intel_dp *intel_dp) rate_per_lane = info->hdmi.max_frl_rate_per_lane; max_frl_rate = max_lanes * rate_per_lane; - if (info->hdmi.dsc_cap.v_1p2) { + /* + * The sink's DSC max FRL rate only applies to compressed video + * transport, which requires a DSC 1.2 encoder in the PCON. Without + * one the HDMI link always carries uncompressed video, for which + * the regular max FRL rate is the limit. + */ + if (drm_dp_pcon_enc_is_dsc_1_2(intel_dp->pcon_dsc_dpcd) && + info->hdmi.dsc_cap.v_1p2) { max_dsc_lanes = info->hdmi.dsc_cap.max_lanes; dsc_rate_per_lane = info->hdmi.dsc_cap.max_frl_rate_per_lane; if (max_dsc_lanes && dsc_rate_per_lane) @@ -5203,8 +5210,9 @@ intel_dp_check_mst_status(struct intel_dp *intel_dp) struct intel_encoder *encoder = &dig_port->base; bool link_ok = true; bool reprobe_needed = false; + int tries = 33; - for (;;) { + while (--tries) { u8 esi[4] = {}; u8 ack[4] = {}; @@ -5247,6 +5255,11 @@ intel_dp_check_mst_status(struct intel_dp *intel_dp) if (!link_ok || intel_dp->link.force_retrain) intel_encoder_link_check_queue_work(encoder, 0); + if (!tries) { + drm_dbg_kms(display->drm, "DPRX ESI not clearing, device may be stuck\n"); + reprobe_needed = true; + } + return !reprobe_needed; } diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index 7daf72b69bae4..a42b0943ed95f 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -242,8 +242,8 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, struct intel_display *display = to_intel_display(helper->dev); struct intel_fbdev *ifbdev = to_intel_fbdev(helper); struct intel_framebuffer *fb = ifbdev->fb; + struct fb_info *info = helper->info; struct ref_tracker *wakeref; - struct fb_info *info; struct i915_vma *vma; unsigned long flags = 0; bool prealloc = false; @@ -296,13 +296,6 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, goto out_unlock; } - info = drm_fb_helper_alloc_info(helper); - if (IS_ERR(info)) { - drm_err(display->drm, "Failed to allocate fb_info (%pe)\n", info); - ret = PTR_ERR(info); - goto out_unpin; - } - helper->funcs = &intel_fb_helper_funcs; helper->fb = &fb->base; diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c index 531ee122bf822..68503a3f945e0 100644 --- a/drivers/gpu/drm/i915/display/intel_hdcp.c +++ b/drivers/gpu/drm/i915/display/intel_hdcp.c @@ -140,6 +140,9 @@ intel_hdcp_required_content_stream(struct intel_atomic_state *state, if (conn_dig_port != dig_port) continue; + if (drm_WARN_ON(display->drm, data->k >= INTEL_NUM_PIPES(display))) + return -EINVAL; + data->streams[data->k].stream_id = intel_conn_to_vcpi(state, connector); data->k++; @@ -150,7 +153,7 @@ intel_hdcp_required_content_stream(struct intel_atomic_state *state, } drm_connector_list_iter_end(&conn_iter); - if (drm_WARN_ON(display->drm, data->k > INTEL_NUM_PIPES(display) || data->k == 0)) + if (drm_WARN_ON(display->drm, !data->k)) return -EINVAL; /* @@ -1794,9 +1797,10 @@ int hdcp2_authenticate_repeater_topology(struct intel_connector *connector) return -EINVAL; } - if (seq_num_v < hdcp->seq_num_v) { - /* Roll over of the seq_num_v from repeater. Reauthenticate. */ - drm_dbg_kms(display->drm, "Seq_num_v roll over.\n"); + if (hdcp->hdcp2_encrypted && seq_num_v <= hdcp->seq_num_v) { + /* Reauthenticate on Seq_num_v repeat or rollover */ + drm_dbg_kms(display->drm, "Seq_num_v %s\n", + seq_num_v == hdcp->seq_num_v ? "repeat" : "rollover"); return -EINVAL; } diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c index 4ab7e2e3bfd42..e77c6422b001f 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.c +++ b/drivers/gpu/drm/i915/display/intel_hdmi.c @@ -2680,6 +2680,32 @@ intel_hdmi_add_properties(struct intel_hdmi *intel_hdmi, struct drm_connector *_ drm_connector_attach_max_bpc_property(&connector->base, 8, 12); } +/* + * HDMI 2.0 spec, section 6.1.3.1 (Scrambling Control): after + * enabling Scrambling_Enable and starting scrambled video + * transmission, poll Scrambling_Status for up to 200 ms. + */ +void +intel_hdmi_poll_for_scrambling_enable(const struct intel_crtc_state *crtc_state, + struct drm_connector *_connector) +{ + struct intel_connector *connector = to_intel_connector(_connector); + struct intel_display *display = to_intel_display(crtc_state); + bool scrambling_enabled = false; + int ret; + + if (!crtc_state->hdmi_scrambling) + return; + + /* Poll for a max of 200 msec as per HDMI spec */ + ret = poll_timeout_us(scrambling_enabled = drm_scdc_get_scrambling_status(&connector->base), + scrambling_enabled, 1000, 200 * 1000, false); + if (ret) + drm_dbg_kms(display->drm, + "[CONNECTOR:%d:%s] Timed out waiting for scrambling enable\n", + connector->base.base.id, connector->base.name); +} + /* * intel_hdmi_handle_sink_scrambling: handle sink scrambling/clock ratio setup * @encoder: intel_encoder diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.h b/drivers/gpu/drm/i915/display/intel_hdmi.h index dec2ad7dd8a22..2de9056367916 100644 --- a/drivers/gpu/drm/i915/display/intel_hdmi.h +++ b/drivers/gpu/drm/i915/display/intel_hdmi.h @@ -69,5 +69,7 @@ void hsw_read_infoframe(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state, unsigned int type, void *frame, ssize_t len); +void intel_hdmi_poll_for_scrambling_enable(const struct intel_crtc_state *crtc_state, + struct drm_connector *_connector); #endif /* __INTEL_HDMI_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c index 04a2ea086fb2e..3ac6cd74ae15b 100644 --- a/drivers/gpu/drm/i915/display/intel_vrr.c +++ b/drivers/gpu/drm/i915/display/intel_vrr.c @@ -22,6 +22,9 @@ bool intel_vrr_is_capable(struct intel_connector *connector) const struct drm_display_info *info = &connector->base.display_info; struct intel_dp *intel_dp; + if (!HAS_VRR(display)) + return false; + /* * DP Sink is capable of VRR video timings if * Ignore MSA bit is set in DPCD. @@ -46,8 +49,11 @@ bool intel_vrr_is_capable(struct intel_connector *connector) return false; } - return HAS_VRR(display) && - info->monitor_range.max_vfreq - info->monitor_range.min_vfreq > 10; + if (!info->monitor_range.min_vfreq || !info->monitor_range.max_vfreq || + info->monitor_range.min_vfreq > info->monitor_range.max_vfreq) + return false; + + return info->monitor_range.max_vfreq - info->monitor_range.min_vfreq > 10; } bool intel_vrr_is_in_range(struct intel_connector *connector, int vrefresh) diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c index a5862998bf144..eb54fffc69886 100644 --- a/drivers/gpu/drm/i915/display/skl_watermark.c +++ b/drivers/gpu/drm/i915/display/skl_watermark.c @@ -3815,7 +3815,7 @@ void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc, return; skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0); - skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0); + skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb_y[plane->id], 0, 0); crtc_state->wm.skl.plane_min_ddb[plane->id] = 0; crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index ed6599694835f..ae5194625508f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -612,6 +612,7 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, return -EINVAL; } + slot = array_index_nospec(slot, set->num_engines); if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) { drm_dbg(&i915->drm, "Invalid placement[%d], already occupied\n", slot); @@ -769,8 +770,8 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, struct intel_engine_cs *engine; if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { - kfree(set.engines); - return -EFAULT; + err = -EFAULT; + goto err; } memset(&set.engines[n], 0, sizeof(set.engines[n])); @@ -786,8 +787,8 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, drm_dbg(&i915->drm, "Invalid engine[%d]: { class:%d, instance:%d }\n", n, ci.engine_class, ci.engine_instance); - kfree(set.engines); - return -ENOENT; + err = -ENOENT; + goto err; } set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL; @@ -800,15 +801,21 @@ static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, set_proto_ctx_engines_extensions, ARRAY_SIZE(set_proto_ctx_engines_extensions), &set); - if (err) { - kfree(set.engines); - return err; - } + if (err) + goto err_extensions; pc->num_user_engines = set.num_engines; pc->user_engines = set.engines; return 0; + +err_extensions: + for (n = 0; n < set.num_engines; n++) + kfree(set.engines[n].siblings); +err: + kfree(set.engines); + + return err; } static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv, @@ -850,7 +857,7 @@ static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv, pe = &pc->user_engines[idx]; /* Only render engine supports RPCS configuration. */ - if (pe->engine->class != RENDER_CLASS) + if (!pe->engine || pe->engine->class != RENDER_CLASS) return -EINVAL; sseu = &pe->sseu; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c index 3d53c54eb48fe..d17695ae0a31c 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c @@ -5,9 +5,10 @@ #include +#include +#include #include #include -#include #include "i915_drv.h" #include "i915_ttm_buddy_manager.h" diff --git a/drivers/gpu/drm/i915/gt/intel_engine_user.c b/drivers/gpu/drm/i915/gt/intel_engine_user.c index 833987015b8bb..ed42b9d05a07d 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_user.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_user.c @@ -257,7 +257,7 @@ void intel_engines_driver_register(struct drm_i915_private *i915) p = &prev->rb_right; } - if (IS_ENABLED(CONFIG_DRM_I915_SELFTESTS) && + if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) && IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) { struct intel_engine_cs *engine; unsigned int isolation; diff --git a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c index 33351deeea4f0..07eaf71955c44 100644 --- a/drivers/gpu/drm/i915/gt/selftest_gt_pm.c +++ b/drivers/gpu/drm/i915/gt/selftest_gt_pm.c @@ -16,9 +16,9 @@ static int cmp_u64(const void *A, const void *B) { const u64 *a = A, *b = B; - if (a < b) + if (*a < *b) return -1; - else if (a > b) + else if (*a > *b) return 1; else return 0; @@ -28,9 +28,9 @@ static int cmp_u32(const void *A, const void *B) { const u32 *a = A, *b = B; - if (a < b) + if (*a < *b) return -1; - else if (a > b) + else if (*a > *b) return 1; else return 0; diff --git a/drivers/gpu/drm/i915/i915_active.c b/drivers/gpu/drm/i915/i915_active.c index 6b0c1162505ac..bbcfdf23c0a7d 100644 --- a/drivers/gpu/drm/i915/i915_active.c +++ b/drivers/gpu/drm/i915/i915_active.c @@ -318,7 +318,7 @@ active_instance(struct i915_active *ref, u64 idx) */ node = kmem_cache_alloc(slab_cache, GFP_ATOMIC); if (!node) - goto out; + goto err; __i915_active_fence_init(&node->base, NULL, node_retire); node->ref = ref; @@ -332,6 +332,11 @@ active_instance(struct i915_active *ref, u64 idx) spin_unlock_irq(&ref->tree_lock); return &node->base; + +err: + spin_unlock_irq(&ref->tree_lock); + + return NULL; } void __i915_active_init(struct i915_active *ref, diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c index b2e311f4791a0..af8d0d6259a14 100644 --- a/drivers/gpu/drm/i915/i915_pci.c +++ b/drivers/gpu/drm/i915/i915_pci.c @@ -958,6 +958,9 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) (struct intel_device_info *) ent->driver_data; int err; + if (!intel_info) + return -ENODEV; + if (intel_info->require_force_probe && !id_forced(pdev->device)) { dev_info(&pdev->dev, "Your graphics device %04x is not properly supported by i915 in this\n" diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 14d9ec0ed7771..f75584f713c23 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -401,8 +401,10 @@ static int query_perf_config_list(struct drm_i915_private *i915, ids = krealloc(oa_config_ids, n_configs * sizeof(*oa_config_ids), GFP_KERNEL); - if (!ids) + if (!ids) { + kfree(oa_config_ids); return -ENOMEM; + } alloc = fetch_and_zero(&n_configs); diff --git a/drivers/gpu/drm/i915/i915_scatterlist.c b/drivers/gpu/drm/i915/i915_scatterlist.c index 4d830740946da..6a34dae13769e 100644 --- a/drivers/gpu/drm/i915/i915_scatterlist.c +++ b/drivers/gpu/drm/i915/i915_scatterlist.c @@ -7,7 +7,7 @@ #include "i915_scatterlist.h" #include "i915_ttm_buddy_manager.h" -#include +#include #include #include @@ -167,9 +167,9 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); const u64 size = res->size; const u32 max_segment = round_down(UINT_MAX, page_alignment); - struct drm_buddy *mm = bman_res->mm; + struct gpu_buddy *mm = bman_res->mm; struct list_head *blocks = &bman_res->blocks; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct i915_refct_sgt *rsgt; struct scatterlist *sg; struct sg_table *st; @@ -202,8 +202,8 @@ struct i915_refct_sgt *i915_rsgt_from_buddy_resource(struct ttm_resource *res, list_for_each_entry(block, blocks, link) { u64 block_size, offset; - block_size = min_t(u64, size, drm_buddy_block_size(mm, block)); - offset = drm_buddy_block_offset(block); + block_size = min_t(u64, size, gpu_buddy_block_size(mm, block)); + offset = gpu_buddy_block_offset(block); while (block_size) { u64 len; diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c index 942345548bc32..c5ca90088705a 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c @@ -5,18 +5,19 @@ #include +#include +#include +#include #include #include -#include - #include "i915_ttm_buddy_manager.h" #include "i915_gem.h" struct i915_ttm_buddy_manager { struct ttm_resource_manager manager; - struct drm_buddy mm; + struct gpu_buddy mm; struct list_head reserved; struct mutex lock; unsigned long visible_size; @@ -38,7 +39,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); struct i915_ttm_buddy_resource *bman_res; - struct drm_buddy *mm = &bman->mm; + struct gpu_buddy *mm = &bman->mm; unsigned long n_pages, lpfn; u64 min_page_size; u64 size; @@ -57,13 +58,13 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, bman_res->mm = mm; if (place->flags & TTM_PL_FLAG_TOPDOWN) - bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + bman_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; if (place->flags & TTM_PL_FLAG_CONTIGUOUS) - bman_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION; + bman_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; if (place->fpfn || lpfn != man->size) - bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION; + bman_res->flags |= GPU_BUDDY_RANGE_ALLOCATION; GEM_BUG_ON(!bman_res->base.size); size = bman_res->base.size; @@ -89,7 +90,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, goto err_free_res; } - err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, (u64)lpfn << PAGE_SHIFT, (u64)n_pages << PAGE_SHIFT, min_page_size, @@ -101,15 +102,15 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, if (lpfn <= bman->visible_size) { bman_res->used_visible_size = PFN_UP(bman_res->base.size); } else { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; list_for_each_entry(block, &bman_res->blocks, link) { unsigned long start = - drm_buddy_block_offset(block) >> PAGE_SHIFT; + gpu_buddy_block_offset(block) >> PAGE_SHIFT; if (start < bman->visible_size) { unsigned long end = start + - (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); bman_res->used_visible_size += min(end, bman->visible_size) - start; @@ -126,7 +127,7 @@ static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, return 0; err_free_blocks: - drm_buddy_free_list(mm, &bman_res->blocks, 0); + gpu_buddy_free_list(mm, &bman_res->blocks, 0); mutex_unlock(&bman->lock); err_free_res: ttm_resource_fini(man, &bman_res->base); @@ -141,7 +142,7 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); mutex_lock(&bman->lock); - drm_buddy_free_list(&bman->mm, &bman_res->blocks, 0); + gpu_buddy_free_list(&bman->mm, &bman_res->blocks, 0); bman->visible_avail += bman_res->used_visible_size; mutex_unlock(&bman->lock); @@ -156,8 +157,8 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man, { struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct drm_buddy *mm = &bman->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &bman->mm; + struct gpu_buddy_block *block; if (!place->fpfn && !place->lpfn) return true; @@ -176,9 +177,9 @@ static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man, /* Check each drm buddy block individually */ list_for_each_entry(block, &bman_res->blocks, link) { unsigned long fpfn = - drm_buddy_block_offset(block) >> PAGE_SHIFT; + gpu_buddy_block_offset(block) >> PAGE_SHIFT; unsigned long lpfn = fpfn + - (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); if (place->fpfn < lpfn && place->lpfn > fpfn) return true; @@ -194,8 +195,8 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man, { struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct drm_buddy *mm = &bman->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &bman->mm; + struct gpu_buddy_block *block; if (!place->fpfn && !place->lpfn) return true; @@ -209,9 +210,9 @@ static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man, /* Check each drm buddy block individually */ list_for_each_entry(block, &bman_res->blocks, link) { unsigned long fpfn = - drm_buddy_block_offset(block) >> PAGE_SHIFT; + gpu_buddy_block_offset(block) >> PAGE_SHIFT; unsigned long lpfn = fpfn + - (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); if (fpfn < place->fpfn || lpfn > place->lpfn) return false; @@ -224,7 +225,7 @@ static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct drm_buddy_block *block; + struct gpu_buddy_block *block; mutex_lock(&bman->lock); drm_printf(printer, "default_page_size: %lluKiB\n", @@ -293,7 +294,7 @@ int i915_ttm_buddy_man_init(struct ttm_device *bdev, if (!bman) return -ENOMEM; - err = drm_buddy_init(&bman->mm, size, chunk_size); + err = gpu_buddy_init(&bman->mm, size, chunk_size); if (err) goto err_free_bman; @@ -333,7 +334,7 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) { struct ttm_resource_manager *man = ttm_manager_type(bdev, type); struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct drm_buddy *mm = &bman->mm; + struct gpu_buddy *mm = &bman->mm; int ret; ttm_resource_manager_set_used(man, false); @@ -345,8 +346,8 @@ int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) ttm_set_driver_manager(bdev, type, NULL); mutex_lock(&bman->lock); - drm_buddy_free_list(mm, &bman->reserved, 0); - drm_buddy_fini(mm); + gpu_buddy_free_list(mm, &bman->reserved, 0); + gpu_buddy_fini(mm); bman->visible_avail += bman->visible_reserved; WARN_ON_ONCE(bman->visible_avail != bman->visible_size); mutex_unlock(&bman->lock); @@ -371,15 +372,15 @@ int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, u64 start, u64 size) { struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); - struct drm_buddy *mm = &bman->mm; + struct gpu_buddy *mm = &bman->mm; unsigned long fpfn = start >> PAGE_SHIFT; unsigned long flags = 0; int ret; - flags |= DRM_BUDDY_RANGE_ALLOCATION; + flags |= GPU_BUDDY_RANGE_ALLOCATION; mutex_lock(&bman->lock); - ret = drm_buddy_alloc_blocks(mm, start, + ret = gpu_buddy_alloc_blocks(mm, start, start + size, size, mm->chunk_size, &bman->reserved, diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h index d646207128303..1cff018c1689a 100644 --- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h +++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.h @@ -13,7 +13,7 @@ struct ttm_device; struct ttm_resource_manager; -struct drm_buddy; +struct gpu_buddy; /** * struct i915_ttm_buddy_resource @@ -33,7 +33,7 @@ struct i915_ttm_buddy_resource { struct list_head blocks; unsigned long flags; unsigned long used_visible_size; - struct drm_buddy *mm; + struct gpu_buddy *mm; }; /** diff --git a/drivers/gpu/drm/i915/selftests/intel_memory_region.c b/drivers/gpu/drm/i915/selftests/intel_memory_region.c index 7b856b5090f9a..8307390943a2f 100644 --- a/drivers/gpu/drm/i915/selftests/intel_memory_region.c +++ b/drivers/gpu/drm/i915/selftests/intel_memory_region.c @@ -6,7 +6,7 @@ #include #include -#include +#include #include "../i915_selftest.h" @@ -371,7 +371,7 @@ static int igt_mock_splintered_region(void *arg) struct drm_i915_private *i915 = mem->i915; struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; - struct drm_buddy *mm; + struct gpu_buddy *mm; unsigned int expected_order; LIST_HEAD(objects); u64 size; @@ -447,8 +447,8 @@ static int igt_mock_max_segment(void *arg) struct drm_i915_private *i915 = mem->i915; struct i915_ttm_buddy_resource *res; struct drm_i915_gem_object *obj; - struct drm_buddy_block *block; - struct drm_buddy *mm; + struct gpu_buddy_block *block; + struct gpu_buddy *mm; struct list_head *blocks; struct scatterlist *sg; I915_RND_STATE(prng); @@ -487,8 +487,8 @@ static int igt_mock_max_segment(void *arg) mm = res->mm; size = 0; list_for_each_entry(block, blocks, link) { - if (drm_buddy_block_size(mm, block) > size) - size = drm_buddy_block_size(mm, block); + if (gpu_buddy_block_size(mm, block) > size) + size = gpu_buddy_block_size(mm, block); } if (size < max_segment) { pr_err("%s: Failed to create a huge contiguous block [> %u], largest block %lld\n", @@ -527,14 +527,14 @@ static u64 igt_object_mappable_total(struct drm_i915_gem_object *obj) struct intel_memory_region *mr = obj->mm.region; struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(obj->mm.res); - struct drm_buddy *mm = bman_res->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = bman_res->mm; + struct gpu_buddy_block *block; u64 total; total = 0; list_for_each_entry(block, &bman_res->blocks, link) { - u64 start = drm_buddy_block_offset(block); - u64 end = start + drm_buddy_block_size(mm, block); + u64 start = gpu_buddy_block_offset(block); + u64 end = start + gpu_buddy_block_size(mm, block); if (start < resource_size(&mr->io)) total += min_t(u64, end, resource_size(&mr->io)) - start; diff --git a/drivers/gpu/drm/imagination/pvr_context.c b/drivers/gpu/drm/imagination/pvr_context.c index 5edc3c01af72b..5befaed562ac6 100644 --- a/drivers/gpu/drm/imagination/pvr_context.c +++ b/drivers/gpu/drm/imagination/pvr_context.c @@ -161,22 +161,24 @@ ctx_fw_data_init(void *cpu_ptr, void *priv) /** * pvr_context_destroy_queues() - Destroy all queues attached to a context. * @ctx: Context to destroy queues on. + * @cleanup_queue_entity: Whether to cleanup the queue entity e.g. context + * creation failure path. * * Should be called when the last reference to a context object is dropped. * It releases all resources attached to the queues bound to this context. */ -static void pvr_context_destroy_queues(struct pvr_context *ctx) +static void pvr_context_destroy_queues(struct pvr_context *ctx, bool cleanup_queue_entity) { switch (ctx->type) { case DRM_PVR_CTX_TYPE_RENDER: - pvr_queue_destroy(ctx->queues.fragment); - pvr_queue_destroy(ctx->queues.geometry); + pvr_queue_destroy(ctx->queues.fragment, cleanup_queue_entity); + pvr_queue_destroy(ctx->queues.geometry, cleanup_queue_entity); break; case DRM_PVR_CTX_TYPE_COMPUTE: - pvr_queue_destroy(ctx->queues.compute); + pvr_queue_destroy(ctx->queues.compute, cleanup_queue_entity); break; case DRM_PVR_CTX_TYPE_TRANSFER_FRAG: - pvr_queue_destroy(ctx->queues.transfer); + pvr_queue_destroy(ctx->queues.transfer, cleanup_queue_entity); break; } } @@ -240,7 +242,7 @@ static int pvr_context_create_queues(struct pvr_context *ctx, return -EINVAL; err_destroy_queues: - pvr_context_destroy_queues(ctx); + pvr_context_destroy_queues(ctx, true); return err; } @@ -307,8 +309,8 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co goto err_free_ctx; ctx->vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); - if (IS_ERR(ctx->vm_ctx)) { - err = PTR_ERR(ctx->vm_ctx); + if (!ctx->vm_ctx) { + err = -EINVAL; goto err_free_ctx; } @@ -318,10 +320,14 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co goto err_put_vm; } - err = pvr_context_create_queues(ctx, args, ctx->data); + err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); if (err) goto err_free_ctx_data; + err = pvr_context_create_queues(ctx, args, ctx->data); + if (err) + goto err_free_ctx_id; + err = init_fw_objs(ctx, args, ctx->data); if (err) goto err_destroy_queues; @@ -329,23 +335,12 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED, ctx_fw_data_init, ctx, &ctx->fw_obj); if (err) - goto err_free_ctx_data; + goto err_destroy_queues; - err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL); + err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); if (err) goto err_destroy_fw_obj; - err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL); - if (err) { - /* - * It's possible that another thread could have taken a reference on the context at - * this point as it is in the ctx_ids xarray. Therefore instead of directly - * destroying the context, drop a reference instead. - */ - pvr_context_put(ctx); - return err; - } - spin_lock(&pvr_dev->ctx_list_lock); list_add_tail(&ctx->file_link, &pvr_file->contexts); spin_unlock(&pvr_dev->ctx_list_lock); @@ -356,7 +351,16 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co pvr_fw_object_destroy(ctx->fw_obj); err_destroy_queues: - pvr_context_destroy_queues(ctx); + pvr_context_destroy_queues(ctx, true); + +err_free_ctx_id: + /* + * Ctx_id is not exposed to userspace and not visible yet within + * the kernel/FW, plus a matching context handle (exposed to userspace) + * hasn't been allocated yet, so it is safe to remove ctx_id + * from the ctx_ids xarray. + */ + xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); err_free_ctx_data: kfree(ctx->data); @@ -382,7 +386,7 @@ pvr_context_release(struct kref *ref_count) spin_unlock(&pvr_dev->ctx_list_lock); xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id); - pvr_context_destroy_queues(ctx); + pvr_context_destroy_queues(ctx, false); pvr_fw_object_destroy(ctx->fw_obj); kfree(ctx->data); pvr_vm_context_put(ctx->vm_ctx); diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c index 916b40ced7eb0..e403411cf9639 100644 --- a/drivers/gpu/drm/imagination/pvr_drv.c +++ b/drivers/gpu/drm/imagination/pvr_drv.c @@ -1252,14 +1252,13 @@ pvr_set_uobj_array(const struct drm_pvr_obj_array *out, u32 min_stride, u32 obj_ if (copy_to_user(out_ptr, in_ptr, cpy_elem_size)) return -EFAULT; - out_ptr += obj_size; - in_ptr += out->stride; - } + if (out->stride > obj_size && + clear_user(out_ptr + cpy_elem_size, out->stride - obj_size)) { + return -EFAULT; + } - if (out->stride > obj_size && - clear_user(u64_to_user_ptr(out->array + obj_size), - out->stride - obj_size)) { - return -EFAULT; + out_ptr += out->stride; + in_ptr += obj_size; } } diff --git a/drivers/gpu/drm/imagination/pvr_queue.c b/drivers/gpu/drm/imagination/pvr_queue.c index fc415dd0d7a73..7497bca2e7e83 100644 --- a/drivers/gpu/drm/imagination/pvr_queue.c +++ b/drivers/gpu/drm/imagination/pvr_queue.c @@ -179,7 +179,7 @@ static const struct dma_fence_ops pvr_queue_job_fence_ops = { /** * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is - * backed by a UFO. + * already backed by a UFO. * @f: The dma_fence to turn into a pvr_queue_fence. * * Return: @@ -356,6 +356,15 @@ static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count) pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len); } +static bool +is_paired_job_fence(struct dma_fence *fence, struct pvr_job *job) +{ + /* This assumes "fence" is one of "job"'s drm_sched_job::dependencies */ + return job->type == DRM_PVR_JOB_TYPE_FRAGMENT && + job->paired_job && + &job->paired_job->base.s_fence->scheduled == fence; +} + /** * job_count_remaining_native_deps() - Count the number of non-signaled native dependencies. * @job: Job to operate on. @@ -371,6 +380,17 @@ static unsigned long job_count_remaining_native_deps(struct pvr_job *job) xa_for_each(&job->base.dependencies, index, fence) { struct pvr_queue_fence *jfence; + if (is_paired_job_fence(fence, job)) { + /* + * A fence between paired jobs won't resolve to a pvr_queue_fence (i.e. + * be backed by a UFO) until the jobs have been submitted, together. + * The submitting code will insert a partial render fence command for this. + */ + WARN_ON(dma_fence_is_signaled(fence)); + remaining_count++; + continue; + } + jfence = to_pvr_queue_job_fence(fence); if (!jfence) continue; @@ -468,10 +488,11 @@ pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job) } static struct dma_fence * -pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job) +pvr_queue_get_paired_frag_job_dep(struct pvr_job *job) { struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ? job->paired_job : NULL; + struct pvr_queue *frag_queue = frag_job ? frag_job->ctx->queues.fragment : NULL; struct dma_fence *f; unsigned long index; @@ -490,7 +511,10 @@ pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job) return dma_fence_get(f); } - return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity); + /* Initialize the paired fragment job's done_fence, so we can signal it. */ + pvr_queue_job_fence_init(frag_job->done_fence, frag_queue); + + return pvr_queue_get_job_cccb_fence(frag_queue, frag_job); } /** @@ -509,11 +533,6 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job, struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity); struct dma_fence *internal_dep = NULL; - /* - * Initialize the done_fence, so we can signal it. This must be done - * here because otherwise by the time of run_job() the job will end up - * in the pending list without a valid fence. - */ if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) { /* * This will be called on a paired fragment job after being @@ -523,18 +542,15 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job, */ if (job->paired_job->has_pm_ref) return NULL; - - /* - * In this case we need to use the job's own ctx to initialise - * the done_fence. The other steps are done in the ctx of the - * paired geometry job. - */ - pvr_queue_job_fence_init(job->done_fence, - job->ctx->queues.fragment); - } else { - pvr_queue_job_fence_init(job->done_fence, queue); } + /* + * Initialize the done_fence, so we can signal it. This must be done + * here because otherwise by the time of run_job() the job will end up + * in the pending list without a valid fence. + */ + pvr_queue_job_fence_init(job->done_fence, queue); + /* CCCB fence is used to make sure we have enough space in the CCCB to * submit our commands. */ @@ -555,7 +571,7 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job, /* The paired job fence should come last, when everything else is ready. */ if (!internal_dep) - internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job); + internal_dep = pvr_queue_get_paired_frag_job_dep(job); return internal_dep; } @@ -630,9 +646,8 @@ static void pvr_queue_submit_job_to_cccb(struct pvr_job *job) if (!jfence) continue; - /* Skip the partial render fence, we will place it at the end. */ - if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job && - &job->paired_job->base.s_fence->scheduled == fence) + /* This fence will be placed last, as partial render fence. */ + if (is_paired_job_fence(fence, job)) continue; if (dma_fence_is_signaled(&jfence->base)) @@ -803,7 +818,9 @@ static void pvr_queue_start(struct pvr_queue *queue) * the scheduler, and re-assign parent fences in the middle. * * Return: - * * DRM_GPU_SCHED_STAT_RESET. + * *%DRM_GPU_SCHED_STAT_NO_HANG if the job fence has already been + * signaled, or + * *%DRM_GPU_SCHED_STAT_RESET otherwise. */ static enum drm_gpu_sched_stat pvr_queue_timedout_job(struct drm_sched_job *s_job) @@ -814,6 +831,9 @@ pvr_queue_timedout_job(struct drm_sched_job *s_job) struct pvr_job *job; u32 job_count = 0; + if (dma_fence_is_signaled(s_job->s_fence->parent)) + return DRM_GPU_SCHED_STAT_NO_HANG; + dev_err(sched->dev, "Job timeout\n"); /* Before we stop the scheduler, make sure the queue is out of any list, so @@ -1386,11 +1406,12 @@ void pvr_queue_kill(struct pvr_queue *queue) /** * pvr_queue_destroy() - Destroy a queue. * @queue: The queue to destroy. + * @cleanup_queue_entity: Whether to cleanup the queue entity. * * Cleanup the queue and free the resources attached to it. Should be * called from the context release function. */ -void pvr_queue_destroy(struct pvr_queue *queue) +void pvr_queue_destroy(struct pvr_queue *queue, bool cleanup_queue_entity) { if (!queue) return; @@ -1400,7 +1421,8 @@ void pvr_queue_destroy(struct pvr_queue *queue) mutex_unlock(&queue->ctx->pvr_dev->queues.lock); drm_sched_fini(&queue->scheduler); - drm_sched_entity_fini(&queue->entity); + if (cleanup_queue_entity) + drm_sched_entity_fini(&queue->entity); if (WARN_ON(queue->last_queued_job_scheduled_fence)) dma_fence_put(queue->last_queued_job_scheduled_fence); diff --git a/drivers/gpu/drm/imagination/pvr_queue.h b/drivers/gpu/drm/imagination/pvr_queue.h index fc1986d73fc88..a205e29437f4e 100644 --- a/drivers/gpu/drm/imagination/pvr_queue.h +++ b/drivers/gpu/drm/imagination/pvr_queue.h @@ -158,7 +158,7 @@ struct pvr_queue *pvr_queue_create(struct pvr_context *ctx, void pvr_queue_kill(struct pvr_queue *queue); -void pvr_queue_destroy(struct pvr_queue *queue); +void pvr_queue_destroy(struct pvr_queue *queue, bool cleanup_queue_entity); void pvr_queue_process(struct pvr_queue *queue); diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index 3d97990170bf6..b2f8e89dafb70 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -746,6 +746,7 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj, pvr_gem_object_get(pvr_obj); + mutex_lock(&vm_ctx->lock); err = drm_gpuvm_exec_lock(&vm_exec); if (err) goto err_cleanup; @@ -755,6 +756,7 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj, drm_gpuvm_exec_unlock(&vm_exec); err_cleanup: + mutex_unlock(&vm_ctx->lock); pvr_vm_bind_op_fini(&bind_op); return err; diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c index 0bf7105c8748b..7c873e62c16da 100644 --- a/drivers/gpu/drm/lima/lima_device.c +++ b/drivers/gpu/drm/lima/lima_device.c @@ -368,12 +368,6 @@ int lima_device_init(struct lima_device *ldev) if (err) goto err_out0; - ldev->empty_vm = lima_vm_create(ldev); - if (!ldev->empty_vm) { - err = -ENOMEM; - goto err_out1; - } - ldev->va_start = 0; if (ldev->id == lima_gpu_mali450) { ldev->va_end = LIMA_VA_RESERVE_START; @@ -387,6 +381,12 @@ int lima_device_init(struct lima_device *ldev) } else ldev->va_end = LIMA_VA_RESERVE_END; + ldev->empty_vm = lima_vm_create(ldev); + if (!ldev->empty_vm) { + err = -ENOMEM; + goto err_out1; + } + ldev->iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(ldev->iomem)) { dev_err(ldev->dev, "fail to ioremap iomem\n"); diff --git a/drivers/gpu/drm/logicvc/Kconfig b/drivers/gpu/drm/logicvc/Kconfig index 579a358ed5cf3..11aae1626199b 100644 --- a/drivers/gpu/drm/logicvc/Kconfig +++ b/drivers/gpu/drm/logicvc/Kconfig @@ -4,7 +4,6 @@ config DRM_LOGICVC depends on OF || COMPILE_TEST select DRM_CLIENT_SELECTION select DRM_KMS_HELPER - select DRM_KMS_DMA_HELPER select DRM_GEM_DMA_HELPER select REGMAP select REGMAP_MMIO diff --git a/drivers/gpu/drm/mediatek/mtk_crtc.c b/drivers/gpu/drm/mediatek/mtk_crtc.c index c4c6d0249df56..aa8e6f6ddcd32 100644 --- a/drivers/gpu/drm/mediatek/mtk_crtc.c +++ b/drivers/gpu/drm/mediatek/mtk_crtc.c @@ -153,10 +153,10 @@ static void mtk_crtc_reset(struct drm_crtc *crtc) { struct mtk_crtc_state *state; - if (crtc->state) + if (crtc->state) { __drm_atomic_helper_crtc_destroy_state(crtc->state); - - kfree(to_mtk_crtc_state(crtc->state)); + kfree(to_mtk_crtc_state(crtc->state)); + } crtc->state = NULL; state = kzalloc(sizeof(*state), GFP_KERNEL); diff --git a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c index c0af3e3b51d55..6a259872b0a91 100644 --- a/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c +++ b/drivers/gpu/drm/mediatek/mtk_disp_ovl_adaptor.c @@ -625,6 +625,7 @@ static void mtk_disp_ovl_adaptor_master_unbind(struct device *dev) struct mtk_disp_ovl_adaptor *priv = dev_get_drvdata(dev); priv->children_bound = false; + component_unbind_all(dev, priv->mmsys_dev); } static const struct component_master_ops mtk_disp_ovl_adaptor_master_ops = { @@ -651,12 +652,15 @@ static int mtk_disp_ovl_adaptor_probe(struct platform_device *pdev) priv->mmsys_dev = pdev->dev.platform_data; - component_master_add_with_match(dev, &mtk_disp_ovl_adaptor_master_ops, match); + ret = component_master_add_with_match(dev, &mtk_disp_ovl_adaptor_master_ops, match); + if (ret) + return dev_err_probe(dev, ret, "Failed to add component master\n"); pm_runtime_enable(dev); ret = component_add(dev, &mtk_disp_ovl_adaptor_comp_ops); if (ret != 0) { + component_master_del(dev, &mtk_disp_ovl_adaptor_master_ops); pm_runtime_disable(dev); return dev_err_probe(dev, ret, "Failed to add component\n"); } @@ -666,6 +670,7 @@ static int mtk_disp_ovl_adaptor_probe(struct platform_device *pdev) static void mtk_disp_ovl_adaptor_remove(struct platform_device *pdev) { + component_del(&pdev->dev, &mtk_disp_ovl_adaptor_comp_ops); component_master_del(&pdev->dev, &mtk_disp_ovl_adaptor_master_ops); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c index acee2227275b7..6845bb8043254 100644 --- a/drivers/gpu/drm/mediatek/mtk_dsi.c +++ b/drivers/gpu/drm/mediatek/mtk_dsi.c @@ -1302,6 +1302,7 @@ static const struct mtk_dsi_driver_data mt8188_dsi_driver_data = { static const struct of_device_id mtk_dsi_of_match[] = { { .compatible = "mediatek,mt2701-dsi", .data = &mt2701_dsi_driver_data }, + { .compatible = "mediatek,mt8167-dsi", .data = &mt2701_dsi_driver_data }, { .compatible = "mediatek,mt8173-dsi", .data = &mt8173_dsi_driver_data }, { .compatible = "mediatek,mt8183-dsi", .data = &mt8183_dsi_driver_data }, { .compatible = "mediatek,mt8186-dsi", .data = &mt8186_dsi_driver_data }, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c index 31974a4d7e14b..b43850966a5f6 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_catalog.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_catalog.c @@ -1167,7 +1167,7 @@ static const struct adreno_reglist a730_hwcg[] = { { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 }, { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 }, { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 }, - { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 }, + { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22220222 }, { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 }, { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 }, { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 }, diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c index 21a3f9b0ab4c2..3f17cfaca8b47 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -564,7 +564,7 @@ static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) int ret; u32 val; - if (test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) + if (!test_and_clear_bit(GMU_STATUS_FW_START, &gmu->status)) return; gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); @@ -584,22 +584,19 @@ static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value) writel(value, ptr + (offset << 2)); } -static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, - const char *name); - static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) { struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct platform_device *pdev = to_platform_device(gmu->dev); - void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc"); + void __iomem *pdcptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc"); u32 seqmem0_drv0_reg = REG_A6XX_RSCC_SEQ_MEM_0_DRV0; void __iomem *seqptr = NULL; uint32_t pdc_address_offset; bool pdc_in_aop = false; if (IS_ERR(pdcptr)) - goto err; + return; if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) @@ -612,9 +609,9 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) pdc_address_offset = 0x30080; if (!pdc_in_aop) { - seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq"); + seqptr = devm_platform_ioremap_resource_byname(pdev, "gmu_pdc_seq"); if (IS_ERR(seqptr)) - goto err; + return; } /* Disable SDE clock gating */ @@ -704,12 +701,6 @@ static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) /* ensure no writes happen before the uCode is fully written */ wmb(); - -err: - if (!IS_ERR_OR_NULL(pdcptr)) - iounmap(pdcptr); - if (!IS_ERR_OR_NULL(seqptr)) - iounmap(seqptr); } /* @@ -1256,6 +1247,9 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu) /* Stop the interrupts and mask the hardware */ a6xx_gmu_irq_disable(gmu); + /* Halt the gmu cm3 core */ + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + /* Tell RPMh to power off the GPU */ a6xx_rpmh_stop(gmu); @@ -1799,27 +1793,6 @@ static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) return 0; } -static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, - const char *name) -{ - void __iomem *ret; - struct resource *res = platform_get_resource_byname(pdev, - IORESOURCE_MEM, name); - - if (!res) { - DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name); - return ERR_PTR(-EINVAL); - } - - ret = ioremap(res->start, resource_size(res)); - if (!ret) { - DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name); - return ERR_PTR(-EINVAL); - } - - return ret; -} - static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, const char *name, irq_handler_t handler) { @@ -1870,7 +1843,6 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) { struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; - struct platform_device *pdev = to_platform_device(gmu->dev); mutex_lock(&gmu->lock); if (!gmu->initialized) { @@ -1899,8 +1871,6 @@ void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) qmp_put(gmu->qmp); iounmap(gmu->mmio); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) - iounmap(gmu->rscc); gmu->mmio = NULL; gmu->rscc = NULL; @@ -1926,10 +1896,38 @@ static int cxpd_notifier_cb(struct notifier_block *nb, return 0; } +static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, + const char *name, resource_size_t *start) +{ + void __iomem *ret; + struct resource *res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, name); + + if (!res) { + DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name); + return ERR_PTR(-EINVAL); + } + + ret = ioremap(res->start, resource_size(res)); + if (!ret) { + DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name); + return ERR_PTR(-EINVAL); + } + + if (start) + *start = res->start; + + return ret; +} + int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) { struct platform_device *pdev = of_find_device_by_node(node); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + resource_size_t start; + struct resource *res; int ret; if (!pdev) @@ -1947,12 +1945,21 @@ int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) gmu->legacy = true; /* Map the GMU registers */ - gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); + gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu", &start); if (IS_ERR(gmu->mmio)) { ret = PTR_ERR(gmu->mmio); goto err_mmio; } + res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory"); + if (!res) { + ret = -EINVAL; + goto err_mmio; + } + + /* Identify gmu base offset from gpu base address */ + gmu->mmio_offset = (u32)(start - res->start); + gmu->cxpd = dev_pm_domain_attach_by_name(gmu->dev, "cx"); if (IS_ERR(gmu->cxpd)) { ret = PTR_ERR(gmu->cxpd); @@ -1993,10 +2000,13 @@ int a6xx_gmu_wrapper_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) { + struct platform_device *pdev = of_find_device_by_node(node); struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; struct a6xx_gmu *gmu = &a6xx_gpu->gmu; - struct platform_device *pdev = of_find_device_by_node(node); struct device_link *link; + resource_size_t start; + struct resource *res; int ret; if (!pdev) @@ -2091,15 +2101,24 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) goto err_memory; /* Map the GMU registers */ - gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); + gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu", &start); if (IS_ERR(gmu->mmio)) { ret = PTR_ERR(gmu->mmio); goto err_memory; } + res = platform_get_resource_byname(gpu->pdev, IORESOURCE_MEM, "kgsl_3d0_reg_memory"); + if (!res) { + ret = -EINVAL; + goto err_mmio; + } + + /* Identify gmu base offset from gpu base address */ + gmu->mmio_offset = (u32)(start - res->start); + if (adreno_is_a650_family(adreno_gpu) || adreno_is_a7xx(adreno_gpu)) { - gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc"); + gmu->rscc = devm_platform_ioremap_resource_byname(pdev, "rscc"); if (IS_ERR(gmu->rscc)) { ret = -ENODEV; goto err_mmio; @@ -2177,8 +2196,6 @@ int a6xx_gmu_init(struct a6xx_gpu *a6xx_gpu, struct device_node *node) err_mmio: iounmap(gmu->mmio); - if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc")) - iounmap(gmu->rscc); free_irq(gmu->gmu_irq, gmu); free_irq(gmu->hfi_irq, gmu); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h index 06cfc294016f5..55b1c78daa8b5 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -68,6 +68,7 @@ struct a6xx_gmu { struct drm_gpuvm *vm; void __iomem *mmio; + u32 mmio_offset; void __iomem *rscc; int hfi_irq; @@ -130,20 +131,23 @@ struct a6xx_gmu { unsigned long status; }; +#define GMU_BYTE_OFFSET(gmu, offset) (((offset) << 2) - (gmu)->mmio_offset) + static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset) { - return readl(gmu->mmio + (offset << 2)); + /* The 'offset' is based on GPU's start address. Adjust it */ + return readl(gmu->mmio + GMU_BYTE_OFFSET(gmu, offset)); } static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value) { - writel(value, gmu->mmio + (offset << 2)); + writel(value, gmu->mmio + GMU_BYTE_OFFSET(gmu, offset)); } static inline void gmu_write_bulk(struct a6xx_gmu *gmu, u32 offset, const u32 *data, u32 size) { - memcpy_toio(gmu->mmio + (offset << 2), data, size); + memcpy_toio(gmu->mmio + GMU_BYTE_OFFSET(gmu, offset), data, size); wmb(); } @@ -160,17 +164,17 @@ static inline u64 gmu_read64(struct a6xx_gmu *gmu, u32 lo, u32 hi) { u64 val; - val = (u64) readl(gmu->mmio + (lo << 2)); - val |= ((u64) readl(gmu->mmio + (hi << 2)) << 32); + val = gmu_read(gmu, lo); + val |= ((u64) gmu_read(gmu, hi) << 32); return val; } #define gmu_poll_timeout(gmu, addr, val, cond, interval, timeout) \ - readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \ - interval, timeout) + readl_poll_timeout((gmu)->mmio + (GMU_BYTE_OFFSET(gmu, addr)), val, \ + cond, interval, timeout) #define gmu_poll_timeout_atomic(gmu, addr, val, cond, interval, timeout) \ - readl_poll_timeout_atomic((gmu)->mmio + ((addr) << 2), val, cond, \ + readl_poll_timeout_atomic((gmu)->mmio + (GMU_BYTE_OFFSET(gmu, addr)), val, cond, \ interval, timeout) static inline u32 gmu_read_rscc(struct a6xx_gmu *gmu, u32 offset) diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c index 918d2e504adec..cdd1a27ab8106 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.c @@ -1244,9 +1244,12 @@ static void a6xx_get_gmu_registers(struct msm_gpu *gpu, _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gmu_reglist[1], &a6xx_state->gmu_registers[1], true); - if (adreno_is_a621(adreno_gpu) || adreno_is_a623(adreno_gpu)) + if (adreno_is_a621(adreno_gpu)) _a6xx_get_gmu_registers(gpu, a6xx_state, &a621_gpucc_reg, &a6xx_state->gmu_registers[2], false); + else if (adreno_is_a623(adreno_gpu) || adreno_is_a663(adreno_gpu)) + _a6xx_get_gmu_registers(gpu, a6xx_state, &a623_gpucc_reg, + &a6xx_state->gmu_registers[2], false); else _a6xx_get_gmu_registers(gpu, a6xx_state, &a6xx_gpucc_reg, &a6xx_state->gmu_registers[2], false); diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h index 1c18499b60bb9..e6fcae8d4bd34 100644 --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu_state.h @@ -343,48 +343,59 @@ static const struct a6xx_registers a6xx_gbif_reglist = static const u32 a6xx_gmu_gx_registers[] = { /* GMU GX */ - 0x0000, 0x0000, 0x0010, 0x0013, 0x0016, 0x0016, 0x0018, 0x001b, - 0x001e, 0x001e, 0x0020, 0x0023, 0x0026, 0x0026, 0x0028, 0x002b, - 0x002e, 0x002e, 0x0030, 0x0033, 0x0036, 0x0036, 0x0038, 0x003b, - 0x003e, 0x003e, 0x0040, 0x0043, 0x0046, 0x0046, 0x0080, 0x0084, - 0x0100, 0x012b, 0x0140, 0x0140, + 0x1a800, 0x1a800, 0x1a810, 0x1a813, 0x1a816, 0x1a816, 0x1a818, 0x1a81b, + 0x1a81e, 0x1a81e, 0x1a820, 0x1a823, 0x1a826, 0x1a826, 0x1a828, 0x1a82b, + 0x1a82e, 0x1a82e, 0x1a830, 0x1a833, 0x1a836, 0x1a836, 0x1a838, 0x1a83b, + 0x1a83e, 0x1a83e, 0x1a840, 0x1a843, 0x1a846, 0x1a846, 0x1a880, 0x1a884, + 0x1a900, 0x1a92b, 0x1a940, 0x1a940, }; static const u32 a6xx_gmu_cx_registers[] = { /* GMU CX */ - 0x4c00, 0x4c07, 0x4c10, 0x4c12, 0x4d00, 0x4d00, 0x4d07, 0x4d0a, - 0x5000, 0x5004, 0x5007, 0x5008, 0x500b, 0x500c, 0x500f, 0x501c, - 0x5024, 0x502a, 0x502d, 0x5030, 0x5040, 0x5053, 0x5087, 0x5089, - 0x50a0, 0x50a2, 0x50a4, 0x50af, 0x50c0, 0x50c3, 0x50d0, 0x50d0, - 0x50e4, 0x50e4, 0x50e8, 0x50ec, 0x5100, 0x5103, 0x5140, 0x5140, - 0x5142, 0x5144, 0x514c, 0x514d, 0x514f, 0x5151, 0x5154, 0x5154, - 0x5157, 0x5158, 0x515d, 0x515d, 0x5162, 0x5162, 0x5164, 0x5165, - 0x5180, 0x5186, 0x5190, 0x519e, 0x51c0, 0x51c0, 0x51c5, 0x51cc, - 0x51e0, 0x51e2, 0x51f0, 0x51f0, 0x5200, 0x5201, + 0x1f400, 0x1f407, 0x1f410, 0x1f412, 0x1f500, 0x1f500, 0x1f507, 0x1f50a, + 0x1f800, 0x1f804, 0x1f807, 0x1f808, 0x1f80b, 0x1f80c, 0x1f80f, 0x1f81c, + 0x1f824, 0x1f82a, 0x1f82d, 0x1f830, 0x1f840, 0x1f853, 0x1f887, 0x1f889, + 0x1f8a0, 0x1f8a2, 0x1f8a4, 0x1f8af, 0x1f8c0, 0x1f8c3, 0x1f8d0, 0x1f8d0, + 0x1f8e4, 0x1f8e4, 0x1f8e8, 0x1f8ec, 0x1f900, 0x1f903, 0x1f940, 0x1f940, + 0x1f942, 0x1f944, 0x1f94c, 0x1f94d, 0x1f94f, 0x1f951, 0x1f954, 0x1f954, + 0x1f957, 0x1f958, 0x1f95d, 0x1f95d, 0x1f962, 0x1f962, 0x1f964, 0x1f965, + 0x1f980, 0x1f986, 0x1f990, 0x1f99e, 0x1f9c0, 0x1f9c0, 0x1f9c5, 0x1f9cc, + 0x1f9e0, 0x1f9e2, 0x1f9f0, 0x1f9f0, 0x1fa00, 0x1fa01, /* GMU AO */ - 0x9300, 0x9316, 0x9400, 0x9400, + 0x23b00, 0x23b16, 0x23c00, 0x23c00, }; static const u32 a6xx_gmu_gpucc_registers[] = { /* GPU CC */ - 0x9800, 0x9812, 0x9840, 0x9852, 0x9c00, 0x9c04, 0x9c07, 0x9c0b, - 0x9c15, 0x9c1c, 0x9c1e, 0x9c2d, 0x9c3c, 0x9c3d, 0x9c3f, 0x9c40, - 0x9c42, 0x9c49, 0x9c58, 0x9c5a, 0x9d40, 0x9d5e, 0xa000, 0xa002, - 0xa400, 0xa402, 0xac00, 0xac02, 0xb000, 0xb002, 0xb400, 0xb402, - 0xb800, 0xb802, + 0x24000, 0x24012, 0x24040, 0x24052, 0x24400, 0x24404, 0x24407, 0x2440b, + 0x24415, 0x2441c, 0x2441e, 0x2442d, 0x2443c, 0x2443d, 0x2443f, 0x24440, + 0x24442, 0x24449, 0x24458, 0x2445a, 0x24540, 0x2455e, 0x24800, 0x24802, + 0x24c00, 0x24c02, 0x25400, 0x25402, 0x25800, 0x25802, 0x25c00, 0x25c02, + 0x26000, 0x26002, /* GPU CC ACD */ - 0xbc00, 0xbc16, 0xbc20, 0xbc27, + 0x26400, 0x26416, 0x26420, 0x26427, }; static const u32 a621_gmu_gpucc_registers[] = { /* GPU CC */ - 0x9800, 0x980e, 0x9c00, 0x9c0e, 0xb000, 0xb004, 0xb400, 0xb404, - 0xb800, 0xb804, 0xbc00, 0xbc05, 0xbc14, 0xbc1d, 0xbc2a, 0xbc30, - 0xbc32, 0xbc32, 0xbc41, 0xbc55, 0xbc66, 0xbc68, 0xbc78, 0xbc7a, - 0xbc89, 0xbc8a, 0xbc9c, 0xbc9e, 0xbca0, 0xbca3, 0xbcb3, 0xbcb5, - 0xbcc5, 0xbcc7, 0xbcd6, 0xbcd8, 0xbce8, 0xbce9, 0xbcf9, 0xbcfc, - 0xbd0b, 0xbd0c, 0xbd1c, 0xbd1e, 0xbd40, 0xbd70, 0xbe00, 0xbe16, - 0xbe20, 0xbe2d, + 0x24000, 0x2400e, 0x24400, 0x2440e, 0x24800, 0x24805, 0x24c00, 0x24cff, + 0x25800, 0x25804, 0x25c00, 0x25c04, 0x26000, 0x26004, 0x26400, 0x26405, + 0x26414, 0x2641d, 0x2642a, 0x26430, 0x26432, 0x26432, 0x26441, 0x26455, + 0x26466, 0x26468, 0x26478, 0x2647a, 0x26489, 0x2648a, 0x2649c, 0x2649e, + 0x264a0, 0x264a3, 0x264b3, 0x264b5, 0x264c5, 0x264c7, 0x264d6, 0x264d8, + 0x264e8, 0x264e9, 0x264f9, 0x264fc, 0x2650b, 0x2650c, 0x2651c, 0x2651e, + 0x26540, 0x26570, 0x26600, 0x26616, 0x26620, 0x2662d, +}; + +static const u32 a623_gmu_gpucc_registers[] = { + /* GPU CC */ + 0x24000, 0x2400e, 0x24400, 0x2440e, 0x25800, 0x25804, 0x25c00, 0x25c04, + 0x26000, 0x26004, 0x26400, 0x26405, 0x26414, 0x2641d, 0x2642a, 0x26430, + 0x26432, 0x26432, 0x26441, 0x26455, 0x26466, 0x26468, 0x26478, 0x2647a, + 0x26489, 0x2648a, 0x2649c, 0x2649e, 0x264a0, 0x264a3, 0x264b3, 0x264b5, + 0x264c5, 0x264c7, 0x264d6, 0x264d8, 0x264e8, 0x264e9, 0x264f9, 0x264fc, + 0x2650b, 0x2650c, 0x2651c, 0x2651e, 0x26540, 0x26570, 0x26600, 0x26616, + 0x26620, 0x2662d, }; static const u32 a6xx_gmu_cx_rscc_registers[] = { @@ -402,6 +413,7 @@ static const struct a6xx_registers a6xx_gmu_reglist[] = { static const struct a6xx_registers a6xx_gpucc_reg = REGS(a6xx_gmu_gpucc_registers, 0, 0); static const struct a6xx_registers a621_gpucc_reg = REGS(a621_gmu_gpucc_registers, 0, 0); +static const struct a6xx_registers a623_gpucc_reg = REGS(a623_gmu_gpucc_registers, 0, 0); static u32 a6xx_get_cp_roq_size(struct msm_gpu *gpu); static u32 a7xx_get_cp_roq_size(struct msm_gpu *gpu); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c index 13cc658065c56..6524531bd8bdc 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_core_perf.c @@ -394,6 +394,10 @@ int dpu_core_perf_crtc_update(struct drm_crtc *crtc, trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate); + /* If we're going offline, PM callbacks will disable the clocks instead */ + if (!clk_rate) + return 0; + clk_rate = min(clk_rate, kms->perf.max_core_clk_rate); ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate); if (ret) { diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 777eab5ad844e..e21aa5cdbaf29 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -709,8 +709,7 @@ void dpu_encoder_update_topology(struct drm_encoder *drm_enc, if (fb && MSM_FORMAT_IS_YUV(msm_framebuffer_format(fb))) topology->num_cdm++; } else if (disp_info->intf_type == INTF_DP) { - if (msm_dp_is_yuv_420_enabled(priv->kms->dp[disp_info->h_tile_instance[0]], - adj_mode)) + if (drm_mode_is_420_only(&connector->display_info, adj_mode)) topology->num_cdm++; } } diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c index 809ca191e9de7..c5cd19d1a486a 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c @@ -405,7 +405,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp4_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to init kms\n"); - goto fail; + return ret; } kms = priv->kms; @@ -416,7 +416,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = regulator_enable(mdp4_kms->vdd); if (ret) { DRM_DEV_ERROR(dev->dev, "failed to enable regulator vdd: %d\n", ret); - goto fail; + return ret; } } @@ -428,7 +428,7 @@ static int mdp4_kms_init(struct drm_device *dev) DRM_DEV_ERROR(dev->dev, "unexpected MDP version: v%d.%d\n", major, minor); ret = -ENXIO; - goto fail; + return ret; } mdp4_kms->rev = minor; @@ -437,7 +437,7 @@ static int mdp4_kms_init(struct drm_device *dev) if (!mdp4_kms->lut_clk) { DRM_DEV_ERROR(dev->dev, "failed to get lut_clk\n"); ret = -ENODEV; - goto fail; + return ret; } clk_set_rate(mdp4_kms->lut_clk, max_clk); } @@ -459,7 +459,7 @@ static int mdp4_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp4_kms->dev, NULL); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -467,7 +467,7 @@ static int mdp4_kms_init(struct drm_device *dev) ret = modeset_init(mdp4_kms); if (ret) { DRM_DEV_ERROR(dev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } mdp4_kms->blank_cursor_bo = msm_gem_new(dev, SZ_16K, MSM_BO_WC | MSM_BO_SCANOUT); @@ -475,14 +475,14 @@ static int mdp4_kms_init(struct drm_device *dev) ret = PTR_ERR(mdp4_kms->blank_cursor_bo); DRM_DEV_ERROR(dev->dev, "could not allocate blank-cursor bo: %d\n", ret); mdp4_kms->blank_cursor_bo = NULL; - goto fail; + return ret; } ret = msm_gem_get_and_pin_iova(mdp4_kms->blank_cursor_bo, kms->vm, &mdp4_kms->blank_cursor_iova); if (ret) { DRM_DEV_ERROR(dev->dev, "could not pin blank-cursor bo: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -491,12 +491,6 @@ static int mdp4_kms_init(struct drm_device *dev) dev->mode_config.max_height = 2048; return 0; - -fail: - if (kms) - mdp4_destroy(kms); - - return ret; } static const struct dev_pm_ops mdp4_pm_ops = { diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c index 61edf68640926..0b6b97be3ce93 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c @@ -514,7 +514,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = mdp_kms_init(&mdp5_kms->base, &kms_funcs); if (ret) { DRM_DEV_ERROR(&pdev->dev, "failed to init kms\n"); - goto fail; + return ret; } config = mdp5_cfg_get_config(mdp5_kms->cfg); @@ -537,7 +537,7 @@ static int mdp5_kms_init(struct drm_device *dev) vm = msm_kms_init_vm(mdp5_kms->dev, pdev->dev.parent); if (IS_ERR(vm)) { ret = PTR_ERR(vm); - goto fail; + return ret; } kms->vm = vm; @@ -547,7 +547,7 @@ static int mdp5_kms_init(struct drm_device *dev) ret = modeset_init(mdp5_kms); if (ret) { DRM_DEV_ERROR(&pdev->dev, "modeset_init failed: %d\n", ret); - goto fail; + return ret; } dev->mode_config.min_width = 0; @@ -559,11 +559,6 @@ static int mdp5_kms_init(struct drm_device *dev) dev->vblank_disable_immediate = true; return 0; -fail: - if (kms) - mdp5_kms_destroy(kms); - - return ret; } static void mdp5_destroy(struct mdp5_kms *mdp5_kms) diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c index 38ed4de8313e3..2ff7db77a82b6 100644 --- a/drivers/gpu/drm/msm/dp/dp_ctrl.c +++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c @@ -1941,13 +1941,12 @@ static int msm_dp_ctrl_reinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_mainlink_disable(ctrl); ctrl->phy_opts.dp.lanes = ctrl->link->link_params.num_lanes; phy_configure(phy, &ctrl->phy_opts); + /* * Disable and re-enable the mainlink clock since the * link clock might have been adjusted as part of the * link maintenance. */ - dev_pm_opp_set_rate(ctrl->dev, 0); - msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -1973,7 +1972,6 @@ static int msm_dp_ctrl_deinitialize_mainlink(struct msm_dp_ctrl_private *ctrl) msm_dp_ctrl_reset(&ctrl->msm_dp_ctrl); - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); @@ -2620,7 +2618,6 @@ void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl) ctrl->stream_clks_on = false; } - dev_pm_opp_set_rate(ctrl->dev, 0); msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl); phy_power_off(phy); diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c index c6f5422b60ddb..68043dbe72bfa 100644 --- a/drivers/gpu/drm/msm/dp/dp_display.c +++ b/drivers/gpu/drm/msm/dp/dp_display.c @@ -930,6 +930,7 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, u32 mode_rate_khz = 0, supported_rate_khz = 0, mode_bpp = 0; struct msm_dp *dp; int mode_pclk_khz = mode->clock; + bool is_yuv_420; dp = to_dp_bridge(bridge)->msm_dp_display; @@ -941,9 +942,16 @@ enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge, msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display); link_info = &msm_dp_display->panel->link_info; - if ((drm_mode_is_420_only(&dp->connector->display_info, mode) && - msm_dp_display->panel->vsc_sdp_supported) || - msm_dp_wide_bus_available(dp)) + is_yuv_420 = drm_mode_is_420_only(&dp->connector->display_info, mode); + + /* + * YUV 420 is carried over DP by signalling the colorimetry through a + * VSC SDP, so a 420-only mode cannot be driven without VSC SDP support. + */ + if (is_yuv_420 && !msm_dp_display->panel->vsc_sdp_supported) + return MODE_NO_420; + + if (is_yuv_420 || msm_dp_display->wide_bus_supported) mode_pclk_khz /= 2; if (mode_pclk_khz > DP_MAX_PIXEL_CLK_KHZ) @@ -1506,22 +1514,10 @@ void __exit msm_dp_unregister(void) platform_driver_unregister(&msm_dp_display_driver); } -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *msm_dp_display, - const struct drm_display_mode *mode) -{ - struct msm_dp_display_private *dp; - const struct drm_display_info *info; - - dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display); - info = &msm_dp_display->connector->display_info; - - return dp->panel->vsc_sdp_supported && drm_mode_is_420_only(info, mode); -} - bool msm_dp_needs_periph_flush(const struct msm_dp *msm_dp_display, const struct drm_display_mode *mode) { - return msm_dp_is_yuv_420_enabled(msm_dp_display, mode); + return drm_mode_is_420_only(&msm_dp_display->connector->display_info, mode); } bool msm_dp_wide_bus_available(const struct msm_dp *msm_dp_display) diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c index 15b7f6c7146e1..9191f0fc3d81a 100644 --- a/drivers/gpu/drm/msm/dp/dp_panel.c +++ b/drivers/gpu/drm/msm/dp/dp_panel.c @@ -254,8 +254,10 @@ int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel, return -EINVAL; } - if (msm_dp_panel->drm_edid) + if (msm_dp_panel->drm_edid) { + drm_edid_connector_update(connector, msm_dp_panel->drm_edid); return drm_edid_connector_add_modes(connector); + } return 0; } diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c index 50474c994d473..5110fe05ca841 100644 --- a/drivers/gpu/drm/msm/dsi/dsi_host.c +++ b/drivers/gpu/drm/msm/dsi/dsi_host.c @@ -549,8 +549,6 @@ int dsi_link_clk_enable_v2(struct msm_dsi_host *msm_host) void dsi_link_clk_disable_6g(struct msm_dsi_host *msm_host) { - /* Drop the performance state vote */ - dev_pm_opp_set_rate(&msm_host->pdev->dev, 0); clk_disable_unprepare(msm_host->esc_clk); clk_disable_unprepare(msm_host->pixel_clk); clk_disable_unprepare(msm_host->byte_intf_clk); @@ -670,12 +668,24 @@ static void dsi_calc_pclk(struct msm_dsi_host *msm_host, bool is_bonded_dsi) int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi) { + long rounded_byte_clk_rate; + if (!msm_host->mode) { pr_err("%s: mode not set\n", __func__); return -EINVAL; } dsi_calc_pclk(msm_host, is_bonded_dsi); + + rounded_byte_clk_rate = clk_round_rate(msm_host->byte_clk, + msm_host->byte_clk_rate); + if (rounded_byte_clk_rate < 0) { + pr_err("%s: failed to round byte clock rate, %ld\n", + __func__, rounded_byte_clk_rate); + return rounded_byte_clk_rate; + } + + msm_host->byte_clk_rate = rounded_byte_clk_rate; msm_host->esc_clk_rate = clk_get_rate(msm_host->esc_clk); return 0; } diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h index 3cbf082314924..e391505fdaf04 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h @@ -109,6 +109,7 @@ struct msm_dsi_phy { struct msm_dsi_dphy_timing timing; const struct msm_dsi_phy_cfg *cfg; void *tuning_cfg; + void *pll_data; enum msm_dsi_phy_usecase usecase; bool regulator_ldo_mode; diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c index c5e1d2016bcca..32f06edd21a9f 100644 --- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c +++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c @@ -426,8 +426,11 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_7nm *pll) u32 data; spin_lock_irqsave(&pll->pll_enable_lock, flags); - pll->pll_enable_cnt++; - WARN_ON(pll->pll_enable_cnt == INT_MAX); + if (pll->pll_enable_cnt++) { + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); + WARN_ON(pll->pll_enable_cnt == INT_MAX); + return; + } data = readl(pll->phy->base + REG_DSI_7nm_PHY_CMN_CTRL_0); data |= DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; @@ -873,6 +876,7 @@ static int dsi_pll_7nm_init(struct msm_dsi_phy *phy) spin_lock_init(&pll_7nm->pll_enable_lock); pll_7nm->phy = phy; + phy->pll_data = pll_7nm; ret = pll_7nm_register(pll_7nm, phy->provided_clocks->hws); if (ret) { @@ -961,8 +965,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, u32 const delay_us = 5; u32 const timeout_us = 1000; struct msm_dsi_dphy_timing *timing = &phy->timing; + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; bool less_than_1500_mhz; + unsigned long flags; u32 vreg_ctrl_0, vreg_ctrl_1, lane_ctrl0; u32 glbl_pemph_ctrl_0; u32 glbl_str_swi_cal_sel_ctrl, glbl_hstx_str_ctrl_0; @@ -1084,10 +1090,13 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy, glbl_rescode_bot_ctrl = 0x3c; } + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 1; /* de-assert digital and pll power down */ data = DSI_7nm_PHY_CMN_CTRL_0_DIGTOP_PWRDN_B | DSI_7nm_PHY_CMN_CTRL_0_PLL_SHUTDOWNB; writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* Assert PLL core reset */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_PLL_CNTRL); @@ -1200,7 +1209,9 @@ static bool dsi_7nm_set_continuous_clock(struct msm_dsi_phy *phy, bool enable) static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) { + struct dsi_pll_7nm *pll = phy->pll_data; void __iomem *base = phy->base; + unsigned long flags; u32 data; DBG(""); @@ -1227,8 +1238,11 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy) writel(data, base + REG_DSI_7nm_PHY_CMN_CTRL_0); writel(0, base + REG_DSI_7nm_PHY_CMN_LANE_CTRL0); + spin_lock_irqsave(&pll->pll_enable_lock, flags); + pll->pll_enable_cnt = 0; /* Turn off all PHY blocks */ writel(0x00, base + REG_DSI_7nm_PHY_CMN_CTRL_0); + spin_unlock_irqrestore(&pll->pll_enable_lock, flags); /* make sure phy is turned off */ wmb(); diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 6d847d593f1ae..d40793845541e 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -357,8 +357,6 @@ void __exit msm_dp_unregister(void); int msm_dp_modeset_init(struct msm_dp *dp_display, struct drm_device *dev, struct drm_encoder *encoder, bool yuv_supported); void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp_display); -bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode); bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode); bool msm_dp_wide_bus_available(const struct msm_dp *dp_display); @@ -383,12 +381,6 @@ static inline void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm { } -static inline bool msm_dp_is_yuv_420_enabled(const struct msm_dp *dp_display, - const struct drm_display_mode *mode) -{ - return false; -} - static inline bool msm_dp_needs_periph_flush(const struct msm_dp *dp_display, const struct drm_display_mode *mode) { diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c index b5969374d53fb..d86c89d03f4b8 100644 --- a/drivers/gpu/drm/msm/msm_fbdev.c +++ b/drivers/gpu/drm/msm/msm_fbdev.c @@ -93,9 +93,9 @@ int msm_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, { struct drm_device *dev = helper->dev; struct msm_drm_private *priv = dev->dev_private; + struct fb_info *fbi = helper->info; struct drm_framebuffer *fb = NULL; struct drm_gem_object *bo; - struct fb_info *fbi = NULL; uint64_t paddr; uint32_t format; int ret, pitch; @@ -128,13 +128,6 @@ int msm_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, goto fail; } - fbi = drm_fb_helper_alloc_info(helper); - if (IS_ERR(fbi)) { - DRM_DEV_ERROR(dev->dev, "failed to allocate fb info\n"); - ret = PTR_ERR(fbi); - goto fail; - } - DBG("fbi=%p, dev=%p", fbi, dev); helper->funcs = &msm_fbdev_helper_funcs; diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c index 9016ef978be5e..701fe4a0b73c9 100644 --- a/drivers/gpu/drm/msm/msm_gem_vma.c +++ b/drivers/gpu/drm/msm/msm_gem_vma.c @@ -452,6 +452,8 @@ msm_gem_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec) return ret; } + drm_gpuvm_bo_evict(vm_bo, false); + return 0; } diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index a4377ee84c723..569a1ba02e608 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -505,6 +505,8 @@ static void recover_worker(struct kthread_work *work) */ if (!vm->managed) msm_gem_vm_unusable(submit->vm); + + put_task_struct(task); } get_comm_cmdline(submit, &comm, &cmd); @@ -548,11 +550,11 @@ static void recover_worker(struct kthread_work *work) msm_update_fence(ring->fctx, fence); } + gpu->funcs->recover(gpu); + /* retire completed submits, plus the one that hung: */ retire_submits(gpu); - gpu->funcs->recover(gpu); - /* * Replay all remaining submits starting with highest priority * ring diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c index b2f612e5dc793..e09ec7f398d58 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.c +++ b/drivers/gpu/drm/msm/msm_ringbuffer.c @@ -109,9 +109,9 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, ring->memptrs_iova = memptrs_iova; ret = drm_sched_init(&ring->sched, &args); - if (ret) { + if (ret) goto fail; - } + ring->sched_initialized = true; INIT_LIST_HEAD(&ring->submits); spin_lock_init(&ring->submit_lock); @@ -133,7 +133,8 @@ void msm_ringbuffer_destroy(struct msm_ringbuffer *ring) if (IS_ERR_OR_NULL(ring)) return; - drm_sched_fini(&ring->sched); + if (ring->sched_initialized) + drm_sched_fini(&ring->sched); msm_fence_context_free(ring->fctx); diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.h b/drivers/gpu/drm/msm/msm_ringbuffer.h index d1e49f701c817..bace58a3dffbe 100644 --- a/drivers/gpu/drm/msm/msm_ringbuffer.h +++ b/drivers/gpu/drm/msm/msm_ringbuffer.h @@ -54,6 +54,7 @@ struct msm_ringbuffer { * The job scheduler for this ring. */ struct drm_gpu_scheduler sched; + bool sched_initialized; /* * List of in-flight submits on this ring. Protected by submit_lock. diff --git a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml index b15a242d974d6..09b8a0b9c0de7 100644 --- a/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml +++ b/drivers/gpu/drm/msm/registers/adreno/a6xx_gmu.xml @@ -40,56 +40,56 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + @@ -99,15 +99,15 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - + - + - + @@ -119,71 +119,71 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -191,27 +191,27 @@ xsi:schemaLocation="https://gitlab.freedesktop.org/freedreno/ rules-fd.xsd"> - - - - - + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c index 1c3b33be6c40f..d9499d8c71288 100644 --- a/drivers/gpu/drm/mxsfb/lcdif_kms.c +++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c @@ -373,14 +373,23 @@ static void lcdif_disable_controller(struct lcdif_drm_private *lcdif) int ret; reg = readl(lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Disable the layer for DMA. */ reg &= ~CTRLDESCL0_5_EN; + /* + * It is necessary to wait for the full frame to finish streaming + * through the DMA engine before we can safely disable it by removing + * the DISP_PARA_DISP_ON bit. Disabling it in-flight can leave the + * hardware confused and unable to resume streaming for the next frame. + */ + reg |= CTRLDESCL0_5_SHADOW_LOAD_EN; writel(reg, lcdif->base + LCDC_V8_CTRLDESCL0_5); + /* Wait for the frame to finish or timeout after 50 ms. */ ret = readl_poll_timeout(lcdif->base + LCDC_V8_CTRLDESCL0_5, - reg, !(reg & CTRLDESCL0_5_EN), - 0, 36000); /* Wait ~2 frame times max */ + reg, !(reg & CTRLDESCL0_5_SHADOW_LOAD_EN), + 200, 50000); if (ret) - drm_err(lcdif->drm, "Failed to disable controller!\n"); + drm_err(lcdif->drm, "Timed out waiting for final vblank!\n"); reg = readl(lcdif->base + LCDC_V8_DISP_PARA); reg &= ~DISP_PARA_DISP_ON; diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h index 7903d7470d194..01145db32c534 100644 --- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h +++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h @@ -87,4 +87,5 @@ int gp102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct int gv100_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); int tu102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); int ga102_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); +int gb202_disp_new(struct nvkm_device *, enum nvkm_subdev_type, int inst, struct nvkm_disp **); #endif diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c index b1e92b1f7a262..adaa6cb5bd2be 100644 --- a/drivers/gpu/drm/nouveau/nouveau_chan.c +++ b/drivers/gpu/drm/nouveau/nouveau_chan.c @@ -90,6 +90,14 @@ nouveau_channel_del(struct nouveau_channel **pchan) { struct nouveau_channel *chan = *pchan; if (chan) { + /* + * Drop the kill-event subscription first. Its handler + * dereferences chan->fence, which the fence context teardown + * below frees, so leaving it armed across the teardown leaves + * a window for a use-after-free. + */ + nvif_event_dtor(&chan->kill); + if (chan->fence) nouveau_fence(chan->cli->drm)->context_del(chan); @@ -100,7 +108,6 @@ nouveau_channel_del(struct nouveau_channel **pchan) nvif_object_dtor(&chan->nvsw); nvif_object_dtor(&chan->gart); nvif_object_dtor(&chan->vram); - nvif_event_dtor(&chan->kill); nvif_object_dtor(&chan->user); nvif_mem_dtor(&chan->mem_userd); nouveau_vma_del(&chan->sema.vma); diff --git a/drivers/gpu/drm/nouveau/nouveau_exec.c b/drivers/gpu/drm/nouveau/nouveau_exec.c index c4949e815eb36..6c31093be1c78 100644 --- a/drivers/gpu/drm/nouveau/nouveau_exec.c +++ b/drivers/gpu/drm/nouveau/nouveau_exec.c @@ -331,10 +331,10 @@ nouveau_exec_ucopy(struct nouveau_exec_job_args *args, return 0; -err_free_pushs: - u_free(args->push.s); err_free_ins: u_free(args->in_sync.s); +err_free_pushs: + u_free(args->push.s); return ret; } diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c index f6cfb6ea4885d..80e2bde9915fd 100644 --- a/drivers/gpu/drm/nouveau/nouveau_gem.c +++ b/drivers/gpu/drm/nouveau/nouveau_gem.c @@ -313,11 +313,20 @@ nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem, rep->offset = nvbo->offset; if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50 && !nouveau_cli_uvmm(cli)) { + int ret; + + ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); + if (ret) + return ret; + vma = nouveau_vma_find(nvbo, vmm); - if (!vma) + if (!vma) { + ttm_bo_unreserve(&nvbo->bo); return -EINVAL; + } rep->offset = vma->addr; + ttm_bo_unreserve(&nvbo->bo); } else rep->offset = 0; diff --git a/drivers/gpu/drm/nouveau/nouveau_sgdma.c b/drivers/gpu/drm/nouveau/nouveau_sgdma.c index bd870028514b6..f3a1902bc7e0d 100644 --- a/drivers/gpu/drm/nouveau/nouveau_sgdma.c +++ b/drivers/gpu/drm/nouveau/nouveau_sgdma.c @@ -72,9 +72,7 @@ nouveau_sgdma_create_ttm(struct ttm_buffer_object *bo, uint32_t page_flags) struct nouveau_sgdma_be *nvbe; enum ttm_caching caching; - if (nvbo->force_coherent) - caching = ttm_uncached; - else if (drm->agp.bridge) + if (nvbo->force_coherent || drm->agp.bridge) caching = ttm_write_combined; else caching = ttm_cached; diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c index 79eefdfd08a26..be3b50e17ff28 100644 --- a/drivers/gpu/drm/nouveau/nouveau_uvmm.c +++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c @@ -1251,6 +1251,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); if (!op->reg || op->reg->dirty) { ret = -ENOENT; + op->reg = NULL; goto unwind_continue; } @@ -1259,6 +1260,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); if (IS_ERR(op->ops)) { ret = PTR_ERR(op->ops); + op->reg = NULL; goto unwind_continue; } @@ -1405,6 +1407,7 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, op->va.range); break; case OP_UNMAP_SPARSE: + op->reg->dirty = false; __nouveau_uvma_region_insert(uvmm, op->reg); nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, op->ops); @@ -1421,7 +1424,8 @@ nouveau_uvmm_bind_job_submit(struct nouveau_job *job, break; } - drm_gpuva_ops_free(&uvmm->base, op->ops); + if (!IS_ERR_OR_NULL(op->ops)) + drm_gpuva_ops_free(&uvmm->base, op->ops); op->ops = NULL; op->reg = NULL; } @@ -1711,10 +1715,10 @@ nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args, return 0; -err_free_ops: - u_free(args->op.s); err_free_ins: u_free(args->in_sync.s); +err_free_ops: + u_free(args->op.s); return ret; } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c index a965914f1c2fb..3738c4664fb6a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c @@ -2529,6 +2529,7 @@ nv170_chipset = { .vfn = { 0x00000001, ga100_vfn_new }, .ce = { 0x000003ff, ga100_ce_new }, .fifo = { 0x00000001, ga100_fifo_new }, + .sec2 = { 0x00000001, tu102_sec2_new }, }; static const struct nvkm_device_chip @@ -2844,7 +2845,7 @@ nv1b2_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2860,7 +2861,7 @@ nv1b3_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2876,7 +2877,7 @@ nv1b5_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2892,7 +2893,7 @@ nv1b6_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; @@ -2908,7 +2909,7 @@ nv1b7_chipset = { .pci = { 0x00000001, gh100_pci_new }, .timer = { 0x00000001, gk20a_timer_new }, .vfn = { 0x00000001, ga100_vfn_new }, - .disp = { 0x00000001, ga102_disp_new }, + .disp = { 0x00000001, gb202_disp_new }, .fifo = { 0x00000001, ga102_fifo_new }, }; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild index e1aecd3fe96c1..98d6ca5ac311e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild @@ -27,6 +27,7 @@ nvkm-y += nvkm/engine/disp/gp102.o nvkm-y += nvkm/engine/disp/gv100.o nvkm-y += nvkm/engine/disp/tu102.o nvkm-y += nvkm/engine/disp/ga102.o +nvkm-y += nvkm/engine/disp/gb202.o nvkm-y += nvkm/engine/disp/udisp.o nvkm-y += nvkm/engine/disp/uconn.o diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c index ab0a85c924304..820834b5ee9b2 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ga102.c @@ -144,12 +144,23 @@ ga102_disp = { }, }; +static const struct nvkm_disp_func +ga102_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + .gsp.intr = tu102_disp_intr, + .gsp.head = &tu102_gsp_head, + .gsp.hdmi_gcp = tu102_sor_hdmi_gcp, + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, +}; + int ga102_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_disp **pdisp) { if (nvkm_gsp_rm(device->gsp)) - return r535_disp_new(&ga102_disp, device, type, inst, pdisp); + return r535_disp_new(&ga102_gsp_disp, device, type, inst, pdisp); return nvkm_disp_new_(&ga102_disp, device, type, inst, pdisp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c new file mode 100644 index 0000000000000..d0360610f9fad --- /dev/null +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gb202.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright 2026 Valve Corp. + */ +#include "priv.h" +#include "head.h" +#include "ior.h" + +#include + +/* GB20x (NVD5.0) reorganised the SF HDMI packet units. The AVI unit is + * unchanged from GV100, but the legacy VSI unit is gone. Vendor infoframes + * are sent through the shared generic infoframe units instead. Register + * layout per NVIDIA's clc971.h/clca71.h, programming sequence per + * nvhdmipkt_C971.c:programAdvancedInfoframeC971(). + */ +static void +gb202_sor_hdmi_infoframe_vsi(struct nvkm_ior *ior, int head, void *data, u32 size) +{ + struct nvkm_device *device = ior->disp->engine.subdev.device; + const u32 hoff = head * 0x400; + /* Generic infoframe unit 1, the slot NVIDIA's driver uses for the VSI. */ + const u32 ctrl = 0x6f0138 + hoff; + u8 buf[36] = {}; + int i; + + /* Disable the unit and wait for it to go idle. */ + nvkm_mask(device, ctrl, 0x00000001, 0x00000000); + if (nvkm_msec(device, 2000, + if (!(nvkm_rd32(device, ctrl) & 0x00400000)) + break; + ) < 0) + return; + + if (!size) + return; + + /* Clear SENT status, and point the data port at unit 1's slot. */ + nvkm_mask(device, ctrl, 0x00800000, 0x00800000); + nvkm_wr32(device, 0x6f03f0 + hoff, 0x00000001); + + /* The data port takes the raw packet, except that a zero is inserted + * in HB3 after the three header bytes. A slot is 9 dwords (HB0-3 plus + * up to 32 payload bytes). An HDMI infoframe carries at most PB0-27, + * so the tail stays zero, and we always write the whole slot. + */ + size = min_t(u32, size, 31); + memcpy(buf, data, min_t(u32, size, 3)); + if (size > 3) + memcpy(&buf[4], (u8 *)data + 3, size - 3); + + for (i = 0; i < 36; i += 4) { + nvkm_wr32(device, 0x6f03f4 + hoff, buf[i + 0] | buf[i + 1] << 8 | + buf[i + 2] << 16 | + (u32)buf[i + 3] << 24); + } + + /* No flip ID or scanline matching. */ + nvkm_wr32(device, 0x6f013c + hoff, 0x00000000); + + /* ENABLE | RUN_MODE=ALWAYS | LOC=VBLANK | OFFSET=1 | SIZE=0. */ + nvkm_wr32(device, ctrl, 0x00000041); + + /* Audio priority low (the init value). */ + nvkm_wr32(device, 0x6f03f8 + hoff, 0x00000002); +} + +/* General Control Packet AVMute bracket. The GCP unit moved to slot 1 on + * NVD5.0. Only SB0 (the AVMute bit) is ours to write so we must not do a + * full write here: SB1 carries the deep-color CD/PP fields, and SB1_CTRL + * (bit 24, new with clc871.h) controls where their generation happens (HW + * or driver) on these chips, with the default being HW. + */ +static void +gb202_sor_hdmi_gcp(struct nvkm_ior *sor, int head, bool enable) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 hdmi = head * 0x400; + + nvkm_mask(device, 0x6f0040 + hdmi, 0x00000001, 0x00000000); + nvkm_mask(device, 0x6f004c + hdmi, 0x000000ff, !enable ? 0x00000001 : + 0x00000010); + nvkm_mask(device, 0x6f0040 + hdmi, 0x00000001, 0x00000001); +} + +/* Same core-channel state mirror as gv100_head_state() (assembly at 0x680000, + * armed at +0x8000, per-head method offsets unchanged), but NVD5.0 spaces + * heads 0x800 apart (see NVCA7D_HEAD_SET_*(a) in clca7d.h). + */ +static void +gb202_head_state(struct nvkm_head *head, struct nvkm_head_state *state) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + const u32 hoff = (state == &head->arm) * 0x8000 + head->id * 0x800; + u32 data; + + data = nvkm_rd32(device, 0x682064 + hoff); + state->vtotal = (data & 0xffff0000) >> 16; + state->htotal = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x682068 + hoff); + state->vsynce = (data & 0xffff0000) >> 16; + state->hsynce = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x68206c + hoff); + state->vblanke = (data & 0xffff0000) >> 16; + state->hblanke = (data & 0x0000ffff); + data = nvkm_rd32(device, 0x682070 + hoff); + state->vblanks = (data & 0xffff0000) >> 16; + state->hblanks = (data & 0x0000ffff); + /* Bit 31 is ADJ1000DIV1001, not a HERTZ bit. We don't have enough bits + * to add the full clock in hz on Blackwell (35 bits), but state->hz + * is unused and obsolete under GSP so this is fine. + */ + state->hz = nvkm_rd32(device, 0x68200c + hoff) & 0x7fffffff; + + data = nvkm_rd32(device, 0x682004 + hoff); + switch ((data & 0x000000f0) >> 4) { + case 5: state->or.depth = 30; break; + case 4: state->or.depth = 24; break; + case 1: state->or.depth = 18; break; + default: + state->or.depth = 18; + WARN_ON(1); + break; + } +} + +/* NVD5.0 (GB20x and later) moved the RM head-timing interrupt enable to + * the low-latency vector's EN1 block. The event latch is unchanged. + */ +static void +gb202_head_vblank_put(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_mask(device, 0x611ef0 + (head->id * 4), 0x00000002, 0x00000000); +} + +static void +gb202_head_vblank_get(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); + nvkm_mask(device, 0x611ef0 + (head->id * 4), 0x00000002, 0x00000002); +} + +static irqreturn_t +gb202_disp_intr(struct nvkm_inth *inth) +{ + struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); + irqreturn_t ret = tu102_disp_intr(inth); + + /* The FE interrupt vectors are message-based on NVD5.0. Re-arm the + * low-latency vector so it fires again for any event that latched + * while we were servicing. + */ + nvkm_wr32(disp->engine.subdev.device, 0x611f34, 0x00000001); + return ret; +} + +static const struct nvkm_head_func +gb202_gsp_head = { + .state = gb202_head_state, + .rgpos = gv100_head_rgpos, + .vblank_get = gb202_head_vblank_get, + .vblank_put = gb202_head_vblank_put, +}; + +/* GB20x is GSP-only. This table supplies the register programming the + * GSP-RM display path needs from the chip. + */ +static const struct nvkm_disp_func +gb202_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + /* Head timing arrives on the dedicated low-latency vector. */ + .gsp.intr = gb202_disp_intr, + .gsp.intr_low_latency = true, + .gsp.head = &gb202_gsp_head, + .gsp.hdmi_gcp = gb202_sor_hdmi_gcp, + /* The legacy AVI unit is unchanged on GB20x. */ + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gb202_sor_hdmi_infoframe_vsi, +}; + +int +gb202_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, + struct nvkm_disp **pdisp) +{ + return r535_disp_new(&gb202_gsp_disp, device, type, inst, pdisp); +} diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c index 614921166fba9..2d5ca155091ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gv100.c @@ -253,7 +253,7 @@ gv100_head_vblank_get(struct nvkm_head *head) nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000004, 0x00000004); } -static void +void gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline) { struct nvkm_device *device = head->disp->engine.subdev.device; @@ -263,7 +263,7 @@ gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline) *hline = nvkm_rd32(device, 0x616334 + hoff) & 0x0000ffff; } -static void +void gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state) { struct nvkm_device *device = head->disp->engine.subdev.device; diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h index 856252bf559a4..5976498da9094 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/head.h @@ -53,6 +53,10 @@ void gf119_head_rgclk(struct nvkm_head *, int); int gv100_head_cnt(struct nvkm_disp *, unsigned long *); int gv100_head_new(struct nvkm_disp *, int id); +void gv100_head_state(struct nvkm_head *head, struct nvkm_head_state *state); +void gv100_head_rgpos(struct nvkm_head *head, u16 *hline, u16 *vline); + +extern const struct nvkm_head_func tu102_gsp_head; #define HEAD_MSG(h,l,f,a...) do { \ struct nvkm_head *_h = (h); \ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h index 3ba04bead2f9c..5d682a774f2dd 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ior.h @@ -194,6 +194,7 @@ void gv100_sor_dp_audio_sym(struct nvkm_ior *, int, u16, u32); void gv100_sor_dp_watermark(struct nvkm_ior *, int, u8); extern const struct nvkm_ior_func_hda gv100_sor_hda; +void tu102_sor_hdmi_gcp(struct nvkm_ior *, int, bool); void tu102_sor_dp_vcpi(struct nvkm_ior *, int, u8, u8, u16, u16); int nv50_pior_cnt(struct nvkm_disp *, unsigned long *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h index a3fd7cb7c4883..fde321dbd7c8f 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/priv.h @@ -5,6 +5,8 @@ #include #include struct nvkm_head; +struct nvkm_head_func; +struct nvkm_ior; struct nvkm_outp; struct dcb_output; @@ -34,6 +36,20 @@ struct nvkm_disp_func { int (*new)(struct nvkm_disp *, int id); } wndw, head, dac, sor, pior; + /* Register programming that the GSP-RM display path (rm/r535) needs from + * the chip, everything else on that path goes through RM. The hooks are + * called unconditionally and the head table is handed to nvkm_head_new_(). + */ + struct { + irqreturn_t (*intr)(struct nvkm_inth *); + /* Head-timing interrupts arrive on a second DISP vector. */ + bool intr_low_latency; + const struct nvkm_head_func *head; + void (*hdmi_gcp)(struct nvkm_ior *, int head, bool enable); + void (*hdmi_infoframe_avi)(struct nvkm_ior *, int head, void *data, u32 size); + void (*hdmi_infoframe_vsi)(struct nvkm_ior *, int head, void *data, u32 size); + } gsp; + u16 ramht_size; struct nvkm_sclass root; @@ -72,6 +88,7 @@ int gv100_disp_wndw_cnt(struct nvkm_disp *, unsigned long *); int gv100_disp_caps_new(const struct nvkm_oclass *, void *, u32, struct nvkm_object **); int tu102_disp_init(struct nvkm_disp *); +irqreturn_t tu102_disp_intr(struct nvkm_inth *); void nv50_disp_dptmds_war_2(struct nvkm_disp *, struct dcb_output *); void nv50_disp_dptmds_war_3(struct nvkm_disp *, struct dcb_output *); diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c index dcb9f8ba374ca..f6c163072ff66 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/tu102.c @@ -30,6 +30,21 @@ #include +/* General Control Packet: bracket an audio enable/disable with AVMute + * through the legacy GCP SF unit. Used by the GSP-RM path, which sends the + * equivalent packet via RM as well but keeps the direct write in sync. + */ +void +tu102_sor_hdmi_gcp(struct nvkm_ior *sor, int head, bool enable) +{ + struct nvkm_device *device = sor->disp->engine.subdev.device; + const u32 hdmi = head * 0x400; + + nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000); + nvkm_wr32(device, 0x6f00cc + hdmi, !enable ? 0x00000001 : 0x00000010); + nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000001); +} + void tu102_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned) { @@ -104,6 +119,64 @@ tu102_sor_new(struct nvkm_disp *disp, int id) return nvkm_ior_new_(&tu102_sor, disp, SOR, id, hda & BIT(id)); } +/* The GSP-RM display path leaves head-timing (vblank) interrupts and their + * enables to us. These program the RM head-timing line (bit 1 of the + * per-head enable, not the bit nvkm's own gv100 path uses). + */ +static void +tu102_head_vblank_put(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000000); +} + +static void +tu102_head_vblank_get(struct nvkm_head *head) +{ + struct nvkm_device *device = head->disp->engine.subdev.device; + + nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); + nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000002); +} + +const struct nvkm_head_func +tu102_gsp_head = { + .state = gv100_head_state, + .rgpos = gv100_head_rgpos, + .vblank_get = tu102_head_vblank_get, + .vblank_put = tu102_head_vblank_put, +}; + +static void +tu102_disp_intr_head_timing(struct nvkm_disp *disp, int head) +{ + struct nvkm_subdev *subdev = &disp->engine.subdev; + struct nvkm_device *device = subdev->device; + u32 stat = nvkm_rd32(device, 0x611c00 + (head * 0x04)); + + if (stat & 0x00000002) { + nvkm_disp_vblank(disp, head); + + nvkm_wr32(device, 0x611800 + (head * 0x04), 0x00000002); + } +} + +irqreturn_t +tu102_disp_intr(struct nvkm_inth *inth) +{ + struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); + struct nvkm_subdev *subdev = &disp->engine.subdev; + struct nvkm_device *device = subdev->device; + unsigned long mask = nvkm_rd32(device, 0x611ec0) & 0x000000ff; + int head; + + for_each_set_bit(head, &mask, 8) + tu102_disp_intr_head_timing(disp, head); + + return IRQ_HANDLED; +} + int tu102_disp_init(struct nvkm_disp *disp) { @@ -230,12 +303,23 @@ tu102_disp = { }, }; +static const struct nvkm_disp_func +tu102_gsp_disp = { + .uevent = &gv100_disp_chan_uevent, + .ramht_size = 0x2000, + .gsp.intr = tu102_disp_intr, + .gsp.head = &tu102_gsp_head, + .gsp.hdmi_gcp = tu102_sor_hdmi_gcp, + .gsp.hdmi_infoframe_avi = gv100_sor_hdmi_infoframe_avi, + .gsp.hdmi_infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, +}; + int tu102_disp_new(struct nvkm_device *device, enum nvkm_subdev_type type, int inst, struct nvkm_disp **pdisp) { if (nvkm_gsp_rm(device->gsp)) - return r535_disp_new(&tu102_disp, device, type, inst, pdisp); + return r535_disp_new(&tu102_gsp_disp, device, type, inst, pdisp); return nvkm_disp_new_(&tu102_disp, device, type, inst, pdisp); } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c index 13d829593180c..8b6274b945b0d 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/acr/base.c @@ -315,6 +315,7 @@ nvkm_acr_oneinit(struct nvkm_subdev *subdev) i, us, fw); } } + nvkm_done(acr->wpr); return -EINVAL; } nvkm_done(acr->wpr); diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowrom.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowrom.c index 39144ceb117b4..9e171b1bad732 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowrom.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/shadowrom.c @@ -24,34 +24,126 @@ #include +#define NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE 0x4947564E /* "NVGI" */ +#define NV_ROM_DIRECTORY_IDENTIFIER 0x44524652 /* "RFRD" */ + +struct priv { + struct nvkm_device *device; + u32 pci_rom_offset; +}; + static u32 nvbios_prom_read(void *data, u32 offset, u32 length, struct nvkm_bios *bios) { - struct nvkm_device *device = data; + struct priv *priv = data; + struct nvkm_device *device = priv->device; u32 i; - if (offset + length <= 0x00100000) { - for (i = offset; i < offset + length; i += 4) - *(u32 *)&bios->data[i] = nvkm_rd32(device, 0x300000 + i); - return length; - } - return 0; + + /* Make sure we don't try to read past the end of data[] */ + if (offset + length > bios->size) + return 0; + + /* Make sure the read falls within the 1MB PROM window */ + if (offset + priv->pci_rom_offset + length > 0x00100000) + return 0; + + for (i = offset; i < offset + length; i += 4) + *(u32 *)&bios->data[i] = nvkm_rd32(device, 0x300000 + priv->pci_rom_offset + i); + return length; } static void nvbios_prom_fini(void *data) { - struct nvkm_device *device = data; + struct priv *priv = data; + struct nvkm_device *device = priv->device; + nvkm_pci_rom_shadow(device->pci, true); + + kfree(data); } static void * nvbios_prom_init(struct nvkm_bios *bios, const char *name) { struct nvkm_device *device = bios->subdev.device; + struct priv *priv; + u32 fixed0; + + /* There is no PROM on NV4x iGPUs */ if (device->card_type == NV_40 && device->chipset >= 0x4c) return ERR_PTR(-ENODEV); + + priv = kzalloc_obj(*priv); + if (!priv) + return ERR_PTR(-ENOMEM); + + /* Disable the PCI ROM shadow so that we can read PROM. */ nvkm_pci_rom_shadow(device->pci, false); - return device; + + /* + * Check for an IFR header. If present, parse it to find the actual PCI ROM header. + * + * The IFR header is documented in Documentation/gpu/nova/core/vbios.rst + */ + fixed0 = nvkm_rd32(device, 0x300000); + if (fixed0 == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE) { + u32 fixed1 = nvkm_rd32(device, 0x300004); + u8 version = (fixed1 >> 8) & 0xff; + u32 fixed2, data_size, offset, signature; + + switch (version) { + case 1: + case 2: + data_size = (fixed1 >> 16) & 0x7fff; + priv->pci_rom_offset = nvkm_rd32(device, 0x300000 + data_size + 4); + break; + case 3: + fixed2 = nvkm_rd32(device, 0x300008); + data_size = fixed2 & 0x000fffff; + + /* ROM directory offset */ + offset = nvkm_rd32(device, 0x300000 + data_size) + 4096; + + signature = nvkm_rd32(device, 0x300000 + offset); + if (signature != NV_ROM_DIRECTORY_IDENTIFIER) { + nvkm_error(&bios->subdev, "could not find IFR ROM directory\n"); + goto fail; + } + + priv->pci_rom_offset = nvkm_rd32(device, 0x300000 + offset + 8); + + break; + default: + nvkm_error(&bios->subdev, "unsupported IFR header version %u\n", + version); + goto fail; + } + + /* Double-check that the offset is valid */ + if (priv->pci_rom_offset >= 0x00100000) { + nvkm_error(&bios->subdev, + "PCI ROM offset of 0x%x is too large\n", priv->pci_rom_offset); + goto fail; + } + + /* If there is an IFR header, there must also be a PCI ROM header. */ + signature = nvkm_rd32(device, 0x300000 + priv->pci_rom_offset) & 0xffff; + if (signature != 0xaa55) { + nvkm_error(&bios->subdev, + "could not find PCI ROM signature at offset 0x%x\n", + priv->pci_rom_offset); + goto fail; + } + } + + priv->device = device; + return priv; + +fail: + nvkm_pci_rom_shadow(device->pci, true); + kfree(priv); + return ERR_PTR(-ENODEV); } const struct nvbios_source diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c index 6e63df816d855..62b23cf53766c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c @@ -400,16 +400,16 @@ r535_sor_dp_audio(struct nvkm_ior *sor, int head, bool enable) r535_sor_dp_audio_mute(sor, false); } -static void -r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +static int +r535_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) { struct nvkm_disp *disp = sor->disp; struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); - if (WARN_ON(IS_ERR(ctrl))) - return; + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); ctrl->subDeviceInstance = 0; ctrl->head = head; @@ -429,12 +429,20 @@ r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u ctrl->MST.sendACT = 0; ctrl->MST.singleHeadMSTPipeline = 0; ctrl->MST.bEnableAudioOverRightPanel = 0; - WARN_ON(nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl)); + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + +static void +r535_sor_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +{ + const struct nvkm_rm_api *rmapi = sor->disp->engine.subdev.device->gsp->rm->api; + + WARN_ON(rmapi->disp->dp.vcpi(sor, head, slot, slot_nr, pbn, aligned_pbn)); } static int -r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, - u32 watermark, u32 hblanksym, u32 vblanksym) +r535_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) { struct nvkm_disp *disp = sor->disp; struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; @@ -461,6 +469,15 @@ r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); } +static int +r535_sor_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) +{ + const struct nvkm_rm_api *rmapi = sor->disp->engine.subdev.device->gsp->rm->api; + + return rmapi->disp->dp.sst(sor, head, ef, watermark, hblanksym, vblanksym); +} + static const struct nvkm_ior_func_dp r535_sor_dp = { .sst = r535_sor_dp_sst, @@ -545,16 +562,21 @@ r535_sor_hdmi_ctrl_audio(struct nvkm_outp *outp, bool enable) static void r535_sor_hdmi_audio(struct nvkm_ior *sor, int head, bool enable) { - struct nvkm_device *device = sor->disp->engine.subdev.device; - const u32 hdmi = head * 0x400; - r535_sor_hdmi_ctrl_audio(sor->asy.outp, enable); r535_sor_hdmi_ctrl_audio_mute(sor->asy.outp, !enable); + sor->disp->func->gsp.hdmi_gcp(sor, head, enable); +} + +static void +r535_sor_hdmi_infoframe_avi(struct nvkm_ior *sor, int head, void *data, u32 size) +{ + sor->disp->func->gsp.hdmi_infoframe_avi(sor, head, data, size); +} - /* General Control (GCP). */ - nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000000); - nvkm_wr32(device, 0x6f00cc + hdmi, !enable ? 0x00000001 : 0x00000010); - nvkm_mask(device, 0x6f00c0 + hdmi, 0x00000001, 0x00000001); +static void +r535_sor_hdmi_infoframe_vsi(struct nvkm_ior *sor, int head, void *data, u32 size) +{ + sor->disp->func->gsp.hdmi_infoframe_vsi(sor, head, data, size); } static void @@ -582,8 +604,8 @@ r535_sor_hdmi = { .ctrl = r535_sor_hdmi_ctrl, .scdc = r535_sor_hdmi_scdc, /*TODO: SF_USER -> KMS. */ - .infoframe_avi = gv100_sor_hdmi_infoframe_avi, - .infoframe_vsi = gv100_sor_hdmi_infoframe_vsi, + .infoframe_avi = r535_sor_hdmi_infoframe_avi, + .infoframe_vsi = r535_sor_hdmi_infoframe_vsi, .audio = r535_sor_hdmi_audio, }; @@ -608,35 +630,6 @@ r535_sor_cnt(struct nvkm_disp *disp, unsigned long *pmask) return 4; } -static void -r535_head_vblank_put(struct nvkm_head *head) -{ - struct nvkm_device *device = head->disp->engine.subdev.device; - - nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000000); -} - -static void -r535_head_vblank_get(struct nvkm_head *head) -{ - struct nvkm_device *device = head->disp->engine.subdev.device; - - nvkm_wr32(device, 0x611800 + (head->id * 4), 0x00000002); - nvkm_mask(device, 0x611d80 + (head->id * 4), 0x00000002, 0x00000002); -} - -static void -r535_head_state(struct nvkm_head *head, struct nvkm_head_state *state) -{ -} - -static const struct nvkm_head_func -r535_head = { - .state = r535_head_state, - .vblank_get = r535_head_vblank_get, - .vblank_put = r535_head_vblank_put, -}; - static struct nvkm_conn * r535_conn_new(struct nvkm_disp *disp, u32 id) { @@ -1408,35 +1401,6 @@ static const struct nvkm_event_func r535_disp_event = { }; -static void -r535_disp_intr_head_timing(struct nvkm_disp *disp, int head) -{ - struct nvkm_subdev *subdev = &disp->engine.subdev; - struct nvkm_device *device = subdev->device; - u32 stat = nvkm_rd32(device, 0x611c00 + (head * 0x04)); - - if (stat & 0x00000002) { - nvkm_disp_vblank(disp, head); - - nvkm_wr32(device, 0x611800 + (head * 0x04), 0x00000002); - } -} - -static irqreturn_t -r535_disp_intr(struct nvkm_inth *inth) -{ - struct nvkm_disp *disp = container_of(inth, typeof(*disp), engine.subdev.inth); - struct nvkm_subdev *subdev = &disp->engine.subdev; - struct nvkm_device *device = subdev->device; - unsigned long mask = nvkm_rd32(device, 0x611ec0) & 0x000000ff; - int head; - - for_each_set_bit(head, &mask, 8) - r535_disp_intr_head_timing(disp, head); - - return IRQ_HANDLED; -} - static void r535_disp_fini(struct nvkm_disp *disp, bool suspend) { @@ -1663,7 +1627,7 @@ r535_disp_oneinit(struct nvkm_disp *disp) nvkm_gsp_rm_ctrl_done(&disp->rm.objcom, ctrl); for_each_set_bit(i, &disp->head.mask, disp->head.nr) { - ret = nvkm_head_new_(&r535_head, disp, i); + ret = nvkm_head_new_(disp->func->gsp.head, disp, i); if (ret) return ret; } @@ -1707,12 +1671,20 @@ r535_disp_oneinit(struct nvkm_disp *disp) if (ret) return ret; - ret = nvkm_gsp_intr_stall(gsp, disp->engine.subdev.type, disp->engine.subdev.inst); + /* Chips that raise head-timing interrupts on a separate low-latency + * vector report it as a second DISP interrupt table entry, exposed + * as instance 1 by the RM engine-index translation (see + * r570_gsp_xlat_mc_engine_idx()). Their high-latency vector + * (instance 0) is left unhandled as no event nouveau enables is + * routed to it, and without a handler it stays masked. + */ + ret = nvkm_gsp_intr_stall(gsp, disp->engine.subdev.type, + disp->func->gsp.intr_low_latency ? 1 : disp->engine.subdev.inst); if (ret < 0) return ret; ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &disp->engine.subdev, - r535_disp_intr, &disp->engine.subdev.inth); + disp->func->gsp.intr, &disp->engine.subdev.inth); if (ret) return ret; @@ -1744,6 +1716,7 @@ r535_disp_new(const struct nvkm_disp_func *hw, struct nvkm_device *device, rm->uevent = hw->uevent; rm->sor.cnt = r535_sor_cnt; rm->sor.new = r535_sor_new; + rm->gsp = hw->gsp; rm->ramht_size = hw->ramht_size; rm->root.oclass = gpu->disp.class.root; @@ -1785,6 +1758,8 @@ r535_disp = { .dp = { .get_caps = r535_dp_get_caps, .set_indexed_link_rates = r535_dp_set_indexed_link_rates, + .sst = r535_dp_sst, + .vcpi = r535_dp_vcpi, }, .chan = { .set_pushbuf = r535_disp_chan_set_pushbuf, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c index a96e31c2d80b2..8a23837f356e9 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/disp.c @@ -5,6 +5,7 @@ #include #include +#include #include #include "nvhw/drf.h" @@ -74,6 +75,67 @@ r570_disp_chan_set_pushbuf(struct nvkm_disp *disp, s32 oclass, int inst, struct return nvkm_gsp_rm_ctrl_wr(&gsp->internal.device.subdevice, ctrl); } +static int +r570_dp_vcpi(struct nvkm_ior *sor, int head, u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn) +{ + struct nvkm_disp *disp = sor->disp; + NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, + NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->subDeviceInstance = 0; + ctrl->head = head; + ctrl->sorIndex = sor->id; + ctrl->dpLink = sor->asy.link == 2; + ctrl->bEnableOverride = 1; + ctrl->bMST = 1; + ctrl->hBlankSym = 0; + ctrl->vBlankSym = 0; + ctrl->colorFormat = 0; + ctrl->bEnableTwoHeadOneOr = 0; + ctrl->singleHeadMultistreamMode = 0; + ctrl->MST.slotStart = slot; + ctrl->MST.slotEnd = slot + slot_nr - 1; + ctrl->MST.PBN = pbn; + ctrl->MST.Timeslice = aligned_pbn; + ctrl->MST.sendACT = 0; + ctrl->MST.singleHeadMSTPipeline = 0; + ctrl->MST.bEnableAudioOverRightPanel = 0; + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + +static int +r570_dp_sst(struct nvkm_ior *sor, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym) +{ + struct nvkm_disp *disp = sor->disp; + NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS *ctrl; + + ctrl = nvkm_gsp_rm_ctrl_get(&disp->rm.objcom, + NV0073_CTRL_CMD_DP_CONFIG_STREAM, sizeof(*ctrl)); + if (IS_ERR(ctrl)) + return PTR_ERR(ctrl); + + ctrl->subDeviceInstance = 0; + ctrl->head = head; + ctrl->sorIndex = sor->id; + ctrl->dpLink = sor->asy.link == 2; + ctrl->bEnableOverride = 1; + ctrl->bMST = 0; + ctrl->hBlankSym = hblanksym; + ctrl->vBlankSym = vblanksym; + ctrl->colorFormat = 0; + ctrl->bEnableTwoHeadOneOr = 0; + ctrl->SST.bEnhancedFraming = ef; + ctrl->SST.tuSize = 64; + ctrl->SST.waterMark = watermark; + ctrl->SST.bEnableAudioOverRightPanel = 0; + return nvkm_gsp_rm_ctrl_wr(&disp->rm.objcom, ctrl); +} + static int r570_dp_set_indexed_link_rates(struct nvkm_outp *outp) { @@ -255,6 +317,8 @@ r570_disp = { .dp = { .get_caps = r570_dp_get_caps, .set_indexed_link_rates = r570_dp_set_indexed_link_rates, + .sst = r570_dp_sst, + .vcpi = r570_dp_vcpi, }, .chan = { .set_pushbuf = r570_disp_chan_set_pushbuf, diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c index 996941c668ba9..1488771c63fc4 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/gsp.c @@ -44,6 +44,15 @@ r570_gsp_xlat_mc_engine_idx(u32 mc_engine_idx, enum nvkm_subdev_type *ptype, int *ptype = NVKM_ENGINE_DISP; *pinst = 0; return true; + case MC_ENGINE_IDX_DISP_LOW: + /* GB20x+ report a separate low-latency display vector, used + * for head-timing interrupts. Expose it as a second DISP + * interrupt instance. r535_disp_oneinit() attaches the + * handler to it when the chip's gsp.intr_low_latency is set. + */ + *ptype = NVKM_ENGINE_DISP; + *pinst = 1; + return true; case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE19: *ptype = NVKM_ENGINE_CE; *pinst = mc_engine_idx - MC_ENGINE_IDX_CE0; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h index 06e972835d778..742b25a2a12df 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r570/nvrm/disp.h @@ -256,6 +256,8 @@ typedef struct NV0073_CTRL_DP_CTRL_PARAMS { NvU32 eightLaneDpcdBaseAddr; } NV0073_CTRL_DP_CTRL_PARAMS; +#define NV0073_CTRL_CMD_DP_CONFIG_STREAM (0x731362U) /* finn: Evaluated from "(FINN_NV04_DISPLAY_COMMON_DP_INTERFACE_ID << 8) | NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS_MESSAGE_ID" */ + typedef struct NV0073_CTRL_CMD_DP_CONFIG_STREAM_PARAMS { NvU32 subDeviceInstance; NvU32 head; diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h index 4f0ae6cc085c3..c069c2d97169e 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/rm.h @@ -6,6 +6,7 @@ #ifndef __NVKM_RM_H__ #define __NVKM_RM_H__ #include "handles.h" +struct nvkm_ior; struct nvkm_outp; struct r535_gr; @@ -93,6 +94,10 @@ struct nvkm_rm_api { struct { int (*get_caps)(struct nvkm_disp *, int *link_bw, bool *mst, bool *wm); int (*set_indexed_link_rates)(struct nvkm_outp *); + int (*sst)(struct nvkm_ior *, int head, bool ef, + u32 watermark, u32 hblanksym, u32 vblanksym); + int (*vcpi)(struct nvkm_ior *, int head, + u8 slot, u8 slot_nr, u16 pbn, u16 aligned_pbn); } dp; struct { diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c index b129e5a8d7915..7bfd7eb32eba3 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.c +++ b/drivers/gpu/drm/omapdrm/dss/dsi.c @@ -455,15 +455,10 @@ static irqreturn_t omap_dsi_irq_handler(int irq, void *arg) timer_delete(&dsi->te_timer); #endif - /* make a copy and unlock, so that isrs can unregister - * themselves */ - memcpy(&dsi->isr_tables_copy, &dsi->isr_tables, - sizeof(dsi->isr_tables)); + dsi_handle_isrs(&dsi->isr_tables, irqstatus, vcstatus, ciostatus); spin_unlock(&dsi->irq_lock); - dsi_handle_isrs(&dsi->isr_tables_copy, irqstatus, vcstatus, ciostatus); - dsi_handle_irq_errors(dsi, irqstatus, vcstatus, ciostatus); dsi_collect_irq_stats(dsi, irqstatus, vcstatus, ciostatus); diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.h b/drivers/gpu/drm/omapdrm/dss/dsi.h index 601707c0ecc4e..2b25247ea8935 100644 --- a/drivers/gpu/drm/omapdrm/dss/dsi.h +++ b/drivers/gpu/drm/omapdrm/dss/dsi.h @@ -379,8 +379,6 @@ struct dsi_data { spinlock_t irq_lock; struct dsi_isr_tables isr_tables; - /* space for a copy used by the interrupt handler */ - struct dsi_isr_tables isr_tables_copy; int update_vc; #ifdef DSI_PERF_MEASURE diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c index 948af7ec1130b..2b435c05bdb18 100644 --- a/drivers/gpu/drm/omapdrm/omap_fbdev.c +++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c @@ -155,9 +155,9 @@ int omap_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, struct drm_device *dev = helper->dev; struct omap_drm_private *priv = dev->dev_private; struct omap_fbdev *fbdev = priv->fbdev; + struct fb_info *fbi = helper->info; struct drm_framebuffer *fb = NULL; union omap_gem_size gsize; - struct fb_info *fbi = NULL; struct drm_mode_fb_cmd2 mode_cmd = {0}; struct drm_gem_object *bo; dma_addr_t dma_addr; @@ -226,13 +226,6 @@ int omap_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, goto fail; } - fbi = drm_fb_helper_alloc_info(helper); - if (IS_ERR(fbi)) { - dev_err(dev->dev, "failed to allocate fb info\n"); - ret = PTR_ERR(fbi); - goto fail; - } - DBG("fbi=%p, dev=%p", fbi, dev); helper->funcs = &omap_fbdev_helper_funcs; diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig index 2d8173fa4acb3..ce23b836d3562 100644 --- a/drivers/gpu/drm/panel/Kconfig +++ b/drivers/gpu/drm/panel/Kconfig @@ -107,6 +107,7 @@ config DRM_PANEL_BOE_TV101WUM_LL2 config DRM_PANEL_EBBG_FT8719 tristate "EBBG FT8719 panel driver" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE @@ -336,7 +337,8 @@ config DRM_PANEL_JADARD_JD9365DA_H3 config DRM_PANEL_JDI_LPM102A188A tristate "JDI LPM102A188A DSI panel" - depends on OF && GPIOLIB + depends on GPIOLIB + depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE help @@ -438,6 +440,7 @@ config DRM_PANEL_LG_LG4573 config DRM_PANEL_LG_SW43408 tristate "LG SW43408 panel" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE @@ -542,6 +545,7 @@ config DRM_PANEL_NOVATEK_NT36523 config DRM_PANEL_NOVATEK_NT36672A tristate "Novatek NT36672A DSI panel" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE @@ -552,6 +556,7 @@ config DRM_PANEL_NOVATEK_NT36672A config DRM_PANEL_NOVATEK_NT36672E tristate "Novatek NT36672E DSI panel" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE @@ -680,7 +685,8 @@ config DRM_PANEL_RAYDIUM_RM692E5 config DRM_PANEL_RAYDIUM_RM69380 tristate "Raydium RM69380-based DSI panel" - depends on OF && GPIOLIB + depends on GPIOLIB + depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE help @@ -834,6 +840,7 @@ config DRM_PANEL_SAMSUNG_S6E3HA8 depends on OF depends on DRM_MIPI_DSI depends on BACKLIGHT_CLASS_DEVICE + select DRM_DISPLAY_HELPER select DRM_DISPLAY_DSC_HELPER help Say Y or M here if you want to enable support for the @@ -1162,6 +1169,7 @@ config DRM_PANEL_VISIONOX_R66451 config DRM_PANEL_VISIONOX_RM69299 tristate "Visionox RM69299" + depends on GPIOLIB depends on OF depends on DRM_MIPI_DSI help diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c index d1e68c464e308..9ad1769885b3c 100644 --- a/drivers/gpu/drm/panel/panel-edp.c +++ b/drivers/gpu/drm/panel/panel-edp.c @@ -831,6 +831,13 @@ static int generic_edp_panel_probe(struct device *dev, struct panel_edp *panel) return 0; } +static void panel_edp_put_adapter(void *_adap) +{ + struct i2c_adapter *adap = _adap; + + put_device(&adap->dev); +} + static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, struct drm_dp_aux *aux) { @@ -878,6 +885,11 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, if (!panel->ddc) return -EPROBE_DEFER; + + err = devm_add_action_or_reset(dev, panel_edp_put_adapter, + panel->ddc); + if (err) + return err; } else if (aux) { panel->ddc = &aux->ddc; } @@ -889,7 +901,7 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, err = drm_panel_of_backlight(&panel->base); if (err) - goto err_finished_ddc_init; + return err; /* * We use runtime PM for prepare / unprepare since those power the panel @@ -936,9 +948,6 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc, err_finished_pm_runtime: pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); -err_finished_ddc_init: - if (panel->ddc && (!panel->aux || panel->ddc != &panel->aux->ddc)) - put_device(&panel->ddc->dev); return err; } @@ -982,8 +991,6 @@ static void panel_edp_remove(struct device *dev) pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); - if (panel->ddc && (!panel->aux || panel->ddc != &panel->aux->ddc)) - put_device(&panel->ddc->dev); drm_edid_free(panel->drm_edid); panel->drm_edid = NULL; @@ -1885,6 +1892,7 @@ static const struct panel_delay delay_80_500_e50_d50 = { * Sort first by vendor, then by product ID. */ static const struct edp_panel_entry edp_panels[] = { + EDP_PANEL_ENTRY('A', 'U', 'O', 0x0290, &delay_200_500_e50, "B140XTN07.5"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"), @@ -1900,17 +1908,20 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x30ed, &delay_200_500_e50, "G156HAN03.0"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x3c9f, &delay_200_500_e50, "B140HAK03.5"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAN04.0"), EDP_PANEL_ENTRY2('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0", &auo_b116xa3_mode), EDP_PANEL_ENTRY('A', 'U', 'O', 0x435c, &delay_200_500_e50, "Unknown"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x49ba, &delay_200_500_e50, "B116XTN02.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x52b0, &delay_200_500_e50, "B116XAK02.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0x67a8, &delay_200_500_e50, "B140XTK02.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"), EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"), @@ -1918,7 +1929,9 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('A', 'U', 'O', 0xa199, &delay_200_500_e50, "B116XAN06.1"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xa7b3, &delay_200_500_e50, "B140UAN04.4"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xc4b4, &delay_200_500_e50, "B116XAT04.1"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0xc7ad, &delay_200_500_e50, "B140HAN07.7"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xc9a8, &delay_200_500_e50, "B140QAN08.H"), + EDP_PANEL_ENTRY('A', 'U', 'O', 0xcb8f, &delay_200_500_e50, "B133HAN06.6"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xcdba, &delay_200_500_e50, "B140UAX01.2"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xd497, &delay_200_500_e50, "B120XAN01.0"), EDP_PANEL_ENTRY('A', 'U', 'O', 0xf390, &delay_200_500_e50, "B140XTN07.7"), @@ -1954,6 +1967,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x08b2, &delay_200_500_e200, "NT140WHM-N49"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0848, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0849, &delay_200_500_e200, "Unknown"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x090d, &delay_200_500_e50, "NT140WHM-N4T"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x09c3, &delay_200_500_e50, "NT116WHM-N21,836X2"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0951, &delay_200_500_e80, "NV116WHM-N47"), @@ -1967,6 +1981,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x09ae, &delay_200_500_e200, "NT140FHM-N45"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a1b, &delay_200_500_e50, "NV133WUM-N63"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a25, &delay_200_500_e50_po2e200, "NV133FHM-N4F V8.0"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a36, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a3e, &delay_200_500_e80_d50, "NV116WHM-N49"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"), @@ -1979,8 +1994,10 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b66, &delay_200_500_e80, "NE140WUM-N6G"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b85, &delay_200_500_e50, "NT140WHM-T05"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c26, &delay_200_500_p2e200, "NV140WUM-T08"), + EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c6f, &delay_200_500_e50, "NV140FHM-N40"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"), EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200, "NV140WUM-N64"), @@ -2029,6 +2046,7 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('C', 'S', 'W', 0x1100, &delay_200_500_e80_d50, "MNB601LS1-1"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1103, &delay_200_500_e80_d50, "MNB601LS1-3"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1104, &delay_200_500_e50_d100, "MNB601LS1-4"), + EDP_PANEL_ENTRY('C', 'S', 'W', 0x110a, &delay_200_500_e50, "PNB601LS1-2"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x143f, &delay_200_500_e50, "MNE007QS3-6"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1448, &delay_200_500_e50, "MNE007QS3-7"), EDP_PANEL_ENTRY('C', 'S', 'W', 0x1457, &delay_80_500_e80_p2e200, "MNE007QS3-8"), @@ -2069,7 +2087,9 @@ static const struct edp_panel_entry edp_panels[] = { EDP_PANEL_ENTRY('L', 'G', 'D', 0x0567, &delay_200_500_e200_d200, "Unknown"), EDP_PANEL_ENTRY('L', 'G', 'D', 0x05af, &delay_200_500_e200_d200, "Unknown"), EDP_PANEL_ENTRY('L', 'G', 'D', 0x05f1, &delay_200_500_e200_d200, "Unknown"), + EDP_PANEL_ENTRY('L', 'G', 'D', 0x06b2, &delay_200_500_e200_d200, "LP129WT232166"), EDP_PANEL_ENTRY('L', 'G', 'D', 0x0778, &delay_200_500_e200_d200, "134WT1"), + EDP_PANEL_ENTRY('L', 'G', 'D', 0x07fe, &delay_200_500_e200_d200, "LP116WHA-SPB1"), EDP_PANEL_ENTRY('S', 'H', 'P', 0x1511, &delay_200_500_e50, "LQ140M1JW48"), EDP_PANEL_ENTRY('S', 'H', 'P', 0x1523, &delay_80_500_e50, "LQ140M1JW46"), diff --git a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c index 5c2530598ddb9..9e5287829b952 100644 --- a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c +++ b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c @@ -1157,6 +1157,8 @@ static int jadard_dsi_probe(struct mipi_dsi_device *dsi) if (ret) return ret; + jadard->panel.prepare_prev_first = true; + drm_panel_add(&jadard->panel); mipi_dsi_set_drvdata(dsi, jadard); diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c index ba1a02000bb9d..04e19b5f22f8b 100644 --- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c +++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c @@ -89,16 +89,22 @@ static int s6d16d0_prepare(struct drm_panel *panel) MIPI_DSI_DCS_TEAR_MODE_VBLANK); if (ret) { dev_err(s6->dev, "failed to enable vblank TE (%d)\n", ret); - return ret; + goto err_power_off; } /* Exit sleep mode and power on */ ret = mipi_dsi_dcs_exit_sleep_mode(dsi); if (ret) { dev_err(s6->dev, "failed to exit sleep mode (%d)\n", ret); - return ret; + goto err_power_off; } return 0; + +err_power_off: + gpiod_set_value_cansleep(s6->reset_gpio, 1); + regulator_disable(s6->supply); + + return ret; } static int s6d16d0_enable(struct drm_panel *panel) diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c index 9ebf6d6e19fc0..1eea6d47529f2 100644 --- a/drivers/gpu/drm/panel/panel-simple.c +++ b/drivers/gpu/drm/panel/panel-simple.c @@ -895,6 +895,31 @@ static const struct panel_desc ampire_am_1280800n3tzqw_t00h = { .connector_type = DRM_MODE_CONNECTOR_LVDS, }; +static const struct drm_display_mode ampire_am_1280800w8tzqw_t00h_mode = { + .clock = 72400, + .hdisplay = 1280, + .hsync_start = 1280 + 40, + .hsync_end = 1280 + 40 + 80, + .htotal = 1280 + 40 + 80 + 40, + .vdisplay = 800, + .vsync_start = 800 + 10, + .vsync_end = 800 + 10 + 18, + .vtotal = 800 + 10 + 18 + 10, +}; + +static const struct panel_desc ampire_am_1280800w8tzqw_t00h = { + .modes = &ire_am_1280800w8tzqw_t00h_mode, + .num_modes = 1, + .bpc = 8, + .size = { + .width = 217, + .height = 136, + }, + .bus_flags = DRM_BUS_FLAG_DE_HIGH, + .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG, + .connector_type = DRM_MODE_CONNECTOR_LVDS, +}; + static const struct drm_display_mode ampire_am_480272h3tmqw_t01h_mode = { .clock = 9000, .hdisplay = 480, @@ -5123,6 +5148,9 @@ static const struct of_device_id platform_of_match[] = { { .compatible = "ampire,am-1280800n3tzqw-t00h", .data = &ire_am_1280800n3tzqw_t00h, + }, { + .compatible = "ampire,am-1280800w8tzqw-t00h", + .data = &ire_am_1280800w8tzqw_t00h, }, { .compatible = "ampire,am-480272h3tmqw-t01h", .data = &ire_am_480272h3tmqw_t01h, diff --git a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c index 227f97f9b136f..13cfe252a838d 100644 --- a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c +++ b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c @@ -162,9 +162,6 @@ static int tdo_tl070wsh30_panel_add(struct tdo_tl070wsh30_panel *tdo_tl070wsh30) return err; } - drm_panel_init(&tdo_tl070wsh30->base, &tdo_tl070wsh30->link->dev, - &tdo_tl070wsh30_panel_funcs, DRM_MODE_CONNECTOR_DSI); - err = drm_panel_of_backlight(&tdo_tl070wsh30->base); if (err) return err; @@ -183,10 +180,13 @@ static int tdo_tl070wsh30_panel_probe(struct mipi_dsi_device *dsi) dsi->format = MIPI_DSI_FMT_RGB888; dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | MIPI_DSI_MODE_LPM; - tdo_tl070wsh30 = devm_kzalloc(&dsi->dev, sizeof(*tdo_tl070wsh30), - GFP_KERNEL); - if (!tdo_tl070wsh30) - return -ENOMEM; + tdo_tl070wsh30 = devm_drm_panel_alloc(&dsi->dev, + __typeof(*tdo_tl070wsh30), base, + &tdo_tl070wsh30_panel_funcs, + DRM_MODE_CONNECTOR_DSI); + + if (IS_ERR(tdo_tl070wsh30)) + return PTR_ERR(tdo_tl070wsh30); mipi_dsi_set_drvdata(dsi, tdo_tl070wsh30); tdo_tl070wsh30->link = dsi; diff --git a/drivers/gpu/drm/panel/panel-waveshare-dsi.c b/drivers/gpu/drm/panel/panel-waveshare-dsi.c index 02e1bd9fa15a2..67cb6caaa6142 100644 --- a/drivers/gpu/drm/panel/panel-waveshare-dsi.c +++ b/drivers/gpu/drm/panel/panel-waveshare-dsi.c @@ -36,6 +36,7 @@ struct ws_panel_data { const struct drm_display_mode *mode; int lanes; unsigned long mode_flags; + int fps_reg_val; /* if non-zero, write to register 0xd0 */ }; /* 2.8inch 480x640 @@ -345,6 +346,47 @@ static const struct ws_panel_data ws_panel_7_0_h_data = { .mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS, }; +/* 10.1inch DSI LCD (E) + * https://www.waveshare.com/10.1inch-dsi-lcd-e.htm + */ +static const struct drm_display_mode ws_panel_1920x1200_60fps_mode = { + .clock = 166666, + .hdisplay = 1920, + .hsync_start = 1920 + 238, + .hsync_end = 1920 + 238 + 4, + .htotal = 1920 + 238 + 4 + 28, + .vdisplay = 1200, + .vsync_start = 1200 + 28, + .vsync_end = 1200 + 28 + 10, + .vtotal = 1200 + 28 + 10 + 30, +}; + +static const struct drm_display_mode ws_panel_1920x1200_30fps_mode = { + .clock = 83333, + .hdisplay = 1920, + .hsync_start = 1920 + 238, + .hsync_end = 1920 + 238 + 4, + .htotal = 1920 + 238 + 4 + 28, + .vdisplay = 1200, + .vsync_start = 1200 + 28, + .vsync_end = 1200 + 28 + 10, + .vtotal = 1200 + 28 + 10 + 30, +}; + +static const struct ws_panel_data ws_panel_10_1_e_4lane_data = { + .mode = &ws_panel_1920x1200_60fps_mode, + .lanes = 4, + .mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS, + .fps_reg_val = 60, +}; + +static const struct ws_panel_data ws_panel_10_1_e_2lane_data = { + .mode = &ws_panel_1920x1200_30fps_mode, + .lanes = 2, + .mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO | MIPI_DSI_CLOCK_NON_CONTINUOUS, + .fps_reg_val = 30, +}; + static struct ws_panel *panel_to_ts(struct drm_panel *panel) { return container_of(panel, struct ws_panel, base); @@ -493,13 +535,15 @@ static int ws_panel_probe(struct i2c_client *i2c) return -EINVAL; ts->mode = _ws_panel_data->mode; + if (!ts->mode) return -EINVAL; - i2c_set_clientdata(i2c, ts); - ts->i2c = i2c; + if (_ws_panel_data->fps_reg_val) + ws_panel_i2c_write(ts, 0xd0, _ws_panel_data->fps_reg_val); + ws_panel_i2c_write(ts, 0xc0, 0x01); ws_panel_i2c_write(ts, 0xc2, 0x01); ws_panel_i2c_write(ts, 0xac, 0x01); @@ -570,10 +614,18 @@ static int ws_panel_probe(struct i2c_client *i2c) return -ENODEV; } +static void ws_panel_reset(struct drm_panel *panel) +{ + struct ws_panel *ts = panel_to_ts(panel); + + ws_panel_i2c_write(ts, 0xd2, 0x5a); +} + static void ws_panel_remove(struct i2c_client *i2c) { struct ws_panel *ts = i2c_get_clientdata(i2c); + ws_panel_reset(&ts->base); ws_panel_disable(&ts->base); drm_panel_remove(&ts->base); @@ -583,6 +635,7 @@ static void ws_panel_shutdown(struct i2c_client *i2c) { struct ws_panel *ts = i2c_get_clientdata(i2c); + ws_panel_reset(&ts->base); ws_panel_disable(&ts->base); } @@ -635,6 +688,12 @@ static const struct of_device_id ws_panel_of_ids[] = { }, { .compatible = "waveshare,7.0inch-h-panel", .data = &ws_panel_7_0_h_data, + }, { + .compatible = "waveshare,10.1inch-e-4lane-panel", + .data = &ws_panel_10_1_e_4lane_data, + }, { + .compatible = "waveshare,10.1inch-e-2lane-panel", + .data = &ws_panel_10_1_e_2lane_data, }, { /* sentinel */ } diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c index 962a10e00848e..cdf82d83de7ab 100644 --- a/drivers/gpu/drm/panthor/panthor_device.c +++ b/drivers/gpu/drm/panthor/panthor_device.c @@ -182,7 +182,10 @@ int panthor_device_init(struct panthor_device *ptdev) return ret; #ifdef CONFIG_DEBUG_FS - drmm_mutex_init(&ptdev->base, &ptdev->gems.lock); + ret = drmm_mutex_init(&ptdev->base, &ptdev->gems.lock); + if (ret) + return ret; + INIT_LIST_HEAD(&ptdev->gems.node); #endif diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c index 4c202fc5ce050..687fa282b093d 100644 --- a/drivers/gpu/drm/panthor/panthor_drv.c +++ b/drivers/gpu/drm/panthor/panthor_drv.c @@ -1630,7 +1630,7 @@ static int panthor_probe(struct platform_device *pdev) ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver, struct panthor_device, base); if (IS_ERR(ptdev)) - return -ENOMEM; + return PTR_ERR(ptdev); platform_set_drvdata(pdev, ptdev); diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c index df767e82148af..3fe101ea58d37 100644 --- a/drivers/gpu/drm/panthor/panthor_fw.c +++ b/drivers/gpu/drm/panthor/panthor_fw.c @@ -495,6 +495,7 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev, struct panthor_fw_binary_section_entry_hdr hdr; struct panthor_fw_section *section; u32 section_size; + u32 data_size; u32 name_len; int ret; @@ -545,6 +546,16 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev, return -EINVAL; } + section_size = hdr.va.end - hdr.va.start; + data_size = hdr.data.end - hdr.data.start; + if (data_size > section_size) { + drm_err(&ptdev->base, "Firmware corrupted, section data exceeds section size\n"); + return -EINVAL; + } + + if (!section_size) + return 0; + name_len = iter->size - iter->offset; section = drmm_kzalloc(&ptdev->base, sizeof(*section), GFP_KERNEL); @@ -553,7 +564,7 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev, list_add_tail(§ion->node, &ptdev->fw->sections); section->flags = hdr.flags; - section->data.size = hdr.data.end - hdr.data.start; + section->data.size = data_size; if (section->data.size > 0) { void *data = drmm_kmalloc(&ptdev->base, section->data.size, GFP_KERNEL); @@ -576,7 +587,6 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev, section->name = name; } - section_size = hdr.va.end - hdr.va.start; if (section_size) { u32 cache_mode = hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_MASK; struct panthor_gem_object *bo; @@ -647,7 +657,8 @@ static int panthor_fw_read_build_info(struct panthor_device *ptdev, return ret; if (hdr.meta_start > fw->size || - hdr.meta_start + hdr.meta_size > fw->size) { + hdr.meta_size > fw->size - hdr.meta_start || + hdr.meta_size <= header_len) { drm_err(&ptdev->base, "Firmware build info corrupt\n"); /* We don't need the build info, so continue */ return 0; @@ -777,6 +788,7 @@ static int panthor_fw_load(struct panthor_device *ptdev) } if (hdr.size > iter.size) { + ret = -EINVAL; drm_err(&ptdev->base, "Firmware image is truncated\n"); goto out; } @@ -804,18 +816,24 @@ static int panthor_fw_load(struct panthor_device *ptdev) * iface_fw_to_cpu_addr() - Turn an MCU address into a CPU address * @ptdev: Device. * @mcu_va: MCU address. + * @size: Size of the object pointed to by @mcu_va. * - * Return: NULL if the address is not part of the shared section, non-NULL otherwise. + * Return: NULL if the object is not part of the shared section, non-NULL otherwise. */ -static void *iface_fw_to_cpu_addr(struct panthor_device *ptdev, u32 mcu_va) +static void *iface_fw_to_cpu_addr(struct panthor_device *ptdev, u32 mcu_va, size_t size) { u64 shared_mem_start = panthor_kernel_bo_gpuva(ptdev->fw->shared_section->mem); - u64 shared_mem_end = shared_mem_start + - panthor_kernel_bo_size(ptdev->fw->shared_section->mem); - if (mcu_va < shared_mem_start || mcu_va >= shared_mem_end) + size_t shared_mem_size = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); + u64 offset; + + if (mcu_va < shared_mem_start) + return NULL; + + offset = mcu_va - shared_mem_start; + if (offset > shared_mem_size || size > shared_mem_size - offset) return NULL; - return ptdev->fw->shared_section->mem->kmap + (mcu_va - shared_mem_start); + return ptdev->fw->shared_section->mem->kmap + offset; } static int panthor_init_cs_iface(struct panthor_device *ptdev, @@ -825,20 +843,23 @@ static int panthor_init_cs_iface(struct panthor_device *ptdev, struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_idx); struct panthor_fw_cs_iface *cs_iface = &ptdev->fw->iface.streams[csg_idx][cs_idx]; u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); - u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + - (csg_idx * glb_iface->control->group_stride) + + u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + + ((u64)csg_idx * glb_iface->control->group_stride) + CSF_STREAM_CONTROL_OFFSET + - (cs_idx * csg_iface->control->stream_stride); + ((u64)cs_idx * csg_iface->control->stream_stride); struct panthor_fw_cs_iface *first_cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0); - if (iface_offset + sizeof(*cs_iface) >= shared_section_sz) + if (iface_offset > shared_section_sz || + sizeof(*cs_iface->control) > shared_section_sz - iface_offset) return -EINVAL; spin_lock_init(&cs_iface->lock); cs_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; - cs_iface->input = iface_fw_to_cpu_addr(ptdev, cs_iface->control->input_va); - cs_iface->output = iface_fw_to_cpu_addr(ptdev, cs_iface->control->output_va); + cs_iface->input = iface_fw_to_cpu_addr(ptdev, cs_iface->control->input_va, + sizeof(*cs_iface->input)); + cs_iface->output = iface_fw_to_cpu_addr(ptdev, cs_iface->control->output_va, + sizeof(*cs_iface->output)); if (!cs_iface->input || !cs_iface->output) { drm_err(&ptdev->base, "Invalid stream control interface input/output VA"); @@ -880,16 +901,20 @@ static int panthor_init_csg_iface(struct panthor_device *ptdev, struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); struct panthor_fw_csg_iface *csg_iface = &ptdev->fw->iface.groups[csg_idx]; u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); - u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + (csg_idx * glb_iface->control->group_stride); + u64 iface_offset = CSF_GROUP_CONTROL_OFFSET + + ((u64)csg_idx * glb_iface->control->group_stride); unsigned int i; - if (iface_offset + sizeof(*csg_iface) >= shared_section_sz) + if (iface_offset > shared_section_sz || + sizeof(*csg_iface->control) > shared_section_sz - iface_offset) return -EINVAL; spin_lock_init(&csg_iface->lock); csg_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; - csg_iface->input = iface_fw_to_cpu_addr(ptdev, csg_iface->control->input_va); - csg_iface->output = iface_fw_to_cpu_addr(ptdev, csg_iface->control->output_va); + csg_iface->input = iface_fw_to_cpu_addr(ptdev, csg_iface->control->input_va, + sizeof(*csg_iface->input)); + csg_iface->output = iface_fw_to_cpu_addr(ptdev, csg_iface->control->output_va, + sizeof(*csg_iface->output)); if (csg_iface->control->stream_num < MIN_CS_PER_CSG || csg_iface->control->stream_num > MAX_CS_PER_CSG) @@ -933,11 +958,15 @@ static u32 panthor_get_instr_features(struct panthor_device *ptdev) static int panthor_fw_init_ifaces(struct panthor_device *ptdev) { struct panthor_fw_global_iface *glb_iface = &ptdev->fw->iface.global; + u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); unsigned int i; if (!ptdev->fw->shared_section->mem->kmap) return -EINVAL; + if (sizeof(*glb_iface->control) > shared_section_sz) + return -EINVAL; + spin_lock_init(&glb_iface->lock); glb_iface->control = ptdev->fw->shared_section->mem->kmap; @@ -946,8 +975,10 @@ static int panthor_fw_init_ifaces(struct panthor_device *ptdev) return -EINVAL; } - glb_iface->input = iface_fw_to_cpu_addr(ptdev, glb_iface->control->input_va); - glb_iface->output = iface_fw_to_cpu_addr(ptdev, glb_iface->control->output_va); + glb_iface->input = iface_fw_to_cpu_addr(ptdev, glb_iface->control->input_va, + sizeof(*glb_iface->input)); + glb_iface->output = iface_fw_to_cpu_addr(ptdev, glb_iface->control->output_va, + sizeof(*glb_iface->output)); if (!glb_iface->input || !glb_iface->output) { drm_err(&ptdev->base, "Invalid global control interface input/output VA"); return -EINVAL; diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c index 07a9c523a17af..59ea0b94932da 100644 --- a/drivers/gpu/drm/radeon/r100.c +++ b/drivers/gpu/drm/radeon/r100.c @@ -906,6 +906,7 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev, { struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]; struct radeon_fence *fence; + uint64_t cur_src_offset, cur_dst_offset; uint32_t cur_pages; uint32_t stride_bytes = RADEON_GPU_PAGE_SIZE; uint32_t pitch; @@ -934,6 +935,10 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev, cur_pages = 8191; } num_gpu_pages -= cur_pages; + cur_src_offset = src_offset + + (uint64_t)num_gpu_pages * RADEON_GPU_PAGE_SIZE; + cur_dst_offset = dst_offset + + (uint64_t)num_gpu_pages * RADEON_GPU_PAGE_SIZE; /* pages are in Y direction - height page width in X direction - width */ @@ -950,13 +955,13 @@ struct radeon_fence *r100_copy_blit(struct radeon_device *rdev, RADEON_DP_SRC_SOURCE_MEMORY | RADEON_GMC_CLR_CMP_CNTL_DIS | RADEON_GMC_WR_MSK_DIS); - radeon_ring_write(ring, (pitch << 22) | (src_offset >> 10)); - radeon_ring_write(ring, (pitch << 22) | (dst_offset >> 10)); + radeon_ring_write(ring, (pitch << 22) | (cur_src_offset >> 10)); + radeon_ring_write(ring, (pitch << 22) | (cur_dst_offset >> 10)); radeon_ring_write(ring, (0x1fff) | (0x1fff << 16)); radeon_ring_write(ring, 0); radeon_ring_write(ring, (0x1fff) | (0x1fff << 16)); - radeon_ring_write(ring, num_gpu_pages); - radeon_ring_write(ring, num_gpu_pages); + radeon_ring_write(ring, 0); + radeon_ring_write(ring, 0); radeon_ring_write(ring, cur_pages | (stride_pixels << 16)); } radeon_ring_write(ring, PACKET0(RADEON_DSTCACHE_CTLSTAT, 0)); diff --git a/drivers/gpu/drm/radeon/radeon_fbdev.c b/drivers/gpu/drm/radeon/radeon_fbdev.c index dc81b0c2dbff3..d5c621d48e20f 100644 --- a/drivers/gpu/drm/radeon/radeon_fbdev.c +++ b/drivers/gpu/drm/radeon/radeon_fbdev.c @@ -206,7 +206,7 @@ int radeon_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, struct radeon_device *rdev = fb_helper->dev->dev_private; const struct drm_format_info *format_info; struct drm_mode_fb_cmd2 mode_cmd = { }; - struct fb_info *info; + struct fb_info *info = fb_helper->info; struct drm_gem_object *gobj; struct radeon_bo *rbo; struct drm_framebuffer *fb; @@ -247,13 +247,6 @@ int radeon_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, fb_helper->funcs = &radeon_fbdev_fb_helper_funcs; fb_helper->fb = fb; - /* okay we have an object now allocate the framebuffer */ - info = drm_fb_helper_alloc_info(fb_helper); - if (IS_ERR(info)) { - ret = PTR_ERR(info); - goto err_drm_framebuffer_unregister_private; - } - info->fbops = &radeon_fbdev_fb_ops; /* radeon resume is fragile and needs a vt switch to help it along */ @@ -279,10 +272,6 @@ int radeon_fbdev_driver_fbdev_probe(struct drm_fb_helper *fb_helper, return 0; -err_drm_framebuffer_unregister_private: - fb_helper->fb = NULL; - drm_framebuffer_unregister_private(fb); - drm_framebuffer_cleanup(fb); err_kfree: kfree(fb); err_radeon_fbdev_destroy_pinned_object: diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c index ba1446acd7032..16adc5d5f67be 100644 --- a/drivers/gpu/drm/radeon/radeon_kms.c +++ b/drivers/gpu/drm/radeon/radeon_kms.c @@ -71,6 +71,7 @@ void radeon_driver_unload_kms(struct drm_device *dev) if (radeon_is_px(dev)) { pm_runtime_get_sync(dev->dev); pm_runtime_forbid(dev->dev); + pm_runtime_dont_use_autosuspend(dev->dev); } radeon_acpi_fini(rdev); diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c index 040d4e4aff000..ab49604d69acf 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -293,6 +294,9 @@ static void rzg2l_du_vsp_cleanup(struct drm_device *dev, void *res) { struct rzg2l_du_vsp *vsp = res; + if (vsp->link) + device_link_del(vsp->link); + put_device(vsp->vsp); } @@ -317,6 +321,18 @@ int rzg2l_du_vsp_init(struct rzg2l_du_vsp *vsp, struct device_node *np, if (ret < 0) return ret; + /* + * Enforce suspend/resume ordering between the DU (consumer) and the + * VSP (supplier). The DU will be suspended before and resume after the + * VSP. + */ + vsp->link = device_link_add(rcdu->dev, vsp->vsp, DL_FLAG_STATELESS); + if (!vsp->link) { + dev_err(rcdu->dev, "Failed to create device link to VSP %s\n", + dev_name(vsp->vsp)); + return -EINVAL; + } + ret = vsp1_du_init(vsp->vsp); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h index 322eb80dcbaff..a22aaf0843ed7 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h @@ -15,6 +15,7 @@ #include struct device; +struct device_link; struct drm_framebuffer; struct rzg2l_du_device; struct rzg2l_du_format_info; @@ -29,6 +30,7 @@ struct rzg2l_du_vsp_plane { struct rzg2l_du_vsp { unsigned int index; struct device *vsp; + struct device_link *link; struct rzg2l_du_device *dev; }; diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c index cbd898a182b56..1c6f29c285b65 100644 --- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c +++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c @@ -228,7 +228,6 @@ static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi, u32 dphytim1; u32 dphytim2; u32 dphytim3; - int ret; /* All DSI global operation timings are set with recommended setting */ for (i = 0; i < ARRAY_SIZE(rzg2l_mipi_dsi_global_timings); ++i) { @@ -268,12 +267,6 @@ static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi, rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM2, dphytim2); rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM3, dphytim3); - ret = reset_control_deassert(dsi->rstc); - if (ret < 0) - return ret; - - udelay(1); - return 0; } @@ -285,8 +278,6 @@ static void rzg2l_mipi_dsi_dphy_exit(struct rzg2l_mipi_dsi *dsi) dphyctrl0 &= ~(DSIDPHYCTRL0_EN_LDO1200 | DSIDPHYCTRL0_EN_BGR); rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYCTRL0, dphyctrl0); - - reset_control_assert(dsi->rstc); } static int rzg2l_dphy_conf_clks(struct rzg2l_mipi_dsi *dsi, unsigned long mode_freq, @@ -393,6 +384,14 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, dsisetr |= FIELD_PREP(DSISETR_MRPSZ, RZG2L_DCS_BUF_SIZE); rzg2l_mipi_dsi_link_write(dsi, DSISETR, dsisetr); + if (dsi->rstc) { + ret = reset_control_deassert(dsi->rstc); + if (ret < 0) + goto err_phy; + + fsleep(1000); + } + return 0; err_phy: @@ -404,6 +403,7 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi, static void rzg2l_mipi_dsi_stop(struct rzg2l_mipi_dsi *dsi) { + reset_control_assert(dsi->rstc); dsi->info->dphy_exit(dsi); pm_runtime_put(dsi->dev); } @@ -604,29 +604,33 @@ static void rzg2l_mipi_dsi_atomic_pre_enable(struct drm_bridge *bridge, const struct drm_display_mode *mode; struct drm_connector *connector; struct drm_crtc *crtc; - int ret; connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder); crtc = drm_atomic_get_new_connector_state(state, connector)->crtc; mode = &drm_atomic_get_new_crtc_state(state, crtc)->adjusted_mode; - ret = rzg2l_mipi_dsi_startup(dsi, mode); - if (ret < 0) - return; - - rzg2l_mipi_dsi_set_display_timing(dsi, mode); + rzg2l_mipi_dsi_startup(dsi, mode); } static void rzg2l_mipi_dsi_atomic_enable(struct drm_bridge *bridge, struct drm_atomic_state *state) { struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge); + const struct drm_display_mode *mode; + struct drm_connector *connector; + struct drm_crtc *crtc; int ret; ret = rzg2l_mipi_dsi_start_hs_clock(dsi); if (ret < 0) goto err_stop; + connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder); + crtc = drm_atomic_get_new_connector_state(state, connector)->crtc; + mode = &drm_atomic_get_new_crtc_state(state, crtc)->adjusted_mode; + + rzg2l_mipi_dsi_set_display_timing(dsi, mode); + ret = rzg2l_mipi_dsi_start_video(dsi); if (ret < 0) goto err_stop_clock; diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig index b7b025814e72b..7a942b5e89657 100644 --- a/drivers/gpu/drm/rockchip/Kconfig +++ b/drivers/gpu/drm/rockchip/Kconfig @@ -64,6 +64,7 @@ config ROCKCHIP_CDN_DP config ROCKCHIP_DW_DP bool "Rockchip specific extensions for Synopsys DW DP" + select DRM_BRIDGE_CONNECTOR help This selects support for Rockchip SoC specific extensions to enable Synopsys DesignWare Cores based DisplayPort transmit @@ -141,6 +142,8 @@ config ROCKCHIP_RGB config ROCKCHIP_RK3066_HDMI bool "Rockchip specific extensions for RK3066 HDMI" depends on DRM_ROCKCHIP + select DRM_DISPLAY_HELPER + select DRM_BRIDGE_CONNECTOR help This selects support for Rockchip SoC specific extensions for the RK3066 HDMI driver. If you want to enable diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c index d30f0983a53a6..a95af0483ab23 100644 --- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c @@ -8,6 +8,7 @@ * Jeff Chen */ +#include #include #include #include @@ -223,7 +224,6 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, struct drm_crtc *crtc; struct drm_crtc_state *old_crtc_state; struct of_endpoint endpoint; - struct device_node *remote_port, *remote_port_parent; char name[32]; u32 port_id; int ret; @@ -247,18 +247,23 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder, if (ret < 0) return; - remote_port_parent = of_graph_get_remote_port_parent(endpoint.local_node); + struct device_node *remote_port_parent __free(device_node) = + of_graph_get_remote_port_parent(endpoint.local_node); if (remote_port_parent) { - if (of_get_child_by_name(remote_port_parent, "ports")) { - remote_port = of_graph_get_remote_port(endpoint.local_node); + struct device_node *ports __free(device_node) = + of_get_child_by_name(remote_port_parent, "ports"); + + if (ports) { + struct device_node *remote_port __free(device_node) = + of_graph_get_remote_port(endpoint.local_node); + of_property_read_u32(remote_port, "reg", &port_id); - of_node_put(remote_port); - sprintf(name, "%s vp%d", remote_port_parent->full_name, port_id); + snprintf(name, sizeof(name), "%s vp%d", + remote_port_parent->full_name, port_id); } else { - sprintf(name, "%s %s", - remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb"); + snprintf(name, sizeof(name), "%s %s", + remote_port_parent->full_name, endpoint.id ? "vopl" : "vopb"); } - of_node_put(remote_port_parent); DRM_DEV_DEBUG(dp->dev, "vop %s output to dp\n", (ret) ? "LIT" : "BIG"); } @@ -328,6 +333,7 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) { struct device *dev = dp->dev; struct device_node *np = dev->of_node; + struct clk *clk; dp->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,grf"); if (IS_ERR(dp->grf)) { @@ -351,6 +357,11 @@ static int rockchip_dp_of_probe(struct rockchip_dp_device *dp) return PTR_ERR(dp->pclk); } + clk = devm_clk_get_optional_enabled(dev, "hclk"); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), + "failed to get hclk property\n"); + dp->rst = devm_reset_control_get(dev, "dp"); if (IS_ERR(dp->rst)) { DRM_DEV_ERROR(dev, "failed to get dp reset control\n"); @@ -469,6 +480,8 @@ static int rockchip_dp_probe(struct platform_device *pdev) return -ENOMEM; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; i = 0; while (dp_data[i].reg) { diff --git a/drivers/gpu/drm/rockchip/cdn-dp-reg.c b/drivers/gpu/drm/rockchip/cdn-dp-reg.c index 924fb1d3ece28..c0866ca105435 100644 --- a/drivers/gpu/drm/rockchip/cdn-dp-reg.c +++ b/drivers/gpu/drm/rockchip/cdn-dp-reg.c @@ -683,6 +683,8 @@ int cdn_dp_config_video(struct cdn_dp_device *dp) val = div_u64(8 * (symbol + 1), bit_per_pix) - val; val += 2; ret = cdn_dp_reg_write(dp, DP_VC_TABLE(15), val); + if (ret) + goto err_config_video; switch (video->color_depth) { case 6: diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c index 6d57e1c746273..410e5f4573306 100644 --- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c +++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c @@ -76,7 +76,7 @@ static const struct drm_encoder_helper_funcs dw_dp_encoder_helper_funcs = { static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *data) { struct platform_device *pdev = to_platform_device(dev); - struct dw_dp_plat_data plat_data; + const struct dw_dp_plat_data *plat_data; struct drm_device *drm_dev = data; struct rockchip_dw_dp *dp; struct drm_encoder *encoder; @@ -90,7 +90,10 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void * dp->dev = dev; platform_set_drvdata(pdev, dp); - plat_data.max_link_rate = 810000; + plat_data = of_device_get_match_data(dev); + if (!plat_data) + return -ENODEV; + encoder = &dp->encoder.encoder; encoder->possible_crtcs = drm_of_find_possible_crtcs(drm_dev, dev->of_node); rockchip_drm_encoder_set_crtc_endpoint_id(&dp->encoder, dev->of_node, 0, 0); @@ -100,32 +103,36 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void * return ret; drm_encoder_helper_add(encoder, &dw_dp_encoder_helper_funcs); - dp->base = dw_dp_bind(dev, encoder, &plat_data); - if (IS_ERR(dp->base)) { - ret = PTR_ERR(dp->base); - return ret; - } + dp->base = dw_dp_bind(dev, encoder, plat_data); + if (IS_ERR(dp->base)) + return PTR_ERR(dp->base); connector = drm_bridge_connector_init(drm_dev, encoder); if (IS_ERR(connector)) { - ret = PTR_ERR(connector); - return dev_err_probe(dev, ret, "Failed to init bridge connector"); + dw_dp_unbind(dp->base); + return dev_err_probe(dev, PTR_ERR(connector), + "Failed to init bridge connector\n"); } - drm_connector_attach_encoder(connector, encoder); + return drm_connector_attach_encoder(connector, encoder); +} - return 0; +static void dw_dp_rockchip_unbind(struct device *dev, struct device *master, + void *data) +{ + struct rockchip_dw_dp *dp = dev_get_drvdata(dev); + + dw_dp_unbind(dp->base); } static const struct component_ops dw_dp_rockchip_component_ops = { .bind = dw_dp_rockchip_bind, + .unbind = dw_dp_rockchip_unbind, }; static int dw_dp_probe(struct platform_device *pdev) { - struct device *dev = &pdev->dev; - - return component_add(dev, &dw_dp_rockchip_component_ops); + return component_add(&pdev->dev, &dw_dp_rockchip_component_ops); } static void dw_dp_remove(struct platform_device *pdev) @@ -133,8 +140,16 @@ static void dw_dp_remove(struct platform_device *pdev) component_del(&pdev->dev, &dw_dp_rockchip_component_ops); } +static const struct dw_dp_plat_data rk3588_dp_plat_data = { + .max_link_rate = 810000, + .pixel_mode = DW_DP_MP_QUAD_PIXEL, +}; + static const struct of_device_id dw_dp_of_match[] = { - { .compatible = "rockchip,rk3588-dp", }, + { + .compatible = "rockchip,rk3588-dp", + .data = &rk3588_dp_plat_data, + }, {} }; MODULE_DEVICE_TABLE(of, dw_dp_of_match); diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c index 7ec7bea5e38e6..3c48984d62448 100644 --- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c +++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c @@ -336,7 +336,8 @@ static bool vop2_output_uv_swap(u32 bus_format, u32 output_mode) static bool vop2_output_rg_swap(struct vop2 *vop2, u32 bus_format) { - if (vop2->version == VOP_VERSION_RK3588) { + if (vop2->version == VOP_VERSION_RK3588 || + vop2->version == VOP_VERSION_RK3576) { if (bus_format == MEDIA_BUS_FMT_YUV8_1X24 || bus_format == MEDIA_BUS_FMT_YUV10_1X30) return true; @@ -350,6 +351,8 @@ static bool is_yuv_output(u32 bus_format) switch (bus_format) { case MEDIA_BUS_FMT_YUV8_1X24: case MEDIA_BUS_FMT_YUV10_1X30: + case MEDIA_BUS_FMT_YUYV10_1X20: + case MEDIA_BUS_FMT_UYVY10_1X20: case MEDIA_BUS_FMT_UYYVYY8_0_5X24: case MEDIA_BUS_FMT_UYYVYY10_0_5X30: case MEDIA_BUS_FMT_YUYV8_2X8: diff --git a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c index f3950e8476a75..7cac271704c10 100644 --- a/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c +++ b/drivers/gpu/drm/rockchip/rockchip_vop2_reg.c @@ -2289,15 +2289,6 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * lead to the configuration of the previous VP being take effect along with the VSYNC * of the new VP. */ - if (layer_sel != old_layer_sel || port_sel != old_port_sel) - ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); - vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); - - if (port_sel != old_port_sel) { - vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); - vop2_cfg_done(vp); - rk3568_vop2_wait_for_port_mux_done(vop2); - } if (layer_sel != old_layer_sel && atv_layer_sel != old_layer_sel) { cfg_done = vop2_readl(vop2, RK3568_REG_CFG_DONE); @@ -2307,7 +2298,17 @@ static void rk3568_vop2_setup_layer_mixer(struct vop2_video_port *vp) * Changes of other VPs' overlays have not taken effect */ if (cfg_done) - rk3568_vop2_wait_for_layer_cfg_done(vop2, vop2->old_layer_sel); + rk3568_vop2_wait_for_layer_cfg_done(vop2, old_layer_sel); + } + + if (layer_sel != old_layer_sel || port_sel != old_port_sel) + ovl_ctrl |= FIELD_PREP(RK3568_OVL_CTRL__LAYERSEL_REGDONE_SEL, vp->id); + vop2_writel(vop2, RK3568_OVL_CTRL, ovl_ctrl); + + if (port_sel != old_port_sel) { + vop2_writel(vop2, RK3568_OVL_PORT_SEL, port_sel); + vop2_cfg_done(vp); + rk3568_vop2_wait_for_port_mux_done(vop2); } vop2_writel(vop2, RK3568_OVL_LAYER_SEL, layer_sel); diff --git a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c index 8e9ae7d980eb2..2dfa3efef2100 100644 --- a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c +++ b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2025 Valve Corporation */ +#include + #include "sched_tests.h" /* @@ -288,6 +290,7 @@ static const struct drm_sched_backend_ops drm_mock_scheduler_ops = { */ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) { + static unsigned int instance; struct drm_sched_init_args args = { .ops = &drm_mock_scheduler_ops, .num_rqs = DRM_SCHED_PRIORITY_COUNT, @@ -297,11 +300,19 @@ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout) .name = "drm-mock-scheduler", }; struct drm_mock_scheduler *sched; + struct device *dev; + char name[64]; int ret; sched = kunit_kzalloc(test, sizeof(*sched), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, sched); + snprintf(name, sizeof(name), "%s-%u", args.name, instance++); + dev = kunit_device_register(test, name); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + + args.dev = dev; + ret = drm_sched_init(&sched->base, &args); KUNIT_ASSERT_EQ(test, ret, 0); diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c index 18d2294c526de..4a7d49f0afe96 100644 --- a/drivers/gpu/drm/solomon/ssd130x.c +++ b/drivers/gpu/drm/solomon/ssd130x.c @@ -863,12 +863,13 @@ static int ssd132x_update_rect(struct ssd130x_device *ssd130x, */ /* Set column start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, x / segment_width, columns - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_COL_RANGE, x / segment_width, + x / segment_width + columns - 1); if (ret < 0) return ret; /* Set row start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, y, rows - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD132X_SET_ROW_RANGE, y, y + rows - 1); if (ret < 0) return ret; @@ -914,12 +915,12 @@ static int ssd133x_update_rect(struct ssd130x_device *ssd130x, */ /* Set column start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, columns - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_COL_RANGE, x, x + columns - 1); if (ret < 0) return ret; /* Set row start and end */ - ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, rows - 1); + ret = ssd130x_write_cmd(ssd130x, 3, SSD133X_SET_ROW_RANGE, y, y + rows - 1); if (ret < 0) return ret; diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c index 1518126e9fe32..bb8952dc6e944 100644 --- a/drivers/gpu/drm/sun4i/sun4i_backend.c +++ b/drivers/gpu/drm/sun4i/sun4i_backend.c @@ -685,7 +685,7 @@ static int sun4i_backend_init_sat(struct device *dev) { ret = clk_prepare_enable(backend->sat_clk); if (ret) { dev_err(dev, "Couldn't enable the SAT clock\n"); - return ret; + goto err_assert_reset; } return 0; diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c index 18e74047b0f56..3e90aeb68c480 100644 --- a/drivers/gpu/drm/sun4i/sun4i_crtc.c +++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c @@ -208,7 +208,7 @@ struct sun4i_crtc *sun4i_crtc_init(struct drm_device *drm, planes = sunxi_engine_layers_init(drm, engine); if (IS_ERR(planes)) { dev_err(drm->dev, "Couldn't create the planes\n"); - return NULL; + return ERR_CAST(planes); } /* find primary and cursor planes for drm_crtc_init_with_planes */ diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c index ab0938ba61f7d..c7b539b3497f6 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c @@ -146,7 +146,7 @@ static void sun4i_hdmi_enable(struct drm_encoder *encoder, drm_atomic_helper_connector_hdmi_update_infoframes(connector, state); - val |= SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); + val = SUN4I_HDMI_PKT_CTRL_TYPE(0, SUN4I_HDMI_PKT_AVI); val |= SUN4I_HDMI_PKT_CTRL_TYPE(1, SUN4I_HDMI_PKT_END); writel(val, hdmi->base + SUN4I_HDMI_PKT_CTRL_REG(0)); diff --git a/drivers/gpu/drm/sun4i/sun4i_tcon.c b/drivers/gpu/drm/sun4i/sun4i_tcon.c index 960e83c8291da..67047e3b657ff 100644 --- a/drivers/gpu/drm/sun4i/sun4i_tcon.c +++ b/drivers/gpu/drm/sun4i/sun4i_tcon.c @@ -717,9 +717,11 @@ void sun4i_tcon_mode_set(struct sun4i_tcon *tcon, case DRM_MODE_ENCODER_DSI: /* DSI is tied to special case of CPU interface */ sun4i_tcon0_mode_set_cpu(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_LVDS: sun4i_tcon0_mode_set_lvds(tcon, encoder, mode); + sun4i_tcon_set_mux(tcon, 0, encoder); break; case DRM_MODE_ENCODER_NONE: sun4i_tcon0_mode_set_rgb(tcon, encoder, mode); @@ -970,6 +972,7 @@ static int sun4i_tcon_of_get_id_from_port(struct device_node *port) continue; ret = of_property_read_u32(remote, "reg", ®); + of_node_put(remote); if (ret) continue; @@ -1407,7 +1410,7 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, { struct device_node *port, *remote; struct platform_device *pdev; - int id, ret; + int id, ret = 0; /* find TCON TOP platform device and TCON id */ @@ -1430,21 +1433,20 @@ static int sun8i_r40_tcon_tv_set_mux(struct sun4i_tcon *tcon, if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP) && encoder->encoder_type == DRM_MODE_ENCODER_TMDS) { ret = sun8i_tcon_top_set_hdmi_src(&pdev->dev, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } if (IS_ENABLED(CONFIG_DRM_SUN8I_TCON_TOP)) { ret = sun8i_tcon_top_de_config(&pdev->dev, tcon->id, id); - if (ret) { - put_device(&pdev->dev); - return ret; - } + if (ret) + goto out_put_device; } - return 0; +out_put_device: + put_device(&pdev->dev); + + return ret; } static const struct sun4i_tcon_quirks sun4i_a10_quirks = { diff --git a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c index 96532709c2a7e..51cdfa1ae3bf4 100644 --- a/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c +++ b/drivers/gpu/drm/sun4i/sun8i_dw_hdmi.c @@ -83,6 +83,8 @@ static u32 sun8i_dw_hdmi_find_possible_crtcs(struct drm_device *drm, of_node_put(remote_port); } } + + of_node_put(port); } else { crtcs = drm_of_find_possible_crtcs(drm, node); } diff --git a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c index 4fa69c463dc46..489ea94693ffa 100644 --- a/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c +++ b/drivers/gpu/drm/sun4i/sun8i_hdmi_phy.c @@ -91,7 +91,7 @@ static const struct dw_hdmi_mpll_config sun50i_h6_mpll_cfg[] = { }, }, { 594000000, { - { 0x1a40, 0x0003 }, + { 0x1a7c, 0x0003 }, { 0x3b4c, 0x0003 }, { 0x5a64, 0x0003 }, }, diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c index 82df6244af885..695491b7739bd 100644 --- a/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c +++ b/drivers/gpu/drm/sun4i/sun8i_vi_scaler.c @@ -893,20 +893,26 @@ static void sun8i_vi_scaler_set_coeff(struct regmap *map, u32 base, lan3coefftab32_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_YHCOEFF1(base, i), lan3coefftab32_right[offset + i]); + } + offset = sun8i_vi_scaler_coef_index(vstep) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) + regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), + lan2coefftab32[offset + i]); + + offset = sun8i_vi_scaler_coef_index(hstep / format->hsub) * + SUN8I_VI_SCALER_COEFF_COUNT; + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF0(base, i), ch_left[offset + i]); regmap_write(map, SUN8I_SCALER_VSU_CHCOEFF1(base, i), ch_right[offset + i]); } - - offset = sun8i_vi_scaler_coef_index(hstep) * + offset = sun8i_vi_scaler_coef_index(vstep / format->vsub) * SUN8I_VI_SCALER_COEFF_COUNT; - for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) { - regmap_write(map, SUN8I_SCALER_VSU_YVCOEFF(base, i), - lan2coefftab32[offset + i]); + for (i = 0; i < SUN8I_VI_SCALER_COEFF_COUNT; i++) regmap_write(map, SUN8I_SCALER_VSU_CVCOEFF(base, i), cy[offset + i]); - } } void sun8i_vi_scaler_enable(struct sun8i_mixer *mixer, int layer, bool enable) diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h index 265a1b8a93493..6caf87ab1f154 100644 --- a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h +++ b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h @@ -39,7 +39,7 @@ struct resource *drm_sysfb_get_memory_si(struct drm_device *dev, int drm_sysfb_get_stride_si(struct drm_device *dev, const struct screen_info *si, const struct drm_format_info *format, unsigned int width, unsigned int height, u64 size); -u64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_info *si, +s64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_info *si, unsigned int height, unsigned int stride, u64 size); const struct drm_format_info *drm_sysfb_get_format_si(struct drm_device *dev, const struct drm_sysfb_format *formats, diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c b/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c index 885864168c54f..a250bea7b19ca 100644 --- a/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c +++ b/drivers/gpu/drm/sysfb/drm_sysfb_screen_info.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -56,18 +57,24 @@ int drm_sysfb_get_stride_si(struct drm_device *dev, const struct screen_info *si unsigned int width, unsigned int height, u64 size) { u64 lfb_linelength = si->lfb_linelength; + s64 stride; if (!lfb_linelength) lfb_linelength = drm_format_info_min_pitch(format, 0, width); - return drm_sysfb_get_validated_int0(dev, "stride", lfb_linelength, div64_u64(size, height)); + stride = drm_sysfb_get_validated_size0(dev, "stride", lfb_linelength, + div64_u64(size, height)); + if (stride < INT_MIN || stride > INT_MAX) + return -EINVAL; + + return (int)stride; /* stride or negative errno code */ } EXPORT_SYMBOL(drm_sysfb_get_stride_si); -u64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_info *si, +s64 drm_sysfb_get_visible_size_si(struct drm_device *dev, const struct screen_info *si, unsigned int height, unsigned int stride, u64 size) { - u64 vsize = PAGE_ALIGN(height * stride); + u64 vsize = mul_u32_u32(height, stride); return drm_sysfb_get_validated_size0(dev, "visible size", vsize, size); } diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c index 97a3711e79337..844d1e8b0ab3c 100644 --- a/drivers/gpu/drm/sysfb/efidrm.c +++ b/drivers/gpu/drm/sysfb/efidrm.c @@ -143,7 +143,8 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, const struct screen_info *si; const struct drm_format_info *format; int width, height, stride; - u64 vsize, mem_flags; + s64 vsize; + u64 mem_flags; struct resource resbuf; struct resource *res; struct efidrm_device *efi; @@ -195,8 +196,8 @@ static struct efidrm_device *efidrm_device_create(struct drm_driver *drv, if (stride < 0) return ERR_PTR(stride); vsize = drm_sysfb_get_visible_size_si(dev, si, height, stride, resource_size(res)); - if (!vsize) - return ERR_PTR(-EINVAL); + if (vsize < 0) + return ERR_PTR(vsize); drm_dbg(dev, "framebuffer format=%p4cc, size=%dx%d, stride=%d bytes\n", &format->format, width, height, stride); diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c index f94fddea897a5..db5e9389d48fd 100644 --- a/drivers/gpu/drm/sysfb/ofdrm.c +++ b/drivers/gpu/drm/sysfb/ofdrm.c @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -235,7 +236,7 @@ static bool is_avivo(u32 vendor, u32 device) /* This will match most R5xx */ return (vendor == PCI_VENDOR_ID_ATI) && ((device >= PCI_VENDOR_ID_ATI_R520 && device < 0x7800) || - (PCI_VENDOR_ID_ATI_R600 >= 0x9400)); + (device >= PCI_VENDOR_ID_ATI_R600)); } static enum ofdrm_model display_get_model_of(struct drm_device *dev, struct device_node *of_node) @@ -910,7 +911,10 @@ static struct ofdrm_device *ofdrm_device_create(struct drm_driver *drv, return ERR_PTR(-EINVAL); } - fb_size = linebytes * height; + if (check_mul_overflow(linebytes, height, &fb_size)) { + drm_err(dev, "framebuffer size exceeds maximum\n"); + return ERR_PTR(-EINVAL); + } /* * Try to figure out the address of the framebuffer. Unfortunately, Open diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c index 0358164a623c9..22f8caab39be0 100644 --- a/drivers/gpu/drm/sysfb/simpledrm.c +++ b/drivers/gpu/drm/sysfb/simpledrm.c @@ -44,13 +44,6 @@ simplefb_get_validated_int(struct drm_device *dev, const char *name, return drm_sysfb_get_validated_int(dev, name, value, INT_MAX); } -static int -simplefb_get_validated_int0(struct drm_device *dev, const char *name, - uint32_t value) -{ - return drm_sysfb_get_validated_int0(dev, name, value, INT_MAX); -} - static const struct drm_format_info * simplefb_get_validated_format(struct drm_device *dev, const char *format_name) { @@ -84,14 +77,14 @@ static int simplefb_get_width_pd(struct drm_device *dev, const struct simplefb_platform_data *pd) { - return simplefb_get_validated_int0(dev, "width", pd->width); + return drm_sysfb_get_validated_int0(dev, "width", pd->width, U16_MAX); } static int simplefb_get_height_pd(struct drm_device *dev, const struct simplefb_platform_data *pd) { - return simplefb_get_validated_int0(dev, "height", pd->height); + return drm_sysfb_get_validated_int0(dev, "height", pd->height, U16_MAX); } static int @@ -140,7 +133,7 @@ simplefb_get_width_of(struct drm_device *dev, struct device_node *of_node) if (ret) return ret; - return simplefb_get_validated_int0(dev, "width", width); + return drm_sysfb_get_validated_int0(dev, "width", width, U16_MAX); } static int @@ -151,7 +144,7 @@ simplefb_get_height_of(struct drm_device *dev, struct device_node *of_node) if (ret) return ret; - return simplefb_get_validated_int0(dev, "height", height); + return drm_sysfb_get_validated_int0(dev, "height", height, U16_MAX); } static int @@ -196,6 +189,39 @@ simplefb_get_memory_of(struct drm_device *dev, struct device_node *of_node) return res; } +static int __simplefb_get_panel_size_mm_of(struct drm_device *dev, + struct device_node *of_panel_node, + const char *name) +{ + int ret; + u32 value; + + ret = of_property_read_u32(of_panel_node, name, &value); + if (ret) { + drm_dbg(dev, "simplefb: cannot parse panel %s: error %d\n", + name, ret); + return ret; + } else if (value > U16_MAX) { + drm_dbg(dev, "simplefb: panel %s of %u exceeds maximum value\n", + name, value); + return -EINVAL; + } + + return value; +} + +static int simplefb_get_panel_width_mm_of(struct drm_device *dev, + struct device_node *of_panel_node) +{ + return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "width-mm"); +} + +static int simplefb_get_panel_height_mm_of(struct drm_device *dev, + struct device_node *of_panel_node) +{ + return __simplefb_get_panel_size_mm_of(dev, of_panel_node, "height-mm"); +} + /* * Simple Framebuffer device */ @@ -597,7 +623,7 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, struct drm_sysfb_device *sysfb; struct drm_device *dev; int width, height, stride; - int width_mm = 0, height_mm = 0; + u16 width_mm = 0, height_mm = 0; struct device_node *panel_node; const struct drm_format_info *format; struct resource *res, *mem = NULL; @@ -661,8 +687,18 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, return ERR_CAST(mem); panel_node = of_parse_phandle(of_node, "panel", 0); if (panel_node) { - simplefb_read_u32_of(dev, panel_node, "width-mm", &width_mm); - simplefb_read_u32_of(dev, panel_node, "height-mm", &height_mm); + /* + * Ignore errors from parsing the physical panel + * size. Using the pre-initialized sizes of 0 will + * make drm_sysfb_mode() calculate a default physical + * size based on a resolution of 96 dpi. + */ + ret = simplefb_get_panel_width_mm_of(dev, panel_node); + if (ret > 0) + width_mm = ret; + ret = simplefb_get_panel_height_mm_of(dev, panel_node); + if (ret > 0) + height_mm = ret; of_node_put(panel_node); } } else { @@ -670,9 +706,15 @@ static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv, return ERR_PTR(-ENODEV); } if (!stride) { - stride = drm_format_info_min_pitch(format, 0, width); - if (drm_WARN_ON(dev, !stride)) + u64 pitch = drm_format_info_min_pitch(format, 0, width); + + if (drm_WARN_ON(dev, !pitch)) { + return ERR_PTR(-EINVAL); /* driver bug */ + } else if (pitch > INT_MAX) { + drm_warn(dev, "stride of %llu exceeds maximum\n", pitch); return ERR_PTR(-EINVAL); + } + stride = pitch; } sysfb->fb_mode = drm_sysfb_mode(width, height, width_mm, height_mm); diff --git a/drivers/gpu/drm/sysfb/vesadrm.c b/drivers/gpu/drm/sysfb/vesadrm.c index 16a4b52d45c6c..37123976e1fbf 100644 --- a/drivers/gpu/drm/sysfb/vesadrm.c +++ b/drivers/gpu/drm/sysfb/vesadrm.c @@ -392,7 +392,7 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv, const struct screen_info *si; const struct drm_format_info *format; int width, height, stride; - u64 vsize; + s64 vsize; struct resource resbuf; struct resource *res; struct vesadrm_device *vesa; @@ -445,8 +445,8 @@ static struct vesadrm_device *vesadrm_device_create(struct drm_driver *drv, if (stride < 0) return ERR_PTR(stride); vsize = drm_sysfb_get_visible_size_si(dev, si, height, stride, resource_size(res)); - if (!vsize) - return ERR_PTR(-EINVAL); + if (vsize < 0) + return ERR_PTR(vsize); drm_dbg(dev, "framebuffer format=%p4cc, size=%dx%d, stride=%d bytes\n", &format->format, width, height, stride); diff --git a/drivers/gpu/drm/tegra/fbdev.c b/drivers/gpu/drm/tegra/fbdev.c index 1b70f5e164afd..50d3186241c6a 100644 --- a/drivers/gpu/drm/tegra/fbdev.c +++ b/drivers/gpu/drm/tegra/fbdev.c @@ -75,10 +75,9 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, struct tegra_drm *tegra = helper->dev->dev_private; struct drm_device *drm = helper->dev; struct drm_mode_fb_cmd2 cmd = { 0 }; + struct fb_info *info = helper->info; unsigned int bytes_per_pixel; struct drm_framebuffer *fb; - unsigned long offset; - struct fb_info *info; struct tegra_bo *bo; size_t size; int err; @@ -99,13 +98,6 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, if (IS_ERR(bo)) return PTR_ERR(bo); - info = drm_fb_helper_alloc_info(helper); - if (IS_ERR(info)) { - dev_err(drm->dev, "failed to allocate framebuffer info\n"); - drm_gem_object_put(&bo->gem); - return PTR_ERR(info); - } - fb = tegra_fb_alloc(drm, drm_get_format_info(drm, cmd.pixel_format, cmd.modifier[0]), &cmd, &bo, 1); @@ -119,15 +111,11 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, helper->funcs = &tegra_fbdev_helper_funcs; helper->fb = fb; - helper->info = info; info->fbops = &tegra_fb_ops; drm_fb_helper_fill_info(info, helper, sizes); - offset = info->var.xoffset * bytes_per_pixel + - info->var.yoffset * fb->pitches[0]; - if (bo->pages) { bo->vaddr = vmap(bo->pages, bo->num_pages, VM_MAP, pgprot_writecombine(PAGE_KERNEL)); @@ -139,9 +127,9 @@ int tegra_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper, } info->flags |= FBINFO_VIRTFB; - info->screen_buffer = bo->vaddr + offset; + info->screen_buffer = bo->vaddr; info->screen_size = size; - info->fix.smem_start = (unsigned long)(bo->iova + offset); + info->fix.smem_start = (unsigned long)(bo->iova); info->fix.smem_len = size; return 0; diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile index c0e952293ad02..524310241e16d 100644 --- a/drivers/gpu/drm/tests/Makefile +++ b/drivers/gpu/drm/tests/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_DRM_KUNIT_TEST) += \ drm_atomic_test.o \ drm_atomic_state_test.o \ drm_bridge_test.o \ - drm_buddy_test.o \ drm_cmdline_parser_test.o \ drm_connector_test.o \ drm_damage_helper_test.o \ diff --git a/drivers/gpu/drm/tests/drm_exec_test.c b/drivers/gpu/drm/tests/drm_exec_test.c index 3a20c788c51f8..2fc47f3b463b7 100644 --- a/drivers/gpu/drm/tests/drm_exec_test.c +++ b/drivers/gpu/drm/tests/drm_exec_test.c @@ -16,8 +16,6 @@ #include #include -#include "../lib/drm_random.h" - struct drm_exec_priv { struct device *dev; struct drm_device *drm; diff --git a/drivers/gpu/drm/tests/drm_gem_shmem_test.c b/drivers/gpu/drm/tests/drm_gem_shmem_test.c index 4b459f21acfd9..b1fa287892fb8 100644 --- a/drivers/gpu/drm/tests/drm_gem_shmem_test.c +++ b/drivers/gpu/drm/tests/drm_gem_shmem_test.c @@ -95,13 +95,9 @@ static void drm_gem_shmem_test_obj_create_private(struct kunit *test) sg_init_one(sgt->sgl, buf, TEST_SIZE); /* - * Set the DMA mask to 64-bits and map the sgtables - * otherwise drm_gem_shmem_free will cause a warning - * on debug kernels. + * Map the sgtables otherwise drm_gem_shmem_free will cause a warning on + * debug kernels. */ - ret = dma_set_mask(drm_dev->dev, DMA_BIT_MASK(64)); - KUNIT_ASSERT_EQ(test, ret, 0); - ret = dma_map_sgtable(drm_dev->dev, sgt, DMA_BIDIRECTIONAL, 0); KUNIT_ASSERT_EQ(test, ret, 0); @@ -352,11 +348,19 @@ static int drm_gem_shmem_test_init(struct kunit *test) { struct device *dev; struct drm_device *drm_dev; + int ret; /* Allocate a parent device */ dev = drm_kunit_helper_alloc_device(test); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev); + /* + * Set the DMA mask to 64-bits to avoid intermittent failures calling + * drm_gem_shmem_get_pages_sgt(). + */ + ret = dma_set_mask(dev, DMA_BIT_MASK(64)); + KUNIT_ASSERT_EQ(test, ret, 0); + /* * The DRM core will automatically initialize the GEM core and create * a DRM Memory Manager object which provides an address space pool diff --git a/drivers/gpu/drm/tests/drm_mm_test.c b/drivers/gpu/drm/tests/drm_mm_test.c index 6174d09290208..99744322ed1a2 100644 --- a/drivers/gpu/drm/tests/drm_mm_test.c +++ b/drivers/gpu/drm/tests/drm_mm_test.c @@ -15,8 +15,6 @@ #include -#include "../lib/drm_random.h" - enum { BEST, BOTTOMUP, diff --git a/drivers/gpu/drm/tidss/tidss_encoder.c b/drivers/gpu/drm/tidss/tidss_encoder.c index 81a04f7677701..db467bbcdb776 100644 --- a/drivers/gpu/drm/tidss/tidss_encoder.c +++ b/drivers/gpu/drm/tidss/tidss_encoder.c @@ -106,6 +106,8 @@ int tidss_encoder_create(struct tidss_device *tidss, enc = &t_enc->encoder; enc->possible_crtcs = possible_crtcs; + devm_drm_bridge_add(tidss->dev, &t_enc->bridge); + /* Attaching first bridge to the encoder */ ret = drm_bridge_attach(enc, &t_enc->bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR); diff --git a/drivers/gpu/drm/tiny/cirrus-qemu.c b/drivers/gpu/drm/tiny/cirrus-qemu.c index 97a93adc5669d..a6ce6aa99477e 100644 --- a/drivers/gpu/drm/tiny/cirrus-qemu.c +++ b/drivers/gpu/drm/tiny/cirrus-qemu.c @@ -570,6 +570,9 @@ static int cirrus_pci_probe(struct pci_dev *pdev, struct cirrus_device *cirrus; int ret; + if (pci_resource_len(pdev, 0) < CIRRUS_VRAM_SIZE) + return -ENODEV; + ret = aperture_remove_conflicting_pci_devices(pdev, cirrus_driver.name); if (ret) return ret; diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c index 1bcc67977f487..3cbdc8101b7c6 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c @@ -251,7 +251,7 @@ static void ttm_bo_validate_basic(struct kunit *test) NULL, &dummy_ttm_bo_destroy); KUNIT_EXPECT_EQ(test, err, 0); - snd_place = ttm_place_kunit_init(test, snd_mem, DRM_BUDDY_TOPDOWN_ALLOCATION); + snd_place = ttm_place_kunit_init(test, snd_mem, GPU_BUDDY_TOPDOWN_ALLOCATION); snd_placement = ttm_placement_kunit_init(test, snd_place, 1); err = ttm_bo_validate(bo, snd_placement, &ctx_val); @@ -263,7 +263,7 @@ static void ttm_bo_validate_basic(struct kunit *test) KUNIT_EXPECT_TRUE(test, ttm_tt_is_populated(bo->ttm)); KUNIT_EXPECT_EQ(test, bo->resource->mem_type, snd_mem); KUNIT_EXPECT_EQ(test, bo->resource->placement, - DRM_BUDDY_TOPDOWN_ALLOCATION); + GPU_BUDDY_TOPDOWN_ALLOCATION); ttm_bo_put(bo); ttm_mock_manager_fini(priv->ttm_dev, snd_mem); diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c index d7eb6471f2eda..a9c233f982f67 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -30,7 +30,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man, { struct ttm_mock_manager *manager = to_mock_mgr(man); struct ttm_mock_resource *mock_res; - struct drm_buddy *mm = &manager->mm; + struct gpu_buddy *mm = &manager->mm; u64 lpfn, fpfn, alloc_size; int err; @@ -46,14 +46,14 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man, INIT_LIST_HEAD(&mock_res->blocks); if (place->flags & TTM_PL_FLAG_TOPDOWN) - mock_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + mock_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; if (place->flags & TTM_PL_FLAG_CONTIGUOUS) - mock_res->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION; + mock_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; alloc_size = (uint64_t)mock_res->base.size; mutex_lock(&manager->lock); - err = drm_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size, + err = gpu_buddy_alloc_blocks(mm, fpfn, lpfn, alloc_size, manager->default_page_size, &mock_res->blocks, mock_res->flags); @@ -66,7 +66,7 @@ static int ttm_mock_manager_alloc(struct ttm_resource_manager *man, return 0; error_free_blocks: - drm_buddy_free_list(mm, &mock_res->blocks, 0); + gpu_buddy_free_list(mm, &mock_res->blocks, 0); ttm_resource_fini(man, &mock_res->base); mutex_unlock(&manager->lock); @@ -78,10 +78,10 @@ static void ttm_mock_manager_free(struct ttm_resource_manager *man, { struct ttm_mock_manager *manager = to_mock_mgr(man); struct ttm_mock_resource *mock_res = to_mock_mgr_resource(res); - struct drm_buddy *mm = &manager->mm; + struct gpu_buddy *mm = &manager->mm; mutex_lock(&manager->lock); - drm_buddy_free_list(mm, &mock_res->blocks, 0); + gpu_buddy_free_list(mm, &mock_res->blocks, 0); mutex_unlock(&manager->lock); ttm_resource_fini(man, res); @@ -105,7 +105,7 @@ int ttm_mock_manager_init(struct ttm_device *bdev, u32 mem_type, u32 size) mutex_init(&manager->lock); - err = drm_buddy_init(&manager->mm, size, PAGE_SIZE); + err = gpu_buddy_init(&manager->mm, size, PAGE_SIZE); if (err) { kfree(manager); @@ -141,7 +141,7 @@ void ttm_mock_manager_fini(struct ttm_device *bdev, u32 mem_type) ttm_resource_manager_set_used(man, false); mutex_lock(&mock_man->lock); - drm_buddy_fini(&mock_man->mm); + gpu_buddy_fini(&mock_man->mm); mutex_unlock(&mock_man->lock); ttm_set_driver_manager(bdev, mem_type, NULL); diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h index e4c95f86a4670..08710756fd8e0 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.h @@ -5,11 +5,11 @@ #ifndef TTM_MOCK_MANAGER_H #define TTM_MOCK_MANAGER_H -#include +#include struct ttm_mock_manager { struct ttm_resource_manager man; - struct drm_buddy mm; + struct gpu_buddy mm; u64 default_page_size; /* protects allocations of mock buffer objects */ struct mutex lock; diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index cf0e785aa9d3c..d7a722d875cbc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -266,8 +266,8 @@ static void ttm_bo_release(struct kref *kref) 30 * HZ); } - if (bo->bdev->funcs->release_notify) - bo->bdev->funcs->release_notify(bo); + if (bdev->funcs->release_notify) + bdev->funcs->release_notify(bo); drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_free(bdev, bo->resource); @@ -281,7 +281,7 @@ static void ttm_bo_release(struct kref *kref) ttm_bo_flush_all_fences(bo); bo->deleted = true; - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bdev->lru_lock); /* * Make pinned bos immediately available to @@ -297,7 +297,7 @@ static void ttm_bo_release(struct kref *kref) } kref_init(&bo->kref); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); @@ -358,7 +358,6 @@ static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - struct ttm_device *bdev = bo->bdev; struct ttm_resource *evict_mem; struct ttm_placement placement; struct ttm_place hop; @@ -369,7 +368,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, dma_resv_assert_held(bo->base.resv); placement.num_placement = 0; - bdev->funcs->evict_flags(bo, &placement); + bo->bdev->funcs->evict_flags(bo, &placement); if (!placement.num_placement) { ret = ttm_bo_wait_ctx(bo, ctx); @@ -422,16 +421,16 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { struct ttm_resource *res = bo->resource; - struct ttm_device *bdev = bo->bdev; dma_resv_assert_held(bo->base.resv); - if (bo->resource->mem_type == TTM_PL_SYSTEM) + + if (res->mem_type == TTM_PL_SYSTEM) return true; /* Don't evict this BO if it's outside of the * requested placement range */ - return ttm_resource_intersects(bdev, res, place, bo->base.size); + return ttm_resource_intersects(bo->bdev, res, place, bo->base.size); } EXPORT_SYMBOL(ttm_bo_eviction_valuable); @@ -1105,10 +1104,12 @@ struct ttm_bo_swapout_walk { static s64 ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_place place = {.mem_type = bo->resource->mem_type}; + struct ttm_place place = { .mem_type = bo->resource->mem_type }; struct ttm_bo_swapout_walk *swapout_walk = container_of(walk, typeof(*swapout_walk), walk); struct ttm_operation_ctx *ctx = walk->arg.ctx; + struct ttm_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; s64 ret; /* @@ -1117,20 +1118,19 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) * The driver may use the fact that we're moving from SYSTEM * as an indication that we're about to swap out. */ - if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, &place)) { + if (bo->pin_count || !bdev->funcs->eviction_valuable(bo, &place)) { ret = -EBUSY; goto out; } - if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || - bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || - bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) { + if (!tt || !ttm_tt_is_populated(tt) || + tt->page_flags & (TTM_TT_FLAG_EXTERNAL | TTM_TT_FLAG_SWAPPED)) { ret = -EBUSY; goto out; } if (bo->deleted) { - pgoff_t num_pages = bo->ttm->num_pages; + pgoff_t num_pages = tt->num_pages; ret = ttm_bo_wait_ctx(bo, ctx); if (ret) @@ -1171,21 +1171,21 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) goto out; ttm_bo_unmap_virtual(bo); - if (bo->bdev->funcs->swap_notify) - bo->bdev->funcs->swap_notify(bo); + if (bdev->funcs->swap_notify) + bdev->funcs->swap_notify(bo); - if (ttm_tt_is_populated(bo->ttm)) { - spin_lock(&bo->bdev->lru_lock); + if (ttm_tt_is_populated(tt)) { + spin_lock(&bdev->lru_lock); ttm_resource_del_bulk_move(bo->resource, bo); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); - ret = ttm_tt_swapout(bo->bdev, bo->ttm, swapout_walk->gfp_flags); + ret = ttm_tt_swapout(bdev, tt, swapout_walk->gfp_flags); - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bdev->lru_lock); if (ret) ttm_resource_add_bulk_move(bo->resource, bo); ttm_resource_move_to_lru_tail(bo->resource); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); } out: @@ -1258,6 +1258,7 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) int ttm_bo_populate(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { + struct ttm_device *bdev = bo->bdev; struct ttm_tt *tt = bo->ttm; bool swapped; int ret; @@ -1268,16 +1269,16 @@ int ttm_bo_populate(struct ttm_buffer_object *bo, return 0; swapped = ttm_tt_is_swapped(tt); - ret = ttm_tt_populate(bo->bdev, tt, ctx); + ret = ttm_tt_populate(bdev, tt, ctx); if (ret) return ret; if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && bo->resource) { - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bdev->lru_lock); ttm_resource_add_bulk_move(bo->resource, bo); ttm_resource_move_to_lru_tail(bo->resource); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); } return 0; diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index acbbca9d5c92f..85bb9f9f1f616 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -174,13 +174,13 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo, dst_iter = ttm_kmap_iter_linear_io_init(&_dst_iter.io, bdev, dst_mem); if (PTR_ERR(dst_iter) == -EINVAL && dst_man->use_tt) - dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, bo->ttm); + dst_iter = ttm_kmap_iter_tt_init(&_dst_iter.tt, ttm); if (IS_ERR(dst_iter)) return PTR_ERR(dst_iter); src_iter = ttm_kmap_iter_linear_io_init(&_src_iter.io, bdev, src_mem); if (PTR_ERR(src_iter) == -EINVAL && src_man->use_tt) - src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, bo->ttm); + src_iter = ttm_kmap_iter_tt_init(&_src_iter.tt, ttm); if (IS_ERR(src_iter)) { ret = PTR_ERR(src_iter); goto out_src_iter; @@ -318,11 +318,11 @@ static int ttm_bo_ioremap(struct ttm_buffer_object *bo, { struct ttm_resource *mem = bo->resource; - if (bo->resource->bus.addr) { + if (mem->bus.addr) { map->bo_kmap_type = ttm_bo_map_premapped; - map->virtual = ((u8 *)bo->resource->bus.addr) + offset; + map->virtual = ((u8 *)mem->bus.addr) + offset; } else { - resource_size_t res = bo->resource->bus.offset + offset; + resource_size_t res = mem->bus.offset + offset; map->bo_kmap_type = ttm_bo_map_iomap; if (mem->bus.caching == ttm_write_combined) @@ -349,7 +349,7 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, }; struct ttm_tt *ttm = bo->ttm; struct ttm_resource_manager *man = - ttm_manager_type(bo->bdev, bo->resource->mem_type); + ttm_manager_type(bo->bdev, mem->mem_type); pgprot_t prot; int ret; @@ -428,20 +428,21 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page, unsigned long num_pages, struct ttm_bo_kmap_obj *map) { + struct ttm_resource *res = bo->resource; unsigned long offset, size; int ret; map->virtual = NULL; map->bo = bo; - if (num_pages > PFN_UP(bo->resource->size)) + if (num_pages > PFN_UP(res->size)) return -EINVAL; - if ((start_page + num_pages) > PFN_UP(bo->resource->size)) + if ((start_page + num_pages) > PFN_UP(res->size)) return -EINVAL; - ret = ttm_mem_io_reserve(bo->bdev, bo->resource); + ret = ttm_mem_io_reserve(bo->bdev, res); if (ret) return ret; - if (!bo->resource->bus.is_iomem) { + if (!res->bus.is_iomem) { return ttm_bo_kmap_ttm(bo, start_page, num_pages, map); } else { offset = start_page << PAGE_SHIFT; @@ -581,7 +582,7 @@ void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map) iounmap(map->vaddr_iomem); iosys_map_clear(map); - ttm_mem_io_free(bo->bdev, bo->resource); + ttm_mem_io_free(bo->bdev, mem); } EXPORT_SYMBOL(ttm_bo_vunmap); @@ -644,10 +645,9 @@ static int ttm_bo_move_to_ghost(struct ttm_buffer_object *bo, static void ttm_bo_move_pipeline_evict(struct ttm_buffer_object *bo, struct dma_fence *fence) { - struct ttm_device *bdev = bo->bdev; struct ttm_resource_manager *from; - from = ttm_manager_type(bdev, bo->resource->mem_type); + from = ttm_manager_type(bo->bdev, bo->resource->mem_type); /** * BO doesn't have a TTM we need to bind/unbind. Just remember @@ -719,8 +719,8 @@ EXPORT_SYMBOL(ttm_bo_move_accel_cleanup); void ttm_bo_move_sync_cleanup(struct ttm_buffer_object *bo, struct ttm_resource *new_mem) { - struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man = ttm_manager_type(bdev, new_mem->mem_type); + struct ttm_resource_manager *man = + ttm_manager_type(bo->bdev, new_mem->mem_type); int ret; ret = ttm_bo_wait_free_node(bo, man->use_tt); @@ -824,13 +824,12 @@ static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, struct ttm_buffer_object *bo) { struct ttm_lru_walk_arg *arg = curs->arg; - struct dma_resv *resv = bo->base.resv; int ret; if (arg->ctx->interruptible) - ret = dma_resv_lock_interruptible(resv, arg->ticket); + ret = dma_resv_lock_interruptible(bo->base.resv, arg->ticket); else - ret = dma_resv_lock(resv, arg->ticket); + ret = dma_resv_lock(bo->base.resv, arg->ticket); if (!ret) { curs->needs_unlock = true; @@ -1074,7 +1073,7 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, .num_placement = 1, .placement = &sys_placement_flags, }; - struct ttm_tt *tt = bo->ttm; + struct ttm_device *bdev = bo->bdev; long lret; dma_resv_assert_held(bo->base.resv); @@ -1096,19 +1095,19 @@ long ttm_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo, return lret; if (bo->bulk_move) { - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bdev->lru_lock); ttm_resource_del_bulk_move(bo->resource, bo); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); } - lret = ttm_tt_backup(bo->bdev, tt, (struct ttm_backup_flags) + lret = ttm_tt_backup(bdev, bo->ttm, (struct ttm_backup_flags) {.purge = flags.purge, .writeback = flags.writeback}); if (lret <= 0 && bo->bulk_move) { - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bdev->lru_lock); ttm_resource_add_bulk_move(bo->resource, bo); - spin_unlock(&bo->bdev->lru_lock); + spin_unlock(&bdev->lru_lock); } if (lret < 0 && lret != -EINTR) diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index e6abc7b40b189..a80510489c456 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -186,7 +186,6 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, { struct vm_area_struct *vma = vmf->vma; struct ttm_buffer_object *bo = vma->vm_private_data; - struct ttm_device *bdev = bo->bdev; unsigned long page_offset; unsigned long page_last; unsigned long pfn; @@ -205,7 +204,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, if (unlikely(ret != 0)) return ret; - err = ttm_mem_io_reserve(bdev, bo->resource); + err = ttm_mem_io_reserve(bo->bdev, bo->resource); if (unlikely(err != 0)) return VM_FAULT_SIGBUS; @@ -293,7 +292,6 @@ vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) { struct vm_area_struct *vma = vmf->vma; struct ttm_buffer_object *bo = vma->vm_private_data; - struct drm_device *ddev = bo->base.dev; vm_fault_t ret = VM_FAULT_NOPAGE; unsigned long address; unsigned long pfn; @@ -305,7 +303,8 @@ vm_fault_t ttm_bo_vm_dummy_page(struct vm_fault *vmf, pgprot_t prot) return VM_FAULT_OOM; /* Set the page to be freed using drmm release action */ - if (drmm_add_action_or_reset(ddev, ttm_bo_release_dummy_page, page)) + if (drmm_add_action_or_reset(bo->base.dev, ttm_bo_release_dummy_page, + page)) return VM_FAULT_OOM; pfn = page_to_pfn(page); @@ -322,10 +321,9 @@ EXPORT_SYMBOL(ttm_bo_vm_dummy_page); vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; - pgprot_t prot; struct ttm_buffer_object *bo = vma->vm_private_data; - struct drm_device *ddev = bo->base.dev; vm_fault_t ret; + pgprot_t prot; int idx; ret = ttm_bo_vm_reserve(bo, vmf); @@ -333,7 +331,7 @@ vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) return ret; prot = vma->vm_page_prot; - if (drm_dev_enter(ddev, &idx)) { + if (drm_dev_enter(bo->base.dev, &idx)) { ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); drm_dev_exit(idx); } else { diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index baf27c70a4193..dd5666eb4692b 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -836,40 +836,51 @@ EXPORT_SYMBOL(ttm_pool_alloc); int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, const struct ttm_operation_ctx *ctx) { + struct ttm_pool_tt_restore *restore = tt->restore; struct ttm_pool_alloc_state alloc; + int ret; if (WARN_ON(!ttm_tt_is_backed_up(tt))) return -EINVAL; - if (!tt->restore) { + if (!restore) { gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; ttm_pool_alloc_state_init(tt, &alloc); if (ctx->gfp_retry_mayfail) gfp |= __GFP_RETRY_MAYFAIL; - tt->restore = kzalloc(sizeof(*tt->restore), gfp); - if (!tt->restore) + restore = kzalloc(sizeof(*restore), gfp); + if (!restore) return -ENOMEM; - tt->restore->snapshot_alloc = alloc; - tt->restore->pool = pool; - tt->restore->restored_pages = 1; - } else { - struct ttm_pool_tt_restore *restore = tt->restore; - int ret; + restore->snapshot_alloc = alloc; + restore->pool = pool; + restore->restored_pages = 1; + tt->restore = restore; + } else { alloc = restore->snapshot_alloc; - if (ttm_pool_restore_valid(tt->restore)) { - ret = ttm_pool_restore_commit(restore, tt->backup, ctx, &alloc); + if (ttm_pool_restore_valid(restore)) { + ret = ttm_pool_restore_commit(restore, tt->backup, + ctx, &alloc); + if (ret) return ret; } - if (!alloc.remaining_pages) + if (!alloc.remaining_pages) { + ret = ttm_pool_apply_caching(&alloc); + if (ret) + return ret; + + kfree(tt->restore); + tt->restore = NULL; + return 0; + } } - return __ttm_pool_alloc(pool, tt, ctx, &alloc, tt->restore); + return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore); } /** @@ -981,9 +992,31 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, return -EBUSY; #ifdef CONFIG_X86 - /* Anything returned to the system needs to be cached. */ - if (tt->caching != ttm_cached) - set_pages_array_wb(tt->pages, tt->num_pages); + /* Anything returned to the system needs to be cached. Walk allocations + * skipping NULL pages and issue set_pages_array_wb() per contiguous run. + */ + if (tt->caching != ttm_cached) { + pgoff_t run_start = 0, run_count = 0; + + for (i = 0; i < tt->num_pages; i += num_pages) { + page = tt->pages[i]; + if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { + if (run_count) { + set_pages_array_wb(&tt->pages[run_start], + run_count); + run_count = 0; + } + num_pages = 1; + continue; + } + num_pages = 1UL << ttm_pool_page_order(pool, page); + if (!run_count) + run_start = i; + run_count += num_pages; + } + if (run_count) + set_pages_array_wb(&tt->pages[run_start], run_count); + } #endif if (tt->dma_address || flags->purge) { @@ -991,7 +1024,7 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, unsigned int order; page = tt->pages[i]; - if (unlikely(!page)) { + if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { num_pages = 1; continue; } @@ -1034,6 +1067,10 @@ long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, if (unlikely(!page)) continue; + /* Already-handled entry from a previous attempt. */ + if (unlikely(ttm_backup_page_ptr_is_handle(page))) + continue; + ttm_pool_split_for_swap(pool, page); shandle = ttm_backup_backup_page(backup, page, flags->writeback, i, diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 2a05fbd075465..8396cab34cb03 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -620,11 +620,11 @@ ttm_resource_cursor_check_bulk(struct ttm_resource_cursor *cursor, struct ttm_lru_item *next_lru) { struct ttm_resource *next = ttm_lru_item_to_res(next_lru); - struct ttm_lru_bulk_move *bulk = NULL; - struct ttm_buffer_object *bo = next->bo; + struct ttm_lru_bulk_move *bulk; lockdep_assert_held(&cursor->man->bdev->lru_lock); - bulk = bo->bulk_move; + + bulk = next->bo->bulk_move; if (cursor->bulk != bulk) { if (bulk) { diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c index a048e37f1c2c1..7d0329292ef6d 100644 --- a/drivers/gpu/drm/tve200/tve200_drv.c +++ b/drivers/gpu/drm/tve200/tve200_drv.c @@ -259,6 +259,7 @@ static const struct of_device_id tve200_of_match[] = { }, {}, }; +MODULE_DEVICE_TABLE(of, tve200_of_match); static struct platform_driver tve200_driver = { .driver = { diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index e684a012281d3..a036d982d6995 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -181,6 +181,12 @@ struct v3d_dev { */ struct mutex reset_lock; + /* Ordered workqueue shared by every queue's scheduler timeout work. + * V3D reset is global to all queues, so the timeout handlers must not + * run concurrently. + */ + struct workqueue_struct *reset_wq; + /* Lock taken when creating and pushing the GPU scheduler * jobs, to keep the sched-fence seqnos in order. */ diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c index d473d24dcd0df..c3dc7d79def18 100644 --- a/drivers/gpu/drm/v3d/v3d_gem.c +++ b/drivers/gpu/drm/v3d/v3d_gem.c @@ -46,7 +46,7 @@ v3d_idle_axi(struct v3d_dev *v3d, int core) (V3D_GMP_STATUS_RD_COUNT_MASK | V3D_GMP_STATUS_WR_COUNT_MASK | V3D_GMP_STATUS_CFG_BUSY)) == 0, 100)) { - DRM_ERROR("Failed to wait for safe GMP shutdown\n"); + drm_err(&v3d->drm, "Failed to wait for safe GMP shutdown\n"); } return; } diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 670805645c4dc..9b98386965796 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -214,15 +214,11 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) struct v3d_dev *v3d = job->base.v3d; struct v3d_queue_state *queue = &v3d->queue[V3D_BIN]; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; unsigned long irqflags; - if (unlikely(job->base.base.s_fence->finished.error)) { - spin_lock_irqsave(&queue->queue_lock, irqflags); - queue->active_job = NULL; - spin_unlock_irqrestore(&queue->queue_lock, irqflags); - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* Lock required around bin_job update vs * v3d_overflow_mem_work(). @@ -239,7 +235,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_BIN); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -267,6 +263,12 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT0QEA, job->end); return fence; + +out_clean_job: + spin_lock_irqsave(&queue->queue_lock, irqflags); + queue->active_job = NULL; + spin_unlock_irqrestore(&queue->queue_lock, irqflags); + return fence; } static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) @@ -274,12 +276,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) struct v3d_render_job *job = to_render_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_RENDER].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_RENDER].active_job = &job->base; @@ -293,7 +293,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_RENDER); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -314,6 +314,10 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, V3D_CLE_CT1QEA, job->end); return fence; + +out_clean_job: + v3d->queue[V3D_RENDER].active_job = NULL; + return fence; } static struct dma_fence * @@ -322,18 +326,16 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) struct v3d_tfu_job *job = to_tfu_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_TFU].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; v3d->queue[V3D_TFU].active_job = &job->base; fence = v3d_fence_create(v3d, V3D_TFU); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -361,6 +363,10 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job) V3D_WRITE(V3D_TFU_ICFG(v3d->ver), job->args.icfg | V3D_TFU_ICFG_IOC); return fence; + +out_clean_job: + v3d->queue[V3D_TFU].active_job = NULL; + return fence; } static struct dma_fence * @@ -369,13 +375,11 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) struct v3d_csd_job *job = to_csd_job(sched_job); struct v3d_dev *v3d = job->base.v3d; struct drm_device *dev = &v3d->drm; - struct dma_fence *fence; + struct dma_fence *fence = NULL; int i, csd_cfg0_reg; - if (unlikely(job->base.base.s_fence->finished.error)) { - v3d->queue[V3D_CSD].active_job = NULL; - return NULL; - } + if (unlikely(job->base.base.s_fence->finished.error)) + goto out_clean_job; /* The HW interprets a workgroup size of 0 as 65536; however, the * user-space driver exposes a maximum of 65535. Therefore, a 0 in @@ -393,7 +397,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) fence = v3d_fence_create(v3d, V3D_CSD); if (IS_ERR(fence)) - return NULL; + goto out_clean_job; if (job->base.irq_fence) dma_fence_put(job->base.irq_fence); @@ -420,6 +424,10 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) V3D_CORE_WRITE(0, csd_cfg0_reg, job->args.cfg[0]); return fence; + +out_clean_job: + v3d->queue[V3D_CSD].active_job = NULL; + return fence; } static void @@ -868,6 +876,7 @@ v3d_queue_sched_init(struct v3d_dev *v3d, const struct drm_sched_backend_ops *op .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 1, .timeout = msecs_to_jiffies(500), + .timeout_wq = v3d->reset_wq, .dev = v3d->drm.dev, }; @@ -882,9 +891,13 @@ v3d_sched_init(struct v3d_dev *v3d) { int ret; + v3d->reset_wq = alloc_ordered_workqueue("v3d_reset", 0); + if (!v3d->reset_wq) + return -ENOMEM; + ret = v3d_queue_sched_init(v3d, &v3d_bin_sched_ops, V3D_BIN, "v3d_bin"); if (ret) - return ret; + goto fail; ret = v3d_queue_sched_init(v3d, &v3d_render_sched_ops, V3D_RENDER, "v3d_render"); @@ -927,4 +940,6 @@ v3d_sched_fini(struct v3d_dev *v3d) if (v3d->queue[q].sched.ready) drm_sched_fini(&v3d->queue[q].sched); } + + destroy_workqueue(v3d->reset_wq); } diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c index a1efda9c39f92..cd7996592db37 100644 --- a/drivers/gpu/drm/vc4/vc4_bo.c +++ b/drivers/gpu/drm/vc4/vc4_bo.c @@ -733,9 +733,13 @@ static int vc4_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct { struct vc4_bo *bo = to_vc4_bo(obj); - if (bo->validated_shader && (vma->vm_flags & VM_WRITE)) { - DRM_DEBUG("mmapping of shader BOs for writing not allowed.\n"); - return -EINVAL; + if (bo->validated_shader) { + if (vma->vm_flags & VM_WRITE) { + DRM_DEBUG("mmapping of shader BOs for writing not allowed.\n"); + return -EINVAL; + } + + vm_flags_clear(vma, VM_MAYWRITE); } mutex_lock(&bo->madv_lock); @@ -1046,7 +1050,7 @@ static void vc4_bo_cache_destroy(struct drm_device *dev, void *unused) struct vc4_dev *vc4 = to_vc4_dev(dev); int i; - timer_delete(&vc4->bo_cache.time_timer); + timer_shutdown_sync(&vc4->bo_cache.time_timer); cancel_work_sync(&vc4->bo_cache.time_work); vc4_bo_cache_purge(dev); diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c index 5833a6997fd66..7e00e13a0de10 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi.c +++ b/drivers/gpu/drm/vc4/vc4_hdmi.c @@ -654,7 +654,7 @@ static int vc4_hdmi_write_infoframe(struct drm_connector *connector, if (!drm_dev_enter(drm, &idx)) return 0; - if (len > sizeof(buffer)) { + if (len > VC4_HDMI_PACKET_SIZE) { ret = -ENOMEM; goto out; } @@ -680,6 +680,9 @@ static int vc4_hdmi_write_infoframe(struct drm_connector *connector, base + packet_reg); packet_reg += 4; + if (packet_reg >= packet_reg_next) + break; + writel(buffer[i + 3] << 0 | buffer[i + 4] << 8 | buffer[i + 5] << 16 | @@ -2164,8 +2167,17 @@ static int vc4_hdmi_audio_cpu_dai_probe(struct snd_soc_dai *dai) return 0; } +static int vc4_hdmi_audio_cpu_dai_startup(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) +{ + /* hdmi-codec has no channel allocation for odd channel counts */ + return snd_pcm_hw_constraint_step(substream->runtime, 0, + SNDRV_PCM_HW_PARAM_CHANNELS, 2); +} + static const struct snd_soc_dai_ops vc4_snd_dai_ops = { .probe = vc4_hdmi_audio_cpu_dai_probe, + .startup = vc4_hdmi_audio_cpu_dai_startup, }; static struct snd_soc_dai_driver vc4_hdmi_audio_cpu_dai_drv = { diff --git a/drivers/gpu/drm/vc4/vc4_hdmi_regs.h b/drivers/gpu/drm/vc4/vc4_hdmi_regs.h index 59bfd69f54d98..fb97e8a72b59f 100644 --- a/drivers/gpu/drm/vc4/vc4_hdmi_regs.h +++ b/drivers/gpu/drm/vc4/vc4_hdmi_regs.h @@ -7,6 +7,12 @@ #define VC4_HDMI_PACKET_STRIDE 0x24 +/* + * A packet RAM slot is 9 words, holding a 3-byte packet header followed by + * four 7-byte subpackets, so only 31 of its 36 bytes are addressable. + */ +#define VC4_HDMI_PACKET_SIZE 31 + enum vc4_hdmi_regs { VC4_INVALID = 0, VC4_HDMI, diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c index aa51aba723a38..97e22d6c226de 100644 --- a/drivers/gpu/drm/vc4/vc4_hvs.c +++ b/drivers/gpu/drm/vc4/vc4_hvs.c @@ -409,30 +409,53 @@ static int vc4_hvs_debugfs_lbm_allocs(struct seq_file *m, void *data) (((c1) & 0x1ff) << 9) | \ (((c2) & 0x1ff) << 18)) -/* The whole filter kernel is arranged as the coefficients 0-16 going +/* The PPF is a 4-tap filter with 64 phases, of which only 8 key phases + * (at 0, 1/8, ..., 7/8 of a source pixel) are stored; the hardware + * linearly interpolates the rest. The 32 coefficients are indexed + * tap * 8 + phase, taps ordered +2, +1, 0, -1 from the sample point. + * + * The whole filter kernel is arranged as the coefficients 0-16 going * up, then a pad, then 17-31 going down and reversed within the - * dwords. This means that a linear phase kernel (where it's - * symmetrical at the boundary between 15 and 16) has the last 5 - * dwords matching the first 5, but reversed. + * dwords. */ -#define VC4_LINEAR_PHASE_KERNEL(c0, c1, c2, c3, c4, c5, c6, c7, c8, \ - c9, c10, c11, c12, c13, c14, c15) \ +#define VC4_KERNEL(c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, \ + c12, c13, c14, c15, c16, c17, c18, c19, c20, c21, \ + c22, c23, c24, c25, c26, c27, c28, c29, c30, c31) \ {VC4_PPF_FILTER_WORD(c0, c1, c2), \ VC4_PPF_FILTER_WORD(c3, c4, c5), \ VC4_PPF_FILTER_WORD(c6, c7, c8), \ VC4_PPF_FILTER_WORD(c9, c10, c11), \ VC4_PPF_FILTER_WORD(c12, c13, c14), \ - VC4_PPF_FILTER_WORD(c15, c15, 0)} + VC4_PPF_FILTER_WORD(c15, c16, 0), \ + VC4_PPF_FILTER_WORD(c19, c18, c17), \ + VC4_PPF_FILTER_WORD(c22, c21, c20), \ + VC4_PPF_FILTER_WORD(c25, c24, c23), \ + VC4_PPF_FILTER_WORD(c28, c27, c26), \ + VC4_PPF_FILTER_WORD(c31, c30, c29)} + +/* A linear phase kernel is symmetrical at the boundary between 15 and + * 16, so only the first half need be given. + */ +#define VC4_LINEAR_PHASE_KERNEL(c0, c1, c2, c3, c4, c5, c6, c7, c8, \ + c9, c10, c11, c12, c13, c14, c15) \ + VC4_KERNEL(c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, \ + c12, c13, c14, c15, c15, c14, c13, c12, c11, c10, \ + c9, c8, c7, c6, c5, c4, c3, c2, c1, c0) -#define VC4_LINEAR_PHASE_KERNEL_DWORDS 6 -#define VC4_KERNEL_DWORDS (VC4_LINEAR_PHASE_KERNEL_DWORDS * 2 - 1) +#define VC4_KERNEL_DWORDS 11 /* Recommended B=1/3, C=1/3 filter choice from Mitchell/Netravali. * http://www.cs.utexas.edu/~fussell/courses/cs384g/lectures/mitchell/Mitchell.pdf + * + * Tabulated at the key phases the hardware actually uses, so it is not + * linear phase: a symmetric table would have to put the key phases at + * (p + 1/2) / 8, half a key phase late. */ static const u32 mitchell_netravali_1_3_1_3_kernel[] = - VC4_LINEAR_PHASE_KERNEL(0, -2, -6, -8, -10, -8, -3, 2, 18, - 50, 82, 119, 155, 187, 213, 227); + VC4_KERNEL(0, -1, -4, -7, -9, -9, -6, 1, + 14, 36, 66, 101, 137, 171, 200, 220, + 228, 220, 200, 171, 137, 101, 66, 36, + 14, 1, -6, -9, -9, -7, -4, -1); static const u32 nearest_neighbour_kernel[] = VC4_LINEAR_PHASE_KERNEL(0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 255, 255, 255, 255); @@ -458,14 +481,8 @@ static int vc4_hvs_upload_linear_kernel(struct vc4_hvs *hvs, dst_kernel = hvs->dlist + space->start; - for (i = 0; i < VC4_KERNEL_DWORDS; i++) { - if (i < VC4_LINEAR_PHASE_KERNEL_DWORDS) - writel(kernel[i], &dst_kernel[i]); - else { - writel(kernel[VC4_KERNEL_DWORDS - i - 1], - &dst_kernel[i]); - } - } + for (i = 0; i < VC4_KERNEL_DWORDS; i++) + writel(kernel[i], &dst_kernel[i]); return 0; } @@ -2256,7 +2273,7 @@ static int vc4_hvs_bind(struct device *dev, struct device *master, void *data) static void vc4_hvs_unbind(struct device *dev, struct device *master, void *data) { - struct drm_device *drm = dev_get_drvdata(master); + struct drm_device *drm = data; struct vc4_dev *vc4 = to_vc4_dev(drm); struct vc4_hvs *hvs = vc4->hvs; struct drm_mm_node *node, *next; diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c index 69b399f3b8027..4fdad88e53e10 100644 --- a/drivers/gpu/drm/vc4/vc4_irq.c +++ b/drivers/gpu/drm/vc4/vc4_irq.c @@ -104,7 +104,7 @@ vc4_overflow_mem_work(struct work_struct *work) vc4->bin_alloc_overflow = BIT(bin_bo_slot); V3D_WRITE(V3D_BPOA, bo->base.dma_addr + bin_bo_slot * vc4->bin_alloc_size); - V3D_WRITE(V3D_BPOS, bo->base.base.size); + V3D_WRITE(V3D_BPOS, vc4->bin_alloc_size); V3D_WRITE(V3D_INTCTL, V3D_INT_OUTOMEM); V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM); spin_unlock_irqrestore(&vc4->job_lock, irqflags); diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c index faee4198f4da0..b8cdabf029fa2 100644 --- a/drivers/gpu/drm/vc4/vc4_plane.c +++ b/drivers/gpu/drm/vc4/vc4_plane.c @@ -688,36 +688,18 @@ static void vc4_write_tpz(struct vc4_plane_state *vc4_state, u32 src, u32 dst) #define PHASE_BITS 6 static void vc4_write_ppf(struct vc4_plane_state *vc4_state, u32 src, u32 dst, - u32 xy, int channel, int chroma_offset, - bool no_interpolate) + u32 xy, int channel, unsigned int subsample, + int chroma_offset, bool no_interpolate) { struct vc4_dev *vc4 = to_vc4_dev(vc4_state->base.plane->dev); + unsigned int sub_shift = subsample == 2 ? 1 : 0; u32 scale = src / dst; s32 offset, offset2; s32 phase; WARN_ON_ONCE(vc4->gen > VC4_GEN_6_D); - /* - * Start the phase at 1/2 pixel from the 1st pixel at src_x. - * 1/4 pixel for YUV, plus the offset for chroma siting. - */ - if (channel) { - /* - * The phase is relative to scale_src->x, so shift it for - * display list's x value - */ - offset = (xy & 0x1ffff) >> (16 - PHASE_BITS) >> 1; - offset -= chroma_offset >> (17 - PHASE_BITS); - offset += -(1 << PHASE_BITS >> 2); - } else { - /* - * The phase is relative to scale_src->x, so shift it for - * display list's x value - */ - offset = (xy & 0xffff) >> (16 - PHASE_BITS); - offset += -(1 << PHASE_BITS >> 1); - + if (!channel) { /* * This is a kludge to make sure the scaling factors are * consistent with YUV's luma scaling. We lose 1-bit precision @@ -726,6 +708,25 @@ static void vc4_write_ppf(struct vc4_plane_state *vc4_state, u32 src, u32 dst, scale &= ~1; } + /* + * Start the phase at 1/2 pixel from the 1st pixel at src_x, less the + * chroma siting offset. The phase is relative to scale_src->x, so + * shift it for the display list's x value. Everything is computed in + * luma pixels and then converted to this channel's pixels, so that a + * subsampled chroma channel lands on the same position as the luma. + */ + offset = (xy & ((0x10000 << sub_shift) - 1)) >> (16 - PHASE_BITS); + offset -= chroma_offset >> (16 - PHASE_BITS); + offset -= 1 << PHASE_BITS >> 1; + offset >>= sub_shift; + + /* + * Output pixel r samples the source at (r + 1/2) * scale - 1/2, so the + * phase the first output pixel starts at needs half a destination + * pixel's worth of source added to it. + */ + offset += (s32)(scale >> (17 - PHASE_BITS)); + /* * There may be a also small error introduced by precision of scale. * Add half of that as a compromise @@ -930,7 +931,13 @@ static void vc4_write_scaling_parameters(struct drm_plane_state *state, { struct vc4_dev *vc4 = to_vc4_dev(state->plane->dev); struct vc4_plane_state *vc4_state = to_vc4_plane_state(state); + const struct drm_format_info *info = state->fb->format; bool no_interpolate = state->scaling_filter == DRM_SCALING_FILTER_NEAREST_NEIGHBOR; + unsigned int hsub = channel ? info->hsub : 1; + unsigned int vsub = channel ? info->vsub : 1; + /* Chroma siting only has any meaning on a subsampled axis */ + int siting_h = hsub > 1 ? state->chroma_siting_h : 0; + int siting_v = vsub > 1 ? state->chroma_siting_v : 0; if (vc4_state->is_yuv444_unity) no_interpolate = 1; @@ -941,16 +948,14 @@ static void vc4_write_scaling_parameters(struct drm_plane_state *state, if (vc4_state->x_scaling[channel] == VC4_SCALING_PPF) { vc4_write_ppf(vc4_state, vc4_state->src_w[channel], vc4_state->crtc_w, vc4_state->src_x, channel, - state->chroma_siting_h, - no_interpolate); + hsub, siting_h, no_interpolate); } /* Ch0 V-PPF Words 0-1: Scaling Parameters, Context */ if (vc4_state->y_scaling[channel] == VC4_SCALING_PPF) { vc4_write_ppf(vc4_state, vc4_state->src_h[channel], vc4_state->crtc_h, vc4_state->src_y, channel, - state->chroma_siting_v, - no_interpolate); + vsub, siting_v, no_interpolate); vc4_dlist_write(vc4_state, 0xc0c0c0c0); } @@ -2838,7 +2843,8 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev, BIT(DRM_SCALING_FILTER_DEFAULT) | BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR)); - drm_plane_create_chroma_siting_properties(plane, 0, 0); + /* MPEG-2 / H.264 / HEVC 4:2:0 siting: H cosited, V interstitial */ + drm_plane_create_chroma_siting_properties(plane, 0, 0x8000); if (type == DRM_PLANE_TYPE_PRIMARY) drm_plane_create_zpos_immutable_property(plane, 0); diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c index e907f3d8bd950..0b162d3444822 100644 --- a/drivers/gpu/drm/vc4/vc4_v3d.c +++ b/drivers/gpu/drm/vc4/vc4_v3d.c @@ -494,7 +494,7 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data) static void vc4_v3d_unbind(struct device *dev, struct device *master, void *data) { - struct drm_device *drm = dev_get_drvdata(master); + struct drm_device *drm = data; struct vc4_dev *vc4 = to_vc4_dev(drm); vc4_irq_uninstall(drm); diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c index 1e7bdda556984..205d360a007a2 100644 --- a/drivers/gpu/drm/vc4/vc4_validate.c +++ b/drivers/gpu/drm/vc4/vc4_validate.c @@ -384,6 +384,23 @@ validate_tile_binning_config(VALIDATE_ARGS) return -EINVAL; } + /* The tile state data array is 48 bytes per tile, and we put it at + * the start of a BO containing both it and the tile alloc. + */ + tile_state_size = 48 * tile_count; + + /* Since the tile alloc array will follow us, align. */ + tile_state_size = roundup(tile_state_size, 4096); + + /* Reject configurations whose tile state would leave no room for + * the tile alloc pool that follows it in the slot. + */ + if (tile_state_size >= vc4->bin_alloc_size) { + DRM_DEBUG("Tile binning config of %dx%d too large\n", + exec->bin_tiles_x, exec->bin_tiles_y); + return -EINVAL; + } + bin_slot = vc4_v3d_get_bin_slot(vc4); if (bin_slot < 0) { if (bin_slot != -EINTR && bin_slot != -ERESTARTSYS) { @@ -399,13 +416,13 @@ validate_tile_binning_config(VALIDATE_ARGS) exec->bin_slots |= BIT(bin_slot); bin_addr = vc4->bin_bo->base.dma_addr + bin_slot * vc4->bin_alloc_size; - /* The tile state data array is 48 bytes per tile, and we put it at - * the start of a BO containing both it and the tile alloc. - */ - tile_state_size = 48 * tile_count; + exec->tile_alloc_offset = bin_addr + tile_state_size; - /* Since the tile alloc array will follow us, align. */ - exec->tile_alloc_offset = bin_addr + roundup(tile_state_size, 4096); + /* The TSDA area must be zeroed out before use, otherwise the PTB might + * consume a stale tile state. + */ + memset(vc4->bin_bo->base.vaddr + bin_slot * vc4->bin_alloc_size, 0, + tile_state_size); *(uint8_t *)(validated + 14) = ((flags & ~(VC4_BIN_CONFIG_ALLOC_INIT_BLOCK_SIZE_MASK | diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c index c3315935d8bc4..e265a06f2c06a 100644 --- a/drivers/gpu/drm/virtio/virtgpu_display.c +++ b/drivers/gpu/drm/virtio/virtgpu_display.c @@ -319,7 +319,7 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev, if (ret) { kfree(virtio_gpu_fb); drm_gem_object_put(obj); - return NULL; + return ERR_PTR(ret); } return &virtio_gpu_fb->base; diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h index 2f3531950aa4d..00c609c3b53a8 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.h +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h @@ -43,6 +43,8 @@ #include #include +#include + #define DRIVER_NAME "virtio_gpu" #define DRIVER_DESC "virtio GPU" @@ -60,6 +62,24 @@ /* See virtio_gpu_ctx_create. One additional character for NULL terminator. */ #define DEBUG_NAME_MAX_LEN 65 +/* + * Whether the host must be told about resource backing pages by DMA address + * rather than guest-physical address. + * + * This mirrors vring_use_map_api() in drivers/virtio/virtio_ring.c, including + * its xen_domain() case. + */ +static inline bool virtio_gpu_use_dma_api(const struct virtio_device *vdev) +{ + if (!virtio_has_dma_quirk(vdev)) + return true; + + if (xen_domain()) + return true; + + return false; +} + struct virtio_gpu_object_params { unsigned long size; bool dumb; diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c index 015b5debd7451..7174086119c51 100644 --- a/drivers/gpu/drm/virtio/virtgpu_gem.c +++ b/drivers/gpu/drm/virtio/virtgpu_gem.c @@ -139,13 +139,15 @@ void virtio_gpu_gem_object_close(struct drm_gem_object *obj, if (!vgdev->has_virgl_3d) return; - objs = virtio_gpu_array_alloc(1); - if (!objs) - return; - virtio_gpu_array_add_obj(objs, obj); + if (vfpriv->context_created) { + objs = virtio_gpu_array_alloc(1); + if (!objs) + return; + virtio_gpu_array_add_obj(objs, obj); - virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id, - objs); + virtio_gpu_cmd_context_detach_resource(vgdev, vfpriv->ctx_id, + objs); + } virtio_gpu_notify(vgdev); } diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c index 1c15cbf326b78..4a25347734fda 100644 --- a/drivers/gpu/drm/virtio/virtgpu_kms.c +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c @@ -48,7 +48,10 @@ static void virtio_gpu_config_changed_work_func(struct work_struct *work) virtio_gpu_cmd_get_edids(vgdev); virtio_gpu_cmd_get_display_info(vgdev); virtio_gpu_notify(vgdev); - drm_helper_hpd_irq_event(vgdev->ddev); + wait_event_timeout(vgdev->resp_wq, + !vgdev->display_info_pending, + 5 * HZ); + drm_kms_helper_hotplug_event(vgdev->ddev); } events_clear |= VIRTIO_GPU_EVENT_DISPLAY; } diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c index e6363c887500a..d228b5be375a3 100644 --- a/drivers/gpu/drm/virtio/virtgpu_object.c +++ b/drivers/gpu/drm/virtio/virtgpu_object.c @@ -161,7 +161,7 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev, struct virtio_gpu_mem_entry **ents, unsigned int *nents) { - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); struct scatterlist *sg; struct sg_table *pages; int si; diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c index 8181b22b9b46a..4a72c69fe28ec 100644 --- a/drivers/gpu/drm/virtio/virtgpu_vq.c +++ b/drivers/gpu/drm/virtio/virtgpu_vq.c @@ -723,7 +723,7 @@ int virtio_gpu_panic_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_to_host_2d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, @@ -754,7 +754,7 @@ void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_to_host_2d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, @@ -839,9 +839,6 @@ static void virtio_gpu_cmd_get_display_info_cb(struct virtio_gpu_device *vgdev, vgdev->display_info_pending = false; spin_unlock(&vgdev->display_info_lock); wake_up(&vgdev->resp_wq); - - if (!drm_helper_hpd_irq_event(vgdev->ddev)) - drm_kms_helper_hotplug_event(vgdev->ddev); } static void virtio_gpu_cmd_get_capset_info_cb(struct virtio_gpu_device *vgdev, @@ -896,7 +893,8 @@ static int virtio_get_edid_block(void *data, u8 *buf, struct virtio_gpu_resp_edid *resp = data; size_t start = block * EDID_LENGTH; - if (start + len > le32_to_cpu(resp->size)) + if (start + len > le32_to_cpu(resp->size) || + start + len > sizeof(resp->edid)) return -EINVAL; memcpy(buf, resp->edid + start, len); return 0; @@ -1192,7 +1190,7 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev, struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(objs->objs[0]); struct virtio_gpu_transfer_host_3d *cmd_p; struct virtio_gpu_vbuffer *vbuf; - bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev); + bool use_dma_api = virtio_gpu_use_dma_api(vgdev->vdev); if (virtio_gpu_is_shmem(bo) && use_dma_api) dma_sync_sgtable_for_device(vgdev->vdev->dev.parent, diff --git a/drivers/gpu/drm/vmwgfx/ttm_object.c b/drivers/gpu/drm/vmwgfx/ttm_object.c index 36d46b79562a3..95f7c5820aab8 100644 --- a/drivers/gpu/drm/vmwgfx/ttm_object.c +++ b/drivers/gpu/drm/vmwgfx/ttm_object.c @@ -547,14 +547,17 @@ int ttm_prime_fd_to_handle(struct ttm_object_file *tfile, if (IS_ERR(dma_buf)) return PTR_ERR(dma_buf); - if (dma_buf->ops != &tdev->ops) - return -ENOSYS; + if (dma_buf->ops != &tdev->ops) { + ret = -ENOSYS; + goto out; + } prime = (struct ttm_prime_object *) dma_buf->priv; base = &prime->base; *handle = base->handle; ret = ttm_ref_object_add(tfile, base, NULL, false); +out: dma_buf_put(dma_buf); return ret; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c index fa5841fda659d..d78874e7f5c76 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c @@ -30,6 +30,7 @@ #include "vmwgfx_bo.h" #include +#include /* * Template that implements find_first_diff() for a generic @@ -463,19 +464,42 @@ static int vmw_external_bo_copy(struct vmw_bo *dst, u32 dst_offset, container_of(dst->tbo.bdev, struct vmw_private, bdev); size_t dst_size = dst->tbo.resource->size; size_t src_size = src->tbo.resource->size; + size_t dst_end, src_end; struct iosys_map dst_map = {0}; struct iosys_map src_map = {0}; + bool dst_mapped = false; + bool src_mapped = false; int ret, i; int x_in_bytes; u8 *vsrc; u8 *vdst; + if (!height || !width_in_bytes) + return 0; + + if (!dst_stride || !src_stride) + return -EINVAL; + if (dst_stride < width_in_bytes || src_stride < width_in_bytes) + return -EINVAL; + if (check_mul_overflow((size_t)dst_stride, (size_t)height - 1, &dst_end) || + check_add_overflow(dst_end, (size_t)width_in_bytes, &dst_end) || + check_add_overflow((size_t)dst_offset, dst_end, &dst_end) || + dst_end > dst_size || + check_mul_overflow((size_t)src_stride, (size_t)height - 1, &src_end) || + check_add_overflow(src_end, (size_t)width_in_bytes, &src_end) || + check_add_overflow((size_t)src_offset, src_end, &src_end) || + src_end > src_size) { + drm_dbg_driver(&vmw->drm, "Out-of-bounds external BO copy\n"); + return -EINVAL; + } + vsrc = map_external(src, &src_map); if (!vsrc) { drm_dbg_driver(&vmw->drm, "Wasn't able to map src\n"); ret = -ENOMEM; goto out; } + src_mapped = true; vdst = map_external(dst, &dst_map); if (!vdst) { @@ -483,16 +507,13 @@ static int vmw_external_bo_copy(struct vmw_bo *dst, u32 dst_offset, ret = -ENOMEM; goto out; } + dst_mapped = true; vsrc += src_offset; vdst += dst_offset; - if (src_stride == dst_stride) { - dst_size -= dst_offset; - src_size -= src_offset; - memcpy(vdst, vsrc, - min(dst_stride * height, min(dst_size, src_size))); + if (src_stride == dst_stride && width_in_bytes == dst_stride) { + memcpy(vdst, vsrc, dst_stride * (size_t)height); } else { - WARN_ON(dst_stride < width_in_bytes); for (i = 0; i < height; ++i) { memcpy(vdst, vsrc, width_in_bytes); vsrc += src_stride; @@ -508,8 +529,10 @@ static int vmw_external_bo_copy(struct vmw_bo *dst, u32 dst_offset, ret = 0; out: - unmap_external(src, &src_map); - unmap_external(dst, &dst_map); + if (src_mapped) + unmap_external(src, &src_map); + if (dst_mapped) + unmap_external(dst, &dst_map); return ret; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c index c46f17ba7236d..c53bb9376b366 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c @@ -432,6 +432,7 @@ vmw_cursor_mob_map(struct vmw_plane_state *vps) u32 size = vmw_cursor_mob_size(vps->cursor.update_type, vps->base.crtc_w, vps->base.crtc_h); struct vmw_bo *vbo = vps->cursor.mob; + void *map; if (!vbo) return -EINVAL; @@ -446,11 +447,15 @@ vmw_cursor_mob_map(struct vmw_plane_state *vps) if (unlikely(ret != 0)) return -ENOMEM; - vmw_bo_map_and_cache(vbo); + map = vmw_bo_map_and_cache(vbo); + if (!map) { + vmw_bo_unmap(vbo); + ret = -ENOMEM; + } ttm_bo_unreserve(&vbo->tbo); - return 0; + return ret; } /** @@ -663,9 +668,15 @@ int vmw_cursor_plane_prepare_fb(struct drm_plane *plane, !vmw_cursor_buffer_changed(vps, old_vps)) { vps->cursor.update_type = VMW_CURSOR_UPDATE_NONE; - } else { - vmw_cursor_mob_get(vcp, vps); - vmw_cursor_mob_map(vps); + } else if (vps->cursor.update_type == + VMW_CURSOR_UPDATE_MOB && + (vmw_cursor_mob_get(vcp, vps) || + vmw_cursor_mob_map(vps))) { + /* + * Reset the cursor to avoid crashes later. + */ + vps->cursor.update_type = + VMW_CURSOR_UPDATE_NONE; } } } @@ -732,6 +743,34 @@ int vmw_cursor_plane_atomic_check(struct drm_plane *plane, "surface not suitable for cursor\n"); return -EINVAL; } + } else if (update_type == VMW_CURSOR_UPDATE_GB_ONLY || + update_type == VMW_CURSOR_UPDATE_MOB) { + u32 cursor_max_dim = + vmw_read(vmw, SVGA_REG_CURSOR_MAX_DIMENSION); + + if (new_state->crtc_w > cursor_max_dim || + new_state->crtc_h > cursor_max_dim) { + drm_warn(&vmw->drm, + "Cursor dimensions (%d, %d) exceed device max %u\n", + new_state->crtc_w, new_state->crtc_h, + cursor_max_dim); + return -EINVAL; + } + + if (update_type == VMW_CURSOR_UPDATE_MOB) { + u32 mob_max_size = + vmw_read(vmw, SVGA_REG_MOB_MAX_SIZE); + u64 mob_size = (u64)new_state->crtc_w * + new_state->crtc_h * sizeof(u32) + + sizeof(SVGAGBCursorHeader); + + if (mob_size > mob_max_size) { + drm_warn(&vmw->drm, + "Cursor MOB size %llu exceeds device max %u\n", + mob_size, mob_max_size); + return -EINVAL; + } + } } return 0; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index e1f18020170ab..a9136a6523cba 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -1272,9 +1272,13 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv, SVGA3dCmdHeader *header) { VMW_DECLARE_CMD_VAR(*cmd, SVGA3dCmdDXBindQuery); + struct vmw_ctx_validation_info *ctx_node = VMW_GET_CTX_NODE(sw_context); struct vmw_bo *vmw_bo; int ret; + if (!ctx_node) + return -EINVAL; + cmd = container_of(header, typeof(*cmd), header); /* @@ -1288,7 +1292,7 @@ static int vmw_cmd_dx_bind_query(struct vmw_private *dev_priv, return ret; sw_context->dx_query_mob = vmw_bo; - sw_context->dx_query_ctx = sw_context->dx_ctx_node->ctx; + sw_context->dx_query_ctx = ctx_node->ctx; return 0; } @@ -1506,6 +1510,12 @@ static int vmw_cmd_dma(struct vmw_private *dev_priv, bool dirty; cmd = container_of(header, typeof(*cmd), header); + + if (unlikely(header->size < sizeof(cmd->body) + sizeof(*suffix))) { + VMW_DEBUG_USER("Illegal SVGA_3D_CMD_SURFACE_DMA size.\n"); + return -EINVAL; + } + suffix = (SVGA3dCmdSurfaceDMASuffix *)((unsigned long) &cmd->body + header->size - sizeof(*suffix)); @@ -1567,11 +1577,17 @@ static int vmw_cmd_draw(struct vmw_private *dev_priv, uint32_t maxnum; int ret; + cmd = container_of(header, typeof(*cmd), header); + + if (unlikely(header->size < sizeof(cmd->body))) { + VMW_DEBUG_USER("Illegal DRAW_PRIMITIVES header size.\n"); + return -EINVAL; + } + ret = vmw_cmd_cid_check(dev_priv, sw_context, header); if (unlikely(ret != 0)) return ret; - cmd = container_of(header, typeof(*cmd), header); maxnum = (header->size - sizeof(cmd->body)) / sizeof(*decl); if (unlikely(cmd->body.numVertexDecls > maxnum)) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c index 00be92da55097..fe59a613e1d05 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c @@ -366,13 +366,24 @@ void vmw_fence_fifo_down(struct vmw_fence_manager *fman) ret = vmw_fence_obj_wait(fence, false, false, VMW_FENCE_WAIT_TIMEOUT); + spin_lock(&fman->lock); if (unlikely(ret != 0)) { + bool cookie = dma_fence_begin_signalling(); + list_del_init(&fence->head); - dma_fence_signal(&fence->base); + if (fence->waiter_added) { + vmw_seqno_waiter_remove(fman->dev_priv); + fence->waiter_added = false; + } + dma_fence_signal_locked(&fence->base); + dma_fence_end_signalling(cookie); } BUG_ON(!list_empty(&fence->head)); + spin_unlock(&fman->lock); + dma_fence_put(&fence->base); + spin_lock(&fman->lock); } spin_unlock(&fman->lock); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index 3e8a2f4a907da..35c7277521a90 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -553,6 +553,9 @@ static int vmw_kms_new_framebuffer_surface(struct vmw_private *dev_priv, memcpy(&vfbs->uo, uo, sizeof(vfbs->uo)); vmw_user_object_ref(&vfbs->uo); + if (vfbs->uo.buffer) + vfbs->base.base.obj[0] = &vfbs->uo.buffer->tbo.base; + *out = &vfbs->base; ret = drm_framebuffer_init(dev, &vfbs->base.base, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c index 45561bc1c9eff..8ab88f3886525 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c @@ -311,7 +311,7 @@ void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res) return; cur = max(res_start, dirty->start); - res_end = max(res_end, dirty->end); + res_end = min(res_end, dirty->end); while (cur < res_end) { unsigned long num; @@ -347,7 +347,7 @@ void vmw_bo_dirty_clear(struct vmw_bo *vbo) return; cur = max(res_start, dirty->start); - res_end = max(res_end, dirty->end); + res_end = min(res_end, dirty->end); while (cur < res_end) { unsigned long num; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c index 3880116969417..e3a187a2c7a1c 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c @@ -136,7 +136,7 @@ static void vmw_resource_release(struct kref *kref) val_buf.num_shared = 0; res->func->unbind(res, false, &val_buf); } - res->guest_memory_size = false; + res->guest_memory_dirty = false; vmw_resource_mob_detach(res); if (res->dirty) res->func->dirty_free(res); @@ -773,7 +773,7 @@ void vmw_resource_unbind_list(struct vmw_bo *vbo) if (!WARN_ON_ONCE(!res->func->unbind)) (void) res->func->unbind(res, res->res_dirty, &val_buf); - res->guest_memory_size = true; + res->guest_memory_dirty = true; res->res_dirty = false; vmw_resource_mob_detach(res); } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c index a8c8c9375d297..bc89a137ab6d9 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c @@ -25,6 +25,8 @@ * **************************************************************************/ +#include + #include #include "vmwgfx_binding.h" @@ -685,7 +687,7 @@ int vmw_shader_destroy_ioctl(struct drm_device *dev, void *data, static int vmw_user_shader_alloc(struct vmw_private *dev_priv, struct vmw_bo *buffer, size_t shader_size, - size_t offset, + u64 offset, SVGA3dShaderType shader_type, uint8_t num_input_sig, uint8_t num_output_sig, @@ -739,7 +741,7 @@ static int vmw_user_shader_alloc(struct vmw_private *dev_priv, static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv, struct vmw_bo *buffer, size_t shader_size, - size_t offset, + u64 offset, SVGA3dShaderType shader_type) { struct vmw_shader *shader; @@ -768,7 +770,7 @@ static struct vmw_resource *vmw_shader_alloc(struct vmw_private *dev_priv, static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv, enum drm_vmw_shader_type shader_type_drm, - u32 buffer_handle, size_t size, size_t offset, + u32 buffer_handle, size_t size, u64 offset, uint8_t num_input_sig, uint8_t num_output_sig, uint32_t *shader_handle) { @@ -779,13 +781,16 @@ static int vmw_shader_define(struct drm_device *dev, struct drm_file *file_priv, int ret; if (buffer_handle != SVGA3D_INVALID_ID) { + u64 end; + ret = vmw_user_bo_lookup(file_priv, buffer_handle, &buffer); if (unlikely(ret != 0)) { VMW_DEBUG_USER("Couldn't find buffer for shader creation.\n"); return ret; } - if ((u64)buffer->tbo.base.size < (u64)size + (u64)offset) { + if (check_add_overflow((u64)size, (u64)offset, &end) || + end > buffer->tbo.base.size) { VMW_DEBUG_USER("Illegal buffer- or shader size.\n"); ret = -EINVAL; goto out_bad_arg; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c index 7e281c3c6bc58..8e01e13eebf4d 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c @@ -77,7 +77,7 @@ static int vmw_gb_surface_unbind(struct vmw_resource *res, static int vmw_gb_surface_destroy(struct vmw_resource *res); static int vmw_gb_surface_define_internal(struct drm_device *dev, - struct drm_vmw_gb_surface_create_ext_req *req, + const struct drm_vmw_gb_surface_create_ext_req *req, struct drm_vmw_gb_surface_create_rep *rep, struct drm_file *file_priv); static int @@ -1503,7 +1503,7 @@ int vmw_gb_surface_reference_ext_ioctl(struct drm_device *dev, void *data, */ static int vmw_gb_surface_define_internal(struct drm_device *dev, - struct drm_vmw_gb_surface_create_ext_req *req, + const struct drm_vmw_gb_surface_create_ext_req *req, struct drm_vmw_gb_surface_create_rep *rep, struct drm_file *file_priv) { @@ -1521,9 +1521,21 @@ vmw_gb_surface_define_internal(struct drm_device *dev, req->base.svga3d_flags); /* array_size must be null for non-GL3 host. */ - if (req->base.array_size > 0 && !has_sm4_context(dev_priv)) { - VMW_DEBUG_USER("SM4 surface not supported.\n"); - return -EINVAL; + if (req->base.array_size > 0) { + if (has_sm5_context(dev_priv)) { + if (req->base.array_size > SVGA3D_SM5_MAX_SURFACE_ARRAYSIZE) { + VMW_DEBUG_USER("Invalid Surface Array Size.\n"); + return -EINVAL; + } + } else if (has_sm4_context(dev_priv)) { + if (req->base.array_size > SVGA3D_SM4_MAX_SURFACE_ARRAYSIZE) { + VMW_DEBUG_USER("Invalid Surface Array Size.\n"); + return -EINVAL; + } + } else { + VMW_DEBUG_USER("SM4+ surface not supported.\n"); + return -EINVAL; + } } if (!has_sm4_1_context(dev_priv)) { diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c index aec774fa4d7bf..87de8e43667db 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c @@ -214,14 +214,14 @@ vmw_vkms_init(struct vmw_private *vmw) vmw->vkms_enabled = false; ret = vmw_host_get_guestinfo(GUESTINFO_VBLANK, buffer, &buf_len); - if (ret || buf_len > max_buf_len) - return; - buffer[buf_len] = '\0'; + if (!ret && buf_len <= max_buf_len) { + buffer[buf_len] = '\0'; - ret = kstrtobool(buffer, &vmw->vkms_enabled); - if (!ret && vmw->vkms_enabled) { - ret = drm_vblank_init(&vmw->drm, VMWGFX_NUM_DISPLAY_UNITS); - vmw->vkms_enabled = (ret == 0); + ret = kstrtobool(buffer, &vmw->vkms_enabled); + if (!ret && vmw->vkms_enabled) { + ret = drm_vblank_init(&vmw->drm, VMWGFX_NUM_DISPLAY_UNITS); + vmw->vkms_enabled = (ret == 0); + } } vmw->crc_workq = alloc_ordered_workqueue("vmwgfx_crc_generator", 0); @@ -236,7 +236,8 @@ vmw_vkms_init(struct vmw_private *vmw) void vmw_vkms_cleanup(struct vmw_private *vmw) { - destroy_workqueue(vmw->crc_workq); + if (vmw->crc_workq) + destroy_workqueue(vmw->crc_workq); } bool diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index ee52b751623f0..c5fc8cc71149c 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -94,6 +94,7 @@ xe-y += xe_bb.o \ xe_nvm.o \ xe_oa.o \ xe_observation.o \ + xe_pagefault.o \ xe_pat.o \ xe_pci.o \ xe_pcode.o \ diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c index b18d15cc3c53d..5bf00505dab59 100644 --- a/drivers/gpu/drm/xe/display/xe_fb_pin.c +++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c @@ -101,30 +101,14 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb, dpt_size = ALIGN(intel_rotation_info_size(&view->rotated) * 8, XE_PAGE_SIZE); - if (IS_DGFX(xe)) - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_VRAM0 | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE, - alignment, false); - else - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_STOLEN | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE, - alignment, false); - if (IS_ERR(dpt)) - dpt = xe_bo_create_pin_map_at_novm(xe, tile0, - dpt_size, ~0ull, - ttm_bo_type_kernel, - XE_BO_FLAG_SYSTEM | - XE_BO_FLAG_GGTT | - XE_BO_FLAG_PAGETABLE, - alignment, false); + dpt = xe_bo_create_pin_map_at_novm(xe, tile0, + dpt_size, ~0ull, + ttm_bo_type_kernel, + XE_BO_FLAG_VRAM_IF_DGFX(tile0) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_PAGETABLE | + XE_BO_FLAG_SCANOUT, + alignment, false); if (IS_ERR(dpt)) return PTR_ERR(dpt); diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h index 8cfcd3360896c..8d05db3fb356d 100644 --- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h +++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h @@ -41,6 +41,7 @@ #define GFX_OP_PIPE_CONTROL(len) ((0x3<<29)|(0x3<<27)|(0x2<<24)|((len)-2)) +#define PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH BIT(11) /* gen12 */ #define PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE BIT(10) /* gen12 */ #define PIPE_CONTROL0_HDC_PIPELINE_FLUSH BIT(9) /* gen12 */ diff --git a/drivers/gpu/drm/xe/regs/xe_i2c_regs.h b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h index af781c8e4a804..88f416322c14a 100644 --- a/drivers/gpu/drm/xe/regs/xe_i2c_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_i2c_regs.h @@ -17,4 +17,6 @@ #define I2C_CONFIG_CMD XE_REG(I2C_CONFIG_SPACE_OFFSET + PCI_COMMAND) #define I2C_CONFIG_PMCSR XE_REG(I2C_CONFIG_SPACE_OFFSET + 0x84) +#define I2C_REG(reg) XE_REG((reg) + I2C_MEM_SPACE_OFFSET) + #endif /* _XE_I2C_REGS_H_ */ diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 5904d658d1f29..e056fac1a86b9 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -197,8 +197,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test, err = xe_bo_vmap(bo); if (err) { - KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", - PTR_ERR(bo)); + KUNIT_FAIL(test, "Failed to vmap our pagetables: %d\n", err); return; } diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 0dabe85351392..5c531b8d92111 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -930,6 +930,21 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, xe_pm_runtime_get_noresume(xe); } + /* + * Attach CCS BBs before submitting the copy job below so a VF + * migration racing the copy sees valid, up to date attach state. + */ + if (IS_VF_CCS_READY(xe) && + ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || + (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && + handle_system_ccs) { + ret = xe_sriov_vf_ccs_attach_bo(bo, new_mem); + if (ret) { + xe_pm_runtime_put(xe); + goto out; + } + } + if (move_lacks_source) { u32 flags = 0; @@ -967,22 +982,19 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, ttm_bo_move_null(ttm_bo, new_mem); } - dma_fence_put(fence); - xe_pm_runtime_put(xe); - /* - * CCS meta data is migrated from TT -> SMEM. So, let us detach the - * BBs from BO as it is no longer needed. + * Detach must wait for the copy above to complete: a VF migration + * racing an in-flight copy must still see valid CCS BBs, so don't + * tear them down until the copy fence has signaled. */ if (IS_VF_CCS_READY(xe) && old_mem_type == XE_PL_TT && - new_mem->mem_type == XE_PL_SYSTEM) + new_mem->mem_type == XE_PL_SYSTEM) { + dma_fence_wait(fence, false); xe_sriov_vf_ccs_detach_bo(bo); + } - if (IS_VF_CCS_READY(xe) && - ((move_lacks_source && new_mem->mem_type == XE_PL_TT) || - (old_mem_type == XE_PL_SYSTEM && new_mem->mem_type == XE_PL_TT)) && - handle_system_ccs) - ret = xe_sriov_vf_ccs_attach_bo(bo); + dma_fence_put(fence); + xe_pm_runtime_put(xe); out: if ((!ttm_bo->resource || ttm_bo->resource->mem_type == XE_PL_SYSTEM) && @@ -1174,7 +1186,7 @@ int xe_bo_notifier_prepare_pinned(struct xe_bo *bo) backup = xe_bo_init_locked(xe, NULL, NULL, bo->ttm.base.resv, NULL, xe_bo_size(bo), DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | - XE_BO_FLAG_PINNED, &exec); + XE_BO_FLAG_PINNED, NULL, &exec); if (IS_ERR(backup)) { drm_exec_retry_on_contention(&exec); ret = PTR_ERR(backup); @@ -1315,7 +1327,7 @@ int xe_bo_evict_pinned(struct xe_bo *bo) xe_bo_size(bo), DRM_XE_GEM_CPU_CACHING_WB, ttm_bo_type_kernel, XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS | - XE_BO_FLAG_PINNED, &exec); + XE_BO_FLAG_PINNED, NULL, &exec); if (IS_ERR(backup)) { drm_exec_retry_on_contention(&exec); ret = PTR_ERR(backup); @@ -1663,6 +1675,8 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) if (bo->ttm.base.import_attach) drm_prime_gem_destroy(&bo->ttm.base, NULL); + if (bo->dma_buf) + dma_buf_put(bo->dma_buf); drm_gem_object_release(&bo->ttm.base); xe_assert(xe, list_empty(&ttm_bo->base.gpuva.list)); @@ -2080,6 +2094,8 @@ void xe_bo_free(struct xe_bo *bo) * @cpu_caching: The cpu caching used for system memory backing store. * @type: The TTM buffer object type. * @flags: XE_BO_FLAG_ flags. + * @dma_buf: The dma-buf to reference for the BO lifetime (imported BOs), + * or NULL. * @exec: The drm_exec transaction to use for exhaustive eviction. * * Initialize or create an xe buffer object. On failure, any allocated buffer @@ -2091,7 +2107,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, struct xe_tile *tile, struct dma_resv *resv, struct ttm_lru_bulk_move *bulk, size_t size, u16 cpu_caching, enum ttm_bo_type type, - u32 flags, struct drm_exec *exec) + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec) { struct ttm_operation_ctx ctx = { .interruptible = true, @@ -2177,6 +2194,17 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, placement = (type == ttm_bo_type_sg || bo->flags & XE_BO_FLAG_DEFER_BACKING) ? &sys_placement : &bo->placement; + + /* + * For imported BOs, keep the exporter dma-buf alive for the BO + * lifetime. Taken before ttm_bo_init_reserved() to also cover a + * creation failure there. Released in xe_ttm_bo_destroy(). + */ + if (dma_buf) { + get_dma_buf(dma_buf); + bo->dma_buf = dma_buf; + } + err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, placement, alignment, &ctx, NULL, resv, xe_ttm_bo_destroy); @@ -2291,7 +2319,7 @@ __xe_bo_create_locked(struct xe_device *xe, vm && !xe_vm_in_fault_mode(vm) && flags & XE_BO_FLAG_USER ? &vm->lru_bulk_move : NULL, size, - cpu_caching, type, flags, exec); + cpu_caching, type, flags, NULL, exec); if (IS_ERR(bo)) return bo; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index a77af42b5f9ea..71169b57fbda4 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -6,6 +6,7 @@ #ifndef _XE_BO_H_ #define _XE_BO_H_ +#include #include #include "xe_bo_types.h" @@ -84,6 +85,28 @@ #define XE_PCI_BARRIER_MMAP_OFFSET (0x50 << XE_PTE_SHIFT) +/** + * enum xe_madv_purgeable_state - Buffer object purgeable state enumeration + * + * This enum defines the possible purgeable states for a buffer object, + * allowing userspace to provide memory usage hints to the kernel for + * better memory management under pressure. + * + * @XE_MADV_PURGEABLE_WILLNEED: The buffer object is needed and should not be purged. + * This is the default state. + * @XE_MADV_PURGEABLE_DONTNEED: The buffer object is not currently needed and can be + * purged by the kernel under memory pressure. + * @XE_MADV_PURGEABLE_PURGED: The buffer object has been purged by the kernel. + * + * Accessing a purged buffer will result in an error. Per i915 semantics, + * once purged, a BO remains permanently invalid and must be destroyed and recreated. + */ +enum xe_madv_purgeable_state { + XE_MADV_PURGEABLE_WILLNEED, + XE_MADV_PURGEABLE_DONTNEED, + XE_MADV_PURGEABLE_PURGED, +}; + struct sg_table; struct xe_bo *xe_bo_alloc(void); @@ -93,7 +116,8 @@ struct xe_bo *xe_bo_init_locked(struct xe_device *xe, struct xe_bo *bo, struct xe_tile *tile, struct dma_resv *resv, struct ttm_lru_bulk_move *bulk, size_t size, u16 cpu_caching, enum ttm_bo_type type, - u32 flags, struct drm_exec *exec); + u32 flags, struct dma_buf *dma_buf, + struct drm_exec *exec); struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags, @@ -212,6 +236,40 @@ static inline bool xe_bo_is_protected(const struct xe_bo *bo) return bo->pxp_key_instance; } +/** + * xe_bo_is_purged() - Check if buffer object has been purged + * @bo: The buffer object to check + * + * Checks if the buffer object's backing store has been discarded by the + * kernel due to memory pressure after being marked as purgeable (DONTNEED). + * Once purged, the BO cannot be restored and any attempt to use it will fail. + * + * Context: Caller must hold the BO's dma-resv lock + * Return: true if the BO has been purged, false otherwise + */ +static inline bool xe_bo_is_purged(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + return bo->madv_purgeable == XE_MADV_PURGEABLE_PURGED; +} + +/** + * xe_bo_madv_is_dontneed() - Check if BO is marked as DONTNEED + * @bo: The buffer object to check + * + * Checks if userspace has marked this BO as DONTNEED (i.e., its contents + * are not currently needed and can be discarded under memory pressure). + * This is used internally to decide whether a BO is eligible for purging. + * + * Context: Caller must hold the BO's dma-resv lock + * Return: true if the BO is marked DONTNEED, false otherwise + */ +static inline bool xe_bo_madv_is_dontneed(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + return bo->madv_purgeable == XE_MADV_PURGEABLE_DONTNEED; +} + static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) { if (likely(bo)) { @@ -398,6 +456,19 @@ void xe_bo_dev_fini(struct xe_bo_dev *bo_device); struct sg_table *xe_bo_sg(struct xe_bo *bo); +/** + * xe_bo_sg_is_contiguous() - Check if a BO's DMA address space is contiguous. + * @bo: the BO to check (must have a valid sg table, i.e. !xe_bo_is_vram()) + * @len: required contiguous length in bytes + * + * Returns true if the first @len bytes of the BO are mapped to a contiguous + * DMA address range. + */ +static inline bool xe_bo_sg_is_contiguous(struct xe_bo *bo, size_t len) +{ + return drm_prime_get_contiguous_size(xe_bo_sg(bo)) >= len; +} + /* * xe_sg_segment_size() - Provides upper limit for sg segment size. * @dev: device pointer diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index d4fe3c8dca5be..efff5519fc588 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -35,6 +35,8 @@ struct xe_bo { struct xe_bo *backup_obj; /** @parent_obj: Ref to parent bo if this a backup_obj */ struct xe_bo *parent_obj; + /** @dma_buf: Imported dma-buf ref to keep its resv alive. */ + struct dma_buf *dma_buf; /** @flags: flags for this buffer object */ u32 flags; /** @vm: VM this BO is attached to, for extobj this will be NULL */ @@ -108,6 +110,12 @@ struct xe_bo { * from default */ u64 min_align; + + /** + * @madv_purgeable: user space advise on BO purgeability, protected + * by BO's dma-resv lock. + */ + u32 madv_purgeable; }; #endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 203e3038cc817..26a294f79e79f 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -150,7 +150,8 @@ static void xe_devcoredump_snapshot_free(struct xe_devcoredump_snapshot *ss) xe_guc_ct_snapshot_free(ss->guc.ct); ss->guc.ct = NULL; - xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); + if (!IS_ERR_OR_NULL(ss->gt)) + xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc); ss->matched_node = NULL; xe_guc_exec_queue_snapshot_free(ss->ge); @@ -255,7 +256,8 @@ static void xe_devcoredump_free(void *data) if (!data || !coredump_to_xe(coredump)) return; - cancel_work_sync(&coredump->snapshot.work); + if (coredump->captured) + cancel_work_sync(&coredump->snapshot.work); mutex_lock(&coredump->lock); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 0d69cd0e4e798..05a84ea6bd6ab 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -962,7 +962,11 @@ int xe_device_probe(struct xe_device *xe) if (err) goto err_unregister_display; - return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); + err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); + if (err) + goto err_unregister_display; + + return 0; err_unregister_display: xe_display_unregister(xe); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 0e80f2940c996..2ed03fd074ff5 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -305,6 +305,8 @@ struct xe_device { * pcode mailbox commands. */ u8 has_mbx_power_limits:1; + /** @info.has_page_reclaim_hw_assist: Device supports page reclamation feature */ + u8 has_page_reclaim_hw_assist:1; /** @info.has_pxp: Device has PXP support */ u8 has_pxp:1; /** @info.has_range_tlb_inval: Has range based TLB invalidations */ diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index eeea2a6e15b5c..07010f097e584 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -251,7 +251,7 @@ xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 0, /* Will require 1way or 2way for vm_bind */ - ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); + ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, dma_buf, &exec); drm_exec_retry_on_contention(&exec); if (IS_ERR(bo)) { ret = PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index 87a32b61bece6..c156cf4661f24 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -286,13 +286,23 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto err_exec; } - /* Wait behind rebinds */ + /* + * Wait behind rebinds and any kernel operations (evictions, defrag + * moves, ...) on the VM and all external BOs. The VM's private BOs + * carry their kernel ops in the VM dma-resv KERNEL slot, while each + * external BO carries them in its own dma-resv KERNEL slot; both are + * covered by iterating every object locked by the exec, mirroring the + * drm_gpuvm_resv_add_fence() below. + */ if (!xe_vm_in_lr_mode(vm)) { - err = xe_sched_job_add_deps(job, - xe_vm_resv(vm), - DMA_RESV_USAGE_KERNEL); - if (err) - goto err_put_job; + struct drm_gem_object *obj; + + drm_exec_for_each_locked_object(exec, obj) { + err = xe_sched_job_add_deps(job, obj->resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_put_job; + } } for (i = 0; i < num_syncs && !err; i++) diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index f253e2df49077..1a4d8d4f44b4d 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -174,8 +174,10 @@ static int register_save_restore(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "\n"); drm_printf(p, "Whitelist\n"); - for_each_hw_engine(hwe, gt, id) + for_each_hw_engine(hwe, gt, id) { xe_reg_whitelist_dump(&hwe->reg_whitelist, p); + xe_reg_whitelist_dump(&hwe->oa_whitelist, p); + } xe_pm_runtime_put(gt_to_xe(gt)); diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c index 3e3d1d52f6302..81e3ec7ff4fdd 100644 --- a/drivers/gpu/drm/xe/xe_gt_idle.c +++ b/drivers/gpu/drm/xe/xe_gt_idle.c @@ -236,7 +236,8 @@ int xe_gt_idle_pg_print(struct xe_gt *gt, struct drm_printer *p) xe_force_wake_put(gt_to_fw(gt), fw_ref); } - if (gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK) { + if (gt->info.engine_mask & + (XE_HW_ENGINE_RCS_MASK | XE_HW_ENGINE_CCS_MASK)) { drm_printf(p, "Render Power Gating Enabled: %s\n", str_yes_no(pg_enabled & RENDER_POWERGATE_ENABLE)); diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index db71823b25380..63017ae2c3818 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -65,10 +65,14 @@ ads_to_map(struct xe_guc_ads *ads) /* * The Additional Data Struct (ADS) has pointers for different buffers used by - * the GuC. One single gem object contains the ADS struct itself (guc_ads) and - * all the extra buffers indirectly linked via the ADS struct's entries. + * the GuC. One gem object (ads->bo) contains the ADS struct itself (guc_ads) + * and most of the extra buffers linked via the ADS struct's entries. The UM + * fault queues (PAGE_FAULT, PAGE_FAULT_RESPONSE, ACCESS_COUNTER rings) are + * kept in a separate BO (ads->um_queue_bo) so that the full memset of ads->bo + * performed on every GT reset does not discard fault descriptors already + * written into the rings by the GPU. * - * Layout of the ADS blob allocated for the GuC: + * Layout of the ADS blob (ads->bo): * * +---------------------------------------+ <== base * | guc_ads | @@ -100,10 +104,6 @@ ads_to_map(struct xe_guc_ads *ads) * +---------------------------------------+ * | padding | * +---------------------------------------+ <== 4K aligned - * | UM queues | - * +---------------------------------------+ - * | padding | - * +---------------------------------------+ <== 4K aligned * | private data | * +---------------------------------------+ * | padding | @@ -157,16 +157,6 @@ static size_t guc_ads_capture_size(struct xe_guc_ads *ads) return PAGE_ALIGN(ads->capture_size); } -static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads) -{ - struct xe_device *xe = ads_to_xe(ads); - - if (!xe->info.has_usm) - return 0; - - return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX; -} - static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) { return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); @@ -207,22 +197,12 @@ static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) return PAGE_ALIGN(offset); } -static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads) -{ - u32 offset; - - offset = guc_ads_capture_offset(ads) + - guc_ads_capture_size(ads); - - return PAGE_ALIGN(offset); -} - static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) { size_t offset; - offset = guc_ads_um_queues_offset(ads) + - guc_ads_um_queues_size(ads); + offset = guc_ads_capture_offset(ads) + + guc_ads_capture_size(ads); return PAGE_ALIGN(offset); } @@ -408,6 +388,48 @@ int xe_guc_ads_init(struct xe_guc_ads *ads) ads->bo = bo; + if (xe->info.has_usm) { + /* + * Allocate a separate BO for the HW fault ring (UM queues). + * + * Round the size up to the next power of two so that on iGPU + * (system memory, no IOMMU) the TTM pool issues a single + * alloc_pages(order=N) call, maximising the chance of getting + * a physically contiguous block. GuC requires contiguous DPA. + */ + size_t um_size = IS_DGFX(xe) ? + GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX : + roundup_pow_of_two(GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX); + + u32 um_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE | + XE_BO_FLAG_NEEDS_UC; + + bo = xe_managed_bo_create_pin_map(xe, tile, um_size, um_flags); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + /* + * On pre-Xe3p platforms, GAM (not GuC) accesses the UM queue + * ring via base_dpa, which must be a contiguous DMA address + * range. Verify that the allocated pages are contiguous in + * DMA address space. + */ + if (!xe_bo_is_vram(bo) && + unlikely(!xe_bo_sg_is_contiguous(bo, + GUC_UM_QUEUE_SIZE * + GUC_UM_HW_QUEUE_MAX))) { + drm_err(&xe->drm, + "UM fault queue memory is not contiguous in DMA address space; GAM requires contiguous DPA\n"); + return -ENOMEM; + } + + ads->um_queue_bo = bo; + } + return 0; } ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */ @@ -824,13 +846,19 @@ static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) static void guc_um_init_params(struct xe_guc_ads *ads) { - u32 um_queue_offset = guc_ads_um_queues_offset(ads); + struct xe_bo *um_bo = ads->um_queue_bo; u64 base_dpa; u32 base_ggtt; int i; - base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset; - base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset; + if (um_bo) { + /* All USM platforms: UM queues in dedicated um_queue_bo */ + base_ggtt = xe_bo_ggtt_addr(um_bo); + base_dpa = xe_bo_main_addr(um_bo, PAGE_SIZE); + } else { + /* Platform does not support USM: no UM queues, nothing to do */ + return; + } for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, diff --git a/drivers/gpu/drm/xe/xe_guc_ads_types.h b/drivers/gpu/drm/xe/xe_guc_ads_types.h index 70c132458ac38..acb5366f76efe 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads_types.h +++ b/drivers/gpu/drm/xe/xe_guc_ads_types.h @@ -16,6 +16,11 @@ struct xe_bo; struct xe_guc_ads { /** @bo: XE BO for GuC ads blob */ struct xe_bo *bo; + /** + * @um_queue_bo: Dedicated BO for the HW fault ring (UM queues). + * NULL if the platform does not support USM. + */ + struct xe_bo *um_queue_bo; /** @golden_lrc_size: golden LRC size */ size_t golden_lrc_size; /** @regset_size: size of register set passed to GuC for save/restore */ diff --git a/drivers/gpu/drm/xe/xe_guc_buf.c b/drivers/gpu/drm/xe/xe_guc_buf.c index 502ca3a4ee606..3de6377ae2d52 100644 --- a/drivers/gpu/drm/xe/xe_guc_buf.c +++ b/drivers/gpu/drm/xe/xe_guc_buf.c @@ -13,6 +13,8 @@ #include "xe_guc_buf.h" #include "xe_sa.h" +#define XE_GUC_BUF_CACHE_DEFAULT_SIZE SZ_8K + static struct xe_guc *cache_to_guc(struct xe_guc_buf_cache *cache) { return container_of(cache, struct xe_guc, buf); @@ -23,21 +25,12 @@ static struct xe_gt *cache_to_gt(struct xe_guc_buf_cache *cache) return guc_to_gt(cache_to_guc(cache)); } -/** - * xe_guc_buf_cache_init() - Initialize the GuC Buffer Cache. - * @cache: the &xe_guc_buf_cache to initialize - * - * The Buffer Cache allows to obtain a reusable buffer that can be used to pass - * indirect H2G data to GuC without a need to create a ad-hoc allocation. - * - * Return: 0 on success or a negative error code on failure. - */ -int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache) +static int guc_buf_cache_init(struct xe_guc_buf_cache *cache, u32 size) { struct xe_gt *gt = cache_to_gt(cache); struct xe_sa_manager *sam; - sam = __xe_sa_bo_manager_init(gt_to_tile(gt), SZ_8K, 0, sizeof(u32)); + sam = __xe_sa_bo_manager_init(gt_to_tile(gt), size, 0, sizeof(u32), 0); if (IS_ERR(sam)) return PTR_ERR(sam); cache->sam = sam; @@ -48,6 +41,35 @@ int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache) return 0; } +/** + * xe_guc_buf_cache_init() - Initialize the GuC Buffer Cache. + * @cache: the &xe_guc_buf_cache to initialize + * + * The Buffer Cache allows to obtain a reusable buffer that can be used to pass + * data to GuC or read data from GuC without a need to create a ad-hoc allocation. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache) +{ + return guc_buf_cache_init(cache, XE_GUC_BUF_CACHE_DEFAULT_SIZE); +} + +/** + * xe_guc_buf_cache_init_with_size() - Initialize the GuC Buffer Cache. + * @cache: the &xe_guc_buf_cache to initialize + * @size: size in bytes + * + * Like xe_guc_buf_cache_init(), except it allows the caller to make the cache + * buffer larger, allowing to accommodate larger objects. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_guc_buf_cache_init_with_size(struct xe_guc_buf_cache *cache, u32 size) +{ + return guc_buf_cache_init(cache, max(XE_GUC_BUF_CACHE_DEFAULT_SIZE, size)); +} + /** * xe_guc_buf_cache_dwords() - Number of dwords the GuC Buffer Cache supports. * @cache: the &xe_guc_buf_cache to query diff --git a/drivers/gpu/drm/xe/xe_guc_buf.h b/drivers/gpu/drm/xe/xe_guc_buf.h index 0d67604d96bdd..8204e58962501 100644 --- a/drivers/gpu/drm/xe/xe_guc_buf.h +++ b/drivers/gpu/drm/xe/xe_guc_buf.h @@ -12,6 +12,7 @@ #include "xe_guc_buf_types.h" int xe_guc_buf_cache_init(struct xe_guc_buf_cache *cache); +int xe_guc_buf_cache_init_with_size(struct xe_guc_buf_cache *cache, u32 size); u32 xe_guc_buf_cache_dwords(struct xe_guc_buf_cache *cache); struct xe_guc_buf xe_guc_buf_reserve(struct xe_guc_buf_cache *cache, u32 dwords); struct xe_guc_buf xe_guc_buf_from_data(struct xe_guc_buf_cache *cache, diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 243dad3e24185..3a1613886495c 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -437,8 +437,15 @@ static void guc_capture_alloc_steered_lists(struct xe_guc *guc) if (!list || guc->capture->extlists) return; - total = bitmap_weight(gt->fuse_topo.g_dss_mask, sizeof(gt->fuse_topo.g_dss_mask) * 8) * - guc_capture_get_steer_reg_num(guc_to_xe(guc)); + { + xe_dss_mask_t all_dss; + + bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, + XE_MAX_DSS_FUSE_BITS); + + total = bitmap_weight(all_dss, XE_MAX_DSS_FUSE_BITS) * + guc_capture_get_steer_reg_num(guc_to_xe(guc)); + } if (!total) return; diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 4158e8af13908..b9d265ad06a0e 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -82,13 +82,16 @@ static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reaso struct g2h_fence { u32 *response_buffer; u32 seqno; + /* fields below this point are setup based on the response */ u32 response_data; u16 response_len; u16 error; u16 hint; u16 reason; + u32 counter; bool cancel; bool retry; + bool wait; bool fail; bool done; }; @@ -102,6 +105,11 @@ static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) g2h_fence->seqno = ~0x0; } +static void g2h_fence_reinit(struct g2h_fence *g2h_fence) +{ + memset_after(g2h_fence, 0, seqno); +} + static void g2h_fence_cancel(struct g2h_fence *g2h_fence) { g2h_fence->cancel = true; @@ -1134,6 +1142,7 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response * and g2h_fence_cancel. */ +wait_again: ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); if (!ret) { LNL_FLUSH_WORK(&ct->g2h_worker); @@ -1159,6 +1168,14 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, return -ETIME; } + if (g2h_fence.wait) { + xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n", + action[0], g2h_fence.counter); + /* we can't leave any response data if we want to wait again */ + g2h_fence_reinit(&g2h_fence); + mutex_unlock(&ct->lock); + goto wait_again; + } if (g2h_fence.retry) { xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", action[0], g2h_fence.reason); @@ -1354,7 +1371,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) return -EPROTO; } - g2h_fence = xa_erase(&ct->fence_lookup, fence); + /* don't erase as we still expect a final response with the same fence */ + if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) + g2h_fence = xa_load(&ct->fence_lookup, fence); + else + g2h_fence = xa_erase(&ct->fence_lookup, fence); + if (unlikely(!g2h_fence)) { /* Don't tear down channel, as send could've timed out */ /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ @@ -1365,6 +1387,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) xe_gt_assert(gt, fence == g2h_fence->seqno); + /* + * reinit as we might have already process this g2h_fence before + * if we received a NO_RESPONSE_BUSY reply + */ + g2h_fence_reinit(g2h_fence); + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { g2h_fence->fail = true; g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); @@ -1372,6 +1400,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { g2h_fence->retry = true; g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { + g2h_fence->wait = true; + g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]); } else if (g2h_fence->response_buffer) { g2h_fence->response_len = hxg_len; memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); @@ -1379,7 +1410,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); } - g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + /* don't release any space if it was an intermediate message */ + if (!g2h_fence->wait) + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ WRITE_ONCE(g2h_fence->done, true); @@ -1416,6 +1449,7 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) case GUC_HXG_TYPE_RESPONSE_SUCCESS: case GUC_HXG_TYPE_RESPONSE_FAILURE: case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + case GUC_HXG_TYPE_NO_RESPONSE_BUSY: ret = parse_g2h_response(ct, msg, len); break; default: diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index d88770370eeb0..66a738f17a3ba 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -565,6 +565,8 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, hw_engine_setup_default_state(hwe); xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_sr_init(&hwe->oa_sr, hwe->name, gt_to_xe(gt)); xe_reg_whitelist_process_engine(hwe); } diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h index e4191a7a2c318..04ca7545d22c3 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine_types.h +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -128,6 +128,14 @@ struct xe_hw_engine { * @reg_whitelist: table with registers to be whitelisted */ struct xe_reg_sr reg_whitelist; + /** + * @oa_whitelist: oa registers to be whitelisted + */ + struct xe_reg_sr oa_whitelist; + /** + * @oa_sr: oa nonpriv whitelist registers, changed on oa stream open/close + */ + struct xe_reg_sr oa_sr; /** * @reg_lrc: LRC workaround registers */ diff --git a/drivers/gpu/drm/xe/xe_i2c.c b/drivers/gpu/drm/xe/xe_i2c.c index 48dfcb41fa08c..25c7eb725ce26 100644 --- a/drivers/gpu/drm/xe/xe_i2c.c +++ b/drivers/gpu/drm/xe/xe_i2c.c @@ -6,7 +6,9 @@ */ #include +#include #include +#include #include #include #include @@ -23,6 +25,8 @@ #include #include +#include + #include "regs/xe_i2c_regs.h" #include "regs/xe_irq_regs.h" @@ -94,18 +98,21 @@ static int xe_i2c_register_adapter(struct xe_i2c *i2c) struct platform_device *pdev; struct fwnode_handle *fwnode; int ret; + u32 id; fwnode = fwnode_create_software_node(xe_i2c_adapter_properties, NULL); if (IS_ERR(fwnode)) return PTR_ERR(fwnode); + id = (pci_domain_nr(pci->bus) << 16) | pci_dev_id(pci); + /* * Not using platform_device_register_full() here because we don't have * a handle to the platform_device before it returns. xe_i2c_notifier() * uses that handle, but it may be called before * platform_device_register_full() is done. */ - pdev = platform_device_alloc(adapter_name, pci_dev_id(pci)); + pdev = platform_device_alloc(adapter_name, id); if (!pdev) { ret = -ENOMEM; goto err_fwnode_remove; @@ -214,11 +221,40 @@ static void xe_i2c_remove_irq(struct xe_i2c *i2c) irq_domain_remove(i2c->irqdomain); } +/* See "Disabling DW_apb_i2c" in the DesignWare DW_abp_i2c databook. */ +static void xe_i2c_disable(struct xe_i2c *i2c) +{ + int timeout = 100; + u32 status; + + xe_mmio_rmw32(i2c->mmio, I2C_REG(DW_IC_ENABLE), DW_IC_ENABLE_ENABLE, 0); + + do { + status = xe_mmio_read32(i2c->mmio, I2C_REG(DW_IC_ENABLE_STATUS)); + if (!(status & DW_IC_ENABLE_ENABLE)) + return; + /* Can't sleep here. */ + udelay(25); + } while (timeout--); + + dev_warn(i2c->drm_dev, "timeout in disabling i2c adapter\n"); +} + static int xe_i2c_read(void *context, unsigned int reg, unsigned int *val) { struct xe_i2c *i2c = context; - *val = xe_mmio_read32(i2c->mmio, XE_REG(reg + I2C_MEM_SPACE_OFFSET)); + *val = xe_mmio_read32(i2c->mmio, I2C_REG(reg)); + + switch (reg) { + case DW_IC_ENABLE: + case DW_IC_ENABLE_STATUS: + FIELD_MODIFY(DW_IC_ENABLE_ENABLE, val, + i2c->ic_enable & DW_IC_ENABLE_ENABLE); + break; + default: + break; + } return 0; } @@ -227,8 +263,32 @@ static int xe_i2c_write(void *context, unsigned int reg, unsigned int val) { struct xe_i2c *i2c = context; - xe_mmio_write32(i2c->mmio, XE_REG(reg + I2C_MEM_SPACE_OFFSET), val); + switch (reg) { + case DW_IC_CON: + case DW_IC_TAR: + case DW_IC_SAR: + /* Disable the controller. */ + xe_i2c_disable(i2c); + + /* Write the register. */ + xe_mmio_write32(i2c->mmio, I2C_REG(reg), val); + + /* Enable the controller. */ + xe_mmio_rmw32(i2c->mmio, I2C_REG(DW_IC_ENABLE), 0, DW_IC_ENABLE_ENABLE); + return 0; + case DW_IC_ENABLE: + i2c->ic_enable = val; + /* Other fields can be updated except the enable bit. */ + val |= DW_IC_ENABLE_ENABLE; + break; + case DW_IC_SMBUS_INTR_MASK: + /* Preserve the interrupt mask requested by the adapter driver. */ + break; + default: + break; + } + xe_mmio_write32(i2c->mmio, I2C_REG(reg), val); return 0; } diff --git a/drivers/gpu/drm/xe/xe_i2c.h b/drivers/gpu/drm/xe/xe_i2c.h index ecd5f10358e29..6192e20feaa3f 100644 --- a/drivers/gpu/drm/xe/xe_i2c.h +++ b/drivers/gpu/drm/xe/xe_i2c.h @@ -34,6 +34,7 @@ struct xe_i2c { struct platform_device *pdev; struct i2c_adapter *adapter; struct i2c_client *client[XE_I2C_MAX_CLIENTS]; + unsigned int ic_enable; struct notifier_block bus_notifier; struct work_struct work; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 9f29dbe890f66..53eadea8ab38d 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -1692,6 +1692,13 @@ void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); } + + /* + * The ring and the LRC context image are both WC, so the ring tail + * update which publishes these writes can become visible to the device + * first. Ensure the ring contents are visible before returning. + */ + xe_device_wmb(xe); } u64 xe_lrc_descriptor(struct xe_lrc *lrc) diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index b6905f35d6c81..1e8c0b92d712a 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -33,6 +33,7 @@ #include "xe_res_cursor.h" #include "xe_sa.h" #include "xe_sched_job.h" +#include "xe_sriov_vf_ccs.h" #include "xe_sync.h" #include "xe_trace_bo.h" #include "xe_validation.h" @@ -764,31 +765,13 @@ static u32 xe_migrate_ccs_copy(struct xe_migrate *m, return flush_flags; } -/** - * xe_migrate_copy() - Copy content of TTM resources. - * @m: The migration context. - * @src_bo: The buffer object @src is currently bound to. - * @dst_bo: If copying between resources created for the same bo, set this to - * the same value as @src_bo. If copying between buffer objects, set it to - * the buffer object @dst is currently bound to. - * @src: The source TTM resource. - * @dst: The dst TTM resource. - * @copy_only_ccs: If true copy only CCS metadata - * - * Copies the contents of @src to @dst: On flat CCS devices, - * the CCS metadata is copied as well if needed, or if not present, - * the CCS metadata of @dst is cleared for security reasons. - * - * Return: Pointer to a dma_fence representing the last copy batch, or - * an error pointer on failure. If there is a failure, any copy operation - * started by the function call has been synced. - */ -struct dma_fence *xe_migrate_copy(struct xe_migrate *m, - struct xe_bo *src_bo, - struct xe_bo *dst_bo, - struct ttm_resource *src, - struct ttm_resource *dst, - bool copy_only_ccs) +static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + struct xe_bo *dst_bo, + struct ttm_resource *src, + struct ttm_resource *dst, + bool copy_only_ccs, + bool is_vram_resolve) { struct xe_gt *gt = m->tile->primary_gt; struct xe_device *xe = gt_to_xe(gt); @@ -809,8 +792,15 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, bool copy_ccs = xe_device_has_flat_ccs(xe) && xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); - bool use_comp_pat = type_device && xe_device_has_flat_ccs(xe) && - GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram; + + /* + * For decompression operation, always use the compression PAT index. + * Otherwise, only use the compression PAT index for device memory + * when copying from VRAM to system memory. + */ + bool use_comp_pat = is_vram_resolve || (type_device && + xe_device_has_flat_ccs(xe) && + GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram); /* Copying CCS between two different BOs is not supported yet. */ if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) @@ -969,6 +959,53 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, return fence; } +/** + * xe_migrate_copy() - Copy content of TTM resources. + * @m: The migration context. + * @src_bo: The buffer object @src is currently bound to. + * @dst_bo: If copying between resources created for the same bo, set this to + * the same value as @src_bo. If copying between buffer objects, set it to + * the buffer object @dst is currently bound to. + * @src: The source TTM resource. + * @dst: The dst TTM resource. + * @copy_only_ccs: If true copy only CCS metadata + * + * Copies the contents of @src to @dst: On flat CCS devices, + * the CCS metadata is copied as well if needed, or if not present, + * the CCS metadata of @dst is cleared for security reasons. + * + * Return: Pointer to a dma_fence representing the last copy batch, or + * an error pointer on failure. If there is a failure, any copy operation + * started by the function call has been synced. + */ +struct dma_fence *xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *src_bo, + struct xe_bo *dst_bo, + struct ttm_resource *src, + struct ttm_resource *dst, + bool copy_only_ccs) +{ + return __xe_migrate_copy(m, src_bo, dst_bo, src, dst, copy_only_ccs, false); +} + +/** + * xe_migrate_resolve() - Resolve and decompress a buffer object if required. + * @m: The migrate context + * @bo: The buffer object to resolve + * @res: The reservation object + * + * Wrapper around __xe_migrate_copy() with is_vram_resolve set to true + * to trigger decompression if needed. + * + * Return: A dma_fence that signals on completion, or an ERR_PTR on failure. + */ +struct dma_fence *xe_migrate_resolve(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *res) +{ + return __xe_migrate_copy(m, bo, bo, res, res, false, true); +} + /** * xe_migrate_lrc() - Get the LRC from migrate context. * @migrate: Migrate context. @@ -1000,6 +1037,8 @@ static int emit_flush_invalidate(struct xe_exec_queue *q, u32 *dw, int i, * @tile: Tile whose migration context to be used. * @q : Execution to be used along with migration context. * @src_bo: The buffer object @src is currently bound to. + * @new_mem: The (not yet committed) destination resource @src_bo is being + * moved into; src_bo->ttm.resource is still the old resource. * @read_write : Creates BB commands for CCS read/write. * * Creates batch buffer instructions to copy CCS metadata from CCS pool to @@ -1011,23 +1050,28 @@ static int emit_flush_invalidate(struct xe_exec_queue *q, u32 *dw, int i, */ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, + struct ttm_resource *new_mem, enum xe_sriov_vf_ccs_rw_ctxs read_write) { bool src_is_pltt = read_write == XE_SRIOV_VF_CCS_READ_CTX; bool dst_is_pltt = read_write == XE_SRIOV_VF_CCS_WRITE_CTX; - struct ttm_resource *src = src_bo->ttm.resource; + struct ttm_resource *src = new_mem; struct xe_migrate *m = tile->migrate; struct xe_gt *gt = tile->primary_gt; u32 batch_size, batch_size_allocated; struct xe_device *xe = gt_to_xe(gt); struct xe_res_cursor src_it, ccs_it; + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_sa_manager *bb_pool; u64 size = xe_bo_size(src_bo); struct xe_bb *bb = NULL; u64 src_L0, src_L0_ofs; u32 src_L0_pt; int err; + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), @@ -1060,11 +1104,15 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, size -= src_L0; } + bb_pool = ctx->mem.ccs_bb_pool; + guard(mutex) (xe_sa_bo_swap_guard(bb_pool)); + xe_sa_bo_swap_shadow(bb_pool); + bb = xe_bb_ccs_new(gt, batch_size, read_write); if (IS_ERR(bb)) { drm_err(&xe->drm, "BB allocation failed.\n"); err = PTR_ERR(bb); - goto err_ret; + return err; } batch_size_allocated = batch_size; @@ -1113,10 +1161,52 @@ int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, xe_assert(xe, (batch_size_allocated == bb->len)); src_bo->bb_ccs[read_write] = bb; + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + xe_sa_bo_sync_shadow(bb->bo); return 0; +} -err_ret: - return err; +/** + * xe_migrate_ccs_rw_copy_clear() - Clear the CCS read/write batch buffer + * content. + * @src_bo: The buffer object @src is currently bound to. + * @read_write : Creates BB commands for CCS read/write. + * + * Directly clearing the BB lacks atomicity and can lead to undefined + * behavior if the vCPU is halted mid-operation during the clearing + * process. To avoid this issue, we use a shadow buffer object approach. + * + * First swap the SA BO address with the shadow BO, perform the clearing + * operation on the BB, update the shadow BO in the ring buffer, then + * sync the shadow and the actual buffer to maintain consistency. + * + * Returns: None. + */ +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write) +{ + struct xe_bb *bb = src_bo->bb_ccs[read_write]; + struct xe_device *xe = xe_bo_device(src_bo); + struct xe_sriov_vf_ccs_ctx *ctx; + struct xe_sa_manager *bb_pool; + u32 *cs; + + xe_assert(xe, IS_SRIOV_VF(xe)); + + ctx = &xe->sriov.vf.ccs.contexts[read_write]; + bb_pool = ctx->mem.ccs_bb_pool; + + guard(mutex) (xe_sa_bo_swap_guard(bb_pool)); + xe_sa_bo_swap_shadow(bb_pool); + + cs = xe_sa_bo_cpu_addr(bb->bo); + memset(cs, MI_NOOP, bb->len * sizeof(u32)); + xe_sriov_vf_ccs_rw_update_bb_addr(ctx); + + xe_sa_bo_sync_shadow(bb->bo); + + xe_bb_free(bb, NULL); + src_bo->bb_ccs[read_write] = NULL; } /** diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index bc55f650204b8..9665a0aff547c 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -127,10 +127,18 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, struct ttm_resource *dst, bool copy_only_ccs); +struct dma_fence *xe_migrate_resolve(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *res); + int xe_migrate_ccs_rw_copy(struct xe_tile *tile, struct xe_exec_queue *q, struct xe_bo *src_bo, + struct ttm_resource *new_mem, enum xe_sriov_vf_ccs_rw_ctxs read_write); +void xe_migrate_ccs_rw_copy_clear(struct xe_bo *src_bo, + enum xe_sriov_vf_ccs_rw_ctxs read_write); + struct xe_lrc *xe_migrate_lrc(struct xe_migrate *migrate); struct xe_exec_queue *xe_migrate_exec_queue(struct xe_migrate *migrate); int xe_migrate_access_memory(struct xe_migrate *m, struct xe_bo *bo, diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c index 7d04591e297a8..a753ac7ecc860 100644 --- a/drivers/gpu/drm/xe/xe_oa.c +++ b/drivers/gpu/drm/xe/xe_oa.c @@ -35,6 +35,7 @@ #include "xe_oa.h" #include "xe_observation.h" #include "xe_pm.h" +#include "xe_reg_whitelist.h" #include "xe_sched_job.h" #include "xe_sriov.h" #include "xe_sync.h" @@ -863,6 +864,9 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream) mutex_destroy(&stream->stream_lock); + if (stream->sample) + xe_reg_dewhitelist_oa_regs(stream->gt); + xe_oa_disable_metric_set(stream); xe_exec_queue_put(stream->k_exec_q); @@ -1563,6 +1567,10 @@ static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) config = xchg(&stream->oa_config, config); drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", stream->oa_config->uuid); + } else { + while (param.num_syncs--) + xe_sync_entry_cleanup(¶m.syncs[param.num_syncs]); + kfree(param.syncs); } err_config_put: @@ -1887,6 +1895,9 @@ static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa, goto err_disable; } + if (stream->sample) + xe_reg_whitelist_oa_regs(stream->gt); + /* Hold a reference on the drm device till stream_fd is released */ drm_dev_get(&stream->oa->xe->drm); @@ -2407,9 +2418,9 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); if (oa_config->id < 0) { - drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); + drm_dbg(&oa->xe->drm, "Failed to allocate id for OA config\n"); err = oa_config->id; - goto sysfs_err; + goto id_alloc_err; } id = oa_config->id; @@ -2420,6 +2431,8 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi return id; +id_alloc_err: + sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); sysfs_err: mutex_unlock(&oa->metrics_lock); reg_err: diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h index daf701b5d48b9..48f9520f560c2 100644 --- a/drivers/gpu/drm/xe/xe_oa_types.h +++ b/drivers/gpu/drm/xe/xe_oa_types.h @@ -125,6 +125,9 @@ struct xe_oa_gt { /** @oa_unit: array of oa_units */ struct xe_oa_unit *oa_unit; + + /** @whitelist_count: number of open streams for which oa registers are whitelisted */ + u32 whitelist_count; }; /** diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c new file mode 100644 index 0000000000000..d509a80cb1f31 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include "xe_pagefault.h" +#include "xe_pagefault_types.h" + +/** + * DOC: Xe page faults + * + * Xe page faults are handled in two layers. The producer layer interacts with + * hardware or firmware to receive and parse faults into struct xe_pagefault, + * then forwards them to the consumer. The consumer layer services the faults + * (e.g., memory migration, page table updates) and acknowledges the result back + * to the producer, which then forwards the results to the hardware or firmware. + * The consumer uses a page fault queue sized to absorb all potential faults and + * a multi-threaded worker to process them. Multiple producers are supported, + * with a single shared consumer. + * + * xe_pagefault.c implements the consumer layer. + */ + +/** + * xe_pagefault_init() - Page fault init + * @xe: xe device instance + * + * Initialize Xe page fault state. Must be done after reading fuses. + * + * Return: 0 on Success, errno on failure + */ +int xe_pagefault_init(struct xe_device *xe) +{ + /* TODO - implement */ + return 0; +} + +/** + * xe_pagefault_reset() - Page fault reset for a GT + * @xe: xe device instance + * @gt: GT being reset + * + * Reset the Xe page fault state for a GT; that is, squash any pending faults on + * the GT. + */ +void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt) +{ + /* TODO - implement */ +} + +/** + * xe_pagefault_handler() - Page fault handler + * @xe: xe device instance + * @pf: Page fault + * + * Sink the page fault to a queue (i.e., a memory buffer) and queue a worker to + * service it. Safe to be called from IRQ or process context. Reclaim safe. + * + * Return: 0 on success, errno on failure + */ +int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf) +{ + /* TODO - implement */ + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_pagefault.h b/drivers/gpu/drm/xe/xe_pagefault.h new file mode 100644 index 0000000000000..bd0cdf9ed37f1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGEFAULT_H_ +#define _XE_PAGEFAULT_H_ + +struct xe_device; +struct xe_gt; +struct xe_pagefault; + +int xe_pagefault_init(struct xe_device *xe); + +void xe_pagefault_reset(struct xe_device *xe, struct xe_gt *gt); + +int xe_pagefault_handler(struct xe_device *xe, struct xe_pagefault *pf); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h new file mode 100644 index 0000000000000..d3b516407d600 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pagefault_types.h @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef _XE_PAGEFAULT_TYPES_H_ +#define _XE_PAGEFAULT_TYPES_H_ + +#include + +struct xe_gt; +struct xe_pagefault; + +/** enum xe_pagefault_access_type - Xe page fault access type */ +enum xe_pagefault_access_type { + /** @XE_PAGEFAULT_ACCESS_TYPE_READ: Read access type */ + XE_PAGEFAULT_ACCESS_TYPE_READ = 0, + /** @XE_PAGEFAULT_ACCESS_TYPE_WRITE: Write access type */ + XE_PAGEFAULT_ACCESS_TYPE_WRITE = 1, + /** @XE_PAGEFAULT_ACCESS_TYPE_ATOMIC: Atomic access type */ + XE_PAGEFAULT_ACCESS_TYPE_ATOMIC = 2, +}; + +/** enum xe_pagefault_type - Xe page fault type */ +enum xe_pagefault_type { + /** @XE_PAGEFAULT_TYPE_NOT_PRESENT: Not present */ + XE_PAGEFAULT_TYPE_NOT_PRESENT = 0, + /** @XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION: Write access violation */ + XE_PAGEFAULT_TYPE_WRITE_ACCESS_VIOLATION = 1, + /** @XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION: Atomic access violation */ + XE_PAGEFAULT_TYPE_ATOMIC_ACCESS_VIOLATION = 2, +}; + +/** struct xe_pagefault_ops - Xe pagefault ops (producer) */ +struct xe_pagefault_ops { + /** + * @ack_fault: Ack fault + * @pf: Page fault + * @err: Error state of fault + * + * Page fault producer receives acknowledgment from the consumer and + * sends the result to the HW/FW interface. + */ + void (*ack_fault)(struct xe_pagefault *pf, int err); +}; + +/** + * struct xe_pagefault - Xe page fault + * + * Generic page fault structure for communication between producer and consumer. + * Carefully sized to be 64 bytes. Upon a device page fault, the producer + * populates this structure, and the consumer copies it into the page-fault + * queue for deferred handling. + */ +struct xe_pagefault { + /** + * @gt: GT of fault + */ + struct xe_gt *gt; + /** + * @consumer: State for the software handling the fault. Populated by + * the producer and may be modified by the consumer to communicate + * information back to the producer upon fault acknowledgment. + */ + struct { + /** @consumer.page_addr: address of page fault */ + u64 page_addr; + /** @consumer.asid: address space ID */ + u32 asid; + /** + * @consumer.access_type: access type, u8 rather than enum to + * keep size compact + */ + u8 access_type; + /** + * @consumer.fault_type: fault type, u8 rather than enum to + * keep size compact + */ + u8 fault_type; +#define XE_PAGEFAULT_LEVEL_NACK 0xff /* Producer indicates nack fault */ + /** @consumer.fault_level: fault level */ + u8 fault_level; + /** @consumer.engine_class: engine class */ + u8 engine_class; + /** @consumer.engine_instance: engine instance */ + u8 engine_instance; + /** consumer.reserved: reserved bits for future expansion */ + u8 reserved[7]; + } consumer; + /** + * @producer: State for the producer (i.e., HW/FW interface). Populated + * by the producer and should not be modified—or even inspected—by the + * consumer, except for calling operations. + */ + struct { + /** @producer.private: private pointer */ + void *private; + /** @producer.ops: operations */ + const struct xe_pagefault_ops *ops; +#define XE_PAGEFAULT_PRODUCER_MSG_LEN_DW 4 + /** + * @producer.msg: page fault message, used by producer in fault + * acknowledgment to formulate response to HW/FW interface. + * Included in the page-fault message because the producer + * typically receives the fault in a context where memory cannot + * be allocated (e.g., atomic context or the reclaim path). + */ + u32 msg[XE_PAGEFAULT_PRODUCER_MSG_LEN_DW]; + } producer; +}; + +/** + * struct xe_pagefault_queue: Xe pagefault queue (consumer) + * + * Used to capture all device page faults for deferred processing. Size this + * queue to absorb the device’s worst-case number of outstanding faults. + */ +struct xe_pagefault_queue { + /** + * @data: Data in queue containing struct xe_pagefault, protected by + * @lock + */ + void *data; + /** @size: Size of queue in bytes */ + u32 size; + /** @head: Head pointer in bytes, moved by producer, protected by @lock */ + u32 head; + /** @tail: Tail pointer in bytes, moved by consumer, protected by @lock */ + u32 tail; + /** @lock: protects page fault queue */ + spinlock_t lock; + /** @worker: to process page faults */ + struct work_struct worker; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 2e7cb99ae87aa..9a321384b73da 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -160,6 +160,12 @@ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) return xe->pat.table[pat_index].coh_mode; } +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) +{ + WARN_ON(pat_index >= xe->pat.n_entries); + return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); +} + static void program_pat(struct xe_gt *gt, const struct xe_pat_table_entry table[], int n_entries) { diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h index fa0dfbe525cd6..12cb6ec39d34c 100644 --- a/drivers/gpu/drm/xe/xe_pat.h +++ b/drivers/gpu/drm/xe/xe_pat.h @@ -58,4 +58,14 @@ void xe_pat_dump(struct xe_gt *gt, struct drm_printer *p); */ u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index); +/** + * xe_pat_index_get_comp_en - Extract the compression enable flag for + * the given pat_index. + * @xe: xe device + * @pat_index: The pat_index to query + * + * Return: true if compression is enabled for this pat_index, false otherwise. + */ +bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index); + #endif diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 89cc6d32f041b..b787292117901 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -591,6 +591,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_heci_cscfi = desc->has_heci_cscfi; xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; + xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; xe->info.has_pxp = desc->has_pxp; xe->info.has_sriov = desc->has_sriov; xe->info.skip_guc_pc = desc->skip_guc_pc; diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h index 9b9766a3baa34..a7b6576725edf 100644 --- a/drivers/gpu/drm/xe/xe_pci_types.h +++ b/drivers/gpu/drm/xe/xe_pci_types.h @@ -42,6 +42,7 @@ struct xe_device_desc { u8 has_late_bind:1; u8 has_llc:1; u8 has_mbx_power_limits:1; + u8 has_page_reclaim_hw_assist:1; u8 has_pxp:1; u8 has_sriov:1; u8 needs_scratch:1; diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 1f0262adc3ae3..1d22af2145bf9 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -715,7 +715,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, .vm = vm, .tile = tile, .curs = &curs, - .va_curs_start = range ? range->base.itree.start : + .va_curs_start = range ? xe_svm_range_start(range) : xe_vma_start(vma), .vma = vma, .wupd.entries = entries, @@ -734,7 +734,7 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, } if (xe_svm_range_has_dma_mapping(range)) { xe_res_first_dma(range->base.pages.dma_addr, 0, - range->base.itree.last + 1 - range->base.itree.start, + xe_svm_range_size(range), &curs); xe_svm_range_debug(range, "BIND PREPARE - MIXED"); } else { @@ -778,8 +778,8 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, walk_pt: ret = xe_pt_walk_range(&pt->base, pt->level, - range ? range->base.itree.start : xe_vma_start(vma), - range ? range->base.itree.last + 1 : xe_vma_end(vma), + range ? xe_svm_range_start(range) : xe_vma_start(vma), + range ? xe_svm_range_end(range) : xe_vma_end(vma), &xe_walk.base); *num_entries = xe_walk.wupd.num_used_entries; @@ -983,8 +983,8 @@ bool xe_pt_zap_ptes_range(struct xe_tile *tile, struct xe_vm *vm, if (!(pt_mask & BIT(tile->id))) return false; - (void)xe_pt_walk_shared(&pt->base, pt->level, range->base.itree.start, - range->base.itree.last + 1, &xe_walk.base); + (void)xe_pt_walk_shared(&pt->base, pt->level, xe_svm_range_start(range), + xe_svm_range_end(range), &xe_walk.base); return xe_walk.needs_invalidate; } @@ -999,12 +999,22 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *t u64 *ptr = data; u32 i; + /* + * @qword_ofs is the absolute entry offset within the page table, while + * @ptes is indexed relative to @update->ofs (its first entry). The GPU + * path (write_pgtable) splits a single update into MAX_PTE_PER_SDI-sized + * chunks, calling this with an advancing @qword_ofs but a fresh @data + * pointer per chunk, so translate back into a @ptes index rather than + * assuming the chunk starts at ptes[0]. + */ for (i = 0; i < num_qwords; i++) { + u32 idx = qword_ofs - update->ofs + i; + if (map) xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * - sizeof(u64), u64, ptes[i].pte); + sizeof(u64), u64, ptes[idx].pte); else - ptr[i] = ptes[i].pte; + ptr[i] = ptes[idx].pte; } } @@ -1701,8 +1711,8 @@ static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_svm_range *range, struct xe_vm_pgtable_update *entries) { - u64 start = range ? range->base.itree.start : xe_vma_start(vma); - u64 end = range ? range->base.itree.last + 1 : xe_vma_end(vma); + u64 start = range ? xe_svm_range_start(range) : xe_vma_start(vma); + u64 end = range ? xe_svm_range_end(range) : xe_vma_end(vma); struct xe_pt_stage_unbind_walk xe_walk = { .base = { .ops = &xe_pt_stage_unbind_ops, @@ -1912,7 +1922,7 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, vm_dbg(&xe_vma_vm(vma)->xe->drm, "Preparing bind, with range [%lx...%lx)\n", - range->base.itree.start, range->base.itree.last); + xe_svm_range_start(range), xe_svm_range_end(range) - 1); pt_op->vma = NULL; pt_op->bind = true; @@ -1927,8 +1937,8 @@ static int bind_range_prepare(struct xe_vm *vm, struct xe_tile *tile, pt_op->num_entries, true); xe_pt_update_ops_rfence_interval(pt_update_ops, - range->base.itree.start, - range->base.itree.last + 1); + xe_svm_range_start(range), + xe_svm_range_end(range)); ++pt_update_ops->current_op; pt_update_ops->needs_svm_lock = true; @@ -2023,7 +2033,7 @@ static int unbind_range_prepare(struct xe_vm *vm, vm_dbg(&vm->xe->drm, "Preparing unbind, with range [%lx...%lx)\n", - range->base.itree.start, range->base.itree.last); + xe_svm_range_start(range), xe_svm_range_end(range) - 1); pt_op->vma = XE_INVALID_VMA; pt_op->bind = false; @@ -2034,8 +2044,8 @@ static int unbind_range_prepare(struct xe_vm *vm, xe_vm_dbg_print_entries(tile_to_xe(tile), pt_op->entries, pt_op->num_entries, false); - xe_pt_update_ops_rfence_interval(pt_update_ops, range->base.itree.start, - range->base.itree.last + 1); + xe_pt_update_ops_rfence_interval(pt_update_ops, xe_svm_range_start(range), + xe_svm_range_end(range)); ++pt_update_ops->current_op; pt_update_ops->needs_svm_lock = true; pt_update_ops->needs_invalidation |= xe_vm_has_scratch(vm) || @@ -2143,8 +2153,11 @@ static void xe_pt_update_ops_init(struct xe_vm_pgtable_update_ops *pt_update_ops) { init_llist_head(&pt_update_ops->deferred); + pt_update_ops->current_op = 0; pt_update_ops->start = ~0x0ull; pt_update_ops->last = 0x0ull; + pt_update_ops->needs_svm_lock = false; + pt_update_ops->needs_invalidation = false; } /** diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c index 21763dc51150b..eb1d6f52f75fb 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.c +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -9,7 +9,8 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_oa_regs.h" #include "regs/xe_regs.h" -#include "xe_gt_types.h" +#include "xe_assert.h" +#include "xe_gt.h" #include "xe_gt_printk.h" #include "xe_platform_types.h" #include "xe_reg_sr.h" @@ -66,33 +67,36 @@ static const struct xe_rtp_entry_sr register_whitelist[] = { ENGINE_CLASS(RENDER)), XE_RTP_ACTIONS(WHITELIST(CSBE_DEBUG_STATUS(RENDER_RING_BASE), 0)) }, - { XE_RTP_NAME("oa_reg_render"), +}; + +static const struct xe_rtp_entry_sr oa_whitelist[] = { + +#define WHITELIST_DENY(r, f) WHITELIST(r, (f) | RING_FORCE_TO_NONPRIV_DENY) + +#define WHITELIST_OA_MMIO_TRG(trg, status, head) \ + WHITELIST_DENY(trg, RING_FORCE_TO_NONPRIV_ACCESS_RW), \ + WHITELIST_DENY(status, RING_FORCE_TO_NONPRIV_ACCESS_RD), \ + WHITELIST_DENY(head, RING_FORCE_TO_NONPRIV_ACCESS_RD | RING_FORCE_TO_NONPRIV_RANGE_4) + +#define WHITELIST_OAG_MMIO_TRG \ + WHITELIST_OA_MMIO_TRG(OAG_MMIOTRIGGER, OAG_OASTATUS, OAG_OAHEADPTR) + + { XE_RTP_NAME("oag_mmio_trg_rcs"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), ENGINE_CLASS(RENDER)), - XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER, - RING_FORCE_TO_NONPRIV_ACCESS_RW), - WHITELIST(OAG_OASTATUS, - RING_FORCE_TO_NONPRIV_ACCESS_RD), - WHITELIST(OAG_OAHEADPTR, - RING_FORCE_TO_NONPRIV_ACCESS_RD | - RING_FORCE_TO_NONPRIV_RANGE_4)) + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) }, - { XE_RTP_NAME("oa_reg_compute"), + { XE_RTP_NAME("oag_mmio_trg_ccs"), XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED), ENGINE_CLASS(COMPUTE)), - XE_RTP_ACTIONS(WHITELIST(OAG_MMIOTRIGGER, - RING_FORCE_TO_NONPRIV_ACCESS_RW), - WHITELIST(OAG_OASTATUS, - RING_FORCE_TO_NONPRIV_ACCESS_RD), - WHITELIST(OAG_OAHEADPTR, - RING_FORCE_TO_NONPRIV_ACCESS_RD | - RING_FORCE_TO_NONPRIV_RANGE_4)) + XE_RTP_ACTIONS(WHITELIST_OAG_MMIO_TRG) }, }; -static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) +static int whitelist_apply_to_hwe(struct xe_hw_engine *hwe, struct xe_reg_sr *in, + struct xe_reg_sr *out, int first_slot) { - struct xe_reg_sr *sr = &hwe->reg_whitelist; + struct xe_reg_sr *sr = in; struct xe_reg_sr_entry *entry; struct drm_printer p; unsigned long reg; @@ -101,7 +105,7 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) xe_gt_dbg(hwe->gt, "Add %s whitelist to engine\n", sr->name); p = xe_gt_dbg_printer(hwe->gt); - slot = 0; + slot = first_slot; xa_for_each(&sr->xa, reg, entry) { struct xe_reg_sr_entry hwe_entry = { .reg = RING_FORCE_TO_NONPRIV(hwe->mmio_base, slot), @@ -118,10 +122,12 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) } xe_reg_whitelist_print_entry(&p, 0, reg, entry); - xe_reg_sr_add(&hwe->reg_sr, &hwe_entry, hwe->gt); + xe_reg_sr_add(out, &hwe_entry, hwe->gt); slot++; } + + return slot; } /** @@ -135,10 +141,79 @@ static void whitelist_apply_to_hwe(struct xe_hw_engine *hwe) void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe) { struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe); + int first_oa_slot; xe_rtp_process_to_sr(&ctx, register_whitelist, ARRAY_SIZE(register_whitelist), &hwe->reg_whitelist); - whitelist_apply_to_hwe(hwe); + first_oa_slot = whitelist_apply_to_hwe(hwe, &hwe->reg_whitelist, &hwe->reg_sr, 0); + + xe_rtp_process_to_sr(&ctx, oa_whitelist, ARRAY_SIZE(oa_whitelist), + &hwe->oa_whitelist); + /* + * Save oa nonpriv registers to hwe->oa_sr, from which oa registers are whitelisted + * or de-whitelisted, by toggling the 'deny' bit on oa stream open/close + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->oa_sr, first_oa_slot); + + /* + * Also save oa nonpriv registers to hwe->reg_sr, to ensure oa registers are not + * whitelisted by default after probe, gt reset, resume and engine reset + */ + whitelist_apply_to_hwe(hwe, &hwe->oa_whitelist, &hwe->reg_sr, first_oa_slot); +} + +static void __whitelist_oa_regs(struct xe_hw_engine *hwe, bool whitelist) +{ + struct xe_reg_sr_entry *entry; + unsigned long reg; + + xa_for_each(&hwe->oa_sr.xa, reg, entry) { + if (whitelist) + entry->set_bits &= ~RING_FORCE_TO_NONPRIV_DENY; + else + entry->set_bits |= RING_FORCE_TO_NONPRIV_DENY; + } + + xe_reg_sr_apply_mmio(&hwe->oa_sr, hwe->gt); +} + +/** + * xe_reg_whitelist_oa_regs - whitelist oa registers for gt + * @gt: gt to whitelist oa registers for + * + * Whitelist OA registers by resetting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_whitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + if (gt->oa.whitelist_count++) + return; + + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, true); +} + +/** + * xe_reg_dewhitelist_oa_regs - dewhitelist oa registers for gt + * @gt: gt to dewhitelist oa registers for + * + * Dewhitelist OA registers by setting RING_FORCE_TO_NONPRIV_DENY + */ +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + lockdep_assert_held(>->oa.gt_lock); + xe_assert(gt_to_xe(gt), gt->oa.whitelist_count); + if (--gt->oa.whitelist_count) + return; + + for_each_hw_engine(hwe, gt, id) + __whitelist_oa_regs(hwe, false); } /** diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h index 69b121d377da0..f243b6ac60b7e 100644 --- a/drivers/gpu/drm/xe/xe_reg_whitelist.h +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h @@ -9,12 +9,16 @@ #include struct drm_printer; +struct xe_gt; struct xe_hw_engine; struct xe_reg_sr; struct xe_reg_sr_entry; void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe); +void xe_reg_whitelist_oa_regs(struct xe_gt *gt); +void xe_reg_dewhitelist_oa_regs(struct xe_gt *gt); + void xe_reg_whitelist_print_entry(struct drm_printer *p, unsigned int indent, u32 reg, struct xe_reg_sr_entry *entry); diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h index 4e00008b70817..5f4ab08c06869 100644 --- a/drivers/gpu/drm/xe/xe_res_cursor.h +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -58,7 +58,7 @@ struct xe_res_cursor { /** @dma_addr: Current element in a struct drm_pagemap_addr array */ const struct drm_pagemap_addr *dma_addr; /** @mm: Buddy allocator for VRAM cursor */ - struct drm_buddy *mm; + struct gpu_buddy *mm; /** * @dma_start: DMA start address for the current segment. * This may be different to @dma_addr.addr since elements in @@ -69,7 +69,7 @@ struct xe_res_cursor { u64 dma_seg_size; }; -static struct drm_buddy *xe_res_get_buddy(struct ttm_resource *res) +static struct gpu_buddy *xe_res_get_buddy(struct ttm_resource *res) { struct ttm_resource_manager *mgr; @@ -104,30 +104,30 @@ static inline void xe_res_first(struct ttm_resource *res, case XE_PL_STOLEN: case XE_PL_VRAM0: case XE_PL_VRAM1: { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct list_head *head, *next; - struct drm_buddy *mm = xe_res_get_buddy(res); + struct gpu_buddy *mm = xe_res_get_buddy(res); head = &to_xe_ttm_vram_mgr_resource(res)->blocks; block = list_first_entry_or_null(head, - struct drm_buddy_block, + struct gpu_buddy_block, link); if (!block) goto fallback; - while (start >= drm_buddy_block_size(mm, block)) { - start -= drm_buddy_block_size(mm, block); + while (start >= gpu_buddy_block_size(mm, block)) { + start -= gpu_buddy_block_size(mm, block); next = block->link.next; if (next != head) - block = list_entry(next, struct drm_buddy_block, + block = list_entry(next, struct gpu_buddy_block, link); } cur->mm = mm; - cur->start = drm_buddy_block_offset(block) + start; - cur->size = min(drm_buddy_block_size(mm, block) - start, + cur->start = gpu_buddy_block_offset(block) + start; + cur->size = min(gpu_buddy_block_size(mm, block) - start, size); cur->remaining = size; cur->node = block; @@ -259,7 +259,7 @@ static inline void xe_res_first_dma(const struct drm_pagemap_addr *dma_addr, */ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct list_head *next; u64 start; @@ -295,18 +295,18 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) block = cur->node; next = block->link.next; - block = list_entry(next, struct drm_buddy_block, link); + block = list_entry(next, struct gpu_buddy_block, link); - while (start >= drm_buddy_block_size(cur->mm, block)) { - start -= drm_buddy_block_size(cur->mm, block); + while (start >= gpu_buddy_block_size(cur->mm, block)) { + start -= gpu_buddy_block_size(cur->mm, block); next = block->link.next; - block = list_entry(next, struct drm_buddy_block, link); + block = list_entry(next, struct gpu_buddy_block, link); } - cur->start = drm_buddy_block_offset(block) + start; - cur->size = min(drm_buddy_block_size(cur->mm, block) - start, + cur->start = gpu_buddy_block_offset(block) + start; + cur->size = min(gpu_buddy_block_size(cur->mm, block) - start, cur->remaining); cur->node = block; break; diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index e0082b55e2162..46c71d2889218 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -176,13 +176,25 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) { struct xe_gt *gt = job->q->gt; + struct xe_device *xe = gt_to_xe(gt); bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); - u32 flags; + u32 flags0, flags; if (XE_GT_WA(gt, 14016712196)) i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH, LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0); + flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH; + /* + * Prior to MTL, HDC Pipeline Flush reliably also flushes the LSC + * untyped L1 dataport cache, provided HDC_CHICKEN0 is programmed + * correctly. Starting with MTL that coupling no longer holds + * regardless of how HDC_CHICKEN0 is programmed, but explicitly + * requesting the flush via PIPE_CONTROL is itself only reliable + * from Xe2 onward, so only gate it in on Xe2+. + */ + if (GRAPHICS_VERx100(xe) >= 2000) + flags0 |= PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH; flags = (PIPE_CONTROL_CS_STALL | PIPE_CONTROL_TILE_CACHE_FLUSH | PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | @@ -198,7 +210,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; - return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0); + return emit_pipe_control(dw, i, flags0, flags, 0, 0); } static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i) diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c index fedd017d6dd36..d9e0d30f8135e 100644 --- a/drivers/gpu/drm/xe/xe_sa.c +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -29,6 +29,7 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) kvfree(sa_manager->cpu_ptr); sa_manager->bo = NULL; + sa_manager->shadow = NULL; } /** @@ -37,12 +38,14 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) * @size: number of bytes to allocate * @guard: number of bytes to exclude from suballocations * @align: alignment for each suballocated chunk + * @flags: flags for suballocator * * Prepares the suballocation manager for suballocations. * * Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure. */ -struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align) +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags) { struct xe_device *xe = tile_to_xe(tile); struct xe_sa_manager *sa_manager; @@ -79,6 +82,26 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3 memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size); } + if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) { + struct xe_bo *shadow; + + ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard); + if (ret) + return ERR_PTR(ret); + + shadow = xe_managed_bo_create_pin_map(xe, tile, size, + XE_BO_FLAG_VRAM_IF_DGFX(tile) | + XE_BO_FLAG_GGTT | + XE_BO_FLAG_GGTT_INVALIDATE | + XE_BO_FLAG_PINNED_NORESTORE); + if (IS_ERR(shadow)) { + drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n", + size / SZ_1K, shadow); + return ERR_CAST(shadow); + } + sa_manager->shadow = shadow; + } + drm_suballoc_manager_init(&sa_manager->base, managed_size, align); ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini, sa_manager); @@ -88,6 +111,48 @@ struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u3 return sa_manager; } +/** + * xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO. + * @sa_manager: the XE sub allocator manager + * + * Swaps the sub-allocator primary buffer object with shadow buffer object. + * + * Return: None. + */ +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager) +{ + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + swap(sa_manager->bo, sa_manager->shadow); + if (!sa_manager->bo->vmap.is_iomem) + sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr; +} + +/** + * xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO. + * @sa_bo: the sub-allocator buffer object. + * + * Synchronize sub-allocator shadow buffer object with primary buffer object. + * + * Return: None. + */ +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); + + xe_assert(xe, sa_manager->shadow); + lockdep_assert_held(&sa_manager->swap_guard); + + xe_map_memcpy_to(xe, &sa_manager->shadow->vmap, + drm_suballoc_soffset(sa_bo), + xe_sa_bo_cpu_addr(sa_bo), + drm_suballoc_size(sa_bo)); +} + /** * __xe_sa_bo_new() - Make a suballocation but use custom gfp flags. * @sa_manager: the &xe_sa_manager diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h index 99dbf0eea5402..90ffa93a000ce 100644 --- a/drivers/gpu/drm/xe/xe_sa.h +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -14,12 +14,14 @@ struct dma_fence; struct xe_tile; -struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 guard, u32 align); +#define XE_SA_BO_MANAGER_FLAG_SHADOW BIT(0) +struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, + u32 guard, u32 align, u32 flags); struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp); static inline struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align) { - return __xe_sa_bo_manager_init(tile, size, SZ_4K, align); + return __xe_sa_bo_manager_init(tile, size, SZ_4K, align, 0); } /** @@ -68,4 +70,18 @@ static inline void *xe_sa_bo_cpu_addr(struct drm_suballoc *sa) drm_suballoc_soffset(sa); } +void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager); +void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo); + +/** + * xe_sa_bo_swap_guard() - Retrieve the SA BO swap guard within sub-allocator. + * @sa_manager: the &xe_sa_manager + * + * Return: Sub alloctor swap guard mutex. + */ +static inline struct mutex *xe_sa_bo_swap_guard(struct xe_sa_manager *sa_manager) +{ + return &sa_manager->swap_guard; +} + #endif diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h index cb7238799dcb2..1085c9c37d6b6 100644 --- a/drivers/gpu/drm/xe/xe_sa_types.h +++ b/drivers/gpu/drm/xe/xe_sa_types.h @@ -12,6 +12,9 @@ struct xe_bo; struct xe_sa_manager { struct drm_suballoc_manager base; struct xe_bo *bo; + struct xe_bo *shadow; + /** @swap_guard: Timeline guard updating @bo and @shadow */ + struct mutex swap_guard; void *cpu_ptr; bool is_iomem; }; diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c index 739a3eb180b53..e03714ef7ae99 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.c @@ -148,7 +148,8 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) xe_sriov_info(xe, "Allocating %s CCS BB pool size = %lldMB\n", ctx->ctx_id ? "Restore" : "Save", bb_pool_size / SZ_1M); - sa_manager = xe_sa_bo_manager_init(tile, bb_pool_size, SZ_16); + sa_manager = __xe_sa_bo_manager_init(tile, bb_pool_size, SZ_4K, SZ_16, + XE_SA_BO_MANAGER_FLAG_SHADOW); if (IS_ERR(sa_manager)) { xe_sriov_err(xe, "Suballocator init failed with error: %pe\n", @@ -160,9 +161,12 @@ static int alloc_bb_pool(struct xe_tile *tile, struct xe_sriov_vf_ccs_ctx *ctx) offset = 0; xe_map_memset(xe, &sa_manager->bo->vmap, offset, MI_NOOP, bb_pool_size); + xe_map_memset(xe, &sa_manager->shadow->vmap, offset, MI_NOOP, + bb_pool_size); offset = bb_pool_size - sizeof(u32); xe_map_wr(xe, &sa_manager->bo->vmap, offset, u32, MI_BATCH_BUFFER_END); + xe_map_wr(xe, &sa_manager->shadow->vmap, offset, u32, MI_BATCH_BUFFER_END); ctx->mem.ccs_bb_pool = sa_manager; @@ -311,9 +315,23 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) return err; } +#define XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET (2 * sizeof(u32)) +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx) +{ + u64 addr = xe_sa_manager_gpu_addr(ctx->mem.ccs_bb_pool); + struct xe_lrc *lrc = xe_exec_queue_lrc(ctx->mig_q); + struct xe_device *xe = gt_to_xe(ctx->mig_q->gt); + + xe_device_wmb(xe); + xe_map_wr(xe, &lrc->bo->vmap, XE_SRIOV_VF_CCS_RW_BB_ADDR_OFFSET, u32, addr); + xe_device_wmb(xe); +} + /** * xe_sriov_vf_ccs_attach_bo - Insert CCS read write commands in the BO. * @bo: the &buffer object to which batch buffer commands will be added. + * @new_mem: the (not yet committed) destination resource @bo is being moved + * into; bo->ttm.resource is still the old resource at this point. * * This function shall be called only by VF. It inserts the PTEs and copy * command instructions in the BO by calling xe_migrate_ccs_rw_copy() @@ -321,7 +339,7 @@ int xe_sriov_vf_ccs_init(struct xe_device *xe) * * Returns: 0 if successful, negative error code on failure. */ -int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem) { struct xe_device *xe = xe_bo_device(bo); enum xe_sriov_vf_ccs_rw_ctxs ctx_id; @@ -340,7 +358,21 @@ int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo) xe_assert(xe, !bb); ctx = &xe->sriov.vf.ccs.contexts[ctx_id]; - err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, ctx_id); + err = xe_migrate_ccs_rw_copy(tile, ctx->mig_q, bo, new_mem, ctx_id); + if (err) + goto err_unwind; + } + return 0; + +err_unwind: + /* + * Clean up any contexts already attached. Can't reuse + * xe_sriov_vf_ccs_detach_bo() here as it requires both contexts + * attached before cleaning up either one. + */ + for_each_ccs_rw_ctx(ctx_id) { + if (bo->bb_ccs[ctx_id]) + xe_migrate_ccs_rw_copy_clear(bo, ctx_id); } return err; } @@ -371,9 +403,7 @@ int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo) if (!bb) continue; - memset(bb->cs, MI_NOOP, bb->len * sizeof(u32)); - xe_bb_free(bb, NULL); - bo->bb_ccs[ctx_id] = NULL; + xe_migrate_ccs_rw_copy_clear(bo, ctx_id); } return 0; } diff --git a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h index 0745c0ff02282..00d2c9dbe00d9 100644 --- a/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h +++ b/drivers/gpu/drm/xe/xe_sriov_vf_ccs.h @@ -11,14 +11,16 @@ #include "xe_sriov_vf_ccs_types.h" struct drm_printer; +struct ttm_resource; struct xe_device; struct xe_bo; int xe_sriov_vf_ccs_init(struct xe_device *xe); -int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo); +int xe_sriov_vf_ccs_attach_bo(struct xe_bo *bo, struct ttm_resource *new_mem); int xe_sriov_vf_ccs_detach_bo(struct xe_bo *bo); int xe_sriov_vf_ccs_register_context(struct xe_device *xe); void xe_sriov_vf_ccs_print(struct xe_device *xe, struct drm_printer *p); +void xe_sriov_vf_ccs_rw_update_bb_addr(struct xe_sriov_vf_ccs_ctx *ctx); static inline bool xe_sriov_vf_ccs_ready(struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c index 65452c20609ed..e1b8030b6bcf5 100644 --- a/drivers/gpu/drm/xe/xe_svm.c +++ b/drivers/gpu/drm/xe/xe_svm.c @@ -689,7 +689,7 @@ static u64 block_offset_to_pfn(struct xe_vram_region *vr, u64 offset) return PHYS_PFN(offset + vr->hpa_base); } -static struct drm_buddy *vram_to_buddy(struct xe_vram_region *vram) +static struct gpu_buddy *vram_to_buddy(struct xe_vram_region *vram) { return &vram->ttm.mm; } @@ -700,16 +700,16 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati struct xe_bo *bo = to_xe_bo(devmem_allocation); struct ttm_resource *res = bo->ttm.resource; struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; int j = 0; list_for_each_entry(block, blocks, link) { struct xe_vram_region *vr = block->private; - struct drm_buddy *buddy = vram_to_buddy(vr); - u64 block_pfn = block_offset_to_pfn(vr, drm_buddy_block_offset(block)); + struct gpu_buddy *buddy = vram_to_buddy(vr); + u64 block_pfn = block_offset_to_pfn(vr, gpu_buddy_block_offset(block)); int i; - for (i = 0; i < drm_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i) + for (i = 0; i < gpu_buddy_block_size(buddy, block) >> PAGE_SHIFT; ++i) pfn[j++] = block_pfn + i; } @@ -877,7 +877,7 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, struct dma_fence *pre_migrate_fence = NULL; struct xe_device *xe = vr->xe; struct device *dev = xe->drm.dev; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; struct xe_validation_ctx vctx; struct list_head *blocks; struct drm_exec exec; @@ -1074,10 +1074,8 @@ static int __xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, xe_svm_range_fault_count_stats_incr(gt, range); - if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem) { - err = -EACCES; - goto out; - } + if (ctx.devmem_only && !range->base.pages.flags.migrate_devmem) + return -EACCES; if (xe_svm_range_is_valid(range, tile, ctx.devmem_only)) { xe_svm_range_valid_fault_count_stats_incr(gt, range); diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 9175b4a2214b8..f3d72773835cc 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -17,16 +18,16 @@ #include "xe_ttm_vram_mgr.h" #include "xe_vram_types.h" -static inline struct drm_buddy_block * +static inline struct gpu_buddy_block * xe_ttm_vram_mgr_first_block(struct list_head *list) { - return list_first_entry_or_null(list, struct drm_buddy_block, link); + return list_first_entry_or_null(list, struct gpu_buddy_block, link); } -static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm, +static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm, struct list_head *head) { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; u64 start, size; block = xe_ttm_vram_mgr_first_block(head); @@ -34,12 +35,12 @@ static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm, return false; while (head != block->link.next) { - start = drm_buddy_block_offset(block); - size = drm_buddy_block_size(mm, block); + start = gpu_buddy_block_offset(block); + size = gpu_buddy_block_size(mm, block); - block = list_entry(block->link.next, struct drm_buddy_block, + block = list_entry(block->link.next, struct gpu_buddy_block, link); - if (start + size != drm_buddy_block_offset(block)) + if (start + size != gpu_buddy_block_offset(block)) return false; } @@ -53,7 +54,7 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, { struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr_resource *vres; - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; u64 size, min_page_size; unsigned long lpfn; int err; @@ -80,10 +81,10 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, INIT_LIST_HEAD(&vres->blocks); if (place->flags & TTM_PL_FLAG_TOPDOWN) - vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) - vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; + vres->flags |= GPU_BUDDY_RANGE_ALLOCATION; if (WARN_ON(!vres->base.size)) { err = -EINVAL; @@ -119,27 +120,27 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn); } - err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, + err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, (u64)lpfn << PAGE_SHIFT, size, min_page_size, &vres->blocks, vres->flags); if (err) goto error_unlock; if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { - if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks)) + if (!gpu_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks)) size = vres->base.size; } if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { vres->used_visible_size = size; } else { - struct drm_buddy_block *block; + struct gpu_buddy_block *block; list_for_each_entry(block, &vres->blocks, link) { - u64 start = drm_buddy_block_offset(block); + u64 start = gpu_buddy_block_offset(block); if (start < mgr->visible_size) { - u64 end = start + drm_buddy_block_size(mm, block); + u64 end = start + gpu_buddy_block_size(mm, block); vres->used_visible_size += min(end, mgr->visible_size) - start; @@ -159,11 +160,11 @@ static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, * the object. */ if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { - struct drm_buddy_block *block = list_first_entry(&vres->blocks, + struct gpu_buddy_block *block = list_first_entry(&vres->blocks, typeof(*block), link); - vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT; + vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT; } else { vres->base.start = XE_BO_INVALID_OFFSET; } @@ -185,10 +186,10 @@ static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; mutex_lock(&mgr->lock); - drm_buddy_free_list(mm, &vres->blocks, 0); + gpu_buddy_free_list(mm, &vres->blocks, 0); mgr->visible_avail += vres->used_visible_size; mutex_unlock(&mgr->lock); @@ -201,7 +202,7 @@ static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); - struct drm_buddy *mm = &mgr->mm; + struct gpu_buddy *mm = &mgr->mm; mutex_lock(&mgr->lock); drm_printf(printer, "default_page_size: %lluKiB\n", @@ -224,8 +225,8 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; if (!place->fpfn && !place->lpfn) return true; @@ -235,9 +236,9 @@ static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, list_for_each_entry(block, &vres->blocks, link) { unsigned long fpfn = - drm_buddy_block_offset(block) >> PAGE_SHIFT; + gpu_buddy_block_offset(block) >> PAGE_SHIFT; unsigned long lpfn = fpfn + - (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); if (place->fpfn < lpfn && place->lpfn > fpfn) return true; @@ -254,8 +255,8 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); - struct drm_buddy *mm = &mgr->mm; - struct drm_buddy_block *block; + struct gpu_buddy *mm = &mgr->mm; + struct gpu_buddy_block *block; if (!place->fpfn && !place->lpfn) return true; @@ -265,9 +266,9 @@ static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, list_for_each_entry(block, &vres->blocks, link) { unsigned long fpfn = - drm_buddy_block_offset(block) >> PAGE_SHIFT; + gpu_buddy_block_offset(block) >> PAGE_SHIFT; unsigned long lpfn = fpfn + - (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); + (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); if (fpfn < place->fpfn || lpfn > place->lpfn) return false; @@ -297,7 +298,7 @@ static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg) WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); - drm_buddy_fini(&mgr->mm); + gpu_buddy_fini(&mgr->mm); ttm_resource_manager_cleanup(&mgr->manager); @@ -328,7 +329,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, mgr->visible_avail = io_size; ttm_resource_manager_init(man, &xe->ttm, size); - err = drm_buddy_init(&mgr->mm, man->size, default_page_size); + err = gpu_buddy_init(&mgr->mm, man->size, default_page_size); if (err) return err; @@ -376,7 +377,7 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, if (!*sgt) return -ENOMEM; - /* Determine the number of DRM_BUDDY blocks to export */ + /* Determine the number of GPU_BUDDY blocks to export */ xe_res_first(res, offset, length, &cursor); while (cursor.remaining) { num_entries++; @@ -393,10 +394,10 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, sg->length = 0; /* - * Walk down DRM_BUDDY blocks to populate scatterlist nodes - * @note: Use iterator api to get first the DRM_BUDDY block + * Walk down GPU_BUDDY blocks to populate scatterlist nodes + * @note: Use iterator api to get first the GPU_BUDDY block * and the number of bytes from it. Access the following - * DRM_BUDDY block(s) if more buffer needs to exported + * GPU_BUDDY block(s) if more buffer needs to exported */ xe_res_first(res, offset, length, &cursor); for_each_sgtable_sg((*sgt), sg, i) { diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h index 1144f9232ebbb..674a846f99862 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h @@ -6,7 +6,7 @@ #ifndef _XE_TTM_VRAM_MGR_TYPES_H_ #define _XE_TTM_VRAM_MGR_TYPES_H_ -#include +#include #include /** @@ -18,7 +18,7 @@ struct xe_ttm_vram_mgr { /** @manager: Base TTM resource manager */ struct ttm_resource_manager manager; /** @mm: DRM buddy allocator which manages the VRAM */ - struct drm_buddy mm; + struct gpu_buddy mm; /** @visible_size: Proped size of the CPU visible portion */ u64 visible_size; /** @visible_avail: CPU visible portion still unallocated */ diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c index 465bda3554439..b799efc838017 100644 --- a/drivers/gpu/drm/xe/xe_uc.c +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -15,6 +15,7 @@ #include "xe_guc.h" #include "xe_guc_pc.h" #include "xe_guc_engine_activity.h" +#include "xe_guc_submit.h" #include "xe_huc.h" #include "xe_sriov.h" #include "xe_uc_fw.h" @@ -160,12 +161,14 @@ static int vf_uc_load_hw(struct xe_uc *uc) if (err) goto err_out; - uc->guc.submission_state.enabled = true; - err = xe_guc_opt_in_features_enable(&uc->guc); if (err) goto err_out; + err = xe_guc_submit_enable(&uc->guc); + if (err) + return err; + err = xe_gt_record_default_lrcs(uc_to_gt(uc)); if (err) goto err_out; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 8f7b8f2da06ba..423705bcd94c1 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -355,7 +355,6 @@ int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, unsigned int num_fences) { struct drm_gem_object *obj; - unsigned long index; int ret; do { @@ -368,7 +367,7 @@ int xe_vm_validate_rebind(struct xe_vm *vm, struct drm_exec *exec, return ret; } while (!list_empty(&vm->gpuvm.evict.list)); - drm_exec_for_each_locked_object(exec, index, obj) { + drm_exec_for_each_locked_object(exec, obj) { ret = dma_resv_reserve_fences(obj->resv, num_fences); if (ret) return ret; @@ -1626,10 +1625,10 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags, struct xe_file *xef) return ERR_PTR(err); err_svm_fini: - if (flags & XE_VM_FLAG_FAULT_MODE) { - vm->size = 0; /* close the vm */ - xe_svm_fini(vm); - } + vm->size = 0; /* close the vm */ + if (flags & XE_VM_FLAG_FAULT_MODE) + xe_svm_close(vm); + xe_svm_fini(vm); err_no_resv: mutex_destroy(&vm->snap_mutex); for_each_tile(tile, xe, id) @@ -2834,8 +2833,22 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, } } +/** + * struct xe_vma_lock_and_validate_flags - Flags for vma_lock_and_validate() + * @res_evict: Allow evicting resources during validation + * @validate: Perform BO validation + * @request_decompress: Request BO decompression + * @check_purged: Reject operation if BO is purged + */ +struct xe_vma_lock_and_validate_flags { + u32 res_evict : 1; + u32 validate : 1; + u32 request_decompress : 1; + u32 check_purged : 1; +}; + static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, - bool res_evict, bool validate) + struct xe_vma_lock_and_validate_flags flags) { struct xe_bo *bo = xe_vma_bo(vma); struct xe_vm *vm = xe_vma_vm(vma); @@ -2844,10 +2857,19 @@ static int vma_lock_and_validate(struct drm_exec *exec, struct xe_vma *vma, if (bo) { if (!bo->vm) err = drm_exec_lock_obj(exec, &bo->ttm.base); - if (!err && validate) + + /* Reject new mappings to DONTNEED/purged BOs; allow cleanup operations */ + if (!err && flags.check_purged) { + if (xe_bo_madv_is_dontneed(bo)) + err = -EBUSY; /* BO marked purgeable */ + else if (xe_bo_is_purged(bo)) + err = -EINVAL; /* BO already purged */ + } + + if (!err && flags.validate) err = xe_bo_validate(bo, vm, !xe_vm_in_preempt_fence_mode(vm) && - res_evict, exec); + flags.res_evict, exec); } return err; @@ -2933,9 +2955,13 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, case DRM_GPUVA_OP_MAP: if (!op->map.invalidate_on_bind) err = vma_lock_and_validate(exec, op->map.vma, - res_evict, - !xe_vm_in_fault_mode(vm) || - op->map.immediate); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = !xe_vm_in_fault_mode(vm) || + op->map.immediate, + .request_decompress = false, + .check_purged = true, + }); break; case DRM_GPUVA_OP_REMAP: err = check_ufence(gpuva_to_vma(op->base.remap.unmap->va)); @@ -2944,13 +2970,28 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.remap.unmap->va), - res_evict, false); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = false, + }); if (!err && op->remap.prev) err = vma_lock_and_validate(exec, op->remap.prev, - res_evict, true); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = true, + .request_decompress = false, + .check_purged = true, + }); if (!err && op->remap.next) err = vma_lock_and_validate(exec, op->remap.next, - res_evict, true); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = true, + .request_decompress = false, + .check_purged = true, + }); break; case DRM_GPUVA_OP_UNMAP: err = check_ufence(gpuva_to_vma(op->base.unmap.va)); @@ -2959,7 +3000,12 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.unmap.va), - res_evict, false); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = false, + }); break; case DRM_GPUVA_OP_PREFETCH: { @@ -2972,14 +3018,39 @@ static int op_lock_and_prep(struct drm_exec *exec, struct xe_vm *vm, region <= ARRAY_SIZE(region_to_mem_type)); } + /* + * Prefetch attempts to migrate BO's backing store without + * repopulating it first. Purged BOs have no backing store + * to migrate, so reject the operation. + */ err = vma_lock_and_validate(exec, gpuva_to_vma(op->base.prefetch.va), - res_evict, false); - if (!err && !xe_vma_has_no_bo(vma)) - err = xe_bo_migrate(xe_vma_bo(vma), - region_to_mem_type[region], - NULL, - exec); + (struct xe_vma_lock_and_validate_flags) { + .res_evict = res_evict, + .validate = false, + .request_decompress = false, + .check_purged = true, + }); + if (!err && !xe_vma_has_no_bo(vma)) { + struct xe_bo *bo = xe_vma_bo(vma); + u32 mem_type; + + if (region == DRM_XE_CONSULT_MEM_ADVISE_PREF_LOC) { + unsigned int i; + + mem_type = XE_PL_TT; + for (i = 0; i < bo->placement.num_placement; i++) { + if (mem_type_is_vram(bo->placements[i].mem_type)) { + mem_type = bo->placements[i].mem_type; + break; + } + } + } else { + mem_type = region_to_mem_type[region]; + } + + err = xe_bo_migrate(bo, mem_type, NULL, exec); + } break; } default: @@ -3393,6 +3464,8 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, !IS_DGFX(xe) && coh_mode == XE_COH_NONE && + is_cpu_addr_mirror) || XE_IOCTL_DBG(xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR && !IS_ENABLED(CONFIG_DRM_GPUSVM)) || XE_IOCTL_DBG(xe, obj && diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c index a4076f56a3c98..8f6f9bcb68506 100644 --- a/drivers/gpu/drm/xe/xe_vram.c +++ b/drivers/gpu/drm/xe/xe_vram.c @@ -206,12 +206,28 @@ static inline u64 get_flat_ccs_offset(struct xe_gt *gt, u64 tile_size) offset = offset_hi << 32; /* HW view bits 39:32 */ offset |= offset_lo << 6; /* HW view bits 31:6 */ offset *= num_enabled; /* convert to SW view */ - offset = round_up(offset, SZ_128K); /* SW must round up to nearest 128K */ - /* We don't expect any holes */ - xe_assert_msg(xe, offset == (xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE) - - ccs_size), - "Hole between CCS and GSM.\n"); + drm_info(&xe->drm, "FLAT_CCS base:%llx, aligned:%s\n", offset, + str_yes_no(IS_ALIGNED(offset, SZ_128K))); + + /* + * Everything below this offset is handed to the VRAM + * allocator, so it has to be the *first* address the + * compression hardware owns, rounded down. Rounding it up + * publishes CCS storage as free memory. + */ + offset = round_down(offset, SZ_4K); + + /* + * CCS storage must not run into GSM. The old check compared + * the offset against GSMBASE - ccs_size for equality, which + * could not fail: that value is 128K aligned, so it agreed + * with the rounded-up offset even when the base was not 128K + * aligned - exactly the case this fixes. + */ + xe_assert_msg(xe, offset + ccs_size <= + xe_mmio_read64_2x32(>_to_tile(gt)->mmio, GSMBASE), + "CCS overlaps GSM.\n"); } else { reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); offset = (u64)REG_FIELD_GET(XEHP_FLAT_CCS_PTR, reg) * SZ_64K; diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c index ada0d0aa6b749..2fd2a82be89f8 100644 --- a/drivers/gpu/drm/xe/xe_wopcm.c +++ b/drivers/gpu/drm/xe/xe_wopcm.c @@ -49,9 +49,9 @@ */ /* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */ -/* FIXME: Larger size require for 2 tile PVC, do a proper probe sooner or later */ +/* FIXME: Larger size require for some platforms, do a proper probe sooner or later */ #define DGFX_WOPCM_SIZE SZ_4M -/* FIXME: Larger size require for MTL, do a proper probe sooner or later */ +#define LNL_WOPCM_SIZE SZ_8M #define MTL_WOPCM_SIZE SZ_4M #define WOPCM_SIZE SZ_2M @@ -181,9 +181,14 @@ static int __wopcm_init_regs(struct xe_device *xe, struct xe_gt *gt, u32 xe_wopcm_size(struct xe_device *xe) { - return IS_DGFX(xe) ? DGFX_WOPCM_SIZE : - xe->info.platform == XE_METEORLAKE ? MTL_WOPCM_SIZE : - WOPCM_SIZE; + if (xe->info.platform >= XE_LUNARLAKE) + return LNL_WOPCM_SIZE; + else if (IS_DGFX(xe)) + return DGFX_WOPCM_SIZE; + else if (xe->info.platform == XE_METEORLAKE) + return MTL_WOPCM_SIZE; + else + return WOPCM_SIZE; } /** diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c index 3801eab63282f..06aa7fcd367e1 100644 --- a/drivers/gpu/host1x/bus.c +++ b/drivers/gpu/host1x/bus.c @@ -221,7 +221,7 @@ int host1x_device_init(struct host1x_device *device) teardown: list_for_each_entry_continue_reverse(client, &device->clients, list) - if (client->ops->exit) + if (client->ops && client->ops->exit) client->ops->exit(client); /* reset client to end of list for late teardown */ @@ -229,7 +229,7 @@ int host1x_device_init(struct host1x_device *device) teardown_late: list_for_each_entry_continue_reverse(client, &device->clients, list) - if (client->ops->late_exit) + if (client->ops && client->ops->late_exit) client->ops->late_exit(client); mutex_unlock(&device->clients_lock); @@ -1001,10 +1001,10 @@ void host1x_bo_clear_cached_mappings(struct host1x_bo *bo) if (WARN_ON(!cache)) continue; - mutex_lock(&mapping->cache->lock); + mutex_lock(&cache->lock); WARN_ON(kref_read(&mapping->ref) != 1); __host1x_bo_unpin(&mapping->ref); - mutex_unlock(&mapping->cache->lock); + mutex_unlock(&cache->lock); } } EXPORT_SYMBOL(host1x_bo_clear_cached_mappings); diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c index 6433c00d5d7e0..b828f773fc065 100644 --- a/drivers/gpu/host1x/debug.c +++ b/drivers/gpu/host1x/debug.c @@ -31,7 +31,7 @@ void host1x_debug_output(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, false); @@ -43,7 +43,7 @@ void host1x_debug_cont(struct output *o, const char *fmt, ...) int len; va_start(args, fmt); - len = vsnprintf(o->buf, sizeof(o->buf), fmt, args); + len = vscnprintf(o->buf, sizeof(o->buf), fmt, args); va_end(args); o->fn(o->ctx, o->buf, len, true); diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c index d44b8de890be0..46b6494768b4d 100644 --- a/drivers/gpu/host1x/hw/channel_hw.c +++ b/drivers/gpu/host1x/hw/channel_hw.c @@ -36,10 +36,9 @@ static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo, for (i = 0; i < words; i += TRACE_MAX_LENGTH) { u32 num_words = min(words - i, TRACE_MAX_LENGTH); - offset += i * sizeof(u32); - trace_host1x_cdma_push_gather(dev_name(dev), bo, - num_words, offset, + num_words, + offset + i * sizeof(u32), mem); } diff --git a/drivers/gpu/tests/Makefile b/drivers/gpu/tests/Makefile new file mode 100644 index 0000000000000..4183e6e2de451 --- /dev/null +++ b/drivers/gpu/tests/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +gpu_buddy_tests-y = gpu_buddy_test.o gpu_random.o +obj-$(CONFIG_GPU_BUDDY_KUNIT_TEST) += gpu_buddy_tests.o diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/tests/gpu_buddy_test.c similarity index 66% rename from drivers/gpu/drm/tests/drm_buddy_test.c rename to drivers/gpu/tests/gpu_buddy_test.c index 7a0e523651f05..3f689e1f3581a 100644 --- a/drivers/gpu/drm/tests/drm_buddy_test.c +++ b/drivers/gpu/tests/gpu_buddy_test.c @@ -10,9 +10,9 @@ #include #include -#include +#include -#include "../lib/drm_random.h" +#include "gpu_random.h" static unsigned int random_seed; @@ -21,14 +21,14 @@ static inline u64 get_size(int order, u64 chunk_size) return (1 << order) * chunk_size; } -static void drm_test_buddy_alloc_range_bias(struct kunit *test) +static void gpu_test_buddy_alloc_range_bias(struct kunit *test) { u32 mm_size, size, ps, bias_size, bias_start, bias_end, bias_rem; - DRM_RND_STATE(prng, random_seed); + GPU_RND_STATE(prng, random_seed); unsigned int i, count, *order; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; unsigned long flags; - struct drm_buddy mm; + struct gpu_buddy mm; LIST_HEAD(allocated); bias_size = SZ_1M; @@ -38,11 +38,11 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) kunit_info(test, "mm_size=%u, ps=%u\n", mm_size, ps); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps), "buddy_init failed\n"); count = mm_size / bias_size; - order = drm_random_order(count, &prng); + order = gpu_random_order(count, &prng); KUNIT_EXPECT_TRUE(test, order); /* @@ -62,79 +62,79 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) /* internal round_up too big */ KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, bias_size + ps, bias_size, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, bias_size, bias_size); /* size too big */ KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, bias_size + ps, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, bias_size + ps, ps); /* bias range too small for size */ KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start + ps, + gpu_buddy_alloc_blocks(&mm, bias_start + ps, bias_end, bias_size, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n", bias_start + ps, bias_end, bias_size, ps); /* bias misaligned */ KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start + ps, + gpu_buddy_alloc_blocks(&mm, bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc h didn't fail with bias(%x-%x), size=%u, ps=%u\n", bias_start + ps, bias_end - ps, bias_size >> 1, bias_size >> 1); /* single big page */ KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, bias_size, bias_size, &tmp, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc i failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, bias_size, bias_size); - drm_buddy_free_list(&mm, &tmp, 0); + gpu_buddy_free_list(&mm, &tmp, 0); /* single page with internal round_up */ KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, ps, bias_size, &tmp, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, ps, bias_size); - drm_buddy_free_list(&mm, &tmp, 0); + gpu_buddy_free_list(&mm, &tmp, 0); /* random size within */ size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps); if (size) KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, size, ps, &tmp, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, size, ps); bias_rem -= size; /* too big for current avail */ KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, bias_rem + ps, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc didn't fail with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, bias_rem + ps, ps); @@ -144,10 +144,10 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) size = max(size, ps); KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, size, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, size, ps); /* @@ -155,15 +155,15 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) * unallocated, and ideally not always on the bias * boundaries. */ - drm_buddy_free_list(&mm, &tmp, 0); + gpu_buddy_free_list(&mm, &tmp, 0); } else { list_splice_tail(&tmp, &allocated); } } kfree(order); - drm_buddy_free_list(&mm, &allocated, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_fini(&mm); /* * Something more free-form. Idea is to pick a random starting bias @@ -174,7 +174,7 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) * allocated nodes in the middle of the address space. */ - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps), "buddy_init failed\n"); bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps); @@ -186,10 +186,10 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) u32 size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps); KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, size, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc failed with bias(%x-%x), size=%u, ps=%u\n", bias_start, bias_end, size, ps); bias_rem -= size; @@ -215,24 +215,24 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) KUNIT_ASSERT_EQ(test, bias_start, 0); KUNIT_ASSERT_EQ(test, bias_end, mm_size); KUNIT_ASSERT_TRUE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, bias_end, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, ps, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc passed with bias(%x-%x), size=%u\n", bias_start, bias_end, ps); - drm_buddy_free_list(&mm, &allocated, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_fini(&mm); /* - * Allocate cleared blocks in the bias range when the DRM buddy's clear avail is + * Allocate cleared blocks in the bias range when the GPU buddy's clear avail is * zero. This will validate the bias range allocation in scenarios like system boot * when no cleared blocks are available and exercise the fallback path too. The resulting * blocks should always be dirty. */ - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, ps), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, ps), "buddy_init failed\n"); bias_start = round_up(prandom_u32_state(&prng) % (mm_size - ps), ps); @@ -240,11 +240,11 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) bias_end = max(bias_end, bias_start + ps); bias_rem = bias_end - bias_start; - flags = DRM_BUDDY_CLEAR_ALLOCATION | DRM_BUDDY_RANGE_ALLOCATION; + flags = GPU_BUDDY_CLEAR_ALLOCATION | GPU_BUDDY_RANGE_ALLOCATION; size = max(round_up(prandom_u32_state(&prng) % bias_rem, ps), ps); KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, bias_start, + gpu_buddy_alloc_blocks(&mm, bias_start, bias_end, size, ps, &allocated, flags), @@ -252,27 +252,27 @@ static void drm_test_buddy_alloc_range_bias(struct kunit *test) bias_start, bias_end, size, ps); list_for_each_entry(block, &allocated, link) - KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false); + KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false); - drm_buddy_free_list(&mm, &allocated, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_clear(struct kunit *test) +static void gpu_test_buddy_alloc_clear(struct kunit *test) { unsigned long n_pages, total, i = 0; const unsigned long ps = SZ_4K; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; const int max_order = 12; LIST_HEAD(allocated); - struct drm_buddy mm; + struct gpu_buddy mm; unsigned int order; u32 mm_size, size; LIST_HEAD(dirty); LIST_HEAD(clean); mm_size = SZ_4K << max_order; - KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps)); + KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps)); KUNIT_EXPECT_EQ(test, mm.max_order, max_order); @@ -285,11 +285,11 @@ static void drm_test_buddy_alloc_clear(struct kunit *test) * is indeed all dirty pages and vice versa. Free it all again, * keeping the dirty/clear status. */ - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 5 * ps, ps, &allocated, - DRM_BUDDY_TOPDOWN_ALLOCATION), + GPU_BUDDY_TOPDOWN_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 5 * ps); - drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED); + gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED); n_pages = 10; do { @@ -302,37 +302,37 @@ static void drm_test_buddy_alloc_clear(struct kunit *test) flags = 0; } else { list = &clean; - flags = DRM_BUDDY_CLEAR_ALLOCATION; + flags = GPU_BUDDY_CLEAR_ALLOCATION; } - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, ps, ps, list, flags), "buddy_alloc hit an error size=%lu\n", ps); } while (++i < n_pages); list_for_each_entry(block, &clean, link) - KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), true); + KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), true); list_for_each_entry(block, &dirty, link) - KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false); + KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false); - drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED); + gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED); /* * Trying to go over the clear limit for some allocation. * The allocation should never fail with reasonable page-size. */ - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 10 * ps, ps, &clean, - DRM_BUDDY_CLEAR_ALLOCATION), + GPU_BUDDY_CLEAR_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 10 * ps); - drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED); - drm_buddy_free_list(&mm, &dirty, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED); + gpu_buddy_free_list(&mm, &dirty, 0); + gpu_buddy_fini(&mm); - KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps)); + KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps)); /* * Create a new mm. Intentionally fragment the address space by creating @@ -354,34 +354,34 @@ static void drm_test_buddy_alloc_clear(struct kunit *test) else list = &clean; - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, ps, ps, list, 0), "buddy_alloc hit an error size=%lu\n", ps); } while (++i < n_pages); - drm_buddy_free_list(&mm, &clean, DRM_BUDDY_CLEARED); - drm_buddy_free_list(&mm, &dirty, 0); + gpu_buddy_free_list(&mm, &clean, GPU_BUDDY_CLEARED); + gpu_buddy_free_list(&mm, &dirty, 0); order = 1; do { size = SZ_4K << order; - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, size, size, &allocated, - DRM_BUDDY_CLEAR_ALLOCATION), + GPU_BUDDY_CLEAR_ALLOCATION), "buddy_alloc hit an error size=%u\n", size); total = 0; list_for_each_entry(block, &allocated, link) { if (size != mm_size) - KUNIT_EXPECT_EQ(test, drm_buddy_block_is_clear(block), false); - total += drm_buddy_block_size(&mm, block); + KUNIT_EXPECT_EQ(test, gpu_buddy_block_is_clear(block), false); + total += gpu_buddy_block_size(&mm, block); } KUNIT_EXPECT_EQ(test, total, size); - drm_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_free_list(&mm, &allocated, 0); } while (++order <= max_order); - drm_buddy_fini(&mm); + gpu_buddy_fini(&mm); /* * Create a new mm with a non power-of-two size. Allocate a random size from each @@ -390,44 +390,44 @@ static void drm_test_buddy_alloc_clear(struct kunit *test) */ mm_size = (SZ_4K << max_order) + (SZ_4K << (max_order - 2)); - KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps)); + KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps)); KUNIT_EXPECT_EQ(test, mm.max_order, max_order); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order, 4 * ps, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 4 * ps); - drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order, + gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED); + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, SZ_4K << max_order, 2 * ps, ps, &allocated, - DRM_BUDDY_CLEAR_ALLOCATION), + GPU_BUDDY_CLEAR_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 2 * ps); - drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, SZ_4K << max_order, mm_size, + gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED); + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, SZ_4K << max_order, mm_size, ps, ps, &allocated, - DRM_BUDDY_RANGE_ALLOCATION), + GPU_BUDDY_RANGE_ALLOCATION), "buddy_alloc hit an error size=%lu\n", ps); - drm_buddy_free_list(&mm, &allocated, DRM_BUDDY_CLEARED); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, GPU_BUDDY_CLEARED); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_contiguous(struct kunit *test) +static void gpu_test_buddy_alloc_contiguous(struct kunit *test) { const unsigned long ps = SZ_4K, mm_size = 16 * 3 * SZ_4K; unsigned long i, n_pages, total; - struct drm_buddy_block *block; - struct drm_buddy mm; + struct gpu_buddy_block *block; + struct gpu_buddy mm; LIST_HEAD(left); LIST_HEAD(middle); LIST_HEAD(right); LIST_HEAD(allocated); - KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, mm_size, ps)); + KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, mm_size, ps)); /* * Idea is to fragment the address space by alternating block * allocations between three different lists; one for left, middle and * right. We can then free a list to simulate fragmentation. In - * particular we want to exercise the DRM_BUDDY_CONTIGUOUS_ALLOCATION, + * particular we want to exercise the GPU_BUDDY_CONTIGUOUS_ALLOCATION, * including the try_harder path. */ @@ -444,66 +444,66 @@ static void drm_test_buddy_alloc_contiguous(struct kunit *test) else list = &right; KUNIT_ASSERT_FALSE_MSG(test, - drm_buddy_alloc_blocks(&mm, 0, mm_size, + gpu_buddy_alloc_blocks(&mm, 0, mm_size, ps, ps, list, 0), "buddy_alloc hit an error size=%lu\n", ps); } while (++i < n_pages); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 3 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc didn't error size=%lu\n", 3 * ps); - drm_buddy_free_list(&mm, &middle, 0); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + gpu_buddy_free_list(&mm, &middle, 0); + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 3 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc didn't error size=%lu\n", 3 * ps); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 2 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc didn't error size=%lu\n", 2 * ps); - drm_buddy_free_list(&mm, &right, 0); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + gpu_buddy_free_list(&mm, &right, 0); + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 3 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc didn't error size=%lu\n", 3 * ps); /* * At this point we should have enough contiguous space for 2 blocks, * however they are never buddies (since we freed middle and right) so * will require the try_harder logic to find them. */ - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 2 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 2 * ps); - drm_buddy_free_list(&mm, &left, 0); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, 0, mm_size, + gpu_buddy_free_list(&mm, &left, 0); + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, 0, mm_size, 3 * ps, ps, &allocated, - DRM_BUDDY_CONTIGUOUS_ALLOCATION), + GPU_BUDDY_CONTIGUOUS_ALLOCATION), "buddy_alloc hit an error size=%lu\n", 3 * ps); total = 0; list_for_each_entry(block, &allocated, link) - total += drm_buddy_block_size(&mm, block); + total += gpu_buddy_block_size(&mm, block); KUNIT_ASSERT_EQ(test, total, ps * 2 + ps * 3); - drm_buddy_free_list(&mm, &allocated, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_pathological(struct kunit *test) +static void gpu_test_buddy_alloc_pathological(struct kunit *test) { u64 mm_size, size, start = 0; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; const int max_order = 3; unsigned long flags = 0; int order, top; - struct drm_buddy mm; + struct gpu_buddy mm; LIST_HEAD(blocks); LIST_HEAD(holes); LIST_HEAD(tmp); @@ -516,7 +516,7 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test) */ mm_size = SZ_4K << max_order; - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K), "buddy_init failed\n"); KUNIT_EXPECT_EQ(test, mm.max_order, max_order); @@ -526,18 +526,18 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test) block = list_first_entry_or_null(&blocks, typeof(*block), link); if (block) { list_del(&block->link); - drm_buddy_free_block(&mm, block); + gpu_buddy_free_block(&mm, block); } for (order = top; order--;) { size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM with order=%d, top=%d\n", order, top); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_move_tail(&block->link, &blocks); @@ -545,45 +545,45 @@ static void drm_test_buddy_alloc_pathological(struct kunit *test) /* There should be one final page for this sub-allocation */ size = get_size(0, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM for hole\n"); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_move_tail(&block->link, &holes); size = get_size(top, mm.chunk_size); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!", top, max_order); } - drm_buddy_free_list(&mm, &holes, 0); + gpu_buddy_free_list(&mm, &holes, 0); /* Nothing larger than blocks of chunk_size now available */ for (order = 1; order <= max_order; order++) { size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc unexpectedly succeeded at order %d, it should be full!", order); } list_splice_tail(&holes, &blocks); - drm_buddy_free_list(&mm, &blocks, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &blocks, 0); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_pessimistic(struct kunit *test) +static void gpu_test_buddy_alloc_pessimistic(struct kunit *test) { u64 mm_size, size, start = 0; - struct drm_buddy_block *block, *bn; + struct gpu_buddy_block *block, *bn; const unsigned int max_order = 16; unsigned long flags = 0; - struct drm_buddy mm; + struct gpu_buddy mm; unsigned int order; LIST_HEAD(blocks); LIST_HEAD(tmp); @@ -595,19 +595,19 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test) */ mm_size = SZ_4K << max_order; - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K), "buddy_init failed\n"); KUNIT_EXPECT_EQ(test, mm.max_order, max_order); for (order = 0; order < max_order; order++) { size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM with order=%d\n", order); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_move_tail(&block->link, &blocks); @@ -615,11 +615,11 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test) /* And now the last remaining block available */ size = get_size(0, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM on final alloc\n"); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_move_tail(&block->link, &blocks); @@ -627,58 +627,58 @@ static void drm_test_buddy_alloc_pessimistic(struct kunit *test) /* Should be completely full! */ for (order = max_order; order--;) { size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc unexpectedly succeeded, it should be full!"); } block = list_last_entry(&blocks, typeof(*block), link); list_del(&block->link); - drm_buddy_free_block(&mm, block); + gpu_buddy_free_block(&mm, block); /* As we free in increasing size, we make available larger blocks */ order = 1; list_for_each_entry_safe(block, bn, &blocks, link) { list_del(&block->link); - drm_buddy_free_block(&mm, block); + gpu_buddy_free_block(&mm, block); size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM with order=%d\n", order); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_del(&block->link); - drm_buddy_free_block(&mm, block); + gpu_buddy_free_block(&mm, block); order++; } /* To confirm, now the whole mm should be available */ size = get_size(max_order, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc (realloc) hit -ENOMEM with order=%d\n", max_order); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_del(&block->link); - drm_buddy_free_block(&mm, block); - drm_buddy_free_list(&mm, &blocks, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_block(&mm, block); + gpu_buddy_free_list(&mm, &blocks, 0); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_optimistic(struct kunit *test) +static void gpu_test_buddy_alloc_optimistic(struct kunit *test) { u64 mm_size, size, start = 0; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; unsigned long flags = 0; const int max_order = 16; - struct drm_buddy mm; + struct gpu_buddy mm; LIST_HEAD(blocks); LIST_HEAD(tmp); int order; @@ -690,19 +690,19 @@ static void drm_test_buddy_alloc_optimistic(struct kunit *test) mm_size = SZ_4K * ((1 << (max_order + 1)) - 1); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, SZ_4K), + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_init(&mm, mm_size, SZ_4K), "buddy_init failed\n"); KUNIT_EXPECT_EQ(test, mm.max_order, max_order); for (order = 0; order <= max_order; order++) { size = get_size(order, mm.chunk_size); - KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_FALSE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc hit -ENOMEM with order=%d\n", order); - block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link); + block = list_first_entry_or_null(&tmp, struct gpu_buddy_block, link); KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n"); list_move_tail(&block->link, &blocks); @@ -710,79 +710,79 @@ static void drm_test_buddy_alloc_optimistic(struct kunit *test) /* Should be completely full! */ size = get_size(0, mm.chunk_size); - KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size, + KUNIT_ASSERT_TRUE_MSG(test, gpu_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags), "buddy_alloc unexpectedly succeeded, it should be full!"); - drm_buddy_free_list(&mm, &blocks, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &blocks, 0); + gpu_buddy_fini(&mm); } -static void drm_test_buddy_alloc_limit(struct kunit *test) +static void gpu_test_buddy_alloc_limit(struct kunit *test) { u64 size = U64_MAX, start = 0; - struct drm_buddy_block *block; + struct gpu_buddy_block *block; unsigned long flags = 0; LIST_HEAD(allocated); - struct drm_buddy mm; + struct gpu_buddy mm; - KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, SZ_4K)); + KUNIT_EXPECT_FALSE(test, gpu_buddy_init(&mm, size, SZ_4K)); - KUNIT_EXPECT_EQ_MSG(test, mm.max_order, DRM_BUDDY_MAX_ORDER, + KUNIT_EXPECT_EQ_MSG(test, mm.max_order, GPU_BUDDY_MAX_ORDER, "mm.max_order(%d) != %d\n", mm.max_order, - DRM_BUDDY_MAX_ORDER); + GPU_BUDDY_MAX_ORDER); size = mm.chunk_size << mm.max_order; - KUNIT_EXPECT_FALSE(test, drm_buddy_alloc_blocks(&mm, start, size, size, + KUNIT_EXPECT_FALSE(test, gpu_buddy_alloc_blocks(&mm, start, size, size, mm.chunk_size, &allocated, flags)); - block = list_first_entry_or_null(&allocated, struct drm_buddy_block, link); + block = list_first_entry_or_null(&allocated, struct gpu_buddy_block, link); KUNIT_EXPECT_TRUE(test, block); - KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), mm.max_order, + KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_order(block), mm.max_order, "block order(%d) != %d\n", - drm_buddy_block_order(block), mm.max_order); + gpu_buddy_block_order(block), mm.max_order); - KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_size(&mm, block), + KUNIT_EXPECT_EQ_MSG(test, gpu_buddy_block_size(&mm, block), BIT_ULL(mm.max_order) * mm.chunk_size, "block size(%llu) != %llu\n", - drm_buddy_block_size(&mm, block), + gpu_buddy_block_size(&mm, block), BIT_ULL(mm.max_order) * mm.chunk_size); - drm_buddy_free_list(&mm, &allocated, 0); - drm_buddy_fini(&mm); + gpu_buddy_free_list(&mm, &allocated, 0); + gpu_buddy_fini(&mm); } -static int drm_buddy_suite_init(struct kunit_suite *suite) +static int gpu_buddy_suite_init(struct kunit_suite *suite) { while (!random_seed) random_seed = get_random_u32(); - kunit_info(suite, "Testing DRM buddy manager, with random_seed=0x%x\n", + kunit_info(suite, "Testing GPU buddy manager, with random_seed=0x%x\n", random_seed); return 0; } -static struct kunit_case drm_buddy_tests[] = { - KUNIT_CASE(drm_test_buddy_alloc_limit), - KUNIT_CASE(drm_test_buddy_alloc_optimistic), - KUNIT_CASE(drm_test_buddy_alloc_pessimistic), - KUNIT_CASE(drm_test_buddy_alloc_pathological), - KUNIT_CASE(drm_test_buddy_alloc_contiguous), - KUNIT_CASE(drm_test_buddy_alloc_clear), - KUNIT_CASE(drm_test_buddy_alloc_range_bias), +static struct kunit_case gpu_buddy_tests[] = { + KUNIT_CASE(gpu_test_buddy_alloc_limit), + KUNIT_CASE(gpu_test_buddy_alloc_optimistic), + KUNIT_CASE(gpu_test_buddy_alloc_pessimistic), + KUNIT_CASE(gpu_test_buddy_alloc_pathological), + KUNIT_CASE(gpu_test_buddy_alloc_contiguous), + KUNIT_CASE(gpu_test_buddy_alloc_clear), + KUNIT_CASE(gpu_test_buddy_alloc_range_bias), {} }; -static struct kunit_suite drm_buddy_test_suite = { - .name = "drm_buddy", - .suite_init = drm_buddy_suite_init, - .test_cases = drm_buddy_tests, +static struct kunit_suite gpu_buddy_test_suite = { + .name = "gpu_buddy", + .suite_init = gpu_buddy_suite_init, + .test_cases = gpu_buddy_tests, }; -kunit_test_suite(drm_buddy_test_suite); +kunit_test_suite(gpu_buddy_test_suite); MODULE_AUTHOR("Intel Corporation"); -MODULE_DESCRIPTION("Kunit test for drm_buddy functions"); +MODULE_DESCRIPTION("Kunit test for gpu_buddy functions"); MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/lib/drm_random.c b/drivers/gpu/tests/gpu_random.c similarity index 59% rename from drivers/gpu/drm/lib/drm_random.c rename to drivers/gpu/tests/gpu_random.c index 0e9dba1ef4af5..6356372f7e52e 100644 --- a/drivers/gpu/drm/lib/drm_random.c +++ b/drivers/gpu/tests/gpu_random.c @@ -6,28 +6,28 @@ #include #include -#include "drm_random.h" +#include "gpu_random.h" -u32 drm_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state) +u32 gpu_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state) { return upper_32_bits((u64)prandom_u32_state(state) * ep_ro); } -EXPORT_SYMBOL(drm_prandom_u32_max_state); +EXPORT_SYMBOL(gpu_prandom_u32_max_state); -void drm_random_reorder(unsigned int *order, unsigned int count, +void gpu_random_reorder(unsigned int *order, unsigned int count, struct rnd_state *state) { unsigned int i, j; for (i = 0; i < count; ++i) { BUILD_BUG_ON(sizeof(unsigned int) > sizeof(u32)); - j = drm_prandom_u32_max_state(count, state); + j = gpu_prandom_u32_max_state(count, state); swap(order[i], order[j]); } } -EXPORT_SYMBOL(drm_random_reorder); +EXPORT_SYMBOL(gpu_random_reorder); -unsigned int *drm_random_order(unsigned int count, struct rnd_state *state) +unsigned int *gpu_random_order(unsigned int count, struct rnd_state *state) { unsigned int *order, i; @@ -38,7 +38,7 @@ unsigned int *drm_random_order(unsigned int count, struct rnd_state *state) for (i = 0; i < count; i++) order[i] = i; - drm_random_reorder(order, count, state); + gpu_random_reorder(order, count, state); return order; } -EXPORT_SYMBOL(drm_random_order); +EXPORT_SYMBOL(gpu_random_order); diff --git a/drivers/gpu/drm/lib/drm_random.h b/drivers/gpu/tests/gpu_random.h similarity index 53% rename from drivers/gpu/drm/lib/drm_random.h rename to drivers/gpu/tests/gpu_random.h index 9f827260a89db..b68cf34482644 100644 --- a/drivers/gpu/drm/lib/drm_random.h +++ b/drivers/gpu/tests/gpu_random.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __DRM_RANDOM_H__ -#define __DRM_RANDOM_H__ +#ifndef __GPU_RANDOM_H__ +#define __GPU_RANDOM_H__ /* This is a temporary home for a couple of utility functions that should * be transposed to lib/ at the earliest convenience. @@ -8,21 +8,21 @@ #include -#define DRM_RND_STATE_INITIALIZER(seed__) ({ \ +#define GPU_RND_STATE_INITIALIZER(seed__) ({ \ struct rnd_state state__; \ prandom_seed_state(&state__, (seed__)); \ state__; \ }) -#define DRM_RND_STATE(name__, seed__) \ - struct rnd_state name__ = DRM_RND_STATE_INITIALIZER(seed__) +#define GPU_RND_STATE(name__, seed__) \ + struct rnd_state name__ = GPU_RND_STATE_INITIALIZER(seed__) -unsigned int *drm_random_order(unsigned int count, +unsigned int *gpu_random_order(unsigned int count, struct rnd_state *state); -void drm_random_reorder(unsigned int *order, +void gpu_random_reorder(unsigned int *order, unsigned int count, struct rnd_state *state); -u32 drm_prandom_u32_max_state(u32 ep_ro, +u32 gpu_prandom_u32_max_state(u32 ep_ro, struct rnd_state *state); -#endif /* !__DRM_RANDOM_H__ */ +#endif /* !__GPU_RANDOM_H__ */ diff --git a/drivers/hid/bpf/hid_bpf_struct_ops.c b/drivers/hid/bpf/hid_bpf_struct_ops.c index 702c22fae136a..049df06477610 100644 --- a/drivers/hid/bpf/hid_bpf_struct_ops.c +++ b/drivers/hid/bpf/hid_bpf_struct_ops.c @@ -250,6 +250,11 @@ static void hid_bpf_unreg(void *kdata, struct bpf_link *link) mutex_lock(&hdev->bpf.prog_list_lock); + if (!ops->hdev) { + mutex_unlock(&hdev->bpf.prog_list_lock); + return; + } + list_del_rcu(&ops->list); synchronize_srcu(&hdev->bpf.srcu); ops->hdev = NULL; @@ -310,13 +315,17 @@ static struct bpf_struct_ops bpf_hid_bpf_ops = { void __hid_bpf_ops_destroy_device(struct hid_device *hdev) { struct hid_bpf_ops *e; + int count = 0; - rcu_read_lock(); - list_for_each_entry_rcu(e, &hdev->bpf.prog_list, list) { - hid_put_device(hdev); + mutex_lock(&hdev->bpf.prog_list_lock); + list_for_each_entry(e, &hdev->bpf.prog_list, list) { e->hdev = NULL; + count++; } - rcu_read_unlock(); + mutex_unlock(&hdev->bpf.prog_list_lock); + + while (count--) + hid_put_device(hdev); } static int __init hid_bpf_struct_ops_init(void) diff --git a/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c b/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c new file mode 100644 index 0000000000000..e6ba2295dc775 --- /dev/null +++ b/drivers/hid/bpf/progs/Huion__Inspiroy-Frego-M.bpf.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include "vmlinux.h" +#include "hid_bpf.h" +#include "hid_bpf_helpers.h" +#include + +/* + * Huion Inspiroy Frego M Pen Tablet + * Model L610 + * 256c:8251 (Bluetooth) + * 256c:2012 (USB) + */ +#define VID_HUION 0x256C +#define PID_INSPIROY_FREGO_M 0x8251 +#define PID_L610 0x2012 + +#define PEN_RDESC_SIZE 125 +#define SECONDARY_SWITCH_OFFSET 17 + +HID_BPF_CONFIG( + HID_DEVICE(BUS_BLUETOOTH, HID_GROUP_GENERIC, VID_HUION, PID_INSPIROY_FREGO_M), + HID_DEVICE(BUS_USB, HID_GROUP_GENERIC, VID_HUION, PID_L610) +); + +/* + * The pen descriptor reports the second side button as Secondary Tip Switch + * instead of Secondary Barrel Switch. + * + * Relevant part of the original pen report descriptor: + * + * 0x09, 0x42, // Usage (Tip Switch) 12 + * 0x09, 0x44, // Usage (Barrel Switch) 14 + * 0x09, 0x43, // Usage (Secondary Tip Switch) 16 <- change to 0x5a + * 0x09, 0x3c, // Usage (Invert) 18 + * 0x09, 0x45, // Usage (Eraser) 20 + * 0x15, 0x00, // Logical Minimum (0) 22 + * 0x25, 0x01, // Logical Maximum (1) 24 + */ +SEC(HID_BPF_RDESC_FIXUP) +int BPF_PROG(fix_secondary_barrel_rdesc, struct hid_bpf_ctx *hctx) +{ + __u8 *data = hid_bpf_get_data(hctx, 0 /* offset */, HID_MAX_DESCRIPTOR_SIZE /* size */); + + if (!data) + return 0; /* EPERM check */ + + if (hctx->size != PEN_RDESC_SIZE) + return 0; + + if (data[0] != 0x05 || data[1] != 0x0d || /* Usage Page (Digitizers) */ + data[2] != 0x09 || data[3] != 0x02 || /* Usage (Pen) */ + data[16] != 0x09 || + data[SECONDARY_SWITCH_OFFSET] != 0x43) /* Secondary Tip Switch */ + return 0; + + data[SECONDARY_SWITCH_OFFSET] = 0x5a; + + return 0; +} + +HID_BPF_OPS(fix_secondary_barrel) = { + .hid_rdesc_fixup = (void *)fix_secondary_barrel_rdesc, +}; + +SEC("syscall") +int probe(struct hid_bpf_probe_args *ctx) +{ + ctx->retval = ctx->rdesc_size != PEN_RDESC_SIZE; + if (ctx->retval) { + ctx->retval = -EINVAL; + return 0; + } + + if (ctx->rdesc[0] != 0x05 || ctx->rdesc[1] != 0x0d || /* Usage Page (Digitizers) */ + ctx->rdesc[2] != 0x09 || ctx->rdesc[3] != 0x02 || /* Usage (Pen) */ + ctx->rdesc[16] != 0x09 || + ctx->rdesc[SECONDARY_SWITCH_OFFSET] != 0x43) { /* Secondary Tip Switch */ + ctx->retval = -EINVAL; + return 0; + } + + ctx->retval = 0; + + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/drivers/hid/hid-apple.c b/drivers/hid/hid-apple.c index cb1bd6dca8356..924dab47849ee 100644 --- a/drivers/hid/hid-apple.c +++ b/drivers/hid/hid-apple.c @@ -91,6 +91,9 @@ struct apple_sc_backlight { struct hid_device *hdev; }; +/* T2 VHCI re-enumerates the internal keyboard across system resume. */ +static int apple_backlight_resume_brightness = -1; + struct apple_backlight_config_report { u8 report_id; u8 version; @@ -823,6 +826,7 @@ static int apple_backlight_led_set(struct led_classdev *led_cdev, static int apple_backlight_init(struct hid_device *hdev) { int ret; + int brightness; struct apple_sc *asc = hid_get_drvdata(hdev); struct apple_backlight_config_report *rep; @@ -858,13 +862,20 @@ static int apple_backlight_init(struct hid_device *hdev) asc->backlight->cdev.name = "apple::kbd_backlight"; asc->backlight->cdev.max_brightness = rep->backlight_on_max; asc->backlight->cdev.brightness_set_blocking = apple_backlight_led_set; - asc->backlight->cdev.flags = LED_CORE_SUSPENDRESUME; + /* VHCI re-enumeration restores the cached brightness in the next probe. */ + + brightness = READ_ONCE(apple_backlight_resume_brightness); + if (brightness < 0) + brightness = LED_OFF; + else + brightness = min_t(int, brightness, rep->backlight_on_max); - ret = apple_backlight_set(hdev, 0, 0); + ret = apple_backlight_set(hdev, brightness, 0); if (ret < 0) { hid_err(hdev, "backlight set request failed: %d\n", ret); goto cleanup_and_exit; } + asc->backlight->cdev.brightness = brightness; ret = devm_led_classdev_register(&hdev->dev, &asc->backlight->cdev); @@ -997,6 +1008,9 @@ static void apple_remove(struct hid_device *hdev) if (asc->quirks & APPLE_RDESC_BATTERY) timer_delete_sync(&asc->battery_timer); + if (asc->backlight) + WRITE_ONCE(apple_backlight_resume_brightness, + asc->backlight->cdev.brightness); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c index 1746e8ea50ddf..185bcdb538f24 100644 --- a/drivers/hid/hid-asus.c +++ b/drivers/hid/hid-asus.c @@ -638,13 +638,20 @@ static int asus_kbd_register_leds(struct hid_device *hdev) unsigned char kbd_func; int ret; - if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD) { - /* Initialize keyboard */ - ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); - if (ret < 0) - return ret; + ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); + if (ret < 0) + return ret; - /* The LED endpoint is initialised in two HID */ + /* Get keyboard functions */ + ret = asus_kbd_get_functions(hdev, &kbd_func, FEATURE_KBD_REPORT_ID); + if (ret < 0) + return ret; + + /* Check for backlight support */ + if (!(kbd_func & SUPPORT_KBD_BACKLIGHT)) + return -ENODEV; + + if (drvdata->quirks & QUIRK_ROG_NKEY_KEYBOARD) { ret = asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID1); if (ret < 0) return ret; @@ -652,34 +659,19 @@ static int asus_kbd_register_leds(struct hid_device *hdev) ret = asus_kbd_init(hdev, FEATURE_KBD_LED_REPORT_ID2); if (ret < 0) return ret; + } - if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) { - ret = asus_kbd_disable_oobe(hdev); - if (ret < 0) - return ret; - } - - if (drvdata->quirks & QUIRK_ROG_ALLY_XPAD) { - intf = to_usb_interface(hdev->dev.parent); - udev = interface_to_usbdev(intf); - validate_mcu_fw_version(hdev, - le16_to_cpu(udev->descriptor.idProduct)); - } - - } else { - /* Initialize keyboard */ - ret = asus_kbd_init(hdev, FEATURE_KBD_REPORT_ID); - if (ret < 0) - return ret; - - /* Get keyboard functions */ - ret = asus_kbd_get_functions(hdev, &kbd_func, FEATURE_KBD_REPORT_ID); + if (dmi_match(DMI_PRODUCT_FAMILY, "ProArt P16")) { + ret = asus_kbd_disable_oobe(hdev); if (ret < 0) return ret; + } - /* Check for backlight support */ - if (!(kbd_func & SUPPORT_KBD_BACKLIGHT)) - return -ENODEV; + if ((drvdata->quirks & QUIRK_ROG_ALLY_XPAD) && hid_is_usb(hdev)) { + intf = to_usb_interface(hdev->dev.parent); + udev = interface_to_usbdev(intf); + validate_mcu_fw_version(hdev, + le16_to_cpu(udev->descriptor.idProduct)); } drvdata->kbd_backlight = devm_kzalloc(&hdev->dev, diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c index 4df51bbd0d54c..78adb5a48a612 100644 --- a/drivers/hid/hid-core.c +++ b/drivers/hid/hid-core.c @@ -379,6 +379,9 @@ static int hid_add_field(struct hid_parser *parser, unsigned report_type, unsign static u32 item_udata(struct hid_item *item) { + if (item->format != HID_ITEM_FORMAT_SHORT) + return 0; + switch (item->size) { case 1: return item->data.u8; case 2: return item->data.u16; @@ -389,6 +392,9 @@ static u32 item_udata(struct hid_item *item) static s32 item_sdata(struct hid_item *item) { + if (item->format != HID_ITEM_FORMAT_SHORT) + return 0; + switch (item->size) { case 1: return item->data.s8; case 2: return item->data.s16; @@ -1925,13 +1931,14 @@ int hid_set_field(struct hid_field *field, unsigned offset, __s32 value) size = field->report_size; - hid_dump_input(field->report->device, field->usage + offset, value); - if (offset >= field->report_count) { hid_err(field->report->device, "offset (%d) exceeds report_count (%d)\n", offset, field->report_count); return -1; } + + hid_dump_input(field->report->device, field->usage + offset, value); + if (field->logical_minimum < 0) { if (value != snto32(s32ton(value, size), size)) { hid_err(field->report->device, "value %d is out of range\n", value); @@ -2443,9 +2450,16 @@ EXPORT_SYMBOL_GPL(hid_hw_start); * * This is usually called from remove function or from probe when something * failed and hid_hw_start was called already. + * + * If the caller enabled HID input via hid_device_io_start() and is unwinding + * without an explicit hid_device_io_stop(), quiesce input first so that + * in-flight reports cannot reach handlers (e.g. hidraw_report_event) whose + * backing objects hid_disconnect() is about to free. */ void hid_hw_stop(struct hid_device *hdev) { + if (hdev->io_started) + hid_device_io_stop(hdev); hid_disconnect(hdev); hdev->ll_driver->stop(hdev); } @@ -2838,6 +2852,8 @@ static int __hid_device_probe(struct hid_device *hdev, struct hid_driver *hdrv) */ if (ret) { + if (hdev->io_started) + hid_device_io_stop(hdev); devres_release_group(&hdev->dev, hdev->devres_group_id); hid_close_report(hdev); hdev->driver = NULL; diff --git a/drivers/hid/hid-corsair-void.c b/drivers/hid/hid-corsair-void.c index 5e9a5b8f7f167..071a663a6c268 100644 --- a/drivers/hid/hid-corsair-void.c +++ b/drivers/hid/hid-corsair-void.c @@ -92,6 +92,9 @@ #define CORSAIR_VOID_STATUS_REPORT_ID 0x64 #define CORSAIR_VOID_FIRMWARE_REPORT_ID 0x66 +#define CORSAIR_VOID_STATUS_REPORT_SIZE 5 +#define CORSAIR_VOID_FIRMWARE_REPORT_SIZE 5 + #define CORSAIR_VOID_USB_SIDETONE_REQUEST 0x1 #define CORSAIR_VOID_USB_SIDETONE_REQUEST_TYPE 0x21 #define CORSAIR_VOID_USB_SIDETONE_VALUE 0x200 @@ -742,6 +745,13 @@ static int corsair_void_raw_event(struct hid_device *hid_dev, /* Description of packets are documented at the top of this file */ if (hid_report->id == CORSAIR_VOID_STATUS_REPORT_ID) { + if (size < CORSAIR_VOID_STATUS_REPORT_SIZE) { + hid_warn_ratelimited(hid_dev, + "unexpected status report of size %d", + size); + return 1; + } + drvdata->mic_up = FIELD_GET(CORSAIR_VOID_MIC_MASK, data[2]); drvdata->connected = (data[3] == CORSAIR_VOID_WIRELESS_CONNECTED) || drvdata->is_wired; @@ -750,6 +760,13 @@ static int corsair_void_raw_event(struct hid_device *hid_dev, FIELD_GET(CORSAIR_VOID_CAPACITY_MASK, data[2]), data[3], data[4]); } else if (hid_report->id == CORSAIR_VOID_FIRMWARE_REPORT_ID) { + if (size < CORSAIR_VOID_FIRMWARE_REPORT_SIZE) { + hid_warn_ratelimited(hid_dev, + "unexpected firmware report of size %d", + size); + return 1; + } + drvdata->fw_receiver_major = data[1]; drvdata->fw_receiver_minor = data[2]; drvdata->fw_headset_major = data[3]; diff --git a/drivers/hid/hid-ft260.c b/drivers/hid/hid-ft260.c index 333341e80b0ec..f479459544aee 100644 --- a/drivers/hid/hid-ft260.c +++ b/drivers/hid/hid-ft260.c @@ -240,6 +240,8 @@ struct ft260_device { struct mutex lock; u8 write_buf[FT260_REPORT_MAX_LENGTH]; unsigned long need_wakeup_at; + /* Protects read_buf, read_idx and read_len against ft260_raw_event() */ + spinlock_t read_lock; u8 *read_buf; u16 read_idx; u16 read_len; @@ -501,6 +503,7 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, int timeout, ret = 0; struct ft260_i2c_read_request_report rep; struct hid_device *hdev = dev->hdev; + unsigned long irqflags; u8 bus_busy = 0; if ((flag & FT260_FLAG_START_REPEATED) == FT260_FLAG_START_REPEATED) @@ -526,9 +529,11 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, reinit_completion(&dev->wait); + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_idx = 0; dev->read_buf = data; dev->read_len = rd_len; + spin_unlock_irqrestore(&dev->read_lock, irqflags); ret = ft260_hid_output_report(hdev, (u8 *)&rep, sizeof(rep)); if (ret < 0) { @@ -543,7 +548,9 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, goto ft260_i2c_read_exit; } + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_buf = NULL; + spin_unlock_irqrestore(&dev->read_lock, irqflags); if (flag & FT260_FLAG_STOP) bus_busy = FT260_I2C_STATUS_BUS_BUSY; @@ -562,7 +569,9 @@ static int ft260_i2c_read(struct ft260_device *dev, u8 addr, u8 *data, } while (len > 0); ft260_i2c_read_exit: + spin_lock_irqsave(&dev->read_lock, irqflags); dev->read_buf = NULL; + spin_unlock_irqrestore(&dev->read_lock, irqflags); return ret; } @@ -1018,6 +1027,7 @@ static int ft260_probe(struct hid_device *hdev, const struct hid_device_id *id) "FT260 usb-i2c bridge"); mutex_init(&dev->lock); + spin_lock_init(&dev->read_lock); init_completion(&dev->wait); ret = ft260_xfer_status(dev, FT260_I2C_STATUS_BUS_BUSY); @@ -1067,14 +1077,36 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, { struct ft260_device *dev = hid_get_drvdata(hdev); struct ft260_i2c_input_report *xfer = (void *)data; + unsigned long irqflags; + + if (size < offsetof(struct ft260_i2c_input_report, data)) { + hid_err(hdev, "short report %d\n", size); + return -1; + } if (xfer->report >= FT260_I2C_REPORT_MIN && xfer->report <= FT260_I2C_REPORT_MAX) { - ft260_dbg("i2c resp: rep %#02x len %d\n", xfer->report, - xfer->length); + bool complete_read; + + ft260_dbg("i2c resp: rep %#02x len %d size %d\n", + xfer->report, xfer->length, size); + + if (xfer->length > size - + offsetof(struct ft260_i2c_input_report, data)) { + hid_err(hdev, "report %#02x: length %d exceeds HID report size\n", + xfer->report, xfer->length); + return -1; + } + + /* + * Hold read_lock so a timed-out ft260_i2c_read() cannot + * clear read_buf between the NULL check and the memcpy. + */ + spin_lock_irqsave(&dev->read_lock, irqflags); if ((dev->read_buf == NULL) || (xfer->length > dev->read_len - dev->read_idx)) { + spin_unlock_irqrestore(&dev->read_lock, irqflags); hid_err(hdev, "unexpected report %#02x, length %d\n", xfer->report, xfer->length); return -1; @@ -1083,8 +1115,11 @@ static int ft260_raw_event(struct hid_device *hdev, struct hid_report *report, memcpy(&dev->read_buf[dev->read_idx], &xfer->data, xfer->length); dev->read_idx += xfer->length; + complete_read = dev->read_idx == dev->read_len; + + spin_unlock_irqrestore(&dev->read_lock, irqflags); - if (dev->read_idx == dev->read_len) + if (complete_read) complete(&dev->wait); } else { diff --git a/drivers/hid/hid-haptic.c b/drivers/hid/hid-haptic.c index fc8a9997f8153..575692580ab90 100644 --- a/drivers/hid/hid-haptic.c +++ b/drivers/hid/hid-haptic.c @@ -187,7 +187,7 @@ static void fill_effect_buf(struct hid_haptic_device *haptic, value = waveform_ordinal; break; default: - break; + continue; } field->value[j] = value; diff --git a/drivers/hid/hid-hyperv.c b/drivers/hid/hid-hyperv.c index 9eafff0b6ea4c..d044bed96accb 100644 --- a/drivers/hid/hid-hyperv.c +++ b/drivers/hid/hid-hyperv.c @@ -171,18 +171,32 @@ static void mousevsc_free_device(struct mousevsc_dev *device) } static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, - struct synthhid_device_info *device_info) + struct synthhid_device_info *device_info, + u32 device_info_size) { int ret = 0; struct hid_descriptor *desc; struct mousevsc_prt_msg ack; + size_t desc_offset; + size_t desc_size; input_device->dev_info_status = -ENOMEM; + if (device_info_size < sizeof(*device_info)) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } + input_device->hid_dev_info = device_info->hid_dev_info; desc = &device_info->hid_descriptor; + desc_offset = offsetof(struct synthhid_device_info, hid_descriptor); + desc_size = device_info_size - desc_offset; if (desc->bLength == 0) goto cleanup; + if (desc->bLength < sizeof(*desc) || desc->bLength > desc_size) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } /* The pointer is not NULL when we resume from hibernation */ kfree(input_device->hid_desc); @@ -197,6 +211,10 @@ static void mousevsc_on_receive_device_info(struct mousevsc_dev *input_device, input_device->dev_info_status = -EINVAL; goto cleanup; } + if (input_device->report_desc_size > desc_size - desc->bLength) { + input_device->dev_info_status = -EINVAL; + goto cleanup; + } /* The pointer is not NULL when we resume from hibernation */ kfree(input_device->report_desc); @@ -273,14 +291,17 @@ static void mousevsc_on_receive(struct hv_device *device, break; case SYNTH_HID_INITIAL_DEVICE_INFO: - WARN_ON(pipe_msg->size < sizeof(struct hv_input_dev_info)); + if (WARN_ON_ONCE(pipe_msg->size < + sizeof(struct synthhid_device_info))) + break; /* * Parse out the device info into device attr, * hid desc and report desc */ mousevsc_on_receive_device_info(input_dev, - (struct synthhid_device_info *)pipe_msg->data); + (struct synthhid_device_info *)pipe_msg->data, + pipe_msg->size); break; case SYNTH_HID_INPUT_REPORT: input_report = diff --git a/drivers/hid/hid-input.c b/drivers/hid/hid-input.c index 32a96ae2946f1..b2fa8c42f09b3 100644 --- a/drivers/hid/hid-input.c +++ b/drivers/hid/hid-input.c @@ -374,6 +374,9 @@ static const struct hid_device_id hid_battery_quirks[] = { { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_APPLE, USB_DEVICE_ID_APPLE_MAGICTRACKPAD), HID_BATTERY_QUIRK_IGNORE }, + { HID_BLUETOOTH_DEVICE(BT_VENDOR_ID_APPLE, + USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC), + HID_BATTERY_QUIRK_AVOID_QUERY }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_ELECOM, USB_DEVICE_ID_ELECOM_BM084), HID_BATTERY_QUIRK_IGNORE }, diff --git a/drivers/hid/hid-lg4ff.c b/drivers/hid/hid-lg4ff.c index 32b711723f2aa..9ddd669d88f16 100644 --- a/drivers/hid/hid-lg4ff.c +++ b/drivers/hid/hid-lg4ff.c @@ -336,6 +336,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, if (entry->wdata.combine) { switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: + if (size < 7) + return 0; rd[5] = rd[3]; rd[6] = 0x7F; return 1; @@ -343,10 +345,14 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2: + if (size < 6) + return 0; rd[4] = rd[3]; rd[5] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: + if (size < 7) + return 0; rd[5] = rd[4]; rd[6] = 0x7F; return 1; @@ -366,6 +372,8 @@ int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, } /* Compute a combined axis when wheel does not supply it */ + if (size <= offset + 1) + return 0; rd[offset] = (0xFF + rd[offset] - rd[offset+1]) >> 1; rd[offset+1] = 0x7F; return 1; diff --git a/drivers/hid/hid-logitech-dj.c b/drivers/hid/hid-logitech-dj.c index 3b5412541c925..7895f27c91351 100644 --- a/drivers/hid/hid-logitech-dj.c +++ b/drivers/hid/hid-logitech-dj.c @@ -1749,7 +1749,8 @@ static int logi_dj_raw_event(struct hid_device *hdev, static int logi_dj_probe(struct hid_device *hdev, const struct hid_device_id *id) { - struct hid_report_enum *rep_enum; + struct hid_report_enum *input_report_enum; + struct hid_report_enum *output_report_enum; struct hid_report *rep; struct dj_receiver_dev *djrcv_dev; struct usb_interface *intf; @@ -1793,10 +1794,25 @@ static int logi_dj_probe(struct hid_device *hdev, } } - rep_enum = &hdev->report_enum[HID_INPUT_REPORT]; + output_report_enum = &hdev->report_enum[HID_OUTPUT_REPORT]; + rep = output_report_enum->report_id_hash[REPORT_ID_DJ_SHORT]; + + if (rep && rep->maxfield < 1) { + hid_err(hdev, "Expected size of DJ short report is %d, but got 0", + DJREPORT_SHORT_LENGTH - 1); + return -EINVAL; + } + + if (rep && rep->field[0]->report_count != DJREPORT_SHORT_LENGTH - 1) { + hid_err(hdev, "Expected size of DJ short report is %d, but got %d", + DJREPORT_SHORT_LENGTH - 1, rep->field[0]->report_count); + return -EINVAL; + } + + input_report_enum = &hdev->report_enum[HID_INPUT_REPORT]; /* no input reports, bail out */ - if (list_empty(&rep_enum->report_list)) + if (list_empty(&input_report_enum->report_list)) return -ENODEV; /* @@ -1804,7 +1820,7 @@ static int logi_dj_probe(struct hid_device *hdev, * Note: we should theoretically check for HID++ and DJ * collections, but this will do. */ - list_for_each_entry(rep, &rep_enum->report_list, list) { + list_for_each_entry(rep, &input_report_enum->report_list, list) { if (rep->application == 0xff000001) has_hidpp = true; } @@ -1817,7 +1833,7 @@ static int logi_dj_probe(struct hid_device *hdev, return -ENODEV; /* get the current application attached to the node */ - rep = list_first_entry(&rep_enum->report_list, struct hid_report, list); + rep = list_first_entry(&input_report_enum->report_list, struct hid_report, list); djrcv_dev = dj_get_receiver_dev(hdev, id->driver_data, rep->application, has_hidpp); if (!djrcv_dev) { @@ -1825,7 +1841,7 @@ static int logi_dj_probe(struct hid_device *hdev, return -ENOMEM; } - if (!rep_enum->numbered) + if (!input_report_enum->numbered) djrcv_dev->unnumbered_application = rep->application; /* Starts the usb device and connects to upper interfaces hiddev and diff --git a/drivers/hid/hid-logitech-hidpp.c b/drivers/hid/hid-logitech-hidpp.c index 4ba652ae54765..83d625f361f49 100644 --- a/drivers/hid/hid-logitech-hidpp.c +++ b/drivers/hid/hid-logitech-hidpp.c @@ -4280,6 +4280,7 @@ static void hidpp_connect_event(struct work_struct *work) ret = input_register_device(input); if (ret) { + hidpp->input = NULL; input_free_device(input); return; } diff --git a/drivers/hid/hid-magicmouse.c b/drivers/hid/hid-magicmouse.c index 9eadf3252d0dc..0480e7d216bc4 100644 --- a/drivers/hid/hid-magicmouse.c +++ b/drivers/hid/hid-magicmouse.c @@ -383,13 +383,17 @@ static void magicmouse_emit_touch(struct magicmouse_sc *msc, int raw_id, u8 *tda } } -static int magicmouse_raw_event(struct hid_device *hdev, - struct hid_report *report, u8 *data, int size) +static int __magicmouse_raw_event(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size, bool nested) { struct magicmouse_sc *msc = hid_get_drvdata(hdev); struct input_dev *input = msc->input; int x = 0, y = 0, ii, clicks = 0, npoints; + /* Protect against zero sized recursive calls from DOUBLE_REPORT_ID */ + if (size < 1) + return 0; + switch (data[0]) { case TRACKPAD_REPORT_ID: case TRACKPAD2_BT_REPORT_ID: @@ -490,9 +494,30 @@ static int magicmouse_raw_event(struct hid_device *hdev, /* Sometimes the trackpad sends two touch reports in one * packet. */ - magicmouse_raw_event(hdev, report, data + 2, data[1]); - magicmouse_raw_event(hdev, report, data + 2 + data[1], - size - 2 - data[1]); + + /* + * A double report only ever wraps two normal reports, so it is + * never nested. Refuse to recurse a second time; otherwise a + * malicious device could chain DOUBLE_REPORT_ID packets to drive + * unbounded recursion and overflow the kernel stack. + */ + if (nested) + return 0; + + /* Ensure that we have at least 2 elements (report type and size) */ + if (size < 2) + return 0; + + if (size < data[1] + 2) { + hid_warn(hdev, + "received report length (%d) was smaller than specified (%d)", + size, data[1] + 2); + return 0; + } + + __magicmouse_raw_event(hdev, report, data + 2, data[1], true); + __magicmouse_raw_event(hdev, report, data + 2 + data[1], + size - 2 - data[1], true); return 0; default: return 0; @@ -518,6 +543,12 @@ static int magicmouse_raw_event(struct hid_device *hdev, return 1; } +static int magicmouse_raw_event(struct hid_device *hdev, + struct hid_report *report, u8 *data, int size) +{ + return __magicmouse_raw_event(hdev, report, data, size, false); +} + static int magicmouse_event(struct hid_device *hdev, struct hid_field *field, struct hid_usage *usage, __s32 value) { @@ -812,6 +843,12 @@ static bool is_usb_magictrackpad2(__u32 vendor, __u32 product) product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; } +static bool is_bt_magictrackpad2(__u32 vendor, __u32 product) +{ + return vendor == BT_VENDOR_ID_APPLE && + product == USB_DEVICE_ID_APPLE_MAGICTRACKPAD2_USBC; +} + static int magicmouse_fetch_battery(struct hid_device *hdev) { #ifdef CONFIG_HID_BATTERY_STRENGTH @@ -820,7 +857,8 @@ static int magicmouse_fetch_battery(struct hid_device *hdev) if (!hdev->battery || (!is_usb_magicmouse2(hdev->vendor, hdev->product) && - !is_usb_magictrackpad2(hdev->vendor, hdev->product))) + !is_usb_magictrackpad2(hdev->vendor, hdev->product) && + !is_bt_magictrackpad2(hdev->vendor, hdev->product))) return -1; report_enum = &hdev->report_enum[hdev->battery_report_type]; @@ -882,6 +920,16 @@ static int magicmouse_probe(struct hid_device *hdev, return ret; } + /* + * When hidinput_connect() fails it frees every input device it + * created, but that does not fail hid_hw_start(): the core simply + * does not claim an input. msc->input, cached in ->input_mapping + * while the report descriptor was parsed, would then be a dangling + * pointer that passes every NULL check. Trust the core's claim. + */ + if (!(hdev->claimed & HID_CLAIMED_INPUT)) + msc->input = NULL; + if (is_usb_magicmouse2(id->vendor, id->product) || is_usb_magictrackpad2(id->vendor, id->product)) { timer_setup(&msc->battery_timer, magicmouse_battery_timer_tick, 0); @@ -953,6 +1001,16 @@ static int magicmouse_probe(struct hid_device *hdev, schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); } + /* + * Query the Bluetooth Magic Trackpad USB-C battery as done for USB. + * Start io first: probe holds driver_input_lock and the synchronous + * GET_REPORT reply would otherwise be dropped. + */ + if (is_bt_magictrackpad2(id->vendor, id->product)) { + hid_device_io_start(hdev); + magicmouse_fetch_battery(hdev); + } + return 0; err_stop_hw: if (is_usb_magicmouse2(id->vendor, id->product) || @@ -977,6 +1035,22 @@ static void magicmouse_remove(struct hid_device *hdev) hid_hw_stop(hdev); } +#ifdef CONFIG_PM +static int magicmouse_reset_resume(struct hid_device *hdev) +{ + struct magicmouse_sc *msc = hid_get_drvdata(hdev); + + /* The device drops out of multitouch mode on resume; re-send the + * enable report. Only the HID_TYPE_USBMOUSE interface accepts it, and + * it must be deferred. Sending it inline here is too early. + */ + if (msc && hdev->type == HID_TYPE_USBMOUSE) + schedule_delayed_work(&msc->work, msecs_to_jiffies(500)); + + return 0; +} +#endif + static const __u8 *magicmouse_report_fixup(struct hid_device *hdev, __u8 *rdesc, unsigned int *rsize) { @@ -1040,6 +1114,9 @@ static struct hid_driver magicmouse_driver = { .event = magicmouse_event, .input_mapping = magicmouse_input_mapping, .input_configured = magicmouse_input_configured, +#ifdef CONFIG_PM + .reset_resume = magicmouse_reset_resume, +#endif }; module_hid_driver(magicmouse_driver); diff --git a/drivers/hid/hid-mcp2221.c b/drivers/hid/hid-mcp2221.c index ef3b5c77c38e3..f10dea1045d15 100644 --- a/drivers/hid/hid-mcp2221.c +++ b/drivers/hid/hid-mcp2221.c @@ -121,6 +121,7 @@ struct mcp2221 { u8 *rxbuf; u8 txbuf[64]; int rxbuf_idx; + int rxbuf_size; int status; u8 cur_i2c_clk_div; struct gpio_chip *gc; @@ -323,17 +324,19 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, mcp->txbuf[3] = (u8)(msg->addr << 1); total_len = msg->len; mcp->rxbuf = msg->buf; + mcp->rxbuf_size = msg->len; } else { mcp->txbuf[1] = smbus_len; mcp->txbuf[2] = 0; mcp->txbuf[3] = (u8)(smbus_addr << 1); total_len = smbus_len; mcp->rxbuf = smbus_buf; + mcp->rxbuf_size = smbus_len; } ret = mcp_send_data_req_status(mcp, mcp->txbuf, 4); if (ret) - return ret; + goto out; mcp->rxbuf_idx = 0; @@ -355,7 +358,7 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, } else { usleep_range(980, 1000); mcp_cancel_last_cmd(mcp); - return ret; + goto out; } } else { retries = 0; @@ -365,6 +368,10 @@ static int mcp_i2c_smbus_read(struct mcp2221 *mcp, usleep_range(980, 1000); ret = mcp_chk_last_cmd_status_free_bus(mcp); +out: + mcp->rxbuf = NULL; + mcp->rxbuf_size = 0; + return ret; } @@ -851,6 +858,9 @@ static int mcp2221_raw_event(struct hid_device *hdev, u8 *buf; struct mcp2221 *mcp = hid_get_drvdata(hdev); + if (size < 4) + return 0; + switch (data[0]) { case MCP2221_I2C_WR_DATA: @@ -912,6 +922,14 @@ static int mcp2221_raw_event(struct hid_device *hdev, mcp->status = -EINVAL; break; } + if (mcp->rxbuf_idx + data[3] > mcp->rxbuf_size) { + mcp->status = -EINVAL; + break; + } + if (4 + data[3] > size) { + mcp->status = -EINVAL; + break; + } buf = mcp->rxbuf; memcpy(&buf[mcp->rxbuf_idx], &data[4], data[3]); mcp->rxbuf_idx = mcp->rxbuf_idx + data[3]; @@ -1035,6 +1053,8 @@ static void mcp2221_hid_unregister(void *ptr) { struct hid_device *hdev = ptr; + if (hdev->io_started) + hid_device_io_stop(hdev); hid_hw_close(hdev); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c index 1959481dc7820..35f9f4f76a550 100644 --- a/drivers/hid/hid-multitouch.c +++ b/drivers/hid/hid-multitouch.c @@ -440,11 +440,14 @@ static const struct mt_class mt_classes[] = { MT_QUIRK_CONTACT_CNT_ACCURATE, }, { .name = MT_CLS_YOGABOOK9I, - .quirks = MT_QUIRK_ALWAYS_VALID | + .quirks = MT_QUIRK_NOT_SEEN_MEANS_UP | + MT_QUIRK_ALWAYS_VALID | + MT_QUIRK_CONTACT_CNT_ACCURATE | MT_QUIRK_FORCE_MULTI_INPUT | MT_QUIRK_SEPARATE_APP_REPORT | MT_QUIRK_HOVERING | MT_QUIRK_YOGABOOK9I, + .maxcontacts = 10, .export_all_inputs = true }, { .name = MT_CLS_EGALAX_P80H84, @@ -1564,6 +1567,144 @@ static int mt_event(struct hid_device *hid, struct hid_field *field, return 0; } +/* + * Yoga Book 9 14IAH10 descriptor fixup. + * + * The device includes a HID_DG_TOUCHPAD application collection designed for + * the Windows inbox HID driver's Win8 PTP touchpad mode. On Linux we want + * only the HID_DG_TOUCHSCREEN collections. The touchpad collection (and the + * HID_DG_BUTTONTYPE and Win8 compliance blob features it contains) must be + * removed so hid-multitouch does not misclassify the touchscreen nodes as + * indirect buttonpads. + * + * The firmware also resets if any USB control request is received while the + * CDC-ACM interface is initialising (~1.18 s after enumeration). Dropping + * the Win8 blob and Contact Count Max feature reports prevents the + * GET_REPORT calls that hid-multitouch issues at probe. + */ +static void mt_yogabook9_fixup(struct hid_device *hdev, __u8 *rdesc, + unsigned int *size) +{ + /* Usage Page (Digitizer), Usage (Touch Pad), Collection (Application) */ + static const __u8 tp_app_hdr[] = { 0x05, 0x0d, 0x09, 0x05, 0xa1, 0x01 }; + /* Vendor Usage Page 0xff00 (Win8 compliance blob header) */ + static const __u8 win8_page[] = { 0x06, 0x00, 0xff }; + /* Usage (Contact Count Max = 0x55) */ + static const __u8 ccmax_usage[] = { 0x09, 0x55 }; + unsigned int i; + + /* + * Step 1: find and remove the Touch Pad application collection. + * Walk HID short items from the collection header to its matching + * End Collection, then close the gap with memmove. + */ + for (i = 0; i + sizeof(tp_app_hdr) <= *size; i++) { + if (memcmp(rdesc + i, tp_app_hdr, sizeof(tp_app_hdr)) == 0) { + __u8 *start = rdesc + i; + __u8 *coll_end = NULL; + __u8 *p = start; + unsigned int drop; + int depth = 0; + + while (p < rdesc + *size) { + __u8 b = *p; + int ds = b & 3; + int item_len; + + if (b == 0xfe) { /* long item */ + if (p + 2 >= rdesc + *size) + break; + item_len = p[1] + 3; + } else { + item_len = (ds == 3) ? 5 : ds + 1; + } + if (p + item_len > rdesc + *size) + break; + + if ((b & 0xfc) == 0xa0) + depth++; /* Collection */ + else if (b == 0xc0) { + depth--; /* End Collection */ + if (depth == 0) { + coll_end = p; + break; + } + } + p += item_len; + } + + if (!coll_end) { + hid_err(hdev, + "Yoga Book 9: Touch Pad End Collection not found\n"); + break; + } + + drop = coll_end - start + 1; + memmove(start, coll_end + 1, rdesc + *size - coll_end - 1); + *size -= drop; + hid_dbg(hdev, + "Yoga Book 9: dropped Touch Pad collection (%u bytes)\n", + drop); + break; + } + } + + /* + * Step 2: neutralize Win8 compliance blob feature reports remaining + * in the touchscreen collections. Change Usage Page 0xff00 to 0x0f00 + * so the case 0xff0000c5 branch in mt_feature_mapping() is not reached + * and no GET_REPORT is issued. + */ + for (i = 0; i + sizeof(win8_page) <= *size; i++) { + if (memcmp(rdesc + i, win8_page, sizeof(win8_page)) == 0) { + rdesc[i + 2] = 0x0f; /* 0xff00 -> 0x0f00 */ + hid_dbg(hdev, + "Yoga Book 9: neutralized Win8 blob at offset %u\n", + i); + } + } + + /* + * Step 3: neutralize Contact Count Max feature reports. Change usage + * 0x55 (HID_DG_CONTACTMAX) to 0x00 so mt_feature_mapping() does not + * issue GET_REPORT. The class maxcontacts field provides the value. + */ + for (i = 0; i + sizeof(ccmax_usage) <= *size; i++) { + if (memcmp(rdesc + i, ccmax_usage, sizeof(ccmax_usage)) == 0) { + rdesc[i + 1] = 0x00; + hid_dbg(hdev, + "Yoga Book 9: neutralized ContactMax at offset %u\n", + i); + } + } + + /* + * Step 4: neutralize Surface Switch (0x57) and Button Switch (0x58) + * feature report usages in the Device Configuration collection. + * mt_set_modes() issues HID_REQ_SET_REPORT for these on every + * input-device open/close; those repeated control requests hit the + * firmware's CDC-ACM init window and trigger resets. + * + * Input Mode (0x52) is intentionally left intact. mt_set_modes() + * sends it once at probe to set the device into touchscreen mode, + * which flushes the firmware's contact buffer and clears a persistent + * ghost contact (cid 2, fixed coordinates) that otherwise appears on + * every enumeration. By probe time cdc_acm has already satisfied the + * CDC-ACM init watchdog (~130 ms), so the single SET_REPORT for Input + * Mode arrives safely after the reset window has closed. + */ + for (i = 0; i + 2 <= *size; i++) { + if (rdesc[i] == 0x09 && + (rdesc[i + 1] == 0x57 || + rdesc[i + 1] == 0x58)) { + hid_dbg(hdev, + "Yoga Book 9: neutralized set-modes usage 0x%02x at offset %u\n", + rdesc[i + 1], i); + rdesc[i + 1] = 0x00; + } + } +} + static const __u8 *mt_report_fixup(struct hid_device *hdev, __u8 *rdesc, unsigned int *size) { @@ -1593,6 +1734,10 @@ got: %x\n", } } + if (hdev->vendor == USB_VENDOR_ID_LENOVO && + hdev->product == USB_DEVICE_ID_LENOVO_YOGABOOK9I) + mt_yogabook9_fixup(hdev, rdesc, size); + return rdesc; } @@ -2567,7 +2712,7 @@ static const struct hid_device_id mt_devices[] = { HID_ANY_ID) }, /* Hantick */ - { .driver_data = MT_CLS_NSMU, + { .driver_data = MT_CLS_WIN_8_FORCE_MULTI_INPUT_NSMU, HID_DEVICE(BUS_I2C, HID_GROUP_MULTITOUCH_WIN_8, I2C_VENDOR_ID_HANTICK, I2C_PRODUCT_ID_HANTICK_5288) }, diff --git a/drivers/hid/hid-nintendo.c b/drivers/hid/hid-nintendo.c index c2849a541f65a..34642352b439d 100644 --- a/drivers/hid/hid-nintendo.c +++ b/drivers/hid/hid-nintendo.c @@ -1453,7 +1453,6 @@ static void joycon_parse_imu_report(struct joycon_ctlr *ctlr, dropped_threshold = ctlr->imu_avg_delta_ms * 3 / 2; dropped_pkts = (delta - min(delta, dropped_threshold)) / ctlr->imu_avg_delta_ms; - ctlr->imu_timestamp_us += 1000 * ctlr->imu_avg_delta_ms; if (dropped_pkts > JC_IMU_DROPPED_PKT_WARNING) { hid_warn_ratelimited(ctlr->hdev, "compensating for %u dropped IMU reports\n", @@ -2138,10 +2137,6 @@ static int joycon_input_create(struct joycon_ctlr *ctlr) ctlr->input->phys = hdev->phys; input_set_drvdata(ctlr->input, ctlr); - ret = input_register_device(ctlr->input); - if (ret) - return ret; - if (joycon_type_is_right_joycon(ctlr)) { joycon_config_right_stick(ctlr->input); joycon_config_buttons(ctlr->input, right_joycon_button_mappings); @@ -2181,6 +2176,10 @@ static int joycon_input_create(struct joycon_ctlr *ctlr) if (joycon_has_rumble(ctlr)) joycon_config_rumble(ctlr); + ret = input_register_device(ctlr->input); + if (ret) + return ret; + return 0; } @@ -2559,7 +2558,12 @@ static int joycon_ctlr_read_handler(struct joycon_ctlr *ctlr, u8 *data, { if (data[0] == JC_INPUT_SUBCMD_REPLY || data[0] == JC_INPUT_IMU_DATA || data[0] == JC_INPUT_MCU_DATA) { - if (size >= 12) /* make sure it contains the input report */ + /* + * The whole struct is cast and parsed below, including the + * IMU/subcmd union, not just the 12-byte partial header this + * used to check for. + */ + if (size >= sizeof(struct joycon_input_report)) joycon_parse_report(ctlr, (struct joycon_input_report *)data); } @@ -2687,14 +2691,14 @@ static int nintendo_hid_probe(struct hid_device *hdev, ret = joycon_init(hdev); if (ret) { hid_err(hdev, "Failed to initialize controller; ret=%d\n", ret); - goto err_close; + goto err_io_stop; } /* Initialize the leds */ ret = joycon_leds_create(ctlr); if (ret) { hid_err(hdev, "Failed to create leds; ret=%d\n", ret); - goto err_close; + goto err_io_stop; } /* Initialize the battery power supply */ @@ -2717,7 +2721,8 @@ static int nintendo_hid_probe(struct hid_device *hdev, err_ida: ida_free(&nintendo_player_id_allocator, ctlr->player_id); -err_close: +err_io_stop: + hid_device_io_stop(hdev); hid_hw_close(hdev); err_stop: hid_hw_stop(hdev); diff --git a/drivers/hid/hid-picolcd_debugfs.c b/drivers/hid/hid-picolcd_debugfs.c index d01176da88962..0d325d69e64b9 100644 --- a/drivers/hid/hid-picolcd_debugfs.c +++ b/drivers/hid/hid-picolcd_debugfs.c @@ -98,6 +98,15 @@ static ssize_t picolcd_debug_eeprom_read(struct file *f, char __user *u, ret = resp->raw_data[2]; if (ret > s) ret = s; + /* + * raw_data[2] is a device-supplied length; also clamp it to + * what picolcd_raw_event() actually stored (raw_size), or a + * hostile device overruns the raw_data[] buffer. + */ + if (ret > resp->raw_size - 3) + ret = resp->raw_size - 3; + if (ret < 0) + ret = 0; if (copy_to_user(u, resp->raw_data+3, ret)) ret = -EFAULT; else diff --git a/drivers/hid/hid-rmi.c b/drivers/hid/hid-rmi.c index d4af17fdba467..d55a0388895fb 100644 --- a/drivers/hid/hid-rmi.c +++ b/drivers/hid/hid-rmi.c @@ -235,7 +235,23 @@ static int rmi_hid_read_block(struct rmi_transport_dev *xport, u16 addr, break; } - read_input_count = data->readReport[1]; + read_input_count = min_t(int, data->readReport[1], + data->input_report_size - 2); + if (!read_input_count) { + /* + * A zero length reply advances neither + * bytes_read nor bytes_needed, and because a + * reply did arrive the wait above does not + * time out either, so a device answering 0 + * forever would spin here indefinitely with + * page_mutex held. + */ + hid_warn(hdev, "%s: zero-length read reply\n", + __func__); + clear_bit(RMI_READ_DATA_PENDING, &data->flags); + ret = -EIO; + break; + } memcpy(buf + bytes_read, &data->readReport[2], min(read_input_count, bytes_needed)); @@ -271,6 +287,11 @@ static int rmi_hid_write_block(struct rmi_transport_dev *xport, u16 addr, goto exit; } + if (len + 4 > data->output_report_size) { + ret = -EINVAL; + goto exit; + } + data->writeReport[0] = RMI_WRITE_REPORT_ID; data->writeReport[1] = len; data->writeReport[2] = addr & 0xFF; @@ -666,8 +687,16 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id) return ret; } - if (id->driver_data) - data->device_flags = id->driver_data; + /* + * RMI_DEVICE can only mean "this probe validated the RMI reports and + * allocated writeReport": every bail-out to start below skips that + * allocation, and device_flags left carrying RMI_DEVICE from + * driver_data would send rmi_input_configured() into rmi_set_page() + * with writeReport still NULL. A bind through the new_id sysfs + * attribute can supply driver_data with the bit set, so do not let + * driver_data grant it. + */ + data->device_flags = id->driver_data & ~RMI_DEVICE; /* * Check for the RMI specific report ids. If they are misisng @@ -696,6 +725,17 @@ static int rmi_probe(struct hid_device *hdev, const struct hid_device_id *id) data->output_report_size = hid_report_len(output_report); + /* + * The write reports built by this driver occupy 6 bytes and the read + * handshake looks at the first 3 bytes of an input report, so refuse + * to drive a device whose reports cannot hold them. + */ + if (data->output_report_size < 6 || data->input_report_size < 3) { + hid_err(hdev, "rmi reports too small (out=%u in=%u)\n", + data->output_report_size, data->input_report_size); + goto start; + } + data->device_flags |= RMI_DEVICE; alloc_size = data->output_report_size + data->input_report_size; diff --git a/drivers/hid/hid-roccat-kone.c b/drivers/hid/hid-roccat-kone.c index fabc08efcfd8c..2510001fead94 100644 --- a/drivers/hid/hid-roccat-kone.c +++ b/drivers/hid/hid-roccat-kone.c @@ -36,6 +36,8 @@ static uint profile_numbers[5] = {0, 1, 2, 3, 4}; static void kone_profile_activated(struct kone_device *kone, uint new_profile) { + if (new_profile < 1 || new_profile > ARRAY_SIZE(kone->profiles)) + new_profile = 1; kone->actual_profile = new_profile; kone->actual_dpi = kone->profiles[new_profile - 1].startup_dpi; } @@ -793,8 +795,10 @@ static void kone_keep_values_up_to_date(struct kone_device *kone, { switch (event->event) { case kone_mouse_event_switch_profile: - kone->actual_dpi = kone->profiles[event->value - 1]. - startup_dpi; + if (event->value >= 1 && + event->value <= ARRAY_SIZE(kone->profiles)) + kone->actual_dpi = + kone->profiles[event->value - 1].startup_dpi; fallthrough; case kone_mouse_event_osd_profile: kone->actual_profile = event->value; diff --git a/drivers/hid/hid-roccat.c b/drivers/hid/hid-roccat.c index e413662f75082..e54b6a8a92ef7 100644 --- a/drivers/hid/hid-roccat.c +++ b/drivers/hid/hid-roccat.c @@ -70,6 +70,15 @@ static struct roccat_device *devices[ROCCAT_MAX_DEVICES]; /* protects modifications of devices array */ static DEFINE_MUTEX(devices_lock); +static void roccat_free_device(struct roccat_device *device) +{ + int i; + + for (i = 0; i < ROCCAT_CBUF_SIZE; i++) + kfree(device->cbuf[i].value); + kfree(device); +} + static ssize_t roccat_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos) { @@ -226,7 +235,7 @@ static int roccat_release(struct inode *inode, struct file *file) hid_hw_power(device->hid, PM_HINT_NORMAL); hid_hw_close(device->hid); } else { - kfree(device); + roccat_free_device(device); } } @@ -374,7 +383,7 @@ void roccat_disconnect(int minor) hid_hw_close(device->hid); wake_up_interruptible(&device->wait); } else { - kfree(device); + roccat_free_device(device); } } EXPORT_SYMBOL_GPL(roccat_disconnect); diff --git a/drivers/hid/hid-sensor-custom.c b/drivers/hid/hid-sensor-custom.c index 761760668f6d9..0b97d61df0c8b 100644 --- a/drivers/hid/hid-sensor-custom.c +++ b/drivers/hid/hid-sensor-custom.c @@ -609,7 +609,7 @@ static int hid_sensor_custom_add_attributes(struct hid_sensor_custom &sensor_inst->fields[i]. hid_custom_attribute_group); if (ret) - break; + goto err_remove_groups; /* For power or report field store indexes */ if (sensor_inst->fields[i].attribute.attrib_id == @@ -621,6 +621,13 @@ static int hid_sensor_custom_add_attributes(struct hid_sensor_custom } return ret; + +err_remove_groups: + while (--i >= 0) + sysfs_remove_group(&sensor_inst->pdev->dev.kobj, + &sensor_inst->fields[i].hid_custom_attribute_group); + kfree(sensor_inst->fields); + return ret; } static void hid_sensor_custom_remove_attributes(struct hid_sensor_custom * @@ -1005,26 +1012,26 @@ static int hid_sensor_custom_probe(struct platform_device *pdev) return ret; } - ret = sysfs_create_group(&sensor_inst->pdev->dev.kobj, - &enable_sensor_attr_group); + ret = hid_sensor_custom_add_attributes(sensor_inst); if (ret) goto err_remove_callback; - ret = hid_sensor_custom_add_attributes(sensor_inst); + ret = sysfs_create_group(&sensor_inst->pdev->dev.kobj, + &enable_sensor_attr_group); if (ret) - goto err_remove_group; + goto err_remove_attributes; ret = hid_sensor_custom_dev_if_add(sensor_inst); if (ret) - goto err_remove_attributes; + goto err_remove_group; return 0; -err_remove_attributes: - hid_sensor_custom_remove_attributes(sensor_inst); err_remove_group: sysfs_remove_group(&sensor_inst->pdev->dev.kobj, &enable_sensor_attr_group); +err_remove_attributes: + hid_sensor_custom_remove_attributes(sensor_inst); err_remove_callback: sensor_hub_remove_callback(hsdev, hsdev->usage); @@ -1042,9 +1049,10 @@ static void hid_sensor_custom_remove(struct platform_device *pdev) } hid_sensor_custom_dev_if_remove(sensor_inst); - hid_sensor_custom_remove_attributes(sensor_inst); + /* Remove enable_sensor first as it uses fields via power_state/report_state. */ sysfs_remove_group(&sensor_inst->pdev->dev.kobj, &enable_sensor_attr_group); + hid_sensor_custom_remove_attributes(sensor_inst); sensor_hub_remove_callback(hsdev, hsdev->usage); } diff --git a/drivers/hid/hid-sensor-hub.c b/drivers/hid/hid-sensor-hub.c index 1b725c8796506..faeb9c4536f52 100644 --- a/drivers/hid/hid-sensor-hub.c +++ b/drivers/hid/hid-sensor-hub.c @@ -239,12 +239,17 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, u32 field_index, int buffer_size, void *buffer) { struct hid_report *report; + struct hid_field *field; struct sensor_hub_data *data = hid_get_drvdata(hsdev->hdev); - int report_size; + size_t field_size; + size_t report_size; + size_t copied = 0; + size_t to_copy; int ret = 0; - u8 *val_ptr; - int buffer_index = 0; - int i; + unsigned int i; + + if (!buffer || buffer_size <= 0) + return -EINVAL; memset(buffer, 0, buffer_size); @@ -258,26 +263,29 @@ int sensor_hub_get_feature(struct hid_sensor_hub_device *hsdev, u32 report_id, hid_hw_request(hsdev->hdev, report, HID_REQ_GET_REPORT); hid_hw_wait(hsdev->hdev); + field = report->field[field_index]; + /* calculate number of bytes required to read this field */ - report_size = DIV_ROUND_UP(report->field[field_index]->report_size, - 8) * - report->field[field_index]->report_count; - if (!report_size) { + field_size = DIV_ROUND_UP(field->report_size, 8); + /* HID core stores each parsed report value in a __s32 slot. */ + if (!field_size || field_size > sizeof(field->value[0])) { + ret = -EINVAL; + goto done_proc; + } + if (field->report_count > SIZE_MAX / field_size) { ret = -EINVAL; goto done_proc; } - ret = min(report_size, buffer_size); - val_ptr = (u8 *)report->field[field_index]->value; - for (i = 0; i < report->field[field_index]->report_count; ++i) { - if (buffer_index >= ret) - break; + report_size = field_size * field->report_count; + report_size = min_t(size_t, report_size, buffer_size); - memcpy(&((u8 *)buffer)[buffer_index], val_ptr, - report->field[field_index]->report_size / 8); - val_ptr += sizeof(__s32); - buffer_index += (report->field[field_index]->report_size / 8); + for (i = 0; i < field->report_count && copied < report_size; ++i) { + to_copy = min(field_size, report_size - copied); + memcpy(&((u8 *)buffer)[copied], &field->value[i], to_copy); + copied += to_copy; } + ret = copied; done_proc: mutex_unlock(&data->mutex); diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c index b966e4044238c..24a184dd28868 100644 --- a/drivers/hid/hid-sony.c +++ b/drivers/hid/hid-sony.c @@ -27,6 +27,7 @@ * There will be no PIN request from the device. */ +#include #include #include #include @@ -520,14 +521,12 @@ static void sony_set_leds(struct sony_sc *sc); static inline void sony_schedule_work(struct sony_sc *sc, enum sony_worker which) { - unsigned long flags; - switch (which) { case SONY_WORKER_STATE: - spin_lock_irqsave(&sc->lock, flags); - if (!sc->defer_initialization && sc->state_worker_initialized) - schedule_work(&sc->state_worker); - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + if (!sc->defer_initialization && sc->state_worker_initialized) + schedule_work(&sc->state_worker); + } break; } } @@ -796,7 +795,6 @@ static const u8 *sony_report_fixup(struct hid_device *hdev, u8 *rdesc, static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size) { static const u8 sixaxis_battery_capacity[] = { 0, 1, 25, 50, 75, 100 }; - unsigned long flags; int offset; u8 battery_capacity; int battery_status; @@ -818,10 +816,10 @@ static void sixaxis_parse_report(struct sony_sc *sc, u8 *rd, int size) battery_status = POWER_SUPPLY_STATUS_DISCHARGING; } - spin_lock_irqsave(&sc->lock, flags); - sc->battery_capacity = battery_capacity; - sc->battery_status = battery_status; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + sc->battery_capacity = battery_capacity; + sc->battery_status = battery_status; + } if (sc->quirks & SIXAXIS_CONTROLLER) { int val; @@ -1625,15 +1623,14 @@ static int sony_battery_get_property(struct power_supply *psy, union power_supply_propval *val) { struct sony_sc *sc = power_supply_get_drvdata(psy); - unsigned long flags; int ret = 0; u8 battery_capacity; int battery_status; - spin_lock_irqsave(&sc->lock, flags); - battery_capacity = sc->battery_capacity; - battery_status = sc->battery_status; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + battery_capacity = sc->battery_capacity; + battery_status = sc->battery_status; + } switch (psp) { case POWER_SUPPLY_PROP_PRESENT: @@ -1715,10 +1712,9 @@ static inline int sony_compare_connection_type(struct sony_sc *sc0, static int sony_check_add_dev_list(struct sony_sc *sc) { struct sony_sc *entry; - unsigned long flags; int ret; - spin_lock_irqsave(&sony_dev_list_lock, flags); + guard(spinlock_irqsave)(&sony_dev_list_lock); list_for_each_entry(entry, &sony_device_list, list_node) { ret = memcmp(sc->mac_address, entry->mac_address, @@ -1732,27 +1728,23 @@ static int sony_check_add_dev_list(struct sony_sc *sc) "controller with MAC address %pMR already connected\n", sc->mac_address); } - goto unlock; + goto out; } } ret = 0; list_add(&(sc->list_node), &sony_device_list); -unlock: - spin_unlock_irqrestore(&sony_dev_list_lock, flags); +out: return ret; } static void sony_remove_dev_list(struct sony_sc *sc) { - unsigned long flags; + guard(spinlock_irqsave)(&sony_dev_list_lock); - if (sc->list_node.next) { - spin_lock_irqsave(&sony_dev_list_lock, flags); - list_del(&(sc->list_node)); - spin_unlock_irqrestore(&sony_dev_list_lock, flags); - } + if (!list_empty(&sc->list_node)) + list_del_init(&sc->list_node); } static int sony_get_bt_devaddr(struct sony_sc *sc) @@ -1879,16 +1871,21 @@ static inline void sony_init_output_report(struct sony_sc *sc, static inline void sony_cancel_work_sync(struct sony_sc *sc) { - unsigned long flags; - if (sc->state_worker_initialized) { - spin_lock_irqsave(&sc->lock, flags); - sc->state_worker_initialized = 0; - spin_unlock_irqrestore(&sc->lock, flags); + scoped_guard(spinlock_irqsave, &sc->lock) { + sc->state_worker_initialized = 0; + } cancel_work_sync(&sc->state_worker); } } +static void sony_cleanup(struct sony_sc *sc) +{ + sony_cancel_work_sync(sc); + sony_remove_dev_list(sc); + sony_release_device_id(sc); +} + static int sony_input_configured(struct hid_device *hdev, struct hid_input *hidinput) { @@ -2047,9 +2044,7 @@ static int sony_input_configured(struct hid_device *hdev, err_close: hid_hw_close(hdev); err_stop: - sony_cancel_work_sync(sc); - sony_remove_dev_list(sc); - sony_release_device_id(sc); + sony_cleanup(sc); return ret; } @@ -2075,6 +2070,8 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) } spin_lock_init(&sc->lock); + INIT_LIST_HEAD(&sc->list_node); + sc->device_id = -1; sc->quirks = quirks; hid_set_drvdata(hdev, sc); @@ -2103,6 +2100,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) ret = hid_hw_start(hdev, connect_mask); if (ret) { hid_err(hdev, "hw start failed\n"); + sony_cleanup(sc); return ret; } @@ -2154,7 +2152,7 @@ static int sony_probe(struct hid_device *hdev, const struct hid_device_id *id) err: usb_free_urb(sc->ghl_urb); - + sony_cleanup(sc); hid_hw_stop(hdev); return ret; } @@ -2164,18 +2162,14 @@ static void sony_remove(struct hid_device *hdev) struct sony_sc *sc = hid_get_drvdata(hdev); if (sc->quirks & (GHL_GUITAR_PS3WIIU | GHL_GUITAR_PS4)) { - timer_delete_sync(&sc->ghl_poke_timer); + /* poison, not kill: a pending timer must not re-submit during teardown */ + usb_poison_urb(sc->ghl_urb); + timer_shutdown_sync(&sc->ghl_poke_timer); usb_free_urb(sc->ghl_urb); } hid_hw_close(hdev); - - sony_cancel_work_sync(sc); - - sony_remove_dev_list(sc); - - sony_release_device_id(sc); - + sony_cleanup(sc); hid_hw_stop(hdev); } diff --git a/drivers/hid/hid-steam.c b/drivers/hid/hid-steam.c index 197126d6e0810..80d2eabc930ec 100644 --- a/drivers/hid/hid-steam.c +++ b/drivers/hid/hid-steam.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "hid-ids.h" MODULE_DESCRIPTION("HID driver for Valve Steam Controller"); @@ -68,13 +69,14 @@ static LIST_HEAD(steam_devices); /* Joystick runs are about 5 mm and 32768 units */ #define STEAM_DECK_JOYSTICK_RESOLUTION 6553 /* Accelerometer has 16 bit resolution and a range of +/- 2g */ -#define STEAM_DECK_ACCEL_RES_PER_G 16384 -#define STEAM_DECK_ACCEL_RANGE 32768 +#define STEAM_ACCEL_RES_PER_G 16384 +#define STEAM_ACCEL_RANGE 32768 +#define STEAM_ACCEL_FUZZ 128 #define STEAM_DECK_ACCEL_FUZZ 32 /* Gyroscope has 16 bit resolution and a range of +/- 2000 dps */ -#define STEAM_DECK_GYRO_RES_PER_DPS 16 -#define STEAM_DECK_GYRO_RANGE 32768 -#define STEAM_DECK_GYRO_FUZZ 1 +#define STEAM_GYRO_RES_PER_DPS 16 +#define STEAM_GYRO_RANGE 32768 +#define STEAM_GYRO_FUZZ 0 #define STEAM_PAD_FUZZ 256 @@ -149,7 +151,7 @@ enum { SETTING_USB_DEBUG_MODE, SETTING_LEFT_TRACKPAD_MODE, SETTING_RIGHT_TRACKPAD_MODE, - SETTING_MOUSE_POINTER_ENABLED, + SETTING_LIZARD_MODE, /* 10 */ SETTING_DPAD_DEADZONE, @@ -243,14 +245,39 @@ enum { /* Input report identifiers */ enum { - ID_CONTROLLER_STATE = 1, - ID_CONTROLLER_DEBUG = 2, - ID_CONTROLLER_WIRELESS = 3, - ID_CONTROLLER_STATUS = 4, - ID_CONTROLLER_DEBUG2 = 5, - ID_CONTROLLER_SECONDARY_STATE = 6, - ID_CONTROLLER_BLE_STATE = 7, - ID_CONTROLLER_DECK_STATE = 9 + ID_CONTROLLER_STATE = 1, + ID_CONTROLLER_DEBUG = 2, + ID_CONTROLLER_WIRELESS = 3, + ID_CONTROLLER_STATUS = 4, + ID_CONTROLLER_DEBUG2 = 5, + ID_CONTROLLER_SECONDARY_STATE = 6, + ID_CONTROLLER_BLE_STATE = 7, + ID_CONTROLLER_DECK_STATE = 9, +}; + +/* Read-only attributes */ +enum { + ATTRIB_UNIQUE_ID, // deprecated + ATTRIB_PRODUCT_ID, + ATTRIB_PRODUCT_REVISON, // deprecated + ATTRIB_CAPABILITIES = ATTRIB_PRODUCT_REVISON, // intentional aliasing + ATTRIB_FIRMWARE_VERSION, // deprecated + ATTRIB_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_FIRMWARE_BUILD_TIME, + ATTRIB_RADIO_DEVICE_ID0, + ATTRIB_RADIO_DEVICE_ID1, + ATTRIB_DONGLE_FIRMWARE_BUILD_TIME, + ATTRIB_HW_ID, // AKA BOARD_REVISION, + ATTRIB_BOOTLOADER_BUILD_TIME, + ATTRIB_CONNECTION_INTERVAL_IN_US, + ATTRIB_SECONDARY_FIRMWARE_BUILD_TIME, + ATTRIB_SECONDARY_BOOTLOADER_BUILD_TIME, + ATTRIB_SECONDARY_HW_ID, // AKA BOARD_REVISION, + ATTRIB_STREAMING, + ATTRIB_TRACKPAD_ID, + ATTRIB_SECONDARY_TRACKPAD_ID, + + ATTRIB_COUNT }; /* String attribute identifiers */ @@ -259,14 +286,14 @@ enum { ATTRIB_STR_UNIT_SERIAL, }; -/* Values for GYRO_MODE (bitmask) */ +/* Values for IMU_MODE (bitmask) */ enum { - SETTING_GYRO_MODE_OFF = 0, - SETTING_GYRO_MODE_STEERING = BIT(0), - SETTING_GYRO_MODE_TILT = BIT(1), - SETTING_GYRO_MODE_SEND_ORIENTATION = BIT(2), - SETTING_GYRO_MODE_SEND_RAW_ACCEL = BIT(3), - SETTING_GYRO_MODE_SEND_RAW_GYRO = BIT(4), + SETTING_IMU_MODE_OFF = 0, + SETTING_IMU_MODE_STEERING = BIT(0), + SETTING_IMU_MODE_TILT = BIT(1), + SETTING_IMU_MODE_SEND_ORIENTATION = BIT(2), + SETTING_IMU_MODE_SEND_RAW_ACCEL = BIT(3), + SETTING_IMU_MODE_SEND_RAW_GYRO = BIT(4), }; /* Trackpad modes */ @@ -282,6 +309,11 @@ enum { TRACKPAD_GESTURE_KEYBOARD, }; +struct steam_controller_attribute { + unsigned char tag; + __le32 value; +} __packed; + /* Pad identifiers for the deck */ #define STEAM_PAD_LEFT 0 #define STEAM_PAD_RIGHT 1 @@ -313,6 +345,7 @@ struct steam_device { u16 rumble_left; u16 rumble_right; unsigned int sensor_timestamp_us; + unsigned int sensor_update_rate_us; struct work_struct unregister_work; }; @@ -323,6 +356,13 @@ static int steam_recv_report(struct steam_device *steam, u8 *buf; int ret; + /* + * All reports start with a two byte header. + * We must read at least two bytes to get a sensible output. + */ + if (size < 2) + return -EINVAL; + r = steam->hdev->report_enum[HID_FEATURE_REPORT].report_id_hash[0]; if (!r) { hid_err(steam->hdev, "No HID_FEATURE_REPORT submitted - nothing to read\n"); @@ -345,10 +385,31 @@ static int steam_recv_report(struct steam_device *steam, ret = hid_hw_raw_request(steam->hdev, 0x00, buf, hid_report_len(r) + 1, HID_FEATURE_REPORT, HID_REQ_GET_REPORT); - if (ret > 0) - memcpy(data, buf + 1, min(size, ret - 1)); + if (ret > 0) { + /* Remove the report ID from the return buffer */ + ret--; + size = min(size, ret); + memcpy(data, buf + 1, size); + } kfree(buf); - return ret; + + if (ret < 0) + hid_err(steam->hdev, "%s: error %d\n", __func__, ret); + else + hid_dbg(steam->hdev, "Received report %*ph\n", size, data); + if (ret < 0) + return ret; + + if (ret < 2) { + hid_err(steam->hdev, "%s: reply too short\n", __func__); + return -EPROTO; + } + if (ret < data[1] + 2) { + hid_err(steam->hdev, "%s: expected %u bytes, read %i\n", + __func__, data[1] + 2, ret); + return -EPROTO; + } + return size; } static int steam_send_report(struct steam_device *steam, @@ -375,6 +436,8 @@ static int steam_send_report(struct steam_device *steam, /* The report ID is always 0 */ memcpy(buf + 1, cmd, size); + hid_dbg(steam->hdev, "Sending report %*ph\n", size, cmd); + /* * Sometimes the wireless controller fails with EPIPE * when sending a feature report. @@ -447,25 +510,59 @@ static int steam_get_serial(struct steam_device *steam) u8 cmd[] = {ID_GET_STRING_ATTRIBUTE, sizeof(steam->serial_no), ATTRIB_STR_UNIT_SERIAL}; u8 reply[3 + STEAM_SERIAL_LEN + 1]; - mutex_lock(&steam->report_mutex); + guard(mutex)(&steam->report_mutex); ret = steam_send_report(steam, cmd, sizeof(cmd)); if (ret < 0) - goto out; + return ret; ret = steam_recv_report(steam, reply, sizeof(reply)); if (ret < 0) - goto out; + return ret; if (reply[0] != ID_GET_STRING_ATTRIBUTE || reply[1] < 1 || reply[1] > sizeof(steam->serial_no) || reply[2] != ATTRIB_STR_UNIT_SERIAL) { - ret = -EIO; - goto out; + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; } reply[3 + STEAM_SERIAL_LEN] = 0; strscpy(steam->serial_no, reply + 3, reply[1]); -out: - mutex_unlock(&steam->report_mutex); return ret; } +static int steam_get_attributes(struct steam_device *steam) +{ + int ret = 0; + u8 cmd[] = {ID_GET_ATTRIBUTES_VALUES, 0}; + u8 reply[64] = {}; + u8 size; + int i; + struct steam_controller_attribute *attr; + + guard(mutex)(&steam->report_mutex); + ret = steam_send_report(steam, cmd, sizeof(cmd)); + if (ret < 0) + return ret; + ret = steam_recv_report(steam, reply, sizeof(reply)); + if (ret < 0) + return ret; + if (reply[0] != ID_GET_ATTRIBUTES_VALUES || reply[1] < 2) { + hid_err(steam->hdev, "%s: invalid reply (%*ph)\n", __func__, + (int)sizeof(reply), reply); + return -EIO; + } + + size = min(reply[1], sizeof(reply) - 2); + for (i = 0; i + sizeof(*attr) <= size; i += sizeof(*attr)) { + attr = (struct steam_controller_attribute *)&reply[i + 2]; + if (attr->tag == ATTRIB_CONNECTION_INTERVAL_IN_US) { + steam->sensor_update_rate_us = get_unaligned_le32(&attr->value); + hid_dbg(steam->hdev, "Sensor update rate: %uus\n", + steam->sensor_update_rate_us); + } + } + + return 0; +} + /* * This command requests the wireless adaptor to post an event * with the connection status. Useful if this driver is loaded when @@ -473,11 +570,8 @@ static int steam_get_serial(struct steam_device *steam) */ static inline int steam_request_conn_status(struct steam_device *steam) { - int ret; - mutex_lock(&steam->report_mutex); - ret = steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); - mutex_unlock(&steam->report_mutex); - return ret; + guard(mutex)(&steam->report_mutex); + return steam_send_report_byte(steam, ID_DONGLE_GET_WIRELESS_STATE); } /* @@ -624,6 +718,42 @@ static void steam_input_close(struct input_dev *dev) } } +static int steam_sensor_open(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return 0; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, + SETTING_IMU_MODE_SEND_RAW_ACCEL | SETTING_IMU_MODE_SEND_RAW_GYRO, + 0); + + return 0; +} + +static void steam_sensor_close(struct input_dev *dev) +{ + struct steam_device *steam = input_get_drvdata(dev); + unsigned long flags; + bool client_opened; + + spin_lock_irqsave(&steam->lock, flags); + client_opened = steam->client_opened; + spin_unlock_irqrestore(&steam->lock, flags); + if (client_opened) + return; + + guard(mutex)(&steam->report_mutex); + steam_write_settings(steam, SETTING_IMU_MODE, 0, 0); +} + static enum power_supply_property steam_battery_props[] = { POWER_SUPPLY_PROP_PRESENT, POWER_SUPPLY_PROP_SCOPE, @@ -837,9 +967,6 @@ static int steam_sensors_register(struct steam_device *steam) struct input_dev *sensors; int ret; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return 0; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -854,8 +981,14 @@ static int steam_sensors_register(struct steam_device *steam) input_set_drvdata(sensors, steam); sensors->dev.parent = &hdev->dev; + if (!(steam->quirks & STEAM_QUIRK_DECK)) { + sensors->open = steam_sensor_open; + sensors->close = steam_sensor_close; + } - sensors->name = "Steam Deck Motion Sensors"; + sensors->name = steam->quirks & STEAM_QUIRK_DECK ? + "Steam Deck Motion Sensors" : + "Steam Controller Motion Sensors"; sensors->phys = hdev->phys; sensors->uniq = steam->serial_no; sensors->id.bustype = hdev->bus; @@ -867,25 +1000,34 @@ static int steam_sensors_register(struct steam_device *steam) __set_bit(EV_MSC, sensors->evbit); __set_bit(MSC_TIMESTAMP, sensors->mscbit); - input_set_abs_params(sensors, ABS_X, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Y, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_set_abs_params(sensors, ABS_Z, -STEAM_DECK_ACCEL_RANGE, - STEAM_DECK_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); - input_abs_set_res(sensors, ABS_X, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Y, STEAM_DECK_ACCEL_RES_PER_G); - input_abs_set_res(sensors, ABS_Z, STEAM_DECK_ACCEL_RES_PER_G); - - input_set_abs_params(sensors, ABS_RX, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RY, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_set_abs_params(sensors, ABS_RZ, -STEAM_DECK_GYRO_RANGE, - STEAM_DECK_GYRO_RANGE, STEAM_DECK_GYRO_FUZZ, 0); - input_abs_set_res(sensors, ABS_RX, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RY, STEAM_DECK_GYRO_RES_PER_DPS); - input_abs_set_res(sensors, ABS_RZ, STEAM_DECK_GYRO_RES_PER_DPS); + if (steam->quirks & STEAM_QUIRK_DECK) { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_DECK_ACCEL_FUZZ, 0); + } else { + input_set_abs_params(sensors, ABS_X, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Y, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + input_set_abs_params(sensors, ABS_Z, -STEAM_ACCEL_RANGE, + STEAM_ACCEL_RANGE, STEAM_ACCEL_FUZZ, 0); + } + input_abs_set_res(sensors, ABS_X, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Y, STEAM_ACCEL_RES_PER_G); + input_abs_set_res(sensors, ABS_Z, STEAM_ACCEL_RES_PER_G); + + input_set_abs_params(sensors, ABS_RX, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RY, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_set_abs_params(sensors, ABS_RZ, -STEAM_GYRO_RANGE, + STEAM_GYRO_RANGE, STEAM_GYRO_FUZZ, 0); + input_abs_set_res(sensors, ABS_RX, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RY, STEAM_GYRO_RES_PER_DPS); + input_abs_set_res(sensors, ABS_RZ, STEAM_GYRO_RES_PER_DPS); ret = input_register_device(sensors); if (ret) @@ -916,9 +1058,6 @@ static void steam_sensors_unregister(struct steam_device *steam) { struct input_dev *sensors; - if (!(steam->quirks & STEAM_QUIRK_DECK)) - return; - rcu_read_lock(); sensors = rcu_dereference(steam->sensors); rcu_read_unlock(); @@ -966,6 +1105,12 @@ static int steam_register(struct steam_device *steam) strscpy(steam->serial_no, "XXXXXXXXXX", sizeof(steam->serial_no)); + ret = steam_get_attributes(steam); + if (ret < 0) + hid_err(steam->hdev, + "%s:steam_get_attributes failed with error %d\n", + __func__, ret); + hid_info(steam->hdev, "Steam Controller '%s' connected", steam->serial_no); @@ -1049,6 +1194,7 @@ static void steam_mode_switch_cb(struct work_struct *work) return; steam->gamepad_mode = !steam->gamepad_mode; + hid_dbg(steam->hdev, "%s: switching gamepad mode to %i\n", __func__, steam->gamepad_mode); if (steam->gamepad_mode) steam_set_lizard_mode(steam, false); else { @@ -1244,6 +1390,10 @@ static int steam_probe(struct hid_device *hdev, INIT_LIST_HEAD(&steam->list); INIT_WORK(&steam->rumble_work, steam_haptic_rumble_cb); steam->sensor_timestamp_us = 0; + if (steam->quirks & STEAM_QUIRK_DECK) + steam->sensor_update_rate_us = 4000; + else + steam->sensor_update_rate_us = 9000; INIT_WORK(&steam->unregister_work, steam_work_unregister_cb); /* @@ -1354,13 +1504,45 @@ static void steam_do_connect_event(struct steam_device *steam, bool connected) * Clamp the values to 32767..-32767 so that the range is * symmetrical and can be negated safely. */ -static inline s16 steam_le16(u8 *data) +static inline s16 steam_le16(const u8 *data) { - s16 x = (s16) le16_to_cpup((__le16 *)data); + s16 x = (s16) get_unaligned_le16((const __le16 *)data); return x == -32768 ? -32767 : x; } +struct steam_button_mapping { + int code; + u8 byte; + u8 bit; +}; + +struct steam_axis_mapping { + int code; + s8 sign; + u8 byte; +}; + +static void steam_map_buttons(struct input_dev *input, + const struct steam_button_mapping *mappings, const u8 *data) +{ + const struct steam_button_mapping *mapping; + + for (mapping = mappings; mapping->code; mapping++) + input_report_key(input, mapping->code, + data[mapping->byte] & BIT(mapping->bit)); +} + +static void steam_map_axes(struct input_dev *input, + const struct steam_axis_mapping *mappings, const u8 *data) +{ + const struct steam_axis_mapping *mapping; + + for (mapping = mappings; mapping->sign; mapping++) + input_report_abs(input, mapping->code, + mapping->sign * steam_le16(&data[mapping->byte])); +} + /* * The size for this message payload is 60. * The known values are: @@ -1427,18 +1609,52 @@ static inline s16 steam_le16(u8 *data) * 10.7 | -- | lpad_and_joy */ +static const struct steam_button_mapping steam_controller_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL, 9, 7 }, + { BTN_GRIPR, 10, 0 }, + { BTN_THUMBR, 10, 2 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMB2, 10, 4 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_axis_mappings[] = { + { ABS_RX, 1, 20 }, + { ABS_RY, -1, 22 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_controller_imu_mappings[] = { + { ABS_X, 1, 28 }, + { ABS_Z, -1, 30 }, + { ABS_Y, 1, 32 }, + { ABS_RX, 1, 34 }, + { ABS_RZ, 1, 36 }, + { ABS_RY, 1, 38 }, + { /* sentinel */ }, +}; + static void steam_do_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - /* 24 bits of buttons */ - u8 b8, b9, b10; s16 x, y; bool lpad_touched, lpad_and_joy; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - input_report_abs(input, ABS_HAT2Y, data[11]); input_report_abs(input, ABS_HAT2X, data[12]); @@ -1450,8 +1666,8 @@ static void steam_do_input_event(struct steam_device *steam, * joystick values. * (lpad_touched || lpad_and_joy) tells if the lpad is really touched. */ - lpad_touched = b10 & BIT(3); - lpad_and_joy = b10 & BIT(7); + lpad_touched = data[10] & BIT(3); + lpad_and_joy = data[10] & BIT(7); x = steam_le16(data + 16); y = -steam_le16(data + 18); @@ -1467,35 +1683,25 @@ static void steam_do_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT0X, 0); input_report_abs(input, ABS_HAT0Y, 0); } + input_report_key(input, BTN_THUMB, lpad_touched || lpad_and_joy); - input_report_abs(input, ABS_RX, steam_le16(data + 20)); - input_report_abs(input, ABS_RY, -steam_le16(data + 22)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMB, lpad_touched || lpad_and_joy); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(4))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); + steam_map_buttons(input, steam_controller_button_mappings, data); + steam_map_axes(input, steam_controller_axis_mappings, data); input_sync(input); } +static void steam_do_sensors_event(struct steam_device *steam, + struct input_dev *sensors, u8 *data) +{ + steam->sensor_timestamp_us += steam->sensor_update_rate_us; + + input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); + steam_map_axes(sensors, steam_controller_imu_mappings, data); + + input_sync(sensors); +} + /* * The size for this message payload is 56. * The known values are: @@ -1594,23 +1800,68 @@ static void steam_do_input_event(struct steam_device *steam, * 15.6 | -- | unknown * 15.7 | -- | unknown */ + +static const struct steam_button_mapping steam_deck_button_mappings[] = { + { BTN_TR2, 8, 0 }, + { BTN_TL2, 8, 1 }, + { BTN_TR, 8, 2 }, + { BTN_TL, 8, 3 }, + { BTN_Y, 8, 4 }, + { BTN_B, 8, 5 }, + { BTN_X, 8, 6 }, + { BTN_A, 8, 7 }, + { BTN_SELECT, 9, 4 }, + { BTN_MODE, 9, 5 }, + { BTN_START, 9, 6 }, + { BTN_GRIPL2, 9, 7 }, + { BTN_GRIPR2, 10, 0 }, + { BTN_THUMBL, 10, 6 }, + { BTN_THUMBR, 11, 2 }, + { BTN_DPAD_UP, 9, 0 }, + { BTN_DPAD_RIGHT, 9, 1 }, + { BTN_DPAD_LEFT, 9, 2 }, + { BTN_DPAD_DOWN, 9, 3 }, + { BTN_THUMB, 10, 1 }, + { BTN_THUMB2, 10, 2 }, + { BTN_GRIPL, 13, 1 }, + { BTN_GRIPR, 13, 2 }, + { BTN_BASE, 14, 2 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_axis_mappings[] = { + { ABS_X, 1, 48 }, + { ABS_Y, -1, 50 }, + { ABS_RX, 1, 52 }, + { ABS_RY, -1, 54 }, + { ABS_HAT2Y, 1, 44 }, + { ABS_HAT2X, 1, 46 }, + { /* sentinel */ }, +}; + +static const struct steam_axis_mapping steam_deck_imu_mappings[] = { + { ABS_X, 1, 24 }, + { ABS_Z, -1, 26 }, + { ABS_Y, 1, 28 }, + { ABS_RX, 1, 30 }, + { ABS_RZ, -1, 32 }, + { ABS_RY, 1, 34 }, + { /* sentinel */ }, +}; + static void steam_do_deck_input_event(struct steam_device *steam, struct input_dev *input, u8 *data) { - u8 b8, b9, b10, b11, b13, b14; + bool start_pressed; bool lpad_touched, rpad_touched; - b8 = data[8]; - b9 = data[9]; - b10 = data[10]; - b11 = data[11]; - b13 = data[13]; - b14 = data[14]; + start_pressed = data[9] & BIT(6); - if (!(b9 & BIT(6)) && steam->did_mode_switch) { + if (!start_pressed && steam->did_mode_switch) { steam->did_mode_switch = false; cancel_delayed_work(&steam->mode_switch); - } else if (!steam->client_opened && (b9 & BIT(6)) && !steam->did_mode_switch) { + } else if (!steam->client_opened && start_pressed && !steam->did_mode_switch) { + hid_dbg(steam->hdev, "%s: doing mode switch\n", __func__); steam->did_mode_switch = true; schedule_delayed_work(&steam->mode_switch, 45 * HZ / 100); } @@ -1618,8 +1869,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, if (!steam->gamepad_mode && lizard_mode) return; - lpad_touched = b10 & BIT(3); - rpad_touched = b10 & BIT(4); + lpad_touched = data[10] & BIT(3); + rpad_touched = data[10] & BIT(4); if (lpad_touched) { input_report_abs(input, ABS_HAT0X, steam_le16(data + 16)); @@ -1637,38 +1888,8 @@ static void steam_do_deck_input_event(struct steam_device *steam, input_report_abs(input, ABS_HAT1Y, 0); } - input_report_abs(input, ABS_X, steam_le16(data + 48)); - input_report_abs(input, ABS_Y, -steam_le16(data + 50)); - input_report_abs(input, ABS_RX, steam_le16(data + 52)); - input_report_abs(input, ABS_RY, -steam_le16(data + 54)); - - input_report_abs(input, ABS_HAT2Y, steam_le16(data + 44)); - input_report_abs(input, ABS_HAT2X, steam_le16(data + 46)); - - input_event(input, EV_KEY, BTN_TR2, !!(b8 & BIT(0))); - input_event(input, EV_KEY, BTN_TL2, !!(b8 & BIT(1))); - input_event(input, EV_KEY, BTN_TR, !!(b8 & BIT(2))); - input_event(input, EV_KEY, BTN_TL, !!(b8 & BIT(3))); - input_event(input, EV_KEY, BTN_Y, !!(b8 & BIT(4))); - input_event(input, EV_KEY, BTN_B, !!(b8 & BIT(5))); - input_event(input, EV_KEY, BTN_X, !!(b8 & BIT(6))); - input_event(input, EV_KEY, BTN_A, !!(b8 & BIT(7))); - input_event(input, EV_KEY, BTN_SELECT, !!(b9 & BIT(4))); - input_event(input, EV_KEY, BTN_MODE, !!(b9 & BIT(5))); - input_event(input, EV_KEY, BTN_START, !!(b9 & BIT(6))); - input_event(input, EV_KEY, BTN_GRIPL2, !!(b9 & BIT(7))); - input_event(input, EV_KEY, BTN_GRIPR2, !!(b10 & BIT(0))); - input_event(input, EV_KEY, BTN_THUMBL, !!(b10 & BIT(6))); - input_event(input, EV_KEY, BTN_THUMBR, !!(b11 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_UP, !!(b9 & BIT(0))); - input_event(input, EV_KEY, BTN_DPAD_RIGHT, !!(b9 & BIT(1))); - input_event(input, EV_KEY, BTN_DPAD_LEFT, !!(b9 & BIT(2))); - input_event(input, EV_KEY, BTN_DPAD_DOWN, !!(b9 & BIT(3))); - input_event(input, EV_KEY, BTN_THUMB, !!(b10 & BIT(1))); - input_event(input, EV_KEY, BTN_THUMB2, !!(b10 & BIT(2))); - input_event(input, EV_KEY, BTN_GRIPL, !!(b13 & BIT(1))); - input_event(input, EV_KEY, BTN_GRIPR, !!(b13 & BIT(2))); - input_event(input, EV_KEY, BTN_BASE, !!(b14 & BIT(2))); + steam_map_buttons(input, steam_deck_button_mappings, data); + steam_map_axes(input, steam_deck_axis_mappings, data); input_sync(input); } @@ -1676,25 +1897,13 @@ static void steam_do_deck_input_event(struct steam_device *steam, static void steam_do_deck_sensors_event(struct steam_device *steam, struct input_dev *sensors, u8 *data) { - /* - * The deck input report is received every 4 ms on average, - * with a jitter of +/- 4 ms even though the USB descriptor claims - * that it uses 1 kHz. - * Since the HID report does not include a sensor timestamp, - * use a fixed increment here. - */ - steam->sensor_timestamp_us += 4000; + steam->sensor_timestamp_us += steam->sensor_update_rate_us; if (!steam->gamepad_mode && lizard_mode) return; input_event(sensors, EV_MSC, MSC_TIMESTAMP, steam->sensor_timestamp_us); - input_report_abs(sensors, ABS_X, steam_le16(data + 24)); - input_report_abs(sensors, ABS_Z, -steam_le16(data + 26)); - input_report_abs(sensors, ABS_Y, steam_le16(data + 28)); - input_report_abs(sensors, ABS_RX, steam_le16(data + 30)); - input_report_abs(sensors, ABS_RZ, -steam_le16(data + 32)); - input_report_abs(sensors, ABS_RY, steam_le16(data + 34)); + steam_map_axes(sensors, steam_deck_imu_mappings, data); input_sync(sensors); } @@ -1773,6 +1982,9 @@ static int steam_raw_event(struct hid_device *hdev, input = rcu_dereference(steam->input); if (likely(input)) steam_do_input_event(steam, input, data); + sensors = rcu_dereference(steam->sensors); + if (likely(sensors)) + steam_do_sensors_event(steam, sensors, data); rcu_read_unlock(); break; case ID_CONTROLLER_DECK_STATE: diff --git a/drivers/hid/hid-uclogic-core.c b/drivers/hid/hid-uclogic-core.c index c6db3e7c5fd30..421600266fc69 100644 --- a/drivers/hid/hid-uclogic-core.c +++ b/drivers/hid/hid-uclogic-core.c @@ -533,7 +533,17 @@ static void uclogic_remove(struct hid_device *hdev) { struct uclogic_drvdata *drvdata = hid_get_drvdata(hdev); - timer_delete_sync(&drvdata->inrange_timer); + /* + * Shut the in-range timer down before stopping the device. + * uclogic_raw_event_pen() re-arms inrange_timer on every pen report + * and keeps running until hid_hw_stop() stops the transport, so a + * plain timer_delete_sync() here can be undone by a report landing in + * the window before hid_hw_stop(). timer_shutdown_sync() cancels the + * timer and makes any later re-arm a no-op, so it is provably dead + * before hid_hw_stop() frees the input device drvdata->pen_input + * points at. + */ + timer_shutdown_sync(&drvdata->inrange_timer); hid_hw_stop(hdev); kfree(drvdata->desc_ptr); uclogic_params_cleanup(&drvdata->params); diff --git a/drivers/hid/hid-universal-pidff.c b/drivers/hid/hid-universal-pidff.c index 549dac555d408..60180467a0bfd 100644 --- a/drivers/hid/hid-universal-pidff.c +++ b/drivers/hid/hid-universal-pidff.c @@ -104,12 +104,14 @@ static int universal_pidff_probe(struct hid_device *hdev, error = init_function(hdev, id->driver_data); if (error) { hid_warn(hdev, "Error initialising force feedback\n"); - goto err; + goto err_stop; } hid_info(hdev, "Universal pidff driver loaded successfully!"); return 0; +err_stop: + hid_hw_stop(hdev); err: return error; } diff --git a/drivers/hid/i2c-hid/Makefile b/drivers/hid/i2c-hid/Makefile index 55bd5e0f35af3..38d5d827f3ce4 100644 --- a/drivers/hid/i2c-hid/Makefile +++ b/drivers/hid/i2c-hid/Makefile @@ -8,7 +8,7 @@ obj-$(CONFIG_I2C_HID_CORE) += i2c-hid.o i2c-hid-objs = i2c-hid-core.o i2c-hid-$(CONFIG_DMI) += i2c-hid-dmi-quirks.o -obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o +obj-$(CONFIG_I2C_HID_ACPI) += i2c-hid-acpi.o i2c-hid-acpi-prp0001.o obj-$(CONFIG_I2C_HID_OF) += i2c-hid-of.o obj-$(CONFIG_I2C_HID_OF_ELAN) += i2c-hid-of-elan.o obj-$(CONFIG_I2C_HID_OF_GOODIX) += i2c-hid-of-goodix.o diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c new file mode 100644 index 0000000000000..d2cf4714ae7f1 --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi-prp0001.c @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * HID over I2C driver for PRP0001 devices missing hid-descr-addr + * + * Some devices, for example the Lenovo KaiTian N60d and Inspur CP300L3, use + * _HID "PRP0001" with _DSD compatible "hid-over-i2c" but lack "hid-descr-addr" + * from the _DSD. The HID descriptor address is provided only through an ACPI + * _DSM. The TPD0 node in the DSDT shows _DSM Function 1 returning 0x20. + * + * Copyright (C) 2026 谢致邦 (XIE Zhibang) + */ + +#include +#include +#include +#include +#include + +#include "i2c-hid.h" +#include "i2c-hid-acpi.h" + +static int i2c_hid_acpi_prp0001_power_up(struct i2chid_ops *ops) +{ + /* give the device time to power up */ + msleep(750); + return 0; +} + +static struct i2chid_ops i2c_hid_acpi_prp0001_ops = { + .power_up = i2c_hid_acpi_prp0001_power_up, + /* + * No .restore_sequence needed: the _DSM on these devices returns a + * constant (0x20) with no side effects, unlike some PNP0C50 _DSM + * implementations that switch the hardware between PS/2 and I2C modes. + */ +}; + +static int i2c_hid_acpi_prp0001_probe(struct i2c_client *client) +{ + struct device *dev = &client->dev; + struct acpi_device *adev; + u16 hid_descriptor_address; + int ret; + + /* If hid-descr-addr is present, let i2c-hid-of handle it */ + if (device_property_present(dev, "hid-descr-addr")) + return -ENODEV; + + adev = ACPI_COMPANION(dev); + if (!adev) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + dev_warn(dev, + "hid-descr-addr device property NOT found, using ACPI _DSM fallback. Contact vendor for firmware update!\n"); + hid_descriptor_address = ret; + + /* + * No acpi_device_fix_up_power() needed: TPD0 has no _PS0, _PS3, _PSC + * or _PRx methods and follows I2C bus power. + */ + return i2c_hid_core_probe(client, &i2c_hid_acpi_prp0001_ops, + hid_descriptor_address, 0); +} + +static const struct of_device_id i2c_hid_acpi_prp0001_of_match[] = { + { .compatible = "hid-over-i2c" }, + {}, +}; +MODULE_DEVICE_TABLE(of, i2c_hid_acpi_prp0001_of_match); + +static const struct i2c_device_id i2c_hid_acpi_prp0001_id[] = { + { .name = "hid-over-i2c" }, + { } +}; +MODULE_DEVICE_TABLE(i2c, i2c_hid_acpi_prp0001_id); + +static struct i2c_driver i2c_hid_acpi_prp0001_driver = { + .driver = { + .name = "i2c_hid_acpi_prp0001", + .pm = &i2c_hid_core_pm, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + /* + * of_match_ptr() makes this NULL when CONFIG_OF=n, but that's + * fine: the I2C id_table with "hid-over-i2c" handles matching + * via client->name (set by acpi_set_modalias() from the _DSD + * compatible property). + */ + .of_match_table = of_match_ptr(i2c_hid_acpi_prp0001_of_match), + }, + + .probe = i2c_hid_acpi_prp0001_probe, + .remove = i2c_hid_core_remove, + .shutdown = i2c_hid_core_shutdown, + .id_table = i2c_hid_acpi_prp0001_id, +}; + +module_i2c_driver(i2c_hid_acpi_prp0001_driver); + +MODULE_DESCRIPTION("HID over I2C driver for PRP0001 devices missing hid-descr-addr"); +MODULE_AUTHOR("谢致邦 (XIE Zhibang) "); +MODULE_LICENSE("GPL"); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.c b/drivers/hid/i2c-hid/i2c-hid-acpi.c index abd700a101f46..13f977d6aab61 100644 --- a/drivers/hid/i2c-hid/i2c-hid-acpi.c +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.c @@ -25,9 +25,9 @@ #include #include #include -#include #include "i2c-hid.h" +#include "i2c-hid-acpi.h" struct i2c_hid_acpi { struct i2chid_ops ops; @@ -48,39 +48,11 @@ static const struct acpi_device_id i2c_hid_acpi_blacklist[] = { { } }; -/* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ -static guid_t i2c_hid_guid = - GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, - 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); - -static int i2c_hid_acpi_get_descriptor(struct i2c_hid_acpi *ihid_acpi) -{ - struct acpi_device *adev = ihid_acpi->adev; - acpi_handle handle = acpi_device_handle(adev); - union acpi_object *obj; - u16 hid_descriptor_address; - - if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) - return -ENODEV; - - obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, 1, 1, NULL, - ACPI_TYPE_INTEGER); - if (!obj) { - acpi_handle_err(handle, "Error _DSM call to get HID descriptor address failed\n"); - return -ENODEV; - } - - hid_descriptor_address = obj->integer.value; - ACPI_FREE(obj); - - return hid_descriptor_address; -} - static void i2c_hid_acpi_restore_sequence(struct i2chid_ops *ops) { struct i2c_hid_acpi *ihid_acpi = container_of(ops, struct i2c_hid_acpi, ops); - i2c_hid_acpi_get_descriptor(ihid_acpi); + i2c_hid_acpi_get_descriptor(ihid_acpi->adev); } static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) @@ -93,24 +65,28 @@ static void i2c_hid_acpi_shutdown_tail(struct i2chid_ops *ops) static int i2c_hid_acpi_probe(struct i2c_client *client) { struct device *dev = &client->dev; + struct acpi_device *adev = ACPI_COMPANION(dev); struct i2c_hid_acpi *ihid_acpi; u16 hid_descriptor_address; int ret; - ihid_acpi = devm_kzalloc(&client->dev, sizeof(*ihid_acpi), GFP_KERNEL); + if (acpi_match_device_ids(adev, i2c_hid_acpi_blacklist) == 0) + return -ENODEV; + + ret = i2c_hid_acpi_get_descriptor(adev); + if (ret < 0) + return ret; + hid_descriptor_address = ret; + + ihid_acpi = devm_kzalloc(dev, sizeof(*ihid_acpi), GFP_KERNEL); if (!ihid_acpi) return -ENOMEM; - ihid_acpi->adev = ACPI_COMPANION(dev); + ihid_acpi->adev = adev; ihid_acpi->ops.shutdown_tail = i2c_hid_acpi_shutdown_tail; ihid_acpi->ops.restore_sequence = i2c_hid_acpi_restore_sequence; - ret = i2c_hid_acpi_get_descriptor(ihid_acpi); - if (ret < 0) - return ret; - hid_descriptor_address = ret; - - acpi_device_fix_up_power(ihid_acpi->adev); + acpi_device_fix_up_power(adev); return i2c_hid_core_probe(client, &ihid_acpi->ops, hid_descriptor_address, 0); diff --git a/drivers/hid/i2c-hid/i2c-hid-acpi.h b/drivers/hid/i2c-hid/i2c-hid-acpi.h new file mode 100644 index 0000000000000..0bbed1853313d --- /dev/null +++ b/drivers/hid/i2c-hid/i2c-hid-acpi.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _I2C_HID_ACPI_H +#define _I2C_HID_ACPI_H + +#include +#include + +static inline int i2c_hid_acpi_get_descriptor(struct acpi_device *adev) +{ + /* HID I²C Device: 3cdff6f7-4267-4555-ad05-b30a3d8938de */ + static const guid_t i2c_hid_guid = + GUID_INIT(0x3CDFF6F7, 0x4267, 0x4555, + 0xAD, 0x05, 0xB3, 0x0A, 0x3D, 0x89, 0x38, 0xDE); + + acpi_handle handle = acpi_device_handle(adev); + union acpi_object *obj; + u16 addr; + + obj = acpi_evaluate_dsm_typed(handle, &i2c_hid_guid, + 1, 1, NULL, ACPI_TYPE_INTEGER); + if (!obj) { + acpi_handle_err(handle, + "Error _DSM call to get HID descriptor address failed\n"); + return -ENODEV; + } + + addr = obj->integer.value; + ACPI_FREE(obj); + return addr; +} + +#endif diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c index e0a302544cef4..6d407cbed1886 100644 --- a/drivers/hid/i2c-hid/i2c-hid-core.c +++ b/drivers/hid/i2c-hid/i2c-hid-core.c @@ -790,7 +790,7 @@ static int i2c_hid_parse(struct hid_device *hid) ihid->hdesc.wReportDescRegister, rdesc, rsize); if (ret) { - hid_err(hid, "reading report descriptor failed\n"); + dev_err(&client->dev, "reading report descriptor failed\n"); goto out; } } diff --git a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c index f1597ad67e7c8..f4dbcd1d1d472 100644 --- a/drivers/hid/i2c-hid/i2c-hid-of-goodix.c +++ b/drivers/hid/i2c-hid/i2c-hid-of-goodix.c @@ -51,8 +51,10 @@ static int goodix_i2c_hid_power_up(struct i2chid_ops *ops) return ret; ret = regulator_enable(ihid_goodix->vddio); - if (ret) + if (ret) { + regulator_disable(ihid_goodix->vdd); return ret; + } if (ihid_goodix->timings->post_power_delay_ms) msleep(ihid_goodix->timings->post_power_delay_ms); diff --git a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c index 0156ab3917789..c6ba6886072ad 100644 --- a/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c +++ b/drivers/hid/intel-thc-hid/intel-quicki2c/pci-quicki2c.c @@ -771,6 +771,7 @@ static void quicki2c_remove(struct pci_dev *pdev) quicki2c_hid_remove(qcdev); quicki2c_dma_deinit(qcdev); + pm_runtime_dont_use_autosuspend(qcdev->dev); pm_runtime_get_noresume(qcdev->dev); quicki2c_dev_deinit(qcdev); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c index f0830a56d556b..1c96d1d85845c 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/pci-quickspi.c @@ -556,7 +556,14 @@ static int quickspi_alloc_report_buf(struct quickspi_device *qsdev) max_report_len = max(le16_to_cpu(qsdev->dev_desc.max_output_len), le16_to_cpu(qsdev->dev_desc.max_input_len)); - qsdev->report_buf = devm_kzalloc(qsdev->dev, max_report_len, GFP_KERNEL); + /* + * write_cmd_to_txdma() writes the output report header ahead of the + * content in this buffer, so it has to hold both. + */ + qsdev->report_buf_size = HIDSPI_OUTPUT_REPORT_SIZE(max_report_len); + + qsdev->report_buf = devm_kzalloc(qsdev->dev, qsdev->report_buf_size, + GFP_KERNEL); if (!qsdev->report_buf) return -ENOMEM; @@ -715,6 +722,7 @@ static void quickspi_remove(struct pci_dev *pdev) quickspi_hid_remove(qsdev); quickspi_dma_deinit(qsdev); + pm_runtime_dont_use_autosuspend(qsdev->dev); pm_runtime_get_noresume(qsdev->dev); quickspi_dev_deinit(qsdev); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h index bf5e18f5a5f42..0ed964bfe3dd2 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-dev.h @@ -157,6 +157,7 @@ struct quickspi_device { u8 *report_descriptor; u8 *input_buf; u8 *report_buf; + u32 report_buf_size; u32 report_len; wait_queue_head_t reset_ack_wq; diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c index ad52e402c28ac..9df948f94d5a3 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-hid.c @@ -61,7 +61,7 @@ static int quickspi_hid_raw_request(struct hid_device *hid, switch (reqtype) { case HID_REQ_GET_REPORT: - ret = quickspi_get_report(qsdev, rtype, reportnum, buf); + ret = quickspi_get_report(qsdev, rtype, reportnum, buf, len); break; case HID_REQ_SET_REPORT: ret = quickspi_set_report(qsdev, rtype, reportnum, buf, len); diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c index cb19057f1191b..847c5ec55569a 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.c @@ -30,6 +30,9 @@ static int write_cmd_to_txdma(struct quickspi_device *qsdev, write_buf = (struct output_report *)qsdev->report_buf; + if (HIDSPI_OUTPUT_REPORT_SIZE(report_buf_len) > qsdev->report_buf_size) + return -EINVAL; + write_buf->output_hdr.report_type = report_type; write_buf->output_hdr.content_len = cpu_to_le16(report_buf_len); write_buf->output_hdr.content_id = report_id; @@ -342,10 +345,12 @@ int reset_tic(struct quickspi_device *qsdev) } int quickspi_get_report(struct quickspi_device *qsdev, - u8 report_type, unsigned int report_id, void *buf) + u8 report_type, unsigned int report_id, void *buf, + u32 buf_len) { int rep_type; int ret; + u32 report_len; if (report_type == HID_INPUT_REPORT) { rep_type = GET_INPUT_REPORT; @@ -372,9 +377,17 @@ int quickspi_get_report(struct quickspi_device *qsdev, } qsdev->get_report_cmpl = false; - memcpy(buf, qsdev->report_buf, qsdev->report_len); + /* quickspi_handle_input_data() updates this from IRQ context. */ + report_len = READ_ONCE(qsdev->report_len); + if (report_len > buf_len) { + dev_err_once(qsdev->dev, "Get report response too big, %u vs %u\n", + report_len, buf_len); + return -EINVAL; + } + + memcpy(buf, qsdev->report_buf, report_len); - return qsdev->report_len; + return report_len; } int quickspi_set_report(struct quickspi_device *qsdev, diff --git a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h index 775e29c1ed136..8a2338bee8085 100644 --- a/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h +++ b/drivers/hid/intel-thc-hid/intel-quickspi/quickspi-protocol.h @@ -12,7 +12,7 @@ struct quickspi_device; void quickspi_handle_input_data(struct quickspi_device *qsdev, u32 buf_len); int quickspi_get_report(struct quickspi_device *qsdev, u8 report_type, - unsigned int report_id, void *buf); + unsigned int report_id, void *buf, u32 buf_len); int quickspi_set_report(struct quickspi_device *qsdev, u8 report_type, unsigned int report_id, void *buf, u32 buf_len); int quickspi_get_report_descriptor(struct quickspi_device *qsdev); diff --git a/drivers/hid/usbhid/hid-pidff.c b/drivers/hid/usbhid/hid-pidff.c index 7eb9e28c0d909..b9a7646e2786a 100644 --- a/drivers/hid/usbhid/hid-pidff.c +++ b/drivers/hid/usbhid/hid-pidff.c @@ -1520,13 +1520,20 @@ static int pidff_check_autocenter(struct pidff_device *pidff, int hid_pidff_init_with_quirks(struct hid_device *hid, u32 initial_quirks) { struct pidff_device *pidff; - struct hid_input *hidinput = - list_entry(hid->inputs.next, struct hid_input, list); - struct input_dev *dev = hidinput->input; + struct hid_input *hidinput; + struct input_dev *dev; struct ff_device *ff; int max_effects; int error; + if (list_empty(&hid->inputs)) { + hid_err(hid, "no inputs found\n"); + return -ENODEV; + } + + hidinput = list_first_entry(&hid->inputs, struct hid_input, list); + dev = hidinput->input; + hid_dbg(hid, "starting pid init\n"); if (list_empty(&hid->report_enum[HID_OUTPUT_REPORT].report_list)) { diff --git a/drivers/hid/wacom_wac.c b/drivers/hid/wacom_wac.c index da1f0ea85625d..af76e49fd0f25 100644 --- a/drivers/hid/wacom_wac.c +++ b/drivers/hid/wacom_wac.c @@ -1548,6 +1548,19 @@ static int wacom_intuos_pro2_bt_irq(struct wacom_wac *wacom, size_t len) return 0; } + if (wacom->features.type == INTUOSP2_BT || + wacom->features.type == INTUOSP2S_BT) { + if (len < 286) { + dev_warn(wacom->pen_input->dev.parent, + "Pro2 BT report too short: %zu bytes\n", len); + return 0; + } + } else if (len < 46) { + dev_warn(wacom->pen_input->dev.parent, + "Pro2 BT report too short: %zu bytes\n", len); + return 0; + } + wacom_intuos_pro2_bt_pen(wacom); if (wacom->features.type == INTUOSP2_BT || wacom->features.type == INTUOSP2S_BT) { diff --git a/drivers/hsi/controllers/omap_ssi_core.c b/drivers/hsi/controllers/omap_ssi_core.c index eeacc427fd659..47826315d7d62 100644 --- a/drivers/hsi/controllers/omap_ssi_core.c +++ b/drivers/hsi/controllers/omap_ssi_core.c @@ -502,6 +502,12 @@ static int ssi_probe(struct platform_device *pd) pm_runtime_enable(&pd->dev); + ssi->device.dma_mask = &ssi->device.coherent_dma_mask; + + err = dma_set_mask_and_coherent(&ssi->device, DMA_BIT_MASK(32)); + if (err) + goto out2; + err = ssi_hw_init(ssi); if (err < 0) goto out2; diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c index fd1d675ae37a4..d5348195e4fb0 100644 --- a/drivers/hv/channel_mgmt.c +++ b/drivers/hv/channel_mgmt.c @@ -844,14 +844,14 @@ static void vmbus_wait_for_unload(void) = per_cpu_ptr(hv_context.cpu_context, cpu); /* - * In a CoCo VM the synic_message_page is not allocated + * In a CoCo VM the hyp_synic_message_page is not allocated * in hv_synic_alloc(). Instead it is set/cleared in * hv_synic_enable_regs() and hv_synic_disable_regs() * such that it is set only when the CPU is online. If * not all present CPUs are online, the message page * might be NULL, so skip such CPUs. */ - page_addr = hv_cpu->synic_message_page; + page_addr = hv_cpu->hyp_synic_message_page; if (!page_addr) continue; @@ -892,7 +892,7 @@ static void vmbus_wait_for_unload(void) struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); - page_addr = hv_cpu->synic_message_page; + page_addr = hv_cpu->hyp_synic_message_page; if (!page_addr) continue; diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c index 1fe3573ae52a4..9be2af772867d 100644 --- a/drivers/hv/connection.c +++ b/drivers/hv/connection.c @@ -71,7 +71,8 @@ module_param(max_version, uint, S_IRUGO); MODULE_PARM_DESC(max_version, "Maximal VMBus protocol version which can be negotiated"); -int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) +static int vmbus_try_connection_id(struct vmbus_channel_msginfo *msginfo, + u32 version, u32 connection_id) { int ret = 0; struct vmbus_channel_initiate_contact *msg; @@ -86,20 +87,20 @@ int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) msg->vmbus_version_requested = version; /* - * VMBus protocol 5.0 (VERSION_WIN10_V5) and higher require that we must - * use VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate Contact Message, - * and for subsequent messages, we must use the Message Connection ID - * field in the host-returned Version Response Message. And, with - * VERSION_WIN10_V5 and higher, we don't use msg->interrupt_page, but we - * tell the host explicitly that we still use VMBUS_MESSAGE_SINT(2) for - * compatibility. + * For VMBus protocol 5.0 (VERSION_WIN10_V5) and higher, use the + * caller-supplied connection_id for the Initiate Contact message so + * the caller can implement the required retry scheme. For subsequent + * messages, use the Message Connection ID field in the host-returned + * Version Response message. With VERSION_WIN10_V5 and higher, we don't + * use msg->interrupt_page, but tell the host explicitly that we still + * use VMBUS_MESSAGE_SINT(2) for compatibility. * * On old hosts, we should always use VMBUS_MESSAGE_CONNECTION_ID (1). */ if (version >= VERSION_WIN10_V5) { msg->msg_sint = VMBUS_MESSAGE_SINT; msg->msg_vtl = ms_hyperv.vtl; - vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID_4; + vmbus_connection.msg_conn_id = connection_id; } else { msg->interrupt_page = virt_to_phys(vmbus_connection.int_page); vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID; @@ -161,6 +162,22 @@ int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) return ret; } +int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) +{ + int ret; + + /* Try the redirect ID first for VTL2 with VMBus protocol 5.0+. */ + if (version >= VERSION_WIN10_V5 && ms_hyperv.vtl == 2) { + ret = vmbus_try_connection_id(msginfo, version, + VMBUS_MESSAGE_CONNECTION_ID_REDIRECT); + if (ret != -ENXIO) + return ret; + } + + return vmbus_try_connection_id(msginfo, version, + VMBUS_MESSAGE_CONNECTION_ID_4); +} + /* * vmbus_connect - Sends a connect request on the partition service connection */ @@ -454,18 +471,10 @@ int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep) switch (ret) { case HV_STATUS_INVALID_CONNECTION_ID: - /* - * See vmbus_negotiate_version(): VMBus protocol 5.0 - * and higher require that we must use - * VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate - * Contact message, but on old hosts that only - * support VMBus protocol 4.0 or lower, here we get - * HV_STATUS_INVALID_CONNECTION_ID and we should - * return an error immediately without retrying. - */ + /* Allow INITIATE_CONTACT to try another connection ID. */ hdr = buffer; if (hdr->msgtype == CHANNELMSG_INITIATE_CONTACT) - return -EINVAL; + return -ENXIO; /* * We could get this if we send messages too * frequently. diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c index b14c5f9e0ef29..90db1e17582d8 100644 --- a/drivers/hv/hv.c +++ b/drivers/hv/hv.c @@ -96,10 +96,70 @@ int hv_post_message(union hv_connection_id connection_id, return hv_result(status); } +static int hv_alloc_page(void **page, bool decrypt, const char *note) +{ + int ret = 0; + + /* + * After the page changes its encryption status, its contents might + * appear scrambled on some hardware. Thus `get_zeroed_page` would + * zero the page out in vain, so do that explicitly exactly once. + * + * By default, the page is allocated encrypted in a CoCo VM. + */ + *page = (void *)__get_free_page(GFP_KERNEL); + if (!*page) + return -ENOMEM; + + if (decrypt) + ret = set_memory_decrypted((unsigned long)*page, 1); + if (ret) + goto failed; + + memset(*page, 0, PAGE_SIZE); + return 0; + +failed: + /* + * Report the failure but don't put the page back on the free list as + * its encryption status is unknown. + */ + pr_err("allocation failed for %s page, error %d, decrypted %d\n", + note, ret, decrypt); + *page = NULL; + return ret; +} + +static int hv_free_page(void **page, bool encrypt, const char *note) +{ + int ret = 0; + + if (!*page) + return 0; + + if (encrypt) + ret = set_memory_encrypted((unsigned long)*page, 1); + + /* + * In the case of the failure, the page is leaked. Something is wrong, + * prefer to lose the page with the unknown encryption status and stay afloat. + */ + if (ret) + pr_err("deallocation failed for %s page, error %d, encrypt %d\n", + note, ret, encrypt); + else + free_page((unsigned long)*page); + + *page = NULL; + + return ret; +} + int hv_synic_alloc(void) { int cpu, ret = -ENOMEM; struct hv_per_cpu_context *hv_cpu; + const bool decrypt = !vmbus_is_confidential(); /* * First, zero all per-cpu memory areas so hv_synic_free() can @@ -125,73 +185,37 @@ int hv_synic_alloc(void) vmbus_on_msg_dpc, (unsigned long)hv_cpu); if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) { - hv_cpu->post_msg_page = (void *)get_zeroed_page(GFP_ATOMIC); - if (!hv_cpu->post_msg_page) { - pr_err("Unable to allocate post msg page\n"); + ret = hv_alloc_page(&hv_cpu->post_msg_page, + decrypt, "post msg"); + if (ret) goto err; - } - - ret = set_memory_decrypted((unsigned long)hv_cpu->post_msg_page, 1); - if (ret) { - pr_err("Failed to decrypt post msg page: %d\n", ret); - /* Just leak the page, as it's unsafe to free the page. */ - hv_cpu->post_msg_page = NULL; - goto err; - } - - memset(hv_cpu->post_msg_page, 0, PAGE_SIZE); } /* - * Synic message and event pages are allocated by paravisor. - * Skip these pages allocation here. + * If these SynIC pages are not allocated, SIEF and SIM pages + * are configured using what the root partition or the paravisor + * provides upon reading the SIEFP and SIMP registers. */ if (!ms_hyperv.paravisor_present && !hv_root_partition()) { - hv_cpu->synic_message_page = - (void *)get_zeroed_page(GFP_ATOMIC); - if (!hv_cpu->synic_message_page) { - pr_err("Unable to allocate SYNIC message page\n"); + ret = hv_alloc_page(&hv_cpu->hyp_synic_message_page, + decrypt, "hypervisor SynIC msg"); + if (ret) goto err; - } - - hv_cpu->synic_event_page = - (void *)get_zeroed_page(GFP_ATOMIC); - if (!hv_cpu->synic_event_page) { - pr_err("Unable to allocate SYNIC event page\n"); - - free_page((unsigned long)hv_cpu->synic_message_page); - hv_cpu->synic_message_page = NULL; + ret = hv_alloc_page(&hv_cpu->hyp_synic_event_page, + decrypt, "hypervisor SynIC event"); + if (ret) goto err; - } } - if (!ms_hyperv.paravisor_present && - (hv_isolation_type_snp() || hv_isolation_type_tdx())) { - ret = set_memory_decrypted((unsigned long) - hv_cpu->synic_message_page, 1); - if (ret) { - pr_err("Failed to decrypt SYNIC msg page: %d\n", ret); - hv_cpu->synic_message_page = NULL; - - /* - * Free the event page here so that hv_synic_free() - * won't later try to re-encrypt it. - */ - free_page((unsigned long)hv_cpu->synic_event_page); - hv_cpu->synic_event_page = NULL; + if (vmbus_is_confidential()) { + ret = hv_alloc_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); + if (ret) goto err; - } - - ret = set_memory_decrypted((unsigned long) - hv_cpu->synic_event_page, 1); - if (ret) { - pr_err("Failed to decrypt SYNIC event page: %d\n", ret); - hv_cpu->synic_event_page = NULL; + ret = hv_alloc_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + if (ret) goto err; - } - - memset(hv_cpu->synic_message_page, 0, PAGE_SIZE); - memset(hv_cpu->synic_event_page, 0, PAGE_SIZE); } } @@ -207,48 +231,28 @@ int hv_synic_alloc(void) void hv_synic_free(void) { - int cpu, ret; + int cpu; + const bool encrypt = !vmbus_is_confidential(); for_each_present_cpu(cpu) { struct hv_per_cpu_context *hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); - /* It's better to leak the page if the encryption fails. */ - if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) { - if (hv_cpu->post_msg_page) { - ret = set_memory_encrypted((unsigned long) - hv_cpu->post_msg_page, 1); - if (ret) { - pr_err("Failed to encrypt post msg page: %d\n", ret); - hv_cpu->post_msg_page = NULL; - } - } + if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) + hv_free_page(&hv_cpu->post_msg_page, + encrypt, "post msg"); + if (!ms_hyperv.paravisor_present && !hv_root_partition()) { + hv_free_page(&hv_cpu->hyp_synic_event_page, + encrypt, "hypervisor SynIC event"); + hv_free_page(&hv_cpu->hyp_synic_message_page, + encrypt, "hypervisor SynIC msg"); } - - if (!ms_hyperv.paravisor_present && - (hv_isolation_type_snp() || hv_isolation_type_tdx())) { - if (hv_cpu->synic_message_page) { - ret = set_memory_encrypted((unsigned long) - hv_cpu->synic_message_page, 1); - if (ret) { - pr_err("Failed to encrypt SYNIC msg page: %d\n", ret); - hv_cpu->synic_message_page = NULL; - } - } - - if (hv_cpu->synic_event_page) { - ret = set_memory_encrypted((unsigned long) - hv_cpu->synic_event_page, 1); - if (ret) { - pr_err("Failed to encrypt SYNIC event page: %d\n", ret); - hv_cpu->synic_event_page = NULL; - } - } + if (vmbus_is_confidential()) { + hv_free_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + hv_free_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); } - - free_page((unsigned long)hv_cpu->post_msg_page); - free_page((unsigned long)hv_cpu->synic_event_page); - free_page((unsigned long)hv_cpu->synic_message_page); } kfree(hv_context.hv_numa_map); @@ -278,12 +282,12 @@ void hv_synic_enable_regs(unsigned int cpu) /* Mask out vTOM bit. ioremap_cache() maps decrypted */ u64 base = (simp.base_simp_gpa << HV_HYP_PAGE_SHIFT) & ~ms_hyperv.shared_gpa_boundary; - hv_cpu->synic_message_page = + hv_cpu->hyp_synic_message_page = (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE); - if (!hv_cpu->synic_message_page) + if (!hv_cpu->hyp_synic_message_page) pr_err("Fail to map synic message page.\n"); } else { - simp.base_simp_gpa = virt_to_phys(hv_cpu->synic_message_page) + simp.base_simp_gpa = virt_to_phys(hv_cpu->hyp_synic_message_page) >> HV_HYP_PAGE_SHIFT; } @@ -297,12 +301,12 @@ void hv_synic_enable_regs(unsigned int cpu) /* Mask out vTOM bit. ioremap_cache() maps decrypted */ u64 base = (siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT) & ~ms_hyperv.shared_gpa_boundary; - hv_cpu->synic_event_page = + hv_cpu->hyp_synic_event_page = (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE); - if (!hv_cpu->synic_event_page) + if (!hv_cpu->hyp_synic_event_page) pr_err("Fail to map synic event page.\n"); } else { - siefp.base_siefp_gpa = virt_to_phys(hv_cpu->synic_event_page) + siefp.base_siefp_gpa = virt_to_phys(hv_cpu->hyp_synic_event_page) >> HV_HYP_PAGE_SHIFT; } @@ -360,8 +364,8 @@ void hv_synic_disable_regs(unsigned int cpu) */ simp.simp_enabled = 0; if (ms_hyperv.paravisor_present || hv_root_partition()) { - iounmap(hv_cpu->synic_message_page); - hv_cpu->synic_message_page = NULL; + iounmap(hv_cpu->hyp_synic_message_page); + hv_cpu->hyp_synic_message_page = NULL; } else { simp.base_simp_gpa = 0; } @@ -372,8 +376,8 @@ void hv_synic_disable_regs(unsigned int cpu) siefp.siefp_enabled = 0; if (ms_hyperv.paravisor_present || hv_root_partition()) { - iounmap(hv_cpu->synic_event_page); - hv_cpu->synic_event_page = NULL; + iounmap(hv_cpu->hyp_synic_event_page); + hv_cpu->hyp_synic_event_page = NULL; } else { siefp.base_siefp_gpa = 0; } @@ -403,7 +407,7 @@ static bool hv_synic_event_pending(void) { struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); union hv_synic_event_flags *event = - (union hv_synic_event_flags *)hv_cpu->synic_event_page + VMBUS_MESSAGE_SINT; + (union hv_synic_event_flags *)hv_cpu->hyp_synic_event_page + VMBUS_MESSAGE_SINT; unsigned long *recv_int_page = event->flags; /* assumes VMBus version >= VERSION_WIN8 */ bool pending; u32 relid; diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h index 34943de7d6ac4..f16395ba5d823 100644 --- a/drivers/hv/hyperv_vmbus.h +++ b/drivers/hv/hyperv_vmbus.h @@ -108,6 +108,8 @@ struct hv_input_post_message { enum { VMBUS_MESSAGE_CONNECTION_ID = 1, VMBUS_MESSAGE_CONNECTION_ID_4 = 4, + /* VTL2 redirect connection ID for INITIATE_CONTACT. */ + VMBUS_MESSAGE_CONNECTION_ID_REDIRECT = 0x800074, VMBUS_MESSAGE_PORT_ID = 1, VMBUS_EVENT_CONNECTION_ID = 2, VMBUS_EVENT_PORT_ID = 2, @@ -120,8 +122,26 @@ enum { * Per cpu state for channel handling */ struct hv_per_cpu_context { - void *synic_message_page; - void *synic_event_page; + /* + * SynIC pages for communicating with the host. + * + * These pages are accessible to the host partition and the hypervisor. + * They may be used for exchanging data with the host partition and the + * hypervisor even when they aren't trusted yet the guest partition + * must be prepared to handle the malicious behavior. + */ + void *hyp_synic_message_page; + void *hyp_synic_event_page; + /* + * SynIC pages for communicating with the paravisor. + * + * These pages may be accessed from within the guest partition only in + * CoCo VMs. Neither the host partition nor the hypervisor can access + * these pages in that case; they are used for exchanging data with the + * paravisor. + */ + void *para_synic_message_page; + void *para_synic_event_page; /* * The page is only used in hv_post_message() for a TDX VM (with the diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c index 40b7179fee805..161a9655aa0b1 100644 --- a/drivers/hv/mshv_eventfd.c +++ b/drivers/hv/mshv_eventfd.c @@ -164,10 +164,19 @@ static int mshv_try_assert_irq_fast(struct mshv_irqfd *irqfd) if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT) return -EOPNOTSUPP; +#if IS_ENABLED(CONFIG_X86) if (irq->lapic_control.logical_dest_mode) return -EOPNOTSUPP; +#endif - vp = partition->pt_vp_array[irq->lapic_apic_id]; + /* + * Pairs with smp_store_release() in mshv_partition_ioctl_create_vp(). + * MSHV_IRQFD does not require the target lapic_apic_id to refer to an + * existing VP, so this read can race a concurrent VP creation; the + * acquire ensures that a non-NULL pointer implies the VP's + * initialising stores are visible. + */ + vp = smp_load_acquire(&partition->pt_vp_array[irq->lapic_apic_id]); if (!vp->vp_register_page) return -EOPNOTSUPP; @@ -197,8 +206,10 @@ static void mshv_assert_irq_slow(struct mshv_irqfd *irqfd) unsigned int seq; int idx; +#if IS_ENABLED(CONFIG_X86) WARN_ON(irqfd->irqfd_resampler && !irq->lapic_control.level_triggered); +#endif idx = srcu_read_lock(&partition->pt_irq_srcu); if (irqfd->irqfd_girq_ent.guest_irq_num) { @@ -280,7 +291,7 @@ static void mshv_irqfd_deactivate(struct mshv_irqfd *irqfd) if (!mshv_irqfd_is_active(irqfd)) return; - hlist_del(&irqfd->irqfd_hnode); + hlist_del_init(&irqfd->irqfd_hnode); queue_work(irqfd_cleanup_wq, &irqfd->irqfd_shutdown); } @@ -469,16 +480,6 @@ static int mshv_irqfd_assign(struct mshv_partition *pt, init_poll_funcptr(&irqfd->irqfd_polltbl, mshv_irqfd_queue_proc); spin_lock_irq(&pt->pt_irqfds_lock); - if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE) && - !irqfd->irqfd_lapic_irq.lapic_control.level_triggered) { - /* - * Resample Fd must be for level triggered interrupt - * Otherwise return with failure - */ - spin_unlock_irq(&pt->pt_irqfds_lock); - ret = -EINVAL; - goto fail; - } ret = 0; hlist_for_each_entry(tmp, &pt->pt_irqfds_list, irqfd_hnode) { if (irqfd->irqfd_eventfd_ctx != tmp->irqfd_eventfd_ctx) @@ -491,6 +492,21 @@ static int mshv_irqfd_assign(struct mshv_partition *pt, idx = srcu_read_lock(&pt->pt_irq_srcu); mshv_irqfd_update(pt, irqfd); + +#if IS_ENABLED(CONFIG_X86) + if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE) && + !irqfd->irqfd_lapic_irq.lapic_control.level_triggered) { + /* + * Resample Fd must be for level triggered interrupt + * Otherwise return with failure + */ + spin_unlock_irq(&pt->pt_irqfds_lock); + srcu_read_unlock(&pt->pt_irq_srcu, idx); + ret = -EINVAL; + goto fail; + } +#endif + hlist_add_head(&irqfd->irqfd_hnode, &pt->pt_irqfds_list); spin_unlock_irq(&pt->pt_irqfds_lock); @@ -535,13 +551,14 @@ static int mshv_irqfd_deassign(struct mshv_partition *pt, if (IS_ERR(eventfd)) return PTR_ERR(eventfd); + spin_lock_irq(&pt->pt_irqfds_lock); hlist_for_each_entry_safe(irqfd, n, &pt->pt_irqfds_list, irqfd_hnode) { if (irqfd->irqfd_eventfd_ctx == eventfd && irqfd->irqfd_irqnum == args->gsi) - mshv_irqfd_deactivate(irqfd); } + spin_unlock_irq(&pt->pt_irqfds_lock); eventfd_ctx_put(eventfd); diff --git a/drivers/hv/mshv_irq.c b/drivers/hv/mshv_irq.c index d0fb9ef734f42..7422a36199b40 100644 --- a/drivers/hv/mshv_irq.c +++ b/drivers/hv/mshv_irq.c @@ -51,7 +51,7 @@ int mshv_update_routing_table(struct mshv_partition *partition, /* * Allow only one to one mapping between GSI and MSI routing. */ - if (girq->guest_irq_num != 0) { + if (girq->girq_entry_valid) { r = -EINVAL; goto out; } @@ -119,6 +119,10 @@ void mshv_copy_girq_info(struct mshv_guest_irq_ent *ent, lirq->lapic_vector = ent->girq_irq_data & 0xFF; lirq->lapic_apic_id = (ent->girq_addr_lo >> 12) & 0xFF; lirq->lapic_control.interrupt_type = (ent->girq_irq_data & 0x700) >> 8; +#if IS_ENABLED(CONFIG_X86) lirq->lapic_control.level_triggered = (ent->girq_irq_data >> 15) & 0x1; lirq->lapic_control.logical_dest_mode = (ent->girq_addr_lo >> 2) & 0x1; +#elif IS_ENABLED(CONFIG_ARM64) + lirq->lapic_control.asserted = 1; +#endif } diff --git a/drivers/hv/mshv_portid_table.c b/drivers/hv/mshv_portid_table.c index c349af1f0aaac..6f59b3e376247 100644 --- a/drivers/hv/mshv_portid_table.c +++ b/drivers/hv/mshv_portid_table.c @@ -40,12 +40,14 @@ mshv_port_table_fini(void) int mshv_portid_alloc(struct port_table_info *info) { - int ret = 0; + int ret; + idr_preload(GFP_KERNEL); idr_lock(&port_table_idr); ret = idr_alloc(&port_table_idr, info, PORTID_MIN, - PORTID_MAX, GFP_KERNEL); + PORTID_MAX, GFP_NOWAIT); idr_unlock(&port_table_idr); + idr_preload_end(); return ret; } diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h index e3931b0f12693..db6b42db2fdc6 100644 --- a/drivers/hv/mshv_root.h +++ b/drivers/hv/mshv_root.h @@ -169,7 +169,7 @@ struct mshv_girq_routing_table { }; struct hv_synic_pages { - struct hv_message_page *synic_message_page; + struct hv_message_page *hyp_synic_message_page; struct hv_synic_event_flags_page *synic_event_flags_page; struct hv_synic_event_ring_page *synic_event_ring_page; }; diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c index c9c274f29c3c6..a23b5ba09fb40 100644 --- a/drivers/hv/mshv_root_hv_call.c +++ b/drivers/hv/mshv_root_hv_call.c @@ -388,7 +388,13 @@ int hv_call_assert_virtual_interrupt(u64 partition_id, u32 vector, memset(input, 0, sizeof(*input)); input->partition_id = partition_id; input->vector = vector; + /* + * NOTE: dest_addr only needs to be provided while asserting an + * interrupt on x86 platform + */ +#if IS_ENABLED(CONFIG_X86) input->dest_addr = dest_addr; +#endif input->control = control; status = hv_do_hypercall(HVCALL_ASSERT_VIRTUAL_INTERRUPT, input, NULL); local_irq_restore(flags); diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c index 4e04bef544379..3a356b34774bd 100644 --- a/drivers/hv/mshv_root_main.c +++ b/drivers/hv/mshv_root_main.c @@ -979,7 +979,13 @@ mshv_partition_ioctl_create_vp(struct mshv_partition *partition, /* already exclusive with the partition mutex for all ioctls */ partition->pt_vp_count++; - partition->pt_vp_array[args.vp_index] = vp; + /* + * Pairs with smp_load_acquire() in mshv_try_assert_irq_fast(), which + * can run concurrently from an irqfd waker without holding pt_mutex. + * The release ensures the VP's initialising stores are visible to any + * reader that observes a non-NULL pointer in pt_vp_array. + */ + smp_store_release(&partition->pt_vp_array[args.vp_index], vp); return ret; diff --git a/drivers/hv/mshv_synic.c b/drivers/hv/mshv_synic.c index 1a822beae2390..3c65b1c4abbe3 100644 --- a/drivers/hv/mshv_synic.c +++ b/drivers/hv/mshv_synic.c @@ -399,7 +399,7 @@ mshv_intercept_isr(struct hv_message *msg) void mshv_isr(void) { struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages); - struct hv_message_page **msg_page = &spages->synic_message_page; + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; struct hv_message *msg; bool handled; @@ -461,7 +461,7 @@ int mshv_synic_init(unsigned int cpu) #endif union hv_synic_scontrol sctrl; struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages); - struct hv_message_page **msg_page = &spages->synic_message_page; + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; struct hv_synic_event_flags_page **event_flags_page = &spages->synic_event_flags_page; struct hv_synic_event_ring_page **event_ring_page = @@ -555,7 +555,7 @@ int mshv_synic_cleanup(unsigned int cpu) union hv_synic_sirbp sirbp; union hv_synic_scontrol sctrl; struct hv_synic_pages *spages = this_cpu_ptr(mshv_root.synic_pages); - struct hv_message_page **msg_page = &spages->synic_message_page; + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; struct hv_synic_event_flags_page **event_flags_page = &spages->synic_event_flags_page; struct hv_synic_event_ring_page **event_ring_page = diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c index 1a0e350c19404..b1552a26b8d3e 100644 --- a/drivers/hv/vmbus_drv.c +++ b/drivers/hv/vmbus_drv.c @@ -1048,7 +1048,7 @@ static void vmbus_onmessage_work(struct work_struct *work) void vmbus_on_msg_dpc(unsigned long data) { struct hv_per_cpu_context *hv_cpu = (void *)data; - void *page_addr = hv_cpu->synic_message_page; + void *page_addr = hv_cpu->hyp_synic_message_page; struct hv_message msg_copy, *msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; struct vmbus_channel_message_header *hdr; @@ -1232,7 +1232,7 @@ static void vmbus_chan_sched(struct hv_per_cpu_context *hv_cpu) * The event page can be directly checked to get the id of * the channel that has the interrupt pending. */ - void *page_addr = hv_cpu->synic_event_page; + void *page_addr = hv_cpu->hyp_synic_event_page; union hv_synic_event_flags *event = (union hv_synic_event_flags *)page_addr + VMBUS_MESSAGE_SINT; @@ -1315,7 +1315,7 @@ static void __vmbus_isr(void) vmbus_chan_sched(hv_cpu); - page_addr = hv_cpu->synic_message_page; + page_addr = hv_cpu->hyp_synic_message_page; msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; /* Check if there are actual msgs to be processed */ @@ -1372,8 +1372,19 @@ static void vmbus_isr(void) if (IS_ENABLED(CONFIG_PREEMPT_RT)) { vmbus_irqd_wake(); } else { - lockdep_hardirq_threaded(); + static DEFINE_WAIT_OVERRIDE_MAP(vmbus_map, LD_WAIT_CONFIG); + + /* + * vmbus_isr is never force-threaded and always invoked at hard + * IRQ level. __vmbus_isr() below can acquire a spinlock_t + * which becomes a sleeping lock and must not be acquired in + * this context. Therefore on PREEMPT_RT this will be threaded + * via vmbus_irqd_wake(). On non-PREEMPT the annotation lets + * lockdep know that acquiring a spinlock_t is not an issue. + */ + lock_map_acquire_try(&vmbus_map); __vmbus_isr(); + lock_map_release(&vmbus_map); } } @@ -2884,7 +2895,7 @@ static void hv_crash_handler(struct pt_regs *regs) hv_synic_disable_regs(cpu); }; -static int hv_synic_suspend(void) +static int hv_synic_suspend(void *data) { /* * When we reach here, all the non-boot CPUs have been offlined. @@ -2911,7 +2922,7 @@ static int hv_synic_suspend(void) return 0; } -static void hv_synic_resume(void) +static void hv_synic_resume(void *data) { hv_synic_enable_regs(0); @@ -2923,11 +2934,15 @@ static void hv_synic_resume(void) } /* The callbacks run only on CPU0, with irqs_disabled. */ -static struct syscore_ops hv_synic_syscore_ops = { +static const struct syscore_ops hv_synic_syscore_ops = { .suspend = hv_synic_suspend, .resume = hv_synic_resume, }; +static struct syscore hv_synic_syscore = { + .ops = &hv_synic_syscore_ops, +}; + static int __init hv_acpi_init(void) { int ret; @@ -2936,6 +2951,13 @@ static int __init hv_acpi_init(void) return -ENODEV; if (hv_root_partition() && !hv_nested) + /* + * A non-nested root partition does not need VMBus client + * functionality. However, the mshv_root module may have + * a dependency on the VMBus module as described in + * commit 840b740a35bf. Return success so the module + * loads even though no VMBus initialization is done. + */ return 0; /* @@ -2970,7 +2992,7 @@ static int __init hv_acpi_init(void) hv_setup_kexec_handler(hv_kexec_handler); hv_setup_crash_handler(hv_crash_handler); - register_syscore_ops(&hv_synic_syscore_ops); + register_syscore(&hv_synic_syscore); return 0; @@ -2984,7 +3006,15 @@ static void __exit vmbus_exit(void) { int cpu; - unregister_syscore_ops(&hv_synic_syscore_ops); + if (hv_root_partition() && !hv_nested) + /* + * If a non-nested root partition loaded the VMBus module, + * hv_acpi_init() did not do any VMBus initialization. + * There's nothing to clean up, so just return. + */ + return; + + unregister_syscore(&hv_synic_syscore); hv_remove_kexec_handler(); hv_remove_crash_handler(); diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig index 750ff36bbe3b5..f3ee974b377b7 100644 --- a/drivers/hwmon/Kconfig +++ b/drivers/hwmon/Kconfig @@ -2275,7 +2275,7 @@ config SENSORS_INA2XX select REGMAP_I2C help If you say yes here you get support for INA219, INA220, INA226, - INA230, INA231, INA260, and SY24655 power monitor chips. + INA230, INA231, INA234, INA260, and SY24655 power monitor chips. The INA2xx driver is configured for the default configuration of the part as described in the datasheet. diff --git a/drivers/hwmon/ads7828.c b/drivers/hwmon/ads7828.c index 436637264056c..20ef4992f5bb8 100644 --- a/drivers/hwmon/ads7828.c +++ b/drivers/hwmon/ads7828.c @@ -106,12 +106,11 @@ static int ads7828_probe(struct i2c_client *client) struct ads7828_data *data; struct device *hwmon_dev; unsigned int vref_mv = ADS7828_INT_VREF_MV; - unsigned int vref_uv; + int vref_uv; bool diff_input = false; bool ext_vref = false; unsigned int regval; enum ads7828_chips chip; - struct regulator *reg; data = devm_kzalloc(dev, sizeof(struct ads7828_data), GFP_KERNEL); if (!data) @@ -125,9 +124,11 @@ static int ads7828_probe(struct i2c_client *client) } else if (dev->of_node) { diff_input = of_property_read_bool(dev->of_node, "ti,differential-input"); - reg = devm_regulator_get_optional(dev, "vref"); - if (!IS_ERR(reg)) { - vref_uv = regulator_get_voltage(reg); + vref_uv = devm_regulator_get_enable_read_voltage(dev, "vref"); + if (vref_uv < 0) { + if (vref_uv != -ENODEV) + return vref_uv; + } else { vref_mv = DIV_ROUND_CLOSEST(vref_uv, 1000); if (vref_mv < ADS7828_EXT_VREF_MV_MIN || vref_mv > ADS7828_EXT_VREF_MV_MAX) diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c index 174dfee47f7a7..f935c3477b364 100644 --- a/drivers/hwmon/adt7462.c +++ b/drivers/hwmon/adt7462.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1814,10 +1815,17 @@ static const struct i2c_device_id adt7462_id[] = { }; MODULE_DEVICE_TABLE(i2c, adt7462_id); +static const struct of_device_id adt7462_of_match[] = { + { .compatible = "onnn,adt7462" }, + { }, +}; +MODULE_DEVICE_TABLE(of, adt7462_of_match); + static struct i2c_driver adt7462_driver = { .class = I2C_CLASS_HWMON, .driver = { .name = "adt7462", + .of_match_table = adt7462_of_match, }, .probe = adt7462_probe, .id_table = adt7462_id, diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c index dbee6926fa055..0fad4bfe53df6 100644 --- a/drivers/hwmon/adt7470.c +++ b/drivers/hwmon/adt7470.c @@ -70,8 +70,8 @@ static const unsigned short normal_i2c[] = { 0x2C, 0x2E, 0x2F, I2C_CLIENT_END }; #define ADT7470_PWM1_AUTO_MASK 0x80 #define ADT7470_PWM_AUTO_MASK 0xC0 #define ADT7470_REG_PWM34_CFG 0x69 -#define ADT7470_PWM3_AUTO_MASK 0x40 -#define ADT7470_PWM4_AUTO_MASK 0x80 +#define ADT7470_PWM4_AUTO_MASK 0x40 +#define ADT7470_PWM3_AUTO_MASK 0x80 #define ADT7470_REG_PWM_MIN_BASE_ADDR 0x6A #define ADT7470_REG_PWM_MIN_MAX_ADDR 0x6D #define ADT7470_REG_PWM_TEMP_MIN_BASE_ADDR 0x6E @@ -110,6 +110,21 @@ static const unsigned short normal_i2c[] = { 0x2C, 0x2E, 0x2F, I2C_CLIENT_END }; #define ALARM2(x) ((x) << 8) +/* TEMP1..TEMP7 (ch 0..6) are, respectively BIT(0)..BIT(6) of reg 0x41 and + * 0x72, or BIT(0)..BIT(6) of data->alarm. + * TEMP8..TEMP9 (ch 7..9) are, respectively BIT(0)..BIT(2) of reg 0x42 and + * 0x73, or BIT(8)..BIT(10) of data->alarm. + */ +#define TEMP_ALARM_BIT(ch) ({ \ + typeof(ch) _ch = (ch); \ + (1 << (_ch < 7 ? _ch : _ch + 1)); \ +}) + +/* FAN1..FAN4 (ch 0..3) are respectively BIT(4)..BIT(7) in + * reg 0x42 and 0x73 or BIT(12)..BIT(15) in data->alarm. + */ +#define FAN_ALARM_BIT(ch) (1 << (12 + (ch))) + #define ADT7470_VENDOR 0x41 #define ADT7470_DEVICE 0x70 /* datasheet only mentions a revision 2 */ @@ -167,6 +182,7 @@ struct adt7470_data { u8 pwm_min[ADT7470_PWM_COUNT]; s8 pwm_tmin[ADT7470_PWM_COUNT]; u8 pwm_auto_temp[ADT7470_PWM_COUNT]; + u32 pwm_freq; struct task_struct *auto_update; unsigned int auto_update_interval; @@ -205,11 +221,12 @@ static inline int adt7470_write_word_data(struct adt7470_data *data, unsigned in /* Probe for temperature sensors. Assumes lock is held */ static int adt7470_read_temperatures(struct adt7470_data *data) { - unsigned long res; + struct device *dev = regmap_get_device(data->regmap); + u8 pwm[ADT7470_FAN_COUNT]; unsigned int pwm_cfg[2]; - int err; + unsigned long res; + int err, err2; int i; - u8 pwm[ADT7470_FAN_COUNT]; /* save pwm[1-4] config register */ err = regmap_read(data->regmap, ADT7470_REG_PWM_CFG(0), &pwm_cfg[0]); @@ -233,19 +250,19 @@ static int adt7470_read_temperatures(struct adt7470_data *data) err = regmap_update_bits(data->regmap, ADT7470_REG_PWM_CFG(2), ADT7470_PWM_AUTO_MASK, 0); if (err < 0) - return err; + goto out_restore; /* write pwm control to whatever it was */ err = regmap_bulk_write(data->regmap, ADT7470_REG_PWM(0), &pwm[0], ADT7470_PWM_COUNT); if (err < 0) - return err; + goto out_restore; /* start reading temperature sensors */ err = regmap_update_bits(data->regmap, ADT7470_REG_CFG, ADT7470_T05_STB_MASK, ADT7470_T05_STB_MASK); if (err < 0) - return err; + goto out_restore; /* Delay is 200ms * number of temp sensors. */ res = msleep_interruptible((data->num_temp_sensors >= 0 ? @@ -256,13 +273,30 @@ static int adt7470_read_temperatures(struct adt7470_data *data) err = regmap_update_bits(data->regmap, ADT7470_REG_CFG, ADT7470_T05_STB_MASK, 0); if (err < 0) - return err; + goto out_restore; +out_restore: /* restore pwm[1-4] config registers */ - err = regmap_write(data->regmap, ADT7470_REG_PWM_CFG(0), pwm_cfg[0]); - if (err < 0) - return err; - err = regmap_write(data->regmap, ADT7470_REG_PWM_CFG(2), pwm_cfg[1]); + err2 = regmap_write(data->regmap, ADT7470_REG_PWM_CFG(0), pwm_cfg[0]); + if (err2 < 0) { + dev_warn_ratelimited(dev, + "failed to restore PWM{1,2} config (%d)\n", + err2); + + if (!err) + err = err2; + } + + err2 = regmap_write(data->regmap, ADT7470_REG_PWM_CFG(2), pwm_cfg[1]); + if (err2 < 0) { + dev_warn_ratelimited(dev, + "failed to restore PWM{3,4} config (%d)\n", + err2); + + if (!err) + err = err2; + } + if (err < 0) return err; @@ -491,7 +525,7 @@ static ssize_t auto_update_interval_store(struct device *dev, if (kstrtol(buf, 10, &temp)) return -EINVAL; - temp = clamp_val(temp, 0, 60000); + temp = clamp_val(temp, 500, 60000); mutex_lock(&data->lock); data->auto_update_interval = temp; @@ -551,7 +585,7 @@ static int adt7470_temp_read(struct device *dev, u32 attr, int channel, long *va *val = 1000 * data->temp_max[channel]; break; case hwmon_temp_alarm: - *val = !!(data->alarm & channel); + *val = !!(data->alarm & TEMP_ALARM_BIT(channel)); break; default: return -EOPNOTSUPP; @@ -571,14 +605,16 @@ static int adt7470_temp_write(struct device *dev, u32 attr, int channel, long va switch (attr) { case hwmon_temp_min: mutex_lock(&data->lock); - data->temp_min[channel] = val; err = regmap_write(data->regmap, ADT7470_TEMP_MIN_REG(channel), val); + if (!err) + data->temp_min[channel] = val; mutex_unlock(&data->lock); break; case hwmon_temp_max: mutex_lock(&data->lock); - data->temp_max[channel] = val; err = regmap_write(data->regmap, ADT7470_TEMP_MAX_REG(channel), val); + if (!err) + data->temp_max[channel] = val; mutex_unlock(&data->lock); break; default: @@ -624,36 +660,33 @@ static ssize_t alarm_mask_store(struct device *dev, static int adt7470_fan_read(struct device *dev, u32 attr, int channel, long *val) { struct adt7470_data *data = adt7470_update_device(dev); + u16 fan_data; if (IS_ERR(data)) return PTR_ERR(data); switch (attr) { case hwmon_fan_input: - if (FAN_DATA_VALID(data->fan[channel])) - *val = FAN_PERIOD_TO_RPM(data->fan[channel]); - else - *val = 0; + fan_data = READ_ONCE(data->fan[channel]); break; case hwmon_fan_min: - if (FAN_DATA_VALID(data->fan_min[channel])) - *val = FAN_PERIOD_TO_RPM(data->fan_min[channel]); - else - *val = 0; + fan_data = READ_ONCE(data->fan_min[channel]); break; case hwmon_fan_max: - if (FAN_DATA_VALID(data->fan_max[channel])) - *val = FAN_PERIOD_TO_RPM(data->fan_max[channel]); - else - *val = 0; + fan_data = READ_ONCE(data->fan_max[channel]); break; case hwmon_fan_alarm: - *val = !!(data->alarm & (1 << (12 + channel))); - break; + *val = !!(data->alarm & FAN_ALARM_BIT(channel)); + return 0; default: return -EOPNOTSUPP; } + if (FAN_DATA_VALID(fan_data)) + *val = FAN_PERIOD_TO_RPM(fan_data); + else + *val = 0; + return 0; } @@ -721,7 +754,7 @@ static ssize_t force_pwm_max_store(struct device *dev, } /* These are the valid PWM frequencies to the nearest Hz */ -static const int adt7470_freq_map[] = { +static const u32 adt7470_freq_map[] = { 11, 15, 22, 29, 35, 44, 59, 88, 1400, 22500 }; @@ -761,7 +794,7 @@ static int adt7470_pwm_read(struct device *dev, u32 attr, int channel, long *val *val = 1 + data->pwm_automatic[channel]; break; case hwmon_pwm_freq: - *val = pwm1_freq_get(dev); + *val = data->pwm_freq; break; default: return -EOPNOTSUPP; @@ -774,12 +807,14 @@ static int pwm1_freq_set(struct device *dev, long freq) { struct adt7470_data *data = dev_get_drvdata(dev); unsigned int low_freq = ADT7470_CFG_LF; + u32 closest_freq; int index; int err; /* Round the user value given to the closest available frequency */ index = find_closest(freq, adt7470_freq_map, ARRAY_SIZE(adt7470_freq_map)); + closest_freq = adt7470_freq_map[index]; if (index >= 8) { index -= 8; @@ -797,6 +832,10 @@ static int pwm1_freq_set(struct device *dev, long freq) err = regmap_update_bits(data->regmap, ADT7470_REG_CFG_2, ADT7470_FREQ_MASK, index << ADT7470_FREQ_SHIFT); + if (err < 0) + goto out; + + data->pwm_freq = closest_freq; out: mutex_unlock(&data->lock); @@ -813,9 +852,10 @@ static int adt7470_pwm_write(struct device *dev, u32 attr, int channel, long val case hwmon_pwm_input: val = clamp_val(val, 0, 255); mutex_lock(&data->lock); - data->pwm[channel] = val; err = regmap_write(data->regmap, ADT7470_REG_PWM(channel), - data->pwm[channel]); + val); + if (!err) + data->pwm[channel] = val; mutex_unlock(&data->lock); break; case hwmon_pwm_enable: @@ -829,10 +869,11 @@ static int adt7470_pwm_write(struct device *dev, u32 attr, int channel, long val val--; mutex_lock(&data->lock); - data->pwm_automatic[channel] = val; err = regmap_update_bits(data->regmap, ADT7470_REG_PWM_CFG(channel), pwm_auto_reg_mask, val ? pwm_auto_reg_mask : 0); + if (!err) + data->pwm_automatic[channel] = val; mutex_unlock(&data->lock); break; case hwmon_pwm_freq: @@ -1008,8 +1049,10 @@ static ssize_t pwm_auto_temp_store(struct device *dev, if (temp < 0) return temp; + if (temp > 0xF) + return -EINVAL; + mutex_lock(&data->lock); - data->pwm_automatic[attr->index] = temp; if (!(attr->index % 2)) { mask = 0xF0; @@ -1020,6 +1063,9 @@ static ssize_t pwm_auto_temp_store(struct device *dev, } err = regmap_update_bits(data->regmap, pwm_auto_reg, mask, val); + if (!err) + data->pwm_auto_temp[attr->index] = temp; + mutex_unlock(&data->lock); return err < 0 ? err : count; @@ -1248,6 +1294,7 @@ static int adt7470_probe(struct i2c_client *client) struct device *dev = &client->dev; struct adt7470_data *data; struct device *hwmon_dev; + int freq_val; int err; data = devm_kzalloc(dev, sizeof(struct adt7470_data), GFP_KERNEL); @@ -1272,6 +1319,14 @@ static int adt7470_probe(struct i2c_client *client) if (err < 0) return err; + freq_val = pwm1_freq_get(dev); + if (freq_val <= 0) { + err = freq_val < 0 ? freq_val : -EINVAL; + return err; + } + + data->pwm_freq = (u32)freq_val; + /* Register sysfs hooks */ hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name, data, &adt7470_chip_info, diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c index fc6d6a9053cee..c8e5bb12ce71f 100644 --- a/drivers/hwmon/applesmc.c +++ b/drivers/hwmon/applesmc.c @@ -1249,12 +1249,17 @@ static void applesmc_release_light_sensor(void) static int applesmc_create_key_backlight(void) { + int ret; + if (!smcreg.has_key_backlight) return 0; applesmc_led_wq = create_singlethread_workqueue("applesmc-led"); if (!applesmc_led_wq) return -ENOMEM; - return led_classdev_register(&pdev->dev, &applesmc_backlight); + ret = led_classdev_register(&pdev->dev, &applesmc_backlight); + if (ret) + destroy_workqueue(applesmc_led_wq); + return ret; } static void applesmc_release_key_backlight(void) diff --git a/drivers/hwmon/aspeed-pwm-tacho.c b/drivers/hwmon/aspeed-pwm-tacho.c index aa159bf158a37..4c62487336db0 100644 --- a/drivers/hwmon/aspeed-pwm-tacho.c +++ b/drivers/hwmon/aspeed-pwm-tacho.c @@ -933,7 +933,9 @@ static int aspeed_pwm_tacho_probe(struct platform_device *pdev) "missing or invalid reset controller device tree entry"); return PTR_ERR(priv->rst); } - reset_control_deassert(priv->rst); + ret = reset_control_deassert(priv->rst); + if (ret) + return ret; ret = devm_add_action_or_reset(dev, aspeed_pwm_tacho_remove, priv); if (ret) diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c index 95c50d3a788ce..13fd212ad4040 100644 --- a/drivers/hwmon/asus-ec-sensors.c +++ b/drivers/hwmon/asus-ec-sensors.c @@ -870,7 +870,7 @@ struct ec_sensors_data { /* sorted list of unique register banks */ u8 banks[ASUS_EC_MAX_BANK + 1]; /* in jiffies */ - unsigned long last_updated; + u64 next_update; struct lock_data lock_data; /* number of board EC sensors */ u8 nr_sensors; @@ -1050,7 +1050,7 @@ static int asus_ec_block_read(const struct device *dev, } for (ireg = 0; ireg < ec->nr_registers; ireg++) { reg_bank = register_bank(ec->registers[ireg]); - if (reg_bank < bank) { + if (reg_bank != bank) { continue; } ec_read(register_index(ec->registers[ireg]), @@ -1139,13 +1139,12 @@ static int get_cached_value_or_update(const struct device *dev, int sensor_index, struct ec_sensors_data *state, s32 *value) { - if (time_after(jiffies, state->last_updated + HZ)) { + if (time_after64(get_jiffies_64(), state->next_update)) { if (update_ec_sensors(dev, state)) { dev_err(dev, "update_ec_sensors() failure\n"); return -EIO; } - - state->last_updated = jiffies; + state->next_update = get_jiffies_64() + HZ; } *value = state->sensors[sensor_index].cached_value; @@ -1263,6 +1262,7 @@ static int asus_ec_probe(struct platform_device *pdev) if (!ec_data) return -ENOMEM; + ec_data->next_update = INITIAL_JIFFIES; dev_set_drvdata(dev, ec_data); ec_data->board_info = pboard_info; @@ -1353,9 +1353,11 @@ static int asus_ec_probe(struct platform_device *pdev) if (!nr_count[type]) continue; - asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev, - nr_count[type], type, - hwmon_attributes[type]); + status = asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev, + nr_count[type], type, + hwmon_attributes[type]); + if (status) + return status; *ptr_asus_ec_ci++ = asus_ec_hwmon_chan++; } diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c index 9d071f7ca9d2e..0e182454779e0 100644 --- a/drivers/hwmon/chipcap2.c +++ b/drivers/hwmon/chipcap2.c @@ -90,11 +90,6 @@ struct cc2_data { bool process_irqs; }; -enum cc2_chan_addr { - CC2_CHAN_TEMP = 0, - CC2_CHAN_HUMIDITY, -}; - /* %RH as a per cent mille from a register value */ static long cc2_rh_convert(u16 data) { @@ -493,7 +488,7 @@ static irqreturn_t cc2_low_interrupt(int irq, void *data) if (cc2->process_irqs) { hwmon_notify_event(cc2->hwmon, hwmon_humidity, - hwmon_humidity_min_alarm, CC2_CHAN_HUMIDITY); + hwmon_humidity_min_alarm, 0); cc2->rh_alarm.low_alarm = true; } @@ -506,7 +501,7 @@ static irqreturn_t cc2_high_interrupt(int irq, void *data) if (cc2->process_irqs) { hwmon_notify_event(cc2->hwmon, hwmon_humidity, - hwmon_humidity_max_alarm, CC2_CHAN_HUMIDITY); + hwmon_humidity_max_alarm, 0); cc2->rh_alarm.high_alarm = true; } diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c index ad79db5a183e0..a84a32ada8bd5 100644 --- a/drivers/hwmon/coretemp.c +++ b/drivers/hwmon/coretemp.c @@ -650,6 +650,7 @@ static void coretemp_device_remove(int zoneid) struct platform_data *pdata = platform_get_drvdata(pdev); ida_destroy(&pdata->ida); + kfree(pdata->core_data); kfree(pdata); platform_device_unregister(pdev); } diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c index b7b911f8359c7..bf7775a731fbe 100644 --- a/drivers/hwmon/corsair-cpro.c +++ b/drivers/hwmon/corsair-cpro.c @@ -566,21 +566,18 @@ static int bootloader_show(struct seq_file *seqf, void *unused) } DEFINE_SHOW_ATTRIBUTE(bootloader); -static void ccp_debugfs_init(struct ccp_device *ccp) +static void ccp_debugfs_init(struct ccp_device *ccp, bool fw_valid, bool bl_valid) { char name[32]; - int ret; scnprintf(name, sizeof(name), "corsaircpro-%s", dev_name(&ccp->hdev->dev)); ccp->debugfs = debugfs_create_dir(name, NULL); - ret = get_fw_version(ccp); - if (!ret) + if (fw_valid) debugfs_create_file("firmware_version", 0444, ccp->debugfs, ccp, &firmware_fops); - ret = get_bl_version(ccp); - if (!ret) + if (bl_valid) debugfs_create_file("bootloader_version", 0444, ccp->debugfs, ccp, &bootloader_fops); } @@ -588,6 +585,7 @@ static void ccp_debugfs_init(struct ccp_device *ccp) static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) { struct ccp_device *ccp; + bool fw_valid, bl_valid; int ret; ccp = devm_kzalloc(&hdev->dev, sizeof(*ccp), GFP_KERNEL); @@ -632,19 +630,30 @@ static int ccp_probe(struct hid_device *hdev, const struct hid_device_id *id) if (ret) goto out_hw_close; - ccp_debugfs_init(ccp); + /* + * Query the versions before registering the hwmon device: they send + * USB commands without holding ccp->mutex, which is only safe while + * nothing else can call send_usb_cmd(). + */ + fw_valid = !get_fw_version(ccp); + bl_valid = !get_bl_version(ccp); ccp->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "corsaircpro", ccp, &ccp_chip_info, NULL); if (IS_ERR(ccp->hwmon_dev)) { ret = PTR_ERR(ccp->hwmon_dev); - goto out_hw_close; + goto out_debugfs_remove; } + ccp_debugfs_init(ccp, fw_valid, bl_valid); + return 0; +out_debugfs_remove: + debugfs_remove_recursive(ccp->debugfs); out_hw_close: hid_hw_close(hdev); + hid_device_io_stop(hdev); out_hw_stop: hid_hw_stop(hdev); return ret; diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c index ea28c9219507a..01f6dfbf65da9 100644 --- a/drivers/hwmon/corsair-psu.c +++ b/drivers/hwmon/corsair-psu.c @@ -140,13 +140,18 @@ struct corsairpsu_data { }; /* some values are SMBus LINEAR11 data which need a conversion */ -static int corsairpsu_linear11_to_int(const u16 val, const int scale) +static long corsairpsu_linear11_to_long(const u16 val, const int scale) { const int exp = ((s16)val) >> 11; - const int mant = (((s16)(val & 0x7ff)) << 5) >> 5; - const int result = mant * scale; + const int mant = ((s16)((val & 0x7ff) << 5)) >> 5; + s64 result = mant * scale; - return (exp >= 0) ? (result << exp) : (result >> -exp); + if (exp >= 0) + result *= (int)(1UL << exp); + else + result >>= -exp; + + return clamp(result, LONG_MIN, LONG_MAX); } /* the micro-controller uses percentage values to control pwm */ @@ -271,13 +276,13 @@ static int corsairpsu_get_value(struct corsairpsu_data *priv, u8 cmd, u8 rail, l case PSU_CMD_RAIL_AMPS: case PSU_CMD_TEMP0: case PSU_CMD_TEMP1: - *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1000); + *val = corsairpsu_linear11_to_long(tmp & 0xFFFF, 1000); break; case PSU_CMD_FAN: - *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1); + *val = corsairpsu_linear11_to_long(tmp & 0xFFFF, 1); break; case PSU_CMD_FAN_PWM_ENABLE: - *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1); + *val = corsairpsu_linear11_to_long(tmp & 0xFFFF, 1); /* * 0 = automatic mode, means the micro-controller controls the fan using a plan * which can be modified, but changing this plan is not supported by this @@ -291,12 +296,12 @@ static int corsairpsu_get_value(struct corsairpsu_data *priv, u8 cmd, u8 rail, l *val = 2; break; case PSU_CMD_FAN_PWM: - *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1); + *val = corsairpsu_linear11_to_long(tmp & 0xFFFF, 1); *val = corsairpsu_dutycycle_to_pwm(*val); break; case PSU_CMD_RAIL_WATTS: case PSU_CMD_TOTAL_WATTS: - *val = corsairpsu_linear11_to_int(tmp & 0xFFFF, 1000000); + *val = corsairpsu_linear11_to_long(tmp & 0xFFFF, 1000000); break; case PSU_CMD_TOTAL_UPTIME: case PSU_CMD_UPTIME: @@ -709,7 +714,7 @@ static int vendor_show(struct seq_file *seqf, void *unused) { struct corsairpsu_data *priv = seqf->private; - seq_printf(seqf, "%s\n", priv->vendor); + seq_printf(seqf, "%.*s\n", REPLY_SIZE, priv->vendor); return 0; } @@ -719,7 +724,7 @@ static int product_show(struct seq_file *seqf, void *unused) { struct corsairpsu_data *priv = seqf->private; - seq_printf(seqf, "%s\n", priv->product); + seq_printf(seqf, "%.*s\n", REPLY_SIZE, priv->product); return 0; } @@ -831,6 +836,7 @@ static int corsairpsu_probe(struct hid_device *hdev, const struct hid_device_id fail_and_close: hid_hw_close(hdev); + hid_device_io_stop(hdev); fail_and_stop: hid_hw_stop(hdev); return ret; diff --git a/drivers/hwmon/cros_ec_hwmon.c b/drivers/hwmon/cros_ec_hwmon.c index 48331703f2f50..f39401606e750 100644 --- a/drivers/hwmon/cros_ec_hwmon.c +++ b/drivers/hwmon/cros_ec_hwmon.c @@ -5,6 +5,7 @@ * Copyright (C) 2024 Thomas Weißschuh */ +#include #include #include #include @@ -25,9 +26,11 @@ struct cros_ec_hwmon_priv { struct cros_ec_device *cros_ec; + struct device *hwmon_dev; const char *temp_sensor_names[EC_TEMP_SENSOR_ENTRIES + EC_TEMP_SENSOR_B_ENTRIES]; u8 usable_fans; bool fan_control_supported; + bool temp_threshold_supported; u8 manual_fans; /* bits to indicate whether the fan is set to manual */ u8 manual_fan_pwm[EC_FAN_SPEED_ENTRIES]; }; @@ -102,6 +105,23 @@ static int cros_ec_hwmon_read_temp(struct cros_ec_device *cros_ec, u8 index, u8 return 0; } +static int cros_ec_hwmon_read_temp_threshold(struct cros_ec_device *cros_ec, u8 index, + enum ec_temp_thresholds threshold, u32 *temp) +{ + struct ec_params_thermal_get_threshold_v1 req = {}; + struct ec_thermal_config resp; + int ret; + + req.sensor_num = index; + ret = cros_ec_cmd(cros_ec, 1, EC_CMD_THERMAL_GET_THRESHOLD, + &req, sizeof(req), &resp, sizeof(resp)); + if (ret < 0) + return ret; + + *temp = resp.temp_host[threshold]; + return 0; +} + static bool cros_ec_hwmon_is_error_fan(u16 speed) { return speed == EC_FAN_SPEED_NOT_PRESENT || speed == EC_FAN_SPEED_STALLED; @@ -120,12 +140,29 @@ static long cros_ec_hwmon_temp_to_millicelsius(u8 temp) return kelvin_to_millicelsius((((long)temp) + EC_TEMP_SENSOR_OFFSET)); } +static bool cros_ec_hwmon_attr_is_temp_threshold(u32 attr) +{ + return attr == hwmon_temp_max || + attr == hwmon_temp_crit || + attr == hwmon_temp_emergency; +} + +static enum ec_temp_thresholds cros_ec_hwmon_attr_to_thres(u32 attr) +{ + if (attr == hwmon_temp_max) + return EC_TEMP_THRESH_WARN; + else if (attr == hwmon_temp_crit) + return EC_TEMP_THRESH_HIGH; + return EC_TEMP_THRESH_HALT; /* attr == hwmon_temp_emergency */ +} + static int cros_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { struct cros_ec_hwmon_priv *priv = dev_get_drvdata(dev); int ret = -EOPNOTSUPP; u8 control_method; + u32 threshold; u8 pwm_value; u16 speed; u8 temp; @@ -168,6 +205,13 @@ static int cros_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type, ret = cros_ec_hwmon_read_temp(priv->cros_ec, channel, &temp); if (ret == 0) *val = cros_ec_hwmon_is_error_temp(temp); + + } else if (cros_ec_hwmon_attr_is_temp_threshold(attr)) { + ret = cros_ec_hwmon_read_temp_threshold(priv->cros_ec, channel, + cros_ec_hwmon_attr_to_thres(attr), + &threshold); + if (ret == 0) + *val = kelvin_to_millicelsius(threshold); } } @@ -267,8 +311,14 @@ static umode_t cros_ec_hwmon_is_visible(const void *data, enum hwmon_sensor_type if (priv->fan_control_supported && priv->usable_fans & BIT(channel)) return 0644; } else if (type == hwmon_temp) { - if (priv->temp_sensor_names[channel]) - return 0444; + if (priv->temp_sensor_names[channel]) { + if (cros_ec_hwmon_attr_is_temp_threshold(attr)) { + if (priv->temp_threshold_supported) + return 0444; + } else { + return 0444; + } + } } return 0; @@ -286,31 +336,33 @@ static const struct hwmon_channel_info * const cros_ec_hwmon_info[] = { HWMON_PWM_INPUT | HWMON_PWM_ENABLE, HWMON_PWM_INPUT | HWMON_PWM_ENABLE, HWMON_PWM_INPUT | HWMON_PWM_ENABLE), +#define CROS_EC_HWMON_TEMP_PARAMS (HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL | \ + HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_EMERGENCY) HWMON_CHANNEL_INFO(temp, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL, - HWMON_T_INPUT | HWMON_T_FAULT | HWMON_T_LABEL), + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS, + CROS_EC_HWMON_TEMP_PARAMS), NULL }; @@ -328,6 +380,8 @@ static int cros_ec_hwmon_cooling_get_cur_state(struct thermal_cooling_device *cd u8 read_val; int ret; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + ret = cros_ec_hwmon_read_pwm_value(priv->hwmon_priv->cros_ec, priv->index, &read_val); if (ret) return ret; @@ -341,6 +395,8 @@ static int cros_ec_hwmon_cooling_set_cur_state(struct thermal_cooling_device *cd { const struct cros_ec_hwmon_cooling_priv *priv = cdev->devdata; + guard(hwmon_lock)(priv->hwmon_priv->hwmon_dev); + return cros_ec_hwmon_write_pwm_input(priv->hwmon_priv->cros_ec, priv->index, val); } @@ -474,7 +530,6 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) struct cros_ec_dev *ec_dev = dev_get_drvdata(dev->parent); struct cros_ec_device *cros_ec = ec_dev->ec_dev; struct cros_ec_hwmon_priv *priv; - struct device *hwmon_dev; u8 thermal_version; int ret; @@ -495,13 +550,19 @@ static int cros_ec_hwmon_probe(struct platform_device *pdev) cros_ec_hwmon_probe_temp_sensors(dev, priv, thermal_version); cros_ec_hwmon_probe_fans(priv); priv->fan_control_supported = cros_ec_hwmon_probe_fan_control_supported(priv->cros_ec); + priv->temp_threshold_supported = is_cros_ec_cmd_available(priv->cros_ec, + EC_CMD_THERMAL_GET_THRESHOLD, 1); + + priv->hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, + &cros_ec_hwmon_chip_info, NULL); + if (IS_ERR(priv->hwmon_dev)) + return PTR_ERR(priv->hwmon_dev); + cros_ec_hwmon_register_fan_cooling_devices(dev, priv); - hwmon_dev = devm_hwmon_device_register_with_info(dev, "cros_ec", priv, - &cros_ec_hwmon_chip_info, NULL); platform_set_drvdata(pdev, priv); - return PTR_ERR_OR_ZERO(hwmon_dev); + return 0; } static int cros_ec_hwmon_suspend(struct platform_device *pdev, pm_message_t state) diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c index 768690a597f4d..9f102589f240c 100644 --- a/drivers/hwmon/dell-smm-hwmon.c +++ b/drivers/hwmon/dell-smm-hwmon.c @@ -1565,6 +1565,14 @@ static const struct dmi_system_id i8k_whitelist_fan_control[] __initconst = { }, .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], }, + { + .ident = "Dell Latitude 7530", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Latitude 7530"), + }, + .driver_data = (void *)&i8k_fan_control_data[I8K_FAN_30A3_31A3], + }, { .ident = "Dell Latitude E6440", .matches = { diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c index eca33220d34a0..39f69adb88c25 100644 --- a/drivers/hwmon/emc1403.c +++ b/drivers/hwmon/emc1403.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include @@ -30,7 +29,6 @@ enum emc1403_chip { emc1402, emc1403, emc1404, emc1428 }; struct thermal_data { enum emc1403_chip chip; struct regmap *regmap; - struct mutex mutex; }; static ssize_t power_state_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -268,8 +266,8 @@ static s8 emc1403_temp_regs_low[][4] = { }, }; -static int __emc1403_get_temp(struct thermal_data *data, int channel, - enum emc1403_reg_map map, long *val) +static int emc1403_get_temp(struct thermal_data *data, int channel, + enum emc1403_reg_map map, long *val) { unsigned int regvalh; unsigned int regvall = 0; @@ -295,38 +293,22 @@ static int __emc1403_get_temp(struct thermal_data *data, int channel, return 0; } -static int emc1403_get_temp(struct thermal_data *data, int channel, - enum emc1403_reg_map map, long *val) -{ - int ret; - - mutex_lock(&data->mutex); - ret = __emc1403_get_temp(data, channel, map, val); - mutex_unlock(&data->mutex); - - return ret; -} - static int emc1403_get_hyst(struct thermal_data *data, int channel, enum emc1403_reg_map map, long *val) { int hyst, ret; long limit; - mutex_lock(&data->mutex); - ret = __emc1403_get_temp(data, channel, map, &limit); + ret = emc1403_get_temp(data, channel, map, &limit); if (ret < 0) - goto unlock; + return ret; ret = regmap_read(data->regmap, 0x21, &hyst); if (ret < 0) - goto unlock; - if (map == temp_min) - *val = limit + hyst * 1000; - else - *val = limit - hyst * 1000; -unlock: - mutex_unlock(&data->mutex); - return ret; + return ret; + + *val = limit - hyst * 1000; + + return 0; } static int emc1403_temp_read(struct thermal_data *data, u32 attr, int channel, long *val) @@ -341,9 +323,6 @@ static int emc1403_temp_read(struct thermal_data *data, u32 attr, int channel, l case hwmon_temp_input: ret = emc1403_get_temp(data, channel, ema1403_temp_map[attr], val); break; - case hwmon_temp_min_hyst: - ret = emc1403_get_hyst(data, channel, temp_min, val); - break; case hwmon_temp_max_hyst: ret = emc1403_get_hyst(data, channel, temp_max, val); break; @@ -451,20 +430,16 @@ static int emc1403_set_hyst(struct thermal_data *data, long val) else val = clamp_val(val, 0, 255000); - mutex_lock(&data->mutex); - ret = __emc1403_get_temp(data, 0, temp_crit, &limit); + ret = emc1403_get_temp(data, 0, temp_crit, &limit); if (ret < 0) - goto unlock; + return ret; hyst = limit - val; if (data->chip == emc1428) hyst = clamp_val(DIV_ROUND_CLOSEST(hyst, 1000), 0, 127); else hyst = clamp_val(DIV_ROUND_CLOSEST(hyst, 1000), 0, 255); - ret = regmap_write(data->regmap, 0x21, hyst); -unlock: - mutex_unlock(&data->mutex); - return ret; + return regmap_write(data->regmap, 0x21, hyst); } static int emc1403_set_temp(struct thermal_data *data, int channel, @@ -478,7 +453,6 @@ static int emc1403_set_temp(struct thermal_data *data, int channel, regh = emc1403_temp_regs[channel][map]; regl = emc1403_temp_regs_low[channel][map]; - mutex_lock(&data->mutex); if (regl >= 0) { if (data->chip == emc1428) val = clamp_val(val, -128000, 127875); @@ -487,7 +461,7 @@ static int emc1403_set_temp(struct thermal_data *data, int channel, regval = DIV_ROUND_CLOSEST(val, 125); ret = regmap_write(data->regmap, regh, (regval >> 3) & 0xff); if (ret < 0) - goto unlock; + return ret; ret = regmap_write(data->regmap, regl, (regval & 0x07) << 5); } else { if (data->chip == emc1428) @@ -497,8 +471,6 @@ static int emc1403_set_temp(struct thermal_data *data, int channel, regval = DIV_ROUND_CLOSEST(val, 1000); ret = regmap_write(data->regmap, regh, regval); } -unlock: - mutex_unlock(&data->mutex); return ret; } @@ -572,7 +544,6 @@ static umode_t emc1403_temp_is_visible(const void *_data, u32 attr, int channel) case hwmon_temp_max_alarm: case hwmon_temp_crit_alarm: case hwmon_temp_fault: - case hwmon_temp_min_hyst: case hwmon_temp_max_hyst: return 0444; case hwmon_temp_min: @@ -615,35 +586,35 @@ static const struct hwmon_channel_info * const emc1403_info[] = { HWMON_CHANNEL_INFO(chip, HWMON_C_UPDATE_INTERVAL), HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT, HWMON_T_INPUT | HWMON_T_MIN | HWMON_T_MAX | - HWMON_T_CRIT | HWMON_T_MIN_HYST | HWMON_T_MAX_HYST | + HWMON_T_CRIT | HWMON_T_MAX_HYST | HWMON_T_CRIT_HYST | HWMON_T_MIN_ALARM | HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_FAULT ), @@ -695,8 +666,6 @@ static int emc1403_probe(struct i2c_client *client) if (IS_ERR(data->regmap)) return PTR_ERR(data->regmap); - mutex_init(&data->mutex); - hwmon_dev = devm_hwmon_device_register_with_info(&client->dev, client->name, data, &emc1403_chip_info, diff --git a/drivers/hwmon/gigabyte_waterforce.c b/drivers/hwmon/gigabyte_waterforce.c index 27487e215bddf..4eea05f8b569c 100644 --- a/drivers/hwmon/gigabyte_waterforce.c +++ b/drivers/hwmon/gigabyte_waterforce.c @@ -371,13 +371,15 @@ static int waterforce_probe(struct hid_device *hdev, const struct hid_device_id if (IS_ERR(priv->hwmon_dev)) { ret = PTR_ERR(priv->hwmon_dev); hid_err(hdev, "hwmon registration failed with %d\n", ret); - goto fail_and_close; + goto fail_and_io_stop; } waterforce_debugfs_init(priv); return 0; +fail_and_io_stop: + hid_device_io_stop(hdev); fail_and_close: hid_hw_close(hdev); fail_and_stop: diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c index a8892ced1e549..ea5ca2dea2a99 100644 --- a/drivers/hwmon/gpio-fan.c +++ b/drivers/hwmon/gpio-fan.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -84,6 +85,7 @@ static DEVICE_ATTR_RO(fan1_alarm); static int fan_alarm_init(struct gpio_fan_data *fan_data) { int alarm_irq; + int err; struct device *dev = fan_data->dev; /* @@ -94,7 +96,11 @@ static int fan_alarm_init(struct gpio_fan_data *fan_data) if (alarm_irq <= 0) return 0; - INIT_WORK(&fan_data->alarm_work, fan_alarm_notify); + err = devm_work_autocancel(dev, &fan_data->alarm_work, + fan_alarm_notify); + if (err) + return err; + irq_set_irq_type(alarm_irq, IRQ_TYPE_EDGE_BOTH); return devm_request_irq(dev, alarm_irq, fan_alarm_irq_handler, IRQF_SHARED, "GPIO fan alarm", fan_data); @@ -604,8 +610,11 @@ static void gpio_fan_shutdown(struct platform_device *pdev) { struct gpio_fan_data *fan_data = platform_get_drvdata(pdev); - if (fan_data->gpios) + if (fan_data->gpios) { + mutex_lock(&fan_data->lock); set_fan_speed(fan_data, 0); + mutex_unlock(&fan_data->lock); + } } static int gpio_fan_runtime_suspend(struct device *dev) diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c index 0b4bdcd33c7b8..0f6200410c868 100644 --- a/drivers/hwmon/hwmon.c +++ b/drivers/hwmon/hwmon.c @@ -371,18 +371,17 @@ static ssize_t pec_store(struct device *dev, struct device_attribute *devattr, * handling is not required. */ hwdev = to_hwmon_device(hdev); - guard(mutex)(&hwdev->lock); - if (hwdev->chip->ops->write) { - err = hwdev->chip->ops->write(hdev, hwmon_chip, hwmon_chip_pec, 0, val); - if (err && err != -EOPNOTSUPP) - goto put; + scoped_guard(mutex, &hwdev->lock) { + if (hwdev->chip->ops->write) { + err = hwdev->chip->ops->write(hdev, hwmon_chip, hwmon_chip_pec, 0, val); + if (err && err != -EOPNOTSUPP) + goto put; + } + if (!val) + client->flags &= ~I2C_CLIENT_PEC; + else + client->flags |= I2C_CLIENT_PEC; } - - if (!val) - client->flags &= ~I2C_CLIENT_PEC; - else - client->flags |= I2C_CLIENT_PEC; - err = count; put: put_device(hdev); diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c index bc3c1f7314b3e..a3dc4fec32f7e 100644 --- a/drivers/hwmon/ina2xx.c +++ b/drivers/hwmon/ina2xx.c @@ -23,6 +23,7 @@ */ #include +#include #include #include #include @@ -31,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -135,28 +137,48 @@ static const struct regmap_config ina2xx_regmap_config = { .writeable_reg = ina2xx_writeable_reg, }; -enum ina2xx_ids { ina219, ina226, ina260, sy24655 }; +enum ina2xx_ids { + ina219, + ina226, + ina234, + ina260, + sy24655 +}; + +enum ina2xx_alert_type { + INA2XX_ALERT_NONE, + INA2XX_ALERT_CURRENT_LOW, + INA2XX_ALERT_CURRENT_HIGH, + INA2XX_ALERT_POWER_HIGH, + INA2XX_ALERT_BUS_VOLTAGE_LOW, + INA2XX_ALERT_BUS_VOLTAGE_HIGH, + INA2XX_ALERT_SHUNT_VOLTAGE_LOW, + INA2XX_ALERT_SHUNT_VOLTAGE_HIGH, +}; struct ina2xx_config { u16 config_default; bool has_alerts; /* chip supports alerts and limits */ bool has_ishunt; /* chip has internal shunt resistor */ - bool has_power_average; /* chip has internal shunt resistor */ + bool has_power_average; /* chip supports average power */ + bool has_update_interval; int calibration_value; int shunt_div; + int shunt_voltage_shift; int bus_voltage_shift; int bus_voltage_lsb; /* uV */ int power_lsb_factor; + int current_shift; }; struct ina2xx_data { const struct ina2xx_config *config; enum ina2xx_ids chip; + enum ina2xx_alert_type active_alert; long rshunt; long current_lsb_uA; long power_lsb_uW; - struct mutex config_lock; struct regmap *regmap; struct i2c_client *client; }; @@ -166,44 +188,70 @@ static const struct ina2xx_config ina2xx_config[] = { .config_default = INA219_CONFIG_DEFAULT, .calibration_value = 4096, .shunt_div = 100, + .shunt_voltage_shift = 0, .bus_voltage_shift = 3, .bus_voltage_lsb = 4000, .power_lsb_factor = 20, .has_alerts = false, .has_ishunt = false, .has_power_average = false, + .current_shift = 0, + .has_update_interval = false, }, [ina226] = { .config_default = INA226_CONFIG_DEFAULT, .calibration_value = 2048, .shunt_div = 400, + .shunt_voltage_shift = 0, .bus_voltage_shift = 0, .bus_voltage_lsb = 1250, .power_lsb_factor = 25, .has_alerts = true, .has_ishunt = false, .has_power_average = false, + .current_shift = 0, + .has_update_interval = true, + }, + [ina234] = { + .config_default = INA226_CONFIG_DEFAULT, + .calibration_value = 2048, + .shunt_div = 25, /* 2.5 µV/LSB raw ADC reading from INA2XX_SHUNT_VOLTAGE */ + .shunt_voltage_shift = 4, + .bus_voltage_shift = 4, + .bus_voltage_lsb = 25600, + .power_lsb_factor = 32, + .has_alerts = true, + .has_ishunt = false, + .has_power_average = false, + .current_shift = 4, + .has_update_interval = true, }, [ina260] = { .config_default = INA260_CONFIG_DEFAULT, .shunt_div = 400, + .shunt_voltage_shift = 0, .bus_voltage_shift = 0, .bus_voltage_lsb = 1250, .power_lsb_factor = 8, .has_alerts = true, .has_ishunt = true, .has_power_average = false, + .current_shift = 0, + .has_update_interval = true, }, [sy24655] = { .config_default = SY24655_CONFIG_DEFAULT, .calibration_value = 4096, .shunt_div = 400, + .shunt_voltage_shift = 0, .bus_voltage_shift = 0, .bus_voltage_lsb = 1250, .power_lsb_factor = 25, .has_alerts = true, .has_ishunt = false, .has_power_average = true, + .current_shift = 0, + .has_update_interval = false, }, }; @@ -248,28 +296,34 @@ static u16 ina226_interval_to_reg(long interval) return FIELD_PREP(INA226_AVG_RD_MASK, avg_bits); } -static int ina2xx_get_value(struct ina2xx_data *data, u8 reg, - unsigned int regval) +static long ina2xx_get_value(struct ina2xx_data *data, u8 reg, + unsigned int regval) { - int val; + s64 val64; + long val; switch (reg) { case INA2XX_SHUNT_VOLTAGE: /* signed register */ - val = DIV_ROUND_CLOSEST((s16)regval, data->config->shunt_div); + val = DIV_ROUND_CLOSEST((s16)regval >> data->config->shunt_voltage_shift, + data->config->shunt_div); break; case INA2XX_BUS_VOLTAGE: - val = (regval >> data->config->bus_voltage_shift) * - data->config->bus_voltage_lsb; - val = DIV_ROUND_CLOSEST(val, 1000); + val = DIV_ROUND_CLOSEST((regval >> data->config->bus_voltage_shift) * + data->config->bus_voltage_lsb, 1000); break; case INA2XX_POWER: - val = regval * data->power_lsb_uW; + val = min_t(u64, (u64)regval * data->power_lsb_uW, LONG_MAX); break; case INA2XX_CURRENT: /* signed register, result in mA */ - val = (s16)regval * data->current_lsb_uA; - val = DIV_ROUND_CLOSEST(val, 1000); + val64 = (s64)((s16)regval >> data->config->current_shift) * + data->current_lsb_uA; + if (val64 < 0) + val64 = -DIV_ROUND_CLOSEST_ULL(-val64, 1000); + else + val64 = DIV_ROUND_CLOSEST_ULL(val64, 1000); + val = clamp_val(val64, LONG_MIN, LONG_MAX); break; case INA2XX_CALIBRATION: val = regval; @@ -358,25 +412,29 @@ static int ina2xx_read_init(struct device *dev, int reg, long *val) */ static u16 ina226_alert_to_reg(struct ina2xx_data *data, int reg, long val) { + long limit; + switch (reg) { case INA2XX_SHUNT_VOLTAGE: - val = clamp_val(val, 0, SHRT_MAX * data->config->shunt_div); - val *= data->config->shunt_div; - return clamp_val(val, 0, SHRT_MAX); + val = min_t(long, val, DIV_ROUND_CLOSEST(SHRT_MAX, data->config->shunt_div)); + return min_t(long, (val * data->config->shunt_div) << data->config->shunt_voltage_shift, + SHRT_MAX); case INA2XX_BUS_VOLTAGE: - val = clamp_val(val, 0, 200000); - val = (val * 1000) << data->config->bus_voltage_shift; - val = DIV_ROUND_CLOSEST(val, data->config->bus_voltage_lsb); - return clamp_val(val, 0, USHRT_MAX); + val = min_t(long, val, 130000); + return min_t(long, + DIV_ROUND_CLOSEST((val * 1000) << data->config->bus_voltage_shift, + data->config->bus_voltage_lsb), + USHRT_MAX); case INA2XX_POWER: - val = clamp_val(val, 0, UINT_MAX - data->power_lsb_uW); - val = DIV_ROUND_CLOSEST(val, data->power_lsb_uW); - return clamp_val(val, 0, USHRT_MAX); + val = min_t(long, val, LONG_MAX - data->power_lsb_uW); + return min_t(long, DIV_ROUND_CLOSEST(val, data->power_lsb_uW), USHRT_MAX); case INA2XX_CURRENT: - val = clamp_val(val, INT_MIN / 1000, INT_MAX / 1000); + limit = (LONG_MAX - (data->current_lsb_uA / 2)) / 1000; + val = min_t(long, val, limit); /* signed register, result in mA */ val = DIV_ROUND_CLOSEST(val * 1000, data->current_lsb_uA); - return clamp_val(val, SHRT_MIN, SHRT_MAX); + limit = SHRT_MAX >> data->config->current_shift; + return (u16)(min_t(long, val, limit) << data->config->current_shift); default: /* programmer goofed */ WARN_ON_ONCE(1); @@ -384,33 +442,94 @@ static u16 ina226_alert_to_reg(struct ina2xx_data *data, int reg, long val) } } -static int ina226_alert_limit_read(struct ina2xx_data *data, u32 mask, int reg, long *val) +static u32 ina2xx_alert_type_to_mask(enum ina2xx_alert_type alert) +{ + switch (alert) { + case INA2XX_ALERT_CURRENT_LOW: + case INA2XX_ALERT_SHUNT_VOLTAGE_LOW: + return INA226_SHUNT_UNDER_VOLTAGE_MASK; + case INA2XX_ALERT_CURRENT_HIGH: + case INA2XX_ALERT_SHUNT_VOLTAGE_HIGH: + return INA226_SHUNT_OVER_VOLTAGE_MASK; + case INA2XX_ALERT_BUS_VOLTAGE_LOW: + return INA226_BUS_UNDER_VOLTAGE_MASK; + case INA2XX_ALERT_BUS_VOLTAGE_HIGH: + return INA226_BUS_OVER_VOLTAGE_MASK; + case INA2XX_ALERT_POWER_HIGH: + return INA226_POWER_OVER_LIMIT_MASK; + case INA2XX_ALERT_NONE: + return 0; + default: + /* programmer error */ + WARN_ON_ONCE(1); + return 0; + } +} + +static enum ina2xx_alert_type ina2xx_mask_to_alert_type(u32 mask) +{ + int top_bit = fls(mask & INA226_ALERT_CONFIG_MASK); + + if (!top_bit) + return INA2XX_ALERT_NONE; + + /* + * Multiple bits may be set, with the highest-set function taking + * precedence according to the datasheet. Shunt voltage masks are + * assumed to map to voltage monitoring rather than current monitoring, + * since the latter isn't directly implemented in the hardware. + */ + switch (BIT(top_bit - 1)) { + case INA226_SHUNT_OVER_VOLTAGE_MASK: + return INA2XX_ALERT_SHUNT_VOLTAGE_HIGH; + case INA226_SHUNT_UNDER_VOLTAGE_MASK: + return INA2XX_ALERT_SHUNT_VOLTAGE_LOW; + case INA226_BUS_OVER_VOLTAGE_MASK: + return INA2XX_ALERT_BUS_VOLTAGE_HIGH; + case INA226_BUS_UNDER_VOLTAGE_MASK: + return INA2XX_ALERT_BUS_VOLTAGE_LOW; + case INA226_POWER_OVER_LIMIT_MASK: + return INA2XX_ALERT_POWER_HIGH; + default: + return INA2XX_ALERT_NONE; + } +} + +static int ina226_alert_limit_read(struct ina2xx_data *data, enum ina2xx_alert_type alert, + int reg, long *val) { struct regmap *regmap = data->regmap; int regval; + u32 mask; int ret; - mutex_lock(&data->config_lock); + /* Avoid nonzero reads from inactive alerts caused by shared limit register */ + if (data->active_alert != alert) { + *val = 0; + return 0; + } + ret = regmap_read(regmap, INA226_MASK_ENABLE, ®val); if (ret) - goto abort; + return ret; + mask = ina2xx_alert_type_to_mask(alert); if (regval & mask) { ret = regmap_read(regmap, INA226_ALERT_LIMIT, ®val); if (ret) - goto abort; + return ret; *val = ina2xx_get_value(data, reg, regval); } else { *val = 0; } -abort: - mutex_unlock(&data->config_lock); - return ret; + return 0; } -static int ina226_alert_limit_write(struct ina2xx_data *data, u32 mask, int reg, long val) +static int ina226_alert_limit_write(struct ina2xx_data *data, enum ina2xx_alert_type alert, + int reg, long val) { struct regmap *regmap = data->regmap; + u32 mask; int ret; if (val < 0) @@ -421,23 +540,27 @@ static int ina226_alert_limit_write(struct ina2xx_data *data, u32 mask, int reg, * due to register write sequence. Then, only enable the alert * if the value is non-zero. */ - mutex_lock(&data->config_lock); ret = regmap_update_bits(regmap, INA226_MASK_ENABLE, INA226_ALERT_CONFIG_MASK, 0); if (ret < 0) - goto abort; + return ret; + data->active_alert = INA2XX_ALERT_NONE; ret = regmap_write(regmap, INA226_ALERT_LIMIT, ina226_alert_to_reg(data, reg, val)); if (ret < 0) - goto abort; + return ret; - if (val) + if (val) { + mask = ina2xx_alert_type_to_mask(alert); ret = regmap_update_bits(regmap, INA226_MASK_ENABLE, INA226_ALERT_CONFIG_MASK, mask); -abort: - mutex_unlock(&data->config_lock); - return ret; + if (ret < 0) + return ret; + data->active_alert = alert; + } + + return 0; } static int ina2xx_chip_read(struct device *dev, u32 attr, long *val) @@ -460,15 +583,26 @@ static int ina2xx_chip_read(struct device *dev, u32 attr, long *val) return 0; } -static int ina226_alert_read(struct regmap *regmap, u32 mask, long *val) +static int ina226_alert_read(struct ina2xx_data *data, enum ina2xx_alert_type alert, long *val) { unsigned int regval; + u32 mask; int ret; - ret = regmap_read_bypassed(regmap, INA226_MASK_ENABLE, ®val); + /* + * With alert latching, reading alerts from hardware also clears the + * alert, so return early if the alert is inactive. + */ + if (data->active_alert != alert) { + *val = 0; + return 0; + } + + ret = regmap_read_bypassed(data->regmap, INA226_MASK_ENABLE, ®val); if (ret) return ret; + mask = ina2xx_alert_type_to_mask(alert); *val = (regval & mask) && (regval & INA226_ALERT_FUNCTION_FLAG); return 0; @@ -477,10 +611,10 @@ static int ina226_alert_read(struct regmap *regmap, u32 mask, long *val) static int ina2xx_in_read(struct device *dev, u32 attr, int channel, long *val) { int voltage_reg = channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE; - u32 under_voltage_mask = channel ? INA226_BUS_UNDER_VOLTAGE_MASK - : INA226_SHUNT_UNDER_VOLTAGE_MASK; - u32 over_voltage_mask = channel ? INA226_BUS_OVER_VOLTAGE_MASK - : INA226_SHUNT_OVER_VOLTAGE_MASK; + enum ina2xx_alert_type under_voltage_alert = channel ? INA2XX_ALERT_BUS_VOLTAGE_LOW + : INA2XX_ALERT_SHUNT_VOLTAGE_LOW; + enum ina2xx_alert_type over_voltage_alert = channel ? INA2XX_ALERT_BUS_VOLTAGE_HIGH + : INA2XX_ALERT_SHUNT_VOLTAGE_HIGH; struct ina2xx_data *data = dev_get_drvdata(dev); struct regmap *regmap = data->regmap; unsigned int regval; @@ -494,15 +628,15 @@ static int ina2xx_in_read(struct device *dev, u32 attr, int channel, long *val) *val = ina2xx_get_value(data, voltage_reg, regval); break; case hwmon_in_lcrit: - return ina226_alert_limit_read(data, under_voltage_mask, + return ina226_alert_limit_read(data, under_voltage_alert, voltage_reg, val); case hwmon_in_crit: - return ina226_alert_limit_read(data, over_voltage_mask, + return ina226_alert_limit_read(data, over_voltage_alert, voltage_reg, val); case hwmon_in_lcrit_alarm: - return ina226_alert_read(regmap, under_voltage_mask, val); + return ina226_alert_read(data, under_voltage_alert, val); case hwmon_in_crit_alarm: - return ina226_alert_read(regmap, over_voltage_mask, val); + return ina226_alert_read(data, over_voltage_alert, val); default: return -EOPNOTSUPP; } @@ -521,6 +655,7 @@ static int sy24655_average_power_read(struct ina2xx_data *data, u8 reg, long *va u8 template[6]; int ret; long accumulator_24, sample_count; + u64 val64; /* 48-bit register read */ ret = i2c_smbus_read_i2c_block_data(data->client, reg, 6, template); @@ -539,7 +674,8 @@ static int sy24655_average_power_read(struct ina2xx_data *data, u8 reg, long *va return 0; } - *val = DIV_ROUND_CLOSEST(accumulator_24, sample_count) * data->power_lsb_uW; + val64 = (u64)DIV_ROUND_CLOSEST(accumulator_24, sample_count) * data->power_lsb_uW; + *val = min_t(u64, val64, LONG_MAX); return 0; } @@ -554,10 +690,10 @@ static int ina2xx_power_read(struct device *dev, u32 attr, long *val) case hwmon_power_average: return sy24655_average_power_read(data, SY24655_EIN, val); case hwmon_power_crit: - return ina226_alert_limit_read(data, INA226_POWER_OVER_LIMIT_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_POWER_HIGH, INA2XX_POWER, val); case hwmon_power_crit_alarm: - return ina226_alert_read(data->regmap, INA226_POWER_OVER_LIMIT_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_POWER_HIGH, val); default: return -EOPNOTSUPP; } @@ -593,15 +729,15 @@ static int ina2xx_curr_read(struct device *dev, u32 attr, long *val) *val = ina2xx_get_value(data, INA2XX_CURRENT, regval); return 0; case hwmon_curr_lcrit: - return ina226_alert_limit_read(data, INA226_SHUNT_UNDER_VOLTAGE_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_CURRENT_LOW, INA2XX_CURRENT, val); case hwmon_curr_crit: - return ina226_alert_limit_read(data, INA226_SHUNT_OVER_VOLTAGE_MASK, + return ina226_alert_limit_read(data, INA2XX_ALERT_CURRENT_HIGH, INA2XX_CURRENT, val); case hwmon_curr_lcrit_alarm: - return ina226_alert_read(regmap, INA226_SHUNT_UNDER_VOLTAGE_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_CURRENT_LOW, val); case hwmon_curr_crit_alarm: - return ina226_alert_read(regmap, INA226_SHUNT_OVER_VOLTAGE_MASK, val); + return ina226_alert_read(data, INA2XX_ALERT_CURRENT_HIGH, val); default: return -EOPNOTSUPP; } @@ -645,12 +781,12 @@ static int ina2xx_in_write(struct device *dev, u32 attr, int channel, long val) switch (attr) { case hwmon_in_lcrit: return ina226_alert_limit_write(data, - channel ? INA226_BUS_UNDER_VOLTAGE_MASK : INA226_SHUNT_UNDER_VOLTAGE_MASK, + channel ? INA2XX_ALERT_BUS_VOLTAGE_LOW : INA2XX_ALERT_SHUNT_VOLTAGE_LOW, channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE, val); case hwmon_in_crit: return ina226_alert_limit_write(data, - channel ? INA226_BUS_OVER_VOLTAGE_MASK : INA226_SHUNT_OVER_VOLTAGE_MASK, + channel ? INA2XX_ALERT_BUS_VOLTAGE_HIGH : INA2XX_ALERT_SHUNT_VOLTAGE_HIGH, channel ? INA2XX_BUS_VOLTAGE : INA2XX_SHUNT_VOLTAGE, val); default: @@ -665,7 +801,7 @@ static int ina2xx_power_write(struct device *dev, u32 attr, long val) switch (attr) { case hwmon_power_crit: - return ina226_alert_limit_write(data, INA226_POWER_OVER_LIMIT_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_POWER_HIGH, INA2XX_POWER, val); default: return -EOPNOTSUPP; @@ -679,10 +815,10 @@ static int ina2xx_curr_write(struct device *dev, u32 attr, long val) switch (attr) { case hwmon_curr_lcrit: - return ina226_alert_limit_write(data, INA226_SHUNT_UNDER_VOLTAGE_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_CURRENT_LOW, INA2XX_CURRENT, val); case hwmon_curr_crit: - return ina226_alert_limit_write(data, INA226_SHUNT_OVER_VOLTAGE_MASK, + return ina226_alert_limit_write(data, INA2XX_ALERT_CURRENT_HIGH, INA2XX_CURRENT, val); default: return -EOPNOTSUPP; @@ -713,7 +849,7 @@ static umode_t ina2xx_is_visible(const void *_data, enum hwmon_sensor_types type const struct ina2xx_data *data = _data; bool has_alerts = data->config->has_alerts; bool has_power_average = data->config->has_power_average; - enum ina2xx_ids chip = data->chip; + bool has_update_interval = data->config->has_update_interval; switch (type) { case hwmon_in: @@ -775,7 +911,7 @@ static umode_t ina2xx_is_visible(const void *_data, enum hwmon_sensor_types type case hwmon_chip: switch (attr) { case hwmon_chip_update_interval: - if (chip == ina226 || chip == ina260) + if (has_update_interval) return 0644; break; default: @@ -843,8 +979,12 @@ static ssize_t shunt_resistor_show(struct device *dev, struct device_attribute *da, char *buf) { struct ina2xx_data *data = dev_get_drvdata(dev); + long rshunt; - return sysfs_emit(buf, "%li\n", data->rshunt); + scoped_guard(hwmon_lock, dev) { + rshunt = data->rshunt; + } + return sysfs_emit(buf, "%li\n", rshunt); } static ssize_t shunt_resistor_store(struct device *dev, @@ -859,9 +999,9 @@ static ssize_t shunt_resistor_store(struct device *dev, if (status < 0) return status; - mutex_lock(&data->config_lock); + hwmon_lock(dev); status = ina2xx_set_shunt(data, val); - mutex_unlock(&data->config_lock); + hwmon_unlock(dev); if (status < 0) return status; return count; @@ -900,6 +1040,16 @@ static int ina2xx_init(struct device *dev, struct ina2xx_data *data) if (data->config->has_alerts) { bool active_high = device_property_read_bool(dev, "ti,alert-polarity-active-high"); + unsigned int mask_enable; + + /* + * Infer active alert from MASK_ENABLE in case it's already + * configured (e.g., by a past probe or firmware) + */ + ret = regmap_read(regmap, INA226_MASK_ENABLE, &mask_enable); + if (ret < 0) + return ret; + data->active_alert = ina2xx_mask_to_alert_type(mask_enable); regmap_update_bits(regmap, INA226_MASK_ENABLE, INA226_ALERT_LATCH_ENABLE | INA226_ALERT_POLARITY, @@ -951,7 +1101,6 @@ static int ina2xx_probe(struct i2c_client *client) data->client = client; data->config = &ina2xx_config[chip]; data->chip = chip; - mutex_init(&data->config_lock); data->regmap = devm_regmap_init_i2c(client, &ina2xx_regmap_config); if (IS_ERR(data->regmap)) { @@ -990,6 +1139,7 @@ static const struct i2c_device_id ina2xx_id[] = { { "ina226", ina226 }, { "ina230", ina226 }, { "ina231", ina226 }, + { "ina234", ina234 }, { "ina260", ina260 }, { "sy24655", sy24655 }, { } @@ -1021,6 +1171,10 @@ static const struct of_device_id __maybe_unused ina2xx_of_match[] = { .compatible = "ti,ina231", .data = (void *)ina226 }, + { + .compatible = "ti,ina234", + .data = (void *)ina234 + }, { .compatible = "ti,ina260", .data = (void *)ina260 diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c index a465a8a7ef5af..e1795ccde876c 100644 --- a/drivers/hwmon/lm90.c +++ b/drivers/hwmon/lm90.c @@ -1196,7 +1196,7 @@ static int lm90_update_alarms_locked(struct lm90_data *data, bool force) check_enable = (client->irq || !(data->config_orig & 0x80)) && (data->config & 0x80); - if (force || check_enable) + if (data->hwmon_dev && (force || check_enable)) schedule_work(&data->report_work); /* @@ -1204,7 +1204,7 @@ static int lm90_update_alarms_locked(struct lm90_data *data, bool force) * alarms are all clear, and alerts are currently disabled. * Otherwise (re)schedule worker if needed. */ - if (check_enable) { + if (check_enable && data->hwmon_dev) { if (!(data->current_alarms & data->alert_alarms)) { dev_dbg(&client->dev, "Re-enabling ALERT#\n"); lm90_update_confreg(data, data->config & ~0x80); diff --git a/drivers/hwmon/ltc4282.c b/drivers/hwmon/ltc4282.c index 58c2d3a624329..a82374ba7fc58 100644 --- a/drivers/hwmon/ltc4282.c +++ b/drivers/hwmon/ltc4282.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -142,7 +143,7 @@ struct ltc4282_state { */ struct ltc4282_cache in0_1_cache[LTC4282_CHAN_VGPIO]; u32 vsense_max; - long power_max; + s64 power_max; u32 rsense; u16 vdd; u16 vfs_out; @@ -384,8 +385,8 @@ static int ltc4282_read_in(struct ltc4282_state *st, u32 attr, long *val, channel, val); case hwmon_in_min_alarm: if (channel == LTC4282_CHAN_VGPIO) - ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, - LTC4282_GPIO_ALARM_L_MASK, val); + return ltc4282_read_alarm(st, LTC4282_ADC_ALERT_LOG, + LTC4282_GPIO_ALARM_L_MASK, val); return ltc4282_vdd_source_read_alm(st, LTC4282_VSOURCE_ALARM_L_MASK, @@ -623,13 +624,12 @@ static int ltc4282_read(struct device *dev, enum hwmon_sensor_types type, } static int ltc4282_write_power_byte(const struct ltc4282_state *st, u32 reg, - long val) + s64 val) { u32 power; u64 temp; - if (val > st->power_max) - val = st->power_max; + val = clamp(val, 0, st->power_max); temp = val * int_pow(U8_MAX, 2) * st->rsense; power = DIV64_U64_ROUND_CLOSEST(temp, @@ -639,7 +639,7 @@ static int ltc4282_write_power_byte(const struct ltc4282_state *st, u32 reg, } static int ltc4282_write_power_word(const struct ltc4282_state *st, u32 reg, - long val) + u64 val) { u64 temp = int_pow(U16_MAX, 2) * st->rsense, temp_2; __be16 __raw; @@ -948,8 +948,11 @@ static int ltc4282_curr_reset_hist(struct ltc4282_state *st) static int ltc4282_write_curr(struct ltc4282_state *st, u32 attr, long val) { + s32 ulimit = min_t(u64, INT_MAX, + div_u64((u64)INT_MAX * DECA * MICRO, st->rsense)); + u64 val64 = clamp(val, 0, ulimit); /* need to pass it in millivolt */ - u32 in = DIV_ROUND_CLOSEST_ULL((u64)val * st->rsense, DECA * MICRO); + u32 in = DIV_ROUND_CLOSEST_ULL(val64 * st->rsense, DECA * MICRO); switch (attr) { case hwmon_curr_max: @@ -1137,7 +1140,7 @@ static const struct clk_ops ltc4282_ops = { static int ltc428_clk_provider_setup(struct ltc4282_state *st, struct device *dev) { - struct clk_init_data init; + struct clk_init_data init = {}; int ret; if (!IS_ENABLED(CONFIG_COMMON_CLK)) @@ -1256,7 +1259,8 @@ static int ltc4282_set_max_limits(struct ltc4282_state *st) return ret; /* Power is given by ISENSE * Vout. */ - st->power_max = DIV_ROUND_CLOSEST(st->vsense_max * DECA * MILLI, st->rsense) * st->vfs_out; + st->power_max = DIV_ROUND_CLOSEST_ULL((u64)st->vsense_max * DECA * MILLI, + st->rsense) * st->vfs_out; ret = ltc4282_write_power_byte(st, LTC4282_POWER_MAX, st->power_max); if (ret) return ret; @@ -1423,7 +1427,7 @@ static int ltc4282_setup(struct ltc4282_state *st, struct device *dev) if (!ret) { int reg_val; - switch (val) { + switch (st->vsense_max) { case 12500: reg_val = 0; break; diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c index a7066f3a0bb4e..3690f2ffe5fcb 100644 --- a/drivers/hwmon/max6621.c +++ b/drivers/hwmon/max6621.c @@ -17,8 +17,8 @@ #define MAX6621_DRV_NAME "max6621" #define MAX6621_TEMP_INPUT_REG_NUM 9 -#define MAX6621_TEMP_INPUT_MIN -127000 -#define MAX6621_TEMP_INPUT_MAX 128000 +#define MAX6621_TEMP_INPUT_MIN -128000 +#define MAX6621_TEMP_INPUT_MAX 127000 #define MAX6621_TEMP_ALERT_CHAN_SHIFT 1 #define MAX6621_TEMP_S0D0_REG 0x00 @@ -239,7 +239,7 @@ max6621_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, if (ret) return ret; - *val = (regval >> MAX6621_REG_TEMP_SHIFT) * + *val = ((s16)regval >> MAX6621_REG_TEMP_SHIFT) * 1000L; break; @@ -254,7 +254,7 @@ max6621_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, if (ret) return ret; - *val = regval * 1000L; + *val = (s16)regval * 1000L; break; case hwmon_temp_crit_alarm: diff --git a/drivers/hwmon/nct6694-hwmon.c b/drivers/hwmon/nct6694-hwmon.c index 6dcf22ca5018a..9a9a4db434c4f 100644 --- a/drivers/hwmon/nct6694-hwmon.c +++ b/drivers/hwmon/nct6694-hwmon.c @@ -159,6 +159,9 @@ static inline s8 temp_to_reg(long val) #define NCT6694_HWMON_TEMP_CONFIG (HWMON_T_INPUT | HWMON_T_ENABLE | \ HWMON_T_MAX | HWMON_T_MAX_HYST | \ HWMON_T_MAX_ALARM) +#define NCT6694_HWMON_DTIN_CONFIG (HWMON_T_INPUT | \ + HWMON_T_MAX | HWMON_T_MAX_HYST | \ + HWMON_T_MAX_ALARM) #define NCT6694_HWMON_FAN_CONFIG (HWMON_F_INPUT | HWMON_F_ENABLE | \ HWMON_F_MIN | HWMON_F_MIN_ALARM) #define NCT6694_HWMON_PWM_CONFIG (HWMON_PWM_INPUT | HWMON_PWM_ENABLE | \ @@ -193,22 +196,22 @@ static const struct hwmon_channel_info *nct6694_info[] = { NCT6694_HWMON_TEMP_CONFIG, /* TDP2 */ NCT6694_HWMON_TEMP_CONFIG, /* TDP3 */ NCT6694_HWMON_TEMP_CONFIG, /* TDP4 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN0 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN1 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN2 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN3 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN4 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN5 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN6 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN7 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN8 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN9 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN10 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN11 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN12 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN13 */ - NCT6694_HWMON_TEMP_CONFIG, /* DTIN14 */ - NCT6694_HWMON_TEMP_CONFIG), /* DTIN15 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN0 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN1 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN2 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN3 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN4 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN5 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN6 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN7 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN8 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN9 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN10 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN11 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN12 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN13 */ + NCT6694_HWMON_DTIN_CONFIG, /* DTIN14 */ + NCT6694_HWMON_DTIN_CONFIG), /* DTIN15 */ HWMON_CHANNEL_INFO(fan, NCT6694_HWMON_FAN_CONFIG, /* FIN0 */ diff --git a/drivers/hwmon/nct6775-core.c b/drivers/hwmon/nct6775-core.c index 79bc67ffb9986..ce2cf1f229004 100644 --- a/drivers/hwmon/nct6775-core.c +++ b/drivers/hwmon/nct6775-core.c @@ -791,12 +791,12 @@ static const u16 NCT6106_REG_TOLERANCE_H[] = { 0x112, 0x122, 0x132 }; static const u16 NCT6106_REG_TARGET[] = { 0x111, 0x121, 0x131 }; -static const u16 NCT6106_REG_WEIGHT_TEMP_SEL[] = { 0x168, 0x178, 0x188 }; -static const u16 NCT6106_REG_WEIGHT_TEMP_STEP[] = { 0x169, 0x179, 0x189 }; -static const u16 NCT6106_REG_WEIGHT_TEMP_STEP_TOL[] = { 0x16a, 0x17a, 0x18a }; -static const u16 NCT6106_REG_WEIGHT_DUTY_STEP[] = { 0x16b, 0x17b, 0x18b }; -static const u16 NCT6106_REG_WEIGHT_TEMP_BASE[] = { 0x16c, 0x17c, 0x18c }; -static const u16 NCT6106_REG_WEIGHT_DUTY_BASE[] = { 0x16d, 0x17d, 0x18d }; +static const u16 NCT6106_REG_WEIGHT_TEMP_SEL[] = { 0x168, 0x178, 0x188, 0, 0 }; +static const u16 NCT6106_REG_WEIGHT_TEMP_STEP[] = { 0x169, 0x179, 0x189, 0, 0 }; +static const u16 NCT6106_REG_WEIGHT_TEMP_STEP_TOL[] = { 0x16a, 0x17a, 0x18a, 0, 0 }; +static const u16 NCT6106_REG_WEIGHT_DUTY_STEP[] = { 0x16b, 0x17b, 0x18b, 0, 0 }; +static const u16 NCT6106_REG_WEIGHT_TEMP_BASE[] = { 0x16c, 0x17c, 0x18c, 0, 0 }; +static const u16 NCT6106_REG_WEIGHT_DUTY_BASE[] = { 0x16d, 0x17d, 0x18d, 0, 0 }; static const u16 NCT6106_REG_AUTO_TEMP[] = { 0x160, 0x170, 0x180 }; static const u16 NCT6106_REG_AUTO_PWM[] = { 0x164, 0x174, 0x184 }; @@ -846,8 +846,6 @@ static const u16 NCT6116_FAN_PULSE_SHIFT[] = { 0, 2, 4, 6, 6 }; static const u16 NCT6116_REG_PWM[] = { 0x119, 0x129, 0x139, 0x199, 0x1a9 }; static const u16 NCT6116_REG_FAN_MODE[] = { 0x113, 0x123, 0x133, 0x193, 0x1a3 }; static const u16 NCT6116_REG_TEMP_SEL[] = { 0x110, 0x120, 0x130, 0x190, 0x1a0 }; -static const u16 NCT6116_REG_TEMP_SOURCE[] = { - 0xb0, 0xb1, 0xb2 }; static const u16 NCT6116_REG_CRITICAL_TEMP[] = { 0x11a, 0x12a, 0x13a, 0x19a, 0x1aa }; @@ -3650,7 +3648,7 @@ int nct6775_probe(struct device *dev, struct nct6775_data *data, = NCT6106_CRITICAL_PWM_ENABLE_MASK; data->REG_CRITICAL_PWM = NCT6116_REG_CRITICAL_PWM; data->REG_TEMP_OFFSET = NCT6106_REG_TEMP_OFFSET; - data->REG_TEMP_SOURCE = NCT6116_REG_TEMP_SOURCE; + data->REG_TEMP_SOURCE = NCT6106_REG_TEMP_SOURCE; data->REG_TEMP_SEL = NCT6116_REG_TEMP_SEL; data->REG_WEIGHT_TEMP_SEL = NCT6106_REG_WEIGHT_TEMP_SEL; data->REG_WEIGHT_TEMP[0] = NCT6106_REG_WEIGHT_TEMP_STEP; @@ -3664,13 +3662,13 @@ int nct6775_probe(struct device *dev, struct nct6775_data *data, reg_temp = NCT6106_REG_TEMP; reg_temp_mon = NCT6106_REG_TEMP_MON; - num_reg_temp = ARRAY_SIZE(NCT6106_REG_TEMP); + num_reg_temp = 3; num_reg_temp_mon = ARRAY_SIZE(NCT6106_REG_TEMP_MON); num_reg_tsi_temp = ARRAY_SIZE(NCT6116_REG_TSI_TEMP); reg_temp_over = NCT6106_REG_TEMP_OVER; reg_temp_hyst = NCT6106_REG_TEMP_HYST; reg_temp_config = NCT6106_REG_TEMP_CONFIG; - num_reg_temp_config = ARRAY_SIZE(NCT6106_REG_TEMP_CONFIG); + num_reg_temp_config = 3; reg_temp_alternate = NCT6106_REG_TEMP_ALTERNATE; reg_temp_crit = NCT6106_REG_TEMP_CRIT; reg_temp_crit_l = NCT6106_REG_TEMP_CRIT_L; diff --git a/drivers/hwmon/npcm750-pwm-fan.c b/drivers/hwmon/npcm750-pwm-fan.c index 802c73def428b..bb390f6c7c3f5 100644 --- a/drivers/hwmon/npcm750-pwm-fan.c +++ b/drivers/hwmon/npcm750-pwm-fan.c @@ -364,6 +364,11 @@ static void npcm7xx_fan_polling(struct timer_list *t) add_timer(&data->fan_timer); } +static void npcm7xx_fan_cleanup(void *timer) +{ + timer_shutdown_sync(timer); +} + static inline void npcm7xx_fan_compute(struct npcm7xx_pwm_fan_data *data, u8 fan, u8 cmp, u8 fan_id, u8 flag_int, u8 flag_mode, u8 flag_clear) @@ -1027,6 +1032,12 @@ static int npcm7xx_pwm_fan_probe(struct platform_device *pdev) msecs_to_jiffies(NPCM7XX_FAN_POLL_TIMER_200MS); timer_setup(&data->fan_timer, npcm7xx_fan_polling, 0); + ret = devm_add_action_or_reset(dev, + npcm7xx_fan_cleanup, + &data->fan_timer); + if (ret) + return ret; + add_timer(&data->fan_timer); break; } diff --git a/drivers/hwmon/nzxt-kraken3.c b/drivers/hwmon/nzxt-kraken3.c index d00409bcab93a..05525406c5fbb 100644 --- a/drivers/hwmon/nzxt-kraken3.c +++ b/drivers/hwmon/nzxt-kraken3.c @@ -948,7 +948,7 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id ret = kraken3_init_device(hdev); if (ret < 0) { hid_err(hdev, "device init failed with %d\n", ret); - goto fail_and_close; + goto fail_and_stop_io; } ret = kraken3_get_fw_ver(hdev); @@ -960,13 +960,15 @@ static int kraken3_probe(struct hid_device *hdev, const struct hid_device_id *id if (IS_ERR(priv->hwmon_dev)) { ret = PTR_ERR(priv->hwmon_dev); hid_err(hdev, "hwmon registration failed with %d\n", ret); - goto fail_and_close; + goto fail_and_stop_io; } kraken3_debugfs_init(priv, device_name); return 0; +fail_and_stop_io: + hid_device_io_stop(hdev); fail_and_close: hid_hw_close(hdev); fail_and_stop: diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c index 58ef9fa0184be..efc78a8f58c88 100644 --- a/drivers/hwmon/nzxt-smart2.c +++ b/drivers/hwmon/nzxt-smart2.c @@ -203,7 +203,7 @@ struct drvdata { */ struct mutex mutex; long update_interval; - u8 output_buffer[OUTPUT_REPORT_SIZE]; + u8 output_buffer[OUTPUT_REPORT_SIZE] __aligned(ARCH_DMA_MINALIGN); }; static long scale_pwm_value(long val, long orig_max, long new_max) @@ -754,7 +754,11 @@ static int nzxt_smart2_hid_probe(struct hid_device *hdev, hid_device_io_start(hdev); - init_device(drvdata, UPDATE_INTERVAL_DEFAULT_MS); + ret = init_device(drvdata, UPDATE_INTERVAL_DEFAULT_MS); + if (ret) { + dev_err(&hdev->dev, "init_device failed: %d\n", ret); + goto out_hw_close; + } drvdata->hwmon = hwmon_device_register_with_info(&hdev->dev, "nzxtsmart2", drvdata, @@ -768,7 +772,7 @@ static int nzxt_smart2_hid_probe(struct hid_device *hdev, out_hw_close: hid_hw_close(hdev); - + hid_device_io_stop(hdev); out_hw_stop: hid_hw_stop(hdev); return ret; diff --git a/drivers/hwmon/occ/common.c b/drivers/hwmon/occ/common.c index e18e80e832fd3..175208d712b06 100644 --- a/drivers/hwmon/occ/common.c +++ b/drivers/hwmon/occ/common.c @@ -1052,32 +1052,49 @@ static int occ_setup_sensor_attrs(struct occ *occ) } /* only need to do this once at startup, as OCC won't change sensors on us */ -static void occ_parse_poll_response(struct occ *occ) +static int occ_parse_poll_response(struct occ *occ) { unsigned int i, old_offset, offset = 0, size = 0; + u16 data_length; struct occ_sensor *sensor; - struct occ_sensors *sensors = &occ->sensors; + struct occ_sensors parsed = {}; + struct occ_sensors *sensors = &parsed; struct occ_response *resp = &occ->resp; struct occ_poll_response *poll = (struct occ_poll_response *)&resp->data[0]; struct occ_poll_response_header *header = &poll->header; struct occ_sensor_data_block *block = &poll->block; + data_length = get_unaligned_be16(&resp->data_length); + if (data_length < sizeof(*header) || data_length > OCC_RESP_DATA_BYTES) { + dev_err(occ->bus_dev, "invalid OCC poll response length %u\n", + data_length); + return -EMSGSIZE; + } + dev_info(occ->bus_dev, "OCC found, code level: %.16s\n", header->occ_code_level); for (i = 0; i < header->num_sensor_data_blocks; ++i) { block = (struct occ_sensor_data_block *)((u8 *)block + offset); + if (size + sizeof(*header) + sizeof(block->header) > + data_length) { + dev_err(occ->bus_dev, + "truncated OCC sensor block header\n"); + return -EMSGSIZE; + } + old_offset = offset; offset = (block->header.num_sensors * block->header.sensor_length) + sizeof(block->header); - size += offset; /* validate all the length/size fields */ - if ((size + sizeof(*header)) >= OCC_RESP_DATA_BYTES) { - dev_warn(occ->bus_dev, "exceeded response buffer\n"); - return; + if (size + sizeof(*header) + offset > data_length) { + dev_err(occ->bus_dev, + "exceeded OCC poll response length\n"); + return -EMSGSIZE; } + size += offset; dev_dbg(occ->bus_dev, " %04x..%04x: %.4s (%d sensors)\n", old_offset, offset - 1, block->header.eye_catcher, @@ -1107,6 +1124,9 @@ static void occ_parse_poll_response(struct occ *occ) dev_dbg(occ->bus_dev, "Max resp size: %u+%zd=%zd\n", size, sizeof(*header), size + sizeof(*header)); + occ->sensors = parsed; + + return 0; } int occ_active(struct occ *occ, bool active) @@ -1138,10 +1158,12 @@ int occ_active(struct occ *occ, bool active) goto unlock; } - occ->active = true; occ->next_update = jiffies + OCC_UPDATE_FREQUENCY; - occ_parse_poll_response(occ); + rc = occ_parse_poll_response(occ); + if (rc) + goto unlock; + occ->active = true; rc = occ_setup_sensor_attrs(occ); if (rc) { dev_err(occ->bus_dev, diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c index dd7275a67a0ab..de88ff5967a32 100644 --- a/drivers/hwmon/pmbus/lm25066.c +++ b/drivers/hwmon/pmbus/lm25066.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "pmbus.h" @@ -132,23 +133,23 @@ static const struct __coeff lm25066_coeff[][PSC_NUM_CLASSES + 2] = { .R = -2, }, [PSC_CURRENT_IN] = { - .m = 10742, - .b = 1552, + .m = 5456, + .b = 2118, .R = -2, }, [PSC_CURRENT_IN_L] = { - .m = 5456, - .b = 2118, + .m = 10742, + .b = 1552, .R = -2, }, [PSC_POWER] = { - .m = 1204, - .b = 8524, + .m = 612, + .b = 11202, .R = -3, }, [PSC_POWER_L] = { - .m = 612, - .b = 11202, + .m = 1204, + .b = 8524, .R = -3, }, [PSC_TEMPERATURE] = { @@ -167,23 +168,23 @@ static const struct __coeff lm25066_coeff[][PSC_NUM_CLASSES + 2] = { .R = -2, }, [PSC_CURRENT_IN] = { - .m = 10753, - .b = -1200, + .m = 5405, + .b = -600, .R = -2, }, [PSC_CURRENT_IN_L] = { - .m = 5405, - .b = -600, + .m = 10753, + .b = -1200, .R = -2, }, [PSC_POWER] = { - .m = 1204, - .b = -6000, + .m = 605, + .b = -8000, .R = -3, }, [PSC_POWER_L] = { - .m = 605, - .b = -8000, + .m = 1204, + .b = -6000, .R = -3, }, [PSC_TEMPERATURE] = { @@ -202,23 +203,23 @@ static const struct __coeff lm25066_coeff[][PSC_NUM_CLASSES + 2] = { .R = -2, }, [PSC_CURRENT_IN] = { - .m = 15076, - .b = -504, + .m = 7645, + .b = 100, .R = -2, }, [PSC_CURRENT_IN_L] = { - .m = 7645, - .b = 100, + .m = 15076, + .b = -504, .R = -2, }, [PSC_POWER] = { - .m = 1701, - .b = -4000, + .m = 861, + .b = -965, .R = -3, }, [PSC_POWER_L] = { - .m = 861, - .b = -965, + .m = 1701, + .b = -4000, .R = -3, }, [PSC_TEMPERATURE] = { @@ -519,18 +520,20 @@ static int lm25066_probe(struct i2c_client *client) info->m[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].m; info->b[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].b; info->R[PSC_VOLTAGE_OUT] = coeff[PSC_VOLTAGE_OUT].R; - info->R[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].R; - info->R[PSC_POWER] = coeff[PSC_POWER].R; if (config & LM25066_DEV_SETUP_CL) { info->m[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN_L].m; info->b[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN_L].b; + info->R[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN_L].R; info->m[PSC_POWER] = coeff[PSC_POWER_L].m; info->b[PSC_POWER] = coeff[PSC_POWER_L].b; + info->R[PSC_POWER] = coeff[PSC_POWER_L].R; } else { info->m[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].m; info->b[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].b; + info->R[PSC_CURRENT_IN] = coeff[PSC_CURRENT_IN].R; info->m[PSC_POWER] = coeff[PSC_POWER].m; info->b[PSC_POWER] = coeff[PSC_POWER].b; + info->R[PSC_POWER] = coeff[PSC_POWER].R; } /* @@ -540,8 +543,8 @@ static int lm25066_probe(struct i2c_client *client) if (of_property_read_u32(client->dev.of_node, "shunt-resistor-micro-ohms", &shunt)) shunt = 1000; - info->m[PSC_CURRENT_IN] = info->m[PSC_CURRENT_IN] * shunt / 1000; - info->m[PSC_POWER] = info->m[PSC_POWER] * shunt / 1000; + info->m[PSC_CURRENT_IN] = DIV_ROUND_CLOSEST_ULL((u64)info->m[PSC_CURRENT_IN] * shunt, 1000); + info->m[PSC_POWER] = DIV_ROUND_CLOSEST_ULL((u64)info->m[PSC_POWER] * shunt, 1000); #if IS_ENABLED(CONFIG_SENSORS_LM25066_REGULATOR) /* LM25056 doesn't support OPERATION */ diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c index ddf64e72751cc..aa578d2723f22 100644 --- a/drivers/hwmon/pmbus/pmbus_core.c +++ b/drivers/hwmon/pmbus/pmbus_core.c @@ -46,7 +46,7 @@ module_param(wp, int, 0444); struct pmbus_sensor { struct pmbus_sensor *next; char name[PMBUS_NAME_SIZE]; /* sysfs sensor name */ - struct device_attribute attribute; + struct sensor_device_attribute attribute; u8 page; /* page number */ u8 phase; /* phase number, 0xff for all phases */ u16 reg; /* register */ @@ -69,7 +69,7 @@ struct pmbus_boolean { struct pmbus_label { char name[PMBUS_NAME_SIZE]; /* sysfs label name */ - struct device_attribute attribute; + struct sensor_device_attribute attribute; char label[PMBUS_NAME_SIZE]; /* label */ }; #define to_pmbus_label(_attr) \ @@ -512,7 +512,7 @@ int pmbus_update_byte_data(struct i2c_client *client, int page, u8 reg, if (tmp != rv) rv = _pmbus_write_byte_data(client, page, reg, tmp); - return rv; + return rv < 0 ? rv : 0; } EXPORT_SYMBOL_NS_GPL(pmbus_update_byte_data, "PMBUS"); @@ -1156,12 +1156,11 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b, int ret, status; u16 regval; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + status = pmbus_get_status(client, page, reg); - if (status < 0) { - ret = status; - goto unlock; - } + if (status < 0) + return status; if (s1) pmbus_update_sensor_data(client, s1); @@ -1170,10 +1169,12 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b, regval = status & mask; if (regval) { - if (data->revision >= PMBUS_REV_12) { + /* Generic STATUS_WORD alarms are not individually clearable. */ + if (data->revision >= PMBUS_REV_12 && + reg != PMBUS_STATUS_WORD) { ret = _pmbus_write_byte_data(client, page, reg, regval); if (ret) - goto unlock; + return ret; } else { pmbus_clear_fault_page(client, page); } @@ -1181,14 +1182,10 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b, if (s1 && s2) { s64 v1, v2; - if (s1->data < 0) { - ret = s1->data; - goto unlock; - } - if (s2->data < 0) { - ret = s2->data; - goto unlock; - } + if (s1->data < 0) + return s1->data; + if (s2->data < 0) + return s2->data; v1 = pmbus_reg2data(data, s1); v2 = pmbus_reg2data(data, s2); @@ -1196,8 +1193,6 @@ static int pmbus_get_boolean(struct i2c_client *client, struct pmbus_boolean *b, } else { ret = !!regval; } -unlock: - mutex_unlock(&data->update_lock); return ret; } @@ -1225,18 +1220,19 @@ static ssize_t pmbus_show_sensor(struct device *dev, struct device_attribute *devattr, char *buf) { struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_sensor *sensor = to_pmbus_sensor(devattr); + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pmbus_sensor *sensor = to_pmbus_sensor(attr); struct pmbus_data *data = i2c_get_clientdata(client); - ssize_t ret; + s64 val; - mutex_lock(&data->update_lock); - pmbus_update_sensor_data(client, sensor); - if (sensor->data < 0) - ret = sensor->data; - else - ret = sysfs_emit(buf, "%lld\n", pmbus_reg2data(data, sensor)); - mutex_unlock(&data->update_lock); - return ret; + scoped_guard(pmbus_lock, client) { + pmbus_update_sensor_data(client, sensor); + if (sensor->data < 0) + return sensor->data; + val = pmbus_reg2data(data, sensor); + } + + return sysfs_emit(buf, "%lld\n", val); } static ssize_t pmbus_set_sensor(struct device *dev, @@ -1245,8 +1241,8 @@ static ssize_t pmbus_set_sensor(struct device *dev, { struct i2c_client *client = to_i2c_client(dev->parent); struct pmbus_data *data = i2c_get_clientdata(client); - struct pmbus_sensor *sensor = to_pmbus_sensor(devattr); - ssize_t rv = count; + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pmbus_sensor *sensor = to_pmbus_sensor(attr); s64 val; int ret; u16 regval; @@ -1254,21 +1250,22 @@ static ssize_t pmbus_set_sensor(struct device *dev, if (kstrtos64(buf, 10, &val) < 0) return -EINVAL; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + regval = pmbus_data2reg(data, sensor, val); ret = _pmbus_write_word_data(client, sensor->page, sensor->reg, regval); if (ret < 0) - rv = ret; - else - sensor->data = -ENODATA; - mutex_unlock(&data->update_lock); - return rv; + return ret; + + sensor->data = -ENODATA; + return count; } static ssize_t pmbus_show_label(struct device *dev, struct device_attribute *da, char *buf) { - struct pmbus_label *label = to_pmbus_label(da); + struct sensor_device_attribute *attr = to_sensor_dev_attr(da); + struct pmbus_label *label = to_pmbus_label(attr); return sysfs_emit(buf, "%s\n", label->label); } @@ -1364,7 +1361,7 @@ static int pmbus_thermal_get_temp(struct thermal_zone_device *tz, int *temp) struct pmbus_data *pmbus_data = tdata->pmbus_data; struct i2c_client *client = to_i2c_client(pmbus_data->dev); struct device *dev = pmbus_data->hwmon_dev; - int ret = 0; + int _temp; if (!dev) { /* May not even get to hwmon yet */ @@ -1372,15 +1369,15 @@ static int pmbus_thermal_get_temp(struct thermal_zone_device *tz, int *temp) return 0; } - mutex_lock(&pmbus_data->update_lock); - pmbus_update_sensor_data(client, sensor); - if (sensor->data < 0) - ret = sensor->data; - else - *temp = (int)pmbus_reg2data(pmbus_data, sensor); - mutex_unlock(&pmbus_data->update_lock); + scoped_guard(pmbus_lock, client) { + pmbus_update_sensor_data(client, sensor); + if (sensor->data < 0) + return sensor->data; + _temp = (int)pmbus_reg2data(pmbus_data, sensor); + } - return ret; + *temp = _temp; + return 0; } static const struct thermal_zone_device_ops pmbus_thermal_ops = { @@ -1421,8 +1418,8 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, bool update, bool readonly, bool writeonly, bool convert) { + struct sensor_device_attribute *a; struct pmbus_sensor *sensor; - struct device_attribute *a; sensor = devm_kzalloc(data->dev, sizeof(*sensor), GFP_KERNEL); if (!sensor) @@ -1446,12 +1443,11 @@ static struct pmbus_sensor *pmbus_add_sensor(struct pmbus_data *data, sensor->update = update; sensor->convert = convert; sensor->data = -ENODATA; - pmbus_dev_attr_init(a, sensor->name, - readonly ? 0444 : 0644, - writeonly ? pmbus_show_zero : pmbus_show_sensor, - pmbus_set_sensor); + pmbus_attr_init(a, sensor->name, readonly ? 0444 : 0644, + writeonly ? pmbus_show_zero : pmbus_show_sensor, + pmbus_set_sensor, -1); - if (pmbus_add_attribute(data, &a->attr)) + if (pmbus_add_attribute(data, &a->dev_attr.attr)) return NULL; sensor->next = data->sensors; @@ -1468,8 +1464,8 @@ static int pmbus_add_label(struct pmbus_data *data, const char *name, int seq, const char *lstring, int index, int phase) { + struct sensor_device_attribute *a; struct pmbus_label *label; - struct device_attribute *a; label = devm_kzalloc(data->dev, sizeof(*label), GFP_KERNEL); if (!label) @@ -1493,8 +1489,8 @@ static int pmbus_add_label(struct pmbus_data *data, lstring, index, phase); } - pmbus_dev_attr_init(a, label->name, 0444, pmbus_show_label, NULL); - return pmbus_add_attribute(data, &a->attr); + pmbus_attr_init(a, label->name, 0444, pmbus_show_label, NULL, -1); + return pmbus_add_attribute(data, &a->dev_attr.attr); } /* @@ -2382,7 +2378,7 @@ struct pmbus_samples_attr { struct pmbus_samples_reg { int page; struct pmbus_samples_attr *attr; - struct device_attribute dev_attr; + struct sensor_device_attribute attribute; }; static struct pmbus_samples_attr pmbus_samples_registers[] = { @@ -2404,21 +2400,21 @@ static struct pmbus_samples_attr pmbus_samples_registers[] = { } }; -#define to_samples_reg(x) container_of(x, struct pmbus_samples_reg, dev_attr) +#define to_samples_reg(x) container_of(x, struct pmbus_samples_reg, attribute) static ssize_t pmbus_show_samples(struct device *dev, struct device_attribute *devattr, char *buf) { int val; struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_samples_reg *reg = to_samples_reg(devattr); - struct pmbus_data *data = i2c_get_clientdata(client); + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pmbus_samples_reg *reg = to_samples_reg(attr); - mutex_lock(&data->update_lock); - val = _pmbus_read_word_data(client, reg->page, 0xff, reg->attr->reg); - mutex_unlock(&data->update_lock); - if (val < 0) - return val; + scoped_guard(pmbus_lock, client) { + val = _pmbus_read_word_data(client, reg->page, 0xff, reg->attr->reg); + if (val < 0) + return val; + } return sysfs_emit(buf, "%d\n", val); } @@ -2430,15 +2426,15 @@ static ssize_t pmbus_set_samples(struct device *dev, int ret; long val; struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_samples_reg *reg = to_samples_reg(devattr); - struct pmbus_data *data = i2c_get_clientdata(client); + struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr); + struct pmbus_samples_reg *reg = to_samples_reg(attr); if (kstrtol(buf, 0, &val) < 0) return -EINVAL; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + ret = _pmbus_write_word_data(client, reg->page, reg->attr->reg, val); - mutex_unlock(&data->update_lock); return ret ? : count; } @@ -2446,6 +2442,7 @@ static ssize_t pmbus_set_samples(struct device *dev, static int pmbus_add_samples_attr(struct pmbus_data *data, int page, struct pmbus_samples_attr *attr) { + struct sensor_device_attribute *a; struct pmbus_samples_reg *reg; reg = devm_kzalloc(data->dev, sizeof(*reg), GFP_KERNEL); @@ -2455,10 +2452,12 @@ static int pmbus_add_samples_attr(struct pmbus_data *data, int page, reg->attr = attr; reg->page = page; - pmbus_dev_attr_init(®->dev_attr, attr->name, 0644, - pmbus_show_samples, pmbus_set_samples); + a = ®->attribute; - return pmbus_add_attribute(data, ®->dev_attr.attr); + pmbus_attr_init(a, attr->name, 0644, + pmbus_show_samples, pmbus_set_samples, -1); + + return pmbus_add_attribute(data, &a->dev_attr.attr); } static int pmbus_add_samples_attributes(struct i2c_client *client, @@ -2950,14 +2949,9 @@ static int _pmbus_is_enabled(struct i2c_client *client, u8 page) static int __maybe_unused pmbus_is_enabled(struct i2c_client *client, u8 page) { - struct pmbus_data *data = i2c_get_clientdata(client); - int ret; + guard(pmbus_lock)(client); - mutex_lock(&data->update_lock); - ret = _pmbus_is_enabled(client, page); - mutex_unlock(&data->update_lock); - - return ret; + return _pmbus_is_enabled(client, page); } #define to_dev_attr(_dev_attr) \ @@ -2971,14 +2965,21 @@ static void pmbus_notify(struct pmbus_data *data, int page, int reg, int flags) struct device_attribute *da = to_dev_attr(data->group.attrs[i]); struct sensor_device_attribute *attr = to_sensor_dev_attr(da); int index = attr->index; - u16 smask = pb_index_to_mask(index); - u8 spage = pb_index_to_page(index); - u16 sreg = pb_index_to_reg(index); + u16 smask, sreg; + u8 spage; + + if (index == -1) + continue; + + smask = pb_index_to_mask(index); + spage = pb_index_to_page(index); + sreg = pb_index_to_reg(index); if (reg == sreg && page == spage && (smask & flags)) { dev_dbg(data->dev, "sysfs notify: %s", da->attr.name); - sysfs_notify(&data->dev->kobj, NULL, da->attr.name); - kobject_uevent(&data->dev->kobj, KOBJ_CHANGE); + sysfs_notify(&data->hwmon_dev->kobj, NULL, + da->attr.name); + kobject_uevent(&data->hwmon_dev->kobj, KOBJ_CHANGE); flags &= ~smask; } @@ -2987,14 +2988,13 @@ static void pmbus_notify(struct pmbus_data *data, int page, int reg, int flags) } } -static int _pmbus_get_flags(struct pmbus_data *data, u8 page, unsigned int *flags, +static int _pmbus_get_flags(struct i2c_client *client, u8 page, unsigned int *flags, unsigned int *event, bool notify) { + struct pmbus_data *data = i2c_get_clientdata(client); int i, status; const struct pmbus_status_category *cat; const struct pmbus_status_assoc *bit; - struct device *dev = data->dev; - struct i2c_client *client = to_i2c_client(dev); int func = data->info->func[page]; *flags = 0; @@ -3070,16 +3070,12 @@ static int _pmbus_get_flags(struct pmbus_data *data, u8 page, unsigned int *flag return 0; } -static int __maybe_unused pmbus_get_flags(struct pmbus_data *data, u8 page, unsigned int *flags, +static int __maybe_unused pmbus_get_flags(struct i2c_client *client, u8 page, unsigned int *flags, unsigned int *event, bool notify) { - int ret; + guard(pmbus_lock)(client); - mutex_lock(&data->update_lock); - ret = _pmbus_get_flags(data, page, flags, event, notify); - mutex_unlock(&data->update_lock); - - return ret; + return _pmbus_get_flags(client, page, flags, event, notify); } #if IS_ENABLED(CONFIG_REGULATOR) @@ -3095,17 +3091,13 @@ static int _pmbus_regulator_on_off(struct regulator_dev *rdev, bool enable) { struct device *dev = rdev_get_dev(rdev); struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_data *data = i2c_get_clientdata(client); u8 page = rdev_get_id(rdev); - int ret; - mutex_lock(&data->update_lock); - ret = pmbus_update_byte_data(client, page, PMBUS_OPERATION, - PB_OPERATION_CONTROL_ON, - enable ? PB_OPERATION_CONTROL_ON : 0); - mutex_unlock(&data->update_lock); + guard(pmbus_lock)(client); - return ret; + return pmbus_update_byte_data(client, page, PMBUS_OPERATION, + PB_OPERATION_CONTROL_ON, + enable ? PB_OPERATION_CONTROL_ON : 0); } static int pmbus_regulator_enable(struct regulator_dev *rdev) @@ -3122,54 +3114,41 @@ static int pmbus_regulator_get_error_flags(struct regulator_dev *rdev, unsigned { struct device *dev = rdev_get_dev(rdev); struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_data *data = i2c_get_clientdata(client); int event; - return pmbus_get_flags(data, rdev_get_id(rdev), flags, &event, false); + return pmbus_get_flags(client, rdev_get_id(rdev), flags, &event, false); } static int pmbus_regulator_get_status(struct regulator_dev *rdev) { struct device *dev = rdev_get_dev(rdev); struct i2c_client *client = to_i2c_client(dev->parent); - struct pmbus_data *data = i2c_get_clientdata(client); u8 page = rdev_get_id(rdev); int status, ret; int event; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + status = pmbus_get_status(client, page, PMBUS_STATUS_WORD); - if (status < 0) { - ret = status; - goto unlock; - } + if (status < 0) + return status; - if (status & PB_STATUS_OFF) { - ret = REGULATOR_STATUS_OFF; - goto unlock; - } + if (status & PB_STATUS_OFF) + return REGULATOR_STATUS_OFF; /* If regulator is ON & reports power good then return ON */ - if (!(status & PB_STATUS_POWER_GOOD_N)) { - ret = REGULATOR_STATUS_ON; - goto unlock; - } + if (!(status & PB_STATUS_POWER_GOOD_N)) + return REGULATOR_STATUS_ON; - ret = _pmbus_get_flags(data, rdev_get_id(rdev), &status, &event, false); + ret = _pmbus_get_flags(client, rdev_get_id(rdev), &status, &event, false); if (ret) - goto unlock; + return ret; if (status & (REGULATOR_ERROR_UNDER_VOLTAGE | REGULATOR_ERROR_OVER_CURRENT | - REGULATOR_ERROR_REGULATION_OUT | REGULATOR_ERROR_FAIL | REGULATOR_ERROR_OVER_TEMP)) { - ret = REGULATOR_STATUS_ERROR; - goto unlock; - } + REGULATOR_ERROR_REGULATION_OUT | REGULATOR_ERROR_FAIL | REGULATOR_ERROR_OVER_TEMP)) + return REGULATOR_STATUS_ERROR; - ret = REGULATOR_STATUS_UNDEFINED; - -unlock: - mutex_unlock(&data->update_lock); - return ret; + return REGULATOR_STATUS_UNDEFINED; } static int pmbus_regulator_get_low_margin(struct i2c_client *client, int page) @@ -3234,19 +3213,16 @@ static int pmbus_regulator_get_voltage(struct regulator_dev *rdev) .class = PSC_VOLTAGE_OUT, .convert = true, }; - int ret; + int voltage; - mutex_lock(&data->update_lock); - s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_READ_VOUT); - if (s.data < 0) { - ret = s.data; - goto unlock; + scoped_guard(pmbus_lock, client) { + s.data = _pmbus_read_word_data(client, s.page, 0xff, PMBUS_READ_VOUT); + if (s.data < 0) + return s.data; + voltage = (int)pmbus_reg2data(data, &s); } - ret = (int)pmbus_reg2data(data, &s) * 1000; /* unit is uV */ -unlock: - mutex_unlock(&data->update_lock); - return ret; + return voltage * 1000; /* unit is uV */ } static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, @@ -3263,22 +3239,18 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, }; int val = DIV_ROUND_CLOSEST(min_uv, 1000); /* convert to mV */ int low, high; - int ret; *selector = 0; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + low = pmbus_regulator_get_low_margin(client, s.page); - if (low < 0) { - ret = low; - goto unlock; - } + if (low < 0) + return low; high = pmbus_regulator_get_high_margin(client, s.page); - if (high < 0) { - ret = high; - goto unlock; - } + if (high < 0) + return high; /* Make sure we are within margins */ if (low > val) @@ -3288,10 +3260,7 @@ static int pmbus_regulator_set_voltage(struct regulator_dev *rdev, int min_uv, val = pmbus_data2reg(data, &s, val); - ret = _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); -unlock: - mutex_unlock(&data->update_lock); - return ret; + return _pmbus_write_word_data(client, s.page, PMBUS_VOUT_COMMAND, (u16)val); } static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, @@ -3301,7 +3270,6 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, struct i2c_client *client = to_i2c_client(dev->parent); struct pmbus_data *data = i2c_get_clientdata(client); int val, low, high; - int ret; if (data->flags & PMBUS_VOUT_PROTECTED) return 0; @@ -3314,29 +3282,20 @@ static int pmbus_regulator_list_voltage(struct regulator_dev *rdev, val = DIV_ROUND_CLOSEST(rdev->desc->min_uV + (rdev->desc->uV_step * selector), 1000); /* convert to mV */ - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); low = pmbus_regulator_get_low_margin(client, rdev_get_id(rdev)); - if (low < 0) { - ret = low; - goto unlock; - } + if (low < 0) + return low; high = pmbus_regulator_get_high_margin(client, rdev_get_id(rdev)); - if (high < 0) { - ret = high; - goto unlock; - } + if (high < 0) + return high; - if (val >= low && val <= high) { - ret = val * 1000; /* unit is uV */ - goto unlock; - } + if (val >= low && val <= high) + return val * 1000; /* unit is uV */ - ret = 0; -unlock: - mutex_unlock(&data->update_lock); - return ret; + return 0; } const struct regulator_ops pmbus_regulator_ops = { @@ -3477,16 +3436,16 @@ static irqreturn_t pmbus_fault_handler(int irq, void *pdata) struct i2c_client *client = to_i2c_client(data->dev); int i, status, event; - mutex_lock(&data->update_lock); + guard(pmbus_lock)(client); + for (i = 0; i < data->info->pages; i++) { - _pmbus_get_flags(data, i, &status, &event, true); + _pmbus_get_flags(client, i, &status, &event, true); if (event) pmbus_regulator_notify(data, i, event); } pmbus_clear_faults(client); - mutex_unlock(&data->update_lock); return IRQ_HANDLED; } @@ -3542,15 +3501,13 @@ static struct dentry *pmbus_debugfs_dir; /* pmbus debugfs directory */ static int pmbus_debugfs_get(void *data, u64 *val) { - int rc; struct pmbus_debugfs_entry *entry = data; - struct pmbus_data *pdata = i2c_get_clientdata(entry->client); + struct i2c_client *client = entry->client; + int rc; - rc = mutex_lock_interruptible(&pdata->update_lock); - if (rc) - return rc; - rc = _pmbus_read_byte_data(entry->client, entry->page, entry->reg); - mutex_unlock(&pdata->update_lock); + guard(pmbus_lock)(client); + + rc = _pmbus_read_byte_data(client, entry->page, entry->reg); if (rc < 0) return rc; @@ -3563,15 +3520,14 @@ DEFINE_DEBUGFS_ATTRIBUTE(pmbus_debugfs_ops, pmbus_debugfs_get, NULL, static int pmbus_debugfs_get_status(void *data, u64 *val) { - int rc; struct pmbus_debugfs_entry *entry = data; - struct pmbus_data *pdata = i2c_get_clientdata(entry->client); + struct i2c_client *client = entry->client; + struct pmbus_data *pdata = i2c_get_clientdata(client); + int rc; - rc = mutex_lock_interruptible(&pdata->update_lock); - if (rc) - return rc; - rc = pdata->read_status(entry->client, entry->page); - mutex_unlock(&pdata->update_lock); + guard(pmbus_lock)(client); + + rc = pdata->read_status(client, entry->page); if (rc < 0) return rc; @@ -3587,17 +3543,14 @@ static ssize_t pmbus_debugfs_block_read(struct file *file, char __user *buf, { int rc; struct pmbus_debugfs_entry *entry = file->private_data; - struct pmbus_data *pdata = i2c_get_clientdata(entry->client); + struct i2c_client *client = entry->client; char data[I2C_SMBUS_BLOCK_MAX + 2] = { 0 }; - rc = mutex_lock_interruptible(&pdata->update_lock); - if (rc) - return rc; - rc = pmbus_read_block_data(entry->client, entry->page, entry->reg, - data); - mutex_unlock(&pdata->update_lock); - if (rc < 0) - return rc; + scoped_guard(pmbus_lock, client) { + rc = pmbus_read_block_data(client, entry->page, entry->reg, data); + if (rc < 0) + return rc; + } /* Add newline at the end of a read data */ data[rc] = '\n'; diff --git a/drivers/hwmon/raspberrypi-hwmon.c b/drivers/hwmon/raspberrypi-hwmon.c index a2938881ccd2f..b0f0f50986b9a 100644 --- a/drivers/hwmon/raspberrypi-hwmon.c +++ b/drivers/hwmon/raspberrypi-hwmon.c @@ -7,7 +7,6 @@ * Copyright (C) 2018 Stefan Wahren */ #include -#include #include #include #include @@ -72,6 +71,13 @@ static void get_values_poll(struct work_struct *work) schedule_delayed_work(&data->get_values_poll_work, 2 * HZ); } +static void rpi_hwmon_cancel_poll_work(void *res) +{ + struct rpi_hwmon_data *data = res; + + disable_delayed_work_sync(&data->get_values_poll_work); +} + static int rpi_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val) { @@ -117,8 +123,8 @@ static int rpi_hwmon_probe(struct platform_device *pdev) if (IS_ERR(data->hwmon_dev)) return PTR_ERR(data->hwmon_dev); - ret = devm_delayed_work_autocancel(dev, &data->get_values_poll_work, - get_values_poll); + INIT_DELAYED_WORK(&data->get_values_poll_work, get_values_poll); + ret = devm_add_action_or_reset(dev, rpi_hwmon_cancel_poll_work, data); if (ret) return ret; platform_set_drvdata(pdev, data); diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c index f7116b4d8b796..d87b81f5769d3 100644 --- a/drivers/hwmon/sht3x.c +++ b/drivers/hwmon/sht3x.c @@ -21,6 +21,7 @@ #include #include #include +#include /* commands (high repeatability mode) */ static const unsigned char sht3x_cmd_measure_single_hpm[] = { 0x24, 0x00 }; @@ -276,9 +277,9 @@ static struct sht3x_data *sht3x_update_client(struct device *dev) if (ret) goto out; - val = be16_to_cpup((__be16 *)buf); + val = get_unaligned_be16(buf); data->temperature = sht3x_extract_temperature(val); - val = be16_to_cpup((__be16 *)(buf + 3)); + val = get_unaligned_be16(buf + 3); data->humidity = sht3x_extract_humidity(val); data->last_update = jiffies; } @@ -336,7 +337,7 @@ static int limits_update(struct sht3x_data *data) if (ret) return ret; - raw = be16_to_cpup((__be16 *)buffer); + raw = get_unaligned_be16(buffer); temperature = sht3x_extract_temperature((raw & 0x01ff) << 7); humidity = sht3x_extract_humidity(raw & 0xfe00); data->temperature_limits[index] = temperature; @@ -389,7 +390,7 @@ static size_t limit_write(struct device *dev, raw = ((u32)(temperature + 45000) * 24543) >> (16 + 7); raw |= ((humidity * 42950) >> 16) & 0xfe00; - *((__be16 *)position) = cpu_to_be16(raw); + put_unaligned_be16(raw, position); position += SHT3X_WORD_LEN; *position = crc8(sht3x_crc8_table, position - SHT3X_WORD_LEN, diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c index 6c9b776237c23..5a76361bb1350 100644 --- a/drivers/hwmon/sht4x.c +++ b/drivers/hwmon/sht4x.c @@ -55,7 +55,6 @@ DECLARE_CRC8_TABLE(sht4x_crc8_table); /** * struct sht4x_data - All the data required to operate an SHT4X chip * @client: the i2c client associated with the SHT4X - * @lock: a mutex that is used to prevent parallel access to the i2c client * @heating_complete: the time that the last heating finished * @data_pending: true if and only if there are measurements to retrieve after heating * @heater_power: the power at which the heater will be started @@ -68,7 +67,6 @@ DECLARE_CRC8_TABLE(sht4x_crc8_table); */ struct sht4x_data { struct i2c_client *client; - struct mutex lock; /* atomic read data updates */ unsigned long heating_complete; /* in jiffies */ bool data_pending; u32 heater_power; /* in milli-watts */ @@ -87,7 +85,7 @@ struct sht4x_data { */ static int sht4x_read_values(struct sht4x_data *data) { - int ret = 0; + int ret; u16 t_ticks, rh_ticks; unsigned long next_update; struct i2c_client *client = data->client; @@ -96,8 +94,6 @@ static int sht4x_read_values(struct sht4x_data *data) u8 raw_data[SHT4X_RESPONSE_LENGTH]; unsigned long curr_jiffies; - mutex_lock(&data->lock); - curr_jiffies = jiffies; if (time_before(curr_jiffies, data->heating_complete)) msleep(jiffies_to_msecs(data->heating_complete - curr_jiffies)); @@ -110,11 +106,11 @@ static int sht4x_read_values(struct sht4x_data *data) msecs_to_jiffies(data->update_interval); if (data->valid && time_before_eq(jiffies, next_update)) - goto unlock; + return 0; ret = i2c_master_send(client, cmd, SHT4X_CMD_LEN); if (ret < 0) - goto unlock; + return ret; usleep_range(SHT4X_MEAS_DELAY_HPM, SHT4X_MEAS_DELAY_HPM + SHT4X_DELAY_EXTRA); } @@ -123,7 +119,7 @@ static int sht4x_read_values(struct sht4x_data *data) if (ret != SHT4X_RESPONSE_LENGTH) { if (ret >= 0) ret = -ENODATA; - goto unlock; + return ret; } t_ticks = raw_data[0] << 8 | raw_data[1]; @@ -132,26 +128,20 @@ static int sht4x_read_values(struct sht4x_data *data) crc = crc8(sht4x_crc8_table, &raw_data[0], SHT4X_WORD_LEN, CRC8_INIT_VALUE); if (crc != raw_data[2]) { dev_err(&client->dev, "data integrity check failed\n"); - ret = -EIO; - goto unlock; + return -EIO; } crc = crc8(sht4x_crc8_table, &raw_data[3], SHT4X_WORD_LEN, CRC8_INIT_VALUE); if (crc != raw_data[5]) { dev_err(&client->dev, "data integrity check failed\n"); - ret = -EIO; - goto unlock; + return -EIO; } data->temperature = ((21875 * (int32_t)t_ticks) >> 13) - 45000; data->humidity = ((15625 * (int32_t)rh_ticks) >> 13) - 6000; data->last_updated = jiffies; data->valid = true; - ret = 0; - -unlock: - mutex_unlock(&data->lock); - return ret; + return 0; } static ssize_t sht4x_interval_write(struct sht4x_data *data, long val) @@ -287,22 +277,18 @@ static ssize_t heater_enable_store(struct device *dev, heating_time_bound = 1100; } - mutex_lock(&data->lock); + guard(hwmon_lock)(dev); - if (time_before(jiffies, data->heating_complete)) { - ret = -EBUSY; - goto unlock; - } + if (time_before(jiffies, data->heating_complete)) + return -EBUSY; ret = i2c_master_send(data->client, &cmd, SHT4X_CMD_LEN); if (ret < 0) - goto unlock; + return ret; data->heating_complete = jiffies + msecs_to_jiffies(heating_time_bound); data->data_pending = true; -unlock: - mutex_unlock(&data->lock); - return ret; + return count; } static ssize_t heater_power_show(struct device *dev, @@ -330,6 +316,8 @@ static ssize_t heater_power_store(struct device *dev, if (power != 20 && power != 110 && power != 200) return -EINVAL; + guard(hwmon_lock)(dev); + data->heater_power = power; return count; @@ -360,6 +348,8 @@ static ssize_t heater_time_store(struct device *dev, if (time != 100 && time != 1000) return -EINVAL; + guard(hwmon_lock)(dev); + data->heater_time = time; return count; @@ -422,8 +412,6 @@ static int sht4x_probe(struct i2c_client *client) data->heater_time = 1000; data->heating_complete = jiffies; - mutex_init(&data->lock); - crc8_populate_msb(sht4x_crc8_table, SHT4X_CRC8_POLYNOMIAL); ret = i2c_master_send(client, cmd, SHT4X_CMD_LEN); diff --git a/drivers/hwspinlock/hwspinlock_core.c b/drivers/hwspinlock/hwspinlock_core.c index cc8e952a67727..a509b73da190d 100644 --- a/drivers/hwspinlock/hwspinlock_core.c +++ b/drivers/hwspinlock/hwspinlock_core.c @@ -472,7 +472,7 @@ static int hwspin_lock_register_single(struct hwspinlock *hwlock, int id) out: mutex_unlock(&hwspinlock_tree_lock); - return 0; + return ret; } static struct hwspinlock *hwspin_lock_unregister_single(unsigned int id) diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c index 911a7b6d4867b..d57000626c060 100644 --- a/drivers/hwtracing/coresight/coresight-core.c +++ b/drivers/hwtracing/coresight/coresight-core.c @@ -79,7 +79,7 @@ struct coresight_device *coresight_get_percpu_sink(int cpu) } EXPORT_SYMBOL_GPL(coresight_get_percpu_sink); -static struct coresight_device *coresight_get_source(struct coresight_path *path) +struct coresight_device *coresight_get_source(struct coresight_path *path) { struct coresight_device *csdev; @@ -378,19 +378,12 @@ static void coresight_disable_helpers(struct coresight_device *csdev, void *data } /* - * Helper function to call source_ops(csdev)->disable and also disable the - * helpers. - * - * There is an imbalance between coresight_enable_path() and - * coresight_disable_path(). Enabling also enables the source's helpers as part - * of the path, but disabling always skips the first item in the path (which is - * the source), so sources and their helpers don't get disabled as part of that - * function and we need the extra step here. + * coresight_disable_source() only disables the source, but do nothing for + * the associated helpers, which are controlled as part of the path. */ void coresight_disable_source(struct coresight_device *csdev, void *data) { source_ops(csdev)->disable(csdev, data); - coresight_disable_helpers(csdev, NULL); } EXPORT_SYMBOL_GPL(coresight_disable_source); @@ -417,9 +410,9 @@ int coresight_resume_source(struct coresight_device *csdev) EXPORT_SYMBOL_GPL(coresight_resume_source); /* - * coresight_disable_path_from : Disable components in the given path beyond - * @nd in the list. If @nd is NULL, all the components, except the SOURCE are - * disabled. + * coresight_disable_path_from : Disable components in the given path starting + * from @nd in the list. If @nd is NULL, all the components, except the SOURCE + * are disabled. */ static void coresight_disable_path_from(struct coresight_path *path, struct coresight_node *nd) @@ -430,7 +423,7 @@ static void coresight_disable_path_from(struct coresight_path *path, if (!nd) nd = list_first_entry(&path->path_list, struct coresight_node, link); - list_for_each_entry_continue(nd, &path->path_list, link) { + list_for_each_entry_from(nd, &path->path_list, link) { csdev = nd->csdev; type = csdev->type; @@ -450,12 +443,6 @@ static void coresight_disable_path_from(struct coresight_path *path, coresight_disable_sink(csdev); break; case CORESIGHT_DEV_TYPE_SOURCE: - /* - * We skip the first node in the path assuming that it - * is the source. So we don't expect a source device in - * the middle of a path. - */ - WARN_ON(1); break; case CORESIGHT_DEV_TYPE_LINK: parent = list_prev_entry(nd, link)->csdev; @@ -561,6 +548,8 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode, err_disable_helpers: coresight_disable_helpers(csdev, path); err_disable_path: + /* Fetch the previous node, the last successfully enabled one */ + nd = list_next_entry(nd, link); coresight_disable_path_from(path, nd); goto out; } diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c index accf101779de8..60f4fde3b398b 100644 --- a/drivers/hwtracing/coresight/coresight-etm-perf.c +++ b/drivers/hwtracing/coresight/coresight-etm-perf.c @@ -312,6 +312,35 @@ static bool sinks_compatible(struct coresight_device *a, (sink_ops(a) == sink_ops(b)); } +/* + * This helper is used for fetching the path pointer via the ctxt. + * + * Perf event callbacks run on the same CPU in atomic context, but AUX pause + * and resume may run in NMI context and preempt other callbacks. Since the + * event stop callback clears ctxt->event_data before the data is released, + * AUX pause/resume will either observe a NULL pointer and stop fetching the + * path pointer, or safely access event_data and the path, as the data has + * not yet been freed. + */ +static struct coresight_path *etm_event_get_ctxt_path(struct etm_ctxt *ctxt) +{ + struct etm_event_data *event_data; + struct coresight_path *path; + + if (!ctxt) + return NULL; + + event_data = READ_ONCE(ctxt->event_data); + if (!event_data) + return NULL; + + path = etm_event_cpu_path(event_data, smp_processor_id()); + if (!path) + return NULL; + + return path; +} + static void *etm_setup_aux(struct perf_event *event, void **pages, int nr_pages, bool overwrite) { @@ -463,13 +492,23 @@ static void *etm_setup_aux(struct perf_event *event, void **pages, goto out; } -static int etm_event_resume(struct coresight_device *csdev, - struct etm_ctxt *ctxt) +static int etm_event_resume(struct coresight_path *path) { - if (!ctxt->event_data) + struct coresight_device *source; + int ret; + + if (!path) return 0; - return coresight_resume_source(csdev); + source = coresight_get_source(path); + if (!source) + return 0; + + ret = coresight_resume_source(source); + if (ret < 0) + dev_err(&source->dev, "Failed to resume ETM event.\n"); + + return ret; } static void etm_event_start(struct perf_event *event, int flags) @@ -478,23 +517,19 @@ static void etm_event_start(struct perf_event *event, int flags) struct etm_event_data *event_data; struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; - struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); + struct coresight_device *source, *sink; struct coresight_path *path; u64 hw_id; - if (!csdev) - goto fail; - if (flags & PERF_EF_RESUME) { - if (etm_event_resume(csdev, ctxt) < 0) { - dev_err(&csdev->dev, "Failed to resume ETM event.\n"); + path = etm_event_get_ctxt_path(ctxt); + if (etm_event_resume(path) < 0) goto fail; - } return; } /* Have we messed up our tracking ? */ - if (WARN_ON(ctxt->event_data)) + if (WARN_ON(READ_ONCE(ctxt->event_data))) goto fail; /* @@ -522,9 +557,10 @@ static void etm_event_start(struct perf_event *event, int flags) path = etm_event_cpu_path(event_data, cpu); path->handle = handle; - /* We need a sink, no need to continue without one */ + /* We need source and sink, no need to continue if any is not set */ + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (WARN_ON_ONCE(!sink)) + if (WARN_ON_ONCE(!source || !sink)) goto fail_end_stop; /* Nothing will happen without a path */ @@ -532,7 +568,7 @@ static void etm_event_start(struct perf_event *event, int flags) goto fail_end_stop; /* Finally enable the tracer */ - if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF, path)) + if (source_ops(source)->enable(source, event, CS_MODE_PERF, path)) goto fail_disable_path; /* @@ -556,7 +592,7 @@ static void etm_event_start(struct perf_event *event, int flags) /* Tell the perf core the event is alive */ event->hw.state = 0; /* Save the event_data for this ETM */ - ctxt->event_data = event_data; + WRITE_ONCE(ctxt->event_data, event_data); return; fail_disable_path: @@ -576,27 +612,26 @@ static void etm_event_start(struct perf_event *event, int flags) return; } -static void etm_event_pause(struct perf_event *event, - struct coresight_device *csdev, +static void etm_event_pause(struct coresight_path *path, + struct perf_event *event, struct etm_ctxt *ctxt) { - int cpu = smp_processor_id(); - struct coresight_device *sink; struct perf_output_handle *handle = &ctxt->handle; - struct coresight_path *path; + struct coresight_device *source, *sink; + struct etm_event_data *event_data; unsigned long size; - if (!ctxt->event_data) + if (!path) return; - /* Stop tracer */ - coresight_pause_source(csdev); - - path = etm_event_cpu_path(ctxt->event_data, cpu); + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (WARN_ON_ONCE(!sink)) + if (WARN_ON_ONCE(!source || !sink)) return; + /* Stop tracer */ + coresight_pause_source(source); + /* * The per CPU sink has own interrupt handling, it might have * race condition with updating buffer on AUX trace pause if @@ -612,8 +647,9 @@ static void etm_event_pause(struct perf_event *event, if (!sink_ops(sink)->update_buffer) return; + event_data = READ_ONCE(ctxt->event_data); size = sink_ops(sink)->update_buffer(sink, handle, - ctxt->event_data->snk_config); + event_data->snk_config); if (READ_ONCE(handle->event)) { if (!size) return; @@ -629,14 +665,14 @@ static void etm_event_stop(struct perf_event *event, int mode) { int cpu = smp_processor_id(); unsigned long size; - struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu); + struct coresight_device *source, *sink; struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt); struct perf_output_handle *handle = &ctxt->handle; + struct coresight_path *path = etm_event_get_ctxt_path(ctxt); struct etm_event_data *event_data; - struct coresight_path *path; if (mode & PERF_EF_PAUSE) - return etm_event_pause(event, csdev, ctxt); + return etm_event_pause(path, event, ctxt); /* * If we still have access to the event_data via handle, @@ -646,9 +682,9 @@ static void etm_event_stop(struct perf_event *event, int mode) WARN_ON(perf_get_aux(handle) != ctxt->event_data)) return; - event_data = ctxt->event_data; + event_data = READ_ONCE(ctxt->event_data); /* Clear the event_data as this ETM is stopping the trace. */ - ctxt->event_data = NULL; + WRITE_ONCE(ctxt->event_data, NULL); if (event->hw.state == PERF_HES_STOPPED) return; @@ -670,19 +706,13 @@ static void etm_event_stop(struct perf_event *event, int mode) return; } - if (!csdev) - return; - - path = etm_event_cpu_path(event_data, cpu); - if (!path) - return; - + source = coresight_get_source(path); sink = coresight_get_sink(path); - if (!sink) + if (!source || !sink) return; /* stop tracer */ - coresight_disable_source(csdev, event); + coresight_disable_source(source, event); /* tell the core */ event->hw.state = PERF_HES_STOPPED; diff --git a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c index 762109307b869..b3c67e96a82a4 100644 --- a/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm3x-sysfs.c @@ -717,26 +717,19 @@ static DEVICE_ATTR_RW(cntr_rld_event); static ssize_t cntr_val_show(struct device *dev, struct device_attribute *attr, char *buf) { - int i, ret = 0; u32 val; struct etm_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etm_config *config = &drvdata->config; if (!coresight_get_mode(drvdata->csdev)) { spin_lock(&drvdata->spinlock); - for (i = 0; i < drvdata->nr_cntr; i++) - ret += sprintf(buf, "counter %d: %x\n", - i, config->cntr_val[i]); + val = config->cntr_val[config->cntr_idx]; spin_unlock(&drvdata->spinlock); - return ret; - } - - for (i = 0; i < drvdata->nr_cntr; i++) { - val = etm_readl(drvdata, ETMCNTVRn(i)); - ret += sprintf(buf, "counter %d: %x\n", i, val); + } else { + val = etm_readl(drvdata, ETMCNTVRn(config->cntr_idx)); } - return ret; + return sysfs_emit(buf, "%#x\n", val); } static ssize_t cntr_val_store(struct device *dev, diff --git a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c index c302072b293a3..e1a59b4345052 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-cfg.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-cfg.c @@ -76,7 +76,7 @@ static int etm4_cfg_map_reg_offset(struct etmv4_drvdata *drvdata, } else if ((offset & GENMASK(11, 4)) == TRCSEQEVRn(0)) { /* sequencer state control registers */ idx = (offset & GENMASK(3, 0)) / 4; - if (idx < ETM_MAX_SEQ_STATES) { + if (idx < ETM_MAX_SEQ_TRANSITIONS) { reg_csdev->driver_regval = &drvcfg->seq_ctrl[idx]; err = 0; } diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c index 2271f5ea81214..522e128174467 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-core.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c @@ -92,7 +92,7 @@ static int etm4_probe_cpu(unsigned int cpu); static bool etm4x_sspcicrn_present(struct etmv4_drvdata *drvdata, int n) { return (n < drvdata->nr_ss_cmp) && - drvdata->nr_pe && + drvdata->nr_pe_cmp && (drvdata->config.ss_status[n] & TRCSSCSRn_PC); } @@ -541,7 +541,8 @@ static int etm4_enable_hw(struct etmv4_drvdata *drvdata) etm4x_relaxed_write32(csa, config->vissctlr, TRCVISSCTLR); if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, config->vipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, config->seq_ctrl[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { etm4x_relaxed_write32(csa, config->seq_rst, TRCSEQRSTEVR); @@ -640,18 +641,33 @@ static void etm4_enable_sysfs_smp_call(void *info) * TRCRSCTLR1 (always true) used to get the counter to decrement. From * there a resource selector is configured with the counter and the * timestamp control register to use the resource selector to trigger the - * event that will insert a timestamp packet in the stream. + * event that will insert a timestamp packet in the stream: + * + * +--------------+ + * | Resource 1 | fixed "always-true" resource + * +--------------+ + * | + * +------v-------+ + * | Counter x | (reload to 1 on underflow) + * +--------------+ + * | + * +------v--------------+ + * | Resource Selector y | (trigger on counter x == 0) + * +---------------------+ + * | + * +------v---------------+ + * | Timestamp Generator | (timestamp on resource y) + * +----------------------+ */ static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) { - int ctridx, ret = -EINVAL; - int counter, rselector; - u32 val = 0; + int ctridx; + int rselector; struct etmv4_config *config = &drvdata->config; /* No point in trying if we don't have at least one counter */ if (!drvdata->nr_cntr) - goto out; + return -EINVAL; /* Find a counter that hasn't been initialised */ for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++) @@ -661,15 +677,19 @@ static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) /* All the counters have been configured already, bail out */ if (ctridx == drvdata->nr_cntr) { pr_debug("%s: no available counter found\n", __func__); - ret = -ENOSPC; - goto out; + return -ENOSPC; } /* - * Searching for an available resource selector to use, starting at - * '2' since every implementation has at least 2 resource selector. - * ETMIDR4 gives the number of resource selector _pairs_, - * hence multiply by 2. + * Searching for an available resource selector to use, starting at '2' + * since resource 0 is the fixed 'always returns false' resource and 1 + * is the fixed 'always returns true' resource. See IHI0064H_b '7.3.64 + * TRCRSCTLRn, Resource Selection Control Registers, n=2-31'. If there + * are no resources, there would also be no counters so wouldn't get + * here. + * + * ETMIDR4 gives the number of resource selector _pairs_, hence multiply + * by 2. */ for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++) if (!config->res_ctrl[rselector]) @@ -678,13 +698,9 @@ static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) if (rselector == drvdata->nr_resource * 2) { pr_debug("%s: no available resource selector found\n", __func__); - ret = -ENOSPC; - goto out; + return -ENOSPC; } - /* Remember what counter we used */ - counter = 1 << ctridx; - /* * Initialise original and reload counter value to the smallest * possible value in order to get as much precision as we can. @@ -692,26 +708,41 @@ static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata) config->cntr_val[ctridx] = 1; config->cntrldvr[ctridx] = 1; - /* Set the trace counter control register */ - val = 0x1 << 16 | /* Bit 16, reload counter automatically */ - 0x0 << 7 | /* Select single resource selector */ - 0x1; /* Resource selector 1, i.e always true */ - - config->cntr_ctrl[ctridx] = val; - - val = 0x2 << 16 | /* Group 0b0010 - Counter and sequencers */ - counter << 0; /* Counter to use */ - - config->res_ctrl[rselector] = val; + /* + * Trace Counter Control Register TRCCNTCTLRn + * + * CNTCHAIN = 0, don't reload on the previous counter + * RLDSELF = true, reload counter automatically on underflow + * RLDEVENT = RES_SEL_FALSE (0), reload on single false resource (never reload) + * CNTEVENT = RES_SEL_TRUE (1), count single fixed 'always true' resource (always decrement) + */ + config->cntr_ctrl[ctridx] = TRCCNTCTLRn_RLDSELF | + FIELD_PREP(TRCCNTCTLRn_RLDEVENT_MASK, + etm4_res_sel_single(ETM4_RES_SEL_FALSE)) | + FIELD_PREP(TRCCNTCTLRn_CNTEVENT_MASK, + etm4_res_sel_single(ETM4_RES_SEL_TRUE)); - val = 0x0 << 7 | /* Select single resource selector */ - rselector; /* Resource selector */ + /* + * Resource Selection Control Register TRCRSCTLRn + * + * PAIRINV = 0, INV = 0, don't invert + * GROUP = 2, SELECT = ctridx, trigger when counter 'ctridx' reaches 0 + * + * Multiple counters can be selected, and each bit signifies a counter, + * so set bit 'ctridx' to select our counter. + */ + config->res_ctrl[rselector] = FIELD_PREP(TRCRSCTLRn_GROUP_MASK, 2) | + FIELD_PREP(TRCRSCTLRn_SELECT_MASK, 1 << ctridx); - config->ts_ctrl = val; + /* + * Global Timestamp Control Register TRCTSCTLR + * + * EVENT = generate timestamp on single resource 'rselector' + */ + config->ts_ctrl = FIELD_PREP(TRCTSCTLR_EVENT_MASK, + etm4_res_sel_single(rselector)); - ret = 0; -out: - return ret; + return 0; } static int etm4_parse_event_config(struct coresight_device *csdev, @@ -879,8 +910,10 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa cscfg_config_sysfs_get_active_cfg(&cfg_hash, &preset); if (cfg_hash) { ret = cscfg_csdev_enable_active_config(csdev, cfg_hash, preset); - if (ret) + if (ret) { + etm4_release_trace_id(drvdata); return ret; + } } raw_spin_lock(&drvdata->spinlock); @@ -1476,6 +1509,8 @@ static void etm4_init_arch_data(void *info) drvdata->lpoverride = (etmidr5 & TRCIDR5_LPOVERRIDE) && (!drvdata->skip_power_up); /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */ drvdata->nrseqstate = FIELD_GET(TRCIDR5_NUMSEQSTATE_MASK, etmidr5); + if (drvdata->nrseqstate) + drvdata->nr_seq_ctrls = ETM_MAX_SEQ_TRANSITIONS; /* NUMCNTR, bits[30:28] number of counters available for tracing */ drvdata->nr_cntr = FIELD_GET(TRCIDR5_NUMCNTR_MASK, etmidr5); @@ -1889,7 +1924,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata) if (drvdata->nr_pe_cmp) state->trcvipcssctlr = etm4x_read32(csa, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) state->trcseqevr[i] = etm4x_read32(csa, TRCSEQEVRn(i)); if (drvdata->nrseqstate) { @@ -2020,7 +2055,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata) if (drvdata->nr_pe_cmp) etm4x_relaxed_write32(csa, state->trcvipcssctlr, TRCVIPCSSCTLR); - for (i = 0; i < drvdata->nrseqstate - 1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) etm4x_relaxed_write32(csa, state->trcseqevr[i], TRCSEQEVRn(i)); if (drvdata->nrseqstate) { diff --git a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c index e9eeea6240d55..cc6cdd3ae29d5 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c +++ b/drivers/hwtracing/coresight/coresight-etm4x-sysfs.c @@ -223,7 +223,7 @@ static ssize_t reset_store(struct device *dev, config->vipcssctlr = 0x0; /* Disable seq events */ - for (i = 0; i < drvdata->nrseqstate-1; i++) + for (i = 0; i < drvdata->nr_seq_ctrls; i++) config->seq_ctrl[i] = 0x0; config->seq_rst = 0x0; config->seq_state = 0x0; @@ -1395,9 +1395,11 @@ static ssize_t seq_idx_store(struct device *dev, struct etmv4_drvdata *drvdata = dev_get_drvdata(dev->parent); struct etmv4_config *config = &drvdata->config; + if (!drvdata->nr_seq_ctrls) + return -ENOTSUPP; if (kstrtoul(buf, 16, &val)) return -EINVAL; - if (val >= drvdata->nrseqstate - 1) + if (val >= drvdata->nr_seq_ctrls) return -EINVAL; /* diff --git a/drivers/hwtracing/coresight/coresight-etm4x.h b/drivers/hwtracing/coresight/coresight-etm4x.h index 012c52fd19338..0489496babb33 100644 --- a/drivers/hwtracing/coresight/coresight-etm4x.h +++ b/drivers/hwtracing/coresight/coresight-etm4x.h @@ -225,6 +225,50 @@ #define TRCRSCTLRn_GROUP_MASK GENMASK(19, 16) #define TRCRSCTLRn_SELECT_MASK GENMASK(15, 0) +#define TRCCNTCTLRn_CNTCHAIN BIT(17) +#define TRCCNTCTLRn_RLDSELF BIT(16) +#define TRCCNTCTLRn_RLDEVENT_MASK GENMASK(15, 8) +#define TRCCNTCTLRn_CNTEVENT_MASK GENMASK(7, 0) + +#define TRCTSCTLR_EVENT_MASK GENMASK(7, 0) + +#define ETM4_RES_SEL_FALSE 0 /* Fixed function 'always false' resource selector */ +#define ETM4_RES_SEL_TRUE 1 /* Fixed function 'always true' resource selector */ + +#define ETM4_RES_SEL_SINGLE_MASK GENMASK(4, 0) +#define ETM4_RES_SEL_PAIR_MASK GENMASK(3, 0) +#define ETM4_RES_SEL_TYPE_PAIR BIT(7) + +/* + * Utilities for programming EVENT resource selectors, e.g. TRCCNTCTLRn_RLDEVENT. + * + * Resource selectors have a common format across registers: + * + * 7 6 5 4 0 + * +------+------+-------+ + * | TYPE | RES0 | SEL | + * +------+------+-------+ + * + * Where TYPE indicates whether the selector is for a single event or a pair. + * When TYPE is pair, SEL is 4 bits wide and using pair 0 is UNPREDICTABLE. + * Otherwise for single it's 5 bits wide. + */ +static inline u32 etm4_res_sel_single(u8 res_sel_idx) +{ + WARN_ON_ONCE(!FIELD_FIT(ETM4_RES_SEL_SINGLE_MASK, res_sel_idx)); + return FIELD_PREP(ETM4_RES_SEL_SINGLE_MASK, res_sel_idx); +} + +static inline u32 etm4_res_sel_pair(u8 res_sel_idx) +{ + if (__builtin_constant_p(res_sel_idx)) + BUILD_BUG_ON(res_sel_idx == 0); + WARN_ON_ONCE(!FIELD_FIT(ETM4_RES_SEL_PAIR_MASK, res_sel_idx) || + (res_sel_idx == 0)); + return FIELD_PREP(ETM4_RES_SEL_PAIR_MASK, res_sel_idx) | + ETM4_RES_SEL_TYPE_PAIR; +} + /* * System instructions to access ETM registers. * See ETMv4.4 spec ARM IHI0064F section 4.3.6 System instructions @@ -570,6 +614,7 @@ #define ETM_MAX_NR_PE 8 #define ETMv4_MAX_CNTR 4 #define ETM_MAX_SEQ_STATES 4 +#define ETM_MAX_SEQ_TRANSITIONS 3 #define ETM_MAX_EXT_INP_SEL 4 #define ETM_MAX_EXT_INP 256 #define ETM_MAX_EXT_OUT 4 @@ -824,8 +869,7 @@ struct etmv4_config { u32 eventctrl0; u32 eventctrl1; u32 stall_ctrl; - u32 ts_ctrl; - u32 syncfreq; + u32 ts_ctrl; /* TRCTSCTLR */ u32 ccctlr; u32 bb_ctrl; u32 vinst_ctrl; @@ -833,15 +877,16 @@ struct etmv4_config { u32 vissctlr; u32 vipcssctlr; u8 seq_idx; - u32 seq_ctrl[ETM_MAX_SEQ_STATES]; + u8 syncfreq; + u32 seq_ctrl[ETM_MAX_SEQ_TRANSITIONS]; u32 seq_rst; u32 seq_state; u8 cntr_idx; - u32 cntrldvr[ETMv4_MAX_CNTR]; - u32 cntr_ctrl[ETMv4_MAX_CNTR]; - u32 cntr_val[ETMv4_MAX_CNTR]; + u32 cntrldvr[ETMv4_MAX_CNTR]; /* TRCCNTRLDVRn */ + u32 cntr_ctrl[ETMv4_MAX_CNTR]; /* TRCCNTCTLRn */ + u32 cntr_val[ETMv4_MAX_CNTR]; /* TRCCNTVRn */ u8 res_idx; - u32 res_ctrl[ETM_MAX_RES_SEL]; + u32 res_ctrl[ETM_MAX_RES_SEL]; /* TRCRSCTLRn */ u8 ss_idx; u32 ss_ctrl[ETM_MAX_SS_CMP]; u32 ss_status[ETM_MAX_SS_CMP]; @@ -884,7 +929,7 @@ struct etmv4_save_state { u32 trcvissctlr; u32 trcvipcssctlr; - u32 trcseqevr[ETM_MAX_SEQ_STATES]; + u32 trcseqevr[ETM_MAX_SEQ_TRANSITIONS]; u32 trcseqrstevr; u32 trcseqstr; u32 trcextinselr; @@ -937,6 +982,7 @@ struct etmv4_save_state { * @numcidc: Number of contextID comparators. * @numvmidc: Number of VMID comparators. * @nrseqstate: The number of sequencer states that are implemented. + * @nr_seq_ctrls: The number of sequence state transition control registers. * @nr_event: Indicates how many events the trace unit support. * @nr_resource:The number of resource selection pairs available for tracing. * @nr_ss_cmp: Number of single-shot comparator controls that are available. @@ -1002,6 +1048,7 @@ struct etmv4_drvdata { u8 numextinsel; u8 numvmidc; u8 nrseqstate; + u8 nr_seq_ctrls; u8 nr_event; u8 nr_resource; u8 nr_ss_cmp; diff --git a/drivers/hwtracing/coresight/coresight-platform.c b/drivers/hwtracing/coresight/coresight-platform.c index 0db64c5f49959..6e57fc5018f47 100644 --- a/drivers/hwtracing/coresight/coresight-platform.c +++ b/drivers/hwtracing/coresight/coresight-platform.c @@ -45,9 +45,8 @@ coresight_add_out_conn(struct device *dev, } } - pdata->nr_outconns++; pdata->out_conns = - devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns, + devm_krealloc_array(dev, pdata->out_conns, pdata->nr_outconns + 1, sizeof(*pdata->out_conns), GFP_KERNEL); if (!pdata->out_conns) return ERR_PTR(-ENOMEM); @@ -63,7 +62,8 @@ coresight_add_out_conn(struct device *dev, * used right away. */ *conn = *new_conn; - pdata->out_conns[pdata->nr_outconns - 1] = conn; + pdata->out_conns[pdata->nr_outconns] = conn; + pdata->nr_outconns++; return conn; } EXPORT_SYMBOL_GPL(coresight_add_out_conn); @@ -86,13 +86,13 @@ int coresight_add_in_conn(struct coresight_connection *out_conn) return 0; } - pdata->nr_inconns++; pdata->in_conns = - devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns, + devm_krealloc_array(dev, pdata->in_conns, pdata->nr_inconns + 1, sizeof(*pdata->in_conns), GFP_KERNEL); if (!pdata->in_conns) return -ENOMEM; - pdata->in_conns[pdata->nr_inconns - 1] = out_conn; + pdata->in_conns[pdata->nr_inconns] = out_conn; + pdata->nr_inconns++; return 0; } EXPORT_SYMBOL_GPL(coresight_add_in_conn); diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h index bcc5db0d9c3c2..7f27ef2ab1619 100644 --- a/drivers/hwtracing/coresight/coresight-priv.h +++ b/drivers/hwtracing/coresight/coresight-priv.h @@ -250,6 +250,7 @@ void coresight_add_helper(struct coresight_device *csdev, void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev); struct coresight_device *coresight_get_percpu_sink(int cpu); +struct coresight_device *coresight_get_source(struct coresight_path *path); void coresight_disable_source(struct coresight_device *csdev, void *data); void coresight_pause_source(struct coresight_device *csdev); int coresight_resume_source(struct coresight_device *csdev); diff --git a/drivers/hwtracing/intel_th/core.c b/drivers/hwtracing/intel_th/core.c index e3a7ab112ea95..dc888a11c7a09 100644 --- a/drivers/hwtracing/intel_th/core.c +++ b/drivers/hwtracing/intel_th/core.c @@ -843,18 +843,8 @@ static int intel_th_output_open(struct inode *inode, struct file *file) return err; } -static int intel_th_output_release(struct inode *inode, struct file *file) -{ - struct intel_th_device *thdev = file->private_data; - - put_device(&thdev->dev); - - return 0; -} - static const struct file_operations intel_th_output_fops = { .open = intel_th_output_open, - .release = intel_th_output_release, .llseek = noop_llseek, }; diff --git a/drivers/hwtracing/intel_th/msu.c b/drivers/hwtracing/intel_th/msu.c index f3a13b300835d..d3170b1476de2 100644 --- a/drivers/hwtracing/intel_th/msu.c +++ b/drivers/hwtracing/intel_th/msu.c @@ -1490,8 +1490,10 @@ static int intel_th_msc_release(struct inode *inode, struct file *file) { struct msc_iter *iter = file->private_data; struct msc *msc = iter->msc; + struct intel_th_device *thdev = msc->thdev; msc_iter_remove(iter, msc); + put_device(&thdev->dev); return 0; } diff --git a/drivers/hwtracing/ptt/hisi_ptt.c b/drivers/hwtracing/ptt/hisi_ptt.c index 3090479a29798..3c3e9fa934183 100644 --- a/drivers/hwtracing/ptt/hisi_ptt.c +++ b/drivers/hwtracing/ptt/hisi_ptt.c @@ -171,13 +171,13 @@ static bool hisi_ptt_wait_trace_hw_idle(struct hisi_ptt *hisi_ptt) HISI_PTT_WAIT_TRACE_TIMEOUT_US); } -static void hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) +static bool hisi_ptt_wait_dma_reset_done(struct hisi_ptt *hisi_ptt) { u32 val; - readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, - val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, - HISI_PTT_RESET_TIMEOUT_US); + return !readl_poll_timeout_atomic(hisi_ptt->iobase + HISI_PTT_TRACE_WR_STS, + val, !val, HISI_PTT_RESET_POLL_INTERVAL_US, + HISI_PTT_RESET_TIMEOUT_US); } static void hisi_ptt_trace_end(struct hisi_ptt *hisi_ptt) @@ -202,14 +202,18 @@ static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) return -EBUSY; } - ctrl->started = true; - /* Reset the DMA before start tracing */ val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val |= HISI_PTT_TRACE_CTRL_RST; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); - hisi_ptt_wait_dma_reset_done(hisi_ptt); + if (!hisi_ptt_wait_dma_reset_done(hisi_ptt)) { + pci_err(hisi_ptt->pdev, "timed out waiting for DMA reset\n"); + val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + val &= ~HISI_PTT_TRACE_CTRL_RST; + writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); + return -ETIMEDOUT; + } val = readl(hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); val &= ~HISI_PTT_TRACE_CTRL_RST; @@ -234,6 +238,8 @@ static int hisi_ptt_trace_start(struct hisi_ptt *hisi_ptt) if (!hisi_ptt->trace_ctrl.is_port) val |= HISI_PTT_TRACE_CTRL_FILTER_MODE; + ctrl->started = true; + /* Start the Trace */ val |= HISI_PTT_TRACE_CTRL_EN; writel(val, hisi_ptt->iobase + HISI_PTT_TRACE_CTRL); diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c index 188e24cc4d35f..9fdd6a5fb8b65 100644 --- a/drivers/i2c/busses/i2c-amd-mp2-plat.c +++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c @@ -316,8 +316,10 @@ static int i2c_amd_probe(struct platform_device *pdev) amd_mp2_pm_runtime_put(mp2_dev); - if (ret < 0) + if (ret < 0) { dev_err(&pdev->dev, "i2c add adapter failed = %d\n", ret); + amd_mp2_unregister_cb(&i2c_dev->common); + } return ret; } diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index e418a4f23f156..19300c098af8f 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -803,6 +803,17 @@ static int bcm_iproc_i2c_xfer_wait(struct bcm_iproc_i2c_dev *iproc_i2c, } if (!time_left && !iproc_i2c->xfer_is_done) { + /* + * The controller may fail to clear START_BUSY after a timeout, + * reset the controller to recover in that case. + */ + if (!!(iproc_i2c_rd_reg(iproc_i2c, M_CMD_OFFSET) & + BIT(M_CMD_START_BUSY_SHIFT))) { + bcm_iproc_i2c_enable_disable(iproc_i2c, false); + bcm_iproc_i2c_init(iproc_i2c); + bcm_iproc_i2c_enable_disable(iproc_i2c, true); + } + /* flush both TX/RX FIFOs */ val = BIT(M_FIFO_RX_FLUSH_SHIFT) | BIT(M_FIFO_TX_FLUSH_SHIFT); iproc_i2c_wr_reg(iproc_i2c, M_FIFO_CTRL_OFFSET, val); diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index b19c1144b58ec..a55690b80df80 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -32,6 +32,8 @@ #include #include +#include + #include "i2c-designware-core.h" static const char *const abort_sources[] = { diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index c8e4c122ed4df..b1b0cd18d9ac6 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -17,6 +17,8 @@ #include #include +#include + #define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C | \ I2C_FUNC_SMBUS_BYTE | \ I2C_FUNC_SMBUS_BYTE_DATA | \ @@ -24,85 +26,10 @@ I2C_FUNC_SMBUS_BLOCK_DATA | \ I2C_FUNC_SMBUS_I2C_BLOCK) -#define DW_IC_CON_MASTER BIT(0) -#define DW_IC_CON_SPEED_STD (1 << 1) -#define DW_IC_CON_SPEED_FAST (2 << 1) -#define DW_IC_CON_SPEED_HIGH (3 << 1) -#define DW_IC_CON_SPEED_MASK GENMASK(2, 1) -#define DW_IC_CON_10BITADDR_SLAVE BIT(3) -#define DW_IC_CON_10BITADDR_MASTER BIT(4) -#define DW_IC_CON_RESTART_EN BIT(5) -#define DW_IC_CON_SLAVE_DISABLE BIT(6) -#define DW_IC_CON_STOP_DET_IFADDRESSED BIT(7) -#define DW_IC_CON_TX_EMPTY_CTRL BIT(8) -#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) -#define DW_IC_CON_BUS_CLEAR_CTRL BIT(11) - -#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) -#define DW_IC_DATA_CMD_FIRST_DATA_BYTE BIT(11) - -/* - * Registers offset - */ -#define DW_IC_CON 0x00 -#define DW_IC_TAR 0x04 -#define DW_IC_SAR 0x08 -#define DW_IC_DATA_CMD 0x10 -#define DW_IC_SS_SCL_HCNT 0x14 -#define DW_IC_SS_SCL_LCNT 0x18 -#define DW_IC_FS_SCL_HCNT 0x1c -#define DW_IC_FS_SCL_LCNT 0x20 -#define DW_IC_HS_SCL_HCNT 0x24 -#define DW_IC_HS_SCL_LCNT 0x28 -#define DW_IC_INTR_STAT 0x2c -#define DW_IC_INTR_MASK 0x30 -#define DW_IC_RAW_INTR_STAT 0x34 -#define DW_IC_RX_TL 0x38 -#define DW_IC_TX_TL 0x3c -#define DW_IC_CLR_INTR 0x40 -#define DW_IC_CLR_RX_UNDER 0x44 -#define DW_IC_CLR_RX_OVER 0x48 -#define DW_IC_CLR_TX_OVER 0x4c -#define DW_IC_CLR_RD_REQ 0x50 -#define DW_IC_CLR_TX_ABRT 0x54 -#define DW_IC_CLR_RX_DONE 0x58 -#define DW_IC_CLR_ACTIVITY 0x5c -#define DW_IC_CLR_STOP_DET 0x60 -#define DW_IC_CLR_START_DET 0x64 -#define DW_IC_CLR_GEN_CALL 0x68 -#define DW_IC_ENABLE 0x6c -#define DW_IC_STATUS 0x70 -#define DW_IC_TXFLR 0x74 -#define DW_IC_RXFLR 0x78 -#define DW_IC_SDA_HOLD 0x7c -#define DW_IC_TX_ABRT_SOURCE 0x80 -#define DW_IC_ENABLE_STATUS 0x9c -#define DW_IC_CLR_RESTART_DET 0xa8 -#define DW_IC_SMBUS_INTR_MASK 0xcc -#define DW_IC_SCL_STUCK_AT_LOW_TIMEOUT 0xac -#define DW_IC_SDA_STUCK_AT_LOW_TIMEOUT 0xb0 -#define DW_IC_COMP_PARAM_1 0xf4 -#define DW_IC_COMP_VERSION 0xf8 #define DW_IC_SDA_HOLD_MIN_VERS 0x3131312A /* "111*" == v1.11* */ #define DW_IC_BUS_CLEAR_MIN_VERS 0x3230302A /* "200*" == v2.00* */ -#define DW_IC_COMP_TYPE 0xfc #define DW_IC_COMP_TYPE_VALUE 0x44570140 /* "DW" + 0x0140 */ -#define DW_IC_INTR_RX_UNDER BIT(0) -#define DW_IC_INTR_RX_OVER BIT(1) -#define DW_IC_INTR_RX_FULL BIT(2) -#define DW_IC_INTR_TX_OVER BIT(3) -#define DW_IC_INTR_TX_EMPTY BIT(4) -#define DW_IC_INTR_RD_REQ BIT(5) -#define DW_IC_INTR_TX_ABRT BIT(6) -#define DW_IC_INTR_RX_DONE BIT(7) -#define DW_IC_INTR_ACTIVITY BIT(8) -#define DW_IC_INTR_STOP_DET BIT(9) -#define DW_IC_INTR_START_DET BIT(10) -#define DW_IC_INTR_GEN_CALL BIT(11) -#define DW_IC_INTR_RESTART_DET BIT(12) -#define DW_IC_INTR_MST_ON_HOLD BIT(13) - #define DW_IC_INTR_DEFAULT_MASK (DW_IC_INTR_RX_FULL | \ DW_IC_INTR_TX_ABRT | \ DW_IC_INTR_STOP_DET) @@ -112,18 +39,6 @@ DW_IC_INTR_RX_UNDER | \ DW_IC_INTR_RD_REQ) -#define DW_IC_ENABLE_ENABLE BIT(0) -#define DW_IC_ENABLE_ABORT BIT(1) -#define DW_IC_ENABLE_BUS_RECOVERY BIT(3) - -#define DW_IC_STATUS_ACTIVITY BIT(0) -#define DW_IC_STATUS_TFE BIT(2) -#define DW_IC_STATUS_RFNE BIT(3) -#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) -#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) -#define DW_IC_STATUS_MASTER_HOLD_TX_FIFO_EMPTY BIT(7) -#define DW_IC_STATUS_SDA_STUCK_NOT_RECOVERED BIT(11) - #define DW_IC_SDA_HOLD_RX_SHIFT 16 #define DW_IC_SDA_HOLD_RX_MASK GENMASK(23, 16) diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index c5d169becc750..177247e74fe7a 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -25,6 +25,8 @@ #include #include +#include + #include "i2c-designware-core.h" #define AMD_TIMEOUT_MIN_US 25 diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index 34d881572351c..2516824798d91 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -364,7 +364,7 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { "HISI02A1", 0 }, { "HISI02A2", 0 }, { "HISI02A3", 0 }, - { "HJMC3001", 0 }, + { "HJMC3001", ACCESS_INTR_MASK }, { "HYGO0010", ACCESS_INTR_MASK }, { "INT33C2", 0 }, { "INT33C3", 0 }, diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c index 6eb16b7d75a6d..4498ca4d72ca6 100644 --- a/drivers/i2c/busses/i2c-designware-slave.c +++ b/drivers/i2c/busses/i2c-designware-slave.c @@ -19,6 +19,8 @@ #include #include +#include + #include "i2c-designware-core.h" static void i2c_dw_configure_fifo_slave(struct dw_i2c_dev *dev) diff --git a/drivers/i2c/busses/i2c-imx.c b/drivers/i2c/busses/i2c-imx.c index d5e6e2eca3b34..9477d814fde94 100644 --- a/drivers/i2c/busses/i2c-imx.c +++ b/drivers/i2c/busses/i2c-imx.c @@ -930,9 +930,6 @@ static int i2c_imx_reg_slave(struct i2c_client *client) if (i2c_imx->slave) return -EBUSY; - i2c_imx->slave = client; - i2c_imx->last_slave_event = I2C_SLAVE_STOP; - /* Resume */ ret = pm_runtime_resume_and_get(i2c_imx->adapter.dev.parent); if (ret < 0) { @@ -940,6 +937,11 @@ static int i2c_imx_reg_slave(struct i2c_client *client) return ret; } + scoped_guard(spinlock_irqsave, &i2c_imx->slave_lock) { + i2c_imx->slave = client; + i2c_imx->last_slave_event = I2C_SLAVE_STOP; + } + i2c_imx_slave_init(i2c_imx); return 0; @@ -958,6 +960,7 @@ static int i2c_imx_unreg_slave(struct i2c_client *client) i2c_imx_reset_regs(i2c_imx); + hrtimer_cancel(&i2c_imx->slave_timer); i2c_imx->slave = NULL; /* Suspend */ @@ -1952,6 +1955,47 @@ static int i2c_imx_runtime_resume(struct device *dev) return 0; } +static int __maybe_unused i2c_imx_suspend_noirq(struct device *dev) +{ + struct imx_i2c_struct *i2c_imx = dev_get_drvdata(dev); + int ret; + + i2c_mark_adapter_suspended(&i2c_imx->adapter); + + /* + * Cancel the slave timer before powering down to prevent + * i2c_imx_slave_timeout() from accessing hardware registers + * while the clock is disabled. + */ + hrtimer_cancel(&i2c_imx->slave_timer); + + ret = pm_runtime_force_suspend(dev); + if (ret) { + i2c_mark_adapter_resumed(&i2c_imx->adapter); + if (i2c_imx->slave) { + hrtimer_forward_now(&i2c_imx->slave_timer, I2C_IMX_CHECK_DELAY); + hrtimer_restart(&i2c_imx->slave_timer); + } + return ret; + } + + return 0; +} + +static int __maybe_unused i2c_imx_resume_noirq(struct device *dev) +{ + struct imx_i2c_struct *i2c_imx = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + + i2c_mark_adapter_resumed(&i2c_imx->adapter); + + return 0; +} + static int i2c_imx_suspend(struct device *dev) { /* @@ -1985,8 +2029,8 @@ static int i2c_imx_resume(struct device *dev) } static const struct dev_pm_ops i2c_imx_pm_ops = { - NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, - pm_runtime_force_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(i2c_imx_suspend_noirq, + i2c_imx_resume_noirq) SYSTEM_SLEEP_PM_OPS(i2c_imx_suspend, i2c_imx_resume) RUNTIME_PM_OPS(i2c_imx_runtime_suspend, i2c_imx_runtime_resume, NULL) }; diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index 664a5471d9335..695be3b214608 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -141,6 +141,7 @@ struct jz4780_i2c { void __iomem *iomem; int irq; struct clk *clk; + unsigned long clk_rate_khz; struct i2c_adapter adap; const struct ingenic_i2c_config *cdata; @@ -246,7 +247,7 @@ static int jz4780_i2c_set_target(struct jz4780_i2c *i2c, unsigned char address) static int jz4780_i2c_set_speed(struct jz4780_i2c *i2c) { - int dev_clk_khz = clk_get_rate(i2c->clk) / 1000; + int dev_clk_khz = i2c->clk_rate_khz; int cnt_high = 0; /* HIGH period count of the SCL clock */ int cnt_low = 0; /* LOW period count of the SCL clock */ int cnt_period = 0; /* period count of the SCL clock */ @@ -796,6 +797,8 @@ static int jz4780_i2c_probe(struct platform_device *pdev) if (IS_ERR(i2c->clk)) return PTR_ERR(i2c->clk); + i2c->clk_rate_khz = clk_get_rate(i2c->clk) / 1000; + ret = of_property_read_u32(pdev->dev.of_node, "clock-frequency", &clk_freq); if (ret) { diff --git a/drivers/i2c/busses/i2c-k1.c b/drivers/i2c/busses/i2c-k1.c index 1c58c39b92795..f2bb04a1b4935 100644 --- a/drivers/i2c/busses/i2c-k1.c +++ b/drivers/i2c/busses/i2c-k1.c @@ -565,11 +565,6 @@ static int spacemit_i2c_probe(struct platform_device *pdev) if (i2c->irq < 0) return dev_err_probe(dev, i2c->irq, "failed to get irq resource"); - ret = devm_request_irq(i2c->dev, i2c->irq, spacemit_i2c_irq_handler, - IRQF_NO_SUSPEND, dev_name(i2c->dev), i2c); - if (ret) - return dev_err_probe(dev, ret, "failed to request irq"); - clk = devm_clk_get_enabled(dev, "func"); if (IS_ERR(clk)) return dev_err_probe(dev, PTR_ERR(clk), "failed to enable func clock"); @@ -592,6 +587,11 @@ static int spacemit_i2c_probe(struct platform_device *pdev) init_completion(&i2c->complete); + ret = devm_request_irq(i2c->dev, i2c->irq, spacemit_i2c_irq_handler, + IRQF_NO_SUSPEND, dev_name(i2c->dev), i2c); + if (ret) + return dev_err_probe(dev, ret, "failed to request irq"); + platform_set_drvdata(pdev, i2c); ret = i2c_add_numbered_adapter(&i2c->adapt); diff --git a/drivers/i2c/busses/i2c-mxs.c b/drivers/i2c/busses/i2c-mxs.c index 08c9091a1e351..72c80107c11af 100644 --- a/drivers/i2c/busses/i2c-mxs.c +++ b/drivers/i2c/busses/i2c-mxs.c @@ -831,7 +831,7 @@ static int mxs_i2c_probe(struct platform_device *pdev) } /* Setup the DMA */ - i2c->dmach = dma_request_chan(dev, "rx-tx"); + i2c->dmach = devm_dma_request_chan(dev, "rx-tx"); if (IS_ERR(i2c->dmach)) { return dev_err_probe(dev, PTR_ERR(i2c->dmach), "Failed to request dma\n"); @@ -869,9 +869,6 @@ static void mxs_i2c_remove(struct platform_device *pdev) i2c_del_adapter(&i2c->adapter); - if (i2c->dmach) - dma_release_channel(i2c->dmach); - writel(MXS_I2C_CTRL0_SFTRST, i2c->regs + MXS_I2C_CTRL0_SET); } diff --git a/drivers/i2c/busses/i2c-ocores.c b/drivers/i2c/busses/i2c-ocores.c index 0f67e57cdeff6..9534e218382ca 100644 --- a/drivers/i2c/busses/i2c-ocores.c +++ b/drivers/i2c/busses/i2c-ocores.c @@ -761,7 +761,11 @@ static int ocores_i2c_resume(struct device *dev) rate = clk_get_rate(i2c->clk) / 1000; if (rate) i2c->ip_clock_khz = rate; - return ocores_init(dev, i2c); + ret = ocores_init(dev, i2c); + if (ret) + clk_disable_unprepare(i2c->clk); + + return ret; } static DEFINE_NOIRQ_DEV_PM_OPS(ocores_i2c_pm, diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c index 9d83358f2ae24..776c3e36e9152 100644 --- a/drivers/i2c/busses/i2c-qcom-cci.c +++ b/drivers/i2c/busses/i2c-qcom-cci.c @@ -246,7 +246,7 @@ static int cci_reset(struct cci *cci) return 0; } -static int cci_init(struct cci *cci) +static void cci_init(struct cci *cci) { u32 val = CCI_IRQ_MASK_0_I2C_M0_RD_DONE | CCI_IRQ_MASK_0_I2C_M0_Q0_REPORT | @@ -287,8 +287,6 @@ static int cci_init(struct cci *cci) val = hw->scl_stretch_en << 8 | hw->trdhld << 4 | hw->tsp; writel(val, cci->base + CCI_I2C_Mm_MISC_CTL(i)); } - - return 0; } static int cci_run_queue(struct cci *cci, u8 master, u8 queue) @@ -496,24 +494,8 @@ static int __maybe_unused cci_resume_runtime(struct device *dev) return 0; } -static int __maybe_unused cci_suspend(struct device *dev) -{ - if (!pm_runtime_suspended(dev)) - return cci_suspend_runtime(dev); - - return 0; -} - -static int __maybe_unused cci_resume(struct device *dev) -{ - cci_resume_runtime(dev); - pm_request_autosuspend(dev); - - return 0; -} - static const struct dev_pm_ops qcom_cci_pm = { - SET_SYSTEM_SLEEP_PM_OPS(cci_suspend, cci_resume) + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) SET_RUNTIME_PM_OPS(cci_suspend_runtime, cci_resume_runtime, NULL) }; @@ -612,16 +594,16 @@ static int cci_probe(struct platform_device *pdev) ret = cci_reset(cci); if (ret < 0) - goto error; + goto disable_clocks; - ret = cci_init(cci); - if (ret < 0) - goto error; + cci_init(cci); pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC); + ret = devm_pm_runtime_set_active_enabled(dev); + if (ret) + goto disable_clocks; + pm_runtime_use_autosuspend(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); for (i = 0; i < cci->data->num_masters; i++) { if (!cci->master[i].cci) @@ -637,8 +619,6 @@ static int cci_probe(struct platform_device *pdev) return 0; error_i2c: - pm_runtime_disable(dev); - pm_runtime_dont_use_autosuspend(dev); for (--i ; i >= 0; i--) { if (cci->master[i].cci) { @@ -646,8 +626,6 @@ static int cci_probe(struct platform_device *pdev) of_node_put(cci->master[i].adap.dev.of_node); } } -error: - disable_irq(cci->irq); disable_clocks: cci_disable_clocks(cci); @@ -666,10 +644,6 @@ static void cci_remove(struct platform_device *pdev) cci_halt(cci, i); } } - - disable_irq(cci->irq); - pm_runtime_disable(&pdev->dev); - pm_runtime_set_suspended(&pdev->dev); } static const struct cci_data cci_v1_data = { diff --git a/drivers/i2c/busses/i2c-qcom-geni.c b/drivers/i2c/busses/i2c-qcom-geni.c index bfb352b04902c..6b2caaaa9437e 100644 --- a/drivers/i2c/busses/i2c-qcom-geni.c +++ b/drivers/i2c/busses/i2c-qcom-geni.c @@ -156,8 +156,8 @@ static const struct geni_i2c_clk_fld geni_i2c_clk_map_19p2mhz[] = { /* source_clock = 32 MHz */ static const struct geni_i2c_clk_fld geni_i2c_clk_map_32mhz[] = { - { I2C_MAX_STANDARD_MODE_FREQ, 8, 14, 18, 38 }, - { I2C_MAX_FAST_MODE_FREQ, 4, 3, 9, 19 }, + { I2C_MAX_STANDARD_MODE_FREQ, 12, 9, 10, 26 }, + { I2C_MAX_FAST_MODE_FREQ, 4, 3, 9, 19 }, { I2C_MAX_FAST_MODE_PLUS_FREQ, 2, 3, 5, 15 }, {} }; diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index e049fa908231e..ba44d2c25630f 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -1814,6 +1814,12 @@ void i2c_del_adapter(struct i2c_adapter *adap) return; } + /* + * This drains any in-flight writers, so all + * clients will be caught by i2c_deregister_clients(). + */ + device_remove_file(&adap->dev, &dev_attr_new_device); + i2c_acpi_remove_space_handler(adap); i2c_deregister_clients(adap); diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c index 71eb1ef56f0c3..ad6acb5ebadc3 100644 --- a/drivers/i2c/i2c-core-smbus.c +++ b/drivers/i2c/i2c-core-smbus.c @@ -566,6 +566,18 @@ s32 __i2c_smbus_xfer(struct i2c_adapter *adapter, u16 addr, if (res) return res; + /* Reject invalid caller-supplied block lengths before any + * tracepoint or native smbus_xfer callback runs. + */ + if (data && + (protocol == I2C_SMBUS_I2C_BLOCK_DATA || + protocol == I2C_SMBUS_BLOCK_PROC_CALL || + (protocol == I2C_SMBUS_BLOCK_DATA && + read_write == I2C_SMBUS_WRITE)) && + (data->block[0] == 0 || + data->block[0] > I2C_SMBUS_BLOCK_MAX)) + return -EINVAL; + /* If enabled, the following two tracepoints are conditional on * read_write and protocol. */ diff --git a/drivers/i2c/i2c-mux.c b/drivers/i2c/i2c-mux.c index 2f0ea4f43bf55..6beb44e87c1e9 100644 --- a/drivers/i2c/i2c-mux.c +++ b/drivers/i2c/i2c-mux.c @@ -413,6 +413,7 @@ int i2c_mux_add_adapter(struct i2c_mux_core *muxc, return 0; err_free_priv: + of_node_put(priv->adap.dev.of_node); kfree(priv); return ret; } diff --git a/drivers/i2c/muxes/i2c-demux-pinctrl.c b/drivers/i2c/muxes/i2c-demux-pinctrl.c index f2a1f47449789..2403c0bf7c431 100644 --- a/drivers/i2c/muxes/i2c-demux-pinctrl.c +++ b/drivers/i2c/muxes/i2c-demux-pinctrl.c @@ -247,6 +247,7 @@ static int i2c_demux_pinctrl_probe(struct platform_device *pdev) props[i].value = devm_kstrdup(&pdev->dev, "ok", GFP_KERNEL); if (!props[i].name || !props[i].value) { err = -ENOMEM; + of_node_put(adap_np); goto err_rollback; } props[i].length = 3; diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c index 588395e7a7706..c8ee43b70d5ec 100644 --- a/drivers/i3c/master.c +++ b/drivers/i3c/master.c @@ -799,6 +799,11 @@ static struct attribute *i3c_masterdev_attrs[] = { }; ATTRIBUTE_GROUPS(i3c_masterdev); +static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) +{ + kfree(dev); +} + static void i3c_masterdev_release(struct device *dev) { struct i3c_master_controller *master = dev_to_i3cmaster(dev); @@ -811,6 +816,8 @@ static void i3c_masterdev_release(struct device *dev) i3c_bus_cleanup(bus); of_node_put(dev->of_node); + + i3c_master_free_i3c_dev(master->this); } static const struct device_type i3c_masterdev_type = { @@ -1023,11 +1030,6 @@ static void i3c_device_release(struct device *dev) kfree(i3cdev); } -static void i3c_master_free_i3c_dev(struct i3c_dev_desc *dev) -{ - kfree(dev); -} - static struct i3c_dev_desc * i3c_master_alloc_i3c_dev(struct i3c_master_controller *master, const struct i3c_device_info *info) @@ -1787,12 +1789,21 @@ static int i3c_master_early_i3c_dev_add(struct i3c_master_controller *master, static void i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) { + struct i3c_device *i3cdev, *tmp; struct i3c_dev_desc *desc; + LIST_HEAD(i3c_unreg_devs); int ret; if (!master->init_done) return; + i3c_bus_maintenance_lock(&master->bus); + + if (master->shutting_down) { + i3c_bus_maintenance_unlock(&master->bus); + return; + } + i3c_bus_for_each_i3cdev(&master->bus, desc) { if (desc->dev || !desc->info.dyn_addr || desc == master->this) continue; @@ -1813,23 +1824,37 @@ i3c_master_register_new_i3c_devs(struct i3c_master_controller *master) if (desc->boardinfo) desc->dev->dev.of_node = desc->boardinfo->of_node; - ret = device_register(&desc->dev->dev); - if (ret) { - dev_err(&master->dev, - "Failed to add I3C device (err = %d)\n", ret); - put_device(&desc->dev->dev); - } + list_add_tail(&desc->dev->node, &i3c_unreg_devs); } + + i3c_bus_maintenance_unlock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + ret = device_register(&i3cdev->dev); + if (ret) + dev_err(&master->dev, "Failed to add I3C device (err = %d)\n", ret); + else + list_del_init(&i3cdev->node); + } + + i3c_bus_maintenance_lock(&master->bus); + + list_for_each_entry_safe(i3cdev, tmp, &i3c_unreg_devs, node) { + list_del(&i3cdev->node); + desc = i3cdev->desc; + i3cdev->desc = NULL; + put_device(&i3cdev->dev); + desc->dev = NULL; + } + + i3c_bus_maintenance_unlock(&master->bus); } static void i3c_master_reg_work_fn(struct work_struct *work) { struct i3c_master_controller *master = container_of(work, typeof(*master), reg_work); - i3c_bus_normaluse_lock(&master->bus); - if (!master->shutting_down) - i3c_master_register_new_i3c_devs(master); - i3c_bus_normaluse_unlock(&master->bus); + i3c_master_register_new_i3c_devs(master); } /** @@ -2023,6 +2048,8 @@ int i3c_master_set_info(struct i3c_master_controller *master, return 0; err_free_dev: + master->bus.cur_master = NULL; + master->this = NULL; i3c_master_free_i3c_dev(i3cdev); return ret; @@ -2043,7 +2070,8 @@ static void i3c_master_detach_free_devs(struct i3c_master_controller *master) i3cdev->boardinfo->init_dyn_addr, I3C_ADDR_SLOT_FREE); - i3c_master_free_i3c_dev(i3cdev); + if (i3cdev != master->this) + i3c_master_free_i3c_dev(i3cdev); } list_for_each_entry_safe(i2cdev, i2ctmp, &master->bus.devs.i2c, @@ -2272,7 +2300,8 @@ i3c_master_search_i3c_dev_duplicate(struct i3c_dev_desc *refdev) struct i3c_dev_desc *i3cdev; i3c_bus_for_each_i3cdev(&master->bus, i3cdev) { - if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid) + if (i3cdev != refdev && i3cdev->info.pid == refdev->info.pid && + i3cdev != master->this) return i3cdev; } @@ -2808,11 +2837,12 @@ static void i3c_master_unregister_i3c_devs(struct i3c_master_controller *master) if (!i3cdev->dev) continue; - i3cdev->dev->desc = NULL; - if (device_is_registered(&i3cdev->dev->dev)) + if (device_is_registered(&i3cdev->dev->dev)) { + get_device(&i3cdev->dev->dev); device_unregister(&i3cdev->dev->dev); - else - put_device(&i3cdev->dev->dev); + } + i3cdev->dev->desc = NULL; + put_device(&i3cdev->dev->dev); i3cdev->dev = NULL; } } diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c index d329faf4b3f96..178029769bab4 100644 --- a/drivers/i3c/master/adi-i3c-master.c +++ b/drivers/i3c/master/adi-i3c-master.c @@ -932,6 +932,7 @@ static const struct of_device_id adi_i3c_master_of_match[] = { { .compatible = "adi,i3c-master-v1" }, {} }; +MODULE_DEVICE_TABLE(of, adi_i3c_master_of_match); static int adi_i3c_master_probe(struct platform_device *pdev) { @@ -967,17 +968,9 @@ static int adi_i3c_master_probe(struct platform_device *pdev) writel(0x00, master->regs + REG_ENABLE); writel(0x00, master->regs + REG_IRQ_MASK); - ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0, - dev_name(&pdev->dev), master); - if (ret) - return ret; - platform_set_drvdata(pdev, master); master->free_rr_slots = GENMASK(ADI_MAX_DEVS, 1); - - writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); - spin_lock_init(&master->ibi.lock); master->ibi.num_slots = 15; master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, @@ -989,6 +982,13 @@ static int adi_i3c_master_probe(struct platform_device *pdev) spin_lock_init(&master->xferqueue.lock); INIT_LIST_HEAD(&master->xferqueue.list); + ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0, + dev_name(&pdev->dev), master); + if (ret) + return ret; + + writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); + return i3c_master_register(&master->base, &pdev->dev, &adi_i3c_master_ops, false); } diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c index 675c257ebe2c2..4a4fae96be370 100644 --- a/drivers/i3c/master/dw-i3c-master.c +++ b/drivers/i3c/master/dw-i3c-master.c @@ -883,7 +883,15 @@ static int dw_i3c_master_daa(struct i3c_master_controller *m) if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) dw_i3c_master_dequeue_xfer(master, xfer); - newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + /* + * cmd->rx_len holds the number of addresses ENTDAA left unassigned. + * On an empty bus rx_len == maxdevs, so avoid GENMASK(-1, 0). + */ + if (cmd->rx_len >= master->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + newdevs &= ~olddevs; for (pos = 0; pos < master->maxdevs; pos++) { diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c index efb7a1f92641c..b6ba18ccdd2d3 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c @@ -362,9 +362,7 @@ static int hci_cmd_v1_daa(struct i3c_hci *hci) * TODO: Extend the subsystem layer to allow for registering * new device and provide BCR/DCR/PID at the same time. */ - ret = i3c_master_add_i3c_dev_locked(&hci->master, next_addr); - if (ret) - break; + i3c_master_add_i3c_dev_locked(&hci->master, next_addr); } if (dat_idx >= 0) diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c index 5fc2e4c55ebb0..d7999d341a227 100644 --- a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c @@ -300,9 +300,7 @@ static int hci_cmd_v2_daa(struct i3c_hci *hci) * TODO: Extend the subsystem layer to allow for registering * new device and provide BCR/DCR/PID at the same time. */ - ret = i3c_master_add_i3c_dev_locked(&hci->master, next_addr); - if (ret) - break; + i3c_master_add_i3c_dev_locked(&hci->master, next_addr); } hci_free_xfer(xfer, 2); diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c index 75e6610cfd812..1eeab337e5e53 100644 --- a/drivers/i3c/master/mipi-i3c-hci/core.c +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -147,7 +147,10 @@ static int i3c_hci_bus_init(struct i3c_master_controller *m) if (hci->quirks & HCI_QUIRK_RESP_BUF_THLD) amd_set_resp_buf_thld(hci); - reg_set(HC_CONTROL, HC_CONTROL_BUS_ENABLE); + WRITE_ONCE(hci->irq_inactive, false); + + /* Enable bus with Hot-Join disabled */ + reg_set(HC_CONTROL, HC_CONTROL_BUS_ENABLE | HC_CONTROL_HOT_JOIN_CTRL); dev_dbg(&hci->master.dev, "HC_CONTROL = %#x", reg_read(HC_CONTROL)); return 0; @@ -536,6 +539,15 @@ static irqreturn_t i3c_hci_irq_handler(int irq, void *dev_id) irqreturn_t result = IRQ_NONE; u32 val; + /* + * The IRQ can be shared, so the handler may be called when the IRQ is + * due to a different device. That could happen before the controller + * has been initialized, so exit immediately if IRQs are not expected + * for this device. + */ + if (READ_ONCE(hci->irq_inactive)) + return IRQ_NONE; + val = reg_read(INTR_STATUS); reg_write(INTR_STATUS, val); dev_dbg(&hci->master.dev, "INTR_STATUS %#x", val); @@ -775,6 +787,8 @@ static int i3c_hci_probe(struct platform_device *pdev) if (ret) return ret; + WRITE_ONCE(hci->irq_inactive, true); + irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, i3c_hci_irq_handler, IRQF_SHARED, NULL, hci); diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h index 32c8aecde9f76..5502ab0a3535d 100644 --- a/drivers/i3c/master/mipi-i3c-hci/hci.h +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -46,6 +46,7 @@ struct i3c_hci { void *io_data; const struct hci_cmd_ops *cmd; spinlock_t lock; + bool irq_inactive; struct mutex control_mutex; atomic_t next_cmd_tid; u32 caps; diff --git a/drivers/i3c/master/renesas-i3c.c b/drivers/i3c/master/renesas-i3c.c index 5b1bf5a0266cc..91c04f3a4c552 100644 --- a/drivers/i3c/master/renesas-i3c.c +++ b/drivers/i3c/master/renesas-i3c.c @@ -428,6 +428,21 @@ static void renesas_i3c_enqueue_xfer(struct renesas_i3c *i3c, struct renesas_i3c } } +static void renesas_i3c_irqs_mask_and_clear_locked(struct renesas_i3c *i3c) +{ + /* Disable all the interrupts. */ + renesas_writel(i3c->regs, BIE, 0); + renesas_writel(i3c->regs, NTIE, 0); + + /* Clear normal transfer status flags. */ + renesas_writel(i3c->regs, NTST, 0); + + /* Clear bus status flags. */ + renesas_writel(i3c->regs, BST, 0); + /* Read back registers to confirm writes have fully propagated. */ + renesas_readl(i3c->regs, BST); +} + static void renesas_i3c_wait_xfer(struct renesas_i3c *i3c, struct renesas_i3c_xfer *xfer) { unsigned long time_left; @@ -632,6 +647,10 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) if (!xfer) return -ENOMEM; + init_completion(&xfer->comp); + cmd = xfer->cmds; + cmd->rx_count = i3c->maxdevs; + /* Enable I3C bus. */ renesas_i3c_bus_enable(m, true); @@ -653,10 +672,6 @@ static int renesas_i3c_daa(struct i3c_master_controller *m) renesas_writel(i3c->regs, DATBAS(pos), datbas_dvdyad_with_parity(ret)); } - init_completion(&xfer->comp); - cmd = xfer->cmds; - cmd->rx_count = 0; - ret = renesas_i3c_get_free_pos(i3c); if (ret < 0) return ret; @@ -745,13 +760,13 @@ static int renesas_i3c_send_ccc_cmd(struct i3c_master_controller *m, if (!xfer) return -ENOMEM; - renesas_i3c_bus_enable(m, true); - init_completion(&xfer->comp); cmd = xfer->cmds; cmd->rnw = ccc->rnw; cmd->cmd0 = 0; + renesas_i3c_bus_enable(m, true); + /* Calculate the command descriptor. */ switch (ccc->id) { case I3C_CCC_SETDASA: @@ -803,15 +818,15 @@ static int renesas_i3c_priv_xfers(struct i3c_dev_desc *dev, struct i3c_priv_xfer struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); int i; - /* Enable I3C bus. */ - renesas_i3c_bus_enable(m, true); - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); if (!xfer) return -ENOMEM; init_completion(&xfer->comp); + /* Enable I3C bus. */ + renesas_i3c_bus_enable(m, true); + for (i = 0; i < i3c_nxfers; i++) { struct renesas_i3c_cmd *cmd = xfer->cmds; @@ -878,10 +893,26 @@ static int renesas_i3c_reattach_i3c_dev(struct i3c_dev_desc *dev, struct i3c_master_controller *m = i3c_dev_get_master(dev); struct renesas_i3c *i3c = to_renesas_i3c(m); struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + int pos; + + pos = renesas_i3c_get_free_pos(i3c); + + if (data->index > pos && pos >= 0) { + renesas_writel(i3c->regs, DATBAS(data->index), 0); + i3c->addrs[data->index] = 0; + i3c->free_pos |= BIT(data->index); + + data->index = pos; + i3c->free_pos &= ~BIT(data->index); + } i3c->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr : dev->info.static_addr; + renesas_writel(i3c->regs, DATBAS(data->index), + DATBAS_DVSTAD(dev->info.static_addr) | + datbas_dvdyad_with_parity(i3c->addrs[data->index])); + return 0; } @@ -891,6 +922,8 @@ static void renesas_i3c_detach_i3c_dev(struct i3c_dev_desc *dev) struct i3c_master_controller *m = i3c_dev_get_master(dev); struct renesas_i3c *i3c = to_renesas_i3c(m); + renesas_writel(i3c->regs, DATBAS(data->index), 0); + i3c_dev_set_master_data(dev, NULL); i3c->addrs[data->index] = 0; i3c->free_pos |= BIT(data->index); @@ -907,19 +940,19 @@ static int renesas_i3c_i2c_xfers(struct i2c_dev_desc *dev, u8 start_bit = CNDCTL_STCND; int i; - struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); - if (!xfer) - return -ENOMEM; - if (!i2c_nxfers) return 0; - renesas_i3c_bus_enable(m, false); + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; init_completion(&xfer->comp); xfer->is_i2c_xfer = true; cmd = xfer->cmds; + renesas_i3c_bus_enable(m, false); + if (!(renesas_readl(i3c->regs, BCST) & BCST_BFREF)) { cmd->err = -EBUSY; return cmd->err; @@ -1000,6 +1033,11 @@ static irqreturn_t renesas_i3c_tx_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1040,6 +1078,11 @@ static irqreturn_t renesas_i3c_resp_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; /* Clear the Respone Queue Full status flag*/ @@ -1124,6 +1167,11 @@ static irqreturn_t renesas_i3c_tend_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1170,6 +1218,11 @@ static irqreturn_t renesas_i3c_rx_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { @@ -1220,15 +1273,13 @@ static irqreturn_t renesas_i3c_stop_isr(int irq, void *data) struct renesas_i3c_xfer *xfer; scoped_guard(spinlock, &i3c->xferqueue.lock) { - xfer = i3c->xferqueue.cur; - - /* read back registers to confirm writes have fully propagated */ - renesas_writel(i3c->regs, BST, 0); - renesas_readl(i3c->regs, BST); - renesas_writel(i3c->regs, BIE, 0); - renesas_clear_bit(i3c->regs, NTST, NTST_TDBEF0 | NTST_RDBFF0); + renesas_i3c_irqs_mask_and_clear_locked(i3c); renesas_clear_bit(i3c->regs, SCSTRCTL, SCSTRCTL_RWE); + xfer = i3c->xferqueue.cur; + if (!xfer) + return IRQ_HANDLED; + xfer->ret = 0; complete(&xfer->comp); } @@ -1245,6 +1296,11 @@ static irqreturn_t renesas_i3c_start_isr(int irq, void *data) scoped_guard(spinlock, &i3c->xferqueue.lock) { xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + cmd = xfer->cmds; if (xfer->is_i2c_xfer) { diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c index d33f4c7654974..1f936196b30cc 100644 --- a/drivers/i3c/master/svc-i3c-master.c +++ b/drivers/i3c/master/svc-i3c-master.c @@ -418,14 +418,22 @@ static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, buf = slot->data; while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS)) && - slot->len < SVC_I3C_FIFO_SIZE) { + slot->len < dev->ibi->max_payload_len) { mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL); count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl); + count = min(count, dev->ibi->max_payload_len - slot->len); readsb(master->regs + SVC_I3C_MRDATAB, buf, count); slot->len += count; buf += count; } + /* + * The device may have sent more than the requested payload. Drop the + * extra bytes so they do not leak into the next transfer. + */ + if (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))) + writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL); + master->ibi.tbq_slot = slot; return 0; @@ -608,6 +616,15 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) break; case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: svc_i3c_master_emit_stop(master); + + /* + * If a target gets stuck holding SDA low, the controller reports a MR. + * On NPCM845, emitting STOP may spuriously set SLVSTART, retriggering + * the interrupt and re-entering MR handling, leading to an IRQ storm. + * Clear SLVSTART after STOP to break the loop. + */ + if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART)) + writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS); break; default: break; diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 9ba83954c2555..922ed32f73ed0 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -52,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,10 @@ static unsigned int preferred_states_mask __read_mostly; static bool force_irq_on __read_mostly; static bool ibrs_off __read_mostly; +/* The maximum allowed length for the 'table' module parameter */ +#define MAX_CMDLINE_TABLE_LEN 256 +static char cmdline_table_str[MAX_CMDLINE_TABLE_LEN] __read_mostly; + static struct cpuidle_device __percpu *intel_idle_cpuidle_devices; static unsigned long auto_demotion_disable_flags; @@ -107,6 +112,9 @@ static struct device *sysfs_root __initdata; static const struct idle_cpu *icpu __initdata; static struct cpuidle_state *cpuidle_state_table __initdata; +/* C-states data from the 'intel_idle.table' cmdline parameter */ +static struct cpuidle_state cmdline_states[CPUIDLE_STATE_MAX] __initdata; + static unsigned int mwait_substates __initdata; /* @@ -2420,6 +2428,152 @@ static void __init intel_idle_sysfs_uninit(void) put_device(sysfs_root); } + /** + * get_cmdline_field - Get the current field from a cmdline string. + * @args: The cmdline string to get the current field from. + * @field: Pointer to the current field upon return. + * @sep: The fields separator character. + * + * Examples: + * Input: args="C1:1:1,C1E:2:10", sep=':' + * Output: field="C1", return "1:1,C1E:2:10" + * Input: args="C1:1:1,C1E:2:10", sep=',' + * Output: field="C1:1:1", return "C1E:2:10" + * Ipnut: args="::", sep=':' + * Output: field="", return ":" + * + * Return: The continuation of the cmdline string after the field or NULL. + */ +static char *get_cmdline_field(char *args, char **field, char sep) +{ + unsigned int i; + + for (i = 0; args[i] && !isspace(args[i]); i++) { + if (args[i] == sep) + break; + } + + *field = args; + + if (args[i] != sep) + return NULL; + + args[i] = '\0'; + return args + i + 1; +} + +/** + * cmdline_table_adjust - Adjust the C-states table with data from cmdline. + * @drv: cpuidle driver (assumed to point to intel_idle_driver). + * + * Adjust the C-states table with data from the 'intel_idle.table' module + * parameter (if specified). + */ +static void __init cmdline_table_adjust(struct cpuidle_driver *drv) +{ + char *args = cmdline_table_str; + struct cpuidle_state *state; + int i; + + if (args[0] == '\0') + /* The 'intel_idle.table' module parameter was not specified */ + return; + + /* Create a copy of the C-states table */ + for (i = 0; i < drv->state_count; i++) + cmdline_states[i] = drv->states[i]; + + /* + * Adjust the C-states table copy with data from the 'intel_idle.table' + * module parameter. + */ + while (args) { + char *fields, *name, *val; + + /* + * Get the next C-state definition, which is expected to be + * '::'. Treat "empty" + * fields as unchanged. For example, + * '::' leaves the latency unchanged. + */ + args = get_cmdline_field(args, &fields, ','); + + /* name */ + fields = get_cmdline_field(fields, &name, ':'); + if (!fields) + goto error; + + if (!strcmp(name, "POLL")) { + pr_err("Cannot adjust POLL\n"); + continue; + } + + /* Find the C-state by its name */ + state = NULL; + for (i = 0; i < drv->state_count; i++) { + if (!strcmp(name, drv->states[i].name)) { + state = &cmdline_states[i]; + break; + } + } + + if (!state) { + pr_err("C-state '%s' was not found\n", name); + continue; + } + + /* Latency */ + fields = get_cmdline_field(fields, &val, ':'); + if (!fields) + goto error; + + if (*val) { + if (kstrtouint(val, 0, &state->exit_latency)) + goto error; + } + + /* Target residency */ + fields = get_cmdline_field(fields, &val, ':'); + + if (*val) { + if (kstrtouint(val, 0, &state->target_residency)) + goto error; + } + + /* + * Allow for 3 more fields, but ignore them. Helps to make + * possible future extensions of the cmdline format backward + * compatible. + */ + for (i = 0; fields && i < 3; i++) { + fields = get_cmdline_field(fields, &val, ':'); + if (!fields) + break; + } + + if (fields) { + pr_err("Too many fields for C-state '%s'\n", state->name); + goto error; + } + + pr_info("C-state from cmdline: name=%s, latency=%u, residency=%u\n", + state->name, state->exit_latency, state->target_residency); + } + + /* Copy the adjusted C-states table back */ + for (i = 1; i < drv->state_count; i++) + drv->states[i] = cmdline_states[i]; + + pr_info("Adjusted C-states with data from 'intel_idle.table'\n"); + return; + +error: + pr_info("Failed to adjust C-states with data from 'intel_idle.table'\n"); +} + +#define INTEL_IDLE_INIT_QOS 20 +static struct pm_qos_request qos_req __initdata; + static int __init intel_idle_init(void) { const struct x86_cpu_id *id; @@ -2485,11 +2639,19 @@ static int __init intel_idle_init(void) if (!intel_idle_cpuidle_devices) return -ENOMEM; + intel_idle_cpuidle_driver_init(&intel_idle_driver); + cmdline_table_adjust(&intel_idle_driver); + retval = intel_idle_sysfs_init(); if (retval) pr_warn("failed to initialized sysfs"); - intel_idle_cpuidle_driver_init(&intel_idle_driver); + /* + * Some platforms, in particular the Intel S1200BTL motherboard, have a + * problem with using package idle states too early, so prevent that + * from taking place until the device_initcall() phase is over. + */ + cpu_latency_qos_add_request(&qos_req, INTEL_IDLE_INIT_QOS); retval = cpuidle_register_driver(&intel_idle_driver); if (retval) { @@ -2515,6 +2677,9 @@ static int __init intel_idle_init(void) intel_idle_cpuidle_devices_uninit(); cpuidle_unregister_driver(&intel_idle_driver); init_driver_fail: + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + intel_idle_sysfs_uninit(); free_percpu(intel_idle_cpuidle_devices); return retval; @@ -2522,6 +2687,15 @@ static int __init intel_idle_init(void) } subsys_initcall_sync(intel_idle_init); +static int __init intel_idle_init_complete(void) +{ + if (cpu_latency_qos_request_active((&qos_req))) + cpu_latency_qos_remove_request(&qos_req); + + return 0; +} +device_initcall_sync(intel_idle_init_complete); + /* * We are not really modular, but we used to support that. Meaning we also * support "intel_idle.max_cstate=..." at boot and also a read-only export of @@ -2560,3 +2734,21 @@ module_param(force_irq_on, bool, 0444); */ module_param(ibrs_off, bool, 0444); MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle"); + +/* + * Define the C-states table from a user input string. Expected format is + * 'name:latency:residency', where: + * - name: The C-state name. + * - latency: The C-state exit latency in us. + * - residency: The C-state target residency in us. + * + * Multiple C-states can be defined by separating them with commas: + * 'name1:latency1:residency1,name2:latency2:residency2' + * + * Example: intel_idle.table=C1:1:1,C1E:5:10,C6:100:600 + * + * To leave latency or residency unchanged, use an empty field, for example: + * 'C1:1:1,C1E::10' - leaves C1E latency unchanged. + */ +module_param_string(table, cmdline_table_str, MAX_CMDLINE_TABLE_LEN, 0444); +MODULE_PARM_DESC(table, "Build the C-states table from a user input string"); diff --git a/drivers/iio/accel/dmard09.c b/drivers/iio/accel/dmard09.c index d9290e3b9c464..5297ab2ee0bbe 100644 --- a/drivers/iio/accel/dmard09.c +++ b/drivers/iio/accel/dmard09.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #define DMARD09_DRV_NAME "dmard09" @@ -79,6 +80,12 @@ static int dmard09_read_raw(struct iio_dev *indio_dev, *val = accel; return IIO_VAL_INT; + case IIO_CHAN_INFO_SCALE: + *val = 0; + /* 1 g / 32 LSB, in m/s^2 */ + *val2 = IIO_G_TO_M_S_2(NANO / 32); + + return IIO_VAL_INT_PLUS_NANO; default: return -EINVAL; } diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c index cefc7cf4bd835..279a9b3648860 100644 --- a/drivers/iio/accel/mma8452.c +++ b/drivers/iio/accel/mma8452.c @@ -1682,18 +1682,16 @@ static int mma8452_probe(struct i2c_client *client) goto trigger_cleanup; if (client->irq) { - ret = devm_request_threaded_irq(&client->dev, - client->irq, - NULL, mma8452_interrupt, - IRQF_TRIGGER_LOW | IRQF_ONESHOT, - client->name, indio_dev); + ret = request_threaded_irq(client->irq, NULL, mma8452_interrupt, + IRQF_TRIGGER_LOW | IRQF_ONESHOT, + client->name, indio_dev); if (ret) goto buffer_cleanup; } ret = pm_runtime_set_active(&client->dev); if (ret < 0) - goto buffer_cleanup; + goto free_irq; pm_runtime_enable(&client->dev); pm_runtime_set_autosuspend_delay(&client->dev, @@ -1702,7 +1700,7 @@ static int mma8452_probe(struct i2c_client *client) ret = iio_device_register(indio_dev); if (ret < 0) - goto buffer_cleanup; + goto free_irq; ret = mma8452_set_freefall_mode(data, false); if (ret < 0) @@ -1713,6 +1711,10 @@ static int mma8452_probe(struct i2c_client *client) unregister_device: iio_device_unregister(indio_dev); +free_irq: + if (client->irq) + free_irq(client->irq, indio_dev); + buffer_cleanup: iio_triggered_buffer_cleanup(indio_dev); @@ -1738,6 +1740,9 @@ static void mma8452_remove(struct i2c_client *client) pm_runtime_disable(&client->dev); pm_runtime_set_suspended(&client->dev); + if (client->irq) + free_irq(client->irq, indio_dev); + iio_triggered_buffer_cleanup(indio_dev); mma8452_trigger_cleanup(indio_dev); mma8452_standby(iio_priv(indio_dev)); diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig index d4965c4b5d86f..555c17ac38e40 100644 --- a/drivers/iio/adc/Kconfig +++ b/drivers/iio/adc/Kconfig @@ -1026,6 +1026,7 @@ config MAX1363 config MAX34408 tristate "Maxim max34408/max344089 ADC driver" depends on I2C + select REGMAP_I2C help Say yes here to build support for Maxim max34408/max34409 current sense monitor with 8-bits ADC interface with overcurrent delay/threshold and diff --git a/drivers/iio/adc/adi-axi-adc.c b/drivers/iio/adc/adi-axi-adc.c index 14fa4238c2b96..61f7a07f501f9 100644 --- a/drivers/iio/adc/adi-axi-adc.c +++ b/drivers/iio/adc/adi-axi-adc.c @@ -54,6 +54,10 @@ #define AXI_AD485X_PACKET_FORMAT_24BIT 0x1 #define AXI_AD485X_PACKET_FORMAT_32BIT 0x2 #define AXI_AD408X_CNTRL_3_FILTER_EN_MSK BIT(0) +#define AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK GENMASK(3, 2) +#define AXI_AD408X_PACKET_FORMAT_20BIT 0x0 +#define AXI_AD408X_PACKET_FORMAT_16BIT 0x1 +#define AXI_AD408X_PACKET_FORMAT_14BIT 0x2 #define ADI_AXI_ADC_REG_SYNC_STATUS 0x0068 #define ADI_AXI_ADC_SYNC_STATUS_ADC_SYNC_MSK BIT(0) @@ -437,6 +441,31 @@ static int axi_adc_ad408x_filter_type_set(struct iio_backend *back, AXI_AD408X_CNTRL_3_FILTER_EN_MSK); } +static int axi_adc_ad408x_data_size_set(struct iio_backend *back, + unsigned int size) +{ + struct adi_axi_adc_state *st = iio_backend_get_priv(back); + unsigned int val; + + switch (size) { + case 20: + val = AXI_AD408X_PACKET_FORMAT_20BIT; + break; + case 16: + val = AXI_AD408X_PACKET_FORMAT_16BIT; + break; + case 14: + val = AXI_AD408X_PACKET_FORMAT_14BIT; + break; + default: + return -EINVAL; + } + + return regmap_update_bits(st->regmap, ADI_AXI_ADC_REG_CNTRL_3, + AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK, + FIELD_PREP(AXI_AD408X_CNTRL_3_PACKET_FORMAT_MSK, val)); +} + static int axi_adc_ad408x_interface_data_align(struct iio_backend *back, u32 timeout_us) { @@ -661,6 +690,7 @@ static const struct iio_backend_ops adi_ad408x_ops = { .free_buffer = axi_adc_free_buffer, .data_sample_trigger = axi_adc_data_sample_trigger, .filter_type_set = axi_adc_ad408x_filter_type_set, + .data_size_set = axi_adc_ad408x_data_size_set, .interface_data_align = axi_adc_ad408x_interface_data_align, .num_lanes_set = axi_adc_num_lanes_set, .debugfs_reg_access = iio_backend_debugfs_ptr(axi_adc_reg_access), diff --git a/drivers/iio/adc/pac1921.c b/drivers/iio/adc/pac1921.c index 35433250b0087..e8d9ced88906c 100644 --- a/drivers/iio/adc/pac1921.c +++ b/drivers/iio/adc/pac1921.c @@ -1037,7 +1037,7 @@ static irqreturn_t pac1921_trigger_handler(int irq, void *p) iio_for_each_active_channel(idev, bit) { u16 val; - ret = pac1921_read_res(priv, idev->channels[ch].address, &val); + ret = pac1921_read_res(priv, idev->channels[bit].address, &val); if (ret) goto done; diff --git a/drivers/iio/adc/qcom-spmi-iadc.c b/drivers/iio/adc/qcom-spmi-iadc.c index b64a8a407168b..0ec3a0c4b1de0 100644 --- a/drivers/iio/adc/qcom-spmi-iadc.c +++ b/drivers/iio/adc/qcom-spmi-iadc.c @@ -481,6 +481,11 @@ static const struct iio_chan_spec iadc_channels[] = { }, }; +static void iadc_disable_irq_wake(void *data) +{ + disable_irq_wake((unsigned long)data); +} + static int iadc_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; @@ -538,9 +543,16 @@ static int iadc_probe(struct platform_device *pdev) if (!iadc->poll_eoc) { ret = devm_request_irq(dev, irq_eoc, iadc_isr, 0, "spmi-iadc", iadc); - if (!ret) - enable_irq_wake(irq_eoc); - else + if (ret) + return ret; + + ret = enable_irq_wake(irq_eoc); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, iadc_disable_irq_wake, + (void *)(unsigned long)irq_eoc); + if (ret) return ret; } else { ret = devm_device_init_wakeup(iadc->dev); diff --git a/drivers/iio/adc/rtq6056.c b/drivers/iio/adc/rtq6056.c index 2bf3a09ac6b04..e2b1da13c0d31 100644 --- a/drivers/iio/adc/rtq6056.c +++ b/drivers/iio/adc/rtq6056.c @@ -728,7 +728,7 @@ static int rtq6056_probe(struct i2c_client *i2c) if (!i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_WORD_DATA)) return -EOPNOTSUPP; - devdata = device_get_match_data(dev); + devdata = i2c_get_match_data(i2c); if (!devdata) return dev_err_probe(dev, -EINVAL, "Invalid dev data\n"); @@ -871,6 +871,13 @@ static const struct richtek_dev_data rtq6059_devdata = { .set_average = rtq6059_adc_set_average, }; +static const struct i2c_device_id rtq6056_id[] = { + { "rtq6056", (kernel_ulong_t)&rtq6056_devdata }, + { "rtq6059", (kernel_ulong_t)&rtq6059_devdata }, + { } +}; +MODULE_DEVICE_TABLE(i2c, rtq6056_id); + static const struct of_device_id rtq6056_device_match[] = { { .compatible = "richtek,rtq6056", .data = &rtq6056_devdata }, { .compatible = "richtek,rtq6059", .data = &rtq6059_devdata }, @@ -885,6 +892,7 @@ static struct i2c_driver rtq6056_driver = { .pm = pm_ptr(&rtq6056_pm_ops), }, .probe = rtq6056_probe, + .id_table = rtq6056_id, }; module_i2c_driver(rtq6056_driver); diff --git a/drivers/iio/adc/ti-ads7138.c b/drivers/iio/adc/ti-ads7138.c index ee5c1b8e3a8e4..c4896161369bb 100644 --- a/drivers/iio/adc/ti-ads7138.c +++ b/drivers/iio/adc/ti-ads7138.c @@ -227,6 +227,26 @@ static int ads7138_osr_to_bits(int osr) return -EINVAL; } +static int ads7138_read_statistics(const struct i2c_client *client, u8 reg, + u8 *out_values, u8 length) +{ + int ret; + + /* Disable statistics update so the value is not updated mid read */ + ret = ads7138_i2c_clear_bit(client, ADS7138_REG_GENERAL_CFG, + ADS7138_GENERAL_CFG_STATS_EN); + if (ret) + return ret; + + ret = ads7138_i2c_read_block(client, reg, out_values, length); + if (ret) + return ret; + + /* Enable statistics update after read */ + return ads7138_i2c_set_bit(client, ADS7138_REG_GENERAL_CFG, + ADS7138_GENERAL_CFG_STATS_EN); +} + static int ads7138_read_raw(struct iio_dev *indio_dev, struct iio_chan_spec const *chan, int *val, int *val2, long mask) @@ -236,28 +256,32 @@ static int ads7138_read_raw(struct iio_dev *indio_dev, u8 values[2]; switch (mask) { + /* + * Reading the statistics registers reinitializes them. This is + * unfortunate but necessary to prevent data races. + */ case IIO_CHAN_INFO_RAW: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_RECENT_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_RECENT_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; *val = get_unaligned_le16(values); return IIO_VAL_INT; case IIO_CHAN_INFO_PEAK: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_MAX_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_MAX_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; *val = get_unaligned_le16(values); return IIO_VAL_INT; case IIO_CHAN_INFO_TROUGH: - ret = ads7138_i2c_read_block(data->client, - ADS7138_REG_MIN_LSB_CH(chan->channel), - values, ARRAY_SIZE(values)); + ret = ads7138_read_statistics(data->client, + ADS7138_REG_MIN_LSB_CH(chan->channel), + values, ARRAY_SIZE(values)); if (ret) return ret; diff --git a/drivers/iio/chemical/atlas-sensor.c b/drivers/iio/chemical/atlas-sensor.c index 8bbba85af699a..8694c173b95f6 100644 --- a/drivers/iio/chemical/atlas-sensor.c +++ b/drivers/iio/chemical/atlas-sensor.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -88,7 +87,6 @@ struct atlas_data { struct iio_trigger *trig; const struct atlas_device *chip; struct regmap *regmap; - struct irq_work work; unsigned int interrupt_enabled; /* 96-bit data + 32-bit pad + 64-bit timestamp */ __be32 buffer[6] __aligned(8); @@ -413,7 +411,11 @@ static int atlas_buffer_postenable(struct iio_dev *indio_dev) if (ret) return ret; - return atlas_set_interrupt(data, true); + ret = atlas_set_interrupt(data, true); + if (ret) + pm_runtime_put_autosuspend(&data->client->dev); + + return ret; } static int atlas_buffer_predisable(struct iio_dev *indio_dev) @@ -437,13 +439,6 @@ static const struct iio_buffer_setup_ops atlas_buffer_setup_ops = { .predisable = atlas_buffer_predisable, }; -static void atlas_work_handler(struct irq_work *work) -{ - struct atlas_data *data = container_of(work, struct atlas_data, work); - - iio_trigger_poll(data->trig); -} - static irqreturn_t atlas_trigger_handler(int irq, void *private) { struct iio_poll_func *pf = private; @@ -470,7 +465,7 @@ static irqreturn_t atlas_interrupt_handler(int irq, void *private) struct iio_dev *indio_dev = private; struct atlas_data *data = iio_priv(indio_dev); - irq_work_queue(&data->work); + iio_trigger_poll_nested(data->trig); return IRQ_HANDLED; } @@ -666,8 +661,6 @@ static int atlas_probe(struct i2c_client *client) goto unregister_trigger; } - init_irq_work(&data->work, atlas_work_handler); - if (client->irq > 0) { /* interrupt pin toggles on new conversion */ ret = devm_request_threaded_irq(&client->dev, client->irq, diff --git a/drivers/iio/chemical/sgp30.c b/drivers/iio/chemical/sgp30.c index 21730d62b5c8d..cb5e7bb3f1919 100644 --- a/drivers/iio/chemical/sgp30.c +++ b/drivers/iio/chemical/sgp30.c @@ -548,6 +548,9 @@ static int sgp_probe(struct i2c_client *client) data->iaq_thread = kthread_run(sgp_iaq_threadfn, data, "%s-iaq", data->client->name); + if (IS_ERR(data->iaq_thread)) + return dev_err_probe(dev, PTR_ERR(data->iaq_thread), + "failed to start IAQ thread\n"); return 0; } diff --git a/drivers/iio/dac/ad3552r-hs.c b/drivers/iio/dac/ad3552r-hs.c index 6bc64f53bce91..5d6c2517a7a80 100644 --- a/drivers/iio/dac/ad3552r-hs.c +++ b/drivers/iio/dac/ad3552r-hs.c @@ -591,7 +591,7 @@ static ssize_t ad3552r_hs_show_data_source_avail(struct file *f, int i; for (i = 0; i < ARRAY_SIZE(dbgfs_attr_source); i++) { - len += scnprintf(buf + len, PAGE_SIZE - len, "%s ", + len += scnprintf(buf + len, sizeof(buf) - len, "%s ", dbgfs_attr_source[i]); } buf[len - 1] = '\n'; diff --git a/drivers/iio/dac/m62332.c b/drivers/iio/dac/m62332.c index 3497513854d7d..2c13feee8d619 100644 --- a/drivers/iio/dac/m62332.c +++ b/drivers/iio/dac/m62332.c @@ -32,6 +32,7 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) { struct m62332_data *data = iio_priv(indio_dev); struct i2c_client *client = data->client; + bool enabling, disabling; u8 outbuf[2]; int res; @@ -43,7 +44,10 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) mutex_lock(&data->mutex); - if (val) { + enabling = val && !data->raw[channel]; + disabling = !val && data->raw[channel]; + + if (enabling) { res = regulator_enable(data->vcc); if (res) goto out; @@ -52,14 +56,17 @@ static int m62332_set_value(struct iio_dev *indio_dev, u8 val, int channel) res = i2c_master_send(client, outbuf, ARRAY_SIZE(outbuf)); if (res >= 0 && res != ARRAY_SIZE(outbuf)) res = -EIO; - if (res < 0) + if (res < 0) { + if (enabling) + regulator_disable(data->vcc); goto out; + } - data->raw[channel] = val; - - if (!val) + if (disabling) regulator_disable(data->vcc); + data->raw[channel] = val; + mutex_unlock(&data->mutex); return 0; diff --git a/drivers/iio/gyro/mpu3050-core.c b/drivers/iio/gyro/mpu3050-core.c index d84e04e4b4314..07c0f9011c637 100644 --- a/drivers/iio/gyro/mpu3050-core.c +++ b/drivers/iio/gyro/mpu3050-core.c @@ -356,7 +356,7 @@ static int mpu3050_read_raw(struct iio_dev *indio_dev, goto out_read_raw_unlock; } - *val = be16_to_cpu(raw_val); + *val = (s16)be16_to_cpu(raw_val); ret = IIO_VAL_INT; goto out_read_raw_unlock; diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index 69191aaaf808c..7d3a8f92d0647 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -47,9 +47,6 @@ struct iio_dmabuf_priv { u64 context; - /* Spinlock used for locking the dma_fence */ - spinlock_t lock; - struct dma_buf_attachment *attach; struct sg_table *sgt; enum dma_data_direction dir; @@ -57,7 +54,12 @@ struct iio_dmabuf_priv { }; struct iio_dma_fence { + /* + * Must remain the first member so the default release callback can pass + * the fence directly to dma_fence_free(). + */ struct dma_fence base; + spinlock_t lock; /* protects base */ struct iio_dmabuf_priv *priv; struct work_struct work; }; @@ -1599,12 +1601,16 @@ static int iio_buffer_chrdev_release(struct inode *inode, struct file *filep) wake_up(&buffer->pollq); - guard(mutex)(&buffer->dmabufs_mutex); - - /* Close all attached DMABUFs */ - list_for_each_entry_safe(priv, tmp, &buffer->dmabufs, entry) { - list_del_init(&priv->entry); - iio_buffer_dmabuf_put(priv->attach); + /* + * The mutex must be unlocked before iio_device_put(), which might drop the + * last reference and free the buffer. + */ + scoped_guard(mutex, &buffer->dmabufs_mutex) { + /* Close all attached DMABUFs */ + list_for_each_entry_safe(priv, tmp, &buffer->dmabufs, entry) { + list_del_init(&priv->entry); + iio_buffer_dmabuf_put(priv->attach); + } } kfree(ib); @@ -1682,7 +1688,6 @@ static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib, if (!priv) return -ENOMEM; - spin_lock_init(&priv->lock); priv->context = dma_fence_context_alloc(1); dmabuf = dma_buf_get(fd); @@ -1807,18 +1812,9 @@ iio_buffer_dma_fence_get_driver_name(struct dma_fence *fence) return "iio"; } -static void iio_buffer_dma_fence_release(struct dma_fence *fence) -{ - struct iio_dma_fence *iio_fence = - container_of(fence, struct iio_dma_fence, base); - - kfree(iio_fence); -} - static const struct dma_fence_ops iio_buffer_dma_fence_ops = { .get_driver_name = iio_buffer_dma_fence_get_driver_name, .get_timeline_name = iio_buffer_dma_fence_get_driver_name, - .release = iio_buffer_dma_fence_release, }; static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, @@ -1872,6 +1868,8 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, goto err_attachment_put; } + spin_lock_init(&fence->lock); + fence->priv = priv; seqno = atomic_add_return(1, &priv->seqno); @@ -1882,7 +1880,7 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib, * the dma_fence. */ dma_fence_init(&fence->base, &iio_buffer_dma_fence_ops, - &priv->lock, priv->context, seqno); + &fence->lock, priv->context, seqno); ret = iio_dma_resv_lock(dmabuf, nonblock); if (ret) diff --git a/drivers/iio/light/apds9306.c b/drivers/iio/light/apds9306.c index 389125675caa4..5ce01037ae554 100644 --- a/drivers/iio/light/apds9306.c +++ b/drivers/iio/light/apds9306.c @@ -471,9 +471,9 @@ static int apds9306_read_data(struct apds9306_data *data, int *val, int reg) int status = 0; u8 buff[3]; - ret = pm_runtime_resume_and_get(data->dev); - if (ret) - return ret; + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(data->dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return PM_RUNTIME_ACQUIRE_ERR(&pm); ret = regmap_field_read(rf->intg_time, &intg_time_idx); if (ret) @@ -537,8 +537,6 @@ static int apds9306_read_data(struct apds9306_data *data, int *val, int reg) *val = get_unaligned_le24(&buff); - pm_runtime_put_autosuspend(data->dev); - return 0; } diff --git a/drivers/iio/light/cm32181.c b/drivers/iio/light/cm32181.c index bb90f738312a3..04c6808c41b36 100644 --- a/drivers/iio/light/cm32181.c +++ b/drivers/iio/light/cm32181.c @@ -369,7 +369,7 @@ static int cm32181_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_CALIBSCALE: cm32181->calibscale = val; - return val; + return 0; case IIO_CHAN_INFO_INT_TIME: ret = cm32181_write_als_it(cm32181, val2); return ret; diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c index 886accf5e8591..bb17ef2a829b7 100644 --- a/drivers/iio/light/gp2ap002.c +++ b/drivers/iio/light/gp2ap002.c @@ -342,6 +342,10 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, bool state) { struct gp2ap002 *gp2ap002 = iio_priv(indio_dev); + int ret; + + if (state == gp2ap002->enabled) + return 0; if (state) { /* @@ -349,13 +353,16 @@ static int gp2ap002_write_event_config(struct iio_dev *indio_dev, * already) and reintialize the sensor by using runtime_pm * callbacks. */ - pm_runtime_get_sync(gp2ap002->dev); - gp2ap002->enabled = true; + ret = pm_runtime_resume_and_get(gp2ap002->dev); + if (ret) + return ret; + } else { pm_runtime_put_autosuspend(gp2ap002->dev); - gp2ap002->enabled = false; } + gp2ap002->enabled = state; + return 0; } @@ -642,6 +649,7 @@ static int gp2ap002_runtime_suspend(struct device *dev) /* Disable chip and IRQ, everything off */ ret = regmap_write(gp2ap002->map, GP2AP002_OPMOD, 0x00); if (ret) { + enable_irq(gp2ap002->irq); dev_err(gp2ap002->dev, "error setting up operation mode\n"); return ret; } @@ -669,7 +677,7 @@ static int gp2ap002_runtime_resume(struct device *dev) ret = regulator_enable(gp2ap002->vio); if (ret) { dev_err(dev, "failed to enable VIO regulator in resume path\n"); - return ret; + goto out_disable_vdd; } msleep(20); @@ -677,13 +685,19 @@ static int gp2ap002_runtime_resume(struct device *dev) ret = gp2ap002_init(gp2ap002); if (ret) { dev_err(dev, "re-initialization failed\n"); - return ret; + goto out_disable_vio; } /* Re-activate the IRQ */ enable_irq(gp2ap002->irq); return 0; + +out_disable_vio: + regulator_disable(gp2ap002->vio); +out_disable_vdd: + regulator_disable(gp2ap002->vdd); + return ret; } static DEFINE_RUNTIME_DEV_PM_OPS(gp2ap002_dev_pm_ops, gp2ap002_runtime_suspend, diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c index 374bccad9119a..f89c5d9ee5240 100644 --- a/drivers/iio/light/isl29028.c +++ b/drivers/iio/light/isl29028.c @@ -409,7 +409,7 @@ static int isl29028_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static int isl29028_read_raw(struct iio_dev *indio_dev, diff --git a/drivers/iio/light/ltrf216a.c b/drivers/iio/light/ltrf216a.c index 5f27f754fe1c4..48296493fc056 100644 --- a/drivers/iio/light/ltrf216a.c +++ b/drivers/iio/light/ltrf216a.c @@ -248,11 +248,10 @@ static int ltrf216a_get_lux(struct ltrf216a_data *data) return ret; greendata = ltrf216a_read_data(data, LTRF216A_ALS_DATA_0); + ltrf216a_set_power_state(data, false); if (greendata < 0) return greendata; - ltrf216a_set_power_state(data, false); - lux = greendata * data->info->lux_multiplier * LTRF216A_WIN_FAC; return lux; diff --git a/drivers/iio/light/opt4001.c b/drivers/iio/light/opt4001.c index 95167273bb904..a24684666c30c 100644 --- a/drivers/iio/light/opt4001.c +++ b/drivers/iio/light/opt4001.c @@ -39,7 +39,7 @@ #define OPT4001_CTRL_OPER_MODE_MASK GENMASK(5, 4) #define OPT4001_CTRL_LATCH_MASK GENMASK(3, 3) #define OPT4001_CTRL_INT_POL_MASK GENMASK(2, 2) -#define OPT4001_CTRL_FAULT_COUNT GENMASK(0, 1) +#define OPT4001_CTRL_FAULT_COUNT_MASK GENMASK(1, 0) /* OPT4001 constants */ #define OPT4001_DEVICE_ID_VAL 0x121 @@ -173,6 +173,7 @@ static int opt4001_read_lux_value(struct iio_dev *indio_dev, u8 crc; u8 calc_crc; u64 lux_raw; + u32 rem; int ret; ret = regmap_read(chip->regmap, OPT4001_LIGHT1_MSB, &light1); @@ -199,8 +200,8 @@ static int opt4001_read_lux_value(struct iio_dev *indio_dev, lux_raw = lux_raw << exp; lux_raw = lux_raw * chip->chip_info->mul; - *val = div_u64_rem(lux_raw, chip->chip_info->div, val2); - *val2 = *val2 * 100; + *val = div_u64_rem(lux_raw, chip->chip_info->div, &rem); + *val2 = rem * 100; return IIO_VAL_INT_PLUS_NANO; } @@ -222,33 +223,14 @@ static int opt4001_set_conf(struct opt4001_chip *chip) return ret; } -static int opt4001_power_down(struct opt4001_chip *chip) -{ - struct device *dev = &chip->client->dev; - int ret; - unsigned int reg; - - ret = regmap_read(chip->regmap, OPT4001_DEVICE_ID, ®); - if (ret) { - dev_err(dev, "Failed to read configuration\n"); - return ret; - } - - /* MODE_OFF is 0x0 so just set bits to 0 */ - reg &= ~OPT4001_CTRL_OPER_MODE_MASK; - - ret = regmap_write(chip->regmap, OPT4001_CTRL, reg); - if (ret) - dev_err(dev, "Failed to set configuration to power down\n"); - - return ret; -} - static void opt4001_chip_off_action(void *data) { struct opt4001_chip *chip = data; + int ret; - opt4001_power_down(chip); + ret = regmap_clear_bits(chip->regmap, OPT4001_CTRL, OPT4001_CTRL_OPER_MODE_MASK); + if (ret) + dev_err(&chip->client->dev, "Failed to power down\n"); } static const struct iio_chan_spec opt4001_channels[] = { @@ -287,6 +269,9 @@ static int opt4001_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_INT_TIME: + if (val) + return -EINVAL; + int_time = opt4001_als_time_to_index(val2); if (int_time < 0) return int_time; diff --git a/drivers/iio/light/opt4060.c b/drivers/iio/light/opt4060.c index 981c704e7df56..9be752535d750 100644 --- a/drivers/iio/light/opt4060.c +++ b/drivers/iio/light/opt4060.c @@ -650,6 +650,9 @@ static int opt4060_write_raw(struct iio_dev *indio_dev, switch (mask) { case IIO_CHAN_INFO_INT_TIME: + if (val) + return -EINVAL; + int_time = opt4060_als_time_to_index(val2); if (int_time < 0) return int_time; @@ -728,6 +731,7 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, { int ret, pers, fault_count, int_time; u64 uval; + u32 rem; int_time = opt4060_int_time_reg[chip->int_time][0]; @@ -753,7 +757,8 @@ static ssize_t opt4060_read_ev_period(struct opt4060_chip *chip, int *val, } uval = mul_u32_u32(int_time, pers); - *val = div_u64_rem(uval, MICRO, val2); + *val = div_u64_rem(uval, MICRO, &rem); + *val2 = rem; return IIO_VAL_INT_PLUS_MICRO; } @@ -823,7 +828,7 @@ static int opt4060_get_thresholds(struct opt4060_chip *chip, u32 *th_lo, u32 *th ret = regmap_read(chip->regmap, OPT4060_THRESHOLD_HIGH, ®val); if (ret) { - dev_err(chip->dev, "Failed to read THRESHOLD_LOW.\n"); + dev_err(chip->dev, "Failed to read THRESHOLD_HIGH.\n"); return ret; } *th_hi = opt4060_calc_val_from_th_reg(regval); diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c index a75a83594a7ee..3be6934218866 100644 --- a/drivers/iio/light/stk3310.c +++ b/drivers/iio/light/stk3310.c @@ -117,6 +117,9 @@ struct stk3310_data { struct mutex lock; bool als_enabled; bool ps_enabled; + bool ps_int_enabled; + uint32_t ps_thdl; + uint32_t ps_thdh; uint32_t ps_near_level; u64 timestamp; struct regmap *regmap; @@ -296,10 +299,17 @@ static int stk3310_write_event(struct iio_dev *indio_dev, buf = cpu_to_be16(val); ret = regmap_bulk_write(data->regmap, reg, &buf, 2); - if (ret < 0) + if (ret < 0) { dev_err(&client->dev, "failed to set PS threshold!\n"); + return ret; + } - return ret; + if (reg == STK3310_REG_THDH_PS) + data->ps_thdh = val; + else + data->ps_thdl = val; + + return 0; } static int stk3310_read_event_config(struct iio_dev *indio_dev, @@ -331,11 +341,17 @@ static int stk3310_write_event_config(struct iio_dev *indio_dev, /* Set INT_PS value */ mutex_lock(&data->lock); ret = regmap_field_write(data->reg_int_ps, state); - if (ret < 0) + if (ret < 0) { dev_err(&client->dev, "failed to set interrupt mode\n"); + mutex_unlock(&data->lock); + return ret; + } + + data->ps_int_enabled = state; + mutex_unlock(&data->lock); - return ret; + return 0; } static int stk3310_read_raw(struct iio_dev *indio_dev, @@ -504,10 +520,15 @@ static int stk3310_init(struct iio_dev *indio_dev) /* Enable PS interrupts */ ret = regmap_field_write(data->reg_int_ps, STK3310_PSINT_EN); - if (ret < 0) + if (ret < 0) { dev_err(&client->dev, "failed to enable interrupts!\n"); + return ret; + } - return ret; + data->ps_int_enabled = true; + data->ps_thdh = STK3310_PS_MAX_VAL; + + return 0; } static bool stk3310_is_volatile_reg(struct device *dev, unsigned int reg) @@ -671,9 +692,18 @@ static void stk3310_remove(struct i2c_client *client) static int stk3310_suspend(struct device *dev) { struct stk3310_data *data; + int ret; data = iio_priv(i2c_get_clientdata(to_i2c_client(dev))); + if (data->ps_int_enabled) { + ret = regmap_field_write(data->reg_int_ps, 0x0); + if (ret < 0) { + dev_err(dev, "failed to disable ps int at suspend.\n"); + return ret; + } + } + return stk3310_set_state(data, STK3310_STATE_STANDBY); } @@ -681,6 +711,8 @@ static int stk3310_resume(struct device *dev) { u8 state = 0; struct stk3310_data *data; + __be16 buf; + int ret; data = iio_priv(i2c_get_clientdata(to_i2c_client(dev))); if (data->ps_enabled) @@ -688,7 +720,37 @@ static int stk3310_resume(struct device *dev) if (data->als_enabled) state |= STK3310_STATE_EN_ALS; - return stk3310_set_state(data, state); + ret = stk3310_set_state(data, state); + if (ret < 0) + return ret; + + if (data->ps_thdl != 0x0) { + buf = cpu_to_be16(data->ps_thdl); + ret = regmap_bulk_write(data->regmap, STK3310_REG_THDL_PS, &buf, 2); + if (ret < 0) { + dev_err(dev, "failed to set reg THDL_PS at resume.\n"); + return ret; + } + } + + if (data->ps_thdh != STK3310_PS_MAX_VAL) { + buf = cpu_to_be16(data->ps_thdh); + ret = regmap_bulk_write(data->regmap, STK3310_REG_THDH_PS, &buf, 2); + if (ret < 0) { + dev_err(dev, "failed to set reg THDH_PS at resume.\n"); + return ret; + } + } + + if (data->ps_int_enabled) { + ret = regmap_field_write(data->reg_int_ps, STK3310_PSINT_EN); + if (ret < 0) { + dev_err(dev, "failed to enable ps int at resume.\n"); + return ret; + } + } + + return 0; } static DEFINE_SIMPLE_DEV_PM_OPS(stk3310_pm_ops, stk3310_suspend, diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c index 8801a491de77b..15693bc3e8e85 100644 --- a/drivers/iio/light/tsl2583.c +++ b/drivers/iio/light/tsl2583.c @@ -794,7 +794,7 @@ static int tsl2583_write_raw(struct iio_dev *indio_dev, if (ret < 0) return ret; - return ret; + return 0; } static const struct iio_info tsl2583_info = { diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c index 0b171106441ab..ffbde15401141 100644 --- a/drivers/iio/light/tsl2772.c +++ b/drivers/iio/light/tsl2772.c @@ -1263,7 +1263,7 @@ static int tsl2772_read_raw(struct iio_dev *indio_dev, } break; case IIO_CHAN_INFO_CALIBSCALE: - if (chan->type == IIO_LIGHT) + if (chan->type == IIO_INTENSITY) *val = tsl2772_als_gain[chip->settings.als_gain]; else *val = tsl2772_prox_gain[chip->settings.prox_gain]; diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c index 83a0b0283605b..061934f0cb999 100644 --- a/drivers/iio/orientation/hid-sensor-rotation.c +++ b/drivers/iio/orientation/hid-sensor-rotation.c @@ -353,12 +353,6 @@ static int hid_dev_rot_probe(struct platform_device *pdev) return ret; } - ret = iio_device_register(indio_dev); - if (ret) { - dev_err(&pdev->dev, "device register failed\n"); - goto error_remove_trigger; - } - rot_state->callbacks.send_event = dev_rot_proc_event; rot_state->callbacks.capture_sample = dev_rot_capture_sample; rot_state->callbacks.pdev = pdev; @@ -366,13 +360,19 @@ static int hid_dev_rot_probe(struct platform_device *pdev) &rot_state->callbacks); if (ret) { dev_err(&pdev->dev, "callback reg failed\n"); - goto error_iio_unreg; + goto error_remove_trigger; + } + + ret = iio_device_register(indio_dev); + if (ret) { + dev_err(&pdev->dev, "device register failed\n"); + goto error_remove_callback; } return 0; -error_iio_unreg: - iio_device_unregister(indio_dev); +error_remove_callback: + sensor_hub_remove_callback(hsdev, hsdev->usage); error_remove_trigger: hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); return ret; @@ -385,8 +385,8 @@ static void hid_dev_rot_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct dev_rot_state *rot_state = iio_priv(indio_dev); - sensor_hub_remove_callback(hsdev, hsdev->usage); iio_device_unregister(indio_dev); + sensor_hub_remove_callback(hsdev, hsdev->usage); hid_sensor_remove_trigger(indio_dev, &rot_state->common_attributes); } diff --git a/drivers/iio/pressure/dps310.c b/drivers/iio/pressure/dps310.c index 8edaa4d10a70e..41b2d0b014e18 100644 --- a/drivers/iio/pressure/dps310.c +++ b/drivers/iio/pressure/dps310.c @@ -845,7 +845,6 @@ static const struct iio_info dps310_info = { static int dps310_probe(struct i2c_client *client) { - const struct i2c_device_id *id = i2c_client_get_device_id(client); struct dps310_data *data; struct iio_dev *iio; int rc; @@ -858,7 +857,7 @@ static int dps310_probe(struct i2c_client *client) data->client = client; mutex_init(&data->lock); - iio->name = id->name; + iio->name = DPS310_DEV_NAME; iio->channels = dps310_channels; iio->num_channels = ARRAY_SIZE(dps310_channels); iio->info = &dps310_info; diff --git a/drivers/iio/pressure/mpl115.c b/drivers/iio/pressure/mpl115.c index 16e112b796ba9..2929ea8e4e01b 100644 --- a/drivers/iio/pressure/mpl115.c +++ b/drivers/iio/pressure/mpl115.c @@ -203,9 +203,9 @@ int mpl115_probe(struct device *dev, const char *name, if (data->shutdown) { /* Enable runtime PM */ - pm_runtime_get_noresume(dev); - pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = pm_runtime_set_active(dev); + if (ret) + return ret; /* * As the device takes 3 ms to come up with a fresh @@ -215,7 +215,10 @@ int mpl115_probe(struct device *dev, const char *name, */ pm_runtime_set_autosuspend_delay(dev, 2000); pm_runtime_use_autosuspend(dev); - pm_runtime_put(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; dev_dbg(dev, "low-power mode enabled"); } else diff --git a/drivers/iio/proximity/srf04.c b/drivers/iio/proximity/srf04.c index e97f9a20ac7a9..337938657449d 100644 --- a/drivers/iio/proximity/srf04.c +++ b/drivers/iio/proximity/srf04.c @@ -331,6 +331,7 @@ static int srf04_probe(struct platform_device *pdev) if (ret) { dev_err(data->dev, "pm_runtime_set_active: %d\n", ret); iio_device_unregister(indio_dev); + return ret; } pm_runtime_enable(data->dev); diff --git a/drivers/iio/temperature/hid-sensor-temperature.c b/drivers/iio/temperature/hid-sensor-temperature.c index 9f628a8e5cfbe..34bff7e9f3a33 100644 --- a/drivers/iio/temperature/hid-sensor-temperature.c +++ b/drivers/iio/temperature/hid-sensor-temperature.c @@ -244,7 +244,7 @@ static int hid_temperature_probe(struct platform_device *pdev) if (ret) goto error_remove_trigger; - ret = devm_iio_device_register(indio_dev->dev.parent, indio_dev); + ret = iio_device_register(indio_dev); if (ret) goto error_remove_callback; @@ -264,6 +264,7 @@ static void hid_temperature_remove(struct platform_device *pdev) struct iio_dev *indio_dev = platform_get_drvdata(pdev); struct temperature_state *temp_st = iio_priv(indio_dev); + iio_device_unregister(indio_dev); sensor_hub_remove_callback(hsdev, HID_USAGE_SENSOR_TEMPERATURE); hid_sensor_remove_trigger(indio_dev, &temp_st->common_attributes); } diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c index ce511800b056c..a73a879706c4f 100644 --- a/drivers/infiniband/core/cma.c +++ b/drivers/infiniband/core/cma.c @@ -3523,10 +3523,12 @@ static void addr_handler(int status, struct sockaddr *src_addr, memcpy(addr, src_addr, rdma_addr_size(src_addr)); if (!status && !id_priv->cma_dev) { status = cma_acquire_dev_by_src_ip(id_priv); - if (status) + if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n", status); - rdma_restrack_add(&id_priv->res); + } else { + rdma_restrack_add(&id_priv->res); + } } else if (status) { pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status); } @@ -5237,7 +5239,7 @@ static int cma_netevent_callback(struct notifier_block *self, list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) { if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr, - neigh->ha, ETH_ALEN)) + neigh->ha, neigh->dev->addr_len)) continue; cma_id_get(current_id); if (!queue_work(cma_wq, ¤t_id->id.net_work)) diff --git a/drivers/infiniband/core/counters.c b/drivers/infiniband/core/counters.c index 5dad5d77ce274..a2c85840c501a 100644 --- a/drivers/infiniband/core/counters.c +++ b/drivers/infiniband/core/counters.c @@ -198,12 +198,20 @@ static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, ret = __rdma_counter_bind_qp(counter, qp, port); if (ret) - goto err_mode; + goto err_bind; rdma_restrack_parent_name(&counter->res, &qp->res); rdma_restrack_add(&counter->res); return counter; +err_bind: + mutex_lock(&port_counter->lock); + port_counter->num_counters--; + if (!port_counter->num_counters && + port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL) + __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, + false); + mutex_unlock(&port_counter->lock); err_mode: rdma_free_hw_stats_struct(counter->stats); err_stats: @@ -226,7 +234,6 @@ static void rdma_counter_free(struct rdma_counter *counter) mutex_unlock(&port_counter->lock); - rdma_restrack_del(&counter->res); rdma_free_hw_stats_struct(counter->stats); kfree(counter); } @@ -321,6 +328,7 @@ static void counter_release(struct kref *kref) counter = container_of(kref, struct rdma_counter, kref); counter_history_stat_update(counter); + rdma_restrack_del(&counter->res); counter->device->ops.counter_dealloc(counter); rdma_counter_free(counter); } @@ -482,7 +490,8 @@ static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, return NULL; counter = container_of(res, struct rdma_counter, res); - kref_get(&counter->kref); + if (!kref_get_unless_zero(&counter->kref)) + counter = NULL; rdma_restrack_put(res); return counter; diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 584537c71545c..4ea0dfcff95ea 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -324,6 +324,7 @@ void ib_free_cq(struct ib_cq *cq) if (WARN_ON_ONCE(cq->cqe_used)) return; + rdma_restrack_del(&cq->res); if (cq->device->ops.pre_destroy_cq) { ret = cq->device->ops.pre_destroy_cq(cq); WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail"); @@ -350,7 +351,6 @@ void ib_free_cq(struct ib_cq *cq) else ret = cq->device->ops.destroy_cq(cq, NULL); WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail"); - rdma_restrack_del(&cq->res); kfree(cq->wc); kfree(cq); } diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c index ac9aaef1e5e61..c8b7942739a0a 100644 --- a/drivers/infiniband/core/device.c +++ b/drivers/infiniband/core/device.c @@ -2848,6 +2848,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops) SET_DEVICE_OP(dev_ops, query_gid); SET_DEVICE_OP(dev_ops, query_pkey); SET_DEVICE_OP(dev_ops, query_port); + SET_DEVICE_OP(dev_ops, query_port_speed); SET_DEVICE_OP(dev_ops, query_qp); SET_DEVICE_OP(dev_ops, query_srq); SET_DEVICE_OP(dev_ops, query_ucontext); @@ -3169,6 +3170,7 @@ static void __exit ib_core_cleanup(void) /* Make sure that any pending umem accounting work is done. */ destroy_workqueue(ib_wq); destroy_workqueue(ib_unreg_wq); + rcu_barrier(); WARN_ON(!xa_empty(&clients)); WARN_ON(!xa_empty(&devices)); } diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c index 8f26bfb695861..5150cd53d4435 100644 --- a/drivers/infiniband/core/mad.c +++ b/drivers/infiniband/core/mad.c @@ -2031,6 +2031,24 @@ void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr) change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP); } +static bool is_kernel_rmpp_data_response(struct ib_mad_agent_private *agent, + struct ib_mad_recv_wc *mad_recv_wc) +{ + const struct ib_mad_hdr *mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr; + struct ib_rmpp_mad *rmpp_mad; + + if (!ib_mad_kernel_rmpp_agent(&agent->agent) || + !ib_response_mad(mad_hdr) || + !ib_is_mad_class_rmpp(mad_hdr->mgmt_class)) + return false; + + rmpp_mad = (struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad; + + return (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & + IB_MGMT_RMPP_FLAG_ACTIVE) && + rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA; +} + static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, struct ib_mad_recv_wc *mad_recv_wc) { @@ -2050,6 +2068,18 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv, } list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list); + if (is_kernel_rmpp_data_response(mad_agent_priv, mad_recv_wc)) { + spin_lock_irqsave(&mad_agent_priv->lock, flags); + mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc); + spin_unlock_irqrestore(&mad_agent_priv->lock, flags); + + if (!mad_send_wr) { + ib_free_recv_mad(mad_recv_wc); + deref_mad_agent(mad_agent_priv); + return; + } + } + if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) { mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv, mad_recv_wc); diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c index 2220a2dfab240..c92c24e681f11 100644 --- a/drivers/infiniband/core/nldev.c +++ b/drivers/infiniband/core/nldev.c @@ -679,7 +679,7 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (has_cap_net_admin) { if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_RKEY, mr->rkey)) @@ -695,9 +695,12 @@ static int fill_res_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) return -EMSGSIZE; - if (!rdma_is_kernel_res(res) && - nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, mr->pd->res.id)) - return -EMSGSIZE; + if (!rdma_is_kernel_res(res)) { + struct ib_pd *pd = READ_ONCE(mr->pd); + + if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_PDN, pd->res.id)) + return -EMSGSIZE; + } if (fill_res_name_pid(msg, res)) return -EMSGSIZE; @@ -711,7 +714,7 @@ static int fill_res_mr_raw_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (!dev->ops.fill_res_mr_entry_raw) return -EINVAL; @@ -1001,7 +1004,7 @@ static int fill_stat_mr_entry(struct sk_buff *msg, bool has_cap_net_admin, struct rdma_restrack_entry *res, uint32_t port) { struct ib_mr *mr = container_of(res, struct ib_mr, res); - struct ib_device *dev = mr->pd->device; + struct ib_device *dev = mr->device; if (nla_put_u32(msg, RDMA_NLDEV_ATTR_RES_MRN, res->id)) goto err; @@ -2100,6 +2103,11 @@ static int nldev_stat_set_counter_dynamic_doit(struct nlattr *tb[], nla_for_each_nested(entry_attr, tb[RDMA_NLDEV_ATTR_STAT_HWCOUNTERS], rem) { + if (nla_len(entry_attr) != sizeof(u32)) { + ret = -EINVAL; + goto out; + } + index = nla_get_u32(entry_attr); if ((index >= stats->num_counters) || !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) { diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c index a7de6f403fcaf..56aa1def56fac 100644 --- a/drivers/infiniband/core/restrack.c +++ b/drivers/infiniband/core/restrack.c @@ -71,6 +71,8 @@ int rdma_restrack_count(struct ib_device *dev, enum rdma_restrack_type type, xa_lock(&rt->xa); xas_for_each(&xas, e, U32_MAX) { + if (xa_is_zero(e)) + continue; if (xa_get_mark(&rt->xa, e->id, RESTRACK_DD) && !show_details) continue; cnt++; @@ -127,6 +129,46 @@ static void rdma_restrack_attach_task(struct rdma_restrack_entry *res, res->user = true; } +static struct rdma_restrack_root *res_to_rt(struct rdma_restrack_entry *res) +{ + struct ib_device *dev = res_to_dev(res); + + if (WARN_ON(!dev)) + return NULL; + + return &dev->res[res->type]; +} + +static void restrack_drain_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, res, XA_ZERO_ENTRY, + GFP_KERNEL); + WARN_ON(old != res); + } + + rdma_restrack_put(res); + wait_for_completion(&res->comp); +} + +static void restrack_restore_res(struct rdma_restrack_root *rt, + struct rdma_restrack_entry *res) +{ + reinit_completion(&res->comp); + kref_init(&res->kref); + + if (rt) { + struct rdma_restrack_entry *old; + + old = xa_cmpxchg(&rt->xa, res->id, XA_ZERO_ENTRY, res, + GFP_KERNEL); + WARN_ON(old); + } +} + /** * rdma_restrack_set_name() - set the task for this resource * @res: resource entry @@ -175,22 +217,23 @@ void rdma_restrack_new(struct rdma_restrack_entry *res, EXPORT_SYMBOL(rdma_restrack_new); /** - * rdma_restrack_add() - add object to the reource tracking database + * rdma_restrack_add() - add object to the resource tracking database. + * If this resource reuses an ID of a resource that was already destroyed + * after calling rdma_restrack_begin() but didn't yet call + * rdma_restrack_commit_del() it can result in an untracked QP. * @res: resource entry */ void rdma_restrack_add(struct rdma_restrack_entry *res) { - struct ib_device *dev = res_to_dev(res); struct rdma_restrack_root *rt; int ret = 0; - if (!dev) - return; - if (res->no_track) goto out; - rt = &dev->res[res->type]; + rt = res_to_rt(res); + if (!rt) + return; if (res->type == RDMA_RESTRACK_QP) { /* Special case to ensure that LQPN points to right QP */ @@ -227,6 +270,28 @@ void rdma_restrack_add(struct rdma_restrack_entry *res) } EXPORT_SYMBOL(rdma_restrack_add); +/** + * rdma_restrack_abort_del() - re-add object to the resource tracking database + * it can only be used after rdma_restrack_begin_del(). + * @res: resource entry + */ +void rdma_restrack_abort_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_abort_del); + int __must_check rdma_restrack_get(struct rdma_restrack_entry *res) { return kref_get_unless_zero(&res->kref); @@ -263,7 +328,7 @@ static void restrack_release(struct kref *kref) struct rdma_restrack_entry *res; res = container_of(kref, struct rdma_restrack_entry, kref); - if (res->task) { + if (res->task && !res->valid) { put_task_struct(res->task); res->task = NULL; } @@ -277,14 +342,43 @@ int rdma_restrack_put(struct rdma_restrack_entry *res) EXPORT_SYMBOL(rdma_restrack_put); /** - * rdma_restrack_del() - delete object from the reource tracking database + * rdma_restrack_sync() - Fence concurrent netlink dumps on an entry + * @res: resource entry + * + * After this returns any concurrent netlink dump threads will see the current + * value of the object. This is useful if the object has to be changed and there + * is not locking to protect the nl side. Eg for mr->pd. This effectively + * destroys the object from a kref/xarray perspective and then immediately + * restores it. The kref is acting like a lock to barrier concurrent nl threads. + * Callers must ensure rdma_restrack_del() is not concurrently called. + */ +void rdma_restrack_sync(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + return; + + rt = res_to_rt(res); + if (!rt) + return; + + if (WARN_ON(xa_get_mark(&rt->xa, res->id, RESTRACK_DD))) + return; + + restrack_drain_res(rt, res); + restrack_restore_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_sync); + +/** + * rdma_restrack_del() - delete object from the resource tracking database * @res: resource entry */ void rdma_restrack_del(struct rdma_restrack_entry *res) { struct rdma_restrack_entry *old; struct rdma_restrack_root *rt; - struct ib_device *dev; if (!res->valid) { if (res->task) { @@ -297,12 +391,10 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) if (res->no_track) goto out; - dev = res_to_dev(res); - if (WARN_ON(!dev)) + rt = res_to_rt(res); + if (!rt) return; - rt = &dev->res[res->type]; - old = xa_erase(&rt->xa, res->id); WARN_ON(old != res); @@ -310,5 +402,61 @@ void rdma_restrack_del(struct rdma_restrack_entry *res) res->valid = false; rdma_restrack_put(res); wait_for_completion(&res->comp); + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } } EXPORT_SYMBOL(rdma_restrack_del); + +/** + * rdma_restrack_begin_del() - invalidate the object from the resource tracking + * database but preserve its index in the array. + * Since this preserves the index in the array until rdma_restrack_commit_del() + * is called, if rdma_restrack_add() is called in between with an old QP ID it + * can result in an untracked QP. + * @res: resource entry + */ +void rdma_restrack_begin_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt = NULL; + + if (!res->valid) + return; + + if (!res->no_track) { + rt = res_to_rt(res); + if (!rt) + return; + } + + restrack_drain_res(rt, res); +} +EXPORT_SYMBOL(rdma_restrack_begin_del); + +/** + * rdma_restrack_commit_del() - delete object from the resource tracking + * database and free the task. + * @res: resource entry + */ +void rdma_restrack_commit_del(struct rdma_restrack_entry *res) +{ + struct rdma_restrack_root *rt; + + if (!res->valid || res->no_track) + goto out; + + rt = res_to_rt(res); + if (!rt) + return; + + xa_erase(&rt->xa, res->id); + +out: + res->valid = false; + if (res->task) { + put_task_struct(res->task); + res->task = NULL; + } +} +EXPORT_SYMBOL(rdma_restrack_commit_del); diff --git a/drivers/infiniband/core/restrack.h b/drivers/infiniband/core/restrack.h index 6a04fc41f7380..2df78e084e107 100644 --- a/drivers/infiniband/core/restrack.h +++ b/drivers/infiniband/core/restrack.h @@ -26,7 +26,11 @@ struct rdma_restrack_root { int rdma_restrack_init(struct ib_device *dev); void rdma_restrack_clean(struct ib_device *dev); void rdma_restrack_add(struct rdma_restrack_entry *res); +void rdma_restrack_abort_del(struct rdma_restrack_entry *res); void rdma_restrack_del(struct rdma_restrack_entry *res); +void rdma_restrack_sync(struct rdma_restrack_entry *res); +void rdma_restrack_begin_del(struct rdma_restrack_entry *res); +void rdma_restrack_commit_del(struct rdma_restrack_entry *res); void rdma_restrack_new(struct rdma_restrack_entry *res, enum rdma_restrack_type type); void rdma_restrack_set_name(struct rdma_restrack_entry *res, diff --git a/drivers/infiniband/core/ucma.c b/drivers/infiniband/core/ucma.c index f86ece701db6e..797efd4a47282 100644 --- a/drivers/infiniband/core/ucma.c +++ b/drivers/infiniband/core/ucma.c @@ -951,7 +951,7 @@ static ssize_t ucma_query_path(struct ucma_context *ctx, resp->num_paths = ctx->cm_id->route.num_pri_alt_paths; for (i = 0, out_len -= sizeof(*resp); - i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data); + i < resp->num_paths && out_len >= sizeof(struct ib_path_rec_data); i++, out_len -= sizeof(struct ib_path_rec_data)) { struct sa_path_rec *rec = &ctx->cm_id->route.path_rec[i]; @@ -1404,7 +1404,10 @@ static int ucma_set_ib_path(struct ucma_context *ctx, memset(&event, 0, sizeof event); event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; - return ucma_event_handler(ctx->cm_id, &event); + rdma_lock_handler(ctx->cm_id); + ret = ucma_event_handler(ctx->cm_id, &event); + rdma_unlock_handler(ctx->cm_id); + return ret; } static int ucma_set_option_ib(struct ucma_context *ctx, int optname, @@ -1776,6 +1779,13 @@ static ssize_t ucma_write_cm_event(struct ucma_file *file, goto out; } + rdma_lock_handler(ctx->cm_id); + if (!ctx->uid) { + kfree(uevent); + ret = -EINVAL; + goto err_unlock; + } + uevent->ctx = ctx; uevent->resp.uid = ctx->uid; uevent->resp.id = ctx->id; @@ -1789,6 +1799,8 @@ static ssize_t ucma_write_cm_event(struct ucma_file *file, mutex_unlock(&ctx->file->mut); wake_up_interruptible(&ctx->file->poll_wait); +err_unlock: + rdma_unlock_handler(ctx->cm_id); out: ucma_put_ctx(ctx); return ret; diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c index fd3a774904f8d..468d662c42c47 100644 --- a/drivers/infiniband/core/umem.c +++ b/drivers/infiniband/core/umem.c @@ -78,14 +78,17 @@ static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int d */ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { unsigned long curr_len = 0; dma_addr_t curr_base = ~0; - unsigned long va, pgoff; + unsigned long pgoff; struct scatterlist *sg; - dma_addr_t mask; + unsigned long mask = 0; + unsigned int bits; dma_addr_t end; + u64 last_va; + u64 va; int i; umem->iova = va = virt; @@ -103,9 +106,12 @@ unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, * number of required pages. Compute the largest page size that could * work based on VA address bits that don't change. */ - mask = pgsz_bitmap & - GENMASK(BITS_PER_LONG - 1, - bits_per((umem->length - 1 + virt) ^ virt)); + if (check_add_overflow(umem->length - 1, virt, &last_va)) + return 0; + bits = bits_per(virt ^ last_va); + if (bits < BITS_PER_LONG) + mask = pgsz_bitmap & GENMASK(BITS_PER_LONG - 1, bits); + /* offset into first SGL */ pgoff = umem->address & ~PAGE_MASK; diff --git a/drivers/infiniband/core/umem_dmabuf.c b/drivers/infiniband/core/umem_dmabuf.c index b239ac09ec292..4fc73f53342b7 100644 --- a/drivers/infiniband/core/umem_dmabuf.c +++ b/drivers/infiniband/core/umem_dmabuf.c @@ -206,6 +206,10 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach) if (umem_dmabuf->revoked) return; + + if (umem_dmabuf->pinned_revoke) + umem_dmabuf->pinned_revoke(umem_dmabuf->private); + ib_umem_dmabuf_unmap_pages(umem_dmabuf); if (umem_dmabuf->pinned) { dma_buf_unpin(umem_dmabuf->attach); @@ -214,6 +218,11 @@ static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach) umem_dmabuf->revoked = 1; } +static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { + .allow_peer2peer = true, + .move_notify = ib_umem_dmabuf_revoke_locked, +}; + static struct ib_umem_dmabuf * ib_umem_dmabuf_get_pinned_and_lock(struct ib_device *device, struct device *dma_device, @@ -266,6 +275,58 @@ ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, } EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_with_dma_device); +/** + * ib_umem_dmabuf_get_pinned_revocable_and_lock - Map & pin a revocable dmabuf + * @device: IB device. + * @offset: Start offset. + * @size: Length. + * @fd: dmabuf fd. + * @access: Access flags. + * + * Obtains a umem from a dmabuf for drivers/devices that can support revocation. + * + * Returns with dma_resv_lock held upon success. The driver must set the revoke + * callback prior to unlock by calling ib_umem_dmabuf_set_revoke_locked(). + * + * When a revocation occurs, the revoke callback will be called. The driver must + * ensure that the region is no longer accessed when the callback returns. Any + * subsequent access attempts should also probably cause an AE for MRs. + * + * If the umem is used for an MR, the driver must ensure that the key remains in + * use such that it cannot be obtained by a new region until this region is + * fully deregistered (i.e., ibv_dereg_mr). If a driver needs to serialize with + * revoke calls, it can use dma_resv_lock. + * + * If successful, then the revoke callback may be called at any time and will + * also be called automatically upon ib_umem_release (serialized). The revoke + * callback will be called one time at most. + * + * Return: A pointer to ib_umem_dmabuf on success, or an ERR_PTR on failure. + */ +struct ib_umem_dmabuf * +ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, + unsigned long offset, size_t size, + int fd, int access) +{ + const struct dma_buf_attach_ops *ops = + &ib_umem_dmabuf_attach_pinned_revocable_ops; + + return ib_umem_dmabuf_get_pinned_and_lock(device, device->dma_device, + offset, size, fd, access, + ops); +} +EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_revocable_and_lock); + +void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, + void (*revoke)(void *priv), void *priv) +{ + dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); + + umem_dmabuf->pinned_revoke = revoke; + umem_dmabuf->private = priv; +} +EXPORT_SYMBOL(ib_umem_dmabuf_set_revoke_locked); + struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, size_t size, int fd, diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c index f4616deeca545..95c1974e02661 100644 --- a/drivers/infiniband/core/uverbs_cmd.c +++ b/drivers/infiniband/core/uverbs_cmd.c @@ -47,6 +47,7 @@ #include "uverbs.h" #include "core_priv.h" +#include "restrack.h" /* * Copy a response to userspace. If the provided 'resp' is larger than the @@ -830,6 +831,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) ret = PTR_ERR(new_pd); goto put_uobjs; } + if (new_pd == orig_pd) { + uobj_put_obj_read(new_pd); + cmd.flags &= ~IB_MR_REREG_PD; + } } else { new_pd = mr->pd; } @@ -875,9 +880,10 @@ static int ib_uverbs_rereg_mr(struct uverbs_attr_bundle *attrs) mr = new_mr; } else { if (cmd.flags & IB_MR_REREG_PD) { - atomic_dec(&orig_pd->usecnt); - mr->pd = new_pd; atomic_inc(&new_pd->usecnt); + WRITE_ONCE(mr->pd, new_pd); + rdma_restrack_sync(&mr->res); + atomic_dec(&orig_pd->usecnt); } if (cmd.flags & IB_MR_REREG_TRANS) { mr->iova = cmd.hca_va; diff --git a/drivers/infiniband/core/uverbs_std_types_device.c b/drivers/infiniband/core/uverbs_std_types_device.c index c0fd283d9d6c9..a28f9f21bed89 100644 --- a/drivers/infiniband/core/uverbs_std_types_device.c +++ b/drivers/infiniband/core/uverbs_std_types_device.c @@ -209,6 +209,39 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)( &resp, sizeof(resp)); } +static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT_SPEED)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_ucontext *ucontext; + struct ib_device *ib_dev; + u32 port_num; + u64 speed; + int ret; + + ucontext = ib_uverbs_get_ucontext(attrs); + if (IS_ERR(ucontext)) + return PTR_ERR(ucontext); + ib_dev = ucontext->device; + + if (!ib_dev->ops.query_port_speed) + return -EOPNOTSUPP; + + ret = uverbs_get_const(&port_num, attrs, + UVERBS_ATTR_QUERY_PORT_SPEED_PORT_NUM); + if (ret) + return ret; + + if (!rdma_is_port_valid(ib_dev, port_num)) + return -EINVAL; + + ret = ib_dev->ops.query_port_speed(ib_dev, port_num, &speed); + if (ret) + return ret; + + return uverbs_copy_to(attrs, UVERBS_ATTR_QUERY_PORT_SPEED_RESP, + &speed, sizeof(speed)); +} + static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)( struct uverbs_attr_bundle *attrs) { @@ -469,6 +502,14 @@ DECLARE_UVERBS_NAMED_METHOD( active_speed_ex), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + UVERBS_METHOD_QUERY_PORT_SPEED, + UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_PORT_SPEED_PORT_NUM, u32, + UA_MANDATORY), + UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_QUERY_PORT_SPEED_RESP, + UVERBS_ATTR_TYPE(u64), + UA_MANDATORY)); + DECLARE_UVERBS_NAMED_METHOD( UVERBS_METHOD_QUERY_GID_TABLE, UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_GID_TABLE_ENTRY_SIZE, u64, @@ -498,6 +539,7 @@ DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE, &UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE), &UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES), &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT), + &UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT_SPEED), &UVERBS_METHOD(UVERBS_METHOD_QUERY_CONTEXT), &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_TABLE), &UVERBS_METHOD(UVERBS_METHOD_QUERY_GID_ENTRY)); diff --git a/drivers/infiniband/core/uverbs_std_types_dmah.c b/drivers/infiniband/core/uverbs_std_types_dmah.c index 97101e0938263..9873ab49a6013 100644 --- a/drivers/infiniband/core/uverbs_std_types_dmah.c +++ b/drivers/infiniband/core/uverbs_std_types_dmah.c @@ -18,11 +18,14 @@ static int uverbs_free_dmah(struct ib_uobject *uobject, if (atomic_read(&dmah->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&dmah->res); ret = dmah->device->ops.dealloc_dmah(dmah, attrs); - if (ret) + if (ret) { + rdma_restrack_abort_del(&dmah->res); return ret; + } - rdma_restrack_del(&dmah->res); + rdma_restrack_commit_del(&dmah->res); kfree(dmah); return 0; } diff --git a/drivers/infiniband/core/uverbs_std_types_mr.c b/drivers/infiniband/core/uverbs_std_types_mr.c index 570b9656801d7..0c72f801e0d32 100644 --- a/drivers/infiniband/core/uverbs_std_types_mr.c +++ b/drivers/infiniband/core/uverbs_std_types_mr.c @@ -364,7 +364,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)( dmah, attrs); else mr = pd->device->ops.reg_user_mr(pd, addr, length, iova, - access_flags, dmah, NULL); + access_flags, dmah, + &attrs->driver_udata); if (IS_ERR(mr)) return PTR_ERR(mr); @@ -527,7 +528,8 @@ DECLARE_UVERBS_NAMED_METHOD( UA_MANDATORY), UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY, UVERBS_ATTR_TYPE(u32), - UA_MANDATORY)); + UA_MANDATORY), + UVERBS_ATTR_UHW()); DECLARE_UVERBS_NAMED_METHOD_DESTROY( UVERBS_METHOD_MR_DESTROY, diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index bc1878da55cd7..4357164a9976c 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -213,6 +213,57 @@ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate) } EXPORT_SYMBOL(ib_rate_to_mbps); +struct ib_speed_attr { + const char *str; + int speed; +}; + +#define IB_SPEED_ATTR(speed_type, _str, _speed) \ + [speed_type] = {.str = _str, .speed = _speed} + +static const struct ib_speed_attr ib_speed_attrs[] = { + IB_SPEED_ATTR(IB_SPEED_SDR, " SDR", 25), + IB_SPEED_ATTR(IB_SPEED_DDR, " DDR", 50), + IB_SPEED_ATTR(IB_SPEED_QDR, " QDR", 100), + IB_SPEED_ATTR(IB_SPEED_FDR10, " FDR10", 100), + IB_SPEED_ATTR(IB_SPEED_FDR, " FDR", 140), + IB_SPEED_ATTR(IB_SPEED_EDR, " EDR", 250), + IB_SPEED_ATTR(IB_SPEED_HDR, " HDR", 500), + IB_SPEED_ATTR(IB_SPEED_NDR, " NDR", 1000), + IB_SPEED_ATTR(IB_SPEED_XDR, " XDR", 2000), +}; + +int ib_port_attr_to_speed_info(struct ib_port_attr *attr, + struct ib_port_speed_info *speed_info) +{ + int speed_idx = attr->active_speed; + + switch (attr->active_speed) { + case IB_SPEED_DDR: + case IB_SPEED_QDR: + case IB_SPEED_FDR10: + case IB_SPEED_FDR: + case IB_SPEED_EDR: + case IB_SPEED_HDR: + case IB_SPEED_NDR: + case IB_SPEED_XDR: + case IB_SPEED_SDR: + break; + default: + speed_idx = IB_SPEED_SDR; /* Default to SDR for invalid rates */ + break; + } + + speed_info->str = ib_speed_attrs[speed_idx].str; + speed_info->rate = ib_speed_attrs[speed_idx].speed; + speed_info->rate *= ib_width_enum_to_int(attr->active_width); + if (speed_info->rate < 0) + return -EINVAL; + + return 0; +} +EXPORT_SYMBOL(ib_port_attr_to_speed_info); + __attribute_const__ enum rdma_transport_type rdma_node_get_transport(unsigned int node_type) { @@ -335,6 +386,7 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) { int ret; + rdma_restrack_begin_del(&pd->res); if (pd->__internal_mr) { ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL); WARN_ON(ret); @@ -342,10 +394,12 @@ int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata) } ret = pd->device->ops.dealloc_pd(pd, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&pd->res); return ret; + } - rdma_restrack_del(&pd->res); + rdma_restrack_commit_del(&pd->res); kfree(pd); return ret; } @@ -1083,16 +1137,20 @@ int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata) if (atomic_read(&srq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&srq->res); + ret = srq->device->ops.destroy_srq(srq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&srq->res); return ret; + } atomic_dec(&srq->pd->usecnt); if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd) atomic_dec(&srq->ext.xrc.xrcd->usecnt); if (ib_srq_has_cq(srq->srq_type)) atomic_dec(&srq->ext.cq->usecnt); - rdma_restrack_del(&srq->res); + rdma_restrack_commit_del(&srq->res); kfree(srq); return ret; @@ -2098,6 +2156,8 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (qp->real_qp != qp) return __ib_destroy_shared_qp(qp); + rdma_restrack_begin_del(&qp->res); + sec = qp->qp_sec; if (sec) ib_destroy_qp_security_begin(sec); @@ -2110,6 +2170,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (ret) { if (sec) ib_destroy_qp_security_abort(sec); + rdma_restrack_abort_del(&qp->res); return ret; } @@ -2122,7 +2183,7 @@ int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata) if (sec) ib_destroy_qp_security_end(sec); - rdma_restrack_del(&qp->res); + rdma_restrack_commit_del(&qp->res); kfree(qp); return ret; } @@ -2187,11 +2248,15 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) if (atomic_read(&cq->usecnt)) return -EBUSY; + rdma_restrack_begin_del(&cq->res); + ret = cq->device->ops.destroy_cq(cq, udata); - if (ret) + if (ret) { + rdma_restrack_abort_del(&cq->res); return ret; + } - rdma_restrack_del(&cq->res); + rdma_restrack_commit_del(&cq->res); kfree(cq); return ret; } diff --git a/drivers/infiniband/hw/bnxt_re/debugfs.c b/drivers/infiniband/hw/bnxt_re/debugfs.c index be5e9b5ca2f06..d72c1693c57f9 100644 --- a/drivers/infiniband/hw/bnxt_re/debugfs.c +++ b/drivers/infiniband/hw/bnxt_re/debugfs.c @@ -10,8 +10,8 @@ #include #include #include +#include -#include "bnxt_ulp.h" #include "roce_hsi.h" #include "qplib_res.h" #include "qplib_sp.h" diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 25d88c2c5735f..64b6e572e7047 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -4570,11 +4570,13 @@ int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma) case BNXT_RE_MMAP_DBR_PAGE: case BNXT_RE_MMAP_TOGGLE_PAGE: /* Driver doesn't expect write access for user space */ - if (vma->vm_flags & VM_WRITE) + if (vma->vm_flags & VM_WRITE) { ret = -EFAULT; - else + } else { + vm_flags_clear(vma, VM_MAYWRITE); ret = vm_insert_page(vma, vma->vm_start, virt_to_page((void *)bnxt_entry->mem_offset)); + } break; default: ret = -EINVAL; diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 4ca2eee6fbc22..f33e006126fd2 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -55,8 +55,8 @@ #include #include #include +#include -#include "bnxt_ulp.h" #include "roce_hsi.h" #include "qplib_res.h" #include "qplib_sp.h" diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index ce90d3d834d4a..9e97d67de1beb 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -46,6 +46,7 @@ #include #include #include +#include #include #include "roce_hsi.h" @@ -55,7 +56,6 @@ #include "qplib_sp.h" #include "qplib_fp.h" #include -#include "bnxt_ulp.h" #include "bnxt_re.h" #include "ib_verbs.h" diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.c b/drivers/infiniband/hw/bnxt_re/qplib_res.c index d321acb07335b..0fcea79eb110d 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.c @@ -721,9 +721,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, break; case BNXT_QPLIB_DPI_TYPE_WC: dpi->dbr = ioremap_wc(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; default: dpi->dbr = ioremap(umaddr, PAGE_SIZE); + if (!dpi->dbr) + goto fail_ioremap; break; } @@ -731,6 +735,13 @@ int bnxt_qplib_alloc_dpi(struct bnxt_qplib_res *res, mutex_unlock(&res->dpi_tbl_lock); return 0; +fail_ioremap: + /* Roll back the bit we just claimed. */ + set_bit(bit_num, dpit->tbl); + dpit->app_tbl[bit_num] = NULL; + mutex_unlock(&res->dpi_tbl_lock); + return -ENOMEM; + } int bnxt_qplib_dealloc_dpi(struct bnxt_qplib_res *res, diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index 2ea3b7f232a3f..ebe8893937f62 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -39,7 +39,7 @@ #ifndef __BNXT_QPLIB_RES_H__ #define __BNXT_QPLIB_RES_H__ -#include "bnxt_ulp.h" +#include extern const struct bnxt_qplib_gid bnxt_qplib_gid_zero; diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index d892f55febe24..0d023c361ef8e 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -934,6 +934,7 @@ static void c4iw_rdev_close(struct c4iw_rdev *rdev) void c4iw_dealloc(struct uld_ctx *ctx) { + debugfs_remove_recursive(ctx->dev->debugfs_root); c4iw_rdev_close(&ctx->dev->rdev); WARN_ON(!xa_empty(&ctx->dev->cqs)); WARN_ON(!xa_empty(&ctx->dev->qps)); @@ -952,7 +953,12 @@ void c4iw_dealloc(struct uld_ctx *ctx) static void c4iw_remove(struct uld_ctx *ctx) { pr_debug("c4iw_dev %p\n", ctx->dev); - debugfs_remove_recursive(ctx->dev->debugfs_root); + + /* c4iw_register_device() may still be using ctx->dev. */ + cancel_work_sync(&ctx->reg_work); + if (!ctx->dev) + return; + c4iw_unregister_device(ctx->dev); c4iw_dealloc(ctx); } diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 40dd6ac5f91af..ac349de9d2e44 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -277,6 +277,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, int err; struct fw_ri_tpte *tpt; u32 stag_idx; + bool stag_idx_allocated = false; static atomic_t key; if (c4iw_fatal_error(rdev)) @@ -299,6 +300,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, return -ENOMEM; } mutex_lock(&rdev->stats.lock); + stag_idx_allocated = true; rdev->stats.stag.cur += 32; if (rdev->stats.stag.cur > rdev->stats.stag.max) rdev->stats.stag.max = rdev->stats.stag.cur; @@ -333,7 +335,7 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, (rdev->lldi.vr->stag.start >> 5), sizeof(*tpt), tpt, skb, wr_waitp); - if (reset_tpt_entry) { + if (reset_tpt_entry || (err && stag_idx_allocated)) { c4iw_put_resource(&rdev->resource.tpt_table, stag_idx); mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur -= 32; diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index 0bb3389d761f3..18db10741cee6 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -1392,9 +1392,11 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) chunk_list->chunks[i].length = EFA_CHUNK_USED_SIZE; } - chunk_list->chunks[chunk_list_size - 1].length = - ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + - EFA_CHUNK_PTR_SIZE; + + if (page_cnt % EFA_PTRS_PER_CHUNK != 0) + chunk_list->chunks[chunk_list_size - 1].length = + ((page_cnt % EFA_PTRS_PER_CHUNK) * EFA_CHUNK_PAYLOAD_PTR_SIZE) + + EFA_CHUNK_PTR_SIZE; /* fill the dma addresses of sg list pages to chunks: */ chunk_idx = 0; @@ -1406,9 +1408,12 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) rdma_block_iter_dma_address(&biter); if (payload_idx == EFA_PTRS_PER_CHUNK) { + payload_idx = 0; chunk_idx++; + if (chunk_idx >= chunk_list_size) + break; + cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; - payload_idx = 0; } } diff --git a/drivers/infiniband/hw/erdma/Kconfig b/drivers/infiniband/hw/erdma/Kconfig index 267fc1f3c42af..745e5551773f5 100644 --- a/drivers/infiniband/hw/erdma/Kconfig +++ b/drivers/infiniband/hw/erdma/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config INFINIBAND_ERDMA tristate "Alibaba Elastic RDMA Adapter (ERDMA) support" - depends on PCI_MSI && 64BIT + depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN depends on INFINIBAND_ADDR_TRANS depends on INFINIBAND_USER_ACCESS help diff --git a/drivers/infiniband/hw/erdma/erdma_cm.c b/drivers/infiniband/hw/erdma/erdma_cm.c index e0acc185e7193..f0c8ebf62a057 100644 --- a/drivers/infiniband/hw/erdma/erdma_cm.c +++ b/drivers/infiniband/hw/erdma/erdma_cm.c @@ -1021,10 +1021,9 @@ int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET) return -EAFNOSUPPORT; - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s); if (ret < 0) @@ -1154,10 +1153,9 @@ int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) return -ECONNRESET; } - qp = find_qp_by_qpn(dev, params->qpn); + qp = erdma_qp_get_by_qpn(dev, params->qpn); if (!qp) return -ENOENT; - erdma_qp_get(qp); down_write(&qp->state_lock); if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) { diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c index 6486234a23600..e4161e85422c6 100644 --- a/drivers/infiniband/hw/erdma/erdma_eq.c +++ b/drivers/infiniband/hw/erdma/erdma_eq.c @@ -52,7 +52,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK, le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) { cqn = le32_to_cpu(aeqe->event_data0); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -62,9 +62,10 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (cq->ibcq.event_handler) cq->ibcq.event_handler(&event, cq->ibcq.cq_context); + erdma_cq_put(cq); } else { qpn = le32_to_cpu(aeqe->event_data0); - qp = find_qp_by_qpn(dev, qpn); + qp = erdma_qp_get_by_qpn(dev, qpn); if (!qp) continue; @@ -74,6 +75,7 @@ void erdma_aeq_event_handler(struct erdma_dev *dev) if (qp->ibqp.event_handler) qp->ibqp.event_handler(&event, qp->ibqp.qp_context); + erdma_qp_put(qp); } } @@ -157,7 +159,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) poll_cnt++; cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe)); - cq = find_cq_by_cqn(dev, cqn); + cq = erdma_cq_get_by_cqn(dev, cqn); if (!cq) continue; @@ -166,6 +168,7 @@ void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) if (cq->ibcq.comp_handler) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); + erdma_cq_put(cq); } notify_eq(&ceq_cb->eq); @@ -219,6 +222,7 @@ static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) irq_set_affinity_hint(eqc->irq.msix_vector, NULL); free_irq(eqc->irq.msix_vector, eqc); + tasklet_kill(&eqc->tasklet); } static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c index 25f6c49aec779..e002343832f74 100644 --- a/drivers/infiniband/hw/erdma/erdma_qp.c +++ b/drivers/infiniband/hw/erdma/erdma_qp.c @@ -734,7 +734,7 @@ int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr *wr = recv_wr; struct erdma_qp *qp = to_eqp(ibqp); unsigned long flags; - int ret; + int ret = 0; spin_lock_irqsave(&qp->lock, flags); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c index 058edc42de58b..fdd67756f3bc3 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.c +++ b/drivers/infiniband/hw/erdma/erdma_verbs.c @@ -1303,8 +1303,15 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); + /* + * A timeout disables the command queue, so retry cannot succeed. Treat + * terminal command failures as diagnostic; propagating them can make + * forced uverbs cleanup discard the last software resource pointers. + */ if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to deregister MR 0x%x: %d\n", + ibmr->lkey, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8); @@ -1320,6 +1327,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) struct erdma_dev *dev = to_edev(ibcq->device); struct erdma_ucontext *ctx = rdma_udata_to_drv_context( udata, struct erdma_ucontext, ibucontext); + unsigned long flags; int err; struct erdma_cmdq_destroy_cq_req req; @@ -1330,7 +1338,16 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy CQ %u: %d\n", + cq->cqn, err); + + xa_lock_irqsave(&dev->cq_xa, flags); + __xa_erase(&dev->cq_xa, cq->cqn); + xa_unlock_irqrestore(&dev->cq_xa, flags); + + erdma_cq_put(cq); + wait_for_completion(&cq->free); if (rdma_is_kernel_res(&cq->ibcq.res)) { dma_free_coherent(&dev->pdev->dev, cq->depth << CQE_SHIFT, @@ -1342,8 +1359,6 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) put_mtt_entries(dev, &cq->user_cq.qbuf_mem); } - xa_erase(&dev->cq_xa, cq->cqn); - return 0; } @@ -1355,6 +1370,7 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) udata, struct erdma_ucontext, ibucontext); struct erdma_cmdq_destroy_qp_req req; union erdma_mod_qp_params params; + unsigned long flags; int err; down_write(&qp->state_lock); @@ -1378,7 +1394,13 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, true); if (err) - return err; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy QP %u: %d\n", + QP_ID(qp), err); + + xa_lock_irqsave(&dev->qp_xa, flags); + __xa_erase(&dev->qp_xa, QP_ID(qp)); + xa_unlock_irqrestore(&dev->qp_xa, flags); erdma_qp_put(qp); wait_for_completion(&qp->safe_free); @@ -1393,7 +1415,6 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) if (qp->cep) erdma_cep_put(qp->cep); - xa_erase(&dev->qp_xa, QP_ID(qp)); return 0; } @@ -1970,6 +1991,8 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, cq->ibcq.cqe = depth; cq->depth = depth; cq->assoc_eqn = attr->comp_vector + 1; + refcount_set(&cq->refcount, 1); + init_completion(&cq->free); ret = xa_alloc_cyclic(&dev->cq_xa, &cq->cqn, cq, XA_LIMIT(1, dev->attrs.max_cq - 1), @@ -2282,7 +2305,9 @@ int erdma_destroy_ah(struct ib_ah *ibah, u32 flags) ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL, flags & RDMA_DESTROY_AH_SLEEPABLE); if (ret) - return ret; + ibdev_warn_ratelimited(&dev->ibdev, + "failed to destroy AH %u: %d\n", + ah->ahn, ret); erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_AH], ah->ahn); diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h index 7d8d3fe501d5c..c73cecf92f611 100644 --- a/drivers/infiniband/hw/erdma/erdma_verbs.h +++ b/drivers/infiniband/hw/erdma/erdma_verbs.h @@ -7,6 +7,9 @@ #ifndef __ERDMA_VERBS_H__ #define __ERDMA_VERBS_H__ +#include +#include + #include "erdma.h" /* RDMA Capability. */ @@ -341,6 +344,8 @@ struct erdma_cq { u32 depth; u32 assoc_eqn; + refcount_t refcount; + struct completion free; union { struct erdma_kcq_info kern_cq; @@ -355,9 +360,40 @@ static inline struct erdma_qp *find_qp_by_qpn(struct erdma_dev *dev, int id) return (struct erdma_qp *)xa_load(&dev->qp_xa, id); } -static inline struct erdma_cq *find_cq_by_cqn(struct erdma_dev *dev, int id) +static inline struct erdma_qp *erdma_qp_get_by_qpn(struct erdma_dev *dev, + int id) +{ + struct erdma_qp *qp; + unsigned long flags; + + xa_lock_irqsave(&dev->qp_xa, flags); + qp = xa_load(&dev->qp_xa, id); + if (qp && !kref_get_unless_zero(&qp->ref)) + qp = NULL; + xa_unlock_irqrestore(&dev->qp_xa, flags); + + return qp; +} + +static inline struct erdma_cq *erdma_cq_get_by_cqn(struct erdma_dev *dev, + int id) +{ + struct erdma_cq *cq; + unsigned long flags; + + xa_lock_irqsave(&dev->cq_xa, flags); + cq = xa_load(&dev->cq_xa, id); + if (cq && !refcount_inc_not_zero(&cq->refcount)) + cq = NULL; + xa_unlock_irqrestore(&dev->cq_xa, flags); + + return cq; +} + +static inline void erdma_cq_put(struct erdma_cq *cq) { - return (struct erdma_cq *)xa_load(&dev->cq_xa, id); + if (refcount_dec_and_test(&cq->refcount)) + complete(&cq->free); } void erdma_qp_get(struct erdma_qp *qp); diff --git a/drivers/infiniband/hw/hfi1/chip.c b/drivers/infiniband/hw/hfi1/chip.c index 0781ab756d441..57c70384d2107 100644 --- a/drivers/infiniband/hw/hfi1/chip.c +++ b/drivers/infiniband/hw/hfi1/chip.c @@ -15047,7 +15047,7 @@ int hfi1_init_dd(struct hfi1_devdata *dd) */ ret = hfi1_pcie_ddinit(dd, pdev); if (ret < 0) - goto bail_free; + goto bail; /* Save PCI space registers to rewrite after device reset */ ret = save_pci_variables(dd); @@ -15302,8 +15302,6 @@ int hfi1_init_dd(struct hfi1_devdata *dd) bail_cleanup: hfi1_free_rx(dd); hfi1_pcie_ddcleanup(dd); -bail_free: - hfi1_free_devdata(dd); bail: return ret; } diff --git a/drivers/infiniband/hw/hfi1/hfi.h b/drivers/infiniband/hw/hfi1/hfi.h index cb630551cf1a3..20942a8ac249e 100644 --- a/drivers/infiniband/hw/hfi1/hfi.h +++ b/drivers/infiniband/hw/hfi1/hfi.h @@ -2044,9 +2044,7 @@ struct cc_state *get_cc_state_protected(struct hfi1_pportdata *ppd) /* waiting for an urgent packet to arrive */ #define HFI1_CTXT_WAITING_URG 4 -/* free up any allocated data at closes */ int hfi1_init_dd(struct hfi1_devdata *dd); -void hfi1_free_devdata(struct hfi1_devdata *dd); /* LED beaconing functions */ void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, diff --git a/drivers/infiniband/hw/hfi1/init.c b/drivers/infiniband/hw/hfi1/init.c index 7e0298c62882e..f0b3d53243687 100644 --- a/drivers/infiniband/hw/hfi1/init.c +++ b/drivers/infiniband/hw/hfi1/init.c @@ -630,8 +630,6 @@ void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd, ppd->sm_trap_qp = 0x0; ppd->sa_qp = 0x1; - ppd->hfi1_wq = NULL; - spin_lock_init(&ppd->cca_timer_lock); for (i = 0; i < OPA_MAX_SLS; i++) { @@ -1163,7 +1161,7 @@ static void finalize_asic_data(struct hfi1_devdata *dd, * It cleans up and frees all data structures set up by * by hfi1_alloc_devdata(). */ -void hfi1_free_devdata(struct hfi1_devdata *dd) +static void hfi1_free_devdata(struct hfi1_devdata *dd) { struct hfi1_asic_data *ad; unsigned long flags; @@ -1227,8 +1225,9 @@ static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev, GFP_KERNEL); if (ret < 0) { dev_err(&pdev->dev, - "Could not allocate unit ID: error %d\n", -ret); - goto bail; + "Could not allocate unit ID: error %pe\n", ERR_PTR(ret)); + rvt_dealloc_device(&dd->verbs_dev.rdi); + return ERR_PTR(ret); } /* @@ -1572,15 +1571,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) /* First, lock the non-writable module parameters */ HFI1_CAP_LOCK(); - /* Validate dev ids */ - if (!(ent->device == PCI_DEVICE_ID_INTEL0 || - ent->device == PCI_DEVICE_ID_INTEL1)) { - dev_err(&pdev->dev, "Failing on unknown Intel deviceid 0x%x\n", - ent->device); - ret = -ENODEV; - goto bail; - } - /* Allocate the dd so we can get to work */ dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS * sizeof(struct hfi1_pportdata)); @@ -1634,17 +1624,17 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) goto bail; + ret = create_workqueues(dd); + if (ret) + goto free_devdata; + /* * Do device-specific initialization, function table setup, dd * allocation, etc. */ ret = hfi1_init_dd(dd); if (ret) - goto clean_bail; /* error already printed */ - - ret = create_workqueues(dd); - if (ret) - goto clean_bail; + goto destroy_workqueues; /* error already printed */ /* do the generic initialization */ initfail = hfi1_init(dd, 0); @@ -1657,11 +1647,8 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) * we still create devices, so diags, etc. can be used * to determine cause of problem. */ - if (!initfail && !ret) { + if (!initfail && !ret) dd->flags |= HFI1_INITTED; - /* create debufs files after init and ib register */ - hfi1_dbg_ibdev_init(&dd->verbs_dev); - } j = hfi1_device_create(dd); if (j) @@ -1670,7 +1657,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (initfail || ret) { msix_clean_up_interrupts(dd); stop_timers(dd); - flush_workqueue(ib_wq); for (pidx = 0; pidx < dd->num_pports; ++pidx) { hfi1_quiet_serdes(dd->pport + pidx); ppd = dd->pport + pidx; @@ -1687,6 +1673,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) hfi1_device_remove(dd); if (!ret) hfi1_unregister_ib_device(dd); + hfi1_free_rx(dd); postinit_cleanup(dd); if (initfail) ret = initfail; @@ -1694,10 +1681,14 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } sdma_start(dd); + hfi1_dbg_ibdev_init(&dd->verbs_dev); return 0; -clean_bail: +destroy_workqueues: + destroy_workqueues(dd); +free_devdata: + hfi1_free_devdata(dd); hfi1_pcie_cleanup(pdev); bail: return ret; @@ -1743,9 +1734,6 @@ static void remove_one(struct pci_dev *pdev) stop_timers(dd); - /* wait until all of our (qsfp) queue_work() calls complete */ - flush_workqueue(ib_wq); - postinit_cleanup(dd); } diff --git a/drivers/infiniband/hw/hfi1/user_sdma.c b/drivers/infiniband/hw/hfi1/user_sdma.c index 9b1aece1b0800..ae8ff7d5a37ce 100644 --- a/drivers/infiniband/hw/hfi1/user_sdma.c +++ b/drivers/infiniband/hw/hfi1/user_sdma.c @@ -1028,6 +1028,7 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, struct user_sdma_txreq *tx, u32 datalen) { u32 ahg[AHG_KDETH_ARRAY_SIZE]; + int ret; int idx = 0; u8 omfactor; /* KDETH.OM */ struct hfi1_user_sdma_pkt_q *pq = req->pq; @@ -1132,11 +1133,13 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, req->info.comp_idx, req->sde->this_idx, req->ahg_idx, ahg, idx, tidval); - sdma_txinit_ahg(&tx->txreq, - SDMA_TXREQ_F_USE_AHG, - datalen, req->ahg_idx, idx, - ahg, sizeof(req->hdr), - user_sdma_txreq_cb); + ret = sdma_txinit_ahg(&tx->txreq, + SDMA_TXREQ_F_USE_AHG, + datalen, req->ahg_idx, idx, + ahg, sizeof(req->hdr), + user_sdma_txreq_cb); + if (ret) + return ret; return idx; } diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 1680d0ac071ed..ae7a3939124f2 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -842,7 +842,7 @@ static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, mhop.bt_chunk_size; for (i = 0; i < table->num_hem; ++i) { - obj = i * buf_chunk_size / table->obj_size; + obj = (u64)i * buf_chunk_size / table->obj_size; if (table->hem[i]) hns_roce_table_mhop_put(hr_dev, table, obj, 0); } diff --git a/drivers/infiniband/hw/ionic/ionic_hw_stats.c b/drivers/infiniband/hw/ionic/ionic_hw_stats.c index 244a80dde08f3..1ed1ee4a209fe 100644 --- a/drivers/infiniband/hw/ionic/ionic_hw_stats.c +++ b/drivers/infiniband/hw/ionic/ionic_hw_stats.c @@ -237,35 +237,34 @@ static int ionic_get_hw_stats(struct ib_device *ibdev, static struct rdma_hw_stats * ionic_counter_alloc_stats(struct rdma_counter *counter) { + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); - struct ionic_counter *cntr; - int err; + struct rdma_hw_stats *stats; + int id; - cntr = kzalloc(sizeof(*cntr), GFP_KERNEL); - if (!cntr) - return NULL; - - /* buffer for current values from the device */ cntr->vals = kzalloc(PAGE_SIZE, GFP_KERNEL); if (!cntr->vals) - goto err_vals; + return NULL; - err = xa_alloc(&dev->counter_stats->xa_counters, &counter->id, - cntr, - XA_LIMIT(0, IONIC_MAX_QPID), - GFP_KERNEL); - if (err) - goto err_xa; + id = ida_alloc_max(&dev->counter_stats->counter_ida, + IONIC_MAX_QPID, GFP_KERNEL); + if (id < 0) + goto err_ida; - INIT_LIST_HEAD(&cntr->qp_list); + counter->id = id; + + stats = rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs, + dev->counter_stats->queue_stats_count, + RDMA_HW_STATS_DEFAULT_LIFESPAN); + if (!stats) + goto err_hw_stats; + + return stats; - return rdma_alloc_hw_stats_struct(dev->counter_stats->stats_hdrs, - dev->counter_stats->queue_stats_count, - RDMA_HW_STATS_DEFAULT_LIFESPAN); -err_xa: +err_hw_stats: + ida_free(&dev->counter_stats->counter_ida, id); +err_ida: kfree(cntr->vals); -err_vals: - kfree(cntr); return NULL; } @@ -273,14 +272,10 @@ ionic_counter_alloc_stats(struct rdma_counter *counter) static int ionic_counter_dealloc(struct rdma_counter *counter) { struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); - struct ionic_counter *cntr; - - cntr = xa_erase(&dev->counter_stats->xa_counters, counter->id); - if (!cntr) - return -EINVAL; + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + ida_free(&dev->counter_stats->counter_ida, counter->id); kfree(cntr->vals); - kfree(cntr); return 0; } @@ -289,13 +284,8 @@ static int ionic_counter_bind_qp(struct rdma_counter *counter, struct ib_qp *ibqp, u32 port) { - struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); struct ionic_qp *qp = to_ionic_qp(ibqp); - struct ionic_counter *cntr; - - cntr = xa_load(&dev->counter_stats->xa_counters, counter->id); - if (!cntr) - return -EINVAL; list_add_tail(&qp->qp_list_counter, &cntr->qp_list); ibqp->counter = counter; @@ -315,29 +305,23 @@ static int ionic_counter_unbind_qp(struct ib_qp *ibqp, u32 port) return 0; } -static int ionic_get_qp_stats(struct ib_device *ibdev, - struct rdma_hw_stats *hw_stats, - u32 counter_id) +static int ionic_counter_update_stats(struct rdma_counter *counter) { - struct ionic_ibdev *dev = to_ionic_ibdev(ibdev); - struct ionic_counter_stats *cs; - struct ionic_counter *cntr; + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + struct ionic_ibdev *dev = to_ionic_ibdev(counter->device); + struct ionic_counter_stats *cs = dev->counter_stats; dma_addr_t hw_stats_dma; struct ionic_qp *qp; int rc, stat_i = 0; - cs = dev->counter_stats; - cntr = xa_load(&cs->xa_counters, counter_id); - if (!cntr) - return -EINVAL; - hw_stats_dma = dma_map_single(dev->lif_cfg.hwdev, cntr->vals, PAGE_SIZE, DMA_FROM_DEVICE); rc = dma_mapping_error(dev->lif_cfg.hwdev, hw_stats_dma); if (rc) return rc; - memset(hw_stats->value, 0, sizeof(u64) * hw_stats->num_counters); + memset(counter->stats->value, 0, + sizeof(u64) * counter->stats->num_counters); list_for_each_entry(qp, &cntr->qp_list, qp_list_counter) { rc = ionic_hw_stats_cmd(dev, hw_stats_dma, PAGE_SIZE, @@ -347,7 +331,7 @@ static int ionic_get_qp_stats(struct ib_device *ibdev, goto err_cmd; for (stat_i = 0; stat_i < cs->queue_stats_count; ++stat_i) - hw_stats->value[stat_i] += + counter->stats->value[stat_i] += ionic_v1_stat_val(&cs->hdr[stat_i], cntr->vals, PAGE_SIZE); @@ -362,11 +346,6 @@ static int ionic_get_qp_stats(struct ib_device *ibdev, return rc; } -static int ionic_counter_update_stats(struct rdma_counter *counter) -{ - return ionic_get_qp_stats(counter->device, counter->stats, counter->id); -} - static int ionic_alloc_counters(struct ionic_ibdev *dev) { struct ionic_counter_stats *cs = dev->counter_stats; @@ -427,12 +406,22 @@ static const struct ib_device_ops ionic_hw_stats_ops = { .get_hw_stats = ionic_get_hw_stats, }; +static void ionic_counter_init(struct rdma_counter *counter) +{ + struct ionic_rdma_counter *cntr = to_ionic_rdma_counter(counter); + + INIT_LIST_HEAD(&cntr->qp_list); +} + static const struct ib_device_ops ionic_counter_stats_ops = { .counter_alloc_stats = ionic_counter_alloc_stats, .counter_dealloc = ionic_counter_dealloc, .counter_bind_qp = ionic_counter_bind_qp, .counter_unbind_qp = ionic_counter_unbind_qp, .counter_update_stats = ionic_counter_update_stats, + .counter_init = ionic_counter_init, + + INIT_RDMA_OBJ_SIZE(rdma_counter, ionic_rdma_counter, rdma_counter), }; void ionic_stats_init(struct ionic_ibdev *dev) @@ -462,7 +451,7 @@ void ionic_stats_init(struct ionic_ibdev *dev) return; } - xa_init_flags(&dev->counter_stats->xa_counters, XA_FLAGS_ALLOC); + ida_init(&dev->counter_stats->counter_ida); ib_set_device_ops(&dev->ibdev, &ionic_counter_stats_ops); } @@ -471,7 +460,7 @@ void ionic_stats_init(struct ionic_ibdev *dev) void ionic_stats_cleanup(struct ionic_ibdev *dev) { if (dev->counter_stats) { - xa_destroy(&dev->counter_stats->xa_counters); + ida_destroy(&dev->counter_stats->counter_ida); kfree(dev->counter_stats->hdr); kfree(dev->counter_stats->stats_hdrs); kfree(dev->counter_stats); diff --git a/drivers/infiniband/hw/ionic/ionic_ibdev.h b/drivers/infiniband/hw/ionic/ionic_ibdev.h index 63828240d6590..1c43344ec93ce 100644 --- a/drivers/infiniband/hw/ionic/ionic_ibdev.h +++ b/drivers/infiniband/hw/ionic/ionic_ibdev.h @@ -331,14 +331,21 @@ struct ionic_counter_stats { int queue_stats_count; struct ionic_v1_stat *hdr; struct rdma_stat_desc *stats_hdrs; - struct xarray xa_counters; + struct ida counter_ida; }; -struct ionic_counter { +struct ionic_rdma_counter { + struct rdma_counter rdma_counter; void *vals; struct list_head qp_list; }; +static inline struct ionic_rdma_counter * +to_ionic_rdma_counter(struct rdma_counter *counter) +{ + return container_of(counter, struct ionic_rdma_counter, rdma_counter); +} + static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev) { return container_of(ibdev, struct ionic_ibdev, ibdev); diff --git a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c index f3cd281c3a2f8..45846dc2e89b2 100644 --- a/drivers/infiniband/hw/ionic/ionic_lif_cfg.c +++ b/drivers/infiniband/hw/ionic/ionic_lif_cfg.c @@ -70,7 +70,7 @@ void ionic_fill_lif_cfg(struct ionic_lif *lif, struct ionic_lif_cfg *cfg) * eq_count is tunable; see ionic_eq_count */ cfg->aq_count = le32_to_cpu(ident->rdma.aq_qtype.qid_count); - cfg->eq_count = le32_to_cpu(ident->rdma.eq_qtype.qid_count); + cfg->eq_count = lif->ionic->neqs_per_lif; cfg->cq_count = le32_to_cpu(ident->rdma.cq_qtype.qid_count); cfg->qp_count = le32_to_cpu(ident->rdma.sq_qtype.qid_count); cfg->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif); diff --git a/drivers/infiniband/hw/irdma/hw.c b/drivers/infiniband/hw/irdma/hw.c index 7bad0e38786a2..cbf7f4d75e5ea 100644 --- a/drivers/infiniband/hw/irdma/hw.c +++ b/drivers/infiniband/hw/irdma/hw.c @@ -167,8 +167,7 @@ static void irdma_complete_cqp_request(struct irdma_cqp *cqp, struct irdma_cqp_request *cqp_request) { if (cqp_request->waiting) { - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } else if (cqp_request->callback_fcn) { cqp_request->callback_fcn(cqp_request); } @@ -1039,9 +1038,9 @@ static int irdma_create_cqp(struct irdma_pci_f *rf) INIT_LIST_HEAD(&cqp->cqp_avail_reqs); INIT_LIST_HEAD(&cqp->cqp_pending_reqs); - /* init the waitqueue of the cqp_requests and add them to the list */ + /* init the completion of the cqp_requests and add them to the list */ for (i = 0; i < sqsize; i++) { - init_waitqueue_head(&cqp->cqp_requests[i].waitq); + init_completion(&cqp->cqp_requests[i].comp); list_add_tail(&cqp->cqp_requests[i].list, &cqp->cqp_avail_reqs); } init_waitqueue_head(&cqp->remove_wq); diff --git a/drivers/infiniband/hw/irdma/main.h b/drivers/infiniband/hw/irdma/main.h index 0c16045a34be6..777d2ed406f8a 100644 --- a/drivers/infiniband/hw/irdma/main.h +++ b/drivers/infiniband/hw/irdma/main.h @@ -160,13 +160,12 @@ struct irdma_cqp_compl_info { struct irdma_cqp_request { struct cqp_cmds_info info; - wait_queue_head_t waitq; + struct completion comp; struct list_head list; refcount_t refcnt; void (*callback_fcn)(struct irdma_cqp_request *cqp_request); void *param; struct irdma_cqp_compl_info compl_info; - bool request_done; /* READ/WRITE_ONCE macros operate on it */ bool waiting:1; bool dynamic:1; bool pending:1; diff --git a/drivers/infiniband/hw/irdma/uk.c b/drivers/infiniband/hw/irdma/uk.c index 4cc81d61be7fa..7aab007619667 100644 --- a/drivers/infiniband/hw/irdma/uk.c +++ b/drivers/infiniband/hw/irdma/uk.c @@ -1540,15 +1540,12 @@ static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = { * irdma_setup_connection_wqes - setup WQEs necessary to complete * connection. * @qp: hw qp (user and kernel) - * @info: qp initialization info */ -static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp, - struct irdma_qp_uk_init_info *info) +static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp) { u16 move_cnt = 1; - if (!info->legacy_mode && - (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)) + if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE) move_cnt = 3; qp->conn_wqes = move_cnt; @@ -1699,7 +1696,7 @@ int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info) sq_ring_size = qp->sq_size << info->sq_shift; IRDMA_RING_INIT(qp->sq_ring, sq_ring_size); if (info->first_sq_wq) { - irdma_setup_connection_wqes(qp, info); + irdma_setup_connection_wqes(qp); qp->swqe_polarity = 1; qp->first_sq_wq = true; } else { diff --git a/drivers/infiniband/hw/irdma/user.h b/drivers/infiniband/hw/irdma/user.h index aeebf768174ab..a0b409a895f0e 100644 --- a/drivers/infiniband/hw/irdma/user.h +++ b/drivers/infiniband/hw/irdma/user.h @@ -560,7 +560,6 @@ struct irdma_qp_uk_init_info { u8 sq_shift; u8 rq_shift; int abi_ver; - bool legacy_mode; struct irdma_srq_uk *srq_uk; }; diff --git a/drivers/infiniband/hw/irdma/utils.c b/drivers/infiniband/hw/irdma/utils.c index e5e226b346211..43734ab827c07 100644 --- a/drivers/infiniband/hw/irdma/utils.c +++ b/drivers/infiniband/hw/irdma/utils.c @@ -442,7 +442,7 @@ struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp, if (cqp_request) { cqp_request->dynamic = true; if (wait) - init_waitqueue_head(&cqp_request->waitq); + init_completion(&cqp_request->comp); } } if (!cqp_request) { @@ -479,7 +479,7 @@ void irdma_free_cqp_request(struct irdma_cqp *cqp, if (cqp_request->dynamic) { kfree(cqp_request); } else { - WRITE_ONCE(cqp_request->request_done, false); + reinit_completion(&cqp_request->comp); cqp_request->callback_fcn = NULL; cqp_request->waiting = false; cqp_request->pending = false; @@ -514,8 +514,7 @@ irdma_free_pending_cqp_request(struct irdma_cqp *cqp, { if (cqp_request->waiting) { cqp_request->compl_info.error = true; - WRITE_ONCE(cqp_request->request_done, true); - wake_up(&cqp_request->waitq); + complete_all(&cqp_request->comp); } wait_event_timeout(cqp->remove_wq, refcount_read(&cqp_request->refcnt) == 1, 1000); @@ -608,9 +607,8 @@ static int irdma_wait_event(struct irdma_pci_f *rf, cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops); do { irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq); - if (wait_event_timeout(cqp_request->waitq, - READ_ONCE(cqp_request->request_done), - msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) + if (wait_for_completion_timeout(&cqp_request->comp, + msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS))) break; if (cqp_request->pending) @@ -1169,6 +1167,12 @@ void irdma_free_qp_rsrc(struct irdma_qp *iwqp) iwqp->kqp.dma_mem.va = NULL; kfree(iwqp->kqp.sq_wrid_mem); kfree(iwqp->kqp.rq_wrid_mem); + + if (iwqp->user_mode && iwqp->iwpbl) { + struct irdma_mr *iwmr = iwqp->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } /** @@ -2461,7 +2465,7 @@ void irdma_generate_flush_completions(struct irdma_qp *iwqp) cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid; sw_wqe = qp->sq_base[wqe_idx].elem; get_64bit_val(sw_wqe, 24, &wqe_qword); - cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE); + cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword); cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ; /* remove the SQ WR by moving SQ tail*/ IRDMA_RING_SET_TAIL(*sq_ring, diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c index 2b85e32134a5b..9ada0bc00bd07 100644 --- a/drivers/infiniband/hw/irdma/verbs.c +++ b/drivers/infiniband/hw/irdma/verbs.c @@ -462,6 +462,9 @@ static struct irdma_pbl *irdma_get_pbl(unsigned long va, list_for_each_entry (iwpbl, pbl_list, list) { if (iwpbl->user_base == va) { + struct irdma_mr *iwmr = iwpbl->iwmr; + + refcount_inc(&iwmr->user_ring_refs); list_del(&iwpbl->list); iwpbl->on_list = false; return iwpbl; @@ -631,18 +634,16 @@ static int irdma_setup_umode_qp(struct ib_udata *udata, iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx; iwqp->user_mode = 1; - if (req.user_wqe_bufs) { - info->qp_uk_init_info.legacy_mode = ucontext->legacy_mode; - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, - &ucontext->qp_reg_mem_list); - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); - - if (!iwqp->iwpbl) { - ret = -ENODATA; - ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); - return ret; - } + + spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); + iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs, + &ucontext->qp_reg_mem_list); + spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + + if (!iwqp->iwpbl) { + ret = -ENODATA; + ibdev_dbg(&iwdev->ibdev, "VERBS: no pbl info\n"); + return ret; } if (!ucontext->use_raw_attrs) { @@ -1889,6 +1890,11 @@ static void irdma_srq_free_rsrc(struct irdma_pci_f *rf, struct irdma_srq *iwsrq) dma_free_coherent(rf->sc_dev.hw->device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); iwsrq->kmem.va = NULL; + } else { + /* Not called in any failure path, so iwpbl is valid. */ + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); } irdma_free_rsrc(rf, rf->allocated_srqs, srq->srq_uk.srq_id); @@ -1911,6 +1917,21 @@ static void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq) iwcq->kmem_shadow.size, iwcq->kmem_shadow.va, iwcq->kmem_shadow.pa); iwcq->kmem_shadow.va = NULL; + } else { + struct irdma_mr *iwmr; + + /* May be called in a failure path before iwpbl is valid. */ + if (iwcq->iwpbl) { + iwmr = iwcq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } + + if (iwcq->iwpbl_shadow) { + iwmr = iwcq->iwpbl_shadow->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } } irdma_free_rsrc(rf, rf->allocated_cqs, cq->cq_uk.cq_id); @@ -2026,7 +2047,7 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, struct irdma_modify_cq_info info = {}; struct irdma_dma_mem kmem_buf; struct irdma_cq_mr *cqmr_buf; - struct irdma_pbl *iwpbl_buf; + struct irdma_pbl *iwpbl_buf = NULL; struct irdma_device *iwdev; struct irdma_pci_f *rf; struct irdma_cq_buf *cq_buf = NULL; @@ -2068,10 +2089,6 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - /* CQ resize not supported with legacy GEN_1 libi40iw */ - if (ucontext->legacy_mode) - return -EOPNOTSUPP; - if (ib_copy_from_udata(&req, udata, min(sizeof(req), udata->inlen))) return -EINVAL; @@ -2136,6 +2153,19 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, goto error; spin_lock_irqsave(&iwcq->lock, flags); + if (udata) { + struct irdma_pbl *old_iwpbl = iwcq->iwpbl; + + /* Only update if the resize was successful. Otherwise, HW is + * still pointing to the old PBL. + */ + iwcq->iwpbl = iwpbl_buf; + if (old_iwpbl) { + struct irdma_mr *old_iwmr = old_iwpbl->iwmr; + + refcount_dec(&old_iwmr->user_ring_refs); + } + } if (cq_buf) { cq_buf->kmem_buf = iwcq->kmem; cq_buf->hw = dev->hw; @@ -2151,6 +2181,11 @@ static int irdma_resize_cq(struct ib_cq *ibcq, int entries, return 0; error: + if (iwpbl_buf) { + struct irdma_mr *iwmr = iwpbl_buf->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } if (!udata) { dma_free_coherent(dev->hw->device, kmem_buf.size, kmem_buf.va, kmem_buf.pa); @@ -2428,6 +2463,11 @@ static int irdma_create_srq(struct ib_srq *ibsrq, dma_free_coherent(rf->hw.device, iwsrq->kmem.size, iwsrq->kmem.va, iwsrq->kmem.pa); free_rsrc: + if (iwsrq->user_mode && iwsrq->iwpbl) { + struct irdma_mr *iwmr = iwsrq->iwpbl->iwmr; + + refcount_dec(&iwmr->user_ring_refs); + } irdma_free_rsrc(rf, rf->allocated_srqs, iwsrq->srq_num); return err_code; } @@ -2506,6 +2546,8 @@ static int irdma_create_cq(struct ib_cq *ibcq, INIT_LIST_HEAD(&iwcq->resize_list); INIT_LIST_HEAD(&iwcq->cmpl_generated); iwcq->cq_num = cq_num; + iwcq->iwpbl = NULL; + iwcq->iwpbl_shadow = NULL; info.dev = dev; ukinfo->cq_size = max(entries, 4); ukinfo->cq_id = cq_num; @@ -2524,8 +2566,6 @@ static int irdma_create_cq(struct ib_cq *ibcq, struct irdma_ucontext *ucontext; struct irdma_create_cq_req req = {}; struct irdma_cq_mr *cqmr; - struct irdma_pbl *iwpbl; - struct irdma_pbl *iwpbl_shadow; struct irdma_cq_mr *cqmr_shadow; iwcq->user_mode = true; @@ -2539,34 +2579,34 @@ static int irdma_create_cq(struct ib_cq *ibcq, } spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, - &ucontext->cq_reg_mem_list); + iwcq->iwpbl = irdma_get_pbl((unsigned long)req.user_cq_buf, + &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl) { + if (!iwcq->iwpbl) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr = &iwpbl->cq_mr; + cqmr = &iwcq->iwpbl->cq_mr; if (rf->sc_dev.hw_attrs.uk_attrs.feature_flags & - IRDMA_FEATURE_CQ_RESIZE && !ucontext->legacy_mode) { + IRDMA_FEATURE_CQ_RESIZE) { spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - iwpbl_shadow = irdma_get_pbl( + iwcq->iwpbl_shadow = irdma_get_pbl( (unsigned long)req.user_shadow_area, &ucontext->cq_reg_mem_list); spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); - if (!iwpbl_shadow) { + if (!iwcq->iwpbl_shadow) { err_code = -EPROTO; goto cq_free_rsrc; } - cqmr_shadow = &iwpbl_shadow->cq_mr; + cqmr_shadow = &iwcq->iwpbl_shadow->cq_mr; info.shadow_area_pa = cqmr_shadow->cq_pbl.addr; } else { info.shadow_area_pa = cqmr->shadow; } - if (iwpbl->pbl_allocated) { + if (iwcq->iwpbl->pbl_allocated) { info.virtual_map = true; info.pbl_chunk_size = 1; info.first_pm_pbl_idx = cqmr->cq_pbl.idx; @@ -2806,7 +2846,7 @@ static bool irdma_check_mem_contiguous(u64 *arr, u32 npages, u32 pg_size) u32 pg_idx; for (pg_idx = 0; pg_idx < npages; pg_idx++) { - if ((*arr + (pg_size * pg_idx)) != arr[pg_idx]) + if ((*arr + ((u64)pg_size * pg_idx)) != arr[pg_idx]) return false; } @@ -2839,7 +2879,7 @@ static bool irdma_check_mr_contiguous(struct irdma_pble_alloc *palloc, for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { arr = leaf->addr; - if ((*start_addr + (i * pg_size * PBLE_PER_PAGE)) != *arr) + if ((*start_addr + ((u64)i * pg_size * PBLE_PER_PAGE)) != *arr) return false; ret = irdma_check_mem_contiguous(arr, leaf->cnt, pg_size); if (!ret) @@ -3369,6 +3409,7 @@ static struct irdma_mr *irdma_alloc_iwmr(struct ib_umem *region, if (!iwmr) return ERR_PTR(-ENOMEM); + refcount_set(&iwmr->user_ring_refs, 1); iwpbl = &iwmr->iwpbl; iwpbl->iwmr = iwmr; iwmr->region = region; @@ -3756,6 +3797,9 @@ static struct ib_mr *irdma_rereg_user_mr(struct ib_mr *ib_mr, int flags, if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) return ERR_PTR(-EOPNOTSUPP); + if (iwmr->type != IRDMA_MEMREG_TYPE_MEM) + return ERR_PTR(-EINVAL); + ret = ib_umem_check_rereg(iwmr->region, flags, new_access); if (ret) return ERR_PTR(ret); @@ -3865,41 +3909,41 @@ static struct ib_mr *irdma_get_dma_mr(struct ib_pd *pd, int acc) * irdma_del_memlist - Deleting pbl list entries for CQ/QP * @iwmr: iwmr for IB's user page addresses * @ucontext: ptr to user context + * + * Return: True if the MR is currently in-use by a QP/CQ/SRQ ring. */ -static void irdma_del_memlist(struct irdma_mr *iwmr, +static bool irdma_del_memlist(struct irdma_mr *iwmr, struct irdma_ucontext *ucontext) { struct irdma_pbl *iwpbl = &iwmr->iwpbl; unsigned long flags; + spinlock_t *lock; + bool in_use = false; switch (iwmr->type) { case IRDMA_MEMREG_TYPE_CQ: - spin_lock_irqsave(&ucontext->cq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->cq_reg_mem_list_lock, flags); + lock = &ucontext->cq_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_QP: - spin_lock_irqsave(&ucontext->qp_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->qp_reg_mem_list_lock, flags); + lock = &ucontext->qp_reg_mem_list_lock; break; case IRDMA_MEMREG_TYPE_SRQ: - spin_lock_irqsave(&ucontext->srq_reg_mem_list_lock, flags); - if (iwpbl->on_list) { - iwpbl->on_list = false; - list_del(&iwpbl->list); - } - spin_unlock_irqrestore(&ucontext->srq_reg_mem_list_lock, flags); + lock = &ucontext->srq_reg_mem_list_lock; break; default: - break; + return false; + } + + spin_lock_irqsave(lock, flags); + if (!refcount_dec_if_one(&iwmr->user_ring_refs)) { + in_use = true; + } else if (iwpbl->on_list) { + iwpbl->on_list = false; + list_del(&iwpbl->list); } + spin_unlock_irqrestore(lock, flags); + + return in_use; } /** @@ -3921,7 +3965,12 @@ static int irdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ucontext = rdma_udata_to_drv_context(udata, struct irdma_ucontext, ibucontext); - irdma_del_memlist(iwmr, ucontext); + + /* Do not allow the MR to be unpinned if it is still + * backing a user ring. + */ + if (irdma_del_memlist(iwmr, ucontext)) + return -EBUSY; } goto done; } diff --git a/drivers/infiniband/hw/irdma/verbs.h b/drivers/infiniband/hw/irdma/verbs.h index 289ebc9b23ca7..a1651641eb714 100644 --- a/drivers/infiniband/hw/irdma/verbs.h +++ b/drivers/infiniband/hw/irdma/verbs.h @@ -120,6 +120,7 @@ struct irdma_mr { u64 len; u64 pgaddrmem[IRDMA_MAX_SAVED_PHY_PGADDR]; struct irdma_pbl iwpbl; + refcount_t user_ring_refs; }; struct irdma_srq { @@ -152,6 +153,8 @@ struct irdma_cq { struct list_head resize_list; struct irdma_cq_poll_info cur_cqe; struct list_head cmpl_generated; + struct irdma_pbl *iwpbl; + struct irdma_pbl *iwpbl_shadow; }; struct irdma_cmpl_gen { diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c index f00bf3b015e73..285b3702198b7 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -459,6 +459,12 @@ static void mana_table_remove_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp xa_erase_irq(&mdev->qp_table_wq, qp->ibqp.qp_num); } +static void mana_table_drain_qp_ref(struct mana_ib_qp *qp) +{ + mana_put_qp_ref(qp); + wait_for_completion(&qp->free); +} + static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp) { u32 qids = qp->ud_qp.queues[MANA_UD_SEND_QUEUE].id | MANA_SENDQ_MASK; @@ -477,6 +483,7 @@ static int mana_table_store_ud_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *q remove_sq: xa_erase_irq(&mdev->qp_table_wq, qids); + mana_table_drain_qp_ref(qp); return err; } @@ -524,8 +531,7 @@ static void mana_table_remove_qp(struct mana_ib_dev *mdev, qp->ibqp.qp_type); return; } - mana_put_qp_ref(qp); - wait_for_completion(&qp->free); + mana_table_drain_qp_ref(qp); } static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd, diff --git a/drivers/infiniband/hw/mana/wr.c b/drivers/infiniband/hw/mana/wr.c index 1813567d3b16c..36a1d506f08f6 100644 --- a/drivers/infiniband/hw/mana/wr.c +++ b/drivers/infiniband/hw/mana/wr.c @@ -144,7 +144,7 @@ static int mana_ib_post_send_ud(struct mana_ib_qp *qp, const struct ib_ud_wr *wr int mana_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, const struct ib_send_wr **bad_wr) { - int err; + int err = 0; struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp); for (; wr; wr = wr->next) { diff --git a/drivers/infiniband/hw/mlx5/cong.c b/drivers/infiniband/hw/mlx5/cong.c index a78a067e3ce7f..e3c4b1c928169 100644 --- a/drivers/infiniband/hw/mlx5/cong.c +++ b/drivers/infiniband/hw/mlx5/cong.c @@ -361,7 +361,7 @@ static int mlx5_ib_set_cc_params(struct mlx5_ib_dev *dev, u32 port_num, MLX5_SET(field_select_r_roce_rp, field, field_select_r_roce_rp, attr_mask); - err = mlx5_cmd_exec_in(dev->mdev, modify_cong_params, in); + err = mlx5_cmd_exec_in(mdev, modify_cong_params, in); kvfree(in); alloc_err: mlx5_ib_put_native_port_mdev(dev, port_num + 1); @@ -399,15 +399,13 @@ static ssize_t get_param(struct file *filp, char __user *buf, size_t count, int offset = param->offset; u32 var = 0; int ret; - char lbuf[11]; + char lbuf[12]; ret = mlx5_ib_get_cc_params(param->dev, param->port_num, offset, &var); if (ret) return ret; - ret = snprintf(lbuf, sizeof(lbuf), "%d\n", var); - if (ret < 0) - return ret; + ret = scnprintf(lbuf, sizeof(lbuf), "%u\n", var); return simple_read_from_buffer(buf, count, pos, lbuf, ret); } diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 651d76bca114d..c6590dde68332 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -169,7 +169,8 @@ enum { static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe, struct mlx5_ib_qp *qp) { - enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1); + enum rdma_link_layer ll = + rdma_port_get_link_layer(qp->ibqp.device, qp->port); struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device); struct mlx5_ib_srq *srq = NULL; struct mlx5_ib_wq *wq; diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index 09709ae5bd471..c6b0bb786739d 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -1585,6 +1585,130 @@ static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index, return 0; } +static int mlx5_ib_query_port_speed_from_port(struct mlx5_ib_dev *dev, + u32 port_num, u64 *speed) +{ + struct ib_port_speed_info speed_info; + struct ib_port_attr attr = {}; + int err; + + err = mlx5_ib_query_port(&dev->ib_dev, port_num, &attr); + if (err) + return err; + + if (attr.state == IB_PORT_DOWN) { + *speed = 0; + return 0; + } + + err = ib_port_attr_to_speed_info(&attr, &speed_info); + if (err) + return err; + + *speed = speed_info.rate; + return 0; +} + +static int mlx5_ib_query_port_speed_from_vport(struct mlx5_core_dev *mdev, + u8 op_mod, u16 vport, + u8 other_vport, u64 *speed, + struct mlx5_ib_dev *dev, + u32 port_num) +{ + u32 max_tx_speed; + u8 vport_state; + int err; + + err = mlx5_query_vport_max_tx_speed(mdev, op_mod, vport, other_vport, + &max_tx_speed, &vport_state); + if (err) + return err; + + if (vport_state == VPORT_STATE_DOWN || max_tx_speed == 0) + /* Value 0 indicates field not supported, fallback */ + return mlx5_ib_query_port_speed_from_port(dev, port_num, + speed); + + *speed = max_tx_speed; + return 0; +} + +static int mlx5_ib_query_port_speed_from_bond(struct mlx5_ib_dev *dev, + u32 port_num, u64 *speed) +{ + struct mlx5_core_dev *mdev = dev->mdev; + u32 bond_speed; + int err; + + err = mlx5_lag_query_bond_speed(mdev, &bond_speed); + if (err) + return err; + + *speed = bond_speed / MLX5_MAX_TX_SPEED_UNIT; + + return 0; +} + +static int mlx5_ib_query_port_speed_non_rep(struct mlx5_ib_dev *dev, + u32 port_num, u64 *speed) +{ + u16 op_mod = MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT; + + if (mlx5_lag_is_roce(dev->mdev)) + return mlx5_ib_query_port_speed_from_bond(dev, port_num, + speed); + + return mlx5_ib_query_port_speed_from_vport(dev->mdev, op_mod, 0, false, + speed, dev, port_num); +} + +static int mlx5_ib_query_port_speed_rep(struct mlx5_ib_dev *dev, u32 port_num, + u64 *speed) +{ + struct mlx5_eswitch_rep *rep; + struct mlx5_core_dev *mdev; + u16 op_mod; + + if (!dev->port[port_num - 1].rep) { + mlx5_ib_warn(dev, "Representor doesn't exist for port %u\n", + port_num); + return -EINVAL; + } + + rep = dev->port[port_num - 1].rep; + mdev = mlx5_eswitch_get_core_dev(rep->esw); + if (!mdev) + return -ENODEV; + + if (rep->vport == MLX5_VPORT_UPLINK) { + if (mlx5_lag_is_sriov(mdev)) + return mlx5_ib_query_port_speed_from_bond(dev, + port_num, + speed); + + return mlx5_ib_query_port_speed_from_port(dev, port_num, + speed); + } + + op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + return mlx5_ib_query_port_speed_from_vport(dev->mdev, op_mod, + rep->vport, true, speed, dev, + port_num); +} + +int mlx5_ib_query_port_speed(struct ib_device *ibdev, u32 port_num, u64 *speed) +{ + struct mlx5_ib_dev *dev = to_mdev(ibdev); + + if (mlx5_ib_port_link_layer(ibdev, port_num) == + IB_LINK_LAYER_INFINIBAND || mlx5_core_mp_enabled(dev->mdev)) + return mlx5_ib_query_port_speed_from_port(dev, port_num, speed); + else if (!dev->is_rep) + return mlx5_ib_query_port_speed_non_rep(dev, port_num, speed); + else + return mlx5_ib_query_port_speed_rep(dev, port_num, speed); +} + static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index, union ib_gid *gid) { @@ -1895,6 +2019,9 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) dev->lb.qps == 1) { if (!dev->lb.enabled) { err = mlx5_nic_vport_update_local_lb(dev->mdev, true); + if (err) + goto err_rollback; + dev->lb.enabled = true; } } @@ -1902,6 +2029,14 @@ int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) mutex_unlock(&dev->lb.mutex); return err; + +err_rollback: + if (td) + dev->lb.user_td--; + if (qp) + dev->lb.qps--; + mutex_unlock(&dev->lb.mutex); + return err; } void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp) @@ -4359,6 +4494,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .query_device = mlx5_ib_query_device, .query_gid = mlx5_ib_query_gid, .query_pkey = mlx5_ib_query_pkey, + .query_port_speed = mlx5_ib_query_port_speed, .query_qp = mlx5_ib_query_qp, .query_srq = mlx5_ib_query_srq, .query_ucontext = mlx5_ib_query_ucontext, @@ -5225,6 +5361,7 @@ static int __init mlx5_ib_init(void) dd_err: mlx5r_rep_cleanup(); rep_err: + rcu_barrier(); mlx5_ib_qp_event_cleanup(); qp_event_err: destroy_workqueue(mlx5_ib_event_wq); @@ -5238,6 +5375,7 @@ static void __exit mlx5_ib_cleanup(void) auxiliary_driver_unregister(&mlx5r_driver); auxiliary_driver_unregister(&mlx5r_mp_driver); mlx5r_rep_cleanup(); + rcu_barrier(); mlx5_ib_qp_event_cleanup(); destroy_workqueue(mlx5_ib_event_wq); diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index ba55e6db3bca7..20591530a17b3 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -1446,6 +1446,8 @@ int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); int mlx5_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props); +int mlx5_ib_query_port_speed(struct ib_device *ibdev, u32 port_num, + u64 *speed); void mlx5_ib_populate_pas(struct ib_umem *umem, size_t page_size, __be64 *pas, u64 access_flags); int mlx5_ib_get_cqe_size(struct ib_cq *ibcq); diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index bd5a7d1ad14c3..0c8cd84964be0 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1679,6 +1679,10 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device, atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); umem_dmabuf->private = mr; if (!pinned_mode) { + err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); + if (err) + goto err_dereg_mr; + err = mlx5r_store_odp_mkey(dev, &mr->mmkey); if (err) goto err_dereg_mr; diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 47bb42576ad5d..5cd1968a33d0d 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -1816,6 +1816,9 @@ int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) struct mlx5_eq_param param = {}; int err = 0; + if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) + return -EOPNOTSUPP; + mutex_lock(&dev->odp_eq_mutex); if (eq->core) goto unlock; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 02a0f4920cabf..50b20264c4bdd 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -637,6 +637,7 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr) { int desc_sz = 1 << qp->sq.wqe_shift; + int rq_buf_size, sq_buf_size; if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n", @@ -661,11 +662,21 @@ static int set_user_buf_size(struct mlx5_ib_dev *dev, if (attr->qp_type == IB_QPT_RAW_PACKET || qp->flags & IB_QP_CREATE_SOURCE_QPN) { - base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift; - qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &base->ubuffer.buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, + &qp->raw_packet_qp.sq.ubuffer.buf_size)) + return -EINVAL; } else { - base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + - (qp->sq.wqe_cnt << 6); + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &rq_buf_size)) + return -EINVAL; + if (check_shl_overflow(qp->sq.wqe_cnt, 6, &sq_buf_size)) + return -EINVAL; + if (check_add_overflow(rq_buf_size, sq_buf_size, + &base->ubuffer.buf_size)) + return -EINVAL; } return 0; @@ -989,7 +1000,11 @@ static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, qp->rq.offset = 0; qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); - qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; + if (check_shl_overflow(qp->rq.wqe_cnt, qp->rq.wqe_shift, + &qp->sq.offset)) { + err = -EINVAL; + goto err_bfreg; + } err = set_user_buf_size(dev, qp, ucmd, base, attr); if (err) diff --git a/drivers/infiniband/sw/rxe/rxe_odp.c b/drivers/infiniband/sw/rxe/rxe_odp.c index 4d4e3b324dd29..16b3fbdf2f867 100644 --- a/drivers/infiniband/sw/rxe/rxe_odp.c +++ b/drivers/infiniband/sw/rxe/rxe_odp.c @@ -115,6 +115,7 @@ int rxe_odp_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, err = rxe_odp_init_pages(mr); if (err) { ib_umem_odp_release(umem_odp); + mr->umem = NULL; return err; } diff --git a/drivers/infiniband/sw/rxe/rxe_qp.c b/drivers/infiniband/sw/rxe/rxe_qp.c index 95f1c1c2949de..c31c0c9a53828 100644 --- a/drivers/infiniband/sw/rxe/rxe_qp.c +++ b/drivers/infiniband/sw/rxe/rxe_qp.c @@ -124,6 +124,7 @@ static void free_rd_atomic_resources(struct rxe_qp *qp) } kfree(qp->resp.resources); qp->resp.resources = NULL; + qp->resp.res = NULL; } } @@ -658,13 +659,24 @@ int rxe_qp_from_attr(struct rxe_qp *qp, struct ib_qp_attr *attr, int mask, int max_dest_rd_atomic = attr->max_dest_rd_atomic ? roundup_pow_of_two(attr->max_dest_rd_atomic) : 0; - qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; - + /* + * Not gated by IB_QP_STATE, so the responder task is live. + * Quiesce recv_task like rxe_qp_reset() before swapping the + * rd_atomic array, so rxe_receiver() cannot race the free/ + * realloc. + */ + rxe_disable_task(&qp->recv_task); free_rd_atomic_resources(qp); - + qp->attr.max_dest_rd_atomic = max_dest_rd_atomic; err = alloc_rd_atomic_resources(qp, max_dest_rd_atomic); + /* + * On ENOMEM leave recv_task quiesced: qp->resp.resources is + * NULL and rxe_prepare_res()/find_resource() would deref it. + * Re-enable only after a fresh array is installed. + */ if (err) return err; + rxe_enable_task(&qp->recv_task); } if (mask & IB_QP_EN_SQD_ASYNC_NOTIFY) diff --git a/drivers/infiniband/sw/rxe/rxe_req.c b/drivers/infiniband/sw/rxe/rxe_req.c index 12d03f390b097..24f5c044363f7 100644 --- a/drivers/infiniband/sw/rxe/rxe_req.c +++ b/drivers/infiniband/sw/rxe/rxe_req.c @@ -701,6 +701,21 @@ int rxe_requester(struct rxe_qp *qp) if (unlikely(!wqe)) goto exit; + /* + * Don't trust user space data: a user QP's WQE comes from an mmap'd + * ring, so num_sge/cur_sge are attacker-controlled. Bound num_sge like + * get_srq_wqe(); bound cur_sge only when payload exists (dma.resid), + * since copy_data() skips dma->sge[] on a zero-length copy (all a + * max_sge == 0 QP can post). + */ + if (unlikely(wqe->dma.num_sge > qp->sq.max_sge || + (wqe->dma.resid && + wqe->dma.cur_sge >= qp->sq.max_sge))) { + rxe_dbg_qp(qp, "invalid num_sge/cur_sge in send wqe\n"); + wqe->status = IB_WC_LOC_QP_OP_ERR; + goto err; + } + if (rxe_wqe_is_fenced(qp, wqe)) { qp->req.wait_fence = 1; goto exit; diff --git a/drivers/infiniband/sw/rxe/rxe_resp.c b/drivers/infiniband/sw/rxe/rxe_resp.c index 995805e16d78b..6aabf1d07ea34 100644 --- a/drivers/infiniband/sw/rxe/rxe_resp.c +++ b/drivers/infiniband/sw/rxe/rxe_resp.c @@ -1203,7 +1203,14 @@ static enum resp_states do_complete(struct rxe_qp *qp, spin_lock_irqsave(&qp->state_lock, flags); if (unlikely(qp_state(qp) == IB_QPS_ERR)) { spin_unlock_irqrestore(&qp->state_lock, flags); - return RESPST_CHK_RESOURCE; + /* The packet was executed and completed before the QP + * moved to ERROR; it must be consumed exactly once. + * Re-entering the request chain with the stale packet + * would copy it into every remaining recv WQE as a new + * completion. Remaining WQEs are flushed by the drain + * path at rxe_receiver() entry. + */ + return pkt ? RESPST_CLEANUP : RESPST_CHK_RESOURCE; } spin_unlock_irqrestore(&qp->state_lock, flags); diff --git a/drivers/infiniband/sw/siw/siw_cm.c b/drivers/infiniband/sw/siw/siw_cm.c index 49ff121f77fe4..bb7d909639071 100644 --- a/drivers/infiniband/sw/siw/siw_cm.c +++ b/drivers/infiniband/sw/siw/siw_cm.c @@ -1701,7 +1701,7 @@ int siw_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) cep->state = SIW_EPSTATE_CLOSED; siw_free_cm_id(cep); - if (qp->cep) { + if (qp->cep == cep) { siw_cep_put(cep); qp->cep = NULL; } diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c index efa2f097b5828..9fbade0b85715 100644 --- a/drivers/infiniband/sw/siw/siw_verbs.c +++ b/drivers/infiniband/sw/siw/siw_verbs.c @@ -316,6 +316,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, struct siw_ucontext *uctx = rdma_udata_to_drv_context(udata, struct siw_ucontext, base_ucontext); + struct siw_uresp_create_qp uresp = {}; unsigned long flags; int num_sqe, num_rqe, rv = 0; size_t length; @@ -369,11 +370,6 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, spin_lock_init(&qp->rq_lock); spin_lock_init(&qp->orq_lock); - rv = siw_qp_add(sdev, qp); - if (rv) - goto err_atomic; - - /* All queue indices are derived from modulo operations * on a free running 'get' (consumer) and 'put' (producer) * unsigned counter. Having queue sizes at power of two @@ -391,14 +387,14 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (qp->sendq == NULL) { rv = -ENOMEM; - goto err_out_xa; + goto err_out; } if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) { if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) qp->attrs.flags |= SIW_SIGNAL_ALL_WR; else { rv = -EINVAL; - goto err_out_xa; + goto err_out; } } qp->pd = pd; @@ -424,7 +420,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (qp->recvq == NULL) { rv = -ENOMEM; - goto err_out_xa; + goto err_out; } qp->attrs.rq_size = num_rqe; } @@ -439,11 +435,8 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, qp->attrs.state = SIW_QP_STATE_IDLE; if (udata) { - struct siw_uresp_create_qp uresp = {}; - uresp.num_sqe = num_sqe; uresp.num_rqe = num_rqe; - uresp.qp_id = qp_id(qp); if (qp->sendq) { length = num_sqe * sizeof(struct siw_sqe); @@ -452,7 +445,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, length, &uresp.sq_key); if (!qp->sq_entry) { rv = -ENOMEM; - goto err_out_xa; + goto err_out; } } @@ -464,9 +457,23 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (!qp->rq_entry) { uresp.sq_key = SIW_INVAL_UOBJ_KEY; rv = -ENOMEM; - goto err_out_xa; + goto err_out; } } + } + qp->tx_cpu = siw_get_tx_cpu(sdev); + if (qp->tx_cpu < 0) { + rv = -EINVAL; + goto err_out; + } + init_completion(&qp->qp_free); + + rv = siw_qp_add(sdev, qp); + if (rv) + goto err_out_tx; + + if (udata) { + uresp.qp_id = qp_id(qp); if (udata->outlen < sizeof(uresp)) { rv = -EINVAL; @@ -476,22 +483,19 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs, if (rv) goto err_out_xa; } - qp->tx_cpu = siw_get_tx_cpu(sdev); - if (qp->tx_cpu < 0) { - rv = -EINVAL; - goto err_out_xa; - } + INIT_LIST_HEAD(&qp->devq); spin_lock_irqsave(&sdev->lock, flags); list_add_tail(&qp->devq, &sdev->qp_list); spin_unlock_irqrestore(&sdev->lock, flags); - init_completion(&qp->qp_free); - return 0; err_out_xa: xa_erase(&sdev->qp_xa, qp_id(qp)); +err_out_tx: + siw_put_tx_cpu(qp->tx_cpu); +err_out: if (uctx) { rdma_user_mmap_entry_remove(qp->sq_entry); rdma_user_mmap_entry_remove(qp->rq_entry); diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c index 5b4d76e97437d..b897c1c3e4c8d 100644 --- a/drivers/infiniband/ulp/ipoib/ipoib_main.c +++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c @@ -2755,6 +2755,7 @@ static int __init ipoib_init_module(void) err_sa: ib_sa_unregister_client(&ipoib_sa_client); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); err_fs: ipoib_unregister_debugfs(); @@ -2772,6 +2773,7 @@ static void __exit ipoib_cleanup_module(void) ib_sa_unregister_client(&ipoib_sa_client); ipoib_unregister_debugfs(); destroy_workqueue(ipoib_workqueue); + rcu_barrier(); } module_init(ipoib_init_module); diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 640634f96d72a..6483a55170cd1 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -59,6 +59,8 @@ static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); +static void isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, + struct ib_device *ib_dev); static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) { @@ -498,6 +500,8 @@ isert_connect_release(struct isert_conn *isert_conn) if (isert_conn->qp) isert_destroy_qp(isert_conn); + isert_unmap_tx_desc(&isert_conn->login_tx_desc, device->ib_device); + if (isert_conn->login_desc) isert_free_login_buf(isert_conn); @@ -945,27 +949,16 @@ isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, } if (!login->login_failed) { if (login->login_complete) { - ret = isert_alloc_rx_descriptors(isert_conn); - if (ret) - return ret; - - ret = isert_post_recvm(isert_conn, - ISERT_QP_MAX_RECV_DTOS); - if (ret) - return ret; - - /* Now we are in FULL_FEATURE phase */ - mutex_lock(&isert_conn->mutex); - isert_conn->state = ISER_CONN_FULL_FEATURE; - mutex_unlock(&isert_conn->mutex); - goto post_send; + /* Posted and sent from isert_get_rx_pdu(). */ + isert_conn->login_rsp_pending = true; + return 0; } ret = isert_login_post_recv(isert_conn); if (ret) return ret; } -post_send: + ret = isert_login_post_send(isert_conn, tx_desc); if (ret) return ret; @@ -973,6 +966,21 @@ isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, return 0; } +static int +isert_check_login_req(struct isert_conn *isert_conn) +{ + struct iscsi_hdr *hdr = isert_get_iscsi_hdr(isert_conn->login_desc); + u32 dlength = ntoh24(hdr->dlength); + + if (unlikely(dlength > (u32)isert_conn->login_req_len)) { + isert_dbg("login PDU declares %u data bytes but only %d were received\n", + dlength, isert_conn->login_req_len); + return -EINVAL; + } + + return 0; +} + static void isert_rx_login_req(struct isert_conn *isert_conn) { @@ -1335,6 +1343,21 @@ isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, ISER_RX_SIZE, DMA_FROM_DEVICE); + /* + * The data segment length declared in the BHS is attacker controlled + * and is used further down to read that many bytes out of the fixed + * size receive descriptor, so it has to be checked against the number + * of bytes that were actually received. Comparing without subtracting + * also rejects PDUs shorter than the iSER and iSCSI headers, which + * would otherwise be parsed out of stale descriptor contents. + */ + if (unlikely(wc->byte_len < ISER_HEADERS_LEN + ntoh24(hdr->dlength))) { + isert_err("PDU declares %u data bytes but only %u bytes were received\n", + ntoh24(hdr->dlength), wc->byte_len); + iscsit_cause_connection_reinstatement(isert_conn->conn, 0); + return; + } + isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, (int)(wc->byte_len - ISER_HEADERS_LEN)); @@ -1396,8 +1419,12 @@ isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) if (isert_conn->conn) { struct iscsi_login *login = isert_conn->conn->conn_login; - if (login && !login->first_request) + if (login && !login->first_request) { + if (isert_check_login_req(isert_conn)) + return; + isert_rx_login_req(isert_conn); + } } mutex_lock(&isert_conn->mutex); @@ -2362,6 +2389,10 @@ isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) if (!login->first_request) return 0; + ret = isert_check_login_req(isert_conn); + if (ret) + return ret; + isert_rx_login_req(isert_conn); isert_info("before login_comp conn: %p\n", conn); @@ -2587,8 +2618,30 @@ static void isert_free_conn(struct iscsit_conn *conn) static void isert_get_rx_pdu(struct iscsit_conn *conn) { + struct isert_conn *isert_conn = conn->context; struct completion comp; + /* The login timeout timer can fail the login after isert_put_login_tx(). */ + if (!isert_conn->login_rsp_pending) + return; + + isert_conn->login_rsp_pending = false; + + /* The session is registered by now; see isert_put_login_tx(). */ + if (isert_alloc_rx_descriptors(isert_conn)) + return; + + if (isert_post_recvm(isert_conn, ISERT_QP_MAX_RECV_DTOS)) + return; + + /* Now we are in FULL_FEATURE phase */ + mutex_lock(&isert_conn->mutex); + isert_conn->state = ISER_CONN_FULL_FEATURE; + mutex_unlock(&isert_conn->mutex); + + if (isert_login_post_send(isert_conn, &isert_conn->login_tx_desc)) + return; + init_completion(&comp); wait_for_completion_interruptible(&comp); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index 0b2dfd6e7e270..0bac5aa66c802 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -178,6 +178,7 @@ struct isert_conn { struct completion login_comp; struct completion login_req_comp; struct iser_tx_desc login_tx_desc; + bool login_rsp_pending; struct rdma_cm_id *cm_id; struct ib_qp *qp; struct ib_cq *cq; diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c index 1fc67fdff9fc2..12a9a888fe0ee 100644 --- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c +++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c @@ -1049,6 +1049,11 @@ static void process_read(struct rtrs_srv_con *con, "Processing read request failed, invalid message\n"); return; } + usr_len = le16_to_cpu(msg->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, READ); id = srv_path->ops_ids[buf_id]; @@ -1056,7 +1061,6 @@ static void process_read(struct rtrs_srv_con *con, id->dir = READ; id->msg_id = buf_id; id->rd_msg = msg; - usr_len = le16_to_cpu(msg->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, @@ -1102,6 +1106,11 @@ static void process_write(struct rtrs_srv_con *con, rtrs_srv_state_str(srv_path->state)); return; } + usr_len = le16_to_cpu(req->usr_len); + if (usr_len > off) { + pr_debug("rtrs-srv: Invalid usr_len %zu > off %u\n", usr_len, off); + return; + } rtrs_srv_get_ops_ids(srv_path); rtrs_srv_update_rdma_stats(srv_path->stats, off, WRITE); id = srv_path->ops_ids[buf_id]; @@ -1109,7 +1118,6 @@ static void process_write(struct rtrs_srv_con *con, id->dir = WRITE; id->msg_id = buf_id; - usr_len = le16_to_cpu(req->usr_len); data_len = off - usr_len; data = page_address(srv->chunks[buf_id]); ret = ctx->ops.rdma_ev(srv->priv, id, data, data_len, diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c index 7b696a07e6033..99131492abf70 100644 --- a/drivers/infiniband/ulp/srp/ib_srp.c +++ b/drivers/infiniband/ulp/srp/ib_srp.c @@ -1943,7 +1943,8 @@ static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp, ch->req_lim += be32_to_cpu(rsp->req_lim_delta); if (rsp->tag == ch->tsk_mgmt_tag) { ch->tsk_mgmt_status = -1; - if (be32_to_cpu(rsp->resp_data_len) >= 4) + if (be32_to_cpu(rsp->resp_data_len) >= 4 && + byte_len >= sizeof(*rsp) + 4) ch->tsk_mgmt_status = rsp->data[3]; complete(&ch->tsk_mgmt_done); } else { @@ -2043,13 +2044,20 @@ static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, } static void srp_process_cred_req(struct srp_rdma_ch *ch, - struct srp_cred_req *req) + struct srp_cred_req *req, u32 byte_len) { - struct srp_cred_rsp rsp = { - .opcode = SRP_CRED_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_cred_rsp rsp = { .opcode = SRP_CRED_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, ch->target->scsi_host, PFX + "dropping truncated SRP_CRED_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) shost_printk(KERN_ERR, ch->target->scsi_host, PFX @@ -2057,14 +2065,21 @@ static void srp_process_cred_req(struct srp_rdma_ch *ch, } static void srp_process_aer_req(struct srp_rdma_ch *ch, - struct srp_aer_req *req) + struct srp_aer_req *req, u32 byte_len) { struct srp_target_port *target = ch->target; - struct srp_aer_rsp rsp = { - .opcode = SRP_AER_RSP, - .tag = req->tag, - }; - s32 delta = be32_to_cpu(req->req_lim_delta); + struct srp_aer_rsp rsp = { .opcode = SRP_AER_RSP }; + s32 delta; + + if (byte_len < sizeof(*req)) { + shost_printk(KERN_ERR, target->scsi_host, PFX + "dropping truncated SRP_AER_REQ (%u bytes received, %zu expected)\n", + byte_len, sizeof(*req)); + return; + } + + rsp.tag = req->tag; + delta = be32_to_cpu(req->req_lim_delta); shost_printk(KERN_ERR, target->scsi_host, PFX "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); @@ -2106,11 +2121,11 @@ static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) break; case SRP_CRED_REQ: - srp_process_cred_req(ch, iu->buf); + srp_process_cred_req(ch, iu->buf, wc->byte_len); break; case SRP_AER_REQ: - srp_process_aer_req(ch, iu->buf); + srp_process_aer_req(ch, iu->buf, wc->byte_len); break; case SRP_T_LOGOUT: diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c index ba70c64a0aa9e..212d4c7662c8a 100644 --- a/drivers/infiniband/ulp/srpt/ib_srpt.c +++ b/drivers/infiniband/ulp/srpt/ib_srpt.c @@ -959,6 +959,7 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, struct srpt_rdma_ch *ch = ioctx->ch; struct scatterlist *prev = NULL; unsigned prev_nents; + u8 n_rdma, n_rw_ctx; int ret, i; if (nbufs == 1) { @@ -970,6 +971,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, return -ENOMEM; } + n_rw_ctx = ioctx->n_rw_ctx; + n_rdma = ioctx->n_rdma; + for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) { struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i]; u64 remote_addr = be64_to_cpu(db->va); @@ -1016,6 +1020,9 @@ static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx, } if (ioctx->rw_ctxs != &ioctx->s_rw_ctx) kfree(ioctx->rw_ctxs); + ioctx->rw_ctxs = NULL; + ioctx->n_rw_ctx = n_rw_ctx; + ioctx->n_rdma = n_rdma; return ret; } @@ -1596,7 +1603,7 @@ static void srpt_handle_cmd(struct srpt_rdma_ch *ch, rc = target_init_cmd(cmd, ch->sess, &send_ioctx->sense_data[0], scsilun_to_int(&srp_cmd->lun), data_len, - TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF); + cmd->sam_task_attr, dir, TARGET_SCF_ACK_KREF); if (rc != 0) { pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc, srp_cmd->tag); diff --git a/drivers/input/evdev.c b/drivers/input/evdev.c index 90ff6be85cf46..1cf5ab25dc0a4 100644 --- a/drivers/input/evdev.c +++ b/drivers/input/evdev.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include "input-compat.h" @@ -67,8 +68,10 @@ static size_t evdev_get_mask_cnt(unsigned int type) [EV_SND] = SND_CNT, [EV_FF] = FF_CNT, }; + unsigned long mask = array_index_mask_nospec(type, EV_CNT); - return (type < EV_CNT) ? counts[type] : 0; + /* Returns 0 for out-of-bounds types, including speculatively */ + return counts[type & mask] & mask; } /* requires the buffer lock to be held */ @@ -146,11 +149,11 @@ static void __evdev_queue_syn_dropped(struct evdev_client *client) struct timespec64 ts = ktime_to_timespec64(ev_time[client->clk_type]); struct input_event ev; + memset(&ev, 0, sizeof(ev)); ev.input_event_sec = ts.tv_sec; ev.input_event_usec = ts.tv_nsec / NSEC_PER_USEC; ev.type = EV_SYN; ev.code = SYN_DROPPED; - ev.value = 0; client->buffer[client->head++] = ev; client->head &= client->bufsize - 1; @@ -218,20 +221,20 @@ static void __pass_event(struct evdev_client *client, client->head &= client->bufsize - 1; if (unlikely(client->head == client->tail)) { + struct input_event ev; + + memset(&ev, 0, sizeof(ev)); + ev.input_event_sec = event->input_event_sec; + ev.input_event_usec = event->input_event_usec; + ev.type = EV_SYN; + ev.code = SYN_DROPPED; + /* * This effectively "drops" all unconsumed events, leaving * EV_SYN/SYN_DROPPED plus the newest event in the queue. */ client->tail = (client->head - 2) & (client->bufsize - 1); - - client->buffer[client->tail] = (struct input_event) { - .input_event_sec = event->input_event_sec, - .input_event_usec = event->input_event_usec, - .type = EV_SYN, - .code = SYN_DROPPED, - .value = 0, - }; - + client->buffer[client->tail] = ev; client->packet_head = client->tail; } @@ -253,6 +256,8 @@ static void evdev_pass_values(struct evdev_client *client, if (client->revoked) return; + memset(&event, 0, sizeof(event)); + ts = ktime_to_timespec64(ev_time[client->clk_type]); event.input_event_sec = ts.tv_sec; event.input_event_usec = ts.tv_nsec / NSEC_PER_USEC; diff --git a/drivers/input/joystick/iforce/iforce-packets.c b/drivers/input/joystick/iforce/iforce-packets.c index effa76bfd8f9a..01fee14054fb2 100644 --- a/drivers/input/joystick/iforce/iforce-packets.c +++ b/drivers/input/joystick/iforce/iforce-packets.c @@ -155,6 +155,9 @@ void iforce_process_packet(struct iforce *iforce, switch (packet_id) { case 0x01: /* joystick position data */ + if (len < 7) + break; + input_report_abs(dev, ABS_X, (__s16) get_unaligned_le16(data)); input_report_abs(dev, ABS_Y, @@ -170,6 +173,9 @@ void iforce_process_packet(struct iforce *iforce, break; case 0x03: /* wheel position data */ + if (len < 7) + break; + input_report_abs(dev, ABS_WHEEL, (__s16) get_unaligned_le16(data)); input_report_abs(dev, ABS_GAS, 255 - data[2]); @@ -181,6 +187,9 @@ void iforce_process_packet(struct iforce *iforce, break; case 0x02: /* status report */ + if (len < 2) + break; + input_report_key(dev, BTN_DEAD, data[0] & 0x02); input_sync(dev); @@ -200,7 +209,7 @@ void iforce_process_packet(struct iforce *iforce, } } - for (j = 3; j < len; j += 2) + for (j = 3; j + sizeof(u16) <= len; j += sizeof(u16)) mark_core_as_ready(iforce, get_unaligned_le16(data + j)); break; diff --git a/drivers/input/joystick/iforce/iforce-usb.c b/drivers/input/joystick/iforce/iforce-usb.c index 1f00f76b01745..5e3a8dbe6c6fb 100644 --- a/drivers/input/joystick/iforce/iforce-usb.c +++ b/drivers/input/joystick/iforce/iforce-usb.c @@ -158,6 +158,9 @@ static void iforce_usb_irq(struct urb *urb) goto exit; } + if (!urb->actual_length) + goto exit; + iforce_process_packet(iforce, iforce_usb->data_in[0], iforce_usb->data_in + 1, urb->actual_length - 1); diff --git a/drivers/input/joystick/psxpad-spi.c b/drivers/input/joystick/psxpad-spi.c index f902a56d011f4..aac77a2cfe468 100644 --- a/drivers/input/joystick/psxpad-spi.c +++ b/drivers/input/joystick/psxpad-spi.c @@ -373,6 +373,7 @@ static int psxpad_spi_probe(struct spi_device *spi) return err; } + spi_set_drvdata(spi, pad); pm_runtime_enable(&spi->dev); return 0; diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c index ebfb841aafee1..1b6f38f629056 100644 --- a/drivers/input/joystick/xpad.c +++ b/drivers/input/joystick/xpad.c @@ -428,6 +428,7 @@ static const struct xpad_device { { 0x3285, 0x0646, "Nacon Pro Compact", 0, XTYPE_XBOXONE }, { 0x3285, 0x0662, "Nacon Revolution5 Pro", 0, XTYPE_XBOX360 }, { 0x3285, 0x0663, "Nacon Evol-X", 0, XTYPE_XBOXONE }, + { 0x3507, 0x000b, "ZENAIM LEVERLESS", 0, XTYPE_XBOX360 }, { 0x3537, 0x1004, "GameSir T4 Kaleid", 0, XTYPE_XBOX360 }, { 0x3537, 0x100f, "GameSir Nova 2 Lite", 0, XTYPE_XBOX360 }, { 0x3537, 0x1010, "GameSir G7 SE", 0, XTYPE_XBOXONE }, @@ -591,6 +592,7 @@ static const struct usb_device_id xpad_table[] = { XPAD_XBOX360_VENDOR(0x31e3), /* Wooting Keyboards */ XPAD_XBOX360_VENDOR(0x3285), /* Nacon GC-100 */ XPAD_XBOXONE_VENDOR(0x3285), /* Nacon Evol-X */ + XPAD_XBOX360_VENDOR(0x3507), /* ZENAIM Controllers */ XPAD_XBOX360_VENDOR(0x3537), /* GameSir Controllers */ XPAD_XBOXONE_VENDOR(0x3537), /* GameSir Controllers */ XPAD_XBOX360_VENDOR(0x3651), /* CRKD Controllers */ diff --git a/drivers/input/keyboard/atkbd.c b/drivers/input/keyboard/atkbd.c index 8eeac27dec545..db7762abfe5c6 100644 --- a/drivers/input/keyboard/atkbd.c +++ b/drivers/input/keyboard/atkbd.c @@ -1937,6 +1937,28 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = { }, .callback = atkbd_deactivate_fixup, }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HONOR"), + DMI_MATCH(DMI_PRODUCT_NAME, "BCC-N"), + }, + .callback = atkbd_deactivate_fixup, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HONOR"), + DMI_MATCH(DMI_PRODUCT_NAME, "FMB-P"), + }, + .callback = atkbd_deactivate_fixup, + }, + { + /* HONOR MagicBook Pro 14 2026 */ + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HONOR"), + DMI_MATCH(DMI_PRODUCT_NAME, "ZQC-P"), + }, + .callback = atkbd_deactivate_fixup, + }, { /* Lenovo Yoga Air 14 (83QK) */ .matches = { @@ -1946,9 +1968,10 @@ static const struct dmi_system_id atkbd_dmi_quirk_table[] __initconst = { .callback = atkbd_deactivate_fixup, }, { + /* Xiaomi Book Pro 14 (TM2424) */ .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "HONOR"), - DMI_MATCH(DMI_PRODUCT_NAME, "BCC-N"), + DMI_MATCH(DMI_SYS_VENDOR, "XIAOMI"), + DMI_MATCH(DMI_PRODUCT_NAME, "Xiaomi Book Pro 14"), }, .callback = atkbd_deactivate_fixup, }, diff --git a/drivers/input/misc/cs40l50-vibra.c b/drivers/input/misc/cs40l50-vibra.c index 7aa7d577e01b1..f4c605eafd111 100644 --- a/drivers/input/misc/cs40l50-vibra.c +++ b/drivers/input/misc/cs40l50-vibra.c @@ -139,10 +139,10 @@ static struct cs40l50_effect *cs40l50_find_effect(int id, struct list_head *effe static int cs40l50_effect_bank_set(struct cs40l50_work *work_data, struct cs40l50_effect *effect) { - s16 bank_type = work_data->custom_data[0] & CS40L50_CUSTOM_DATA_MASK; + u32 bank_type = work_data->custom_data[0] & CS40L50_CUSTOM_DATA_MASK; if (bank_type >= CS40L50_WVFRM_BANK_NUM) { - dev_err(work_data->vib->dev, "Invalid bank (%d)\n", bank_type); + dev_err(work_data->vib->dev, "Invalid bank (%u)\n", bank_type); return -EINVAL; } @@ -327,6 +327,12 @@ static int cs40l50_add(struct input_dev *dev, struct ff_effect *effect, return -EINVAL; } + if (periodic->custom_len < CS40L50_OWT_CUSTOM_DATA_SIZE) { + dev_err(vib->dev, "Invalid custom data length (%u)\n", + periodic->custom_len); + return -EINVAL; + } + work_data.custom_data = memdup_array_user(effect->u.periodic.custom_data, effect->u.periodic.custom_len, sizeof(s16)); diff --git a/drivers/input/misc/ims-pcu.c b/drivers/input/misc/ims-pcu.c index 94113e14bf599..168662a2cc890 100644 --- a/drivers/input/misc/ims-pcu.c +++ b/drivers/input/misc/ims-pcu.c @@ -448,6 +448,14 @@ static void ims_pcu_handle_response(struct ims_pcu *pcu) } } +static void ims_pcu_reset_packet(struct ims_pcu *pcu) +{ + pcu->have_stx = false; + pcu->have_dle = false; + pcu->read_pos = 0; + pcu->check_sum = 0; +} + static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb) { int i; @@ -460,6 +468,14 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb) continue; if (pcu->have_dle) { + if (pcu->read_pos >= IMS_PCU_BUF_SIZE) { + dev_warn(pcu->dev, + "Packet too long (%d bytes), discarding\n", + pcu->read_pos); + ims_pcu_reset_packet(pcu); + continue; + } + pcu->have_dle = false; pcu->read_buf[pcu->read_pos++] = data; pcu->check_sum += data; @@ -472,10 +488,8 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb) dev_warn(pcu->dev, "Unexpected STX at byte %d, discarding old data\n", pcu->read_pos); + ims_pcu_reset_packet(pcu); pcu->have_stx = true; - pcu->have_dle = false; - pcu->read_pos = 0; - pcu->check_sum = 0; break; case IMS_PCU_PROTOCOL_DLE: @@ -495,12 +509,18 @@ static void ims_pcu_process_data(struct ims_pcu *pcu, struct urb *urb) ims_pcu_handle_response(pcu); } - pcu->have_stx = false; - pcu->have_dle = false; - pcu->read_pos = 0; + ims_pcu_reset_packet(pcu); break; default: + if (pcu->read_pos >= IMS_PCU_BUF_SIZE) { + dev_warn(pcu->dev, + "Packet too long (%d bytes), discarding\n", + pcu->read_pos); + ims_pcu_reset_packet(pcu); + continue; + } + pcu->read_buf[pcu->read_pos++] = data; pcu->check_sum += data; break; diff --git a/drivers/input/mouse/byd.c b/drivers/input/mouse/byd.c index 71aa23dd7d8d0..91bcff433898b 100644 --- a/drivers/input/mouse/byd.c +++ b/drivers/input/mouse/byd.c @@ -425,7 +425,7 @@ static void byd_disconnect(struct psmouse *psmouse) struct byd_data *priv = psmouse->private; if (priv) { - timer_delete(&priv->timer); + timer_shutdown_sync(&priv->timer); kfree(psmouse->private); psmouse->private = NULL; } diff --git a/drivers/input/mouse/focaltech.c b/drivers/input/mouse/focaltech.c index 356b99d485443..d1a3b48ee891f 100644 --- a/drivers/input/mouse/focaltech.c +++ b/drivers/input/mouse/focaltech.c @@ -197,7 +197,7 @@ static void focaltech_process_rel_packet(struct psmouse *psmouse, { struct focaltech_data *priv = psmouse->private; struct focaltech_hw_state *state = &priv->state; - int finger1, finger2; + unsigned int finger1, finger2; state->pressed = packet[0] >> 7; finger1 = ((packet[0] >> 4) & 0x7) - 1; diff --git a/drivers/input/rmi4/rmi_f54.c b/drivers/input/rmi4/rmi_f54.c index 61909e1a39e24..6c6cdec7da9e3 100644 --- a/drivers/input/rmi4/rmi_f54.c +++ b/drivers/input/rmi4/rmi_f54.c @@ -104,7 +104,9 @@ struct f54_data { enum rmi_f54_report_type report_type; u8 *report_data; + size_t max_report_size; int report_size; + int report_error; bool is_busy; struct mutex status_mutex; @@ -339,6 +341,12 @@ static void rmi_f54_buffer_queue(struct vb2_buffer *vb) mutex_lock(&f54->data_mutex); } + if (f54->report_error) { + dev_err(&f54->fn->dev, "Error acquiring report: %d\n", f54->report_error); + state = VB2_BUF_STATE_ERROR; + goto data_done; + } + ptr = vb2_plane_vaddr(vb, 0); if (!ptr) { dev_err(&f54->fn->dev, "Error acquiring frame ptr\n"); @@ -444,7 +452,12 @@ static int rmi_f54_set_input(struct f54_data *f54, unsigned int i) static int rmi_f54_vidioc_s_input(struct file *file, void *priv, unsigned int i) { - return rmi_f54_set_input(video_drvdata(file), i); + struct f54_data *f54 = video_drvdata(file); + + if (vb2_is_busy(&f54->queue)) + return -EBUSY; + + return rmi_f54_set_input(f54, i); } static int rmi_f54_vidioc_g_input(struct file *file, void *priv, @@ -545,7 +558,14 @@ static void rmi_f54_work(struct work_struct *work) dev_err(&fn->dev, "Bad report size, report type=%d\n", f54->report_type); error = -EINVAL; - goto error; /* retry won't help */ + goto out; /* retry won't help */ + } + + if (report_size > f54->max_report_size) { + dev_err(&fn->dev, "Report size %d exceeds buffer size %zu\n", + report_size, f54->max_report_size); + error = -EINVAL; + goto out; } /* @@ -556,7 +576,7 @@ static void rmi_f54_work(struct work_struct *work) &command); if (error) { dev_err(&fn->dev, "Failed to read back command\n"); - goto error; + goto out; } if (command & F54_GET_REPORT) { if (time_after(jiffies, f54->timeout)) { @@ -564,7 +584,7 @@ static void rmi_f54_work(struct work_struct *work) error = -ETIMEDOUT; } report_size = 0; - goto error; + goto out; } rmi_dbg(RMI_DEBUG_FN, &fn->dev, "Get report command completed, reading data\n"); @@ -579,7 +599,7 @@ static void rmi_f54_work(struct work_struct *work) fifo, sizeof(fifo)); if (error) { dev_err(&fn->dev, "Failed to set fifo start offset\n"); - goto abort; + goto out; } error = rmi_read_block(fn->rmi_dev, fn->fd.data_base_addr + @@ -588,16 +608,17 @@ static void rmi_f54_work(struct work_struct *work) if (error) { dev_err(&fn->dev, "%s: read [%d bytes] returned %d\n", __func__, size, error); - goto abort; + goto out; } } -abort: - f54->report_size = error ? 0 : report_size; -error: +out: if (error) report_size = 0; + f54->report_size = report_size; + f54->report_error = error; + if (report_size == 0 && !error) { queue_delayed_work(f54->workqueue, &f54->work, msecs_to_jiffies(1)); @@ -678,8 +699,8 @@ static int rmi_f54_probe(struct rmi_function *fn) rx = f54->num_rx_electrodes; tx = f54->num_tx_electrodes; - f54->report_data = devm_kzalloc(&fn->dev, - array3_size(tx, rx, sizeof(u16)), + f54->max_report_size = array3_size(tx, rx, sizeof(u16)); + f54->report_data = devm_kzalloc(&fn->dev, f54->max_report_size, GFP_KERNEL); if (f54->report_data == NULL) return -ENOMEM; diff --git a/drivers/input/rmi4/rmi_f55.c b/drivers/input/rmi4/rmi_f55.c index 488adaca4dd00..a0877d32a9141 100644 --- a/drivers/input/rmi4/rmi_f55.c +++ b/drivers/input/rmi4/rmi_f55.c @@ -54,10 +54,10 @@ static int rmi_f55_detect(struct rmi_function *fn) f55->num_tx_electrodes = f55->qry[F55_NUM_TX_OFFSET]; f55->cfg_num_rx_electrodes = f55->num_rx_electrodes; - f55->cfg_num_tx_electrodes = f55->num_rx_electrodes; + f55->cfg_num_tx_electrodes = f55->num_tx_electrodes; drv_data->num_rx_electrodes = f55->cfg_num_rx_electrodes; - drv_data->num_tx_electrodes = f55->cfg_num_rx_electrodes; + drv_data->num_tx_electrodes = f55->cfg_num_tx_electrodes; if (f55->qry[F55_PHYS_CHAR_OFFSET] & F55_CAP_SENSOR_ASSIGN) { int i, total; diff --git a/drivers/input/tablet/aiptek.c b/drivers/input/tablet/aiptek.c index 2b3fbb0455d5c..9e78105bace77 100644 --- a/drivers/input/tablet/aiptek.c +++ b/drivers/input/tablet/aiptek.c @@ -658,6 +658,8 @@ static void aiptek_irq(struct urb *urb) pck = (data[1] & aiptek->curSetting.stylusButtonUpper) != 0 ? 1 : 0; macro = dv && p && tip && !(data[3] & 1) ? (data[3] >> 1) : -1; + if (macro >= ARRAY_SIZE(macroKeyEvents)) + macro = -1; z = get_unaligned_le16(data + 4); if (dv) { @@ -699,7 +701,9 @@ static void aiptek_irq(struct urb *urb) left = (data[1]& aiptek->curSetting.mouseButtonLeft) != 0 ? 1 : 0; right = (data[1] & aiptek->curSetting.mouseButtonRight) != 0 ? 1 : 0; middle = (data[1] & aiptek->curSetting.mouseButtonMiddle) != 0 ? 1 : 0; - macro = dv && p && left && !(data[3] & 1) ? (data[3] >> 1) : 0; + macro = dv && p && left && !(data[3] & 1) ? (data[3] >> 1) : -1; + if (macro >= ARRAY_SIZE(macroKeyEvents)) + macro = -1; if (dv) { /* If the selected tool changed, reset the old @@ -737,11 +741,11 @@ static void aiptek_irq(struct urb *urb) */ else if (data[0] == 6) { macro = get_unaligned_le16(data + 1); - if (macro > 0) { + if (macro > 0 && macro - 1 < ARRAY_SIZE(macroKeyEvents)) { input_report_key(inputdev, macroKeyEvents[macro - 1], 0); } - if (macro < 25) { + if (macro + 1 < ARRAY_SIZE(macroKeyEvents)) { input_report_key(inputdev, macroKeyEvents[macro + 1], 0); } @@ -760,7 +764,8 @@ static void aiptek_irq(struct urb *urb) aiptek->curSetting.toolMode; } - input_report_key(inputdev, macroKeyEvents[macro], 1); + if (macro < ARRAY_SIZE(macroKeyEvents)) + input_report_key(inputdev, macroKeyEvents[macro], 1); input_report_abs(inputdev, ABS_MISC, 1 | AIPTEK_REPORT_TOOL_UNKNOWN); input_sync(inputdev); diff --git a/drivers/input/touchscreen/hynitron_cstxxx.c b/drivers/input/touchscreen/hynitron_cstxxx.c index 1d8ca90dcda69..af66d91848b30 100644 --- a/drivers/input/touchscreen/hynitron_cstxxx.c +++ b/drivers/input/touchscreen/hynitron_cstxxx.c @@ -313,6 +313,12 @@ static void cst3xx_touch_report(struct i2c_client *client) return; touch_cnt = buf[5] & CST3XX_TOUCH_COUNT_MASK; + if (touch_cnt > ts_data->chip->max_touch_num) { + dev_err(&client->dev, "cst3xx invalid touch count (%d vs %d max)\n", + touch_cnt, ts_data->chip->max_touch_num); + return; + } + /* * Check the check bit of the last touch slot. The check bit is * always present after touch point 1 for valid data, and then @@ -335,9 +341,10 @@ static void cst3xx_touch_report(struct i2c_client *client) finger_id = (buf[idx] >> 4) & 0x0f; /* Sanity check we don't have more fingers than we expect */ - if (ts_data->chip->max_touch_num < finger_id) { - dev_err(&client->dev, "cst3xx touch read failure\n"); - break; + if (finger_id >= ts_data->chip->max_touch_num) { + dev_err(&client->dev, + "cst3xx invalid finger id %d\n", finger_id); + return; } /* sw value of 0 means no touch, 0x03 means touch */ diff --git a/drivers/input/touchscreen/sur40.c b/drivers/input/touchscreen/sur40.c index 7b3b10cbfcfc4..0a4e40b85ab86 100644 --- a/drivers/input/touchscreen/sur40.c +++ b/drivers/input/touchscreen/sur40.c @@ -725,21 +725,13 @@ static int sur40_probe(struct usb_interface *interface, goto err_free_input; } - /* register the polled input device */ - error = input_register_device(input); - if (error) { - dev_err(&interface->dev, - "Unable to register polled input device."); - goto err_free_buffer; - } - /* register the video master device */ snprintf(sur40->v4l2.name, sizeof(sur40->v4l2.name), "%s", DRIVER_LONG); error = v4l2_device_register(sur40->dev, &sur40->v4l2); if (error) { dev_err(&interface->dev, "Unable to register video master device."); - goto err_unreg_v4l2; + goto err_free_buffer; } /* initialize the lock and subdevice */ @@ -795,6 +787,14 @@ static int sur40_probe(struct usb_interface *interface, if (error) { dev_err(&interface->dev, "Unable to register video subdevice."); + goto err_free_ctrl; + } + + /* register the polled input device */ + error = input_register_device(input); + if (error) { + dev_err(&interface->dev, + "Unable to register polled input device."); goto err_unreg_video; } @@ -806,6 +806,8 @@ static int sur40_probe(struct usb_interface *interface, err_unreg_video: video_unregister_device(&sur40->vdev); +err_free_ctrl: + v4l2_ctrl_handler_free(&sur40->hdl); err_unreg_v4l2: v4l2_device_unregister(&sur40->v4l2); err_free_buffer: @@ -823,11 +825,12 @@ static void sur40_disconnect(struct usb_interface *interface) { struct sur40_state *sur40 = usb_get_intfdata(interface); + input_unregister_device(sur40->input); + v4l2_ctrl_handler_free(&sur40->hdl); video_unregister_device(&sur40->vdev); v4l2_device_unregister(&sur40->v4l2); - input_unregister_device(sur40->input); kfree(sur40->bulk_in_buffer); kfree(sur40); diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c index 6cc979b26151e..cb99cb13089a7 100644 --- a/drivers/interconnect/core.c +++ b/drivers/interconnect/core.c @@ -528,7 +528,7 @@ struct icc_path *of_icc_get_by_index(struct device *dev, int idx) path->name = kasprintf(GFP_KERNEL, "%s-%s", src_data->node->name, dst_data->node->name); if (!path->name) { - kfree(path); + icc_put(path); path = ERR_PTR(-ENOMEM); } @@ -626,8 +626,9 @@ struct icc_path *icc_get(struct device *dev, const char *src, const char *dst) path->name = kasprintf(GFP_KERNEL, "%s-%s", src_node->name, dst_node->name); if (!path->name) { - kfree(path); - path = ERR_PTR(-ENOMEM); + mutex_unlock(&icc_lock); + icc_put(path); + return ERR_PTR(-ENOMEM); } out: mutex_unlock(&icc_lock); diff --git a/drivers/iommu/amd/debugfs.c b/drivers/iommu/amd/debugfs.c index 3909a1fb218e9..f7bb551d285f6 100644 --- a/drivers/iommu/amd/debugfs.c +++ b/drivers/iommu/amd/debugfs.c @@ -176,19 +176,13 @@ static ssize_t devid_write(struct file *filp, const char __user *ubuf, kfree(srcid_ptr); return -ENODEV; } - break; - } - - if (pci_seg->id != seg) { + sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); kfree(srcid_ptr); - return -EINVAL; + return cnt; } - sbdf = PCI_SEG_DEVID_TO_SBDF(seg, devid); - kfree(srcid_ptr); - - return cnt; + return -EINVAL; } static int devid_show(struct seq_file *m, void *unused) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 32175daa3dd8a..c07c3a01b978b 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -854,10 +854,11 @@ static void __init free_command_buffer(struct amd_iommu *iommu) void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp, size_t size) { + int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; void *buf; size = PAGE_ALIGN(size); - buf = iommu_alloc_pages_sz(gfp, size); + buf = iommu_alloc_pages_node_sz(nid, gfp, size); if (!buf) return NULL; if (check_feature(FEATURE_SNP) && @@ -930,7 +931,9 @@ static void free_ga_log(struct amd_iommu *iommu) { #ifdef CONFIG_IRQ_REMAP iommu_free_pages(iommu->ga_log); + iommu->ga_log = NULL; iommu_free_pages(iommu->ga_log_tail); + iommu->ga_log_tail = NULL; #endif } @@ -972,14 +975,19 @@ static int iommu_ga_log_enable(struct amd_iommu *iommu) static int iommu_init_ga_log(struct amd_iommu *iommu) { + int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; + if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) return 0; - iommu->ga_log = iommu_alloc_pages_sz(GFP_KERNEL, GA_LOG_SIZE); + if (iommu->ga_log && iommu->ga_log_tail) + return 0; + + iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE); if (!iommu->ga_log) goto err_out; - iommu->ga_log_tail = iommu_alloc_pages_sz(GFP_KERNEL, 8); + iommu->ga_log_tail = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, 8); if (!iommu->ga_log_tail) goto err_out; @@ -1930,19 +1938,20 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h, else iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; + if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { + pr_warn_once("Host Address Translation is not supported.\n"); + amd_iommu_hatdis = true; + } + /* XT and GAM require GA mode. */ if ((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0) { amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; + break; } else { if (h->efr_reg & BIT(IOMMU_EFR_XTSUP_SHIFT)) amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; } - if (h->efr_attr & BIT(IOMMU_IVHD_ATTR_HATDIS_SHIFT)) { - pr_warn_once("Host Address Translation is not supported.\n"); - amd_iommu_hatdis = true; - } - early_iommu_features_init(iommu, h); break; @@ -3059,7 +3068,7 @@ static void disable_iommus(void) * disable suspend until real resume implemented */ -static void amd_iommu_resume(void) +static void amd_iommu_resume(void *data) { struct amd_iommu *iommu; @@ -3074,7 +3083,7 @@ static void amd_iommu_resume(void) amd_iommu_enable_interrupts(); } -static int amd_iommu_suspend(void) +static int amd_iommu_suspend(void *data) { /* disable IOMMUs to go out of the way for BIOS */ disable_iommus(); @@ -3082,11 +3091,15 @@ static int amd_iommu_suspend(void) return 0; } -static struct syscore_ops amd_iommu_syscore_ops = { +static const struct syscore_ops amd_iommu_syscore_ops = { .suspend = amd_iommu_suspend, .resume = amd_iommu_resume, }; +static struct syscore amd_iommu_syscore = { + .ops = &amd_iommu_syscore_ops, +}; + static void __init free_iommu_resources(void) { free_iommu_all(); @@ -3096,14 +3109,31 @@ static void __init free_iommu_resources(void) /* SB IOAPIC is always on this device in AMD systems */ #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) +/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */ +#define IOAPIC_SB_DEVID_FAM18H_M4H ((0x00 << 8) | PCI_DEVFN(0xb, 0)) + +/* + * The Southbridge IOAPIC is assigned a GSI Base of 0 (handling interrupts + * 0 through 23). + */ +static int __init get_sb_ioapic_id(void) +{ + int idx = mp_find_ioapic(0); + + if (idx < 0) + return -ENODEV; + + return mpc_ioapic_id(idx); +} + static bool __init check_ioapic_information(void) { const char *fw_bug = FW_BUG; bool ret, has_sb_ioapic; - int idx; + int idx, sb_apicid; has_sb_ioapic = false; - ret = false; + ret = true; /* * If we have map overrides on the kernel command line the @@ -3113,6 +3143,16 @@ static bool __init check_ioapic_information(void) if (cmdline_maps) fw_bug = ""; + sb_apicid = get_sb_ioapic_id(); + if (sb_apicid < 0) { + /* + * Lack of SB IOAPIC registration is not a firmware bug, + * e.g. kernel booted with noapic or noacpi. + */ + fw_bug = ""; + goto out; + } + for (idx = 0; idx < nr_ioapics; idx++) { int devid, id = mpc_ioapic_id(idx); @@ -3121,12 +3161,16 @@ static bool __init check_ioapic_information(void) pr_err("%s: IOAPIC[%d] not in IVRS table\n", fw_bug, id); ret = false; - } else if (devid == IOAPIC_SB_DEVID) { + } else if (id == sb_apicid && (devid == IOAPIC_SB_DEVID || + (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON && + boot_cpu_data.x86 == 0x18 && + boot_cpu_data.x86_model >= 0x4 && + boot_cpu_data.x86_model <= 0xf && + devid == IOAPIC_SB_DEVID_FAM18H_M4H))) { has_sb_ioapic = true; - ret = true; } } - +out: if (!has_sb_ioapic) { /* * We expect the SB IOAPIC to be listed in the IVRS @@ -3137,6 +3181,7 @@ static bool __init check_ioapic_information(void) * device id for the IOAPIC in the system. */ pr_err("%s: No southbridge IOAPIC found\n", fw_bug); + ret = false; } if (!ret) @@ -3458,7 +3503,7 @@ static int __init state_next(void) init_state = IOMMU_ENABLED; break; case IOMMU_ENABLED: - register_syscore_ops(&amd_iommu_syscore_ops); + register_syscore(&amd_iommu_syscore); iommu_snp_enable(); amd_iommu_apply_erratum_snp(); @@ -3576,12 +3621,12 @@ int __init amd_iommu_enable(void) void amd_iommu_disable(void) { - amd_iommu_suspend(); + amd_iommu_suspend(NULL); } int amd_iommu_reenable(int mode) { - amd_iommu_resume(); + amd_iommu_resume(NULL); return 0; } @@ -3860,6 +3905,12 @@ static int __init parse_ivrs_acpihid(char *str) return 1; found: + if (early_acpihid_map_size == EARLY_MAP_SIZE) { + pr_err("Early ACPI HID map overflow - ignoring ivrs_acpihid%s\n", + str); + return 1; + } + p = acpiid; hid = strsep(&p, ":"); uid = p; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index c458eec8a5bba..12302647e2675 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1408,11 +1408,23 @@ static int iommu_completion_wait(struct amd_iommu *iommu) int ret; u64 data; - if (!iommu->need_sync) - return 0; - raw_spin_lock_irqsave(&iommu->lock, flags); + if (!iommu->need_sync) { + /* + * No command has been queued since the last completion-wait. + * A concurrent CPU may have already queued that CWAIT and + * cleared need_sync; need_sync == false only means a covering + * CWAIT is queued, not that all prior commands have completed. + * Wait for the last allocated sequence number so that any + * command queued before this call (possibly on another CPU) + * is guaranteed to have completed before returning. + */ + data = iommu->cmd_sem_val; + raw_spin_unlock_irqrestore(&iommu->lock, flags); + return wait_on_sem(iommu, data); + } + data = get_cmdsem_val(iommu); build_completion_wait(&cmd, iommu, data); @@ -1422,9 +1434,7 @@ static int iommu_completion_wait(struct amd_iommu *iommu) if (ret) return ret; - ret = wait_on_sem(iommu, data); - - return ret; + return wait_on_sem(iommu, data); } static void domain_flush_complete(struct protection_domain *domain) diff --git a/drivers/iommu/amd/ppr.c b/drivers/iommu/amd/ppr.c index 1f8d2823bea42..1dde19a79e80f 100644 --- a/drivers/iommu/amd/ppr.c +++ b/drivers/iommu/amd/ppr.c @@ -140,7 +140,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) if (event.fault.prm.pasid == 0 || event.fault.prm.pasid >= dev_data->max_pasids) { pr_info_ratelimited("Invalid PASID : 0x%x, device : 0x%x\n", - event.fault.prm.pasid, pdev->dev.id); + event.fault.prm.pasid, dev_data->devid); goto out; } @@ -151,7 +151,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) /* Submit event */ iommu_report_device_fault(&pdev->dev, &event); - + pci_dev_put(pdev); return; out: @@ -159,6 +159,7 @@ static void iommu_call_iopf_notifier(struct amd_iommu *iommu, u64 *raw) amd_iommu_complete_ppr(&pdev->dev, PPR_PASID(raw[0]), IOMMU_PAGE_RESP_FAILURE, PPR_TAG(raw[0]) & 0x1FF); + pci_dev_put(pdev); } void amd_iommu_poll_ppr_log(struct amd_iommu *iommu) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 9ec5591565083..a52f86454f77d 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -285,6 +285,20 @@ static int arm_vsmmu_vsid_to_sid(struct arm_vsmmu *vsmmu, u32 vsid, u32 *sid) return ret; } +static int arm_vsmmu_vdevice_init(struct iommufd_vdevice *vdev) +{ + struct device *dev = iommufd_vdevice_to_device(vdev); + struct arm_smmu_master *master = dev_iommu_priv_get(dev); + + /* + * arm_vsmmu_vsid_to_sid() maps a vSID to master->streams[0] alone, so + * more streams would leave the rest stale and none reads out of bounds. + */ + if (master->num_streams != 1) + return -EOPNOTSUPP; + return 0; +} + /* This is basically iommu_viommu_arm_smmuv3_invalidate in u64 for conversion */ struct arm_vsmmu_invalidation_cmd { union { @@ -391,6 +405,7 @@ int arm_vsmmu_cache_invalidate(struct iommufd_viommu *viommu, static const struct iommufd_viommu_ops arm_vsmmu_ops = { .alloc_domain_nested = arm_vsmmu_alloc_domain_nested, .cache_invalidate = arm_vsmmu_cache_invalidate, + .vdevice_init = arm_vsmmu_vdevice_init, }; size_t arm_smmu_get_viommu_size(struct device *dev, diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 59a480974d80f..80b56d1501b14 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -104,6 +104,17 @@ void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, target->data[0] &= cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); } + /* + * Enable Hardware Access and Dirty updates (DBM) if supported. This is + * safe to enable by default, as PTE_WRITE and PTE_DBM share the same bit, + * while the EPD0 config can't get as far as fetching any PTEs anyway. + */ + if (master->smmu->features & ARM_SMMU_FEAT_HA) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA); + if (master->smmu->features & ARM_SMMU_FEAT_HD) + target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HD); + if (master->smmu->features & ARM_SMMU_FEAT_HAFT && system_supports_haft()) + target->data[1] |= cpu_to_le64(CTXDESC_CD_1_HAFT); /* * MAIR value is pretty much constant and global, so we can just get it @@ -223,6 +234,9 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) if (system_supports_bbml2_noabort()) feat_mask |= ARM_SMMU_FEAT_BBML2; + if (system_supports_haft()) + feat_mask |= ARM_SMMU_FEAT_HAFT; + if ((smmu->features & feat_mask) != feat_mask) return false; diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 10c8a29db8028..3d796ff6c2c62 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -1097,9 +1097,9 @@ VISIBLE_IF_KUNIT void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target, __le64 *safe_bits) { - const __le64 eats_s1chk = + const u64 eats_s1chk = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK); - const __le64 eats_trans = + const u64 eats_trans = FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS); /* @@ -3837,6 +3837,20 @@ int arm_smmu_cmdq_init(struct arm_smmu_device *smmu, return 0; } +static void arm_smmu_free_iopf_action(void *data) +{ + struct iopf_queue *queue = data; + + iopf_queue_free(queue); +} + +static void arm_smmu_destroy_vmid_map(void *data) +{ + struct ida *ida = data; + + ida_destroy(ida); +} + static int arm_smmu_init_queues(struct arm_smmu_device *smmu) { int ret; @@ -3864,6 +3878,11 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev)); if (!smmu->evtq.iopf) return -ENOMEM; + ret = devm_add_action_or_reset(smmu->dev, + arm_smmu_free_iopf_action, + smmu->evtq.iopf); + if (ret) + return ret; } /* priq */ @@ -3942,7 +3961,8 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) ida_init(&smmu->vmid_map); - return 0; + return devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_vmid_map, + &smmu->vmid_map); } static int arm_smmu_init_structures(struct arm_smmu_device *smmu) @@ -3972,8 +3992,9 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, u32 reg; writel_relaxed(val, smmu->base + reg_off); - return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, - 1, ARM_SMMU_POLL_TIMEOUT_US); + return readl_relaxed_poll_timeout_atomic(smmu->base + ack_off, reg, + reg == val, 1, + ARM_SMMU_POLL_TIMEOUT_US); } /* GBPA is "special" */ @@ -4155,6 +4176,15 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu) return ret; } +static void arm_smmu_disable_action(void *data) +{ + struct arm_smmu_device *smmu = data; + + if (smmu->impl_ops && smmu->impl_ops->device_disable) + smmu->impl_ops->device_disable(smmu); + arm_smmu_device_disable(smmu); +} + static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) { struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; @@ -4352,10 +4382,14 @@ static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) { - u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD); + u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD | + ARM_SMMU_FEAT_HAFT); u32 hw_features = 0; switch (FIELD_GET(IDR0_HTTU, reg)) { + case IDR0_HTTU_ACCESS_DIRTY_HAFT: + hw_features |= ARM_SMMU_FEAT_HAFT; + fallthrough; case IDR0_HTTU_ACCESS_DIRTY: hw_features |= ARM_SMMU_FEAT_HD; fallthrough; @@ -4665,6 +4699,9 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev, smmu->features |= ARM_SMMU_FEAT_COHERENCY; switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) { + case IDR0_HTTU_ACCESS_DIRTY_HAFT: + smmu->features |= ARM_SMMU_FEAT_HAFT; + fallthrough; case IDR0_HTTU_ACCESS_DIRTY: smmu->features |= ARM_SMMU_FEAT_HD; fallthrough; @@ -4878,7 +4915,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Initialise in-memory data structures */ ret = arm_smmu_init_structures(smmu); if (ret) - goto err_free_iopf; + return ret; /* Record our private device structure */ platform_set_drvdata(pdev, smmu); @@ -4888,30 +4925,30 @@ static int arm_smmu_device_probe(struct platform_device *pdev) /* Reset the device */ ret = arm_smmu_device_reset(smmu); + if (ret) { + arm_smmu_device_disable(smmu); + return ret; + } + + /* Register last so it unwinds first, while the CMDQ is still up. */ + ret = devm_add_action_or_reset(smmu->dev, arm_smmu_disable_action, smmu); if (ret) - goto err_disable; + return ret; /* And we're up. Go go go! */ ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, "smmu3.%pa", &ioaddr); if (ret) - goto err_disable; + return ret; ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_free_sysfs; + iommu_device_sysfs_remove(&smmu->iommu); + return ret; } return 0; - -err_free_sysfs: - iommu_device_sysfs_remove(&smmu->iommu); -err_disable: - arm_smmu_device_disable(smmu); -err_free_iopf: - iopf_queue_free(smmu->evtq.iopf); - return ret; } static void arm_smmu_device_remove(struct platform_device *pdev) @@ -4920,9 +4957,6 @@ static void arm_smmu_device_remove(struct platform_device *pdev) iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); - arm_smmu_device_disable(smmu); - iopf_queue_free(smmu->evtq.iopf); - ida_destroy(&smmu->vmid_map); } static void arm_smmu_device_shutdown(struct platform_device *pdev) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index 287e223c054d1..d65bbda8aac3b 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -40,6 +40,7 @@ struct arm_vsmmu; #define IDR0_HTTU GENMASK(7, 6) #define IDR0_HTTU_ACCESS 1 #define IDR0_HTTU_ACCESS_DIRTY 2 +#define IDR0_HTTU_ACCESS_DIRTY_HAFT 3 #define IDR0_COHACC (1 << 4) #define IDR0_TTF GENMASK(3, 2) #define IDR0_TTF_AARCH64 2 @@ -370,6 +371,7 @@ static inline unsigned int arm_smmu_cdtab_l2_idx(unsigned int ssid) #define CTXDESC_CD_0_ASET (1UL << 47) #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48) +#define CTXDESC_CD_1_HAFT (1UL << 3) #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4) /* @@ -718,6 +720,7 @@ struct arm_smmu_strtab_cfg { struct arm_smmu_impl_ops { int (*device_reset)(struct arm_smmu_device *smmu); + void (*device_disable)(struct arm_smmu_device *smmu); void (*device_remove)(struct arm_smmu_device *smmu); int (*init_structures)(struct arm_smmu_device *smmu); struct arm_smmu_cmdq *(*get_secondary_cmdq)( @@ -768,6 +771,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_HD (1 << 22) #define ARM_SMMU_FEAT_S2FWB (1 << 23) #define ARM_SMMU_FEAT_BBML2 (1 << 24) +#define ARM_SMMU_FEAT_HAFT (1 << 25) u32 features; #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0) diff --git a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c index 17591d8eb64b8..2b55a221c35a6 100644 --- a/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c +++ b/drivers/iommu/arm/arm-smmu-v3/tegra241-cmdqv.c @@ -58,6 +58,8 @@ #define VINTF_ENABLED BIT(0) #define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s)) +#define VINTF_SID_MATCH_VIRT_SID GENMASK(20, 1) +#define VINTF_SID_MATCH_ENABLE BIT(0) #define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s)) #define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \ @@ -320,16 +322,30 @@ static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf) while (map) { unsigned long lidx = __ffs64(map); - struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx]; - u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); + struct tegra241_vcmdq *vcmdq; + u32 gerror; + map &= ~BIT_ULL(lidx); + + /* Pairs with smp_store_release() publishing it */ + vcmdq = smp_load_acquire(&vintf->lvcmdqs[lidx]); + if (!vcmdq) + continue; + + gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)); __arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq); writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN)); - map &= ~BIT_ULL(lidx); } } } +/* + * The CMDQV error interrupt is edge-triggered, so a pending VINTF error fires + * this ISR once and does not re-assert. An unacked guest therefore cannot + * storm the host. The HW latches and forwards each new error event on its + * own, so an already-set ERR_MAP bit does not suppress the interrupt for a + * new error. + */ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) { struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid; @@ -352,16 +368,27 @@ static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid) /* Handle VINTF0 and its LVCMDQs */ if (vintf_map & BIT_ULL(0)) { - tegra241_vintf0_handle_error(cmdqv->vintfs[0]); + struct tegra241_vintf *vintf0; + vintf_map &= ~BIT_ULL(0); + + /* NULL until tegra241_cmdqv_init_structures() publishes it */ + vintf0 = smp_load_acquire(&cmdqv->vintfs[0]); + if (vintf0) + tegra241_vintf0_handle_error(vintf0); } /* Handle other user VINTFs and their LVCMDQs */ while (vintf_map) { unsigned long idx = __ffs64(vintf_map); + struct tegra241_vintf *vintf; - tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]); vintf_map &= ~BIT_ULL(idx); + + /* The slot may be published or torn down (NULL'd) concurrently */ + vintf = smp_load_acquire(&cmdqv->vintfs[idx]); + if (vintf) + tegra241_vintf_user_handle_error(vintf); } return IRQ_HANDLED; @@ -668,7 +695,6 @@ static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx, vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx); vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx); - vintf->lvcmdqs[lidx] = vcmdq; return 0; } @@ -684,7 +710,7 @@ static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) dev_dbg(vintf->cmdqv->dev, "%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64)); /* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */ - if (vcmdq->vintf->hyp_own) + if (!vcmdq->vintf->idx) kfree(vcmdq); } @@ -707,14 +733,15 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) /* Build an arm_smmu_cmdq for each LVCMDQ */ ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq); if (ret) - goto deinit_lvcmdq; + goto free_vcmdq; + + /* Pairs with the smp_load_acquire() in the error ISR */ + smp_store_release(&vintf->lvcmdqs[lidx], vcmdq); dev_dbg(cmdqv->dev, "%sallocated\n", lvcmdq_error_header(vcmdq, header, 64)); return vcmdq; -deinit_lvcmdq: - tegra241_vintf_deinit_lvcmdq(vintf, lidx); free_vcmdq: kfree(vcmdq); return ERR_PTR(ret); @@ -725,8 +752,18 @@ tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx) static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) { kfree(cmdqv->vintfs[idx]->lvcmdqs); + /* + * Clear the slot and drain any in-flight ISR before returning idx to + * the IDA, so a concurrent create that reuses idx cannot have its + * freshly published VINTF erased here. A plain WRITE_ONCE() suffices + * since clearing the slot publishes no data. This also covers the + * init-failure unwind, which reaches deinit_vintf() without the + * destroy callback. + */ + WRITE_ONCE(cmdqv->vintfs[idx], NULL); + if (cmdqv->irq > 0) + synchronize_irq(cmdqv->irq); ida_free(&cmdqv->vintf_ids, idx); - cmdqv->vintfs[idx] = NULL; } static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, @@ -752,7 +789,8 @@ static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx, return -ENOMEM; } - cmdqv->vintfs[idx] = vintf; + /* Pairs with the smp_load_acquire() in tegra241_cmdqv_isr() */ + smp_store_release(&cmdqv->vintfs[idx], vintf); return ret; } @@ -763,8 +801,6 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) struct tegra241_vintf *vintf = cmdqv->vintfs[idx]; u16 lidx; - tegra241_vintf_hw_deinit(vintf); - /* Remove LVCMDQ resources */ for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++) if (vintf->lvcmdqs[lidx]) @@ -772,7 +808,7 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx); tegra241_cmdqv_deinit_vintf(cmdqv, idx); - if (!vintf->hyp_own) { + if (vintf->idx) { mutex_destroy(&vintf->lvcmdq_mutex); ida_destroy(&vintf->sids); /* Guest-owned VINTF is free-ed with viommu by iommufd core */ @@ -781,12 +817,31 @@ static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx) } } +static void tegra241_cmdqv_hw_disable(struct arm_smmu_device *smmu) +{ + struct tegra241_cmdqv *cmdqv = + container_of(smmu, struct tegra241_cmdqv, smmu); + u16 idx; + + for (idx = 0; idx < cmdqv->num_vintfs; idx++) + if (cmdqv->vintfs[idx]) + tegra241_vintf_hw_deinit(cmdqv->vintfs[idx]); +} + static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) { struct tegra241_cmdqv *cmdqv = container_of(smmu, struct tegra241_cmdqv, smmu); u16 idx; + /* + * Free the IRQ before tearing down the VINTFs. free_irq() waits for any + * in-flight tegra241_cmdqv_isr() to finish and blocks new ones, so the + * ISR cannot dereference a VINTF that is freed by the loop below. + */ + if (cmdqv->irq > 0) + free_irq(cmdqv->irq, cmdqv); + /* Remove VINTF resources */ for (idx = 0; idx < cmdqv->num_vintfs; idx++) { if (cmdqv->vintfs[idx]) { @@ -799,8 +854,6 @@ static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu) /* Remove cmdqv resources */ ida_destroy(&cmdqv->vintf_ids); - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); iounmap(cmdqv->base); kfree(cmdqv->vintfs); put_device(cmdqv->dev); /* smmu->impl_dev */ @@ -846,6 +899,7 @@ static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = { /* For in-kernel use */ .get_secondary_cmdq = tegra241_cmdqv_get_cmdq, .device_reset = tegra241_cmdqv_hw_reset, + .device_disable = tegra241_cmdqv_hw_disable, .device_remove = tegra241_cmdqv_remove, /* For user-space use */ .hw_info = tegra241_cmdqv_hw_info, @@ -934,6 +988,12 @@ static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu) ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf); if (ret) { dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret); + /* + * tegra241_cmdqv_init_vintf() failed to publish the vintf0 to + * cmdqv->vintfs[], so the probe unwind path that goes through + * cmdqv->vintfs[] would miss it. Free it here. + */ + kfree(vintf); return ret; } @@ -994,17 +1054,6 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, cmdqv->dev = smmu->impl_dev; cmdqv->base_phys = res->start; - if (cmdqv->irq > 0) { - ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, - IRQF_ONESHOT, "tegra241-cmdqv", - cmdqv); - if (ret) { - dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", - cmdqv->irq, ret); - goto iounmap; - } - } - regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM)); cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval); cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval); @@ -1015,10 +1064,25 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, cmdqv->vintfs = kcalloc(cmdqv->num_vintfs, sizeof(*cmdqv->vintfs), GFP_KERNEL); if (!cmdqv->vintfs) - goto free_irq; + goto iounmap; ida_init(&cmdqv->vintf_ids); + /* + * Request the IRQ only after cmdqv->vintfs is allocated and zeroed, so + * the ISR would not walk an uninitialized array. + */ + if (cmdqv->irq > 0) { + ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr, + IRQF_ONESHOT, "tegra241-cmdqv", + cmdqv); + if (ret) { + dev_err(cmdqv->dev, "failed to request irq (%d): %d\n", + cmdqv->irq, ret); + goto free_vintfs; + } + } + #ifdef CONFIG_IOMMU_DEBUGFS if (!cmdqv_debugfs_dir) { cmdqv_debugfs_dir = @@ -1033,9 +1097,9 @@ __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res, return new_smmu; -free_irq: - if (cmdqv->irq > 0) - free_irq(cmdqv->irq, cmdqv); +free_vintfs: + ida_destroy(&cmdqv->vintf_ids); + kfree(cmdqv->vintfs); iounmap: iounmap(base); return NULL; @@ -1190,13 +1254,15 @@ static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue, if (ret) goto unmap_lvcmdq; + /* No lockless reader of a user VINTF's lvcmdqs[]; mutex-serialized */ + vintf->lvcmdqs[lidx] = vcmdq; + hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user; mutex_unlock(&vintf->lvcmdq_mutex); return 0; unmap_lvcmdq: tegra241_vcmdq_unmap_lvcmdq(vcmdq); - tegra241_vintf_deinit_lvcmdq(vintf, lidx); undepend_vcmdq: if (vcmdq->prev) iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core); @@ -1212,6 +1278,7 @@ static void tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu *viommu) if (vintf->mmap_offset) iommufd_viommu_destroy_mmap(&vintf->vsmmu.core, vintf->mmap_offset); + tegra241_vintf_hw_deinit(vintf); tegra241_cmdqv_remove_vintf(vintf->cmdqv, vintf->idx); } @@ -1238,10 +1305,11 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) u64 virt_sid = vdev->virt_id; int sidx; - if (virt_sid > UINT_MAX) + if (virt_sid > FIELD_MAX(VINTF_SID_MATCH_VIRT_SID)) return -EINVAL; - WARN_ON_ONCE(master->num_streams != 1); + if (master->num_streams != 1) + return -EOPNOTSUPP; /* Find an empty pair of SID_REPLACE and SID_MATCH */ sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1, @@ -1250,7 +1318,9 @@ static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev) return sidx; writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx))); - writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx))); + writel(FIELD_PREP(VINTF_SID_MATCH_VIRT_SID, virt_sid) | + VINTF_SID_MATCH_ENABLE, + REG_VINTF(vintf, SID_MATCH(sidx))); dev_dbg(vintf->cmdqv->dev, "VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n", vintf->idx, sidx, stream->id, (u32)virt_sid); diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index 3b9fb1c71e5f9..f7cb7de80cfb9 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -231,6 +231,7 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops); struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; + int ret; /* The domain must have split pagetables already enabled */ if (cb->tcr[0] & ARM_SMMU_TCR_EPD1) @@ -260,8 +261,16 @@ static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie, cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid); } + ret = pm_runtime_resume_and_get(smmu_domain->smmu->dev); + if (ret < 0) { + dev_err(smmu_domain->smmu->dev, "failed to get runtime PM: %d\n", ret); + return -ENODEV; + } + arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx); + pm_runtime_put_autosuspend(smmu_domain->smmu->dev); + return 0; } diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index 9c1166a3af6c9..82c3ffe0c7836 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -200,7 +200,7 @@ static irqreturn_t qcom_iommu_fault(int irq, void *dev) fsynr = iommu_readl(ctx, ARM_SMMU_CB_FSYNR0); iova = iommu_readq(ctx, ARM_SMMU_CB_FAR); - if (!report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { + if (report_iommu_fault(ctx->domain, ctx->dev, iova, 0)) { dev_err_ratelimited(ctx->dev, "Unhandled context fault: fsr=0x%x, " "iova=0x%016llx, fsynr=0x%x, cb=%d\n", @@ -858,7 +858,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) ret = iommu_device_register(&qcom_iommu->iommu, &qcom_iommu_ops, dev); if (ret) { dev_err(dev, "Failed to register iommu\n"); - goto err_pm_disable; + goto err_sysfs_remove; } if (qcom_iommu->local_base) { @@ -869,6 +869,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev) return 0; +err_sysfs_remove: + iommu_device_sysfs_remove(&qcom_iommu->iommu); err_pm_disable: pm_runtime_disable(dev); return ret; diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index a613010cab2bc..402573b9c3b66 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1707,13 +1707,16 @@ void *iommu_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, } if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) && - !gfpflags_allow_blocking(gfp) && !coherent) + !gfpflags_allow_blocking(gfp) && !coherent) { page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr, - gfp, NULL); - else + gfp, NULL); + if (!page) + return NULL; + } else { cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs); - if (!cpu_addr) - return NULL; + if (!cpu_addr) + return NULL; + } *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot, dev->coherent_dma_mask); @@ -2220,6 +2223,19 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev, dma_addr_t iova; int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; size_t size = cookie_msi_granule(domain); + static DEFINE_MUTEX(msi_prepare_lock); + + /* + * Normally a device's default domain is only ever attached to that + * device's own group, and the group mutex held by + * iommu_group_mutex_assert()'s callers is enough on its own. A VFIO + * type1 container is the one case that breaks that assumption: it + * can merge devices from different groups onto one domain, so two + * devices' group mutexes don't serialize each other here. A static + * lock is sufficient due to the expectation that this is a corner + * case that will never be contended in practice. + */ + guard(mutex)(&msi_prepare_lock); msi_addr &= ~(phys_addr_t)(size - 1); list_for_each_entry(msi_page, msi_page_list, list) diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 6938800e98845..f4bdc5ffa9d9b 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -915,6 +915,18 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg) return 0; } +static bool dmar_required(void) +{ + /* tboot supersedes any user/platform opt */ + if (!intel_iommu_tboot_noforce && tboot_enabled()) + return true; + + if (!no_iommu && (!dmar_disabled || dmar_platform_optin())) + return true; + + return false; +} + void __init detect_intel_iommu(void) { int ret; @@ -928,8 +940,7 @@ void __init detect_intel_iommu(void) if (!ret) ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl, &validate_drhd_cb); - if (!ret && !no_iommu && !iommu_detected && - (!dmar_disabled || dmar_platform_optin())) { + if (!ret && !iommu_detected && dmar_required()) { iommu_detected = 1; /* Make sure ACS will be enabled */ pci_request_acs(); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index cee1851b69245..77e51712040b0 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -64,7 +64,7 @@ static int rwbf_quirk; * (used when kernel is launched w/ TXT) */ static int force_on = 0; -static int intel_iommu_tboot_noforce; +int intel_iommu_tboot_noforce; static int no_platform_optin; #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) @@ -1421,7 +1421,8 @@ static void copied_context_tear_down(struct intel_iommu *iommu, assert_spin_locked(&iommu->lock); did_old = context_domain_id(context); - context_clear_entry(context); + context_clear_present(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); if (did_old < cap_ndoms(iommu->cap)) { iommu->flush.flush_context(iommu, did_old, @@ -1432,6 +1433,9 @@ static void copied_context_tear_down(struct intel_iommu *iommu, DMA_TLB_DSI_FLUSH); } + context_clear_entry(context); + __iommu_flush_cache(iommu, context, sizeof(*context)); + clear_context_copied(iommu, bus, devfn); } @@ -1728,7 +1732,7 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -1924,7 +1928,7 @@ static int copy_context_table(struct intel_iommu *iommu, struct context_entry **tbl, int bus, bool ext) { - int tbl_idx, pos = 0, idx, devfn, ret = 0, did; + int tbl_idx, tbl_slot = 0, idx, devfn, ret = 0, did; struct context_entry *new_ce = NULL, ce; struct context_entry *old_ce = NULL; struct root_entry re; @@ -1940,10 +1944,9 @@ static int copy_context_table(struct intel_iommu *iommu, if (idx == 0) { /* First save what we may have and clean up */ if (new_ce) { - tbl[tbl_idx] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); - pos = 1; } if (old_ce) @@ -1965,6 +1968,9 @@ static int copy_context_table(struct intel_iommu *iommu, } } + /* Track if saving UCTP or LCTP entries in scalable mode */ + tbl_slot = ext && devfn >= 0x80 ? 1 : 0; + ret = -ENOMEM; old_ce = memremap(old_ce_phys, PAGE_SIZE, MEMREMAP_WB); @@ -1993,7 +1999,7 @@ static int copy_context_table(struct intel_iommu *iommu, new_ce[idx] = ce; } - tbl[tbl_idx + pos] = new_ce; + tbl[tbl_idx + tbl_slot] = new_ce; __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); @@ -2303,7 +2309,7 @@ static void iommu_flush_all(void) } } -static int iommu_suspend(void) +static int iommu_suspend(void *data) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu = NULL; @@ -2330,7 +2336,7 @@ static int iommu_suspend(void) return 0; } -static void iommu_resume(void) +static void iommu_resume(void *data) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu = NULL; @@ -2361,14 +2367,18 @@ static void iommu_resume(void) } } -static struct syscore_ops iommu_syscore_ops = { +static const struct syscore_ops iommu_syscore_ops = { .resume = iommu_resume, .suspend = iommu_suspend, }; +static struct syscore iommu_syscore = { + .ops = &iommu_syscore_ops, +}; + static void __init init_iommu_pm_ops(void) { - register_syscore_ops(&iommu_syscore_ops); + register_syscore(&iommu_syscore); } #else @@ -2956,10 +2966,11 @@ static bool has_external_pci(void) static int __init platform_optin_force_iommu(void) { - if (!dmar_platform_optin() || no_platform_optin || !has_external_pci()) + if (no_iommu || !dmar_platform_optin() || no_platform_optin || + !has_external_pci()) return 0; - if (no_iommu || dmar_disabled) + if (dmar_disabled) pr_info("Intel-IOMMU force enabled due to platform opt in\n"); /* @@ -2970,7 +2981,6 @@ static int __init platform_optin_force_iommu(void) iommu_set_default_passthrough(false); dmar_disabled = 0; - no_iommu = 0; return 1; } @@ -3620,6 +3630,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather) { + struct iommu_pages_list freelist = IOMMU_PAGES_LIST_INIT(freelist); struct dmar_domain *dmar_domain = to_dmar_domain(domain); unsigned long start_pfn, last_pfn; int level = 0; @@ -3636,7 +3647,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, start_pfn = iova >> VTD_PAGE_SHIFT; last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT; - domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist); + domain_unmap(dmar_domain, start_pfn, last_pfn, &freelist); if (dmar_domain->max_addr == iova + size) dmar_domain->max_addr = iova; @@ -3648,6 +3659,14 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, if (!iommu_iotlb_gather_queued(gather)) iommu_iotlb_gather_add_page(domain, gather, iova, size); + /* + * iommu_iotlb_gather_add_page() may have synced, which frees + * gather->freelist. Hand this range's page tables over only after + * that call. A queued gather frees them from the flush queue + * instead. + */ + iommu_pages_list_splice(&freelist, &gather->freelist); + return size; } @@ -3849,6 +3868,7 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) return &iommu->iommu; free_table: + intel_pasid_teardown_sm_context(dev); intel_pasid_free_table(dev); clear_rbtree: device_rbtree_remove(info); diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 6b11081e690fe..e696e019e9697 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -1320,7 +1320,7 @@ void cache_tag_flush_range_np(struct dmar_domain *domain, unsigned long start, unsigned long end); void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did); + struct context_entry *context, u16 did, u16 sid); int intel_iommu_enable_prq(struct intel_iommu *iommu); int intel_iommu_finish_prq(struct intel_iommu *iommu); @@ -1434,6 +1434,7 @@ static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu) extern int dmar_disabled; extern int intel_iommu_enabled; +extern int intel_iommu_tboot_noforce; #else static inline int iommu_calculate_agaw(struct intel_iommu *iommu) { @@ -1446,6 +1447,7 @@ static inline int iommu_calculate_max_sagaw(struct intel_iommu *iommu) #define dmar_disabled (1) #define intel_iommu_enabled (0) #define intel_iommu_sm (0) +#define intel_iommu_tboot_noforce (0) #endif static inline const char *decode_prq_descriptor(char *str, size_t size, diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 85c3116e351c6..48e21743f2077 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -749,7 +749,7 @@ static void device_pasid_table_teardown(struct device *dev, u8 bus, u8 devfn) context_clear_present(context); __iommu_flush_cache(iommu, context, sizeof(*context)); spin_unlock(&iommu->lock); - intel_context_flush_no_pasid(info, context, did); + intel_context_flush_no_pasid(info, context, did, PCI_DEVID(bus, devfn)); context_clear_entry(context); __iommu_flush_cache(iommu, context, sizeof(*context)); } @@ -953,9 +953,12 @@ static void __context_flush_dev_iotlb(struct device_domain_info *info) * This helper can only be used when IOMMU is working in the legacy mode or * IOMMU is in scalable mode but all PASID table entries of the device are * non-present. + * + * @sid identifies the context entry that was modified, which may be a DMA + * alias of @info->dev rather than its own requester ID. */ void intel_context_flush_no_pasid(struct device_domain_info *info, - struct context_entry *context, u16 did) + struct context_entry *context, u16 did, u16 sid) { struct intel_iommu *iommu = info->iommu; @@ -965,7 +968,7 @@ void intel_context_flush_no_pasid(struct device_domain_info *info, * when operating in scalable mode. Therefore the @did value doesn't * matter in scalable mode. */ - iommu->flush.flush_context(iommu, did, PCI_DEVID(info->bus, info->devfn), + iommu->flush.flush_context(iommu, did, sid, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); /* diff --git a/drivers/iommu/intel/perf.c b/drivers/iommu/intel/perf.c index dceeadc3ee7cd..6afb55073e9f1 100644 --- a/drivers/iommu/intel/perf.c +++ b/drivers/iommu/intel/perf.c @@ -63,7 +63,7 @@ void dmar_latency_disable(struct intel_iommu *iommu, enum latency_type type) return; spin_lock_irqsave(&latency_lock, flags); - memset(&lstat[type], 0, sizeof(*lstat) * DMAR_LATENCY_NUM); + memset(&lstat[type], 0, sizeof(*lstat)); spin_unlock_irqrestore(&latency_lock, flags); } diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index e147f71f91b72..6c275c7679459 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -27,7 +27,7 @@ void intel_svm_check(struct intel_iommu *iommu) { - if (!pasid_supported(iommu)) + if (!pasid_supported(iommu) || !ecap_smpwc(iommu->ecap)) return; if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && diff --git a/drivers/iommu/io-pgfault.c b/drivers/iommu/io-pgfault.c index 8b5926c1452ed..afe977235d72f 100644 --- a/drivers/iommu/io-pgfault.c +++ b/drivers/iommu/io-pgfault.c @@ -332,16 +332,30 @@ void iopf_group_response(struct iopf_group *group, .code = status, }; - /* Only send response if there is a fault report pending */ mutex_lock(&fault_param->lock); - if (!list_empty(&group->pending_node)) { - ops->page_response(dev, &group->last_fault, &resp); - list_del_init(&group->pending_node); - } + ops->page_response(dev, &group->last_fault, &resp); + list_del_init(&group->pending_node); mutex_unlock(&fault_param->lock); } EXPORT_SYMBOL_GPL(iopf_group_response); +/** + * iopf_group_dequeue - Dequeue a page fault group from the pending list + * @group: the group to dequeue + * + * The fault handler is responsible for responding to the group after + * this function returns. + */ +void iopf_group_dequeue(struct iopf_group *group) +{ + struct iommu_fault_param *fault_param = group->fault_param; + + mutex_lock(&fault_param->lock); + list_del_init(&group->pending_node); + mutex_unlock(&fault_param->lock); +} +EXPORT_SYMBOL_GPL(iopf_group_dequeue); + /** * iopf_queue_discard_partial - Remove all pending partial fault * @queue: the queue whose partial faults need to be discarded diff --git a/drivers/iommu/iommu-sva.c b/drivers/iommu/iommu-sva.c index fd735aaae9e3f..838563fc44313 100644 --- a/drivers/iommu/iommu-sva.c +++ b/drivers/iommu/iommu-sva.c @@ -145,8 +145,8 @@ struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm list_add(&domain->next, &iommu_mm->sva_domains); out: refcount_set(&handle->users, 1); - mutex_unlock(&iommu_sva_lock); handle->dev = dev; + mutex_unlock(&iommu_sva_lock); return handle; out_free_domain: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 59244c744eabd..1547f3587a4ec 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -360,7 +360,7 @@ int iommu_mock_device_add(struct device *dev, struct iommu_device *iommu) rc = device_add(dev); if (rc) - iommu_fwspec_free(dev); + dev_iommu_free(dev); return rc; } EXPORT_SYMBOL_GPL(iommu_mock_device_add); diff --git a/drivers/iommu/iommufd/device.c b/drivers/iommu/iommufd/device.c index 4c842368289f0..3216deab98e39 100644 --- a/drivers/iommu/iommufd/device.c +++ b/drivers/iommu/iommufd/device.c @@ -148,29 +148,22 @@ static void iommufd_device_remove_vdev(struct iommufd_device *idev) if (!idev->vdev) goto out_unlock; - vdev = iommufd_get_vdevice(idev->ictx, idev->vdev->obj.id); + vdev = idev->vdev; + /* * An ongoing vdev destroy ioctl has removed the vdev from the object * xarray, but has not finished iommufd_vdevice_destroy() yet as it * needs the same mutex. We exit the locking then wait on wait_cnt * reference for the vdev destruction. */ - if (IS_ERR(vdev)) - goto out_unlock; - - /* Should never happen */ - if (WARN_ON(vdev != idev->vdev)) { - iommufd_put_object(idev->ictx, &vdev->obj); + if (iommufd_try_inc_users(idev->ictx, &vdev->obj)) goto out_unlock; - } /* * vdev is still alive. Hold a users refcount to prevent racing with * userspace destruction, then use iommufd_object_tombstone_user() to * destroy it and leave a tombstone. */ - refcount_inc(&vdev->obj.users); - iommufd_put_object(idev->ictx, &vdev->obj); mutex_unlock(&idev->igroup->lock); iommufd_object_tombstone_user(idev->ictx, &vdev->obj); return; @@ -589,7 +582,7 @@ static int iommufd_hwpt_replace_device(struct iommufd_device *idev, if (rc) goto out_free_handle; - iommufd_auto_response_faults(hwpt, old_handle); + iommufd_auto_response_faults(old, old_handle); kfree(old_handle); return 0; @@ -1307,8 +1300,8 @@ void iommufd_access_notify_unmap(struct io_pagetable *iopt, unsigned long iova, xa_lock(&ioas->iopt.access_list); xa_for_each(&ioas->iopt.access_list, index, access) { - if (!iommufd_lock_obj(&access->obj) || - iommufd_access_is_internal(access)) + if (iommufd_access_is_internal(access) || + !iommufd_lock_obj(&access->obj)) continue; xa_unlock(&ioas->iopt.access_list); diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c index a934f02528b0b..8971360c955ae 100644 --- a/drivers/iommu/iommufd/eventq.c +++ b/drivers/iommu/iommufd/eventq.c @@ -484,6 +484,8 @@ int iommufd_fault_iopf_handler(struct iopf_group *group) hwpt = group->attach_handle->domain->iommufd_hwpt; fault = hwpt->fault; + iopf_group_dequeue(group); + spin_lock(&fault->common.lock); list_add_tail(&group->node, &fault->common.deliver); spin_unlock(&fault->common.lock); diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c index 459a7c5169154..164612195905a 100644 --- a/drivers/iommu/iommufd/ioas.c +++ b/drivers/iommu/iommufd/ioas.c @@ -433,6 +433,8 @@ static int iommufd_take_all_iova_rwsem(struct iommufd_ctx *ictx, rc = xa_err(xa_store(ioas_list, index, ioas, GFP_KERNEL)); if (rc) { + up_write(&ioas->iopt.iova_rwsem); + refcount_dec(&ioas->obj.users); iommufd_release_all_iova_rwsem(ictx, ioas_list); return rc; } @@ -541,6 +543,10 @@ int iommufd_ioas_change_process(struct iommufd_ucmd *ucmd) return rc; for_each_ioas_area(&ioas_list, index, ioas, area) { + if (!area->pages) { + rc = -EBUSY; + goto out; + } if (area->pages->type != IOPT_ADDRESS_FILE) { rc = -EINVAL; goto out; diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 53d37e77c4b98..c208563eb367f 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -181,6 +181,7 @@ static inline bool iommufd_lock_obj(struct iommufd_object *obj) return true; } +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj); struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, enum iommufd_object_type type); static inline void iommufd_put_object(struct iommufd_ctx *ictx, @@ -695,14 +696,6 @@ void iommufd_vdevice_abort(struct iommufd_object *obj); int iommufd_hw_queue_alloc_ioctl(struct iommufd_ucmd *ucmd); void iommufd_hw_queue_destroy(struct iommufd_object *obj); -static inline struct iommufd_vdevice * -iommufd_get_vdevice(struct iommufd_ctx *ictx, u32 id) -{ - return container_of(iommufd_get_object(ictx, id, - IOMMUFD_OBJ_VDEVICE), - struct iommufd_vdevice, obj); -} - #ifdef CONFIG_IOMMUFD_TEST int iommufd_test(struct iommufd_ucmd *ucmd); void iommufd_selftest_destroy(struct iommufd_object *obj); diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c index ce775fbbae94e..02d06a02d5e67 100644 --- a/drivers/iommu/iommufd/main.c +++ b/drivers/iommu/iommufd/main.c @@ -180,6 +180,26 @@ struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, return obj; } +/* + * Increment the users count of an object outside the context of an ioctl that + * has already locked it. The users refcount cannot be increased on an already + * created object unless the object is installed in the xarray, otherwise things + * are racing with a parallel destruction. + */ +int iommufd_try_inc_users(struct iommufd_ctx *ictx, struct iommufd_object *obj) +{ + struct iommufd_object *cur; + + xa_lock(&ictx->objects); + cur = xa_load(&ictx->objects, obj->id); + if (cur == obj) + refcount_inc(&obj->users); + xa_unlock(&ictx->objects); + if (cur != obj) + return -EBUSY; + return 0; +} + static int iommufd_object_dec_wait(struct iommufd_ctx *ictx, struct iommufd_object *to_destroy) { diff --git a/drivers/iommu/iommufd/selftest.c b/drivers/iommu/iommufd/selftest.c index 8fd27f65d6237..cd3d1380cf6c5 100644 --- a/drivers/iommu/iommufd/selftest.c +++ b/drivers/iommu/iommufd/selftest.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -183,6 +184,7 @@ struct mock_dev { struct device dev; struct mock_viommu *viommu; struct rw_semaphore viommu_rwsem; + struct rw_semaphore iopf_rwsem; unsigned long flags; unsigned long vdev_id; int id; @@ -1101,6 +1103,7 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags) return ERR_PTR(-ENOMEM); init_rwsem(&mdev->viommu_rwsem); + init_rwsem(&mdev->iopf_rwsem); device_initialize(&mdev->dev); mdev->flags = dev_flags; mdev->dev.release = mock_dev_release; @@ -1126,7 +1129,9 @@ static struct mock_dev *mock_dev_create(unsigned long dev_flags) goto err_put; } + down_write(&mdev->iopf_rwsem); rc = iommu_mock_device_add(&mdev->dev, &mock_iommu.iommu_dev); + up_write(&mdev->iopf_rwsem); if (rc) goto err_put; return mdev; @@ -1181,7 +1186,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, } sobj->idev.idev = idev; + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_attach(idev, IOMMU_NO_PASID, &pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_unbind; @@ -1196,7 +1203,9 @@ static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, return 0; out_detach: + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(idev, IOMMU_NO_PASID); + up_write(&sobj->idev.mock_dev->iopf_rwsem); out_unbind: iommufd_device_unbind(idev); out_mdev: @@ -1240,7 +1249,9 @@ static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_replace(sobj->idev.idev, IOMMU_NO_PASID, &pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; @@ -1796,6 +1807,9 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, if (!page_size || !length || iova % page_size || length % page_size || !uptr) return -EINVAL; + max = length / page_size; + if (max > SZ_16M * BITS_PER_BYTE) + return -EOVERFLOW; hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); if (IS_ERR(hwpt)) @@ -1806,7 +1820,6 @@ static int iommufd_test_dirty(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, goto out_put; } - max = length / page_size; tmp = kvzalloc(DIV_ROUND_UP(max, BITS_PER_LONG) * sizeof(unsigned long), GFP_KERNEL_ACCOUNT); if (!tmp) { @@ -1852,10 +1865,16 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, { struct iopf_fault event = {}; struct iommufd_device *idev; + struct mock_dev *mdev; idev = iommufd_get_device(ucmd, cmd->trigger_iopf.dev_id); if (IS_ERR(idev)) return PTR_ERR(idev); + if (!iommufd_selftest_is_mock_dev(idev->dev)) { + iommufd_put_object(ucmd->ictx, &idev->obj); + return -EINVAL; + } + mdev = to_mock_dev(idev->dev); event.fault.prm.flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; if (cmd->trigger_iopf.pasid != IOMMU_NO_PASID) @@ -1866,7 +1885,9 @@ static int iommufd_test_trigger_iopf(struct iommufd_ucmd *ucmd, event.fault.prm.grpid = cmd->trigger_iopf.grpid; event.fault.prm.perm = cmd->trigger_iopf.perm; + down_read(&mdev->iopf_rwsem); iommu_report_device_fault(idev->dev, &event); + up_read(&mdev->iopf_rwsem); iommufd_put_object(ucmd->ictx, &idev->obj); return 0; @@ -1974,14 +1995,19 @@ static int iommufd_test_pasid_attach(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_attach(sobj->idev.idev, cmd->pasid_attach.pasid, &cmd->pasid_attach.pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); - if (rc) + if (rc) { + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, cmd->pasid_attach.pasid); + up_write(&sobj->idev.mock_dev->iopf_rwsem); + } out_sobj: iommufd_put_object(ucmd->ictx, &sobj->obj); @@ -1998,8 +2024,10 @@ static int iommufd_test_pasid_replace(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); rc = iommufd_device_replace(sobj->idev.idev, cmd->pasid_attach.pasid, &cmd->pasid_attach.pt_id); + up_write(&sobj->idev.mock_dev->iopf_rwsem); if (rc) goto out_sobj; @@ -2019,7 +2047,9 @@ static int iommufd_test_pasid_detach(struct iommufd_ucmd *ucmd, if (IS_ERR(sobj)) return PTR_ERR(sobj); + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, cmd->pasid_detach.pasid); + up_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_put_object(ucmd->ictx, &sobj->obj); return 0; } @@ -2030,7 +2060,9 @@ void iommufd_selftest_destroy(struct iommufd_object *obj) switch (sobj->type) { case TYPE_IDEV: + down_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_detach(sobj->idev.idev, IOMMU_NO_PASID); + up_write(&sobj->idev.mock_dev->iopf_rwsem); iommufd_device_unbind(sobj->idev.idev); mock_dev_destroy(sobj->idev.mock_dev); break; diff --git a/drivers/iommu/iommufd/viommu.c b/drivers/iommu/iommufd/viommu.c index 462b457ffd0c4..f80582f25e1ea 100644 --- a/drivers/iommu/iommufd/viommu.c +++ b/drivers/iommu/iommufd/viommu.c @@ -143,7 +143,7 @@ void iommufd_vdevice_destroy(struct iommufd_object *obj) int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd) { struct iommu_vdevice_alloc *cmd = ucmd->cmd; - struct iommufd_vdevice *vdev, *curr; + struct iommufd_vdevice *vdev; size_t vdev_size = sizeof(*vdev); struct iommufd_viommu *viommu; struct iommufd_device *idev; @@ -189,7 +189,7 @@ int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd) if (WARN_ON_ONCE(viommu->ops->vdevice_size < vdev_size || !viommu->ops->vdevice_init)) { rc = -EOPNOTSUPP; - goto out_put_idev; + goto out_unlock_igroup; } vdev_size = viommu->ops->vdevice_size; } @@ -218,18 +218,28 @@ int iommufd_vdevice_alloc_ioctl(struct iommufd_ucmd *ucmd) */ idev->vdev = vdev; - curr = xa_cmpxchg(&viommu->vdevs, virt_id, NULL, vdev, GFP_KERNEL); - if (curr) { - rc = xa_err(curr) ?: -EEXIST; + /* + * Reserve the slot with a zero entry (reads back as NULL) until the + * vdevice_init() op accepts the vDEVICE. Only the xa_* helpers hide a + * reserved entry, so never use a raw xas_* iterator on this xarray. + */ + rc = xa_insert(&viommu->vdevs, virt_id, NULL, GFP_KERNEL); + if (rc) { + if (rc == -EBUSY) + rc = -EEXIST; goto out_abort; } if (viommu->ops && viommu->ops->vdevice_init) { rc = viommu->ops->vdevice_init(vdev); - if (rc) + if (rc) { + xa_release(&viommu->vdevs, virt_id); goto out_abort; + } } + xa_store(&viommu->vdevs, virt_id, vdev, GFP_KERNEL); + cmd->out_vdevice_id = vdev->obj.id; rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); if (rc) diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 43a61ba021a51..1dfa6f69c28f2 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -719,7 +719,7 @@ static int msm_iommu_probe(struct platform_device *pdev) iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) - return -ENODEV; + return -ENOMEM; iommu->dev = &pdev->dev; INIT_LIST_HEAD(&iommu->ctx_list); @@ -783,19 +783,25 @@ static int msm_iommu_probe(struct platform_device *pdev) "msm-smmu.%pa", &ioaddr); if (ret) { pr_err("Could not add msm-smmu at %pa to sysfs\n", &ioaddr); - return ret; + goto err_remove_list; } ret = iommu_device_register(&iommu->iommu, &msm_iommu_ops, &pdev->dev); if (ret) { pr_err("Could not register msm-smmu at %pa\n", &ioaddr); - return ret; + goto err_remove_sysfs; } pr_info("device mapped at %p, irq %d with %d ctx banks\n", iommu->base, iommu->irq, iommu->ncb); return ret; + +err_remove_sysfs: + iommu_device_sysfs_remove(&iommu->iommu); +err_remove_list: + list_del(&iommu->dev_node); + return ret; } static const struct of_device_id msm_iommu_dt_match[] = { diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 44b965a2db923..ad2dbd29fef8d 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -88,7 +88,7 @@ struct dma_iommu_mapping { /* MTK generation one iommu HW only support 4K size mapping */ #define MT2701_IOMMU_PAGE_SHIFT 12 #define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) -#define MT2701_LARB_NR_MAX 3 +#define MT2701_LARB_NR_MAX 4 /* * MTK m4u support 4GB iova address space, and only support 4K page diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c index de286563bd44e..1f7b8e46812c9 100644 --- a/drivers/iommu/riscv/iommu.c +++ b/drivers/iommu/riscv/iommu.c @@ -813,7 +813,6 @@ struct riscv_iommu_domain { struct list_head bonds; spinlock_t lock; /* protect bonds list updates. */ int pscid; - bool amo_enabled; int numa_node; unsigned int pgd_mode; unsigned long *pgd_root; @@ -1275,8 +1274,6 @@ static int riscv_iommu_map_pages(struct iommu_domain *iommu_domain, if (!(prot & IOMMU_WRITE)) pte_prot = _PAGE_BASE | _PAGE_READ; - else if (domain->amo_enabled) - pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE; else pte_prot = _PAGE_BASE | _PAGE_READ | _PAGE_WRITE | _PAGE_DIRTY; @@ -1460,7 +1457,6 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev) INIT_LIST_HEAD_RCU(&domain->bonds); spin_lock_init(&domain->lock); domain->numa_node = dev_to_node(iommu->dev); - domain->amo_enabled = !!(iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD); domain->pgd_mode = pgd_mode; domain->pgd_root = iommu_alloc_pages_node_sz(domain->numa_node, GFP_KERNEL_ACCOUNT, SZ_4K); @@ -1583,8 +1579,6 @@ static struct iommu_device *riscv_iommu_probe_device(struct device *dev) * the device directory. Do not mark the context valid yet. */ tc = 0; - if (iommu->caps & RISCV_IOMMU_CAPABILITIES_AMO_HWAD) - tc |= RISCV_IOMMU_DC_TC_SADE; for (i = 0; i < fwspec->num_ids; i++) { dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]); if (!dc) { @@ -1687,6 +1681,7 @@ int riscv_iommu_init(struct riscv_iommu_device *iommu) int rc; RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ); + raw_spin_lock_init(&iommu->cmdq.lock); RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ); rc = riscv_iommu_init_check(iommu); diff --git a/drivers/iommu/riscv/iommu.h b/drivers/iommu/riscv/iommu.h index 46df79dd54957..5676001548cc7 100644 --- a/drivers/iommu/riscv/iommu.h +++ b/drivers/iommu/riscv/iommu.h @@ -12,6 +12,7 @@ #define _RISCV_IOMMU_H_ #include +#include #include #include @@ -23,6 +24,7 @@ struct riscv_iommu_queue { atomic_t prod; /* unbounded producer allocation index */ atomic_t head; /* unbounded shadow ring buffer consumer index */ atomic_t tail; /* unbounded shadow ring buffer producer index */ + raw_spinlock_t lock; /* serialize queue publishing */ unsigned int mask; /* index mask, queue length - 1 */ unsigned int irq; /* allocated interrupt number */ struct riscv_iommu_device *iommu; /* iommu device handling the queue when active */ diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 0861dd469bd86..2d0dabb0d101a 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -76,6 +76,8 @@ #define SPAGE_ORDER 12 #define SPAGE_SIZE (1 << SPAGE_ORDER) +#define DISABLE_FETCH_DTE_TIME_LIMIT BIT(31) + /* * Support mapping any size that fits in one page table: * 4 KiB to 4 MiB @@ -930,6 +932,7 @@ static int rk_iommu_enable(struct rk_iommu *iommu) struct iommu_domain *domain = iommu->domain; struct rk_iommu_domain *rk_domain = to_rk_domain(domain); int ret, i; + u32 auto_gate; ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks); if (ret) @@ -948,6 +951,11 @@ static int rk_iommu_enable(struct rk_iommu *iommu) rk_ops->mk_dtentries(rk_domain->dt_dma)); rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); + + /* Workaround for iommu blocked, BIT(31) default to 1 */ + auto_gate = rk_iommu_read(iommu->bases[i], RK_MMU_AUTO_GATING); + auto_gate |= DISABLE_FETCH_DTE_TIME_LIMIT; + rk_iommu_write(iommu->bases[i], RK_MMU_AUTO_GATING, auto_gate); } ret = rk_iommu_enable_paging(iommu); diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index aa576736d60ba..e36d08a0fd1df 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -972,6 +972,8 @@ static unsigned long *get_rto_from_iova(struct s390_domain *domain, case ZPCI_TABLE_TYPE_RFX: case ZPCI_TABLE_TYPE_RSX: rso = get_rso_from_iova(domain, iova); + if (!rso) + return NULL; rsx = calc_rsx(iova); rse = READ_ONCE(rso[rsx]); if (!reg_entry_isvalid(rse)) diff --git a/drivers/ipack/devices/ipoctal.c b/drivers/ipack/devices/ipoctal.c index ba2e9e52d72bf..474b58858dca6 100644 --- a/drivers/ipack/devices/ipoctal.c +++ b/drivers/ipack/devices/ipoctal.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,8 @@ static const struct tty_operations ipoctal_fops; +static void ipoctal_release(struct kref *kref); + struct ipoctal_channel { struct ipoctal_stats stats; unsigned int nb_bytes; @@ -49,6 +52,9 @@ struct ipoctal { struct tty_driver *tty_drv; u8 __iomem *mem8_space; u8 __iomem *int_space; + struct kref kref; + struct module *carrier_owner; + bool removed; }; static inline struct ipoctal *chan_to_ipoctal(struct ipoctal_channel *chan, @@ -70,8 +76,14 @@ static void ipoctal_reset_channel(struct ipoctal_channel *channel) static int ipoctal_port_activate(struct tty_port *port, struct tty_struct *tty) { struct ipoctal_channel *channel; + struct ipoctal *ipoctal; channel = dev_get_drvdata(tty->dev); + ipoctal = chan_to_ipoctal(channel, tty->index); + + + if (ipoctal->removed) + return -ENODEV; /* * Enable RX. TX will be enabled when @@ -95,6 +107,7 @@ static int ipoctal_install(struct tty_driver *driver, struct tty_struct *tty) if (res) goto err_put_carrier; + kref_get(&ipoctal->kref); tty->driver_data = channel; return 0; @@ -460,8 +473,13 @@ static ssize_t ipoctal_write_tty(struct tty_struct *tty, const u8 *buf, size_t count) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); size_t char_copied; + + if (ipoctal->removed || !channel->tty_port.xmit_buf) + return 0; + char_copied = ipoctal_copy_write_buffer(channel, buf, count); /* As the IP-OCTAL 485 only supports half duplex, do it manually */ @@ -501,8 +519,13 @@ static void ipoctal_set_termios(struct tty_struct *tty, unsigned char mr2 = 0; unsigned char csr = 0; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); speed_t baud; + + if (ipoctal->removed) + return; + cflag = tty->termios.c_cflag; /* Disable and reset everything before change the setup */ @@ -631,10 +654,16 @@ static void ipoctal_hangup(struct tty_struct *tty) { unsigned long flags; struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + spin_lock_irqsave(&channel->lock, flags); channel->nb_bytes = 0; channel->pointer_read = 0; @@ -651,10 +680,16 @@ static void ipoctal_hangup(struct tty_struct *tty) static void ipoctal_shutdown(struct tty_struct *tty) { struct ipoctal_channel *channel = tty->driver_data; + struct ipoctal *ipoctal; if (channel == NULL) return; + ipoctal = chan_to_ipoctal(channel, tty->index); + + if (ipoctal->removed) + return; + ipoctal_reset_channel(channel); tty_port_set_initialized(&channel->tty_port, false); } @@ -664,8 +699,9 @@ static void ipoctal_cleanup(struct tty_struct *tty) struct ipoctal_channel *channel = tty->driver_data; struct ipoctal *ipoctal = chan_to_ipoctal(channel, tty->index); - /* release the carrier driver */ - ipack_put_carrier(ipoctal->dev); + /* release the carrier driver via cached owner */ + module_put(ipoctal->carrier_owner); + kref_put(&ipoctal->kref, ipoctal_release); } static const struct tty_operations ipoctal_fops = { @@ -683,6 +719,13 @@ static const struct tty_operations ipoctal_fops = { .cleanup = ipoctal_cleanup, }; +static void ipoctal_release(struct kref *kref) +{ + struct ipoctal *ipoctal = container_of(kref, struct ipoctal, kref); + + kfree(ipoctal); +} + static int ipoctal_probe(struct ipack_device *dev) { int res; @@ -692,7 +735,10 @@ static int ipoctal_probe(struct ipack_device *dev) if (ipoctal == NULL) return -ENOMEM; + kref_init(&ipoctal->kref); + ipoctal->dev = dev; + ipoctal->carrier_owner = dev->bus->owner; res = ipoctal_inst_slot(ipoctal, dev->bus->bus_nr, dev->slot); if (res) goto out_uninst; @@ -701,7 +747,7 @@ static int ipoctal_probe(struct ipack_device *dev) return 0; out_uninst: - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); return res; } @@ -709,6 +755,8 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) { int i; + ipoctal->removed = true; + ipoctal->dev->bus->ops->free_irq(ipoctal->dev); for (i = 0; i < NR_CHANNELS; i++) { @@ -725,7 +773,7 @@ static void __ipoctal_remove(struct ipoctal *ipoctal) tty_unregister_driver(ipoctal->tty_drv); kfree(ipoctal->tty_drv->name); tty_driver_kref_put(ipoctal->tty_drv); - kfree(ipoctal); + kref_put(&ipoctal->kref, ipoctal_release); } static void ipoctal_remove(struct ipack_device *idev) diff --git a/drivers/irqchip/exynos-combiner.c b/drivers/irqchip/exynos-combiner.c index e7dfcf0cda438..495848442b355 100644 --- a/drivers/irqchip/exynos-combiner.c +++ b/drivers/irqchip/exynos-combiner.c @@ -200,12 +200,13 @@ static void __init combiner_init(void __iomem *combiner_base, /** * combiner_suspend - save interrupt combiner state before suspend + * @data: syscore context * * Save the interrupt enable set register for all combiner groups since * the state is lost when the system enters into a sleep state. * */ -static int combiner_suspend(void) +static int combiner_suspend(void *data) { int i; @@ -218,12 +219,13 @@ static int combiner_suspend(void) /** * combiner_resume - restore interrupt combiner state after resume + * @data: syscore context * * Restore the interrupt enable set register for all combiner groups since * the state is lost when the system enters into a sleep state on suspend. * */ -static void combiner_resume(void) +static void combiner_resume(void *data) { int i; @@ -240,11 +242,15 @@ static void combiner_resume(void) #define combiner_resume NULL #endif -static struct syscore_ops combiner_syscore_ops = { +static const struct syscore_ops combiner_syscore_ops = { .suspend = combiner_suspend, .resume = combiner_resume, }; +static struct syscore combiner_syscore = { + .ops = &combiner_syscore_ops, +}; + static int __init combiner_of_init(struct device_node *np, struct device_node *parent) { @@ -264,7 +270,7 @@ static int __init combiner_of_init(struct device_node *np, combiner_init(combiner_base, np); - register_syscore_ops(&combiner_syscore_ops); + register_syscore(&combiner_syscore); return 0; } diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index a44c49e985b75..a4d03a2d15699 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -726,7 +726,7 @@ static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) } while (1); } -static int mpic_suspend(void) +static int mpic_suspend(void *data) { struct mpic *mpic = mpic_data; @@ -735,7 +735,7 @@ static int mpic_suspend(void) return 0; } -static void mpic_resume(void) +static void mpic_resume(void *data) { struct mpic *mpic = mpic_data; bool src0, src1; @@ -788,11 +788,15 @@ static void mpic_resume(void) mpic_ipi_resume(mpic); } -static struct syscore_ops mpic_syscore_ops = { +static const struct syscore_ops mpic_syscore_ops = { .suspend = mpic_suspend, .resume = mpic_resume, }; +static struct syscore mpic_syscore = { + .ops = &mpic_syscore_ops, +}; + static int __init mpic_map_region(struct device_node *np, int index, void __iomem **base, phys_addr_t *phys_base) { @@ -905,7 +909,7 @@ static int __init mpic_of_init(struct device_node *node, struct device_node *par mpic_handle_cascade_irq, mpic); } - register_syscore_ops(&mpic_syscore_ops); + register_syscore(&mpic_syscore); return 0; } diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index 821b288587cab..674138668f1c6 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -291,7 +291,7 @@ static int bcm7038_l1_init_one(struct device_node *dn, unsigned int idx, static LIST_HEAD(bcm7038_l1_intcs_list); static DEFINE_RAW_SPINLOCK(bcm7038_l1_intcs_lock); -static int bcm7038_l1_suspend(void) +static int bcm7038_l1_suspend(void *data) { struct bcm7038_l1_chip *intc; int boot_cpu, word; @@ -317,7 +317,7 @@ static int bcm7038_l1_suspend(void) return 0; } -static void bcm7038_l1_resume(void) +static void bcm7038_l1_resume(void *data) { struct bcm7038_l1_chip *intc; int boot_cpu, word; @@ -338,11 +338,15 @@ static void bcm7038_l1_resume(void) } } -static struct syscore_ops bcm7038_l1_syscore_ops = { +static const struct syscore_ops bcm7038_l1_syscore_ops = { .suspend = bcm7038_l1_suspend, .resume = bcm7038_l1_resume, }; +static struct syscore bcm7038_l1_syscore = { + .ops = &bcm7038_l1_syscore_ops, +}; + static int bcm7038_l1_set_wake(struct irq_data *d, unsigned int on) { struct bcm7038_l1_chip *intc = irq_data_get_irq_chip_data(d); @@ -430,7 +434,7 @@ static int bcm7038_l1_probe(struct platform_device *pdev, struct device_node *pa raw_spin_unlock(&bcm7038_l1_intcs_lock); if (list_is_singular(&bcm7038_l1_intcs_list)) - register_syscore_ops(&bcm7038_l1_syscore_ops); + register_syscore(&bcm7038_l1_syscore); #endif pr_info("registered BCM7038 L1 intc (%pOF, IRQs: %d)\n", diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index a1661657391d6..8607ed0d98841 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -4593,6 +4593,13 @@ static int its_vpe_init(struct its_vpe *vpe) static void its_vpe_teardown(struct its_vpe *vpe) { + /* + * If vpt_page is NULL, then its_vpe_init() has failed, and + * there is nothing to do as no resource has been allocated. + */ + if (vpe->vpt_page == NULL) + return; + its_vpe_db_proxy_unmap(vpe); its_vpe_id_free(vpe->vpe_id); its_free_pending_table(vpe->vpt_page); @@ -4673,8 +4680,10 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i)); } - if (err) + if (err) { + its_vpe_teardown(vm->vpes[i]); its_vpe_irq_domain_free(domain, virq, i); + } return err; } @@ -4994,7 +5003,7 @@ static void its_enable_quirks(struct its_node *its) its_quirks, its); } -static int its_save_disable(void) +static int its_save_disable(void *data) { struct its_node *its; int err = 0; @@ -5030,7 +5039,7 @@ static int its_save_disable(void) return err; } -static void its_restore_enable(void) +static void its_restore_enable(void *data) { struct its_node *its; int ret; @@ -5090,11 +5099,15 @@ static void its_restore_enable(void) raw_spin_unlock(&its_lock); } -static struct syscore_ops its_syscore_ops = { +static const struct syscore_ops its_syscore_ops = { .suspend = its_save_disable, .resume = its_restore_enable, }; +static struct syscore its_syscore = { + .ops = &its_syscore_ops, +}; + static void __init __iomem *its_map_one(struct resource *res, int *err) { void __iomem *its_base; @@ -5322,7 +5335,7 @@ static int __init its_probe_one(struct its_node *its) err = its_init_domain(its); if (err) - goto out_free_tables; + goto out_free_collection; raw_spin_lock(&its_lock); list_add(&its->entry, &its_nodes); @@ -5330,6 +5343,8 @@ static int __init its_probe_one(struct its_node *its) return 0; +out_free_collection: + kfree(its->collections); out_free_tables: its_free_tables(its); out_free_cmd: @@ -5742,9 +5757,13 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; err = its_probe_one(its); - if (!err) - return 0; + if (err) + goto probe_err; + + return 0; +probe_err: + its_node_destroy(its); node_err: iort_deregister_domain_token(its_entry->translation_id); dom_err: @@ -5834,6 +5853,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, its_acpi_probe(); if (list_empty(&its_nodes)) { + rdists->has_vlpis = false; pr_warn("ITS: No ITS available, not enabling LPIs\n"); return -ENXIO; } @@ -5866,7 +5886,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, } } - register_syscore_ops(&its_syscore_ops); + register_syscore(&its_syscore); return 0; } diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c index ce2732d649a3e..a650eccdb37a8 100644 --- a/drivers/irqchip/irq-gic-v5-irs.c +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -198,7 +198,7 @@ int gicv5_irs_iste_alloc(const u32 lpi) if (!gicv5_global_data.ist.l2) return 0; - irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); + irs_data = per_cpu(per_cpu_irs_data, 0); if (!irs_data) return -ENOENT; diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c index fa86dd03e307c..707c5423e97d0 100644 --- a/drivers/irqchip/irq-gic-v5.c +++ b/drivers/irqchip/irq-gic-v5.c @@ -207,17 +207,13 @@ static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type) FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, hwirq_type); gic_insn(cddi, CDDI); - - gic_insn(0, CDEOI); } static void gicv5_ppi_irq_eoi(struct irq_data *d) { /* Skip deactivate for forwarded PPI interrupts */ - if (irqd_is_forwarded_to_vcpu(d)) { - gic_insn(0, CDEOI); + if (irqd_is_forwarded_to_vcpu(d)) return; - } gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_PPI); } @@ -811,6 +807,9 @@ void __init gicv5_init_lpi_domain(void) void __init gicv5_free_lpi_domain(void) { + if (!gicv5_global_data.lpi_domain) + return; + irq_domain_remove(gicv5_global_data.lpi_domain); gicv5_global_data.lpi_domain = NULL; } @@ -914,6 +913,13 @@ static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs) */ isb(); + /* + * Ensure that we can receive the next interrupts in the event that we + * have a long running handler or directly enter a guest by doing the + * priority drop immediately. + */ + gic_insn(0, CDEOI); + hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia); handle_irq_per_domain(hwirq); @@ -923,8 +929,10 @@ static void gicv5_cpu_disable_interrupts(void) { u64 cr0; - cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0); + cr0 = read_sysreg_s(SYS_ICC_CR0_EL1); + cr0 &= ~ICC_CR0_EL1_EN_MASK; write_sysreg_s(cr0, SYS_ICC_CR0_EL1); + isb(); } static void gicv5_cpu_enable_interrupts(void) @@ -939,7 +947,8 @@ static void gicv5_cpu_enable_interrupts(void) pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI); write_sysreg_s(pcr, SYS_ICC_PCR_EL1); - cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1); + cr0 = read_sysreg_s(SYS_ICC_CR0_EL1); + cr0 |= ICC_CR0_EL1_EN_MASK; write_sysreg_s(cr0, SYS_ICC_CR0_EL1); } @@ -1106,7 +1115,7 @@ static int __init gicv5_of_init(struct device_node *node, struct device_node *pa ret = gicv5_starting_cpu(smp_processor_id()); if (ret) - goto out_dom; + goto out_int; ret = set_handle_irq(gicv5_handle_irq); if (ret) @@ -1114,7 +1123,7 @@ static int __init gicv5_of_init(struct device_node *node, struct device_node *pa ret = gicv5_irs_enable(); if (ret) - goto out_int; + goto out_handle; gicv5_smp_init(); @@ -1124,9 +1133,10 @@ static int __init gicv5_of_init(struct device_node *node, struct device_node *pa return 0; +out_handle: + set_handle_irq(NULL); out_int: gicv5_cpu_disable_interrupts(); -out_dom: gicv5_free_domains(); out_irs: gicv5_irs_remove(); diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c index 91b2f587119ce..cca77f9948a38 100644 --- a/drivers/irqchip/irq-i8259.c +++ b/drivers/irqchip/irq-i8259.c @@ -202,13 +202,13 @@ static void mask_and_ack_8259A(struct irq_data *d) } } -static void i8259A_resume(void) +static void i8259A_resume(void *data) { if (i8259A_auto_eoi >= 0) init_8259A(i8259A_auto_eoi); } -static void i8259A_shutdown(void) +static void i8259A_shutdown(void *data) { /* Put the i8259A into a quiescent state that * the kernel initialization code can get it @@ -220,11 +220,15 @@ static void i8259A_shutdown(void) } } -static struct syscore_ops i8259_syscore_ops = { +static const struct syscore_ops i8259_syscore_ops = { .resume = i8259A_resume, .shutdown = i8259A_shutdown, }; +static struct syscore i8259_syscore = { + .ops = &i8259_syscore_ops, +}; + static void init_8259A(int auto_eoi) { unsigned long flags; @@ -320,7 +324,7 @@ struct irq_domain * __init __init_i8259_irqs(struct device_node *node) if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) pr_err("Failed to register cascade interrupt\n"); - register_syscore_ops(&i8259_syscore_ops); + register_syscore(&i8259_syscore); return domain; } diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c index b91f5c14b4057..04f7ba0657be7 100644 --- a/drivers/irqchip/irq-imx-gpcv2.c +++ b/drivers/irqchip/irq-imx-gpcv2.c @@ -33,7 +33,7 @@ static void __iomem *gpcv2_idx_to_reg(struct gpcv2_irqchip_data *cd, int i) return cd->gpc_base + cd->cpu2wakeup + i * 4; } -static int gpcv2_wakeup_source_save(void) +static int gpcv2_wakeup_source_save(void *data) { struct gpcv2_irqchip_data *cd; void __iomem *reg; @@ -52,7 +52,7 @@ static int gpcv2_wakeup_source_save(void) return 0; } -static void gpcv2_wakeup_source_restore(void) +static void gpcv2_wakeup_source_restore(void *data) { struct gpcv2_irqchip_data *cd; int i; @@ -65,9 +65,13 @@ static void gpcv2_wakeup_source_restore(void) writel_relaxed(cd->saved_irq_mask[i], gpcv2_idx_to_reg(cd, i)); } -static struct syscore_ops imx_gpcv2_syscore_ops = { - .suspend = gpcv2_wakeup_source_save, - .resume = gpcv2_wakeup_source_restore, +static const struct syscore_ops gpcv2_syscore_ops = { + .suspend = gpcv2_wakeup_source_save, + .resume = gpcv2_wakeup_source_restore, +}; + +static struct syscore gpcv2_syscore = { + .ops = &gpcv2_syscore_ops, }; static int imx_gpcv2_irq_set_wake(struct irq_data *d, unsigned int on) @@ -276,7 +280,7 @@ static int __init imx_gpcv2_irqchip_init(struct device_node *node, writel_relaxed(~0x1, cd->gpc_base + cd->cpu2wakeup); imx_gpcv2_instance = cd; - register_syscore_ops(&imx_gpcv2_syscore_ops); + register_syscore(&gpcv2_syscore); /* * Clear the OF_POPULATED flag set in of_irq_init so that diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index 39e5a72ccd3c8..ad2105685b48b 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -407,21 +407,25 @@ static struct irq_domain *acpi_get_vec_parent(int node, struct acpi_vector_group return NULL; } -static int eiointc_suspend(void) +static int eiointc_suspend(void *data) { return 0; } -static void eiointc_resume(void) +static void eiointc_resume(void *data) { eiointc_router_init(0); } -static struct syscore_ops eiointc_syscore_ops = { +static const struct syscore_ops eiointc_syscore_ops = { .suspend = eiointc_suspend, .resume = eiointc_resume, }; +static struct syscore eiointc_syscore = { + .ops = &eiointc_syscore_ops, +}; + static int __init pch_pic_parse_madt(union acpi_subtable_headers *header, const unsigned long end) { @@ -540,7 +544,7 @@ static int __init eiointc_init(struct eiointc_priv *priv, int parent_irq, eiointc_router_init(0); if (nr_pics == 1) { - register_syscore_ops(&eiointc_syscore_ops); + register_syscore(&eiointc_syscore); cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_EIOINTC_STARTING, "irqchip/loongarch/eiointc:starting", eiointc_router_init, NULL); diff --git a/drivers/irqchip/irq-loongson-htpic.c b/drivers/irqchip/irq-loongson-htpic.c index f4abdf156de71..1c691c4be9894 100644 --- a/drivers/irqchip/irq-loongson-htpic.c +++ b/drivers/irqchip/irq-loongson-htpic.c @@ -71,15 +71,19 @@ static void htpic_reg_init(void) writel(0xffff, htpic->base + HTINT_EN_OFF); } -static void htpic_resume(void) +static void htpic_resume(void *data) { htpic_reg_init(); } -struct syscore_ops htpic_syscore_ops = { +static const struct syscore_ops htpic_syscore_ops = { .resume = htpic_resume, }; +static struct syscore htpic_syscore = { + .ops = &htpic_syscore_ops, +}; + static int __init htpic_of_init(struct device_node *node, struct device_node *parent) { unsigned int parent_irq[4]; @@ -130,7 +134,7 @@ static int __init htpic_of_init(struct device_node *node, struct device_node *pa htpic_irq_dispatch, htpic); } - register_syscore_ops(&htpic_syscore_ops); + register_syscore(&htpic_syscore); return 0; diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index d8558eb350445..d2be8e954e92f 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -159,7 +159,7 @@ static void htvec_reset(struct htvec *priv) } } -static int htvec_suspend(void) +static int htvec_suspend(void *data) { int i; @@ -169,7 +169,7 @@ static int htvec_suspend(void) return 0; } -static void htvec_resume(void) +static void htvec_resume(void *data) { int i; @@ -177,11 +177,15 @@ static void htvec_resume(void) writel(htvec_priv->saved_vec_en[i], htvec_priv->base + HTVEC_EN_OFF + 4 * i); } -static struct syscore_ops htvec_syscore_ops = { +static const struct syscore_ops htvec_syscore_ops = { .suspend = htvec_suspend, .resume = htvec_resume, }; +static struct syscore htvec_syscore = { + .ops = &htvec_syscore_ops, +}; + static int htvec_init(phys_addr_t addr, unsigned long size, int num_parents, int parent_irq[], struct fwnode_handle *domain_handle) { @@ -214,7 +218,7 @@ static int htvec_init(phys_addr_t addr, unsigned long size, htvec_priv = priv; - register_syscore_ops(&htvec_syscore_ops); + register_syscore(&htvec_syscore); return 0; diff --git a/drivers/irqchip/irq-loongson-pch-lpc.c b/drivers/irqchip/irq-loongson-pch-lpc.c index 912bf50a5c7ca..3a125f3e42873 100644 --- a/drivers/irqchip/irq-loongson-pch-lpc.c +++ b/drivers/irqchip/irq-loongson-pch-lpc.c @@ -151,7 +151,7 @@ static int pch_lpc_disabled(struct pch_lpc *priv) (readl(priv->base + LPC_INT_STS) == 0xffffffff); } -static int pch_lpc_suspend(void) +static int pch_lpc_suspend(void *data) { pch_lpc_priv->saved_reg_ctl = readl(pch_lpc_priv->base + LPC_INT_CTL); pch_lpc_priv->saved_reg_ena = readl(pch_lpc_priv->base + LPC_INT_ENA); @@ -159,18 +159,22 @@ static int pch_lpc_suspend(void) return 0; } -static void pch_lpc_resume(void) +static void pch_lpc_resume(void *data) { writel(pch_lpc_priv->saved_reg_ctl, pch_lpc_priv->base + LPC_INT_CTL); writel(pch_lpc_priv->saved_reg_ena, pch_lpc_priv->base + LPC_INT_ENA); writel(pch_lpc_priv->saved_reg_pol, pch_lpc_priv->base + LPC_INT_POL); } -static struct syscore_ops pch_lpc_syscore_ops = { +static const struct syscore_ops pch_lpc_syscore_ops = { .suspend = pch_lpc_suspend, .resume = pch_lpc_resume, }; +static struct syscore pch_lpc_syscore = { + .ops = &pch_lpc_syscore_ops, +}; + int __init pch_lpc_acpi_init(struct irq_domain *parent, struct acpi_madt_lpc_pic *acpi_pchlpc) { @@ -222,7 +226,7 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent, pch_lpc_priv = priv; pch_lpc_handle = irq_handle; - register_syscore_ops(&pch_lpc_syscore_ops); + register_syscore(&pch_lpc_syscore); return 0; diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index 62e6bf3a0611a..c6b369a974a7d 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -278,7 +278,7 @@ static void pch_pic_reset(struct pch_pic *priv) } } -static int pch_pic_suspend(void) +static int pch_pic_suspend(void *data) { int i, j; @@ -296,7 +296,7 @@ static int pch_pic_suspend(void) return 0; } -static void pch_pic_resume(void) +static void pch_pic_resume(void *data) { int i, j; @@ -313,11 +313,15 @@ static void pch_pic_resume(void) } } -static struct syscore_ops pch_pic_syscore_ops = { +static const struct syscore_ops pch_pic_syscore_ops = { .suspend = pch_pic_suspend, .resume = pch_pic_resume, }; +static struct syscore pch_pic_syscore = { + .ops = &pch_pic_syscore_ops, +}; + static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, struct irq_domain *parent_domain, struct fwnode_handle *domain_handle, u32 gsi_base) @@ -356,7 +360,7 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, pch_pic_priv[nr_pics++] = priv; if (nr_pics == 1) - register_syscore_ops(&pch_pic_syscore_ops); + register_syscore(&pch_pic_syscore); return 0; diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 6f69f4e5dbacd..12919836dadba 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -64,20 +64,6 @@ struct mbigen_device { void __iomem *base; }; -static inline unsigned int get_mbigen_node_offset(unsigned int nid) -{ - unsigned int offset = nid * MBIGEN_NODE_OFFSET; - - /* - * To avoid touched clear register in unexpected way, we need to directly - * skip clear register when access to more than 10 mbigen nodes. - */ - if (nid >= (REG_MBIGEN_CLEAR_OFFSET / MBIGEN_NODE_OFFSET)) - offset += MBIGEN_NODE_OFFSET; - - return offset; -} - static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) { unsigned int nid, pin; @@ -86,7 +72,8 @@ static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) nid = hwirq / IRQS_PER_MBIGEN_NODE + 1; pin = hwirq % IRQS_PER_MBIGEN_NODE; - return pin * 4 + get_mbigen_node_offset(nid) + REG_MBIGEN_VEC_OFFSET; + return pin * 4 + nid * MBIGEN_NODE_OFFSET + + REG_MBIGEN_VEC_OFFSET; } static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, @@ -101,7 +88,8 @@ static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, *mask = 1 << (irq_ofst % 32); ofst = irq_ofst / 32 * 4; - *addr = ofst + get_mbigen_node_offset(nid) + REG_MBIGEN_TYPE_OFFSET; + *addr = ofst + nid * MBIGEN_NODE_OFFSET + + REG_MBIGEN_TYPE_OFFSET; } static inline void get_mbigen_clear_reg(irq_hw_number_t hwirq, diff --git a/drivers/irqchip/irq-mchp-eic.c b/drivers/irqchip/irq-mchp-eic.c index 979bb86929f8e..31093a8ab67c3 100644 --- a/drivers/irqchip/irq-mchp-eic.c +++ b/drivers/irqchip/irq-mchp-eic.c @@ -109,7 +109,7 @@ static int mchp_eic_irq_set_wake(struct irq_data *d, unsigned int on) return 0; } -static int mchp_eic_irq_suspend(void) +static int mchp_eic_irq_suspend(void *data) { unsigned int hwirq; @@ -123,7 +123,7 @@ static int mchp_eic_irq_suspend(void) return 0; } -static void mchp_eic_irq_resume(void) +static void mchp_eic_irq_resume(void *data) { unsigned int hwirq; @@ -135,11 +135,15 @@ static void mchp_eic_irq_resume(void) MCHP_EIC_SCFG(hwirq)); } -static struct syscore_ops mchp_eic_syscore_ops = { +static const struct syscore_ops mchp_eic_syscore_ops = { .suspend = mchp_eic_irq_suspend, .resume = mchp_eic_irq_resume, }; +static struct syscore mchp_eic_syscore = { + .ops = &mchp_eic_syscore_ops, +}; + static struct irq_chip mchp_eic_chip = { .name = "eic", .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SET_TYPE_MASKED, @@ -258,7 +262,7 @@ static int mchp_eic_probe(struct platform_device *pdev, struct device_node *pare goto clk_unprepare; } - register_syscore_ops(&mchp_eic_syscore_ops); + register_syscore(&mchp_eic_syscore); pr_info("%pOF: EIC registered, nr_irqs %u\n", node, MCHP_EIC_NIRQ); diff --git a/drivers/irqchip/irq-mst-intc.c b/drivers/irqchip/irq-mst-intc.c index 9643cc3a77d77..7f760f555a76a 100644 --- a/drivers/irqchip/irq-mst-intc.c +++ b/drivers/irqchip/irq-mst-intc.c @@ -143,7 +143,7 @@ static void mst_intc_polarity_restore(struct mst_intc_chip_data *cd) writew_relaxed(cd->saved_polarity_conf[i], addr + i * 4); } -static void mst_irq_resume(void) +static void mst_irq_resume(void *data) { struct mst_intc_chip_data *cd; @@ -151,7 +151,7 @@ static void mst_irq_resume(void) mst_intc_polarity_restore(cd); } -static int mst_irq_suspend(void) +static int mst_irq_suspend(void *data) { struct mst_intc_chip_data *cd; @@ -160,14 +160,18 @@ static int mst_irq_suspend(void) return 0; } -static struct syscore_ops mst_irq_syscore_ops = { +static const struct syscore_ops mst_irq_syscore_ops = { .suspend = mst_irq_suspend, .resume = mst_irq_resume, }; +static struct syscore mst_irq_syscore = { + .ops = &mst_irq_syscore_ops, +}; + static int __init mst_irq_pm_init(void) { - register_syscore_ops(&mst_irq_syscore_ops); + register_syscore(&mst_irq_syscore); return 0; } late_initcall(mst_irq_pm_init); diff --git a/drivers/irqchip/irq-mtk-cirq.c b/drivers/irqchip/irq-mtk-cirq.c index de481ba340f83..9571f622774e3 100644 --- a/drivers/irqchip/irq-mtk-cirq.c +++ b/drivers/irqchip/irq-mtk-cirq.c @@ -199,7 +199,7 @@ static const struct irq_domain_ops cirq_domain_ops = { }; #ifdef CONFIG_PM_SLEEP -static int mtk_cirq_suspend(void) +static int mtk_cirq_suspend(void *data) { void __iomem *reg; u32 value, mask; @@ -257,7 +257,7 @@ static int mtk_cirq_suspend(void) return 0; } -static void mtk_cirq_resume(void) +static void mtk_cirq_resume(void *data) { void __iomem *reg = mtk_cirq_reg(cirq_data, CIRQ_CONTROL); u32 value; @@ -272,14 +272,18 @@ static void mtk_cirq_resume(void) writel_relaxed(value, reg); } -static struct syscore_ops mtk_cirq_syscore_ops = { +static const struct syscore_ops mtk_cirq_syscore_ops = { .suspend = mtk_cirq_suspend, .resume = mtk_cirq_resume, }; +static struct syscore mtk_cirq_syscore = { + .ops = &mtk_cirq_syscore_ops, +}; + static void mtk_cirq_syscore_init(void) { - register_syscore_ops(&mtk_cirq_syscore_ops); + register_syscore(&mtk_cirq_syscore); } #else static inline void mtk_cirq_syscore_init(void) {} diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 942c1f8c363d7..90fb1a14c1f13 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -1,5 +1,10 @@ // SPDX-License-Identifier: GPL-2.0-only /* + * Realtek Interrupt controller. + * + * The Realtek Interrupt controller is a big endian device found in the + * Realtek MIPS SoCs. + * * Copyright (C) 2020 Birger Koblitz * Copyright (C) 2020 Bert Vermeulen * Copyright (C) 2020 John Crispin @@ -23,10 +28,15 @@ #define RTL_ICTL_NUM_INPUTS 32 -#define REG(x) (realtek_ictl_base + x) +#define REG(cpu, x) (realtek_ictl_base[cpu] + x) + +struct realtek_ictl_output { + struct irq_domain *domain; + u32 mask; +}; static DEFINE_RAW_SPINLOCK(irq_lock); -static void __iomem *realtek_ictl_base; +static void __iomem *realtek_ictl_base[NR_CPUS]; /* * IRR0-IRR3 store 4 bits per interrupt, but Realtek uses inverted numbering, @@ -37,60 +47,85 @@ static void __iomem *realtek_ictl_base; #define IRR_OFFSET(idx) (4 * (3 - (idx * 4) / 32)) #define IRR_SHIFT(idx) ((idx * 4) % 32) -static void write_irr(void __iomem *irr0, int idx, u32 value) +static inline void enable_gimr(unsigned int cpu, unsigned int hw_irq) +{ + u32 gimr; + + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); + gimr |= BIT(hw_irq); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); +} + +static inline void disable_gimr(unsigned int cpu, unsigned int hw_irq) +{ + u32 gimr; + + gimr = readl_be(REG(cpu, RTL_ICTL_GIMR)); + gimr &= ~BIT(hw_irq); + writel_be(gimr, REG(cpu, RTL_ICTL_GIMR)); +} + +static void write_irr(unsigned int cpu, int hw_irq, u32 value) { - unsigned int offset = IRR_OFFSET(idx); - unsigned int shift = IRR_SHIFT(idx); + void __iomem *irr0 = REG(cpu, RTL_ICTL_IRR0); + unsigned int offset = IRR_OFFSET(hw_irq); + unsigned int shift = IRR_SHIFT(hw_irq); u32 irr; - irr = readl(irr0 + offset) & ~(0xf << shift); + irr = readl_be(irr0 + offset) & ~(0xf << shift); irr |= (value & 0xf) << shift; - writel(irr, irr0 + offset); + writel_be(irr, irr0 + offset); } static void realtek_ictl_unmask_irq(struct irq_data *i) { - unsigned long flags; - u32 value; + unsigned int cpu; - raw_spin_lock_irqsave(&irq_lock, flags); + guard(raw_spinlock)(&irq_lock); + for_each_cpu(cpu, irq_data_get_effective_affinity_mask(i)) + enable_gimr(cpu, i->hwirq); +} - value = readl(REG(RTL_ICTL_GIMR)); - value |= BIT(i->hwirq); - writel(value, REG(RTL_ICTL_GIMR)); +static void realtek_ictl_mask_irq(struct irq_data *i) +{ + unsigned int cpu; - raw_spin_unlock_irqrestore(&irq_lock, flags); + guard(raw_spinlock)(&irq_lock); + for_each_cpu(cpu, irq_data_get_effective_affinity_mask(i)) + disable_gimr(cpu, i->hwirq); } -static void realtek_ictl_mask_irq(struct irq_data *i) +static int realtek_ictl_irq_affinity(struct irq_data *i, const struct cpumask *dest, bool force) { - unsigned long flags; - u32 value; + if (!irqd_irq_masked(i)) + realtek_ictl_mask_irq(i); - raw_spin_lock_irqsave(&irq_lock, flags); + irq_data_update_effective_affinity(i, dest); - value = readl(REG(RTL_ICTL_GIMR)); - value &= ~BIT(i->hwirq); - writel(value, REG(RTL_ICTL_GIMR)); + if (!irqd_irq_masked(i)) + realtek_ictl_unmask_irq(i); - raw_spin_unlock_irqrestore(&irq_lock, flags); + return IRQ_SET_MASK_OK; } static struct irq_chip realtek_ictl_irq = { - .name = "realtek-rtl-intc", - .irq_mask = realtek_ictl_mask_irq, - .irq_unmask = realtek_ictl_unmask_irq, + .name = "realtek-rtl-intc", + .irq_mask = realtek_ictl_mask_irq, + .irq_unmask = realtek_ictl_unmask_irq, + .irq_set_affinity = realtek_ictl_irq_affinity, }; -static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw) +static int intc_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hw_irq) { - unsigned long flags; + struct realtek_ictl_output *output = d->host_data; + unsigned int cpu; irq_set_chip_and_handler(irq, &realtek_ictl_irq, handle_level_irq); - raw_spin_lock_irqsave(&irq_lock, flags); - write_irr(REG(RTL_ICTL_IRR0), hw, 1); - raw_spin_unlock_irqrestore(&irq_lock, flags); + guard(raw_spinlock_irqsave)(&irq_lock); + output->mask |= BIT(hw_irq); + for_each_present_cpu(cpu) + write_irr(cpu, hw_irq, 1); return 0; } @@ -102,73 +137,99 @@ static const struct irq_domain_ops irq_domain_ops = { static void realtek_irq_dispatch(struct irq_desc *desc) { + struct realtek_ictl_output *output = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); - struct irq_domain *domain; + unsigned int cpu = smp_processor_id(); unsigned long pending; - unsigned int soc_int; + unsigned int hw_irq; chained_irq_enter(chip, desc); - pending = readl(REG(RTL_ICTL_GIMR)) & readl(REG(RTL_ICTL_GISR)); + pending = readl_be(REG(cpu, RTL_ICTL_GIMR)) & + readl_be(REG(cpu, RTL_ICTL_GISR)) & output->mask; if (unlikely(!pending)) { spurious_interrupt(); goto out; } - domain = irq_desc_get_handler_data(desc); - for_each_set_bit(soc_int, &pending, 32) - generic_handle_domain_irq(domain, soc_int); + for_each_set_bit(hw_irq, &pending, RTL_ICTL_NUM_INPUTS) + generic_handle_domain_irq(output->domain, hw_irq); out: chained_irq_exit(chip, desc); } -static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +static int __init realtek_setup_parents(struct device_node *node) { + int err, parent_irq, num_parents = of_irq_count(node); + struct realtek_ictl_output *output; struct of_phandle_args oirq; struct irq_domain *domain; - unsigned int soc_irq; - int parent_irq; - realtek_ictl_base = of_iomap(node, 0); - if (!realtek_ictl_base) - return -ENXIO; - - /* Disable all cascaded interrupts and clear routing */ - writel(0, REG(RTL_ICTL_GIMR)); - for (soc_irq = 0; soc_irq < RTL_ICTL_NUM_INPUTS; soc_irq++) - write_irr(REG(RTL_ICTL_IRR0), soc_irq, 0); + output = kcalloc(1, sizeof(*output), GFP_KERNEL); + if (!output) + return -ENOMEM; - if (WARN_ON(!of_irq_count(node))) { + if (WARN_ON(!num_parents)) { /* - * If DT contains no parent interrupts, assume MIPS CPU IRQ 2 - * (HW0) is connected to the first output. This is the case for - * all known hardware anyway. "interrupt-map" is deprecated, so - * don't bother trying to parse that. + * If DT contains no parent interrupts, assume MIPS IRQ 2 (HW0) is + * connected to the first output. This is the case for all known hardware. */ - oirq.np = of_find_compatible_node(NULL, NULL, "mti,cpu-interrupt-controller"); + oirq.np = of_find_compatible_node(NULL, NULL, + "mti,cpu-interrupt-controller"); + if (!oirq.np) { + err = -EINVAL; + goto err_out; + } + oirq.args_count = 1; oirq.args[0] = 2; - parent_irq = irq_create_of_mapping(&oirq); - of_node_put(oirq.np); } else { parent_irq = of_irq_get(node, 0); } - if (parent_irq < 0) - return parent_irq; - else if (!parent_irq) - return -ENODEV; + if (parent_irq <= 0) { + err = parent_irq ? parent_irq : -ENODEV; + goto err_out; + } - domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); - if (!domain) - return -ENOMEM; + domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, + &irq_domain_ops, output); + if (!domain) { + err = -ENOMEM; + goto err_out; + } - irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, domain); + output->domain = domain; + irq_set_chained_handler_and_data(parent_irq, realtek_irq_dispatch, output); return 0; + +err_out: + kfree(output); + + return err; +} + +static int __init realtek_rtl_of_init(struct device_node *node, struct device_node *parent) +{ + unsigned int cpu; + + for_each_present_cpu(cpu) { + realtek_ictl_base[cpu] = of_iomap(node, cpu); + if (!realtek_ictl_base[cpu]) + return -ENXIO; + + /* Disable all cascaded interrupts and clear routing */ + for (unsigned int hw_irq = 0; hw_irq < RTL_ICTL_NUM_INPUTS; hw_irq++) { + disable_gimr(cpu, hw_irq); + write_irr(cpu, hw_irq, 0); + } + } + + return realtek_setup_parents(node); } IRQCHIP_DECLARE(realtek_rtl_intc, "realtek,rtl-intc", realtek_rtl_of_init); diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c index a20a6471b0e48..1ff3535a4617f 100644 --- a/drivers/irqchip/irq-renesas-irqc.c +++ b/drivers/irqchip/irq-renesas-irqc.c @@ -176,6 +176,7 @@ static int irqc_probe(struct platform_device *pdev) goto err_runtime_pm_disable; } + p->irq_domain->flags |= IRQ_DOMAIN_FLAG_DESTROY_GC; ret = irq_alloc_domain_generic_chips(p->irq_domain, p->number_of_irqs, 1, "irqc", handle_level_irq, 0, 0, IRQ_GC_INIT_NESTED_LOCK); diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index c938ab1592895..709a84247918a 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -103,7 +103,7 @@ static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq * falling/rising-edge. */ if ((iscr & bit) && (iitsr & IITSR_IITSEL_MASK(hw_irq))) { - writel_relaxed(iscr & ~bit, priv->base + ISCR); + writel_relaxed(~bit, priv->base + ISCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. @@ -119,7 +119,7 @@ static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwir reg = readl_relaxed(priv->base + TSCR); if (reg & bit) { - writel_relaxed(reg & ~bit, priv->base + TSCR); + writel_relaxed(~bit, priv->base + TSCR); /* * Enforce that the posted write is flushed to prevent that the * just handled interrupt is raised again. @@ -398,7 +398,7 @@ static int rzg2l_irqc_set_type(struct irq_data *d, unsigned int type) return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH); } -static int rzg2l_irqc_irq_suspend(void) +static int rzg2l_irqc_irq_suspend(void *data) { struct rzg2l_irqc_reg_cache *cache = &rzg2l_irqc_data->cache; void __iomem *base = rzg2l_irqc_data->base; @@ -410,7 +410,7 @@ static int rzg2l_irqc_irq_suspend(void) return 0; } -static void rzg2l_irqc_irq_resume(void) +static void rzg2l_irqc_irq_resume(void *data) { struct rzg2l_irqc_reg_cache *cache = &rzg2l_irqc_data->cache; void __iomem *base = rzg2l_irqc_data->base; @@ -425,11 +425,15 @@ static void rzg2l_irqc_irq_resume(void) writel_relaxed(cache->iitsr, base + IITSR); } -static struct syscore_ops rzg2l_irqc_syscore_ops = { +static const struct syscore_ops rzg2l_irqc_syscore_ops = { .suspend = rzg2l_irqc_irq_suspend, .resume = rzg2l_irqc_irq_resume, }; +static struct syscore rzg2l_irqc_syscore = { + .ops = &rzg2l_irqc_syscore_ops, +}; + static const struct irq_chip rzg2l_irqc_chip = { .name = "rzg2l-irqc", .irq_eoi = rzg2l_irqc_eoi, @@ -577,7 +581,7 @@ static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_n return -ENOMEM; } - register_syscore_ops(&rzg2l_irqc_syscore_ops); + register_syscore(&rzg2l_irqc_syscore); return 0; } diff --git a/drivers/irqchip/irq-sa11x0.c b/drivers/irqchip/irq-sa11x0.c index d8d4dff16276b..e5f24c5f3f416 100644 --- a/drivers/irqchip/irq-sa11x0.c +++ b/drivers/irqchip/irq-sa11x0.c @@ -85,7 +85,7 @@ static struct sa1100irq_state { unsigned int iccr; } sa1100irq_state; -static int sa1100irq_suspend(void) +static int sa1100irq_suspend(void *data) { struct sa1100irq_state *st = &sa1100irq_state; @@ -102,7 +102,7 @@ static int sa1100irq_suspend(void) return 0; } -static void sa1100irq_resume(void) +static void sa1100irq_resume(void *data) { struct sa1100irq_state *st = &sa1100irq_state; @@ -114,14 +114,18 @@ static void sa1100irq_resume(void) } } -static struct syscore_ops sa1100irq_syscore_ops = { +static const struct syscore_ops sa1100irq_syscore_ops = { .suspend = sa1100irq_suspend, .resume = sa1100irq_resume, }; +static struct syscore sa1100irq_syscore = { + .ops = &sa1100irq_syscore_ops, +}; + static int __init sa1100irq_init_devicefs(void) { - register_syscore_ops(&sa1100irq_syscore_ops); + register_syscore(&sa1100irq_syscore); return 0; } diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index 0799c15c745d4..ca70641982ed6 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -250,7 +250,7 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type) return IRQ_SET_MASK_OK; } -static int plic_irq_suspend(void) +static int plic_irq_suspend(void *data) { unsigned int i, cpu; unsigned long flags; @@ -282,7 +282,7 @@ static int plic_irq_suspend(void) return 0; } -static void plic_irq_resume(void) +static void plic_irq_resume(void *data) { unsigned int i, index, cpu; unsigned long flags; @@ -313,11 +313,15 @@ static void plic_irq_resume(void) } } -static struct syscore_ops plic_irq_syscore_ops = { +static const struct syscore_ops plic_irq_syscore_ops = { .suspend = plic_irq_suspend, .resume = plic_irq_resume, }; +static struct syscore plic_irq_syscore = { + .ops = &plic_irq_syscore_ops, +}; + static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) { @@ -683,7 +687,7 @@ static int plic_probe(struct fwnode_handle *fwnode) cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, "irqchip/sifive/plic:starting", plic_starting_cpu, plic_dying_cpu); - register_syscore_ops(&plic_irq_syscore_ops); + register_syscore(&plic_irq_syscore); plic_global_setup_done = true; } } diff --git a/drivers/irqchip/irq-stm32mp-exti.c b/drivers/irqchip/irq-stm32mp-exti.c index a24f4f1a4f8fd..ce4e03ee1ffbb 100644 --- a/drivers/irqchip/irq-stm32mp-exti.c +++ b/drivers/irqchip/irq-stm32mp-exti.c @@ -23,7 +23,7 @@ #define IRQS_PER_BANK 32 -#define HWSPNLCK_TIMEOUT 1000 /* usec */ +#define HWSPNLCK_TIMEOUT_MS 1 #define EXTI_EnCIDCFGR(n) (0x180 + (n) * 4) #define EXTI_HWCFGR1 0x3f0 @@ -377,7 +377,7 @@ static int stm32mp_exti_set_type(struct irq_data *d, unsigned int type) raw_spin_lock(&chip_data->rlock); if (hwlock) { - err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT); + err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT_MS); if (err) { pr_err("%s can't get hwspinlock (%d)\n", __func__, err); goto unlock; diff --git a/drivers/irqchip/irq-sun6i-r.c b/drivers/irqchip/irq-sun6i-r.c index 37d4b29763bc4..23251831c06e2 100644 --- a/drivers/irqchip/irq-sun6i-r.c +++ b/drivers/irqchip/irq-sun6i-r.c @@ -268,7 +268,7 @@ static const struct irq_domain_ops sun6i_r_intc_domain_ops = { .free = irq_domain_free_irqs_common, }; -static int sun6i_r_intc_suspend(void) +static int sun6i_r_intc_suspend(void *data) { u32 buf[BITS_TO_U32(MAX(SUN6I_NR_TOP_LEVEL_IRQS, SUN6I_NR_MUX_BITS))]; int i; @@ -284,7 +284,7 @@ static int sun6i_r_intc_suspend(void) return 0; } -static void sun6i_r_intc_resume(void) +static void sun6i_r_intc_resume(void *data) { int i; @@ -294,17 +294,21 @@ static void sun6i_r_intc_resume(void) writel_relaxed(0, base + SUN6I_IRQ_ENABLE(i)); } -static void sun6i_r_intc_shutdown(void) +static void sun6i_r_intc_shutdown(void *data) { - sun6i_r_intc_suspend(); + sun6i_r_intc_suspend(data); } -static struct syscore_ops sun6i_r_intc_syscore_ops = { +static const struct syscore_ops sun6i_r_intc_syscore_ops = { .suspend = sun6i_r_intc_suspend, .resume = sun6i_r_intc_resume, .shutdown = sun6i_r_intc_shutdown, }; +static struct syscore sun6i_r_intc_syscore = { + .ops = &sun6i_r_intc_syscore_ops, +}; + static int __init sun6i_r_intc_init(struct device_node *node, struct device_node *parent, const struct sun6i_r_intc_variant *v) @@ -346,10 +350,10 @@ static int __init sun6i_r_intc_init(struct device_node *node, return -ENOMEM; } - register_syscore_ops(&sun6i_r_intc_syscore_ops); + register_syscore(&sun6i_r_intc_syscore); sun6i_r_intc_ack_nmi(); - sun6i_r_intc_resume(); + sun6i_r_intc_resume(NULL); return 0; } diff --git a/drivers/irqchip/irq-tegra.c b/drivers/irqchip/irq-tegra.c index 66cbb9f77ff30..b6382cf6359ad 100644 --- a/drivers/irqchip/irq-tegra.c +++ b/drivers/irqchip/irq-tegra.c @@ -132,7 +132,7 @@ static int tegra_set_wake(struct irq_data *d, unsigned int enable) return 0; } -static int tegra_ictlr_suspend(void) +static int tegra_ictlr_suspend(void *data) { unsigned long flags; unsigned int i; @@ -161,7 +161,7 @@ static int tegra_ictlr_suspend(void) return 0; } -static void tegra_ictlr_resume(void) +static void tegra_ictlr_resume(void *data) { unsigned long flags; unsigned int i; @@ -184,14 +184,18 @@ static void tegra_ictlr_resume(void) local_irq_restore(flags); } -static struct syscore_ops tegra_ictlr_syscore_ops = { +static const struct syscore_ops tegra_ictlr_syscore_ops = { .suspend = tegra_ictlr_suspend, .resume = tegra_ictlr_resume, }; +static struct syscore tegra_ictlr_syscore = { + .ops = &tegra_ictlr_syscore_ops, +}; + static void tegra_ictlr_syscore_init(void) { - register_syscore_ops(&tegra_ictlr_syscore_ops); + register_syscore(&tegra_ictlr_syscore); } #else #define tegra_set_wake NULL diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c index 2bcdf216a0003..e38104c5064e6 100644 --- a/drivers/irqchip/irq-vic.c +++ b/drivers/irqchip/irq-vic.c @@ -120,7 +120,7 @@ static void resume_one_vic(struct vic_device *vic) writel(~vic->soft_int, base + VIC_INT_SOFT_CLEAR); } -static void vic_resume(void) +static void vic_resume(void *data) { int id; @@ -146,7 +146,7 @@ static void suspend_one_vic(struct vic_device *vic) writel(~vic->resume_irqs, base + VIC_INT_ENABLE_CLEAR); } -static int vic_suspend(void) +static int vic_suspend(void *data) { int id; @@ -156,11 +156,15 @@ static int vic_suspend(void) return 0; } -static struct syscore_ops vic_syscore_ops = { +static const struct syscore_ops vic_syscore_ops = { .suspend = vic_suspend, .resume = vic_resume, }; +static struct syscore vic_syscore = { + .ops = &vic_syscore_ops, +}; + /** * vic_pm_init - initcall to register VIC pm * @@ -171,7 +175,7 @@ static struct syscore_ops vic_syscore_ops = { static int __init vic_pm_init(void) { if (vic_id > 0) - register_syscore_ops(&vic_syscore_ops); + register_syscore(&vic_syscore); return 0; } diff --git a/drivers/leds/leds-gpio.c b/drivers/leds/leds-gpio.c index c4e0ceda70a7a..b29629a2ae1fd 100644 --- a/drivers/leds/leds-gpio.c +++ b/drivers/leds/leds-gpio.c @@ -293,6 +293,7 @@ static int gpio_led_probe(struct platform_device *pdev) if (IS_ERR(led_dat->gpiod)) { dev_info(dev, "Skipping unavailable LED gpio %d (%s)\n", template->gpio, template->name); + led_dat->gpiod = NULL; continue; } diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c index 0344189bb991c..ba0fea0add2b7 100644 --- a/drivers/leds/leds-pca9532.c +++ b/drivers/leds/leds-pca9532.c @@ -182,11 +182,13 @@ static int pca9532_set_brightness(struct led_classdev *led_cdev, int err = 0; struct pca9532_led *led = ldev_to_led(led_cdev); - if (value == LED_OFF) + if (value == LED_OFF) { led->state = PCA9532_OFF; - else if (value == LED_FULL) + } else if (led->state == PCA9532_PWM1) { + return 0; /* Non-zero brightness shall not stop HW blinking */ + } else if (value == LED_FULL) { led->state = PCA9532_ON; - else { + } else { led->state = PCA9532_PWM0; /* Thecus: hardcode one pwm */ err = pca9532_calcpwm(led->client, PCA9532_PWM_ID_0, 0, value); if (err) @@ -325,9 +327,9 @@ static int pca9532_gpio_set_value(struct gpio_chip *gc, unsigned int offset, struct pca9532_led *led = &data->leds[offset]; if (val) - led->state = PCA9532_ON; - else led->state = PCA9532_OFF; + else + led->state = PCA9532_ON; pca9532_setled(led); @@ -347,7 +349,7 @@ static int pca9532_gpio_get_value(struct gpio_chip *gc, unsigned offset) static int pca9532_gpio_direction_input(struct gpio_chip *gc, unsigned offset) { /* To use as input ensure pin is not driven */ - pca9532_gpio_set_value(gc, offset, 0); + pca9532_gpio_set_value(gc, offset, 1); return 0; } @@ -395,10 +397,14 @@ static int pca9532_configure(struct i2c_client *client, for (i = 0; i < 2; i++) { data->pwm[i] = pdata->pwm[i]; data->psc[i] = pdata->psc[i]; - i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), - data->pwm[i]); - i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), - data->psc[i]); + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PWM(maxleds, i), + data->pwm[i]); + if (err < 0) + return err; + err = i2c_smbus_write_byte_data(client, PCA9532_REG_PSC(maxleds, i), + data->psc[i]); + if (err < 0) + return err; } data->hw_blink = true; diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c index 4e5dd76d714d8..684d77fbe1d81 100644 --- a/drivers/leds/leds-st1202.c +++ b/drivers/leds/leds-st1202.c @@ -32,10 +32,11 @@ #define ST1202_ILED_REG0 0x09 #define ST1202_MAX_LEDS 12 #define ST1202_MAX_PATTERNS 8 -#define ST1202_MILLIS_PATTERN_DUR_MAX 5660 +#define ST1202_MILLIS_PATTERN_DUR_MAX (ST1202_MILLIS_PATTERN_DUR_MIN * U8_MAX) #define ST1202_MILLIS_PATTERN_DUR_MIN 22 #define ST1202_PATTERN_DUR 0x16 #define ST1202_PATTERN_PWM 0x1E +#define ST1202_PATTERN_PWM_FULL 0x0FFF #define ST1202_PATTERN_REP 0x15 struct st1202_led { @@ -86,7 +87,7 @@ static int st1202_write_reg(struct st1202_chip *chip, int reg, uint8_t val) static uint8_t st1202_prescalar_to_miliseconds(unsigned int value) { - return value / ST1202_MILLIS_PATTERN_DUR_MIN - 1; + return value / ST1202_MILLIS_PATTERN_DUR_MIN; } static int st1202_pwm_pattern_write(struct st1202_chip *chip, int led_num, @@ -128,37 +129,11 @@ static int st1202_duration_pattern_write(struct st1202_chip *chip, int pattern, st1202_prescalar_to_miliseconds(value)); } -static void st1202_brightness_set(struct led_classdev *led_cdev, - enum led_brightness value) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - - guard(mutex)(&chip->lock); - - st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); -} - -static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) -{ - struct st1202_led *led = cdev_to_st1202_led(led_cdev); - struct st1202_chip *chip = led->chip; - u8 value = 0; - - guard(mutex)(&chip->lock); - - st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); - - return value; -} - -static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +static int __st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) { u8 chan_low, chan_high; int ret; - guard(mutex)(&chip->lock); - if (led_num <= 7) { ret = st1202_read_reg(chip, ST1202_CHAN_ENABLE_LOW, &chan_low); if (ret < 0) @@ -186,6 +161,40 @@ static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active return 0; } +static int st1202_channel_set(struct st1202_chip *chip, int led_num, bool active) +{ + guard(mutex)(&chip->lock); + + return __st1202_channel_set(chip, led_num, active); +} + +static void st1202_brightness_set(struct led_classdev *led_cdev, + enum led_brightness value) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + + guard(mutex)(&chip->lock); + + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) + st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); + st1202_write_reg(chip, ST1202_ILED_REG0 + led->led_num, value); + __st1202_channel_set(chip, led->led_num, !!value); +} + +static enum led_brightness st1202_brightness_get(struct led_classdev *led_cdev) +{ + struct st1202_led *led = cdev_to_st1202_led(led_cdev); + struct st1202_chip *chip = led->chip; + u8 value = 0; + + guard(mutex)(&chip->lock); + + st1202_read_reg(chip, ST1202_ILED_REG0 + led->led_num, &value); + + return value; +} + static int st1202_led_set(struct led_classdev *ldev, enum led_brightness value) { struct st1202_led *led = cdev_to_st1202_led(ldev); @@ -201,12 +210,16 @@ static int st1202_led_pattern_clear(struct led_classdev *ldev) guard(mutex)(&chip->lock); + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + for (int patt = 0; patt < ST1202_MAX_PATTERNS; patt++) { - ret = st1202_pwm_pattern_write(chip, led->led_num, patt, LED_OFF); + ret = st1202_pwm_pattern_write(chip, led->led_num, patt, ST1202_PATTERN_PWM_FULL); if (ret != 0) return ret; - ret = st1202_duration_pattern_write(chip, patt, ST1202_MILLIS_PATTERN_DUR_MIN); + ret = st1202_write_reg(chip, ST1202_PATTERN_DUR + patt, 0); if (ret != 0) return ret; } @@ -225,13 +238,19 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, if (len > ST1202_MAX_PATTERNS) return -EINVAL; - guard(mutex)(&chip->lock); - for (int patt = 0; patt < len; patt++) { if (pattern[patt].delta_t < ST1202_MILLIS_PATTERN_DUR_MIN || pattern[patt].delta_t > ST1202_MILLIS_PATTERN_DUR_MAX) return -EINVAL; + } + guard(mutex)(&chip->lock); + + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, ST1202_CONFIG_REG_SHFT); + if (ret != 0) + return ret; + + for (int patt = 0; patt < len; patt++) { ret = st1202_pwm_pattern_write(chip, led->led_num, patt, pattern[patt].brightness); if (ret != 0) return ret; @@ -245,6 +264,10 @@ static int st1202_led_pattern_set(struct led_classdev *ldev, if (ret != 0) return ret; + ret = __st1202_channel_set(chip, led->led_num, true); + if (ret != 0) + return ret; + ret = st1202_write_reg(chip, ST1202_CONFIG_REG, (ST1202_CONFIG_REG_PATSR | ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_SHFT)); if (ret != 0) @@ -257,13 +280,19 @@ static int st1202_dt_init(struct st1202_chip *chip) { struct device *dev = &chip->client->dev; struct st1202_led *led; - int err, reg; + int err; + u32 reg; for_each_available_child_of_node_scoped(dev_of_node(dev), child) { err = of_property_read_u32(child, "reg", ®); if (err) return dev_err_probe(dev, err, "Invalid register\n"); + if (reg >= ST1202_MAX_LEDS) + return dev_err_probe(dev, -EINVAL, + "LED reg %u out of range [0, %d]\n", + reg, ST1202_MAX_LEDS - 1); + led = &chip->leds[reg]; led->is_active = true; led->fwnode = of_fwnode_handle(child); @@ -323,11 +352,6 @@ static int st1202_setup(struct st1202_chip *chip) if (ret < 0) return ret; - ret = st1202_write_reg(chip, ST1202_CONFIG_REG, - ST1202_CONFIG_REG_PATS | ST1202_CONFIG_REG_PATSR); - if (ret < 0) - return ret; - return 0; } diff --git a/drivers/leds/trigger/ledtrig-cpu.c b/drivers/leds/trigger/ledtrig-cpu.c index 05848a2fecff6..679323c2ccda0 100644 --- a/drivers/leds/trigger/ledtrig-cpu.c +++ b/drivers/leds/trigger/ledtrig-cpu.c @@ -94,28 +94,32 @@ void ledtrig_cpu(enum cpu_led_event ledevt) } EXPORT_SYMBOL(ledtrig_cpu); -static int ledtrig_cpu_syscore_suspend(void) +static int ledtrig_cpu_syscore_suspend(void *data) { ledtrig_cpu(CPU_LED_STOP); return 0; } -static void ledtrig_cpu_syscore_resume(void) +static void ledtrig_cpu_syscore_resume(void *data) { ledtrig_cpu(CPU_LED_START); } -static void ledtrig_cpu_syscore_shutdown(void) +static void ledtrig_cpu_syscore_shutdown(void *data) { ledtrig_cpu(CPU_LED_HALTED); } -static struct syscore_ops ledtrig_cpu_syscore_ops = { +static const struct syscore_ops ledtrig_cpu_syscore_ops = { .shutdown = ledtrig_cpu_syscore_shutdown, .suspend = ledtrig_cpu_syscore_suspend, .resume = ledtrig_cpu_syscore_resume, }; +static struct syscore ledtrig_cpu_syscore = { + .ops = &ledtrig_cpu_syscore_ops, +}; + static int ledtrig_online_cpu(unsigned int cpu) { ledtrig_cpu(CPU_LED_START); @@ -157,7 +161,7 @@ static int __init ledtrig_cpu_init(void) led_trigger_register_simple(trig->name, &trig->_trig); } - register_syscore_ops(&ledtrig_cpu_syscore_ops); + register_syscore(&ledtrig_cpu_syscore); ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "leds/trigger:starting", ledtrig_online_cpu, ledtrig_prepare_down_cpu); diff --git a/drivers/leds/trigger/ledtrig-gpio.c b/drivers/leds/trigger/ledtrig-gpio.c index 7f6a2352b0acd..3199275fd250b 100644 --- a/drivers/leds/trigger/ledtrig-gpio.c +++ b/drivers/leds/trigger/ledtrig-gpio.c @@ -86,7 +86,8 @@ static int gpio_trig_activate(struct led_classdev *led) * The generic property "trigger-sources" is followed, * and we hope that this is a GPIO. */ - gpio_data->gpiod = gpiod_get_optional(dev, "trigger-sources", GPIOD_IN); + gpio_data->gpiod = gpiod_get_optional(dev, "trigger-sources", + GPIOD_IN | GPIOD_FLAGS_BIT_NONEXCLUSIVE); if (IS_ERR(gpio_data->gpiod)) { ret = PTR_ERR(gpio_data->gpiod); kfree(gpio_data); diff --git a/drivers/leds/uleds.c b/drivers/leds/uleds.c index 48ca73c106bc0..8e250838a15e5 100644 --- a/drivers/leds/uleds.c +++ b/drivers/leds/uleds.c @@ -148,10 +148,13 @@ static ssize_t uleds_read(struct file *file, char __user *buffer, size_t count, } else if (!udev->new_data && (file->f_flags & O_NONBLOCK)) { retval = -EAGAIN; } else if (udev->new_data) { - retval = copy_to_user(buffer, &udev->brightness, - sizeof(udev->brightness)); - udev->new_data = false; - retval = sizeof(udev->brightness); + if (copy_to_user(buffer, &udev->brightness, + sizeof(udev->brightness))) { + retval = -EFAULT; + } else { + udev->new_data = false; + retval = sizeof(udev->brightness); + } } mutex_unlock(&udev->mutex); diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c index b0f09c70f1ff8..5fe47e784d437 100644 --- a/drivers/macintosh/via-pmu.c +++ b/drivers/macintosh/via-pmu.c @@ -2600,7 +2600,7 @@ void pmu_blink(int n) #if defined(CONFIG_SUSPEND) && defined(CONFIG_PPC32) int pmu_sys_suspended; -static int pmu_syscore_suspend(void) +static int pmu_syscore_suspend(void *data) { /* Suspend PMU event interrupts */ pmu_suspend(); @@ -2614,7 +2614,7 @@ static int pmu_syscore_suspend(void) return 0; } -static void pmu_syscore_resume(void) +static void pmu_syscore_resume(void *data) { struct adb_request req; @@ -2634,14 +2634,18 @@ static void pmu_syscore_resume(void) pmu_sys_suspended = 0; } -static struct syscore_ops pmu_syscore_ops = { +static const struct syscore_ops pmu_syscore_ops = { .suspend = pmu_syscore_suspend, .resume = pmu_syscore_resume, }; +static struct syscore pmu_syscore = { + .ops = &pmu_syscore_ops, +}; + static int pmu_syscore_register(void) { - register_syscore_ops(&pmu_syscore_ops); + register_syscore(&pmu_syscore); return 0; } diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index a80cb2c9df6ee..445cd50c1b667 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -82,6 +82,7 @@ struct imx_mu_con_priv { enum imx_mu_chan_type type; struct mbox_chan *chan; struct work_struct txdb_work; + bool shutdown; }; struct imx_mu_priv { @@ -221,6 +222,36 @@ static u32 imx_mu_xcr_rmw(struct imx_mu_priv *priv, enum imx_mu_xcr type, u32 se return val; } +static void imx_mu_xcr_clr_shut(struct imx_mu_priv *priv, struct imx_mu_con_priv *cp, + enum imx_mu_xcr type, u32 clr) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&priv->xcr_lock, flags); + cp->shutdown = true; + + val = imx_mu_read(priv, priv->dcfg->xCR[type]); + val &= ~clr; + imx_mu_write(priv, val, priv->dcfg->xCR[type]); + spin_unlock_irqrestore(&priv->xcr_lock, flags); +} + +static void imx_mu_xcr_set_act(struct imx_mu_priv *priv, struct imx_mu_con_priv *cp, + enum imx_mu_xcr type, u32 set) +{ + unsigned long flags; + u32 val; + + spin_lock_irqsave(&priv->xcr_lock, flags); + if (!cp->shutdown) { + val = imx_mu_read(priv, priv->dcfg->xCR[type]); + val |= set; + imx_mu_write(priv, val, priv->dcfg->xCR[type]); + } + spin_unlock_irqrestore(&priv->xcr_lock, flags); +} + static int imx_mu_generic_tx(struct imx_mu_priv *priv, struct imx_mu_con_priv *cp, void *data) @@ -379,7 +410,7 @@ static int imx_mu_specific_rx(struct imx_mu_priv *priv, struct imx_mu_con_priv * *data++ = imx_mu_read(priv, priv->dcfg->xRR + (i % num_rr) * 4); } - imx_mu_xcr_rmw(priv, IMX_MU_RCR, IMX_MU_xCR_RIEn(priv->dcfg->type, 0), 0); + imx_mu_xcr_set_act(priv, cp, IMX_MU_RCR, IMX_MU_xCR_RIEn(priv->dcfg->type, 0)); mbox_chan_received_data(cp->chan, (void *)priv->msg); return 0; @@ -607,6 +638,7 @@ static int imx_mu_startup(struct mbox_chan *chan) return ret; } + cp->shutdown = false; switch (cp->type) { case IMX_MU_TYPE_RX: imx_mu_xcr_rmw(priv, IMX_MU_RCR, IMX_MU_xCR_RIEn(priv->dcfg->type, cp->idx), 0); @@ -641,13 +673,13 @@ static void imx_mu_shutdown(struct mbox_chan *chan) switch (cp->type) { case IMX_MU_TYPE_TX: - imx_mu_xcr_rmw(priv, IMX_MU_TCR, 0, IMX_MU_xCR_TIEn(priv->dcfg->type, cp->idx)); + imx_mu_xcr_clr_shut(priv, cp, IMX_MU_TCR, IMX_MU_xCR_TIEn(priv->dcfg->type, cp->idx)); break; case IMX_MU_TYPE_RX: - imx_mu_xcr_rmw(priv, IMX_MU_RCR, 0, IMX_MU_xCR_RIEn(priv->dcfg->type, cp->idx)); + imx_mu_xcr_clr_shut(priv, cp, IMX_MU_RCR, IMX_MU_xCR_RIEn(priv->dcfg->type, cp->idx)); break; case IMX_MU_TYPE_RXDB: - imx_mu_xcr_rmw(priv, IMX_MU_GIER, 0, IMX_MU_xCR_GIEn(priv->dcfg->type, cp->idx)); + imx_mu_xcr_clr_shut(priv, cp, IMX_MU_GIER, IMX_MU_xCR_GIEn(priv->dcfg->type, cp->idx)); break; case IMX_MU_TYPE_RST: imx_mu_xcr_rmw(priv, IMX_MU_CR, IMX_MU_xCR_RST(priv->dcfg->type), 0); @@ -936,38 +968,36 @@ static int imx_mu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - ret = devm_mbox_controller_register(dev, &priv->mbox); - if (ret) + ret = devm_pm_runtime_enable(dev); + if (ret < 0) goto disable_clk; - of_platform_populate(dev->of_node, NULL, NULL, dev); - - pm_runtime_enable(dev); - ret = pm_runtime_resume_and_get(dev); if (ret < 0) - goto disable_runtime_pm; + goto disable_clk; ret = pm_runtime_put_sync(dev); if (ret < 0) - goto disable_runtime_pm; + goto disable_clk; clk_disable_unprepare(priv->clk); + ret = devm_mbox_controller_register(dev, &priv->mbox); + if (ret) + goto err_out; + + devm_of_platform_populate(dev); + return 0; -disable_runtime_pm: - pm_runtime_disable(dev); disable_clk: clk_disable_unprepare(priv->clk); +err_out: return ret; } static void imx_mu_remove(struct platform_device *pdev) { - struct imx_mu_priv *priv = platform_get_drvdata(pdev); - - pm_runtime_disable(priv->dev); } static const struct imx_mu_dcfg imx_mu_cfg_imx6sx = { diff --git a/drivers/mailbox/mailbox-mchp-ipc-sbi.c b/drivers/mailbox/mailbox-mchp-ipc-sbi.c index b87bf2fb4b9b9..f081f8a9bcf8e 100644 --- a/drivers/mailbox/mailbox-mchp-ipc-sbi.c +++ b/drivers/mailbox/mailbox-mchp-ipc-sbi.c @@ -378,6 +378,8 @@ static int mchp_ipc_get_cluster_aggr_irq(struct mchp_ipc_sbi_mbox *ipc) for_each_online_cpu(cpuid) { hartid = cpuid_to_hartid_map(cpuid); irq_name = devm_kasprintf(ipc->dev, GFP_KERNEL, "hart-%lu", hartid); + if (!irq_name) + return -ENOMEM; ret = platform_get_irq_byname_optional(pdev, irq_name); if (ret <= 0) continue; diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c index 60866e8417c9b..0f2019aadc216 100644 --- a/drivers/mailbox/mailbox.c +++ b/drivers/mailbox/mailbox.c @@ -100,8 +100,10 @@ static void tx_tick(struct mbox_chan *chan, int r) if (chan->cl->tx_done) chan->cl->tx_done(chan->cl, mssg, r); - if (r != -ETIME && chan->cl->tx_block) + if (r != -ETIME && chan->cl->tx_block) { + chan->tx_status = r; complete(&chan->tx_complete); + } } static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer) @@ -270,6 +272,8 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg) if (ret == 0) { t = -ETIME; tx_tick(chan, t); + } else if (chan->tx_status < 0) { + t = chan->tx_status; } } diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 713022aed2e2f..5189791c2790d 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -93,12 +93,11 @@ struct pcc_chan_reg { * @plat_irq: platform interrupt * @type: PCC subspace type * @plat_irq_flags: platform interrupt flags - * @chan_in_use: this flag is used just to check if the interrupt needs - * handling when it is shared. Since only one transfer can occur - * at a time and mailbox takes care of locking, this flag can be - * accessed without a lock. Note: the type only support the - * communication from OSPM to Platform, like type3, use it, and - * other types completely ignore it. + * @chan_in_use: lockless flag used by type 3 initiator subspaces to filter + * platform interrupts. Only one transfer can occur at a time, but + * the interrupt handler may sample the flag on another CPU, so all + * accesses must use READ_ONCE() or WRITE_ONCE(). Other subspace + * types do not test it. */ struct pcc_chan_info { struct pcc_mbox_chan chan; @@ -322,8 +321,13 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) if (pcc_chan_reg_read_modify_write(&pchan->plat_irq_ack)) return IRQ_NONE; + /* + * Initiator subspaces use this flag to filter shared interrupts. Use + * READ_ONCE() to sample the lockless flag written by pcc_send_data() + * on another CPU. + */ if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_MASTER_SUBSPACE && - !pchan->chan_in_use) + !READ_ONCE(pchan->chan_in_use)) return IRQ_NONE; if (!pcc_mbox_cmd_complete_check(pchan)) @@ -333,12 +337,12 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) return IRQ_NONE; /* - * Clear this flag after updating interrupt ack register and just - * before mbox_chan_received_data() which might call pcc_send_data() - * where the flag is set again to start new transfer. This is - * required to avoid any possible race in updatation of this flag. + * Clear this flag after updating the interrupt ack register and before + * notifying the client and mailbox core. mbox_chan_txdone() may submit + * the next queued transfer and set the flag again. Use WRITE_ONCE() for + * the lockless update observed by the send and interrupt paths. */ - pchan->chan_in_use = false; + WRITE_ONCE(pchan->chan_in_use, false); mbox_chan_received_data(chan, NULL); pcc_chan_acknowledge(pchan); @@ -437,9 +441,18 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) if (ret) return ret; + /* + * Set chan_in_use before ringing the doorbell so a fast completion + * interrupt is not mistaken for a shared interrupt from another + * subspace. Use WRITE_ONCE() for the lockless flag update. The + * ordered I/O accessor used to ring the doorbell orders this store + * before the platform is notified. + */ + if (pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, true); ret = pcc_chan_reg_read_modify_write(&pchan->db); - if (!ret && pchan->plat_irq > 0) - pchan->chan_in_use = true; + if (ret && pchan->plat_irq > 0) + WRITE_ONCE(pchan->chan_in_use, false); return ret; } diff --git a/drivers/mailbox/qcom-cpucp-mbox.c b/drivers/mailbox/qcom-cpucp-mbox.c index 44f4ed15f8184..57f731418fafb 100644 --- a/drivers/mailbox/qcom-cpucp-mbox.c +++ b/drivers/mailbox/qcom-cpucp-mbox.c @@ -56,14 +56,25 @@ static irqreturn_t qcom_cpucp_mbox_irq_fn(int irq, void *data) for_each_set_bit(i, (unsigned long *)&status, APSS_CPUCP_IPC_CHAN_SUPPORTED) { u32 val = readl(cpucp->rx_base + APSS_CPUCP_RX_MBOX_CMD(i) + APSS_CPUCP_MBOX_CMD_OFF); struct mbox_chan *chan = &cpucp->chans[i]; + struct mbox_client *cl; unsigned long flags; - /* Provide mutual exclusion with changes to chan->cl */ + /* + * Provide mutual exclusion with changes to chan->cl. + * Save cl locally and clear the HW interrupt inside the lock, + * then invoke mbox_chan_received_data() outside the lock to + * avoid a PREEMPT_RT self-deadlock: mbox_chan_received_data() + * can call back into mbox_send_message() via scmi_rx_callback() + * -> mailbox_clear_channel(), which re-acquires chan->lock + * (converted to an rt_spinlock under PREEMPT_RT). + */ spin_lock_irqsave(&chan->lock, flags); - if (chan->cl) - mbox_chan_received_data(chan, &val); + cl = chan->cl; writeq(BIT(i), cpucp->rx_base + APSS_CPUCP_RX_MBOX_CLEAR); spin_unlock_irqrestore(&chan->lock, flags); + + if (cl) + mbox_chan_received_data(chan, &val); } return IRQ_HANDLED; @@ -99,6 +110,14 @@ static int qcom_cpucp_mbox_send_data(struct mbox_chan *chan, void *data) unsigned long chan_id = channel_number(chan); u32 *val = data; + /* + * mailbox_clear_channel() calls mbox_send_message() with NULL data to + * signal the remote side that the channel has been cleared. Nothing + * needs to be written to the TX register in that case, so just return. + */ + if (!val) + return 0; + writel(*val, cpucp->tx_base + APSS_CPUCP_TX_MBOX_CMD(chan_id) + APSS_CPUCP_MBOX_CMD_OFF); return 0; diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index d957d989c0cee..dead907aa7a28 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -167,7 +167,7 @@ static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, { struct qcom_ipcc *ipcc = to_qcom_ipcc(mbox); struct qcom_ipcc_chan_info *mchan; - struct mbox_chan *chan; + struct mbox_chan *chan, *free_chan = NULL; struct device *dev; int chan_id; @@ -180,16 +180,21 @@ static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, chan = &ipcc->chans[chan_id]; mchan = chan->con_priv; - if (!mchan) - break; - else if (mchan->client_id == ph->args[0] && - mchan->signal_id == ph->args[1]) + if (!mchan) { + /* Keep scanning past holes to reject duplicate channel requests. */ + if (!free_chan) + free_chan = chan; + } else if (mchan->client_id == ph->args[0] && + mchan->signal_id == ph->args[1]) { return ERR_PTR(-EBUSY); + } } - if (chan_id >= mbox->num_chans) + if (!free_chan) return ERR_PTR(-EBUSY); + chan = free_chan; + mchan = devm_kzalloc(dev, sizeof(*mchan), GFP_KERNEL); if (!mchan) return ERR_PTR(-ENOMEM); diff --git a/drivers/mailbox/riscv-sbi-mpxy-mbox.c b/drivers/mailbox/riscv-sbi-mpxy-mbox.c index 7c9c006b7244a..714f7fb97a2fd 100644 --- a/drivers/mailbox/riscv-sbi-mpxy-mbox.c +++ b/drivers/mailbox/riscv-sbi-mpxy-mbox.c @@ -314,8 +314,13 @@ static int mpxy_get_notifications(u32 channel_id, channel_id, 0, 0, 0, 0, 0); if (sret.error) goto err_put_cpu; + if (sret.value < 0 || mpxy_shmem_size < sizeof(*notif_data) || + sret.value > mpxy_shmem_size - sizeof(*notif_data)) { + put_cpu(); + return -EOVERFLOW; + } - memcpy(notif_data, mpxy->shmem, sret.value + 16); + memcpy(notif_data, mpxy->shmem, sret.value + sizeof(*notif_data)); *events_data_len = sret.value; err_put_cpu: @@ -480,11 +485,14 @@ static void mpxy_mbox_peek_rpmi_data(struct mbox_chan *chan, struct rpmi_mbox_message msg; unsigned long pos = 0; - while (pos < events_data_len && (events_data_len - pos) <= sizeof(*event)) { + while (events_data_len - pos >= sizeof(*event)) { event = (struct rpmi_notification_event *)(notif->events_data + pos); msg.type = RPMI_MBOX_MSG_TYPE_NOTIFICATION_EVENT; msg.notif.event_datalen = le16_to_cpu(event->event_datalen); + if (msg.notif.event_datalen > + events_data_len - pos - sizeof(*event)) + break; msg.notif.event_id = event->event_id; msg.notif.event_data = event->event_data; msg.error = 0; diff --git a/drivers/mailbox/rockchip-mailbox.c b/drivers/mailbox/rockchip-mailbox.c index 4d966cb2ed036..5775362de508a 100644 --- a/drivers/mailbox/rockchip-mailbox.c +++ b/drivers/mailbox/rockchip-mailbox.c @@ -201,19 +201,10 @@ static int rockchip_mbox_probe(struct platform_device *pdev) /* Each channel has two buffers for A2B and B2A */ mb->buf_size = (size_t)resource_size(res) / (drv_data->num_chans * 2); - mb->pclk = devm_clk_get(&pdev->dev, "pclk_mailbox"); - if (IS_ERR(mb->pclk)) { - ret = PTR_ERR(mb->pclk); - dev_err(&pdev->dev, "failed to get pclk_mailbox clock: %d\n", - ret); - return ret; - } - - ret = clk_prepare_enable(mb->pclk); - if (ret) { - dev_err(&pdev->dev, "failed to enable pclk: %d\n", ret); - return ret; - } + mb->pclk = devm_clk_get_enabled(&pdev->dev, "pclk_mailbox"); + if (IS_ERR(mb->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(mb->pclk), + "failed to get and enable pclk_mailbox clock\n"); for (i = 0; i < mb->mbox.num_chans; i++) { irq = platform_get_irq(pdev, i); diff --git a/drivers/md/dm-bufio.c b/drivers/md/dm-bufio.c index b7cbe58740947..59e5d8aa42b71 100644 --- a/drivers/md/dm-bufio.c +++ b/drivers/md/dm-bufio.c @@ -1282,11 +1282,11 @@ static void free_buffer(struct dm_buffer *b) * dm-io completion routine. It just calls b->bio.bi_end_io, pretending * that the request was handled directly with bio interface. */ -static void dmio_complete(unsigned long error, void *context) +static void dmio_complete(unsigned long error, unsigned long unsup, void *context) { struct dm_buffer *b = context; - b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0); + b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : unlikely(unsup != 0) ? BLK_STS_NOTSUPP : 0); } static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, @@ -1314,7 +1314,7 @@ static void use_dmio(struct dm_buffer *b, enum req_op op, sector_t sector, io_req.mem.ptr.vma = (char *)b->data + offset; } - r = dm_io(&io_req, 1, ®ion, NULL, ioprio); + r = dm_io(&io_req, 1, ®ion, NULL, NULL, ioprio); if (unlikely(r)) b->end_io(b, errno_to_blk_status(r)); } @@ -2197,7 +2197,7 @@ int dm_bufio_issue_flush(struct dm_bufio_client *c) if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; - return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_flush); @@ -2223,7 +2223,7 @@ int dm_bufio_issue_discard(struct dm_bufio_client *c, sector_t block, sector_t c if (WARN_ON_ONCE(dm_bufio_in_request())) return -EINVAL; /* discards are optional */ - return dm_io(&io_req, 1, &io_reg, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &io_reg, NULL, NULL, IOPRIO_DEFAULT); } EXPORT_SYMBOL_GPL(dm_bufio_issue_discard); diff --git a/drivers/md/dm-era-target.c b/drivers/md/dm-era-target.c index 883d1549cfc0e..6f87e2c4e7da6 100644 --- a/drivers/md/dm-era-target.c +++ b/drivers/md/dm-era-target.c @@ -1034,6 +1034,7 @@ static int metadata_checkpoint(struct era_metadata *md) static int metadata_take_snap(struct era_metadata *md) { int r, inc; + dm_block_t location; struct dm_block *clone; if (md->metadata_snap != SUPERBLOCK_LOCATION) { @@ -1071,7 +1072,9 @@ static int metadata_take_snap(struct era_metadata *md) r = dm_sm_inc_block(md->sm, md->writeset_tree_root); if (r) { DMERR("%s: couldn't inc writeset tree root", __func__); + location = dm_block_location(clone); dm_tm_unlock(md->tm, clone); + dm_sm_dec_block(md->sm, location); return r; } @@ -1079,7 +1082,9 @@ static int metadata_take_snap(struct era_metadata *md) if (r) { DMERR("%s: couldn't inc era tree root", __func__); dm_sm_dec_block(md->sm, md->writeset_tree_root); + location = dm_block_location(clone); dm_tm_unlock(md->tm, clone); + dm_sm_dec_block(md->sm, location); return r; } diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index cd602b39481d6..6536143f89122 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -65,6 +65,7 @@ #define SB_VERSION_4 4 #define SB_VERSION_5 5 #define SB_VERSION_6 6 +#define SB_VERSION_7 7 #define SB_SECTORS 8 #define MAX_SECTORS_PER_BLOCK 8 @@ -90,6 +91,7 @@ struct superblock { #define SB_FLAG_FIXED_PADDING 0x8 #define SB_FLAG_FIXED_HMAC 0x10 #define SB_FLAG_INLINE 0x20 +#define SB_FLAG_DISCARD_KEYED 0x40 #define JOURNAL_ENTRY_ROUNDUP 8 @@ -276,6 +278,7 @@ struct dm_integrity_c { bool recalculate_flag; bool reset_recalculate_flag; bool discard; + bool discard_keyed; bool fix_padding; bool fix_hmac; bool legacy_recalculate; @@ -482,7 +485,9 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) static void sb_set_version(struct dm_integrity_c *ic) { - if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) + if (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED)) + ic->sb->version = SB_VERSION_7; + else if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) ic->sb->version = SB_VERSION_6; else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ic->sb->version = SB_VERSION_5; @@ -561,7 +566,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) } } - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) return r; @@ -1034,12 +1039,14 @@ static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned in return crypt_journal(ic, encrypt, section, n_sections, comp); } -static void complete_journal_io(unsigned long error, void *context) +static void complete_journal_io(unsigned long error, unsigned long unsup, void *context) { struct journal_completion *comp = context; if (unlikely(error != 0)) dm_integrity_io_error(comp->ic, "writing journal", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(comp->ic, "writing journal", -EOPNOTSUPP); complete_journal_op(comp); } @@ -1054,7 +1061,7 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, if (unlikely(dm_integrity_failed(ic))) { if (comp) - complete_journal_io(-1UL, comp); + complete_journal_io(-1UL, -1UL, comp); return; } @@ -1079,13 +1086,13 @@ static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? "reading journal" : "writing journal", r); if (comp) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); - complete_journal_io(-1UL, comp); + complete_journal_io(-1UL, -1UL, comp); } } } @@ -1176,7 +1183,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1)); if (unlikely(dm_integrity_failed(ic))) { - fn(-1UL, data); + fn(-1UL, -1UL, data); return; } @@ -1196,10 +1203,10 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, u io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); - fn(-1UL, data); + fn(-1UL, -1UL, data); } } @@ -1413,7 +1420,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se { unsigned int hash_offset = 0; unsigned char mismatch_hash = 0; - unsigned char mismatch_filler = !ic->discard; + unsigned char mismatch_filler = !ic->discard || ic->discard_keyed; do { unsigned char *data, *dp; @@ -1465,7 +1472,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se } hash_offset = 0; mismatch_hash = 0; - mismatch_filler = !ic->discard; + mismatch_filler = !ic->discard || ic->discard_keyed; } } } @@ -1492,12 +1499,14 @@ struct flush_request { struct completion comp; }; -static void flush_notify(unsigned long error, void *fr_) +static void flush_notify(unsigned long error, unsigned long unsup, void *fr_) { struct flush_request *fr = fr_; if (unlikely(error != 0)) dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(fr->ic, "flushing disk cache", -EOPNOTSUPP); complete(&fr->comp); } @@ -1520,7 +1529,7 @@ static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_dat fr.io_reg.count = 0; fr.ic = ic; init_completion(&fr.comp); - r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, NULL, IOPRIO_DEFAULT); BUG_ON(r); } @@ -1641,7 +1650,8 @@ static void integrity_end_io(struct bio *bio) } static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector, - const char *data, unsigned offset, char *result) + const char *data, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); SHASH_DESC_ON_STACK(req, ic->internal_shash); @@ -1670,10 +1680,12 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t goto failed; } - r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT); - if (unlikely(r < 0)) { - dm_integrity_io_error(ic, "crypto_shash_update", r); - goto failed; + if (likely(len)) { + r = crypto_shash_update(req, data + offset, len); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto failed; + } } r = crypto_shash_final(req, result); @@ -1694,7 +1706,8 @@ static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t } static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req, - sector_t sector, struct page *page, unsigned offset, char *result) + sector_t sector, struct page *page, unsigned offset, + unsigned int len, char *result) { __le64 sector_le = cpu_to_le64(sector); struct ahash_request *req; @@ -1703,6 +1716,7 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah int r; unsigned int digest_size; unsigned int nbytes = 0; + unsigned int nents = 1 + (len ? 1 : 0); might_sleep(); @@ -1716,12 +1730,12 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait); if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { - sg_init_table(sg, 3); + sg_init_table(sg, nents + 1); sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE); nbytes += SALT_SIZE; s++; } else { - sg_init_table(sg, 2); + sg_init_table(sg, nents); } if (likely(!is_vmalloc_addr(§or_le))) { @@ -1734,8 +1748,10 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah nbytes += sizeof(sector_le); s++; - sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset); - nbytes += ic->sectors_per_block << SECTOR_SHIFT; + if (likely(len)) { + sg_set_page(s, page, len, offset); + nbytes += len; + } ahash_request_set_crypt(req, sg, result, nbytes); @@ -1758,11 +1774,41 @@ static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ah static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, sector_t sector, const char *data, unsigned offset, char *result) +{ + unsigned int len = ic->sectors_per_block << SECTOR_SHIFT; + + if (likely(ic->internal_shash != NULL)) + integrity_sector_checksum_shash(ic, sector, data, offset, len, result); + else + integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, + offset, len, result); +} + +/* + * Authenticated marker for a discarded block: HMAC_key(salt || sector), with + * no data payload. Because a real data tag's input always covers a full + * block, its length differs from this marker's, so the two can never + * collide structurally, regardless of block content. + */ +static void integrity_discard_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + sector_t sector, char *result) { if (likely(ic->internal_shash != NULL)) - integrity_sector_checksum_shash(ic, sector, data, offset, result); + integrity_sector_checksum_shash(ic, sector, NULL, 0, 0, result); else - integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result); + integrity_sector_checksum_ahash(ic, ahash_req, sector, NULL, 0, 0, result); +} + +static void integrity_discard_fill_tags(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + unsigned char *checksums, sector_t *sector, + unsigned int blocks) +{ + unsigned int i; + + for (i = 0; i < blocks; i++) { + integrity_discard_checksum(ic, ahash_req, *sector, checksums + i * ic->tag_size); + *sector += ic->sectors_per_block; + } } static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p) @@ -1791,6 +1837,29 @@ static void *integrity_identity(struct dm_integrity_c *ic, void *data) return virt_to_page(data); } +static int integrity_recheck_verify_tag(struct dm_integrity_io *dio, char *checksum, + char *on_disk_tag, sector_t logical_sector) +{ + struct dm_integrity_c *ic = dio->ic; + int r; + + if (!ic->discard_keyed) + return dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_CMP); + + r = dm_integrity_rw_tag(ic, on_disk_tag, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_READ); + if (unlikely(r)) + return r; + + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + if (unlikely(r)) { + integrity_discard_checksum(ic, &dio->ahash_req, logical_sector, checksum); + r = crypto_memneq(on_disk_tag, checksum, ic->tag_size); + } + return r; +} + static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) { struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); @@ -1816,6 +1885,7 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks char *mem; char *buffer = page_to_virt(page); unsigned int buffer_offset; + char on_disk_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; int r; struct dm_io_request io_req; struct dm_io_region io_loc; @@ -1836,15 +1906,15 @@ static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checks buffer_offset = (sector - io_loc.sector) << SECTOR_SHIFT; io_loc.count = round_up(io_loc.count, alignment); - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dio->bi_status = errno_to_blk_status(r); goto free_ret; } integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); - r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, - &dio->metadata_offset, ic->tag_size, TAG_CMP); + r = integrity_recheck_verify_tag(dio, checksum, on_disk_tag, + logical_sector); if (r) { if (r > 0) { DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", @@ -1909,14 +1979,19 @@ static void integrity_metadata(struct work_struct *w) if (unlikely(dio->op == REQ_OP_DISCARD)) { unsigned int bi_size = dio->bio_details.bi_iter.bi_size; unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE; - unsigned int max_blocks = max_size / ic->tag_size; + unsigned int max_blocks = (max_size - extra_space) / ic->tag_size; + sector_t sector = dio->range.logical_sector; - memset(checksums, DISCARD_FILLER, max_size); + if (!ic->discard_keyed) + memset(checksums, DISCARD_FILLER, max_size); while (bi_size) { unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); this_step_blocks = min(this_step_blocks, max_blocks); + if (ic->discard_keyed) + integrity_discard_fill_tags(ic, &dio->ahash_req, checksums, + §or, this_step_blocks); r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, this_step_blocks * ic->tag_size, TAG_WRITE); if (unlikely(r)) { @@ -2889,7 +2964,7 @@ static void integrity_commit(struct work_struct *w) } } -static void complete_copy_from_journal(unsigned long error, void *context) +static void complete_copy_from_journal(unsigned long error, unsigned long unsup, void *context) { struct journal_io *io = context; struct journal_completion *comp = io->comp; @@ -2899,6 +2974,8 @@ static void complete_copy_from_journal(unsigned long error, void *context) mempool_free(io, &ic->journal_io_mempool); if (unlikely(error != 0)) dm_integrity_io_error(ic, "copying from journal", -EIO); + else if (unlikely(unsup != 0)) + dm_integrity_io_error(ic, "copying from journal", -EOPNOTSUPP); complete_journal_op(comp); } @@ -3214,7 +3291,7 @@ static void integrity_recalc(struct work_struct *w) io_loc.sector = get_data_sector(ic, area, offset); io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; @@ -3791,6 +3868,8 @@ static void dm_integrity_resume(struct dm_target *ti) ic->wrote_to_journal = false; flags = ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING); + if (ic->discard_keyed) + flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); r = sync_rw_sb(ic, REQ_OP_READ); if (r) dm_integrity_io_error(ic, "reading superblock", r); @@ -3938,7 +4017,8 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, arg_count += ic->sectors_per_block != 1; arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); arg_count += ic->reset_recalculate_flag; - arg_count += ic->discard; + arg_count += ic->discard && !ic->discard_keyed; + arg_count += ic->discard_keyed; arg_count += ic->mode != 'I'; /* interleave_sectors */ arg_count += ic->mode == 'J'; /* journal_sectors */ arg_count += ic->mode == 'J'; /* journal_watermark */ @@ -3961,8 +4041,10 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(" recalculate"); if (ic->reset_recalculate_flag) DMEMIT(" reset_recalculate"); - if (ic->discard) + if (ic->discard && !ic->discard_keyed) DMEMIT(" allow_discards"); + if (ic->discard_keyed) + DMEMIT(" allow_discards_keyed"); if (ic->mode != 'I') DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); @@ -4012,6 +4094,7 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? 'y' : 'n'); DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); + DMEMIT(",allow_discards_keyed=%c", ic->discard_keyed ? 'y' : 'n'); DMEMIT(",fix_padding=%c", ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); DMEMIT(",fix_hmac=%c", @@ -4173,6 +4256,9 @@ static int initialize_superblock(struct dm_integrity_c *ic, get_random_bytes(ic->sb->salt, SALT_SIZE); } + if (ic->discard_keyed) + ic->sb->flags |= cpu_to_le32(SB_FLAG_DISCARD_KEYED); + if (!ic->meta_dev) { if (ic->fix_padding) ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); @@ -4832,6 +4918,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ic->reset_recalculate_flag = true; } else if (!strcmp(opt_string, "allow_discards")) { ic->discard = true; + } else if (!strcmp(opt_string, "allow_discards_keyed")) { + ic->discard = true; + ic->discard_keyed = true; } else if (!strcmp(opt_string, "fix_padding")) { ic->fix_padding = true; } else if (!strcmp(opt_string, "fix_hmac")) { @@ -4960,6 +5049,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv ti->error = "Discard can be only used with internal hash"; goto bad; } + if (ic->discard_keyed && !ic->internal_hash_alg.key) { + r = -EINVAL; + ti->error = "Keyed discard can only be used with keyed internal hash"; + goto bad; + } ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); ic->autocommit_msec = sync_msec; @@ -5078,7 +5172,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv should_write_sb = true; } - if (!ic->sb->version || ic->sb->version > SB_VERSION_6) { + if (!ic->sb->version || ic->sb->version > SB_VERSION_7) { r = -EINVAL; ti->error = "Unknown version"; goto bad; @@ -5126,6 +5220,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv goto bad; } } + if (!ic->discard_keyed && (ic->sb->flags & cpu_to_le32(SB_FLAG_DISCARD_KEYED))) { + r = -EINVAL; + ti->error = "Keyed discard cannot be disabled once enabled"; + goto bad; + } if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { r = -EINVAL; ti->error = "Journal mac mismatch"; @@ -5426,7 +5525,7 @@ static void dm_integrity_dtr(struct dm_target *ti) static struct target_type integrity_target = { .name = "integrity", - .version = {1, 14, 0}, + .version = {1, 15, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, diff --git a/drivers/md/dm-io.c b/drivers/md/dm-io.c index c37668790577f..c6684e63ceb07 100644 --- a/drivers/md/dm-io.c +++ b/drivers/md/dm-io.c @@ -33,6 +33,7 @@ struct dm_io_client { */ struct io { unsigned long error_bits; + unsigned long unsup_bits; atomic_t count; struct dm_io_client *client; io_notify_fn callback; @@ -119,6 +120,7 @@ static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io, static void complete_io(struct io *io) { unsigned long error_bits = io->error_bits; + unsigned long unsup_bits = io->unsup_bits; io_notify_fn fn = io->callback; void *context = io->context; @@ -127,13 +129,17 @@ static void complete_io(struct io *io) io->vma_invalidate_size); mempool_free(io, &io->client->pool); - fn(error_bits, context); + fn(error_bits, unsup_bits, context); } static void dec_count(struct io *io, unsigned int region, blk_status_t error) { - if (error) - set_bit(region, &io->error_bits); + if (unlikely(error)) { + if (error == BLK_STS_NOTSUPP || error == BLK_STS_INVAL) + set_bit(region, &io->unsup_bits); + else + set_bit(region, &io->error_bits); + } if (atomic_dec_and_test(&io->count)) complete_io(io); @@ -170,12 +176,11 @@ struct dpages { struct page **p, unsigned long *len, unsigned int *offset); void (*next_page)(struct dpages *dp); - union { - unsigned int context_u; - struct bvec_iter context_bi; - }; + unsigned int context_u; void *context_ptr; + struct bio *orig_bio; + void *vma_invalidate_address; unsigned long vma_invalidate_size; }; @@ -210,44 +215,6 @@ static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned int o dp->context_ptr = pl; } -/* - * Functions for getting the pages from a bvec. - */ -static void bio_get_page(struct dpages *dp, struct page **p, - unsigned long *len, unsigned int *offset) -{ - struct bio_vec bvec = bvec_iter_bvec((struct bio_vec *)dp->context_ptr, - dp->context_bi); - - *p = bvec.bv_page; - *len = bvec.bv_len; - *offset = bvec.bv_offset; - - /* avoid figuring it out again in bio_next_page() */ - dp->context_bi.bi_sector = (sector_t)bvec.bv_len; -} - -static void bio_next_page(struct dpages *dp) -{ - unsigned int len = (unsigned int)dp->context_bi.bi_sector; - - bvec_iter_advance((struct bio_vec *)dp->context_ptr, - &dp->context_bi, len); -} - -static void bio_dp_init(struct dpages *dp, struct bio *bio) -{ - dp->get_page = bio_get_page; - dp->next_page = bio_next_page; - - /* - * We just use bvec iterator to retrieve pages, so it is ok to - * access the bvec table directly here - */ - dp->context_ptr = bio->bi_io_vec; - dp->context_bi = bio->bi_iter; -} - /* * Functions for getting the pages from a VMA. */ @@ -332,6 +299,21 @@ static void do_region(const blk_opf_t opf, unsigned int region, return; } + if (dp->orig_bio) { + bio = bio_alloc_clone(where->bdev, dp->orig_bio, GFP_NOIO, + &io->client->bios); + bio->bi_iter.bi_sector = where->sector; + bio->bi_iter.bi_size = where->count << SECTOR_SHIFT; + bio->bi_opf = opf; + bio->bi_end_io = endio; + bio->bi_ioprio = ioprio; + store_io_and_region_in_bio(bio, io, region); + + atomic_inc(&io->count); + submit_bio(bio); + return; + } + /* * where->count may be zero if op holds a flush and we need to * send a zero-sized flush. @@ -418,6 +400,7 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions, io = mempool_alloc(&client->pool, GFP_NOIO); io->error_bits = 0; + io->unsup_bits = 0; atomic_set(&io->count, 1); /* see dispatch_io() */ io->client = client; io->callback = fn; @@ -431,20 +414,23 @@ static void async_io(struct dm_io_client *client, unsigned int num_regions, struct sync_io { unsigned long error_bits; + unsigned long unsup_bits; struct completion wait; }; -static void sync_io_complete(unsigned long error, void *context) +static void sync_io_complete(unsigned long error, unsigned long unsup, void *context) { struct sync_io *sio = context; sio->error_bits = error; + sio->unsup_bits = unsup; complete(&sio->wait); } static int sync_io(struct dm_io_client *client, unsigned int num_regions, struct dm_io_region *where, blk_opf_t opf, struct dpages *dp, - unsigned long *error_bits, unsigned short ioprio) + unsigned long *error_bits, unsigned long *unsup_bits, + unsigned short ioprio) { struct sync_io sio; @@ -457,8 +443,10 @@ static int sync_io(struct dm_io_client *client, unsigned int num_regions, if (error_bits) *error_bits = sio.error_bits; + if (unsup_bits) + *unsup_bits = sio.unsup_bits; - return sio.error_bits ? -EIO : 0; + return sio.error_bits ? -EIO : sio.unsup_bits ? -EOPNOTSUPP : 0; } static int dp_init(struct dm_io_request *io_req, struct dpages *dp, @@ -468,6 +456,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, dp->vma_invalidate_address = NULL; dp->vma_invalidate_size = 0; + dp->orig_bio = NULL; switch (io_req->mem.type) { case DM_IO_PAGE_LIST: @@ -475,7 +464,11 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, break; case DM_IO_BIO: - bio_dp_init(dp, io_req->mem.ptr.bio); + /* + * The destination bios clone this bio's biovec directly, so + * there are no per-page accessors to set up here. + */ + dp->orig_bio = io_req->mem.ptr.bio; break; case DM_IO_VMA: @@ -500,7 +493,7 @@ static int dp_init(struct dm_io_request *io_req, struct dpages *dp, int dm_io(struct dm_io_request *io_req, unsigned int num_regions, struct dm_io_region *where, unsigned long *sync_error_bits, - unsigned short ioprio) + unsigned long *sync_unsup_bits, unsigned short ioprio) { int r; struct dpages dp; @@ -516,7 +509,8 @@ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, - io_req->bi_opf, &dp, sync_error_bits, ioprio); + io_req->bi_opf, &dp, sync_error_bits, + sync_unsup_bits, ioprio); async_io(io_req->client, num_regions, where, io_req->bi_opf, &dp, io_req->notify.fn, io_req->notify.context, ioprio); diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index cec9a60227b6f..7eecce8711ce1 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -517,16 +517,16 @@ static int run_complete_job(struct kcopyd_job *job) return 0; } -static void complete_io(unsigned long error, void *context) +static void complete_io(unsigned long error, unsigned long unsup, void *context) { struct kcopyd_job *job = context; struct dm_kcopyd_client *kc = job->kc; io_job_finish(kc->throttle); - if (error) { + if (unlikely((error | unsup) != 0)) { if (op_is_write(job->op)) - job->write_err |= error; + job->write_err |= error | unsup; else job->read_err = 1; @@ -578,9 +578,9 @@ static int run_io_job(struct kcopyd_job *job) io_job_start(job->kc->throttle); if (job->op == REQ_OP_READ) - r = dm_io(&io_req, 1, &job->source, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &job->source, NULL, NULL, IOPRIO_DEFAULT); else - r = dm_io(&io_req, job->num_dests, job->dests, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, job->num_dests, job->dests, NULL, NULL, IOPRIO_DEFAULT); return r; } diff --git a/drivers/md/dm-log-userspace-base.c b/drivers/md/dm-log-userspace-base.c index 607436804a8b2..40e21bec47926 100644 --- a/drivers/md/dm-log-userspace-base.c +++ b/drivers/md/dm-log-userspace-base.c @@ -139,6 +139,7 @@ static int build_constructor_string(struct dm_target *ti, str_size += strlen(argv[i]) + 1; /* +1 for space between args */ str_size += 20; /* Max number of chars in a printed u64 number */ + str_size++; /* For NUL-terminator */ str = kzalloc(str_size, GFP_KERNEL); if (!str) { diff --git a/drivers/md/dm-log.c b/drivers/md/dm-log.c index c8a1f9b6385fc..38b686a7a4876 100644 --- a/drivers/md/dm-log.c +++ b/drivers/md/dm-log.c @@ -300,7 +300,7 @@ static int rw_header(struct log_c *lc, enum req_op op) { lc->io_req.bi_opf = op; - return dm_io(&lc->io_req, 1, &lc->header_location, NULL, IOPRIO_DEFAULT); + return dm_io(&lc->io_req, 1, &lc->header_location, NULL, NULL, IOPRIO_DEFAULT); } static int flush_header(struct log_c *lc) @@ -313,7 +313,7 @@ static int flush_header(struct log_c *lc) lc->io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH; - return dm_io(&lc->io_req, 1, &null_location, NULL, IOPRIO_DEFAULT); + return dm_io(&lc->io_req, 1, &null_location, NULL, NULL, IOPRIO_DEFAULT); } static int read_header(struct log_c *log) diff --git a/drivers/md/dm-pcache/cache.c b/drivers/md/dm-pcache/cache.c index 534bf07b794f1..8635fbf780aa0 100644 --- a/drivers/md/dm-pcache/cache.c +++ b/drivers/md/dm-pcache/cache.c @@ -119,6 +119,10 @@ int cache_pos_decode(struct pcache_cache *cache, return -EIO; pos->cache_seg = &cache->segments[latest.cache_seg_id]; + + if (latest.seg_off >= pos->cache_seg->segment.data_size) + return -EIO; + pos->seg_off = latest.seg_off; *seq = latest.header.seq; *index = (latest_addr - pos_onmedia); @@ -193,6 +197,7 @@ static int cache_tail_init(struct pcache_cache *cache) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); bool new_cache = !(cache->cache_info.flags & PCACHE_CACHE_FLAGS_INIT_DONE); + int ret; if (new_cache) { __set_bit(0, cache->seg_map); @@ -209,6 +214,12 @@ static int cache_tail_init(struct pcache_cache *cache) pcache_dev_err(pcache, "Corrupted key tail or dirty tail.\n"); return -EIO; } + + ret = cache_verify_dirty_tail(cache); + if (ret) { + pcache_dev_err(pcache, "dirty tail chain does not terminate (crafted cache image?)\n"); + return ret; + } } return 0; @@ -246,6 +257,13 @@ static int get_seg_id(struct pcache_cache *cache, } else { *seg_id = cache->cache_info.seg_id; } + + if (*seg_id >= cache_dev->seg_num) { + pcache_dev_err(pcache, "invalid segment id %u from cache device (seg_num %u)\n", + *seg_id, cache_dev->seg_num); + ret = -EIO; + goto err; + } } return 0; err: @@ -261,6 +279,13 @@ static int cache_segs_init(struct pcache_cache *cache) int ret; u32 i; + if (cache_info->n_segs > cache->cache_dev->seg_num) { + pcache_dev_err(CACHE_TO_PCACHE(cache), + "cache_info n_segs %u exceeds cache device segments %u\n", + cache_info->n_segs, cache->cache_dev->seg_num); + return -EIO; + } + for (i = 0; i < cache_info->n_segs; i++) { ret = get_seg_id(cache, prev_cache_seg, new_cache, &seg_id); if (ret) diff --git a/drivers/md/dm-pcache/cache.h b/drivers/md/dm-pcache/cache.h index 27613b56be54c..bf61a4a487bc0 100644 --- a/drivers/md/dm-pcache/cache.h +++ b/drivers/md/dm-pcache/cache.h @@ -180,6 +180,7 @@ struct pcache_cache { u32 advance; int ret; } writeback_ctx; + atomic_t writeback_errors; char gc_kset_onmedia_buf[PCACHE_KSET_ONMEDIA_SIZE_MAX]; struct delayed_work gc_work; @@ -491,6 +492,27 @@ static inline u32 cache_key_data_crc(struct pcache_cache_key *key) return crc32c(PCACHE_CRC_SEED, data, key->len); } +/** + * kset_onmedia_valid - Validate a kset header read from the cache device. + * @kset_onmedia: Pointer to the kset copied from on-media metadata. + * + * The magic and CRC are attacker-computable (fixed public seed). A non-last + * kset stores key_num keys inline, and cache_kset_crc() and the replay loop + * read struct_size(.., data, key_num) bytes from a buffer sized for + * PCACHE_KSET_KEYS_MAX keys, so key_num must be bounded before any such use. + */ +static inline bool kset_onmedia_valid(struct pcache_cache_kset_onmedia *kset_onmedia) +{ + if (kset_onmedia->magic != PCACHE_KSET_MAGIC) + return false; + + if (!(kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) && + kset_onmedia->key_num > PCACHE_KSET_KEYS_MAX) + return false; + + return true; +} + static inline u32 cache_kset_crc(struct pcache_cache_kset_onmedia *kset_onmedia) { u32 crc_size; @@ -630,6 +652,8 @@ static inline int cache_decode_dirty_tail(struct pcache_cache *cache) &cache->dirty_tail_index); } +int cache_verify_dirty_tail(struct pcache_cache *cache); + int pcache_cache_init(void); void pcache_cache_exit(void); #endif /* _PCACHE_CACHE_H */ diff --git a/drivers/md/dm-pcache/cache_dev.c b/drivers/md/dm-pcache/cache_dev.c index ece689e6ce59d..f0259353ee397 100644 --- a/drivers/md/dm-pcache/cache_dev.c +++ b/drivers/md/dm-pcache/cache_dev.c @@ -242,6 +242,8 @@ int cache_dev_start(struct dm_pcache *pcache) struct pcache_cache_dev *cache_dev = &pcache->cache_dev; struct pcache_sb sb; bool format = false; + u32 seg_num; + u64 max_segs; int ret; mutex_init(&cache_dev->seg_lock); @@ -269,7 +271,25 @@ int cache_dev_start(struct dm_pcache *pcache) goto dax_release; cache_dev->sb_flags = le32_to_cpu(sb.flags); - ret = cache_dev_init(cache_dev, le32_to_cpu(sb.seg_num)); + + /* + * seg_num is read from the crc32c-only superblock, so whoever supplies + * the cache device controls it. It is the ceiling every later on-media + * segment id is validated against, so bound it against what the device + * physically holds before it is trusted, or a forged seg_num lets a + * segment id address past the DAX mapping. + */ + seg_num = le32_to_cpu(sb.seg_num); + max_segs = (bdev_nr_bytes(cache_dev->dm_dev->bdev) - PCACHE_SEGMENTS_OFF) / + PCACHE_SEG_SIZE; + if (seg_num == 0 || seg_num > max_segs || seg_num > PCACHE_CACHE_SEGS_MAX) { + pcache_dev_err(pcache, "invalid seg_num %u from cache device (device holds %llu, max %u)\n", + seg_num, max_segs, (u32)PCACHE_CACHE_SEGS_MAX); + ret = -EIO; + goto dax_release; + } + + ret = cache_dev_init(cache_dev, seg_num); if (ret) goto dax_release; diff --git a/drivers/md/dm-pcache/cache_gc.c b/drivers/md/dm-pcache/cache_gc.c index 94f8b276a0212..99999a92c4e67 100644 --- a/drivers/md/dm-pcache/cache_gc.c +++ b/drivers/md/dm-pcache/cache_gc.c @@ -37,18 +37,18 @@ static bool need_gc(struct pcache_cache *cache, struct pcache_cache_pos *dirty_t kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->gc_kset_onmedia_buf; - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - key_tail->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(key_tail)); ret = copy_mc_to_kernel(kset_onmedia, key_addr, to_copy); if (ret) { pcache_dev_err(pcache, "error to read kset: %d", ret); return false; } - /* Check if kset_onmedia is corrupted */ - if (kset_onmedia->magic != PCACHE_KSET_MAGIC) { - pcache_dev_debug(pcache, "gc error: magic is not as expected. key_tail: %u:%u magic: %llx, expected: %llx\n", + /* Reject a corrupted or out-of-bounds kset before reading its keys */ + if (!kset_onmedia_valid(kset_onmedia)) { + pcache_dev_debug(pcache, "gc error: invalid kset. key_tail: %u:%u magic: %llx, key_num: %u\n", key_tail->cache_seg->cache_seg_id, key_tail->seg_off, - kset_onmedia->magic, PCACHE_KSET_MAGIC); + kset_onmedia->magic, kset_onmedia->key_num); return false; } @@ -134,6 +134,11 @@ void pcache_cache_gc_fn(struct work_struct *work) continue; } + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&key_tail)) { + atomic_inc(&cache->gc_errors); + return; + } + for (i = 0; i < kset_onmedia->key_num; i++) { struct pcache_cache_key key_tmp = { 0 }; diff --git a/drivers/md/dm-pcache/cache_key.c b/drivers/md/dm-pcache/cache_key.c index 2b77e121f89be..c5a8ac02b216b 100644 --- a/drivers/md/dm-pcache/cache_key.c +++ b/drivers/md/dm-pcache/cache_key.c @@ -90,13 +90,30 @@ int cache_key_decode(struct pcache_cache *cache, struct pcache_cache_key *key) { struct dm_pcache *pcache = CACHE_TO_PCACHE(cache); + u64 dev_bytes = (u64)cache->dev_size << SECTOR_SHIFT; key->off = key_onmedia->off; key->len = key_onmedia->len; + if (key_onmedia->len == 0 || + key_onmedia->len > dev_bytes || + key_onmedia->off > dev_bytes - key_onmedia->len) { + pcache_dev_err(pcache, "key off %llu + len %u exceeds device size\n", + key_onmedia->off, key_onmedia->len); + return -EIO; + } + key->cache_pos.cache_seg = &cache->segments[key_onmedia->cache_seg_id]; key->cache_pos.seg_off = key_onmedia->cache_seg_off; + if ((u64)key->cache_pos.seg_off + key->len > + key->cache_pos.cache_seg->segment.data_size) { + pcache_dev_err(pcache, "key seg_off %u + len %u exceeds segment data size %u\n", + key->cache_pos.seg_off, key->len, + key->cache_pos.cache_seg->segment.data_size); + return -EIO; + } + key->seg_gen = key_onmedia->seg_gen; key->flags = key_onmedia->flags; @@ -728,18 +745,17 @@ static int kset_replay(struct pcache_cache *cache, struct pcache_cache_kset_onme goto err; } - __set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map); - /* Check if the segment generation is valid for insertion. */ if (key->seg_gen < key->cache_pos.cache_seg->gen) { cache_key_put(key); - } else { - cache_subtree = get_subtree(&cache->req_key_tree, key->off); - spin_lock(&cache_subtree->tree_lock); - cache_key_insert(&cache->req_key_tree, key, true); - spin_unlock(&cache_subtree->tree_lock); + continue; } + __set_bit(key->cache_pos.cache_seg->cache_seg_id, cache->seg_map); + cache_subtree = get_subtree(&cache->req_key_tree, key->off); + spin_lock(&cache_subtree->tree_lock); + cache_key_insert(&cache->req_key_tree, key, true); + spin_unlock(&cache_subtree->tree_lock); cache_seg_get(key->cache_pos.cache_seg); } @@ -754,7 +770,7 @@ int cache_replay(struct pcache_cache *cache) struct pcache_cache_pos pos_tail; struct pcache_cache_pos *pos; struct pcache_cache_kset_onmedia *kset_onmedia; - u32 to_copy, count = 0; + u32 to_copy, count = 0, last_hops = 0; int ret = 0; kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL); @@ -771,14 +787,14 @@ int cache_replay(struct pcache_cache *cache) __set_bit(pos->cache_seg->cache_seg_id, cache->seg_map); while (true) { - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - pos->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(pos)); ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(pos), to_copy); if (ret) { ret = -EIO; goto out; } - if (kset_onmedia->magic != PCACHE_KSET_MAGIC || + if (!kset_onmedia_valid(kset_onmedia) || kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { break; } @@ -789,6 +805,11 @@ int cache_replay(struct pcache_cache *cache) pcache_dev_debug(pcache, "last kset replay, next: %u\n", kset_onmedia->next_cache_seg_id); + if (++last_hops > cache->n_segs) { + ret = -EIO; + goto out; + } + next_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; pos->cache_seg = next_seg; @@ -799,6 +820,11 @@ int cache_replay(struct pcache_cache *cache) } /* Replay the kset and check for errors. */ + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(pos)) { + ret = -EIO; + goto out; + } + ret = kset_replay(cache, kset_onmedia); if (ret) goto out; @@ -820,6 +846,75 @@ int cache_replay(struct pcache_cache *cache) return ret; } +/* + * cache_verify_dirty_tail - reject a persisted dirty_tail whose last-kset + * chain does not terminate. + * + * dirty_tail is decoded independently of the key_tail chain cache_replay() + * walks, so replay's hop cap does not cover it. A crafted chain that loops + * back on itself makes the writeback worker re-arm forever; walk it once here + * with the same cap and fail the load if it does not end within n_segs hops. + */ +int cache_verify_dirty_tail(struct pcache_cache *cache) +{ + struct pcache_cache_pos pos; + struct pcache_cache_kset_onmedia *kset_onmedia; + u32 to_copy, last_hops = 0, count = 0; + int ret = 0; + + kset_onmedia = kzalloc(PCACHE_KSET_ONMEDIA_SIZE_MAX, GFP_KERNEL); + if (!kset_onmedia) + return -ENOMEM; + + cache_pos_copy(&pos, &cache->dirty_tail); + + while (true) { + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(&pos)); + ret = copy_mc_to_kernel(kset_onmedia, cache_pos_addr(&pos), to_copy); + if (ret) { + ret = -EIO; + goto out; + } + + /* A missing, short or corrupt kset is the normal end of the chain. */ + if (!kset_onmedia_valid(kset_onmedia) || + kset_onmedia->crc != cache_kset_crc(kset_onmedia)) { + ret = 0; + goto out; + } + + if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) { + if (kset_onmedia->next_cache_seg_id >= cache->cache_info.n_segs) { + ret = -EIO; + goto out; + } + + if (++last_hops > cache->n_segs) { + ret = -EIO; + goto out; + } + + pos.cache_seg = &cache->segments[kset_onmedia->next_cache_seg_id]; + pos.seg_off = 0; + continue; + } + + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&pos)) { + ret = -EIO; + goto out; + } + + cache_pos_advance(&pos, get_kset_onmedia_size(kset_onmedia)); + if (++count > 512) { + cond_resched(); + count = 0; + } + } +out: + kfree(kset_onmedia); + return ret; +} + int cache_tree_init(struct pcache_cache *cache, struct pcache_cache_tree *cache_tree, u32 n_subtrees) { int ret; diff --git a/drivers/md/dm-pcache/cache_segment.c b/drivers/md/dm-pcache/cache_segment.c index 9d92e2b067ed0..c698ebbc626d2 100644 --- a/drivers/md/dm-pcache/cache_segment.c +++ b/drivers/md/dm-pcache/cache_segment.c @@ -243,8 +243,16 @@ struct pcache_cache_segment *get_cache_segment(struct pcache_cache *cache) spin_lock(&cache->seg_map_lock); again: - seg_id = find_next_zero_bit(cache->seg_map, cache->n_segs, cache->last_cache_seg); - if (seg_id == cache->n_segs) { + /* + * Only allocate initialized segments. cache_segs_init() initializes + * cache_info.n_segs of the cache->n_segs device segments; a forged + * smaller cache_info.n_segs leaves the rest as zeroed structs whose data + * pointer is NULL. Bounding the search to cache_info.n_segs keeps such a + * segment from reaching cache_kset_close(), which writes through it. + */ + seg_id = find_next_zero_bit(cache->seg_map, cache->cache_info.n_segs, + cache->last_cache_seg); + if (seg_id == cache->cache_info.n_segs) { /* reset the hint of ->last_cache_seg and retry */ if (cache->last_cache_seg) { cache->last_cache_seg = 0; diff --git a/drivers/md/dm-pcache/cache_writeback.c b/drivers/md/dm-pcache/cache_writeback.c index 87a82b3fe8363..656e4d0489121 100644 --- a/drivers/md/dm-pcache/cache_writeback.c +++ b/drivers/md/dm-pcache/cache_writeback.c @@ -48,18 +48,18 @@ static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cach addr = cache_pos_addr(dirty_tail); kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; - to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, PCACHE_SEG_SIZE - dirty_tail->seg_off); + to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail)); ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy); if (ret) { pcache_dev_err(pcache, "error to read kset: %d", ret); return true; } - /* Check if the magic number matches the expected value */ - if (kset_onmedia->magic != PCACHE_KSET_MAGIC) { - pcache_dev_debug(pcache, "dirty_tail: %u:%u magic: %llx, not expected: %llx\n", + /* Reject a corrupted or out-of-bounds kset before reading its keys */ + if (!kset_onmedia_valid(kset_onmedia)) { + pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n", dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off, - kset_onmedia->magic, PCACHE_KSET_MAGIC); + kset_onmedia->magic, kset_onmedia->key_num); return true; } @@ -229,6 +229,9 @@ void cache_writeback_fn(struct work_struct *work) if (pcache_is_stopping(pcache)) goto unlock; + if (atomic_read(&cache->writeback_errors)) + goto unlock; + kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf; mutex_lock(&cache->dirty_tail_lock); @@ -246,6 +249,11 @@ void cache_writeback_fn(struct work_struct *work) goto queue_work; } + if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) { + atomic_inc(&cache->writeback_errors); + goto unlock; + } + ret = cache_kset_insert_tree(cache, kset_onmedia); if (ret) { delay = PCACHE_CACHE_WRITEBACK_INTERVAL; diff --git a/drivers/md/dm-pcache/dm_pcache.c b/drivers/md/dm-pcache/dm_pcache.c index 87bcc1c515a6f..f6da50b57c678 100644 --- a/drivers/md/dm-pcache/dm_pcache.c +++ b/drivers/md/dm-pcache/dm_pcache.c @@ -439,13 +439,13 @@ static int dm_pcache_message(struct dm_target *ti, unsigned int argc, char **argv, char *result, unsigned int maxlen) { struct dm_pcache *pcache = ti->private; - unsigned long val; + u8 val; if (argc != 2) goto err; if (!strcasecmp(argv[0], "gc_percent")) { - if (kstrtoul(argv[1], 10, &val)) + if (kstrtou8(argv[1], 10, &val)) goto err; return pcache_cache_set_gc_percent(&pcache->cache, val); diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 4bacdc499984b..9eee490c029e7 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -3831,6 +3831,7 @@ static void raid_presuspend(struct dm_target *ti) * resume, raid_postsuspend() is too late. */ set_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags); + set_bit(MD_DM_SUSPENDING, &mddev->flags); if (!reshape_interrupted(mddev)) return; @@ -3847,13 +3848,16 @@ static void raid_presuspend(struct dm_target *ti) static void raid_presuspend_undo(struct dm_target *ti) { struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; + clear_bit(MD_DM_SUSPENDING, &mddev->flags); clear_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags); } static void raid_postsuspend(struct dm_target *ti) { struct raid_set *rs = ti->private; + struct mddev *mddev = &rs->md; if (!test_and_set_bit(RT_FLAG_RS_SUSPENDED, &rs->runtime_flags)) { /* @@ -3864,6 +3868,8 @@ static void raid_postsuspend(struct dm_target *ti) mddev_suspend(&rs->md, false); rs->md.ro = MD_RDONLY; } + clear_bit(MD_DM_SUSPENDING, &mddev->flags); + } static void attempt_restore_of_faulty_devices(struct raid_set *rs) diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index d9ec783a37ddc..9b66863991d12 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -258,7 +258,7 @@ static void fail_mirror(struct mirror *m, enum dm_raid1_error error_type) static int mirror_flush(struct dm_target *ti) { struct mirror_set *ms = ti->private; - unsigned long error_bits; + unsigned long error_bits, unsup_bits; unsigned int i; struct dm_io_region io[MAX_NR_MIRRORS]; @@ -277,8 +277,8 @@ static int mirror_flush(struct dm_target *ti) } error_bits = -1; - dm_io(&io_req, ms->nr_mirrors, io, &error_bits, IOPRIO_DEFAULT); - if (unlikely(error_bits != 0)) { + dm_io(&io_req, ms->nr_mirrors, io, &error_bits, &unsup_bits, IOPRIO_DEFAULT); + if (unlikely((error_bits | unsup_bits) != 0)) { for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error_bits)) fail_mirror(ms->mirror + i, @@ -511,7 +511,7 @@ static void hold_bio(struct mirror_set *ms, struct bio *bio) * Reads *--------------------------------------------------------------- */ -static void read_callback(unsigned long error, void *context) +static void read_callback(unsigned long error, unsigned long unsup, void *context) { struct bio *bio = context; struct mirror *m; @@ -520,6 +520,8 @@ static void read_callback(unsigned long error, void *context) bio_set_m(bio, NULL); if (likely(!error)) { + if (unlikely(unsup != 0)) + bio->bi_status = BLK_STS_INVAL; bio_endio(bio); return; } @@ -553,7 +555,7 @@ static void read_async_bio(struct mirror *m, struct bio *bio) map_region(&io, m, bio); bio_set_m(bio, m); - BUG_ON(dm_io(&io_req, 1, &io, NULL, IOPRIO_DEFAULT)); + BUG_ON(dm_io(&io_req, 1, &io, NULL, NULL, IOPRIO_DEFAULT)); } static inline int region_in_sync(struct mirror_set *ms, region_t region, @@ -600,7 +602,7 @@ static void do_reads(struct mirror_set *ms, struct bio_list *reads) * NOSYNC: increment pending, just write to the default mirror *--------------------------------------------------------------------- */ -static void write_callback(unsigned long error, void *context) +static void write_callback(unsigned long error, unsigned long unsup, void *context) { unsigned int i; struct bio *bio = context; @@ -617,7 +619,7 @@ static void write_callback(unsigned long error, void *context) * This way we handle both writes to SYNC and NOSYNC * regions with the same code. */ - if (likely(!error)) { + if (likely(!(error | unsup))) { bio_endio(bio); return; } @@ -632,6 +634,12 @@ static void write_callback(unsigned long error, void *context) return; } + if (!error && unsup) { + bio->bi_status = BLK_STS_INVAL; + bio_endio(bio); + return; + } + for (i = 0; i < ms->nr_mirrors; i++) if (test_bit(i, &error)) fail_mirror(ms->mirror + i, DM_RAID1_WRITE_ERROR); @@ -680,7 +688,7 @@ static void do_write(struct mirror_set *ms, struct bio *bio) */ bio_set_m(bio, get_default_mirror(ms)); - BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, IOPRIO_DEFAULT)); + BUG_ON(dm_io(&io_req, ms->nr_mirrors, io, NULL, NULL, IOPRIO_DEFAULT)); } static void do_writes(struct mirror_set *ms, struct bio_list *writes) @@ -1262,7 +1270,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, return DM_ENDIO_DONE; } - if (*error == BLK_STS_NOTSUPP) + if (*error == BLK_STS_NOTSUPP || *error == BLK_STS_INVAL) goto out; if (bio->bi_opf & REQ_RAHEAD) diff --git a/drivers/md/dm-snap-persistent.c b/drivers/md/dm-snap-persistent.c index 0e13d60bfdd12..0dc429eb26924 100644 --- a/drivers/md/dm-snap-persistent.c +++ b/drivers/md/dm-snap-persistent.c @@ -223,7 +223,7 @@ static void do_metadata(struct work_struct *work) { struct mdata_req *req = container_of(work, struct mdata_req, work); - req->result = dm_io(req->io_req, 1, req->where, NULL, IOPRIO_DEFAULT); + req->result = dm_io(req->io_req, 1, req->where, NULL, NULL, IOPRIO_DEFAULT); } /* @@ -247,7 +247,7 @@ static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, blk_opf_t opf, struct mdata_req req; if (!metadata) - return dm_io(&io_req, 1, &where, NULL, IOPRIO_DEFAULT); + return dm_io(&io_req, 1, &where, NULL, NULL, IOPRIO_DEFAULT); req.where = &where; req.io_req = &io_req; diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index dbff4922e0682..8019eeb4cc918 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -178,8 +178,10 @@ static void dm_stat_free(struct rcu_head *head) kfree(s->program_id); kfree(s->aux_data); for_each_possible_cpu(cpu) { - dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); - dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); + if (s->stat_percpu[cpu]) { + dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); + dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); + } } dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size); dm_kvfree(s, s->shared_alloc_size); diff --git a/drivers/md/dm-switch.c b/drivers/md/dm-switch.c index 50a52ca50b34c..65d169ffa7537 100644 --- a/drivers/md/dm-switch.c +++ b/drivers/md/dm-switch.c @@ -184,7 +184,7 @@ static void switch_region_table_write(struct switch_ctx *sctx, unsigned long reg pte = sctx->region_table[region_index]; pte &= ~((((region_table_slot_t)1 << sctx->region_table_entry_bits) - 1) << bit); pte |= (region_table_slot_t)value << bit; - sctx->region_table[region_index] = pte; + WRITE_ONCE(sctx->region_table[region_index], pte); } /* diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 03e59b85132aa..d1e4fbc5a21d9 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -39,36 +39,6 @@ static inline u64 fec_interleave(struct dm_verity *v, u64 offset) return offset + mod * (v->fec->rounds << v->data_dev_block_bits); } -/* - * Read error-correcting codes for the requested RS block. Returns a pointer - * to the data block. Caller is responsible for releasing buf. - */ -static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, - unsigned int *offset, unsigned int par_buf_offset, - struct dm_buffer **buf, unsigned short ioprio) -{ - u64 position, block, rem; - u8 *res; - - /* We have already part of parity bytes read, skip to the next block */ - if (par_buf_offset) - index++; - - position = (index + rsb) * v->fec->roots; - block = div64_u64_rem(position, v->fec->io_size, &rem); - *offset = par_buf_offset ? 0 : (unsigned int)rem; - - res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); - if (IS_ERR(res)) { - DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", - v->data_dev->name, (unsigned long long)rsb, - (unsigned long long)block, PTR_ERR(res)); - *buf = NULL; - } - - return res; -} - /* Loop over each preallocated buffer slot. */ #define fec_for_each_prealloc_buffer(__i) \ for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++) @@ -116,15 +86,29 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, { int r, corrected = 0, res; struct dm_buffer *buf; - unsigned int n, i, j, offset, par_buf_offset = 0; - uint16_t par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; + unsigned int n, i, j, parity_pos, to_copy; + uint16_t par_buf[DM_VERITY_FEC_MAX_ROOTS]; u8 *par, *block; + u64 parity_block; struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); - par = fec_read_parity(v, rsb, block_offset, &offset, - par_buf_offset, &buf, bio->bi_ioprio); - if (IS_ERR(par)) + /* + * Compute the index of the first parity block that will be needed and + * the starting position in that block. Then read that block. + * + * io_size is always a power of 2, but roots might not be. Note that + * when it's not, a codeword's parity bytes can span a block boundary. + */ + parity_block = (rsb + block_offset) * v->fec->roots; + parity_pos = parity_block & (v->fec->io_size - 1); + parity_block >>= v->data_dev_block_bits; + par = dm_bufio_read_with_ioprio(v->fec->bufio, parity_block, &buf, + bio->bi_ioprio); + if (IS_ERR(par)) { + DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", + v->data_dev->name, rsb, parity_block, PTR_ERR(par)); return PTR_ERR(par); + } /* * Decode the RS blocks we have in bufs. Each RS block results in @@ -132,8 +116,32 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, */ fec_for_each_buffer_rs_block(fio, n, i) { block = fec_buffer_rs_block(v, fio, n, i); - for (j = 0; j < v->fec->roots - par_buf_offset; j++) - par_buf[par_buf_offset + j] = par[offset + j]; + + /* + * Copy the next 'roots' parity bytes to 'par_buf', reading + * another parity block if needed. + */ + to_copy = min(v->fec->io_size - parity_pos, v->fec->roots); + for (j = 0; j < to_copy; j++) + par_buf[j] = par[parity_pos++]; + if (to_copy < v->fec->roots) { + parity_block++; + parity_pos = 0; + + dm_bufio_release(buf); + par = dm_bufio_read_with_ioprio(v->fec->bufio, + parity_block, &buf, + bio->bi_ioprio); + if (IS_ERR(par)) { + DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", + v->data_dev->name, rsb, parity_block, + PTR_ERR(par)); + return PTR_ERR(par); + } + for (; j < v->fec->roots; j++) + par_buf[j] = par[parity_pos++]; + } + /* Decode an RS block using Reed-Solomon */ res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn, NULL, neras, fio->erasures, 0, NULL); @@ -148,26 +156,6 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, block_offset++; if (block_offset >= 1 << v->data_dev_block_bits) goto done; - - /* Read the next block when we run out of parity bytes */ - offset += (v->fec->roots - par_buf_offset); - /* Check if parity bytes are split between blocks */ - if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) { - par_buf_offset = v->fec->io_size - offset; - for (j = 0; j < par_buf_offset; j++) - par_buf[j] = par[offset + j]; - offset += par_buf_offset; - } else - par_buf_offset = 0; - - if (offset >= v->fec->io_size) { - dm_bufio_release(buf); - - par = fec_read_parity(v, rsb, block_offset, &offset, - par_buf_offset, &buf, bio->bi_ioprio); - if (IS_ERR(par)) - return PTR_ERR(par); - } } done: r = corrected; @@ -262,7 +250,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, (unsigned long long)block, PTR_ERR(bbuf)); /* assume the block is corrupted */ - if (neras && *neras <= v->fec->roots) + if (neras && *neras < v->fec->roots) fio->erasures[(*neras)++] = i; continue; @@ -280,7 +268,7 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, * skip if we have already found the theoretical * maximum number (i.e. fec->roots) of erasures */ - if (neras && *neras <= v->fec->roots && + if (neras && *neras < v->fec->roots && fec_is_erasure(v, io, want_digest, bbuf)) fio->erasures[(*neras)++] = i; } @@ -617,8 +605,8 @@ int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) { if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c || - num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || - num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { + num_c < DM_VERITY_FEC_MIN_ROOTS || + num_c > DM_VERITY_FEC_MAX_ROOTS) { ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; return -EINVAL; } diff --git a/drivers/md/dm-verity-fec.h b/drivers/md/dm-verity-fec.h index ec37e607cb3f0..8552b5d3c9152 100644 --- a/drivers/md/dm-verity-fec.h +++ b/drivers/md/dm-verity-fec.h @@ -13,8 +13,8 @@ /* Reed-Solomon(M, N) parameters */ #define DM_VERITY_FEC_RSM 255 -#define DM_VERITY_FEC_MAX_RSN 253 -#define DM_VERITY_FEC_MIN_RSN 231 /* ~10% space overhead */ +#define DM_VERITY_FEC_MIN_ROOTS 2 /* RS(255, 253): ~0.8% space overhead */ +#define DM_VERITY_FEC_MAX_ROOTS 24 /* RS(255, 231): ~10% space overhead */ /* buffers for deinterleaving and decoding */ #define DM_VERITY_FEC_BUF_PREALLOC 1 /* buffers to preallocate */ @@ -50,7 +50,7 @@ struct dm_verity_fec { /* per-bio data */ struct dm_verity_fec_io { struct rs_control *rs; /* Reed-Solomon state */ - int erasures[DM_VERITY_FEC_MAX_RSN]; /* erasures for decode_rs8 */ + int erasures[DM_VERITY_FEC_MAX_ROOTS]; /* erasures for decode_rs8 */ u8 *bufs[DM_VERITY_FEC_BUF_MAX]; /* bufs for deinterleaving */ unsigned int nbufs; /* number of buffers allocated */ u8 *output; /* buffer for corrected output */ diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c index 40c0664fa0342..2c754144e929a 100644 --- a/drivers/md/dm-verity-target.c +++ b/drivers/md/dm-verity-target.c @@ -379,7 +379,7 @@ static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, io_loc.bdev = v->data_dev->bdev; io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); - r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + r = dm_io(&io_req, 1, &io_loc, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) goto free_ret; diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index af54e289bcebe..34148aa4df799 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -474,12 +474,14 @@ struct io_notify { atomic_t count; }; -static void writecache_notify_io(unsigned long error, void *context) +static void writecache_notify_io(unsigned long error, unsigned long unsup, void *context) { struct io_notify *endio = context; if (unlikely(error != 0)) writecache_error(endio->wc, -EIO, "error writing metadata"); + else if (unlikely(unsup != 0)) + writecache_error(endio->wc, -EOPNOTSUPP, "error writing metadata"); BUG_ON(atomic_read(&endio->count) <= 0); if (atomic_dec_and_test(&endio->count)) complete(&endio->c); @@ -530,11 +532,11 @@ static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios) req.notify.context = &endio; /* writing via async dm-io (implied by notify.fn above) won't return an error */ - (void) dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + (void) dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); i = j; } - writecache_notify_io(0, &endio); + writecache_notify_io(0, 0, &endio); wait_for_completion_io(&endio.c); if (wait_for_ios) @@ -567,7 +569,7 @@ static void ssd_commit_superblock(struct dm_writecache *wc) req.notify.fn = NULL; req.notify.context = NULL; - r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error writing superblock"); } @@ -595,7 +597,7 @@ static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev) req.client = wc->dm_io; req.notify.fn = NULL; - r = dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + r = dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); if (unlikely(r)) writecache_error(wc, r, "error flushing metadata: %d", r); } @@ -989,7 +991,7 @@ static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors req.client = wc->dm_io; req.notify.fn = NULL; - return dm_io(&req, 1, ®ion, NULL, IOPRIO_DEFAULT); + return dm_io(&req, 1, ®ion, NULL, NULL, IOPRIO_DEFAULT); } static void writecache_resume(struct dm_target *ti) diff --git a/drivers/md/dm.c b/drivers/md/dm.c index fb6bcd598c5b8..f587b7229ce89 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -735,7 +735,16 @@ static struct table_device *open_table_device(struct mapped_device *md, return ERR_PTR(-ENOMEM); refcount_set(&td->count, 1); - bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); + /* + * Open the backing device with kernel rather than caller + * credentials. Otherwise the caller's credentials would be + * pinned in bdev_file->f_cred until the table device is closed. + * That would keep the caller's thread keyring alive long beyond the + * lifetime of the caller, breaking userspace expectation (e.g. + * cryptsetup(8) leaking the LUKS volume key). + */ + scoped_with_kernel_creds() + bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); if (IS_ERR(bdev_file)) { r = PTR_ERR(bdev_file); goto out_free_td; @@ -2606,9 +2615,10 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) */ mutex_lock(&md->table_devices_lock); r = add_disk(md->disk); - mutex_unlock(&md->table_devices_lock); - if (r) + if (r) { + mutex_unlock(&md->table_devices_lock); return r; + } /* * Register the holder relationship for devices added before the disk @@ -2619,18 +2629,21 @@ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) if (r) goto out_undo_holders; } + mutex_unlock(&md->table_devices_lock); r = dm_sysfs_init(md); if (r) - goto out_undo_holders; + goto lock_out_undo_holders; md->type = type; + return 0; +lock_out_undo_holders: + mutex_lock(&md->table_devices_lock); out_undo_holders: list_for_each_entry_continue_reverse(td, &md->table_devices, list) bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); - mutex_lock(&md->table_devices_lock); del_gendisk(md->disk); mutex_unlock(&md->table_devices_lock); return r; @@ -3106,7 +3119,7 @@ int dm_resume(struct mapped_device *md) r = -EINVAL; mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); - if (!dm_suspended_md(md)) + if (!dm_suspended_md(md) || test_bit(DMF_FREEING, &md->flags)) goto out; if (dm_suspended_internally_md(md)) { diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 2a95840782927..781cbfb2d3481 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2629,10 +2629,12 @@ static ssize_t location_store(struct mddev *mddev, const char *buf, size_t len) { int rv; + unsigned int noio_flags; rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (mddev->recovery || mddev->sync_thread) { @@ -2719,6 +2721,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } rv = 0; out: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (rv) return rv; @@ -2862,7 +2865,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) if (!has_write_mostly) { pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n", mdname(mddev)); - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return -EINVAL; } diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index f46674bdfeb91..7535742982fc0 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -29,6 +29,7 @@ enum bitmap_state { BITMAP_FIRST_USE = 3, /* llbitmap is just created */ BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */ BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */ + BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */ BITMAP_HOSTENDIAN =15, }; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index dc9b72494a81a..5c60b26ad2468 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -662,6 +662,7 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, if (state == BitNeedSync) need_resync = !mddev->degraded; else if (state == BitDirty && + !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ); @@ -854,7 +855,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } - llbitmap->flags = le32_to_cpu(sb->state); + llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; @@ -910,6 +911,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer); + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), @@ -930,6 +934,9 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) bool restart; int idx; + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (llbitmap->mddev->degraded) return; retry: @@ -969,7 +976,7 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) goto retry; /* If some page is dirty but not expired, setup timer again */ - if (restart) + if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); } @@ -999,10 +1006,11 @@ static int llbitmap_create(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); + if (ret) + mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); - mddev->bitmap = NULL; } return ret; @@ -1051,7 +1059,9 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); - timer_delete_sync(&llbitmap->pending_timer); + set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); + timer_shutdown_sync(&llbitmap->pending_timer); + cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq); @@ -1375,7 +1385,7 @@ static void llbitmap_update_sb(void *data) sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); - sb->state = cpu_to_le32(llbitmap->flags); + sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); diff --git a/drivers/md/md.c b/drivers/md/md.c index 0d60c211e7c08..d03d0fdf763db 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -97,7 +97,7 @@ static struct workqueue_struct *md_misc_wq; static int remove_and_add_spares(struct mddev *mddev, struct md_rdev *this); static void mddev_detach(struct mddev *mddev); -static void export_rdev(struct md_rdev *rdev, struct mddev *mddev); +static void export_rdev(struct md_rdev *rdev); static void md_wakeup_thread_directly(struct md_thread __rcu **thread); /* @@ -234,23 +234,22 @@ static int rdev_need_serial(struct md_rdev *rdev) void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) { int ret = 0; + unsigned int noio_flags; - if (rdev && !rdev_need_serial(rdev) && + if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) && + rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; + noio_flags = memalloc_noio_save(); if (!rdev) ret = rdevs_init_serial(mddev); else ret = rdev_init_serial(rdev); if (ret) - return; + goto out; if (mddev->serial_info_pool == NULL) { - /* - * already in memalloc noio context by - * mddev_suspend() - */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); @@ -259,6 +258,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) pr_err("can't alloc memory pool for serialization\n"); } } +out: + memalloc_noio_restore(noio_flags); } /* @@ -517,9 +518,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) */ WRITE_ONCE(mddev->suspended, mddev->suspended + 1); - /* restrict memory reclaim I/O during raid array is suspend */ - mddev->noio_flag = memalloc_noio_save(); - mutex_unlock(&mddev->suspend_mutex); return 0; } @@ -536,9 +534,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed) return; } - /* entred the memalloc scope from mddev_suspend() */ - memalloc_noio_restore(mddev->noio_flag); - percpu_ref_resurrect(&mddev->active_io); wake_up(&mddev->sb_wait); @@ -972,7 +967,7 @@ void mddev_unlock(struct mddev *mddev) list_for_each_entry_safe(rdev, tmp, &delete, same_set) { list_del_init(&rdev->same_set); kobject_del(&rdev->kobj); - export_rdev(rdev, mddev); + export_rdev(rdev); } if (!legacy_async_del_gendisk) { @@ -1917,6 +1912,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ rdev->bb_page, REQ_OP_READ, true)) return -EIO; bbp = (__le64 *)page_address(rdev->bb_page); + + /* check for badblocks api. */ + if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) { + pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n", + rdev->bdev, sb->bblog_shift); + return -EINVAL; + } rdev->badblocks.shift = sb->bblog_shift; for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) { u64 bb = le64_to_cpu(*bbp); @@ -2640,7 +2642,7 @@ void md_autodetect_dev(dev_t dev); /* just for claiming the bdev */ static struct md_rdev claim_rdev; -static void export_rdev(struct md_rdev *rdev, struct mddev *mddev) +static void export_rdev(struct md_rdev *rdev) { pr_debug("md: export_rdev(%pg)\n", rdev->bdev); md_rdev_clear(rdev); @@ -4040,6 +4042,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) char clevel[16]; ssize_t rv; size_t slen = len; + unsigned int noio_flags; struct md_personality *pers, *oldpers; long level; void *priv, *oldpriv; @@ -4051,6 +4054,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers == NULL) { memcpy(mddev->clevel, buf, slen); @@ -4226,6 +4230,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) md_new_event(); rv = len; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return rv; } @@ -4405,6 +4410,7 @@ static ssize_t raid_disks_store(struct mddev *mddev, const char *buf, size_t len) { unsigned int n; + unsigned int noio_flags; int err; err = kstrtouint(buf, 10, &n); @@ -4414,9 +4420,11 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; - if (mddev->pers) - err = update_raid_disks(mddev, n); - else if (mddev->reshape_position != MaxSector) { + noio_flags = memalloc_noio_save(); + if (mddev->pers) { + if (n != mddev->raid_disks) + err = update_raid_disks(mddev, n); + } else if (mddev->reshape_position != MaxSector) { struct md_rdev *rdev; int olddisks = mddev->raid_disks - mddev->delta_disks; @@ -4436,6 +4444,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ? err : len; } @@ -4816,6 +4825,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) int minor; dev_t dev; struct md_rdev *rdev; + unsigned int noio_flags; int err; if (!*buf || *e != ':' || !e[1] || e[1] == '\n') @@ -4831,6 +4841,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->persistent) { rdev = md_import_device(dev, mddev->major_version, mddev->minor_version); @@ -4849,13 +4860,15 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) rdev = md_import_device(dev, -1, -1); if (IS_ERR(rdev)) { + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return PTR_ERR(rdev); } err = bind_rdev_to_array(rdev, mddev); out: if (err) - export_rdev(rdev, mddev); + export_rdev(rdev); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (!err) md_new_event(); @@ -6944,8 +6957,8 @@ static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; - md_bitmap_destroy(mddev); mddev_detach(mddev); + md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); @@ -7181,7 +7194,7 @@ static void autorun_devices(int part) rdev_for_each_list(rdev, tmp, &candidates) { list_del_init(&rdev->same_set); if (bind_rdev_to_array(rdev, mddev)) - export_rdev(rdev, mddev); + export_rdev(rdev); } autorun_array(mddev); mddev_unlock_and_resume(mddev); @@ -7191,7 +7204,7 @@ static void autorun_devices(int part) */ rdev_for_each_list(rdev, tmp, &candidates) { list_del_init(&rdev->same_set); - export_rdev(rdev, mddev); + export_rdev(rdev); } mddev_put(mddev); } @@ -7379,13 +7392,13 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) pr_warn("md: %pg has different UUID to %pg\n", rdev->bdev, rdev0->bdev); - export_rdev(rdev, mddev); + export_rdev(rdev); return -EINVAL; } } err = bind_rdev_to_array(rdev, mddev); if (err) - export_rdev(rdev, mddev); + export_rdev(rdev); return err; } @@ -7428,7 +7441,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) /* This was a hot-add request, but events doesn't * match, so reject it. */ - export_rdev(rdev, mddev); + export_rdev(rdev); return -EINVAL; } @@ -7454,7 +7467,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) } } if (has_journal || mddev->bitmap) { - export_rdev(rdev, mddev); + export_rdev(rdev); return -EBUSY; } set_bit(Journal, &rdev->flags); @@ -7469,7 +7482,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) /* --add initiated by this node */ err = mddev->cluster_ops->add_new_disk(mddev, rdev); if (err) { - export_rdev(rdev, mddev); + export_rdev(rdev); return err; } } @@ -7479,7 +7492,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) err = bind_rdev_to_array(rdev, mddev); if (err) - export_rdev(rdev, mddev); + export_rdev(rdev); if (mddev_is_clustered(mddev)) { if (info->state & (1 << MD_DISK_CANDIDATE)) { @@ -7542,7 +7555,7 @@ int md_add_new_disk(struct mddev *mddev, struct mdu_disk_info_s *info) err = bind_rdev_to_array(rdev, mddev); if (err) { - export_rdev(rdev, mddev); + export_rdev(rdev); return err; } } @@ -7654,7 +7667,7 @@ static int hot_add_disk(struct mddev *mddev, dev_t dev) return 0; abort_export: - export_rdev(rdev, mddev); + export_rdev(rdev); return err; } @@ -8202,8 +8215,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { int err = 0; + unsigned int noio_flags = 0; void __user *argp = (void __user *)arg; struct mddev *mddev = NULL; + bool suspend; err = md_ioctl_valid(cmd); if (err) @@ -8253,13 +8268,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, if (!md_is_rdwr(mddev)) flush_work(&mddev->sync_work); - err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) : - mddev_lock(mddev); + suspend = md_ioctl_need_suspend(cmd); + err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev); if (err) { pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n", err, cmd); goto out; } + if (suspend) + noio_flags = memalloc_noio_save(); if (cmd == SET_ARRAY_INFO) { err = __md_set_array_info(mddev, argp); @@ -8384,8 +8401,12 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, err != -EINVAL) mddev->hold_active = 0; - md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) : - mddev_unlock(mddev); + if (suspend) { + memalloc_noio_restore(noio_flags); + mddev_unlock_and_resume(mddev); + } else { + mddev_unlock(mddev); + } out: if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY)) @@ -9292,6 +9313,8 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone->mddev = mddev; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); + else + md_io_clone->start_time = 0; if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; @@ -10064,19 +10087,34 @@ static void md_start_sync(struct work_struct *ws) struct mddev *mddev = container_of(ws, struct mddev, sync_work); int spares = 0; bool suspend = false; + unsigned int noio_flags = 0; char *name; /* * If reshape is still in progress, spares won't be added or removed * from conf until reshape is done. */ - if (mddev->reshape_position == MaxSector && + if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); + noio_flags = memalloc_noio_save(); } mddev_lock_nointr(mddev); + + /* + * The spare configuration can change before reconfig_mutex is acquired. + * Recheck while holding the lock and suspend if needed. + */ + if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && + md_spares_need_change(mddev)) { + mddev_unlock(mddev); + mddev_suspend_and_lock_nointr(mddev); + suspend = true; + noio_flags = memalloc_noio_save(); + } + if (!md_is_rdwr(mddev)) { /* * On a read-only array we can: @@ -10120,8 +10158,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); md_new_event(); @@ -10140,8 +10180,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } wake_up(&resync_wait); if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && diff --git a/drivers/md/md.h b/drivers/md/md.h index 2960a98747607..bba0f6790c035 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -343,6 +343,7 @@ struct md_cluster_operations; * @MD_HAS_SUPERBLOCK: There is persistence sb in member disks. * @MD_FAILLAST_DEV: Allow last rdev to be removed. * @MD_SERIALIZE_POLICY: Enforce write IO is not reordered, just used by raid1. + * @MD_DM_SUSPENDING: This DM raid device is suspending. * * change UNSUPPORTED_MDDEV_FLAGS for each array type if new flag is added */ @@ -362,6 +363,7 @@ enum mddev_flags { MD_HAS_SUPERBLOCK, MD_FAILLAST_DEV, MD_SERIALIZE_POLICY, + MD_DM_SUSPENDING, }; enum mddev_sb_flags { @@ -617,7 +619,6 @@ struct mddev { struct md_cluster_info *cluster_info; struct md_cluster_operations *cluster_ops; unsigned int good_device_nr; /* good device num within cluster raid */ - unsigned int noio_flag; /* for memalloc scope API */ /* * Temporarily store rdev that will be finally removed when diff --git a/drivers/md/persistent-data/dm-array.c b/drivers/md/persistent-data/dm-array.c index 8f8792e558063..961fa3c1439a5 100644 --- a/drivers/md/persistent-data/dm-array.c +++ b/drivers/md/persistent-data/dm-array.c @@ -38,6 +38,14 @@ struct array_block { */ #define CSUM_XOR 595846735 +/* + * Each array block can hold this many values. + */ +static uint32_t calc_max_entries(size_t value_size, size_t size_of_block) +{ + return (size_of_block - sizeof(struct array_block)) / value_size; +} + static void array_block_prepare_for_write(const struct dm_block_validator *v, struct dm_block *b, size_t size_of_block) @@ -55,6 +63,7 @@ static int array_block_check(const struct dm_block_validator *v, size_t size_of_block) { struct array_block *bh_le = dm_block_data(b); + uint32_t nr_entries, max_entries, value_size; __le32 csum_disk; if (dm_block_location(b) != le64_to_cpu(bh_le->blocknr)) { @@ -74,6 +83,26 @@ static int array_block_check(const struct dm_block_validator *v, return -EILSEQ; } + nr_entries = le32_to_cpu(bh_le->nr_entries); + max_entries = le32_to_cpu(bh_le->max_entries); + value_size = le32_to_cpu(bh_le->value_size); + + if (!value_size) { + DMERR_LIMIT("%s failed: value_size is zero", __func__); + return -EILSEQ; + } + + if (max_entries != calc_max_entries(value_size, size_of_block)) { + DMERR_LIMIT("%s failed: max_entries %u invalid for value_size %u", + __func__, max_entries, value_size); + return -EILSEQ; + } + + if (nr_entries > max_entries) { + DMERR_LIMIT("%s failed: too many entries", __func__); + return -EILSEQ; + } + return 0; } @@ -138,14 +167,6 @@ static void dec_ablock_entries(struct dm_array_info *info, struct array_block *a on_entries(info, ab, vt->dec); } -/* - * Each array block can hold this many values. - */ -static uint32_t calc_max_entries(size_t value_size, size_t size_of_block) -{ - return (size_of_block - sizeof(struct array_block)) / value_size; -} - /* * Allocate a new array block. The caller will need to unlock block. */ @@ -225,6 +246,14 @@ static int get_ablock(struct dm_array_info *info, dm_block_t b, return r; *ab = dm_block_data(*block); + if (le32_to_cpu((*ab)->value_size) != info->value_type.size) { + DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__, + le32_to_cpu((*ab)->value_size), + info->value_type.size); + dm_tm_unlock(info->btree_info.tm, *block); + return -EILSEQ; + } + return 0; } @@ -287,6 +316,14 @@ static int __shadow_ablock(struct dm_array_info *info, dm_block_t b, return r; *ab = dm_block_data(*block); + if (le32_to_cpu((*ab)->value_size) != info->value_type.size) { + DMERR_LIMIT("%s failed: value_size %u != wanted %u", __func__, + le32_to_cpu((*ab)->value_size), + info->value_type.size); + dm_tm_unlock(info->btree_info.tm, *block); + return -EILSEQ; + } + if (inc) inc_ablock_entries(info, *ab); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 16625b79788bd..1370be1e5cc7b 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3480,8 +3480,6 @@ static void *raid1_takeover(struct mddev *mddev) mddev->new_chunk_sectors = 0; conf = setup_conf(mddev); if (!IS_ERR(conf)) { - /* Array must appear to be quiesced */ - conf->array_frozen = 1; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index d6b174454b1ae..a60e4ba919a76 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1348,6 +1348,7 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, int i, k; sector_t sectors; int max_sectors; + bool atomic = bio->bi_opf & REQ_ATOMIC; if ((mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1454,16 +1455,6 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; - good_sectors = first_bad - dev_sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1483,6 +1474,9 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, r10_bio->sectors = max_sectors; if (r10_bio->sectors < bio_sectors(bio)) { + if (atomic) + goto err_handle; + allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); @@ -1625,6 +1619,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { bio_wouldblock_error(bio); + md_write_end(mddev); return 0; } @@ -1667,6 +1662,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -1684,6 +1681,8 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (IS_ERR(split)) { bio->bi_status = errno_to_blk_status(PTR_ERR(split)); bio_endio(bio); + md_write_end(mddev); + allow_barrier(conf); return 0; } @@ -3384,7 +3383,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, struct md_rdev *rdev = conf->mirrors[j].rdev; if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { - still_degraded = false; + still_degraded = true; break; } } diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 56b234683ee6b..2678e82bfa52d 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io) log->disk_flush_bitmap = 0; for (i = flushed_disks ; i < raid_disks; i++) { - if (atomic_dec_and_test(&io->pending_flushes)) + if (atomic_dec_and_test(&io->pending_flushes)) { ppl_io_unit_finished(io); + break; + } } } diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index b19d870e41719..03d11f4420b09 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -996,7 +996,7 @@ static void stripe_add_to_batch_list(struct r5conf *conf, if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { int seq = sh->bm_seq; if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && - sh->batch_head->bm_seq > seq) + sh->batch_head->bm_seq - seq > 0) seq = sh->batch_head->bm_seq; set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); sh->batch_head->bm_seq = seq; @@ -2456,11 +2456,6 @@ static int scribble_alloc(struct raid5_percpu *percpu, sizeof(unsigned int) * (num + 2); void *scribble; - /* - * If here is in raid array suspend context, it is in memalloc noio - * context as well, there is no potential recursive memory reclaim - * I/Os with the GFP_KERNEL flag. - */ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2475,6 +2470,7 @@ static int scribble_alloc(struct raid5_percpu *percpu, static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) { unsigned long cpu; + unsigned int noio_flags; int err = 0; /* Never shrink. */ @@ -2483,6 +2479,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; raid5_quiesce(conf->mddev, true); + noio_flags = memalloc_noio_save(); cpus_read_lock(); for_each_present_cpu(cpu) { @@ -2496,6 +2493,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) } cpus_read_unlock(); + memalloc_noio_restore(noio_flags); raid5_quiesce(conf->mddev, false); if (!err) { @@ -3541,7 +3539,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, sh->dev[dd_idx].sector); if (conf->mddev->bitmap && firstwrite && !sh->batch_head) { - sh->bm_seq = conf->seq_flush+1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } } @@ -3740,11 +3738,10 @@ static int want_replace(struct stripe_head *sh, int disk_idx) int rv = 0; rdev = sh->raid_conf->disks[disk_idx].replacement; - if (rdev - && !test_bit(Faulty, &rdev->flags) - && !test_bit(In_sync, &rdev->flags) - && (rdev->recovery_offset <= sh->sector - || rdev->mddev->resync_offset <= sh->sector)) + if (rdev && !test_bit(Faulty, &rdev->flags) && + !test_bit(In_sync, &rdev->flags) && + (READ_ONCE(rdev->recovery_offset) <= sh->sector || + rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -4673,7 +4670,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) */ rdev = conf->disks[i].replacement; if (rdev && !test_bit(Faulty, &rdev->flags) && - rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && + READ_ONCE(rdev->recovery_offset) >= + sh->sector + RAID5_STRIPE_SECTORS(conf) && !rdev_has_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf))) set_bit(R5_ReadRepl, &dev->flags); @@ -4715,7 +4713,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) } else if (test_bit(In_sync, &rdev->flags)) set_bit(R5_Insync, &dev->flags); else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= - rdev->recovery_offset) { + READ_ONCE(rdev->recovery_offset)) { /* * in sync if: * - normal IO, or @@ -5448,13 +5446,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) rdev = conf->disks[dd_idx].replacement; if (!rdev || test_bit(Faulty, &rdev->flags) || - rdev->recovery_offset < end_sector) { + READ_ONCE(rdev->recovery_offset) < end_sector) { rdev = conf->disks[dd_idx].rdev; if (!rdev) return 0; if (test_bit(Faulty, &rdev->flags) || !(test_bit(In_sync, &rdev->flags) || - rdev->recovery_offset >= end_sector)) + READ_ONCE(rdev->recovery_offset) >= end_sector)) return 0; } @@ -5702,7 +5700,10 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) { struct r5conf *conf = mddev->private; sector_t logical_sector, last_sector; + sector_t first_stripe, last_stripe; struct stripe_head *sh; + struct bvec_iter bi_iter; + struct bio *orig_bi = bi; int stripe_sectors; /* We need to handle this when io_uring supports discard/trim */ @@ -5713,19 +5714,29 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) /* Skip discard while reshape is happening */ return; - logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1); - last_sector = bio_end_sector(bi); - - bi->bi_next = NULL; - stripe_sectors = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, - stripe_sectors); - sector_div(last_sector, stripe_sectors); + first_stripe = DIV_ROUND_UP_SECTOR_T(bi->bi_iter.bi_sector, + stripe_sectors); + last_stripe = bio_end_sector(bi); + sector_div(last_stripe, stripe_sectors); + + if (first_stripe >= last_stripe) { + bio_endio(bi); + return; + } - logical_sector *= conf->chunk_sectors; - last_sector *= conf->chunk_sectors; + bi_iter = bi->bi_iter; + bi->bi_iter.bi_sector = first_stripe * stripe_sectors; + bi->bi_iter.bi_size = ((last_stripe - first_stripe) * + stripe_sectors) << 9; + md_account_bio(mddev, &bi); + orig_bi->bi_iter = bi_iter; + bi->bi_iter = bi_iter; + bi->bi_next = NULL; + + logical_sector = first_stripe * conf->chunk_sectors; + last_sector = last_stripe * conf->chunk_sectors; for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) { @@ -5767,7 +5778,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } spin_unlock_irq(&sh->stripe_lock); if (conf->mddev->bitmap) { - sh->bm_seq = conf->seq_flush + 1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } @@ -5924,8 +5935,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - start = round_down(start, sectors_per_chunk); - end = round_up(end, sectors_per_chunk); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); @@ -5943,8 +5957,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->prev_chunk_sectors * (conf->previous_raid_disks - conf->max_degraded); - prev_start = round_down(prev_start, sectors_per_chunk); - prev_end = round_down(prev_end, sectors_per_chunk); + sector_div(prev_start, sectors_per_chunk); + prev_start *= sectors_per_chunk; + sector_div(prev_end, sectors_per_chunk); + prev_end *= sectors_per_chunk; prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); @@ -6054,8 +6070,11 @@ static enum stripe_result make_stripe_request(struct mddev *mddev, raid5_release_stripe(sh); out: if (ret == STRIPE_SCHEDULE_AND_RETRY && reshape_interrupted(mddev)) { - bi->bi_status = BLK_STS_RESOURCE; - ret = STRIPE_WAIT_RESHAPE; + if (!mddev_is_dm(mddev) || + test_bit(MD_DM_SUSPENDING, &mddev->flags)) { + bi->bi_status = BLK_STS_RESOURCE; + ret = STRIPE_WAIT_RESHAPE; + } pr_err_ratelimited("dm-raid456: io across reshape position while reshape can't make progress"); } return ret; @@ -6380,8 +6399,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); @@ -6489,8 +6508,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); @@ -6802,12 +6821,14 @@ static void raid5d(struct md_thread *thread) if ( !list_empty(&conf->bitmap_list)) { /* Now is a good time to flush some bitmap updates */ - conf->seq_flush++; + int seq = conf->seq_flush + 1; + + WRITE_ONCE(conf->seq_flush, seq); spin_unlock_irq(&conf->device_lock); if (md_bitmap_enabled(mddev, true)) mddev->bitmap_ops->unplug(mddev, true); spin_lock_irq(&conf->device_lock); - conf->seq_write = conf->seq_flush; + conf->seq_write = seq; activate_bit_delay(conf, conf->temp_inactive_list); } raid5_activate_delayed(conf); @@ -6992,6 +7013,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) { struct r5conf *conf; unsigned long new; + unsigned int noio_flags = 0; int err; int size; @@ -7032,6 +7054,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) goto out_unlock; } + noio_flags = memalloc_noio_save(); mutex_lock(&conf->cache_size_mutex); size = conf->max_nr_stripes; @@ -7048,6 +7071,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) mutex_unlock(&conf->cache_size_mutex); out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ?: len; } @@ -7999,9 +8023,9 @@ static int raid5_run(struct mddev *mddev) /* Hack because v0.91 doesn't store recovery_offset properly. */ if (mddev->major_version == 0 && mddev->minor_version > 90) - rdev->recovery_offset = reshape_offset; + WRITE_ONCE(rdev->recovery_offset, reshape_offset); - if (rdev->recovery_offset < reshape_offset) { + if (READ_ONCE(rdev->recovery_offset) < reshape_offset) { /* We need to check old and new layout */ if (!only_parity(rdev->raid_disk, conf->algorithm, @@ -8152,10 +8176,10 @@ static int raid5_spare_active(struct mddev *mddev) for (i = 0; i < conf->raid_disks; i++) { rdev = conf->disks[i].rdev; replacement = conf->disks[i].replacement; - if (replacement - && replacement->recovery_offset == MaxSector - && !test_bit(Faulty, &replacement->flags) - && !test_and_set_bit(In_sync, &replacement->flags)) { + if (replacement && + READ_ONCE(replacement->recovery_offset) == MaxSector && + !test_bit(Faulty, &replacement->flags) && + !test_and_set_bit(In_sync, &replacement->flags)) { /* Replacement has just become active. */ if (!rdev || !test_and_clear_bit(In_sync, &rdev->flags)) @@ -8170,10 +8194,10 @@ static int raid5_spare_active(struct mddev *mddev) rdev->sysfs_state); } sysfs_notify_dirent_safe(replacement->sysfs_state); - } else if (rdev - && rdev->recovery_offset == MaxSector - && !test_bit(Faulty, &rdev->flags) - && !test_and_set_bit(In_sync, &rdev->flags)) { + } else if (rdev && + READ_ONCE(rdev->recovery_offset) == MaxSector && + !test_bit(Faulty, &rdev->flags) && + !test_and_set_bit(In_sync, &rdev->flags)) { count++; sysfs_notify_dirent_safe(rdev->sysfs_state); } @@ -8545,7 +8569,7 @@ static int raid5_start_reshape(struct mddev *mddev) >= conf->previous_raid_disks) set_bit(In_sync, &rdev->flags); else - rdev->recovery_offset = 0; + WRITE_ONCE(rdev->recovery_offset, 0); /* Failure here is OK */ sysfs_link_rdev(mddev, rdev); @@ -8597,7 +8621,7 @@ static void end_reshape(struct r5conf *conf) if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags)) - rdev->recovery_offset = MaxSector; + WRITE_ONCE(rdev->recovery_offset, MaxSector); spin_unlock_irq(&conf->device_lock); wake_up(&conf->wait_for_reshape); @@ -8917,6 +8941,7 @@ static void *raid6_takeover(struct mddev *mddev) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) { struct r5conf *conf; + unsigned int noio_flags; int err; err = mddev_suspend_and_lock(mddev); @@ -8928,6 +8953,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) return -ENODEV; } + noio_flags = memalloc_noio_save(); if (strncmp(buf, "ppl", 3) == 0) { /* ppl only works with RAID 5 */ if (!raid5_has_ppl(conf) && conf->level == 5) { @@ -8967,6 +8993,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) if (!err) md_update_sb(mddev, 1); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err; diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c index ba6828ef540e0..84eaf9780db25 100644 --- a/drivers/media/cec/core/cec-adap.c +++ b/drivers/media/cec/core/cec-adap.c @@ -965,7 +965,7 @@ int cec_transmit_msg_fh(struct cec_adapter *adap, struct cec_msg *msg, */ mutex_unlock(&adap->lock); err = wait_for_completion_killable(&data->c); - cancel_delayed_work_sync(&data->work); + disable_delayed_work_sync(&data->work); mutex_lock(&adap->lock); if (err) @@ -2210,9 +2210,13 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg, * Unprocessed messages are aborted if userspace isn't doing * any processing either. */ + mutex_lock(&adap->lock); if (!is_broadcast && !is_reply && !adap->follower_cnt && - !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT) + !adap->cec_follower && msg->msg[1] != CEC_MSG_FEATURE_ABORT) { + mutex_unlock(&adap->lock); return cec_feature_abort(adap, msg); + } + mutex_unlock(&adap->lock); break; } @@ -2225,10 +2229,12 @@ static int cec_receive_notify(struct cec_adapter *adap, struct cec_msg *msg, * Send to the exclusive follower if there is one, otherwise send * to all followers. */ + mutex_lock(&adap->lock); if (adap->cec_follower) cec_queue_msg_fh(adap->cec_follower, msg); else cec_queue_msg_followers(adap, msg); + mutex_unlock(&adap->lock); return 0; } diff --git a/drivers/media/cec/core/cec-core.c b/drivers/media/cec/core/cec-core.c index 1b8a33c05b3c9..d7b632c836883 100644 --- a/drivers/media/cec/core/cec-core.c +++ b/drivers/media/cec/core/cec-core.c @@ -371,9 +371,7 @@ void cec_unregister_adapter(struct cec_adapter *adap) return; #ifdef CONFIG_MEDIA_CEC_RC - /* Note: rc_unregister also calls rc_free */ rc_unregister_device(adap->rc); - adap->rc = NULL; #endif debugfs_remove_recursive(adap->cec_dir); #ifdef CONFIG_CEC_NOTIFIER diff --git a/drivers/media/cec/core/cec-pin.c b/drivers/media/cec/core/cec-pin.c index 4d7155281daae..754a4bfcc87c7 100644 --- a/drivers/media/cec/core/cec-pin.c +++ b/drivers/media/cec/core/cec-pin.c @@ -115,7 +115,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) return; pin->adap->cec_pin_is_high = v; - if (atomic_read(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { + if (atomic_read_acquire(&pin->work_pin_num_events) < CEC_NUM_PIN_EVENTS) { u8 ev = v; if (pin->work_pin_events_dropped) { @@ -126,7 +126,7 @@ static void cec_pin_update(struct cec_pin *pin, bool v, bool force) pin->work_pin_ts[pin->work_pin_events_wr] = ktime_get(); pin->work_pin_events_wr = (pin->work_pin_events_wr + 1) % CEC_NUM_PIN_EVENTS; - atomic_inc(&pin->work_pin_num_events); + atomic_inc_return_release(&pin->work_pin_num_events); } else { pin->work_pin_events_dropped = true; pin->work_pin_events_dropped_cnt++; @@ -1101,7 +1101,7 @@ static int cec_pin_thread_func(void *_adap) pin->work_tx_ts); } - while (atomic_read(&pin->work_pin_num_events)) { + while (atomic_read_acquire(&pin->work_pin_num_events)) { unsigned int idx = pin->work_pin_events_rd; u8 v = pin->work_pin_events[idx]; @@ -1110,7 +1110,7 @@ static int cec_pin_thread_func(void *_adap) v & CEC_PIN_EVENT_FL_DROPPED, pin->work_pin_ts[idx]); pin->work_pin_events_rd = (idx + 1) % CEC_NUM_PIN_EVENTS; - atomic_dec(&pin->work_pin_num_events); + atomic_dec_return_release(&pin->work_pin_num_events); } switch (atomic_xchg(&pin->work_irq_change, diff --git a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c index 419b9a7abcceb..4b3f9bff6067a 100644 --- a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c +++ b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c @@ -334,6 +334,10 @@ static const struct cec_dmi_match cec_dmi_match_table[] = { { "Google", "Dirks", "0000:00:02.0", port_ab_conns }, /* Google Moxie */ { "Google", "Moxie", "0000:00:02.0", port_b_conns }, + /* Google Kulnex */ + { "Google", "Kulnex", "0000:00:02.0", port_b_conns }, + /* Google Moxoe */ + { "Google", "Moxoe", "0000:00:02.0", port_b_conns }, }; static struct device *cros_ec_cec_find_hdmi_dev(struct device *dev, diff --git a/drivers/media/cec/platform/meson/ao-cec-g12a.c b/drivers/media/cec/platform/meson/ao-cec-g12a.c index 41f5b8669cb01..2c914f0008b50 100644 --- a/drivers/media/cec/platform/meson/ao-cec-g12a.c +++ b/drivers/media/cec/platform/meson/ao-cec-g12a.c @@ -405,6 +405,7 @@ static int meson_ao_cec_g12a_write(void *context, unsigned int addr, } static const struct regmap_config meson_ao_cec_g12a_cec_regmap_conf = { + .name = "core", .reg_bits = 8, .val_bits = 8, .reg_read = meson_ao_cec_g12a_read, diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c index b7bb49f023957..97ed9654c78a1 100644 --- a/drivers/media/cec/platform/seco/seco-cec.c +++ b/drivers/media/cec/platform/seco/seco-cec.c @@ -649,7 +649,7 @@ static int secocec_probe(struct platform_device *pdev) ret = secocec_ir_probe(secocec); if (ret) - goto err_notifier; + goto err_unregister_adapter; platform_set_drvdata(pdev, secocec); @@ -657,6 +657,10 @@ static int secocec_probe(struct platform_device *pdev) return ret; +err_unregister_adapter: + cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap); + cec_unregister_adapter(secocec->cec_adap); + goto err; err_notifier: cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap); err_delete_adapter: diff --git a/drivers/media/cec/platform/stm32/stm32-cec.c b/drivers/media/cec/platform/stm32/stm32-cec.c index 1ec0cece0a5b7..8c2fc232202de 100644 --- a/drivers/media/cec/platform/stm32/stm32-cec.c +++ b/drivers/media/cec/platform/stm32/stm32-cec.c @@ -132,7 +132,8 @@ static void stm32_rx_done(struct stm32_cec *cec, u32 status) u32 val; regmap_read(cec->regmap, CEC_RXDR, &val); - cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF; + if (cec->rx_msg.len < CEC_MAX_MSG_SIZE) + cec->rx_msg.msg[cec->rx_msg.len++] = val & 0xFF; } if (cec->irq_status & RXEND) { diff --git a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c index e2eff17952ab2..48410ce33490c 100644 --- a/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c +++ b/drivers/media/cec/usb/extron-da-hd-4k-plus/extron-da-hd-4k-plus.c @@ -656,7 +656,8 @@ static void extron_process_received(struct extron_port *port, const char *data) if (!port || port->disconnected) return; - if (len < 5 || (len - 2) % 3 || data[len - 2] != '*') + if (len < 5 || ((len - 2) / 3 > sizeof(msg.msg)) || + (len - 2) % 3 || data[len - 2] != '*') goto malformed; while (*data != '*') { diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c index f316864d99fc5..a20b2b3df5355 100644 --- a/drivers/media/common/videobuf2/videobuf2-core.c +++ b/drivers/media/common/videobuf2/videobuf2-core.c @@ -3021,8 +3021,8 @@ static int __vb2_cleanup_fileio(struct vb2_queue *q) * @nonblock: mode selector (1 means blocking calls, 0 means nonblocking) * @read: access mode selector (1 means read, 0 means write) */ -static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count, - loff_t *ppos, int nonblock, int read) +static ssize_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count, + loff_t *ppos, int nonblock, int read) { struct vb2_fileio_data *fileio; struct vb2_fileio_buf *buf; @@ -3185,15 +3185,15 @@ static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_ return ret; } -size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count, - loff_t *ppos, int nonblocking) +ssize_t vb2_read(struct vb2_queue *q, char __user *data, size_t count, + loff_t *ppos, int nonblocking) { return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1); } EXPORT_SYMBOL_GPL(vb2_read); -size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count, - loff_t *ppos, int nonblocking) +ssize_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count, + loff_t *ppos, int nonblocking) { return __vb2_perform_fileio(q, (char __user *) data, count, ppos, nonblocking, 0); diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c index 3b4e46dac1bf1..16f5f7e430922 100644 --- a/drivers/media/dvb-frontends/rtl2832.c +++ b/drivers/media/dvb-frontends/rtl2832.c @@ -1115,10 +1115,10 @@ static void rtl2832_remove(struct i2c_client *client) dev_dbg(&client->dev, "\n"); - cancel_delayed_work_sync(&dev->i2c_gate_work); - i2c_mux_del_adapters(dev->muxc); + cancel_delayed_work_sync(&dev->i2c_gate_work); + regmap_exit(dev->regmap); kfree(dev); diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c index 0357624968f1b..83c472f61d390 100644 --- a/drivers/media/dvb-frontends/rtl2832_sdr.c +++ b/drivers/media/dvb-frontends/rtl2832_sdr.c @@ -399,7 +399,8 @@ static int rtl2832_sdr_alloc_urbs(struct rtl2832_sdr_dev *dev) } /* Must be called with vb_queue_lock hold */ -static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev) +static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev, + enum vb2_buffer_state state) { struct platform_device *pdev = dev->pdev; unsigned long flags; @@ -413,7 +414,7 @@ static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev) buf = list_entry(dev->queued_bufs.next, struct rtl2832_sdr_frame_buf, list); list_del(&buf->list); - vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + vb2_buffer_done(&buf->vb.vb2_buf, state); } spin_unlock_irqrestore(&dev->queued_bufs_lock, flags); } @@ -855,11 +856,15 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count) dev_dbg(&pdev->dev, "\n"); - if (!dev->udev) + if (!dev->udev) { + rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED); return -ENODEV; + } - if (mutex_lock_interruptible(&dev->v4l2_lock)) + if (mutex_lock_interruptible(&dev->v4l2_lock)) { + rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED); return -ERESTARTSYS; + } if (d->props->power_ctrl) d->props->power_ctrl(d, 1); @@ -900,7 +905,14 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count) if (ret) goto err; + mutex_unlock(&dev->v4l2_lock); + + return 0; + err: + rtl2832_sdr_free_urbs(dev); + rtl2832_sdr_free_stream_bufs(dev); + rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED); mutex_unlock(&dev->v4l2_lock); return ret; @@ -920,7 +932,7 @@ static void rtl2832_sdr_stop_streaming(struct vb2_queue *vq) rtl2832_sdr_kill_urbs(dev); rtl2832_sdr_free_urbs(dev); rtl2832_sdr_free_stream_bufs(dev); - rtl2832_sdr_cleanup_queued_bufs(dev); + rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR); rtl2832_sdr_unset_adc(dev); /* sleep tuner */ @@ -1468,14 +1480,22 @@ static void rtl2832_sdr_remove(struct platform_device *pdev) dev_dbg(&pdev->dev, "\n"); - mutex_lock(&dev->vb_queue_lock); + /* + * vb2_video_unregister_device() releases the vb2 queue, which + * triggers rtl2832_sdr_stop_streaming() if streaming is active. + * stop_streaming() uses dev->udev to free URBs and coherent DMA + * stream buffers via usb_free_coherent(), so it must run before + * dev->udev is cleared. vb2_video_unregister_device() locks + * vb_queue_lock internally and stop_streaming() locks v4l2_lock, + * so neither may be held by the caller. + */ + v4l2_device_disconnect(&dev->v4l2_dev); + vb2_video_unregister_device(&dev->vdev); + mutex_lock(&dev->v4l2_lock); - /* No need to keep the urbs around after disconnection */ dev->udev = NULL; - v4l2_device_disconnect(&dev->v4l2_dev); - video_unregister_device(&dev->vdev); mutex_unlock(&dev->v4l2_lock); - mutex_unlock(&dev->vb_queue_lock); + v4l2_device_put(&dev->v4l2_dev); module_put(pdev->dev.parent->driver->owner); } diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c index 1f088acecf367..3d399af013222 100644 --- a/drivers/media/i2c/alvium-csi2.c +++ b/drivers/media/i2c/alvium-csi2.c @@ -1290,7 +1290,7 @@ static int alvium_set_ctrl_auto_exposure(struct alvium_dev *alvium, bool on) struct device *dev = &alvium->i2c_client->dev; int ret; - ret = alvium_write_hshake(alvium, REG_BCRM_WHITE_BALANCE_AUTO_RW, + ret = alvium_write_hshake(alvium, REG_BCRM_EXPOSURE_AUTO_RW, on ? 0x02 : 0x00); if (ret) { dev_err(dev, "Fail to set autoexposure reg\n"); @@ -2100,20 +2100,21 @@ static int alvium_ctrl_init(struct alvium_dev *alvium) V4L2_CID_PIXEL_RATE, 0, ALVIUM_DEFAULT_PIXEL_RATE_MHZ, 1, ALVIUM_DEFAULT_PIXEL_RATE_MHZ); - ctrls->pixel_rate->flags |= V4L2_CTRL_FLAG_READ_ONLY; /* Link freq is fixed */ ctrls->link_freq = v4l2_ctrl_new_int_menu(hdl, ops, V4L2_CID_LINK_FREQ, 0, 0, &alvium->link_freq); - ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY; + if (ctrls->link_freq) + ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY; /* Auto/manual white balance */ if (alvium->avail_ft.auto_whiteb) { ctrls->auto_wb = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_AUTO_WHITE_BALANCE, 0, 1, 1, 1); - v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false); + if (ctrls->auto_wb) + v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false); } ctrls->blue_balance = v4l2_ctrl_new_std(hdl, ops, @@ -2122,6 +2123,7 @@ static int alvium_ctrl_init(struct alvium_dev *alvium) alvium->max_bbalance, alvium->inc_bbalance, alvium->dft_bbalance); + ctrls->red_balance = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_RED_BALANCE, alvium->min_rbalance, @@ -2136,7 +2138,9 @@ static int alvium_ctrl_init(struct alvium_dev *alvium) V4L2_CID_EXPOSURE_AUTO, V4L2_EXPOSURE_MANUAL, 0, V4L2_EXPOSURE_AUTO); - v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp, 1, true); + if (ctrls->auto_exp) + v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp, + V4L2_EXPOSURE_MANUAL, true); } ctrls->exposure = v4l2_ctrl_new_std(hdl, ops, @@ -2145,14 +2149,16 @@ static int alvium_ctrl_init(struct alvium_dev *alvium) alvium->max_exp, alvium->inc_exp, alvium->dft_exp); - ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE; + if (ctrls->exposure) + ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE; /* Auto/manual gain */ if (alvium->avail_ft.auto_gain) { ctrls->auto_gain = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_AUTOGAIN, 0, 1, 1, 1); - v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true); + if (ctrls->auto_gain) + v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true); } if (alvium->avail_ft.gain) { @@ -2162,7 +2168,8 @@ static int alvium_ctrl_init(struct alvium_dev *alvium) alvium->max_gain, alvium->inc_gain, alvium->dft_gain); - ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE; + if (ctrls->gain) + ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE; } if (alvium->avail_ft.sat) diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index 4734c7b4d37b9..00aec91e3d499 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -71,7 +71,7 @@ /* V_TIMING internal */ #define IMX219_REG_FRM_LENGTH_A CCI_REG16(0x0160) -#define IMX219_FLL_MAX 0xffff +#define IMX219_FLL_MAX 0xfffe #define IMX219_VBLANK_MIN 32 #define IMX219_REG_LINE_LENGTH_A CCI_REG16(0x0162) #define IMX219_LLP_MIN 0x0d78 diff --git a/drivers/media/i2c/imx296.c b/drivers/media/i2c/imx296.c index 3ced3af3897fe..fde64a4e7458f 100644 --- a/drivers/media/i2c/imx296.c +++ b/drivers/media/i2c/imx296.c @@ -24,6 +24,14 @@ static int trigger_mode; module_param(trigger_mode, int, 0644); MODULE_PARM_DESC(trigger_mode, "Set trigger mode: 0=default, 1=XTRIG"); +static int tout_width; +module_param(tout_width, int, 0644); +MODULE_PARM_DESC(tout_width, "Strobe output width in scanlines; 0=disabled"); + +static int tout_delay; +module_param(tout_delay, int, 0644); +MODULE_PARM_DESC(tout_delay, "Strobe output delay in scanlines (wrt XVS)"); + #define IMX296_PIXEL_ARRAY_WIDTH 1456 #define IMX296_PIXEL_ARRAY_HEIGHT 1088 @@ -89,6 +97,7 @@ MODULE_PARM_DESC(trigger_mode, "Set trigger mode: 0=default, 1=XTRIG"); #define IMX296_PULSE2_EN_TRIG BIT(1) #define IMX296_PULSE2_POL_HIGH (0 << 2) #define IMX296_PULSE2_POL_LOW (1 << 2) +#define IMX296_PULSE2_FIX1 BIT(3) #define IMX296_PULSE2_UP IMX296_REG_24BIT(0x307c) #define IMX296_PULSE2_DN IMX296_REG_24BIT(0x3080) #define IMX296_INCKSEL(n) IMX296_REG_8BIT(0x3089 + (n)) @@ -644,6 +653,30 @@ static int imx296_setup(struct imx296 *sensor, struct v4l2_subdev_state *state) return ret; } +static void imx296_enable_tout(struct imx296 *sensor, bool enable) +{ + imx296_write(sensor, IMX296_CTRLTOUT, + enable ? + (IMX296_CTRLTOUT_TOUT1SEL_PULSE | IMX296_CTRLTOUT_TOUT2SEL_PULSE) : 0, + 0); + imx296_write(sensor, IMX296_CTRLTRIG, + enable ? + (IMX296_CTRLTRIG_TOUT1_SEL_PULSE1 | IMX296_CTRLTRIG_TOUT2_SEL_PULSE2) : 0, + 0); + imx296_write(sensor, IMX296_PULSE1, enable ? + (IMX296_PULSE1_EN_NOR | IMX296_PULSE1_POL_HIGH) : 0, + 0); + imx296_write(sensor, IMX296_PULSE2, enable ? + (IMX296_PULSE2_EN_NOR | IMX296_PULSE2_POL_HIGH | IMX296_PULSE2_FIX1) : + IMX296_PULSE2_FIX1, 0); + if (enable) { + imx296_write(sensor, IMX296_PULSE1_UP, tout_delay, 0); + imx296_write(sensor, IMX296_PULSE1_DN, tout_delay + tout_width, 0); + imx296_write(sensor, IMX296_PULSE2_UP, tout_delay, 0); + imx296_write(sensor, IMX296_PULSE2_DN, tout_delay + tout_width, 0); + } +} + static int imx296_stream_on(struct imx296 *sensor) { int ret = 0, tm; @@ -658,6 +691,9 @@ static int imx296_stream_on(struct imx296 *sensor) imx296_write(sensor, IMX296_LOWLAGTRG, (tm == 1) ? IMX296_LOWLAGTRG_FAST : 0, &ret); + /* Enable TOUT[12] when running in "normal" mode only */ + imx296_enable_tout(sensor, tout_width > 0 && tm == 0); + imx296_write(sensor, IMX296_CTRL0A, 0, &ret); /* vflip and hflip cannot change during streaming */ diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c index 6ccb087da4f02..8e7110d5855ec 100644 --- a/drivers/media/i2c/imx355.c +++ b/drivers/media/i2c/imx355.c @@ -1275,14 +1275,15 @@ static int imx355_probe(struct i2c_client *client) */ pm_runtime_set_active(imx355->dev); pm_runtime_enable(imx355->dev); - pm_runtime_idle(imx355->dev); - pm_runtime_set_autosuspend_delay(imx355->dev, 1000); - pm_runtime_use_autosuspend(imx355->dev); ret = v4l2_async_register_subdev_sensor(&imx355->sd); if (ret < 0) goto error_media_entity_runtime_pm; + pm_runtime_idle(imx355->dev); + pm_runtime_set_autosuspend_delay(imx355->dev, 1000); + pm_runtime_use_autosuspend(imx355->dev); + return 0; error_media_entity_runtime_pm: diff --git a/drivers/media/i2c/imx415.c b/drivers/media/i2c/imx415.c index 0b424c17e8804..c3b22b238ea0b 100644 --- a/drivers/media/i2c/imx415.c +++ b/drivers/media/i2c/imx415.c @@ -686,7 +686,7 @@ static int imx415_set_testpattern(struct imx415 *sensor, int val) cci_write(sensor->regmap, IMX415_DIG_CLP_MODE, 0x01, &ret); cci_write(sensor->regmap, IMX415_WRJ_OPEN, 0x01, &ret); } - return 0; + return ret; } static int imx415_s_ctrl(struct v4l2_ctrl *ctrl) @@ -720,7 +720,7 @@ static int imx415_s_ctrl(struct v4l2_ctrl *ctrl) ret = cci_write(sensor->regmap, IMX415_VMAX, format->height + ctrl->val, NULL); if (ret) - return ret; + break; /* * Exposure is set based on VMAX which has just changed, so * program exposure register as well diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c index 1500ee4db47ec..a0375945e7898 100644 --- a/drivers/media/i2c/mt9p031.c +++ b/drivers/media/i2c/mt9p031.c @@ -453,11 +453,7 @@ static int mt9p031_set_params(struct mt9p031 *mt9p031) ret = mt9p031_write(client, MT9P031_HORIZONTAL_BLANK, hblank - 1); if (ret < 0) return ret; - ret = mt9p031_write(client, MT9P031_VERTICAL_BLANK, vblank - 1); - if (ret < 0) - return ret; - - return ret; + return mt9p031_write(client, MT9P031_VERTICAL_BLANK, vblank - 1); } static int mt9p031_s_stream(struct v4l2_subdev *subdev, int enable) diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c index 70d9d7c43f187..895f41dc38ee2 100644 --- a/drivers/media/i2c/ov02a10.c +++ b/drivers/media/i2c/ov02a10.c @@ -820,18 +820,16 @@ static int ov02a10_check_hwcfg(struct device *dev, struct ov02a10 *ov02a10) if (!ep) return -ENXIO; + /* Optional indication of MIPI clock voltage unit */ + if (!fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage", + &clk_volt)) + ov02a10->mipi_clock_voltage = clk_volt; + ret = v4l2_fwnode_endpoint_alloc_parse(ep, &bus_cfg); fwnode_handle_put(ep); if (ret) return ret; - /* Optional indication of MIPI clock voltage unit */ - ret = fwnode_property_read_u32(ep, "ovti,mipi-clock-voltage", - &clk_volt); - - if (!ret) - ov02a10->mipi_clock_voltage = clk_volt; - for (i = 0; i < ARRAY_SIZE(link_freq_menu_items); i++) { for (j = 0; j < bus_cfg.nr_of_link_frequencies; j++) { if (link_freq_menu_items[i] == diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c index 632fb80469bef..62c124a1353a2 100644 --- a/drivers/media/i2c/ov7740.c +++ b/drivers/media/i2c/ov7740.c @@ -1116,10 +1116,8 @@ static void ov7740_remove(struct i2c_client *client) struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); - mutex_destroy(&ov7740->mutex); - v4l2_ctrl_handler_free(ov7740->subdev.ctrl_handler); - media_entity_cleanup(&ov7740->subdev.entity); v4l2_async_unregister_subdev(sd); + media_entity_cleanup(&ov7740->subdev.entity); ov7740_free_controls(ov7740); pm_runtime_get_sync(&client->dev); diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index bcab462708c70..ece8a410e7ced 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -588,10 +588,12 @@ static int rdacm21_probe(struct i2c_client *client) ret = v4l2_async_register_subdev(&dev->sd); if (ret) - goto error_free_ctrls; + goto error_entity_cleanup; return 0; +error_entity_cleanup: + media_entity_cleanup(&dev->sd.entity); error_free_ctrls: v4l2_ctrl_handler_free(&dev->ctrls); error: @@ -606,6 +608,7 @@ static void rdacm21_remove(struct i2c_client *client) v4l2_async_unregister_subdev(&dev->sd); v4l2_ctrl_handler_free(&dev->ctrls); + media_entity_cleanup(&dev->sd.entity); i2c_unregister_device(dev->isp); } diff --git a/drivers/media/i2c/vd55g1.c b/drivers/media/i2c/vd55g1.c index f09d6bf326413..6f458f611f634 100644 --- a/drivers/media/i2c/vd55g1.c +++ b/drivers/media/i2c/vd55g1.c @@ -29,9 +29,7 @@ /* Register Map */ #define VD55G1_REG_MODEL_ID CCI_REG32_LE(0x0000) -#define VD55G1_MODEL_ID 0x53354731 -#define VD55G1_REG_REVISION CCI_REG16_LE(0x0004) -#define VD55G1_REVISION_CCB 0x2020 +#define VD55G1_REG_COLOR_VERSION CCI_REG32_LE(0x0670) #define VD55G1_REG_FWPATCH_REVISION CCI_REG16_LE(0x0012) #define VD55G1_REG_FWPATCH_START_ADDR CCI_REG8(0x2000) #define VD55G1_REG_SYSTEM_FSM CCI_REG8(0x001c) @@ -39,7 +37,8 @@ #define VD55G1_SYSTEM_FSM_SW_STBY 0x02 #define VD55G1_SYSTEM_FSM_STREAMING 0x03 #define VD55G1_REG_BOOT CCI_REG8(0x0200) -#define VD55G1_BOOT_PATCH_SETUP 2 +#define VD55G1_BOOT_BOOT 1 +#define VD55G1_BOOT_PATCH_AND_BOOT 2 #define VD55G1_REG_STBY CCI_REG8(0x0201) #define VD55G1_STBY_START_STREAM 1 #define VD55G1_REG_STREAMING CCI_REG8(0x0202) @@ -57,7 +56,10 @@ #define VD55G1_PATGEN_ENABLE BIT(0) #define VD55G1_REG_MANUAL_ANALOG_GAIN CCI_REG8(0x0501) #define VD55G1_REG_MANUAL_COARSE_EXPOSURE CCI_REG16_LE(0x0502) -#define VD55G1_REG_MANUAL_DIGITAL_GAIN CCI_REG16_LE(0x0504) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0 CCI_REG16_LE(0x0504) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1 CCI_REG16_LE(0x0506) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2 CCI_REG16_LE(0x0508) +#define VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3 CCI_REG16_LE(0x050a) #define VD55G1_REG_APPLIED_COARSE_EXPOSURE CCI_REG16_LE(0x00e8) #define VD55G1_REG_APPLIED_ANALOG_GAIN CCI_REG16_LE(0x00ea) #define VD55G1_REG_APPLIED_DIGITAL_GAIN CCI_REG16_LE(0x00ec) @@ -111,9 +113,8 @@ #define VD55G1_WIDTH 804 #define VD55G1_HEIGHT 704 -#define VD55G1_MODE_DEF 0 +#define VD55G1_MODE_IDX_DEF 0 #define VD55G1_NB_GPIOS 4 -#define VD55G1_MBUS_CODE_DEF 0 #define VD55G1_DGAIN_DEF 256 #define VD55G1_AGAIN_DEF 19 #define VD55G1_EXPO_MAX_TERM 64 @@ -132,7 +133,41 @@ #define VD55G1_MIPI_RATE_MIN (250 * MEGA) #define VD55G1_MIPI_RATE_MAX (1200 * MEGA) -static const u8 patch_array[] = { +enum vd55g1_model_id { + VD55G1_MODEL_ID_2 = 0x53354731, + VD55G1_MODEL_ID_3 = 0x53354733, +}; + +enum vd55g1_color_version { + VD55G1_COLOR_VERSION_MONO, + VD55G1_COLOR_VERSION_BAYER, +}; + +struct vd55g1_version { + char *name; + enum vd55g1_model_id id; + enum vd55g1_color_version color; +}; + +static const struct vd55g1_version vd55g1_versions[] = { + { + .name = "vd55g1", + .id = VD55G1_MODEL_ID_2, + .color = VD55G1_COLOR_VERSION_MONO, + }, + { + .name = "vd55g4", + .id = VD55G1_MODEL_ID_3, + .color = VD55G1_COLOR_VERSION_MONO, + }, + { + .name = "vd65g4", + .id = VD55G1_MODEL_ID_3, + .color = VD55G1_COLOR_VERSION_BAYER, + }, +}; + +static const u8 vd55g1_patch_array[] = { 0x44, 0x03, 0x09, 0x02, 0xe6, 0x01, 0x42, 0x00, 0xea, 0x01, 0x42, 0x00, 0xf0, 0x01, 0x42, 0x00, 0xe6, 0x01, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -466,22 +501,24 @@ struct vd55g1_mode { u32 height; }; -struct vd55g1_fmt_desc { - u32 code; - u8 bpp; - u8 data_type; +static const u32 vd55g1_mbus_formats_mono[] = { + MEDIA_BUS_FMT_Y8_1X8, + MEDIA_BUS_FMT_Y10_1X10, }; -static const struct vd55g1_fmt_desc vd55g1_mbus_codes[] = { +/* Format order is : no flip, hflip, vflip, both */ +static const u32 vd55g1_mbus_formats_bayer[][4] = { { - .code = MEDIA_BUS_FMT_Y8_1X8, - .bpp = 8, - .data_type = MIPI_CSI2_DT_RAW8, + MEDIA_BUS_FMT_SRGGB8_1X8, + MEDIA_BUS_FMT_SGRBG8_1X8, + MEDIA_BUS_FMT_SGBRG8_1X8, + MEDIA_BUS_FMT_SBGGR8_1X8, }, { - .code = MEDIA_BUS_FMT_Y10_1X10, - .bpp = 10, - .data_type = MIPI_CSI2_DT_RAW10, + MEDIA_BUS_FMT_SRGGB10_1X10, + MEDIA_BUS_FMT_SGRBG10_1X10, + MEDIA_BUS_FMT_SGBRG10_1X10, + MEDIA_BUS_FMT_SBGGR10_1X10, }, }; @@ -524,6 +561,7 @@ struct vd55g1_vblank_limits { struct vd55g1 { struct device *dev; + const struct vd55g1_version *version; struct v4l2_subdev sd; struct media_pad pad; struct regulator_bulk_data supplies[ARRAY_SIZE(vd55g1_supply_name)]; @@ -572,27 +610,80 @@ static inline struct vd55g1 *ctrl_to_vd55g1(struct v4l2_ctrl *ctrl) return to_vd55g1(sd); } -static const struct vd55g1_fmt_desc *vd55g1_get_fmt_desc(struct vd55g1 *sensor, - u32 code) +static unsigned int vd55g1_get_fmt_bpp(u32 code) { - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_codes); i++) { - if (vd55g1_mbus_codes[i].code == code) - return &vd55g1_mbus_codes[i]; + switch (code) { + case MEDIA_BUS_FMT_Y8_1X8: + case MEDIA_BUS_FMT_SRGGB8_1X8: + case MEDIA_BUS_FMT_SGRBG8_1X8: + case MEDIA_BUS_FMT_SGBRG8_1X8: + case MEDIA_BUS_FMT_SBGGR8_1X8: + default: + return 8; + + case MEDIA_BUS_FMT_Y10_1X10: + case MEDIA_BUS_FMT_SRGGB10_1X10: + case MEDIA_BUS_FMT_SGRBG10_1X10: + case MEDIA_BUS_FMT_SGBRG10_1X10: + case MEDIA_BUS_FMT_SBGGR10_1X10: + return 10; } +} - /* Should never happen */ - dev_warn(sensor->dev, "Unsupported code %d. default to 8 bpp\n", code); - - return &vd55g1_mbus_codes[0]; +static unsigned int vd55g1_get_fmt_data_type(u32 code) +{ + switch (code) { + case MEDIA_BUS_FMT_Y8_1X8: + case MEDIA_BUS_FMT_SRGGB8_1X8: + case MEDIA_BUS_FMT_SGRBG8_1X8: + case MEDIA_BUS_FMT_SGBRG8_1X8: + case MEDIA_BUS_FMT_SBGGR8_1X8: + default: + return MIPI_CSI2_DT_RAW8; + + case MEDIA_BUS_FMT_Y10_1X10: + case MEDIA_BUS_FMT_SRGGB10_1X10: + case MEDIA_BUS_FMT_SGRBG10_1X10: + case MEDIA_BUS_FMT_SGBRG10_1X10: + case MEDIA_BUS_FMT_SBGGR10_1X10: + return MIPI_CSI2_DT_RAW10; + } } -static s32 vd55g1_get_pixel_rate(struct vd55g1 *sensor, - struct v4l2_mbus_framefmt *format) +static u32 vd55g1_get_fmt_code(struct vd55g1 *sensor, u32 code) { - return sensor->mipi_rate / - vd55g1_get_fmt_desc(sensor, format->code)->bpp; + u32 fallback_code; + unsigned int i, j; + + if (sensor->version->color == VD55G1_COLOR_VERSION_MONO) { + fallback_code = vd55g1_mbus_formats_mono[0]; + for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_formats_mono); i++) + if (vd55g1_mbus_formats_mono[i] == code) + return code; + } else { + fallback_code = vd55g1_mbus_formats_bayer[0][0]; + for (i = 0; i < ARRAY_SIZE(vd55g1_mbus_formats_bayer); i++) { + for (j = 0; + j < ARRAY_SIZE(vd55g1_mbus_formats_bayer[i]); + j++) { + if (vd55g1_mbus_formats_bayer[i][j] == code) + goto adapt_bayer_pattern; + } + } + } + dev_warn(sensor->dev, "Unsupported mbus format: 0x%x\n", code); + + return fallback_code; + +adapt_bayer_pattern: + j = 0; + /* In first init_state() call, controls might not be initialized yet */ + if (sensor->hflip_ctrl && sensor->vflip_ctrl) { + j = (sensor->hflip_ctrl->val ? 1 : 0) + + (sensor->vflip_ctrl->val ? 2 : 0); + } + + return vd55g1_mbus_formats_bayer[i][j]; } static unsigned int vd55g1_get_hblank_min(struct vd55g1 *sensor, @@ -605,7 +696,7 @@ static unsigned int vd55g1_get_hblank_min(struct vd55g1 *sensor, /* MIPI required time */ mipi_req_line_time = (crop->width * - vd55g1_get_fmt_desc(sensor, format->code)->bpp + + vd55g1_get_fmt_bpp(format->code) + VD55G1_MIPI_MARGIN) / (sensor->mipi_rate / MEGA); mipi_req_line_length = mipi_req_line_time * sensor->pixel_clock / @@ -790,9 +881,16 @@ static int vd55g1_update_expo_cluster(struct vd55g1 *sensor, bool is_auto) vd55g1_write(sensor, VD55G1_REG_MANUAL_ANALOG_GAIN, sensor->again_ctrl->val, &ret); - if (!is_auto && sensor->dgain_ctrl->is_new) - vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN, + if (!is_auto && sensor->dgain_ctrl->is_new) { + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH0, + sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH1, sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH2, + sensor->dgain_ctrl->val, &ret); + vd55g1_write(sensor, VD55G1_REG_MANUAL_DIGITAL_GAIN_CH3, + sensor->dgain_ctrl->val, &ret); + } return ret; } @@ -887,7 +985,7 @@ static void vd55g1_update_pad_fmt(struct vd55g1 *sensor, const struct vd55g1_mode *mode, u32 code, struct v4l2_mbus_framefmt *fmt) { - fmt->code = code; + fmt->code = vd55g1_get_fmt_code(sensor, code); fmt->width = mode->width; fmt->height = mode->height; fmt->colorspace = V4L2_COLORSPACE_RAW; @@ -951,10 +1049,9 @@ static int vd55g1_set_framefmt(struct vd55g1 *sensor, int ret = 0; vd55g1_write(sensor, VD55G1_REG_FORMAT_CTRL, - vd55g1_get_fmt_desc(sensor, format->code)->bpp, &ret); + vd55g1_get_fmt_bpp(format->code), &ret); vd55g1_write(sensor, VD55G1_REG_OIF_IMG_CTRL, - vd55g1_get_fmt_desc(sensor, format->code)->data_type, - &ret); + vd55g1_get_fmt_data_type(format->code), &ret); switch (crop->width / format->width) { case 1: @@ -1114,26 +1211,45 @@ static int vd55g1_patch(struct vd55g1 *sensor) u64 patch; int ret = 0; - vd55g1_write_array(sensor, VD55G1_REG_FWPATCH_START_ADDR, - sizeof(patch_array), patch_array, &ret); - vd55g1_write(sensor, VD55G1_REG_BOOT, VD55G1_BOOT_PATCH_SETUP, &ret); - vd55g1_poll_reg(sensor, VD55G1_REG_BOOT, 0, &ret); - if (ret) { - dev_err(sensor->dev, "Failed to apply patch\n"); - return ret; - } + /* Version 2 needs a patch while version 3 does not */ + if (sensor->version->id == VD55G1_MODEL_ID_2) { + vd55g1_write_array(sensor, VD55G1_REG_FWPATCH_START_ADDR, + sizeof(vd55g1_patch_array), + vd55g1_patch_array, &ret); + vd55g1_write(sensor, VD55G1_REG_BOOT, + VD55G1_BOOT_PATCH_AND_BOOT, &ret); + vd55g1_poll_reg(sensor, VD55G1_REG_BOOT, 0, &ret); + if (ret) { + dev_err(sensor->dev, "Failed to apply patch\n"); + return ret; + } - vd55g1_read(sensor, VD55G1_REG_FWPATCH_REVISION, &patch, &ret); - if (patch != (VD55G1_FWPATCH_REVISION_MAJOR << 8) + - VD55G1_FWPATCH_REVISION_MINOR) { - dev_err(sensor->dev, "Bad patch version expected %d.%d got %d.%d\n", - VD55G1_FWPATCH_REVISION_MAJOR, - VD55G1_FWPATCH_REVISION_MINOR, + vd55g1_read(sensor, VD55G1_REG_FWPATCH_REVISION, &patch, &ret); + if (patch != (VD55G1_FWPATCH_REVISION_MAJOR << 8) + + VD55G1_FWPATCH_REVISION_MINOR) { + dev_err(sensor->dev, "Bad patch version expected %d.%d got %d.%d\n", + VD55G1_FWPATCH_REVISION_MAJOR, + VD55G1_FWPATCH_REVISION_MINOR, + (u8)(patch >> 8), (u8)(patch & 0xff)); + return -ENODEV; + } + dev_dbg(sensor->dev, "patch %d.%d applied\n", (u8)(patch >> 8), (u8)(patch & 0xff)); - return -ENODEV; + + } else { + vd55g1_write(sensor, VD55G1_REG_BOOT, VD55G1_BOOT_BOOT, &ret); + vd55g1_poll_reg(sensor, VD55G1_REG_BOOT, 0, &ret); + if (ret) { + dev_err(sensor->dev, "Failed to boot\n"); + return ret; + } + } + + ret = vd55g1_wait_state(sensor, VD55G1_SYSTEM_FSM_SW_STBY, NULL); + if (ret) { + dev_err(sensor->dev, "Sensor waiting after boot failed\n"); + return ret; } - dev_dbg(sensor->dev, "patch %d.%d applied\n", - (u8)(patch >> 8), (u8)(patch & 0xff)); return 0; } @@ -1165,10 +1281,19 @@ static int vd55g1_enum_mbus_code(struct v4l2_subdev *sd, struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { - if (code->index >= ARRAY_SIZE(vd55g1_mbus_codes)) - return -EINVAL; + struct vd55g1 *sensor = to_vd55g1(sd); + u32 base_code; - code->code = vd55g1_mbus_codes[code->index].code; + if (sensor->version->color != VD55G1_COLOR_VERSION_BAYER) { + if (code->index >= ARRAY_SIZE(vd55g1_mbus_formats_mono)) + return -EINVAL; + base_code = vd55g1_mbus_formats_mono[code->index]; + } else { + if (code->index >= ARRAY_SIZE(vd55g1_mbus_formats_bayer)) + return -EINVAL; + base_code = vd55g1_mbus_formats_bayer[code->index][0]; + } + code->code = vd55g1_get_fmt_code(sensor, base_code); return 0; } @@ -1198,12 +1323,6 @@ static int vd55g1_new_format_change_controls(struct vd55g1 *sensor, if (ret) return ret; - /* Update pixel rate to reflect new bpp */ - ret = __v4l2_ctrl_s_ctrl_int64(sensor->pixel_rate_ctrl, - vd55g1_get_pixel_rate(sensor, format)); - if (ret) - return ret; - /* Update hblank according to new width */ hblank = vd55g1_get_hblank_min(sensor, format, crop); ret = __v4l2_ctrl_modify_range(sensor->hblank_ctrl, hblank, hblank, 1, @@ -1260,6 +1379,7 @@ static int vd55g1_init_state(struct v4l2_subdev *sd, { struct vd55g1 *sensor = to_vd55g1(sd); struct v4l2_subdev_format fmt = { 0 }; + int code; struct v4l2_subdev_route routes[] = { { .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE } }; @@ -1274,9 +1394,13 @@ static int vd55g1_init_state(struct v4l2_subdev *sd, if (ret) return ret; - vd55g1_update_pad_fmt(sensor, &vd55g1_supported_modes[VD55G1_MODE_DEF], - vd55g1_mbus_codes[VD55G1_MBUS_CODE_DEF].code, - &fmt.format); + if (sensor->version->color != VD55G1_COLOR_VERSION_BAYER) + code = vd55g1_mbus_formats_mono[0]; + else + code = vd55g1_mbus_formats_bayer[0][0]; + fmt.format.code = vd55g1_get_fmt_code(sensor, code); + fmt.format.width = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].width; + fmt.format.height = vd55g1_supported_modes[VD55G1_MODE_IDX_DEF].height; return vd55g1_set_pad_fmt(sd, sd_state, &fmt); } @@ -1285,9 +1409,16 @@ static int vd55g1_enum_frame_size(struct v4l2_subdev *sd, struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { + struct vd55g1 *sensor = to_vd55g1(sd); + u32 code; + if (fse->index >= ARRAY_SIZE(vd55g1_supported_modes)) return -EINVAL; + code = vd55g1_get_fmt_code(sensor, fse->code); + if (fse->code != code) + return -EINVAL; + fse->min_width = vd55g1_supported_modes[fse->index].width; fse->max_width = fse->min_width; fse->min_height = vd55g1_supported_modes[fse->index].height; @@ -1455,7 +1586,6 @@ static int vd55g1_init_ctrls(struct vd55g1 *sensor) v4l2_subdev_state_get_crop(state, 0); struct v4l2_mbus_framefmt *format = v4l2_subdev_state_get_format(state, 0); - s32 pixel_rate = vd55g1_get_pixel_rate(sensor, format); int ret; v4l2_ctrl_handler_init(hdl, 16); @@ -1463,8 +1593,12 @@ static int vd55g1_init_ctrls(struct vd55g1 *sensor) /* Flip cluster */ sensor->hflip_ctrl = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_HFLIP, 0, 1, 1, 0); + if (sensor->hflip_ctrl) + sensor->hflip_ctrl->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT; sensor->vflip_ctrl = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_VFLIP, 0, 1, 1, 0); + if (sensor->vflip_ctrl) + sensor->vflip_ctrl->flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT; v4l2_ctrl_cluster(2, &sensor->hflip_ctrl); /* Exposition cluster */ @@ -1493,7 +1627,7 @@ static int vd55g1_init_ctrls(struct vd55g1 *sensor) sensor->pixel_rate_ctrl = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_PIXEL_RATE, 1, INT_MAX, 1, - pixel_rate); + sensor->pixel_clock); if (sensor->pixel_rate_ctrl) sensor->pixel_rate_ctrl->flags |= V4L2_CTRL_FLAG_READ_ONLY; sensor->ae_lock_ctrl = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_3A_LOCK, @@ -1546,31 +1680,47 @@ static int vd55g1_init_ctrls(struct vd55g1 *sensor) return ret; } +static const struct vd55g1_version * +vd55g1_get_version(enum vd55g1_model_id id, + enum vd55g1_color_version color) +{ + for (unsigned int i = 0; i < ARRAY_SIZE(vd55g1_versions); i++) { + if (vd55g1_versions[i].id == id && + vd55g1_versions[i].color == color) + return &vd55g1_versions[i]; + } + + return NULL; +} + static int vd55g1_detect(struct vd55g1 *sensor) { - u64 device_rev; - u64 id; - int ret; + const struct vd55g1_version *dt_version = + device_get_match_data(sensor->dev); + const struct vd55g1_version *version; + u64 color, id; + int ret = 0; - ret = vd55g1_read(sensor, VD55G1_REG_MODEL_ID, &id, NULL); + vd55g1_read(sensor, VD55G1_REG_MODEL_ID, &id, &ret); + vd55g1_read(sensor, VD55G1_REG_COLOR_VERSION, &color, &ret); if (ret) return ret; - if (id != VD55G1_MODEL_ID) { - dev_warn(sensor->dev, "Unsupported sensor id %x\n", (u32)id); + version = vd55g1_get_version(id, color); + if (!version) { + dev_warn(sensor->dev, "Unsupported sensor version, expected %s\n", + dt_version->name); return -ENODEV; } - - ret = vd55g1_read(sensor, VD55G1_REG_REVISION, &device_rev, NULL); - if (ret) - return ret; - - if (device_rev != VD55G1_REVISION_CCB) { - dev_err(sensor->dev, "Unsupported sensor revision (0x%x)\n", - (u16)device_rev); + if (version->id != dt_version->id || + version->color != dt_version->color) { + dev_err(sensor->dev, "Probed sensor version %s and device tree definition %s mismatch", + version->name, dt_version->name); return -ENODEV; } + sensor->version = version; + return 0; } @@ -1616,13 +1766,6 @@ static int vd55g1_power_on(struct device *dev) goto disable_clock; } - ret = vd55g1_wait_state(sensor, VD55G1_SYSTEM_FSM_SW_STBY, NULL); - if (ret) { - dev_err(dev, "Sensor waiting after patch failed %d\n", - ret); - goto disable_clock; - } - return 0; disable_clock: @@ -1934,7 +2077,9 @@ static void vd55g1_remove(struct i2c_client *client) } static const struct of_device_id vd55g1_dt_ids[] = { - { .compatible = "st,vd55g1" }, + { .compatible = "st,vd55g1", .data = (void *)&vd55g1_versions[0] }, + { .compatible = "st,vd55g4", .data = (void *)&vd55g1_versions[1] }, + { .compatible = "st,vd65g4", .data = (void *)&vd55g1_versions[2] }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, vd55g1_dt_ids); diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c index 1eee2d4f5b40e..20889f94610cb 100644 --- a/drivers/media/i2c/video-i2c.c +++ b/drivers/media/i2c/video-i2c.c @@ -522,8 +522,12 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) data->kthread_vid_cap = kthread_run(video_i2c_thread_vid_cap, data, "%s-vid-cap", data->v4l2_dev.name); ret = PTR_ERR_OR_ZERO(data->kthread_vid_cap); - if (!ret) - return 0; + if (ret) { + data->kthread_vid_cap = NULL; + goto error_rpm_put; + } + + return 0; error_rpm_put: pm_runtime_put_autosuspend(dev); @@ -888,7 +892,7 @@ static void video_i2c_remove(struct i2c_client *client) if (data->chip->set_power) data->chip->set_power(data, false); - video_unregister_device(&data->vdev); + vb2_video_unregister_device(&data->vdev); } #ifdef CONFIG_PM diff --git a/drivers/media/pci/cobalt/cobalt-alsa-main.c b/drivers/media/pci/cobalt/cobalt-alsa-main.c index c57f87a682690..1356353d931c4 100644 --- a/drivers/media/pci/cobalt/cobalt-alsa-main.c +++ b/drivers/media/pci/cobalt/cobalt-alsa-main.c @@ -135,7 +135,6 @@ int cobalt_alsa_init(struct cobalt_stream *s) err_exit_free: if (sc != NULL) snd_card_free(sc); - kfree(cobsc); err_exit: return ret; } diff --git a/drivers/media/pci/cx23885/cimax2.c b/drivers/media/pci/cx23885/cimax2.c index 06e41f92092dd..a4a319087238d 100644 --- a/drivers/media/pci/cx23885/cimax2.c +++ b/drivers/media/pci/cx23885/cimax2.c @@ -528,6 +528,7 @@ void netup_ci_exit(struct cx23885_tsport *port) if (NULL == state->ca.data) return; + cancel_work_sync(&state->work); dvb_ca_en50221_release(&state->ca); kfree(state); } diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c index a39f445ce22a0..43d658a90f08e 100644 --- a/drivers/media/pci/cx23885/cx23885-core.c +++ b/drivers/media/pci/cx23885/cx23885-core.c @@ -990,8 +990,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev) } /* PCIe stuff */ - dev->lmmio = ioremap(pci_resource_start(dev->pci, 0), - pci_resource_len(dev->pci, 0)); + dev->lmmio = pci_ioremap_bar(dev->pci, 0); + if (!dev->lmmio) { + dev_err(&dev->pci->dev, "CORE %s: can't ioremap MMIO memory\n", + dev->name); + goto err_release_region; + } dev->bmmio = (u8 __iomem *)dev->lmmio; @@ -1096,6 +1100,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev) } return 0; + +err_release_region: + release_mem_region(pci_resource_start(dev->pci, 0), + pci_resource_len(dev->pci, 0)); + cx23885_devcount--; + return -ENODEV; } static void cx23885_dev_unregister(struct cx23885_dev *dev) diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c index e1185aa669f48..abd86488064b5 100644 --- a/drivers/media/pci/dm1105/dm1105.c +++ b/drivers/media/pci/dm1105/dm1105.c @@ -768,6 +768,8 @@ static void dm1105_ir_exit(struct dm1105_dev *dm1105) static int dm1105_hw_init(struct dm1105_dev *dev) { + int ret; + dm1105_disable_irqs(dev); dm_writeb(DM1105_HOST_CTR, 0); @@ -778,7 +780,10 @@ static int dm1105_hw_init(struct dm1105_dev *dev) dm_writew(DM1105_TSCTR, 0xc10a); /* map DMA and set address */ - dm1105_dma_map(dev); + ret = dm1105_dma_map(dev); + if (ret) + return -ENOMEM; + dm1105_set_dma_addr(dev); /* big buffer */ dm_writel(DM1105_RLEN, 5 * DM1105_DMA_BYTES); @@ -1194,6 +1199,7 @@ static void dm1105_remove(struct pci_dev *pdev) dm1105_hw_exit(dev); free_irq(pdev->irq, dev); + destroy_workqueue(dev->wq); pci_iounmap(pdev, dev->io_mem); pci_release_regions(pdev); pci_disable_device(pdev); diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c index db28748434530..237906cb131ac 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c @@ -288,15 +288,27 @@ static const struct dwc_dphy_freq_range freqranges[DPHY_FREQ_RANGE_NUM] = { static u16 get_hsfreq_by_mbps(u32 mbps) { - unsigned int i = DPHY_FREQ_RANGE_NUM; - - while (i--) { - if (freqranges[i].default_mbps == mbps || - (mbps >= freqranges[i].min && mbps <= freqranges[i].max)) - return i; + u16 best = DPHY_FREQ_RANGE_INVALID_INDEX; + unsigned int i; + + for (i = 0; i < DPHY_FREQ_RANGE_NUM; i++) { + if (mbps > freqranges[i].max) + continue; + + if (mbps < freqranges[i].min) + break; + + if (best == DPHY_FREQ_RANGE_INVALID_INDEX || + freqranges[i].osc_freq_target > + freqranges[best].osc_freq_target || + (freqranges[i].osc_freq_target == + freqranges[best].osc_freq_target && + abs((int)mbps - (int)freqranges[i].default_mbps) < + abs((int)mbps - (int)freqranges[best].default_mbps))) + best = i; } - return DPHY_FREQ_RANGE_INVALID_INDEX; + return best; } static int ipu6_isys_dwc_phy_config(struct ipu6_isys *isys, diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys.c b/drivers/media/pci/intel/ipu6/ipu6-isys.c index fc0ec0a4b8f54..9a9e245edc1a2 100644 --- a/drivers/media/pci/intel/ipu6/ipu6-isys.c +++ b/drivers/media/pci/intel/ipu6/ipu6-isys.c @@ -761,6 +761,7 @@ static int isys_notifier_init(struct ipu6_isys *isys) err_parse: fwnode_handle_put(ep); + v4l2_async_nf_cleanup(&isys->notifier); return ret; } diff --git a/drivers/media/pci/intel/ipu6/ipu6.c b/drivers/media/pci/intel/ipu6/ipu6.c index fab7783c664b9..f900de14ba600 100644 --- a/drivers/media/pci/intel/ipu6/ipu6.c +++ b/drivers/media/pci/intel/ipu6/ipu6.c @@ -400,7 +400,6 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(isys_adev->mmu)) { put_device(&isys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(dev, isys_adev->mmu, "ipu6_mmu_init(isys_adev->mmu) failed\n"); } @@ -408,10 +407,8 @@ ipu6_isys_init(struct pci_dev *pdev, struct device *parent, isys_adev->mmu->dev = &isys_adev->auxdev.dev; ret = ipu6_bus_add_device(isys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return isys_adev; } @@ -444,7 +441,6 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, &ipdata->hw_variant); if (IS_ERR(psys_adev->mmu)) { put_device(&psys_adev->auxdev.dev); - kfree(pdata); return dev_err_cast_probe(&pdev->dev, psys_adev->mmu, "ipu6_mmu_init(psys_adev->mmu) failed\n"); } @@ -452,10 +448,8 @@ ipu6_psys_init(struct pci_dev *pdev, struct device *parent, psys_adev->mmu->dev = &psys_adev->auxdev.dev; ret = ipu6_bus_add_device(psys_adev); - if (ret) { - kfree(pdata); + if (ret) return ERR_PTR(ret); - } return psys_adev; } diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c index 4a51b873e47ad..2b1672737d841 100644 --- a/drivers/media/pci/saa7134/saa7134-video.c +++ b/drivers/media/pci/saa7134/saa7134-video.c @@ -1714,8 +1714,10 @@ int saa7134_video_init1(struct saa7134_dev *dev) q->dev = &dev->pci->dev; ret = vb2_queue_init(q); if (ret) - return ret; - saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt); + goto err_free_ctrl; + ret = saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt); + if (ret) + goto err_free_ctrl; q = &dev->vbi_vbq; q->type = V4L2_BUF_TYPE_VBI_CAPTURE; @@ -1732,11 +1734,24 @@ int saa7134_video_init1(struct saa7134_dev *dev) q->lock = &dev->lock; q->dev = &dev->pci->dev; ret = vb2_queue_init(q); - if (ret) - return ret; - saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt); + if (ret) { + saa7134_pgtable_free(dev->pci, &dev->video_q.pt); + goto err_free_ctrl; + } + + ret = saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt); + if (ret) { + saa7134_pgtable_free(dev->pci, &dev->video_q.pt); + goto err_free_ctrl; + } return 0; + +err_free_ctrl: + v4l2_ctrl_handler_free(&dev->ctrl_handler); + if (card_has_radio(dev)) + v4l2_ctrl_handler_free(&dev->radio_ctrl_handler); + return ret; } void saa7134_video_fini(struct saa7134_dev *dev) diff --git a/drivers/media/pci/saa7164/saa7164-core.c b/drivers/media/pci/saa7164/saa7164-core.c index ac290f5464131..49dca57181d83 100644 --- a/drivers/media/pci/saa7164/saa7164-core.c +++ b/drivers/media/pci/saa7164/saa7164-core.c @@ -878,6 +878,9 @@ static int get_resources(struct saa7164_dev *dev) if (request_mem_region(pci_resource_start(dev->pci, 2), pci_resource_len(dev->pci, 2), dev->name)) return 0; + + release_mem_region(pci_resource_start(dev->pci, 0), + pci_resource_len(dev->pci, 0)); } printk(KERN_ERR "%s: can't get MMIO memory @ 0x%llx or 0x%llx\n", @@ -1000,8 +1003,7 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) dev->name, dev->pci->subsystem_vendor, dev->pci->subsystem_device); - saa7164_devcount--; - return -ENODEV; + goto err_devlist; } /* PCI/e allocations */ @@ -1039,7 +1041,7 @@ static int saa7164_dev_setup(struct saa7164_dev *dev) iounmap(dev->lmmio); err_ioremap_bar0: release_resources(dev); - +err_devlist: scoped_guard(mutex, &devlist) { list_del(&dev->devlist); } diff --git a/drivers/media/pci/zoran/zoran_card.c b/drivers/media/pci/zoran/zoran_card.c index f707bdc1fb0f1..b8285df644738 100644 --- a/drivers/media/pci/zoran/zoran_card.c +++ b/drivers/media/pci/zoran/zoran_card.c @@ -885,7 +885,7 @@ static int zoran_init_video_device(struct zoran *zr, struct video_device *video_ static void zoran_exit_video_devices(struct zoran *zr) { video_unregister_device(zr->video_dev); - kfree(zr->video_dev); + zr->video_dev = NULL; } static int zoran_init_video_devices(struct zoran *zr) diff --git a/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c b/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c index c80667dd76621..ec25299e52abc 100644 --- a/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c +++ b/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c @@ -107,6 +107,8 @@ static void c3_isp_params_awb_wt(struct c3_isp_device *isp, c3_isp_write(isp, ISP_AWB_BLK_WT_ADDR, 0); zones_num = cfg->horiz_zones_num * cfg->vert_zones_num; + if (zones_num > C3_ISP_AWB_MAX_ZONES) + zones_num = C3_ISP_AWB_MAX_ZONES; /* Need to write 8 weights at once */ for (i = 0; i < zones_num / 8; i++) { @@ -223,6 +225,8 @@ static void c3_isp_params_ae_wt(struct c3_isp_device *isp, c3_isp_write(isp, ISP_AE_BLK_WT_ADDR, 0); zones_num = cfg->horiz_zones_num * cfg->vert_zones_num; + if (zones_num > C3_ISP_AE_MAX_ZONES) + zones_num = C3_ISP_AE_MAX_ZONES; /* Need to write 8 weights at once */ for (i = 0; i < zones_num / 8; i++) { diff --git a/drivers/media/platform/amphion/venc.c b/drivers/media/platform/amphion/venc.c index c5c1f1fbaa803..762de437e800c 100644 --- a/drivers/media/platform/amphion/venc.c +++ b/drivers/media/platform/amphion/venc.c @@ -977,8 +977,6 @@ static int venc_start_session(struct vpu_inst *inst, u32 type) venc->ready_count = 0; venc->stopped = false; vpu_process_output_buffer(inst); - if (venc->frame_count == 0) - dev_err(inst->dev, "[%d] there is no input when starting\n", inst->id); return 0; error: diff --git a/drivers/media/platform/aspeed/aspeed-video.c b/drivers/media/platform/aspeed/aspeed-video.c index b83e43245277b..88ccd2f374698 100644 --- a/drivers/media/platform/aspeed/aspeed-video.c +++ b/drivers/media/platform/aspeed/aspeed-video.c @@ -2327,6 +2327,7 @@ static int aspeed_video_probe(struct platform_device *pdev) rc = aspeed_video_setup_video(video); if (rc) { aspeed_video_free_buf(video, &video->jpeg); + of_reserved_mem_device_release(&pdev->dev); clk_unprepare(video->vclk); clk_unprepare(video->eclk); return rc; diff --git a/drivers/media/platform/bcm2835/bcm2835-unicam.c b/drivers/media/platform/bcm2835/bcm2835-unicam.c index 1c4ae29d69a07..2ca8a9c459492 100644 --- a/drivers/media/platform/bcm2835/bcm2835-unicam.c +++ b/drivers/media/platform/bcm2835/bcm2835-unicam.c @@ -142,22 +142,6 @@ enum pad_types { MAX_NODES }; -#define MASK_CS_DEFAULT BIT(V4L2_COLORSPACE_DEFAULT) -#define MASK_CS_SMPTE170M BIT(V4L2_COLORSPACE_SMPTE170M) -#define MASK_CS_SMPTE240M BIT(V4L2_COLORSPACE_SMPTE240M) -#define MASK_CS_REC709 BIT(V4L2_COLORSPACE_REC709) -#define MASK_CS_BT878 BIT(V4L2_COLORSPACE_BT878) -#define MASK_CS_470_M BIT(V4L2_COLORSPACE_470_SYSTEM_M) -#define MASK_CS_470_BG BIT(V4L2_COLORSPACE_470_SYSTEM_BG) -#define MASK_CS_JPEG BIT(V4L2_COLORSPACE_JPEG) -#define MASK_CS_SRGB BIT(V4L2_COLORSPACE_SRGB) -#define MASK_CS_OPRGB BIT(V4L2_COLORSPACE_OPRGB) -#define MASK_CS_BT2020 BIT(V4L2_COLORSPACE_BT2020) -#define MASK_CS_RAW BIT(V4L2_COLORSPACE_RAW) -#define MASK_CS_DCI_P3 BIT(V4L2_COLORSPACE_DCI_P3) - -#define MAX_COLORSPACE 32 - /* * struct unicam_fmt - Unicam media bus format information * @pixelformat: V4L2 pixel format FCC identifier. 0 if n/a. @@ -166,9 +150,6 @@ enum pad_types { * @code: V4L2 media bus format code. * @depth: Bits per pixel as delivered from the source. * @csi_dt: CSI data type. - * @valid_colorspaces: Bitmask of valid colorspaces so that the Media Controller - * centric try_fmt can validate the colorspace and pass - * v4l2-compliance. * @check_variants: Flag to denote that there are multiple mediabus formats * still in the list that could match this V4L2 format. * @mc_skip: Media Controller shouldn't list this format via ENUM_FMT as it is @@ -181,7 +162,6 @@ struct unicam_fmt { u32 code; u8 depth; u8 csi_dt; - u32 valid_colorspaces; u8 check_variants:1; u8 mc_skip:1; u8 metadata_fmt:1; @@ -195,240 +175,197 @@ static const struct unicam_fmt formats[] = { .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .check_variants = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_UYVY, .code = MEDIA_BUS_FMT_UYVY8_2X8, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .check_variants = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_YVYU, .code = MEDIA_BUS_FMT_YVYU8_2X8, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .check_variants = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_VYUY, .code = MEDIA_BUS_FMT_VYUY8_2X8, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .check_variants = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_YUYV, .code = MEDIA_BUS_FMT_YUYV8_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .mc_skip = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_UYVY, .code = MEDIA_BUS_FMT_UYVY8_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .mc_skip = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_YVYU, .code = MEDIA_BUS_FMT_YVYU8_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .mc_skip = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { .fourcc = V4L2_PIX_FMT_VYUY, .code = MEDIA_BUS_FMT_VYUY8_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_YUV422_8B, .mc_skip = 1, - .valid_colorspaces = MASK_CS_SMPTE170M | MASK_CS_REC709 | - MASK_CS_JPEG, }, { /* RGB Formats */ .fourcc = V4L2_PIX_FMT_RGB565, /* gggbbbbb rrrrrggg */ .code = MEDIA_BUS_FMT_RGB565_2X8_LE, .depth = 16, .csi_dt = MIPI_CSI2_DT_RGB565, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_RGB565X, /* rrrrrggg gggbbbbb */ .code = MEDIA_BUS_FMT_RGB565_2X8_BE, .depth = 16, .csi_dt = MIPI_CSI2_DT_RGB565, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_RGB555, /* gggbbbbb arrrrrgg */ .code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_LE, .depth = 16, .csi_dt = MIPI_CSI2_DT_RGB555, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_RGB555X, /* arrrrrgg gggbbbbb */ .code = MEDIA_BUS_FMT_RGB555_2X8_PADHI_BE, .depth = 16, .csi_dt = MIPI_CSI2_DT_RGB555, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_RGB24, /* rgb */ .code = MEDIA_BUS_FMT_RGB888_1X24, .depth = 24, .csi_dt = MIPI_CSI2_DT_RGB888, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_BGR24, /* bgr */ .code = MEDIA_BUS_FMT_BGR888_1X24, .depth = 24, .csi_dt = MIPI_CSI2_DT_RGB888, - .valid_colorspaces = MASK_CS_SRGB, }, { .fourcc = V4L2_PIX_FMT_RGB32, /* argb */ .code = MEDIA_BUS_FMT_ARGB8888_1X32, .depth = 32, .csi_dt = 0x0, - .valid_colorspaces = MASK_CS_SRGB, }, { /* Bayer Formats */ .fourcc = V4L2_PIX_FMT_SBGGR8, .code = MEDIA_BUS_FMT_SBGGR8_1X8, .depth = 8, .csi_dt = MIPI_CSI2_DT_RAW8, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGBRG8, .code = MEDIA_BUS_FMT_SGBRG8_1X8, .depth = 8, .csi_dt = MIPI_CSI2_DT_RAW8, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGRBG8, .code = MEDIA_BUS_FMT_SGRBG8_1X8, .depth = 8, .csi_dt = MIPI_CSI2_DT_RAW8, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SRGGB8, .code = MEDIA_BUS_FMT_SRGGB8_1X8, .depth = 8, .csi_dt = MIPI_CSI2_DT_RAW8, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SBGGR10P, .repacked_fourcc = V4L2_PIX_FMT_SBGGR10, .code = MEDIA_BUS_FMT_SBGGR10_1X10, .depth = 10, .csi_dt = MIPI_CSI2_DT_RAW10, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGBRG10P, .repacked_fourcc = V4L2_PIX_FMT_SGBRG10, .code = MEDIA_BUS_FMT_SGBRG10_1X10, .depth = 10, .csi_dt = MIPI_CSI2_DT_RAW10, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGRBG10P, .repacked_fourcc = V4L2_PIX_FMT_SGRBG10, .code = MEDIA_BUS_FMT_SGRBG10_1X10, .depth = 10, .csi_dt = MIPI_CSI2_DT_RAW10, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SRGGB10P, .repacked_fourcc = V4L2_PIX_FMT_SRGGB10, .code = MEDIA_BUS_FMT_SRGGB10_1X10, .depth = 10, .csi_dt = MIPI_CSI2_DT_RAW10, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SBGGR12P, .repacked_fourcc = V4L2_PIX_FMT_SBGGR12, .code = MEDIA_BUS_FMT_SBGGR12_1X12, .depth = 12, .csi_dt = MIPI_CSI2_DT_RAW12, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGBRG12P, .repacked_fourcc = V4L2_PIX_FMT_SGBRG12, .code = MEDIA_BUS_FMT_SGBRG12_1X12, .depth = 12, .csi_dt = MIPI_CSI2_DT_RAW12, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGRBG12P, .repacked_fourcc = V4L2_PIX_FMT_SGRBG12, .code = MEDIA_BUS_FMT_SGRBG12_1X12, .depth = 12, .csi_dt = MIPI_CSI2_DT_RAW12, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SRGGB12P, .repacked_fourcc = V4L2_PIX_FMT_SRGGB12, .code = MEDIA_BUS_FMT_SRGGB12_1X12, .depth = 12, .csi_dt = MIPI_CSI2_DT_RAW12, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SBGGR14P, .repacked_fourcc = V4L2_PIX_FMT_SBGGR14, .code = MEDIA_BUS_FMT_SBGGR14_1X14, .depth = 14, .csi_dt = MIPI_CSI2_DT_RAW14, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGBRG14P, .repacked_fourcc = V4L2_PIX_FMT_SGBRG14, .code = MEDIA_BUS_FMT_SGBRG14_1X14, .depth = 14, .csi_dt = MIPI_CSI2_DT_RAW14, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGRBG14P, .repacked_fourcc = V4L2_PIX_FMT_SGRBG14, .code = MEDIA_BUS_FMT_SGRBG14_1X14, .depth = 14, .csi_dt = MIPI_CSI2_DT_RAW14, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SRGGB14P, .repacked_fourcc = V4L2_PIX_FMT_SRGGB14, .code = MEDIA_BUS_FMT_SRGGB14_1X14, .depth = 14, .csi_dt = MIPI_CSI2_DT_RAW14, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SBGGR16, .code = MEDIA_BUS_FMT_SBGGR16_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_RAW16, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGBRG16, .code = MEDIA_BUS_FMT_SGBRG16_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_RAW16, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SGRBG16, .code = MEDIA_BUS_FMT_SGRBG16_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_RAW16, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_SRGGB16, .code = MEDIA_BUS_FMT_SRGGB16_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_RAW16, - .valid_colorspaces = MASK_CS_RAW, }, { /* Greyscale formats */ @@ -436,14 +373,12 @@ static const struct unicam_fmt formats[] = { .code = MEDIA_BUS_FMT_Y8_1X8, .depth = 8, .csi_dt = MIPI_CSI2_DT_RAW8, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_Y10P, .repacked_fourcc = V4L2_PIX_FMT_Y10, .code = MEDIA_BUS_FMT_Y10_1X10, .depth = 10, .csi_dt = MIPI_CSI2_DT_RAW10, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_Y12P, .repacked_fourcc = V4L2_PIX_FMT_Y12, @@ -456,13 +391,11 @@ static const struct unicam_fmt formats[] = { .code = MEDIA_BUS_FMT_Y14_1X14, .depth = 14, .csi_dt = MIPI_CSI2_DT_RAW14, - .valid_colorspaces = MASK_CS_RAW, }, { .fourcc = V4L2_PIX_FMT_Y16, .code = MEDIA_BUS_FMT_Y16_1X16, .depth = 16, .csi_dt = MIPI_CSI2_DT_RAW16, - .valid_colorspaces = MASK_CS_RAW, }, /* Embedded data format */ { @@ -1913,7 +1846,6 @@ static void unicam_mc_try_fmt(struct unicam_node *node, struct v4l2_format *f, struct v4l2_pix_format *v4l2_format = &f->fmt.pix; struct unicam_device *dev = node->dev; const struct unicam_fmt *fmt; - int is_rgb; /* * Default to the first format if the requested pixel format code isn't @@ -1933,21 +1865,6 @@ static void unicam_mc_try_fmt(struct unicam_node *node, struct v4l2_format *f, if (ret_fmt) *ret_fmt = fmt; - if (v4l2_format->colorspace >= MAX_COLORSPACE || - !(fmt->valid_colorspaces & (1 << v4l2_format->colorspace))) { - v4l2_format->colorspace = __ffs(fmt->valid_colorspaces); - - v4l2_format->xfer_func = - V4L2_MAP_XFER_FUNC_DEFAULT(v4l2_format->colorspace); - v4l2_format->ycbcr_enc = - V4L2_MAP_YCBCR_ENC_DEFAULT(v4l2_format->colorspace); - is_rgb = v4l2_format->colorspace == V4L2_COLORSPACE_SRGB; - v4l2_format->quantization = - V4L2_MAP_QUANTIZATION_DEFAULT(is_rgb, - v4l2_format->colorspace, - v4l2_format->ycbcr_enc); - } - unicam_dbg(3, dev, "%s: %08x %ux%u (bytesperline %u sizeimage %u)\n", __func__, v4l2_format->pixelformat, v4l2_format->width, v4l2_format->height, diff --git a/drivers/media/platform/broadcom/bcm2835-unicam.c b/drivers/media/platform/broadcom/bcm2835-unicam.c index d47098b2361dc..fc27f8264c1e8 100644 --- a/drivers/media/platform/broadcom/bcm2835-unicam.c +++ b/drivers/media/platform/broadcom/bcm2835-unicam.c @@ -2618,6 +2618,7 @@ static int unicam_async_nf_init(struct unicam_device *unicam) return 0; error: + v4l2_async_nf_cleanup(&unicam->notifier); fwnode_handle_put(ep_handle); return ret; } @@ -2750,6 +2751,7 @@ static void unicam_remove(struct platform_device *pdev) v4l2_device_unregister(&unicam->v4l2_dev); media_device_unregister(&unicam->mdev); v4l2_async_nf_unregister(&unicam->notifier); + v4l2_async_nf_cleanup(&unicam->notifier); unicam_subdev_cleanup(unicam); diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.c b/drivers/media/platform/chips-media/wave5/wave5-helper.c index 53a0ac068c2e2..c3d34be833ff0 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-helper.c +++ b/drivers/media/platform/chips-media/wave5/wave5-helper.c @@ -68,7 +68,6 @@ int wave5_vpu_release_device(struct file *filp, int ret = 0; unsigned long flags; - v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx); /* * To prevent Null reference exception, the existing irq handler were * separated to two modules. @@ -89,6 +88,9 @@ int wave5_vpu_release_device(struct file *filp, list_del_init(&inst->list); spin_unlock_irqrestore(&inst->dev->irq_spinlock, flags); mutex_unlock(&inst->dev->irq_lock); + + v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx); + if (inst->state != VPU_INST_STATE_NONE) { u32 fail_res; diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c index d94cf84c3ee5f..fd3c3c2168158 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c @@ -1762,6 +1762,9 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst) (p_param->skip_intra_trans << 25) | (p_param->strong_intra_smooth_enable << 27) | (p_param->en_still_picture << 30); + else if (inst->std == W_AVC_ENC) + reg_val |= (p_param->constraint_set1_flag << 29); + vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_SPS_PARAM, reg_val); reg_val = (p_param->lossless_enable) | diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c index 7a4625acc0478..30f523a5edb15 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c @@ -5,6 +5,7 @@ * Copyright (C) 2021-2023 CHIPS&MEDIA INC */ +#include #include #include "wave5-helper.h" @@ -270,10 +271,23 @@ static void send_eos_event(struct vpu_instance *inst) inst->eos = false; } +static void wave5_update_min_bufs_ctrl(struct vpu_instance *inst, u32 fbc_buf_count) +{ + struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; + struct v4l2_ctrl *ctrl; + + if (!fbc_buf_count || fbc_buf_count == v4l2_m2m_num_dst_bufs_ready(m2m_ctx)) + return; + + ctrl = v4l2_ctrl_find(&inst->v4l2_ctrl_hdl, + V4L2_CID_MIN_BUFFERS_FOR_CAPTURE); + if (ctrl) + v4l2_ctrl_s_ctrl(ctrl, fbc_buf_count); +} + static int handle_dynamic_resolution_change(struct vpu_instance *inst) { struct v4l2_fh *fh = &inst->v4l2_fh; - struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; static const struct v4l2_event vpu_event_src_ch = { .type = V4L2_EVENT_SOURCE_CHANGE, @@ -292,14 +306,6 @@ static int handle_dynamic_resolution_change(struct vpu_instance *inst) inst->needs_reallocation = true; inst->fbc_buf_count = initial_info->min_frame_buffer_count + 1; - if (inst->fbc_buf_count != v4l2_m2m_num_dst_bufs_ready(m2m_ctx)) { - struct v4l2_ctrl *ctrl; - - ctrl = v4l2_ctrl_find(&inst->v4l2_ctrl_hdl, - V4L2_CID_MIN_BUFFERS_FOR_CAPTURE); - if (ctrl) - v4l2_ctrl_s_ctrl(ctrl, inst->fbc_buf_count); - } if (p_dec_info->initial_info_obtained) { const struct vpu_format *vpu_fmt; @@ -427,19 +433,24 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst) if ((dec_info.index_frame_display == DISPLAY_IDX_FLAG_SEQ_END || dec_info.sequence_changed)) { unsigned long flags; + u32 fbc_buf_count = 0; spin_lock_irqsave(&inst->state_spinlock, flags); if (!v4l2_m2m_has_stopped(m2m_ctx)) { switch_state(inst, VPU_INST_STATE_STOP); - if (dec_info.sequence_changed) + if (dec_info.sequence_changed) { handle_dynamic_resolution_change(inst); - else + fbc_buf_count = inst->fbc_buf_count; + } else { send_eos_event(inst); + } flag_last_buffer_done(inst); } spin_unlock_irqrestore(&inst->state_spinlock, flags); + + wave5_update_min_bufs_ctrl(inst, fbc_buf_count); } /* @@ -1352,6 +1363,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count } else if (q->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { struct dec_initial_info *initial_info = &inst->codec_info->dec_info.initial_info; + struct dec_info *p_dec_info = &inst->codec_info->dec_info; if (inst->state == VPU_INST_STATE_STOP) ret = switch_state(inst, VPU_INST_STATE_INIT_SEQ); @@ -1359,6 +1371,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count goto return_buffers; if (inst->state == VPU_INST_STATE_INIT_SEQ && + p_dec_info->initial_info_obtained && inst->dev->product_code == WAVE521C_CODE) { if (initial_info->luma_bitdepth != 8) { dev_info(inst->dev->dev, "%s: no support for %d bit depth", @@ -1367,7 +1380,6 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count goto return_buffers; } } - } pm_runtime_put_autosuspend(inst->dev->dev); return ret; @@ -1479,25 +1491,24 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q) { struct vpu_instance *inst = vb2_get_drv_priv(q); struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx; - bool check_cmd = TRUE; + unsigned long timeout; dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type); pm_runtime_resume_and_get(inst->dev->dev); - while (check_cmd) { + timeout = jiffies + msecs_to_jiffies(VPU_DEC_STOP_TIMEOUT); + while (true) { struct queue_status_info q_status; - struct dec_output_info dec_output_info; wave5_vpu_dec_give_command(inst, DEC_GET_QUEUE_STATUS, &q_status); if (q_status.report_queue_count == 0) break; - if (wave5_vpu_wait_interrupt(inst, VPU_DEC_TIMEOUT) < 0) + if (time_after(jiffies, timeout)) break; - if (wave5_vpu_dec_get_output_info(inst, &dec_output_info)) - dev_dbg(inst->dev->dev, "there is no output info\n"); + usleep_range(1000, 2000); } v4l2_m2m_update_stop_streaming_state(m2m_ctx, q); @@ -1543,8 +1554,9 @@ static const struct vpu_instance_ops wave5_vpu_dec_inst_ops = { static int initialize_sequence(struct vpu_instance *inst) { struct dec_initial_info initial_info; - int ret = 0; unsigned long flags; + u32 fbc_buf_count; + int ret = 0; memset(&initial_info, 0, sizeof(struct dec_initial_info)); @@ -1568,8 +1580,11 @@ static int initialize_sequence(struct vpu_instance *inst) spin_lock_irqsave(&inst->state_spinlock, flags); handle_dynamic_resolution_change(inst); + fbc_buf_count = inst->fbc_buf_count; spin_unlock_irqrestore(&inst->state_spinlock, flags); + wave5_update_min_bufs_ctrl(inst, fbc_buf_count); + return 0; } @@ -1602,6 +1617,7 @@ static void wave5_vpu_dec_device_run(void *priv) ret = initialize_sequence(inst); if (ret) { unsigned long flags; + u32 fbc_buf_count = 0; spin_lock_irqsave(&inst->state_spinlock, flags); if (wave5_is_draining_or_eos(inst) && @@ -1610,14 +1626,18 @@ static void wave5_vpu_dec_device_run(void *priv) switch_state(inst, VPU_INST_STATE_STOP); - if (vb2_is_streaming(dst_vq)) + if (vb2_is_streaming(dst_vq)) { send_eos_event(inst); - else + } else { handle_dynamic_resolution_change(inst); + fbc_buf_count = inst->fbc_buf_count; + } flag_last_buffer_done(inst); } spin_unlock_irqrestore(&inst->state_spinlock, flags); + + wave5_update_min_bufs_ctrl(inst, fbc_buf_count); } else { switch_state(inst, VPU_INST_STATE_INIT_SEQ); } diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c index 24fc0d0d3f4aa..87c145f1fd1a0 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c @@ -226,13 +226,6 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res) } else { dev_dbg(inst->dev->dev, "%s: wave5_vpu_enc_start_one_frame success\n", __func__); - /* - * Remove the source buffer from the ready-queue now and finish - * it in the videobuf2 framework once the index is returned by the - * firmware in finish_encode - */ - if (src_buf) - v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, src_buf->vb2_buf.index); } return 0; @@ -259,27 +252,13 @@ static void wave5_vpu_enc_finish_encode(struct vpu_instance *inst) __func__, enc_output_info.pic_type, enc_output_info.recon_frame_index, enc_output_info.enc_src_idx, enc_output_info.enc_pic_byte, enc_output_info.pts); - /* - * The source buffer will not be found in the ready-queue as it has been - * dropped after sending of the encode firmware command, locate it in - * the videobuf2 queue directly - */ if (enc_output_info.enc_src_idx >= 0) { - struct vb2_buffer *vb = vb2_get_buffer(v4l2_m2m_get_src_vq(m2m_ctx), - enc_output_info.enc_src_idx); - if (vb->state != VB2_BUF_STATE_ACTIVE) - dev_warn(inst->dev->dev, - "%s: encoded buffer (%d) was not in ready queue %i.", - __func__, enc_output_info.enc_src_idx, vb->state); - else - src_buf = to_vb2_v4l2_buffer(vb); - - if (src_buf) { + src_buf = v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, enc_output_info.enc_src_idx); + if (!src_buf) { + dev_warn(inst->dev->dev, "%s: no source buffer found\n", __func__); + } else { inst->timestamp = src_buf->vb2_buf.timestamp; v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE); - } else { - dev_warn(inst->dev->dev, "%s: no source buffer with index: %d found\n", - __func__, enc_output_info.enc_src_idx); } } @@ -936,6 +915,8 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl) case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE: inst->enc_param.profile = H264_PROFILE_BP; inst->bit_depth = 8; + if (ctrl->val == V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) + inst->enc_param.constraint_set1_flag = 1; break; case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN: inst->enc_param.profile = H264_PROFILE_MP; @@ -1211,6 +1192,7 @@ static int wave5_set_enc_openparam(struct enc_open_param *open_param, open_param->wave_param.intra_period = input.avc_idr_period; } } else { + open_param->wave_param.constraint_set1_flag = input.constraint_set1_flag; open_param->wave_param.avc_idr_period = input.avc_idr_period; } open_param->wave_param.entropy_coding_mode = input.entropy_coding_mode; @@ -1476,7 +1458,8 @@ static const struct vb2_ops wave5_vpu_enc_vb2_ops = { .stop_streaming = wave5_vpu_enc_stop_streaming, }; -static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt, +static void wave5_set_default_format(struct vpu_instance *inst, + struct v4l2_pix_format_mplane *src_fmt, struct v4l2_pix_format_mplane *dst_fmt) { src_fmt->pixelformat = enc_fmt_list[VPU_FMT_TYPE_RAW][0].v4l2_pix_fmt; @@ -1488,6 +1471,7 @@ static void wave5_set_default_format(struct v4l2_pix_format_mplane *src_fmt, wave5_update_pix_fmt(dst_fmt, VPU_FMT_TYPE_CODEC, W5_DEF_ENC_PIC_WIDTH, W5_DEF_ENC_PIC_HEIGHT, &enc_frmsize[VPU_FMT_TYPE_CODEC]); + inst->std = wave5_to_vpu_std(dst_fmt->pixelformat, inst->type); } static int wave5_vpu_enc_queue_init(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq) @@ -1683,7 +1667,7 @@ static int wave5_vpu_open_enc(struct file *filp) -6, 6, 1, 0); v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM, - 0, 1, 1, 1); + 0, 1, 1, 0); v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops, V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION, 0, 1, 1, 0); @@ -1749,7 +1733,7 @@ static int wave5_vpu_open_enc(struct file *filp) inst->v4l2_fh.ctrl_handler = v4l2_ctrl_hdl; v4l2_ctrl_handler_setup(v4l2_ctrl_hdl); - wave5_set_default_format(&inst->src_fmt, &inst->dst_fmt); + wave5_set_default_format(inst, &inst->src_fmt, &inst->dst_fmt); inst->conf_win.width = inst->dst_fmt.width; inst->conf_win.height = inst->dst_fmt.height; inst->colorspace = V4L2_COLORSPACE_REC709; diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c index e94d6ebc9f816..e7ea2f6a99d5b 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c @@ -559,8 +559,15 @@ int wave5_vpu_dec_get_output_info(struct vpu_instance *inst, struct dec_output_i p_dec_info->num_of_decoding_fbs : p_dec_info->num_of_display_fbs; if (info->index_frame_display >= 0 && - info->index_frame_display < (int)max_dec_index) - info->disp_frame = inst->frame_buf[val + info->index_frame_display]; + info->index_frame_display < (int)max_dec_index) { + u32 idx = val + info->index_frame_display; + + if (WARN_ON(idx >= MAX_REG_FRAME)) { + ret = -EINVAL; + goto err_out; + } + info->disp_frame = inst->frame_buf[idx]; + } info->rd_ptr = p_dec_info->stream_rd_ptr; info->wr_ptr = p_dec_info->stream_wr_ptr; diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h index bc101397204da..8112d4899ce46 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h @@ -570,6 +570,7 @@ struct enc_wave_param { u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8 transform */ u32 mb_level_rc_enable: 1; /* enable MB-level rate control */ u32 forced_idr_header_enable: 1; /* enable header encoding before IDR frame */ + u32 constraint_set1_flag: 1; /* enable CBP */ }; struct enc_open_param { diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h index 1ea9f5f314995..e04f2dbf3b655 100644 --- a/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuconfig.h @@ -59,6 +59,7 @@ // application specific configuration #define VPU_ENC_TIMEOUT 60000 #define VPU_DEC_TIMEOUT 60000 +#define VPU_DEC_STOP_TIMEOUT 300 // for WAVE encoder #define USE_SRC_PRP_AXI 0 diff --git a/drivers/media/platform/marvell/cafe-driver.c b/drivers/media/platform/marvell/cafe-driver.c index f9796de92aa77..39967ec5e7ee3 100644 --- a/drivers/media/platform/marvell/cafe-driver.c +++ b/drivers/media/platform/marvell/cafe-driver.c @@ -609,6 +609,7 @@ static void cafe_pci_remove(struct pci_dev *pdev) return; } cafe_shutdown(cam); + pci_disable_device(pdev); kfree(cam); } diff --git a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c index 6d26d4aa1eef2..f72ad840c2ee6 100644 --- a/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c +++ b/drivers/media/platform/mediatek/mdp3/mtk-mdp3-core.c @@ -292,7 +292,11 @@ static int mdp_probe(struct platform_device *pdev) goto err_destroy_clock_wq; } mdp->scp = platform_get_drvdata(mm_pdev); - put_device(&mm_pdev->dev); + } + + if (!mdp->scp) { + ret = -EPROBE_DEFER; + goto err_destroy_clock_wq; } mdp->rproc_handle = scp_get_rproc(mdp->scp); diff --git a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c index bf21f2467a0fc..22fd6b6714431 100644 --- a/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c +++ b/drivers/media/platform/mediatek/vcodec/decoder/vdec/vdec_av1_req_lat_if.c @@ -1299,11 +1299,12 @@ static void vdec_av1_slice_setup_tile(struct vdec_av1_slice_frame *frame, tile->uniform_tile_spacing_flag = BIT_FLAG(ctrl_tile, V4L2_AV1_TILE_INFO_FLAG_UNIFORM_TILE_SPACING); - for (i = 0; i < tile->tile_cols + 1; i++) + /* Bound the copy to the mi_col_starts[]/mi_row_starts[] capacity. */ + for (i = 0; i < tile->tile_cols + 1 && i < V4L2_AV1_MAX_TILE_COLS + 1; i++) tile->mi_col_starts[i] = ALIGN(ctrl_tile->mi_col_starts[i], BIT(mib_size_log2)) >> mib_size_log2; - for (i = 0; i < tile->tile_rows + 1; i++) + for (i = 0; i < tile->tile_rows + 1 && i < V4L2_AV1_MAX_TILE_ROWS + 1; i++) tile->mi_row_starts[i] = ALIGN(ctrl_tile->mi_row_starts[i], BIT(mib_size_log2)) >> mib_size_log2; } diff --git a/drivers/media/platform/microchip/microchip-sama7g5-isc.c b/drivers/media/platform/microchip/microchip-sama7g5-isc.c index b0302dfc3278d..7383341ec51d6 100644 --- a/drivers/media/platform/microchip/microchip-sama7g5-isc.c +++ b/drivers/media/platform/microchip/microchip-sama7g5-isc.c @@ -598,13 +598,8 @@ static int __maybe_unused xisc_runtime_suspend(struct device *dev) static int __maybe_unused xisc_runtime_resume(struct device *dev) { struct isc_device *isc = dev_get_drvdata(dev); - int ret; - - ret = clk_prepare_enable(isc->hclock); - if (ret) - return ret; - return ret; + return clk_prepare_enable(isc->hclock); } static const struct dev_pm_ops microchip_xisc_dev_pm_ops = { diff --git a/drivers/media/platform/nuvoton/npcm-video.c b/drivers/media/platform/nuvoton/npcm-video.c index 44e904e61801e..c4c5c318f7865 100644 --- a/drivers/media/platform/nuvoton/npcm-video.c +++ b/drivers/media/platform/nuvoton/npcm-video.c @@ -1720,10 +1720,12 @@ static int npcm_video_init(struct npcm_video *video) if (rc) { dev_err(dev, "Failed to set DMA mask\n"); of_reserved_mem_device_release(dev); + return rc; } rc = npcm_video_ece_init(video); if (rc) { + of_reserved_mem_device_release(dev); dev_err(dev, "Failed to initialize ECE\n"); return rc; } @@ -1748,42 +1750,55 @@ static int npcm_video_probe(struct platform_device *pdev) regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(regs)) { dev_err(&pdev->dev, "Failed to parse VCD reg in DTS\n"); - return PTR_ERR(regs); + rc = PTR_ERR(regs); + goto err_free; } video->vcd_regmap = devm_regmap_init_mmio(&pdev->dev, regs, &npcm_video_regmap_cfg); if (IS_ERR(video->vcd_regmap)) { dev_err(&pdev->dev, "Failed to initialize VCD regmap\n"); - return PTR_ERR(video->vcd_regmap); + rc = PTR_ERR(video->vcd_regmap); + goto err_free; } video->reset = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(video->reset)) { dev_err(&pdev->dev, "Failed to get VCD reset control in DTS\n"); - return PTR_ERR(video->reset); + rc = PTR_ERR(video->reset); + goto err_free; } video->gcr_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "nuvoton,sysgcr"); - if (IS_ERR(video->gcr_regmap)) - return PTR_ERR(video->gcr_regmap); + if (IS_ERR(video->gcr_regmap)) { + rc = PTR_ERR(video->gcr_regmap); + goto err_free; + } video->gfx_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node, "nuvoton,sysgfxi"); - if (IS_ERR(video->gfx_regmap)) - return PTR_ERR(video->gfx_regmap); + if (IS_ERR(video->gfx_regmap)) { + rc = PTR_ERR(video->gfx_regmap); + goto err_free; + } rc = npcm_video_init(video); if (rc) - return rc; + goto err_free; rc = npcm_video_setup_video(video); if (rc) - return rc; + goto err_release_mem; dev_info(video->dev, "NPCM video driver probed\n"); return 0; + +err_release_mem: + of_reserved_mem_device_release(&pdev->dev); +err_free: + kfree(video); + return rc; } static void npcm_video_remove(struct platform_device *pdev) @@ -1798,6 +1813,7 @@ static void npcm_video_remove(struct platform_device *pdev) v4l2_device_unregister(v4l2_dev); if (video->ece.enable) npcm_video_ece_stop(video); + kfree(video); of_reserved_mem_device_release(dev); } diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c index 60fed2fe200dc..a6ed1d124472a 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c @@ -517,6 +517,8 @@ static int mxc_isi_probe(struct platform_device *pdev) return 0; err_xbar: + while (i--) + mxc_isi_pipe_cleanup(&isi->pipes[i]); mxc_isi_crossbar_cleanup(&isi->crossbar); return ret; diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h index e84af5127e4e7..681a05045cd28 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h @@ -11,6 +11,7 @@ #define __MXC_ISI_CORE_H__ #include +#include #include #include #include @@ -412,4 +413,19 @@ static inline void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi) } #endif +/* + * ISI scaling engine works in two parts: it performs pre-decimation of + * the image followed by bilinear filtering to achieve the desired + * downscaling factor. + * + * The decimation filter provides a maximum downscaling factor of 8, and + * the subsequent bilinear filter provides a maximum downscaling factor + * of 2. Combined, the maximum scaling factor can be up to 16. + */ +static inline unsigned int +mxc_isi_clamp_downscale_16(unsigned int val, unsigned int max_val) +{ + return clamp(val, max(1U, DIV_ROUND_UP(max_val, 16)), max_val); +} + #endif /* __MXC_ISI_CORE_H__ */ diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c index ede6cc74c0234..37c7ed23eef1a 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c @@ -107,7 +107,7 @@ static int __mxc_isi_crossbar_set_routing(struct v4l2_subdev *sd, return ret; /* The memory input can be routed to the first pipeline only. */ - for_each_active_route(&state->routing, route) { + for_each_active_route(routing, route) { if (route->sink_pad == xbar->num_sinks - 1 && route->source_pad != xbar->num_sinks) { dev_dbg(xbar->isi->dev, @@ -145,10 +145,10 @@ mxc_isi_crossbar_xlate_streams(struct mxc_isi_crossbar *xbar, */ for_each_active_route(&state->routing, route) { if (route->source_pad != source_pad || - !(source_streams & BIT(route->source_stream))) + !(source_streams & BIT_ULL(route->source_stream))) continue; - sink_streams |= BIT(route->sink_stream); + sink_streams |= BIT_ULL(route->sink_stream); sink_pad = route->sink_pad; } @@ -492,6 +492,7 @@ int mxc_isi_crossbar_init(struct mxc_isi_dev *isi) void mxc_isi_crossbar_cleanup(struct mxc_isi_crossbar *xbar) { + v4l2_subdev_cleanup(&xbar->sd); media_entity_cleanup(&xbar->sd.entity); kfree(xbar->pads); kfree(xbar->inputs); diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c index 9225a7ac1c3ee..0bfd9b8f5f108 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c @@ -112,7 +112,14 @@ static u32 mxc_isi_channel_scaling_ratio(unsigned int from, unsigned int to, else *dec = 8; - return min_t(u32, from * 0x1000 / (to * *dec), ISI_DOWNSCALE_THRESHOLD); + /* + * The ISI rounds output dimensions up to the next integer (i.MX93 RM + * section 57.7.8). Calculate the scale factor such that the theoretical + * output (input / scale_factor) rounds up to exactly the desired + * output. + */ + return min_t(u32, DIV_ROUND_UP(from * 0x1000, to * *dec), + ISI_DOWNSCALE_THRESHOLD); } static void mxc_isi_channel_set_scaling(struct mxc_isi_pipe *pipe, diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c index 00afcbfbdde48..f2a73f0f1ac77 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c @@ -509,9 +509,14 @@ __mxc_isi_m2m_try_fmt_vid(struct mxc_isi_m2m_ctx *ctx, const enum mxc_isi_video_type type) { if (type == MXC_ISI_VIDEO_M2M_CAP) { - /* Downscaling only */ - pix->width = min(pix->width, ctx->queues.out.format.width); - pix->height = min(pix->height, ctx->queues.out.format.height); + const struct v4l2_pix_format_mplane *format = + &ctx->queues.out.format; + + /* Downscaling only, by up to 16. */ + pix->width = mxc_isi_clamp_downscale_16(pix->width, + format->width); + pix->height = mxc_isi_clamp_downscale_16(pix->height, + format->height); } return mxc_isi_format_try(ctx->m2m->pipe, pix, type); diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c index a41c51dd9ce0f..2d0843c86534c 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c @@ -641,16 +641,19 @@ static int mxc_isi_pipe_set_selection(struct v4l2_subdev *sd, /* Composing is supported on the sink only. */ return -EINVAL; - /* The sink crop is bound by the sink format downscaling only). */ + /* + * The ISI supports downscaling only, with a factor up to 16. + * Clamp the compose rectangle size accordingly. + */ format = mxc_isi_pipe_get_pad_format(pipe, state, MXC_ISI_PIPE_PAD_SINK); sel->r.left = 0; sel->r.top = 0; - sel->r.width = clamp(sel->r.width, MXC_ISI_MIN_WIDTH, - format->width); - sel->r.height = clamp(sel->r.height, MXC_ISI_MIN_HEIGHT, - format->height); + sel->r.width = mxc_isi_clamp_downscale_16(sel->r.width, + format->width); + sel->r.height = mxc_isi_clamp_downscale_16(sel->r.height, + format->height); rect = mxc_isi_pipe_get_pad_compose(pipe, state, MXC_ISI_PIPE_PAD_SINK); @@ -796,18 +799,20 @@ int mxc_isi_pipe_init(struct mxc_isi_dev *isi, unsigned int id) irq = platform_get_irq(to_platform_device(isi->dev), id); if (irq < 0) { ret = irq; - goto error; + goto error_subdev; } ret = devm_request_irq(isi->dev, irq, mxc_isi_pipe_irq_handler, 0, dev_name(isi->dev), pipe); if (ret < 0) { dev_err(isi->dev, "failed to request IRQ (%d)\n", ret); - goto error; + goto error_subdev; } return 0; +error_subdev: + v4l2_subdev_cleanup(sd); error: media_entity_cleanup(&sd->entity); mutex_destroy(&pipe->lock); @@ -819,6 +824,7 @@ void mxc_isi_pipe_cleanup(struct mxc_isi_pipe *pipe) { struct v4l2_subdev *sd = &pipe->sd; + v4l2_subdev_cleanup(sd); media_entity_cleanup(&sd->entity); mutex_destroy(&pipe->lock); } diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c index 1be3a728f32f8..000c112ce6cb7 100644 --- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c +++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c @@ -151,7 +151,7 @@ static const struct mxc_isi_format_info mxc_isi_formats[] = { .fourcc = V4L2_PIX_FMT_XBGR32, .type = MXC_ISI_VIDEO_CAP | MXC_ISI_VIDEO_M2M_OUT | MXC_ISI_VIDEO_M2M_CAP, - .isi_in_format = CHNL_MEM_RD_CTRL_IMG_TYPE_XBGR8, + .isi_in_format = CHNL_MEM_RD_CTRL_IMG_TYPE_XRGB8, .isi_out_format = CHNL_IMG_CTRL_FORMAT_XRGB888, .mem_planes = 1, .color_planes = 1, diff --git a/drivers/media/platform/qcom/camss/camss-csid-340.c b/drivers/media/platform/qcom/camss/camss-csid-340.c index 22a30510fb792..0231985746edf 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-340.c +++ b/drivers/media/platform/qcom/camss/camss-csid-340.c @@ -55,8 +55,7 @@ #define CSID_RDI_CTRL_HALT_AT_FRAME_BOUNDARY 0 #define CSID_RDI_CTRL_RESUME_AT_FRAME_BOUNDARY 1 -static void __csid_configure_rx(struct csid_device *csid, - struct csid_phy_config *phy, int vc) +static void __csid_configure_rx(struct csid_device *csid, struct csid_phy_config *phy) { u32 val; @@ -75,32 +74,28 @@ static void __csid_ctrl_rdi(struct csid_device *csid, int enable, u8 rdi) writel_relaxed(!!enable, csid->base + CSID_RDI_CTRL(rdi)); } -static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc) +static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc) { - struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc]; + struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port]; const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); - u8 lane_cnt = csid->phy.lane_cnt; u8 dt_id; u32 val; - if (!lane_cnt) - lane_cnt = 4; - /* * DT_ID is a two bit bitfield that is concatenated with * the four least significant bits of the five bit VC * bitfield to generate an internal CID value. * - * CSID_RDI_CFG0(vc) + * CSID_RDI_CFG0(port) * DT_ID : 28:27 * VC : 26:22 * DT : 21:16 * * CID : VC 3:0 << 2 | DT_ID 1:0 */ - dt_id = vc & 0x03; + dt_id = port & 0x03; val = CSID_RDI_CFG0_DECODE_FORMAT_NOP; /* only for RDI path */ val |= FIELD_PREP(CSID_RDI_CFG0_DT_MASK, format->data_type); @@ -110,20 +105,23 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 if (enable) val |= CSID_RDI_CFG0_ENABLE; - dev_dbg(csid->camss->dev, "CSID%u: Stream %s (dt:0x%x vc=%u)\n", - csid->id, enable ? "enable" : "disable", format->data_type, vc); + dev_dbg(csid->camss->dev, "CSID%u: Stream %s (dt:0x%x port=%u vc=%u)\n", + csid->id, enable ? "enable" : "disable", format->data_type, + port, vc); - writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc)); + writel_relaxed(val, csid->base + CSID_RDI_CFG0(port)); } static void csid_configure_stream(struct csid_device *csid, u8 enable) { int i; + __csid_configure_rx(csid, &csid->phy); + + /* Loop through all enabled ports and configure a stream for each */ for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) { if (csid->phy.en_vc & BIT(i)) { - __csid_configure_rdi_stream(csid, enable, i); - __csid_configure_rx(csid, &csid->phy, i); + __csid_configure_rdi_stream(csid, enable, i, 0); __csid_ctrl_rdi(csid, enable, i); } } diff --git a/drivers/media/platform/qcom/camss/camss-csid-680.c b/drivers/media/platform/qcom/camss/camss-csid-680.c index 3ad3a174bcfb8..edf01ba79907d 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-680.c +++ b/drivers/media/platform/qcom/camss/camss-csid-680.c @@ -219,9 +219,9 @@ static void __csid_configure_top(struct csid_device *csid) CSID_TOP_IO_PATH_CFG0(csid->id)); } -static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc) +static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc) { - struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc]; + struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port]; const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); @@ -233,28 +233,28 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 lane_cnt = 4; val = 0; - writel(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc)); + writel(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(port)); /* * DT_ID is a two bit bitfield that is concatenated with * the four least significant bits of the five bit VC * bitfield to generate an internal CID value. * - * CSID_RDI_CFG0(vc) + * CSID_RDI_CFG0(port) * DT_ID : 28:27 * VC : 26:22 * DT : 21:16 * * CID : VC 3:0 << 2 | DT_ID 1:0 */ - dt_id = vc & 0x03; + dt_id = port & 0x03; /* note: for non-RDI path, this should be format->decode_format */ val |= DECODE_FORMAT_PAYLOAD_ONLY << RDI_CFG0_DECODE_FORMAT; val |= format->data_type << RDI_CFG0_DATA_TYPE; val |= vc << RDI_CFG0_VIRTUAL_CHANNEL; val |= dt_id << RDI_CFG0_DT_ID; - writel(val, csid->base + CSID_RDI_CFG0(vc)); + writel(val, csid->base + CSID_RDI_CFG0(port)); val = RDI_CFG1_TIMESTAMP_STB_FRAME; val |= RDI_CFG1_BYTE_CNTR_EN; @@ -265,23 +265,23 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 val |= RDI_CFG1_CROP_V_EN; val |= RDI_CFG1_PACKING_MIPI; - writel(val, csid->base + CSID_RDI_CFG1(vc)); + writel(val, csid->base + CSID_RDI_CFG1(port)); val = 0; - writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc)); + writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port)); val = 1; - writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc)); + writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port)); val = 0; - writel(val, csid->base + CSID_RDI_CTRL(vc)); + writel(val, csid->base + CSID_RDI_CTRL(port)); - val = readl(csid->base + CSID_RDI_CFG0(vc)); + val = readl(csid->base + CSID_RDI_CFG0(port)); if (enable) val |= RDI_CFG0_ENABLE; else val &= ~RDI_CFG0_ENABLE; - writel(val, csid->base + CSID_RDI_CFG0(vc)); + writel(val, csid->base + CSID_RDI_CFG0(port)); } static void csid_configure_stream(struct csid_device *csid, u8 enable) @@ -290,11 +290,11 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable) __csid_configure_top(csid); - /* Loop through all enabled VCs and configure stream for each */ + /* Loop through all enabled ports and configure a stream for each */ for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) { if (csid->phy.en_vc & BIT(i)) { - __csid_configure_rdi_stream(csid, enable, i); - __csid_configure_rx(csid, &csid->phy, i); + __csid_configure_rdi_stream(csid, enable, i, 0); + __csid_configure_rx(csid, &csid->phy, 0); __csid_ctrl_rdi(csid, enable, i); } } diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen2.c b/drivers/media/platform/qcom/camss/camss-csid-gen2.c index 2a1746dcc1c5b..eadcb2f7e3aaa 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-gen2.c +++ b/drivers/media/platform/qcom/camss/camss-csid-gen2.c @@ -203,10 +203,10 @@ static void __csid_ctrl_rdi(struct csid_device *csid, int enable, u8 rdi) writel_relaxed(val, csid->base + CSID_RDI_CTRL(rdi)); } -static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc) +static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 port, u8 vc) { struct csid_testgen_config *tg = &csid->testgen; - struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc]; + struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port]; const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); @@ -253,10 +253,10 @@ static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc) writel_relaxed(val, csid->base + CSID_TPG_CTRL); } -static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc) +static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc) { /* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */ - struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc]; + struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port]; const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); @@ -267,14 +267,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 * the four least significant bits of the five bit VC * bitfield to generate an internal CID value. * - * CSID_RDI_CFG0(vc) + * CSID_RDI_CFG0(port) * DT_ID : 28:27 * VC : 26:22 * DT : 21:16 * * CID : VC 3:0 << 2 | DT_ID 1:0 */ - u8 dt_id = vc & 0x03; + u8 dt_id = port & 0x03; val = 1 << RDI_CFG0_BYTE_CNTR_EN; val |= 1 << RDI_CFG0_FORMAT_MEASURE_EN; @@ -284,56 +284,57 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 val |= format->data_type << RDI_CFG0_DATA_TYPE; val |= vc << RDI_CFG0_VIRTUAL_CHANNEL; val |= dt_id << RDI_CFG0_DT_ID; - writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc)); + writel_relaxed(val, csid->base + CSID_RDI_CFG0(port)); /* CSID_TIMESTAMP_STB_POST_IRQ */ val = 2 << RDI_CFG1_TIMESTAMP_STB_SEL; - writel_relaxed(val, csid->base + CSID_RDI_CFG1(vc)); + writel_relaxed(val, csid->base + CSID_RDI_CFG1(port)); val = 1; - writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc)); + writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(port)); val = 0; - writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(vc)); + writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(port)); val = 1; - writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc)); + writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port)); val = 0; - writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc)); + writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port)); val = 1; - writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(vc)); + writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(port)); val = 0; - writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(vc)); + writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(port)); val = 1; - writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(vc)); + writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(port)); val = 0; - writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(vc)); + writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(port)); val = 0; - writel_relaxed(val, csid->base + CSID_RDI_CTRL(vc)); + writel_relaxed(val, csid->base + CSID_RDI_CTRL(port)); - val = readl_relaxed(csid->base + CSID_RDI_CFG0(vc)); + val = readl_relaxed(csid->base + CSID_RDI_CFG0(port)); val |= enable << RDI_CFG0_ENABLE; - writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc)); + writel_relaxed(val, csid->base + CSID_RDI_CFG0(port)); } static void csid_configure_stream(struct csid_device *csid, u8 enable) { struct csid_testgen_config *tg = &csid->testgen; u8 i; - /* Loop through all enabled VCs and configure stream for each */ + + /* Loop through all enabled ports and configure a stream for each */ for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) if (csid->phy.en_vc & BIT(i)) { if (tg->enabled) - __csid_configure_testgen(csid, enable, i); + __csid_configure_testgen(csid, enable, i, 0); - __csid_configure_rdi_stream(csid, enable, i); - __csid_configure_rx(csid, &csid->phy, i); + __csid_configure_rdi_stream(csid, enable, i, 0); + __csid_configure_rx(csid, &csid->phy, 0); __csid_ctrl_rdi(csid, enable, i); } } diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen3.c b/drivers/media/platform/qcom/camss/camss-csid-gen3.c index bd059243790ed..ed5c5766efd36 100644 --- a/drivers/media/platform/qcom/camss/camss-csid-gen3.c +++ b/drivers/media/platform/qcom/camss/camss-csid-gen3.c @@ -145,12 +145,12 @@ static void __csid_configure_wrapper(struct csid_device *csid) writel(val, csid->camss->csid_wrapper_base + CSID_IO_PATH_CFG0(csid->id)); } -static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc) +static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc) { u32 val; u8 lane_cnt = csid->phy.lane_cnt; /* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */ - struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc]; + struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port]; const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats, csid->res->formats->nformats, input_format->code); @@ -163,14 +163,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 * the four least significant bits of the five bit VC * bitfield to generate an internal CID value. * - * CSID_RDI_CFG0(vc) + * CSID_RDI_CFG0(port) * DT_ID : 28:27 * VC : 26:22 * DT : 21:16 * * CID : VC 3:0 << 2 | DT_ID 1:0 */ - u8 dt_id = vc & 0x03; + u8 dt_id = port & 0x03; val = RDI_CFG0_TIMESTAMP_EN; val |= RDI_CFG0_TIMESTAMP_STB_SEL; @@ -180,7 +180,7 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 val |= format->data_type << RDI_CFG0_DT; val |= dt_id << RDI_CFG0_DT_ID; - writel(val, csid->base + CSID_RDI_CFG0(vc)); + writel(val, csid->base + CSID_RDI_CFG0(port)); val = RDI_CFG1_PACKING_FORMAT_MIPI; val |= RDI_CFG1_PIX_STORE; @@ -189,22 +189,22 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 val |= RDI_CFG1_CROP_H_EN; val |= RDI_CFG1_CROP_V_EN; - writel(val, csid->base + CSID_RDI_CFG1(vc)); + writel(val, csid->base + CSID_RDI_CFG1(port)); val = 0; - writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc)); + writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port)); val = 1; - writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc)); + writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port)); val = 0; - writel(val, csid->base + CSID_RDI_CTRL(vc)); + writel(val, csid->base + CSID_RDI_CTRL(port)); - val = readl(csid->base + CSID_RDI_CFG0(vc)); + val = readl(csid->base + CSID_RDI_CFG0(port)); if (enable) val |= RDI_CFG0_EN; - writel(val, csid->base + CSID_RDI_CFG0(vc)); + writel(val, csid->base + CSID_RDI_CFG0(port)); } static void csid_configure_stream(struct csid_device *csid, u8 enable) @@ -213,11 +213,11 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable) __csid_configure_wrapper(csid); - /* Loop through all enabled VCs and configure stream for each */ + /* Loop through all enabled ports and configure a stream for each */ for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) if (csid->phy.en_vc & BIT(i)) { - __csid_configure_rdi_stream(csid, enable, i); - __csid_configure_rx(csid, &csid->phy, i); + __csid_configure_rdi_stream(csid, enable, i, 0); + __csid_configure_rx(csid, &csid->phy, 0); __csid_ctrl_rdi(csid, enable, i); } } diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c index 2de97f58f9ae4..74c77969db7ef 100644 --- a/drivers/media/platform/qcom/camss/camss-csiphy.c +++ b/drivers/media/platform/qcom/camss/camss-csiphy.c @@ -558,12 +558,16 @@ static int csiphy_init_formats(struct v4l2_subdev *sd, return csiphy_set_format(sd, fh ? fh->state : NULL, &format); } -static bool csiphy_match_clock_name(const char *clock_name, const char *format, - int index) +static bool __printf(2, 3) +csiphy_match_clock_name(const char *clock_name, const char *format, ...) { char name[16]; /* csiphyXXX_timer\0 */ + va_list args; + + va_start(args, format); + vsnprintf(name, sizeof(name), format, args); + va_end(args); - snprintf(name, sizeof(name), format, index); return !strcmp(clock_name, name); } diff --git a/drivers/media/platform/qcom/camss/camss-vfe-340.c b/drivers/media/platform/qcom/camss/camss-vfe-340.c index 30d7630b3e8b3..d129b0d3a6edb 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe-340.c +++ b/drivers/media/platform/qcom/camss/camss-vfe-340.c @@ -69,24 +69,19 @@ #define TFE_BUS_FRAMEDROP_CFG_0(c) BUS_REG(0x238 + (c) * 0x100) #define TFE_BUS_FRAMEDROP_CFG_1(c) BUS_REG(0x23c + (c) * 0x100) -/* - * TODO: differentiate the port id based on requested type of RDI, BHIST etc - * - * TFE write master IDs (clients) - * - * BAYER 0 - * IDEAL_RAW 1 - * STATS_TINTLESS_BG 2 - * STATS_BHIST 3 - * STATS_AWB_BG 4 - * STATS_AEC_BG 5 - * STATS_BAF 6 - * RDI0 7 - * RDI1 8 - * RDI2 9 - */ -#define RDI_WM(n) (7 + (n)) -#define TFE_WM_NUM 10 +enum tfe_client { + TFE_CLI_BAYER, + TFE_CLI_IDEAL_RAW, + TFE_CLI_STATS_TINTLESS_BG, + TFE_CLI_STATS_BHIST, + TFE_CLI_STATS_AWB_BG, + TFE_CLI_STATS_AEC_BG, + TFE_CLI_STATS_BAF, + TFE_CLI_RDI0, + TFE_CLI_RDI1, + TFE_CLI_RDI2, + TFE_CLI_NUM +}; enum tfe_iface { TFE_IFACE_PIX, @@ -108,6 +103,13 @@ enum tfe_subgroups { TFE_SUBGROUP_NUM }; +static enum tfe_client tfe_wm_client_map[VFE_LINE_NUM_MAX] = { + [VFE_LINE_RDI0] = TFE_CLI_RDI0, + [VFE_LINE_RDI1] = TFE_CLI_RDI1, + [VFE_LINE_RDI2] = TFE_CLI_RDI2, + [VFE_LINE_PIX] = TFE_CLI_BAYER, +}; + static enum tfe_iface tfe_line_iface_map[VFE_LINE_NUM_MAX] = { [VFE_LINE_RDI0] = TFE_IFACE_RDI0, [VFE_LINE_RDI1] = TFE_IFACE_RDI1, @@ -209,10 +211,10 @@ static irqreturn_t vfe_isr(int irq, void *dev) status = readl_relaxed(vfe->base + TFE_BUS_OVERFLOW_STATUS); if (status) { writel_relaxed(status, vfe->base + TFE_BUS_STATUS_CLEAR); - for (i = 0; i < TFE_WM_NUM; i++) { + for (i = 0; i < TFE_CLI_NUM; i++) { if (status & BIT(i)) dev_err_ratelimited(vfe->camss->dev, - "VFE%u: bus overflow for wm %u\n", + "VFE%u: bus overflow for client %u\n", vfe->id, i); } } @@ -235,49 +237,49 @@ static void vfe_enable_irq(struct vfe_device *vfe) TFE_BUS_IRQ_MASK_0_IMG_VIOL, vfe->base + TFE_BUS_IRQ_MASK_0); } -static void vfe_wm_update(struct vfe_device *vfe, u8 rdi, u32 addr, +static void vfe_wm_update(struct vfe_device *vfe, u8 wm, u32 addr, struct vfe_line *line) { - u8 wm = RDI_WM(rdi); + u8 client = tfe_wm_client_map[wm]; - writel_relaxed(addr, vfe->base + TFE_BUS_IMAGE_ADDR(wm)); + writel_relaxed(addr, vfe->base + TFE_BUS_IMAGE_ADDR(client)); } -static void vfe_wm_start(struct vfe_device *vfe, u8 rdi, struct vfe_line *line) +static void vfe_wm_start(struct vfe_device *vfe, u8 wm, struct vfe_line *line) { struct v4l2_pix_format_mplane *pix = &line->video_out.active_fmt.fmt.pix_mp; u32 stride = pix->plane_fmt[0].bytesperline; - u8 wm = RDI_WM(rdi); + u8 client = tfe_wm_client_map[wm]; /* Configuration for plain RDI frames */ - writel_relaxed(TFE_BUS_IMAGE_CFG_0_DEFAULT, vfe->base + TFE_BUS_IMAGE_CFG_0(wm)); - writel_relaxed(0u, vfe->base + TFE_BUS_IMAGE_CFG_1(wm)); - writel_relaxed(TFE_BUS_IMAGE_CFG_2_DEFAULT, vfe->base + TFE_BUS_IMAGE_CFG_2(wm)); - writel_relaxed(stride * pix->height, vfe->base + TFE_BUS_FRAME_INCR(wm)); - writel_relaxed(TFE_BUS_PACKER_CFG_FMT_PLAIN64, vfe->base + TFE_BUS_PACKER_CFG(wm)); + writel_relaxed(TFE_BUS_IMAGE_CFG_0_DEFAULT, vfe->base + TFE_BUS_IMAGE_CFG_0(client)); + writel_relaxed(0u, vfe->base + TFE_BUS_IMAGE_CFG_1(client)); + writel_relaxed(TFE_BUS_IMAGE_CFG_2_DEFAULT, vfe->base + TFE_BUS_IMAGE_CFG_2(client)); + writel_relaxed(stride * pix->height, vfe->base + TFE_BUS_FRAME_INCR(client)); + writel_relaxed(TFE_BUS_PACKER_CFG_FMT_PLAIN64, vfe->base + TFE_BUS_PACKER_CFG(client)); /* No dropped frames, one irq per frame */ - writel_relaxed(0, vfe->base + TFE_BUS_FRAMEDROP_CFG_0(wm)); - writel_relaxed(1, vfe->base + TFE_BUS_FRAMEDROP_CFG_1(wm)); - writel_relaxed(0, vfe->base + TFE_BUS_IRQ_SUBSAMPLE_CFG_0(wm)); - writel_relaxed(1, vfe->base + TFE_BUS_IRQ_SUBSAMPLE_CFG_1(wm)); + writel_relaxed(0, vfe->base + TFE_BUS_FRAMEDROP_CFG_0(client)); + writel_relaxed(1, vfe->base + TFE_BUS_FRAMEDROP_CFG_1(client)); + writel_relaxed(0, vfe->base + TFE_BUS_IRQ_SUBSAMPLE_CFG_0(client)); + writel_relaxed(1, vfe->base + TFE_BUS_IRQ_SUBSAMPLE_CFG_1(client)); vfe_enable_irq(vfe); writel(TFE_BUS_CLIENT_CFG_EN | TFE_BUS_CLIENT_CFG_MODE_FRAME, - vfe->base + TFE_BUS_CLIENT_CFG(wm)); + vfe->base + TFE_BUS_CLIENT_CFG(client)); - dev_dbg(vfe->camss->dev, "VFE%u: Started RDI%u width %u height %u stride %u\n", - vfe->id, rdi, pix->width, pix->height, stride); + dev_dbg(vfe->camss->dev, "VFE%u: Started client %u width %u height %u stride %u\n", + vfe->id, client, pix->width, pix->height, client); } -static void vfe_wm_stop(struct vfe_device *vfe, u8 rdi) +static void vfe_wm_stop(struct vfe_device *vfe, u8 wm) { - u8 wm = RDI_WM(rdi); + u8 client = tfe_wm_client_map[wm]; - writel(0, vfe->base + TFE_BUS_CLIENT_CFG(wm)); + writel(0, vfe->base + TFE_BUS_CLIENT_CFG(client)); - dev_dbg(vfe->camss->dev, "VFE%u: Stopped RDI%u\n", vfe->id, rdi); + dev_dbg(vfe->camss->dev, "VFE%u: Stopped client %u\n", vfe->id, client); } static const struct camss_video_ops vfe_video_ops_520 = { diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c index 8406c48d635b6..e337f8b7e6f07 100644 --- a/drivers/media/platform/qcom/iris/iris_core.c +++ b/drivers/media/platform/qcom/iris/iris_core.c @@ -12,18 +12,24 @@ void iris_core_deinit(struct iris_core *core) { - pm_runtime_resume_and_get(core->dev); + int ret; + + ret = pm_runtime_resume_and_get(core->dev); mutex_lock(&core->lock); if (core->state != IRIS_CORE_DEINIT) { iris_fw_unload(core); - iris_vpu_power_off(core); + + if (!ret) + iris_vpu_power_off(core); + iris_hfi_queues_deinit(core); core->state = IRIS_CORE_DEINIT; } mutex_unlock(&core->lock); - pm_runtime_put_sync(core->dev); + if (!ret) + pm_runtime_put_sync(core->dev); } static int iris_wait_for_system_response(struct iris_core *core) diff --git a/drivers/media/platform/qcom/iris/iris_platform_common.h b/drivers/media/platform/qcom/iris/iris_platform_common.h index 58d05e0a112ee..69720d6790508 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_common.h +++ b/drivers/media/platform/qcom/iris/iris_platform_common.h @@ -41,11 +41,11 @@ enum pipe_type { PIPE_4 = 4, }; -extern struct iris_platform_data qcs8300_data; -extern struct iris_platform_data sm8250_data; -extern struct iris_platform_data sm8550_data; -extern struct iris_platform_data sm8650_data; -extern struct iris_platform_data sm8750_data; +extern const struct iris_platform_data qcs8300_data; +extern const struct iris_platform_data sm8250_data; +extern const struct iris_platform_data sm8550_data; +extern const struct iris_platform_data sm8650_data; +extern const struct iris_platform_data sm8750_data; enum platform_clk_type { IRIS_AXI_CLK, /* AXI0 in case of platforms with multiple AXI clocks */ diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen2.c b/drivers/media/platform/qcom/iris/iris_platform_gen2.c index 85beb80476de8..768362683a744 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_gen2.c +++ b/drivers/media/platform/qcom/iris/iris_platform_gen2.c @@ -737,7 +737,7 @@ static const u32 sm8550_enc_op_int_buf_tbl[] = { BUF_SCRATCH_2, }; -struct iris_platform_data sm8550_data = { +const struct iris_platform_data sm8550_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, @@ -827,7 +827,7 @@ struct iris_platform_data sm8550_data = { * - controller_rst_tbl to sm8650_controller_reset_table * - fwname to "qcom/vpu/vpu33_p4.mbn" */ -struct iris_platform_data sm8650_data = { +const struct iris_platform_data sm8650_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, @@ -912,7 +912,7 @@ struct iris_platform_data sm8650_data = { .enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl), }; -struct iris_platform_data sm8750_data = { +const struct iris_platform_data sm8750_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, @@ -1000,7 +1000,7 @@ struct iris_platform_data sm8750_data = { * - inst_caps to platform_inst_cap_qcs8300 * - inst_fw_caps to inst_fw_cap_qcs8300 */ -struct iris_platform_data qcs8300_data = { +const struct iris_platform_data qcs8300_data = { .get_instance = iris_hfi_gen2_get_instance, .init_hfi_command_ops = iris_hfi_gen2_command_ops_init, .init_hfi_response_ops = iris_hfi_gen2_response_ops_init, diff --git a/drivers/media/platform/qcom/iris/iris_platform_sm8250.c b/drivers/media/platform/qcom/iris/iris_platform_sm8250.c index af841f6cd9d2a..d130d18c6543b 100644 --- a/drivers/media/platform/qcom/iris/iris_platform_sm8250.c +++ b/drivers/media/platform/qcom/iris/iris_platform_sm8250.c @@ -314,7 +314,7 @@ static const u32 sm8250_enc_ip_int_buf_tbl[] = { BUF_SCRATCH_2, }; -struct iris_platform_data sm8250_data = { +const struct iris_platform_data sm8250_data = { .get_instance = iris_hfi_gen1_get_instance, .init_hfi_command_ops = &iris_hfi_gen1_command_ops_init, .init_hfi_response_ops = iris_hfi_gen1_response_ops_init, diff --git a/drivers/media/platform/qcom/iris/iris_probe.c b/drivers/media/platform/qcom/iris/iris_probe.c index a197833bac618..cdeb78ba605ae 100644 --- a/drivers/media/platform/qcom/iris/iris_probe.c +++ b/drivers/media/platform/qcom/iris/iris_probe.c @@ -256,12 +256,12 @@ static int iris_probe(struct platform_device *pdev) core->iris_platform_data = of_device_get_match_data(core->dev); ret = devm_request_threaded_irq(core->dev, core->irq, iris_hfi_isr, - iris_hfi_isr_handler, IRQF_TRIGGER_HIGH, "iris", core); + iris_hfi_isr_handler, + IRQF_TRIGGER_HIGH | IRQF_NO_AUTOEN, + "iris", core); if (ret) return ret; - disable_irq_nosync(core->irq); - iris_init_ops(core); core->iris_platform_data->init_hfi_command_ops(core); core->iris_platform_data->init_hfi_response_ops(core); diff --git a/drivers/media/platform/qcom/iris/iris_resources.c b/drivers/media/platform/qcom/iris/iris_resources.c index cf32f268b703c..dcec4f3460d88 100644 --- a/drivers/media/platform/qcom/iris/iris_resources.c +++ b/drivers/media/platform/qcom/iris/iris_resources.c @@ -66,24 +66,21 @@ int iris_enable_power_domains(struct iris_core *core, struct device *pd_dev) if (ret) return ret; - ret = pm_runtime_get_sync(pd_dev); - if (ret < 0) - return ret; - - return ret; + return pm_runtime_resume_and_get(pd_dev); } int iris_disable_power_domains(struct iris_core *core, struct device *pd_dev) { int ret; + int pm_ret; ret = dev_pm_opp_set_rate(core->dev, 0); - if (ret) - return ret; - pm_runtime_put_sync(pd_dev); + pm_ret = pm_runtime_put_sync(pd_dev); + if (!ret) + ret = pm_ret; - return 0; + return ret; } static struct clk *iris_get_clk_by_type(struct iris_core *core, enum platform_clk_type clk_type) diff --git a/drivers/media/platform/qcom/iris/iris_state.c b/drivers/media/platform/qcom/iris/iris_state.c index d14472414750d..5552725c614ea 100644 --- a/drivers/media/platform/qcom/iris/iris_state.c +++ b/drivers/media/platform/qcom/iris/iris_state.c @@ -60,9 +60,9 @@ int iris_inst_change_state(struct iris_inst *inst, return -EINVAL; change_state: - inst->state = request_state; dev_dbg(inst->core->dev, "state changed from %x to %x\n", inst->state, request_state); + inst->state = request_state; return 0; } @@ -269,7 +269,7 @@ bool iris_allow_cmd(struct iris_inst *inst, u32 cmd) return true; } else if (cmd == V4L2_DEC_CMD_STOP || cmd == V4L2_ENC_CMD_STOP) { if (vb2_is_streaming(src_q)) - if (inst->sub_state != IRIS_INST_SUB_DRAIN) + if (!(inst->sub_state & IRIS_INST_SUB_DRAIN)) return true; } diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c index 07682400de690..4c43d816e8309 100644 --- a/drivers/media/platform/qcom/iris/iris_vidc.c +++ b/drivers/media/platform/qcom/iris/iris_vidc.c @@ -450,14 +450,21 @@ static int iris_enum_frameintervals(struct file *filp, void *fh, static int iris_querycap(struct file *filp, void *fh, struct v4l2_capability *cap) { + struct iris_core *core = video_drvdata(filp); struct iris_inst *inst = iris_get_inst(filp); + char *info; strscpy(cap->driver, IRIS_DRV_NAME, sizeof(cap->driver)); - if (inst->domain == DECODER) + if (inst->domain == DECODER) { strscpy(cap->card, "Iris Decoder", sizeof(cap->card)); - else + info = "dec"; + } else { strscpy(cap->card, "Iris Encoder", sizeof(cap->card)); + info = "enc"; + } + snprintf(cap->bus_info, sizeof(cap->bus_info), + "plat:%s:%s", dev_name(core->dev), info); return 0; } diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c index bb98950e018fa..6839e293d9b50 100644 --- a/drivers/media/platform/qcom/iris/iris_vpu_common.c +++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c @@ -272,7 +272,7 @@ void iris_vpu_power_off(struct iris_core *core) iris_unset_icc_bw(core); if (!iris_vpu_watchdog(core, core->intr_status)) - disable_irq_nosync(core->irq); + disable_irq(core->irq); } int iris_vpu_power_on_controller(struct iris_core *core) diff --git a/drivers/media/platform/qcom/venus/hfi_parser.c b/drivers/media/platform/qcom/venus/hfi_parser.c index c4cf6cd50a9a0..e88a658196260 100644 --- a/drivers/media/platform/qcom/venus/hfi_parser.c +++ b/drivers/media/platform/qcom/venus/hfi_parser.c @@ -85,7 +85,7 @@ parse_alloc_mode(struct venus_core *core, u32 codecs, u32 domain, void *data) type++; } - return sizeof(*mode); + return mode->num_entries * sizeof(u32) + sizeof(*mode); } static void fill_profile_level(struct hfi_plat_caps *cap, const void *data, @@ -146,7 +146,7 @@ parse_caps(struct venus_core *core, u32 codecs, u32 domain, void *data) for_each_codec(core->caps, ARRAY_SIZE(core->caps), codecs, domain, fill_caps, caps_arr, num_caps); - return sizeof(*caps); + return num_caps * sizeof(*cap) + sizeof(u32); } static void fill_raw_fmts(struct hfi_plat_caps *cap, const void *fmts, @@ -171,7 +171,7 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) u32 entries = fmt->format_entries; unsigned int i = 0; u32 num_planes = 0; - u32 size; + u32 size = 2 * sizeof(u32); while (entries) { num_planes = pinfo->num_planes; @@ -186,6 +186,7 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) if (pinfo->num_planes > MAX_PLANES) break; + size += sizeof(*constr) * num_planes + 2 * sizeof(u32); pinfo = (void *)pinfo + sizeof(*constr) * num_planes + 2 * sizeof(u32); entries--; @@ -193,8 +194,6 @@ parse_raw_formats(struct venus_core *core, u32 codecs, u32 domain, void *data) for_each_codec(core->caps, ARRAY_SIZE(core->caps), codecs, domain, fill_raw_fmts, rawfmts, i); - size = fmt->format_entries * (sizeof(*constr) * num_planes + 2 * sizeof(u32)) - + 2 * sizeof(u32); return size; } diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index 14a4e8311a643..a60af81c17c37 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -645,7 +645,6 @@ static int decide_core(struct venus_inst *inst) unsigned long max_freq = ULONG_MAX; struct device *dev = core->dev; struct dev_pm_opp *opp; - int ret = 0; if (legacy_binding) { if (inst->session_type == VIDC_SESSION_TYPE_DEC) @@ -693,11 +692,7 @@ static int decide_core(struct venus_inst *inst) } done: - ret = hfi_session_set_property(inst, ptype, &cu); - if (ret) - return ret; - - return ret; + return hfi_session_set_property(inst, ptype, &cu); } static int acquire_core(struct venus_inst *inst) diff --git a/drivers/media/platform/raspberrypi/hevc_dec/Kconfig b/drivers/media/platform/raspberrypi/hevc_dec/Kconfig index ae1fd079e5c91..fc4d0baf5c0f8 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/Kconfig +++ b/drivers/media/platform/raspberrypi/hevc_dec/Kconfig @@ -1,8 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 config VIDEO_RPI_HEVC_DEC - tristate "Rasperry Pi HEVC decoder" - depends on VIDEO_DEV && VIDEO_DEV + tristate "Raspberry Pi HEVC decoder" + depends on VIDEO_DEV depends on OF select MEDIA_CONTROLLER select MEDIA_CONTROLLER_REQUEST_API diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.c b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.c index 0d06bab61af23..392fd40119fcf 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.c +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.c @@ -52,6 +52,7 @@ static const struct v4l2_ctrl_config hevc_d_ctrls[] = { }, { .name = "Slice param array", .id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, + .ops = &hevc_d_hevc_slice_params_ctrl_ops, .type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS, .flags = V4L2_CTRL_FLAG_DYNAMIC_ARRAY, .dims = { 600 }, @@ -78,7 +79,6 @@ void *hevc_d_find_control_data(struct hevc_d_ctx *ctx, u32 id) static int hevc_d_init_ctrls(struct hevc_d_dev *dev, struct hevc_d_ctx *ctx) { struct v4l2_ctrl_handler *hdl = &ctx->hdl; - struct v4l2_ctrl *ctrl; unsigned int i; v4l2_ctrl_handler_init(hdl, ARRAY_SIZE(hevc_d_ctrls)); @@ -89,7 +89,7 @@ static int hevc_d_init_ctrls(struct hevc_d_dev *dev, struct hevc_d_ctx *ctx) } for (i = 0; i < ARRAY_SIZE(hevc_d_ctrls); i++) { - ctrl = v4l2_ctrl_new_custom(hdl, &hevc_d_ctrls[i], ctx); + v4l2_ctrl_new_custom(hdl, &hevc_d_ctrls[i], ctx); if (hdl->error) { v4l2_err(&dev->v4l2_dev, "Failed to create new custom control id=%#x\n", @@ -154,16 +154,13 @@ static int hevc_d_release(struct file *file) struct hevc_d_ctx *ctx = container_of(file->private_data, struct hevc_d_ctx, fh); - v4l2_fh_del(&ctx->fh, file); - - v4l2_ctrl_handler_free(&ctx->hdl); - v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); - + v4l2_fh_del(&ctx->fh, file); v4l2_fh_exit(&ctx->fh); + v4l2_ctrl_handler_free(&ctx->hdl); mutex_destroy(&ctx->ctx_mutex); - kfree(ctx); + return 0; } @@ -221,14 +218,14 @@ static int hevc_d_probe(struct platform_device *pdev) return ret; } - mutex_init(&dev->dev_mutex); - ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); if (ret) { dev_err_probe(&pdev->dev, ret, "Failed to register V4L2 device\n"); - return ret; + goto err_hw; } + mutex_init(&dev->dev_mutex); + vfd = &dev->vfd; vfd->lock = &dev->dev_mutex; vfd->v4l2_dev = &dev->v4l2_dev; @@ -293,9 +290,13 @@ static int hevc_d_probe(struct platform_device *pdev) err_video: video_unregister_device(&dev->vfd); err_m2m: + media_device_cleanup(&dev->mdev); v4l2_m2m_release(dev->m2m_dev); err_v4l2: + mutex_destroy(&dev->dev_mutex); v4l2_device_unregister(&dev->v4l2_dev); +err_hw: + hevc_d_hw_remove(dev); return ret; } @@ -310,6 +311,7 @@ static void hevc_d_remove(struct platform_device *pdev) v4l2_m2m_release(dev->m2m_dev); video_unregister_device(&dev->vfd); + mutex_destroy(&dev->dev_mutex); v4l2_device_unregister(&dev->v4l2_dev); hevc_d_hw_remove(dev); diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.h b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.h index 367c9624c63d3..6d40d91229a6b 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.h +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d.h @@ -133,6 +133,9 @@ struct hevc_d_ctx { unsigned int p2idx; struct hevc_d_hwbuf pu_bufs[HEVC_D_P2BUF_COUNT]; struct hevc_d_hwbuf coeff_bufs[HEVC_D_P2BUF_COUNT]; + /* Last sizes we expanded to */ + size_t pu_size_max; + size_t coeff_size_max; /* Aux structures only used in setup so no locking needed */ struct hevc_d_q_aux *aux_free; @@ -194,7 +197,6 @@ extern int hevc_d_v4l2_debug; #define hevc_d_dbg(level, dev, fmt, arg...)\ v4l2_dbg((level), hevc_d_v4l2_debug, (dev), fmt, ## arg) -struct v4l2_ctrl *hevc_d_find_ctrl(struct hevc_d_ctx *ctx, u32 id); void *hevc_d_find_control_data(struct hevc_d_ctx *ctx, u32 id); #endif diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.c b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.c index 1d4a7fbd9d4aa..83ce62afd158b 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.c +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.c @@ -11,7 +11,6 @@ * Copyright (C) 2018 Bootlin */ -//#include #include #include @@ -132,7 +131,7 @@ static size_t next_size(const size_t x) #define PROB_BACKUP ((20 << 12) + (20 << 6) + (0 << 0)) #define PROB_RELOAD ((20 << 12) + (20 << 0) + (0 << 6)) -#define HEVC_MAX_REFS V4L2_HEVC_DPB_ENTRIES_NUM_MAX +#define NUM_REF_IDX_ACTIVE_MAX 15 struct hevc_d_q_aux { unsigned int refcount; @@ -181,7 +180,7 @@ struct hevc_d_dec_env { unsigned int entry_ctb_y; unsigned int entry_tile_x; unsigned int entry_tile_y; - unsigned int entry_qp; + int entry_qp; u32 entry_slice; u32 rpi_config2; @@ -204,7 +203,7 @@ struct hevc_d_dec_env { u32 pu_stride; u32 coeff_stride; -#define SLICE_MSGS_MAX (2 * HEVC_MAX_REFS * 8 + 3) +#define SLICE_MSGS_MAX (2 * NUM_REF_IDX_ACTIVE_MAX * 8 + 3) u16 slice_msgs[SLICE_MSGS_MAX]; u8 scaling_factors[NUM_SCALING_FACTORS]; @@ -229,13 +228,13 @@ struct hevc_d_dec_state { unsigned int tile_width; /* Width in tiles */ unsigned int tile_height; /* Height in tiles */ - int *col_bd; - int *row_bd; + unsigned int *col_bd; + unsigned int *row_bd; int *ctb_addr_rs_to_ts; int *ctb_addr_ts_to_rs; /* Aux storage for DPB */ - struct hevc_d_q_aux *ref_aux[HEVC_MAX_REFS]; + struct hevc_d_q_aux *ref_aux[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; struct hevc_d_q_aux *frame_aux; /* Slice vars */ @@ -250,7 +249,7 @@ struct hevc_d_dec_state { const struct v4l2_ctrl_hevc_slice_params *sh; const struct v4l2_ctrl_hevc_decode_params *dec; unsigned int nb_refs[2]; - unsigned int slice_qp; + int slice_qp; unsigned int max_num_merge_cand; /* 0 if I-slice */ bool dependent_slice_segment_flag; u32 data_len; @@ -297,7 +296,7 @@ static void p1_apb_write(struct hevc_d_dec_env *const de, const u16 addr, de->cmd_len++; } -static int ctb_to_tile(unsigned int ctb, unsigned int *bd, int num) +static int ctb_to_tile(unsigned int ctb, unsigned int *bd) { int i; @@ -310,13 +309,13 @@ static int ctb_to_tile(unsigned int ctb, unsigned int *bd, int num) static unsigned int ctb_to_tile_x(const struct hevc_d_dec_state *const s, const unsigned int ctb_x) { - return ctb_to_tile(ctb_x, s->col_bd, s->tile_width); + return ctb_to_tile(ctb_x, s->col_bd); } static unsigned int ctb_to_tile_y(const struct hevc_d_dec_state *const s, const unsigned int ctb_y) { - return ctb_to_tile(ctb_y, s->row_bd, s->tile_height); + return ctb_to_tile(ctb_y, s->row_bd); } static void aux_q_free(struct hevc_d_ctx *const ctx, @@ -337,6 +336,15 @@ static struct hevc_d_q_aux *aux_q_alloc(struct hevc_d_ctx *const ctx, if (!aq) return NULL; + /* + * Calculate col mv size from capture size as that provides an + * upper bound on actual size. Size in SPS might vary but must + * be less than the capture format + */ + ctx->colmv_stride = ALIGN(ctx->dst_fmt.width, 64); + ctx->colmv_picsize = ctx->colmv_stride * + (ALIGN(ctx->dst_fmt.height, 64) >> 4); + if (hwbuf_alloc(dev, &aq->col, ctx->colmv_picsize, DMA_ATTR_FORCE_CONTIGUOUS | DMA_ATTR_NO_KERNEL_MAPPING)) goto fail; @@ -493,11 +501,11 @@ static void write_prob(struct hevc_d_dec_env *const de, const struct hevc_d_dec_state *const s) { const unsigned int init_type = - ((s->sh->flags & V4L2_HEVC_SLICE_PARAMS_FLAG_CABAC_INIT) != 0 && - s->sh->slice_type != HEVC_SLICE_I) ? - s->sh->slice_type + 1 : - 2 - s->sh->slice_type; - const int q = clamp((int)s->slice_qp, 0, 51); + s->sh->slice_type == HEVC_SLICE_I ? 0 : + (s->sh->flags & V4L2_HEVC_SLICE_PARAMS_FLAG_CABAC_INIT) ? + (s->sh->slice_type == HEVC_SLICE_P ? 2 : 1) : + (s->sh->slice_type == HEVC_SLICE_P ? 1 : 2); + const int q = clamp(s->slice_qp, 0, 51); const u8 *p = prob_init[init_type]; u8 dst[RPI_PROB_ARRAY_SIZE]; unsigned int i; @@ -531,7 +539,7 @@ static void write_prob(struct hevc_d_dec_env *const de, p1_apb_write(de, RPI_TRANSFER, PROB_BACKUP); } -#define CMDS_WRITE_SCALING_FACTORS NUM_SCALING_FACTORS +#define CMDS_WRITE_SCALING_FACTORS (NUM_SCALING_FACTORS / 4) static void write_scaling_factors(struct hevc_d_dec_env *const de) { const u8 *p = (u8 *)de->scaling_factors; @@ -548,7 +556,7 @@ static inline __u32 dma_to_axi_addr(dma_addr_t a) } #define CMDS_WRITE_BITSTREAM 4 -static int write_bitstream(struct hevc_d_dec_env *const de, +static void write_bitstream(struct hevc_d_dec_env *const de, const struct hevc_d_dec_state *const s) { /* V4L2 always has emulation prevention bytes in the stream */ @@ -561,7 +569,6 @@ static int write_bitstream(struct hevc_d_dec_env *const de, p1_apb_write(de, RPI_BFNUM, len); p1_apb_write(de, RPI_BFCONTROL, offset + (1 << 7)); /* Stop */ p1_apb_write(de, RPI_BFCONTROL, offset + (rpi_use_emu << 6)); - return 0; } /* @@ -700,6 +707,32 @@ static int has_backward(const struct v4l2_hevc_dpb_entry *const dpb, return 1; } +static void pre_slice_weights(struct hevc_d_dec_env *const de, + const struct v4l2_hevc_pred_weight_table *const w, + const __s8 delta_luma_weight[V4L2_HEVC_DPB_ENTRIES_NUM_MAX], + const __s8 luma_offset[V4L2_HEVC_DPB_ENTRIES_NUM_MAX], + const __s8 delta_chroma_weight[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2], + const __s8 chroma_offset[V4L2_HEVC_DPB_ENTRIES_NUM_MAX][2], + const unsigned int idx) +{ + const int luma_log2_weight_denom = min(w->luma_log2_weight_denom, 7); + const int chroma_log2_weight_denom = clamp(luma_log2_weight_denom + + w->delta_chroma_log2_weight_denom, + 0, 7); + const int luma_weight_denom = (1 << luma_log2_weight_denom); + const int chroma_weight_denom = (1 << chroma_log2_weight_denom); + + msg_slice(de, luma_log2_weight_denom | + (((delta_luma_weight[idx] + luma_weight_denom) & 0x1ff) << 3)); + msg_slice(de, luma_offset[idx] & 0xff); + msg_slice(de, chroma_log2_weight_denom | + (((delta_chroma_weight[idx][0] + chroma_weight_denom) & 0x1ff) << 3)); + msg_slice(de, chroma_offset[idx][0] & 0xff); + msg_slice(de, chroma_log2_weight_denom | + (((delta_chroma_weight[idx][1] + chroma_weight_denom) & 0x1ff) << 3)); + msg_slice(de, chroma_offset[idx][1] & 0xff); +} + static void pre_slice_decode(struct hevc_d_dec_env *const de, const struct hevc_d_dec_state *const s) { @@ -738,14 +771,14 @@ static void pre_slice_decode(struct hevc_d_dec_env *const de, cmd_slice |= no_backward_pred_flag << 10; msg_slice(de, cmd_slice); - if (s->slice_temporal_mvp) { - const u8 *const rpl = collocated_from_l0_flag ? - sh->ref_idx_l0 : sh->ref_idx_l1; - if (sh->collocated_ref_idx >= dec->num_active_dpb_entries) - de->dpbno_col = rpl[0]; - else - de->dpbno_col = rpl[sh->collocated_ref_idx]; - } + if (s->slice_temporal_mvp) + de->dpbno_col = collocated_from_l0_flag ? + (sh->collocated_ref_idx < s->nb_refs[L0] ? + sh->ref_idx_l0[sh->collocated_ref_idx] : + sh->ref_idx_l0[0]) : + (sh->collocated_ref_idx < s->nb_refs[L1] ? + sh->ref_idx_l1[sh->collocated_ref_idx] : + sh->ref_idx_l1[0]); /* Write reference picture descriptions */ weighted_pred_flag = @@ -764,38 +797,14 @@ static void pre_slice_decode(struct hevc_d_dec_env *const de, (weighted_pred_flag ? (3 << 5) : 0)); msg_slice(de, dec->dpb[dpb_no].pic_order_cnt_val & 0xffff); - if (weighted_pred_flag) { - const struct v4l2_hevc_pred_weight_table - *const w = &sh->pred_weight_table; - const int luma_weight_denom = - (1 << w->luma_log2_weight_denom); - const unsigned int chroma_log2_weight_denom = - (w->luma_log2_weight_denom + - w->delta_chroma_log2_weight_denom); - const int chroma_weight_denom = - (1 << chroma_log2_weight_denom); - - msg_slice(de, - w->luma_log2_weight_denom | - (((w->delta_luma_weight_l0[idx] + - luma_weight_denom) & 0x1ff) - << 3)); - msg_slice(de, w->luma_offset_l0[idx] & 0xff); - msg_slice(de, - chroma_log2_weight_denom | - (((w->delta_chroma_weight_l0[idx][0] + - chroma_weight_denom) & 0x1ff) - << 3)); - msg_slice(de, - w->chroma_offset_l0[idx][0] & 0xff); - msg_slice(de, - chroma_log2_weight_denom | - (((w->delta_chroma_weight_l0[idx][1] + - chroma_weight_denom) & 0x1ff) - << 3)); - msg_slice(de, - w->chroma_offset_l0[idx][1] & 0xff); - } + if (weighted_pred_flag) + pre_slice_weights(de, + &sh->pred_weight_table, + sh->pred_weight_table.delta_luma_weight_l0, + sh->pred_weight_table.luma_offset_l0, + sh->pred_weight_table.delta_chroma_weight_l0, + sh->pred_weight_table.chroma_offset_l0, + idx); } for (idx = 0; idx < s->nb_refs[L1]; ++idx) { @@ -808,37 +817,15 @@ static void pre_slice_decode(struct hevc_d_dec_env *const de, (1 << 4) : 0) | (weighted_pred_flag ? (3 << 5) : 0)); msg_slice(de, dec->dpb[dpb_no].pic_order_cnt_val & 0xffff); - if (weighted_pred_flag) { - const struct v4l2_hevc_pred_weight_table - *const w = &sh->pred_weight_table; - const int luma_weight_denom = - (1 << w->luma_log2_weight_denom); - const unsigned int chroma_log2_weight_denom = - (w->luma_log2_weight_denom + - w->delta_chroma_log2_weight_denom); - const int chroma_weight_denom = - (1 << chroma_log2_weight_denom); - - msg_slice(de, - w->luma_log2_weight_denom | - (((w->delta_luma_weight_l1[idx] + - luma_weight_denom) & 0x1ff) << 3)); - msg_slice(de, w->luma_offset_l1[idx] & 0xff); - msg_slice(de, - chroma_log2_weight_denom | - (((w->delta_chroma_weight_l1[idx][0] + - chroma_weight_denom) & 0x1ff) - << 3)); - msg_slice(de, - w->chroma_offset_l1[idx][0] & 0xff); - msg_slice(de, - chroma_log2_weight_denom | - (((w->delta_chroma_weight_l1[idx][1] + - chroma_weight_denom) & 0x1ff) - << 3)); - msg_slice(de, - w->chroma_offset_l1[idx][1] & 0xff); - } + + if (weighted_pred_flag) + pre_slice_weights(de, + &sh->pred_weight_table, + sh->pred_weight_table.delta_luma_weight_l1, + sh->pred_weight_table.luma_offset_l1, + sh->pred_weight_table.delta_chroma_weight_l1, + sh->pred_weight_table.chroma_offset_l1, + idx); } } else { msg_slice(de, cmd_slice); @@ -898,7 +885,7 @@ static void new_entry_point(struct hevc_d_dec_env *const de, const unsigned int tile_y, const unsigned int ctb_col, const unsigned int ctb_row, - const unsigned int slice_qp, + const int slice_qp, const u32 slice_const) { const unsigned int endx = s->col_bd[tile_x + 1] - 1; @@ -915,10 +902,11 @@ static void new_entry_point(struct hevc_d_dec_env *const de, write_slice(de, s, slice_const, endx, endy); if (reset_qp_y) { - unsigned int sps_qp_bd_offset = + int sps_qp_bd_offset = 6 * s->sps.bit_depth_luma_minus8; - p1_apb_write(de, RPI_QP, sps_qp_bd_offset + slice_qp); + p1_apb_write(de, RPI_QP, clamp(sps_qp_bd_offset + slice_qp, + 0, sps_qp_bd_offset + 51)); } p1_apb_write(de, RPI_MODE, @@ -1035,9 +1023,7 @@ static int wpp_decode_slice(struct hevc_d_dec_env *const de, if (rv) return rv; - rv = write_bitstream(de, s); - if (rv) - return rv; + write_bitstream(de, s); if (!s->start_ts || indep || s->ctb_width == 1) write_prob(de, s); @@ -1152,9 +1138,7 @@ static int decode_slice(struct hevc_d_dec_env *const de, return rv; pre_slice_decode(de, s); - rv = write_bitstream(de, s); - if (rv) - return rv; + write_bitstream(de, s); reset_qp_y = !s->start_ts || !s->dependent_slice_segment_flag || @@ -1337,12 +1321,15 @@ static void fill_rs_to_ts(struct hevc_d_dec_state *const s) static int updated_ps(struct hevc_d_dec_state *const s) { unsigned int i; + int rv = -ENOMEM; free_ps_info(s); /* Inferred parameters */ s->log2_ctb_size = s->sps.log2_min_luma_coding_block_size_minus3 + 3 + s->sps.log2_diff_max_min_luma_coding_block_size; + if (s->log2_ctb_size < s->pps.diff_cu_qp_delta_depth) + goto fail_inval; s->ctb_width = (s->sps.pic_width_in_luma_samples + (1 << s->log2_ctb_size) - 1) >> @@ -1380,34 +1367,35 @@ static int updated_ps(struct hevc_d_dec_state *const s) if (!s->row_bd) goto fail; + /* Can't check col widths/row heights in PPS validation so do here */ s->col_bd[0] = 0; - for (i = 1; i < s->tile_width; i++) + for (i = 1; i < s->tile_width; i++) { s->col_bd[i] = s->col_bd[i - 1] + s->pps.column_width_minus1[i - 1] + 1; + if (s->col_bd[i] >= s->ctb_width) + goto fail_inval; + } s->col_bd[s->tile_width] = s->ctb_width; s->row_bd[0] = 0; - for (i = 1; i < s->tile_height; i++) + for (i = 1; i < s->tile_height; i++) { s->row_bd[i] = s->row_bd[i - 1] + s->pps.row_height_minus1[i - 1] + 1; + if (s->row_bd[i] >= s->ctb_height) + goto fail_inval; + } s->row_bd[s->tile_height] = s->ctb_height; fill_rs_to_ts(s); return 0; +fail_inval: + rv = -EINVAL; fail: free_ps_info(s); /* Set invalid to force reload */ s->sps.pic_width_in_luma_samples = 0; - return -ENOMEM; -} - -static void setup_colmv(struct hevc_d_ctx *const ctx, struct hevc_d_run *run, - struct hevc_d_dec_state *const s) -{ - ctx->colmv_stride = ALIGN(s->sps.pic_width_in_luma_samples, 64); - ctx->colmv_picsize = ctx->colmv_stride * - (ALIGN(s->sps.pic_height_in_luma_samples, 64) >> 4); + return rv; } static struct hevc_d_dec_env *dec_env_new(struct hevc_d_ctx *const ctx) @@ -1462,7 +1450,7 @@ static int dec_env_init(struct hevc_d_ctx *const ctx) { unsigned int i; - ctx->dec_pool = kzalloc(sizeof(*ctx->dec_pool) * HEVC_D_DEC_ENV_COUNT, + ctx->dec_pool = kcalloc(HEVC_D_DEC_ENV_COUNT, sizeof(*ctx->dec_pool), GFP_KERNEL); if (!ctx->dec_pool) return -ENOMEM; @@ -1518,7 +1506,7 @@ static u32 mk_config2(const struct hevc_d_dec_state *const s) c |= BIT(14); if (s->mk_aux) c |= BIT(15); /* Write motion vectors to external memory */ - c |= (pps->log2_parallel_merge_level_minus2 + 2) << 16; + c |= min(s->log2_ctb_size, pps->log2_parallel_merge_level_minus2 + 2) << 16; if (s->slice_temporal_mvp) c |= BIT(19); if (sps->flags & V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED) @@ -1549,7 +1537,7 @@ static int hevc_d_h265_setup(struct hevc_d_ctx *ctx, struct hevc_d_run *run) struct hevc_d_q_aux *dpb_q_aux[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; struct hevc_d_dec_state *const s = ctx->state; struct vb2_queue *vq; - struct hevc_d_dec_env *de = ctx->dec0; + struct hevc_d_dec_env *de; unsigned int prev_rs; unsigned int i; int rv; @@ -1715,20 +1703,8 @@ static int hevc_d_h265_setup(struct hevc_d_ctx *ctx, struct hevc_d_run *run) s->sps.pic_width_in_luma_samples; de->rpi_currpoc = sh0->slice_pic_order_cnt; - if (s->sps.flags & - V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED) { - setup_colmv(ctx, run, s); - } - s->slice_idx = 0; - if (sh0->slice_segment_addr != 0) { - v4l2_warn(&dev->v4l2_dev, - "New frame but segment_addr=%d\n", - sh0->slice_segment_addr); - goto fail; - } - /* Either map src buffer or use directly */ s->src_addr = 0; @@ -1746,21 +1722,32 @@ static int hevc_d_h265_setup(struct hevc_d_ctx *ctx, struct hevc_d_run *run) const bool last_slice = i + 1 == run->h265.slice_ents; unsigned int bit_size = old_bits ? sh->bit_size - 8 * sh->data_byte_offset : sh->bit_size; - const u32 byte_size = DIV_ROUND_UP(bit_size, 8); + const u32 byte_size = DIV_ROUND_UP_POW2(bit_size, 8); unsigned int j; s->sh = sh; - if (old_bits && sh->bit_size <= 8 * sh->data_byte_offset) { + /* + * slice_segment_addr indexes ctb_addr_rs_to_ts so must be + * checked before use. + */ + if (sh->slice_segment_addr >= s->ctb_size) { + v4l2_warn(&dev->v4l2_dev, + "Slice %u: segment addr %u >= pic size in CTBs %u\n", + i, sh->slice_segment_addr, s->ctb_size); + goto fail; + } + + if (old_bits && sh->bit_size <= 8 * (u64)sh->data_byte_offset) { v4l2_warn(&dev->v4l2_dev, - "data_byte_offset %d * 8 >= bits %d\n", + "data_byte_offset %u * 8 >= bits %u\n", sh->data_byte_offset, sh->bit_size); goto fail; } - if (sh->data_byte_offset + byte_size > run->src->planes[0].bytesused) { + if ((u64)sh->data_byte_offset + byte_size > run->src->planes[0].bytesused) { v4l2_warn(&dev->v4l2_dev, - "data_byte_offset %d + bits %d (= %d bytes) > bytesused %d\n", + "data_byte_offset %u + bits %u (= %u bytes) > bytesused %u\n", sh->data_byte_offset, bit_size, byte_size, run->src->planes[0].bytesused); goto fail; @@ -2097,24 +2084,25 @@ static void phase1_thread(struct hevc_d_dev *const dev, void *v) struct hevc_d_hwbuf *const coeff_hwbuf = ctx->coeff_bufs + ctx->p2idx; if (de->p1_status & STATUS_PU_EXHAUSTED) { - if (hwbuf_realloc_new(dev, pu_hwbuf, next_size(pu_hwbuf->size))) { - v4l2_err(&dev->v4l2_dev, - "%s: PU realloc (%zx) failed\n", + if (hwbuf_realloc_new(dev, pu_hwbuf, + max(next_size(pu_hwbuf->size), ctx->pu_size_max))) { + v4l2_err(&dev->v4l2_dev, "%s: PU realloc (%zx) failed\n", __func__, pu_hwbuf->size); goto fail; } + ctx->pu_size_max = pu_hwbuf->size; hevc_d_dbg(1, &dev->v4l2_dev, "%s: PU realloc (%zx) OK\n", __func__, pu_hwbuf->size); } if (de->p1_status & STATUS_COEFF_EXHAUSTED) { if (hwbuf_realloc_new(dev, coeff_hwbuf, - next_size(coeff_hwbuf->size))) { - v4l2_err(&dev->v4l2_dev, - "%s: Coeff realloc (%zx) failed\n", + max(next_size(coeff_hwbuf->size), ctx->coeff_size_max))) { + v4l2_err(&dev->v4l2_dev, "%s: Coeff realloc (%zx) failed\n", __func__, coeff_hwbuf->size); goto fail; } + ctx->coeff_size_max = coeff_hwbuf->size; hevc_d_dbg(1, &dev->v4l2_dev, "%s: Coeff realloc (%zx) OK\n", __func__, coeff_hwbuf->size); } @@ -2218,7 +2206,7 @@ static void dec_state_delete(struct hevc_d_ctx *const ctx) free_ps_info(s); - for (i = 0; i != HEVC_MAX_REFS; ++i) + for (i = 0; i != V4L2_HEVC_DPB_ENTRIES_NUM_MAX; ++i) aux_q_release(ctx, &s->ref_aux[i]); aux_q_release(ctx, &s->frame_aux); @@ -2283,6 +2271,8 @@ static void h265_ctx_uninit(struct hevc_d_dev *const dev, struct hevc_d_ctx *ctx hwbuf_free(dev, ctx->pu_bufs + i); for (i = 0; i != ARRAY_SIZE(ctx->coeff_bufs); ++i) hwbuf_free(dev, ctx->coeff_bufs + i); + ctx->pu_size_max = 0; + ctx->coeff_size_max = 0; } void hevc_d_h265_stop(struct hevc_d_ctx *ctx) @@ -2302,8 +2292,11 @@ int hevc_d_h265_start(struct hevc_d_ctx *ctx) size_t pu_size; size_t coeff_size; + /* Reset values that must be zeroed on a second run */ ctx->fatal_err = 0; ctx->dec0 = NULL; + memset(ctx->slots, 0, sizeof(ctx->slots)); + ctx->state = kzalloc(sizeof(*ctx->state), GFP_KERNEL); if (!ctx->state) { v4l2_err(&dev->v4l2_dev, "Failed to allocate decode state\n"); @@ -2375,6 +2368,13 @@ static int try_ctrl_sps(struct v4l2_ctrl *ctrl) const struct v4l2_ctrl_hevc_sps *const sps = ctrl->p_new.p_hevc_sps; struct hevc_d_ctx *const ctx = ctrl->priv; struct hevc_d_dev *const dev = ctx->dev; + const unsigned int ctb_log2_size_y = + sps->log2_min_luma_coding_block_size_minus3 + 3 + + sps->log2_diff_max_min_luma_coding_block_size; + const unsigned int min_tb_log2_size_y = + sps->log2_min_luma_transform_block_size_minus2 + 2; + const unsigned int max_tb_log2_size_y = min_tb_log2_size_y + + sps->log2_diff_max_min_luma_transform_block_size; if (sps->chroma_format_idc != 1) { v4l2_warn(&dev->v4l2_dev, @@ -2410,6 +2410,31 @@ static int try_ctrl_sps(struct v4l2_ctrl *ctrl) return -EINVAL; } + /* Limits from H.265 7.4.3.2.1 */ + if (sps->log2_max_pic_order_cnt_lsb_minus4 > 12) + return -EINVAL; + if (sps->sps_max_dec_pic_buffering_minus1 > 15) + return -EINVAL; + if (sps->sps_max_num_reorder_pics > + sps->sps_max_dec_pic_buffering_minus1) + return -EINVAL; + if (ctb_log2_size_y > 6) + return -EINVAL; + if (max_tb_log2_size_y > 5) + return -EINVAL; + if (max_tb_log2_size_y > ctb_log2_size_y) + return -EINVAL; + if (sps->max_transform_hierarchy_depth_inter > + (ctb_log2_size_y - min_tb_log2_size_y)) + return -EINVAL; + if (sps->max_transform_hierarchy_depth_intra > + (ctb_log2_size_y - min_tb_log2_size_y)) + return -EINVAL; + if (sps->num_short_term_ref_pic_sets > 64) + return -EINVAL; + if (sps->num_long_term_ref_pics_sps > 32) + return -EINVAL; + return 0; } @@ -2425,14 +2450,25 @@ static int try_ctrl_pps(struct v4l2_ctrl *ctrl) if ((pps->flags & V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED) && - (pps->flags & - V4L2_HEVC_PPS_FLAG_TILES_ENABLED) && + (pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) && (pps->num_tile_columns_minus1 || pps->num_tile_rows_minus1)) { v4l2_warn(&dev->v4l2_dev, "WPP + Tiles not supported\n"); return -EINVAL; } + if ((pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED) && + (pps->num_tile_columns_minus1 > + ARRAY_SIZE(pps->column_width_minus1) || + pps->num_tile_rows_minus1 > + ARRAY_SIZE(pps->row_height_minus1))) { + v4l2_warn(&dev->v4l2_dev, "Tiles cols/rows too big\n"); + return -EINVAL; + } + + if (pps->log2_parallel_merge_level_minus2 > 4) + return -EINVAL; + return 0; } @@ -2440,6 +2476,66 @@ const struct v4l2_ctrl_ops hevc_d_hevc_pps_ctrl_ops = { .try_ctrl = try_ctrl_pps, }; +/* Check the DPB indices that decode will use are all in range */ +static bool ref_idx_valid(const __u8 *const ref_idx, const unsigned int n_minus_1) +{ + unsigned int i; + + if (n_minus_1 > NUM_REF_IDX_ACTIVE_MAX - 1) + return false; + for (i = 0; i <= n_minus_1; ++i) + if (ref_idx[i] >= V4L2_HEVC_DPB_ENTRIES_NUM_MAX) + return false; + + return true; +} + +static int try_ctrl_slice_params(struct v4l2_ctrl *ctrl) +{ + struct hevc_d_ctx *const ctx = ctrl->priv; + struct hevc_d_dev *const dev = ctx->dev; + const struct v4l2_ctrl_hevc_slice_params *sh = ctrl->p_new.p_hevc_slice_params; + unsigned int i; + + /* + * Cannot know the size from the slice header alone so can only test + * slice_segment_address for slice 0 + */ + if (sh->slice_segment_addr != 0) { + v4l2_warn(&dev->v4l2_dev, "New frame but segment_addr=%d\n", + sh->slice_segment_addr); + return -EINVAL; + } + + for (i = 0; i != ctrl->new_elems; ++i, ++sh) { + if (sh->slice_type != HEVC_SLICE_B && + sh->slice_type != HEVC_SLICE_P && + sh->slice_type != HEVC_SLICE_I) { + v4l2_warn(&dev->v4l2_dev, + "Slice %u: bad slice type %u\n", + i, sh->slice_type); + return -EINVAL; + } + + if (sh->slice_type != HEVC_SLICE_I) { + if (!ref_idx_valid(sh->ref_idx_l0, sh->num_ref_idx_l0_active_minus1)) + return -EINVAL; + if (sh->five_minus_max_num_merge_cand > 4) + return -EINVAL; + } + if (sh->slice_type == HEVC_SLICE_B) { + if (!ref_idx_valid(sh->ref_idx_l1, sh->num_ref_idx_l1_active_minus1)) + return -EINVAL; + } + } + + return 0; +} + +const struct v4l2_ctrl_ops hevc_d_hevc_slice_params_ctrl_ops = { + .try_ctrl = try_ctrl_slice_params, +}; + void hevc_d_device_run(void *priv) { struct hevc_d_ctx *const ctx = priv; @@ -2468,12 +2564,7 @@ void hevc_d_device_run(void *priv) ctrl = v4l2_ctrl_find(ctx->fh.ctrl_handler, V4L2_CID_STATELESS_HEVC_SLICE_PARAMS); - if (!ctrl || !ctrl->elems) { - v4l2_err(&dev->v4l2_dev, "%s: Missing slice params\n", - __func__); - goto fail; - } - run.h265.slice_ents = ctrl->elems; + run.h265.slice_ents = ctrl->elems; /* Framework ensures >= 1 */ run.h265.slice_params = ctrl->p_cur.p; run.h265.scaling_matrix = @@ -2493,6 +2584,7 @@ void hevc_d_device_run(void *priv) fail: /* We really shouldn't get here but tidy up what we can */ + v4l2_ctrl_request_complete(src_req, &ctx->hdl); v4l2_m2m_buf_done_and_job_finish(dev->m2m_dev, ctx->fh.m2m_ctx, VB2_BUF_STATE_ERROR); media_request_manual_complete(src_req); diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.h b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.h index 48f08d37362b1..9961917bf2040 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.h +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_h265.h @@ -12,6 +12,7 @@ extern const struct v4l2_ctrl_ops hevc_d_hevc_sps_ctrl_ops; extern const struct v4l2_ctrl_ops hevc_d_hevc_pps_ctrl_ops; +extern const struct v4l2_ctrl_ops hevc_d_hevc_slice_params_ctrl_ops; int hevc_d_h265_start(struct hevc_d_ctx *ctx); void hevc_d_h265_stop(struct hevc_d_ctx *ctx); diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.c b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.c index 9758d7310ec74..27e92ccad24df 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.c +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.c @@ -29,9 +29,9 @@ static inline struct hevc_d_ctx *hevc_d_file2ctx(struct file *file) size_t hevc_d_round_up_size(const size_t x) { /* Admit no size < 256 */ - const unsigned int n = x < 256 ? 8 : ilog2(x); + const unsigned int n = x < 256 ? 7 : ilog2(x) - 1; - return x >= (3 << n) ? 4 << n : (3 << n); + return x >= ((size_t)3 << n) ? (size_t)4 << n : ((size_t)3 << n); } static u32 bit_buf_size(unsigned int w, unsigned int h, unsigned int bits_minus8) @@ -83,6 +83,11 @@ void hevc_d_prepare_src_format(struct v4l2_pix_format_mplane *pix_fmt) /* Zero bytes per line for encoded source. */ pix_fmt->plane_fmt[0].bytesperline = 0; pix_fmt->plane_fmt[0].sizeimage = sizeimage; + + /* Zero all unused planes */ + memset(pix_fmt->plane_fmt + pix_fmt->num_planes, 0, + (ARRAY_SIZE(pix_fmt->plane_fmt) - pix_fmt->num_planes) * + sizeof(*pix_fmt->plane_fmt)); } /* Take any pix_format and make it valid */ @@ -171,6 +176,11 @@ static void hevc_d_prepare_dst_format(struct v4l2_pix_format_mplane *pix_fmt) pix_fmt->num_planes = 1; break; } + + /* Zero all unused planes */ + memset(pix_fmt->plane_fmt + pix_fmt->num_planes, 0, + (ARRAY_SIZE(pix_fmt->plane_fmt) - pix_fmt->num_planes) * + sizeof(*pix_fmt->plane_fmt)); } static int hevc_d_querycap(struct file *file, void *priv, @@ -199,61 +209,8 @@ static int hevc_d_enum_fmt_vid_out(struct file *file, void *priv, return -EINVAL; } -static int hevc_d_hevc_validate_sps(const struct v4l2_ctrl_hevc_sps * const sps) -{ - const unsigned int ctb_log2_size_y = - sps->log2_min_luma_coding_block_size_minus3 + 3 + - sps->log2_diff_max_min_luma_coding_block_size; - const unsigned int min_tb_log2_size_y = - sps->log2_min_luma_transform_block_size_minus2 + 2; - const unsigned int max_tb_log2_size_y = min_tb_log2_size_y + - sps->log2_diff_max_min_luma_transform_block_size; - - /* Local limitations */ - if (sps->pic_width_in_luma_samples < 32 || - sps->pic_width_in_luma_samples > 4096) - return 0; - if (sps->pic_height_in_luma_samples < 32 || - sps->pic_height_in_luma_samples > 4096) - return 0; - if (!(sps->bit_depth_luma_minus8 == 0 || - sps->bit_depth_luma_minus8 == 2)) - return 0; - if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) - return 0; - if (sps->chroma_format_idc != 1) - return 0; - - /* Limits from H.265 7.4.3.2.1 */ - if (sps->log2_max_pic_order_cnt_lsb_minus4 > 12) - return 0; - if (sps->sps_max_dec_pic_buffering_minus1 > 15) - return 0; - if (sps->sps_max_num_reorder_pics > - sps->sps_max_dec_pic_buffering_minus1) - return 0; - if (ctb_log2_size_y > 6) - return 0; - if (max_tb_log2_size_y > 5) - return 0; - if (max_tb_log2_size_y > ctb_log2_size_y) - return 0; - if (sps->max_transform_hierarchy_depth_inter > - (ctb_log2_size_y - min_tb_log2_size_y)) - return 0; - if (sps->max_transform_hierarchy_depth_intra > - (ctb_log2_size_y - min_tb_log2_size_y)) - return 0; - /* Check pcm stuff */ - if (sps->num_short_term_ref_pic_sets > 64) - return 0; - if (sps->num_long_term_ref_pics_sps > 32) - return 0; - return 1; -} - static u32 pixelformat_from_sps(const struct v4l2_ctrl_hevc_sps * const sps, - const int index) + const unsigned int index) { static const u32 all_formats[] = { V4L2_PIX_FMT_NV12MT_COL128, @@ -263,7 +220,7 @@ static u32 pixelformat_from_sps(const struct v4l2_ctrl_hevc_sps * const sps, }; u32 pf = 0; - if (!is_sps_set(sps) || !hevc_d_hevc_validate_sps(sps)) { + if (!is_sps_set(sps)) { /* Treat this as an error? For now return both */ if (index < ARRAY_SIZE(all_formats)) @@ -315,7 +272,7 @@ hevc_d_hevc_default_dst_fmt(struct hevc_d_ctx * const ctx) } static u32 hevc_d_hevc_get_dst_pixelformat(struct hevc_d_ctx * const ctx, - const int index) + const unsigned int index) { const struct v4l2_ctrl_hevc_sps * const sps = hevc_d_find_control_data(ctx, V4L2_CID_STATELESS_HEVC_SPS); @@ -373,7 +330,7 @@ static int hevc_d_try_fmt_vid_cap(struct file *file, void *priv, const struct v4l2_ctrl_hevc_sps * const sps = hevc_d_find_control_data(ctx, V4L2_CID_STATELESS_HEVC_SPS); u32 pixelformat; - int i; + unsigned int i; for (i = 0; (pixelformat = pixelformat_from_sps(sps, i)) != 0; i++) { if (f->fmt.pix_mp.pixelformat == pixelformat) diff --git a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.h b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.h index e7952937ed865..5a1ca7e4f06d4 100644 --- a/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.h +++ b/drivers/media/platform/raspberrypi/hevc_dec/hevc_d_video.h @@ -14,12 +14,6 @@ #ifndef _HEVC_D_VIDEO_H_ #define _HEVC_D_VIDEO_H_ -struct hevc_d_format { - u32 pixelformat; - u32 directions; - unsigned int capabilities; -}; - static inline int is_sps_set(const struct v4l2_ctrl_hevc_sps * const sps) { return sps && sps->pic_width_in_luma_samples; diff --git a/drivers/media/platform/renesas/rcar-csi2.c b/drivers/media/platform/renesas/rcar-csi2.c index d1b31ab8b8c4a..9948a3aa89476 100644 --- a/drivers/media/platform/renesas/rcar-csi2.c +++ b/drivers/media/platform/renesas/rcar-csi2.c @@ -2265,11 +2265,7 @@ static int rcsi2_init_phtw_v3u(struct rcar_csi2 *priv, return ret; } - ret = rcsi2_phtw_write_array(priv, step4, ARRAY_SIZE(step4)); - if (ret) - return ret; - - return ret; + return rcsi2_phtw_write_array(priv, step4, ARRAY_SIZE(step4)); } /* ----------------------------------------------------------------------------- diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h index a5a57369ef0eb..10e62f2646d08 100644 --- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h +++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h @@ -60,6 +60,7 @@ #define ICnMC_CSCTHR BIT(5) #define ICnMC_INF(x) ((x) << 16) #define ICnMC_VCSEL(x) ((x) << 22) +#define ICnMC_VCSEL_MASK GENMASK(23, 22) #define ICnMC_INF_MASK GENMASK(21, 16) #define ICnMS_IA BIT(2) diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c index 162e2ace69318..4beedd7d0610d 100644 --- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c +++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c @@ -262,19 +262,24 @@ static void rzg2l_cru_csi2_setup(struct rzg2l_cru_dev *cru, u8 csi_vc) { const struct rzg2l_cru_info *info = cru->info; - u32 icnmc = ICnMC_INF(ip_fmt->datatype); + u32 icnmc = rzg2l_cru_read(cru, info->image_conv) & ~(ICnMC_INF_MASK | + ICnMC_VCSEL_MASK); + icnmc |= ICnMC_INF(ip_fmt->datatype); + /* + * VC filtering goes through SVC register on G3E/V2H. + * + * FIXME: virtual channel filtering is likely broken and only VC=0 + * works. + */ if (cru->info->regs[ICnSVC]) { rzg2l_cru_write(cru, ICnSVCNUM, csi_vc); rzg2l_cru_write(cru, ICnSVC, ICnSVC_SVC0(0) | ICnSVC_SVC1(1) | ICnSVC_SVC2(2) | ICnSVC_SVC3(3)); + } else { + icnmc |= ICnMC_VCSEL(csi_vc); } - icnmc |= rzg2l_cru_read(cru, info->image_conv) & ~ICnMC_INF_MASK; - - /* Set virtual channel CSI2 */ - icnmc |= ICnMC_VCSEL(csi_vc); - rzg2l_cru_write(cru, info->image_conv, icnmc); } @@ -929,6 +934,11 @@ static void rzg2l_cru_format_align(struct rzg2l_cru_dev *cru, v4l2_fill_pixfmt(pix, pix->pixelformat, pix->width, pix->height); + if (info->has_stride) { + pix->bytesperline = ALIGN(pix->bytesperline, RZG2L_CRU_STRIDE_ALIGN); + pix->sizeimage = pix->bytesperline * pix->height; + } + dev_dbg(cru->dev, "Format %ux%u bpl: %u size: %u\n", pix->width, pix->height, pix->bytesperline, pix->sizeimage); } diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec.c b/drivers/media/platform/rockchip/rkvdec/rkvdec.c index 6e606d73ff519..e56128b77da64 100644 --- a/drivers/media/platform/rockchip/rkvdec/rkvdec.c +++ b/drivers/media/platform/rockchip/rkvdec/rkvdec.c @@ -1160,8 +1160,8 @@ static int rkvdec_probe(struct platform_device *pdev) vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return -ENXIO; + if (irq < 0) + return irq; ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, rkvdec_irq_handler, IRQF_ONESHOT, diff --git a/drivers/media/platform/samsung/s3c-camif/camif-core.c b/drivers/media/platform/samsung/s3c-camif/camif-core.c index 221e3c447f361..a3c8b0a10eb3c 100644 --- a/drivers/media/platform/samsung/s3c-camif/camif-core.c +++ b/drivers/media/platform/samsung/s3c-camif/camif-core.c @@ -302,7 +302,6 @@ static int camif_media_dev_init(struct camif_dev *camif) struct media_device *md = &camif->media_dev; struct v4l2_device *v4l2_dev = &camif->v4l2_dev; unsigned int ip_rev = camif->variant->ip_revision; - int ret; memset(md, 0, sizeof(*md)); snprintf(md->model, sizeof(md->model), "Samsung S3C%s CAMIF", @@ -317,11 +316,7 @@ static int camif_media_dev_init(struct camif_dev *camif) media_device_init(md); - ret = v4l2_device_register(camif->dev, v4l2_dev); - if (ret < 0) - return ret; - - return ret; + return v4l2_device_register(camif->dev, v4l2_dev); } static void camif_clk_put(struct camif_dev *camif) diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c index 13762861b7694..200b498127919 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmi.c +++ b/drivers/media/platform/st/stm32/stm32-dcmi.c @@ -2063,6 +2063,7 @@ static int dcmi_probe(struct platform_device *pdev) return 0; err_cleanup: + v4l2_async_nf_unregister(&dcmi->notifier); v4l2_async_nf_cleanup(&dcmi->notifier); err_media_entity_cleanup: media_entity_cleanup(&dcmi->vdev->entity); diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c index b18e273ef4a3e..e9cab99775bc9 100644 --- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c +++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c @@ -398,8 +398,10 @@ static int dcmipp_bytecap_start_streaming(struct vb2_queue *vq, */ if (!vcap->s_subdev) { pad = media_pad_remote_pad_first(&vcap->vdev.entity.pads[0]); - if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) - return -EINVAL; + if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) { + ret = -EINVAL; + goto err_buffer_done; + } vcap->s_subdev = media_entity_to_v4l2_subdev(pad->entity); vcap->s_subdev_pad_nb = pad->index; } diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c index e911c7f7acc52..4781db21c2055 100644 --- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c +++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c @@ -234,8 +234,10 @@ static int sun4i_csi_start_streaming(struct vb2_queue *vq, unsigned int count) int ret; csi_fmt = sun4i_csi_find_format(&csi->fmt.pixelformat, NULL); - if (!csi_fmt) - return -EINVAL; + if (!csi_fmt) { + ret = -EINVAL; + goto err_clear_dma_queue; + } dev_dbg(csi->dev, "Starting capture\n"); diff --git a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c index b7d278b3889f2..8e6f452f76c73 100644 --- a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c +++ b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c @@ -504,9 +504,9 @@ static void hdmirx_hpd_ctrl(struct snps_hdmirx_dev *hdmirx_dev, bool en) hdmirx_writel(hdmirx_dev, CORE_CONFIG, hdmirx_dev->hpd_trigger_level_high ? en : !en); - /* 100ms delay as per HDMI spec */ + /* 100ms delay as per HDMI spec + extra 50ms to cover internal delay */ if (!en) - msleep(100); + msleep(100 + 50); } static void hdmirx_write_edid_data(struct snps_hdmirx_dev *hdmirx_dev, diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c index 243c6196b0247..9fbef24025744 100644 --- a/drivers/media/platform/ti/davinci/vpif_capture.c +++ b/drivers/media/platform/ti/davinci/vpif_capture.c @@ -1499,7 +1499,7 @@ vpif_capture_get_pdata(struct platform_device *pdev, * video ports & endpoints data. */ if (pdev->dev.parent && pdev->dev.parent->of_node) - pdev->dev.of_node = pdev->dev.parent->of_node; + device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent); if (!IS_ENABLED(CONFIG_OF) || !pdev->dev.of_node) return pdev->dev.platform_data; diff --git a/drivers/media/platform/ti/vpe/vpe.c b/drivers/media/platform/ti/vpe/vpe.c index 6029d4e8e0bd3..e6c3aee946c16 100644 --- a/drivers/media/platform/ti/vpe/vpe.c +++ b/drivers/media/platform/ti/vpe/vpe.c @@ -2546,7 +2546,8 @@ static int vpe_probe(struct platform_device *pdev) "vpe_top"); if (!dev->res) { dev_err(&pdev->dev, "missing 'vpe_top' resources data\n"); - return -ENODEV; + ret = -ENODEV; + goto v4l2_dev_unreg; } /* diff --git a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c index e8c2e83379def..e7a7c7a42467a 100644 --- a/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c +++ b/drivers/media/platform/verisilicon/hantro_g2_hevc_dec.c @@ -5,6 +5,8 @@ * Copyright (C) 2020 Safran Passenger Innovations LLC */ +#include + #include "hantro_hw.h" #include "hantro_g2_regs.h" @@ -15,8 +17,8 @@ static void prepare_tile_info_buffer(struct hantro_ctx *ctx) const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; u16 *p = (u16 *)((u8 *)ctx->hevc_dec.tile_sizes.cpu); - unsigned int num_tile_rows = pps->num_tile_rows_minus1 + 1; - unsigned int num_tile_cols = pps->num_tile_columns_minus1 + 1; + unsigned int num_tile_rows = v4l2_hevc_pps_num_tile_rows(pps); + unsigned int num_tile_cols = v4l2_hevc_pps_num_tile_columns(pps); unsigned int pic_width_in_ctbs, pic_height_in_ctbs; unsigned int max_log2_ctb_size, ctb_size; bool tiles_enabled, uniform_spacing; diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c index fcf3bd9bcda2d..83af9fa1ce949 100644 --- a/drivers/media/platform/verisilicon/hantro_v4l2.c +++ b/drivers/media/platform/verisilicon/hantro_v4l2.c @@ -222,6 +222,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv, unsigned int num_fmts, i, j = 0; bool skip_mode_none, enum_all_formats; u32 index = f->index & ~V4L2_FMTDESC_FLAG_ENUM_ALL; + bool need_postproc = ctx->need_postproc; /* * If the V4L2_FMTDESC_FLAG_ENUM_ALL flag is set, we want to enumerate all @@ -230,6 +231,9 @@ static int vidioc_enum_fmt(struct file *file, void *priv, enum_all_formats = !!(f->index & V4L2_FMTDESC_FLAG_ENUM_ALL); f->index = index; + if (enum_all_formats) + need_postproc = HANTRO_AUTO_POSTPROC; + /* * When dealing with an encoder: * - on the capture side we want to filter out all MODE_NONE formats. @@ -242,7 +246,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv, */ skip_mode_none = capture == ctx->is_encoder; - formats = hantro_get_formats(ctx, &num_fmts, HANTRO_AUTO_POSTPROC); + formats = hantro_get_formats(ctx, &num_fmts, need_postproc); for (i = 0; i < num_fmts; i++) { bool mode_none = formats[i].codec_mode == HANTRO_MODE_NONE; fmt = &formats[i]; diff --git a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c index e4e21ad373233..00aa566a4ccdb 100644 --- a/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c +++ b/drivers/media/platform/verisilicon/rockchip_vpu981_hw_av1_dec.c @@ -431,20 +431,39 @@ static int rockchip_vpu981_av1_dec_prepare_run(struct hantro_ctx *ctx) { struct hantro_av1_dec_hw_ctx *av1_dec = &ctx->av1_dec; struct hantro_av1_dec_ctrls *ctrls = &av1_dec->ctrls; + const struct v4l2_av1_tile_info *tile_info; + struct v4l2_ctrl *tge; + u32 num_tiles; ctrls->sequence = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_SEQUENCE); if (WARN_ON(!ctrls->sequence)) return -EINVAL; - ctrls->tile_group_entry = - hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY); - if (WARN_ON(!ctrls->tile_group_entry)) + tge = v4l2_ctrl_find(&ctx->ctrl_handler, + V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY); + if (WARN_ON(!tge)) return -EINVAL; + ctrls->tile_group_entry = tge->p_cur.p; ctrls->frame = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_FRAME); if (WARN_ON(!ctrls->frame)) return -EINVAL; + /* + * rockchip_vpu981_av1_dec_set_tile_info() indexes the tile group + * entry array by tile1 * tile_cols + tile0, so it reads up to + * tile_cols * tile_rows entries, and lays out one descriptor per tile + * in the AV1_MAX_TILES tile_info buffer while programming the real + * tile geometry into the hardware. Reject a frame that claims more + * tiles than userspace submitted, or more than the hardware tile + * buffer holds, so the read stays in bounds and the programmed + * geometry matches the descriptors written. + */ + tile_info = &ctrls->frame->tile_info; + num_tiles = (u32)tile_info->tile_cols * tile_info->tile_rows; + if (num_tiles > tge->elems || num_tiles > AV1_MAX_TILES) + return -EINVAL; + ctrls->film_grain = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_AV1_FILM_GRAIN); @@ -578,16 +597,30 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) const struct v4l2_av1_tile_info *tile_info = &ctrls->frame->tile_info; const struct v4l2_ctrl_av1_tile_group_entry *group_entry = ctrls->tile_group_entry; - int context_update_y = - tile_info->context_update_tile_id / tile_info->tile_cols; - int context_update_x = - tile_info->context_update_tile_id % tile_info->tile_cols; - int context_update_tile_id = - context_update_x * tile_info->tile_rows + context_update_y; + int context_update_y = 0; + int context_update_x = 0; + int context_update_tile_id = 0; u8 *dst = av1_dec->tile_info.cpu; + u8 *dst_end = dst + av1_dec->tile_info.size; struct hantro_dev *vpu = ctx->dev; int tile0, tile1; + /* + * tile_cols and tile_rows are bounded by the V4L2 control validation + * (V4L2_AV1_MAX_TILE_{COLS,ROWS} and V4L2_AV1_MAX_TILE_COUNT). Guard + * the divisor here, and keep the descriptor writes within the + * AV1_MAX_TILES tile_info buffer below; the register values use the + * unmodified tile geometry. + */ + if (tile_info->tile_cols) { + context_update_y = + tile_info->context_update_tile_id / tile_info->tile_cols; + context_update_x = + tile_info->context_update_tile_id % tile_info->tile_cols; + context_update_tile_id = + context_update_x * tile_info->tile_rows + context_update_y; + } + memset(dst, 0, av1_dec->tile_info.size); for (tile0 = 0; tile0 < tile_info->tile_cols; tile0++) { @@ -598,6 +631,10 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) tile_info->height_in_sbs_minus_1[tile1] + 1; u32 x0 = tile_info->width_in_sbs_minus_1[tile0] + 1; + /* Stop once the tile_info descriptor buffer is full. */ + if (dst + 16 > dst_end) + break; + /* tile size in SB units (width,height) */ *dst++ = x0; *dst++ = 0; @@ -622,6 +659,8 @@ static void rockchip_vpu981_av1_dec_set_tile_info(struct hantro_ctx *ctx) *dst++ = (end >> 16) & 255; *dst++ = (end >> 24) & 255; } + if (dst + 16 > dst_end) + break; } hantro_reg_write(vpu, &av1_multicore_expect_context_update, !!(context_update_x == 0)); diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c index 9980346cb5ea3..bfe89782dce43 100644 --- a/drivers/media/radio/radio-si476x.c +++ b/drivers/media/radio/radio-si476x.c @@ -1493,6 +1493,7 @@ static int si476x_radio_probe(struct platform_device *pdev) return 0; exit: v4l2_ctrl_handler_free(radio->videodev.ctrl_handler); + v4l2_device_unregister(&radio->v4l2dev); return rval; } diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c index 48534bb52e4d0..4f974b88628f0 100644 --- a/drivers/media/rc/imon.c +++ b/drivers/media/rc/imon.c @@ -290,6 +290,10 @@ static const struct imon_usb_dev_descr imon_OEM_VFD = { { 0x000100000000ffeell, KEY_VOLUMEUP }, { 0x010000000000ffeell, KEY_VOLUMEDOWN }, { 0x000000000100ffeell, KEY_MUTE }, + /* iMON VFD HID OEM v1.2 */ + { 0x000000000a00ffeell, KEY_VOLUMEUP }, + { 0x000000000b00ffeell, KEY_VOLUMEDOWN }, + { 0x000000000c00ffeell, KEY_MUTE }, /* 0xffdc iMON MCE VFD */ { 0x00010000ffffffeell, KEY_VOLUMEUP }, { 0x01000000ffffffeell, KEY_VOLUMEDOWN }, diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c index a4c94fdf767ca..bee5cd15796fb 100644 --- a/drivers/media/rc/mceusb.c +++ b/drivers/media/rc/mceusb.c @@ -397,6 +397,8 @@ static const struct usb_device_id mceusb_dev_table[] = { { USB_DEVICE(VENDOR_COMPRO, 0x3082) }, /* Northstar Systems, Inc. eHome Infrared Transceiver */ { USB_DEVICE(VENDOR_NORTHSTAR, 0xe004) }, + /* Northstar Systems, Inc. eHome Infrared Transceiver - variant */ + { USB_DEVICE(VENDOR_NORTHSTAR, 0xe033) }, /* TiVo PC IR Receiver */ { USB_DEVICE(VENDOR_TIVO, 0x2000), .driver_info = TIVO_KIT }, diff --git a/drivers/media/rc/sunxi-cir.c b/drivers/media/rc/sunxi-cir.c index cb4c56bf0752a..28e840a7e5b8c 100644 --- a/drivers/media/rc/sunxi-cir.c +++ b/drivers/media/rc/sunxi-cir.c @@ -344,22 +344,25 @@ static int sunxi_ir_probe(struct platform_device *pdev) ir->irq = platform_get_irq(pdev, 0); if (ir->irq < 0) { ret = ir->irq; - goto exit_free_dev; + goto exit_unregister_dev; } ret = devm_request_irq(dev, ir->irq, sunxi_ir_irq, 0, SUNXI_IR_DEV, ir); if (ret) { dev_err(dev, "failed request irq\n"); - goto exit_free_dev; + goto exit_unregister_dev; } ret = sunxi_ir_hw_init(dev); if (ret) - goto exit_free_dev; + goto exit_unregister_dev; dev_info(dev, "initialized sunXi IR driver\n"); return 0; +exit_unregister_dev: + rc_unregister_device(ir->rc); + exit_free_dev: rc_free_device(ir->rc); diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c index a9becd8578fef..12bf7a0890343 100644 --- a/drivers/media/test-drivers/vicodec/vicodec-core.c +++ b/drivers/media/test-drivers/vicodec/vicodec-core.c @@ -63,11 +63,11 @@ struct pixfmt_info { }; static const struct v4l2_fwht_pixfmt_info pixfmt_fwht = { - V4L2_PIX_FMT_FWHT, 0, 3, 1, 1, 1, 1, 1, 0, 1 + V4L2_PIX_FMT_FWHT, 0, 4, 1, 1, 1, 1, 1, 0, 1 }; static const struct v4l2_fwht_pixfmt_info pixfmt_stateless_fwht = { - V4L2_PIX_FMT_FWHT_STATELESS, 0, 3, 1, 1, 1, 1, 1, 0, 1 + V4L2_PIX_FMT_FWHT_STATELESS, 0, 4, 1, 1, 1, 1, 1, 0, 1 }; static void vicodec_dev_release(struct device *dev) diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c index 52b2abe16dcf5..8a8e097d2ac96 100644 --- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c +++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c @@ -594,8 +594,10 @@ static int __init vidtv_bridge_init(void) int ret; ret = platform_device_register(&vidtv_bridge_dev); - if (ret) + if (ret) { + platform_device_put(&vidtv_bridge_dev); return ret; + } ret = platform_driver_register(&vidtv_bridge_driver); if (ret) diff --git a/drivers/media/test-drivers/vimc/vimc-capture.c b/drivers/media/test-drivers/vimc/vimc-capture.c index 7f6124025fc9d..cd28237a348c6 100644 --- a/drivers/media/test-drivers/vimc/vimc-capture.c +++ b/drivers/media/test-drivers/vimc/vimc-capture.c @@ -175,8 +175,8 @@ static int vimc_capture_enum_framesizes(struct file *file, void *priv, if (fsize->index) return -EINVAL; - /* Only accept code in the pix map table */ - vpix = vimc_pix_map_by_code(fsize->pixel_format); + /* Only accept pixel_format in the pix map table */ + vpix = vimc_pix_map_by_pixelformat(fsize->pixel_format); if (!vpix) return -EINVAL; diff --git a/drivers/media/test-drivers/vimc/vimc-core.c b/drivers/media/test-drivers/vimc/vimc-core.c index f632c77e52f5d..bc0474d8059ae 100644 --- a/drivers/media/test-drivers/vimc/vimc-core.c +++ b/drivers/media/test-drivers/vimc/vimc-core.c @@ -422,6 +422,7 @@ static int __init vimc_init(void) if (ret) { dev_err(&vimc_pdev.dev, "platform device registration failed (err=%d)\n", ret); + platform_device_put(&vimc_pdev); return ret; } diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c index 86506be36acb0..f44930502bdaf 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.c +++ b/drivers/media/test-drivers/vivid/vivid-core.c @@ -2289,8 +2289,10 @@ static int __init vivid_init(void) } } ret = platform_device_register(&vivid_pdev); - if (ret) + if (ret) { + platform_device_put(&vivid_pdev); goto free_output_strings; + } ret = platform_driver_register(&vivid_pdrv); if (ret) goto unreg_device; @@ -2311,7 +2313,7 @@ static int __init vivid_init(void) destroy_hdmi_wq: destroy_workqueue(update_hdmi_ctrls_workqueue); unreg_driver: - platform_driver_register(&vivid_pdrv); + platform_driver_unregister(&vivid_pdrv); unreg_device: platform_device_unregister(&vivid_pdev); free_output_strings: diff --git a/drivers/media/test-drivers/vivid/vivid-ctrls.c b/drivers/media/test-drivers/vivid/vivid-ctrls.c index f94c15ff84f78..a8a134b36720e 100644 --- a/drivers/media/test-drivers/vivid/vivid-ctrls.c +++ b/drivers/media/test-drivers/vivid/vivid-ctrls.c @@ -609,17 +609,24 @@ static int vivid_vid_cap_s_ctrl(struct v4l2_ctrl *ctrl) break; case VIVID_CID_REDUCED_FPS: dev->reduced_fps = ctrl->val; - vivid_update_format_cap(dev, true); + if (dev->input_type[dev->input] == HDMI) + vivid_update_reduced_fps(dev); break; case VIVID_CID_HAS_CROP_CAP: + if (vb2_is_busy(&dev->vb_vid_cap_q)) + return -EBUSY; dev->has_crop_cap = ctrl->val; vivid_update_format_cap(dev, true); break; case VIVID_CID_HAS_COMPOSE_CAP: + if (vb2_is_busy(&dev->vb_vid_cap_q)) + return -EBUSY; dev->has_compose_cap = ctrl->val; vivid_update_format_cap(dev, true); break; case VIVID_CID_HAS_SCALER_CAP: + if (vb2_is_busy(&dev->vb_vid_cap_q)) + return -EBUSY; dev->has_scaler_cap = ctrl->val; vivid_update_format_cap(dev, true); break; @@ -1116,14 +1123,20 @@ static int vivid_vid_out_s_ctrl(struct v4l2_ctrl *ctrl) switch (ctrl->id) { case VIVID_CID_HAS_CROP_OUT: + if (vb2_is_busy(&dev->vb_vid_out_q)) + return -EBUSY; dev->has_crop_out = ctrl->val; vivid_update_format_out(dev); break; case VIVID_CID_HAS_COMPOSE_OUT: + if (vb2_is_busy(&dev->vb_vid_out_q)) + return -EBUSY; dev->has_compose_out = ctrl->val; vivid_update_format_out(dev); break; case VIVID_CID_HAS_SCALER_OUT: + if (vb2_is_busy(&dev->vb_vid_out_q)) + return -EBUSY; dev->has_scaler_out = ctrl->val; vivid_update_format_out(dev); break; diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index 8b3162e820327..d1c09bdb2f3b0 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -362,6 +362,24 @@ static enum tpg_pixel_aspect vivid_get_pixel_aspect(const struct vivid_dev *dev) return TPG_PIXEL_ASPECT_SQUARE; } +void vivid_update_reduced_fps(struct vivid_dev *dev) +{ + struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt; + unsigned int size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt); + u64 pixelclock; + + if (dev->reduced_fps && can_reduce_fps(bt)) { + pixelclock = div_u64(bt->pixelclock * 1000, 1001); + bt->flags |= V4L2_DV_FL_REDUCED_FPS; + } else { + pixelclock = bt->pixelclock; + bt->flags &= ~V4L2_DV_FL_REDUCED_FPS; + } + dev->timeperframe_vid_cap = (struct v4l2_fract) { + size / 100, (u32)pixelclock / 100 + }; +} + /* * Called whenever the format has to be reset which can occur when * changing inputs, standard, timings, etc. @@ -370,8 +388,12 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) { struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt; u32 dims[V4L2_CTRL_MAX_DIMS] = {}; - unsigned size; - u64 pixelclock; + + /* + * This resets the format, so must never be called while vb2_is_busy(). + */ + if (WARN_ON(vb2_is_busy(&dev->vb_vid_cap_q))) + return; switch (dev->input_type[dev->input]) { case WEBCAM: @@ -400,17 +422,7 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls) case HDMI: dev->src_rect.width = bt->width; dev->src_rect.height = bt->height; - size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt); - if (dev->reduced_fps && can_reduce_fps(bt)) { - pixelclock = div_u64(bt->pixelclock * 1000, 1001); - bt->flags |= V4L2_DV_FL_REDUCED_FPS; - } else { - pixelclock = bt->pixelclock; - bt->flags &= ~V4L2_DV_FL_REDUCED_FPS; - } - dev->timeperframe_vid_cap = (struct v4l2_fract) { - size / 100, (u32)pixelclock / 100 - }; + vivid_update_reduced_fps(dev); if (bt->interlaced) dev->field_cap = V4L2_FIELD_ALTERNATE; else diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.h b/drivers/media/test-drivers/vivid/vivid-vid-cap.h index 38a99f7e038ea..d08a85927510f 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.h +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.h @@ -9,6 +9,7 @@ #define _VIVID_VID_CAP_H_ void vivid_update_quality(struct vivid_dev *dev); +void vivid_update_reduced_fps(struct vivid_dev *dev); void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls); void vivid_update_outputs(struct vivid_dev *dev); void vivid_update_connected_outputs(struct vivid_dev *dev); diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c b/drivers/media/test-drivers/vivid/vivid-vid-out.c index 8c037b90833e7..23e1d5a189eed 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-out.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c @@ -214,6 +214,12 @@ void vivid_update_format_out(struct vivid_dev *dev) unsigned size, p; u64 pixelclock; + /* + * This resets the format, so must never be called while vb2_is_busy(). + */ + if (WARN_ON(vb2_is_busy(&dev->vb_vid_out_q))) + return; + switch (dev->output_type[dev->output]) { case SVID: default: diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c index 68d0275f29e1b..d32dffe4a573b 100644 --- a/drivers/media/tuners/tda18250.c +++ b/drivers/media/tuners/tda18250.c @@ -440,8 +440,8 @@ static int tda18250_pll_calc(struct dvb_frontend *fe, u8 *rdiv, goto err; exp = (uval & 0x70) >> 4; - if (exp > 5) - exp = 0; + if (exp == 0 || exp > 5) + exp = 1; lopd = 1 << (exp - 1); scale = uval & 0x0f; fvco = lopd * scale * ((c->frequency / 1000) + dev->if_frequency); diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c index 08f0920cf6ca2..f63c441c9110f 100644 --- a/drivers/media/usb/airspy/airspy.c +++ b/drivers/media/usb/airspy/airspy.c @@ -464,14 +464,21 @@ static void airspy_disconnect(struct usb_interface *intf) dev_dbg(s->dev, "\n"); - mutex_lock(&s->vb_queue_lock); + /* + * vb2_video_unregister_device() releases the vb2 queue, which + * triggers airspy_stop_streaming() if streaming is active. + * stop_streaming() dereferences s->udev via airspy_ctrl_msg() and + * airspy_free_stream_bufs(), so it must run before s->udev is + * cleared. vb2_video_unregister_device() locks vb_queue_lock + * internally and stop_streaming() locks v4l2_lock, so neither may + * be held by the caller. + */ + v4l2_device_disconnect(&s->v4l2_dev); + vb2_video_unregister_device(&s->vdev); + mutex_lock(&s->v4l2_lock); - /* No need to keep the urbs around after disconnection */ s->udev = NULL; - v4l2_device_disconnect(&s->v4l2_dev); - video_unregister_device(&s->vdev); mutex_unlock(&s->v4l2_lock); - mutex_unlock(&s->vb_queue_lock); v4l2_device_put(&s->v4l2_dev); } @@ -522,11 +529,13 @@ static int airspy_start_streaming(struct vb2_queue *vq, unsigned int count) dev_dbg(s->dev, "\n"); - if (!s->udev) - return -ENODEV; - mutex_lock(&s->v4l2_lock); + if (!s->udev) { + ret = -ENODEV; + goto err_clear_bit; + } + s->sequence = 0; set_bit(POWER_ON, &s->flags); diff --git a/drivers/media/usb/as102/as102_usb_drv.c b/drivers/media/usb/as102/as102_usb_drv.c index 44565f0297cdb..01f6164002dcc 100644 --- a/drivers/media/usb/as102/as102_usb_drv.c +++ b/drivers/media/usb/as102/as102_usb_drv.c @@ -24,35 +24,33 @@ static void as102_usb_stop_stream(struct as102_dev_t *dev); static int as102_open(struct inode *inode, struct file *file); static int as102_release(struct inode *inode, struct file *file); -static const struct usb_device_id as102_usb_id_table[] = { - { USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001) }, - { USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID) }, - { USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID) }, - { USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID) }, - { USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID) }, - { } /* Terminating entry */ +struct as102_dev_info { + const char *name; + /* + * eLNA configuration: devices built on the reference design work best + * with 0xA0, while custom designs seem to require 0xC0 + */ + uint8_t elna_cfg; }; -/* Note that this table must always have the same number of entries as the - as102_usb_id_table struct */ -static const char * const as102_device_names[] = { - AS102_REFERENCE_DESIGN, - AS102_PCTV_74E, - AS102_ELGATO_EYETV_DTT_NAME, - AS102_NBOX_DVBT_DONGLE_NAME, - AS102_SKY_IT_DIGITAL_KEY_NAME, - NULL /* Terminating entry */ -}; +#define DRIVER_INFO(dev_name, dev_elna_cfg) \ + .driver_info = (kernel_ulong_t)&(const struct as102_dev_info){ \ + .name = (dev_name), \ + .elna_cfg = (dev_elna_cfg), \ + } -/* eLNA configuration: devices built on the reference design work best - with 0xA0, while custom designs seem to require 0xC0 */ -static uint8_t const as102_elna_cfg[] = { - 0xA0, - 0xC0, - 0xC0, - 0xA0, - 0xA0, - 0x00 /* Terminating entry */ +static const struct usb_device_id as102_usb_id_table[] = { + { USB_DEVICE(AS102_USB_DEVICE_VENDOR_ID, AS102_USB_DEVICE_PID_0001), + DRIVER_INFO(AS102_REFERENCE_DESIGN, 0xA0) }, + { USB_DEVICE(PCTV_74E_USB_VID, PCTV_74E_USB_PID), + DRIVER_INFO(AS102_PCTV_74E, 0xC0) }, + { USB_DEVICE(ELGATO_EYETV_DTT_USB_VID, ELGATO_EYETV_DTT_USB_PID), + DRIVER_INFO(AS102_ELGATO_EYETV_DTT_NAME, 0xC0) }, + { USB_DEVICE(NBOX_DVBT_DONGLE_USB_VID, NBOX_DVBT_DONGLE_USB_PID), + DRIVER_INFO(AS102_NBOX_DVBT_DONGLE_NAME, 0xA0) }, + { USB_DEVICE(SKY_IT_DIGITAL_KEY_USB_VID, SKY_IT_DIGITAL_KEY_USB_PID), + DRIVER_INFO(AS102_SKY_IT_DIGITAL_KEY_NAME, 0xA0) }, + { } /* Terminating entry */ }; struct usb_driver as102_usb_driver = { @@ -336,29 +334,18 @@ static int as102_usb_probe(struct usb_interface *intf, { int ret; struct as102_dev_t *as102_dev; - int i; - - /* This should never actually happen */ - if (ARRAY_SIZE(as102_usb_id_table) != - (sizeof(as102_device_names) / sizeof(const char *))) { - pr_err("Device names table invalid size"); - return -EINVAL; - } + const struct as102_dev_info *info = (const struct as102_dev_info *)id->driver_info; as102_dev = kzalloc(sizeof(struct as102_dev_t), GFP_KERNEL); if (as102_dev == NULL) return -ENOMEM; - /* Assign the user-friendly device name */ - for (i = 0; i < ARRAY_SIZE(as102_usb_id_table); i++) { - if (id == &as102_usb_id_table[i]) { - as102_dev->name = as102_device_names[i]; - as102_dev->elna_cfg = as102_elna_cfg[i]; - } - } - - if (as102_dev->name == NULL) + if (info) { + as102_dev->name = info->name; + as102_dev->elna_cfg = info->elna_cfg; + } else { as102_dev->name = "Unknown AS102 device"; + } /* set private callback functions */ as102_dev->bus_adap.ops = &as102_priv_ops; diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c index 691f073892b36..d96b9ab276b76 100644 --- a/drivers/media/usb/cx231xx/cx231xx-cards.c +++ b/drivers/media/usb/cx231xx/cx231xx-cards.c @@ -1575,7 +1575,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev, dev->video_mode.end_point_addr, dev->video_mode.num_alt); - dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->video_mode.num_alt, GFP_KERNEL); + dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32, + dev->video_mode.num_alt, GFP_KERNEL); if (dev->video_mode.alt_max_pkt_size == NULL) return -ENOMEM; @@ -1616,7 +1617,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev, dev->vbi_mode.num_alt); /* compute alternate max packet sizes for vbi */ - dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->vbi_mode.num_alt, GFP_KERNEL); + dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32, + dev->vbi_mode.num_alt, GFP_KERNEL); if (dev->vbi_mode.alt_max_pkt_size == NULL) return -ENOMEM; @@ -1658,7 +1660,9 @@ static int cx231xx_init_v4l2(struct cx231xx *dev, "sliced CC EndPoint Addr 0x%x, Alternate settings: %i\n", dev->sliced_cc_mode.end_point_addr, dev->sliced_cc_mode.num_alt); - dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->sliced_cc_mode.num_alt, GFP_KERNEL); + dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32, + dev->sliced_cc_mode.num_alt, + GFP_KERNEL); if (dev->sliced_cc_mode.alt_max_pkt_size == NULL) return -ENOMEM; @@ -1722,7 +1726,7 @@ static int cx231xx_usb_probe(struct usb_interface *interface, udev = usb_get_dev(interface_to_usbdev(interface)); /* allocate memory for our device state and initialize it */ - dev = devm_kzalloc(&udev->dev, sizeof(*dev), GFP_KERNEL); + dev = devm_kzalloc(&interface->dev, sizeof(*dev), GFP_KERNEL); if (dev == NULL) { retval = -ENOMEM; goto err_if; @@ -1852,7 +1856,9 @@ static int cx231xx_usb_probe(struct usb_interface *interface, dev->ts1_mode.end_point_addr, dev->ts1_mode.num_alt); - dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->ts1_mode.num_alt, GFP_KERNEL); + dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32, + dev->ts1_mode.num_alt, + GFP_KERNEL); if (dev->ts1_mode.alt_max_pkt_size == NULL) { retval = -ENOMEM; goto err_video_alt; diff --git a/drivers/media/usb/cx231xx/cx231xx-video.c b/drivers/media/usb/cx231xx/cx231xx-video.c index 2cd4e333bc4b0..70aa99fead278 100644 --- a/drivers/media/usb/cx231xx/cx231xx-video.c +++ b/drivers/media/usb/cx231xx/cx231xx-video.c @@ -898,7 +898,7 @@ static int vidioc_s_fmt_vid_cap(struct file *file, void *priv, if (rc) return rc; - if (vb2_is_busy(&dev->vidq)) { + if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq)) { dev_err(dev->dev, "%s: queue busy\n", __func__); return -EBUSY; } @@ -933,7 +933,7 @@ static int vidioc_s_std(struct file *file, void *priv, v4l2_std_id norm) if (dev->norm == norm) return 0; - if (vb2_is_busy(&dev->vidq)) + if (vb2_is_busy(&dev->vidq) || vb2_is_busy(&dev->vbiq)) return -EBUSY; dev->norm = norm; diff --git a/drivers/media/usb/dvb-usb-v2/mxl111sf.c b/drivers/media/usb/dvb-usb-v2/mxl111sf.c index 870ac3c8b085d..6404eb74db32f 100644 --- a/drivers/media/usb/dvb-usb-v2/mxl111sf.c +++ b/drivers/media/usb/dvb-usb-v2/mxl111sf.c @@ -987,11 +987,7 @@ static int mxl111sf_frontend_attach_atsc_mh(struct dvb_usb_adapter *adap) if (ret < 0) return ret; - ret = mxl111sf_lg2160_frontend_attach(adap, 2); - if (ret < 0) - return ret; - - return ret; + return mxl111sf_lg2160_frontend_attach(adap, 2); } static int mxl111sf_frontend_attach_mercury(struct dvb_usb_adapter *adap) @@ -1007,11 +1003,7 @@ static int mxl111sf_frontend_attach_mercury(struct dvb_usb_adapter *adap) if (ret < 0) return ret; - ret = mxl111sf_lg2161_ep6_frontend_attach(adap, 2); - if (ret < 0) - return ret; - - return ret; + return mxl111sf_lg2161_ep6_frontend_attach(adap, 2); } static int mxl111sf_frontend_attach_mercury_mh(struct dvb_usb_adapter *adap) diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c index a51cbcf429e13..b8e207443770e 100644 --- a/drivers/media/usb/em28xx/em28xx-cards.c +++ b/drivers/media/usb/em28xx/em28xx-cards.c @@ -3448,6 +3448,7 @@ static void request_module_async(struct work_struct *work) * intf. Don't register extensions twice on those devices. */ if (dev->is_audio_only) { + em28xx_init_extension(dev); #if defined(CONFIG_MODULES) && defined(MODULE) request_module("em28xx-alsa"); #endif @@ -3676,8 +3677,6 @@ static int em28xx_init_dev(struct em28xx *dev, struct usb_device *udev, retval = -ENODEV; goto err_deinit_media; } - em28xx_init_extension(dev); - return 0; } diff --git a/drivers/media/usb/em28xx/em28xx-core.c b/drivers/media/usb/em28xx/em28xx-core.c index 29a7f3f19b56e..1012bfb186460 100644 --- a/drivers/media/usb/em28xx/em28xx-core.c +++ b/drivers/media/usb/em28xx/em28xx-core.c @@ -1134,6 +1134,8 @@ void em28xx_close_extension(struct em28xx *dev) ops->fini(dev); } } + if (dev->dev_next) + list_del(&dev->dev_next->devlist); list_del(&dev->devlist); mutex_unlock(&em28xx_devlist_mutex); } diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c index 14c35995cd952..c8514682bccda 100644 --- a/drivers/media/usb/em28xx/em28xx-video.c +++ b/drivers/media/usb/em28xx/em28xx-video.c @@ -1105,8 +1105,10 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count) dev->max_pkt_size, dev->packet_multiplier, em28xx_urb_data_copy); - if (rc < 0) + if (rc < 0) { + res_free(dev, vq->type); return rc; + } /* * djh: it's not clear whether this code is still needed. I'm diff --git a/drivers/media/usb/go7007/snd-go7007.c b/drivers/media/usb/go7007/snd-go7007.c index 9a6bd87fce03c..64bf668b7ca0d 100644 --- a/drivers/media/usb/go7007/snd-go7007.c +++ b/drivers/media/usb/go7007/snd-go7007.c @@ -195,6 +195,13 @@ static const struct snd_device_ops go7007_snd_device_ops = { .dev_free = go7007_snd_free, }; +static void go7007_snd_card_free(struct snd_card *card) +{ + struct go7007 *go = card->private_data; + + v4l2_device_put(&go->v4l2_dev); +} + int go7007_snd_init(struct go7007 *go) { static int dev; @@ -245,6 +252,8 @@ int go7007_snd_init(struct go7007 *go) gosnd->substream = NULL; go->snd_context = gosnd; v4l2_device_get(&go->v4l2_dev); + gosnd->card->private_data = go; + gosnd->card->private_free = go7007_snd_card_free; ++dev; return 0; @@ -263,7 +272,6 @@ int go7007_snd_remove(struct go7007 *go) snd_card_disconnect(gosnd->card); snd_card_free_when_closed(gosnd->card); - v4l2_device_put(&go->v4l2_dev); return 0; } EXPORT_SYMBOL(go7007_snd_remove); diff --git a/drivers/media/usb/gspca/jl2005bcd.c b/drivers/media/usb/gspca/jl2005bcd.c index a408fcc3a0608..4988fbf5005e6 100644 --- a/drivers/media/usb/gspca/jl2005bcd.c +++ b/drivers/media/usb/gspca/jl2005bcd.c @@ -148,17 +148,12 @@ static int jl2005c_start_new_frame(struct gspca_dev *gspca_dev) static int jl2005c_write_reg(struct gspca_dev *gspca_dev, unsigned char reg, unsigned char value) { - int retval; u8 instruction[2]; instruction[0] = reg; instruction[1] = value; - retval = jl2005c_write2(gspca_dev, instruction); - if (retval < 0) - return retval; - - return retval; + return jl2005c_write2(gspca_dev, instruction); } static int jl2005c_get_firmware_id(struct gspca_dev *gspca_dev) diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c index 33099f39146af..235b685f85364 100644 --- a/drivers/media/usb/msi2500/msi2500.c +++ b/drivers/media/usb/msi2500/msi2500.c @@ -541,7 +541,8 @@ static int msi2500_isoc_init(struct msi2500_dev *dev) } /* Must be called with vb_queue_lock hold */ -static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev) +static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev, + enum vb2_buffer_state state) { unsigned long flags; @@ -554,7 +555,7 @@ static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev) buf = list_entry(dev->queued_bufs.next, struct msi2500_frame_buf, list); list_del(&buf->list); - vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR); + vb2_buffer_done(&buf->vb.vb2_buf, state); } spin_unlock_irqrestore(&dev->queued_bufs_lock, flags); } @@ -830,25 +831,40 @@ static int msi2500_start_streaming(struct vb2_queue *vq, unsigned int count) dev_dbg(dev->dev, "\n"); - if (!dev->udev) - return -ENODEV; + if (!dev->udev) { + ret = -ENODEV; + goto err_cleanup; + } - if (mutex_lock_interruptible(&dev->v4l2_lock)) - return -ERESTARTSYS; + if (mutex_lock_interruptible(&dev->v4l2_lock)) { + ret = -ERESTARTSYS; + goto err_cleanup; + } /* wake-up tuner */ v4l2_subdev_call(dev->v4l2_subdev, core, s_power, 1); ret = msi2500_set_usb_adc(dev); + if (ret) + goto err_unlock_cleanup; ret = msi2500_isoc_init(dev); if (ret) - msi2500_cleanup_queued_bufs(dev); + goto err_unlock_cleanup; ret = msi2500_ctrl_msg(dev, CMD_START_STREAMING, 0); + if (ret) + goto err_isoc_cleanup; mutex_unlock(&dev->v4l2_lock); + return 0; +err_isoc_cleanup: + msi2500_isoc_cleanup(dev); +err_unlock_cleanup: + mutex_unlock(&dev->v4l2_lock); +err_cleanup: + msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED); return ret; } @@ -863,7 +879,7 @@ static void msi2500_stop_streaming(struct vb2_queue *vq) if (dev->udev) msi2500_isoc_cleanup(dev); - msi2500_cleanup_queued_bufs(dev); + msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR); /* according to tests, at least 700us delay is required */ msleep(20); diff --git a/drivers/media/usb/pwc/pwc-if.c b/drivers/media/usb/pwc/pwc-if.c index c6e5d031f068b..4cc278d711aee 100644 --- a/drivers/media/usb/pwc/pwc-if.c +++ b/drivers/media/usb/pwc/pwc-if.c @@ -710,11 +710,15 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) struct pwc_device *pdev = vb2_get_drv_priv(vq); int r; - if (!pdev->udev) + if (!pdev->udev) { + pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED); return -ENODEV; + } - if (mutex_lock_interruptible(&pdev->v4l2_lock)) + if (mutex_lock_interruptible(&pdev->v4l2_lock)) { + pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED); return -ERESTARTSYS; + } /* Turn on camera and set LEDS on */ pwc_camera_power(pdev, 1); pwc_set_leds(pdev, leds[0], leds[1]); @@ -726,6 +730,11 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) pwc_camera_power(pdev, 0); /* And cleanup any queued bufs!! */ pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED); + if (pdev->fill_buf) { + vb2_buffer_done(&pdev->fill_buf->vb.vb2_buf, + VB2_BUF_STATE_QUEUED); + pdev->fill_buf = NULL; + } } mutex_unlock(&pdev->v4l2_lock); diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c index 8332f2c5aed70..ac66b8d0cb81a 100644 --- a/drivers/media/usb/s2255/s2255drv.c +++ b/drivers/media/usb/s2255/s2255drv.c @@ -617,6 +617,12 @@ static void s2255_fillbuff(struct s2255_vc *vc, break; case V4L2_PIX_FMT_JPEG: case V4L2_PIX_FMT_MJPEG: + if (jpgsize < 0 || + jpgsize > vb2_plane_size(&buf->vb.vb2_buf, 0)) { + dprintk(dev, 1, "bad JPEG frame size %d\n", + jpgsize); + break; + } vb2_set_plane_payload(&buf->vb.vb2_buf, 0, jpgsize); memcpy(vbuf, tmpbuf, jpgsize); break; @@ -2281,6 +2287,11 @@ static int s2255_probe(struct usb_interface *interface, } /* check the firmware is valid */ fw_size = dev->fw_data->fw->size; + if (fw_size < 8) { + dev_err(&interface->dev, "Firmware invalid: too small.\n"); + retval = -ENODEV; + goto errorFWMARKER; + } pdata = (__le32 *) &dev->fw_data->fw->data[fw_size - 8]; if (*pdata != S2255_FW_MARKER) { diff --git a/drivers/media/usb/usbtv/usbtv-audio.c b/drivers/media/usb/usbtv/usbtv-audio.c index 333bd305a4f9f..ae0a14e5ed2ab 100644 --- a/drivers/media/usb/usbtv/usbtv-audio.c +++ b/drivers/media/usb/usbtv/usbtv-audio.c @@ -317,6 +317,13 @@ static const struct snd_pcm_ops snd_usbtv_pcm_ops = { .pointer = snd_usbtv_pointer, }; +static void usbtv_audio_card_free(struct snd_card *card) +{ + struct usbtv *usbtv = card->private_data; + + v4l2_device_put(&usbtv->v4l2_dev); +} + int usbtv_audio_init(struct usbtv *usbtv) { int rv; @@ -331,6 +338,10 @@ int usbtv_audio_init(struct usbtv *usbtv) if (rv < 0) return rv; + v4l2_device_get(&usbtv->v4l2_dev); + card->private_data = usbtv; + card->private_free = usbtv_audio_card_free; + strscpy(card->driver, usbtv->dev->driver->name, sizeof(card->driver)); strscpy(card->shortname, "usbtv", sizeof(card->shortname)); snprintf(card->longname, sizeof(card->longname), diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index 1c08bba9ecb91..5f6685e34d90f 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -392,7 +392,6 @@ static int v4l2_async_match_notify(struct v4l2_async_notifier *notifier, err_call_unbind: v4l2_async_nf_call_unbind(notifier, sd, asc); - list_del(&asc->asc_subdev_entry); err_unregister_subdev: if (registered) @@ -898,9 +897,18 @@ void v4l2_async_unregister_subdev(struct v4l2_subdev *sd) sd->subdev_notifier = NULL; if (sd->asc_list.next) { - list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, - asc_subdev_entry) { - v4l2_async_unbind_subdev_one(asc->notifier, asc); + if (list_empty(&sd->asc_list)) { + /* + * If the sub-device was registered through other means + * than v4l2-async, there are no async connections but + * the sub-device may still well be registered. + * Unregister it now. + */ + v4l2_device_unregister_subdev(sd); + } else { + list_for_each_entry_safe(asc, asc_tmp, &sd->asc_list, + asc_subdev_entry) + v4l2_async_unbind_subdev_one(asc->notifier, asc); } } diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c index c1e3bcb4c1c40..524c11e92aec5 100644 --- a/drivers/media/v4l2-core/v4l2-common.c +++ b/drivers/media/v4l2-core/v4l2-common.c @@ -764,14 +764,15 @@ struct clk *__devm_v4l2_sensor_clk_get(struct device *dev, const char *id, if (ret) return ERR_PTR(ret == -EINVAL ? -EPROBE_DEFER : ret); - if (!id) { + if (id) + clk_id = kasprintf(GFP_KERNEL, "clk-%s-%s", dev_name(dev), id); + else clk_id = kasprintf(GFP_KERNEL, "clk-%s", dev_name(dev)); - if (!clk_id) - return ERR_PTR(-ENOMEM); - id = clk_id; - } - clk_hw = devm_clk_hw_register_fixed_rate(dev, id, NULL, 0, rate); + if (!clk_id) + return ERR_PTR(-ENOMEM); + + clk_hw = devm_clk_hw_register_fixed_rate(dev, clk_id, NULL, 0, rate); if (IS_ERR(clk_hw)) return ERR_CAST(clk_hw); diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c index 85d07ef44f626..02e2ae93d1764 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-core.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -778,10 +778,30 @@ static int validate_av1_film_grain(struct v4l2_ctrl_av1_film_grain *fg) return 0; } +static int validate_av1_tile_info(struct v4l2_av1_tile_info *t) +{ + /* + * tile_cols and tile_rows index the per-tile descriptor arrays and + * bound the tile loops in the stateless AV1 drivers; the product + * bounds the total tile descriptor count. + */ + if (t->tile_cols > V4L2_AV1_MAX_TILE_COLS || + t->tile_rows > V4L2_AV1_MAX_TILE_ROWS) + return -EINVAL; + + if ((u32)t->tile_cols * t->tile_rows > V4L2_AV1_MAX_TILE_COUNT) + return -EINVAL; + + return 0; +} + static int validate_av1_frame(struct v4l2_ctrl_av1_frame *f) { int ret = 0; + ret = validate_av1_tile_info(&f->tile_info); + if (ret) + return ret; ret = validate_av1_quantization(&f->quantization); if (ret) return ret; @@ -882,6 +902,7 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, struct v4l2_ctrl_h264_decode_params *p_h264_dec_params; struct v4l2_ctrl_hevc_sps *p_hevc_sps; struct v4l2_ctrl_hevc_pps *p_hevc_pps; + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering; struct v4l2_ctrl_hevc_decode_params *p_hevc_decode_params; struct v4l2_area *area; @@ -1153,6 +1174,18 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, p_hevc_pps->flags &= ~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; + } else { + /* + * These count the entries the stateless HEVC drivers + * read from column_width_minus1[] / row_height_minus1[] + * and use as tile-loop bounds. + */ + if (p_hevc_pps->num_tile_columns_minus1 >= + ARRAY_SIZE(p_hevc_pps->column_width_minus1)) + return -EINVAL; + if (p_hevc_pps->num_tile_rows_minus1 >= + ARRAY_SIZE(p_hevc_pps->row_height_minus1)) + return -EINVAL; } if (p_hevc_pps->flags & @@ -1171,6 +1204,18 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, break; case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: + p_hevc_slice_params = p; + + if (p_hevc_slice_params->num_ref_idx_l0_active_minus1 >= + V4L2_HEVC_DPB_ENTRIES_NUM_MAX) + return -EINVAL; + + if (p_hevc_slice_params->slice_type != V4L2_HEVC_SLICE_TYPE_B) + break; + + if (p_hevc_slice_params->num_ref_idx_l1_active_minus1 >= + V4L2_HEVC_DPB_ENTRIES_NUM_MAX) + return -EINVAL; break; case V4L2_CTRL_TYPE_HDR10_CLL_INFO: @@ -1191,24 +1236,41 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, return -EINVAL; } - if (p_hdr10_mastering->white_point_x < - V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || - p_hdr10_mastering->white_point_x > - V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || - p_hdr10_mastering->white_point_y < - V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || - p_hdr10_mastering->white_point_y > - V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) + /* + * SMPTE ST 2086 Annex A documents that CTA 861-G uses + * (0, 0) to indicate that the white point chromaticity + * is unknown. + */ + if (p_hdr10_mastering->white_point_x || + p_hdr10_mastering->white_point_y) { + if (p_hdr10_mastering->white_point_x < + V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || + p_hdr10_mastering->white_point_x > + V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || + p_hdr10_mastering->white_point_y < + V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || + p_hdr10_mastering->white_point_y > + V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) + return -EINVAL; + } + + /* + * SMPTE ST 2086 Annex A documents that CTA 861-G uses zero + * maximum and minimum luminance values to indicate that + * the corresponding values are unknown. + */ + if (p_hdr10_mastering->max_display_mastering_luminance && + (p_hdr10_mastering->max_display_mastering_luminance < + V4L2_HDR10_MASTERING_MAX_LUMA_LOW || + p_hdr10_mastering->max_display_mastering_luminance > + V4L2_HDR10_MASTERING_MAX_LUMA_HIGH)) return -EINVAL; - if (p_hdr10_mastering->max_display_mastering_luminance < - V4L2_HDR10_MASTERING_MAX_LUMA_LOW || - p_hdr10_mastering->max_display_mastering_luminance > - V4L2_HDR10_MASTERING_MAX_LUMA_HIGH || - p_hdr10_mastering->min_display_mastering_luminance < - V4L2_HDR10_MASTERING_MIN_LUMA_LOW || - p_hdr10_mastering->min_display_mastering_luminance > - V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) + if (p_hdr10_mastering->min_display_mastering_luminance && + (p_hdr10_mastering->min_display_mastering_luminance < + V4L2_HDR10_MASTERING_MIN_LUMA_LOW || + p_hdr10_mastering->min_display_mastering_luminance > + V4L2_HDR10_MASTERING_MIN_LUMA_HIGH)) return -EINVAL; /* The following restriction comes from ITU-T Rec. H.265 spec */ diff --git a/drivers/media/v4l2-core/v4l2-ctrls-request.c b/drivers/media/v4l2-core/v4l2-ctrls-request.c index e77f722b36a48..aa5566a8fb7d3 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls-request.c +++ b/drivers/media/v4l2-core/v4l2-ctrls-request.c @@ -348,13 +348,12 @@ void v4l2_ctrl_request_complete(struct media_request *req, ret = v4l2_ctrl_handler_init(hdl, (main_hdl->nr_of_buckets - 1) * 8); if (!ret) ret = v4l2_ctrl_request_bind(req, hdl, main_hdl); - if (ret) { - v4l2_ctrl_handler_free(hdl); - kfree(hdl); - return; - } + if (ret) + goto error; hdl->request_is_queued = true; obj = media_request_object_find(req, &req_ops, main_hdl); + if (!obj) + goto error; } hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); @@ -389,6 +388,11 @@ void v4l2_ctrl_request_complete(struct media_request *req, mutex_unlock(main_hdl->lock); media_request_object_complete(obj); media_request_object_put(obj); + return; + +error: + v4l2_ctrl_handler_free(hdl); + kfree(hdl); } EXPORT_SYMBOL(v4l2_ctrl_request_complete); diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c index 10a126e50c1ca..406915afcb2f7 100644 --- a/drivers/media/v4l2-core/v4l2-dev.c +++ b/drivers/media/v4l2-core/v4l2-dev.c @@ -1066,25 +1066,25 @@ int __video_register_device(struct video_device *vdev, vdev->dev.class = &video_class; vdev->dev.devt = MKDEV(VIDEO_MAJOR, vdev->minor); vdev->dev.parent = vdev->dev_parent; - vdev->dev.release = v4l2_device_release; dev_set_name(&vdev->dev, "%s%d", name_base, vdev->num); - - /* Increase v4l2_device refcount */ - v4l2_device_get(vdev->v4l2_dev); - mutex_lock(&videodev_lock); ret = device_register(&vdev->dev); if (ret < 0) { mutex_unlock(&videodev_lock); pr_err("%s: device_register failed\n", __func__); - put_device(&vdev->dev); - return ret; + goto cleanup; } + /* Register the release callback that will be called when the last + reference to the device goes away. */ + vdev->dev.release = v4l2_device_release; if (nr != -1 && nr != vdev->num && warn_if_nr_in_use) pr_warn("%s: requested %s%d, got %s\n", __func__, name_base, nr, video_device_node_name(vdev)); + /* Increase v4l2_device refcount */ + v4l2_device_get(vdev->v4l2_dev); + /* Part 5: Register the entity. */ ret = video_register_media_controller(vdev); diff --git a/drivers/media/v4l2-core/v4l2-fwnode.c b/drivers/media/v4l2-core/v4l2-fwnode.c index cb153ce42c45d..bbe52ec8ef609 100644 --- a/drivers/media/v4l2-core/v4l2-fwnode.c +++ b/drivers/media/v4l2-core/v4l2-fwnode.c @@ -623,6 +623,8 @@ int v4l2_fwnode_parse_link(struct fwnode_handle *fwnode, if (!link->remote_node) goto err_put_remote_endpoint; + fwnode_handle_put(fwnode); + return 0; err_put_remote_endpoint: @@ -1246,7 +1248,7 @@ v4l2_async_nf_parse_fwnode_sensor(struct device *dev, return 0; } -int v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd) +int __v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd, struct module *module) { struct v4l2_async_notifier *notifier; int ret; @@ -1272,7 +1274,7 @@ int v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd) if (ret < 0) goto out_cleanup; - ret = v4l2_async_register_subdev(sd); + ret = __v4l2_async_register_subdev(sd, module); if (ret < 0) goto out_unregister; @@ -1290,7 +1292,7 @@ int v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd) return ret; } -EXPORT_SYMBOL_GPL(v4l2_async_register_subdev_sensor); +EXPORT_SYMBOL_GPL(__v4l2_async_register_subdev_sensor); MODULE_DESCRIPTION("V4L2 fwnode binding parsing library"); MODULE_LICENSE("GPL"); diff --git a/drivers/media/v4l2-core/v4l2-h264.c b/drivers/media/v4l2-core/v4l2-h264.c index c00197d095e75..2323f559c6a3b 100644 --- a/drivers/media/v4l2-core/v4l2-h264.c +++ b/drivers/media/v4l2-core/v4l2-h264.c @@ -440,7 +440,8 @@ v4l2_h264_build_b_ref_lists(const struct v4l2_h264_reflist_builder *builder, } if (builder->num_valid > 1 && - !memcmp(b1_reflist, b0_reflist, builder->num_valid)) + !memcmp(b1_reflist, b0_reflist, + sizeof(b1_reflist[0]) * builder->num_valid)) swap(b1_reflist[0], b1_reflist[1]); print_ref_list_b(builder, b0_reflist, 0); diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c index 25e66bf18f5fc..349022153b18e 100644 --- a/drivers/media/v4l2-core/v4l2-subdev.c +++ b/drivers/media/v4l2-core/v4l2-subdev.c @@ -2506,6 +2506,10 @@ int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable) u64 source_mask = 0; int pad_index = -1; + if (WARN_ON(!v4l2_subdev_has_op(sd, pad, enable_streams) || + !v4l2_subdev_has_op(sd, pad, disable_streams))) + return -ENOIOCTLCMD; + /* * Find the source pad. This helper is meant for subdevs that have a * single source pad, so failures shouldn't happen, but catch them diff --git a/drivers/mfd/ab8500-core.c b/drivers/mfd/ab8500-core.c index f0bc0b5a6f4ad..86fa99022cb35 100644 --- a/drivers/mfd/ab8500-core.c +++ b/drivers/mfd/ab8500-core.c @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include /* diff --git a/drivers/mfd/cgbc-core.c b/drivers/mfd/cgbc-core.c index 4782ff1114a9b..33a32063050f7 100644 --- a/drivers/mfd/cgbc-core.c +++ b/drivers/mfd/cgbc-core.c @@ -363,9 +363,9 @@ static void cgbc_remove(struct platform_device *pdev) { struct cgbc_device_data *cgbc = platform_get_drvdata(pdev); - cgbc_session_release(cgbc); - mfd_remove_devices(&pdev->dev); + + cgbc_session_release(cgbc); } static struct platform_driver cgbc_driver = { diff --git a/drivers/mfd/db8500-prcmu.c b/drivers/mfd/db8500-prcmu.c index 21e68a382b114..6672c55f2ebcc 100644 --- a/drivers/mfd/db8500-prcmu.c +++ b/drivers/mfd/db8500-prcmu.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include #include @@ -2285,7 +2285,7 @@ void db8500_prcmu_system_reset(u16 reset_code) /** * db8500_prcmu_get_reset_code - Retrieve SW reset reason code * - * Retrieves the reset reason code stored by prcmu_system_reset() before + * Retrieves the reset reason code stored by db8500_prcmu_system_reset() before * last restart. */ u16 db8500_prcmu_get_reset_code(void) @@ -3041,7 +3041,7 @@ static int db8500_prcmu_probe(struct platform_device *pdev) db8500_irq_init(np); - prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET); + db8500_prcmu_config_esram0_deep_sleep(ESRAM0_DEEP_SLEEP_STATE_RET); err = mfd_add_devices(&pdev->dev, 0, common_prcmu_devs, ARRAY_SIZE(common_prcmu_devs), NULL, 0, db8500_irq_domain); diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c index ee017617d1d1b..412ae7777f729 100644 --- a/drivers/mfd/iqs62x.c +++ b/drivers/mfd/iqs62x.c @@ -237,7 +237,7 @@ static int iqs62x_firmware_parse(struct iqs62x_core *iqs62x, fw_rec = (struct iqs62x_fw_rec *)(fw->data + pos); pos += sizeof(*fw_rec); - if (pos + fw_rec->len - 1 > fw->size) { + if (!fw_rec->len || fw_rec->len - 1 > fw->size - pos) { ret = -EINVAL; break; } diff --git a/drivers/mfd/macsmc.c b/drivers/mfd/macsmc.c index 3228e79c86eb5..90c434c9215bb 100644 --- a/drivers/mfd/macsmc.c +++ b/drivers/mfd/macsmc.c @@ -406,7 +406,7 @@ static int apple_smc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct apple_smc *smc; - u32 count; + __be32 count; int ret; smc = devm_kzalloc(dev, sizeof(*smc), GFP_KERNEL); @@ -457,8 +457,10 @@ static int apple_smc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, smc); BLOCKING_INIT_NOTIFIER_HEAD(&smc->event_handlers); - ret = apple_smc_read_u32(smc, SMC_KEY(#KEY), &count); - if (ret) + ret = apple_smc_read(smc, SMC_KEY(#KEY), &count, sizeof(count)); + if (ret >= 0 && ret != sizeof(count)) + ret = -EINVAL; + if (ret < 0) return dev_err_probe(smc->dev, ret, "Failed to get key count"); smc->key_count = be32_to_cpu(count); diff --git a/drivers/mfd/qnap-mcu.c b/drivers/mfd/qnap-mcu.c index 4ec1f4cf902f6..22a7a77fdeaf9 100644 --- a/drivers/mfd/qnap-mcu.c +++ b/drivers/mfd/qnap-mcu.c @@ -55,6 +55,7 @@ struct qnap_mcu_reply { * @reply: Reply data structure * @variant: Device variant specific information * @version: MCU firmware version + * @rx: Receive buffer the reply is assembled in */ struct qnap_mcu { struct serdev_device *serdev; @@ -62,6 +63,7 @@ struct qnap_mcu { struct qnap_mcu_reply reply; const struct qnap_mcu_variant *variant; u8 version[QNAP_MCU_VERSION_LEN]; + u8 rx[QNAP_MCU_RX_BUFFER_SIZE]; }; /* @@ -146,20 +148,19 @@ int qnap_mcu_exec(struct qnap_mcu *mcu, const u8 *cmd_data, size_t cmd_data_size, u8 *reply_data, size_t reply_data_size) { - unsigned char rx[QNAP_MCU_RX_BUFFER_SIZE]; size_t length = reply_data_size + QNAP_MCU_CHECKSUM_SIZE; struct qnap_mcu_reply *reply = &mcu->reply; int ret = 0; u8 crc; - if (length > sizeof(rx)) { + if (length > sizeof(mcu->rx)) { dev_err(&mcu->serdev->dev, "expected data too big for receive buffer"); return -EINVAL; } guard(mutex)(&mcu->bus_lock); - reply->data = rx; + reply->data = mcu->rx; reply->length = length; reply->received = 0; reinit_completion(&reply->done); @@ -175,13 +176,13 @@ int qnap_mcu_exec(struct qnap_mcu *mcu, return -ETIMEDOUT; } - crc = qnap_mcu_csum(rx, reply_data_size); - if (crc != rx[reply_data_size]) { + crc = qnap_mcu_csum(mcu->rx, reply_data_size); + if (crc != mcu->rx[reply_data_size]) { dev_err(&mcu->serdev->dev, "Invalid Checksum received\n"); return -EIO; } - memcpy(reply_data, rx, reply_data_size); + memcpy(reply_data, mcu->rx, reply_data_size); return 0; } diff --git a/drivers/mfd/rave-sp.c b/drivers/mfd/rave-sp.c index c1b78d127a261..05d26d92df03e 100644 --- a/drivers/mfd/rave-sp.c +++ b/drivers/mfd/rave-sp.c @@ -63,6 +63,12 @@ #define RAVE_SP_TX_BUFFER_SIZE \ (RAVE_SP_STX_ETX_SIZE + 2 * RAVE_SP_RX_BUFFER_SIZE) +enum rave_sp_frame_offset { + RAVE_SP_FRAME_CODE_OFFSET, + RAVE_SP_FRAME_ACK_ID_OFFSET, + RAVE_SP_FRAME_DATA_OFFSET, +}; + /** * enum rave_sp_deframer_state - Possible state for de-framer * @@ -352,7 +358,7 @@ int rave_sp_exec(struct rave_sp *sp, int command, ret = 0; u8 ackid; - command = sp->variant->cmd.translate(data[0]); + command = sp->variant->cmd.translate(data[RAVE_SP_FRAME_CODE_OFFSET]); if (command < 0) return command; @@ -366,8 +372,8 @@ int rave_sp_exec(struct rave_sp *sp, sp->reply = &reply; mutex_unlock(&sp->reply_lock); - data[0] = command; - data[1] = ackid; + data[RAVE_SP_FRAME_CODE_OFFSET] = command; + data[RAVE_SP_FRAME_ACK_ID_OFFSET] = ackid; rave_sp_write(sp, data, data_size); @@ -388,16 +394,23 @@ EXPORT_SYMBOL_GPL(rave_sp_exec); static void rave_sp_receive_event(struct rave_sp *sp, const unsigned char *data, size_t length) { - u8 cmd[] = { - [0] = rave_sp_reply_code(data[0]), - [1] = data[1], - }; + unsigned long action; + u8 cmd[RAVE_SP_FRAME_DATA_OFFSET]; + + if (length < RAVE_SP_FRAME_DATA_OFFSET + 1) { + dev_warn(&sp->serdev->dev, "Dropping short event frame\n"); + return; + } + + cmd[RAVE_SP_FRAME_CODE_OFFSET] = + rave_sp_reply_code(data[RAVE_SP_FRAME_CODE_OFFSET]); + cmd[RAVE_SP_FRAME_ACK_ID_OFFSET] = data[RAVE_SP_FRAME_ACK_ID_OFFSET]; rave_sp_write(sp, cmd, sizeof(cmd)); - blocking_notifier_call_chain(&sp->event_notifier_list, - rave_sp_action_pack(data[0], data[2]), - NULL); + action = rave_sp_action_pack(data[RAVE_SP_FRAME_CODE_OFFSET], + data[RAVE_SP_FRAME_DATA_OFFSET]); + blocking_notifier_call_chain(&sp->event_notifier_list, action, NULL); } static void rave_sp_receive_reply(struct rave_sp *sp, @@ -405,27 +418,35 @@ static void rave_sp_receive_reply(struct rave_sp *sp, { struct device *dev = &sp->serdev->dev; struct rave_sp_reply *reply; - const size_t payload_length = length - 2; + size_t payload_length; + + if (length < RAVE_SP_FRAME_DATA_OFFSET) { + dev_warn(dev, "Dropping short reply frame\n"); + return; + } + payload_length = length - RAVE_SP_FRAME_DATA_OFFSET; mutex_lock(&sp->reply_lock); reply = sp->reply; if (reply) { - if (reply->code == data[0] && reply->ackid == data[1] && + if (reply->code == data[RAVE_SP_FRAME_CODE_OFFSET] && + reply->ackid == data[RAVE_SP_FRAME_ACK_ID_OFFSET] && payload_length >= reply->length) { /* * We are relying on memcpy(dst, src, 0) to be a no-op * when handling commands that have a no-payload reply */ - memcpy(reply->data, &data[2], reply->length); + memcpy(reply->data, &data[RAVE_SP_FRAME_DATA_OFFSET], + reply->length); complete(&reply->received); sp->reply = NULL; } else { dev_err(dev, "Ignoring incorrect reply\n"); dev_dbg(dev, "Code: expected = 0x%08x received = 0x%08x\n", - reply->code, data[0]); + reply->code, data[RAVE_SP_FRAME_CODE_OFFSET]); dev_dbg(dev, "ACK ID: expected = 0x%08x received = 0x%08x\n", - reply->ackid, data[1]); + reply->ackid, data[RAVE_SP_FRAME_ACK_ID_OFFSET]); dev_dbg(dev, "Length: expected = %zu received = %zu\n", reply->length, payload_length); } @@ -439,10 +460,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, size_t length) { const size_t checksum_length = sp->variant->checksum->length; - const size_t payload_length = length - checksum_length; - const u8 *crc_reported = &data[payload_length]; struct device *dev = &sp->serdev->dev; u8 crc_calculated[RAVE_SP_CHECKSUM_SIZE]; + const u8 *crc_reported; + size_t payload_length; if (unlikely(checksum_length > sizeof(crc_calculated))) { dev_warn(dev, "Checksum too long, dropping\n"); @@ -457,6 +478,9 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } + payload_length = length - checksum_length; + crc_reported = &data[payload_length]; + sp->variant->checksum->subroutine(data, payload_length, crc_calculated); @@ -465,10 +489,10 @@ static void rave_sp_receive_frame(struct rave_sp *sp, return; } - if (rave_sp_id_is_event(data[0])) - rave_sp_receive_event(sp, data, length); + if (rave_sp_id_is_event(data[RAVE_SP_FRAME_CODE_OFFSET])) + rave_sp_receive_event(sp, data, payload_length); else - rave_sp_receive_reply(sp, data, length); + rave_sp_receive_reply(sp, data, payload_length); } static size_t rave_sp_receive_buf(struct serdev_device *serdev, diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c index 9e5fc8259eec2..c57795ed20f4e 100644 --- a/drivers/mfd/rsmu_i2c.c +++ b/drivers/mfd/rsmu_i2c.c @@ -330,6 +330,7 @@ static void rsmu_i2c_remove(struct i2c_client *client) static const struct i2c_device_id rsmu_i2c_id[] = { { "8a34000", RSMU_CM }, { "8a34001", RSMU_CM }, + { "8a34002", RSMU_CM }, { "82p33810", RSMU_SABRE }, { "82p33811", RSMU_SABRE }, { "8v19n850", RSMU_SL }, @@ -341,6 +342,7 @@ MODULE_DEVICE_TABLE(i2c, rsmu_i2c_id); static const struct of_device_id rsmu_i2c_of_match[] = { { .compatible = "idt,8a34000", .data = (void *)RSMU_CM }, { .compatible = "idt,8a34001", .data = (void *)RSMU_CM }, + { .compatible = "idt,8a34002", .data = (void *)RSMU_CM }, { .compatible = "idt,82p33810", .data = (void *)RSMU_SABRE }, { .compatible = "idt,82p33811", .data = (void *)RSMU_SABRE }, { .compatible = "idt,8v19n850", .data = (void *)RSMU_SL }, diff --git a/drivers/mfd/rsmu_spi.c b/drivers/mfd/rsmu_spi.c index c931d8cea0a1a..e07f21482439e 100644 --- a/drivers/mfd/rsmu_spi.c +++ b/drivers/mfd/rsmu_spi.c @@ -241,6 +241,7 @@ static void rsmu_spi_remove(struct spi_device *client) static const struct spi_device_id rsmu_spi_id[] = { { "8a34000", RSMU_CM }, { "8a34001", RSMU_CM }, + { "8a34002", RSMU_CM }, { "82p33810", RSMU_SABRE }, { "82p33811", RSMU_SABRE }, {} @@ -250,6 +251,7 @@ MODULE_DEVICE_TABLE(spi, rsmu_spi_id); static const struct of_device_id rsmu_spi_of_match[] = { { .compatible = "idt,8a34000", .data = (void *)RSMU_CM }, { .compatible = "idt,8a34001", .data = (void *)RSMU_CM }, + { .compatible = "idt,8a34002", .data = (void *)RSMU_CM }, { .compatible = "idt,82p33810", .data = (void *)RSMU_SABRE }, { .compatible = "idt,82p33811", .data = (void *)RSMU_SABRE }, {} diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c index c56f5005eb913..35bcb856595dc 100644 --- a/drivers/mfd/sm501.c +++ b/drivers/mfd/sm501.c @@ -1627,6 +1627,7 @@ static void sm501_pci_remove(struct pci_dev *dev) release_mem_region(sm->io_res->start, 0x100); pci_disable_device(dev); + kfree(sm); } static void sm501_plat_remove(struct platform_device *dev) @@ -1637,6 +1638,7 @@ static void sm501_plat_remove(struct platform_device *dev) iounmap(sm->regs); release_mem_region(sm->io_res->start, 0x100); + kfree(sm); } static const struct pci_device_id sm501_pci_tbl[] = { diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c index 7275dcdb7c44f..e52fbf1481fef 100644 --- a/drivers/mfd/tps65219.c +++ b/drivers/mfd/tps65219.c @@ -541,13 +541,15 @@ static int tps65219_probe(struct i2c_client *client) return ret; } - ret = devm_register_power_off_handler(tps->dev, - tps65219_power_off_handler, - tps); - if (ret) { - dev_err(tps->dev, "failed to register power-off handler: %d\n", ret); - return ret; + if (of_device_is_system_power_controller(tps->dev->of_node)) { + ret = devm_register_power_off_handler(tps->dev, + tps65219_power_off_handler, + tps); + if (ret) + return dev_err_probe(tps->dev, ret, + "Failed to register power-off handler\n"); } + return 0; } diff --git a/drivers/misc/ad525x_dpot-i2c.c b/drivers/misc/ad525x_dpot-i2c.c index 469478f7a1d33..896ad61bb9e17 100644 --- a/drivers/misc/ad525x_dpot-i2c.c +++ b/drivers/misc/ad525x_dpot-i2c.c @@ -105,6 +105,7 @@ MODULE_DEVICE_TABLE(i2c, ad_dpot_id); static struct i2c_driver ad_dpot_i2c_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_i2c_probe, .remove = ad_dpot_i2c_remove, diff --git a/drivers/misc/ad525x_dpot-spi.c b/drivers/misc/ad525x_dpot-spi.c index 263055bda48b7..1ebe629715a84 100644 --- a/drivers/misc/ad525x_dpot-spi.c +++ b/drivers/misc/ad525x_dpot-spi.c @@ -131,6 +131,7 @@ MODULE_DEVICE_TABLE(spi, ad_dpot_spi_id); static struct spi_driver ad_dpot_spi_driver = { .driver = { .name = "ad_dpot", + .dev_groups = ad_dpot_groups, }, .probe = ad_dpot_spi_probe, .remove = ad_dpot_spi_remove, diff --git a/drivers/misc/ad525x_dpot.c b/drivers/misc/ad525x_dpot.c index 04683b981e54c..3b55b3f6c7f49 100644 --- a/drivers/misc/ad525x_dpot.c +++ b/drivers/misc/ad525x_dpot.c @@ -630,66 +630,132 @@ static struct attribute *ad525x_attributes_commands[] = { NULL }; -static const struct attribute_group ad525x_group_commands = { - .attrs = ad525x_attributes_commands, +static struct attribute *ad525x_attributes[] = { + &dev_attr_rdac0.attr, + &dev_attr_rdac1.attr, + &dev_attr_rdac2.attr, + &dev_attr_rdac3.attr, + &dev_attr_rdac4.attr, + &dev_attr_rdac5.attr, + &dev_attr_eeprom0.attr, + &dev_attr_eeprom1.attr, + &dev_attr_eeprom2.attr, + &dev_attr_eeprom3.attr, + &dev_attr_eeprom4.attr, + &dev_attr_eeprom5.attr, + &dev_attr_tolerance0.attr, + &dev_attr_tolerance1.attr, + &dev_attr_tolerance2.attr, + &dev_attr_tolerance3.attr, + &dev_attr_tolerance4.attr, + &dev_attr_tolerance5.attr, + &dev_attr_otp0.attr, + &dev_attr_otp1.attr, + &dev_attr_otp2.attr, + &dev_attr_otp3.attr, + &dev_attr_otp4.attr, + &dev_attr_otp5.attr, + &dev_attr_otp0en.attr, + &dev_attr_otp1en.attr, + &dev_attr_otp2en.attr, + &dev_attr_otp3en.attr, + &dev_attr_otp4en.attr, + &dev_attr_otp5en.attr, + &dev_attr_inc_all.attr, + &dev_attr_dec_all.attr, + &dev_attr_inc_all_6db.attr, + &dev_attr_dec_all_6db.attr, + NULL }; -static int ad_dpot_add_files(struct device *dev, - unsigned int features, unsigned int rdac) +static int ad525x_attr_index(struct attribute *attr, + const struct attribute * const *attrs) { - int err = sysfs_create_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - err |= sysfs_create_file(&dev->kobj, - dpot_attrib_otp[rdac]); - } + int i; - if (err) - dev_err(dev, "failed to register sysfs hooks for RDAC%d\n", - rdac); + for (i = 0; attrs[i]; i++) + if (attr == attrs[i]) + return i; - return err; + return -ENOENT; } -static inline void ad_dpot_remove_files(struct device *dev, - unsigned int features, unsigned int rdac) +static bool ad525x_is_command_attr(struct attribute *attr) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_wipers[rdac]); - if (features & F_CMD_EEP) - sysfs_remove_file(&dev->kobj, - dpot_attrib_eeprom[rdac]); - if (features & F_CMD_TOL) - sysfs_remove_file(&dev->kobj, - dpot_attrib_tolerance[rdac]); - if (features & F_CMD_OTP) { - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp_en[rdac]); - sysfs_remove_file(&dev->kobj, - dpot_attrib_otp[rdac]); + int i; + + for (i = 0; ad525x_attributes_commands[i]; i++) { + if (attr == ad525x_attributes_commands[i]) + return true; } + + return false; +} + +static umode_t ad525x_is_visible(struct kobject *kobj, struct attribute *attr, + int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct dpot_data *data = dev_get_drvdata(dev); + int rdac; + + if (!data) + return 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_wipers); + if (rdac >= 0) + return data->wipers & BIT(rdac) ? attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_eeprom); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_EEP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_tolerance); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_TOL) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + rdac = ad525x_attr_index(attr, dpot_attrib_otp_en); + if (rdac >= 0) + return (data->wipers & BIT(rdac)) && (data->feat & F_CMD_OTP) ? + attr->mode : 0; + + if (ad525x_is_command_attr(attr)) + return data->feat & F_CMD_INC ? attr->mode : 0; + + return attr->mode; } +static const struct attribute_group ad525x_group = { + .attrs = ad525x_attributes, + .is_visible = ad525x_is_visible, +}; + +const struct attribute_group *ad_dpot_groups[] = { + &ad525x_group, + NULL +}; +EXPORT_SYMBOL(ad_dpot_groups); + int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name) { struct dpot_data *data; - int i, err = 0; + int i; data = kzalloc(sizeof(struct dpot_data), GFP_KERNEL); if (!data) { - err = -ENOMEM; - goto exit; + dev_err(dev, "failed to create client for %s ID 0x%lX\n", + name, devid); + return -ENOMEM; } dev_set_drvdata(dev, data); @@ -705,51 +771,22 @@ int ad_dpot_probe(struct device *dev, data->wipers = DPOT_WIPERS(devid); for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) { - err = ad_dpot_add_files(dev, data->feat, i); - if (err) - goto exit_remove_files; + if (data->wipers & BIT(i)) { /* power-up midscale */ if (data->feat & F_RDACS_WONLY) data->rdac_cache[i] = data->max_pos / 2; } - if (data->feat & F_CMD_INC) - err = sysfs_create_group(&dev->kobj, &ad525x_group_commands); - - if (err) { - dev_err(dev, "failed to register sysfs hooks\n"); - goto exit_free; - } - dev_info(dev, "%s %d-Position Digital Potentiometer registered\n", name, data->max_pos); return 0; - -exit_remove_files: - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); - -exit_free: - kfree(data); - dev_set_drvdata(dev, NULL); -exit: - dev_err(dev, "failed to create client for %s ID 0x%lX\n", - name, devid); - return err; } EXPORT_SYMBOL(ad_dpot_probe); void ad_dpot_remove(struct device *dev) { struct dpot_data *data = dev_get_drvdata(dev); - int i; - - for (i = DPOT_RDAC0; i < MAX_RDACS; i++) - if (data->wipers & (1 << i)) - ad_dpot_remove_files(dev, data->feat, i); kfree(data); } diff --git a/drivers/misc/ad525x_dpot.h b/drivers/misc/ad525x_dpot.h index 72a9d6801937c..2e877c89523b5 100644 --- a/drivers/misc/ad525x_dpot.h +++ b/drivers/misc/ad525x_dpot.h @@ -10,6 +10,8 @@ #include +struct attribute_group; + #define DPOT_CONF(features, wipers, max_pos, uid) \ (((features) << 18) | (((wipers) & 0xFF) << 10) | \ ((max_pos & 0xF) << 6) | (uid & 0x3F)) @@ -210,5 +212,6 @@ struct ad_dpot_bus_data { int ad_dpot_probe(struct device *dev, struct ad_dpot_bus_data *bdata, unsigned long devid, const char *name); void ad_dpot_remove(struct device *dev); +extern const struct attribute_group *ad_dpot_groups[]; #endif diff --git a/drivers/misc/amd-sbi/rmi-core.c b/drivers/misc/amd-sbi/rmi-core.c index 3dec2fc001244..3d907022f8162 100644 --- a/drivers/misc/amd-sbi/rmi-core.c +++ b/drivers/misc/amd-sbi/rmi-core.c @@ -471,6 +471,8 @@ int create_misc_rmi_device(struct sbrmi_data *data, GFP_KERNEL, "sbrmi-%x", data->dev_static_addr); + if (!data->sbrmi_misc_dev.name) + return -ENOMEM; data->sbrmi_misc_dev.minor = MISC_DYNAMIC_MINOR; data->sbrmi_misc_dev.fops = &sbrmi_fops; data->sbrmi_misc_dev.parent = dev; @@ -478,6 +480,8 @@ int create_misc_rmi_device(struct sbrmi_data *data, GFP_KERNEL, "sbrmi-%x", data->dev_static_addr); + if (!data->sbrmi_misc_dev.nodename) + return -ENOMEM; data->sbrmi_misc_dev.mode = 0600; return misc_register(&data->sbrmi_misc_dev); diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c index 665a3888708ac..a36a6c7d384a9 100644 --- a/drivers/misc/bcm-vk/bcm_vk_msg.c +++ b/drivers/misc/bcm-vk/bcm_vk_msg.c @@ -108,7 +108,8 @@ u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq, bool bcm_vk_drv_access_ok(struct bcm_vk *vk) { - return (!!atomic_read(&vk->msgq_inited)); + /* Pair with the release store after message queue initialization. */ + return !!atomic_read_acquire(&vk->msgq_inited); } void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask) @@ -501,7 +502,8 @@ int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync) msgq++; } } - atomic_set(&vk->msgq_inited, 1); + /* Publish message queue info before allowing driver access. */ + atomic_set_release(&vk->msgq_inited, 1); return ret; } diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index f9952d76d6ed7..84a49156a2475 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -1196,6 +1196,8 @@ static int rtsx_pci_init_hw(struct rtsx_pcr *pcr) /* Gating real mcu clock */ err = rtsx_pci_write_register(pcr, RTS5261_FW_CFG1, RTS5261_MCU_CLOCK_GATING, 0); + if (err < 0) + return err; err = rtsx_pci_write_register(pcr, RTS5261_REG_FPDCTL, SSC_POWER_DOWN, 0); } else { diff --git a/drivers/misc/cardreader/rtsx_usb.c b/drivers/misc/cardreader/rtsx_usb.c index 1830e9ed25216..a127744918f42 100644 --- a/drivers/misc/cardreader/rtsx_usb.c +++ b/drivers/misc/cardreader/rtsx_usb.c @@ -312,6 +312,9 @@ int rtsx_usb_get_card_status(struct rtsx_ucr *ucr, u16 *status) if (ret < 0) return ret; + ucr->card_status_cache = *status; + ucr->card_status_valid = true; + return 0; } EXPORT_SYMBOL_GPL(rtsx_usb_get_card_status); @@ -623,6 +626,7 @@ static int rtsx_usb_probe(struct usb_interface *intf, { struct usb_device *usb_dev = interface_to_usbdev(intf); struct rtsx_ucr *ucr; + u16 status; int ret; dev_dbg(&intf->dev, @@ -659,6 +663,9 @@ static int rtsx_usb_probe(struct usb_interface *intf, if (ret) goto out_init_fail; + /* Prime cached status for runtime autosuspend decisions. */ + rtsx_usb_get_card_status(ucr, &status); + /* initialize USB SG transfer timer */ timer_setup(&ucr->sg_timer, rtsx_usb_sg_timed_out, 0); @@ -713,22 +720,29 @@ static int rtsx_usb_suspend(struct usb_interface *intf, pm_message_t message) struct rtsx_ucr *ucr = (struct rtsx_ucr *)usb_get_intfdata(intf); u16 val = 0; + bool valid = false; dev_dbg(&intf->dev, "%s called with pm message 0x%04x\n", __func__, message.event); if (PMSG_IS_AUTO(message)) { if (mutex_trylock(&ucr->dev_mutex)) { - rtsx_usb_get_card_status(ucr, &val); + valid = ucr->card_status_valid; + if (valid) + val = ucr->card_status_cache; mutex_unlock(&ucr->dev_mutex); - /* Defer the autosuspend if card exists */ - if (val & (SD_CD | MS_CD)) { + /* + * Do not issue USB commands from runtime autosuspend. + * Raw SD_CD is not authoritative on tray-based readers, + * while a real SD card is protected by the SD/MMC child + * runtime-PM reference once the card is powered. Keep + * the historical Memory Stick autosuspend deferral when + * the cached status says MS media is present. + */ + if (valid && (val & MS_CD)) { device_for_each_child(&intf->dev, NULL, rtsx_usb_resume_child); return -EAGAIN; - } else { - /* if the card does not exists, clear OCP status */ - rtsx_usb_write_register(ucr, OCPCTL, MS_OCP_CLEAR, MS_OCP_CLEAR); } } else { /* There is an ongoing operation*/ diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 46bbe05d1da59..29f9003999a07 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -466,6 +466,7 @@ static void fastrpc_channel_ctx_free(struct kref *ref) cctx = container_of(ref, struct fastrpc_channel_ctx, refcount); + idr_destroy(&cctx->ctx_idr); kfree(cctx); } @@ -1276,10 +1277,12 @@ static int fastrpc_internal_invoke(struct fastrpc_user *fl, u32 kernel, } if (err == -ERESTARTSYS) { + spin_lock(&fl->lock); list_for_each_entry_safe(buf, b, &fl->mmaps, node) { list_del(&buf->node); list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps); } + spin_unlock(&fl->lock); } if (err) @@ -1341,7 +1344,9 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, err = PTR_ERR(name); goto err; } - + inbuf.client_id = fl->client_id; + inbuf.namelen = init.namelen; + inbuf.pageslen = 0; if (!fl->cctx->remote_heap) { err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen, &fl->cctx->remote_heap); @@ -1362,12 +1367,10 @@ static int fastrpc_init_create_static_process(struct fastrpc_user *fl, goto err_map; } scm_done = true; + inbuf.pageslen = 1; } } - inbuf.client_id = fl->client_id; - inbuf.namelen = init.namelen; - inbuf.pageslen = 0; fl->pd = USER_PD; args[0].ptr = (u64)(uintptr_t)&inbuf; @@ -1642,7 +1645,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) dev_err(&cctx->rpdev->dev, "No session available\n"); mutex_destroy(&fl->mutex); kfree(fl); - + fastrpc_channel_ctx_put(cctx); return -EBUSY; } @@ -1678,24 +1681,20 @@ static int fastrpc_dmabuf_alloc(struct fastrpc_user *fl, char __user *argp) return err; } - bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE); + bp.fd = get_unused_fd_flags(O_ACCMODE); if (bp.fd < 0) { dma_buf_put(buf->dmabuf); - return -EINVAL; + return bp.fd; } if (copy_to_user(argp, &bp, sizeof(bp))) { - /* - * The usercopy failed, but we can't do much about it, as - * dma_buf_fd() already called fd_install() and made the - * file descriptor accessible for the current process. It - * might already be closed and dmabuf no longer valid when - * we reach this point. Therefore "leak" the fd and rely on - * the process exit path to do any required cleanup. - */ + put_unused_fd(bp.fd); + dma_buf_put(buf->dmabuf); return -EFAULT; } + dma_buf_fd_install(buf->dmabuf, bp.fd); + return 0; } @@ -1858,9 +1857,6 @@ static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_buf * &args[0]); if (!err) { dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); - spin_lock(&fl->lock); - list_del(&buf->node); - spin_unlock(&fl->lock); fastrpc_buf_free(buf); } else { dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); @@ -1874,6 +1870,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) struct fastrpc_buf *buf = NULL, *iter, *b; struct fastrpc_req_munmap req; struct device *dev = fl->sctx->dev; + int err; if (copy_from_user(&req, argp, sizeof(req))) return -EFAULT; @@ -1881,6 +1878,7 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) spin_lock(&fl->lock); list_for_each_entry_safe(iter, b, &fl->mmaps, node) { if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { + list_del(&iter->node); buf = iter; break; } @@ -1893,7 +1891,14 @@ static int fastrpc_req_munmap(struct fastrpc_user *fl, char __user *argp) return -EINVAL; } - return fastrpc_req_munmap_impl(fl, buf); + err = fastrpc_req_munmap_impl(fl, buf); + if (err) { + spin_lock(&fl->lock); + list_add_tail(&buf->node, &fl->mmaps); + spin_unlock(&fl->lock); + } + + return err; } static int fastrpc_req_mmap(struct fastrpc_user *fl, char __user *argp) diff --git a/drivers/misc/lan966x_pci.c b/drivers/misc/lan966x_pci.c index 9c79b58137e52..f6e358389af77 100644 --- a/drivers/misc/lan966x_pci.c +++ b/drivers/misc/lan966x_pci.c @@ -183,6 +183,7 @@ static int lan966x_pci_probe(struct pci_dev *pdev, const struct pci_device_id *i return 0; err_unload_overlay: + of_platform_depopulate(dev); lan966x_pci_unload_overlay(data); return ret; } diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 2c810ab12e620..a6f0bc150d770 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -4,6 +4,7 @@ * Intel Management Engine Interface (Intel MEI) Linux driver */ +#include #include #include #include @@ -1317,15 +1318,16 @@ static void mei_dev_bus_put(struct mei_device *bus) static void mei_cl_bus_dev_release(struct device *dev) { struct mei_cl_device *cldev = to_mei_cl_device(dev); - struct mei_device *mdev = cldev->cl->dev; + struct mei_device *bus = cldev->bus; struct mei_cl *cl; - mei_cl_flush_queues(cldev->cl, NULL); - mei_me_cl_put(cldev->me_cl); - mei_dev_bus_put(cldev->bus); - - list_for_each_entry(cl, &mdev->file_list, link) - WARN_ON(cl == cldev->cl); + scoped_guard(mutex, &bus->device_lock) { + mei_cl_flush_queues(cldev->cl, NULL); + mei_me_cl_put(cldev->me_cl); + list_for_each_entry(cl, &bus->file_list, link) + WARN_ON(cl == cldev->cl); + } + mei_dev_bus_put(bus); kfree(cldev->cl); kfree(cldev); diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 159e8b841564f..a9a70c851448a 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -425,18 +425,24 @@ static void mei_io_tx_list_free_cl(struct list_head *head, } /** - * mei_io_list_free_fp - free cb from a list that matches file pointer + * mei_io_rd_list_free_fp - free cb from a rd_completed list that matches file pointer * - * @head: io list + * @cl: host client * @fp: file pointer (matching cb file object), may be NULL */ -static void mei_io_list_free_fp(struct list_head *head, const struct file *fp) +static void mei_io_rd_list_free_fp(struct mei_cl *cl, const struct file *fp) { struct mei_cl_cb *cb, *next; + LIST_HEAD(cmpl_list); - list_for_each_entry_safe(cb, next, head, list) + spin_lock(&cl->rd_completed_lock); + list_for_each_entry_safe(cb, next, &cl->rd_completed, list) if (!fp || fp == cb->fp) - mei_io_cb_free(cb); + list_move(&cb->list, &cmpl_list); + spin_unlock(&cl->rd_completed_lock); + + list_for_each_entry_safe(cb, next, &cmpl_list, list) + mei_io_cb_free(cb); } /** @@ -565,9 +571,7 @@ int mei_cl_flush_queues(struct mei_cl *cl, const struct file *fp) mei_io_list_flush_cl(&cl->dev->ctrl_rd_list, cl); mei_cl_free_pending(cl); } - spin_lock(&cl->rd_completed_lock); - mei_io_list_free_fp(&cl->rd_completed, fp); - spin_unlock(&cl->rd_completed_lock); + mei_io_rd_list_free_fp(cl, fp); return 0; } @@ -1404,7 +1408,7 @@ void mei_cl_add_rd_completed(struct mei_cl *cl, struct mei_cl_cb *cb) } /** - * mei_cl_del_rd_completed - free read completed callback with lock + * mei_cl_del_rd_completed - unlink read completed callback with lock and free it * * @cl: host client * @cb: callback block @@ -1413,8 +1417,9 @@ void mei_cl_add_rd_completed(struct mei_cl *cl, struct mei_cl_cb *cb) void mei_cl_del_rd_completed(struct mei_cl *cl, struct mei_cl_cb *cb) { spin_lock(&cl->rd_completed_lock); - mei_io_cb_free(cb); + list_del_init(&cb->list); spin_unlock(&cl->rd_completed_lock); + mei_io_cb_free(cb); } /** diff --git a/drivers/misc/nsm.c b/drivers/misc/nsm.c index ef7b327423409..c3b474450d159 100644 --- a/drivers/misc/nsm.c +++ b/drivers/misc/nsm.c @@ -243,7 +243,7 @@ static int nsm_sendrecv_msg_locked(struct nsm *nsm) goto cleanup; } - msg->resp.len = len; + msg->resp.len = min_t(unsigned int, len, sizeof(msg->resp.data)); rc = 0; @@ -367,7 +367,7 @@ static long nsm_dev_ioctl(struct file *file, unsigned int cmd, /* Copy user argument struct to kernel argument struct */ r = -EFAULT; if (copy_from_user(&raw, argp, _IOC_SIZE(cmd))) - goto out; + return r; mutex_lock(&nsm->lock); @@ -413,6 +413,7 @@ static int nsm_device_init_vq(struct virtio_device *vdev) } static const struct file_operations nsm_dev_fops = { + .owner = THIS_MODULE, .unlocked_ioctl = nsm_dev_ioctl, .compat_ioctl = compat_ptr_ioctl, }; diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c index 1c0fd185114fc..fdce31f9fa43d 100644 --- a/drivers/misc/pci_endpoint_test.c +++ b/drivers/misc/pci_endpoint_test.c @@ -859,6 +859,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) struct pci_dev *pdev = test->pdev; struct device *dev = &pdev->dev; int irq_type = test->irq_type; + int ret = 0; enum pci_barno bar; u32 data, status; u32 addr; @@ -900,8 +901,11 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) status = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS); - if (!left || !(status & STATUS_DOORBELL_SUCCESS)) + if (!left || !(status & STATUS_DOORBELL_SUCCESS)) { dev_err(dev, "Failed to trigger doorbell in endpoint\n"); + /* Store error code, but continue to disable doorbell. */ + ret = -EINVAL; + } pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND, COMMAND_DISABLE_DOORBELL); @@ -915,10 +919,7 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test) return -EINVAL; } - if (!(status & STATUS_DOORBELL_SUCCESS)) - return -EINVAL; - - return 0; + return ret; } static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd, diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c index a40395fb840d1..62f9d3fcf09fa 100644 --- a/drivers/misc/rp1-pio.c +++ b/drivers/misc/rp1-pio.c @@ -95,6 +95,7 @@ struct dma_info { struct dma_chan *chan; size_t buf_size; size_t buf_count; + bool cyclic; unsigned int head_idx; unsigned int tail_idx; struct dma_buf_info bufs[DMA_BOUNCE_BUFFER_COUNT]; @@ -940,6 +941,8 @@ static void rp1_pio_sm_dma_callback(void *param) { struct dma_info *dma = param; + if (dma->cyclic) + WRITE_ONCE(dma->head_idx, dma->head_idx + 1); up(&dma->buf_sem); } @@ -961,18 +964,26 @@ static void rp1_pio_sm_kernel_dma_callback(void *param) static void rp1_pio_sm_dma_free(struct device *dev, struct dma_info *dma) { dmaengine_terminate_all(dma->chan); - while (dma->buf_count > 0) { - dma->buf_count--; + if (dma->cyclic) { + dma->buf_count = 0; dma_free_coherent(dev, ROUND_UP(dma->buf_size, PAGE_SIZE), - dma->bufs[dma->buf_count].buf, - dma->bufs[dma->buf_count].dma_addr); + dma->bufs[0].buf, + dma->bufs[0].dma_addr); + } else { + while (dma->buf_count > 0) { + dma->buf_count--; + dma_free_coherent(dev, ROUND_UP(dma->buf_size, PAGE_SIZE), + dma->bufs[dma->buf_count].buf, + dma->bufs[dma->buf_count].dma_addr); + } } dma_release_channel(dma->chan); dma->chan = NULL; + dma->cyclic = false; } static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint sm, uint dir, - uint buf_size, uint buf_count) + uint buf_size, uint buf_count, uint flags) { struct rp1_pio_sm_set_dmactrl_args set_dmactrl_args; struct rp1_pio_device *pio = client->pio; @@ -981,10 +992,13 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s struct device *dev = &pdev->dev; struct dma_slave_caps dma_caps; struct dma_info *dma = NULL; + bool cyclic = flags & RP1_PIO_SM_CONFIG_XFER_FL_DMA_CYCLE; + bool prefer_light_dma = flags & BIT(0); + bool force_dma_type = flags & BIT(1); bool reconfigure = false; phys_addr_t fifo_addr; uint32_t dma_mask; - char chan_name[4]; + char chan_name[5]; int ret = 0; if (sm >= RP1_PIO_SMS_COUNT || dir >= RP1_PIO_DIR_COUNT) @@ -993,6 +1007,9 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s (!buf_size || (buf_size & 3) || !buf_count || buf_count > DMA_BOUNCE_BUFFER_COUNT)) return -EINVAL; + /* Cyclic DMA is currently only supported for FROM_SM */ + if (cyclic && dir == RP1_PIO_DIR_TO_SM) + return -EINVAL; dma_mask = 1 << (sm * 2 + dir); @@ -1012,9 +1029,6 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s if (reconfigure) rp1_pio_sm_dma_free(dev, dma); - dma->buf_size = buf_size; - /* Round up the allocations */ - buf_size = ROUND_UP(buf_size, PAGE_SIZE); sema_init(&dma->buf_sem, 0); /* Allocate and configure a DMA channel */ @@ -1022,17 +1036,33 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s chan_name[0] = (dir == RP1_PIO_DIR_TO_SM) ? 't' : 'r'; chan_name[1] = 'x'; chan_name[2] = '0' + sm; - chan_name[3] = '\0'; + if (prefer_light_dma) + chan_name[3] = 'l'; + else + chan_name[3] = '\0'; + chan_name[4] = '\0'; + dma->cyclic = false; dma->chan = dma_request_chan(dev, chan_name); if (IS_ERR(dma->chan)) { ret = PTR_ERR(dma->chan); goto err_unclaim; } + dma_get_slave_caps(dma->chan, &dma_caps); + if (force_dma_type && + dma_caps.max_burst != (prefer_light_dma ? 4 : 8)) { + ret = -EBUSY; + goto err_dma_free; + } - /* Alloc and map bounce buffers */ - for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) { - struct dma_buf_info *dbi = &dma->bufs[dma->buf_count]; + if (cyclic) { + dma->buf_size = buf_size * buf_count; + dma->buf_count = buf_count; + /* Round up the allocations */ + buf_size = ROUND_UP(dma->buf_size, PAGE_SIZE); + + /* Alloc and map bounce buffer */ + struct dma_buf_info *dbi = &dma->bufs[0]; dbi->buf = dma_alloc_coherent(dma->chan->device->dev, buf_size, &dbi->dma_addr, GFP_KERNEL); @@ -1042,6 +1072,24 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s } sg_init_table(&dbi->sgl, 1); sg_dma_address(&dbi->sgl) = dbi->dma_addr; + } else { + dma->buf_size = buf_size; + /* Round up the allocations */ + buf_size = ROUND_UP(buf_size, PAGE_SIZE); + + /* Alloc and map bounce buffers */ + for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) { + struct dma_buf_info *dbi = &dma->bufs[dma->buf_count]; + + dbi->buf = dma_alloc_coherent(dma->chan->device->dev, buf_size, + &dbi->dma_addr, GFP_KERNEL); + if (!dbi->buf) { + ret = -ENOMEM; + goto err_dma_free; + } + sg_init_table(&dbi->sgl, 1); + sg_dma_address(&dbi->sgl) = dbi->dma_addr; + } } dma->head_idx = 0; @@ -1081,6 +1129,31 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s if (ret) goto err_dma_free; + if (cyclic) { + struct dma_buf_info *dbi = &dma->bufs[0]; + struct dma_async_tx_descriptor *desc; + + sg_dma_len(&dbi->sgl) = dma->buf_size; + desc = dmaengine_prep_dma_cyclic(dma->chan, dbi->dma_addr, + dma->buf_size, dma->buf_size / dma->buf_count, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) { + dev_err(dev, "DMA preparation failed\n"); + goto err_dma_free; + } + + desc->callback = rp1_pio_sm_dma_callback; + desc->callback_param = dma; + + /* Submit the buffer - the callback will kick the semaphore */ + ret = dmaengine_submit(desc); + if (ret < 0) + goto err_dma_free; + + dma->cyclic = cyclic; + dma_async_issue_pending(dma->chan); + } return 0; err_dma_free: @@ -1100,7 +1173,8 @@ static int rp1_pio_sm_config_xfer_user(struct rp1_pio_client *client, void *para struct rp1_pio_sm_config_xfer_args *args = param; return rp1_pio_sm_config_xfer_internal(client, args->sm, args->dir, - args->buf_size, args->buf_count); + args->buf_size, args->buf_count, + RP1_PIO_SM_CONFIG_XFER_FL_DMA_PREFER_HEAVY); } static int rp1_pio_sm_config_xfer32_user(struct rp1_pio_client *client, void *param) @@ -1108,7 +1182,17 @@ static int rp1_pio_sm_config_xfer32_user(struct rp1_pio_client *client, void *pa struct rp1_pio_sm_config_xfer32_args *args = param; return rp1_pio_sm_config_xfer_internal(client, args->sm, args->dir, - args->buf_size, args->buf_count); + args->buf_size, args->buf_count, + RP1_PIO_SM_CONFIG_XFER_FL_DMA_PREFER_HEAVY); +} + +static int rp1_pio_sm_config_xfer_v2_user(struct rp1_pio_client *client, void *param) +{ + struct rp1_pio_sm_config_xfer_v2_args *args = param; + + return rp1_pio_sm_config_xfer_internal(client, args->sm, args->dir, + args->buf_size, args->buf_count, + args->flags); } static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma, @@ -1221,6 +1305,35 @@ static int rp1_pio_sm_rx_user(struct rp1_pio_device *pio, struct dma_info *dma, if (!bytes) return -EINVAL; + if (dma->cyclic) { + size_t period = dma->buf_size / dma->buf_count; + struct dma_buf_info *dbi = &dma->bufs[0]; + + if (bytes > period) + return -EINVAL; + + /* + * The callback posts the semaphore once per period, so consume + * exactly one 'period' worth of data per read. + */ + if (down_interruptible(&dma->buf_sem)) + return -ERESTARTSYS; + + /* The DMA has wrapped onto the period we were about to read. */ + if (READ_ONCE(dma->head_idx) - dma->tail_idx > dma->buf_count) + return -EOVERFLOW; + + /* Pair with the DMAC's writes into the period we are about to copy. */ + dma_rmb(); + + if (copy_to_user(userbuf, + dbi->buf + (dma->tail_idx % dma->buf_count) * period, + bytes)) + return -EFAULT; + dma->tail_idx++; + return 0; + } + if (!userbuf) { if (dma->head_idx - dma->tail_idx == dma->buf_count) return -EBUSY; @@ -1302,7 +1415,8 @@ static int rp1_pio_sm_xfer_data_user(struct rp1_pio_client *client, void *param) int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir, uint buf_size, uint buf_count) { - return rp1_pio_sm_config_xfer_internal(client, sm, dir, buf_size, buf_count); + return rp1_pio_sm_config_xfer_internal(client, sm, dir, buf_size, buf_count, + RP1_PIO_SM_CONFIG_XFER_FL_DMA_PREFER_HEAVY); } EXPORT_SYMBOL_GPL(rp1_pio_sm_config_xfer); @@ -1400,6 +1514,7 @@ struct handler_info { HANDLER(SM_XFER_DATA, sm_xfer_data_user), HANDLER(SM_XFER_DATA32, sm_xfer_data32_user), HANDLER(SM_CONFIG_XFER32, sm_config_xfer32_user), + HANDLER(SM_CONFIG_XFER_V2, sm_config_xfer_v2_user), HANDLER(CAN_ADD_PROGRAM, can_add_program), HANDLER(ADD_PROGRAM, add_program), @@ -1512,6 +1627,17 @@ void rp1_pio_close(struct rp1_pio_client *client) } spin_unlock(&pio->lock); + if (client->claimed_sms) { + struct rp1_pio_sm_set_enabled_args se_args = { + .mask = client->claimed_sms, .enable = 0 + }; + struct rp1_pio_sm_claim_args uc_args = { + .mask = client->claimed_sms + }; + rp1_pio_sm_set_enabled(client, &se_args); + rp1_pio_sm_unclaim(client, &uc_args); + } + claimed = client->claimed_dmas; for (i = 0; claimed; i++) { @@ -1529,18 +1655,6 @@ void rp1_pio_close(struct rp1_pio_client *client) pio->claimed_dmas &= ~client->claimed_dmas; spin_unlock(&pio->lock); - if (client->claimed_sms) { - struct rp1_pio_sm_set_enabled_args se_args = { - .mask = client->claimed_sms, .enable = 0 - }; - struct rp1_pio_sm_claim_args uc_args = { - .mask = client->claimed_sms - }; - - rp1_pio_sm_set_enabled(client, &se_args); - rp1_pio_sm_unclaim(client, &uc_args); - } - if (client->claimed_instrs) rp1_pio_remove_instrs(pio, client->claimed_instrs); diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 3557d78ee47a2..5a87c12f444a3 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -166,13 +166,8 @@ static void get_clear_fault_map(struct gru_state *gru, } /* - * Atomic (interrupt context) & non-atomic (user context) functions to - * convert a vaddr into a physical address. The size of the page - * is returned in pageshift. - * returns: - * 0 - successful - * < 0 - error code - * 1 - (atomic only) try again in non-atomic context + * Convert a user virtual address to a physical address in process context. + * The size of the page is returned in pageshift. */ static int non_atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, int write, @@ -192,87 +187,25 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma, return 0; } -/* - * atomic_pte_lookup - * - * Convert a user virtual address to a physical address - * Only supports Intel large pages (2MB only) on x86_64. - * ZZZ - hugepage support is incomplete - * - * NOTE: mmap_lock is already held on entry to this function. This - * guarantees existence of the page tables. - */ -static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr, - int write, unsigned long *paddr, int *pageshift) -{ - pgd_t *pgdp; - p4d_t *p4dp; - pud_t *pudp; - pmd_t *pmdp; - pte_t pte; - - pgdp = pgd_offset(vma->vm_mm, vaddr); - if (unlikely(pgd_none(*pgdp))) - goto err; - - p4dp = p4d_offset(pgdp, vaddr); - if (unlikely(p4d_none(*p4dp))) - goto err; - - pudp = pud_offset(p4dp, vaddr); - if (unlikely(pud_none(*pudp))) - goto err; - - pmdp = pmd_offset(pudp, vaddr); - if (unlikely(pmd_none(*pmdp))) - goto err; -#ifdef CONFIG_X86_64 - if (unlikely(pmd_leaf(*pmdp))) - pte = ptep_get((pte_t *)pmdp); - else -#endif - pte = *pte_offset_kernel(pmdp, vaddr); - - if (unlikely(!pte_present(pte) || - (write && (!pte_write(pte) || !pte_dirty(pte))))) - return 1; - - *paddr = pte_pfn(pte) << PAGE_SHIFT; -#ifdef CONFIG_HUGETLB_PAGE - *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT; -#else - *pageshift = PAGE_SHIFT; -#endif - return 0; - -err: - return 1; -} - static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr, int write, int atomic, unsigned long *gpa, int *pageshift) { struct mm_struct *mm = gts->ts_mm; struct vm_area_struct *vma; unsigned long paddr; - int ret, ps; + int ps; vma = find_vma(mm, vaddr); if (!vma) goto inval; - /* - * Atomic lookup is faster & usually works even if called in non-atomic - * context. - */ - rmb(); /* Must/check ms_range_active before loading PTEs */ - ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps); - if (ret) { - if (atomic) - goto upm; - if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) - goto inval; - } + if (atomic) + goto upm; + + /* Order the caller's ms_range_active check before loading PTEs. */ + rmb(); + if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps)) + goto inval; if (is_gru_paddr(paddr)) goto inval; paddr = paddr & ~((1UL << ps) - 1); @@ -569,19 +502,9 @@ static irqreturn_t gru_intr(int chiplet, int blade) continue; } - /* - * This is running in interrupt context. Trylock the mmap_lock. - * If it fails, retry the fault in user context. - */ + /* Address translation may sleep, so retry the fault in user context. */ gts->ustats.fmm_tlbmiss++; - if (!gts->ts_force_cch_reload && - mmap_read_trylock(gts->ts_mm)) { - gru_try_dropin(gru, gts, tfh, NULL); - mmap_read_unlock(gts->ts_mm); - } else { - tfh_user_polling_mode(tfh); - STAT(intr_mm_lock_failed); - } + tfh_user_polling_mode(tfh); } return IRQ_HANDLED; } diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 97b8b38ab47df..b8139c27bc7f8 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -54,7 +54,6 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, intr_cbr); printstat(s, intr_tfh); printstat(s, intr_spurious); - printstat(s, intr_mm_lock_failed); printstat(s, call_os); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 640daf1994df7..3348552925c61 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -182,7 +182,6 @@ struct gru_stats_s { atomic_long_t intr_cbr; atomic_long_t intr_tfh; atomic_long_t intr_spurious; - atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; diff --git a/drivers/misc/vmw_vmci/vmci_event.c b/drivers/misc/vmw_vmci/vmci_event.c index 9a41ab65378de..89ab9f05b88c5 100644 --- a/drivers/misc/vmw_vmci/vmci_event.c +++ b/drivers/misc/vmw_vmci/vmci_event.c @@ -179,16 +179,16 @@ int vmci_event_subscribe(u32 event, } } + *new_subscription_id = sub->id; if (have_new_id) { list_add_rcu(&sub->node, &subscriber_array[event]); retval = VMCI_SUCCESS; } else { + kfree(sub); retval = VMCI_ERROR_NO_RESOURCES; } mutex_unlock(&subscriber_mutex); - - *new_subscription_id = sub->id; return retval; } EXPORT_SYMBOL_GPL(vmci_event_subscribe); diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c index cc96338f6f56e..5f4da86a017cf 100644 --- a/drivers/mmc/core/queue.c +++ b/drivers/mmc/core/queue.c @@ -214,8 +214,14 @@ static int mmc_mq_init_request(struct blk_mq_tag_set *set, struct request *req, struct mmc_queue *mq = set->driver_data; struct mmc_card *card = mq->card; struct mmc_host *host = card->host; + u16 sg_len = mmc_get_max_segments(host); - mq_rq->sg = mmc_alloc_sg(mmc_get_max_segments(host), GFP_KERNEL); + if (!sg_len) { + dev_err(mmc_dev(host), "Wrong max_segs assigned\n"); + return -EINVAL; + } + + mq_rq->sg = mmc_alloc_sg(sg_len, GFP_KERNEL); if (!mq_rq->sg) return -ENOMEM; diff --git a/drivers/mmc/host/atmel-mci.c b/drivers/mmc/host/atmel-mci.c index d1fbc6811563a..d631a6ea2eab5 100644 --- a/drivers/mmc/host/atmel-mci.c +++ b/drivers/mmc/host/atmel-mci.c @@ -2616,6 +2616,8 @@ static void atmci_remove(struct platform_device *pdev) free_irq(platform_get_irq(pdev, 0), host); + cancel_work_sync(&host->bh_work); + clk_disable_unprepare(host->mck); pm_runtime_disable(dev); diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c index 2b7d6d9bcde51..3d0d05ab1fe48 100644 --- a/drivers/mmc/host/davinci_mmc.c +++ b/drivers/mmc/host/davinci_mmc.c @@ -928,7 +928,7 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id) } } - if (qstatus & MMCST0_TOUTRD) { + if (data && (qstatus & MMCST0_TOUTRD)) { /* Read data timeout */ data->error = -ETIMEDOUT; end_transfer = 1; @@ -940,7 +940,7 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id) davinci_abort_data(host, data); } - if (qstatus & (MMCST0_CRCWR | MMCST0_CRCRD)) { + if (data && (qstatus & (MMCST0_CRCWR | MMCST0_CRCRD))) { /* Data CRC error */ data->error = -EILSEQ; end_transfer = 1; @@ -1294,14 +1294,10 @@ static int davinci_mmcsd_probe(struct platform_device *pdev) goto cpu_freq_fail; } - ret = mmc_add_host(mmc); - if (ret < 0) - goto mmc_add_host_fail; - ret = devm_request_irq(&pdev->dev, irq, mmc_davinci_irq, 0, mmc_hostname(mmc), host); if (ret) - goto request_irq_fail; + goto mmc_add_host_fail; if (host->sdio_irq >= 0) { ret = devm_request_irq(&pdev->dev, host->sdio_irq, @@ -1311,6 +1307,10 @@ static int davinci_mmcsd_probe(struct platform_device *pdev) mmc->caps |= MMC_CAP_SDIO_IRQ; } + ret = mmc_add_host(mmc); + if (ret < 0) + goto mmc_add_host_fail; + rename_region(mem, mmc_hostname(mmc)); if (mmc->caps & MMC_CAP_8_BIT_DATA) @@ -1324,8 +1324,6 @@ static int davinci_mmcsd_probe(struct platform_device *pdev) return 0; -request_irq_fail: - mmc_remove_host(mmc); mmc_add_host_fail: mmc_davinci_cpufreq_deregister(host); cpu_freq_fail: diff --git a/drivers/mmc/host/loongson2-mmc.c b/drivers/mmc/host/loongson2-mmc.c index da3daab5f3d6d..7e5c6f00badd0 100644 --- a/drivers/mmc/host/loongson2-mmc.c +++ b/drivers/mmc/host/loongson2-mmc.c @@ -628,8 +628,8 @@ static void ls2k0500_mmc_reorder_cmd_data(struct loongson2_mmc_host *host, return; for_each_sg(cmd->data->sg, sg, cmd->data->sg_len, i) { - data = sg_virt(&sg[i]); - for (j = 0; j < (sg_dma_len(&sg[i]) / 4); j++) + data = sg_virt(sg); + for (j = 0; j < (sg_dma_len(sg) / 4); j++) if (cmd->opcode == SD_SWITCH) data[j] = bitrev8x4(data[j]); else @@ -761,8 +761,8 @@ static void ls2k2000_mmc_reorder_cmd_data(struct loongson2_mmc_host *host, return; for_each_sg(cmd->data->sg, sg, cmd->data->sg_len, i) { - data = sg_virt(&sg[i]); - for (j = 0; j < (sg_dma_len(&sg[i]) / 4); j++) + data = sg_virt(sg); + for (j = 0; j < (sg_dma_len(sg) / 4); j++) data[j] = bitrev8x4(data[j]); } } diff --git a/drivers/mmc/host/omap_hsmmc.c b/drivers/mmc/host/omap_hsmmc.c index 09e4354d1f1db..e09bdf690064e 100644 --- a/drivers/mmc/host/omap_hsmmc.c +++ b/drivers/mmc/host/omap_hsmmc.c @@ -1357,7 +1357,7 @@ omap_hsmmc_prepare_data(struct omap_hsmmc_host *host, struct mmc_request *req) if (req->data == NULL) { OMAP_HSMMC_WRITE(host->base, BLK, 0); if (req->cmd->flags & MMC_RSP_BUSY) { - timeout = req->cmd->busy_timeout * NSEC_PER_MSEC; + timeout = (u64)req->cmd->busy_timeout * NSEC_PER_MSEC; /* * Set an arbitrary 100ms data timeout for commands with diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c index 6816d491b0bf0..6cca38e32b991 100644 --- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c +++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c @@ -279,6 +279,8 @@ static const struct renesas_sdhi_of_data_with_quirks of_rza2_compatible = { static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = { { .compatible = "renesas,sdhi-r7s9210", .data = &of_rza2_compatible, }, { .compatible = "renesas,sdhi-mmc-r8a77470", .data = &of_rcar_gen3_compatible, }, + { .compatible = "renesas,sdhi-r8a774b1", .data = &of_r8a77965_compatible, }, + { .compatible = "renesas,sdhi-r8a774c0", .data = &of_r8a77990_compatible, }, { .compatible = "renesas,sdhi-r8a774e1", .data = &of_r8a7795_compatible, }, { .compatible = "renesas,sdhi-r8a7795", .data = &of_r8a7795_compatible, }, { .compatible = "renesas,sdhi-r8a77961", .data = &of_r8a77961_compatible, }, diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c index 97f065cdede56..ce545f63ee915 100644 --- a/drivers/mmc/host/sdhci.c +++ b/drivers/mmc/host/sdhci.c @@ -4208,6 +4208,14 @@ void __sdhci_read_caps(struct sdhci_host *host, const u16 *ver, } EXPORT_SYMBOL_GPL(__sdhci_read_caps); +static void sdhci_unmap_bounce_buffer(void *data) +{ + struct sdhci_host *host = data; + + dma_unmap_single(mmc_dev(host->mmc), host->bounce_addr, + host->bounce_buffer_size, DMA_BIDIRECTIONAL); +} + static void sdhci_allocate_bounce_buffer(struct sdhci_host *host) { struct mmc_host *mmc = host->mmc; @@ -4262,6 +4270,14 @@ static void sdhci_allocate_bounce_buffer(struct sdhci_host *host) } host->bounce_buffer_size = bounce_size; + ret = devm_add_action_or_reset(mmc_dev(mmc), + sdhci_unmap_bounce_buffer, host); + if (ret) { + devm_kfree(mmc_dev(mmc), host->bounce_buffer); + host->bounce_buffer = NULL; + host->bounce_buffer_size = 0; + return; + } /* Lie about this since we're bouncing */ mmc->max_segs = max_blocks; diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h index 847b72ba7aecb..b32f3055679f5 100644 --- a/drivers/mmc/host/sdhci.h +++ b/drivers/mmc/host/sdhci.h @@ -667,7 +667,7 @@ struct sdhci_host { unsigned int tuning_count; /* Timer count for re-tuning */ unsigned int tuning_mode; /* Re-tuning mode supported by host */ - unsigned int tuning_err; /* Error code for re-tuning */ + int tuning_err; /* Error code for re-tuning */ #define SDHCI_TUNING_MODE_1 0 #define SDHCI_TUNING_MODE_2 1 #define SDHCI_TUNING_MODE_3 2 diff --git a/drivers/mmc/host/via-sdmmc.c b/drivers/mmc/host/via-sdmmc.c index c628b3bbfd7aa..cf7b74835aa39 100644 --- a/drivers/mmc/host/via-sdmmc.c +++ b/drivers/mmc/host/via-sdmmc.c @@ -1154,10 +1154,16 @@ static int via_sd_probe(struct pci_dev *pcidev, ret = mmc_add_host(mmc); if (ret) - goto unmap; + goto free_irq; return 0; +free_irq: + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); + free_irq(pcidev->irq, sdhost); + cancel_work_sync(&sdhost->carddet_work); + /* carddet_work may re-enable the interrupt via via_reset_pcictrl(). */ + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); unmap: iounmap(sdhost->mmiobase); release: @@ -1200,6 +1206,10 @@ static void via_sd_remove(struct pci_dev *pcidev) free_irq(pcidev->irq, sdhost); + cancel_work_sync(&sdhost->carddet_work); + /* carddet_work may re-enable the interrupt via via_reset_pcictrl(). */ + writeb(0x0, sdhost->pcictrl_mmiobase + VIA_CRDR_PCIINTCTRL); + timer_delete_sync(&sdhost->timer); cancel_work_sync(&sdhost->finish_bh_work); diff --git a/drivers/mmc/host/vub300.c b/drivers/mmc/host/vub300.c index 93c4e145009e0..1614e9995ee68 100644 --- a/drivers/mmc/host/vub300.c +++ b/drivers/mmc/host/vub300.c @@ -2115,19 +2115,19 @@ static int vub300_probe(struct usb_interface *interface, command_out_urb = usb_alloc_urb(0, GFP_KERNEL); if (!command_out_urb) { retval = -ENOMEM; - goto error0; + goto err_put_udev; } command_res_urb = usb_alloc_urb(0, GFP_KERNEL); if (!command_res_urb) { retval = -ENOMEM; - goto error1; + goto err_free_out_urb; } /* this also allocates memory for our VUB300 mmc host device */ mmc = mmc_alloc_host(sizeof(*vub300), &udev->dev); if (!mmc) { retval = -ENOMEM; dev_err(&udev->dev, "not enough memory for the mmc_host\n"); - goto error4; + goto err_free_res_urb; } /* MMC core transfer sizes tunable parameters */ mmc->caps = 0; @@ -2344,23 +2344,29 @@ static int vub300_probe(struct usb_interface *interface, interface_to_InterfaceNumber(interface)); retval = mmc_add_host(mmc); if (retval) - goto error6; + goto err_stop_io; return 0; -error6: - timer_delete_sync(&vub300->inactivity_timer); + +err_stop_io: + vub300->interface = NULL; + kref_put(&vub300->kref, vub300_delete); + + return retval; + err_free_host: mmc_free_host(mmc); /* * and hence also frees vub300 * which is contained at the end of struct mmc */ -error4: +err_free_res_urb: usb_free_urb(command_res_urb); -error1: +err_free_out_urb: usb_free_urb(command_out_urb); -error0: +err_put_udev: usb_put_dev(udev); + return retval; } diff --git a/drivers/mtd/devices/mtd_intel_dg.c b/drivers/mtd/devices/mtd_intel_dg.c index 114e69135b8d9..c9855d6f1f95e 100644 --- a/drivers/mtd/devices/mtd_intel_dg.c +++ b/drivers/mtd/devices/mtd_intel_dg.c @@ -15,14 +15,18 @@ #include #include #include +#include #include #include #include #include +#define INTEL_DG_NVM_RPM_TIMEOUT_MS 500 + struct intel_dg_nvm { struct kref refcnt; struct mtd_info mtd; + struct device *dev; struct mutex lock; /* region access lock */ void __iomem *base; void __iomem *base2; @@ -421,6 +425,8 @@ static int intel_dg_nvm_init(struct intel_dg_nvm *nvm, struct device *device, unsigned int i, n; int ret; + nvm->dev = device; + /* clean error register, previous errors are ignored */ idg_nvm_error(nvm); @@ -498,6 +504,7 @@ static int intel_dg_mtd_erase(struct mtd_info *mtd, struct erase_info *info) size_t len; u8 region; u64 addr; + int ret; if (WARN_ON(!nvm)) return -EINVAL; @@ -512,20 +519,29 @@ static int intel_dg_mtd_erase(struct mtd_info *mtd, struct erase_info *info) total_len = info->len; addr = info->addr; + ret = pm_runtime_resume_and_get(nvm->dev); + if (ret < 0) { + dev_err(&mtd->dev, "rpm: get failed %d\n", ret); + return ret; + } + + ret = 0; guard(mutex)(&nvm->lock); while (total_len > 0) { if (!IS_ALIGNED(addr, SZ_4K) || !IS_ALIGNED(total_len, SZ_4K)) { dev_err(&mtd->dev, "unaligned erase %llx %zx\n", addr, total_len); info->fail_addr = addr; - return -ERANGE; + ret = -ERANGE; + break; } idx = idg_nvm_get_region(nvm, addr); if (idx >= nvm->nregions) { dev_err(&mtd->dev, "out of range"); info->fail_addr = MTD_FAIL_ADDR_UNKNOWN; - return -ERANGE; + ret = -ERANGE; + break; } from = addr - nvm->regions[idx].offset; @@ -541,14 +557,16 @@ static int intel_dg_mtd_erase(struct mtd_info *mtd, struct erase_info *info) if (bytes < 0) { dev_dbg(&mtd->dev, "erase failed with %zd\n", bytes); info->fail_addr += nvm->regions[idx].offset; - return bytes; + ret = bytes; + break; } addr += len; total_len -= len; } - return 0; + pm_runtime_put_autosuspend(nvm->dev); + return ret; } static int intel_dg_mtd_read(struct mtd_info *mtd, loff_t from, size_t len, @@ -577,17 +595,24 @@ static int intel_dg_mtd_read(struct mtd_info *mtd, loff_t from, size_t len, if (len > nvm->regions[idx].size - from) len = nvm->regions[idx].size - from; + ret = pm_runtime_resume_and_get(nvm->dev); + if (ret < 0) { + dev_err(&mtd->dev, "rpm: get failed %zd\n", ret); + return ret; + } + guard(mutex)(&nvm->lock); ret = idg_read(nvm, region, from, len, buf); if (ret < 0) { dev_dbg(&mtd->dev, "read failed with %zd\n", ret); - return ret; + } else { + *retlen = ret; + ret = 0; } - *retlen = ret; - - return 0; + pm_runtime_put_autosuspend(nvm->dev); + return ret; } static int intel_dg_mtd_write(struct mtd_info *mtd, loff_t to, size_t len, @@ -616,17 +641,24 @@ static int intel_dg_mtd_write(struct mtd_info *mtd, loff_t to, size_t len, if (len > nvm->regions[idx].size - to) len = nvm->regions[idx].size - to; + ret = pm_runtime_resume_and_get(nvm->dev); + if (ret < 0) { + dev_err(&mtd->dev, "rpm: get failed %zd\n", ret); + return ret; + } + guard(mutex)(&nvm->lock); ret = idg_write(nvm, region, to, len, buf); if (ret < 0) { dev_dbg(&mtd->dev, "write failed with %zd\n", ret); - return ret; + } else { + *retlen = ret; + ret = 0; } - *retlen = ret; - - return 0; + pm_runtime_put_autosuspend(nvm->dev); + return ret; } static void intel_dg_nvm_release(struct kref *kref) @@ -748,7 +780,7 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, dev_name(&aux_dev->dev), invm->regions[i].name); if (!name) { ret = -ENOMEM; - goto err; + goto err_norpm; } nvm->regions[n].name = name; @@ -756,6 +788,21 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, n++; } + ret = devm_pm_runtime_enable(device); + if (ret < 0) { + dev_err(device, "rpm: enable failed %d\n", ret); + goto err_norpm; + } + + pm_runtime_set_autosuspend_delay(device, INTEL_DG_NVM_RPM_TIMEOUT_MS); + pm_runtime_use_autosuspend(device); + + ret = pm_runtime_resume_and_get(device); + if (ret < 0) { + dev_err(device, "rpm: get failed %d\n", ret); + goto err_norpm; + } + nvm->base = devm_ioremap_resource(device, &invm->bar); if (IS_ERR(nvm->base)) { ret = PTR_ERR(nvm->base); @@ -784,9 +831,12 @@ static int intel_dg_mtd_probe(struct auxiliary_device *aux_dev, dev_set_drvdata(&aux_dev->dev, nvm); + pm_runtime_put(device); return 0; err: + pm_runtime_put(device); +err_norpm: kref_put(&nvm->refcnt, intel_dg_nvm_release); return ret; } diff --git a/drivers/mtd/maps/vmu-flash.c b/drivers/mtd/maps/vmu-flash.c index 3667faefe6c58..e992f4d1417e6 100644 --- a/drivers/mtd/maps/vmu-flash.c +++ b/drivers/mtd/maps/vmu-flash.c @@ -610,7 +610,7 @@ static int vmu_connect(struct maple_device *mdev) basic_flash_data = be32_to_cpu(mdev->devinfo.function_data[c - 1]); - card = kmalloc(sizeof(struct memcard), GFP_KERNEL); + card = kzalloc_obj(struct memcard); if (!card) { error = -ENOMEM; goto fail_nomem; @@ -628,15 +628,13 @@ static int vmu_connect(struct maple_device *mdev) * Not sure there are actually any multi-partition devices in the * real world, but the hardware supports them, so, so will we */ - card->parts = kmalloc_array(card->partitions, sizeof(struct vmupart), - GFP_KERNEL); + card->parts = kzalloc_objs(struct vmupart, card->partitions); if (!card->parts) { error = -ENOMEM; goto fail_partitions; } - card->mtd = kmalloc_array(card->partitions, sizeof(struct mtd_info), - GFP_KERNEL); + card->mtd = kzalloc_objs(struct mtd_info, card->partitions); if (!card->mtd) { error = -ENOMEM; goto fail_mtd_info; diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c index 64808493b4f5e..42cabd3c1efad 100644 --- a/drivers/mtd/mtdcore.c +++ b/drivers/mtd/mtdcore.c @@ -104,6 +104,15 @@ static void mtd_release(struct device *dev) device_destroy(&mtd_class, index + 1); } +/* + * No-op device release used in add_mtd_device() error paths. + * Prevents mtd_release() from being called via device_release(), + * which would free the mtd_info that the caller still manages. + */ +static void mtd_dev_release_nop(struct device *dev) +{ +} + static void mtd_device_release(struct kref *kref) { struct mtd_info *mtd = container_of(kref, struct mtd_info, refcnt); @@ -798,10 +807,8 @@ int add_mtd_device(struct mtd_info *mtd) mtd_check_of_node(mtd); of_node_get(mtd_get_of_node(mtd)); error = device_register(&mtd->dev); - if (error) { - put_device(&mtd->dev); + if (error) goto fail_added; - } /* Add the nvmem provider */ error = mtd_nvmem_add(mtd); @@ -839,8 +846,16 @@ int add_mtd_device(struct mtd_info *mtd) return 0; fail_nvmem_add: - device_unregister(&mtd->dev); + device_del(&mtd->dev); fail_added: + /* + * Clear type and set nop release to prevent mtd_release() -> + * release_mtd_partition() -> free_partition() from freeing mtd. + * The caller handles cleanup on failure. + */ + mtd->dev.type = NULL; + mtd->dev.release = mtd_dev_release_nop; + put_device(&mtd->dev); of_node_put(mtd_get_of_node(mtd)); fail_devname: idr_remove(&mtd_idr, i); diff --git a/drivers/mtd/mtdoops.c b/drivers/mtd/mtdoops.c index b88083751a0c7..81d11c45a15c6 100644 --- a/drivers/mtd/mtdoops.c +++ b/drivers/mtd/mtdoops.c @@ -392,6 +392,9 @@ static void mtdoops_notify_remove(struct mtd_info *mtd) cxt->mtd = NULL; flush_work(&cxt->work_erase); flush_work(&cxt->work_write); + vfree(cxt->oops_page_used); + cxt->oops_page_used = NULL; + cxt->oops_pages = 0; } diff --git a/drivers/mtd/mtdpart.c b/drivers/mtd/mtdpart.c index 2876501a78145..fb92b7a0ed5f7 100644 --- a/drivers/mtd/mtdpart.c +++ b/drivers/mtd/mtdpart.c @@ -254,7 +254,8 @@ int mtd_add_partition(struct mtd_info *parent, const char *name, /* the direct offset is expected */ if (offset == MTDPART_OFS_APPEND || - offset == MTDPART_OFS_NXTBLK) + offset == MTDPART_OFS_NXTBLK || + offset == MTDPART_OFS_RETAIN) return -EINVAL; if (length == MTDPART_SIZ_FULL) diff --git a/drivers/mtd/mtdswap.c b/drivers/mtd/mtdswap.c index d8f2e5be2d315..d38528210350c 100644 --- a/drivers/mtd/mtdswap.c +++ b/drivers/mtd/mtdswap.c @@ -125,6 +125,7 @@ struct mtdswap_dev { char *page_buf; char *oob_buf; + struct dentry *debugfs_stats; }; struct mtdswap_oobdata { @@ -1262,7 +1263,8 @@ static int mtdswap_add_debugfs(struct mtdswap_dev *d) if (IS_ERR_OR_NULL(root)) return -1; - debugfs_create_file("mtdswap_stats", S_IRUSR, root, d, &mtdswap_fops); + d->debugfs_stats = debugfs_create_file("mtdswap_stats", 0400, root, + d, &mtdswap_fops); return 0; } @@ -1450,6 +1452,7 @@ static void mtdswap_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd) debugfs_failed: del_mtd_blktrans_dev(mbd_dev); + mbd_dev = NULL; cleanup: mtdswap_cleanup(d); @@ -1463,6 +1466,7 @@ static void mtdswap_remove_dev(struct mtd_blktrans_dev *dev) { struct mtdswap_dev *d = MTDSWAP_MBD_TO_MTDSWAP(dev); + debugfs_remove(d->debugfs_stats); del_mtd_blktrans_dev(dev); mtdswap_cleanup(d); kfree(d); diff --git a/drivers/mtd/nand/ecc-mtk.c b/drivers/mtd/nand/ecc-mtk.c index c75bb8b80cc1e..96703f0a418ea 100644 --- a/drivers/mtd/nand/ecc-mtk.c +++ b/drivers/mtd/nand/ecc-mtk.c @@ -123,8 +123,8 @@ static int mt7622_ecc_regs[] = { [ECC_DECIRQ_STA] = 0x144, }; -static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc, - enum mtk_ecc_operation op) +static inline int mtk_ecc_wait_idle(struct mtk_ecc *ecc, + enum mtk_ecc_operation op) { struct device *dev = ecc->dev; u32 val; @@ -136,6 +136,8 @@ static inline void mtk_ecc_wait_idle(struct mtk_ecc *ecc, if (ret) dev_warn(dev, "%s NOT idle\n", op == ECC_ENCODE ? "encoder" : "decoder"); + + return ret; } static irqreturn_t mtk_ecc_irq(int irq, void *id) @@ -312,7 +314,11 @@ int mtk_ecc_enable(struct mtk_ecc *ecc, struct mtk_ecc_config *config) return ret; } - mtk_ecc_wait_idle(ecc, op); + ret = mtk_ecc_wait_idle(ecc, op); + if (ret) { + mutex_unlock(&ecc->lock); + return ret; + } ret = mtk_ecc_config(ecc, config); if (ret) { @@ -412,7 +418,9 @@ int mtk_ecc_encode(struct mtk_ecc *ecc, struct mtk_ecc_config *config, if (ret) goto timeout; - mtk_ecc_wait_idle(ecc, ECC_ENCODE); + ret = mtk_ecc_wait_idle(ecc, ECC_ENCODE); + if (ret) + goto timeout; /* Program ECC bytes to OOB: per sector oob = FDM + ECC + SPARE */ len = (config->strength * ecc->caps->parity_bits + 7) >> 3; diff --git a/drivers/mtd/nand/ecc-realtek.c b/drivers/mtd/nand/ecc-realtek.c index 0046da37ea3e5..e2153b1789be7 100644 --- a/drivers/mtd/nand/ecc-realtek.c +++ b/drivers/mtd/nand/ecc-realtek.c @@ -448,6 +448,7 @@ static const struct of_device_id rtl_ecc_of_ids[] = { }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, rtl_ecc_of_ids); static struct platform_driver rtl_ecc_driver = { .driver = { diff --git a/drivers/mtd/nand/raw/nand_onfi.c b/drivers/mtd/nand/raw/nand_onfi.c index 861975e44b552..f03553eb1324b 100644 --- a/drivers/mtd/nand/raw/nand_onfi.c +++ b/drivers/mtd/nand/raw/nand_onfi.c @@ -35,16 +35,21 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip, struct nand_onfi_params *p) { struct nand_device *base = &chip->base; + struct mtd_info *mtd = nand_to_mtd(chip); struct nand_ecc_props requirements; struct onfi_ext_param_page *ep; struct onfi_ext_section *s; struct onfi_ext_ecc_info *ecc; + size_t remaining, section_len; uint8_t *cursor; int ret; int len; int i; len = le16_to_cpu(p->ext_param_page_length) * 16; + if (len < sizeof(*ep)) + return -EINVAL; + ep = kmalloc(len, GFP_KERNEL); if (!ep) return -ENOMEM; @@ -77,11 +82,29 @@ static int nand_flash_detect_ext_param_page(struct nand_chip *chip, /* find the ECC section. */ cursor = (uint8_t *)(ep + 1); + remaining = len - sizeof(*ep); for (i = 0; i < ONFI_EXT_SECTION_MAX; i++) { s = ep->sections + i; - if (s->type == ONFI_SECTION_TYPE_2) + section_len = s->length * 16; + if (section_len > remaining) { + dev_dbg(&mtd->dev, + "ONFI extended parameter section %d exceeds page\n", + i); + goto ext_out; + } + + if (s->type == ONFI_SECTION_TYPE_2) { + if (section_len < sizeof(*ecc)) { + dev_dbg(&mtd->dev, + "ONFI extended parameter ECC section %d is too short\n", + i); + goto ext_out; + } break; - cursor += s->length * 16; + } + + cursor += section_len; + remaining -= section_len; } if (i == ONFI_EXT_SECTION_MAX) { pr_debug("We can not find the ECC section.\n"); diff --git a/drivers/mtd/nand/raw/pl35x-nand-controller.c b/drivers/mtd/nand/raw/pl35x-nand-controller.c index 81dde993ee9b0..12b9e0936c8f1 100644 --- a/drivers/mtd/nand/raw/pl35x-nand-controller.c +++ b/drivers/mtd/nand/raw/pl35x-nand-controller.c @@ -914,7 +914,6 @@ static int pl35x_nand_init_hw_ecc_controller(struct pl35x_nandc *nfc, chip->ecc.steps = mtd->writesize / chip->ecc.size; chip->ecc.read_page = pl35x_nand_read_page_hwecc; chip->ecc.write_page = pl35x_nand_write_page_hwecc; - chip->ecc.write_page_raw = nand_monolithic_write_page_raw; pl35x_smc_set_ecc_pg_size(nfc, chip, mtd->writesize); nfc->ecc_buf = devm_kmalloc(nfc->dev, chip->ecc.bytes * chip->ecc.steps, @@ -973,15 +972,19 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) switch (chip->ecc.engine_type) { case NAND_ECC_ENGINE_TYPE_ON_DIE: + dev_dbg(nfc->dev, "Using on-die hardware ECC\n"); /* Keep these legacy BBT descriptors for ON_DIE situations */ chip->bbt_td = &bbt_main_descr; chip->bbt_md = &bbt_mirror_descr; fallthrough; case NAND_ECC_ENGINE_TYPE_NONE: + dev_dbg(nfc->dev, "Using no ECC engine\n"); + break; case NAND_ECC_ENGINE_TYPE_SOFT: - chip->ecc.write_page_raw = nand_monolithic_write_page_raw; + dev_dbg(nfc->dev, "Using software ECC\n"); break; case NAND_ECC_ENGINE_TYPE_ON_HOST: + dev_dbg(nfc->dev, "Using on-host hardware ECC\n"); ret = pl35x_nand_init_hw_ecc_controller(nfc, chip); if (ret) return ret; @@ -992,6 +995,9 @@ static int pl35x_nand_attach_chip(struct nand_chip *chip) return -EINVAL; } + chip->ecc.read_page_raw = nand_monolithic_read_page_raw; + chip->ecc.write_page_raw = nand_monolithic_write_page_raw; + return 0; } diff --git a/drivers/mtd/parsers/afs.c b/drivers/mtd/parsers/afs.c index 26116694c821b..7ab3d50f565ec 100644 --- a/drivers/mtd/parsers/afs.c +++ b/drivers/mtd/parsers/afs.c @@ -235,6 +235,9 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, pr_debug("Parsing v2 partition @%08x-%08x\n", off, off + mtd->erasesize); + if (mtd->erasesize < sizeof(footer)) + return -EINVAL; + /* First read the footer */ ptr = off + mtd->erasesize - sizeof(footer); ret = mtd_read(mtd, ptr, sizeof(footer), &sz, (u_char *)footer); @@ -245,6 +248,8 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, } name = (char *) &footer[0]; version = footer[9]; + if (footer[8] > mtd->erasesize - sizeof(footer)) + return -EINVAL; ptr = off + mtd->erasesize - sizeof(footer) - footer[8]; pr_debug("found image \"%s\", version %08x, info @%08x\n", @@ -278,6 +283,8 @@ static int afs_parse_v2_partition(struct mtd_info *mtd, entrypoint = imginfo[pad]; attributes = imginfo[pad+1]; region_count = imginfo[pad+2]; + if (region_count > (ARRAY_SIZE(imginfo) - pad - 3) / 4) + return -EINVAL; block_start = imginfo[20]; block_end = imginfo[21]; diff --git a/drivers/mtd/ubi/attach.c b/drivers/mtd/ubi/attach.c index adc47b87b38a5..2b1629fdb41bb 100644 --- a/drivers/mtd/ubi/attach.c +++ b/drivers/mtd/ubi/attach.c @@ -771,7 +771,7 @@ void ubi_remove_av(struct ubi_attach_info *ai, struct ubi_ainf_volume *av) static int early_erase_peb(struct ubi_device *ubi, const struct ubi_attach_info *ai, int pnum, int ec) { - int err; + int err, torture = 0; struct ubi_ec_hdr *ec_hdr; if ((long long)ec >= UBI_MAX_ERASECOUNTER) { @@ -790,7 +790,7 @@ static int early_erase_peb(struct ubi_device *ubi, ec_hdr->ec = cpu_to_be64(ec); - err = ubi_io_sync_erase(ubi, pnum, 0); + err = ubi_io_sync_erase(ubi, pnum, &torture); if (err < 0) goto out_free; @@ -1600,7 +1600,7 @@ int ubi_attach(struct ubi_device *ubi, int force_scan) err = ubi_read_volume_table(ubi, ai); if (err) - goto out_ai; + goto out_fm; err = ubi_wl_init(ubi, ai); if (err) @@ -1642,6 +1642,8 @@ int ubi_attach(struct ubi_device *ubi, int force_scan) out_vtbl: ubi_free_all_volumes(ubi); vfree(ubi->vtbl); +out_fm: + ubi_free_fastmap(ubi); out_ai: destroy_ai(ai); return err; diff --git a/drivers/mtd/ubi/build.c b/drivers/mtd/ubi/build.c index ef6a22f372f95..fb121b6d08484 100644 --- a/drivers/mtd/ubi/build.c +++ b/drivers/mtd/ubi/build.c @@ -1105,6 +1105,7 @@ int ubi_detach_mtd_dev(int ubi_num, int anyway) ubi->ref_count -= 1; if (ubi->ref_count) { if (!anyway) { + put_device(&ubi->dev); spin_unlock(&ubi_devices_lock); return -EBUSY; } @@ -1317,10 +1318,10 @@ static int __init ubi_init_attach(void) return 0; out_detach: - for (k = 0; k < i; k++) + for (k = 0; k < UBI_MAX_DEVICES; k++) if (ubi_devices[k]) { mutex_lock(&ubi_devices_mutex); - ubi_detach_mtd_dev(ubi_devices[k]->ubi_num, 1); + ubi_detach_mtd_dev(k, 1); mutex_unlock(&ubi_devices_mutex); } return err; @@ -1574,7 +1575,7 @@ static int ubi_mtd_param_parse(const char *val, const struct kernel_param *kp) } module_param_call(mtd, ubi_mtd_param_parse, NULL, NULL, 0400); -MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num]]].\n" +MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=[,[,max_beb_per1024[,ubi_num[,enable_fm[,need_resv_pool]]]]].\n" "Multiple \"mtd\" parameters may be specified.\n" "MTD devices may be specified by their number, name, or path to the MTD character device node.\n" "Optional \"vid_hdr_offs\" parameter specifies UBI VID header position to be used by UBI. (default value if 0)\n" @@ -1587,9 +1588,9 @@ MODULE_PARM_DESC(mtd, "MTD devices to attach. Parameter format: mtd=fm_pool); return_unused_pool_pebs(ubi, &ubi->fm_wl_pool); @@ -540,11 +538,7 @@ static void ubi_fastmap_close(struct ubi_device *ubi) ubi->fm_anchor = NULL; } - if (ubi->fm) { - for (i = 0; i < ubi->fm->used_blocks; i++) - kfree(ubi->fm->e[i]); - } - kfree(ubi->fm); + ubi_free_fastmap(ubi); } /** diff --git a/drivers/mtd/ubi/fastmap.c b/drivers/mtd/ubi/fastmap.c index 9a4940874be5b..12dd05fd37789 100644 --- a/drivers/mtd/ubi/fastmap.c +++ b/drivers/mtd/ubi/fastmap.c @@ -1526,7 +1526,8 @@ int ubi_update_fastmap(struct ubi_device *ubi) if (!tmp_e) { if (old_fm && old_fm->e[i]) { - ret = ubi_sync_erase(ubi, old_fm->e[i], 0); + ret = ubi_sync_erase(ubi, old_fm->e[i], + &old_fm->to_be_tortured[i]); if (ret < 0) { ubi_err(ubi, "could not erase old fastmap PEB"); @@ -1578,7 +1579,8 @@ int ubi_update_fastmap(struct ubi_device *ubi) if (old_fm) { /* no fresh anchor PEB was found, reuse the old one */ if (!tmp_e) { - ret = ubi_sync_erase(ubi, old_fm->e[0], 0); + ret = ubi_sync_erase(ubi, old_fm->e[0], + &old_fm->to_be_tortured[0]); if (ret < 0) { ubi_err(ubi, "could not erase old anchor PEB"); diff --git a/drivers/mtd/ubi/io.c b/drivers/mtd/ubi/io.c index a4999bce435f5..d70d1313ad4e8 100644 --- a/drivers/mtd/ubi/io.c +++ b/drivers/mtd/ubi/io.c @@ -519,8 +519,10 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum) /** * ubi_io_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object - * @pnum: physical eraseblock number to erase - * @torture: if this physical eraseblock has to be tortured + * @pnum: the physical eraseblock number to erase + * @torture: if this physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function synchronously erases physical eraseblock @pnum. If @torture * flag is not zero, the physical eraseblock is checked by means of writing @@ -532,7 +534,7 @@ static int nor_erase_prepare(struct ubi_device *ubi, int pnum) * codes in case of other errors. Note, %-EIO means that the physical * eraseblock is bad. */ -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture) { int err, ret = 0; @@ -560,10 +562,11 @@ int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture) return err; } - if (torture) { + if (*torture) { ret = torture_peb(ubi, pnum); if (ret < 0) return ret; + *torture = 0; } err = do_sync_erase(ubi, pnum); @@ -868,6 +871,8 @@ int ubi_io_write_ec_hdr(struct ubi_device *ubi, int pnum, return -EROFS; } + memset((char *)ec_hdr + UBI_EC_HDR_SIZE, 0xFF, ubi->ec_hdr_alsize - UBI_EC_HDR_SIZE); + err = ubi_io_write(ubi, ec_hdr, pnum, 0, ubi->ec_hdr_alsize); return err; } @@ -1150,6 +1155,14 @@ int ubi_io_write_vid_hdr(struct ubi_device *ubi, int pnum, return -EROFS; } + if (ubi->vid_hdr_shift) { + memset((char *)p, 0xFF, ubi->vid_hdr_shift); + memset((char *)p + ubi->vid_hdr_shift + UBI_VID_HDR_SIZE, 0xFF, + ubi->vid_hdr_alsize - (ubi->vid_hdr_shift + UBI_VID_HDR_SIZE)); + } else { + memset((char *)p + UBI_VID_HDR_SIZE, 0xFF, ubi->vid_hdr_alsize - UBI_VID_HDR_SIZE); + } + err = ubi_io_write(ubi, p, pnum, ubi->vid_hdr_aloffset, ubi->vid_hdr_alsize); return err; diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h index c792b9bcab9bc..edc68c1603e5c 100644 --- a/drivers/mtd/ubi/ubi.h +++ b/drivers/mtd/ubi/ubi.h @@ -901,7 +901,7 @@ int self_check_eba(struct ubi_device *ubi, struct ubi_attach_info *ai_fastmap, struct ubi_attach_info *ai_scan); /* wl.c */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture); +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture); int ubi_wl_get_peb(struct ubi_device *ubi); int ubi_wl_put_peb(struct ubi_device *ubi, int vol_id, int lnum, int pnum, int torture); @@ -923,7 +923,7 @@ int ubi_io_read(const struct ubi_device *ubi, void *buf, int pnum, int offset, int len); int ubi_io_write(struct ubi_device *ubi, const void *buf, int pnum, int offset, int len); -int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int torture); +int ubi_io_sync_erase(struct ubi_device *ubi, int pnum, int *torture); int ubi_io_is_bad(const struct ubi_device *ubi, int pnum); int ubi_io_mark_bad(const struct ubi_device *ubi, int pnum); int ubi_io_read_ec_hdr(struct ubi_device *ubi, int pnum, @@ -969,10 +969,22 @@ int ubi_scan_fastmap(struct ubi_device *ubi, struct ubi_attach_info *ai, struct ubi_attach_info *scan_ai); int ubi_fastmap_init_checkmap(struct ubi_volume *vol, int leb_count); void ubi_fastmap_destroy_checkmap(struct ubi_volume *vol); +static inline void ubi_free_fastmap(struct ubi_device *ubi) +{ + if (ubi->fm) { + int i; + + for (i = 0; i < ubi->fm->used_blocks; i++) + kmem_cache_free(ubi_wl_entry_slab, ubi->fm->e[i]); + kfree(ubi->fm); + ubi->fm = NULL; + } +} #else static inline int ubi_update_fastmap(struct ubi_device *ubi) { return 0; } static inline int ubi_fastmap_init_checkmap(struct ubi_volume *vol, int leb_count) { return 0; } static inline void ubi_fastmap_destroy_checkmap(struct ubi_volume *vol) {} +static inline void ubi_free_fastmap(struct ubi_device *ubi) { } #endif /* block.c */ diff --git a/drivers/mtd/ubi/wl.c b/drivers/mtd/ubi/wl.c index fbd399cf65033..fefcbaf250024 100644 --- a/drivers/mtd/ubi/wl.c +++ b/drivers/mtd/ubi/wl.c @@ -443,12 +443,14 @@ static int prot_queue_del(struct ubi_device *ubi, int pnum) * ubi_sync_erase - synchronously erase a physical eraseblock. * @ubi: UBI device description object * @e: the physical eraseblock to erase - * @torture: if the physical eraseblock has to be tortured + * @torture: if the physical eraseblock has to be tortured; cleared to zero + * once the torture test has completed successfully so that a retry + * of the erase does not torture the physical eraseblock again * * This function returns zero in case of success and a negative error code in * case of failure. */ -int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int torture) +int ubi_sync_erase(struct ubi_device *ubi, struct ubi_wl_entry *e, int *torture) { int err; struct ubi_ec_hdr *ec_hdr; @@ -1113,7 +1115,7 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) dbg_wl("erase PEB %d EC %d LEB %d:%d", pnum, e->ec, wl_wrk->vol_id, wl_wrk->lnum); - err = ubi_sync_erase(ubi, e, wl_wrk->torture); + err = ubi_sync_erase(ubi, e, &wl_wrk->torture); if (!err) { spin_lock(&ubi->wl_lock); @@ -1150,7 +1152,8 @@ static int __erase_worker(struct ubi_device *ubi, struct ubi_work *wl_wrk) int err1; /* Re-schedule the LEB for erasure */ - err1 = schedule_erase(ubi, e, vol_id, lnum, 0, true); + err1 = schedule_erase(ubi, e, vol_id, lnum, wl_wrk->torture, + true); if (err1) { spin_lock(&ubi->wl_lock); wl_entry_destroy(ubi, e); @@ -1757,7 +1760,7 @@ static void shutdown_work(struct ubi_device *ubi) static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync) { struct ubi_wl_entry *e; - int err; + int err, torture = 0; e = kmem_cache_alloc(ubi_wl_entry_slab, GFP_KERNEL); if (!e) @@ -1768,7 +1771,7 @@ static int erase_aeb(struct ubi_device *ubi, struct ubi_ainf_peb *aeb, bool sync ubi->lookuptbl[e->pnum] = e; if (sync) { - err = ubi_sync_erase(ubi, e, false); + err = ubi_sync_erase(ubi, e, &torture); if (err) goto out_free; diff --git a/drivers/net/amt.c b/drivers/net/amt.c index e4d1aa26d7bbd..01d2223fce09b 100644 --- a/drivers/net/amt.c +++ b/drivers/net/amt.c @@ -1211,7 +1211,7 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) data = true; } v6 = false; - group.ip4 = iph->daddr; + group.ip4 = ip_hdr(skb)->daddr; #if IS_ENABLED(CONFIG_IPV6) } else if (iph->version == 6) { ip6h = ipv6_hdr(skb); @@ -1235,7 +1235,7 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) data = true; } v6 = true; - group.ip6 = ip6h->daddr; + group.ip6 = ipv6_hdr(skb)->daddr; #endif } else { dev->stats.tx_errors++; @@ -1278,12 +1278,12 @@ static netdev_tx_t amt_dev_xmit(struct sk_buff *skb, struct net_device *dev) hlist_for_each_entry_rcu(gnode, &tunnel->groups[hash], node) { if (!v6) { - if (gnode->group_addr.ip4 == iph->daddr) + if (gnode->group_addr.ip4 == group.ip4) goto found; #if IS_ENABLED(CONFIG_IPV6) } else { if (ipv6_addr_equal(&gnode->group_addr.ip6, - &ip6h->daddr)) + &group.ip6)) goto found; #endif } @@ -2000,14 +2000,18 @@ static void amt_igmpv3_report_handler(struct amt_dev *amt, struct sk_buff *skb, struct igmpv3_report *ihrv3 = igmpv3_report_hdr(skb); int len = skb_transport_offset(skb) + sizeof(*ihrv3); void *zero_grec = (void *)&igmpv3_zero_grec; - struct iphdr *iph = ip_hdr(skb); struct amt_group_node *gnode; union amt_addr group, host; struct igmpv3_grec *grec; + __be32 saddr; u16 nsrcs; + u16 ngrec; int i; - for (i = 0; i < ntohs(ihrv3->ngrec); i++) { + saddr = ip_hdr(skb)->saddr; + ngrec = ntohs(ihrv3->ngrec); + + for (i = 0; i < ngrec; i++) { len += sizeof(*grec); if (!ip_mc_may_pull(skb, len)) break; @@ -2019,10 +2023,13 @@ static void amt_igmpv3_report_handler(struct amt_dev *amt, struct sk_buff *skb, if (!ip_mc_may_pull(skb, len)) break; + grec = (void *)(skb->data + len - sizeof(*grec) - + nsrcs * sizeof(__be32)); + memset(&group, 0, sizeof(union amt_addr)); group.ip4 = grec->grec_mca; memset(&host, 0, sizeof(union amt_addr)); - host.ip4 = iph->saddr; + host.ip4 = saddr; gnode = amt_lookup_group(tunnel, &group, &host, false); if (!gnode) { gnode = amt_add_group(amt, tunnel, &group, &host, @@ -2162,14 +2169,18 @@ static void amt_mldv2_report_handler(struct amt_dev *amt, struct sk_buff *skb, struct mld2_report *mld2r = (struct mld2_report *)icmp6_hdr(skb); int len = skb_transport_offset(skb) + sizeof(*mld2r); void *zero_grec = (void *)&mldv2_zero_grec; - struct ipv6hdr *ip6h = ipv6_hdr(skb); struct amt_group_node *gnode; union amt_addr group, host; struct mld2_grec *grec; + struct in6_addr saddr; u16 nsrcs; + u16 ngrec; int i; - for (i = 0; i < ntohs(mld2r->mld2r_ngrec); i++) { + saddr = ipv6_hdr(skb)->saddr; + ngrec = ntohs(mld2r->mld2r_ngrec); + + for (i = 0; i < ngrec; i++) { len += sizeof(*grec); if (!ipv6_mc_may_pull(skb, len)) break; @@ -2181,10 +2192,13 @@ static void amt_mldv2_report_handler(struct amt_dev *amt, struct sk_buff *skb, if (!ipv6_mc_may_pull(skb, len)) break; + grec = (void *)(skb->data + len - sizeof(*grec) - + nsrcs * sizeof(struct in6_addr)); + memset(&group, 0, sizeof(union amt_addr)); group.ip6 = grec->grec_mca; memset(&host, 0, sizeof(union amt_addr)); - host.ip6 = ip6h->saddr; + host.ip6 = saddr; gnode = amt_lookup_group(tunnel, &group, &host, true); if (!gnode) { gnode = amt_add_group(amt, tunnel, &group, &host, @@ -2305,7 +2319,9 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb) skb_push(skb, sizeof(*eth)); skb_reset_mac_header(skb); skb_pull(skb, sizeof(*eth)); - eth = eth_hdr(skb); + + if (skb_cow_head(skb, 0)) + return true; if (!pskb_may_pull(skb, sizeof(*iph))) return true; @@ -2315,6 +2331,7 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb) if (!ipv4_is_multicast(iph->daddr)) return true; skb->protocol = htons(ETH_P_IP); + eth = eth_hdr(skb); eth->h_proto = htons(ETH_P_IP); ip_eth_mc_map(iph->daddr, eth->h_dest); #if IS_ENABLED(CONFIG_IPV6) @@ -2328,6 +2345,7 @@ static bool amt_multicast_data_handler(struct amt_dev *amt, struct sk_buff *skb) if (!ipv6_addr_is_multicast(&ip6h->daddr)) return true; skb->protocol = htons(ETH_P_IPV6); + eth = eth_hdr(skb); eth->h_proto = htons(ETH_P_IPV6); ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest); #endif @@ -2351,10 +2369,12 @@ static bool amt_membership_query_handler(struct amt_dev *amt, struct sk_buff *skb) { struct amt_header_membership_query *amtmq; - struct igmpv3_query *ihv3; struct ethhdr *eth, *oeth; + struct igmpv3_query *ihv3; + u8 h_source[ETH_ALEN]; struct iphdr *iph; int hdr_size, len; + u64 response_mac; hdr_size = sizeof(*amtmq) + sizeof(struct udphdr); if (!pskb_may_pull(skb, hdr_size)) @@ -2367,6 +2387,8 @@ static bool amt_membership_query_handler(struct amt_dev *amt, if (amtmq->nonce != amt->nonce) return true; + response_mac = amtmq->response_mac; + hdr_size -= sizeof(*eth); if (iptunnel_pull_header(skb, hdr_size, htons(ETH_P_TEB), false)) return true; @@ -2376,6 +2398,9 @@ static bool amt_membership_query_handler(struct amt_dev *amt, skb_pull(skb, sizeof(*eth)); skb_reset_network_header(skb); eth = eth_hdr(skb); + ether_addr_copy(h_source, oeth->h_source); + if (skb_cow_head(skb, 0)) + return true; if (!pskb_may_pull(skb, sizeof(*iph))) return true; @@ -2388,6 +2413,7 @@ static bool amt_membership_query_handler(struct amt_dev *amt, sizeof(*ihv3))) return true; + iph = ip_hdr(skb); if (!ipv4_is_multicast(iph->daddr)) return true; @@ -2395,10 +2421,11 @@ static bool amt_membership_query_handler(struct amt_dev *amt, skb_reset_transport_header(skb); skb_push(skb, sizeof(*iph) + AMT_IPHDR_OPTS); WRITE_ONCE(amt->ready4, true); - amt->mac = amtmq->response_mac; + amt->mac = response_mac; amt->req_cnt = 0; amt->qi = ihv3->qqic; skb->protocol = htons(ETH_P_IP); + eth = eth_hdr(skb); eth->h_proto = htons(ETH_P_IP); ip_eth_mc_map(iph->daddr, eth->h_dest); #if IS_ENABLED(CONFIG_IPV6) @@ -2421,10 +2448,11 @@ static bool amt_membership_query_handler(struct amt_dev *amt, skb_reset_transport_header(skb); skb_push(skb, sizeof(*ip6h) + AMT_IP6HDR_OPTS); WRITE_ONCE(amt->ready6, true); - amt->mac = amtmq->response_mac; + amt->mac = response_mac; amt->req_cnt = 0; amt->qi = mld2q->mld2q_qqic; skb->protocol = htons(ETH_P_IPV6); + eth = eth_hdr(skb); eth->h_proto = htons(ETH_P_IPV6); ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest); #endif @@ -2432,7 +2460,7 @@ static bool amt_membership_query_handler(struct amt_dev *amt, return true; } - ether_addr_copy(eth->h_source, oeth->h_source); + ether_addr_copy(eth->h_source, h_source); skb->pkt_type = PACKET_MULTICAST; skb->ip_summed = CHECKSUM_NONE; len = skb->len; @@ -2455,8 +2483,11 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) struct ethhdr *eth; struct iphdr *iph; int len, hdr_size; + u64 response_mac; + __be32 saddr; + __be32 nonce; - iph = ip_hdr(skb); + saddr = ip_hdr(skb)->saddr; hdr_size = sizeof(*amtmu) + sizeof(struct udphdr); if (!pskb_may_pull(skb, hdr_size)) @@ -2466,15 +2497,18 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) if (amtmu->reserved || amtmu->version) return true; + nonce = amtmu->nonce; + response_mac = amtmu->response_mac; + if (iptunnel_pull_header(skb, hdr_size, skb->protocol, false)) return true; skb_reset_network_header(skb); list_for_each_entry_rcu(tunnel, &amt->tunnel_list, list) { - if (tunnel->ip4 == iph->saddr) { - if ((amtmu->nonce == tunnel->nonce && - amtmu->response_mac == tunnel->mac)) { + if (tunnel->ip4 == saddr) { + if ((nonce == tunnel->nonce && + response_mac == tunnel->mac)) { mod_delayed_work(amt_wq, &tunnel->gc_wq, msecs_to_jiffies(amt_gmi(amt)) * 3); @@ -2492,6 +2526,9 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) if (!pskb_may_pull(skb, sizeof(*iph))) return true; + if (skb_cow_head(skb, 0)) + return true; + iph = ip_hdr(skb); if (iph->version == 4) { if (ip_mc_check_igmp(skb)) { @@ -2508,6 +2545,7 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) eth = eth_hdr(skb); skb->protocol = htons(ETH_P_IP); eth->h_proto = htons(ETH_P_IP); + iph = ip_hdr(skb); ip_eth_mc_map(iph->daddr, eth->h_dest); #if IS_ENABLED(CONFIG_IPV6) } else if (iph->version == 6) { @@ -2527,6 +2565,7 @@ static bool amt_update_handler(struct amt_dev *amt, struct sk_buff *skb) eth = eth_hdr(skb); skb->protocol = htons(ETH_P_IPV6); eth->h_proto = htons(ETH_P_IPV6); + ip6h = ipv6_hdr(skb); ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest); #endif } else { @@ -2772,7 +2811,7 @@ static void amt_gw_rcv(struct amt_dev *amt, struct sk_buff *skb) static int amt_rcv(struct sock *sk, struct sk_buff *skb) { struct amt_dev *amt; - struct iphdr *iph; + __be32 saddr; int type; bool err; @@ -2785,7 +2824,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) } skb->dev = amt->dev; - iph = ip_hdr(skb); + saddr = ip_hdr(skb)->saddr; type = amt_parse_type(skb); if (type == -1) { err = true; @@ -2795,7 +2834,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) if (amt->mode == AMT_MODE_GATEWAY) { switch (type) { case AMT_MSG_ADVERTISEMENT: - if (iph->saddr != amt->discovery_ip) { + if (saddr != amt->discovery_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); err = true; goto drop; @@ -2807,7 +2846,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) } goto out; case AMT_MSG_MULTICAST_DATA: - if (iph->saddr != amt->remote_ip) { + if (saddr != amt->remote_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); err = true; goto drop; @@ -2818,7 +2857,7 @@ static int amt_rcv(struct sock *sk, struct sk_buff *skb) else goto out; case AMT_MSG_MEMBERSHIP_QUERY: - if (iph->saddr != amt->remote_ip) { + if (saddr != amt->remote_ip) { netdev_dbg(amt->dev, "Invalid Relay IP\n"); err = true; goto drop; @@ -2995,9 +3034,15 @@ static int amt_dev_open(struct net_device *dev) amt->event_idx = 0; amt->nr_events = 0; + enable_delayed_work(&amt->discovery_wq); + enable_delayed_work(&amt->req_wq); + err = amt_socket_create(amt); - if (err) + if (err) { + disable_delayed_work(&amt->req_wq); + disable_delayed_work(&amt->discovery_wq); return err; + } amt->req_cnt = 0; amt->remote_ip = 0; @@ -3023,8 +3068,8 @@ static int amt_dev_stop(struct net_device *dev) struct sk_buff *skb; int i; - cancel_delayed_work_sync(&amt->req_wq); - cancel_delayed_work_sync(&amt->discovery_wq); + disable_delayed_work_sync(&amt->req_wq); + disable_delayed_work_sync(&amt->discovery_wq); cancel_delayed_work_sync(&amt->secret_wq); /* shutdown */ @@ -3176,6 +3221,9 @@ static int amt_newlink(struct net_device *dev, struct nlattr **tb = params->tb; int err = -EINVAL; + if (!net_eq(link_net, dev_net(dev))) + return err; + amt->net = link_net; amt->mode = nla_get_u32(data[IFLA_AMT_MODE]); @@ -3278,6 +3326,8 @@ static int amt_newlink(struct net_device *dev, INIT_DELAYED_WORK(&amt->req_wq, amt_req_work); INIT_DELAYED_WORK(&amt->secret_wq, amt_secret_work); INIT_WORK(&amt->event_wq, amt_event_work); + disable_delayed_work(&amt->req_wq); + disable_delayed_work(&amt->discovery_wq); INIT_LIST_HEAD(&amt->tunnel_list); return 0; err: diff --git a/drivers/net/bonding/bond_alb.c b/drivers/net/bonding/bond_alb.c index 2d37b07c8215c..120673c8967fc 100644 --- a/drivers/net/bonding/bond_alb.c +++ b/drivers/net/bonding/bond_alb.c @@ -678,9 +678,15 @@ static struct slave *rlb_arp_xmit(struct sk_buff *skb, struct bonding *bond) if (arp->op_code == htons(ARPOP_REPLY)) { /* the arp must be sent on the selected rx channel */ tx_slave = rlb_choose_channel(skb, bond, arp); - if (tx_slave) + if (tx_slave && + !ether_addr_equal_64bits(arp->mac_src, + tx_slave->dev->dev_addr)) { + if (unlikely(skb_cow_head(skb, 0))) + return NULL; + arp = (struct arp_pkt *)skb_network_header(skb); bond_hw_addr_copy(arp->mac_src, tx_slave->dev->dev_addr, tx_slave->dev->addr_len); + } netdev_dbg(bond->dev, "(slave %s): Server sent ARP Reply packet\n", tx_slave ? tx_slave->dev->name : "NULL"); } else if (arp->op_code == htons(ARPOP_REQUEST)) { @@ -1281,10 +1287,10 @@ static int alb_set_mac_address(struct bonding *bond, void *addr) } /* determine if the packet is NA or NS */ -static bool alb_determine_nd(struct sk_buff *skb, struct bonding *bond) +static bool alb_determine_nd(struct sk_buff *skb) { - struct ipv6hdr *ip6hdr; - struct icmp6hdr *hdr; + const struct ipv6hdr *ip6hdr; + const struct icmp6hdr *hdr; if (!pskb_network_may_pull(skb, sizeof(*ip6hdr))) return true; @@ -1296,7 +1302,8 @@ static bool alb_determine_nd(struct sk_buff *skb, struct bonding *bond) if (!pskb_network_may_pull(skb, sizeof(*ip6hdr) + sizeof(*hdr))) return true; - hdr = icmp6_hdr(skb); + ip6hdr = ipv6_hdr(skb); + hdr = (const struct icmp6hdr *)(ip6hdr + 1); return hdr->icmp6_type == NDISC_NEIGHBOUR_ADVERTISEMENT || hdr->icmp6_type == NDISC_NEIGHBOUR_SOLICITATION; } @@ -1339,7 +1346,6 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, struct slave *tx_slave) { struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond)); - struct ethhdr *eth_data = eth_hdr(skb); if (!tx_slave) { /* unbalanced or unassigned, send through primary */ @@ -1350,7 +1356,9 @@ static netdev_tx_t bond_do_alb_xmit(struct sk_buff *skb, struct bonding *bond, if (tx_slave && bond_slave_can_tx(tx_slave)) { if (tx_slave != rcu_access_pointer(bond->curr_active_slave)) { - ether_addr_copy(eth_data->h_source, + if (unlikely(skb_cow_head(skb, 0))) + return bond_tx_drop(bond->dev, skb); + ether_addr_copy(skb_eth_hdr(skb)->h_source, tx_slave->dev->dev_addr); } @@ -1374,14 +1382,13 @@ struct slave *bond_xmit_tlb_slave_get(struct bonding *bond, struct ethhdr *eth_data; u32 hash_index; - skb_reset_mac_header(skb); - eth_data = eth_hdr(skb); + eth_data = skb_eth_hdr(skb); /* Do not TX balance any multicast or broadcast */ if (!is_multicast_ether_addr(eth_data->h_dest)) { switch (skb->protocol) { case htons(ETH_P_IPV6): - if (alb_determine_nd(skb, bond)) + if (alb_determine_nd(skb)) break; fallthrough; case htons(ETH_P_IP): @@ -1427,8 +1434,7 @@ struct slave *bond_xmit_alb_slave_get(struct bonding *bond, u32 hash_index = 0; int hash_size = 0; - skb_reset_mac_header(skb); - eth_data = eth_hdr(skb); + eth_data = skb_eth_hdr(skb); switch (ntohs(skb->protocol)) { case ETH_P_IP: { @@ -1467,7 +1473,7 @@ struct slave *bond_xmit_alb_slave_get(struct bonding *bond, break; } - if (alb_determine_nd(skb, bond)) { + if (alb_determine_nd(skb)) { do_tx_balance = false; break; } @@ -1534,8 +1540,8 @@ void bond_alb_monitor(struct work_struct *work) struct bonding *bond = container_of(work, struct bonding, alb_work.work); struct alb_bond_info *bond_info = &(BOND_ALB_INFO(bond)); + struct slave *slave, *curr; struct list_head *iter; - struct slave *slave; if (!bond_has_slaves(bond)) { atomic_set(&bond_info->tx_rebalance_counter, 0); @@ -1597,9 +1603,11 @@ void bond_alb_monitor(struct work_struct *work) * because a slave was disabled then * it can now leave promiscuous mode. */ - dev_set_promiscuity(rtnl_dereference(bond->curr_active_slave)->dev, - -1); - bond_info->primary_is_promisc = 0; + curr = rtnl_dereference(bond->curr_active_slave); + if (bond_info->primary_is_promisc && curr) { + dev_set_promiscuity(curr->dev, -1); + bond_info->primary_is_promisc = 0; + } rtnl_unlock(); rcu_read_lock(); diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index e38cae22fe025..6b9509e939289 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -2481,9 +2481,7 @@ static int __bond_release_one(struct net_device *bond_dev, bond_alb_deinit_slave(bond, slave); } - if (all) { - RCU_INIT_POINTER(bond->curr_active_slave, NULL); - } else if (oldcurrent == slave) { + if (!all && oldcurrent == slave) { /* Note that we hold RTNL over this sequence, so there * is no concern that another slave add/remove event * will interfere. @@ -3422,7 +3420,8 @@ static void bond_send_validate(struct bonding *bond, struct slave *slave) { bond_arp_send_all(bond, slave); #if IS_ENABLED(CONFIG_IPV6) - bond_ns_send_all(bond, slave); + if (likely(ipv6_mod_enabled())) + bond_ns_send_all(bond, slave); #endif } diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 08d4d0814f67a..4367cac1f4e80 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -220,7 +220,7 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], struct bonding *bond = netdev_priv(bond_dev); struct bond_opt_value newval; int miimon = 0; - int err; + int err = 0; if (!data) return 0; diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c index cc371d0c9f3c7..2ea5224ebfcb5 100644 --- a/drivers/net/can/c_can/c_can_main.c +++ b/drivers/net/can/c_can/c_can_main.c @@ -597,20 +597,20 @@ static int c_can_chip_config(struct net_device *dev) return err; /* enable automatic retransmission */ - priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_ENABLE_AR); + priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_ENABLE_AR | CONTROL_INIT); if ((priv->can.ctrlmode & CAN_CTRLMODE_LISTENONLY) && (priv->can.ctrlmode & CAN_CTRLMODE_LOOPBACK)) { /* loopback + silent mode : useful for hot self-test */ - priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST); + priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST | CONTROL_INIT); priv->write_reg(priv, C_CAN_TEST_REG, TEST_LBACK | TEST_SILENT); } else if (priv->can.ctrlmode & CAN_CTRLMODE_LOOPBACK) { /* loopback mode : useful for self-test function */ - priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST); + priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST | CONTROL_INIT); priv->write_reg(priv, C_CAN_TEST_REG, TEST_LBACK); } else if (priv->can.ctrlmode & CAN_CTRLMODE_LISTENONLY) { /* silent mode : bus-monitoring mode */ - priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST); + priv->write_reg(priv, C_CAN_CTRL_REG, CONTROL_TEST | CONTROL_INIT); priv->write_reg(priv, C_CAN_TEST_REG, TEST_SILENT); } diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c index 33a5ec4346278..178fa05bac66e 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_base.c +++ b/drivers/net/can/ctucanfd/ctucanfd_base.c @@ -340,8 +340,8 @@ static void ctucan_set_mode(struct ctucan_priv *priv, const struct can_ctrlmode (mode_reg & ~REG_MODE_FDE); mode_reg = (mode->flags & CAN_CTRLMODE_PRESUME_ACK) ? - (mode_reg | REG_MODE_ACF) : - (mode_reg & ~REG_MODE_ACF); + (mode_reg | REG_MODE_STM) : + (mode_reg & ~REG_MODE_STM); mode_reg = (mode->flags & CAN_CTRLMODE_FD_NON_ISO) ? (mode_reg | REG_MODE_NISOFD) : @@ -869,7 +869,7 @@ static void ctucan_err_interrupt(struct net_device *ndev, u32 isr) break; case CAN_STATE_ERROR_ACTIVE: if (skb) { - cf->can_id |= CAN_ERR_CNT; + cf->can_id |= CAN_ERR_CRTL | CAN_ERR_CNT; cf->data[1] = CAN_ERR_CRTL_ACTIVE; cf->data[6] = bec.txerr; cf->data[7] = bec.rxerr; @@ -1136,8 +1136,12 @@ static irqreturn_t ctucan_interrupt(int irq, void *dev_id) /* Error interrupts */ if (FIELD_GET(REG_INT_STAT_EWLI, isr) || FIELD_GET(REG_INT_STAT_FCSI, isr) || - FIELD_GET(REG_INT_STAT_ALI, isr)) { - icr = isr & (REG_INT_STAT_EWLI | REG_INT_STAT_FCSI | REG_INT_STAT_ALI); + FIELD_GET(REG_INT_STAT_ALI, isr) || + FIELD_GET(REG_INT_STAT_BEI, isr)) { + icr = isr & (REG_INT_STAT_EWLI | + REG_INT_STAT_FCSI | + REG_INT_STAT_ALI | + REG_INT_STAT_BEI); ctucan_netdev_dbg(ndev, "some ERR interrupt: clearing 0x%08x\n", icr); ctucan_write32(priv, CTUCANFD_INT_STAT, icr); diff --git a/drivers/net/can/ctucanfd/ctucanfd_pci.c b/drivers/net/can/ctucanfd/ctucanfd_pci.c index 9da09e7dd63a0..8cc9956492874 100644 --- a/drivers/net/can/ctucanfd/ctucanfd_pci.c +++ b/drivers/net/can/ctucanfd/ctucanfd_pci.c @@ -202,7 +202,7 @@ static int ctucan_pci_probe(struct pci_dev *pdev, pci_set_drvdata(pdev, NULL); kfree(bdata); err_pci_iounmap_bar0: - pci_iounmap(pdev, cra_addr); + pci_iounmap(pdev, bar0_base); err_pci_iounmap_bar1: pci_iounmap(pdev, addr); err_release_regions: @@ -274,6 +274,7 @@ static const struct pci_device_id ctucan_pci_tbl[] = { CTUCAN_WITH_CTUCAN_ID)}, {}, }; +MODULE_DEVICE_TABLE(pci, ctucan_pci_tbl); static struct pci_driver ctucan_pci_driver = { .name = KBUILD_MODNAME, diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c index 3ebd4f779b9bd..0da615afa04de 100644 --- a/drivers/net/can/dev/skb.c +++ b/drivers/net/can/dev/skb.c @@ -202,7 +202,6 @@ static void init_can_skb_reserve(struct sk_buff *skb) skb_reset_transport_header(skb); can_skb_reserve(skb); - can_skb_prv(skb)->skbcnt = 0; } struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf) @@ -312,7 +311,6 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb) if (skb->ip_summed == CHECKSUM_NONE) { /* init headroom */ can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; skb->ip_summed = CHECKSUM_UNNECESSARY; diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c index 4c0d7d26df9f3..45af1d121a845 100644 --- a/drivers/net/can/rcar/rcar_canfd.c +++ b/drivers/net/can/rcar/rcar_canfd.c @@ -468,6 +468,7 @@ struct rcar_canfd_global { struct platform_device *pdev; /* Respective platform device */ struct clk *clkp; /* Peripheral clock */ struct clk *can_clk; /* fCAN clock */ + struct clk *clk_ram; /* Clock RAM */ unsigned long channels_mask; /* Enabled channels mask */ bool extclk; /* CANFD or Ext clock */ bool fdmode; /* CAN FD or Classical CAN only mode */ @@ -1970,22 +1971,120 @@ static void rcar_canfd_channel_remove(struct rcar_canfd_global *gpriv, u32 ch) } } +static int rcar_canfd_global_init(struct rcar_canfd_global *gpriv) +{ + struct device *dev = &gpriv->pdev->dev; + u32 rule_entry = 0; + u32 ch, sts; + int err; + + /* Enable peripheral clock for register access */ + err = clk_prepare_enable(gpriv->clkp); + if (err) { + dev_err(dev, "failed to enable peripheral clock: %pe\n", + ERR_PTR(err)); + return err; + } + + /* Enable RAM clock */ + err = clk_prepare_enable(gpriv->clk_ram); + if (err) { + dev_err(dev, + "failed to enable RAM clock, error %d\n", err); + goto fail_clk; + } + + err = reset_control_reset(gpriv->rstc1); + if (err) + goto fail_ram_clk; + + err = reset_control_reset(gpriv->rstc2); + if (err) + goto fail_reset1; + + err = rcar_canfd_reset_controller(gpriv); + if (err) { + dev_err(dev, "reset controller failed: %pe\n", ERR_PTR(err)); + goto fail_reset2; + } + + /* Controller in Global reset & Channel reset mode */ + rcar_canfd_configure_controller(gpriv); + + /* Configure per channel attributes */ + for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { + /* Configure Channel's Rx fifo */ + rcar_canfd_configure_rx(gpriv, ch); + + /* Configure Channel's Tx (Common) fifo */ + rcar_canfd_configure_tx(gpriv, ch); + + /* Configure receive rules */ + rcar_canfd_configure_afl_rules(gpriv, ch, rule_entry); + rule_entry += RCANFD_CHANNEL_NUMRULES; + } + + /* Configure common interrupts */ + rcar_canfd_enable_global_interrupts(gpriv); + + /* Start Global operation mode */ + rcar_canfd_update_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GMDC_MASK, + RCANFD_GCTR_GMDC_GOPM); + + /* Verify mode change */ + err = readl_poll_timeout((gpriv->base + RCANFD_GSTS), sts, + !(sts & RCANFD_GSTS_GNOPM), 2, 500000); + if (err) { + dev_err(dev, "global operational mode failed\n"); + goto fail_mode; + } + + return 0; + +fail_mode: + rcar_canfd_disable_global_interrupts(gpriv); +fail_reset2: + reset_control_assert(gpriv->rstc2); +fail_reset1: + reset_control_assert(gpriv->rstc1); +fail_ram_clk: + clk_disable_unprepare(gpriv->clk_ram); +fail_clk: + clk_disable_unprepare(gpriv->clkp); + return err; +} + +static void rcar_canfd_global_deinit(struct rcar_canfd_global *gpriv, bool full) +{ + rcar_canfd_disable_global_interrupts(gpriv); + + if (full) { + rcar_canfd_reset_controller(gpriv); + + /* Enter global sleep mode */ + rcar_canfd_set_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GSLPR); + } + + reset_control_assert(gpriv->rstc2); + reset_control_assert(gpriv->rstc1); + clk_disable_unprepare(gpriv->clk_ram); + clk_disable_unprepare(gpriv->clkp); +} + static int rcar_canfd_probe(struct platform_device *pdev) { struct phy *transceivers[RCANFD_NUM_CHANNELS] = { NULL, }; const struct rcar_canfd_hw_info *info; struct device *dev = &pdev->dev; void __iomem *addr; - u32 sts, ch, fcan_freq; struct rcar_canfd_global *gpriv; struct device_node *of_child; unsigned long channels_mask = 0; int err, ch_irq, g_irq; int g_err_irq, g_recc_irq; - u32 rule_entry = 0; bool fdmode = true; /* CAN FD only mode - default */ char name[9] = "channelX"; - struct clk *clk_ram; + u32 ch, fcan_freq; int i; info = of_device_get_match_data(dev); @@ -2075,10 +2174,10 @@ static int rcar_canfd_probe(struct platform_device *pdev) gpriv->extclk = gpriv->info->external_clk; } - clk_ram = devm_clk_get_optional_enabled(dev, "ram_clk"); - if (IS_ERR(clk_ram)) - return dev_err_probe(dev, PTR_ERR(clk_ram), - "cannot get enabled ram clock\n"); + gpriv->clk_ram = devm_clk_get_optional(dev, "ram_clk"); + if (IS_ERR(gpriv->clk_ram)) + return dev_err_probe(dev, PTR_ERR(gpriv->clk_ram), + "cannot get ram clock\n"); addr = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(addr)) { @@ -2127,59 +2226,9 @@ static int rcar_canfd_probe(struct platform_device *pdev) } } - err = reset_control_reset(gpriv->rstc1); + err = rcar_canfd_global_init(gpriv); if (err) - goto fail_dev; - err = reset_control_reset(gpriv->rstc2); - if (err) { - reset_control_assert(gpriv->rstc1); - goto fail_dev; - } - - /* Enable peripheral clock for register access */ - err = clk_prepare_enable(gpriv->clkp); - if (err) { - dev_err(dev, "failed to enable peripheral clock: %pe\n", - ERR_PTR(err)); - goto fail_reset; - } - - err = rcar_canfd_reset_controller(gpriv); - if (err) { - dev_err(dev, "reset controller failed: %pe\n", ERR_PTR(err)); - goto fail_clk; - } - - /* Controller in Global reset & Channel reset mode */ - rcar_canfd_configure_controller(gpriv); - - /* Configure per channel attributes */ - for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) { - /* Configure Channel's Rx fifo */ - rcar_canfd_configure_rx(gpriv, ch); - - /* Configure Channel's Tx (Common) fifo */ - rcar_canfd_configure_tx(gpriv, ch); - - /* Configure receive rules */ - rcar_canfd_configure_afl_rules(gpriv, ch, rule_entry); - rule_entry += RCANFD_CHANNEL_NUMRULES; - } - - /* Configure common interrupts */ - rcar_canfd_enable_global_interrupts(gpriv); - - /* Start Global operation mode */ - rcar_canfd_update_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GMDC_MASK, - RCANFD_GCTR_GMDC_GOPM); - - /* Verify mode change */ - err = readl_poll_timeout((gpriv->base + RCANFD_GSTS), sts, - !(sts & RCANFD_GSTS_GNOPM), 2, 500000); - if (err) { - dev_err(dev, "global operational mode failed\n"); goto fail_mode; - } for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) { err = rcar_canfd_channel_probe(gpriv, ch, fcan_freq, @@ -2198,12 +2247,7 @@ static int rcar_canfd_probe(struct platform_device *pdev) for_each_set_bit(ch, &gpriv->channels_mask, info->max_channels) rcar_canfd_channel_remove(gpriv, ch); fail_mode: - rcar_canfd_disable_global_interrupts(gpriv); -fail_clk: - clk_disable_unprepare(gpriv->clkp); -fail_reset: - reset_control_assert(gpriv->rstc1); - reset_control_assert(gpriv->rstc2); + rcar_canfd_global_deinit(gpriv, false); fail_dev: return err; } @@ -2213,19 +2257,12 @@ static void rcar_canfd_remove(struct platform_device *pdev) struct rcar_canfd_global *gpriv = platform_get_drvdata(pdev); u32 ch; - rcar_canfd_reset_controller(gpriv); - rcar_canfd_disable_global_interrupts(gpriv); - for_each_set_bit(ch, &gpriv->channels_mask, gpriv->info->max_channels) { rcar_canfd_disable_channel_interrupts(gpriv->ch[ch]); rcar_canfd_channel_remove(gpriv, ch); } - /* Enter global sleep mode */ - rcar_canfd_set_bit(gpriv->base, RCANFD_GCTR, RCANFD_GCTR_GSLPR); - clk_disable_unprepare(gpriv->clkp); - reset_control_assert(gpriv->rstc1); - reset_control_assert(gpriv->rstc2); + rcar_canfd_global_deinit(gpriv, true); } static int __maybe_unused rcar_canfd_suspend(struct device *dev) diff --git a/drivers/net/can/softing/softing_fw.c b/drivers/net/can/softing/softing_fw.c index 721df91cdbfbc..282570daf3efe 100644 --- a/drivers/net/can/softing/softing_fw.c +++ b/drivers/net/can/softing/softing_fw.c @@ -91,12 +91,12 @@ int softing_bootloader_command(struct softing *card, int16_t cmd, return ret; } -static int fw_parse(const uint8_t **pmem, uint16_t *ptype, uint32_t *paddr, - uint16_t *plen, const uint8_t **pdat) +static int fw_parse(const u8 **pmem, const u8 *limit, u16 *ptype, + u32 *paddr, u16 *plen, const u8 **pdat) { uint16_t checksum[2]; - const uint8_t *mem; - const uint8_t *end; + const u8 *mem; + const u8 *record_end; /* * firmware records are a binary, unaligned stream composed of: @@ -114,14 +114,21 @@ static int fw_parse(const uint8_t **pmem, uint16_t *ptype, uint32_t *paddr, * endianness & alignment. */ mem = *pmem; + /* A record needs an 8-byte prefix and a 2-byte checksum. */ + if (mem > limit || limit - mem < 10) + return -EINVAL; + *ptype = le16_to_cpup((void *)&mem[0]); *paddr = le32_to_cpup((void *)&mem[2]); *plen = le16_to_cpup((void *)&mem[6]); + if (*plen > limit - mem - 10) + return -EINVAL; + *pdat = &mem[8]; /* verify checksum */ - end = &mem[8 + *plen]; - checksum[0] = le16_to_cpup((void *)end); - for (checksum[1] = 0; mem < end; ++mem) + record_end = &mem[8 + *plen]; + checksum[0] = le16_to_cpup((void *)record_end); + for (checksum[1] = 0; mem < record_end; ++mem) checksum[1] += *mem; if (checksum[0] != checksum[1]) return -EINVAL; @@ -139,6 +146,7 @@ int softing_load_fw(const char *file, struct softing *card, uint16_t type, len; uint32_t addr; uint8_t *buf = NULL, *new_buf; + s64 dpram_offset; int buflen = 0; int8_t type_end = 0; @@ -153,7 +161,7 @@ int softing_load_fw(const char *file, struct softing *card, mem = fw->data; end = &mem[fw->size]; /* look for header record */ - ret = fw_parse(&mem, &type, &addr, &len, &dat); + ret = fw_parse(&mem, end, &type, &addr, &len, &dat); if (ret < 0) goto failed; if (type != 0xffff) @@ -164,7 +172,7 @@ int softing_load_fw(const char *file, struct softing *card, } /* ok, we had a header */ while (mem < end) { - ret = fw_parse(&mem, &type, &addr, &len, &dat); + ret = fw_parse(&mem, end, &type, &addr, &len, &dat); if (ret < 0) goto failed; if (type == 3) { @@ -179,9 +187,13 @@ int softing_load_fw(const char *file, struct softing *card, goto failed; } - if ((addr + len + offset) > size) + dpram_offset = (s64)addr + offset; + if (dpram_offset < 0 || dpram_offset > size || + len > size - dpram_offset) { + ret = -EINVAL; goto failed; - memcpy_toio(&dpram[addr + offset], dat, len); + } + memcpy_toio(&dpram[dpram_offset], dat, len); /* be sure to flush caches from IO space */ mb(); if (len > buflen) { @@ -195,7 +207,7 @@ int softing_load_fw(const char *file, struct softing *card, buf = new_buf; } /* verify record data */ - memcpy_fromio(buf, &dpram[addr + offset], len); + memcpy_fromio(buf, &dpram[dpram_offset], len); if (memcmp(buf, dat, len)) { /* is not ok */ dev_alert(&card->pdev->dev, "DPRAM readback failed\n"); @@ -237,7 +249,7 @@ int softing_load_app_fw(const char *file, struct softing *card) mem = fw->data; end = &mem[fw->size]; /* look for header record */ - ret = fw_parse(&mem, &type, &addr, &len, &dat); + ret = fw_parse(&mem, end, &type, &addr, &len, &dat); if (ret) goto failed; ret = -EINVAL; @@ -253,7 +265,7 @@ int softing_load_app_fw(const char *file, struct softing *card) } /* ok, we had a header */ while (mem < end) { - ret = fw_parse(&mem, &type, &addr, &len, &dat); + ret = fw_parse(&mem, end, &type, &addr, &len, &dat); if (ret) goto failed; @@ -279,6 +291,12 @@ int softing_load_app_fw(const char *file, struct softing *card) /* work in 16bit (target) */ sum &= 0xffff; + if (card->pdat->app.offs > card->dpram_size || + len > card->dpram_size - card->pdat->app.offs) { + ret = -EINVAL; + goto failed; + } + memcpy_toio(&card->dpram[card->pdat->app.offs], dat, len); iowrite32(card->pdat->app.offs + card->pdat->app.addr, &card->dpram[DPRAM_COMMAND + 2]); diff --git a/drivers/net/can/usb/ems_usb.c b/drivers/net/can/usb/ems_usb.c index d8c881130e900..4b130c96c9ab2 100644 --- a/drivers/net/can/usb/ems_usb.c +++ b/drivers/net/can/usb/ems_usb.c @@ -409,6 +409,40 @@ static void ems_usb_rx_err(struct ems_usb *dev, struct ems_cpc_msg *msg) netif_rx(skb); } +static bool ems_usb_rx_msg_len_valid(struct ems_cpc_msg *msg) +{ + size_t len = msg->length; + size_t can_len; + + switch (msg->type) { + case CPC_MSG_TYPE_CAN_STATE: + return len >= sizeof(msg->msg.can_state); + + case CPC_MSG_TYPE_CAN_FRAME: + case CPC_MSG_TYPE_EXT_CAN_FRAME: + case CPC_MSG_TYPE_RTR_FRAME: + case CPC_MSG_TYPE_EXT_RTR_FRAME: + if (len < CPC_CAN_MSG_MIN_SIZE) + return false; + + if (msg->type == CPC_MSG_TYPE_RTR_FRAME || + msg->type == CPC_MSG_TYPE_EXT_RTR_FRAME) + return true; + + can_len = can_cc_dlc2len(msg->msg.can_msg.length & 0xf); + return len >= CPC_CAN_MSG_MIN_SIZE + can_len; + + case CPC_MSG_TYPE_CAN_FRAME_ERROR: + return len >= sizeof(msg->msg.error); + + case CPC_MSG_TYPE_OVERRUN: + return len >= sizeof(msg->msg.overrun); + + default: + return true; + } +} + /* * callback for bulk IN urb */ @@ -451,6 +485,15 @@ static void ems_usb_read_bulk_callback(struct urb *urb) } msg = (struct ems_cpc_msg *)&ibuf[start]; + if (msg->length > + urb->actual_length - start - CPC_MSG_HEADER_LEN) { + netdev_err(netdev, "format error\n"); + break; + } + if (!ems_usb_rx_msg_len_valid(msg)) { + netdev_err(netdev, "format error\n"); + break; + } switch (msg->type) { case CPC_MSG_TYPE_CAN_STATE: diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.c b/drivers/net/can/usb/etas_es58x/es58x_core.c index 1e44bd1dfee9b..70d9cd11cb855 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_core.c +++ b/drivers/net/can/usb/etas_es58x/es58x_core.c @@ -1476,7 +1476,6 @@ static void es58x_read_bulk_callback(struct urb *urb) dev_err_ratelimited(dev, "Failed resubmitting read bulk urb: %pe\n", ERR_PTR(ret)); - return; free_urb: usb_free_coherent(urb->dev, urb->transfer_buffer_length, diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c index e336703b941ef..241ffa8daaee5 100644 --- a/drivers/net/can/usb/gs_usb.c +++ b/drivers/net/can/usb/gs_usb.c @@ -674,7 +674,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) if (hf->flags & GS_CAN_FLAG_FD) { skb = alloc_canfd_skb(netdev, &cfd); if (!skb) - return; + goto resubmit_urb; cfd->can_id = le32_to_cpu(hf->can_id); cfd->len = data_length; @@ -687,7 +687,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb) } else { skb = alloc_can_skb(netdev, &cf); if (!skb) - return; + goto resubmit_urb; cf->can_id = le32_to_cpu(hf->can_id); can_frame_set_cc_len(cf, hf->can_dlc, dev->can.ctrlmode); diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c index a59f20dad692a..6f8d3bc42733c 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c @@ -1626,6 +1626,7 @@ static int kvaser_usb_hydra_get_busparams(struct kvaser_usb_net_priv *priv, reinit_completion(&priv->get_busparams_comp); err = kvaser_usb_send_cmd(dev, cmd, cmd_len); + kfree(cmd); if (err) return err; diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c index 1167d38344f1d..c9322f279e3e8 100644 --- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c +++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_leaf.c @@ -691,13 +691,22 @@ static int kvaser_usb_leaf_wait_cmd(const struct kvaser_usb *dev, u8 id, continue; } - if (pos + tmp->len > actual_len) { + if (tmp->len < CMD_HEADER_LEN || + tmp->len > actual_len - pos) { dev_err_ratelimited(&dev->intf->dev, "Format error\n"); break; } if (tmp->id == id) { + if (tmp->len > sizeof(*cmd)) { + dev_err_ratelimited(&dev->intf->dev, + "Received command %u too large (%u)\n", + tmp->id, tmp->len); + err = -EIO; + goto end; + } + memcpy(cmd, tmp, tmp->len); goto end; } @@ -1737,7 +1746,7 @@ static void kvaser_usb_leaf_read_bulk_callback(struct kvaser_usb *dev, continue; } - if (pos + cmd->len > len) { + if (cmd->len < CMD_HEADER_LEN || cmd->len > len - pos) { dev_err_ratelimited(&dev->intf->dev, "Format error\n"); break; } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c index c74302ca7cee9..529e9cb2570fe 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c @@ -470,7 +470,6 @@ static int peak_usb_start(struct peak_usb_device *dev) netif_device_detach(dev->netdev); usb_unanchor_urb(urb); - kfree(buf); usb_free_urb(urb); break; } diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c index be84191cde56f..f92a3b404ec18 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_fd.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_fd.c @@ -566,6 +566,13 @@ static int pcan_usb_fd_decode_canmsg(struct pcan_usb_fd_if *usb_if, dev->can.ctrlmode); } + if (!(rx_msg_flags & PUCAN_MSG_RTR) && + le16_to_cpu(rx_msg->size) - offsetof(struct pucan_rx_msg, d) < + cfd->len) { + kfree_skb(skb); + return -EBADMSG; + } + cfd->can_id = le32_to_cpu(rm->can_id); if (rx_msg_flags & PUCAN_MSG_EXT_ID) @@ -714,6 +721,24 @@ static void pcan_usb_fd_decode_ts(struct pcan_usb_fd_if *usb_if, peak_usb_set_ts_now(&usb_if->time_ref, le32_to_cpu(ts->ts_low)); } +static size_t pcan_usb_fd_rx_msg_min_size(u16 rx_msg_type) +{ + switch (rx_msg_type) { + case PUCAN_MSG_CAN_RX: + return offsetof(struct pucan_rx_msg, d); + case PCAN_UFD_MSG_CALIBRATION: + return sizeof(struct pcan_ufd_ts_msg); + case PUCAN_MSG_ERROR: + return sizeof(struct pucan_error_msg); + case PUCAN_MSG_STATUS: + return sizeof(struct pucan_status_msg); + case PCAN_UFD_MSG_OVERRUN: + return sizeof(struct pcan_ufd_ovr_msg); + default: + return sizeof(struct pucan_msg); + } +} + /* callback for bulk IN urb */ static int pcan_usb_fd_decode_buf(struct peak_usb_device *dev, struct urb *urb) { @@ -728,6 +753,12 @@ static int pcan_usb_fd_decode_buf(struct peak_usb_device *dev, struct urb *urb) msg_end = urb->transfer_buffer + urb->actual_length; for (; msg_ptr < msg_end;) { u16 rx_msg_type, rx_msg_size; + size_t rx_msg_min_size; + + if (msg_end - msg_ptr < sizeof(*rx_msg)) { + err = -EBADMSG; + break; + } rx_msg = (struct pucan_msg *)msg_ptr; if (!rx_msg->size) { @@ -739,13 +770,20 @@ static int pcan_usb_fd_decode_buf(struct peak_usb_device *dev, struct urb *urb) rx_msg_type = le16_to_cpu(rx_msg->type); /* check if the record goes out of current packet */ - if (msg_ptr + rx_msg_size > msg_end) { + if (rx_msg_size > msg_end - msg_ptr) { netdev_err(netdev, "got frag rec: should inc usb rx buf sze\n"); err = -EBADMSG; break; } + rx_msg_min_size = pcan_usb_fd_rx_msg_min_size(rx_msg_type); + if (rx_msg_size < rx_msg_min_size) { + netdev_err(netdev, "got short rec\n"); + err = -EBADMSG; + break; + } + switch (rx_msg_type) { case PUCAN_MSG_CAN_RX: err = pcan_usb_fd_decode_canmsg(usb_if, rx_msg); diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c index 7be286293b1a2..c6023acd05e5a 100644 --- a/drivers/net/can/usb/peak_usb/pcan_usb_pro.c +++ b/drivers/net/can/usb/peak_usb/pcan_usb_pro.c @@ -534,12 +534,18 @@ static int pcan_usb_pro_handle_canmsg(struct pcan_usb_pro_interface *usb_if, struct pcan_usb_pro_rxmsg *rx) { const unsigned int ctrl_idx = (rx->len >> 4) & 0x0f; - struct peak_usb_device *dev = usb_if->dev[ctrl_idx]; - struct net_device *netdev = dev->netdev; + struct peak_usb_device *dev; + struct net_device *netdev; struct can_frame *can_frame; struct sk_buff *skb; struct skb_shared_hwtstamps *hwts; + if (ctrl_idx >= ARRAY_SIZE(usb_if->dev)) + return -EINVAL; + + dev = usb_if->dev[ctrl_idx]; + netdev = dev->netdev; + skb = alloc_can_skb(netdev, &can_frame); if (!skb) return -ENOMEM; @@ -573,14 +579,20 @@ static int pcan_usb_pro_handle_error(struct pcan_usb_pro_interface *usb_if, { const u16 raw_status = le16_to_cpu(er->status); const unsigned int ctrl_idx = (er->channel >> 4) & 0x0f; - struct peak_usb_device *dev = usb_if->dev[ctrl_idx]; - struct net_device *netdev = dev->netdev; + struct peak_usb_device *dev; + struct net_device *netdev; struct can_frame *can_frame; enum can_state new_state = CAN_STATE_ERROR_ACTIVE; u8 err_mask = 0; struct sk_buff *skb; struct skb_shared_hwtstamps *hwts; + if (ctrl_idx >= ARRAY_SIZE(usb_if->dev)) + return -EINVAL; + + dev = usb_if->dev[ctrl_idx]; + netdev = dev->netdev; + /* nothing should be sent while in BUS_OFF state */ if (dev->can.state == CAN_STATE_BUS_OFF) return 0; diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index 7d6ec2eb7c75e..508b46237011d 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -2204,7 +2204,7 @@ int b53_fdb_dump(struct dsa_switch *ds, int port, mutex_unlock(&priv->arl_mutex); - return 0; + return ret; } EXPORT_SYMBOL(b53_fdb_dump); diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c index e22362e6f0cd1..7b5b747bb1311 100644 --- a/drivers/net/dsa/bcm_sf2_cfp.c +++ b/drivers/net/dsa/bcm_sf2_cfp.c @@ -1088,6 +1088,8 @@ static int bcm_sf2_cfp_rule_get_all(struct bcm_sf2_priv *priv, unsigned int index = 1, rules_cnt = 0; for_each_set_bit_from(index, priv->cfp.unique, priv->num_cfp_rules) { + if (rules_cnt == nfc->rule_cnt) + return -EMSGSIZE; rule_locs[rules_cnt] = index; rules_cnt++; } diff --git a/drivers/net/dsa/lantiq/lantiq_gswip.h b/drivers/net/dsa/lantiq/lantiq_gswip.h index 2df9c8e8cfd0b..a15ca4e6d0726 100644 --- a/drivers/net/dsa/lantiq/lantiq_gswip.h +++ b/drivers/net/dsa/lantiq/lantiq_gswip.h @@ -41,7 +41,7 @@ #define GSWIP_MDIO_PHY_FDUP_DIS 0x0600 #define GSWIP_MDIO_PHY_FCONTX_MASK 0x0180 #define GSWIP_MDIO_PHY_FCONTX_AUTO 0x0000 -#define GSWIP_MDIO_PHY_FCONTX_EN 0x0100 +#define GSWIP_MDIO_PHY_FCONTX_EN 0x0080 #define GSWIP_MDIO_PHY_FCONTX_DIS 0x0180 #define GSWIP_MDIO_PHY_FCONRX_MASK 0x0060 #define GSWIP_MDIO_PHY_FCONRX_AUTO 0x0000 diff --git a/drivers/net/dsa/mt7530-mdio.c b/drivers/net/dsa/mt7530-mdio.c index 0286a6cecb6f5..6cc2c8975af66 100644 --- a/drivers/net/dsa/mt7530-mdio.c +++ b/drivers/net/dsa/mt7530-mdio.c @@ -55,8 +55,15 @@ mt7530_regmap_read(void *context, unsigned int reg, unsigned int *val) if (ret < 0) return ret; - lo = bus->read(bus, priv->mdiodev->addr, r); - hi = bus->read(bus, priv->mdiodev->addr, 0x10); + ret = bus->read(bus, priv->mdiodev->addr, r); + if (ret < 0) + return ret; + lo = ret; + + ret = bus->read(bus, priv->mdiodev->addr, 0x10); + if (ret < 0) + return ret; + hi = ret; *val = (hi << 16) | (lo & 0xffff); diff --git a/drivers/net/dsa/mt7530-mmio.c b/drivers/net/dsa/mt7530-mmio.c index 1dc8b93fb51a1..1065671bcdd13 100644 --- a/drivers/net/dsa/mt7530-mmio.c +++ b/drivers/net/dsa/mt7530-mmio.c @@ -13,6 +13,7 @@ static const struct of_device_id mt7988_of_match[] = { { .compatible = "airoha,an7583-switch", .data = &mt753x_table[ID_AN7583], }, { .compatible = "airoha,en7581-switch", .data = &mt753x_table[ID_EN7581], }, + { .compatible = "econet,en7528-switch", .data = &mt753x_table[ID_EN7528], }, { .compatible = "mediatek,mt7988-switch", .data = &mt753x_table[ID_MT7988], }, { /* sentinel */ }, }; diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 4571da0d7a8f6..d7ebb7b8ee72c 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -188,12 +188,6 @@ mt7530_write(struct mt7530_priv *priv, u32 reg, u32 val) mt7530_mutex_unlock(priv); } -static u32 -_mt7530_unlocked_read(struct mt7530_dummy_poll *p) -{ - return mt7530_mii_read(p->priv, p->reg); -} - static u32 _mt7530_read(struct mt7530_dummy_poll *p) { @@ -546,16 +540,13 @@ static int mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad, int regnum) { - struct mt7530_dummy_poll p; u32 reg, val; int ret; - INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC); - mt7530_mutex_lock(priv); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -565,8 +556,8 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad, MT7531_MDIO_DEV_ADDR(devad) | regnum; mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -576,8 +567,8 @@ mt7531_ind_c45_phy_read(struct mt7530_priv *priv, int port, int devad, MT7531_MDIO_DEV_ADDR(devad); mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -594,16 +585,13 @@ static int mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad, int regnum, u16 data) { - struct mt7530_dummy_poll p; u32 val, reg; int ret; - INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC); - mt7530_mutex_lock(priv); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -613,8 +601,8 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad, MT7531_MDIO_DEV_ADDR(devad) | regnum; mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -624,8 +612,8 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad, MT7531_MDIO_DEV_ADDR(devad) | data; mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -640,16 +628,13 @@ mt7531_ind_c45_phy_write(struct mt7530_priv *priv, int port, int devad, static int mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum) { - struct mt7530_dummy_poll p; int ret; u32 val; - INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC); - mt7530_mutex_lock(priv); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -660,8 +645,8 @@ mt7531_ind_c22_phy_read(struct mt7530_priv *priv, int port, int regnum) mt7530_mii_write(priv, MT7531_PHY_IAC, val | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, val, - !(val & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, val, + !(val & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -678,16 +663,13 @@ static int mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum, u16 data) { - struct mt7530_dummy_poll p; int ret; u32 reg; - INIT_MT7530_DUMMY_POLL(&p, priv, MT7531_PHY_IAC); - mt7530_mutex_lock(priv); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg, - !(reg & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, reg, + !(reg & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -698,8 +680,8 @@ mt7531_ind_c22_phy_write(struct mt7530_priv *priv, int port, int regnum, mt7530_mii_write(priv, MT7531_PHY_IAC, reg | MT7531_PHY_ACS_ST); - ret = readx_poll_timeout(_mt7530_unlocked_read, &p, reg, - !(reg & MT7531_PHY_ACS_ST), 20, 100000); + ret = regmap_read_poll_timeout(priv->regmap, MT7531_PHY_IAC, reg, + !(reg & MT7531_PHY_ACS_ST), 20, 100000); if (ret < 0) { dev_err(priv->dev, "poll timeout\n"); goto out; @@ -2847,6 +2829,30 @@ static void en7581_mac_port_get_caps(struct dsa_switch *ds, int port, } } +static void en7528_mac_port_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) +{ + switch (port) { + /* Ports which are connected to switch PHYs. There is no MII pinout. */ + case 1 ... 4: + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + + config->mac_capabilities |= MAC_10 | MAC_100 | MAC_1000FD; + break; + + /* Port 6 is connected to SoC's GMAC at 1000 Mbps full duplex. There + * is no MII pinout. + */ + case 6: + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + + config->mac_capabilities |= MAC_1000FD; + break; + } +} + static void mt7530_mac_config(struct dsa_switch *ds, int port, unsigned int mode, phy_interface_t interface) @@ -3036,19 +3042,34 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port, struct phylink_config *config) { struct mt7530_priv *priv = ds->priv; - u32 eeecr; config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE; - config->lpi_capabilities = MAC_100FD | MAC_1000FD | MAC_2500FD; + priv->info->mac_port_get_caps(ds, port, config); - eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port)); - /* tx_lpi_timer should be in microseconds. The time units for - * LPI threshold are unspecified. + /* The EN7528 GPHYs report EEE capability, but negotiating EEE with + * common link partners (e.g. Realtek GbE NICs) results in an unstable + * link with dropped frames. Leave the LPI capabilities empty so that + * phylink disables EEE on these PHYs and refuses to enable it from + * userspace. */ - config->lpi_timer_default = FIELD_GET(LPI_THRESH_MASK, eeecr); - - priv->info->mac_port_get_caps(ds, port, config); + if (priv->id != ID_EN7528 && + config->mac_capabilities & (MAC_100FD | MAC_1000FD)) { + u32 eeecr = mt7530_read(priv, MT753X_PMEEECR_P(port)); + + /* LPI above 1 Gbps is not supported */ + config->lpi_capabilities = config->mac_capabilities & + (MAC_100FD | MAC_1000FD); + phy_interface_copy(config->lpi_interfaces, + config->supported_interfaces); + __clear_bit(PHY_INTERFACE_MODE_2500BASEX, + config->lpi_interfaces); + + /* tx_lpi_timer should be in microseconds. The time units for + * LPI threshold are unspecified. + */ + config->lpi_timer_default = FIELD_GET(LPI_THRESH_MASK, eeecr); + } } static int mt753x_pcs_validate(struct phylink_pcs *pcs, @@ -3174,7 +3195,8 @@ mt753x_conduit_state_change(struct dsa_switch *ds, * forwarded to the numerically smallest CPU port whose conduit * interface is up. */ - if (priv->id != ID_MT7530 && priv->id != ID_MT7621) + if (priv->id != ID_MT7530 && priv->id != ID_MT7621 && + priv->id != ID_EN7528) return; mask = BIT(cpu_dp->index); @@ -3377,6 +3399,16 @@ const struct mt753x_info mt753x_table[] = { .phy_write_c45 = mt7531_ind_c45_phy_write, .mac_port_get_caps = en7581_mac_port_get_caps, }, + [ID_EN7528] = { + .id = ID_EN7528, + .pcs_ops = &mt7530_pcs_ops, + .sw_setup = mt7988_setup, + .phy_read_c22 = mt7531_ind_c22_phy_read, + .phy_write_c22 = mt7531_ind_c22_phy_write, + .phy_read_c45 = mt7531_ind_c45_phy_read, + .phy_write_c45 = mt7531_ind_c45_phy_write, + .mac_port_get_caps = en7528_mac_port_get_caps, + }, }; EXPORT_SYMBOL_GPL(mt753x_table); diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h index 7e47cd9af2564..69be713bab800 100644 --- a/drivers/net/dsa/mt7530.h +++ b/drivers/net/dsa/mt7530.h @@ -21,6 +21,7 @@ enum mt753x_id { ID_MT7988 = 3, ID_EN7581 = 4, ID_AN7583 = 5, + ID_EN7528 = 6, }; #define NUM_TRGMII_CTRL 5 diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 09002c853b78e..e9270c266884c 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -2439,6 +2439,7 @@ static int mv88e6xxx_get_rxnfc(struct dsa_switch *ds, int port, struct ethtool_rx_flow_spec *fs = &rxnfc->fs; struct mv88e6xxx_chip *chip = ds->priv; struct mv88e6xxx_policy *policy; + u32 cnt = 0; int err; int id; @@ -2464,11 +2465,18 @@ static int mv88e6xxx_get_rxnfc(struct dsa_switch *ds, int port, break; case ETHTOOL_GRXCLSRLALL: rxnfc->data = 0; - rxnfc->rule_cnt = 0; - idr_for_each_entry(&chip->policies, policy, id) - if (policy->port == port) - rule_locs[rxnfc->rule_cnt++] = id; err = 0; + idr_for_each_entry(&chip->policies, policy, id) { + if (policy->port != port) + continue; + if (cnt == rxnfc->rule_cnt) { + err = -EMSGSIZE; + break; + } + rule_locs[cnt++] = id; + } + if (!err) + rxnfc->rule_cnt = cnt; break; default: err = -EOPNOTSUPP; @@ -5179,6 +5187,7 @@ static const struct mv88e6xxx_ops mv88e6320_ops = { .hardware_reset_pre = mv88e6xxx_g2_eeprom_wait, .hardware_reset_post = mv88e6xxx_g2_eeprom_wait, .reset = mv88e6352_g1_reset, + .rmu_disable = mv88e6352_g1_rmu_disable, .vtu_getnext = mv88e6352_g1_vtu_getnext, .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, @@ -5228,9 +5237,11 @@ static const struct mv88e6xxx_ops mv88e6321_ops = { .set_egress_port = mv88e6095_g1_set_egress_port, .watchdog_ops = &mv88e6390_watchdog_ops, .mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu, + .pot_clear = mv88e6xxx_g2_pot_clear, .hardware_reset_pre = mv88e6xxx_g2_eeprom_wait, .hardware_reset_post = mv88e6xxx_g2_eeprom_wait, .reset = mv88e6352_g1_reset, + .rmu_disable = mv88e6352_g1_rmu_disable, .vtu_getnext = mv88e6352_g1_vtu_getnext, .vtu_loadpurge = mv88e6352_g1_vtu_loadpurge, .stu_getnext = mv88e6352_g1_stu_getnext, @@ -6261,7 +6272,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .global1_addr = 0x1b, .global2_addr = 0x1c, .age_time_coeff = 15000, - .g1_irqs = 8, + .g1_irqs = 9, .g2_irqs = 10, .stats_type = STATS_TYPE_BANK0 | STATS_TYPE_BANK1, .atu_move_port_mask = 0xf, @@ -6289,7 +6300,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = { .global1_addr = 0x1b, .global2_addr = 0x1c, .age_time_coeff = 15000, - .g1_irqs = 8, + .g1_irqs = 9, .g2_irqs = 10, .stats_type = STATS_TYPE_BANK0 | STATS_TYPE_BANK1, .atu_move_port_mask = 0xf, diff --git a/drivers/net/dsa/mv88e6xxx/pcs-6352.c b/drivers/net/dsa/mv88e6xxx/pcs-6352.c index 36993400837ea..e09019ed19fb6 100644 --- a/drivers/net/dsa/mv88e6xxx/pcs-6352.c +++ b/drivers/net/dsa/mv88e6xxx/pcs-6352.c @@ -305,13 +305,16 @@ static bool mv88e6352_pcs_link_check(struct marvell_c22_pcs *mpcs) struct mv88e6xxx_port *port = mpcs->port; struct mv88e6xxx_chip *chip = port->chip; u8 cmode; + int err; /* Port 4 can be in auto-media mode. Check that the port is * associated with the mpcs. */ mv88e6xxx_reg_lock(chip); - chip->info->ops->port_get_cmode(chip, port->port, &cmode); + err = chip->info->ops->port_get_cmode(chip, port->port, &cmode); mv88e6xxx_reg_unlock(chip); + if (err) + return false; return cmode == MV88E6XXX_PORT_STS_CMODE_100BASEX || cmode == MV88E6XXX_PORT_STS_CMODE_1000BASEX || diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c index a36b8b07030e3..4c928983b8623 100644 --- a/drivers/net/dsa/qca/qca8k-8xxx.c +++ b/drivers/net/dsa/qca/qca8k-8xxx.c @@ -1538,7 +1538,7 @@ static int qca8k_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, { struct qca8k_priv *priv = pcs_to_qca8k_pcs(pcs)->priv; int cpu_port_index, ret, port; - u32 reg, val; + u32 mask, reg, val; port = pcs_to_qca8k_pcs(pcs)->port; switch (port) { @@ -1611,11 +1611,21 @@ static int qca8k_pcs_config(struct phylink_pcs *pcs, unsigned int neg_mode, if (priv->ports_config.sgmii_tx_clk_falling_edge) val |= QCA8K_PORT0_PAD_SGMII_TXCLK_FALLING_EDGE; - if (val) - ret = qca8k_rmw(priv, reg, - QCA8K_PORT0_PAD_SGMII_RXCLK_FALLING_EDGE | - QCA8K_PORT0_PAD_SGMII_TXCLK_FALLING_EDGE, - val); + mask = (val) ? (QCA8K_PORT0_PAD_SGMII_RXCLK_FALLING_EDGE | + QCA8K_PORT0_PAD_SGMII_TXCLK_FALLING_EDGE) : 0; + + /* + * (Un)set force mode on QCA8337 only, don't include it in the mask for + * others. It is written to the PORT0 PAD register for both port 0 and 6. + */ + if (priv->switch_id == QCA8K_ID_QCA8337) { + if (neg_mode == PHYLINK_PCS_NEG_OUTBAND) + val |= QCA8K_PORT_PAD_SGMII_FORCE_MODE; + mask |= QCA8K_PORT_PAD_SGMII_FORCE_MODE; + } + + if (mask) + ret = qca8k_rmw(priv, reg, mask, val); return 0; } diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h index d046679265fa4..ed8f0b49812c4 100644 --- a/drivers/net/dsa/qca/qca8k.h +++ b/drivers/net/dsa/qca/qca8k.h @@ -58,6 +58,7 @@ #define QCA8K_PORT_PAD_RGMII_TX_DELAY_EN BIT(25) #define QCA8K_PORT_PAD_RGMII_RX_DELAY_EN BIT(24) #define QCA8K_PORT_PAD_SGMII_EN BIT(7) +#define QCA8K_PORT_PAD_SGMII_FORCE_MODE BIT(3) #define QCA8K_REG_PWS 0x010 #define QCA8K_PWS_POWER_ON_SEL BIT(31) /* This reg is only valid for QCA832x and toggle the package diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c index e10a789e22022..c806a24dcd031 100644 --- a/drivers/net/dsa/realtek/rtl8365mb.c +++ b/drivers/net/dsa/realtek/rtl8365mb.c @@ -544,6 +544,20 @@ static const struct rtl8365mb_chip_info rtl8365mb_chip_infos[] = { .jam_table = rtl8365mb_init_jam_8365mb_vc, .jam_size = ARRAY_SIZE(rtl8365mb_init_jam_8365mb_vc), }, + { + .name = "RTL8367SB", + .chip_id = 0x6367, + .chip_ver = 0x0010, + .extints = { + { 6, 1, PHY_INTF(MII) | PHY_INTF(TMII) | + PHY_INTF(RMII) | PHY_INTF(RGMII) | + PHY_INTF(SGMII) | PHY_INTF(HSGMII) }, + { 7, 2, PHY_INTF(MII) | PHY_INTF(TMII) | + PHY_INTF(RMII) | PHY_INTF(RGMII) }, + }, + .jam_table = rtl8365mb_init_jam_8365mb_vc, + .jam_size = ARRAY_SIZE(rtl8365mb_init_jam_8365mb_vc), + }, { .name = "RTL8367RB-VB", .chip_id = 0x6367, diff --git a/drivers/net/dsa/realtek/rtl83xx.c b/drivers/net/dsa/realtek/rtl83xx.c index 2b9bd4462714b..0df45c758beb4 100644 --- a/drivers/net/dsa/realtek/rtl83xx.c +++ b/drivers/net/dsa/realtek/rtl83xx.c @@ -309,7 +309,7 @@ void rtl83xx_reset_assert(struct realtek_priv *priv) "Failed to assert the switch reset control: %pe\n", ERR_PTR(ret)); - gpiod_set_value(priv->reset, true); + gpiod_set_value_cansleep(priv->reset, true); } void rtl83xx_reset_deassert(struct realtek_priv *priv) @@ -322,7 +322,7 @@ void rtl83xx_reset_deassert(struct realtek_priv *priv) "Failed to deassert the switch reset control: %pe\n", ERR_PTR(ret)); - gpiod_set_value(priv->reset, false); + gpiod_set_value_cansleep(priv->reset, false); } MODULE_AUTHOR("Luiz Angelo Daros de Luca "); diff --git a/drivers/net/dsa/sja1105/sja1105_flower.c b/drivers/net/dsa/sja1105/sja1105_flower.c index 05d8ed3121e7f..172fc163e1850 100644 --- a/drivers/net/dsa/sja1105/sja1105_flower.c +++ b/drivers/net/dsa/sja1105/sja1105_flower.c @@ -391,9 +391,9 @@ int sja1105_cls_flower_add(struct dsa_switch *ds, int port, struct dsa_port *to_dp; to_dp = dsa_port_from_netdev(act->dev); - if (IS_ERR(to_dp)) { + if (IS_ERR(to_dp) || to_dp->ds != ds) { NL_SET_ERR_MSG_MOD(extack, - "Destination not a switch port"); + "Destination not a local switch port"); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 5efd03c2deea7..e016f7521af59 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -568,9 +568,10 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) q->queued++; nframes++; + offset += AIROHA_RX_HEADROOM; e->buf = page_address(page) + offset; e->dma_addr = page_pool_get_dma_addr(page) + offset; - e->dma_len = SKB_WITH_OVERHEAD(q->buf_size); + e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size)); val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len); WRITE_ONCE(desc->ctrl, cpu_to_le32(val)); @@ -635,13 +636,12 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) q->tail = (q->tail + 1) % q->ndesc; q->queued--; - dma_sync_single_for_cpu(eth->dev, e->dma_addr, - SKB_WITH_OVERHEAD(q->buf_size), dir); + dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len, + dir); page = virt_to_head_page(e->buf); len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); - data_len = q->skb ? q->buf_size - : SKB_WITH_OVERHEAD(q->buf_size); + data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len; if (!len || data_len < len) goto free_frag; @@ -651,10 +651,12 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) port = eth->ports[p]; if (!q->skb) { /* first buffer */ - q->skb = napi_build_skb(e->buf, q->buf_size); + q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM, + q->buf_size); if (!q->skb) goto free_frag; + skb_reserve(q->skb, AIROHA_RX_HEADROOM); __skb_put(q->skb, len); skb_mark_for_recycle(q->skb); q->skb->dev = port->dev; @@ -2326,8 +2328,7 @@ static int airoha_tc_setup_qdisc_ets(struct airoha_gdm_port *port, if (opt->parent == TC_H_ROOT) return -EINVAL; - channel = TC_H_MAJ(opt->handle) >> 16; - channel = channel % AIROHA_NUM_QOS_CHANNELS; + channel = TC_H_MIN(opt->parent) % AIROHA_NUM_QOS_CHANNELS; switch (opt->command) { case TC_ETS_REPLACE: diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index 57e8ddb30a9c5..ee0899104c3ba 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -32,6 +32,8 @@ #define AIROHA_FE_MC_MAX_VLAN_TABLE 64 #define AIROHA_FE_MC_MAX_VLAN_PORT 16 #define AIROHA_NUM_TX_IRQ 2 +#define AIROHA_RX_HEADROOM (NET_SKB_PAD + NET_IP_ALIGN) +#define AIROHA_RX_LEN(_n) ((_n) - AIROHA_RX_HEADROOM) #define HW_DSCP_NUM 2048 #define IRQ_QUEUE_LEN(_n) ((_n) ? 1024 : 2048) #define TX_DSCP_NUM 1024 @@ -500,7 +502,7 @@ struct airoha_wdma_info { /* RX queue to IRQ mapping: BIT(q) in IRQ(n) */ #define RX_IRQ0_BANK_PIN_MASK 0x839f -#define RX_IRQ1_BANK_PIN_MASK 0x7fe00000 +#define RX_IRQ1_BANK_PIN_MASK 0xffe00000 #define RX_IRQ2_BANK_PIN_MASK 0x20 #define RX_IRQ3_BANK_PIN_MASK 0x40 #define RX_IRQ_BANK_PIN_MASK(_n) \ diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c index 8c883f2b2d36b..ddc5a6a7681b1 100644 --- a/drivers/net/ethernet/airoha/airoha_npu.c +++ b/drivers/net/ethernet/airoha/airoha_npu.c @@ -154,7 +154,7 @@ static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id, dma_addr_t dma_addr; int ret; - dma_addr = dma_map_single(npu->dev, p, size, DMA_TO_DEVICE); + dma_addr = dma_map_single(npu->dev, p, size, DMA_BIDIRECTIONAL); ret = dma_mapping_error(npu->dev, dma_addr); if (ret) return ret; @@ -177,7 +177,7 @@ static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id, spin_unlock_bh(&npu->cores[core].lock); - dma_unmap_single(npu->dev, dma_addr, size, DMA_TO_DEVICE); + dma_unmap_single(npu->dev, dma_addr, size, DMA_BIDIRECTIONAL); return ret; } @@ -686,7 +686,7 @@ static int airoha_npu_probe(struct platform_device *pdev) npu->irqs[i] = irq; } - err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); if (err) return err; diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index 798839aa1010b..8defd194568f2 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -1575,6 +1575,7 @@ void airoha_ppe_deinit(struct airoha_eth *eth) npu = rcu_replace_pointer(eth->npu, NULL, lockdep_is_held(&flow_offload_mutex)); if (npu) { + synchronize_rcu(); npu->ops.ppe_deinit(npu); airoha_npu_put(npu); } diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c index 2f516b950f4ea..de4a3f1976216 100644 --- a/drivers/net/ethernet/allwinner/sun4i-emac.c +++ b/drivers/net/ethernet/allwinner/sun4i-emac.c @@ -996,9 +996,9 @@ static int emac_probe(struct platform_device *pdev) /* fill in parameters for net-dev structure */ ndev->base_addr = (unsigned long)db->membase; ndev->irq = irq_of_parse_and_map(np, 0); - if (ndev->irq == -ENXIO) { + if (!ndev->irq) { netdev_err(ndev, "No irq resource\n"); - ret = ndev->irq; + ret = -ENXIO; goto out_iounmap; } @@ -1067,6 +1067,7 @@ static int emac_probe(struct platform_device *pdev) return 0; out_release_sram: + of_node_put(db->phy_node); sunxi_sram_release(&pdev->dev); out_clk_disable_unprepare: clk_disable_unprepare(db->clk); @@ -1094,6 +1095,7 @@ static void emac_remove(struct platform_device *pdev) } unregister_netdev(ndev); + of_node_put(db->phy_node); sunxi_sram_release(&pdev->dev); clk_disable_unprepare(db->clk); irq_dispose_mapping(ndev->irq); diff --git a/drivers/net/ethernet/amd/au1000_eth.c b/drivers/net/ethernet/amd/au1000_eth.c index 9d35ac348ebe3..5a04056e38fa3 100644 --- a/drivers/net/ethernet/amd/au1000_eth.c +++ b/drivers/net/ethernet/amd/au1000_eth.c @@ -943,9 +943,10 @@ static int au1000_close(struct net_device *dev) /* stop the device */ netif_stop_queue(dev); + spin_unlock_irqrestore(&aup->lock, flags); + /* disable the interrupt */ free_irq(dev->irq, dev); - spin_unlock_irqrestore(&aup->lock, flags); return 0; } diff --git a/drivers/net/ethernet/amd/pds_core/adminq.c b/drivers/net/ethernet/amd/pds_core/adminq.c index 097bb092bdb8c..eadb4b604fbe3 100644 --- a/drivers/net/ethernet/amd/pds_core/adminq.c +++ b/drivers/net/ethernet/amd/pds_core/adminq.c @@ -18,7 +18,13 @@ static int pdsc_process_notifyq(struct pdsc_qcq *qcq) comp = cq_info->comp; eid = le64_to_cpu(comp->event.eid); while (eid > pdsc->last_eid) { - u16 ecode = le16_to_cpu(comp->event.ecode); + u16 ecode; + + /* Order the payload read after the event id, the field the + * driver uses to detect a new completion. + */ + dma_rmb(); + ecode = le16_to_cpu(comp->event.ecode); switch (ecode) { case PDS_EVENT_LINK_CHANGE: @@ -101,6 +107,10 @@ void pdsc_process_adminq(struct pdsc_qcq *qcq) spin_lock_irqsave(&pdsc->adminq_lock, irqflags); comp = cq->info[cq->tail_idx].comp; while (pdsc_color_match(comp->color, cq->done_color)) { + /* Order the payload reads after the color bit, the field the + * driver uses to detect a new completion. + */ + dma_rmb(); q_info = &q->info[q->tail_idx]; q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); diff --git a/drivers/net/ethernet/amd/pds_core/auxbus.c b/drivers/net/ethernet/amd/pds_core/auxbus.c index 92f359f2b4492..874812bafc1d3 100644 --- a/drivers/net/ethernet/amd/pds_core/auxbus.c +++ b/drivers/net/ethernet/amd/pds_core/auxbus.c @@ -177,17 +177,21 @@ void pdsc_auxbus_dev_del(struct pdsc *cf, struct pdsc *pf, { struct pds_auxiliary_dev *padev; - if (!*pd_ptr) - return; - mutex_lock(&pf->config_lock); + /* A concurrent del may have already torn this device down and + * cleared it. + */ padev = *pd_ptr; + if (!padev) + goto out_unlock; + pds_client_unregister(pf, padev->client_id); auxiliary_device_delete(&padev->aux_dev); auxiliary_device_uninit(&padev->aux_dev); *pd_ptr = NULL; +out_unlock: mutex_unlock(&pf->config_lock); } @@ -210,6 +214,13 @@ int pdsc_auxbus_dev_add(struct pdsc *cf, struct pdsc *pf, mutex_lock(&pf->config_lock); + /* Nothing to do if the aux device is already present. This also + * guards against a second add overwriting *pd_ptr and leaking the + * first, symmetric with the check in pdsc_auxbus_dev_del(). + */ + if (*pd_ptr) + goto out_unlock; + mask = BIT_ULL(PDSC_S_FW_DEAD) | BIT_ULL(PDSC_S_STOPPING_DRIVER); if (cf->state & mask) { diff --git a/drivers/net/ethernet/amd/pds_core/core.c b/drivers/net/ethernet/amd/pds_core/core.c index 076dfe2910c77..c8b64c79fde58 100644 --- a/drivers/net/ethernet/amd/pds_core/core.c +++ b/drivers/net/ethernet/amd/pds_core/core.c @@ -110,7 +110,6 @@ static void pdsc_qcq_intr_free(struct pdsc *pdsc, struct pdsc_qcq *qcq) return; pdsc_intr_free(pdsc, qcq->intx); - qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED; } static int pdsc_qcq_intr_alloc(struct pdsc *pdsc, struct pdsc_qcq *qcq) @@ -145,6 +144,12 @@ void pdsc_qcq_free(struct pdsc *pdsc, struct pdsc_qcq *qcq) pdsc_qcq_intr_free(pdsc, qcq); + /* Drain any work queued by ISR before it was freed above */ + if (qcq->work.func) + cancel_work_sync(&qcq->work); + + qcq->intx = PDS_CORE_INTR_INDEX_NOT_ASSIGNED; + if (qcq->q_base) dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa); @@ -304,8 +309,11 @@ int pdsc_qcq_alloc(struct pdsc *pdsc, unsigned int type, unsigned int index, static void pdsc_core_uninit(struct pdsc *pdsc) { - pdsc_qcq_free(pdsc, &pdsc->notifyqcq); + /* Free adminqcq first: its work accesses notifyqcq, so we must + * disable its IRQ and drain its work before freeing notifyqcq. + */ pdsc_qcq_free(pdsc, &pdsc->adminqcq); + pdsc_qcq_free(pdsc, &pdsc->notifyqcq); if (pdsc->kern_dbpage) { iounmap(pdsc->kern_dbpage); @@ -447,6 +455,8 @@ int pdsc_setup(struct pdsc *pdsc, bool init) { int err; + pci_set_master(pdsc->pdev); + err = pdsc_dev_init(pdsc); if (err) return err; @@ -478,8 +488,8 @@ void pdsc_teardown(struct pdsc *pdsc, bool removing) { if (!pdsc->pdev->is_virtfn) pdsc_devcmd_reset(pdsc); - if (pdsc->adminqcq.work.func) - cancel_work_sync(&pdsc->adminqcq.work); + + pci_clear_master(pdsc->pdev); pdsc_core_uninit(pdsc); @@ -530,6 +540,7 @@ static void pdsc_adminq_wait_and_dec_once_unused(struct pdsc *pdsc) dev_dbg_ratelimited(pdsc->dev, "%s: adminq in use\n", __func__); cpu_relax(); + cond_resched(); } } @@ -603,9 +614,10 @@ void pdsc_pci_reset_thread(struct work_struct *work) struct pdsc *pdsc = container_of(work, struct pdsc, pci_reset_work); struct pci_dev *pdev = pdsc->pdev; - pci_dev_get(pdev); - pci_reset_function(pdev); - pci_dev_put(pdev); + /* Use try variant to avoid deadlock with pdsc_remove(). + * If lock is contended, the watchdog timer will retry. + */ + pci_try_reset_function(pdev); } static void pdsc_check_pci_health(struct pdsc *pdsc) diff --git a/drivers/net/ethernet/amd/pds_core/core.h b/drivers/net/ethernet/amd/pds_core/core.h index 0b53a1fab46d0..e2a86ef581dfc 100644 --- a/drivers/net/ethernet/amd/pds_core/core.h +++ b/drivers/net/ethernet/amd/pds_core/core.h @@ -171,6 +171,7 @@ struct pdsc { struct timer_list wdtimer; unsigned int wdtimer_period; struct work_struct health_work; + bool health_stopped; struct devlink_health_reporter *fw_reporter; u32 fw_recoveries; diff --git a/drivers/net/ethernet/amd/pds_core/devlink.c b/drivers/net/ethernet/amd/pds_core/devlink.c index 621791a3c543b..e35572f099c88 100644 --- a/drivers/net/ethernet/amd/pds_core/devlink.c +++ b/drivers/net/ethernet/amd/pds_core/devlink.c @@ -89,6 +89,12 @@ int pdsc_dl_flash_update(struct devlink *dl, { struct pdsc *pdsc = devlink_priv(dl); + if (params->component) { + NL_SET_ERR_MSG_MOD(extack, + "Component update not supported by this device"); + return -EOPNOTSUPP; + } + return pdsc_firmware_update(pdsc, params->fw, extack); } diff --git a/drivers/net/ethernet/amd/pds_core/fw.c b/drivers/net/ethernet/amd/pds_core/fw.c index fa626719e68d1..59d262b0d29c5 100644 --- a/drivers/net/ethernet/amd/pds_core/fw.c +++ b/drivers/net/ethernet/amd/pds_core/fw.c @@ -107,8 +107,10 @@ int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw, dev_info(pdsc->dev, "Installing firmware\n"); - if (!pdsc->cmd_regs) + if (!pdsc->cmd_regs) { + NL_SET_ERR_MSG_MOD(extack, "BARs not mapped"); return -ENXIO; + } dl = priv_to_devlink(pdsc); devlink_flash_update_status_notify(dl, "Preparing to flash", @@ -134,6 +136,12 @@ int pdsc_firmware_update(struct pdsc *pdsc, const struct firmware *fw, copy_sz = min_t(unsigned int, buf_sz, fw->size - offset); mutex_lock(&pdsc->devcmd_lock); + if (!pdsc->cmd_regs) { + mutex_unlock(&pdsc->devcmd_lock); + err = -ENXIO; + NL_SET_ERR_MSG_MOD(extack, "Device reset during flash"); + goto err_out; + } memcpy_toio(&pdsc->cmd_regs->data, fw->data + offset, copy_sz); err = pdsc_devcmd_fw_download_locked(pdsc, data_addr, offset, copy_sz); diff --git a/drivers/net/ethernet/amd/pds_core/main.c b/drivers/net/ethernet/amd/pds_core/main.c index c7a2eff576325..886b862b54b02 100644 --- a/drivers/net/ethernet/amd/pds_core/main.c +++ b/drivers/net/ethernet/amd/pds_core/main.c @@ -239,6 +239,10 @@ static int pdsc_init_pf(struct pdsc *pdsc) /* General workqueue and timer, but don't start timer yet */ snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); pdsc->wq = create_singlethread_workqueue(wq_name); + if (!pdsc->wq) { + err = -ENOMEM; + goto err_out_unmap_bars; + } INIT_WORK(&pdsc->health_work, pdsc_health_thread); INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread); timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); @@ -254,7 +258,7 @@ static int pdsc_init_pf(struct pdsc *pdsc) err = pdsc_setup(pdsc, PDSC_SETUP_INIT); if (err) { mutex_unlock(&pdsc->config_lock); - goto err_out_unmap_bars; + goto err_out_shutdown_timer; } err = pdsc_start(pdsc); @@ -306,13 +310,14 @@ static int pdsc_init_pf(struct pdsc *pdsc) pdsc_stop(pdsc); err_out_teardown: pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); -err_out_unmap_bars: +err_out_shutdown_timer: timer_shutdown_sync(&pdsc->wdtimer); if (pdsc->wq) destroy_workqueue(pdsc->wq); mutex_destroy(&pdsc->config_lock); mutex_destroy(&pdsc->devcmd_lock); pci_free_irq_vectors(pdsc->pdev); +err_out_unmap_bars: pdsc_unmap_bars(pdsc); err_out_release_regions: pci_release_regions(pdsc->pdev); @@ -436,8 +441,6 @@ static void pdsc_remove(struct pci_dev *pdev) pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); timer_shutdown_sync(&pdsc->wdtimer); - if (pdsc->wq) - destroy_workqueue(pdsc->wq); mutex_lock(&pdsc->config_lock); set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); @@ -445,6 +448,9 @@ static void pdsc_remove(struct pci_dev *pdev) pdsc_stop(pdsc); pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); mutex_unlock(&pdsc->config_lock); + + if (pdsc->wq) + destroy_workqueue(pdsc->wq); mutex_destroy(&pdsc->config_lock); mutex_destroy(&pdsc->devcmd_lock); @@ -465,8 +471,10 @@ static void pdsc_stop_health_thread(struct pdsc *pdsc) return; timer_shutdown_sync(&pdsc->wdtimer); - if (pdsc->health_work.func) - cancel_work_sync(&pdsc->health_work); + if (pdsc->health_work.func && !pdsc->health_stopped) { + disable_work_sync(&pdsc->health_work); + pdsc->health_stopped = true; + } } static void pdsc_restart_health_thread(struct pdsc *pdsc) @@ -474,6 +482,10 @@ static void pdsc_restart_health_thread(struct pdsc *pdsc) if (pdsc->pdev->is_virtfn) return; + if (pdsc->health_stopped) { + enable_work(&pdsc->health_work); + pdsc->health_stopped = false; + } timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); mod_timer(&pdsc->wdtimer, jiffies + 1); } @@ -496,8 +508,12 @@ static void pdsc_reset_prepare(struct pci_dev *pdev) pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev); } - pdsc_unmap_bars(pdsc); - pci_release_regions(pdev); + if (!pdev->is_virtfn) { + mutex_lock(&pdsc->devcmd_lock); + pdsc_unmap_bars(pdsc); + mutex_unlock(&pdsc->devcmd_lock); + pci_release_regions(pdev); + } if (pci_is_enabled(pdev)) pci_disable_device(pdev); } @@ -525,7 +541,9 @@ static void pdsc_reset_done(struct pci_dev *pdev) return; } + mutex_lock(&pdsc->devcmd_lock); err = pdsc_map_bars(pdsc); + mutex_unlock(&pdsc->devcmd_lock); if (err) return; } @@ -550,7 +568,11 @@ static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error) { if (error == pci_channel_io_frozen) { + struct pdsc *pdsc = pci_get_drvdata(pdev); + pdsc_reset_prepare(pdev); + if (!pdev->is_virtfn) + cancel_work_sync(&pdsc->pci_reset_work); return PCI_ERS_RESULT_NEED_RESET; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index e5391a2eca51d..bf10ad7cc9666 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -3299,7 +3299,7 @@ static void xgbe_enable_rx(struct xgbe_prv_data *pdata) XGMAC_IOWRITE(pdata, MAC_RQC0R, reg_val); /* Enable MAC Rx */ - XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 1); + XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 1); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 1); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 1); @@ -3310,7 +3310,6 @@ static void xgbe_disable_rx(struct xgbe_prv_data *pdata) unsigned int i; /* Disable MAC Rx */ - XGMAC_IOWRITE_BITS(pdata, MAC_RCR, DCRCC, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, CST, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, ACS, 0); XGMAC_IOWRITE_BITS(pdata, MAC_RCR, RE, 0); diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c index 7675bb98f0295..c2d22fdb6c182 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c @@ -267,9 +267,14 @@ static void xgbe_an37_set(struct xgbe_prv_data *pdata, bool enable, XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_CTRL1, reg); - reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL); - reg |= XGBE_VEND2_MAC_AUTO_SW; - XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL, reg); + if (pdata->an_mode == XGBE_AN_MODE_CL37_SGMII) { + reg = XMDIO_READ(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL); + if (enable) + reg |= XGBE_VEND2_MAC_AUTO_SW; + else + reg &= ~XGBE_VEND2_MAC_AUTO_SW; + XMDIO_WRITE(pdata, MDIO_MMD_VEND2, MDIO_PCS_DIG_CTRL, reg); + } } static void xgbe_an37_restart(struct xgbe_prv_data *pdata) diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c index d23d23bed39fe..1579b19c36a71 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.c @@ -336,6 +336,35 @@ bool aq_ring_tx_clean(struct aq_ring_s *self) return !!budget; } +void aq_ring_tx_deinit(struct aq_ring_s *self) +{ + if (!self) + return; + + for (; self->sw_head != self->sw_tail; + self->sw_head = aq_ring_next_dx(self, self->sw_head)) { + struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; + struct device *ndev = aq_nic_get_dev(self->aq_nic); + + if (buff->is_mapped) { + if (buff->is_sop) { + dma_unmap_single(ndev, buff->pa, buff->len, + DMA_TO_DEVICE); + } else { + dma_unmap_page(ndev, buff->pa, buff->len, + DMA_TO_DEVICE); + } + } + + if (buff->is_eop) { + if (buff->skb) + dev_kfree_skb_any(buff->skb); + else if (buff->xdpf) + xdp_return_frame(buff->xdpf); + } + } +} + static void aq_rx_checksum(struct aq_ring_s *self, struct aq_ring_buff_s *buff, struct sk_buff *skb) @@ -897,15 +926,29 @@ int aq_ring_rx_fill(struct aq_ring_s *self) void aq_ring_rx_deinit(struct aq_ring_s *self) { - if (!self) + unsigned int i; + + if (!self || !self->buff_ring) return; - for (; self->sw_head != self->sw_tail; - self->sw_head = aq_ring_next_dx(self, self->sw_head)) { - struct aq_ring_buff_s *buff = &self->buff_ring[self->sw_head]; + /* Release every page still owned by the ring. + * + * Walking [sw_head, sw_tail) is not enough: refill is batched + * (aq_ring_rx_fill() waits for AQ_CFG_RX_REFILL_THRES free slots), + * so slots that were cleaned but not yet reposted accumulate in the + * [sw_tail, sw_head) gap, and they keep their page for reuse. Walk + * the whole ring and release whatever is left. + */ + for (i = 0; i < self->size; i++) { + struct aq_ring_buff_s *buff = &self->buff_ring[i]; + + if (!buff->rxdata.page) + continue; aq_free_rxpage(&buff->rxdata, aq_nic_get_dev(self->aq_nic)); } + + self->sw_head = self->sw_tail; } void aq_ring_free(struct aq_ring_s *self) diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h index d627ace850ff5..67503c5126760 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ring.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ring.h @@ -199,6 +199,7 @@ void aq_ring_update_queue_state(struct aq_ring_s *ring); void aq_ring_queue_wake(struct aq_ring_s *ring); void aq_ring_queue_stop(struct aq_ring_s *ring); bool aq_ring_tx_clean(struct aq_ring_s *self); +void aq_ring_tx_deinit(struct aq_ring_s *self); int aq_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags); int aq_ring_rx_clean(struct aq_ring_s *self, diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c index 9769ab4f9bef0..62b5967b6dcc6 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c @@ -275,7 +275,7 @@ void aq_vec_deinit(struct aq_vec_s *self) for (i = 0U; self->tx_rings > i; ++i) { ring = self->ring[i]; - aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]); + aq_ring_tx_deinit(&ring[AQ_VEC_TX_ID]); aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]); } diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c index ceb6c11431dd9..d4de48c3132a3 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -147,8 +147,9 @@ static int tx_spb_ring_full(struct bcmasp_intf *intf, int cnt) int next_index, i; /* Check if we have enough room for cnt descriptors */ + next_index = intf->tx_spb_index; for (i = 0; i < cnt; i++) { - next_index = incr_ring(intf->tx_spb_index, DESC_RING_COUNT); + next_index = incr_ring(next_index, DESC_RING_COUNT); if (next_index == intf->tx_spb_clean_index) return 1; } @@ -333,6 +334,7 @@ static netdev_tx_t bcmasp_xmit(struct sk_buff *skb, struct net_device *dev) txcb->bytes_sent = total_bytes; dma_unmap_addr_set(txcb, dma_addr, mapping); dma_unmap_len_set(txcb, dma_len, size); + txcb->last = false; if (!i) { desc->flags |= DESC_SOF; if (csum_hw) diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c index 832eeb960bd2d..02be751f3d8fc 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c @@ -2154,8 +2154,6 @@ static int bnge_init_nic(struct bnge_net *bn) err_free_ring_grps: bnge_free_ring_grps(bn); - return rc; - err_free_rx_ring_pair_bufs: bnge_free_rx_ring_pair_bufs(bn); return rc; diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c index 62ebe03a0dcf3..39cc392bd8c4b 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_resc.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_resc.c @@ -151,7 +151,8 @@ static int bnge_adjust_rings(struct bnge_dev *bd, u16 *rx, u16 tx_chunks = bnge_num_tx_to_cp(bd, *tx); if (tx_chunks != *tx) { - u16 tx_saved = tx_chunks, rc; + u16 tx_saved = tx_chunks; + int rc; rc = bnge_fix_rings_count(rx, &tx_chunks, max_nq, sh); if (rc) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 8e6eec828d482..c4ea14f8fbe96 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -13474,10 +13474,13 @@ static int bnx2x_init_firmware(struct bnx2x *bp) iro_alloc_err: kfree(bp->init_ops_offsets); + bp->init_ops_offsets = NULL; init_offsets_alloc_err: kfree(bp->init_ops); + bp->init_ops = NULL; init_ops_alloc_err: kfree(bp->init_data); + bp->init_data = NULL; request_firmware_exit: release_firmware(bp->firmware); bp->firmware = NULL; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 98a911eae8eac..52cb2a25600bd 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -59,10 +59,10 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" -#include "bnxt_ulp.h" #include "bnxt_sriov.h" #include "bnxt_ethtool.h" #include "bnxt_dcb.h" @@ -499,11 +499,8 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) if (skb_shinfo(skb)->nr_frags > TX_MAX_FRAGS) { netdev_warn_once(dev, "SKB has too many (%d) fragments, max supported is %d. SKB will be linearized.\n", skb_shinfo(skb)->nr_frags, TX_MAX_FRAGS); - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - dev_core_stats_tx_dropped_inc(dev); - return NETDEV_TX_OK; - } + if (skb_linearize(skb)) + goto tx_free; } #endif free_size = bnxt_tx_avail(bp, txr); @@ -905,7 +902,7 @@ static void bnxt_tx_int(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) static bool bnxt_separate_head_pool(struct bnxt_rx_ring_info *rxr) { - return rxr->need_head_pool || PAGE_SIZE > BNXT_RX_PAGE_SIZE; + return rxr->need_head_pool || rxr->rx_page_size < PAGE_SIZE; } static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, @@ -915,9 +912,9 @@ static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, { struct page *page; - if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { + if (rxr->rx_page_size < PAGE_SIZE) { page = page_pool_dev_alloc_frag(rxr->page_pool, offset, - BNXT_RX_PAGE_SIZE); + rxr->rx_page_size); } else { page = page_pool_dev_alloc_pages(rxr->page_pool); *offset = 0; @@ -936,8 +933,9 @@ static netmem_ref __bnxt_alloc_rx_netmem(struct bnxt *bp, dma_addr_t *mapping, { netmem_ref netmem; - if (PAGE_SIZE > BNXT_RX_PAGE_SIZE) { - netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, BNXT_RX_PAGE_SIZE, gfp); + if (rxr->rx_page_size < PAGE_SIZE) { + netmem = page_pool_alloc_frag_netmem(rxr->page_pool, offset, + rxr->rx_page_size, gfp); } else { netmem = page_pool_alloc_netmems(rxr->page_pool, gfp); *offset = 0; @@ -983,6 +981,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, mapping += bp->rx_dma_offset; rx_buf->data = page; rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; + rx_buf->offset = offset; } else { u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); @@ -991,6 +990,7 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, rx_buf->data = data; rx_buf->data_ptr = data + bp->rx_offset; + rx_buf->offset = 0; } rx_buf->mapping = mapping; @@ -1012,6 +1012,7 @@ void bnxt_reuse_rx_data(struct bnxt_rx_ring_info *rxr, u16 cons, void *data) prod_rx_buf->data_ptr = cons_rx_buf->data_ptr; prod_rx_buf->mapping = cons_rx_buf->mapping; + prod_rx_buf->offset = cons_rx_buf->offset; prod_bd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; cons_bd = &rxr->rx_desc_ring[RX_RING(bp, cons)][RX_IDX(cons)]; @@ -1147,23 +1148,26 @@ static struct sk_buff *bnxt_rx_multi_page_skb(struct bnxt *bp, struct page *page = data; u16 prod = rxr->rx_prod; struct sk_buff *skb; + void *frag_start; int err; + frag_start = page_address(page) + rxr->rx_buf_ring[cons].offset; + err = bnxt_alloc_rx_data(bp, rxr, prod, GFP_ATOMIC); if (unlikely(err)) { bnxt_reuse_rx_data(rxr, cons, data); return NULL; } dma_addr -= bp->rx_dma_offset; - dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, + dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, rxr->rx_page_size, bp->rx_dir); - skb = napi_build_skb(data_ptr - bp->rx_offset, BNXT_RX_PAGE_SIZE); + skb = napi_build_skb(frag_start, rxr->rx_page_size); if (!skb) { page_pool_recycle_direct(rxr->page_pool, page); return NULL; } skb_mark_for_recycle(skb); - skb_reserve(skb, bp->rx_offset); + skb_reserve(skb, data_ptr - (u8 *)frag_start); __skb_put(skb, len); return skb; @@ -1189,7 +1193,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, return NULL; } dma_addr -= bp->rx_dma_offset; - dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, BNXT_RX_PAGE_SIZE, + dma_sync_single_for_cpu(&bp->pdev->dev, dma_addr, rxr->rx_page_size, bp->rx_dir); if (unlikely(!payload)) @@ -1203,7 +1207,7 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, skb_mark_for_recycle(skb); off = (void *)data_ptr - page_address(page); - skb_add_rx_frag(skb, 0, page, off, len, BNXT_RX_PAGE_SIZE); + skb_add_rx_frag(skb, 0, page, off, len, rxr->rx_page_size); memcpy(skb->data - NET_IP_ALIGN, data_ptr - NET_IP_ALIGN, payload + NET_IP_ALIGN); @@ -1288,7 +1292,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, if (skb) { skb_add_rx_frag_netmem(skb, i, cons_rx_buf->netmem, cons_rx_buf->offset, - frag_len, BNXT_RX_PAGE_SIZE); + frag_len, rxr->rx_page_size); } else { skb_frag_t *frag = &shinfo->frags[i]; @@ -1313,7 +1317,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, if (skb) { skb->len -= frag_len; skb->data_len -= frag_len; - skb->truesize -= BNXT_RX_PAGE_SIZE; + skb->truesize -= rxr->rx_page_size; } --shinfo->nr_frags; @@ -1328,7 +1332,7 @@ static u32 __bnxt_rx_agg_netmems(struct bnxt *bp, } page_pool_dma_sync_netmem_for_cpu(rxr->page_pool, netmem, 0, - BNXT_RX_PAGE_SIZE); + rxr->rx_page_size); total_frag_len += frag_len; prod = NEXT_RX_AGG(prod); @@ -2295,8 +2299,7 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (!skb) goto oom_next_rx; } else { - skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, - rxr->page_pool, &xdp); + skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr, &xdp); if (!skb) { /* we should be able to free the old skb here */ bnxt_xdp_buff_frags_free(rxr, &xdp); @@ -3844,11 +3847,13 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, pp.pool_size = bp->rx_agg_ring_size / agg_size_fac; if (BNXT_RX_PAGE_MODE(bp)) pp.pool_size += bp->rx_ring_size / rx_size_fac; + + pp.order = get_order(rxr->rx_page_size); pp.nid = numa_node; pp.netdev = bp->dev; pp.dev = &bp->pdev->dev; pp.dma_dir = bp->rx_dir; - pp.max_len = PAGE_SIZE; + pp.max_len = PAGE_SIZE << pp.order; pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV | PP_FLAG_ALLOW_UNREADABLE_NETMEM; pp.queue_idx = rxr->bnapi->index; @@ -3859,7 +3864,10 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, rxr->page_pool = pool; rxr->need_head_pool = page_pool_is_unreadable(pool); + rxr->need_head_pool |= !!pp.order; if (bnxt_separate_head_pool(rxr)) { + pp.order = 0; + pp.max_len = PAGE_SIZE; pp.pool_size = min(bp->rx_ring_size / rx_size_fac, 1024); pp.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; pool = page_pool_create(&pp); @@ -4318,6 +4326,7 @@ static void bnxt_init_ring_struct(struct bnxt *bp) struct bnxt_rx_ring_info *rxr; struct bnxt_tx_ring_info *txr; struct bnxt_ring_struct *ring; + struct netdev_rx_queue *rxq; if (!bnapi) continue; @@ -4335,6 +4344,9 @@ static void bnxt_init_ring_struct(struct bnxt *bp) if (!rxr) goto skip_rx; + rxq = __netif_get_rx_queue(bp->dev, i); + rxr->rx_page_size = rxq->qcfg.rx_page_size; + ring = &rxr->rx_ring_struct; rmem = &ring->ring_mem; rmem->nr_pages = bp->rx_nr_pages; @@ -4494,8 +4506,18 @@ static void bnxt_init_one_rx_agg_ring_rxbd(struct bnxt *bp, ring = &rxr->rx_agg_ring_struct; ring->fw_ring_id = INVALID_HW_RING_ID; if ((bp->flags & BNXT_FLAG_AGG_RINGS)) { - type = ((u32)BNXT_RX_PAGE_SIZE << RX_BD_LEN_SHIFT) | - RX_BD_TYPE_RX_AGG_BD | RX_BD_FLAGS_SOP; + type = ((u32)rxr->rx_page_size << RX_BD_LEN_SHIFT) | + RX_BD_TYPE_RX_AGG_BD; + + /* Disable EOP if TPA is enabled to prevent overlapping zero + * padding with the next segment's data. On P7_PLUS, EOP will + * automatically disable Relaxed Ordering (RO) to prevent + * potential data corruption (and may degrade performance). On + * older chips, RO will not be automatically disabled and may + * cause corruption. + */ + if (!(bp->flags & BNXT_FLAG_TPA)) + type |= RX_BD_FLAGS_AGG_EOP; bnxt_init_rxbd_pages(ring, type); } @@ -4878,7 +4900,8 @@ void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) bnxt_get_max_rings(bp, &rx, &tx, true); if (rx > 1) { bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS; - bp->dev->hw_features |= NETIF_F_LRO; + if (BNXT_SUPPORTS_TPA(bp)) + bp->dev->hw_features |= NETIF_F_LRO; } } @@ -6577,22 +6600,36 @@ int bnxt_get_nr_rss_ctxs(struct bnxt *bp, int rx_rings) static void bnxt_fill_hw_rss_tbl(struct bnxt *bp, struct bnxt_vnic_info *vnic) { bool no_rss = !(vnic->flags & BNXT_VNIC_RSS_FLAG); - u16 i, j; + u16 i, j, min_j = bp->rx_nr_rings - 1; + + if (!vnic->rss_table) + goto skip_rss_tbl; /* Fill the RSS indirection table with ring group ids */ for (i = 0, j = 0; i < HW_HASH_INDEX_SIZE; i++) { if (!no_rss) j = bp->rss_indir_tbl[i]; + min_j = min(j, min_j); vnic->rss_table[i] = cpu_to_le16(vnic->fw_grp_ids[j]); } + +skip_rss_tbl: + if (vnic->rss_table && !no_rss) + vnic->default_rx_ring = min_j; + else if (vnic->flags & BNXT_VNIC_RFS_FLAG) + vnic->default_rx_ring = vnic->vnic_id - 1; + else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) + vnic->default_rx_ring = bp->rx_nr_rings - 1; + else + vnic->default_rx_ring = 0; } static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic) { + u16 tbl_size, i, min_j = bp->rx_nr_rings - 1; __le16 *ring_tbl = vnic->rss_table; struct bnxt_rx_ring_info *rxr; - u16 tbl_size, i; tbl_size = bnxt_get_rxfh_indir_size(bp->dev); @@ -6605,6 +6642,7 @@ static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, j = ethtool_rxfh_context_indir(vnic->rss_ctx)[i]; else j = bp->rss_indir_tbl[i]; + min_j = min(j, min_j); rxr = &bp->rx_ring[j]; ring_id = rxr->rx_ring_struct.fw_ring_id; @@ -6612,19 +6650,15 @@ static void bnxt_fill_hw_rss_tbl_p5(struct bnxt *bp, ring_id = bnxt_cp_ring_for_rx(bp, rxr); *ring_tbl++ = cpu_to_le16(ring_id); } + vnic->default_rx_ring = min_j; } static void __bnxt_hwrm_vnic_set_rss(struct bnxt *bp, struct hwrm_vnic_rss_cfg_input *req, struct bnxt_vnic_info *vnic) { - if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { - bnxt_fill_hw_rss_tbl_p5(bp, vnic); - if (bp->flags & BNXT_FLAG_CHIP_P7) - req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; - } else { - bnxt_fill_hw_rss_tbl(bp, vnic); - } + if (bp->flags & BNXT_FLAG_CHIP_P7) + req->flags |= VNIC_RSS_CFG_REQ_FLAGS_IPSEC_HASH_TYPE_CFG_SUPPORT; if (bp->rss_hash_delta) { req->hash_type = cpu_to_le32(bp->rss_hash_delta); @@ -6676,6 +6710,7 @@ static int bnxt_hwrm_vnic_set_rss_p5(struct bnxt *bp, if (!set_rss) return hwrm_req_send(bp, req); + bnxt_fill_hw_rss_tbl_p5(bp, vnic); __bnxt_hwrm_vnic_set_rss(bp, req, vnic); ring_tbl_map = vnic->rss_table_dma_addr; nr_ctxs = bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings); @@ -6812,8 +6847,9 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) return rc; if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) { - struct bnxt_rx_ring_info *rxr = &bp->rx_ring[0]; + struct bnxt_rx_ring_info *rxr; + rxr = &bp->rx_ring[vnic->default_rx_ring]; req->default_rx_ring_id = cpu_to_le16(rxr->rx_ring_struct.fw_ring_id); req->default_cmpl_ring_id = @@ -6846,13 +6882,7 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic) req->cos_rule = cpu_to_le16(0xffff); } - if (vnic->flags & BNXT_VNIC_RSS_FLAG) - ring = 0; - else if (vnic->flags & BNXT_VNIC_RFS_FLAG) - ring = vnic->vnic_id - 1; - else if ((vnic->vnic_id == 1) && BNXT_CHIP_TYPE_NITRO_A0(bp)) - ring = bp->rx_nr_rings - 1; - + ring = vnic->default_rx_ring; grp_idx = bp->rx_ring[ring].bnapi->index; req->dflt_ring_grp = cpu_to_le16(bp->grp_info[grp_idx].fw_grp_id); req->lb_rule = cpu_to_le16(0xffff); @@ -7064,6 +7094,7 @@ static void bnxt_hwrm_ring_grp_free(struct bnxt *bp) static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, struct hwrm_ring_alloc_input *req, + struct bnxt_rx_ring_info *rxr, struct bnxt_ring_struct *ring) { struct bnxt_ring_grp_info *grp_info = &bp->grp_info[ring->grp_idx]; @@ -7073,7 +7104,7 @@ static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, if (ring_type == HWRM_RING_ALLOC_AGG) { req->ring_type = RING_ALLOC_REQ_RING_TYPE_RX_AGG; req->rx_ring_id = cpu_to_le16(grp_info->rx_fw_ring_id); - req->rx_buf_size = cpu_to_le16(BNXT_RX_PAGE_SIZE); + req->rx_buf_size = cpu_to_le16(rxr->rx_page_size); enables |= RING_ALLOC_REQ_ENABLES_RX_RING_ID_VALID; } else { req->rx_buf_size = cpu_to_le16(bp->rx_buf_use_size); @@ -7087,6 +7118,7 @@ static void bnxt_set_rx_ring_params_p5(struct bnxt *bp, u32 ring_type, } static int hwrm_ring_alloc_send_msg(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, struct bnxt_ring_struct *ring, u32 ring_type, u32 map_index) { @@ -7143,7 +7175,8 @@ static int hwrm_ring_alloc_send_msg(struct bnxt *bp, cpu_to_le32(bp->rx_ring_mask + 1) : cpu_to_le32(bp->rx_agg_ring_mask + 1); if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) - bnxt_set_rx_ring_params_p5(bp, ring_type, req, ring); + bnxt_set_rx_ring_params_p5(bp, ring_type, req, + rxr, ring); break; case HWRM_RING_ALLOC_CMPL: req->ring_type = RING_ALLOC_REQ_RING_TYPE_L2_CMPL; @@ -7291,7 +7324,7 @@ static int bnxt_hwrm_rx_ring_alloc(struct bnxt *bp, u32 map_idx = bnapi->index; int rc; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, rxr, ring, type, map_idx); if (rc) return rc; @@ -7311,7 +7344,7 @@ static int bnxt_hwrm_rx_agg_ring_alloc(struct bnxt *bp, int rc; map_idx = grp_idx + bp->rx_nr_rings; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, rxr, ring, type, map_idx); if (rc) return rc; @@ -7335,7 +7368,7 @@ static int bnxt_hwrm_cp_ring_alloc_p5(struct bnxt *bp, ring = &cpr->cp_ring_struct; ring->handle = BNXT_SET_NQ_HDL(cpr); - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, map_idx); if (rc) return rc; bnxt_set_db(bp, &cpr->cp_db, type, map_idx, ring->fw_ring_id); @@ -7350,7 +7383,7 @@ static int bnxt_hwrm_tx_ring_alloc(struct bnxt *bp, const u32 type = HWRM_RING_ALLOC_TX; int rc; - rc = hwrm_ring_alloc_send_msg(bp, ring, type, tx_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, tx_idx); if (rc) return rc; bnxt_set_db(bp, &txr->tx_db, type, tx_idx, ring->fw_ring_id); @@ -7376,7 +7409,7 @@ static int bnxt_hwrm_ring_alloc(struct bnxt *bp) vector = bp->irq_tbl[map_idx].vector; disable_irq_nosync(vector); - rc = hwrm_ring_alloc_send_msg(bp, ring, type, map_idx); + rc = hwrm_ring_alloc_send_msg(bp, NULL, ring, type, map_idx); if (rc) { enable_irq(vector); goto err_out; @@ -10703,6 +10736,7 @@ static int __bnxt_setup_vnic(struct bnxt *bp, struct bnxt_vnic_info *vnic) } skip_rss_ctx: + bnxt_fill_hw_rss_tbl(bp, vnic); /* configure default vnic, ring grp */ rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { @@ -10927,6 +10961,11 @@ static int bnxt_set_vnic_mru_p5(struct bnxt *bp, struct bnxt_vnic_info *vnic, vnic->vnic_id, rc); return rc; } + if (rxr_id == vnic->default_rx_ring) { + rc = bnxt_hwrm_vnic_cfg(bp, vnic); + if (rc) + return rc; + } } vnic->mru = mru; bnxt_hwrm_vnic_update(bp, vnic, @@ -11008,6 +11047,9 @@ static int bnxt_setup_nitroa0_vnic(struct bnxt *bp) return rc; } + /* Setup the proper default RX ring */ + bnxt_fill_hw_rss_tbl(bp, vnic); + rc = bnxt_hwrm_vnic_cfg(bp, vnic); if (rc) { netdev_err(bp->dev, "Cannot allocate special vnic for NS2 A0: %x\n", @@ -11150,8 +11192,13 @@ static int bnxt_shutdown_nic(struct bnxt *bp, bool irq_re_init) static int bnxt_init_nic(struct bnxt *bp, bool irq_re_init) { + int rc; + bnxt_init_cp_rings(bp); - bnxt_init_rx_rings(bp); + rc = bnxt_init_rx_rings(bp); + if (rc) + return rc; + bnxt_init_tx_rings(bp); bnxt_init_ring_grps(bp, irq_re_init); bnxt_init_vnics(bp); @@ -11605,6 +11652,16 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify, irq = container_of(notify, struct bnxt_irq, affinity_notify); +#ifdef CONFIG_RFS_ACCEL + if (irq->bp->dev->rx_cpu_rmap && irq->ring_nr < irq->bp->rx_nr_rings) { + err = cpu_rmap_update(irq->bp->dev->rx_cpu_rmap, irq->ring_nr, + mask); + if (err) + netdev_warn(irq->bp->dev, + "aRFS rmap update failed: %d\n", err); + } +#endif + if (!irq->bp->tph_mode) return; @@ -11678,10 +11735,6 @@ static void bnxt_free_irq(struct bnxt *bp) struct bnxt_irq *irq; int i; -#ifdef CONFIG_RFS_ACCEL - free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); - bp->dev->rx_cpu_rmap = NULL; -#endif if (!bp->irq_tbl || !bp->bnapi) return; @@ -11690,14 +11743,14 @@ static void bnxt_free_irq(struct bnxt *bp) irq = &bp->irq_tbl[map_idx]; if (irq->requested) { + bnxt_release_irq_notifier(irq); + if (irq->have_cpumask) { irq_update_affinity_hint(irq->vector, NULL); free_cpumask_var(irq->cpu_mask); irq->have_cpumask = 0; } - bnxt_release_irq_notifier(irq); - free_irq(irq->vector, bp->bnapi[i]); } @@ -11707,6 +11760,11 @@ static void bnxt_free_irq(struct bnxt *bp) /* Disable TPH support */ pcie_disable_tph(bp->pdev); bp->tph_mode = 0; + +#ifdef CONFIG_RFS_ACCEL + free_irq_cpu_rmap(bp->dev->rx_cpu_rmap); + bp->dev->rx_cpu_rmap = NULL; +#endif } static int bnxt_request_irq(struct bnxt *bp) @@ -11726,9 +11784,11 @@ static int bnxt_request_irq(struct bnxt *bp) #endif /* Enable TPH support as part of IRQ request */ - rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); - if (!rc) - bp->tph_mode = PCI_TPH_ST_IV_MODE; + if (BNXT_SUPPORTS_QUEUE_API(bp)) { + rc = pcie_enable_tph(bp->pdev, PCI_TPH_ST_IV_MODE); + if (!rc) + bp->tph_mode = PCI_TPH_ST_IV_MODE; + } for (i = 0, j = 0; i < bp->cp_nr_rings; i++) { int map_idx = bnxt_cp_num_to_irq_num(bp, i); @@ -14223,31 +14283,32 @@ static void bnxt_timer(struct timer_list *t) mod_timer(&bp->timer, jiffies + bp->current_interval); } -static void bnxt_lock_sp(struct bnxt *bp) +static void bnxt_rtnl_lock_sp(struct bnxt *bp) { /* We are called from bnxt_sp_task which has BNXT_STATE_IN_SP_TASK * set. If the device is being closed, bnxt_close() may be holding - * netdev instance lock and waiting for BNXT_STATE_IN_SP_TASK to clear. - * So we must clear BNXT_STATE_IN_SP_TASK before holding netdev - * instance lock. + * RTNL and the netdev instance lock while waiting for + * BNXT_STATE_IN_SP_TASK to clear. Clear it before taking either lock. */ clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + rtnl_lock(); netdev_lock(bp->dev); } -static void bnxt_unlock_sp(struct bnxt *bp) +static void bnxt_rtnl_unlock_sp(struct bnxt *bp) { set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); netdev_unlock(bp->dev); + rtnl_unlock(); } /* Only called from bnxt_sp_task() */ static void bnxt_reset(struct bnxt *bp, bool silent) { - bnxt_lock_sp(bp); + bnxt_rtnl_lock_sp(bp); if (test_bit(BNXT_STATE_OPEN, &bp->state)) bnxt_reset_task(bp, silent); - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); } /* Only called from bnxt_sp_task() */ @@ -14255,9 +14316,9 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) { int i; - bnxt_lock_sp(bp); + bnxt_rtnl_lock_sp(bp); if (!test_bit(BNXT_STATE_OPEN, &bp->state)) { - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); return; } /* Disable and flush TPA before resetting the RX ring */ @@ -14287,7 +14348,14 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) rxr->rx_sw_agg_prod = 0; rxr->rx_next_cons = 0; rxr->bnapi->in_reset = false; - bnxt_alloc_one_rx_ring(bp, i); + rc = bnxt_alloc_one_rx_ring(bp, i); + if (rc) { + netdev_warn(bp->dev, "RX ring reset failed to allocate buffers, rc = %d, falling back to global reset\n", + rc); + bnxt_reset_task(bp, true); + bnxt_rtnl_unlock_sp(bp); + return; + } cpr = &rxr->bnapi->cp_ring; cpr->sw_stats->rx.rx_resets++; if (bp->flags & BNXT_FLAG_AGG_RINGS) @@ -14296,7 +14364,7 @@ static void bnxt_rx_ring_reset(struct bnxt *bp) } if (bp->flags & BNXT_FLAG_TPA) bnxt_set_tpa(bp, true); - bnxt_unlock_sp(bp); + bnxt_rtnl_unlock_sp(bp); } static void bnxt_fw_fatal_close(struct bnxt *bp) @@ -14395,9 +14463,11 @@ void bnxt_fw_exception(struct bnxt *bp) netdev_warn(bp->dev, "Detected firmware fatal condition, initiating reset\n"); set_bit(BNXT_STATE_FW_FATAL_COND, &bp->state); bnxt_ulp_stop(bp); - bnxt_lock_sp(bp); + clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + netdev_lock(bp->dev); bnxt_force_fw_reset(bp); - bnxt_unlock_sp(bp); + set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + netdev_unlock(bp->dev); } /* Returns the number of registered VFs, or 1 if VF configuration is pending, or @@ -14427,7 +14497,8 @@ static int bnxt_get_registered_vfs(struct bnxt *bp) void bnxt_fw_reset(struct bnxt *bp) { bnxt_ulp_stop(bp); - bnxt_lock_sp(bp); + clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + netdev_lock(bp->dev); if (test_bit(BNXT_STATE_OPEN, &bp->state) && !test_bit(BNXT_STATE_IN_FW_RESET, &bp->state)) { struct bnxt_ptp_cfg *ptp = bp->ptp_cfg; @@ -14473,7 +14544,8 @@ void bnxt_fw_reset(struct bnxt *bp) bnxt_queue_fw_reset_work(bp, tmo); } fw_reset_exit: - bnxt_unlock_sp(bp); + set_bit(BNXT_STATE_IN_SP_TASK, &bp->state); + netdev_unlock(bp->dev); } static void bnxt_chk_missed_irq(struct bnxt *bp) @@ -15209,15 +15281,17 @@ static void bnxt_fw_reset_task(struct work_struct *work) bp->fw_reset_state = BNXT_FW_RESET_STATE_OPENING; fallthrough; case BNXT_FW_RESET_STATE_OPENING: - while (!netdev_trylock(bp->dev)) { + while (!rtnl_trylock()) { bnxt_queue_fw_reset_work(bp, HZ / 10); return; } + netdev_lock(bp->dev); rc = bnxt_open(bp->dev); if (rc) { netdev_err(bp->dev, "bnxt_open() failed during FW reset\n"); bnxt_fw_reset_abort(bp, rc); netdev_unlock(bp->dev); + rtnl_unlock(); goto ulp_start; } @@ -15237,6 +15311,7 @@ static void bnxt_fw_reset_task(struct work_struct *work) bnxt_dl_health_fw_status_update(bp, true); } netdev_unlock(bp->dev); + rtnl_unlock(); bnxt_ulp_start(bp, 0); bnxt_reenable_sriov(bp); netdev_lock(bp->dev); @@ -15947,7 +16022,32 @@ static const struct netdev_stat_ops bnxt_stat_ops = { .get_base_stats = bnxt_get_base_stats, }; -static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) +static void bnxt_queue_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = BNXT_RX_PAGE_SIZE; +} + +static int bnxt_validate_qcfg(struct bnxt *bp, struct netdev_queue_config *qcfg) +{ + /* Older chips need MSS calc so rx_page_size is not supported */ + if (!(bp->flags & BNXT_FLAG_CHIP_P5_PLUS) && + qcfg->rx_page_size != BNXT_RX_PAGE_SIZE) + return -EINVAL; + + if (!is_power_of_2(qcfg->rx_page_size)) + return -ERANGE; + + if (qcfg->rx_page_size < BNXT_RX_PAGE_SIZE || + qcfg->rx_page_size > BNXT_MAX_RX_PAGE_SIZE) + return -ERANGE; + + return 0; +} + +static int bnxt_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct bnxt_rx_ring_info *rxr, *clone; struct bnxt *bp = netdev_priv(dev); @@ -15957,6 +16057,10 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) if (!bp->rx_ring) return -ENETDOWN; + rc = bnxt_validate_qcfg(bp, qcfg); + if (rc < 0) + return rc; + rxr = &bp->rx_ring[idx]; clone = qmem; memcpy(clone, rxr, sizeof(*rxr)); @@ -15968,6 +16072,10 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) clone->rx_sw_agg_prod = 0; clone->rx_next_cons = 0; clone->need_head_pool = false; + clone->rx_page_size = qcfg->rx_page_size; + clone->rx_agg_bmap = NULL; + clone->rx_tpa = NULL; + clone->rx_tpa_idx_map = NULL; rc = bnxt_alloc_rx_page_pool(bp, clone, rxr->page_pool->p.nid); if (rc) @@ -16011,15 +16119,22 @@ static int bnxt_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) bnxt_alloc_one_rx_ring_skb(bp, clone, idx); if (bp->flags & BNXT_FLAG_AGG_RINGS) bnxt_alloc_one_rx_ring_netmem(bp, clone, idx); - if (bp->flags & BNXT_FLAG_TPA) - bnxt_alloc_one_tpa_info_data(bp, clone); + if (bp->flags & BNXT_FLAG_TPA) { + rc = bnxt_alloc_one_tpa_info_data(bp, clone); + if (rc) + goto err_free_rx_ring_skbs; + } return 0; +err_free_rx_ring_skbs: + bnxt_free_one_rx_ring_skbs(bp, clone); err_free_tpa_info: bnxt_free_one_tpa_info(bp, clone); err_free_rx_agg_ring: bnxt_free_ring(bp, &clone->rx_agg_ring_struct.ring_mem); + kfree(clone->rx_agg_bmap); + clone->rx_agg_bmap = NULL; err_free_rx_ring: bnxt_free_ring(bp, &clone->rx_ring_struct.ring_mem); err_rxq_info_unreg: @@ -16094,6 +16209,8 @@ static void bnxt_copy_rx_ring(struct bnxt *bp, src_ring = &src->rx_agg_ring_struct; src_rmem = &src_ring->ring_mem; + dst->rx_page_size = src->rx_page_size; + WARN_ON(dst_rmem->nr_pages != src_rmem->nr_pages); WARN_ON(dst_rmem->page_size != src_rmem->page_size); WARN_ON(dst_rmem->flags != src_rmem->flags); @@ -16113,7 +16230,9 @@ static void bnxt_copy_rx_ring(struct bnxt *bp, dst->rx_agg_bmap = src->rx_agg_bmap; } -static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) +static int bnxt_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct bnxt *bp = netdev_priv(dev); struct bnxt_rx_ring_info *rxr, *clone; @@ -16188,7 +16307,7 @@ static int bnxt_queue_start(struct net_device *dev, void *qmem, int idx) rc); napi_enable_locked(&bnapi->napi); bnxt_db_nq_arm(bp, &cpr->cp_db, cpr->cp_raw_cons); - bnxt_reset_task(bp, true); + netif_close(dev); return rc; } @@ -16246,6 +16365,12 @@ static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops = { .ndo_queue_mem_free = bnxt_queue_mem_free, .ndo_queue_start = bnxt_queue_start, .ndo_queue_stop = bnxt_queue_stop, + .ndo_default_qcfg = bnxt_queue_default_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, +}; + +static const struct netdev_queue_mgmt_ops bnxt_queue_mgmt_ops_unsupp = { + .ndo_default_qcfg = bnxt_queue_default_qcfg, }; static void bnxt_remove_one(struct pci_dev *pdev) @@ -16900,9 +17025,10 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (BNXT_SUPPORTS_NTUPLE_VNIC(bp)) bp->rss_cap |= BNXT_RSS_CAP_MULTI_RSS_CTX; + + dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops_unsupp; if (BNXT_SUPPORTS_QUEUE_API(bp)) dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops; - dev->request_ops_lock = true; dev->netmem_tx = true; rc = register_netdev(dev); @@ -17009,6 +17135,7 @@ static int bnxt_resume(struct device *device) struct bnxt *bp = netdev_priv(dev); int rc = 0; + rtnl_lock(); netdev_lock(dev); rc = pci_enable_device(bp->pdev); if (rc) { @@ -17053,6 +17180,7 @@ static int bnxt_resume(struct device *device) resume_exit: netdev_unlock(bp->dev); + rtnl_unlock(); bnxt_ulp_start(bp, rc); if (!rc) bnxt_reenable_sriov(bp); @@ -17218,6 +17346,7 @@ static void bnxt_io_resume(struct pci_dev *pdev) int err; netdev_info(bp->dev, "PCI Slot Resume\n"); + rtnl_lock(); netdev_lock(netdev); err = bnxt_hwrm_func_qcaps(bp); @@ -17235,6 +17364,7 @@ static void bnxt_io_resume(struct pci_dev *pdev) netif_device_attach(netdev); netdev_unlock(netdev); + rtnl_unlock(); bnxt_ulp_start(bp, err); if (!err) bnxt_reenable_sriov(bp); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index d55987e24ad6e..f02508600c1d2 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -131,6 +131,7 @@ struct rx_bd { #define RX_BD_TYPE_48B_BD_SIZE (2 << 4) #define RX_BD_TYPE_64B_BD_SIZE (3 << 4) #define RX_BD_FLAGS_SOP (1 << 6) + #define RX_BD_FLAGS_AGG_EOP (1 << 6) #define RX_BD_FLAGS_EOP (1 << 7) #define RX_BD_FLAGS_BUFFERS (3 << 8) #define RX_BD_FLAGS_1_BUFFER_PACKET (0 << 8) @@ -759,6 +760,7 @@ struct nqe_cn { #endif #define BNXT_RX_PAGE_SIZE (1 << BNXT_RX_PAGE_SHIFT) +#define BNXT_MAX_RX_PAGE_SIZE BIT(15) #define BNXT_MAX_MTU 9500 @@ -900,6 +902,7 @@ struct bnxt_sw_rx_bd { void *data; u8 *data_ptr; dma_addr_t mapping; + unsigned int offset; }; struct bnxt_sw_rx_agg_bd { @@ -1104,6 +1107,7 @@ struct bnxt_rx_ring_info { unsigned long *rx_agg_bmap; u16 rx_agg_bmap_size; + u32 rx_page_size; bool need_head_pool; dma_addr_t rx_desc_mapping[MAX_RX_PAGES]; @@ -1315,6 +1319,7 @@ struct bnxt_vnic_info { #define BNXT_VNIC_RSSCTX_FLAG 0x40 struct ethtool_rxfh_context *rss_ctx; u32 vnic_id; + u16 default_rx_ring; }; struct bnxt_rss_ctx { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c index 67ca02d84c979..0147b3a9ad865 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_devlink.c @@ -13,12 +13,12 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" #include "bnxt_vfr.h" #include "bnxt_devlink.h" #include "bnxt_ethtool.h" -#include "bnxt_ulp.h" #include "bnxt_ptp.h" #include "bnxt_coredump.h" #include "bnxt_nvm_defs.h" diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index a71025f51370d..8035c32802e87 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -27,9 +27,9 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" -#include "bnxt_ulp.h" #include "bnxt_xdp.h" #include "bnxt_ptp.h" #include "bnxt_ethtool.h" diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c index de3427c6c6aaf..faed0cf726e1e 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_hwmon.c @@ -40,7 +40,7 @@ void bnxt_hwmon_notify_event(struct bnxt *bp) return; } - hwmon_notify_event(&bp->pdev->dev, hwmon_temp, attr, 0); + hwmon_notify_event(bp->hwmon_dev, hwmon_temp, attr, 0); } static int bnxt_hwrm_temp_query(struct bnxt *bp, u8 *temp) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c index 0abaa2bbe3577..44b7afe9a6f56 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c @@ -514,12 +514,15 @@ static int bnxt_ptp_enable(struct ptp_clock_info *ptp_info, return rc; case PTP_CLK_REQ_PPS: /* Configure PHC PPS IN */ - rc = bnxt_ptp_cfg_pin(bp, 0, BNXT_PPS_PIN_PPS_IN); + pin_id = 0; + if (!on) + break; + rc = bnxt_ptp_cfg_pin(bp, pin_id, BNXT_PPS_PIN_PPS_IN); if (rc) return rc; rc = bnxt_ptp_cfg_event(bp, BNXT_PPS_EVENT_INTERNAL); if (!rc) - ptp->pps_info.pins[0].event = BNXT_PPS_EVENT_INTERNAL; + ptp->pps_info.pins[pin_id].event = BNXT_PPS_EVENT_INTERNAL; return rc; default: netdev_err(ptp->bp->dev, "Unrecognized PIN function\n"); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c index 80fed2c07b9e8..84c43f83193aa 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_sriov.c @@ -17,9 +17,9 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" -#include "bnxt_ulp.h" #include "bnxt_sriov.h" #include "bnxt_vfr.h" #include "bnxt_ethtool.h" diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c index f8c2c72b382d8..56a35c65d1938 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c @@ -22,10 +22,10 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_hwrm.h" -#include "bnxt_ulp.h" static DEFINE_IDA(bnxt_aux_dev_ids); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index c94a391b1ba5b..85cbeb35681c2 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -183,7 +183,7 @@ void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, u8 *data_ptr, unsigned int len, struct xdp_buff *xdp) { - u32 buflen = BNXT_RX_PAGE_SIZE; + u32 buflen = rxr->rx_page_size; struct bnxt_sw_rx_bd *rx_buf; struct pci_dev *pdev; dma_addr_t mapping; @@ -460,7 +460,7 @@ int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) struct sk_buff * bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, - struct page_pool *pool, struct xdp_buff *xdp) + struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); @@ -468,7 +468,7 @@ bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, return NULL; xdp_update_skb_frags_info(skb, num_frags, sinfo->xdp_frags_size, - BNXT_RX_PAGE_SIZE * num_frags, + rxr->rx_page_size * num_frags, xdp_buff_get_skb_flags(xdp)); return skb; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 220285e190fcd..8933a0dec09a6 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -32,6 +32,6 @@ void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp); struct sk_buff *bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, - u8 num_frags, struct page_pool *pool, + u8 num_frags, struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp); #endif diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 8ef7995db60c6..d046a922a1a4e 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -1373,6 +1373,7 @@ struct macb { struct gpio_desc *phy_reset_gpio; int phy_reset_ms; + int phy_reset_post_ms; /* AT91RM9200 transmit queue (1 on wire + 1 queued) */ struct macb_tx_skb rm9200_txq[2]; @@ -1391,7 +1392,7 @@ struct macb { struct macb_ptp_info *ptp_info; /* macb-ptp interface */ - struct phy *sgmii_phy; /* for ZynqMP SGMII mode */ + struct phy *phy; #ifdef MACB_EXT_DESC uint8_t hw_dma_cap; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index b6e824531081e..95bc3839173c1 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -502,6 +502,7 @@ static int macb_mdio_reset(struct mii_bus *bus) gpiod_set_value_cansleep(bp->phy_reset_gpio, 1); msleep(bp->phy_reset_ms); gpiod_set_value_cansleep(bp->phy_reset_gpio, 0); + msleep(bp->phy_reset_post_ms); } return 0; @@ -1192,6 +1193,8 @@ static int macb_mii_init(struct macb *bp) if (err) goto err_out_unregister_bus; + of_node_put(mdio_np); + return 0; err_out_unregister_bus: @@ -2157,10 +2160,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) queue_writel(queue, ISR, MACB_BIT(RCOMP)); - if (napi_schedule_prep(&queue->napi_rx)) { - netdev_vdbg(bp->dev, "scheduling RX softirq\n"); - __napi_schedule(&queue->napi_rx); - } + napi_schedule(&queue->napi_rx); } if (status & (MACB_BIT(TCOMP) | @@ -2176,10 +2176,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) wmb(); // ensure softirq can see update } - if (napi_schedule_prep(&queue->napi_tx)) { - netdev_vdbg(bp->dev, "scheduling TX softirq\n"); - __napi_schedule(&queue->napi_tx); - } + napi_schedule(&queue->napi_tx); } if (unlikely(status & (MACB_TX_ERR_FLAGS))) { @@ -3311,7 +3308,7 @@ static int macb_open(struct net_device *dev) macb_init_hw(bp); - err = phy_power_on(bp->sgmii_phy); + err = phy_power_on(bp->phy); if (err) goto reset_hw; @@ -3327,7 +3324,7 @@ static int macb_open(struct net_device *dev) return 0; phy_off: - phy_power_off(bp->sgmii_phy); + phy_power_off(bp->phy); reset_hw: macb_reset_hw(bp); @@ -3361,7 +3358,7 @@ static int macb_close(struct net_device *dev) phylink_stop(bp->phylink); phylink_disconnect_phy(bp->phylink); - phy_power_off(bp->sgmii_phy); + phy_power_off(bp->phy); spin_lock_irqsave(&bp->lock, flags); macb_reset_hw(bp); @@ -4920,7 +4917,7 @@ static int macb_init(struct platform_device *pdev) queue->bp = bp; spin_lock_init(&queue->tx_ptr_lock); netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); - netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); + netif_napi_add_tx(dev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); @@ -5560,13 +5557,13 @@ static int init_reset_optional(struct platform_device *pdev) if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { /* Ensure PHY device used in SGMII mode is ready */ - bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL); + bp->phy = devm_phy_optional_get(&pdev->dev, NULL); - if (IS_ERR(bp->sgmii_phy)) - return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy), + if (IS_ERR(bp->phy)) + return dev_err_probe(&pdev->dev, PTR_ERR(bp->phy), "failed to get SGMII PHY\n"); - ret = phy_init(bp->sgmii_phy); + ret = phy_init(bp->phy); if (ret) return dev_err_probe(&pdev->dev, ret, "failed to init SGMII PHY\n"); @@ -5595,7 +5592,7 @@ static int init_reset_optional(struct platform_device *pdev) /* Fully reset controller at hardware level if mapped in device tree */ ret = device_reset_optional(&pdev->dev); if (ret) { - phy_exit(bp->sgmii_phy); + phy_exit(bp->phy); return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); } @@ -5603,7 +5600,7 @@ static int init_reset_optional(struct platform_device *pdev) err_out_phy_exit: if (ret) - phy_exit(bp->sgmii_phy); + phy_exit(bp->phy); return ret; } @@ -5934,6 +5931,7 @@ static int macb_probe(struct platform_device *pdev) spin_lock_init(&bp->lock); spin_lock_init(&bp->stats_lock); + spin_lock_init(&bp->tsu_clk_lock); /* setup capabilities */ macb_configure_caps(bp, macb_config); @@ -6007,6 +6005,12 @@ static int macb_probe(struct platform_device *pdev) if (bp->phy_reset_ms > 1000) bp->phy_reset_ms = 1000; + bp->phy_reset_post_ms = 0; + of_property_read_u32(np, "phy-reset-post-delay", &bp->phy_reset_post_ms); + /* A sane post-reset delay should not be longer than 1s */ + if (bp->phy_reset_post_ms > 1000) + bp->phy_reset_post_ms = 1000; + /* IP specific init */ err = init(pdev); if (err) @@ -6037,11 +6041,14 @@ static int macb_probe(struct platform_device *pdev) return 0; err_out_unregister_mdio: - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); + if (bp->mii_bus) { + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + } + phylink_destroy(bp->phylink); err_out_phy_exit: - phy_exit(bp->sgmii_phy); + phy_exit(bp->phy); err_out_free_netdev: free_netdev(dev); @@ -6065,9 +6072,11 @@ static void macb_remove(struct platform_device *pdev) if (dev) { bp = netdev_priv(dev); unregister_netdev(dev); - phy_exit(bp->sgmii_phy); - mdiobus_unregister(bp->mii_bus); - mdiobus_free(bp->mii_bus); + phy_exit(bp->phy); + if (bp->mii_bus) { + mdiobus_unregister(bp->mii_bus); + mdiobus_free(bp->mii_bus); + } device_set_wakeup_enable(&bp->pdev->dev, 0); cancel_delayed_work_sync(&bp->tx_lpi_work); @@ -6093,7 +6102,7 @@ static int __maybe_unused macb_suspend(struct device *dev) int err; if (!device_may_wakeup(&bp->dev->dev)) - phy_exit(bp->sgmii_phy); + phy_exit(bp->phy); if (!netif_running(netdev)) return 0; @@ -6231,7 +6240,7 @@ static int __maybe_unused macb_resume(struct device *dev) int err; if (!device_may_wakeup(&bp->dev->dev)) - phy_init(bp->sgmii_phy); + phy_init(bp->phy); if (!netif_running(netdev)) return 0; diff --git a/drivers/net/ethernet/cadence/macb_ptp.c b/drivers/net/ethernet/cadence/macb_ptp.c index f2b09100f710e..d961a5faf355d 100644 --- a/drivers/net/ethernet/cadence/macb_ptp.c +++ b/drivers/net/ethernet/cadence/macb_ptp.c @@ -322,9 +322,9 @@ void gem_ptp_txstamp(struct macb *bp, struct sk_buff *skb, skb_tstamp_tx(skb, &shhwtstamps); } -void gem_ptp_init(struct net_device *dev) +void gem_ptp_init(struct net_device *netdev) { - struct macb *bp = netdev_priv(dev); + struct macb *bp = netdev_priv(netdev); bp->ptp_clock_info = gem_ptp_caps_template; @@ -332,7 +332,8 @@ void gem_ptp_init(struct net_device *dev) bp->tsu_rate = bp->ptp_info->get_tsu_rate(bp); bp->ptp_clock_info.max_adj = bp->ptp_info->get_ptp_max_adj(); gem_ptp_init_timer(bp); - bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &dev->dev); + gem_ptp_init_tsu(bp); + bp->ptp_clock = ptp_clock_register(&bp->ptp_clock_info, &netdev->dev); if (IS_ERR(bp->ptp_clock)) { pr_err("ptp clock register failed: %ld\n", PTR_ERR(bp->ptp_clock)); @@ -343,10 +344,6 @@ void gem_ptp_init(struct net_device *dev) return; } - spin_lock_init(&bp->tsu_clk_lock); - - gem_ptp_init_tsu(bp); - dev_info(&bp->pdev->dev, "%s ptp clock registered.\n", GEM_PTP_TIMER_NAME); } diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c index 1be2dc40a1a63..e4dfd5cc15802 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c @@ -2104,8 +2104,10 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } err = pci_enable_device(pdev); - if (err) - return dev_err_probe(dev, err, "Failed to enable PCI device\n"); + if (err) { + err = dev_err_probe(dev, err, "Failed to enable PCI device\n"); + goto err_put_ptp; + } err = pci_request_regions(pdev, DRV_NAME); if (err) { @@ -2259,6 +2261,8 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_release_regions(pdev); err_disable_device: pci_disable_device(pdev); +err_put_ptp: + cavium_ptp_put(ptp_clock); return err; } diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index 6bc8dfdb3d4be..2a8b41d8eb883 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -2959,8 +2959,9 @@ static void enic_remove(struct pci_dev *pdev) if (netdev) { struct enic *enic = netdev_priv(netdev); - cancel_work_sync(&enic->reset); - cancel_work_sync(&enic->change_mtu_work); + disable_work_sync(&enic->reset); + disable_work_sync(&enic->tx_hang_reset); + disable_work_sync(&enic->change_mtu_work); unregister_netdev(netdev); enic_dev_deinit(enic); vnic_dev_close(enic->vdev); diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 40bec34f06a7b..3215d26aa2d39 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -124,6 +124,7 @@ struct gemini_ethernet_port { unsigned int rx_coalesce_nsecs; struct sk_buff *rx_skb; unsigned int rx_frag_nr; + bool rx_dropping; unsigned int freeq_refill; struct gmac_txq txq[TX_QUEUE_NUM]; @@ -1440,7 +1441,8 @@ static struct sk_buff *gmac_skb_if_good_frame(struct gemini_ethernet_port *port, return skb; } -static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) +static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget, + unsigned int *freeq_consumed) { struct gemini_ethernet_port *port = netdev_priv(netdev); unsigned short m = (1 << port->rxq_order) - 1; @@ -1448,9 +1450,12 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) void __iomem *ptr_reg = port->rxq_rwptr; unsigned int frag_nr = port->rx_frag_nr; struct sk_buff *skb = port->rx_skb; + unsigned int consumed = 0; unsigned int frame_len, frag_len; struct gmac_rxdesc *rx = NULL; struct gmac_queue_page *gpage; + unsigned int received = 0; + bool dropping = port->rx_dropping; union gmac_rxdesc_0 word0; union gmac_rxdesc_1 word1; union gmac_rxdesc_3 word3; @@ -1472,6 +1477,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) w = rw.bits.wptr; while (budget && w != r) { + page = NULL; rx = port->rxq_ring + r; word0 = rx->word0; word1 = rx->word1; @@ -1480,11 +1486,22 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) r++; r &= m; + consumed++; frag_len = word0.bits.buffer_size; frame_len = word1.bits.byte_count; page_offs = mapping & ~PAGE_MASK; + if (word3.bits32 & SOF_BIT) { + if (skb) { + napi_free_frags(&port->napi); + port->stats.rx_dropped++; + skb = NULL; + frag_nr = 0; + } + dropping = false; + } + if (!mapping) { netdev_err(netdev, "rxq[%u]: HW BUG: zero DMA desc\n", r); @@ -1495,24 +1512,11 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) gpage = gmac_get_queue_page(geth, port, mapping + PAGE_SIZE); if (!gpage) { dev_err(geth->dev, "could not find mapping\n"); - if (skb) { - napi_free_frags(&port->napi); - port->stats.rx_dropped++; - skb = NULL; - frag_nr = 0; - } - continue; + goto err_drop; } page = gpage->page; if (word3.bits32 & SOF_BIT) { - if (skb) { - napi_free_frags(&port->napi); - port->stats.rx_dropped++; - skb = NULL; - frag_nr = 0; - } - skb = gmac_skb_if_good_frame(port, word0, frame_len); if (!skb) goto err_drop; @@ -1522,8 +1526,7 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) frag_nr = 0; } else if (!skb) { - put_page(page); - continue; + goto err_drop; } if (word3.bits32 & EOF_BIT) @@ -1546,9 +1549,8 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) napi_gro_frags(&port->napi); skb = NULL; frag_nr = 0; - --budget; } - continue; + goto next_desc; err_drop: if (skb) { @@ -1557,16 +1559,29 @@ static unsigned int gmac_rx(struct net_device *netdev, unsigned int budget) frag_nr = 0; } - if (mapping) + if (page) put_page(page); - port->stats.rx_dropped++; + if (!dropping) { + port->stats.rx_dropped++; + dropping = true; + } + +next_desc: + /* Final or single-descriptor fragment, advance things */ + if (word3.bits32 & EOF_BIT) { + budget--; + received++; + dropping = false; + } } port->rx_skb = skb; port->rx_frag_nr = frag_nr; + port->rx_dropping = dropping; + *freeq_consumed = consumed; writew(r, ptr_reg); - return budget; + return received; } static int gmac_napi_poll(struct napi_struct *napi, int budget) @@ -1574,26 +1589,27 @@ static int gmac_napi_poll(struct napi_struct *napi, int budget) struct gemini_ethernet_port *port = netdev_priv(napi->dev); struct gemini_ethernet *geth = port->geth; unsigned int freeq_threshold; + unsigned int freeq_consumed; unsigned int received; freeq_threshold = 1 << (geth->freeq_order - 1); u64_stats_update_begin(&port->rx_stats_syncp); - received = gmac_rx(napi->dev, budget); - if (received < budget) { - napi_gro_flush(napi, false); - napi_complete_done(napi, received); - gmac_enable_rx_irq(napi->dev, 1); + received = gmac_rx(napi->dev, budget, &freeq_consumed); + if (received < budget) ++port->rx_napi_exits; - } - port->freeq_refill += (budget - received); + u64_stats_update_end(&port->rx_stats_syncp); + + port->freeq_refill += freeq_consumed; if (port->freeq_refill > freeq_threshold) { port->freeq_refill -= freeq_threshold; geth_fill_freeq(geth, true); } - u64_stats_update_end(&port->rx_stats_syncp); + if (received < budget && napi_complete_done(napi, received)) + gmac_enable_rx_irq(napi->dev, 1); + return received; } @@ -1894,6 +1910,7 @@ static int gmac_stop(struct net_device *netdev) napi_disable(&port->napi); port->rx_skb = NULL; port->rx_frag_nr = 0; + port->rx_dropping = false; gmac_enable_irq(netdev, 0); gmac_cleanup_rxq(netdev); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index 18d86badd6ea7..6fd478572ef8f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -4732,6 +4732,7 @@ static void dpaa2_eth_disconnect_mac(struct dpaa2_eth_priv *priv) dpaa2_mac_disconnect(mac); dpaa2_mac_close(mac); + put_device(&mac->mc_dev->dev); kfree(mac); } diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c index 789a46f6938ee..ff187e2202966 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c @@ -54,27 +54,44 @@ dpaa2_switch_filter_block_get_unused(struct ethsw_core *ethsw) static u16 dpaa2_switch_port_set_fdb(struct ethsw_port_priv *port_priv, struct net_device *bridge_dev) { + struct ethsw_core *ethsw = port_priv->ethsw_data; struct ethsw_port_priv *other_port_priv = NULL; struct dpaa2_switch_fdb *fdb; struct net_device *other_dev; + bool last_fdb_user = true; struct list_head *iter; + int i; /* If we leave a bridge (bridge_dev is NULL), find an unused * FDB and use that. */ if (!bridge_dev) { - fdb = dpaa2_switch_fdb_get_unused(port_priv->ethsw_data); - - /* If there is no unused FDB, we must be the last port that - * leaves the last bridge, all the others are standalone. We - * can just keep the FDB that we already have. - */ + /* First verify if this is the last port to leave this bridge */ + for (i = 0; i < ethsw->sw_attr.num_ifs; i++) { + if (!ethsw->ports[i] || ethsw->ports[i] == port_priv) + continue; + if (ethsw->ports[i]->fdb == port_priv->fdb) { + last_fdb_user = false; + break; + } + } - if (!fdb) { + /* If this is the last user of the FDB, just keep using it. */ + if (last_fdb_user) { port_priv->fdb->bridge_dev = NULL; return 0; } + /* Since we are not the last port which leaves a bridge, + * acquire a new FDB and use it. The number of FDBs is sized to + * accommodate all switch ports as standalone, each with its + * private FDB, which means that dpaa2_switch_fdb_get_unused() + * must succeed here. WARN if not. + */ + fdb = dpaa2_switch_fdb_get_unused(port_priv->ethsw_data); + if (WARN_ON(!fdb)) + return 0; + port_priv->fdb = fdb; port_priv->fdb->in_use = true; port_priv->fdb->bridge_dev = NULL; @@ -1511,6 +1528,7 @@ static void dpaa2_switch_port_disconnect_mac(struct ethsw_port_priv *port_priv) dpaa2_mac_disconnect(mac); dpaa2_mac_close(mac); + put_device(&mac->mc_dev->dev); kfree(mac); } @@ -2420,18 +2438,13 @@ static int dpaa2_switch_port_blocking_event(struct notifier_block *nb, /* Build a linear skb based on a single-buffer frame descriptor */ static struct sk_buff *dpaa2_switch_build_linear_skb(struct ethsw_core *ethsw, - const struct dpaa2_fd *fd) + const struct dpaa2_fd *fd, + void *fd_vaddr) { u16 fd_offset = dpaa2_fd_get_offset(fd); - dma_addr_t addr = dpaa2_fd_get_addr(fd); u32 fd_length = dpaa2_fd_get_len(fd); struct device *dev = ethsw->dev; struct sk_buff *skb = NULL; - void *fd_vaddr; - - fd_vaddr = dpaa2_iova_to_virt(ethsw->iommu_domain, addr); - dma_unmap_page(dev, addr, DPAA2_SWITCH_RX_BUF_SIZE, - DMA_FROM_DEVICE); skb = build_skb(fd_vaddr, DPAA2_SWITCH_RX_BUF_SIZE + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))); @@ -2457,6 +2470,7 @@ static void dpaa2_switch_tx_conf(struct dpaa2_switch_fq *fq, static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq, const struct dpaa2_fd *fd) { + dma_addr_t addr = dpaa2_fd_get_addr(fd); struct ethsw_core *ethsw = fq->ethsw; struct ethsw_port_priv *port_priv; struct net_device *netdev; @@ -2464,10 +2478,14 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq, struct sk_buff *skb; u16 vlan_tci, vid; int if_id, err; + void *vaddr; + + vaddr = dpaa2_iova_to_virt(ethsw->iommu_domain, addr); + dma_unmap_page(ethsw->dev, addr, DPAA2_SWITCH_RX_BUF_SIZE, + DMA_FROM_DEVICE); /* get switch ingress interface ID */ if_id = upper_32_bits(dpaa2_fd_get_flc(fd)) & 0x0000FFFF; - if (if_id >= ethsw->sw_attr.num_ifs) { dev_err(ethsw->dev, "Frame received from unknown interface!\n"); goto err_free_fd; @@ -2483,7 +2501,7 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq, } } - skb = dpaa2_switch_build_linear_skb(ethsw, fd); + skb = dpaa2_switch_build_linear_skb(ethsw, fd, vaddr); if (unlikely(!skb)) goto err_free_fd; @@ -2501,7 +2519,8 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq, err = __skb_vlan_pop(skb, &vlan_tci); if (err) { dev_info(ethsw->dev, "__skb_vlan_pop() returned %d", err); - goto err_free_fd; + kfree_skb(skb); + return; } } @@ -2516,7 +2535,7 @@ static void dpaa2_switch_rx(struct dpaa2_switch_fq *fq, return; err_free_fd: - dpaa2_switch_free_fd(ethsw, fd); + free_pages((unsigned long)vaddr, 0); } static void dpaa2_switch_detect_features(struct ethsw_core *ethsw) @@ -3288,7 +3307,6 @@ static void dpaa2_switch_teardown(struct fsl_mc_device *sw_dev) static void dpaa2_switch_remove(struct fsl_mc_device *sw_dev) { - struct ethsw_port_priv *port_priv; struct ethsw_core *ethsw; struct device *dev; int i; @@ -3300,11 +3318,17 @@ static void dpaa2_switch_remove(struct fsl_mc_device *sw_dev) dpsw_disable(ethsw->mc_io, 0, ethsw->dpsw_handle); - for (i = 0; i < ethsw->sw_attr.num_ifs; i++) { - port_priv = ethsw->ports[i]; - unregister_netdev(port_priv->netdev); + /* Unregister all the netdevs so that they are brought down and the + * shared NAPI instances gets disabled. + */ + for (i = 0; i < ethsw->sw_attr.num_ifs; i++) + unregister_netdev(ethsw->ports[i]->netdev); + + for (i = 0; i < DPAA2_SWITCH_RX_NUM_FQS; i++) + netif_napi_del(ðsw->fq[i].napi); + + for (i = 0; i < ethsw->sw_attr.num_ifs; i++) dpaa2_switch_remove_port(ethsw, i); - } kfree(ethsw->fdbs); kfree(ethsw->filter_blocks); diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c index 6603ed6962967..dec047c8128d5 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.c +++ b/drivers/net/ethernet/freescale/enetc/enetc.c @@ -72,6 +72,33 @@ void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter) } EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable) +{ + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; + + spin_lock(&si->gen_lock); + + if (enable) + set_bit(ENETC_RXBDR_CM, &priv->flags); + else + clear_bit(ENETC_RXBDR_CM, &priv->flags); + + for (int i = 0; i < priv->num_rx_rings; i++) { + u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR); + u32 rbmr; + + rbmr = u32_replace_bits(old_rbmr, enable, ENETC_RBMR_CM); + if (rbmr == old_rbmr) + continue; + + enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr); + } + + spin_unlock(&si->gen_lock); +} +EXPORT_SYMBOL_GPL(enetc_set_congestion_mode); + static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv) { int num_tx_rings = priv->num_tx_rings; @@ -2624,7 +2651,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, bool extended) { int idx = rx_ring->index; - u32 rbmr = 0; enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0, lower_32_bits(rx_ring->bd_dma_base)); @@ -2652,12 +2678,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1); rx_ring->ext_en = extended; - if (rx_ring->ext_en) - rbmr |= ENETC_RBMR_BDS; - - if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) - rbmr |= ENETC_RBMR_VTE; - rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR); rx_ring->idr = hw->reg + ENETC_SIRXIDR; @@ -2668,8 +2688,6 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring, enetc_lock_mdio(); enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring)); enetc_unlock_mdio(); - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended) @@ -2696,21 +2714,34 @@ static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring) static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) { + struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev); int idx = rx_ring->index; - u32 rbmr; + u32 rbmr = ENETC_RBMR_EN; + + if (rx_ring->ext_en) + rbmr |= ENETC_RBMR_BDS; + + if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX) + rbmr |= ENETC_RBMR_VTE; + + if (test_bit(ENETC_RXBDR_CM, &priv->flags)) + rbmr |= ENETC_RBMR_CM; - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - rbmr |= ENETC_RBMR_EN; enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); } static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_enable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -2740,11 +2771,16 @@ static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring) static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv) { - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_disable_rxbdr(hw, priv->rx_ring[i]); + + spin_unlock(&si->gen_lock); } static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv) @@ -3336,11 +3372,16 @@ EXPORT_SYMBOL_GPL(enetc_get_stats); static void enetc_enable_rxvlan(struct net_device *ndev, bool en) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i; + spin_lock(&si->gen_lock); + for (i = 0; i < priv->num_rx_rings; i++) enetc_bdr_enable_rxvlan(hw, i, en); + + spin_unlock(&si->gen_lock); } static void enetc_enable_txvlan(struct net_device *ndev, bool en) @@ -3670,6 +3711,7 @@ int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv) si = PTR_ALIGN(p, ENETC_SI_ALIGN); si->pad = (char *)si - (char *)p; + spin_lock_init(&si->gen_lock); pci_set_drvdata(pdev, si); si->pdev = pdev; diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index 60c7205ea9ff5..0e2effa947b2d 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -309,6 +309,13 @@ struct enetc_si { struct net_device *ndev; /* back ref. */ + /* General-purpose lock serializing updates that must not race, + * e.g. read-modify-write of shared hardware registers and of + * selected priv->flags bits between the phylink link callbacks + * and the ring (re)configuration path. + */ + spinlock_t gen_lock; + union { struct enetc_cbdr cbd_ring; /* Only ENETC 1.0 */ struct ntmp_user ntmp_user; /* ENETC 4.1 and later */ @@ -413,6 +420,7 @@ enum enetc_active_offloads { enum enetc_flags_bit { ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS = 0, ENETC_TX_DOWN, + ENETC_RXBDR_CM, }; /* interrupt coalescing modes */ @@ -509,6 +517,7 @@ int enetc_get_driver_data(struct enetc_si *si); void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter, const unsigned char *addr); void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter); +void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable); int enetc_open(struct net_device *ndev); int enetc_close(struct net_device *ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index de0fb272c8474..ba630f0982104 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -691,8 +691,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, struct enetc_hw *hw = &pf->si->hw; struct enetc_si *si = pf->si; struct enetc_ndev_priv *priv; - u32 rbmr, cmd_cfg; - int idx; + u32 cmd_cfg; priv = netdev_priv(pf->si->ndev); @@ -704,16 +703,7 @@ static void enetc_pl_mac_link_up(struct phylink_config *config, enetc_force_rgmii_mac(si, speed, duplex); /* Flow control */ - for (idx = 0; idx < priv->num_rx_rings; idx++) { - rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR); - - if (tx_pause) - rbmr |= ENETC_RBMR_CM; - else - rbmr &= ~ENETC_RBMR_CM; - - enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr); - } + enetc_set_congestion_mode(priv, tx_pause); if (tx_pause) { /* When the port first enters congestion, send a PAUSE request diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index a2cf8cbe2539e..b37ab4f826bd2 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -3341,8 +3341,15 @@ static void fec_enet_free_buffers(struct net_device *ndev) for (q = 0; q < fep->num_rx_queues; q++) { rxq = fep->rx_queue[q]; - for (i = 0; i < rxq->bd.ring_size; i++) - page_pool_put_full_page(rxq->page_pool, rxq->rx_skb_info[i].page, false); + for (i = 0; i < rxq->bd.ring_size; i++) { + struct page *page = rxq->rx_skb_info[i].page; + + if (!page) + continue; + + page_pool_put_full_page(rxq->page_pool, page, false); + rxq->rx_skb_info[i].page = NULL; + } for (i = 0; i < XDP_STATS_TOTAL; i++) rxq->stats[i] = 0; @@ -4635,7 +4642,8 @@ fec_probe(struct platform_device *pdev) failed_irq: fec_enet_deinit(ndev); failed_init: - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); failed_reset: pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); @@ -4677,7 +4685,8 @@ fec_drv_remove(struct platform_device *pdev) ERR_PTR(ret)); cancel_work_sync(&fep->tx_timeout_work); - fec_ptp_stop(pdev); + if (fep->bufdesc_ex) + fec_ptp_stop(pdev); unregister_netdev(ndev); fec_enet_mii_remove(fep); if (fep->reg_phy) diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c index 51402dff72c5f..52f13b61bc699 100644 --- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c +++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c @@ -900,22 +900,28 @@ static void dtsec_mac_config(struct phylink_config *config, unsigned int mode, { struct mac_device *mac_dev = fman_config_to_mac(config); struct dtsec_regs __iomem *regs = mac_dev->fman_mac->regs; - u32 tmp; + u32 ecntrl, maccfg2; + + maccfg2 = ioread32be(®s->maccfg2); + maccfg2 &= ~(MACCFG2_NIBBLE_MODE | MACCFG2_BYTE_MODE); switch (state->interface) { case PHY_INTERFACE_MODE_RMII: - tmp = DTSEC_ECNTRL_RMM; + ecntrl = DTSEC_ECNTRL_RMM; + maccfg2 |= MACCFG2_NIBBLE_MODE; break; case PHY_INTERFACE_MODE_RGMII: case PHY_INTERFACE_MODE_RGMII_ID: case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: - tmp = DTSEC_ECNTRL_GMIIM | DTSEC_ECNTRL_RPM; + ecntrl = DTSEC_ECNTRL_GMIIM | DTSEC_ECNTRL_RPM; + maccfg2 |= MACCFG2_BYTE_MODE; break; case PHY_INTERFACE_MODE_SGMII: case PHY_INTERFACE_MODE_1000BASEX: case PHY_INTERFACE_MODE_2500BASEX: - tmp = DTSEC_ECNTRL_TBIM | DTSEC_ECNTRL_SGMIIM; + ecntrl = DTSEC_ECNTRL_TBIM | DTSEC_ECNTRL_SGMIIM; + maccfg2 |= MACCFG2_BYTE_MODE; break; default: dev_warn(mac_dev->dev, "cannot configure dTSEC for %s\n", @@ -923,7 +929,8 @@ static void dtsec_mac_config(struct phylink_config *config, unsigned int mode, return; } - iowrite32be(tmp, ®s->ecntrl); + iowrite32be(ecntrl, ®s->ecntrl); + iowrite32be(maccfg2, ®s->maccfg2); } static void dtsec_link_up(struct phylink_config *config, struct phy_device *phy, diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h index 4cc6dcbfd367b..9e7d6860b83fa 100644 --- a/drivers/net/ethernet/google/gve/gve.h +++ b/drivers/net/ethernet/google/gve/gve.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ /* Interval to schedule a stats report update, 20000ms. */ #define GVE_STATS_REPORT_TIMER_PERIOD 20000 +#define GVE_RX_NAPI_RESCHED_MS 20 /* msecs */ /* Numbers of NIC tx/rx stats in stats report. */ #define NIC_TX_STATS_REPORT_NUM 0 @@ -338,6 +340,7 @@ struct gve_rx_ring { struct xdp_rxq_info xdp_rxq; struct xsk_buff_pool *xsk_pool; struct page_frag_cache page_cache; /* Page cache to allocate XDP frames */ + struct timer_list starvation_timer; /* for queue starvation recovery */ }; /* A TX desc ring entry */ diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 030800776ead1..9751049849432 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -2580,8 +2580,9 @@ static void gve_rx_queue_mem_free(struct net_device *dev, void *per_q_mem) gve_rx_free_ring_dqo(priv, gve_per_q_mem, &cfg); } -static int gve_rx_queue_mem_alloc(struct net_device *dev, void *per_q_mem, - int idx) +static int gve_rx_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_q_mem, int idx) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_alloc_rings_cfg cfg = {0}; @@ -2602,7 +2603,9 @@ static int gve_rx_queue_mem_alloc(struct net_device *dev, void *per_q_mem, return err; } -static int gve_rx_queue_start(struct net_device *dev, void *per_q_mem, int idx) +static int gve_rx_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_q_mem, int idx) { struct gve_priv *priv = netdev_priv(dev); struct gve_rx_ring *gve_per_q_mem; diff --git a/drivers/net/ethernet/google/gve/gve_ptp.c b/drivers/net/ethernet/google/gve/gve_ptp.c index a384a9ed4914e..1a46e4a9a3cb7 100644 --- a/drivers/net/ethernet/google/gve/gve_ptp.c +++ b/drivers/net/ethernet/google/gve/gve_ptp.c @@ -26,6 +26,11 @@ int gve_clock_nic_ts_read(struct gve_priv *priv) return 0; } +static int gve_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + return -EOPNOTSUPP; +} + static int gve_ptp_gettimex64(struct ptp_clock_info *info, struct timespec64 *ts, struct ptp_system_timestamp *sts) @@ -60,6 +65,7 @@ static long gve_ptp_do_aux_work(struct ptp_clock_info *info) static const struct ptp_clock_info gve_ptp_caps = { .owner = THIS_MODULE, .name = "gve clock", + .adjfine = gve_ptp_adjfine, .gettimex64 = gve_ptp_gettimex64, .settime64 = gve_ptp_settime64, .do_aux_work = gve_ptp_do_aux_work, diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index 6a0430a93ed75..0c7ab6f4a540a 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -18,6 +18,16 @@ #include #include +static void gve_rx_starvation_timer(struct timer_list *t) +{ + struct gve_rx_ring *rx = timer_container_of(rx, t, starvation_timer); + struct gve_priv *priv = rx->gve; + struct gve_notify_block *block; + + block = &priv->ntfy_blocks[rx->ntfy_id]; + napi_schedule(&block->napi); +} + static void gve_rx_free_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx) { struct device *hdev = &priv->pdev->dev; @@ -120,6 +130,7 @@ void gve_rx_stop_ring_dqo(struct gve_priv *priv, int idx) if (rx->dqo.page_pool) page_pool_disable_direct_recycling(rx->dqo.page_pool); + timer_shutdown_sync(&rx->starvation_timer); gve_remove_napi(priv, ntfy_idx); gve_rx_remove_from_block(priv, idx); gve_rx_reset_ring_dqo(priv, idx); @@ -208,8 +219,10 @@ static int gve_rx_alloc_hdr_bufs(struct gve_priv *priv, struct gve_rx_ring *rx, void gve_rx_start_ring_dqo(struct gve_priv *priv, int idx) { int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); + struct gve_rx_ring *rx = &priv->rx[idx]; gve_rx_add_to_block(priv, idx); + timer_setup(&rx->starvation_timer, gve_rx_starvation_timer, 0); gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo); } @@ -363,6 +376,7 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) struct gve_rx_compl_queue_dqo *complq = &rx->dqo.complq; struct gve_rx_buf_queue_dqo *bufq = &rx->dqo.bufq; struct gve_priv *priv = rx->gve; + u32 num_bufs_avail_to_hw; u32 num_avail_slots; u32 num_full_slots; u32 num_posted = 0; @@ -398,6 +412,26 @@ void gve_rx_post_buffers_dqo(struct gve_rx_ring *rx) } rx->fill_cnt += num_posted; + + /* If the queue has fewer than GVE_RX_BUF_THRESH_DQO descriptors + * visible to the hardware, the hardware is in danger of starving + * and cannot trigger interrupts. + * + * We use a threshold of 32 because a single maximum-sized RSC + * packet can consume up to 19 descriptors in the Rx path. Lower + * thresholds (e.g., 8 or 16) would be unsafe as they could cause + * the device to drop/stall on a maximum-sized RSC packet. + * + * Start the timer to periodically reschedule NAPI and recover. + */ + num_bufs_avail_to_hw = + ((bufq->tail & ~(GVE_RX_BUF_THRESH_DQO - 1)) - + bufq->head) & bufq->mask; + + if (num_bufs_avail_to_hw < GVE_RX_BUF_THRESH_DQO) { + mod_timer(&rx->starvation_timer, + jiffies + msecs_to_jiffies(GVE_RX_NAPI_RESCHED_MS)); + } } static void gve_rx_skb_csum(struct sk_buff *skb, @@ -825,6 +859,11 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx, rx->rx_hsplit_unsplit_pkt += unsplit; rx->rx_hsplit_bytes += hdr_len; u64_stats_update_end(&rx->statss); + + if (!buf_len) { + gve_free_buffer(rx, buf_state); + return 0; + } } else if (!rx->ctx.skb_head && rx->dqo.page_pool && netmem_is_net_iov(buf_state->page_info.netmem)) { /* when header split is disabled, the header went to the packet diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c index 18376bcc718a2..fc2c47dcfaabe 100644 --- a/drivers/net/ethernet/hisilicon/hip04_eth.c +++ b/drivers/net/ethernet/hisilicon/hip04_eth.c @@ -594,7 +594,11 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget) skb = build_skb(buf, priv->rx_buf_size); if (unlikely(!skb)) { net_dbg_ratelimited("build_skb failed\n"); - goto refill; + /* Retain the slot; return budget so NAPI retries this + * buffer. Refill would overwrite rx_buf[]/rx_phys[] + * and leak them. + */ + return budget; } dma_unmap_single(priv->dev, priv->rx_phys[priv->rx_head], @@ -622,14 +626,15 @@ static int hip04_rx_poll(struct napi_struct *napi, int budget) rx++; } -refill: buf = netdev_alloc_frag(priv->rx_buf_size); if (!buf) goto done; phys = dma_map_single(priv->dev, buf, RX_BUF_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(priv->dev, phys)) + if (dma_mapping_error(priv->dev, phys)) { + skb_free_frag(buf); goto done; + } priv->rx_buf[priv->rx_head] = buf; priv->rx_phys[priv->rx_head] = phys; hip04_set_recv_desc(priv, phys); diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c index e3e7f2270560c..02282dc86faf4 100644 --- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c +++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c @@ -1282,7 +1282,6 @@ static void hix5hd2_dev_remove(struct platform_device *pdev) struct net_device *ndev = platform_get_drvdata(pdev); struct hix5hd2_priv *priv = netdev_priv(ndev); - netif_napi_del(&priv->napi); unregister_netdev(ndev); mdiobus_unregister(priv->bus); mdiobus_free(priv->bus); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index 7f174453928c1..f6fa60f3126f5 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -7261,6 +7261,10 @@ static void hclge_get_cls_key_mac(const struct flow_rule *flow, ether_addr_copy(rule->tuples_mask.dst_mac, match.mask->dst); ether_addr_copy(rule->tuples.src_mac, match.key->src); ether_addr_copy(rule->tuples_mask.src_mac, match.mask->src); + if (is_zero_ether_addr(match.mask->dst)) + rule->unused_tuple |= BIT(INNER_DST_MAC); + if (is_zero_ether_addr(match.mask->src)) + rule->unused_tuple |= BIT(INNER_SRC_MAC); } else { rule->unused_tuple |= BIT(INNER_DST_MAC); rule->unused_tuple |= BIT(INNER_SRC_MAC); @@ -7309,6 +7313,10 @@ static int hclge_get_cls_key_ip(const struct flow_rule *flow, rule->tuples.dst_ip[IPV4_INDEX] = be32_to_cpu(match.key->dst); rule->tuples_mask.dst_ip[IPV4_INDEX] = be32_to_cpu(match.mask->dst); + if (!match.mask->src) + rule->unused_tuple |= BIT(INNER_SRC_IP); + if (!match.mask->dst) + rule->unused_tuple |= BIT(INNER_DST_IP); } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { struct flow_match_ipv6_addrs match; @@ -7321,6 +7329,10 @@ static int hclge_get_cls_key_ip(const struct flow_rule *flow, match.key->dst.s6_addr32); ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip, match.mask->dst.s6_addr32); + if (ipv6_addr_any(&match.mask->src)) + rule->unused_tuple |= BIT(INNER_SRC_IP); + if (ipv6_addr_any(&match.mask->dst)) + rule->unused_tuple |= BIT(INNER_DST_IP); } else { rule->unused_tuple |= BIT(INNER_SRC_IP); rule->unused_tuple |= BIT(INNER_DST_IP); @@ -11930,12 +11942,8 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev) if (ret) goto err_ptp_uninit; - if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_COPPER) { + if (hdev->hw.mac.media_type != HNAE3_MEDIA_TYPE_COPPER) hdev->hw.mac.req_autoneg = hdev->hw.mac.autoneg; - if (hdev->hw.mac.autoneg == AUTONEG_DISABLE && - hdev->hw.mac.speed != SPEED_UNKNOWN) - hdev->hw.mac.req_speed = hdev->hw.mac.speed; - } ret = hclge_set_autoneg_speed_dup(hdev); if (ret) { diff --git a/drivers/net/ethernet/huawei/hinic/hinic_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_dev.h index 52ea97c818b8e..d9ab94910a2a7 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_dev.h +++ b/drivers/net/ethernet/huawei/hinic/hinic_dev.h @@ -104,8 +104,6 @@ struct hinic_dev { u16 num_rss; u16 rss_limit; struct hinic_rss_type rss_type; - u8 *rss_hkey_user; - s32 *rss_indir_user; struct hinic_intr_coal_info *rx_intr_coalesce; struct hinic_intr_coal_info *tx_intr_coalesce; struct hinic_sriov_info sriov_info; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index e9f338e9dbe7a..cd4c295ec7aeb 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -1064,17 +1064,6 @@ static int __set_rss_rxfh(struct net_device *netdev, int err; if (indir) { - if (!nic_dev->rss_indir_user) { - nic_dev->rss_indir_user = - kzalloc(sizeof(u32) * HINIC_RSS_INDIR_SIZE, - GFP_KERNEL); - if (!nic_dev->rss_indir_user) - return -ENOMEM; - } - - memcpy(nic_dev->rss_indir_user, indir, - sizeof(u32) * HINIC_RSS_INDIR_SIZE); - err = hinic_rss_set_indir_tbl(nic_dev, nic_dev->rss_tmpl_idx, indir); if (err) @@ -1082,16 +1071,6 @@ static int __set_rss_rxfh(struct net_device *netdev, } if (key) { - if (!nic_dev->rss_hkey_user) { - nic_dev->rss_hkey_user = - kzalloc(HINIC_RSS_KEY_SIZE * 2, GFP_KERNEL); - - if (!nic_dev->rss_hkey_user) - return -ENOMEM; - } - - memcpy(nic_dev->rss_hkey_user, key, HINIC_RSS_KEY_SIZE); - err = hinic_rss_set_template_tbl(nic_dev, nic_dev->rss_tmpl_idx, key); if (err) diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c index 97c1584dc05bc..1868a13588c00 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c @@ -128,6 +128,7 @@ enum hinic_mbox_tx_status { #define SEQ_ID_START_VAL 0 #define SEQ_ID_MAX_VAL 42 +#define MBOX_LAST_SEG_MAX_LEN (MBOX_MAX_BUF_SZ - SEQ_ID_MAX_VAL * MBOX_SEG_LEN) #define NO_DMA_ATTRIBUTE_VAL 0 @@ -372,7 +373,8 @@ recv_pf_from_vf_mbox_handler(struct hinic_mbox_func_to_func *func_to_func, static bool check_mbox_seq_id_and_seg_len(struct hinic_recv_mbox *recv_mbox, u8 seq_id, u8 seg_len) { - if (seq_id > SEQ_ID_MAX_VAL || seg_len > MBOX_SEG_LEN) + if (seq_id > SEQ_ID_MAX_VAL || seg_len > MBOX_SEG_LEN || + (seq_id == SEQ_ID_MAX_VAL && seg_len > MBOX_LAST_SEG_MAX_LEN)) return false; if (seq_id == 0) { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c index 92c43c05e3f2d..6e29c892a97d8 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_tx.c @@ -240,8 +240,7 @@ static int hinic3_tx_csum(struct hinic3_txq *txq, struct hinic3_sq_task *task, ((struct udphdr *)skb_transport_header(skb))->dest != VXLAN_OFFLOAD_PORT_LE) { /* Unsupported tunnel packet, disable csum offload */ - skb_checksum_help(skb); - return 0; + return skb_checksum_help(skb); } } @@ -391,6 +390,10 @@ static u32 hinic3_tx_offload(struct sk_buff *skb, struct hinic3_sq_task *task, offload |= HINIC3_TX_OFFLOAD_TSO; } else { tso_cs_en = hinic3_tx_csum(txq, task, skb); + if (tso_cs_en < 0) { + offload = HINIC3_TX_OFFLOAD_INVALID; + return offload; + } if (tso_cs_en) offload |= HINIC3_TX_OFFLOAD_CSUM; } @@ -524,6 +527,7 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, skb->len = MIN_SKB_LEN; } + offload = hinic3_tx_offload(skb, &task, &queue_info, txq); num_sge = skb_shinfo(skb)->nr_frags + 1; /* assume normal wqe format + 1 wqebb for task info */ wqebb_cnt = num_sge + 1; @@ -539,7 +543,6 @@ static netdev_tx_t hinic3_send_one_skb(struct sk_buff *skb, return NETDEV_TX_BUSY; } - offload = hinic3_tx_offload(skb, &task, &queue_info, txq); if (unlikely(offload == HINIC3_TX_OFFLOAD_INVALID)) { goto err_drop_pkt; } else if (!offload) { diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index e1047915e101e..074cc6d1050eb 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -457,7 +458,7 @@ static inline u32 emac_iff2rmr(struct net_device *ndev) if (emac_has_feature(dev, EMAC_APM821XX_REQ_JUMBO_FRAME_SIZE)) { r &= ~EMAC4_RMR_MJS_MASK; - r |= EMAC4_RMR_MJS(ndev->mtu); + r |= EMAC4_RMR_MJS(ndev->mtu + VLAN_HLEN); } return r; @@ -3043,6 +3044,11 @@ static int emac_probe(struct platform_device *ofdev) /* Setup error IRQ handler */ dev->emac_irq = platform_get_irq(ofdev, 0); + if (dev->emac_irq < 0) { + err = dev->emac_irq; + goto err_gone; + } + err = devm_request_irq(&ofdev->dev, dev->emac_irq, emac_irq, 0, "EMAC", dev); if (err) { diff --git a/drivers/net/ethernet/ibm/emac/mal.c b/drivers/net/ethernet/ibm/emac/mal.c index 7d70056e90081..a089e7b64bb2b 100644 --- a/drivers/net/ethernet/ibm/emac/mal.c +++ b/drivers/net/ethernet/ibm/emac/mal.c @@ -35,6 +35,7 @@ int mal_register_commac(struct mal_instance *mal, struct mal_commac *commac) { unsigned long flags; + netdev_lock(mal->napi.dev); spin_lock_irqsave(&mal->lock, flags); MAL_DBG(mal, "reg(%08x, %08x)" NL, @@ -44,18 +45,20 @@ int mal_register_commac(struct mal_instance *mal, struct mal_commac *commac) if ((mal->tx_chan_mask & commac->tx_chan_mask) || (mal->rx_chan_mask & commac->rx_chan_mask)) { spin_unlock_irqrestore(&mal->lock, flags); + netdev_unlock(mal->napi.dev); printk(KERN_WARNING "mal%d: COMMAC channels conflict!\n", mal->index); return -EBUSY; } if (list_empty(&mal->list)) - napi_enable(&mal->napi); + napi_enable_locked(&mal->napi); mal->tx_chan_mask |= commac->tx_chan_mask; mal->rx_chan_mask |= commac->rx_chan_mask; list_add(&commac->list, &mal->list); spin_unlock_irqrestore(&mal->lock, flags); + netdev_unlock(mal->napi.dev); return 0; } @@ -64,7 +67,9 @@ void mal_unregister_commac(struct mal_instance *mal, struct mal_commac *commac) { unsigned long flags; + bool disable_napi; + netdev_lock(mal->napi.dev); spin_lock_irqsave(&mal->lock, flags); MAL_DBG(mal, "unreg(%08x, %08x)" NL, @@ -73,10 +78,12 @@ void mal_unregister_commac(struct mal_instance *mal, mal->tx_chan_mask &= ~commac->tx_chan_mask; mal->rx_chan_mask &= ~commac->rx_chan_mask; list_del_init(&commac->list); - if (list_empty(&mal->list)) - napi_disable(&mal->napi); + disable_napi = list_empty(&mal->list); spin_unlock_irqrestore(&mal->lock, flags); + if (disable_napi) + napi_disable_locked(&mal->napi); + netdev_unlock(mal->napi.dev); } int mal_set_rcbs(struct mal_instance *mal, int channel, unsigned long size) @@ -703,13 +710,13 @@ static void mal_remove(struct platform_device *ofdev) MAL_DBG(mal, "remove" NL); /* Synchronize with scheduled polling */ - napi_disable(&mal->napi); - - if (!list_empty(&mal->list)) + if (!list_empty(&mal->list)) { + napi_disable(&mal->napi); /* This is *very* bad */ WARN(1, KERN_EMERG "mal%d: commac list is not empty on remove!\n", mal->index); + } mal_reset(mal); diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index 15160427c8b30..a668afc183938 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -1222,11 +1222,11 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (hw->flash_address) iounmap(hw->flash_address); +err_mdio_ioremap: kfree(adapter->tx_ring); kfree(adapter->rx_ring); err_dma: err_sw_init: -err_mdio_ioremap: iounmap(hw->ce4100_gbe_mdio_base_virt); iounmap(hw->hw_addr); err_ioremap: diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index 8e40bb50a01e1..77f3e7991e75c 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -585,20 +585,25 @@ static int e1000_set_eeprom(struct net_device *netdev, /* need read/modify/write of first changed EEPROM word */ /* only the second byte of the word is being modified */ ret_val = e1000_read_nvm(hw, first_word, 1, &eeprom_buff[0]); + if (ret_val) + goto out; + + /* Device's eeprom is always little-endian, word addressable */ + le16_to_cpus(&eeprom_buff[0]); + ptr++; } - if (((eeprom->offset + eeprom->len) & 1) && (!ret_val)) + if ((eeprom->offset + eeprom->len) & 1) { /* need read/modify/write of last changed EEPROM word */ /* only the first byte of the word is being modified */ ret_val = e1000_read_nvm(hw, last_word, 1, &eeprom_buff[last_word - first_word]); + if (ret_val) + goto out; - if (ret_val) - goto out; - - /* Device's eeprom is always little-endian, word addressable */ - for (i = 0; i < last_word - first_word + 1; i++) - le16_to_cpus(&eeprom_buff[i]); + /* Device's eeprom is always little-endian, word addressable */ + le16_to_cpus(&eeprom_buff[last_word - first_word]); + } memcpy(ptr, bytes, eeprom->len); diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 839b7bfa19359..4bd377a668421 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -1884,27 +1884,18 @@ static int ice_devlink_nvm_snapshot(struct devlink *devlink, */ for (i = 0; i < num_blks; i++) { u32 read_sz = min_t(u32, ICE_DEVLINK_READ_BLK_SIZE, left); - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - vfree(nvm_data); - return -EIO; - } + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; status = ice_read_flat_nvm(hw, i * ICE_DEVLINK_READ_BLK_SIZE, - &read_sz, tmp, read_shadow_ram); + &read_sz, tmp, read_shadow_ram, + &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - read_sz, status, hw->adminq.sq_last_status); + read_sz, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); vfree(nvm_data); return -EIO; } - ice_release_nvm(hw); tmp += read_sz; left -= read_sz; @@ -1937,6 +1928,7 @@ static int ice_devlink_nvm_read(struct devlink *devlink, struct netlink_ext_ack *extack, u64 offset, u32 size, u8 *data) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; struct ice_pf *pf = devlink_priv(devlink); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; @@ -1960,24 +1952,14 @@ static int ice_devlink_nvm_read(struct devlink *devlink, return -ERANGE; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) { - dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n", - status, hw->adminq.sq_last_status); - NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore"); - return -EIO; - } - status = ice_read_flat_nvm(hw, (u32)offset, &size, data, - read_shadow_ram); + read_shadow_ram, &read_aq_err); if (status) { dev_dbg(dev, "ice_read_flat_nvm failed after reading %u bytes, err %d aq_err %d\n", - size, status, hw->adminq.sq_last_status); + size, status, read_aq_err); NL_SET_ERR_MSG_MOD(extack, "Failed to read NVM contents"); - ice_release_nvm(hw); return -EIO; } - ice_release_nvm(hw); return 0; } diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 6886188043764..fc4eae0a00cd3 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -765,6 +765,9 @@ static inline bool ice_is_txtime_ena(const struct ice_tx_ring *ring) struct ice_vsi *vsi = ring->vsi; struct ice_pf *pf = vsi->back; + if (vsi->type != ICE_VSI_PF) + return false; + return test_bit(ring->q_index, pf->txtime_txqs); } diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index f71e45ab809e0..dfc9a11631a3b 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -1086,14 +1086,13 @@ int ice_init_hw(struct ice_hw *hw) hw->evb_veb = true; - /* init xarray for identifying scheduling nodes uniquely */ - xa_init_flags(&hw->port_info->sched_node_ids, XA_FLAGS_ALLOC); + xa_init_flags(&hw->sched_node_ids, XA_FLAGS_ALLOC); /* Query the allocated resources for Tx scheduler */ status = ice_sched_query_res_alloc(hw); if (status) { ice_debug(hw, ICE_DBG_SCHED, "Failed to get scheduler allocated resources\n"); - goto err_unroll_alloc; + goto err_unroll_xarray; } ice_sched_get_psm_clk_freq(hw); @@ -1184,6 +1183,8 @@ int ice_init_hw(struct ice_hw *hw) ice_cleanup_fltr_mgmt_struct(hw); err_unroll_sched: ice_sched_cleanup_all(hw); +err_unroll_xarray: + xa_destroy(&hw->sched_node_ids); err_unroll_alloc: devm_kfree(ice_hw_to_dev(hw), hw->port_info); err_unroll_cqinit: @@ -1224,6 +1225,8 @@ void ice_deinit_hw(struct ice_hw *hw) /* Clear VSI contexts if not already cleared */ ice_clear_all_vsi_ctx(hw); + + xa_destroy(&hw->sched_node_ids); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index 023d0de12670c..2a47d9f3cf7ef 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -703,7 +703,7 @@ ice_dpll_pin_state_update(struct ice_pf *pf, struct ice_dpll_pin *pin, ret, libie_aq_str(pf->hw.adminq.sq_last_status), pin_type_name[pin_type], pin->idx); - else + else if (pf->hw.adminq.sq_last_status != LIBIE_AQ_RC_EBUSY) dev_err_ratelimited(ice_pf_to_dev(pf), "err:%d %s failed to update %s pin:%u\n", ret, @@ -2669,7 +2669,8 @@ static int ice_dpll_pps_update_phase_offsets(struct ice_pf *pf, *phase_offset_pins_updated = 0; ret = ice_aq_get_cgu_input_pin_measure(&pf->hw, DPLL_TYPE_PPS, meas, ARRAY_SIZE(meas)); - if (ret && pf->hw.adminq.sq_last_status == LIBIE_AQ_RC_EAGAIN) { + if (ret && (pf->hw.adminq.sq_last_status == LIBIE_AQ_RC_EAGAIN || + pf->hw.adminq.sq_last_status == LIBIE_AQ_RC_EBUSY)) { return 0; } else if (ret) { dev_err(ice_pf_to_dev(pf), @@ -2731,10 +2732,12 @@ ice_dpll_update_state(struct ice_pf *pf, struct ice_dpll *d, bool init) d->dpll_idx, d->prev_input_idx, d->input_idx, d->dpll_state, d->prev_dpll_state, d->mode); if (ret) { - dev_err(ice_pf_to_dev(pf), - "update dpll=%d state failed, ret=%d %s\n", - d->dpll_idx, ret, - libie_aq_str(pf->hw.adminq.sq_last_status)); + /* EBUSY is expected during reset recovery, don't log error */ + if (pf->hw.adminq.sq_last_status != LIBIE_AQ_RC_EBUSY) + dev_err(ice_pf_to_dev(pf), + "update dpll=%d state failed, ret=%d %s\n", + d->dpll_idx, ret, + libie_aq_str(pf->hw.adminq.sq_last_status)); return ret; } if (init) { @@ -2803,7 +2806,9 @@ static void ice_dpll_periodic_work(struct kthread_work *work) d->periodic_counter % dp->phase_offset_monitor_period == 0) ret = ice_dpll_pps_update_phase_offsets(pf, &phase_offset_ntf); if (ret) { - d->cgu_state_acq_err_num++; + /* EBUSY is expected during reset recovery */ + if (pf->hw.adminq.sq_last_status != LIBIE_AQ_RC_EBUSY) + d->cgu_state_acq_err_num++; /* stop rescheduling this worker */ if (d->cgu_state_acq_err_num > ICE_CGU_STATE_ACQ_ERR_THRESHOLD) { diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index c30e27bbfe6e2..b069e6c514fb1 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -512,9 +512,6 @@ int ice_eswitch_attach_vf(struct ice_pf *pf, struct ice_vf *vf) struct ice_repr *repr; int err; - if (!ice_is_eswitch_mode_switchdev(pf)) - return 0; - repr = ice_repr_create_vf(vf); if (IS_ERR(repr)) return PTR_ERR(repr); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index ce0f8106dcd4e..c10d0b92e63e5 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -794,8 +794,7 @@ static int ice_get_extended_regs(struct net_device *netdev, void *p) static void ice_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_hw *hw = &pf->hw; u32 *regs_buf = (u32 *)p; unsigned int i; @@ -810,8 +809,7 @@ ice_get_regs(struct net_device *netdev, struct ethtool_regs *regs, void *p) static u32 ice_get_msglevel(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); #ifndef CONFIG_DYNAMIC_DEBUG if (pf->hw.debug_mask) @@ -824,8 +822,7 @@ static u32 ice_get_msglevel(struct net_device *netdev) static void ice_set_msglevel(struct net_device *netdev, u32 data) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); #ifndef CONFIG_DYNAMIC_DEBUG if (ICE_DBG_USER & data) @@ -840,16 +837,14 @@ static void ice_set_msglevel(struct net_device *netdev, u32 data) static void ice_get_link_ext_stats(struct net_device *netdev, struct ethtool_link_ext_stats *stats) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); stats->link_down_events = pf->link_down_events; } static int ice_get_eeprom_len(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); return (int)pf->hw.flash.flash_size; } @@ -858,9 +853,8 @@ static int ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, u8 *bytes) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; + struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_hw *hw = &pf->hw; struct device *dev; int ret; @@ -876,24 +870,15 @@ ice_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *eeprom, if (!buf) return -ENOMEM; - ret = ice_acquire_nvm(hw, ICE_RES_READ); - if (ret) { - dev_err(dev, "ice_acquire_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto out; - } - ret = ice_read_flat_nvm(hw, eeprom->offset, &eeprom->len, buf, - false); + false, &read_aq_err); if (ret) { dev_err(dev, "ice_read_flat_nvm failed, err %d aq_err %s\n", - ret, libie_aq_str(hw->adminq.sq_last_status)); - goto release; + ret, libie_aq_str(read_aq_err)); + goto out; } memcpy(bytes, buf, eeprom->len); -release: - ice_release_nvm(hw); out: kfree(buf); return ret; @@ -959,8 +944,7 @@ static u64 ice_link_test(struct net_device *netdev) */ static u64 ice_eeprom_test(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); netdev_info(netdev, "EEPROM test\n"); return !!(ice_nvm_validate_checksum(&pf->hw)); @@ -1077,18 +1061,18 @@ static int ice_lbtest_prepare_rings(struct ice_vsi *vsi) status = ice_vsi_cfg_lan(vsi); if (status) - goto err_setup_rx_ring; + goto err_cfg_lan; status = ice_vsi_start_all_rx_rings(vsi); if (status) - goto err_start_rx_ring; + goto err_cfg_lan; return 0; -err_start_rx_ring: - ice_vsi_free_rx_rings(vsi); -err_setup_rx_ring: +err_cfg_lan: ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); +err_setup_rx_ring: + ice_vsi_free_rx_rings(vsi); err_setup_tx_ring: ice_vsi_free_tx_rings(vsi); @@ -1274,9 +1258,8 @@ static int ice_lbtest_receive_frames(struct ice_rx_ring *rx_ring) */ static u64 ice_loopback_test(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *orig_vsi = np->vsi, *test_vsi; - struct ice_pf *pf = orig_vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); + struct ice_vsi *test_vsi; u8 *tx_frame __free(kfree) = NULL; u8 broadcast[ETH_ALEN], ret = 0; int num_frames, valid_frames; @@ -1365,8 +1348,7 @@ static u64 ice_loopback_test(struct net_device *netdev) */ static u64 ice_intr_test(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); u16 swic_old = pf->sw_int_count; netdev_info(netdev, "interrupt test\n"); @@ -1394,9 +1376,8 @@ static void ice_self_test(struct net_device *netdev, struct ethtool_test *eth_test, u64 *data) { - struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_pf *pf = ice_netdev_to_pf(netdev); bool if_running = netif_running(netdev); - struct ice_pf *pf = np->vsi->back; struct device *dev; dev = ice_pf_to_dev(pf); @@ -1720,9 +1701,7 @@ static int ice_nway_reset(struct net_device *netdev) */ static u32 ice_get_priv_flags(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); u32 i, ret_flags = 0; for (i = 0; i < ICE_PRIV_FLAG_ARRAY_SIZE; i++) { @@ -4432,9 +4411,7 @@ static int ice_get_module_info(struct net_device *netdev, struct ethtool_modinfo *modinfo) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_hw *hw = &pf->hw; u8 sff8472_comp = 0; u8 sff8472_swap = 0; @@ -4506,12 +4483,10 @@ static int ice_get_module_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, u8 *data) { - struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_pf *pf = ice_netdev_to_pf(netdev); #define SFF_READ_BLOCK_SIZE 8 u8 value[SFF_READ_BLOCK_SIZE] = { 0 }; u8 addr = ICE_I2C_EEPROM_DEV_ADDR; - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; bool is_sfp = false; unsigned int i; @@ -4690,8 +4665,7 @@ static void ice_get_fec_stats(struct net_device *netdev, */ static int ice_ethtool_reset(struct net_device *dev, u32 *flags) { - struct ice_netdev_priv *np = netdev_priv(dev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(dev); enum ice_reset_req reset; switch (*flags) { diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c index 013c93b6605ed..c8cb492fddf46 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c @@ -574,9 +574,7 @@ ice_destroy_tunnel(struct ice_hw *hw, u16 index, enum ice_tunnel_type type, int ice_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, unsigned int idx, struct udp_tunnel_info *ti) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); enum ice_tunnel_type tnl_type; int status; u16 index; @@ -598,9 +596,7 @@ int ice_udp_tunnel_set_port(struct net_device *netdev, unsigned int table, int ice_udp_tunnel_unset_port(struct net_device *netdev, unsigned int table, unsigned int idx, struct udp_tunnel_info *ti) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); enum ice_tunnel_type tnl_type; int status; diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index aebf8e08a297b..b2ea0532f07b2 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -2177,8 +2177,7 @@ static void ice_lag_chk_disabled_bond(struct ice_lag *lag, void *ptr) */ static void ice_lag_disable_sriov_bond(struct ice_lag *lag) { - struct ice_netdev_priv *np = netdev_priv(lag->netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(lag->netdev); ice_clear_feature_support(pf, ICE_F_SRIOV_LAG); ice_clear_feature_support(pf, ICE_F_SRIOV_AA_LAG); @@ -2624,7 +2623,7 @@ int ice_init_lag(struct ice_pf *pf) goto free_lport_res; /* associate recipes to profiles */ - for (n = 0; n < ICE_PROFID_IPV6_GTPU_IPV6_TCP_INNER; n++) { + for (n = 0; n < ICE_MAX_NUM_PROFILES; n++) { err = ice_aq_get_recipe_to_profile(&pf->hw, n, &recipe_bits, NULL); if (err) diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index 3d14932871c58..851fab1812510 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -2865,6 +2865,9 @@ int ice_vsi_release(struct ice_vsi *vsi) return -ENODEV; pf = vsi->back; + if (ice_is_vsi_dflt_vsi(vsi)) + ice_clear_dflt_vsi(vsi); + if (test_bit(ICE_FLAG_RSS_ENA, pf->flags)) ice_rss_clean(vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index b89041743f6fc..f85051072ba2c 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -5713,6 +5713,16 @@ static int ice_resume(struct device *dev) /* Restart the service task */ mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); + /* Best-effort wait for the scheduled reset to finish so that the + * device is operational before returning. Without this, userspace + * (e.g. NetworkManager) may try to open the net device while the + * asynchronous reset is still in progress, hitting -EBUSY. + */ + ret = ice_wait_for_reset(pf, secs_to_jiffies(10)); + if (ret) + dev_err(dev, "Wait for reset timed out (10s) during resume: %d\n", + ret); + return 0; } @@ -8106,9 +8116,7 @@ static int ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, struct net_device *dev, u32 filter_mask, int nlflags) { - struct ice_netdev_priv *np = netdev_priv(dev); - struct ice_vsi *vsi = np->vsi; - struct ice_pf *pf = vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(dev); u16 bmode; bmode = pf->first_sw->bridge_mode; @@ -8178,8 +8186,7 @@ ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, u16 __always_unused flags, struct netlink_ext_ack __always_unused *extack) { - struct ice_netdev_priv *np = netdev_priv(dev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(dev); struct nlattr *attr, *br_spec; struct ice_hw *hw = &pf->hw; struct ice_sw *pf_sw; @@ -9613,8 +9620,7 @@ ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, */ int ice_open(struct net_device *netdev) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); if (ice_is_reset_in_progress(pf->state)) { netdev_err(netdev, "can't open net device while reset is in progress"); diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c index 7e187a804dfa1..21f3b615dbbf1 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.c +++ b/drivers/net/ethernet/intel/ice/ice_nvm.c @@ -53,17 +53,27 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, * @length: (in) number of bytes to read; (out) number of bytes actually read * @data: buffer to return data in (sized to fit the specified length) * @read_shadow_ram: if true, read from shadow RAM instead of NVM + * @read_aq_err: if non-NULL, receives the AQ error status of the failing read * * Reads a portion of the NVM, as a flat memory space. This function correctly * breaks read requests across Shadow RAM sectors and ensures that no single * read request exceeds the maximum 4KB read for a single AdminQ command. * + * FW caps the read lock at a maximum of 3000ms, so a read spanning multiple + * 4KB sectors cannot be done under a single lock without FW reclaiming it + * mid-read. The NVM lock is therefore acquired and released around each AQ + * read, so this function must be called without the lock held. + * + * Since ice_release_nvm() issues an AQ command that overwrites + * hw->adminq.sq_last_status, callers that need the failing read's AQ error + * must use @read_aq_err rather than inspecting sq_last_status afterwards. + * * Returns a status code on failure. Note that the data pointer may be * partially updated if some reads succeed before a failure. */ int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram) + bool read_shadow_ram, enum libie_aq_err *read_aq_err) { u32 inlen = *length; u32 bytes_read = 0; @@ -92,12 +102,30 @@ ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, last_cmd = !(bytes_read + read_size < inlen); + status = ice_acquire_nvm(hw, ICE_RES_READ); + if (status) { + ice_debug(hw, ICE_DBG_NVM, "Failed to acquire NVM lock, err %d aq_err %s\n", + status, libie_aq_str(hw->adminq.sq_last_status)); + break; + } + status = ice_aq_read_nvm(hw, ICE_AQC_NVM_START_POINT, offset, read_size, data + bytes_read, last_cmd, read_shadow_ram, NULL); - if (status) + if (status) { + /* Capture the read's AQ error before ice_release_nvm() + * issues its own AQ command and overwrites + * sq_last_status. + */ + if (read_aq_err) + *read_aq_err = hw->adminq.sq_last_status; + + ice_release_nvm(hw); break; + } + + ice_release_nvm(hw); bytes_read += read_size; offset += read_size; @@ -177,14 +205,19 @@ int ice_aq_erase_nvm(struct ice_hw *hw, u16 module_typeid, struct ice_sq_cd *cd) } /** - * ice_read_sr_word_aq - Reads Shadow RAM via AQ + * ice_read_sr_word - Reads Shadow RAM word * @hw: pointer to the HW structure * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) * @data: word read from the Shadow RAM * * Reads one 16 bit word from the Shadow RAM using ice_read_flat_nvm. + * + * The NVM lock is acquired and released internally by ice_read_flat_nvm() + * around the FW read, so this function must be called without the lock held. + * + * Return: zero on success, or a negative error code on failure. */ -static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) +int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) { u32 bytes = sizeof(u16); __le16 data_local; @@ -194,7 +227,7 @@ static int ice_read_sr_word_aq(struct ice_hw *hw, u16 offset, u16 *data) * Shadow RAM sector restrictions necessary when reading from the NVM. */ status = ice_read_flat_nvm(hw, offset * sizeof(u16), &bytes, - (__force u8 *)&data_local, true); + (__force u8 *)&data_local, true, NULL); if (status) return status; @@ -330,13 +363,8 @@ ice_read_flash_module(struct ice_hw *hw, enum ice_bank_select bank, u16 module, return -EINVAL; } - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; - - status = ice_read_flat_nvm(hw, start + offset, &length, data, false); - - ice_release_nvm(hw); + status = ice_read_flat_nvm(hw, start + offset, &length, data, false, + NULL); return status; } @@ -418,27 +446,6 @@ ice_read_netlist_module(struct ice_hw *hw, enum ice_bank_select bank, u32 offset return status; } -/** - * ice_read_sr_word - Reads Shadow RAM word and acquire NVM if necessary - * @hw: pointer to the HW structure - * @offset: offset of the Shadow RAM word to read (0x000000 - 0x001FFF) - * @data: word read from the Shadow RAM - * - * Reads one 16 bit word from the Shadow RAM using the ice_read_sr_word_aq. - */ -int ice_read_sr_word(struct ice_hw *hw, u16 offset, u16 *data) -{ - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (!status) { - status = ice_read_sr_word_aq(hw, offset, data); - ice_release_nvm(hw); - } - - return status; -} - /** * ice_get_pfa_module_tlv - Reads sub module TLV from NVM PFA * @hw: pointer to hardware structure @@ -856,20 +863,18 @@ int ice_get_inactive_netlist_ver(struct ice_hw *hw, struct ice_netlist_info *net static int ice_discover_flash_size(struct ice_hw *hw) { u32 min_size = 0, max_size = ICE_AQC_NVM_MAX_OFFSET + 1; - int status; - - status = ice_acquire_nvm(hw, ICE_RES_READ); - if (status) - return status; + int status = 0; while ((max_size - min_size) > 1) { + enum libie_aq_err read_aq_err = LIBIE_AQ_RC_OK; u32 offset = (max_size + min_size) / 2; u32 len = 1; u8 data; - status = ice_read_flat_nvm(hw, offset, &len, &data, false); + status = ice_read_flat_nvm(hw, offset, &len, &data, false, + &read_aq_err); if (status == -EIO && - hw->adminq.sq_last_status == LIBIE_AQ_RC_EINVAL) { + read_aq_err == LIBIE_AQ_RC_EINVAL) { ice_debug(hw, ICE_DBG_NVM, "%s: New upper bound of %u bytes\n", __func__, offset); status = 0; @@ -880,7 +885,7 @@ static int ice_discover_flash_size(struct ice_hw *hw) min_size = offset; } else { /* an unexpected error occurred */ - goto err_read_flat_nvm; + return status; } } @@ -888,9 +893,6 @@ static int ice_discover_flash_size(struct ice_hw *hw) hw->flash.flash_size = max_size; -err_read_flat_nvm: - ice_release_nvm(hw); - return status; } diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h index 63cdc6bdac589..e1d1a11f5ca41 100644 --- a/drivers/net/ethernet/intel/ice/ice_nvm.h +++ b/drivers/net/ethernet/intel/ice/ice_nvm.h @@ -19,7 +19,7 @@ int ice_aq_read_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset, bool read_shadow_ram, struct ice_sq_cd *cd); int ice_read_flat_nvm(struct ice_hw *hw, u32 offset, u32 *length, u8 *data, - bool read_shadow_ram); + bool read_shadow_ram, enum libie_aq_err *read_aq_err); int ice_get_pfa_module_tlv(struct ice_hw *hw, u16 *module_tlv, u16 *module_tlv_len, u16 module_type); diff --git a/drivers/net/ethernet/intel/ice/ice_parser.c b/drivers/net/ethernet/intel/ice/ice_parser.c index 664beb64f5570..0c240e90a54e1 100644 --- a/drivers/net/ethernet/intel/ice/ice_parser.c +++ b/drivers/net/ethernet/intel/ice/ice_parser.c @@ -2368,6 +2368,9 @@ int ice_parser_profile_init(struct ice_parser_result *rslt, u16 proto_off = 0; u16 off; + if (rslt->ptype >= ICE_FLOW_PTYPE_MAX) + return -EINVAL; + memset(prof, 0, sizeof(*prof)); set_bit(rslt->ptype, prof->ptypes); if (blk == ICE_BLK_SW) { diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c index 86eb3d0315a27..7808d583b9ee1 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp.c @@ -350,7 +350,7 @@ static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp) return 0; } - return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time, + return ice_ptp_extend_32b_ts(READ_ONCE(pf->ptp.cached_phc_time), (in_tstamp >> 8) & mask); } @@ -2164,8 +2164,7 @@ static int ice_ptp_getcrosststamp(struct ptp_clock_info *info, int ice_ptp_hwtstamp_get(struct net_device *netdev, struct kernel_hwtstamp_config *config) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); if (pf->ptp.state != ICE_PTP_READY) return -EIO; @@ -2236,8 +2235,7 @@ int ice_ptp_hwtstamp_set(struct net_device *netdev, struct kernel_hwtstamp_config *config, struct netlink_ext_ack *extack) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); int err; if (pf->ptp.state != ICE_PTP_READY) @@ -2994,6 +2992,11 @@ void ice_ptp_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) struct ice_ptp *ptp = &pf->ptp; int err; + if (ptp->state == ICE_PTP_UNINIT) { + dev_dbg(ice_pf_to_dev(pf), "PTP was not initialized, skipping rebuild\n"); + return; + } + if (ptp->state == ICE_PTP_READY) { ice_ptp_prepare_for_reset(pf, reset_type); } else if (ptp->state != ICE_PTP_RESETTING) { diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c index 99bf38cf352a2..f67e53c5dc87f 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c @@ -4771,15 +4771,12 @@ static int ice_ptp_prep_phy_adj_ll_e810(struct ice_hw *hw, s32 adj) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer adjustment using low latency interface\n"); + + return err; } /** @@ -4800,8 +4797,12 @@ static int ice_ptp_prep_phy_adj_e810(struct ice_hw *hw, s32 adj) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_adj_ll_e810(hw, adj); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL adj timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; @@ -4864,15 +4865,12 @@ static int ice_ptp_prep_phy_incval_ll_e810(struct ice_hw *hw, u64 incval) !FIELD_GET(REG_LL_PROXY_H_EXEC, val), 10, REG_LL_PROXY_H_TIMEOUT_US, false, hw, REG_LL_PROXY_H); - if (err) { - ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); - spin_unlock_irq(¶ms->atqbal_wq.lock); - return err; - } - spin_unlock_irq(¶ms->atqbal_wq.lock); - return 0; + if (err) + ice_debug(hw, ICE_DBG_PTP, "Failed to prepare PHY timer increment using low latency interface\n"); + + return err; } /** @@ -4890,8 +4888,12 @@ static int ice_ptp_prep_phy_incval_e810(struct ice_hw *hw, u64 incval) u8 tmr_idx; int err; - if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) - return ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (hw->dev_caps.ts_dev_info.ll_phy_tmr_update) { + err = ice_ptp_prep_phy_incval_ll_e810(hw, incval); + if (err != -ETIMEDOUT) + return err; + ice_debug(hw, ICE_DBG_PTP, "LL incval timed out, falling back to SBQ\n"); + } tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; low = lower_32_bits(incval); diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index fff0c1afdb414..ffa18d86729a0 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -371,7 +371,7 @@ void ice_free_sched_node(struct ice_port_info *pi, struct ice_sched_node *node) devm_kfree(ice_hw_to_dev(hw), node->children); kfree(node->name); - xa_erase(&pi->sched_node_ids, node->id); + xa_erase(&hw->sched_node_ids, node->id); devm_kfree(ice_hw_to_dev(hw), node); } @@ -977,7 +977,7 @@ ice_sched_add_elems(struct ice_port_info *pi, struct ice_sched_node *tc_node, if (!new_node->name) return -ENOMEM; - status = xa_alloc(&pi->sched_node_ids, &new_node->id, NULL, XA_LIMIT(0, UINT_MAX), + status = xa_alloc(&hw->sched_node_ids, &new_node->id, NULL, XA_LIMIT(0, UINT_MAX), GFP_KERNEL); if (status) { ice_debug(hw, ICE_DBG_SCHED, "xa_alloc failed for sched node status =%d\n", diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c index 843e82fd3bf93..38f8356470131 100644 --- a/drivers/net/ethernet/intel/ice/ice_sriov.c +++ b/drivers/net/ethernet/intel/ice/ice_sriov.c @@ -484,12 +484,14 @@ static int ice_start_vfs(struct ice_pf *pf) goto teardown; } - retval = ice_eswitch_attach_vf(pf, vf); - if (retval) { - dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d", - vf->vf_id, retval); - ice_vf_vsi_release(vf); - goto teardown; + if (ice_is_eswitch_mode_switchdev(pf)) { + retval = ice_eswitch_attach_vf(pf, vf); + if (retval) { + dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d", + vf->vf_id, retval); + ice_vf_vsi_release(vf); + goto teardown; + } } set_bit(ICE_VF_STATE_INIT, vf->vf_states); @@ -1190,8 +1192,7 @@ ice_vf_lan_overflow_event(struct ice_pf *pf, struct ice_rq_event_info *event) */ int ice_set_vf_spoofchk(struct net_device *netdev, int vf_id, bool ena) { - struct ice_netdev_priv *np = netdev_priv(netdev); - struct ice_pf *pf = np->vsi->back; + struct ice_pf *pf = ice_netdev_to_pf(netdev); struct ice_vsi *vf_vsi; struct device *dev; struct ice_vf *vf; diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h index 4f35ef8d6b299..7568c917cdbe1 100644 --- a/drivers/net/ethernet/intel/ice/ice_trace.h +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -63,23 +63,33 @@ DECLARE_EVENT_CLASS(ice_rx_dim_template, TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), TP_ARGS(q_vector, dim), - TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) - __field(struct dim *, dim) + TP_STRUCT__entry(__field(u16, q_index) + __field(u8, state) + __field(u8, profile_ix) + __field(u8, tune_state) + __field(u8, steps_right) + __field(u8, steps_left) + __field(u8, tired) __string(devname, q_vector->rx.rx_ring->netdev->name)), - TP_fast_assign(__entry->q_vector = q_vector; - __entry->dim = dim; + TP_fast_assign(__entry->q_index = q_vector->rx.rx_ring->q_index; + __entry->state = dim->state; + __entry->profile_ix = dim->profile_ix; + __entry->tune_state = dim->tune_state; + __entry->steps_right = dim->steps_right; + __entry->steps_left = dim->steps_left; + __entry->tired = dim->tired; __assign_str(devname);), TP_printk("netdev: %s Rx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", __get_str(devname), - __entry->q_vector->rx.rx_ring->q_index, - __entry->dim->state, - __entry->dim->profile_ix, - __entry->dim->tune_state, - __entry->dim->steps_right, - __entry->dim->steps_left, - __entry->dim->tired) + __entry->q_index, + __entry->state, + __entry->profile_ix, + __entry->tune_state, + __entry->steps_right, + __entry->steps_left, + __entry->tired) ); DEFINE_EVENT(ice_rx_dim_template, ice_rx_dim_work, @@ -90,23 +100,33 @@ DEFINE_EVENT(ice_rx_dim_template, ice_rx_dim_work, DECLARE_EVENT_CLASS(ice_tx_dim_template, TP_PROTO(struct ice_q_vector *q_vector, struct dim *dim), TP_ARGS(q_vector, dim), - TP_STRUCT__entry(__field(struct ice_q_vector *, q_vector) - __field(struct dim *, dim) + TP_STRUCT__entry(__field(u16, q_index) + __field(u8, state) + __field(u8, profile_ix) + __field(u8, tune_state) + __field(u8, steps_right) + __field(u8, steps_left) + __field(u8, tired) __string(devname, q_vector->tx.tx_ring->netdev->name)), - TP_fast_assign(__entry->q_vector = q_vector; - __entry->dim = dim; + TP_fast_assign(__entry->q_index = q_vector->tx.tx_ring->q_index; + __entry->state = dim->state; + __entry->profile_ix = dim->profile_ix; + __entry->tune_state = dim->tune_state; + __entry->steps_right = dim->steps_right; + __entry->steps_left = dim->steps_left; + __entry->tired = dim->tired; __assign_str(devname);), TP_printk("netdev: %s Tx-Q: %d dim-state: %d dim-profile: %d dim-tune: %d dim-st-right: %d dim-st-left: %d dim-tired: %d", __get_str(devname), - __entry->q_vector->tx.tx_ring->q_index, - __entry->dim->state, - __entry->dim->profile_ix, - __entry->dim->tune_state, - __entry->dim->steps_right, - __entry->dim->steps_left, - __entry->dim->tired) + __entry->q_index, + __entry->state, + __entry->profile_ix, + __entry->tune_state, + __entry->steps_right, + __entry->steps_left, + __entry->tired) ); DEFINE_EVENT(ice_tx_dim_template, ice_tx_dim_work, diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 73f08d02f9c76..b843f66c4a6e0 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -2081,7 +2081,7 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) ret = ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto, &frag_off); if (ret < 0) - return -1; + goto checksum_sw_fb; } /* define outer transport */ @@ -2100,11 +2100,7 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) l4.hdr = skb_inner_network_header(skb); break; default: - if (first->tx_flags & ICE_TX_FLAGS_TSO) - return -1; - - skb_checksum_help(skb); - return 0; + goto checksum_sw_fb; } /* compute outer L3 header size */ @@ -2163,7 +2159,7 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto, &frag_off); } else { - return -1; + goto checksum_sw_fb; } /* compute inner L3 header size */ @@ -2216,15 +2212,17 @@ int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off) break; default: - if (first->tx_flags & ICE_TX_FLAGS_TSO) - return -1; - skb_checksum_help(skb); - return 0; + goto checksum_sw_fb; } off->td_cmd |= cmd; off->td_offset |= offset; return 1; + +checksum_sw_fb: + if (first->tx_flags & ICE_TX_FLAGS_TSO) + return -1; + return skb_checksum_help(skb); } /** diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index b0a1b67071c5e..94b807a066ecc 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -759,7 +759,6 @@ struct ice_port_info { /* List contain profile ID(s) and other params per layer */ struct list_head rl_prof_list[ICE_AQC_TOPO_MAX_LEVEL_NUM]; struct ice_qos_cfg qos_cfg; - struct xarray sched_node_ids; u8 is_vf:1; u8 is_custom_tx_enabled:1; }; @@ -922,6 +921,7 @@ struct ice_hw { u8 sw_entry_point_layer; u16 max_children[ICE_AQC_TOPO_MAX_LEVEL_NUM]; struct list_head agg_list; /* lists all aggregator */ + struct xarray sched_node_ids; struct ice_vsi_ctx *vsi_ctx[ICE_MAX_VSI]; u8 evb_veb; /* true for VEB, false for VEPA */ diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index 4f86412a9c0c3..54f59bed9645e 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -812,7 +812,8 @@ void ice_reset_all_vfs(struct ice_pf *pf) } ice_vf_post_vsi_rebuild(vf); - ice_eswitch_attach_vf(pf, vf); + if (ice_is_eswitch_mode_switchdev(pf)) + ice_eswitch_attach_vf(pf, vf); mutex_unlock(&vf->cfg_lock); } @@ -847,6 +848,30 @@ static void ice_notify_vf_reset(struct ice_vf *vf) NULL); } +/** + * ice_reset_interrupts - clear all queue interrupt configuration for a VSI + * @vsi: the VSI whose interrupt registers should be cleared + * + * Zero the QINT_RQCTL and QINT_TQCTL registers for all allocated queues + * in the VSI. This clears the entire register including MSIX_INDX, ITR_INDX, + * CAUSE_ENA and NEXTQ fields, unlike ice_vf_dis_rxq_interrupt() which only + * clears the CAUSE_ENA bit. + */ +void ice_reset_interrupts(struct ice_vsi *vsi) +{ + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + int i; + + ice_for_each_alloc_rxq(vsi, i) + wr32(hw, QINT_RQCTL(vsi->rxq_map[i]), 0); + + ice_for_each_alloc_txq(vsi, i) + wr32(hw, QINT_TQCTL(vsi->txq_map[i]), 0); + + ice_flush(hw); +} + /** * ice_reset_vf - Reset a particular VF * @vf: pointer to the VF structure @@ -918,6 +943,9 @@ int ice_reset_vf(struct ice_vf *vf, u32 flags) ice_dis_vf_qs(vf); + /* cleanup interrupt registers */ + ice_reset_interrupts(vsi); + /* Call Disable LAN Tx queue AQ whether or not queues are * enabled. This is needed for successful completion of VFR. */ diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib_private.h b/drivers/net/ethernet/intel/ice/ice_vf_lib_private.h index 5392b04049862..321d29c25b7c1 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib_private.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib_private.h @@ -26,6 +26,7 @@ void ice_initialize_vf_entry(struct ice_vf *vf); void ice_deinitialize_vf_entry(struct ice_vf *vf); void ice_dis_vf_qs(struct ice_vf *vf); +void ice_reset_interrupts(struct ice_vsi *vsi); int ice_check_vf_init(struct ice_vf *vf); enum virtchnl_status_code ice_err_to_virt_err(int err); struct ice_port_info *ice_vf_get_port_info(struct ice_vf *vf); diff --git a/drivers/net/ethernet/intel/ice/virt/queues.c b/drivers/net/ethernet/intel/ice/virt/queues.c index a82024bf388bf..0765ee901b7c7 100644 --- a/drivers/net/ethernet/intel/ice/virt/queues.c +++ b/drivers/net/ethernet/intel/ice/virt/queues.c @@ -224,6 +224,24 @@ void ice_vf_ena_rxq_interrupt(struct ice_vsi *vsi, u32 q_idx) wr32(hw, QINT_RQCTL(pfq), reg | QINT_RQCTL_CAUSE_ENA_M); } +/** + * ice_vf_dis_rxq_interrupt - disable Rx queue interrupt via QINT_RQCTL + * @vsi: VSI of the VF to configure + * @q_idx: VF queue index used to determine the queue in the PF's space + */ +static void ice_vf_dis_rxq_interrupt(struct ice_vsi *vsi, u32 q_idx) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 pfq = vsi->rxq_map[q_idx]; + u32 reg; + + reg = rd32(hw, QINT_RQCTL(pfq)); + reg &= ~QINT_RQCTL_CAUSE_ENA_M; + wr32(hw, QINT_RQCTL(pfq), reg); + + ice_flush(hw); +} + /** * ice_vc_ena_qs_msg * @vf: pointer to the VF info @@ -416,6 +434,8 @@ int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg) goto error_param; } + for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) + ice_vf_dis_rxq_interrupt(vsi, vf_q_id); bitmap_zero(vf->rxq_ena, ICE_MAX_RSS_QS_PER_VF); } else if (q_map) { for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { @@ -436,6 +456,7 @@ int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg) goto error_param; } + ice_vf_dis_rxq_interrupt(vsi, vf_q_id); /* Clear enabled queues flag */ clear_bit(vf_q_id, vf->rxq_ena); } diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index c74c47bc0b9b9..562fe26ae4629 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -8,6 +8,7 @@ struct idpf_adapter; struct idpf_vport; struct idpf_vport_max_q; +struct idpf_q_vec_rsrc; #include #include @@ -201,7 +202,8 @@ struct idpf_vport_max_q { struct idpf_reg_ops { void (*ctlq_reg_init)(struct idpf_adapter *adapter, struct idpf_ctlq_create_info *cq); - int (*intr_reg_init)(struct idpf_vport *vport); + int (*intr_reg_init)(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); void (*mb_intr_reg_init)(struct idpf_adapter *adapter); void (*reset_reg_init)(struct idpf_adapter *adapter); void (*trigger_reset)(struct idpf_adapter *adapter, @@ -287,8 +289,28 @@ struct idpf_fsteer_fltr { struct ethtool_rx_flow_spec fs; }; +/** + * struct idpf_q_vec_rsrc - handle for queue and vector resources + * @q_vectors: array of queue vectors + * @q_vector_idxs: starting index of queue vectors + * @num_q_vectors: number of IRQ vectors allocated + * @noirq_v_idx: ID of the NOIRQ vector + * @noirq_dyn_ctl_ena: value to write to the above to enable it + * @noirq_dyn_ctl: register to enable/disable the vector for NOIRQ queues + */ +struct idpf_q_vec_rsrc { + struct idpf_q_vector *q_vectors; + u16 *q_vector_idxs; + u16 num_q_vectors; + u16 noirq_v_idx; + u32 noirq_dyn_ctl_ena; + void __iomem *noirq_dyn_ctl; + +}; + /** * struct idpf_vport - Handle for netdevices and queue resources + * @dflt_qv_rsrc: contains default queue and vector resources * @num_txq: Number of allocated TX queues * @num_complq: Number of allocated completion queues * @txq_desc_count: TX queue descriptor count @@ -325,12 +347,6 @@ struct idpf_fsteer_fltr { * @idx: Software index in adapter vports struct * @default_vport: Use this vport if one isn't specified * @base_rxd: True if the driver should use base descriptors instead of flex - * @num_q_vectors: Number of IRQ vectors allocated - * @q_vectors: Array of queue vectors - * @q_vector_idxs: Starting index of queue vectors - * @noirq_dyn_ctl: register to enable/disable the vector for NOIRQ queues - * @noirq_dyn_ctl_ena: value to write to the above to enable it - * @noirq_v_idx: ID of the NOIRQ vector * @max_mtu: device given max possible MTU * @default_mac_addr: device will give a default MAC to use * @rx_itr_profile: RX profiles for Dynamic Interrupt Moderation @@ -343,6 +359,7 @@ struct idpf_fsteer_fltr { * @tstamp_stats: Tx timestamping statistics */ struct idpf_vport { + struct idpf_q_vec_rsrc dflt_qv_rsrc; u16 num_txq; u16 num_complq; u32 txq_desc_count; @@ -380,14 +397,6 @@ struct idpf_vport { bool default_vport; bool base_rxd; - u16 num_q_vectors; - struct idpf_q_vector *q_vectors; - u16 *q_vector_idxs; - - void __iomem *noirq_dyn_ctl; - u32 noirq_dyn_ctl_ena; - u16 noirq_v_idx; - u16 max_mtu; u8 default_mac_addr[ETH_ALEN]; u16 rx_itr_profile[IDPF_DIM_PROFILE_SLOTS]; @@ -549,11 +558,38 @@ struct idpf_vector_lifo { u16 *vec_idx; }; +/** + * struct idpf_queue_id_reg_chunk - individual queue ID and register chunk + * @qtail_reg_start: queue tail register offset + * @qtail_reg_spacing: queue tail register spacing + * @type: queue type of the queues in the chunk + * @start_queue_id: starting queue ID in the chunk + * @num_queues: number of queues in the chunk + */ +struct idpf_queue_id_reg_chunk { + u64 qtail_reg_start; + u32 qtail_reg_spacing; + u32 type; + u32 start_queue_id; + u32 num_queues; +}; + +/** + * struct idpf_queue_id_reg_info - queue ID and register chunk info received + * over the mailbox + * @num_chunks: number of chunks + * @queue_chunks: array of chunks + */ +struct idpf_queue_id_reg_info { + u16 num_chunks; + struct idpf_queue_id_reg_chunk *queue_chunks; +}; + /** * struct idpf_vport_config - Vport configuration data * @user_config: see struct idpf_vport_user_config_data * @max_q: Maximum possible queues - * @req_qs_chunks: Queue chunk data for requested queues + * @qid_reg_info: Struct to store the queue ID and register info * @mac_filter_list_lock: Lock to protect mac filters * @flow_steer_list_lock: Lock to protect fsteer filters * @flags: See enum idpf_vport_config_flags @@ -561,7 +597,7 @@ struct idpf_vector_lifo { struct idpf_vport_config { struct idpf_vport_user_config_data user_config; struct idpf_vport_max_q max_q; - struct virtchnl2_add_queues *req_qs_chunks; + struct idpf_queue_id_reg_info qid_reg_info; spinlock_t mac_filter_list_lock; spinlock_t flow_steer_list_lock; DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS); diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c index 3a04a6bd0d7cc..ee93987f90185 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c @@ -70,11 +70,13 @@ static void idpf_mb_intr_reg_init(struct idpf_adapter *adapter) /** * idpf_intr_reg_init - Initialize interrupt registers * @vport: virtual port structure + * @rsrc: pointer to queue and vector resources */ -static int idpf_intr_reg_init(struct idpf_vport *vport) +static int idpf_intr_reg_init(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_adapter *adapter = vport->adapter; - int num_vecs = vport->num_q_vectors; + u16 num_vecs = rsrc->num_q_vectors; struct idpf_vec_regs *reg_vals; int num_regs, i, err = 0; u32 rx_itr, tx_itr, val; @@ -93,8 +95,8 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) } for (i = 0; i < num_vecs; i++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[i]; - u16 vec_id = vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC; + struct idpf_q_vector *q_vector = &rsrc->q_vectors[i]; + u16 vec_id = rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC; struct idpf_intr_reg *intr = &q_vector->intr_reg; u32 spacing; @@ -123,12 +125,12 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) /* Data vector for NOIRQ queues */ - val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; - vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + val = reg_vals[rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; + rsrc->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); val = PF_GLINT_DYN_CTL_WB_ON_ITR_M | PF_GLINT_DYN_CTL_INTENA_MSK_M | FIELD_PREP(PF_GLINT_DYN_CTL_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); - vport->noirq_dyn_ctl_ena = val; + rsrc->noirq_dyn_ctl_ena = val; free_reg_vals: kfree(reg_vals); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 131a8121839bd..fd41d4bba8e7a 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -139,7 +139,7 @@ static int idpf_mb_intr_req_irq(struct idpf_adapter *adapter) if (err) { dev_err(&adapter->pdev->dev, "IRQ request for mailbox failed, error: %d\n", err); - + kfree(name); return err; } @@ -975,6 +975,8 @@ static void idpf_remove_features(struct idpf_vport *vport) static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); + struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; + struct idpf_queue_id_reg_info *chunks; if (!test_bit(IDPF_VPORT_UP, np->state)) return; @@ -985,6 +987,8 @@ static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) netif_carrier_off(vport->netdev); netif_tx_disable(vport->netdev); + chunks = &vport->adapter->vport_config[vport->idx]->qid_reg_info; + idpf_send_disable_vport_msg(vport); idpf_send_disable_queues_msg(vport); idpf_send_map_unmap_queue_vector_msg(vport, false); @@ -994,15 +998,15 @@ static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) * instead of deleting and reallocating the vport. */ if (test_and_clear_bit(IDPF_VPORT_DEL_QUEUES, vport->flags)) - idpf_send_delete_queues_msg(vport); + idpf_send_delete_queues_msg(vport, chunks); idpf_remove_features(vport); vport->link_up = false; - idpf_vport_intr_deinit(vport); + idpf_vport_intr_deinit(vport, rsrc); idpf_xdp_rxq_info_deinit_all(vport); idpf_vport_queues_rel(vport); - idpf_vport_intr_rel(vport); + idpf_vport_intr_rel(rsrc); clear_bit(IDPF_VPORT_UP, np->state); if (rtnl) @@ -1065,6 +1069,7 @@ static void idpf_decfg_netdev(struct idpf_vport *vport) */ static void idpf_vport_rel(struct idpf_vport *vport) { + struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; struct idpf_adapter *adapter = vport->adapter; struct idpf_vport_config *vport_config; struct idpf_vector_info vec_info; @@ -1089,24 +1094,23 @@ static void idpf_vport_rel(struct idpf_vport *vport) /* Release all the allocated vectors on the stack */ vec_info.num_req_vecs = 0; - vec_info.num_curr_vecs = vport->num_q_vectors; + vec_info.num_curr_vecs = rsrc->num_q_vectors; vec_info.default_vport = vport->default_vport; - idpf_req_rel_vector_indexes(adapter, vport->q_vector_idxs, &vec_info); + idpf_req_rel_vector_indexes(adapter, rsrc->q_vector_idxs, &vec_info); - kfree(vport->q_vector_idxs); - vport->q_vector_idxs = NULL; + kfree(rsrc->q_vector_idxs); + rsrc->q_vector_idxs = NULL; + + idpf_vport_deinit_queue_reg_chunks(vport_config); kfree(adapter->vport_params_recvd[idx]); adapter->vport_params_recvd[idx] = NULL; kfree(adapter->vport_params_reqd[idx]); adapter->vport_params_reqd[idx] = NULL; - if (adapter->vport_config[idx]) { - kfree(adapter->vport_config[idx]->req_qs_chunks); - adapter->vport_config[idx]->req_qs_chunks = NULL; - } kfree(vport->rx_ptype_lkup); vport->rx_ptype_lkup = NULL; + kfree(vport); adapter->num_alloc_vports--; } @@ -1224,6 +1228,7 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, { struct idpf_rss_data *rss_data; u16 idx = adapter->next_vport; + struct idpf_q_vec_rsrc *rsrc; struct idpf_vport *vport; u16 num_max_q; int err; @@ -1271,11 +1276,14 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, vport->default_vport = adapter->num_alloc_vports < idpf_get_default_vports(adapter); - vport->q_vector_idxs = kcalloc(num_max_q, sizeof(u16), GFP_KERNEL); - if (!vport->q_vector_idxs) + rsrc = &vport->dflt_qv_rsrc; + rsrc->q_vector_idxs = kcalloc(num_max_q, sizeof(u16), GFP_KERNEL); + if (!rsrc->q_vector_idxs) goto free_vport; - idpf_vport_init(vport, max_q); + err = idpf_vport_init(vport, max_q); + if (err) + goto free_vector_idxs; /* LUT and key are both initialized here. Key is not strictly dependent * on how many queues we have. If we change number of queues and soft @@ -1286,7 +1294,7 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, rss_data = &adapter->vport_config[idx]->user_config.rss_data; rss_data->rss_key = kzalloc(rss_data->rss_key_size, GFP_KERNEL); if (!rss_data->rss_key) - goto free_vector_idxs; + goto free_qreg_chunks; /* Initialize default rss key */ netdev_rss_key_fill((void *)rss_data->rss_key, rss_data->rss_key_size); @@ -1308,8 +1316,10 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, free_rss_key: kfree(rss_data->rss_key); +free_qreg_chunks: + idpf_vport_deinit_queue_reg_chunks(adapter->vport_config[idx]); free_vector_idxs: - kfree(vport->q_vector_idxs); + kfree(rsrc->q_vector_idxs); free_vport: kfree(vport); @@ -1483,7 +1493,10 @@ static void idpf_rx_init_buf_tail(struct idpf_vport *vport) static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); + struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; struct idpf_adapter *adapter = vport->adapter; + struct idpf_vport_config *vport_config; + struct idpf_queue_id_reg_info *chunks; int err; if (test_bit(IDPF_VPORT_UP, np->state)) @@ -1495,7 +1508,7 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) /* we do not allow interface up just yet */ netif_carrier_off(vport->netdev); - err = idpf_vport_intr_alloc(vport); + err = idpf_vport_intr_alloc(vport, rsrc); if (err) { dev_err(&adapter->pdev->dev, "Failed to allocate interrupts for vport %u: %d\n", vport->vport_id, err); @@ -1506,21 +1519,24 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) if (err) goto intr_rel; - err = idpf_vport_queue_ids_init(vport); + vport_config = adapter->vport_config[vport->idx]; + chunks = &vport_config->qid_reg_info; + + err = idpf_vport_queue_ids_init(vport, chunks); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize queue ids for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } - err = idpf_vport_intr_init(vport); + err = idpf_vport_intr_init(vport, rsrc); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize interrupts for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } - err = idpf_queue_reg_init(vport); + err = idpf_queue_reg_init(vport, chunks); if (err) { dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", vport->vport_id, err); @@ -1544,7 +1560,7 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) goto intr_deinit; } - idpf_vport_intr_ena(vport); + idpf_vport_intr_ena(vport, rsrc); err = idpf_send_config_queues_msg(vport); if (err) { @@ -1605,11 +1621,11 @@ static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) rxq_deinit: idpf_xdp_rxq_info_deinit_all(vport); intr_deinit: - idpf_vport_intr_deinit(vport); + idpf_vport_intr_deinit(vport, rsrc); queues_rel: idpf_vport_queues_rel(vport); intr_rel: - idpf_vport_intr_rel(vport); + idpf_vport_intr_rel(rsrc); err_rtnl_unlock: if (rtnl) @@ -1997,8 +2013,9 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, struct idpf_netdev_priv *np = netdev_priv(vport->netdev); bool vport_is_up = test_bit(IDPF_VPORT_UP, np->state); struct idpf_adapter *adapter = vport->adapter; + struct idpf_vport_config *vport_config; struct idpf_vport *new_vport; - int err; + int err, tmp_err = 0; /* If the system is low on memory, we can end up in bad state if we * free all the memory for queue resources and try to allocate them @@ -2046,8 +2063,10 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, goto free_vport; } + vport_config = adapter->vport_config[vport->idx]; + if (!vport_is_up) { - idpf_send_delete_queues_msg(vport); + idpf_send_delete_queues_msg(vport, &vport_config->qid_reg_info); } else { set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags); idpf_vport_stop(vport, false); @@ -2072,7 +2091,7 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, memcpy(vport, new_vport, offsetof(struct idpf_vport, link_up)); if (reset_cause == IDPF_SR_Q_CHANGE) - idpf_vport_alloc_vec_indexes(vport); + idpf_vport_alloc_vec_indexes(vport, &vport->dflt_qv_rsrc); err = idpf_set_real_num_queues(vport); if (err) @@ -2088,11 +2107,12 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, goto free_vport; err_reset: - idpf_send_add_queues_msg(vport, vport->num_txq, vport->num_complq, - vport->num_rxq, vport->num_bufq); + tmp_err = idpf_send_add_queues_msg(vport, vport->num_txq, + vport->num_complq, vport->num_rxq, + vport->num_bufq); err_open: - if (vport_is_up) + if (!tmp_err && vport_is_up) idpf_vport_open(vport, false); free_vport: diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index c859665b2dc89..8042254f08856 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -19,6 +19,8 @@ LIBETH_SQE_CHECK_PRIV(u32); * Make sure we don't exceed maximum scatter gather buffers for a single * packet. * TSO case has been handled earlier from idpf_features_check(). + * + * Return: %true if skb exceeds max descriptors per packet, %false otherwise. */ static bool idpf_chk_linearize(const struct sk_buff *skb, unsigned int max_bufs, @@ -172,7 +174,7 @@ static void idpf_tx_desc_rel_all(struct idpf_vport *vport) * idpf_tx_buf_alloc_all - Allocate memory for all buffer resources * @tx_q: queue for which the buffers are allocated * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) { @@ -196,7 +198,7 @@ static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) * @vport: vport to allocate resources for * @tx_q: the tx ring to set up * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, struct idpf_tx_queue *tx_q) @@ -297,7 +299,7 @@ static int idpf_compl_desc_alloc(const struct idpf_vport *vport, * idpf_tx_desc_alloc_all - allocate all queues Tx resources * @vport: virtual port private structure * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_tx_desc_alloc_all(struct idpf_vport *vport) { @@ -548,7 +550,7 @@ static void idpf_rx_buf_hw_update(struct idpf_buf_queue *bufq, u32 val) * idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers * @bufq: ring to use * - * Returns 0 on success, negative on failure. + * Return: 0 on success, negative on failure. */ static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq) { @@ -600,7 +602,7 @@ static void idpf_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id) * @bufq: buffer queue to post to * @buf_id: buffer id to post * - * Returns false if buffer could not be allocated, true otherwise. + * Return: %false if buffer could not be allocated, %true otherwise. */ static bool idpf_rx_post_buf_desc(struct idpf_buf_queue *bufq, u16 buf_id) { @@ -649,7 +651,7 @@ static bool idpf_rx_post_buf_desc(struct idpf_buf_queue *bufq, u16 buf_id) * @bufq: buffer queue to post working set to * @working_set: number of buffers to put in working set * - * Returns true if @working_set bufs were posted successfully, false otherwise. + * Return: %true if @working_set bufs were posted successfully, %false otherwise. */ static bool idpf_rx_post_init_bufs(struct idpf_buf_queue *bufq, u16 working_set) @@ -718,7 +720,7 @@ static int idpf_rx_bufs_init_singleq(struct idpf_rx_queue *rxq) * idpf_rx_buf_alloc_all - Allocate memory for all buffer resources * @rxbufq: queue for which the buffers are allocated * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_rx_buf_alloc_all(struct idpf_buf_queue *rxbufq) { @@ -746,7 +748,7 @@ static int idpf_rx_buf_alloc_all(struct idpf_buf_queue *rxbufq) * @bufq: buffer queue to create page pool for * @type: type of Rx buffers to allocate * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, enum libeth_fqe_type type) @@ -781,7 +783,7 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, * idpf_rx_bufs_init_all - Initialize all RX bufs * @vport: virtual port struct * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ int idpf_rx_bufs_init_all(struct idpf_vport *vport) { @@ -836,7 +838,7 @@ int idpf_rx_bufs_init_all(struct idpf_vport *vport) * @vport: vport to allocate resources for * @rxq: Rx queue for which the resources are setup * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, struct idpf_rx_queue *rxq) @@ -898,7 +900,7 @@ static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, * idpf_rx_desc_alloc_all - allocate all RX queues resources * @vport: virtual port structure * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_rx_desc_alloc_all(struct idpf_vport *vport) { @@ -1121,7 +1123,7 @@ idpf_vector_to_queue_set(struct idpf_q_vector *qv) if (!num) return NULL; - qs = idpf_alloc_queue_set(vport, num); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, num); if (!qs) return NULL; @@ -1198,7 +1200,7 @@ static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid) goto config; q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp, - vport->num_q_vectors), + qs->qv_rsrc->num_q_vectors), sizeof(*q_vector->xsksq), GFP_KERNEL); if (!q_vector->xsksq) return -ENOMEM; @@ -1426,7 +1428,7 @@ void idpf_vport_queues_rel(struct idpf_vport *vport) * dereference the queue from queue groups. This allows us to quickly pull a * txq based on a queue index. * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport) { @@ -1559,7 +1561,7 @@ void idpf_vport_calc_num_q_desc(struct idpf_vport *vport) * @vport_msg: message to fill with data * @max_q: vport max queue info * - * Return 0 on success, error value on failure. + * Return: 0 on success, error value on failure. */ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, struct virtchnl2_create_vport *vport_msg, @@ -1694,7 +1696,7 @@ static void idpf_rxq_set_descids(const struct idpf_vport *vport, * @vport: vport to allocate txq groups for * @num_txq: number of txqs to allocate for each group * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) { @@ -1786,7 +1788,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) * @vport: vport to allocate rxq groups for * @num_rxq: number of rxqs to allocate for each group * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq) { @@ -1915,7 +1917,7 @@ static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq) * idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources * @vport: vport with qgrps to allocate * - * Returns 0 on success, negative on failure + * Return: 0 on success, negative on failure */ static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport) { @@ -1944,8 +1946,9 @@ static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport) * idpf_vport_queues_alloc - Allocate memory for all queues * @vport: virtual port * - * Allocate memory for queues associated with a vport. Returns 0 on success, - * negative on failure. + * Allocate memory for queues associated with a vport. + * + * Return: 0 on success, negative on failure. */ int idpf_vport_queues_alloc(struct idpf_vport *vport) { @@ -2172,7 +2175,7 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * - * Returns true if there's any budget left (e.g. the clean is finished) + * Return: %true if there's any budget left (e.g. the clean is finished) */ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, int *cleaned) @@ -2392,13 +2395,13 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, struct idpf_tx_splitq_params *params, u16 td_cmd, u16 size) { - *(u32 *)&desc->flow.qw1.cmd_dtype = (u8)(params->dtype | td_cmd); + *(__le32 *)&desc->flow.qw1.cmd_dtype = cpu_to_le32((u8)(params->dtype | td_cmd)); desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size); desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag); } /** - * idpf_tx_splitq_has_room - check if enough Tx splitq resources are available + * idpf_txq_has_room - check if enough Tx splitq resources are available * @tx_q: the queue to be checked * @descs_needed: number of descriptors required for this packet * @bufs_needed: number of Tx buffers required for this packet @@ -2529,6 +2532,8 @@ unsigned int idpf_tx_res_count_required(struct idpf_tx_queue *txq, * idpf_tx_splitq_bump_ntu - adjust NTU and generation * @txq: the tx ring to wrap * @ntu: ring index to bump + * + * Return: the next ring index hopping to 0 when wraps around */ static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_tx_queue *txq, u16 ntu) { @@ -2797,7 +2802,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, * @skb: pointer to skb * @off: pointer to struct that holds offload parameters * - * Returns error (negative) if TSO was requested but cannot be applied to the + * Return: error (negative) if TSO was requested but cannot be applied to the * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise. */ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) @@ -2875,6 +2880,8 @@ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) * * Since the TX buffer rings mimics the descriptor ring, update the tx buffer * ring entry to reflect that this index is a context descriptor + * + * Return: pointer to the next descriptor */ static union idpf_flex_tx_ctx_desc * idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) @@ -2893,6 +2900,8 @@ idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) * idpf_tx_drop_skb - free the SKB and bump tail if necessary * @tx_q: queue to send buffer on * @skb: pointer to skb + * + * Return: always NETDEV_TX_OK */ netdev_tx_t idpf_tx_drop_skb(struct idpf_tx_queue *tx_q, struct sk_buff *skb) { @@ -2994,7 +3003,7 @@ static bool idpf_tx_splitq_need_re(struct idpf_tx_queue *tx_q) * @skb: send buffer * @tx_q: queue to send buffer on * - * Returns NETDEV_TX_OK if sent, else an error code + * Return: NETDEV_TX_OK if sent, else an error code */ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q) @@ -3075,10 +3084,7 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE; tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP; - /* Set the RE bit to periodically "clean" the descriptor ring. - * MIN_GAP is set to MIN_RING size to ensure it will be set at - * least once each time around the ring. - */ + /* Set the RE bit periodically to "clean" the descriptor ring */ if (idpf_tx_splitq_need_re(tx_q)) { tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE; tx_q->txq_grp->num_completions_pending++; @@ -3120,7 +3126,7 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, * @skb: send buffer * @netdev: network interface device structure * - * Returns NETDEV_TX_OK if sent, else an error code + * Return: NETDEV_TX_OK if sent, else an error code */ netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) { @@ -3270,16 +3276,17 @@ idpf_rx_splitq_extract_csum_bits(const struct virtchnl2_rx_flex_desc_adv_nic_3 * * @rx_desc: Receive descriptor * @decoded: Decoded Rx packet type related fields * - * Return 0 on success and error code on failure - * * Populate the skb fields with the total number of RSC segments, RSC payload * length and packet type. + * + * Return: 0 on success and error code on failure */ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc, struct libeth_rx_pt decoded) { u16 rsc_segments, rsc_seg_len; + u16 l3_start = 0; bool ipv4, ipv6; int len; @@ -3302,7 +3309,10 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, NAPI_GRO_CB(skb)->count = rsc_segments; skb_shinfo(skb)->gso_size = rsc_seg_len; - skb_reset_network_header(skb); + if (unlikely(eth_type_vlan(skb->protocol))) + l3_start = VLAN_HLEN; + + skb_set_network_header(skb, l3_start); if (ipv4) { struct iphdr *ipv4h = ip_hdr(skb); @@ -3310,7 +3320,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; /* Reset and set transport header offset in skb */ - skb_set_transport_header(skb, sizeof(struct iphdr)); + skb_set_transport_header(skb, l3_start + sizeof(struct iphdr)); len = skb->len - skb_transport_offset(skb); /* Compute the TCP pseudo header checksum*/ @@ -3320,7 +3330,7 @@ static int idpf_rx_rsc(struct idpf_rx_queue *rxq, struct sk_buff *skb, struct ipv6hdr *ipv6h = ipv6_hdr(skb); skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; - skb_set_transport_header(skb, sizeof(struct ipv6hdr)); + skb_set_transport_header(skb, l3_start + sizeof(struct ipv6hdr)); len = skb->len - skb_transport_offset(skb); tcp_hdr(skb)->check = ~tcp_v6_check(len, &ipv6h->saddr, &ipv6h->daddr, 0); @@ -3371,6 +3381,8 @@ idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq, * This function checks the ring, descriptor, and packet information in * order to populate the hash, checksum, protocol, and * other fields within the skb. + * + * Return: 0 on success and error code on failure */ static int __idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, @@ -3465,6 +3477,7 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr, * @stat_err_field: field from descriptor to test bits in * @stat_err_bits: value to mask * + * Return: %true if any of given @stat_err_bits are set, %false otherwise. */ static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field, const u8 stat_err_bits) @@ -3476,8 +3489,8 @@ static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field, * idpf_rx_splitq_is_eop - process handling of EOP buffers * @rx_desc: Rx descriptor for current buffer * - * If the buffer is an EOP buffer, this function exits returning true, - * otherwise return false indicating that this is in fact a non-EOP buffer. + * Return: %true if the buffer is an EOP buffer, %false otherwise, indicating + * that this is in fact a non-EOP buffer. */ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) { @@ -3496,7 +3509,7 @@ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_de * expensive overhead for IOMMU access this provides a means of avoiding * it by maintaining the mapping of the page to the system. * - * Returns amount of work completed + * Return: amount of work completed */ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { @@ -3626,7 +3639,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) * @buf_id: buffer ID * @buf_desc: Buffer queue descriptor * - * Return 0 on success and negative on failure. + * Return: 0 on success and negative on failure. */ static int idpf_rx_update_bufq_desc(struct idpf_buf_queue *bufq, u32 buf_id, struct virtchnl2_splitq_rx_buf_desc *buf_desc) @@ -3753,6 +3766,7 @@ static void idpf_rx_clean_refillq_all(struct idpf_buf_queue *bufq, int nid) * @irq: interrupt number * @data: pointer to a q_vector * + * Return: always IRQ_HANDLED */ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, void *data) @@ -3767,39 +3781,34 @@ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, /** * idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport - * @vport: virtual port structure - * + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport) +static void idpf_vport_intr_napi_del_all(struct idpf_q_vec_rsrc *rsrc) { - u16 v_idx; - - for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) - netif_napi_del(&vport->q_vectors[v_idx].napi); + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) + netif_napi_del(&rsrc->q_vectors[v_idx].napi); } /** * idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport - * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport) +static void idpf_vport_intr_napi_dis_all(struct idpf_q_vec_rsrc *rsrc) { - int v_idx; - - for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) - napi_disable(&vport->q_vectors[v_idx].napi); + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) + napi_disable(&rsrc->q_vectors[v_idx].napi); } /** * idpf_vport_intr_rel - Free memory allocated for interrupt vectors - * @vport: virtual port + * @rsrc: pointer to queue and vector resources * * Free the memory allocated for interrupt vectors associated to a vport */ -void idpf_vport_intr_rel(struct idpf_vport *vport) +void idpf_vport_intr_rel(struct idpf_q_vec_rsrc *rsrc) { - for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[v_idx]; kfree(q_vector->xsksq); q_vector->xsksq = NULL; @@ -3813,8 +3822,8 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) q_vector->rx = NULL; } - kfree(vport->q_vectors); - vport->q_vectors = NULL; + kfree(rsrc->q_vectors); + rsrc->q_vectors = NULL; } static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link) @@ -3834,21 +3843,22 @@ static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link) /** * idpf_vport_intr_rel_irq - Free the IRQ association with the OS * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_rel_irq(struct idpf_vport *vport) +static void idpf_vport_intr_rel_irq(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_adapter *adapter = vport->adapter; - int vector; - for (vector = 0; vector < vport->num_q_vectors; vector++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[vector]; + for (u16 vector = 0; vector < rsrc->num_q_vectors; vector++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[vector]; int irq_num, vidx; /* free only the irqs that were actually requested */ if (!q_vector) continue; - vidx = vport->q_vector_idxs[vector]; + vidx = rsrc->q_vector_idxs[vector]; irq_num = adapter->msix_entries[vidx].vector; idpf_q_vector_set_napi(q_vector, false); @@ -3858,22 +3868,23 @@ static void idpf_vport_intr_rel_irq(struct idpf_vport *vport) /** * idpf_vport_intr_dis_irq_all - Disable all interrupt - * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport) +static void idpf_vport_intr_dis_irq_all(struct idpf_q_vec_rsrc *rsrc) { - struct idpf_q_vector *q_vector = vport->q_vectors; - int q_idx; + struct idpf_q_vector *q_vector = rsrc->q_vectors; - writel(0, vport->noirq_dyn_ctl); + writel(0, rsrc->noirq_dyn_ctl); - for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) + for (u16 q_idx = 0; q_idx < rsrc->num_q_vectors; q_idx++) writel(0, q_vector[q_idx].intr_reg.dyn_ctl); } /** * idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings * @q_vector: pointer to q_vector + * + * Return: value to be written back to HW to enable interrupt generation */ static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector) { @@ -4011,8 +4022,12 @@ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) /** * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport * @vport: main vport structure + * @rsrc: pointer to queue and vector resources + * + * Return: 0 on success, negative on failure */ -static int idpf_vport_intr_req_irq(struct idpf_vport *vport) +static int idpf_vport_intr_req_irq(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_adapter *adapter = vport->adapter; const char *drv_name, *if_name, *vec_name; @@ -4021,11 +4036,11 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) drv_name = dev_driver_string(&adapter->pdev->dev); if_name = netdev_name(vport->netdev); - for (vector = 0; vector < vport->num_q_vectors; vector++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[vector]; + for (vector = 0; vector < rsrc->num_q_vectors; vector++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[vector]; char *name; - vidx = vport->q_vector_idxs[vector]; + vidx = rsrc->q_vector_idxs[vector]; irq_num = adapter->msix_entries[vidx].vector; if (q_vector->num_rxq && q_vector->num_txq) @@ -4055,9 +4070,9 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) free_q_irqs: while (--vector >= 0) { - vidx = vport->q_vector_idxs[vector]; + vidx = rsrc->q_vector_idxs[vector]; irq_num = adapter->msix_entries[vidx].vector; - kfree(free_irq(irq_num, &vport->q_vectors[vector])); + kfree(free_irq(irq_num, &rsrc->q_vectors[vector])); } return err; @@ -4086,15 +4101,16 @@ void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx) /** * idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport) +static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { bool dynamic; - int q_idx; u16 itr; - for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) { - struct idpf_q_vector *qv = &vport->q_vectors[q_idx]; + for (u16 q_idx = 0; q_idx < rsrc->num_q_vectors; q_idx++) { + struct idpf_q_vector *qv = &rsrc->q_vectors[q_idx]; /* Set the initial ITR values */ if (qv->num_txq) { @@ -4117,19 +4133,42 @@ static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport) idpf_vport_intr_update_itr_ena_irq(qv); } - writel(vport->noirq_dyn_ctl_ena, vport->noirq_dyn_ctl); + writel(rsrc->noirq_dyn_ctl_ena, rsrc->noirq_dyn_ctl); +} + +/** + * idpf_vport_intr_dis_dim_all - Disable DIM work for all q_vectors + * @rsrc: pointer to queue and vector resources + * + * The DIM works are embedded in the q_vector array that + * idpf_vport_intr_rel() frees, and the poll arms them after + * napi_complete_done() has already cleared NAPI_STATE_SCHED. Disable + * rather than just cancel, so that a poll tail still running past + * napi_disable() cannot queue them again behind the drain. + */ +static void idpf_vport_intr_dis_dim_all(struct idpf_q_vec_rsrc *rsrc) +{ + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[v_idx]; + + disable_work_sync(&q_vector->tx_dim.work); + disable_work_sync(&q_vector->rx_dim.work); + } } /** * idpf_vport_intr_deinit - Release all vector associations for the vport * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -void idpf_vport_intr_deinit(struct idpf_vport *vport) +void idpf_vport_intr_deinit(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { - idpf_vport_intr_dis_irq_all(vport); - idpf_vport_intr_napi_dis_all(vport); - idpf_vport_intr_napi_del_all(vport); - idpf_vport_intr_rel_irq(vport); + idpf_vport_intr_dis_irq_all(rsrc); + idpf_vport_intr_napi_dis_all(rsrc); + idpf_vport_intr_dis_dim_all(rsrc); + idpf_vport_intr_napi_del_all(rsrc); + idpf_vport_intr_rel_irq(vport, rsrc); } /** @@ -4201,16 +4240,13 @@ static void idpf_init_dim(struct idpf_q_vector *qv) /** * idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport - * @vport: main vport structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport) +static void idpf_vport_intr_napi_ena_all(struct idpf_q_vec_rsrc *rsrc) { - int q_idx; + for (u16 q_idx = 0; q_idx < rsrc->num_q_vectors; q_idx++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[q_idx]; - for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[q_idx]; - - idpf_init_dim(q_vector); napi_enable(&q_vector->napi); } } @@ -4221,7 +4257,7 @@ static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport) * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * - * Returns false if clean is not complete else returns true + * Return: %false if clean is not complete else returns %true */ static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, int *cleaned) @@ -4248,7 +4284,7 @@ static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec, * @budget: Used to determine if we are in netpoll * @cleaned: returns number of packets cleaned * - * Returns false if clean is not complete else returns true + * Return: %false if clean is not complete else returns %true */ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, int *cleaned) @@ -4291,6 +4327,8 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, * idpf_vport_splitq_napi_poll - NAPI handler * @napi: struct from which you get q_vector * @budget: budget provided by stack + * + * Return: how many packets were cleaned */ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) { @@ -4336,10 +4374,12 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) /** * idpf_vport_intr_map_vector_to_qs - Map vectors to queues * @vport: virtual port + * @rsrc: pointer to queue and vector resources * * Mapping for vectors to queues */ -static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) +static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { u16 num_txq_grp = vport->num_txq_grp - vport->num_xdp_txq; bool split = idpf_is_queue_model_split(vport->rxq_model); @@ -4350,7 +4390,7 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) { u16 num_rxq; - if (qv_idx >= vport->num_q_vectors) + if (qv_idx >= rsrc->num_q_vectors) qv_idx = 0; rx_qgrp = &vport->rxq_grps[i]; @@ -4366,7 +4406,7 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) q = &rx_qgrp->splitq.rxq_sets[j]->rxq; else q = rx_qgrp->singleq.rxqs[j]; - q->q_vector = &vport->q_vectors[qv_idx]; + q->q_vector = &rsrc->q_vectors[qv_idx]; q_index = q->q_vector->num_rxq; q->q_vector->rx[q_index] = q; q->q_vector->num_rxq++; @@ -4380,7 +4420,7 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) struct idpf_buf_queue *bufq; bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; - bufq->q_vector = &vport->q_vectors[qv_idx]; + bufq->q_vector = &rsrc->q_vectors[qv_idx]; q_index = bufq->q_vector->num_bufq; bufq->q_vector->bufq[q_index] = bufq; bufq->q_vector->num_bufq++; @@ -4395,7 +4435,7 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) for (i = 0, qv_idx = 0; i < num_txq_grp; i++) { u16 num_txq; - if (qv_idx >= vport->num_q_vectors) + if (qv_idx >= rsrc->num_q_vectors) qv_idx = 0; tx_qgrp = &vport->txq_grps[i]; @@ -4405,14 +4445,14 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) struct idpf_tx_queue *q; q = tx_qgrp->txqs[j]; - q->q_vector = &vport->q_vectors[qv_idx]; + q->q_vector = &rsrc->q_vectors[qv_idx]; q->q_vector->tx[q->q_vector->num_txq++] = q; } if (split) { struct idpf_compl_queue *q = tx_qgrp->complq; - q->q_vector = &vport->q_vectors[qv_idx]; + q->q_vector = &rsrc->q_vectors[qv_idx]; q->q_vector->complq[q->q_vector->num_complq++] = q; } @@ -4438,10 +4478,14 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) /** * idpf_vport_intr_init_vec_idx - Initialize the vector indexes * @vport: virtual port + * @rsrc: pointer to queue and vector resources + * + * Initialize vector indexes with values returned over mailbox. * - * Initialize vector indexes with values returened over mailbox + * Return: 0 on success, negative on failure */ -static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) +static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_adapter *adapter = vport->adapter; struct virtchnl2_alloc_vectors *ac; @@ -4450,10 +4494,10 @@ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) ac = adapter->req_vec_chunks; if (!ac) { - for (i = 0; i < vport->num_q_vectors; i++) - vport->q_vectors[i].v_idx = vport->q_vector_idxs[i]; + for (i = 0; i < rsrc->num_q_vectors; i++) + rsrc->q_vectors[i].v_idx = rsrc->q_vector_idxs[i]; - vport->noirq_v_idx = vport->q_vector_idxs[i]; + rsrc->noirq_v_idx = rsrc->q_vector_idxs[i]; return 0; } @@ -4465,10 +4509,10 @@ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) idpf_get_vec_ids(adapter, vecids, total_vecs, &ac->vchunks); - for (i = 0; i < vport->num_q_vectors; i++) - vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]]; + for (i = 0; i < rsrc->num_q_vectors; i++) + rsrc->q_vectors[i].v_idx = vecids[rsrc->q_vector_idxs[i]]; - vport->noirq_v_idx = vecids[vport->q_vector_idxs[i]]; + rsrc->noirq_v_idx = vecids[rsrc->q_vector_idxs[i]]; kfree(vecids); @@ -4478,21 +4522,24 @@ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) /** * idpf_vport_intr_napi_add_all- Register napi handler for all qvectors * @vport: virtual port structure + * @rsrc: pointer to queue and vector resources */ -static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport) +static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { int (*napi_poll)(struct napi_struct *napi, int budget); - u16 v_idx, qv_idx; int irq_num; + u16 qv_idx; if (idpf_is_queue_model_split(vport->txq_model)) napi_poll = idpf_vport_splitq_napi_poll; else napi_poll = idpf_vport_singleq_napi_poll; - for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; - qv_idx = vport->q_vector_idxs[v_idx]; + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) { + struct idpf_q_vector *q_vector = &rsrc->q_vectors[v_idx]; + + qv_idx = rsrc->q_vector_idxs[v_idx]; irq_num = vport->adapter->msix_entries[qv_idx].vector; netif_napi_add_config(vport->netdev, &q_vector->napi, @@ -4504,40 +4551,46 @@ static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport) /** * idpf_vport_intr_alloc - Allocate memory for interrupt vectors * @vport: virtual port + * @rsrc: pointer to queue and vector resources + * + * Allocate one q_vector per queue interrupt. * - * We allocate one q_vector per queue interrupt. If allocation fails we - * return -ENOMEM. + * Return: 0 on success, if allocation fails we return -ENOMEM. */ -int idpf_vport_intr_alloc(struct idpf_vport *vport) +int idpf_vport_intr_alloc(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { u16 txqs_per_vector, rxqs_per_vector, bufqs_per_vector; struct idpf_vport_user_config_data *user_config; struct idpf_q_vector *q_vector; struct idpf_q_coalesce *q_coal; - u32 complqs_per_vector, v_idx; + u32 complqs_per_vector; u16 idx = vport->idx; user_config = &vport->adapter->vport_config[idx]->user_config; - vport->q_vectors = kcalloc(vport->num_q_vectors, - sizeof(struct idpf_q_vector), GFP_KERNEL); - if (!vport->q_vectors) + + rsrc->q_vectors = kcalloc(rsrc->num_q_vectors, + sizeof(struct idpf_q_vector), GFP_KERNEL); + if (!rsrc->q_vectors) return -ENOMEM; txqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp, - vport->num_q_vectors); + rsrc->num_q_vectors); rxqs_per_vector = DIV_ROUND_UP(vport->num_rxq_grp, - vport->num_q_vectors); + rsrc->num_q_vectors); bufqs_per_vector = vport->num_bufqs_per_qgrp * DIV_ROUND_UP(vport->num_rxq_grp, - vport->num_q_vectors); + rsrc->num_q_vectors); complqs_per_vector = DIV_ROUND_UP(vport->num_txq_grp, - vport->num_q_vectors); + rsrc->num_q_vectors); - for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { - q_vector = &vport->q_vectors[v_idx]; + for (u16 v_idx = 0; v_idx < rsrc->num_q_vectors; v_idx++) { + q_vector = &rsrc->q_vectors[v_idx]; q_coal = &user_config->q_coalesce[v_idx]; q_vector->vport = vport; + idpf_init_dim(q_vector); + q_vector->tx_itr_value = q_coal->tx_coalesce_usecs; q_vector->tx_intr_mode = q_coal->tx_intr_mode; q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1; @@ -4584,7 +4637,7 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) return 0; error: - idpf_vport_intr_rel(vport); + idpf_vport_intr_rel(rsrc); return -ENOMEM; } @@ -4592,47 +4645,48 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) /** * idpf_vport_intr_init - Setup all vectors for the given vport * @vport: virtual port + * @rsrc: pointer to queue and vector resources * - * Returns 0 on success or negative on failure + * Return: 0 on success or negative on failure */ -int idpf_vport_intr_init(struct idpf_vport *vport) +int idpf_vport_intr_init(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc) { int err; - err = idpf_vport_intr_init_vec_idx(vport); + err = idpf_vport_intr_init_vec_idx(vport, rsrc); if (err) return err; - idpf_vport_intr_map_vector_to_qs(vport); - idpf_vport_intr_napi_add_all(vport); + idpf_vport_intr_map_vector_to_qs(vport, rsrc); + idpf_vport_intr_napi_add_all(vport, rsrc); - err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport); + err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport, rsrc); if (err) goto unroll_vectors_alloc; - err = idpf_vport_intr_req_irq(vport); + err = idpf_vport_intr_req_irq(vport, rsrc); if (err) goto unroll_vectors_alloc; return 0; unroll_vectors_alloc: - idpf_vport_intr_napi_del_all(vport); + idpf_vport_intr_napi_del_all(rsrc); return err; } -void idpf_vport_intr_ena(struct idpf_vport *vport) +void idpf_vport_intr_ena(struct idpf_vport *vport, struct idpf_q_vec_rsrc *rsrc) { - idpf_vport_intr_napi_ena_all(vport); - idpf_vport_intr_ena_irq_all(vport); + idpf_vport_intr_napi_ena_all(rsrc); + idpf_vport_intr_ena_irq_all(vport, rsrc); } /** * idpf_config_rss - Send virtchnl messages to configure RSS * @vport: virtual port * - * Return 0 on success, negative on failure + * Return: 0 on success, negative on failure */ int idpf_config_rss(struct idpf_vport *vport) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 423cc9486dce7..1ce2100249be2 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -21,7 +21,7 @@ /* Mailbox Queue */ #define IDPF_MAX_MBXQ 1 -#define IDPF_MIN_TXQ_DESC 64 +#define IDPF_MIN_TXQ_DESC 128 #define IDPF_MIN_RXQ_DESC 64 #define IDPF_MIN_TXQ_COMPLQ_DESC 256 #define IDPF_MAX_QIDS 256 @@ -1080,12 +1080,16 @@ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_index, void idpf_vport_calc_num_q_groups(struct idpf_vport *vport); int idpf_vport_queues_alloc(struct idpf_vport *vport); void idpf_vport_queues_rel(struct idpf_vport *vport); -void idpf_vport_intr_rel(struct idpf_vport *vport); -int idpf_vport_intr_alloc(struct idpf_vport *vport); +void idpf_vport_intr_rel(struct idpf_q_vec_rsrc *rsrc); +int idpf_vport_intr_alloc(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector); -void idpf_vport_intr_deinit(struct idpf_vport *vport); -int idpf_vport_intr_init(struct idpf_vport *vport); -void idpf_vport_intr_ena(struct idpf_vport *vport); +void idpf_vport_intr_deinit(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); +int idpf_vport_intr_init(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); +void idpf_vport_intr_ena(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); void idpf_fill_dflt_rss_lut(struct idpf_vport *vport); int idpf_config_rss(struct idpf_vport *vport); int idpf_init_rss_lut(struct idpf_vport *vport); diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c index 4cc58c83688ca..bc47e194fbd25 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c @@ -69,11 +69,13 @@ static void idpf_vf_mb_intr_reg_init(struct idpf_adapter *adapter) /** * idpf_vf_intr_reg_init - Initialize interrupt registers * @vport: virtual port structure + * @rsrc: pointer to queue and vector resources */ -static int idpf_vf_intr_reg_init(struct idpf_vport *vport) +static int idpf_vf_intr_reg_init(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_adapter *adapter = vport->adapter; - int num_vecs = vport->num_q_vectors; + u16 num_vecs = rsrc->num_q_vectors; struct idpf_vec_regs *reg_vals; int num_regs, i, err = 0; u32 rx_itr, tx_itr, val; @@ -92,8 +94,8 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) } for (i = 0; i < num_vecs; i++) { - struct idpf_q_vector *q_vector = &vport->q_vectors[i]; - u16 vec_id = vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC; + struct idpf_q_vector *q_vector = &rsrc->q_vectors[i]; + u16 vec_id = rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC; struct idpf_intr_reg *intr = &q_vector->intr_reg; u32 spacing; @@ -122,12 +124,12 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) /* Data vector for NOIRQ queues */ - val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; - vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + val = reg_vals[rsrc->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; + rsrc->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); val = VF_INT_DYN_CTLN_WB_ON_ITR_M | VF_INT_DYN_CTLN_INTENA_MSK_M | FIELD_PREP(VF_INT_DYN_CTLN_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); - vport->noirq_dyn_ctl_ena = val; + rsrc->noirq_dyn_ctl_ena = val; free_reg_vals: kfree(reg_vals); diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 3c8ceff0e669f..963ffb1b62cc2 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -732,7 +732,9 @@ struct idpf_chunked_msg_params { u32 vc_op; }; -struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num) +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *qv_rsrc, + u32 num) { struct idpf_queue_set *qp; @@ -741,6 +743,7 @@ struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num) return NULL; qp->vport = vport; + qp->qv_rsrc = qv_rsrc; qp->num = num; return qp; @@ -847,7 +850,7 @@ static int idpf_wait_for_marker_event(struct idpf_vport *vport) { struct idpf_queue_set *qs __free(kfree) = NULL; - qs = idpf_alloc_queue_set(vport, vport->num_txq); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, vport->num_txq); if (!qs) return -ENOMEM; @@ -1264,6 +1267,45 @@ static void idpf_init_avail_queues(struct idpf_adapter *adapter) avail_queues->avail_complq = le16_to_cpu(caps->max_tx_complq); } +/** + * idpf_vport_init_queue_reg_chunks - initialize queue register chunks + * @vport_config: persistent vport structure to store the queue register info + * @schunks: source chunks to copy data from + * + * Return: 0 on success, negative on failure. + */ +static int +idpf_vport_init_queue_reg_chunks(struct idpf_vport_config *vport_config, + struct virtchnl2_queue_reg_chunks *schunks) +{ + struct idpf_queue_id_reg_info *q_info = &vport_config->qid_reg_info; + u16 num_chunks = le16_to_cpu(schunks->num_chunks); + + kfree(q_info->queue_chunks); + + q_info->queue_chunks = kcalloc(num_chunks, sizeof(*q_info->queue_chunks), + GFP_KERNEL); + if (!q_info->queue_chunks) { + q_info->num_chunks = 0; + return -ENOMEM; + } + + q_info->num_chunks = num_chunks; + + for (u16 i = 0; i < num_chunks; i++) { + struct idpf_queue_id_reg_chunk *dchunk = &q_info->queue_chunks[i]; + struct virtchnl2_queue_reg_chunk *schunk = &schunks->chunks[i]; + + dchunk->qtail_reg_start = le64_to_cpu(schunk->qtail_reg_start); + dchunk->qtail_reg_spacing = le32_to_cpu(schunk->qtail_reg_spacing); + dchunk->type = le32_to_cpu(schunk->type); + dchunk->start_queue_id = le32_to_cpu(schunk->start_queue_id); + dchunk->num_queues = le32_to_cpu(schunk->num_queues); + } + + return 0; +} + /** * idpf_get_reg_intr_vecs - Get vector queue register offset * @vport: virtual port structure @@ -1324,25 +1366,25 @@ int idpf_get_reg_intr_vecs(struct idpf_vport *vport, * are filled. */ static int idpf_vport_get_q_reg(u32 *reg_vals, int num_regs, u32 q_type, - struct virtchnl2_queue_reg_chunks *chunks) + struct idpf_queue_id_reg_info *chunks) { - u16 num_chunks = le16_to_cpu(chunks->num_chunks); + u16 num_chunks = chunks->num_chunks; int reg_filled = 0, i; u32 reg_val; while (num_chunks--) { - struct virtchnl2_queue_reg_chunk *chunk; + struct idpf_queue_id_reg_chunk *chunk; u16 num_q; - chunk = &chunks->chunks[num_chunks]; - if (le32_to_cpu(chunk->type) != q_type) + chunk = &chunks->queue_chunks[num_chunks]; + if (chunk->type != q_type) continue; - num_q = le32_to_cpu(chunk->num_queues); - reg_val = le64_to_cpu(chunk->qtail_reg_start); + num_q = chunk->num_queues; + reg_val = chunk->qtail_reg_start; for (i = 0; i < num_q && reg_filled < num_regs ; i++) { reg_vals[reg_filled++] = reg_val; - reg_val += le32_to_cpu(chunk->qtail_reg_spacing); + reg_val += chunk->qtail_reg_spacing; } } @@ -1412,15 +1454,13 @@ static int __idpf_queue_reg_init(struct idpf_vport *vport, u32 *reg_vals, /** * idpf_queue_reg_init - initialize queue registers * @vport: virtual port structure + * @chunks: queue registers received over mailbox * - * Return 0 on success, negative on failure + * Return: 0 on success, negative on failure */ -int idpf_queue_reg_init(struct idpf_vport *vport) +int idpf_queue_reg_init(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks) { - struct virtchnl2_create_vport *vport_params; - struct virtchnl2_queue_reg_chunks *chunks; - struct idpf_vport_config *vport_config; - u16 vport_idx = vport->idx; int num_regs, ret = 0; u32 *reg_vals; @@ -1429,16 +1469,6 @@ int idpf_queue_reg_init(struct idpf_vport *vport) if (!reg_vals) return -ENOMEM; - vport_config = vport->adapter->vport_config[vport_idx]; - if (vport_config->req_qs_chunks) { - struct virtchnl2_add_queues *vc_aq = - (struct virtchnl2_add_queues *)vport_config->req_qs_chunks; - chunks = &vc_aq->chunks; - } else { - vport_params = vport->adapter->vport_params_recvd[vport_idx]; - chunks = &vport_params->chunks; - } - /* Initialize Tx queue tail register address */ num_regs = idpf_vport_get_q_reg(reg_vals, IDPF_LARGE_MAX_Q, VIRTCHNL2_QUEUE_TYPE_TX, @@ -1833,7 +1863,7 @@ static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) u32 totqs = vport->num_txq + vport->num_complq; u32 k = 0; - qs = idpf_alloc_queue_set(vport, totqs); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, totqs); if (!qs) return -ENOMEM; @@ -2018,7 +2048,7 @@ static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) u32 totqs = vport->num_rxq + vport->num_bufq; u32 k = 0; - qs = idpf_alloc_queue_set(vport, totqs); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, totqs); if (!qs) return -ENOMEM; @@ -2163,7 +2193,7 @@ static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool en) num_txq = vport->num_txq + vport->num_complq; num_q = num_txq + vport->num_rxq + vport->num_bufq; - qs = idpf_alloc_queue_set(vport, num_q); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, num_q); if (!qs) return -ENOMEM; @@ -2301,7 +2331,7 @@ idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, v_idx = vec->v_idx; itr_idx = vec->rx_itr_idx; } else { - v_idx = qs->vport->noirq_v_idx; + v_idx = qs->qv_rsrc->noirq_v_idx; itr_idx = VIRTCHNL2_ITR_IDX_0; } break; @@ -2321,7 +2351,7 @@ idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, v_idx = vec->v_idx; itr_idx = vec->tx_itr_idx; } else { - v_idx = qs->vport->noirq_v_idx; + v_idx = qs->qv_rsrc->noirq_v_idx; itr_idx = VIRTCHNL2_ITR_IDX_1; } break; @@ -2351,7 +2381,7 @@ int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) u32 num_q = vport->num_txq + vport->num_rxq; u32 k = 0; - qs = idpf_alloc_queue_set(vport, num_q); + qs = idpf_alloc_queue_set(vport, &vport->dflt_qv_rsrc, num_q); if (!qs) return -ENOMEM; @@ -2484,46 +2514,35 @@ int idpf_send_disable_queues_msg(struct idpf_vport *vport) * @num_chunks: number of chunks to copy */ static void idpf_convert_reg_to_queue_chunks(struct virtchnl2_queue_chunk *dchunks, - struct virtchnl2_queue_reg_chunk *schunks, + struct idpf_queue_id_reg_chunk *schunks, u16 num_chunks) { u16 i; for (i = 0; i < num_chunks; i++) { - dchunks[i].type = schunks[i].type; - dchunks[i].start_queue_id = schunks[i].start_queue_id; - dchunks[i].num_queues = schunks[i].num_queues; + dchunks[i].type = cpu_to_le32(schunks[i].type); + dchunks[i].start_queue_id = cpu_to_le32(schunks[i].start_queue_id); + dchunks[i].num_queues = cpu_to_le32(schunks[i].num_queues); } } /** * idpf_send_delete_queues_msg - send delete queues virtchnl message - * @vport: Virtual port private data structure + * @vport: virtual port private data structure + * @chunks: queue ids received over mailbox * - * Will send delete queues virtchnl message. Return 0 on success, negative on - * failure. + * Return: 0 on success, negative on failure. */ -int idpf_send_delete_queues_msg(struct idpf_vport *vport) +int idpf_send_delete_queues_msg(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks) { struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; - struct virtchnl2_create_vport *vport_params; - struct virtchnl2_queue_reg_chunks *chunks; struct idpf_vc_xn_params xn_params = {}; - struct idpf_vport_config *vport_config; - u16 vport_idx = vport->idx; ssize_t reply_sz; u16 num_chunks; int buf_size; - vport_config = vport->adapter->vport_config[vport_idx]; - if (vport_config->req_qs_chunks) { - chunks = &vport_config->req_qs_chunks->chunks; - } else { - vport_params = vport->adapter->vport_params_recvd[vport_idx]; - chunks = &vport_params->chunks; - } - - num_chunks = le16_to_cpu(chunks->num_chunks); + num_chunks = chunks->num_chunks; buf_size = struct_size(eq, chunks.chunks, num_chunks); eq = kzalloc(buf_size, GFP_KERNEL); @@ -2533,7 +2552,7 @@ int idpf_send_delete_queues_msg(struct idpf_vport *vport) eq->vport_id = cpu_to_le32(vport->vport_id); eq->chunks.num_chunks = cpu_to_le16(num_chunks); - idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->chunks, + idpf_convert_reg_to_queue_chunks(eq->chunks.chunks, chunks->queue_chunks, num_chunks); xn_params.vc_op = VIRTCHNL2_OP_DEL_QUEUES; @@ -2571,8 +2590,7 @@ int idpf_send_config_queues_msg(struct idpf_vport *vport) * @num_rx_q: number of receive queues * @num_rx_bufq: number of receive buffer queues * - * Returns 0 on success, negative on failure. vport _MUST_ be const here as - * we should not change any fields within vport itself in this function. + * Return: 0 on success, negative on failure. */ int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, u16 num_complq, u16 num_rx_q, u16 num_rx_bufq) @@ -2590,8 +2608,6 @@ int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, return -ENOMEM; vport_config = vport->adapter->vport_config[vport_idx]; - kfree(vport_config->req_qs_chunks); - vport_config->req_qs_chunks = NULL; aq.vport_id = cpu_to_le32(vport->vport_id); aq.num_tx_q = cpu_to_le16(num_tx_q); @@ -2621,11 +2637,7 @@ int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, if (reply_sz < size) return -EIO; - vport_config->req_qs_chunks = kmemdup(vc_msg, size, GFP_KERNEL); - if (!vport_config->req_qs_chunks) - return -ENOMEM; - - return 0; + return idpf_vport_init_queue_reg_chunks(vport_config, &vc_msg->chunks); } /** @@ -3615,20 +3627,22 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter) /** * idpf_vport_alloc_vec_indexes - Get relative vector indexes * @vport: virtual port data struct + * @rsrc: pointer to queue and vector resources * * This function requests the vector information required for the vport and * stores the vector indexes received from the 'global vector distribution' * in the vport's queue vectors array. * - * Return 0 on success, error on failure + * Return: 0 on success, error on failure */ -int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) +int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc) { struct idpf_vector_info vec_info; int num_alloc_vecs; u32 req; - vec_info.num_curr_vecs = vport->num_q_vectors; + vec_info.num_curr_vecs = rsrc->num_q_vectors; if (vec_info.num_curr_vecs) vec_info.num_curr_vecs += IDPF_RESERVED_VECS; @@ -3641,7 +3655,7 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) vec_info.index = vport->idx; num_alloc_vecs = idpf_req_rel_vector_indexes(vport->adapter, - vport->q_vector_idxs, + rsrc->q_vector_idxs, &vec_info); if (num_alloc_vecs <= 0) { dev_err(&vport->adapter->pdev->dev, "Vector distribution failed: %d\n", @@ -3649,7 +3663,7 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) return -EINVAL; } - vport->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; + rsrc->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; return 0; } @@ -3660,9 +3674,12 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) * @max_q: vport max queue info * * Will initialize vport with the info received through MB earlier + * + * Return: 0 on success, negative on failure. */ -void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) +int idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) { + struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc; struct idpf_adapter *adapter = vport->adapter; struct virtchnl2_create_vport *vport_msg; struct idpf_vport_config *vport_config; @@ -3676,6 +3693,11 @@ void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) rss_data = &vport_config->user_config.rss_data; vport_msg = adapter->vport_params_recvd[idx]; + err = idpf_vport_init_queue_reg_chunks(vport_config, + &vport_msg->chunks); + if (err) + return err; + vport_config->max_q.max_txq = max_q->max_txq; vport_config->max_q.max_rxq = max_q->max_rxq; vport_config->max_q.max_complq = max_q->max_complq; @@ -3702,21 +3724,24 @@ void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q) idpf_vport_init_num_qs(vport, vport_msg); idpf_vport_calc_num_q_desc(vport); idpf_vport_calc_num_q_groups(vport); - idpf_vport_alloc_vec_indexes(vport); + idpf_vport_alloc_vec_indexes(vport, rsrc); vport->crc_enable = adapter->crc_enable; if (!(vport_msg->vport_flags & cpu_to_le16(VIRTCHNL2_VPORT_UPLINK_PORT))) - return; + return 0; err = idpf_ptp_get_vport_tstamps_caps(vport); if (err) { + /* Do not error on timestamp failure */ pci_dbg(vport->adapter->pdev, "Tx timestamping not supported\n"); - return; + return 0; } INIT_WORK(&vport->tstamp_task, idpf_tstamp_task); + + return 0; } /** @@ -3775,21 +3800,21 @@ int idpf_get_vec_ids(struct idpf_adapter *adapter, * Returns number of ids filled */ static int idpf_vport_get_queue_ids(u32 *qids, int num_qids, u16 q_type, - struct virtchnl2_queue_reg_chunks *chunks) + struct idpf_queue_id_reg_info *chunks) { - u16 num_chunks = le16_to_cpu(chunks->num_chunks); + u16 num_chunks = chunks->num_chunks; u32 num_q_id_filled = 0, i; u32 start_q_id, num_q; while (num_chunks--) { - struct virtchnl2_queue_reg_chunk *chunk; + struct idpf_queue_id_reg_chunk *chunk; - chunk = &chunks->chunks[num_chunks]; - if (le32_to_cpu(chunk->type) != q_type) + chunk = &chunks->queue_chunks[num_chunks]; + if (chunk->type != q_type) continue; - num_q = le32_to_cpu(chunk->num_queues); - start_q_id = le32_to_cpu(chunk->start_queue_id); + num_q = chunk->num_queues; + start_q_id = chunk->start_queue_id; for (i = 0; i < num_q; i++) { if ((num_q_id_filled + i) < num_qids) { @@ -3882,30 +3907,19 @@ static int __idpf_vport_queue_ids_init(struct idpf_vport *vport, /** * idpf_vport_queue_ids_init - Initialize queue ids from Mailbox parameters * @vport: virtual port for which the queues ids are initialized + * @chunks: queue ids received over mailbox * * Will initialize all queue ids with ids received as mailbox parameters. - * Returns 0 on success, negative if all the queues are not initialized. + * + * Return: 0 on success, negative if all the queues are not initialized. */ -int idpf_vport_queue_ids_init(struct idpf_vport *vport) +int idpf_vport_queue_ids_init(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks) { - struct virtchnl2_create_vport *vport_params; - struct virtchnl2_queue_reg_chunks *chunks; - struct idpf_vport_config *vport_config; - u16 vport_idx = vport->idx; int num_ids, err = 0; u16 q_type; u32 *qids; - vport_config = vport->adapter->vport_config[vport_idx]; - if (vport_config->req_qs_chunks) { - struct virtchnl2_add_queues *vc_aq = - (struct virtchnl2_add_queues *)vport_config->req_qs_chunks; - chunks = &vc_aq->chunks; - } else { - vport_params = vport->adapter->vport_params_recvd[vport_idx]; - chunks = &vport_params->chunks; - } - qids = kcalloc(IDPF_MAX_QIDS, sizeof(u32), GFP_KERNEL); if (!qids) return -ENOMEM; diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index 0a50a08eb9af7..dc2d50cfb3383 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -93,6 +93,7 @@ struct idpf_netdev_priv; struct idpf_vec_regs; struct idpf_vport; struct idpf_vport_max_q; +struct idpf_vport_config; struct idpf_vport_user_config_data; ssize_t idpf_vc_xn_exec(struct idpf_adapter *adapter, @@ -104,8 +105,16 @@ void idpf_vc_core_deinit(struct idpf_adapter *adapter); int idpf_get_reg_intr_vecs(struct idpf_vport *vport, struct idpf_vec_regs *reg_vals); -int idpf_queue_reg_init(struct idpf_vport *vport); -int idpf_vport_queue_ids_init(struct idpf_vport *vport); +int idpf_queue_reg_init(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks); +int idpf_vport_queue_ids_init(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks); +static inline void +idpf_vport_deinit_queue_reg_chunks(struct idpf_vport_config *vport_cfg) +{ + kfree(vport_cfg->qid_reg_info.queue_chunks); + vport_cfg->qid_reg_info.queue_chunks = NULL; +} bool idpf_vport_is_cap_ena(struct idpf_vport *vport, u16 flag); bool idpf_sideband_flow_type_ena(struct idpf_vport *vport, u32 flow_type); @@ -129,12 +138,15 @@ struct idpf_queue_ptr { struct idpf_queue_set { struct idpf_vport *vport; + struct idpf_q_vec_rsrc *qv_rsrc; u32 num; struct idpf_queue_ptr qs[] __counted_by(num); }; -struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num); +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc, + u32 num); int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs); int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs); @@ -144,7 +156,7 @@ int idpf_send_disable_queues_msg(struct idpf_vport *vport); int idpf_send_config_queues_msg(struct idpf_vport *vport); int idpf_send_enable_queues_msg(struct idpf_vport *vport); -void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q); +int idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q); u32 idpf_get_vport_id(struct idpf_vport *vport); int idpf_send_create_vport_msg(struct idpf_adapter *adapter, struct idpf_vport_max_q *max_q); @@ -159,9 +171,11 @@ void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter, struct idpf_vport_max_q *max_q); int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, u16 num_complq, u16 num_rx_q, u16 num_rx_bufq); -int idpf_send_delete_queues_msg(struct idpf_vport *vport); +int idpf_send_delete_queues_msg(struct idpf_vport *vport, + struct idpf_queue_id_reg_info *chunks); -int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport); +int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport, + struct idpf_q_vec_rsrc *rsrc); int idpf_get_vec_ids(struct idpf_adapter *adapter, u16 *vecids, int num_vecids, struct virtchnl2_vector_chunks *chunks); diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c index 61dfcd8cb370a..c1f2eff615f11 100644 --- a/drivers/net/ethernet/intel/igbvf/netdev.c +++ b/drivers/net/ethernet/intel/igbvf/netdev.c @@ -2191,8 +2191,6 @@ static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter, buffer_info->time_stamp = 0; buffer_info->length = 0; buffer_info->mapped_as_page = false; - if (count) - count--; /* clear timestamp and dma mappings for remaining portion of packet */ while (count--) { diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index ebc00ce2e3a6b..a8e7f4895e29e 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -3074,7 +3074,8 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring) olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT; dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len); bi = &ring->tx_buffer_info[ntu]; @@ -3082,7 +3083,7 @@ static void igc_xdp_xmit_zc(struct igc_ring *ring) meta_req.tx_buffer = bi; meta_req.meta = meta; meta_req.used_desc = 0; - xsk_tx_metadata_request(meta, &igc_xsk_tx_metadata_ops, + xsk_tx_metadata_request(pool, &meta, &igc_xsk_tx_metadata_ops, &meta_req); /* xsk_tx_metadata_request() may have updated next_to_use */ @@ -5354,9 +5355,8 @@ void igc_down(struct igc_adapter *adapter) for (i = 0; i < adapter->num_q_vectors; i++) { if (adapter->q_vector[i]) { - napi_synchronize(&adapter->q_vector[i]->napi); - igc_set_queue_napi(adapter, i, NULL); napi_disable(&adapter->q_vector[i]->napi); + igc_set_queue_napi(adapter, i, NULL); } } @@ -7574,11 +7574,13 @@ static int __igc_resume(struct device *dev, bool rpm) err = __igc_open(netdev, true); if (!rpm) rtnl_unlock(); - if (!err) - netif_device_attach(netdev); + if (err) + return err; } - return err; + netif_device_attach(netdev); + + return 0; } static int igc_resume(struct device *dev) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c index a07e0b3d8d000..211c10aa5880f 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c @@ -1417,6 +1417,16 @@ static int mcs_x2p_calibration(struct mcs *mcs) int i, err = 0; u64 val; + /* Clear any stale calibration state left by firmware/bootloader. + * Some firmware versions may leave MCSX_MIL_GLOBAL bit 5 set, + * preventing the hardware from detecting the rising edge needed to + * trigger X2P calibration. + */ + val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); + val &= ~BIT_ULL(5); + mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val); + usleep_range(100, 200); + /* set X2P calibration */ val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL); val |= BIT_ULL(5); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c index a80c8e7c94f28..d6c69ea7f10d7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c @@ -856,7 +856,7 @@ int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu, static void rvu_mcs_set_lmac_bmap(struct rvu *rvu) { struct mcs *mcs = mcs_get_pdata(0); - unsigned long lmac_bmap; + unsigned long lmac_bmap = 0; int cgx, lmac, port; for (port = 0; port < mcs->hw->lmac_cnt; port++) { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index ad733f4da3c18..0590313bcd81c 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -435,7 +435,7 @@ struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc) return &rvu->pf[rvu_get_pf(rvu->pdev, pcifunc)]; } -static bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc) +bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc) { int pf, vf, nvfs; u64 cfg; @@ -3377,6 +3377,8 @@ int rvu_get_num_lbk_chans(void) return ret; } +#define PCI_DEVID_OCTEONTX2_RVU_AFVF 0xA0F8 + static int rvu_enable_sriov(struct rvu *rvu) { struct pci_dev *pdev = rvu->pdev; @@ -3395,24 +3397,27 @@ static int rvu_enable_sriov(struct rvu *rvu) return 0; pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &rvu->vf_devid); - chans = rvu_get_num_lbk_chans(); - if (chans < 0) - return chans; - vfs = pci_sriov_get_totalvfs(pdev); - - /* Limit VFs in case we have more VFs than LBK channels available. */ - if (vfs > chans) - vfs = chans; - if (!vfs) return 0; - /* LBK channel number 63 is used for switching packets between - * CGX mapped VFs. Hence limit LBK pairs till 62 only. - */ - if (vfs > 62) - vfs = 62; + if (rvu->vf_devid == PCI_DEVID_OCTEONTX2_RVU_AFVF) { + chans = rvu_get_num_lbk_chans(); + if (chans < 0) + return chans; + + /* The last LBK channel is reserved for switching packets between + * CGX mapped VFs. Also, since LBK VFs work in pairs, limit VF + * count to available LBK channels minus 2. + */ + vfs = min(vfs, chans - 2); + + if (vfs <= 0) { + dev_warn(&pdev->dev, + "Skipping SRIOV enablement, not enough LBK channels available\n"); + return 0; + } + } /* Save VFs number for reference in VF interrupts handlers. * Since interrupts might start arriving during SRIOV enablement diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index 914ba2b691ec8..c9ed5276690d3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -900,6 +900,7 @@ u16 rvu_get_rsrc_mapcount(struct rvu_pfvf *pfvf, int blkaddr); struct rvu_pfvf *rvu_get_pfvf(struct rvu *rvu, int pcifunc); void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf); bool is_block_implemented(struct rvu_hwinfo *hw, int blkaddr); +bool is_pf_func_valid(struct rvu *rvu, u16 pcifunc); bool is_pffunc_map_valid(struct rvu *rvu, u16 pcifunc, int blktype); int rvu_get_lf(struct rvu *rvu, struct rvu_block *block, u16 pcifunc, u16 slot); int rvu_lf_reset(struct rvu *rvu, struct rvu_block *block, int lf); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c index d2163da28d18f..fa4ea1258d29d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_cn10k.c @@ -178,6 +178,15 @@ int rvu_mbox_handler_lmtst_tbl_setup(struct rvu *rvu, * pcifunc (will be the one who is calling this mailbox). */ if (req->base_pcifunc) { + /* A VF is untrusted and must not redirect its LMTLINE to + * another PF's region, so confine VF callers to their own PF. + */ + if (is_vf(req->hdr.pcifunc) && + (!is_pf_func_valid(rvu, req->base_pcifunc) || + rvu_get_pf(rvu->pdev, req->hdr.pcifunc) != + rvu_get_pf(rvu->pdev, req->base_pcifunc))) + return -EPERM; + /* Calculating the LMT table index equivalent to primary * pcifunc. */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index beb514ebda029..0a1c778d14b69 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -825,19 +825,25 @@ static int rvu_dbg_rvu_pf_cgx_map_display(struct seq_file *filp, void *unused) int pf, domain, blkid; u8 cgx_id, lmac_id; u16 pcifunc; + u8 start; - domain = 2; + domain = pci_domain_nr(rvu->pdev->bus); mac_ops = get_mac_ops(rvu_first_cgx_pdata(rvu)); /* There can be no CGX devices at all */ if (!mac_ops) return 0; seq_printf(filp, "PCI dev\t\tRVU PF Func\tNIX block\t%s\tLMAC\tCHAN\n", mac_ops->name); + + /* All the PF devices are on contiguous PCI bus numbers, but the PF0(AF) + * may not start from 1 always. Hence get domain and bus from PCI device. + */ + start = rvu->pdev->bus->number; for (pf = 0; pf < rvu->hw->total_pfs; pf++) { if (!is_pf_cgxmapped(rvu, pf)) continue; - pdev = pci_get_domain_bus_and_slot(domain, pf + 1, 0); + pdev = pci_get_domain_bus_and_slot(domain, pf + start, 0); if (!pdev) continue; @@ -1689,6 +1695,12 @@ static int rvu_dbg_nix_tm_tree_display(struct seq_file *m, void *unused) return -EINVAL; pfvf = rvu_get_pfvf(rvu, pcifunc); + + if (!pfvf->sq_ctx) { + seq_printf(m, "SQ context is not initialized for pcifunc 0x%x\n", pcifunc); + return -EINVAL; + } + max_id = pfvf->sq_ctx->qsize; memset(&aq_req, 0, sizeof(struct nix_aq_enq_req)); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 7de06eb8284ac..adedca065210a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -2436,8 +2436,8 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, int pf = rvu_get_pf(rvu->pdev, pcifunc); u8 cgx_id = 0, lmac_id = 0; u16 tl2_tl3_link_schq; - u8 link, link_level; u64 cfg, bmap = 0; + u8 link_level; if (!is_rvu_otx2(rvu)) { /* Skip SMQ flush if pkt count is zero */ @@ -2469,7 +2469,6 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, link_level = rvu_read64(rvu, blkaddr, NIX_AF_PSE_CHANNEL_LEVEL) & 0x01 ? NIX_TXSCH_LVL_TL3 : NIX_TXSCH_LVL_TL2; tl2_tl3_link_schq = smq_flush_ctx->smq_tree_ctx[link_level].schq; - link = smq_flush_ctx->smq_tree_ctx[NIX_TXSCH_LVL_TL1].schq; /* SMQ set enqueue xoff */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_SMQX_CFG(smq)); @@ -2479,13 +2478,13 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, /* Clear all NIX_AF_TL3_TL2_LINK_CFG[ENA] for the TL3/TL2 queue */ for (i = 0; i < (rvu->hw->cgx_links + rvu->hw->lbk_links); i++) { cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); if (!(cfg & BIT_ULL(12))) continue; bmap |= BIT_ULL(i); cfg &= ~BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* Do SMQ flush and set enqueue xoff */ @@ -2506,10 +2505,10 @@ static int nix_smq_flush(struct rvu *rvu, int blkaddr, if (!(bmap & BIT_ULL(i))) continue; cfg = rvu_read64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link)); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i)); cfg |= BIT_ULL(12); rvu_write64(rvu, blkaddr, - NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, link), cfg); + NIX_AF_TL3_TL2X_LINKX_CFG(tl2_tl3_link_schq, i), cfg); } /* clear XOFF on TL2s */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c index 4415d0ce9aef4..d43c034b0264a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_rep.c @@ -97,6 +97,14 @@ int rvu_mbox_handler_rep_event_notify(struct rvu *rvu, struct rep_event *req, { struct rep_evtq_ent *qentry; + /* The mailbox dispatcher normalises only the header pcifunc; the + * nested struct rep_event::pcifunc body field is sender-controlled + * and is later used by rvu_rep_up_notify() to index rvu->pf[] / + * rvu->hwvf[]. Reject out-of-range body selectors before queueing. + */ + if (!is_pf_func_valid(rvu, req->pcifunc)) + return -EINVAL; + qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); if (!qentry) return -ENOMEM; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c index 062bc899d9550..83a51de891547 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c @@ -333,7 +333,8 @@ int otx2_set_rss_table(struct otx2_nic *pfvf, int ctx_id, const u32 *ind_tbl) /* Get memory to put this msg */ for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore the queue if AF_XDP zero copy is enabled */ - if (test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox); @@ -1496,7 +1497,8 @@ int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id, if (type != AURA_NIX_RQ) return 0; - if (!test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { + if (!pfvf->af_xdp_zc_qidx || + !test_bit(pool_id, pfvf->af_xdp_zc_qidx)) { pp_params.order = get_order(buf_size); pp_params.flags = PP_FLAG_DMA_MAP; pp_params.pool_size = min(OTX2_PAGE_POOL_SZ, numptrs); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index b6449f0a9e7dd..95081cfb5338a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -934,7 +934,8 @@ static int otx2_get_rxfh(struct net_device *dev, for (idx = 0; idx < rss->rss_size; idx++) { /* Ignore if the rx queue is AF_XDP zero copy enabled */ - if (test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(rss->ind_tbl[idx], pfvf->af_xdp_zc_qidx)) continue; indir[idx] = rss->ind_tbl[idx]; } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 64c6d9162ef64..17e8f6e51808b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -146,6 +146,7 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) if (allocated) { pfvf->flags |= OTX2_FLAG_MCAM_ENTRIES_ALLOC; pfvf->flags |= OTX2_FLAG_NTUPLE_SUPPORT; + pfvf->flags |= OTX2_FLAG_TC_FLOWER_SUPPORT; } if (allocated != count) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index ae7bded777688..7e42d3291a9fd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -889,8 +889,8 @@ static void otx2_handle_link_event(struct otx2_nic *pf) netif_carrier_on(netdev); netif_tx_start_all_queues(netdev); } else { - netif_tx_stop_all_queues(netdev); netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); } } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 26a08d2cfbb1b..bc4e53964620b 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -30,6 +30,7 @@ #define OTX2_UNSUPP_LSE_DEPTH GENMASK(6, 4) #define MCAST_INVALID_GRP (-1U) +#define RATE_MANTISSA_BITS 8 struct otx2_tc_flow_stats { u64 bytes; @@ -77,10 +78,12 @@ static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, if (burst) { *burst_exp = ilog2(burst) ? ilog2(burst) - 1 : 0; tmp = burst - rounddown_pow_of_two(burst); - if (burst < max_mantissa) + if (burst <= max_mantissa) { *burst_mantissa = tmp * 2; - else + } else { + WARN_ON(*burst_exp < 7); *burst_mantissa = tmp / (1ULL << (*burst_exp - 7)); + } } else { *burst_exp = MAX_BURST_EXPONENT; *burst_mantissa = max_mantissa; @@ -90,28 +93,30 @@ static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, static void otx2_get_egress_rate_cfg(u64 maxrate, u32 *exp, u32 *mantissa, u32 *div_exp) { - u64 tmp; - /* Rate calculation by hardware * * PIR_ADD = ((256 + mantissa) << exp) / 256 * rate = (2 * PIR_ADD) / ( 1 << div_exp) * The resultant rate is in Mbps. + * + * Use div_exp = 0 and compute exp/mantissa for maxrate / 2; the + * leading factor of two yields the full rate. Rates below 2 Mbps + * are floored to the smallest step (exp = 0, mantissa = 0). */ - /* 2Mbps to 100Gbps can be expressed with div_exp = 0. - * Setting this to '0' will ease the calculation of - * exponent and mantissa. - */ *div_exp = 0; - if (maxrate) { - *exp = ilog2(maxrate) ? ilog2(maxrate) - 1 : 0; - tmp = maxrate - rounddown_pow_of_two(maxrate); - if (maxrate < MAX_RATE_MANTISSA) - *mantissa = tmp * 2; - else - *mantissa = tmp / (1ULL << (*exp - 7)); + maxrate = maxrate / 2; + if (!maxrate) { + /* Rates below 2 Mbps map to the smallest step */ + *exp = 0; + *mantissa = 0; + } else { + *exp = ilog2(maxrate); + /* Clear MSB and derive fractional bits */ + maxrate &= ~BIT(*exp); + *mantissa = (maxrate << RATE_MANTISSA_BITS) >> *exp; + } } else { /* Instead of disabling rate limiting, set all values to max */ *exp = MAX_RATE_EXPONENT; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 5534c2c8db0fb..67f01bb6073b1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -736,16 +736,16 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_ptp_destroy; + err = otx2_vf_wq_init(vf); + if (err) + goto err_ipsec_clean; + err = register_netdev(netdev); if (err) { dev_err(dev, "Failed to register netdevice\n"); - goto err_ipsec_clean; + goto err_wq_destroy; } - err = otx2_vf_wq_init(vf); - if (err) - goto err_unreg_netdev; - otx2vf_set_ethtool_ops(netdev); err = otx2vf_mcam_flow_init(vf); @@ -788,6 +788,10 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) otx2_shutdown_tc(vf); err_unreg_netdev: unregister_netdev(netdev); +err_wq_destroy: + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); + destroy_workqueue(vf->otx2_wq); err_ipsec_clean: cn10k_ipsec_clean(vf); err_ptp_destroy: @@ -835,11 +839,13 @@ static void otx2vf_remove(struct pci_dev *pdev) } #endif - cancel_work_sync(&vf->reset_task); otx2_unregister_dl(vf); unregister_netdev(netdev); - if (vf->otx2_wq) + if (vf->otx2_wq) { + cancel_work_sync(&vf->reset_task); + cancel_work_sync(&vf->rx_mode_work); destroy_workqueue(vf->otx2_wq); + } cn10k_ipsec_clean(vf); otx2_ptp_destroy(vf); otx2_mcam_flow_del(vf); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c index 7d67b4cbaf716..0e8a6a6486c4a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c @@ -193,7 +193,8 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx) { - if (test_bit(qidx, pfvf->af_xdp_zc_qidx)) + if (pfvf->af_xdp_zc_qidx && + test_bit(qidx, pfvf->af_xdp_zc_qidx)) sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx); } diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c index 5765bac119f0e..69dfb34008572 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/qos.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/qos.c @@ -235,13 +235,63 @@ static int otx2_qos_txschq_set_parent_topology(struct otx2_nic *pfvf, return rc; } +static int otx2_qos_reset_schq_topology(struct otx2_nic *pfvf, u16 lvl, + u16 schq) +{ + struct mbox *mbox = &pfvf->mbox; + struct nix_txschq_config *cfg; + int rc; + + if (lvl < NIX_TXSCH_LVL_TL4 || lvl >= NIX_TXSCH_LVL_TL1) + return 0; + + mutex_lock(&mbox->lock); + + cfg = otx2_mbox_alloc_msg_nix_txschq_cfg(mbox); + if (!cfg) { + mutex_unlock(&mbox->lock); + return -ENOMEM; + } + + cfg->lvl = lvl; + cfg->num_regs = 1; + + if (lvl == NIX_TXSCH_LVL_TL4) + cfg->reg[0] = NIX_AF_TL4X_TOPOLOGY(schq); + else if (lvl == NIX_TXSCH_LVL_TL3) + cfg->reg[0] = NIX_AF_TL3X_TOPOLOGY(schq); + else if (lvl == NIX_TXSCH_LVL_TL2) + cfg->reg[0] = NIX_AF_TL2X_TOPOLOGY(schq); + + cfg->regval[0] = 0; + + rc = otx2_sync_mbox_msg(mbox); + + mutex_unlock(&mbox->lock); + + return rc; +} + +static void otx2_qos_free_hw_schq(struct otx2_nic *pfvf, u16 lvl, u16 schq) +{ + int err; + + err = otx2_qos_reset_schq_topology(pfvf, lvl, schq); + if (err) + netdev_warn(pfvf->netdev, + "QoS: failed to reset topology for schq %u at level %u: %d\n", + schq, lvl, err); + + otx2_txschq_free_one(pfvf, lvl, schq); +} + static void otx2_qos_free_hw_node_schq(struct otx2_nic *pfvf, struct otx2_qos_node *parent) { struct otx2_qos_node *node; list_for_each_entry_reverse(node, &parent->child_schq_list, list) - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); } static void otx2_qos_free_hw_node(struct otx2_nic *pfvf, @@ -252,7 +302,7 @@ static void otx2_qos_free_hw_node(struct otx2_nic *pfvf, list_for_each_entry_safe(node, tmp, &parent->child_list, list) { otx2_qos_free_hw_node(pfvf, node); otx2_qos_free_hw_node_schq(pfvf, node); - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); } } @@ -266,7 +316,7 @@ static void otx2_qos_free_hw_cfg(struct otx2_nic *pfvf, otx2_qos_free_hw_node_schq(pfvf, node); /* free node hw mappings */ - otx2_txschq_free_one(pfvf, node->level, node->schq); + otx2_qos_free_hw_schq(pfvf, node->level, node->schq); mutex_unlock(&pfvf->qos.qos_lock); } @@ -913,7 +963,7 @@ static void otx2_qos_free_cfg(struct otx2_nic *pfvf, struct otx2_qos_cfg *cfg) for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) { for (idx = 0; idx < cfg->schq[lvl]; idx++) { schq = cfg->schq_list[lvl][idx]; - otx2_txschq_free_one(pfvf, lvl, schq); + otx2_qos_free_hw_schq(pfvf, lvl, schq); } } @@ -921,7 +971,7 @@ static void otx2_qos_free_cfg(struct otx2_nic *pfvf, struct otx2_qos_cfg *cfg) for (idx = 0; idx < cfg->schq_contig[lvl]; idx++) { if (cfg->schq_index_used[lvl][idx]) { schq = cfg->schq_contig_list[lvl][idx]; - otx2_txschq_free_one(pfvf, lvl, schq); + otx2_qos_free_hw_schq(pfvf, lvl, schq); } } } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_pci.c b/drivers/net/ethernet/marvell/prestera/prestera_pci.c index 3e13322470da6..bacc869551dfd 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_pci.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_pci.c @@ -684,6 +684,9 @@ static int prestera_fw_hdr_parse(struct prestera_fw *fw) struct prestera_fw_header *hdr; u32 magic; + if (fw->bin->size < sizeof(*hdr)) + return -EINVAL; + hdr = (struct prestera_fw_header *)fw->bin->data; magic = be32_to_cpu(hdr->magic_number); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 065f969ee44ef..31eee8739c575 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -562,9 +562,7 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode, int val, ge_mode, err = 0; u32 i; - /* MT76x8 has no hardware settings between for the MAC */ - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) && - mac->interface != state->interface) { + if (mac->interface != state->interface) { /* Setup soc pin functions */ switch (state->interface) { case PHY_INTERFACE_MODE_TRGMII: @@ -956,6 +954,30 @@ static const struct phylink_mac_ops mtk_phylink_ops = { .mac_enable_tx_lpi = mtk_mac_enable_tx_lpi, }; +static void rt5350_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ +} + +static void rt5350_mac_link_down(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ +} + +static void rt5350_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, bool tx_pause, bool rx_pause) +{ +} + +/* MT76x8 (rt5350-eth) does not expose any MAC control registers */ +static const struct phylink_mac_ops rt5350_phylink_ops = { + .mac_config = rt5350_mac_config, + .mac_link_down = rt5350_mac_link_down, + .mac_link_up = rt5350_mac_link_up, +}; + static void mtk_mdio_config(struct mtk_eth *eth) { u32 val; @@ -3446,7 +3468,7 @@ static void mtk_poll_controller(struct net_device *dev) mtk_tx_irq_disable(eth, MTK_TX_DONE_INT); mtk_rx_irq_disable(eth, eth->soc->rx.irq_done_mask); - mtk_handle_irq_rx(eth->irq[MTK_FE_IRQ_RX], dev); + mtk_handle_irq_rx(eth->irq[MTK_FE_IRQ_RX], eth); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); mtk_rx_irq_enable(eth, eth->soc->rx.irq_done_mask); } @@ -4798,11 +4820,12 @@ static const struct net_device_ops mtk_netdev_ops = { static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) { + const struct phylink_mac_ops *mac_ops = &mtk_phylink_ops; const __be32 *_id = of_get_property(np, "reg", NULL); phy_interface_t phy_mode; struct phylink *phylink; struct mtk_mac *mac; - int id, err; + int id, err, i; int txqs = 1; u32 val; @@ -4881,8 +4904,8 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.type = PHYLINK_NETDEV; mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD; - mac->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD | - MAC_2500FD; + /* LPI above 1 Gbps is not supported */ + mac->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD; mac->phylink_config.lpi_timer_default = 1000; /* MT7623 gmac0 is now missing its speed-specific PLL configuration @@ -4932,9 +4955,29 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.supported_interfaces); } + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) + mac_ops = &rt5350_phylink_ops; + + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_2P5GPHY) && + id == MTK_GMAC2_ID) + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + mac->phylink_config.supported_interfaces); + + /* LPI wake-up timing is only verified on MTK_GMAC_EEE SoCs */ + if (MTK_HAS_CAPS(eth->soc->caps, MTK_GMAC_EEE)) { + phy_interface_copy(mac->phylink_config.lpi_interfaces, + mac->phylink_config.supported_interfaces); + __clear_bit(PHY_INTERFACE_MODE_2500BASEX, + mac->phylink_config.lpi_interfaces); + for (i = 0; i < PHY_INTERFACE_MODE_MAX; i++) + if (mtk_interface_mode_is_xgmii(eth, i)) + __clear_bit(i, + mac->phylink_config.lpi_interfaces); + } + phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), - phy_mode, &mtk_phylink_ops); + phy_mode, mac_ops); if (IS_ERR(phylink)) { err = PTR_ERR(phylink); goto free_netdev; @@ -4942,11 +4985,6 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink = phylink; - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_2P5GPHY) && - id == MTK_GMAC2_ID) - __set_bit(PHY_INTERFACE_MODE_INTERNAL, - mac->phylink_config.supported_interfaces); - SET_NETDEV_DEV(eth->netdev[id], eth->dev); eth->netdev[id]->watchdog_timeo = 5 * HZ; eth->netdev[id]->netdev_ops = &mtk_netdev_ops; diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h index 0168e2fbc6197..88a9b3b23bea5 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h @@ -994,6 +994,7 @@ enum mkt_eth_capabilities { MTK_U3_COPHY_V2_BIT, MTK_SRAM_BIT, MTK_36BIT_DMA_BIT, + MTK_GMAC_EEE_BIT, /* MUX BITS*/ MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT, @@ -1034,6 +1035,7 @@ enum mkt_eth_capabilities { #define MTK_U3_COPHY_V2 BIT_ULL(MTK_U3_COPHY_V2_BIT) #define MTK_SRAM BIT_ULL(MTK_SRAM_BIT) #define MTK_36BIT_DMA BIT_ULL(MTK_36BIT_DMA_BIT) +#define MTK_GMAC_EEE BIT_ULL(MTK_GMAC_EEE_BIT) #define MTK_ETH_MUX_GDM1_TO_GMAC1_ESW \ BIT_ULL(MTK_ETH_MUX_GDM1_TO_GMAC1_ESW_BIT) @@ -1117,7 +1119,7 @@ enum mkt_eth_capabilities { #define MT7981_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | MTK_GMAC2_GEPHY | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ MTK_MUX_U3_GMAC2_TO_QPHY | MTK_U3_COPHY_V2 | \ - MTK_RSTCTRL_PPE1 | MTK_SRAM) + MTK_RSTCTRL_PPE1 | MTK_SRAM | MTK_GMAC_EEE) #define MT7986_CAPS (MTK_GMAC1_SGMII | MTK_GMAC2_SGMII | \ MTK_MUX_GMAC12_TO_GEPHY_SGMII | MTK_QDMA | \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 3c3e84100d5aa..925ee25d05b47 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -143,7 +143,7 @@ config MLX5_CORE_IPOIB config MLX5_MACSEC bool "Connect-X support for MACSec offload" depends on MLX5_CORE_EN - depends on MACSEC + depends on MACSEC=y || MACSEC=MLX5_CORE default n help Build support for MACsec cryptography-offload acceleration in the NIC. diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index c9555119a6617..e49c7df79f124 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -14,6 +14,10 @@ enum mlx5_devlink_resource_id { MLX5_ID_RES_MAX = __MLX5_ID_RES_MAX - 1, }; +enum mlx5_devlink_port_resource_id { + MLX5_DL_PORT_RES_MAX_SFS = 1, +}; + enum mlx5_devlink_param_id { MLX5_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c index 0b82a6a133d6c..74f68b70ad1c1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fw_tracer.c @@ -1025,13 +1025,11 @@ struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) tracer = kvzalloc(sizeof(*tracer), GFP_KERNEL); if (!tracer) - return ERR_PTR(-ENOMEM); + return NULL; tracer->work_queue = create_singlethread_workqueue("mlx5_fw_tracer"); - if (!tracer->work_queue) { - err = -ENOMEM; + if (!tracer->work_queue) goto free_tracer; - } tracer->dev = dev; @@ -1073,7 +1071,7 @@ struct mlx5_fw_tracer *mlx5_fw_tracer_create(struct mlx5_core_dev *dev) destroy_workqueue(tracer->work_queue); free_tracer: kvfree(tracer); - return ERR_PTR(err); + return NULL; } static int fw_tracer_event(struct notifier_block *nb, unsigned long action, void *data); @@ -1084,7 +1082,7 @@ int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) struct mlx5_core_dev *dev; int err; - if (IS_ERR_OR_NULL(tracer)) + if (!tracer) return 0; if (!tracer->str_db.loaded) @@ -1134,7 +1132,7 @@ int mlx5_fw_tracer_init(struct mlx5_fw_tracer *tracer) /* Stop tracer + Cleanup HW resources */ void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) { - if (IS_ERR_OR_NULL(tracer)) + if (!tracer) return; mutex_lock(&tracer->state_lock); @@ -1163,7 +1161,7 @@ void mlx5_fw_tracer_cleanup(struct mlx5_fw_tracer *tracer) /* Free software resources (Buffers, etc ..) */ void mlx5_fw_tracer_destroy(struct mlx5_fw_tracer *tracer) { - if (IS_ERR_OR_NULL(tracer)) + if (!tracer) return; mlx5_core_dbg(tracer->dev, "FWTracer: Destroy\n"); @@ -1215,7 +1213,7 @@ int mlx5_fw_tracer_reload(struct mlx5_fw_tracer *tracer) struct mlx5_core_dev *dev; int err; - if (IS_ERR_OR_NULL(tracer)) + if (!tracer) return 0; dev = tracer->dev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 3d8276a0692fc..91653b69328ee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -477,7 +477,7 @@ typedef int (*mlx5e_fp_xmit_xdp_frame_check)(struct mlx5e_xdpsq *); typedef bool (*mlx5e_fp_xmit_xdp_frame)(struct mlx5e_xdpsq *, struct mlx5e_xmit_data *, int, - struct xsk_tx_metadata *); + struct xsk_tx_metadata **); struct mlx5e_xdpsq { /* data path */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c index 6049ccf475bc4..a4c096a4fed25 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/port.c @@ -557,6 +557,7 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) u32 in[MLX5_ST_SZ_DW(pplm_reg)] = {}; int sz = MLX5_ST_SZ_BYTES(pplm_reg); u16 fec_policy_auto = 0; + bool fec_set = false; int err; int i; @@ -569,9 +570,6 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) if (fec_policy >= (1 << MLX5E_FEC_LLRS_272_257_1) && !fec_50g_per_lane) return -EOPNOTSUPP; - if (fec_policy && !mlx5e_fec_in_caps(dev, fec_policy)) - return -EOPNOTSUPP; - MLX5_SET(pplm_reg, in, local_port, 1); err = mlx5_core_access_reg(dev, in, sz, out, sz, MLX5_REG_PPLM, 0, 0); if (err) @@ -591,12 +589,17 @@ int mlx5e_set_fec_mode(struct mlx5_core_dev *dev, u16 fec_policy) mlx5e_get_fec_cap_field(out, &fec_caps, i); /* policy supported for link speed */ - if (fec_caps & conf_fec) + if (fec_caps & conf_fec) { mlx5e_fec_admin_field(out, &conf_fec, 1, i); - else - /* set FEC to auto*/ + fec_set = true; + } else { + /* set FEC to auto */ mlx5e_fec_admin_field(out, &fec_policy_auto, 1, i); + } } + if (fec_policy && !fec_set) + return -EOPNOTSUPP; + return mlx5_core_access_reg(dev, out, sz, out, sz, MLX5_REG_PPLM, 0, 1); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c index 8d9a3b5ec973b..c845ed8f9ebfa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c @@ -8,13 +8,28 @@ static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, unsigned int size) { unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); - int i; + unsigned int unique_count = 0; + int i, j; + + /* Count unique vhca_ids */ + for (i = 0; i < size; i++) { + bool is_unique = true; + + /* Check if vhca_ids[i] was already seen */ + for (j = 0; j < i; j++) { + if (vhca_ids[j] == vhca_ids[i]) { + is_unique = false; + break; + } + } + if (is_unique) + unique_count++; + } - /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */ - for (i = 0; i < size; i++) - if (vhca_ids[i] >= max_num_vhca_id) - return false; - return true; + /* Verify that number of unique vhca_ids doesn't exceed + * max_num_vhca_id + */ + return unique_count <= max_num_vhca_id; } static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c index 5db239cae8145..5ae13af900308 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c @@ -311,12 +311,15 @@ sample_restore_get(struct mlx5e_tc_psample *tc_psample, u32 obj_id, static void sample_restore_put(struct mlx5e_tc_psample *tc_psample, struct mlx5e_sample_restore *restore) { + bool last; + mutex_lock(&tc_psample->restore_lock); - if (--restore->count == 0) + last = --restore->count == 0; + if (last) hash_del(&restore->hlist); mutex_unlock(&tc_psample->restore_lock); - if (!restore->count) { + if (last) { mlx5_del_flow_rules(restore->rule); mlx5_modify_header_dealloc(tc_psample->esw->dev, restore->modify_hdr); kfree(restore); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index efb34de4cb7ad..692ac2f21d94e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -30,6 +30,7 @@ enum { MLX5E_TC_FLOW_FLAG_FAILED = MLX5E_TC_FLOW_BASE + 9, MLX5E_TC_FLOW_FLAG_SAMPLE = MLX5E_TC_FLOW_BASE + 10, MLX5E_TC_FLOW_FLAG_USE_ACT_STATS = MLX5E_TC_FLOW_BASE + 11, + MLX5E_TC_FLOW_FLAG_PEER = MLX5E_TC_FLOW_BASE + 12, }; struct mlx5e_tc_flow_parse_attr { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 5322964214b22..2f52e04b28de2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -451,11 +451,11 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta); + int check_result, struct xsk_tx_metadata **meta); INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta) + int check_result, struct xsk_tx_metadata **meta) { struct mlx5e_tx_mpwqe *session = &sq->mpwqe; struct mlx5e_xdpsq_stats *stats = sq->stats; @@ -503,7 +503,10 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx * and it's safe to complete it at any time. */ mlx5e_xdp_mpwqe_session_start(sq); - xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, &session->wqe->eth); + if (meta) + xsk_tx_metadata_request(sq->xsk_pool, meta, + &mlx5e_xsk_tx_metadata_ops, + &session->wqe->eth); } mlx5e_xdp_mpwqe_add_dseg(sq, p, stats); @@ -534,7 +537,7 @@ INDIRECT_CALLABLE_SCOPE int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq) INDIRECT_CALLABLE_SCOPE bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, - int check_result, struct xsk_tx_metadata *meta) + int check_result, struct xsk_tx_metadata **meta) { struct mlx5e_xmit_data_frags *xdptxdf = container_of(xdptxd, struct mlx5e_xmit_data_frags, xd); @@ -648,7 +651,9 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, sq->pc += num_wqebbs; - xsk_tx_metadata_request(meta, &mlx5e_xsk_tx_metadata_ops, eseg); + if (meta) + xsk_tx_metadata_request(sq->xsk_pool, meta, + &mlx5e_xsk_tx_metadata_ops, eseg); sq->doorbell_cseg = cseg; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index 46ab0a9e8cddb..1a9eb77981cc3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -113,11 +113,11 @@ extern const struct xsk_tx_metadata_ops mlx5e_xsk_tx_metadata_ops; INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, int check_result, - struct xsk_tx_metadata *meta)); + struct xsk_tx_metadata **meta)); INDIRECT_CALLABLE_DECLARE(bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd, int check_result, - struct xsk_tx_metadata *meta)); + struct xsk_tx_metadata **meta)); INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check_mpwqe(struct mlx5e_xdpsq *sq)); INDIRECT_CALLABLE_DECLARE(int mlx5e_xmit_xdp_frame_check(struct mlx5e_xdpsq *sq)); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c index a59199ed590da..a7e3fe275d141 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c @@ -97,13 +97,13 @@ bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); xdptxd.len = desc.len; - meta = xsk_buff_get_metadata(pool, desc.addr); + meta = xsk_buff_get_metadata(pool, desc.addr, desc.options); xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, mlx5e_xmit_xdp_frame, sq, &xdptxd, - check_result, meta); + check_result, &meta); if (unlikely(!ret)) { if (sq->mpwqe.wqe) mlx5e_xdp_mpwqe_complete(sq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c index cf8f14ce4cd50..bad8ef5716069 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_dcbnl.c @@ -133,7 +133,7 @@ static int mlx5e_dcbnl_ieee_getets(struct net_device *netdev, if (err) return err; - if (ets->tc_tx_bw[i] < MLX5E_MAX_BW_ALLOC && + if (priv->dcbx.tc_tsa[i] == IEEE_8021QAZ_TSA_ETS && tc_group[i] == (MLX5E_LOWEST_PRIO_GROUP + 1)) is_zero_bw_ets_tc = true; @@ -158,6 +158,13 @@ static int mlx5e_dcbnl_ieee_getets(struct net_device *netdev, } memcpy(ets->tc_tsa, priv->dcbx.tc_tsa, sizeof(ets->tc_tsa)); + /* Report 0 for non ETS TSA */ + for (i = 0; i < ets->ets_cap; i++) { + if (ets->tc_tx_bw[i] == MLX5E_MAX_BW_ALLOC && + priv->dcbx.tc_tsa[i] != IEEE_8021QAZ_TSA_ETS) + ets->tc_tx_bw[i] = 0; + } + return err; } @@ -302,6 +309,14 @@ static int mlx5e_dbcnl_validate_ets(struct net_device *netdev, } } + for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { + if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_CB_SHAPER) { + netdev_err(netdev, + "Failed to validate ETS: CB Shaper is not supported\n"); + return -EOPNOTSUPP; + } + } + /* Validate Bandwidth Sum */ for (i = 0; i < IEEE_8021QAZ_MAX_TCS; i++) { if (ets->tc_tsa[i] == IEEE_8021QAZ_TSA_ETS) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 893e1380a7c97..1aef0f550b58b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -985,6 +985,7 @@ static const u32 pplm_fec_2_ethtool_linkmodes[] = { [MLX5E_FEC_NOFEC] = ETHTOOL_LINK_MODE_FEC_NONE_BIT, [MLX5E_FEC_FIRECODE] = ETHTOOL_LINK_MODE_FEC_BASER_BIT, [MLX5E_FEC_RS_528_514] = ETHTOOL_LINK_MODE_FEC_RS_BIT, + [MLX5E_FEC_RS_544_514_INTERLEAVED_QUAD] = ETHTOOL_LINK_MODE_FEC_RS_BIT, [MLX5E_FEC_RS_544_514] = ETHTOOL_LINK_MODE_FEC_RS_BIT, [MLX5E_FEC_LLRS_272_257_1] = ETHTOOL_LINK_MODE_FEC_LLRS_BIT, }; @@ -2027,7 +2028,7 @@ static int mlx5e_get_module_info(struct net_device *netdev, int size_read = 0; u8 data[4] = {0}; - size_read = mlx5_query_module_eeprom(dev, 0, 2, data); + size_read = mlx5_query_module_eeprom(dev, 0, 2, data, NULL); if (size_read < 2) return -EIO; @@ -2069,6 +2070,7 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev, struct mlx5_core_dev *mdev = priv->mdev; int offset = ee->offset; int size_read; + u8 status = 0; int i = 0; if (!ee->len) @@ -2078,15 +2080,15 @@ static int mlx5e_get_module_eeprom(struct net_device *netdev, while (i < ee->len) { size_read = mlx5_query_module_eeprom(mdev, offset, ee->len - i, - data + i); - + data + i, &status); if (!size_read) /* Done reading */ return 0; if (size_read < 0) { - netdev_err(priv->netdev, "%s: mlx5_query_eeprom failed:0x%x\n", - __func__, size_read); + netdev_err(netdev, + "%s: mlx5_query_eeprom failed:0x%x, status %u\n", + __func__, size_read, status); return size_read; } @@ -2106,6 +2108,7 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev, struct mlx5_core_dev *mdev = priv->mdev; u8 *data = page_data->data; int size_read; + u8 status = 0; int i = 0; if (!page_data->length) @@ -2119,7 +2122,8 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev, query.page = page_data->page; while (i < page_data->length) { query.size = page_data->length - i; - size_read = mlx5_query_module_eeprom_by_page(mdev, &query, data + i); + size_read = mlx5_query_module_eeprom_by_page(mdev, &query, + data + i, &status); /* Done reading, return how many bytes was read */ if (!size_read) @@ -2128,8 +2132,8 @@ static int mlx5e_get_module_eeprom_by_page(struct net_device *netdev, if (size_read < 0) { NL_SET_ERR_MSG_FMT_MOD( extack, - "Query module eeprom by page failed, read %u bytes, err %d", - i, size_read); + "Query module eeprom by page failed, read %u bytes, err %d, status %u", + i, size_read, status); return size_read; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index f660ef37b0a63..a7d84b540488a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -211,11 +211,11 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv) static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) { - struct mlx5e_priv *slave_priv = my_data; + struct mlx5e_priv *master_priv = event_data; switch (event) { case MPV_DEVCOM_MASTER_UP: - mlx5_devcom_comp_set_ready(slave_priv->devcom, true); + mlx5_devcom_comp_set_ready(master_priv->devcom, true); break; case MPV_DEVCOM_MASTER_DOWN: /* no need for comp set ready false since we unregister after @@ -1926,8 +1926,10 @@ int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) { sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); + /* Reset BQL only when the SQ has no bytes in flight. */ + if (sq->cc == sq->pc) + netdev_tx_reset_queue(sq->txq); set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); - netdev_tx_reset_queue(sq->txq); netif_tx_start_queue(sq->txq); netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); } @@ -5540,8 +5542,9 @@ struct mlx5_qmgmt_data { struct mlx5e_channel_param cparam; }; -static int mlx5e_queue_mem_alloc(struct net_device *dev, void *newq, - int queue_index) +static int mlx5e_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *newq, int queue_index) { struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; struct mlx5e_priv *priv = netdev_priv(dev); @@ -5602,8 +5605,9 @@ static int mlx5e_queue_stop(struct net_device *dev, void *oldq, int queue_index) return 0; } -static int mlx5e_queue_start(struct net_device *dev, void *newq, - int queue_index) +static int mlx5e_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *newq, int queue_index) { struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; struct mlx5e_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 8b65441246244..a47fef49ada97 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -57,7 +57,6 @@ #include "lib/vxlan.h" #define CREATE_TRACE_POINTS #include "diag/en_rep_tracepoint.h" -#include "diag/reporter_vnic.h" #include "en_accel/ipsec.h" #include "en/tc/int_port.h" #include "en/ptp.h" @@ -687,11 +686,13 @@ static int mlx5e_rep_open(struct net_device *dev) if (err) goto unlock; + mutex_lock(&rep->esw->state_lock); if (!mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP)) netif_carrier_on(dev); + mutex_unlock(&rep->esw->state_lock); unlock: mutex_unlock(&priv->state_lock); @@ -706,10 +707,12 @@ static int mlx5e_rep_close(struct net_device *dev) int ret; mutex_lock(&priv->state_lock); + mutex_lock(&rep->esw->state_lock); mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); + mutex_unlock(&rep->esw->state_lock); ret = mlx5e_close_locked(dev); mutex_unlock(&priv->state_lock); return ret; @@ -781,22 +784,25 @@ static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_rep_priv *rpriv = priv->ppriv; struct mlx5_eswitch_rep *rep = rpriv->rep; - int err; + int err = 0; + mutex_lock(&rep->esw->state_lock); if (new_carrier) { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); if (err) - return err; + goto unlock; netif_carrier_on(dev); } else { err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); if (err) - return err; + goto unlock; netif_carrier_off(dev); } - return 0; +unlock: + mutex_unlock(&rep->esw->state_lock); + return err; } static const struct net_device_ops mlx5e_netdev_ops_rep = { @@ -1335,9 +1341,12 @@ static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) mlx5e_rep_tc_enable(priv); - if (MLX5_CAP_GEN(mdev, uplink_follow)) + if (MLX5_CAP_GEN(mdev, uplink_follow)) { + mutex_lock(&mdev->priv.eswitch->state_lock); mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); + mutex_unlock(&mdev->priv.eswitch->state_lock); + } mlx5_lag_add_netdev(mdev, netdev); priv->events_nb.notifier_call = uplink_rep_async_event; mlx5_notifier_register(mdev, &priv->events_nb); @@ -1421,51 +1430,6 @@ static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv) return ARRAY_SIZE(mlx5e_ul_rep_stats_grps); } -static int -mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, - struct devlink_fmsg *fmsg, - struct netlink_ext_ack *extack) -{ - struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter); - struct mlx5_eswitch_rep *rep = rpriv->rep; - - mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg, rep->vport, - true); - return 0; -} - -static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = { - .name = "vnic", - .diagnose = mlx5e_rep_vnic_reporter_diagnose, -}; - -static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv, - struct devlink_port *dl_port) -{ - struct mlx5e_rep_priv *rpriv = priv->ppriv; - struct devlink_health_reporter *reporter; - - reporter = devl_port_health_reporter_create(dl_port, - &mlx5_rep_vnic_reporter_ops, - rpriv); - if (IS_ERR(reporter)) { - mlx5_core_err(priv->mdev, - "Failed to create representor vnic reporter, err = %pe\n", - reporter); - return; - } - - rpriv->rep_vnic_reporter = reporter; -} - -static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv) -{ - struct mlx5e_rep_priv *rpriv = priv->ppriv; - - if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter)) - devl_health_reporter_destroy(rpriv->rep_vnic_reporter); -} - static const struct mlx5e_profile mlx5e_rep_profile = { .init = mlx5e_init_rep, .cleanup = mlx5e_cleanup_rep, @@ -1589,10 +1553,8 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, rpriv->rep->vport); - if (!IS_ERR(dl_port)) { + if (!IS_ERR(dl_port)) SET_NETDEV_DEVLINK_PORT(netdev, dl_port); - mlx5e_rep_vnic_reporter_create(priv, dl_port); - } err = register_netdev(netdev); if (err) { @@ -1605,7 +1567,6 @@ mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) return 0; err_detach_netdev: - mlx5e_rep_vnic_reporter_destroy(priv); mlx5e_detach_netdev(netdev_priv(netdev)); err_cleanup_profile: priv->profile->cleanup(priv); @@ -1663,7 +1624,6 @@ mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) } unregister_netdev(netdev); - mlx5e_rep_vnic_reporter_destroy(priv); mlx5e_detach_netdev(priv); priv->profile->cleanup(priv); mlx5e_destroy_netdev(netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index 70640fa1ad7bc..bcd7b4e814d06 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -118,7 +118,6 @@ struct mlx5e_rep_priv { struct rtnl_link_stats64 prev_vf_vport_stats; struct mlx5_flow_handle *send_to_vport_meta_rule; struct rhashtable tc_ht; - struct devlink_health_reporter *rep_vnic_reporter; }; static inline diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 2ffa4e6b8c373..17b74a97138b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -2014,11 +2014,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_frag_page *frag_page, u32 data_bcnt, u32 data_offset) { + u32 page_size = BIT(rq->mpwqe.page_shift); + net_prefetchw(skb->data); do { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ - u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt); + u32 pg_consumed_bytes = min_t(u32, page_size - data_offset, + data_bcnt); unsigned int truesize = pg_consumed_bytes; mlx5e_add_skb_frag(rq, skb, frag_page, data_offset, @@ -2039,6 +2042,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); struct mlx5e_frag_page *head_page = frag_page; struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; + u32 page_size = BIT(rq->mpwqe.page_shift); u32 frag_offset = head_offset; u32 byte_cnt = cqe_bcnt; struct skb_shared_info *sinfo; @@ -2084,9 +2088,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w linear_hr = skb_headroom(skb); linear_data_len = headlen; linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb)); - if (unlikely(frag_offset >= PAGE_SIZE)) { + if (unlikely(frag_offset >= page_size)) { frag_page++; - frag_offset -= PAGE_SIZE; + frag_offset -= page_size; } } @@ -2098,7 +2102,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w while (byte_cnt) { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ pg_consumed_bytes = - min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); + min_t(u32, page_size - frag_offset, byte_cnt); if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) truesize += pg_consumed_bytes; @@ -2135,7 +2139,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w new_nr_frags = sinfo->nr_frags; nr_frags_free = old_nr_frags - new_nr_frags; if (unlikely(nr_frags_free)) - truesize -= (nr_frags_free - 1) * PAGE_SIZE + + truesize -= (nr_frags_free - 1) * page_size + ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); @@ -2349,15 +2353,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match) rq->hw_gro_data->skb = NULL; } -static bool -mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt) +static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, + u16 data_bcnt, + u32 page_size) { int nr_frags = skb_shinfo(skb)->nr_frags; - if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE) + if (page_size >= GRO_LEGACY_MAX_SIZE) return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE; else - return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; + return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; } static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) @@ -2366,18 +2371,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); u32 wqe_offset = be32_to_cpu(cqe->shampo.data_offset); u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); - u32 data_offset = wqe_offset & (PAGE_SIZE - 1); u32 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); - u32 page_idx = wqe_offset >> PAGE_SHIFT; u16 head_size = cqe->shampo.header_size; struct sk_buff **skb = &rq->hw_gro_data->skb; bool flush = cqe->shampo.flush; bool match = cqe->shampo.match; + u32 page_size = BIT(rq->mpwqe.page_shift); struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; + u32 data_offset; + u32 page_idx; wi = mlx5e_get_mpw_info(rq, wqe_id); wi->consumed_strides += cstrides; @@ -2393,7 +2399,16 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq goto mpwrq_cqe_out; } - if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) { + data_offset = wqe_offset & (page_size - 1); + page_idx = wqe_offset >> rq->mpwqe.page_shift; + if (unlikely(cqe_bcnt <= ETH_ZLEN + 2 * VLAN_HLEN)) { + match = false; + flush = true; + } + + if (*skb && + !(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt, + page_size))) { match = false; mlx5e_shampo_flush_skb(rq, cqe, match); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 5ae501f448338..8450b07b0146e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -1170,50 +1170,39 @@ void mlx5e_stats_rmon_get(struct mlx5e_priv *priv, void mlx5e_stats_ts_get(struct mlx5e_priv *priv, struct ethtool_ts_stats *ts_stats) { - int i, j; + u16 nch = mlx5e_stats_nch_read(priv); + int i, tc; - mutex_lock(&priv->state_lock); + ts_stats->pkts = 0; - if (priv->tx_ptp_opened) { - struct mlx5e_ptp *ptp = priv->channels.ptp; + for (i = 0; i < nch; i++) { + struct mlx5e_channel_stats *channel_stats = + priv->channel_stats[i]; - ts_stats->pkts = 0; + for (tc = 0; tc < priv->max_opened_tc; tc++) + ts_stats->pkts += channel_stats->sq[tc].timestamps; + } + + /* Accumulate DMA and port timestamp counters so values stay monotonic + * across channel teardown and mode switches. + */ + if (priv->tx_ptp_opened) { + /* Err and Lost stats are only relevant for port timestamping, + * as the DMA layer will always successfully timestamp packets. + */ ts_stats->err = 0; ts_stats->lost = 0; - if (!ptp) - goto out; - - /* Aggregate stats across all TCs */ - for (i = 0; i < ptp->num_tc; i++) { + for (tc = 0; tc < priv->max_opened_tc; tc++) { struct mlx5e_ptp_cq_stats *stats = - ptp->ptpsq[i].cq_stats; + &priv->ptp_stats.cq[tc]; ts_stats->pkts += stats->cqe; ts_stats->err += stats->abort + stats->err_cqe + - stats->late_cqe; + stats->late_cqe; ts_stats->lost += stats->lost_cqe; } - } else { - /* DMA layer will always successfully timestamp packets. Other - * counters do not make sense for this layer. - */ - ts_stats->pkts = 0; - - /* Aggregate stats across all SQs */ - for (j = 0; j < priv->channels.num; j++) { - struct mlx5e_channel *c = priv->channels.c[j]; - - for (i = 0; i < c->num_tc; i++) { - struct mlx5e_sq_stats *stats = c->sq[i].stats; - - ts_stats->pkts += stats->timestamps; - } - } } - -out: - mutex_unlock(&priv->state_lock); } #define PPORT_PHY_LAYER_OFF(c) \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index ebea43c235cc3..e2de3a9a647c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -2165,7 +2165,8 @@ static void mlx5e_tc_del_flow(struct mlx5e_priv *priv, if (mlx5e_is_eswitch_flow(flow)) { struct mlx5_devcom_comp_dev *devcom = flow->priv->mdev->priv.eswitch->devcom; - if (!mlx5_devcom_for_each_peer_begin(devcom)) { + if (flow_flag_test(flow, PEER) || + !mlx5_devcom_for_each_peer_begin(devcom)) { mlx5e_tc_del_fdb_flow(priv, flow); return; } @@ -4609,6 +4610,7 @@ static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f, else in_mdev = priv->mdev; + flow_flags |= BIT(MLX5E_TC_FLOW_FLAG_PEER); parse_attr = flow->attr->parse_attr; peer_flow = __mlx5e_add_fdb_flow(peer_priv, f, flow_flags, parse_attr->filter_dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c index 76108299ea57d..65758e7211747 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c @@ -247,7 +247,7 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget) } if (unlikely(aff_change && busy_xsk)) { - mlx5e_trigger_irq(&c->icosq); + mlx5e_trigger_napi_icosq(c); ch_stats->force_irq++; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c index 3ce455c2535c4..53f064ebd1c18 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/acl/helper.c @@ -71,7 +71,7 @@ int esw_egress_acl_vlan_create(struct mlx5_eswitch *esw, flow_act.action = flow_action; vport->egress.allowed_vlan = mlx5_add_flow_rules(vport->egress.acl, spec, - &flow_act, fwd_dest, 0); + &flow_act, fwd_dest, fwd_dest ? 1 : 0); if (IS_ERR(vport->egress.allowed_vlan)) { err = PTR_ERR(vport->egress.allowed_vlan); esw_warn(esw->dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c index 0091ba697baec..8eaef73f3d77d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/adj_vport.c @@ -4,16 +4,33 @@ #include "fs_core.h" #include "eswitch.h" -enum { - MLX5_ADJ_VPORT_DISCONNECT = 0x0, - MLX5_ADJ_VPORT_CONNECT = 0x1, -}; - -static int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, - u16 vport, bool connect) +int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, u16 vport, + bool connect) { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; + lockdep_assert_held(&dev->priv.eswitch->state_lock); + + if (MLX5_CAP_ESW(dev, esw_vport_state_max_tx_speed)) { + u8 op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + struct mlx5_vport *esw_vport; + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(dev, op_mod, vport, + true, &speed, NULL); + if (err) { + esw_vport = mlx5_eswitch_get_vport(dev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; + mlx5_core_dbg(dev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, @@ -24,7 +41,7 @@ static int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, MLX5_SET(modify_vport_state_in, in, egress_connect_valid, 1); MLX5_SET(modify_vport_state_in, in, ingress_connect, connect); MLX5_SET(modify_vport_state_in, in, egress_connect, connect); - + MLX5_SET(modify_vport_state_in, in, admin_state, connect); return mlx5_cmd_exec_in(dev, modify_vport_state, in); } @@ -96,7 +113,6 @@ static int mlx5_esw_adj_vport_create(struct mlx5_eswitch *esw, u16 vhca_id, if (err) goto acl_ns_remove; - mlx5_esw_adj_vport_modify(esw->dev, vport_num, MLX5_ADJ_VPORT_CONNECT); return 0; acl_ns_remove: @@ -117,8 +133,7 @@ static void mlx5_esw_adj_vport_destroy(struct mlx5_eswitch *esw, esw_debug(esw->dev, "Destroying adjacent vport %d for vhca_id 0x%x\n", vport_num, vport->vhca_id); - mlx5_esw_adj_vport_modify(esw->dev, vport_num, - MLX5_ADJ_VPORT_DISCONNECT); + mlx5_esw_offloads_rep_remove(esw, vport); mlx5_fs_vport_egress_acl_ns_remove(esw->dev->priv.steering, vport->index); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index cf88a106d80d7..bac24f158978b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -3,6 +3,27 @@ #include #include "eswitch.h" +#include "devlink.h" +#include "diag/reporter_vnic.h" + +static int +mlx5_esw_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, + struct netlink_ext_ack *extack) +{ + struct mlx5_vport *vport = devlink_health_reporter_priv(reporter); + + mlx5_reporter_vnic_diagnose_counters(vport->dev, fmsg, vport->vport, + true); + + return 0; +} + +static const +struct devlink_health_reporter_ops mlx5_esw_rep_vnic_reporter_ops = { + .name = "vnic", + .diagnose = mlx5_esw_rep_vnic_reporter_diagnose, +}; static void mlx5_esw_get_port_parent_id(struct mlx5_core_dev *dev, struct netdev_phys_item_id *ppid) @@ -160,8 +181,35 @@ static const struct devlink_port_ops mlx5_esw_dl_sf_port_ops = { .port_fn_max_io_eqs_set = mlx5_devlink_port_fn_max_io_eqs_set, }; +static int mlx5_esw_devlink_port_res_register(struct mlx5_eswitch *esw, + struct devlink_port *dl_port) +{ + struct devlink_resource_size_params size_params; + struct mlx5_core_dev *dev = esw->dev; + u16 max_sfs, sf_base_id; + int err; + + err = mlx5_esw_sf_max_hpf_functions(dev, &max_sfs, &sf_base_id); + if (err) + return err; + + devlink_resource_size_params_init(&size_params, max_sfs, max_sfs, 1, + DEVLINK_RESOURCE_UNIT_ENTRY); + + return devl_port_resource_register(dl_port, "max_SFs", max_sfs, + MLX5_DL_PORT_RES_MAX_SFS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); +} + +static void mlx5_esw_devlink_port_res_unregister(struct devlink_port *dl_port) +{ + devl_port_resources_unregister(dl_port); +} + int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { + struct devlink_health_reporter *reporter; struct mlx5_core_dev *dev = esw->dev; const struct devlink_port_ops *ops; struct mlx5_devlink_port *dl_port; @@ -191,6 +239,25 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx if (err) goto rate_err; + if (vport_num == MLX5_VPORT_PF) { + err = mlx5_esw_devlink_port_res_register(esw, + &dl_port->dl_port); + if (err) + mlx5_core_dbg(dev, + "Failed to register port resources: %d\n", + err); + } + + reporter = devl_port_health_reporter_create( + &dl_port->dl_port, &mlx5_esw_rep_vnic_reporter_ops, + vport); + if (IS_ERR(reporter)) + mlx5_core_err(dev, + "Failed to create vnic health reporter for vport %d: %pe\n", + vport_num, reporter); + else + dl_port->vnic_reporter = reporter; + return 0; rate_err: @@ -206,6 +273,13 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_vport *vport) return; dl_port = vport->dl_port; + if (dl_port->vnic_reporter) { + devl_health_reporter_destroy(dl_port->vnic_reporter); + dl_port->vnic_reporter = NULL; + } + + mlx5_esw_devlink_port_res_unregister(&dl_port->dl_port); + mlx5_esw_qos_vport_update_parent(vport, NULL, NULL); devl_rate_leaf_destroy(&dl_port->dl_port); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c index da10e04777cf7..1dbafeddd4b2f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c @@ -12,7 +12,7 @@ static int esw_ipsec_vf_query_generic(struct mlx5_core_dev *dev, u16 vport_num, void *hca_cap, *query_cap; int err; - if (!MLX5_CAP_GEN(dev, vhca_resource_manager)) + if (!MLX5_CAP_GEN(dev, vport_group_manager)) return -EOPNOTSUPP; if (!mlx5_esw_ipsec_vf_offload_supported(dev)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index c38deabcb7b96..19d203b049512 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -799,7 +799,7 @@ static int mlx5_esw_vport_caps_get(struct mlx5_eswitch *esw, struct mlx5_vport * void *hca_caps; int err; - if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) return 0; query_ctx = kzalloc(query_out_sz, GFP_KERNEL); @@ -904,6 +904,24 @@ static void esw_vport_cleanup(struct mlx5_eswitch *esw, struct mlx5_vport *vport esw_vport_cleanup_acl(esw, vport); } +static void mlx5_esw_vport_set_max_tx_speed(struct mlx5_eswitch *esw, + struct mlx5_vport *vport) +{ + int ret; + + if (!MLX5_CAP_ESW(esw->dev, esw_vport_state_max_tx_speed)) + return; + + ret = mlx5_modify_vport_max_tx_speed(esw->dev, + MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, + vport->vport, true, + vport->agg_max_tx_speed); + if (ret) + mlx5_core_dbg(esw->dev, + "Failed to set vport %d speed %d, err=%d\n", + vport->vport, vport->agg_max_tx_speed, ret); +} + int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, enum mlx5_eswitch_vport_event enabled_events) { @@ -934,7 +952,7 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, vport->info.trusted = true; if (!mlx5_esw_is_manager_vport(esw, vport_num) && - MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { + MLX5_CAP_GEN(esw->dev, vport_group_manager)) { ret = mlx5_esw_vport_vhca_id_map(esw, vport); if (ret) goto err_vhca_mapping; @@ -950,6 +968,9 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, esw->enabled_vports++; esw_debug(esw->dev, "Enabled VPORT(%d)\n", vport_num); + + if (vport->agg_max_tx_speed) + mlx5_esw_vport_set_max_tx_speed(esw, vport); done: mutex_unlock(&esw->state_lock); return ret; @@ -978,20 +999,26 @@ void mlx5_esw_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport) arm_vport_context_events_cmd(esw->dev, vport_num, 0); if (!mlx5_esw_is_manager_vport(esw, vport_num) && - MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) + MLX5_CAP_GEN(esw->dev, vport_group_manager)) mlx5_esw_vport_vhca_id_unmap(esw, vport); if (vport->vport != MLX5_VPORT_PF && (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled)) esw->enabled_ipsec_vf_count--; + /* Clear rx-mode before esw_vport_change_handle_locked(): on + * MLX5_VPORT_PROMISC_CHANGE it calls esw_update_vport_mc_promisc() + * when vport->allmulti_rule is set, repopulating mc_list with FDB + * rules that dangle once the FDB is destroyed. NULL allmulti_rule + * here skips that path. + */ + esw_apply_vport_rx_mode(esw, vport, false, false); /* We don't assume VFs will cleanup after themselves. * Calling vport change handler while vport is disabled will cleanup * the vport resources. */ esw_vport_change_handle_locked(vport); vport->enabled_events = 0; - esw_apply_vport_rx_mode(esw, vport, false, false); esw_vport_cleanup(esw, vport); esw->enabled_vports--; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 558055d214e10..01f56519334cc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -179,6 +179,7 @@ struct mlx5_vport; struct mlx5_devlink_port { struct devlink_port dl_port; struct mlx5_vport *vport; + struct devlink_health_reporter *vnic_reporter; }; static inline void mlx5_devlink_port_init(struct mlx5_devlink_port *dl_port, @@ -246,6 +247,7 @@ struct mlx5_vport { enum mlx5_eswitch_vport_event enabled_events; int index; struct mlx5_devlink_port *dl_port; + u32 agg_max_tx_speed; }; struct mlx5_esw_indir_table; @@ -264,6 +266,9 @@ struct mlx5_eswitch_fdb { struct offloads_fdb { struct mlx5_flow_namespace *ns; + struct mlx5_flow_table *drop_root; + struct mlx5_flow_handle *drop_root_rule; + struct mlx5_fc *drop_root_fc; struct mlx5_flow_table *tc_miss_table; struct mlx5_flow_table *slow_fdb; struct mlx5_flow_group *send_to_vport_grp; @@ -394,6 +399,7 @@ struct mlx5_eswitch { struct mlx5_esw_offload offloads; u32 last_vport_idx; int mode; + bool offloads_inactive; u16 manager_vport; u16 first_host_vport; u8 num_peers; @@ -636,6 +642,8 @@ const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev); void mlx5_esw_adjacent_vhcas_setup(struct mlx5_eswitch *esw); void mlx5_esw_adjacent_vhcas_cleanup(struct mlx5_eswitch *esw); +int mlx5_esw_adj_vport_modify(struct mlx5_core_dev *dev, u16 vport, + bool connect); #define MLX5_DEBUG_ESWITCH_MASK BIT(3) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 8be0961cb6c7e..fa833c9ba4744 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1575,6 +1575,7 @@ esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) attr.max_grp_num = esw->params.large_group_num; attr.default_ft = miss_fdb; attr.mapping = esw->offloads.reg_c0_obj_pool; + attr.fs_base_prio = FDB_BYPASS_PATH; chains = mlx5_chains_create(dev, &attr); if (IS_ERR(chains)) { @@ -2353,6 +2354,135 @@ static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode) mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp); } +static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw) +{ + if (!esw->fdb_table.offloads.drop_root) + return; + + esw_debug(esw->dev, "Destroying FDB drop root table %#x fc %#x\n", + esw->fdb_table.offloads.drop_root->id, + esw->fdb_table.offloads.drop_root_fc->id); + mlx5_del_flow_rules(esw->fdb_table.offloads.drop_root_rule); + /* Don't free flow counter here, can be reused on a later activation */ + mlx5_destroy_flow_table(esw->fdb_table.offloads.drop_root); + esw->fdb_table.offloads.drop_root_rule = NULL; + esw->fdb_table.offloads.drop_root = NULL; +} + +static int mlx5_esw_fdb_drop_create(struct mlx5_eswitch *esw) +{ + struct mlx5_flow_destination drop_fc_dst = {}; + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_destination *dst = NULL; + struct mlx5_core_dev *dev = esw->dev; + struct mlx5_flow_namespace *root_ns; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *flow_rule; + struct mlx5_flow_table *table; + int err = 0, dst_num = 0; + + if (esw->fdb_table.offloads.drop_root) + return 0; + + root_ns = esw->fdb_table.offloads.ns; + + ft_attr.prio = FDB_DROP_ROOT; + ft_attr.max_fte = 1; + ft_attr.autogroup.max_num_groups = 1; + table = mlx5_create_auto_grouped_flow_table(root_ns, &ft_attr); + if (IS_ERR(table)) { + esw_warn(dev, "Failed to create fdb drop root table, err %pe\n", + table); + return PTR_ERR(table); + } + + /* Drop FC reusable, create once on first deactivation of FDB */ + if (!esw->fdb_table.offloads.drop_root_fc) { + struct mlx5_fc *counter = mlx5_fc_create(dev, 0); + + err = PTR_ERR_OR_ZERO(counter); + if (err) + esw_warn(esw->dev, "create fdb drop fc err %d\n", err); + else + esw->fdb_table.offloads.drop_root_fc = counter; + } + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; + + if (esw->fdb_table.offloads.drop_root_fc) { + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + drop_fc_dst.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + drop_fc_dst.counter = esw->fdb_table.offloads.drop_root_fc; + dst = &drop_fc_dst; + dst_num++; + } + + flow_rule = mlx5_add_flow_rules(table, NULL, &flow_act, dst, dst_num); + err = PTR_ERR_OR_ZERO(flow_rule); + if (err) { + esw_warn(esw->dev, + "fs offloads: Failed to add vport rx drop rule err %d\n", + err); + goto err_flow_rule; + } + + esw->fdb_table.offloads.drop_root = table; + esw->fdb_table.offloads.drop_root_rule = flow_rule; + esw_debug(esw->dev, "Created FDB drop root table %#x fc %#x\n", + table->id, dst ? dst->counter->id : 0); + return 0; + +err_flow_rule: + /* no need to free drop fc, esw_offloads_steering_cleanup will do it */ + mlx5_destroy_flow_table(table); + return err; +} + +static void mlx5_esw_fdb_active(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + unsigned long i; + + mlx5_esw_fdb_drop_destroy(esw); + mlx5_mpfs_enable(esw->dev); + + mutex_lock(&esw->state_lock); + mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { + if (!vport->adjacent) + continue; + esw_debug(esw->dev, "Connecting vport %d to eswitch\n", + vport->vport); + mlx5_esw_adj_vport_modify(esw->dev, vport->vport, true); + } + mutex_unlock(&esw->state_lock); + + esw->offloads_inactive = false; + esw_warn(esw->dev, "MPFS/FDB active\n"); +} + +static void mlx5_esw_fdb_inactive(struct mlx5_eswitch *esw) +{ + struct mlx5_vport *vport; + unsigned long i; + + mlx5_mpfs_disable(esw->dev); + mlx5_esw_fdb_drop_create(esw); + + mutex_lock(&esw->state_lock); + mlx5_esw_for_each_vf_vport(esw, i, vport, U16_MAX) { + if (!vport->adjacent) + continue; + esw_debug(esw->dev, "Disconnecting vport %u from eswitch\n", + vport->vport); + + mlx5_esw_adj_vport_modify(esw->dev, vport->vport, false); + } + mutex_unlock(&esw->state_lock); + + esw->offloads_inactive = true; + esw_warn(esw->dev, "MPFS/FDB inactive\n"); +} + static int esw_offloads_start(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) { @@ -3436,6 +3566,10 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw) static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) { + mlx5_esw_fdb_drop_destroy(esw); + if (esw->fdb_table.offloads.drop_root_fc) + mlx5_fc_destroy(esw->dev, esw->fdb_table.offloads.drop_root_fc); + esw->fdb_table.offloads.drop_root_fc = NULL; esw_destroy_vport_rx_drop_rule(esw); esw_destroy_vport_rx_drop_group(esw); esw_destroy_vport_rx_group(esw); @@ -3604,6 +3738,11 @@ int esw_offloads_enable(struct mlx5_eswitch *esw) if (err) goto err_steering_init; + if (esw->offloads_inactive) + mlx5_esw_fdb_inactive(esw); + else + mlx5_esw_fdb_active(esw); + /* Representor will control the vport link state */ mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; @@ -3672,6 +3811,9 @@ void esw_offloads_disable(struct mlx5_eswitch *esw) esw_offloads_metadata_uninit(esw); mlx5_rdma_disable_roce(esw->dev); mlx5_esw_adjacent_vhcas_cleanup(esw); + /* must be done after vhcas cleanup to avoid adjacent vports connect */ + if (esw->offloads_inactive) + mlx5_esw_fdb_active(esw); /* legacy mode always active */ mutex_destroy(&esw->offloads.termtbl_mutex); } @@ -3682,6 +3824,7 @@ static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) *mlx5_mode = MLX5_ESWITCH_LEGACY; break; case DEVLINK_ESWITCH_MODE_SWITCHDEV: + case DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE: *mlx5_mode = MLX5_ESWITCH_OFFLOADS; break; default: @@ -3691,14 +3834,17 @@ static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) return 0; } -static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) +static int esw_mode_to_devlink(struct mlx5_eswitch *esw, u16 *mode) { - switch (mlx5_mode) { + switch (esw->mode) { case MLX5_ESWITCH_LEGACY: *mode = DEVLINK_ESWITCH_MODE_LEGACY; break; case MLX5_ESWITCH_OFFLOADS: - *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; + if (esw->offloads_inactive) + *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE; + else + *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; break; default: return -EINVAL; @@ -3804,6 +3950,45 @@ static bool mlx5_devlink_netdev_netns_immutable_set(struct devlink *devlink, return ret; } +/* Returns true when only changing between active and inactive switchdev mode */ +static bool mlx5_devlink_switchdev_active_mode_change(struct mlx5_eswitch *esw, + u16 devlink_mode) +{ + /* current mode is not switchdev */ + if (esw->mode != MLX5_ESWITCH_OFFLOADS) + return false; + + /* new mode is not switchdev */ + if (devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV && + devlink_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE) + return false; + + /* already inactive: no change in current state */ + if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE && + esw->offloads_inactive) + return false; + + /* already active: no change in current state */ + if (devlink_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && + !esw->offloads_inactive) + return false; + + down_write(&esw->mode_lock); + esw->offloads_inactive = !esw->offloads_inactive; + esw->eswitch_operation_in_progress = true; + up_write(&esw->mode_lock); + + if (esw->offloads_inactive) + mlx5_esw_fdb_inactive(esw); + else + mlx5_esw_fdb_active(esw); + + down_write(&esw->mode_lock); + esw->eswitch_operation_in_progress = false; + up_write(&esw->mode_lock); + return true; +} + int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3823,12 +4008,16 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; - if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && mlx5_get_sd(esw->dev)) { + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && mlx5_get_sd(esw->dev)) { NL_SET_ERR_MSG_MOD(extack, "Can't change E-Switch mode to switchdev when multi-PF netdev (Socket Direct) is configured."); return -EPERM; } + /* Avoid try_lock, active/inactive mode change is not restricted */ + if (mlx5_devlink_switchdev_active_mode_change(esw, mode)) + return 0; + mlx5_lag_disable_change(esw->dev); err = mlx5_esw_try_lock(esw); if (err < 0) { @@ -3851,7 +4040,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, esw->eswitch_operation_in_progress = true; up_write(&esw->mode_lock); - if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) { NL_SET_ERR_MSG_MOD(extack, "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); @@ -3859,27 +4048,29 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, goto skip; } - if (mode == DEVLINK_ESWITCH_MODE_LEGACY) + if (mlx5_mode == MLX5_ESWITCH_LEGACY) esw->dev->priv.flags |= MLX5_PRIV_FLAGS_SWITCH_LEGACY; if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_SWITCH_LEGACY; mlx5_eswitch_disable_locked(esw); - if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { NL_SET_ERR_MSG_MOD(extack, "Can't change mode while devlink traps are active"); err = -EOPNOTSUPP; goto skip; } + esw->offloads_inactive = + (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE); err = esw_offloads_start(esw, extack); - } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { + } else if (mlx5_mode == MLX5_ESWITCH_LEGACY) { err = esw_offloads_stop(esw, extack); } else { err = -EINVAL; } skip: - if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && err) + if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err) mlx5_devlink_netdev_netns_immutable_set(devlink, false); down_write(&esw->mode_lock); esw->eswitch_operation_in_progress = false; @@ -3898,7 +4089,7 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) if (IS_ERR(esw)) return PTR_ERR(esw); - return esw_mode_to_devlink(esw->mode, mode); + return esw_mode_to_devlink(esw, mode); } static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, @@ -4344,8 +4535,9 @@ int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enab return -EOPNOTSUPP; } - if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { - NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { + NL_SET_ERR_MSG_MOD(extack, + "Device doesn't support vport group management"); return -EOPNOTSUPP; } @@ -4420,8 +4612,9 @@ int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); - if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { - NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { + NL_SET_ERR_MSG_MOD(extack, + "Device doesn't support vport group management"); return -EOPNOTSUPP; } @@ -4685,9 +4878,9 @@ mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, int err; esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); - if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) { NL_SET_ERR_MSG_MOD(extack, - "Device doesn't support VHCA management"); + "Device doesn't support vport group management"); return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c index 40bdc677f051d..e282f65b8756a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads_termtbl.c @@ -163,12 +163,15 @@ void mlx5_eswitch_termtbl_put(struct mlx5_eswitch *esw, struct mlx5_termtbl_handle *tt) { + bool last; + mutex_lock(&esw->offloads.termtbl_mutex); - if (--tt->ref_count == 0) + last = (--tt->ref_count == 0); + if (last) hash_del(&tt->termtbl_hlist); mutex_unlock(&esw->offloads.termtbl_mutex); - if (!tt->ref_count) { + if (last) { mlx5_del_flow_rules(tt->rule); mlx5_destroy_flow_table(tt->termtbl); kfree(tt); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 2db3ffb0a2b28..2ca3bddbdf050 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -3520,6 +3520,11 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) if (!steering->fdb_root_ns) return -ENOMEM; + maj_prio = fs_create_prio(&steering->fdb_root_ns->ns, FDB_DROP_ROOT, 1); + err = PTR_ERR_OR_ZERO(maj_prio); + if (err) + goto out_err; + err = create_fdb_bypass(steering); if (err) goto out_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 2d6019a32669c..d20f9d3e30c79 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -62,14 +62,15 @@ static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags) return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY; } -static u8 lag_active_port_bits(struct mlx5_lag *ldev) +static u8 lag_active_port_bits(struct mlx5_lag *ldev, + struct lag_tracker *tracker) { u8 enabled_ports[MLX5_MAX_PORTS] = {}; u8 active_port = 0; int num_enabled; int idx; - mlx5_infer_tx_enabled(&ldev->tracker, ldev, enabled_ports, + mlx5_infer_tx_enabled(tracker, ldev, enabled_ports, &num_enabled); for (idx = 0; idx < num_enabled; idx++) active_port |= BIT_MASK(enabled_ports[idx]); @@ -78,7 +79,8 @@ static u8 lag_active_port_bits(struct mlx5_lag *ldev) } static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev, - int mode, unsigned long flags) + struct lag_tracker *tracker, int mode, + unsigned long flags) { bool fdb_sel_mode = test_bit(MLX5_LAG_MODE_FLAG_FDB_SEL_MODE_NATIVE, &flags); @@ -107,7 +109,7 @@ static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, struct mlx5_lag *ldev, break; MLX5_SET(lagc, lag_ctx, active_port, - lag_active_port_bits(mlx5_lag_dev(dev))); + lag_active_port_bits(ldev, tracker)); break; default: break; @@ -233,14 +235,25 @@ static void mlx5_ldev_free(struct kref *ref) { struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref); struct net *net; + int i; if (ldev->nb.notifier_call) { net = read_pnet(&ldev->net); unregister_netdevice_notifier_net(net, &ldev->nb); } + mlx5_ldev_for_each(i, 0, ldev) { + if (ldev->pf[i].dev && + ldev->pf[i].port_change_nb.nb.notifier_call) { + struct mlx5_nb *nb = &ldev->pf[i].port_change_nb; + + mlx5_eq_notifier_unregister(ldev->pf[i].dev, nb); + } + } + mlx5_lag_mp_cleanup(ldev); cancel_delayed_work_sync(&ldev->bond_work); + cancel_work_sync(&ldev->speed_update_work); destroy_workqueue(ldev->wq); mutex_destroy(&ldev->lock); kfree(ldev); @@ -274,6 +287,7 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev) kref_init(&ldev->ref); mutex_init(&ldev->lock); INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work); + INIT_WORK(&ldev->speed_update_work, mlx5_mpesw_speed_update_work); ldev->nb.notifier_call = mlx5_lag_netdev_event; write_pnet(&ldev->net, mlx5_core_net(dev)); @@ -482,7 +496,8 @@ static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports) return mlx5_cmd_exec_in(dev, modify_lag, in); } -static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) +static int _mlx5_modify_lag(struct mlx5_lag *ldev, + struct lag_tracker *tracker, u8 *ports) { int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; @@ -499,7 +514,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass)) return ret; - active_ports = lag_active_port_bits(ldev); + active_ports = lag_active_port_bits(ldev, tracker); return mlx5_cmd_modify_active_port(dev0, active_ports); } @@ -559,7 +574,7 @@ void mlx5_modify_lag(struct mlx5_lag *ldev, idx = i * ldev->buckets + j; if (ports[idx] == ldev->v2p_map[idx]) continue; - err = _mlx5_modify_lag(ldev, ports); + err = _mlx5_modify_lag(ldev, tracker, ports); if (err) { mlx5_core_err(dev0, "Failed to modify LAG (%d)\n", @@ -696,7 +711,7 @@ static int mlx5_create_lag(struct mlx5_lag *ldev, mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n", shared_fdb, mlx5_get_str_port_sel_mode(mode, flags)); - err = mlx5_cmd_create_lag(dev0, ldev, mode, flags); + err = mlx5_cmd_create_lag(dev0, ldev, tracker, mode, flags); if (err) { mlx5_core_err(dev0, "Failed to create LAG (%d)\n", @@ -996,6 +1011,144 @@ static bool mlx5_lag_should_disable_lag(struct mlx5_lag *ldev, bool do_bond) ldev->mode != MLX5_LAG_MODE_MPESW; } +#ifdef CONFIG_MLX5_ESWITCH +static int +mlx5_lag_sum_devices_speed(struct mlx5_lag *ldev, u32 *sum_speed, + int (*get_speed)(struct mlx5_core_dev *, u32 *)) +{ + struct mlx5_core_dev *pf_mdev; + int pf_idx; + u32 speed; + int ret; + + *sum_speed = 0; + mlx5_ldev_for_each(pf_idx, 0, ldev) { + pf_mdev = ldev->pf[pf_idx].dev; + if (!pf_mdev) + continue; + + ret = get_speed(pf_mdev, &speed); + if (ret) { + mlx5_core_dbg(pf_mdev, + "Failed to get device speed using %ps. Device %s speed is not available (err=%d)\n", + get_speed, dev_name(pf_mdev->device), + ret); + return ret; + } + + *sum_speed += speed; + } + + return 0; +} + +static int mlx5_lag_sum_devices_max_speed(struct mlx5_lag *ldev, u32 *max_speed) +{ + return mlx5_lag_sum_devices_speed(ldev, max_speed, + mlx5_port_max_linkspeed); +} + +static int mlx5_lag_sum_devices_oper_speed(struct mlx5_lag *ldev, + u32 *oper_speed) +{ + return mlx5_lag_sum_devices_speed(ldev, oper_speed, + mlx5_port_oper_linkspeed); +} + +static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, + u32 speed) +{ + u16 op_mod = MLX5_VPORT_STATE_OP_MOD_ESW_VPORT; + struct mlx5_eswitch *esw = mdev->priv.eswitch; + struct mlx5_vport *vport; + unsigned long i; + int ret; + + if (!esw) + return; + + if (!MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed)) + return; + + mutex_lock(&esw->state_lock); + mlx5_esw_for_each_vport(esw, i, vport) { + if (!vport) + continue; + + if (vport->vport == MLX5_VPORT_UPLINK) + continue; + + vport->agg_max_tx_speed = speed; + + if (!vport->enabled) + continue; + + ret = mlx5_modify_vport_max_tx_speed(mdev, op_mod, + vport->vport, true, speed); + if (ret) + mlx5_core_dbg(mdev, + "Failed to set vport %d speed %d, err=%d\n", + vport->vport, speed, ret); + } + mutex_unlock(&esw->state_lock); +} + +void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) +{ + struct mlx5_core_dev *mdev; + u32 speed; + int pf_idx; + + if (ldev->mode == MLX5_LAG_MODE_MPESW) { + if (mlx5_lag_sum_devices_oper_speed(ldev, &speed)) + return; + } else { + speed = ldev->tracker.bond_speed_mbps; + if (speed == SPEED_UNKNOWN) + return; + } + + /* If speed is not set, use the sum of max speeds of all PFs */ + if (!speed && mlx5_lag_sum_devices_max_speed(ldev, &speed)) + return; + + speed = speed / MLX5_MAX_TX_SPEED_UNIT; + + mlx5_ldev_for_each(pf_idx, 0, ldev) { + mdev = ldev->pf[pf_idx].dev; + if (!mdev) + continue; + + mlx5_lag_modify_device_vports_speed(mdev, speed); + } +} + +void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev) +{ + struct mlx5_core_dev *mdev; + u32 speed; + int pf_idx; + int ret; + + mlx5_ldev_for_each(pf_idx, 0, ldev) { + mdev = ldev->pf[pf_idx].dev; + if (!mdev) + continue; + + ret = mlx5_port_oper_linkspeed(mdev, &speed); + if (ret) { + mlx5_core_dbg(mdev, + "Failed to reset vports speed for device %s. Oper speed is not available (err=%d)\n", + dev_name(mdev->device), ret); + continue; + } + + speed = speed / MLX5_MAX_TX_SPEED_UNIT; + mlx5_lag_modify_device_vports_speed(mdev, speed); + } +} +#endif + static void mlx5_do_bond(struct mlx5_lag *ldev) { int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); @@ -1083,9 +1236,12 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) ndev); dev_put(ndev); } + mlx5_lag_set_vports_agg_speed(ldev); } else if (mlx5_lag_should_modify_lag(ldev, do_bond)) { mlx5_modify_lag(ldev, &tracker); + mlx5_lag_set_vports_agg_speed(ldev); } else if (mlx5_lag_should_disable_lag(ldev, do_bond)) { + mlx5_lag_reset_vports_speed(ldev); mlx5_disable_lag(ldev); } } @@ -1286,6 +1442,65 @@ static int mlx5_handle_changeinfodata_event(struct mlx5_lag *ldev, return 1; } +static void mlx5_lag_update_tracker_speed(struct lag_tracker *tracker, + struct net_device *ndev) +{ + struct ethtool_link_ksettings lksettings; + struct net_device *bond_dev; + int err; + + if (netif_is_lag_master(ndev)) + bond_dev = ndev; + else + bond_dev = netdev_master_upper_dev_get(ndev); + + if (!bond_dev) { + tracker->bond_speed_mbps = SPEED_UNKNOWN; + return; + } + + err = __ethtool_get_link_ksettings(bond_dev, &lksettings); + if (err) { + netdev_dbg(bond_dev, + "Failed to get speed for bond dev %s, err=%d\n", + bond_dev->name, err); + tracker->bond_speed_mbps = SPEED_UNKNOWN; + return; + } + + if (lksettings.base.speed == SPEED_UNKNOWN) + tracker->bond_speed_mbps = 0; + else + tracker->bond_speed_mbps = lksettings.base.speed; +} + +/* Returns speed in Mbps. */ +int mlx5_lag_query_bond_speed(struct mlx5_core_dev *mdev, u32 *speed) +{ + struct mlx5_lag *ldev; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&lag_lock, flags); + ldev = mlx5_lag_dev(mdev); + if (!ldev) { + ret = -ENODEV; + goto unlock; + } + + *speed = ldev->tracker.bond_speed_mbps; + + if (*speed == SPEED_UNKNOWN) { + mlx5_core_dbg(mdev, "Bond speed is unknown\n"); + ret = -EINVAL; + } + +unlock: + spin_unlock_irqrestore(&lag_lock, flags); + return ret; +} +EXPORT_SYMBOL_GPL(mlx5_lag_query_bond_speed); + /* this handler is always registered to netdev events */ static int mlx5_lag_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) @@ -1317,6 +1532,9 @@ static int mlx5_lag_netdev_event(struct notifier_block *this, break; } + if (changed) + mlx5_lag_update_tracker_speed(&tracker, ndev); + ldev->tracker = tracker; if (changed) @@ -1362,6 +1580,10 @@ static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev, ldev->pf[fn].dev = dev; dev->priv.lag = ldev; + + MLX5_NB_INIT(&ldev->pf[fn].port_change_nb, + mlx5_lag_mpesw_port_change_event, PORT_CHANGE); + mlx5_eq_notifier_register(dev, &ldev->pf[fn].port_change_nb); } static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, @@ -1373,6 +1595,9 @@ static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, if (ldev->pf[fn].dev != dev) return; + if (ldev->pf[fn].port_change_nb.nb.notifier_call) + mlx5_eq_notifier_unregister(dev, &ldev->pf[fn].port_change_nb); + ldev->pf[fn].dev = NULL; dev->priv.lag = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index 4918eee2b3da5..be1afece5fdc1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -39,6 +39,7 @@ struct lag_func { struct mlx5_core_dev *dev; struct net_device *netdev; bool has_drop; + struct mlx5_nb port_change_nb; }; /* Used for collection of netdev event info. */ @@ -48,6 +49,7 @@ struct lag_tracker { unsigned int is_bonded:1; unsigned int has_inactive:1; enum netdev_lag_hash hash_type; + u32 bond_speed_mbps; }; /* LAG data of a ConnectX card. @@ -66,6 +68,7 @@ struct mlx5_lag { struct lag_tracker tracker; struct workqueue_struct *wq; struct delayed_work bond_work; + struct work_struct speed_update_work; struct notifier_block nb; possible_net_t net; struct lag_mp lag_mp; @@ -116,6 +119,14 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev); void mlx5_lag_add_devices(struct mlx5_lag *ldev); struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev); +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev); +void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev); +#else +static inline void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) {} +static inline void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev) {} +#endif + static inline bool mlx5_lag_is_supported(struct mlx5_core_dev *dev) { if (!MLX5_CAP_GEN(dev, vport_group_manager) || diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index fc914ee259869..567e51f0e5dad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -110,6 +110,8 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) goto err_rescan_drivers; } + mlx5_lag_set_vports_agg_speed(ldev); + return 0; err_rescan_drivers: @@ -226,3 +228,40 @@ bool mlx5_lag_is_mpesw(struct mlx5_core_dev *dev) return ldev && ldev->mode == MLX5_LAG_MODE_MPESW; } EXPORT_SYMBOL(mlx5_lag_is_mpesw); + +void mlx5_mpesw_speed_update_work(struct work_struct *work) +{ + struct mlx5_lag *ldev = container_of(work, struct mlx5_lag, + speed_update_work); + + mutex_lock(&ldev->lock); + if (ldev->mode == MLX5_LAG_MODE_MPESW) { + if (ldev->mode_changes_in_progress) + queue_work(ldev->wq, &ldev->speed_update_work); + else + mlx5_lag_set_vports_agg_speed(ldev); + } + + mutex_unlock(&ldev->lock); +} + +int mlx5_lag_mpesw_port_change_event(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct mlx5_nb *mlx5_nb = container_of(nb, struct mlx5_nb, nb); + struct lag_func *lag_func = container_of(mlx5_nb, + struct lag_func, + port_change_nb); + struct mlx5_core_dev *dev = lag_func->dev; + struct mlx5_lag *ldev = dev->priv.lag; + struct mlx5_eqe *eqe = data; + + if (!ldev) + return NOTIFY_DONE; + + if (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_DOWN || + eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) + queue_work(ldev->wq, &ldev->speed_update_work); + + return NOTIFY_OK; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h index 46de93ed790de..b767dbb4f4571 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.h @@ -37,4 +37,18 @@ void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev); static inline void mlx5_lag_disable_mpesw(struct mlx5_lag *ldev) {} #endif /* CONFIG_MLX5_ESWITCH */ +#ifdef CONFIG_MLX5_ESWITCH +void mlx5_mpesw_speed_update_work(struct work_struct *work); +int mlx5_lag_mpesw_port_change_event(struct notifier_block *nb, + unsigned long event, void *data); +#else +static inline void mlx5_mpesw_speed_update_work(struct work_struct *work) {} +static inline int mlx5_lag_mpesw_port_change_event(struct notifier_block *nb, + unsigned long event, + void *data) +{ + return NOTIFY_DONE; +} +#endif /* CONFIG_MLX5_ESWITCH */ + #endif /* __MLX5_LAG_MPESW_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c index 4450091e181a1..4a88a42ae4f7a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.c @@ -65,13 +65,14 @@ static int del_l2table_entry_cmd(struct mlx5_core_dev *dev, u32 index) /* UC L2 table hash node */ struct l2table_node { struct l2addr_node node; - u32 index; /* index in HW l2 table */ + int index; /* index in HW l2 table */ int ref_count; }; struct mlx5_mpfs { struct hlist_head hash[MLX5_L2_ADDR_HASH_SIZE]; struct mutex lock; /* Synchronize l2 table access */ + bool enabled; u32 size; unsigned long *bitmap; }; @@ -114,6 +115,8 @@ int mlx5_mpfs_init(struct mlx5_core_dev *dev) return -ENOMEM; } + mpfs->enabled = true; + dev->priv.mpfs = mpfs; return 0; } @@ -135,7 +138,7 @@ int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac) struct mlx5_mpfs *mpfs = dev->priv.mpfs; struct l2table_node *l2addr; int err = 0; - u32 index; + int index; if (!mpfs) return 0; @@ -148,30 +151,34 @@ int mlx5_mpfs_add_mac(struct mlx5_core_dev *dev, u8 *mac) goto out; } - err = alloc_l2table_index(mpfs, &index); - if (err) - goto out; - l2addr = l2addr_hash_add(mpfs->hash, mac, struct l2table_node, GFP_KERNEL); if (!l2addr) { err = -ENOMEM; - goto hash_add_err; + goto out; } - err = set_l2table_entry_cmd(dev, index, mac); - if (err) - goto set_table_entry_err; + index = -1; + + if (mpfs->enabled) { + err = alloc_l2table_index(mpfs, &index); + if (err) + goto hash_del; + err = set_l2table_entry_cmd(dev, index, mac); + if (err) + goto free_l2table_index; + mlx5_core_dbg(dev, "MPFS entry %pM, set @index (%d)\n", + l2addr->node.addr, index); + } l2addr->index = index; l2addr->ref_count = 1; mlx5_core_dbg(dev, "MPFS mac added %pM, index (%d)\n", mac, index); goto out; - -set_table_entry_err: - l2addr_hash_del(l2addr); -hash_add_err: +free_l2table_index: free_l2table_index(mpfs, index); +hash_del: + l2addr_hash_del(l2addr); out: mutex_unlock(&mpfs->lock); return err; @@ -183,7 +190,7 @@ int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) struct mlx5_mpfs *mpfs = dev->priv.mpfs; struct l2table_node *l2addr; int err = 0; - u32 index; + int index; if (!mpfs) return 0; @@ -200,12 +207,87 @@ int mlx5_mpfs_del_mac(struct mlx5_core_dev *dev, u8 *mac) goto unlock; index = l2addr->index; - del_l2table_entry_cmd(dev, index); + if (index >= 0) { + del_l2table_entry_cmd(dev, index); + free_l2table_index(mpfs, index); + mlx5_core_dbg(dev, "MPFS entry %pM, deleted @index (%d)\n", + mac, index); + } l2addr_hash_del(l2addr); - free_l2table_index(mpfs, index); mlx5_core_dbg(dev, "MPFS mac deleted %pM, index (%d)\n", mac, index); unlock: mutex_unlock(&mpfs->lock); return err; } EXPORT_SYMBOL(mlx5_mpfs_del_mac); + +int mlx5_mpfs_enable(struct mlx5_core_dev *dev) +{ + struct mlx5_mpfs *mpfs = dev->priv.mpfs; + struct l2table_node *l2addr; + struct hlist_node *n; + int err = 0, i; + + if (!mpfs) + return -ENODEV; + + mutex_lock(&mpfs->lock); + if (mpfs->enabled) + goto out; + mpfs->enabled = true; + mlx5_core_dbg(dev, "MPFS enabling mpfs\n"); + + mlx5_mpfs_foreach(l2addr, n, mpfs, i) { + u32 index; + + err = alloc_l2table_index(mpfs, &index); + if (err) { + mlx5_core_err(dev, "Failed to allocated MPFS index for %pM, err(%d)\n", + l2addr->node.addr, err); + goto out; + } + + err = set_l2table_entry_cmd(dev, index, l2addr->node.addr); + if (err) { + mlx5_core_err(dev, "Failed to set MPFS l2table entry for %pM index=%d, err(%d)\n", + l2addr->node.addr, index, err); + free_l2table_index(mpfs, index); + goto out; + } + + l2addr->index = index; + mlx5_core_dbg(dev, "MPFS entry %pM, set @index (%d)\n", + l2addr->node.addr, l2addr->index); + } +out: + mutex_unlock(&mpfs->lock); + return err; +} + +void mlx5_mpfs_disable(struct mlx5_core_dev *dev) +{ + struct mlx5_mpfs *mpfs = dev->priv.mpfs; + struct l2table_node *l2addr; + struct hlist_node *n; + int i; + + if (!mpfs) + return; + + mutex_lock(&mpfs->lock); + if (!mpfs->enabled) + goto unlock; + mlx5_mpfs_foreach(l2addr, n, mpfs, i) { + if (l2addr->index < 0) + continue; + del_l2table_entry_cmd(dev, l2addr->index); + free_l2table_index(mpfs, l2addr->index); + mlx5_core_dbg(dev, "MPFS entry %pM, deleted @index (%d)\n", + l2addr->node.addr, l2addr->index); + l2addr->index = -1; + } + mpfs->enabled = false; + mlx5_core_dbg(dev, "MPFS disabled\n"); +unlock: + mutex_unlock(&mpfs->lock); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h index 4a293542a7aa1..9c63838ce1f32 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mpfs.h @@ -45,6 +45,10 @@ struct l2addr_node { u8 addr[ETH_ALEN]; }; +#define mlx5_mpfs_foreach(hs, tmp, mpfs, i) \ + for (i = 0; i < MLX5_L2_ADDR_HASH_SIZE; i++) \ + hlist_for_each_entry_safe(hs, tmp, &(mpfs)->hash[i], node.hlist) + #define for_each_l2hash_node(hn, tmp, hash, i) \ for (i = 0; i < MLX5_L2_ADDR_HASH_SIZE; i++) \ hlist_for_each_entry_safe(hn, tmp, &(hash)[i], hlist) @@ -82,11 +86,16 @@ struct l2addr_node { }) #ifdef CONFIG_MLX5_MPFS +struct mlx5_core_dev; int mlx5_mpfs_init(struct mlx5_core_dev *dev); void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev); +int mlx5_mpfs_enable(struct mlx5_core_dev *dev); +void mlx5_mpfs_disable(struct mlx5_core_dev *dev); #else /* #ifndef CONFIG_MLX5_MPFS */ static inline int mlx5_mpfs_init(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_mpfs_cleanup(struct mlx5_core_dev *dev) {} +static inline int mlx5_mpfs_enable(struct mlx5_core_dev *dev) { return 0; } +static inline void mlx5_mpfs_disable(struct mlx5_core_dev *dev) {} #endif #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 22bdefe5696c9..42bc553d034b5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1427,7 +1427,6 @@ static int mlx5_load(struct mlx5_core_dev *dev) static void mlx5_unload(struct mlx5_core_dev *dev) { - mlx5_eswitch_disable(dev->priv.eswitch); mlx5_devlink_traps_unregister(priv_to_devlink(dev)); mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); @@ -1536,6 +1535,7 @@ void mlx5_uninit_one(struct mlx5_core_dev *dev) mlx5_hwmon_dev_unregister(dev); mlx5_crdump_disable(dev); + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_unregister_device(dev); if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { @@ -1620,6 +1620,7 @@ void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend) devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&dev->intf_state_mutex); + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_detach_device(dev, suspend); if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 09c544bdf70da..f9195f0345df2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -358,11 +358,11 @@ int mlx5_set_port_fcs(struct mlx5_core_dev *mdev, u8 enable); void mlx5_query_port_fcs(struct mlx5_core_dev *mdev, bool *supported, bool *enabled); int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, - u16 offset, u16 size, u8 *data); + u16 offset, u16 size, u8 *data, u8 *status); int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev, struct mlx5_module_eeprom_query_params *params, - u8 *data); + u8 *data, u8 *status); int mlx5_query_port_dcbx_param(struct mlx5_core_dev *mdev, u32 *out); int mlx5_set_port_dcbx_param(struct mlx5_core_dev *mdev, u32 *in); @@ -382,6 +382,7 @@ const struct mlx5_link_info *mlx5_port_ptys2info(struct mlx5_core_dev *mdev, u32 mlx5_port_info2linkmodes(struct mlx5_core_dev *mdev, struct mlx5_link_info *info, bool force_legacy); +int mlx5_port_oper_linkspeed(struct mlx5_core_dev *mdev, u32 *speed); int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed); #define MLX5_PPS_CAP(mdev) (MLX5_CAP_GEN((mdev), pps) && \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c index 876e648c91ba8..296a011df02fb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c @@ -289,11 +289,11 @@ int mlx5_query_module_num(struct mlx5_core_dev *dev, int *module_num) } static int mlx5_query_module_id(struct mlx5_core_dev *dev, int module_num, - u8 *module_id) + u8 *module_id, u8 *status) { u32 in[MLX5_ST_SZ_DW(mcia_reg)] = {}; u32 out[MLX5_ST_SZ_DW(mcia_reg)]; - int err, status; + int err; u8 *ptr; MLX5_SET(mcia_reg, in, i2c_device_address, MLX5_I2C_ADDR_LOW); @@ -308,13 +308,13 @@ static int mlx5_query_module_id(struct mlx5_core_dev *dev, int module_num, if (err) return err; - status = MLX5_GET(mcia_reg, out, status); - if (status) { - mlx5_core_err(dev, "query_mcia_reg failed: status: 0x%x\n", - status); + if (MLX5_GET(mcia_reg, out, status)) { + if (status) + *status = MLX5_GET(mcia_reg, out, status); return -EIO; } - ptr = MLX5_ADDR_OF(mcia_reg, out, dword_0); + + ptr = MLX5_ADDR_OF(mcia_reg, out, dwords); *module_id = ptr[0]; @@ -370,13 +370,14 @@ static int mlx5_mcia_max_bytes(struct mlx5_core_dev *dev) } static int mlx5_query_mcia(struct mlx5_core_dev *dev, - struct mlx5_module_eeprom_query_params *params, u8 *data) + struct mlx5_module_eeprom_query_params *params, + u8 *data, u8 *status) { u32 in[MLX5_ST_SZ_DW(mcia_reg)] = {}; u32 out[MLX5_ST_SZ_DW(mcia_reg)]; - int status, err; void *ptr; u16 size; + int err; size = min_t(int, params->size, mlx5_mcia_max_bytes(dev)); @@ -392,21 +393,20 @@ static int mlx5_query_mcia(struct mlx5_core_dev *dev, if (err) return err; - status = MLX5_GET(mcia_reg, out, status); - if (status) { - mlx5_core_err(dev, "query_mcia_reg failed: status: 0x%x\n", - status); + if (MLX5_GET(mcia_reg, out, status)) { + if (status) + *status = MLX5_GET(mcia_reg, out, status); return -EIO; } - ptr = MLX5_ADDR_OF(mcia_reg, out, dword_0); + ptr = MLX5_ADDR_OF(mcia_reg, out, dwords); memcpy(data, ptr, size); return size; } int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, - u16 offset, u16 size, u8 *data) + u16 offset, u16 size, u8 *data, u8 *status) { struct mlx5_module_eeprom_query_params query = {0}; u8 module_id; @@ -416,7 +416,8 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, if (err) return err; - err = mlx5_query_module_id(dev, query.module_number, &module_id); + err = mlx5_query_module_id(dev, query.module_number, &module_id, + status); if (err) return err; @@ -442,12 +443,12 @@ int mlx5_query_module_eeprom(struct mlx5_core_dev *dev, query.size = size; query.offset = offset; - return mlx5_query_mcia(dev, &query, data); + return mlx5_query_mcia(dev, &query, data, status); } int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev, struct mlx5_module_eeprom_query_params *params, - u8 *data) + u8 *data, u8 *status) { int err; @@ -461,7 +462,7 @@ int mlx5_query_module_eeprom_by_page(struct mlx5_core_dev *dev, return -EINVAL; } - return mlx5_query_mcia(dev, params, data); + return mlx5_query_mcia(dev, params, data, status); } static int mlx5_query_port_pvlc(struct mlx5_core_dev *dev, u32 *pvlc, @@ -1201,6 +1202,30 @@ u32 mlx5_port_info2linkmodes(struct mlx5_core_dev *mdev, return link_modes; } +int mlx5_port_oper_linkspeed(struct mlx5_core_dev *mdev, u32 *speed) +{ + const struct mlx5_link_info *table; + struct mlx5_port_eth_proto eproto; + u32 oper_speed = 0; + u32 max_size; + bool ext; + int err; + int i; + + ext = mlx5_ptys_ext_supported(mdev); + err = mlx5_port_query_eth_proto(mdev, 1, ext, &eproto); + if (err) + return err; + + mlx5e_port_get_link_mode_info_arr(mdev, &table, &max_size, false); + for (i = 0; i < max_size; ++i) + if (eproto.oper & MLX5E_PROT_MASK(i)) + oper_speed = max(oper_speed, table[i].speed); + + *speed = oper_speed; + return 0; +} + int mlx5_port_max_linkspeed(struct mlx5_core_dev *mdev, u32 *speed) { const struct mlx5_link_info *table; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c index 564ae98316bc9..756b87a3d68dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c @@ -423,6 +423,18 @@ int mlx5hws_bwc_queue_poll(struct mlx5hws_context *ctx, if (!got_comp && !drain) return 0; + if (unlikely(ctx->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) { + /* If the device is down for any reason (e.g. FLR), the HW will + * no longer generate completions. + * Note that ETIMEDOUT is returned here because the BWC layer + * already has a special handling for timeouts - it breaks the + * rehash / resize / shrink loops to avoid chain of timeouts. + */ + mlx5_core_warn_once(ctx->mdev, + "BWC poll: device is down, polling for completion aborted\n"); + return -ETIMEDOUT; + } + queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]); while (queue_full || ((got_comp || drain) && *pending_rules)) { ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c index 6113383ae47bb..07a3dd1248c77 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c @@ -282,6 +282,9 @@ int mlx5hws_table_destroy(struct mlx5hws_table *tbl) goto unlock_err; } + if (tbl->default_miss.miss_tbl) + list_del_init(&tbl->default_miss.next); + list_del_init(&tbl->tbl_list_node); mutex_unlock(&ctx->ctrl_lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index a44214c660b03..5d895287df54b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -62,11 +62,61 @@ u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport) return MLX5_GET(query_vport_state_out, out, state); } +static int mlx5_query_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, + u16 vport, u8 other_vport, + u8 *admin_state) +{ + u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; + int err; + + MLX5_SET(query_vport_state_in, in, opcode, + MLX5_CMD_OP_QUERY_VPORT_STATE); + MLX5_SET(query_vport_state_in, in, op_mod, opmod); + MLX5_SET(query_vport_state_in, in, vport_number, vport); + MLX5_SET(query_vport_state_in, in, other_vport, other_vport); + + err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); + if (err) + return err; + + *admin_state = MLX5_GET(query_vport_state_out, out, admin_state); + return 0; +} + int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u8 state) { u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + + if (MLX5_CAP_ESW(mdev, esw_vport_state_max_tx_speed) && + opmod == MLX5_VPORT_STATE_OP_MOD_ESW_VPORT && + vport != MLX5_VPORT_UPLINK) { + u32 speed = 0; + int err; + + err = mlx5_query_vport_max_tx_speed(mdev, opmod, vport, + other_vport, &speed, NULL); + if (err) { +#ifdef CONFIG_MLX5_ESWITCH + struct mlx5_vport *esw_vport; + + esw_vport = mlx5_eswitch_get_vport(mdev->priv.eswitch, + vport); + speed = IS_ERR(esw_vport) ? 0 : + esw_vport->agg_max_tx_speed; +#endif + mlx5_core_dbg(mdev, + "Failed to query vport %d max tx speed, err=%d, using cached %u\n", + vport, err, speed); + } + MLX5_SET(modify_vport_state_in, in, max_tx_speed, speed); + } + MLX5_SET(modify_vport_state_in, in, opcode, MLX5_CMD_OP_MODIFY_VPORT_STATE); MLX5_SET(modify_vport_state_in, in, op_mod, opmod); @@ -77,6 +127,58 @@ int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, return mlx5_cmd_exec_in(mdev, modify_vport_state, in); } +int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, + u16 vport, u8 other_vport, u16 max_tx_speed) +{ + u32 in[MLX5_ST_SZ_DW(modify_vport_state_in)] = {}; + u8 admin_state; + int err; + +#ifdef CONFIG_MLX5_ESWITCH + lockdep_assert_held(&mdev->priv.eswitch->state_lock); +#endif + + err = mlx5_query_vport_admin_state(mdev, opmod, vport, other_vport, + &admin_state); + if (err) + return err; + + MLX5_SET(modify_vport_state_in, in, opcode, + MLX5_CMD_OP_MODIFY_VPORT_STATE); + MLX5_SET(modify_vport_state_in, in, op_mod, opmod); + MLX5_SET(modify_vport_state_in, in, vport_number, vport); + MLX5_SET(modify_vport_state_in, in, other_vport, other_vport); + MLX5_SET(modify_vport_state_in, in, admin_state, admin_state); + MLX5_SET(modify_vport_state_in, in, max_tx_speed, max_tx_speed); + + return mlx5_cmd_exec_in(mdev, modify_vport_state, in); +} + +int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state) +{ + u32 out[MLX5_ST_SZ_DW(query_vport_state_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_vport_state_in)] = {}; + int err; + + MLX5_SET(query_vport_state_in, in, opcode, + MLX5_CMD_OP_QUERY_VPORT_STATE); + MLX5_SET(query_vport_state_in, in, op_mod, op_mod); + MLX5_SET(query_vport_state_in, in, vport_number, vport); + MLX5_SET(query_vport_state_in, in, other_vport, other_vport); + + err = mlx5_cmd_exec_inout(mdev, query_vport_state, in, out); + if (err) + return err; + + *max_tx_speed = MLX5_GET(query_vport_state_out, out, max_tx_speed); + if (state) + *state = MLX5_GET(query_vport_state_out, out, state); + return 0; +} +EXPORT_SYMBOL_GPL(mlx5_query_vport_max_tx_speed); + static int mlx5_query_nic_vport_context(struct mlx5_core_dev *mdev, u16 vport, u32 *out) { @@ -1279,7 +1381,7 @@ int mlx5_vport_get_vhca_id(struct mlx5_core_dev *dev, u16 vport, u16 *vhca_id) if (mlx5_esw_vport_vhca_id(dev->priv.eswitch, vport, vhca_id)) return 0; - query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + query_ctx = kvzalloc(query_out_sz, GFP_KERNEL); if (!query_ctx) return -ENOMEM; @@ -1291,7 +1393,7 @@ int mlx5_vport_get_vhca_id(struct mlx5_core_dev *dev, u16 vport, u16 *vhca_id) *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); out_free: - kfree(query_ctx); + kvfree(query_ctx); return err; } EXPORT_SYMBOL_GPL(mlx5_vport_get_vhca_id); @@ -1306,7 +1408,7 @@ int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap void *set_ctx; int ret; - set_ctx = kzalloc(set_sz, GFP_KERNEL); + set_ctx = kvzalloc(set_sz, GFP_KERNEL); if (!set_ctx) return -ENOMEM; @@ -1335,6 +1437,6 @@ int mlx5_vport_set_other_func_cap(struct mlx5_core_dev *dev, const void *hca_cap MLX5_SET(set_hca_cap_in, set_ctx, function_id, function_id); ret = mlx5_cmd_exec_in(dev, set_hca_cap, set_ctx); - kfree(set_ctx); + kvfree(set_ctx); return ret; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index 5b9f0844b8f60..9b705a11363a0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -572,6 +572,38 @@ mlxsw_sp1_ptp_unmatched_remove(struct mlxsw_sp *mlxsw_sp, mlxsw_sp1_ptp_unmatched_ht_params); } +/* mlxsw_sp1_ptp_packet_finish() is reached both from the NAPI poll context + * (mlxsw_sp1_ptp_got_packet(), mlxsw_sp1_ptp_got_piece() and + * mlxsw_sp1_packet_timestamp()) and from process context, by way of the GC + * workqueue (mlxsw_sp1_ptp_ht_gc_collect() -> + * mlxsw_sp1_ptp_unmatched_finish()). + * + * mlxsw_sp_rx_listener_no_mark_func() ends in napi_gro_receive(), using the + * NAPI pointer that was placed in the SKB control block when the trapped + * packet was received in the NAPI context. That pointer may only be used + * from its own poll context, which this call site cannot guarantee. + * + * netif_receive_skb(), unlike napi_gro_receive(), can be called from outside + * of the NAPI instance's poll context. RX stats accounting and the skb->dev + * assignment are still preserved; the only change is the delivery call. + */ +static void mlxsw_sp1_ptp_rx_finish(struct mlxsw_sp_port *mlxsw_sp_port, + struct sk_buff *skb) +{ + struct mlxsw_sp_port_pcpu_stats *pcpu_stats; + + skb->dev = mlxsw_sp_port->dev; + + pcpu_stats = this_cpu_ptr(mlxsw_sp_port->pcpu_stats); + u64_stats_update_begin(&pcpu_stats->syncp); + pcpu_stats->rx_packets++; + pcpu_stats->rx_bytes += skb->len; + u64_stats_update_end(&pcpu_stats->syncp); + + skb->protocol = eth_type_trans(skb, skb->dev); + netif_receive_skb(skb); +} + /* This function is called in the following scenarios: * * 1) When a packet is matched with its timestamp. @@ -600,7 +632,7 @@ static void mlxsw_sp1_ptp_packet_finish(struct mlxsw_sp *mlxsw_sp, if (ingress) { if (hwtstamps) *skb_hwtstamps(skb) = *hwtstamps; - mlxsw_sp_rx_listener_no_mark_func(skb, local_port, mlxsw_sp); + mlxsw_sp1_ptp_rx_finish(mlxsw_sp_port, skb); } else { /* skb_tstamp_tx() allows hwtstamps to be NULL. */ skb_tstamp_tx(skb, hwtstamps); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index 14885c3d70697..dbe0855ecb575 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -2811,7 +2811,9 @@ void fbnic_napi_depletion_check(struct net_device *netdev) fbnic_wrfl(fbd); } -static int fbnic_queue_mem_alloc(struct net_device *dev, void *qmem, int idx) +static int fbnic_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct fbnic_net *fbn = netdev_priv(dev); const struct fbnic_q_triad *real; @@ -2863,7 +2865,9 @@ static void __fbnic_nv_restart(struct fbnic_net *fbn, netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); } -static int fbnic_queue_start(struct net_device *dev, void *qmem, int idx) +static int fbnic_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) { struct fbnic_net *fbn = netdev_priv(dev); struct fbnic_napi_vector *nv; diff --git a/drivers/net/ethernet/micrel/ks8842.c b/drivers/net/ethernet/micrel/ks8842.c index 541c41a9077a4..b7cf2ee9115fd 100644 --- a/drivers/net/ethernet/micrel/ks8842.c +++ b/drivers/net/ethernet/micrel/ks8842.c @@ -242,7 +242,7 @@ static void ks8842_reset(struct ks8842_adapter *adapter) msleep(10); iowrite16(0, adapter->hw_addr + REG_GRR); } else { - /* The KS8842 goes haywire when doing softare reset + /* The KS8842 goes haywire when doing software reset * a work around in the timberdale IP is implemented to * do a hardware reset instead ks8842_write16(adapter, 3, 1, REG_GRR); @@ -312,7 +312,7 @@ static void ks8842_reset_hw(struct ks8842_adapter *adapter) /* aggressive back off in half duplex */ ks8842_enable_bits(adapter, 32, 1 << 8, REG_SGCR1); - /* enable no excessive collison drop */ + /* enable no excessive collision drop */ ks8842_enable_bits(adapter, 32, 1 << 3, REG_SGCR2); /* Enable port 1 force flow control / back pressure / transmit / recv */ diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c index 0369e80b01bec..d49f281c78676 100644 --- a/drivers/net/ethernet/micrel/ks8851_common.c +++ b/drivers/net/ethernet/micrel/ks8851_common.c @@ -143,9 +143,6 @@ static int ks8851_write_mac_addr(struct net_device *dev) ks8851_wrreg16(ks, KS_MAR(i), val); } - if (!netif_running(dev)) - ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); - ks8851_unlock(ks); return 0; @@ -477,9 +474,8 @@ static int ks8851_net_open(struct net_device *dev) * ks8851_net_stop - close network device * @dev: The device being closed. * - * Called to close down a network device which has been active. Cancell any - * work, shutdown the RX and TX process and then place the chip into a low - * power state whilst it is not being used. + * Called to close down a network device which has been active. Cancel any + * work and shutdown the RX and TX process. */ static int ks8851_net_stop(struct net_device *dev) { @@ -506,8 +502,6 @@ static int ks8851_net_stop(struct net_device *dev) /* shutdown TX process */ ks8851_wrreg16(ks, KS_TXCR, 0x0000); - /* set powermode to soft power down to save power */ - ks8851_set_powermode(ks, PMECR_PM_SOFTDOWN); ks8851_unlock(ks); /* ensure any queued tx buffers are dumped */ diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c index 28a7ff7644bb5..b9e68520278d0 100644 --- a/drivers/net/ethernet/micrel/ks8851_spi.c +++ b/drivers/net/ethernet/micrel/ks8851_spi.c @@ -39,7 +39,7 @@ static int msg_enable; * * The @lock ensures that the chip is protected when certain operations are * in progress. When the read or write packet transfer is in progress, most - * of the chip registers are not ccessible until the transfer is finished and + * of the chip registers are not accessible until the transfer is finished and * the DMA has been de-asserted. */ struct ks8851_net_spi { @@ -296,7 +296,7 @@ static unsigned int calc_txlen(unsigned int len) /** * ks8851_tx_work - process tx packet(s) - * @work: The work strucutre what was scheduled. + * @work: The work structure what was scheduled. * * This is called when a number of packets have been scheduled for * transmission and need to be sent to the device. diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index cdde19b8edc46..60223f03482da 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -1166,7 +1166,7 @@ struct ksz_port_info { * @tx_cfg: Cached transmit control settings. * @rx_cfg: Cached receive control settings. * @intr_mask: Current interrupt mask. - * @intr_set: Current interrup set. + * @intr_set: Current interrupt set. * @intr_blocked: Interrupt blocked. * @rx_desc_info: Receive descriptor information. * @tx_desc_info: Transmit descriptor information. @@ -2096,7 +2096,7 @@ static void sw_dis_prio_rate(struct ksz_hw *hw, int port) } /** - * sw_init_prio_rate - initialize switch prioirty rate + * sw_init_prio_rate - initialize switch priority rate * @hw: The hardware instance. * * This routine initializes the priority rate function of the switch. diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c index 646f3d65274e3..b13c60438b978 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c @@ -816,6 +816,7 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) struct page *(*old_pages)[FDMA_RX_DCB_MAX_DBS]; struct page_pool *page_pool; struct fdma fdma_rx_old; + int page_order, max_mtu; int err, i, j; old_pages = kmemdup(lan966x->rx.page, sizeof(lan966x->rx.page), @@ -826,6 +827,8 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) /* Store these for later to free them */ memcpy(&fdma_rx_old, &lan966x->rx.fdma, sizeof(struct fdma)); page_pool = lan966x->rx.page_pool; + page_order = lan966x->rx.page_order; + max_mtu = lan966x->rx.max_mtu; napi_synchronize(&lan966x->napi); napi_disable(&lan966x->napi); @@ -855,7 +858,24 @@ static int lan966x_fdma_reload(struct lan966x *lan966x, int new_mtu) return 0; restore: lan966x->rx.page_pool = page_pool; + lan966x->rx.page_order = page_order; + lan966x->rx.max_mtu = max_mtu; memcpy(&lan966x->rx.fdma, &fdma_rx_old, sizeof(struct fdma)); + /* + * lan966x_fdma_rx_alloc_page_pool() registered the new pool with + * each port's XDP RXQ before the allocation failed. The new pool is + * destroyed by lan966x_fdma_rx_alloc(), so restore the old pool's + * registration before restarting RX. + */ + for (i = 0; i < lan966x->num_phys_ports; i++) { + if (!lan966x->ports[i]) + continue; + + xdp_rxq_info_unreg_mem_model(&lan966x->ports[i]->xdp_rxq); + xdp_rxq_info_reg_mem_model(&lan966x->ports[i]->xdp_rxq, + MEM_TYPE_PAGE_POOL, page_pool); + } + lan966x_fdma_rx_start(&lan966x->rx); lan966x_fdma_wakeup_netdev(lan966x); diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_psfp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_psfp.c index cd4f42c3f7ebf..83b37f95ee467 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_psfp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_psfp.c @@ -277,6 +277,9 @@ int sparx5_psfp_fm_add(struct sparx5 *sparx5, u32 uidx, ret = sparx5_psfp_fm_get(sparx5, uidx, &fm->pol.idx); if (ret < 0) return ret; + + *id = fm->pol.idx; + /* Was already in use, no need to reconfigure */ if (ret > 1) return 0; @@ -291,8 +294,6 @@ int sparx5_psfp_fm_add(struct sparx5 *sparx5, u32 uidx, if (ret < 0) return ret; - *id = fm->pol.idx; - return 0; } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c index 4dc1ebd5d510d..e5022d783ee68 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc_flower.c @@ -807,7 +807,7 @@ static int sparx5_tc_flower_psfp_setup(struct sparx5 *sparx5, /* Add new flow-meter */ ret = sparx5_psfp_fm_add(sparx5, pol_idx, fm, &psfp_fmid); if (ret < 0) - return ret; + goto err_sg_del; } /* Map stream filter to stream gate */ @@ -816,7 +816,7 @@ static int sparx5_tc_flower_psfp_setup(struct sparx5 *sparx5, /* Add new stream-filter and map it to a steam gate */ ret = sparx5_psfp_sf_add(sparx5, sf, &psfp_sfid); if (ret < 0) - return ret; + goto err_fm_del; /* Streams are classified by ISDX - map ISDX 1:1 to sfid for now. */ sparx5_isdx_conf_set(sparx5, psfp_sfid, psfp_sfid, psfp_fmid); @@ -824,13 +824,23 @@ static int sparx5_tc_flower_psfp_setup(struct sparx5 *sparx5, ret = vcap_rule_add_action_bit(vrule, VCAP_AF_ISDX_ADD_REPLACE_SEL, VCAP_BIT_1); if (ret) - return ret; + goto err_sf_del; ret = vcap_rule_add_action_u32(vrule, VCAP_AF_ISDX_VAL, psfp_sfid); if (ret) - return ret; + goto err_sf_del; return 0; + +err_sf_del: + sparx5_isdx_conf_set(sparx5, psfp_sfid, 0, 0); + sparx5_psfp_sf_del(sparx5, psfp_sfid); +err_fm_del: + if (pol_idx >= 0) + sparx5_psfp_fm_del(sparx5, psfp_fmid); +err_sg_del: + sparx5_psfp_sg_del(sparx5, psfp_sgid); + return ret; } /* Handle the action trap for a VCAP rule */ diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 243f8d11be684..81cbc47de0c62 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -125,6 +125,8 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev) } else { /* If dynamic allocation is enabled we have already allocated * hwc msi + * Also, we make sure in this case the following is always true + * (num_msix_usable - 1 HWC) <= num_online_cpus() */ gc->num_msix_usable = min(resp.max_msix, num_online_cpus() + 1); } @@ -1587,8 +1589,8 @@ void mana_gd_free_res_map(struct gdma_resource *r) * do the same thing. */ -static int irq_setup(unsigned int *irqs, unsigned int len, int node, - bool skip_first_cpu) +static int mana_irq_setup_numa_aware(unsigned int *irqs, unsigned int len, + int node, bool skip_first_cpu) { const struct cpumask *next, *prev = cpu_none_mask; cpumask_var_t cpus __free(free_cpumask_var); @@ -1624,11 +1626,24 @@ static int irq_setup(unsigned int *irqs, unsigned int len, int node, return 0; } +/* must be called with cpus_read_lock() held */ +static void mana_irq_setup_linear(unsigned int *irqs, unsigned int len) +{ + int cpu; + + for_each_online_cpu(cpu) { + if (len == 0) + break; + + irq_set_affinity_and_hint(*irqs++, cpumask_of(cpu)); + len--; + } +} + static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) { struct gdma_context *gc = pci_get_drvdata(pdev); struct gdma_irq_context *gic; - bool skip_first_cpu = false; int *irqs, irq, err, i; irqs = kmalloc_array(nvec, sizeof(int), GFP_KERNEL); @@ -1636,10 +1651,12 @@ static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) return -ENOMEM; /* + * In this function, num_msix_usable = HWC IRQ + Queue IRQ. + * nvec is only Queue IRQ (HWC already setup). * While processing the next pci irq vector, we start with index 1, * as IRQ vector at index 0 is already processed for HWC. * However, the population of irqs array starts with index 0, to be - * further used in irq_setup() + * further used in mana_irq_setup_numa_aware() */ for (i = 1; i <= nvec; i++) { gic = kzalloc(sizeof(*gic), GFP_KERNEL); @@ -1669,18 +1686,51 @@ static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) } /* - * When calling irq_setup() for dynamically added IRQs, if number of - * CPUs is more than or equal to allocated MSI-X, we need to skip the - * first CPU sibling group since they are already affinitized to HWC IRQ + * When calling mana_irq_setup_numa_aware() for dynamically added IRQs, + * if number of CPUs is more than or equal to allocated MSI-X, we need to + * skip the first CPU sibling group since they are already affinitized to + * HWC IRQ */ cpus_read_lock(); - if (gc->num_msix_usable <= num_online_cpus()) - skip_first_cpu = true; + if (gc->num_msix_usable <= num_online_cpus()) { + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, + true); + if (err) { + cpus_read_unlock(); + goto free_irq; + } + } else { + /* + * When num_msix_usable are more than num_online_cpus, our + * queue IRQs should be equal to num of online vCPUs. + * We try to make sure queue IRQs spread across all vCPUs. + * In such a case NUMA or CPU core affinity does not matter. + * Note: in this case the total mana IRQ should always be + * num_online_cpus + 1. The first HWC IRQ is already handled + * in HWC setup calls + * However, if CPUs went offline since num_msix_usable was + * computed, queue IRQs will be more than num_online_cpus(). + * In such cases remaining extra IRQs will retain their default + * affinity. + */ + int first_unassigned = num_online_cpus(); - err = irq_setup(irqs, nvec, gc->numa_node, skip_first_cpu); - if (err) { - cpus_read_unlock(); - goto free_irq; + if (nvec > first_unassigned) { + char buf[32]; + + if (first_unassigned == nvec - 1) + snprintf(buf, sizeof(buf), "%d", + first_unassigned); + else + snprintf(buf, sizeof(buf), "%d-%d", + first_unassigned, nvec - 1); + + dev_dbg(&pdev->dev, + "MANA IRQ indices #%s will retain the default CPU affinity\n", + buf); + } + + mana_irq_setup_linear(irqs, nvec); } cpus_read_unlock(); @@ -1766,7 +1816,7 @@ static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) nvec -= 1; } - err = irq_setup(irqs, nvec, gc->numa_node, false); + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, false); if (err) { cpus_read_unlock(); goto free_irq; diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c index 7697c9b52ed34..7bb8382940b38 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -208,6 +208,7 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, if (err) { NL_SET_ERR_MSG_MOD(extack, "XDP: Insufficient memory for tx/rx re-config"); + apc->bpf_prog = old_prog; return err; } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 62418f164dbac..c5e13eb662f41 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -1262,6 +1262,12 @@ static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, *max_sq = resp.max_num_sq; *max_rq = resp.max_num_rq; + + if (*max_sq == 0 || *max_rq == 0) { + netdev_err(apc->ndev, "Invalid max queues from vPort config\n"); + return -EPROTO; + } + if (resp.num_indirection_ent > 0 && resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && is_power_of_2(resp.num_indirection_ent)) { @@ -2160,6 +2166,19 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, rxbuf_oob = &rxq->rx_oobs[curr]; WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); + if (unlikely(pktlen > rxq->datasize)) { + /* Increase it even if mana_rx_skb() isn't called. */ + rxq->rx_cq.work_done++; + + ++ndev->stats.rx_dropped; + netdev_warn_once(ndev, + "Dropped oversized RX packet: len=%u, datasize=%u\n", + pktlen, rxq->datasize); + + /* Reuse the RX buffer since rxbuf_oob is unchanged. */ + goto drop; + } + mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); /* Unsuccessful refill will have old_buf == NULL. @@ -2597,6 +2616,10 @@ static int mana_alloc_rx_wqe(struct mana_port_context *apc, *cq_size += COMP_ENTRY_SIZE; } + /* Reserve an extra slot for Fence completion + * event (CQE_RX_OBJECT_FENCE) in case RX CQ is full. + */ + *cq_size += COMP_ENTRY_SIZE; return 0; } @@ -2692,7 +2715,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, goto out; rq_size = MANA_PAGE_ALIGN(rq_size); - cq_size = MANA_PAGE_ALIGN(cq_size); + cq_size = MANA_PAGE_ALIGN(roundup_pow_of_two(cq_size)); /* Create RQ */ memset(&spec, 0, sizeof(spec)); @@ -3522,7 +3545,8 @@ static void mana_rdma_service_handle(struct work_struct *work) struct device *dev = gd->gdma_context->dev; int ret; - if (READ_ONCE(gd->rdma_teardown)) + /* Pairs with the smp_store_release() in mana_rdma_probe(). */ + if (smp_load_acquire(&gd->rdma_teardown)) goto out; switch (serv_work->event) { @@ -3804,6 +3828,21 @@ int mana_rdma_probe(struct gdma_dev *gd) if (err) return err; + /* Clear the state left by a previous mana_rdma_remove() so servicing + * events are handled again after a reset cycle. + */ + gd->is_suspended = false; + + /* Publish is_suspended before re-opening the gate, so the handler + * cannot observe an open gate with a stale is_suspended. Pairs + * with the smp_load_acquire() in mana_rdma_service_handle(). This + * matters on the reset path, where mana_rdma_remove() closed the + * gate and drained the workqueue; on the initial probe path the + * gate was never closed and both flags are already clear. It does + * not order gd->adev, which add_adev() publishes below. + */ + smp_store_release(&gd->rdma_teardown, false); + err = add_adev(gd, "rdma"); if (err) mana_gd_deregister_device(gd); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index 16c828dd5c1a3..2e8343e647546 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -1421,7 +1421,8 @@ static int nfp_net_get_fs_rule(struct nfp_net *nn, struct ethtool_rxnfc *cmd) return -ENOENT; } -static int nfp_net_get_fs_loc(struct nfp_net *nn, u32 *rule_locs) +static int nfp_net_get_fs_loc(struct nfp_net *nn, struct ethtool_rxnfc *cmd, + u32 *rule_locs) { struct nfp_fs_entry *entry; u32 count = 0; @@ -1429,8 +1430,12 @@ static int nfp_net_get_fs_loc(struct nfp_net *nn, u32 *rule_locs) if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FLOW_STEER)) return -EOPNOTSUPP; - list_for_each_entry(entry, &nn->fs.list, node) + list_for_each_entry(entry, &nn->fs.list, node) { + if (count == cmd->rule_cnt) + return -EMSGSIZE; rule_locs[count++] = entry->loc; + } + cmd->rule_cnt = count; return 0; } @@ -1451,7 +1456,7 @@ static int nfp_net_get_rxnfc(struct net_device *netdev, return nfp_net_get_fs_rule(nn, cmd); case ETHTOOL_GRXCLSRLALL: cmd->data = NFP_FS_MAX_ENTRY; - return nfp_net_get_fs_loc(nn, rule_locs); + return nfp_net_get_fs_loc(nn, cmd, rule_locs); default: return -EOPNOTSUPP; } @@ -1694,8 +1699,14 @@ static int nfp_net_fs_add(struct nfp_net *nn, struct ethtool_rxnfc *cmd) nn->fs.count--; err = nfp_net_fs_add_hw(nn, new); - if (err) + if (err) { + /* mbox broken, adding the old rule back will + * likely also fail. + */ + list_del(&entry->node); + kfree(entry); goto err; + } nn->fs.count++; list_replace(&entry->node, &new->node); diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c index 279ea0b569557..55525f45e447b 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c @@ -96,6 +96,9 @@ static int nfp_cpp_resource_find(struct nfp_cpp *cpp, struct nfp_resource *res) res->mutex = nfp_cpp_mutex_alloc(cpp, NFP_RESOURCE_TBL_TARGET, addr, key); + if (!res->mutex) + return -ENOMEM; + res->cpp_id = NFP_CPP_ID(entry.region.cpp_target, entry.region.cpp_action, entry.region.cpp_token); diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 19aa1f1538aa3..c16baa28ad756 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -2740,7 +2740,7 @@ static void nv_tx_timeout(struct net_device *dev, unsigned int txqueue) netdev_info(dev, "Ring at %lx\n", (unsigned long)np->ring_addr); netdev_info(dev, "Dumping tx registers\n"); - for (i = 0; i <= np->register_size; i += 32) { + for (i = 0; i + 32 <= np->register_size; i += 32) { netdev_info(dev, "%3x: %08x %08x %08x %08x " "%08x %08x %08x %08x\n", @@ -6187,10 +6187,10 @@ static void nv_remove(struct pci_dev *pci_dev) struct net_device *dev = pci_get_drvdata(pci_dev); struct fe_priv *np = netdev_priv(dev); - free_percpu(np->txrx_stats); - unregister_netdev(dev); + free_percpu(np->txrx_stats); + nv_restore_mac_addr(pci_dev); /* restore any phy related changes */ @@ -6221,7 +6221,7 @@ static int nv_suspend(struct device *device) netif_device_detach(dev); /* save non-pci configuration space */ - for (i = 0; i <= np->register_size/sizeof(u32); i++) + for (i = 0; i < np->register_size/sizeof(u32); i++) np->saved_config_space[i] = readl(base + i*sizeof(u32)); return 0; @@ -6236,7 +6236,7 @@ static int nv_resume(struct device *device) int i, rc = 0; /* restore non-pci configuration space */ - for (i = 0; i <= np->register_size/sizeof(u32); i++) + for (i = 0; i < np->register_size/sizeof(u32); i++) writel(np->saved_config_space[i], base+i*sizeof(u32)); if (np->driver_data & DEV_NEED_MSI_FIX) diff --git a/drivers/net/ethernet/oa_tc6.c b/drivers/net/ethernet/oa_tc6.c index cfe1f6e3cd12c..b6743710608fe 100644 --- a/drivers/net/ethernet/oa_tc6.c +++ b/drivers/net/ethernet/oa_tc6.c @@ -7,48 +7,11 @@ #include #include +#include #include #include #include -/* OPEN Alliance TC6 registers */ -/* Standard Capabilities Register */ -#define OA_TC6_REG_STDCAP 0x0002 -#define STDCAP_DIRECT_PHY_REG_ACCESS BIT(8) - -/* Reset Control and Status Register */ -#define OA_TC6_REG_RESET 0x0003 -#define RESET_SWRESET BIT(0) /* Software Reset */ - -/* Configuration Register #0 */ -#define OA_TC6_REG_CONFIG0 0x0004 -#define CONFIG0_SYNC BIT(15) -#define CONFIG0_ZARFE_ENABLE BIT(12) - -/* Status Register #0 */ -#define OA_TC6_REG_STATUS0 0x0008 -#define STATUS0_RESETC BIT(6) /* Reset Complete */ -#define STATUS0_HEADER_ERROR BIT(5) -#define STATUS0_LOSS_OF_FRAME_ERROR BIT(4) -#define STATUS0_RX_BUFFER_OVERFLOW_ERROR BIT(3) -#define STATUS0_TX_PROTOCOL_ERROR BIT(0) - -/* Buffer Status Register */ -#define OA_TC6_REG_BUFFER_STATUS 0x000B -#define BUFFER_STATUS_TX_CREDITS_AVAILABLE GENMASK(15, 8) -#define BUFFER_STATUS_RX_CHUNKS_AVAILABLE GENMASK(7, 0) - -/* Interrupt Mask Register #0 */ -#define OA_TC6_REG_INT_MASK0 0x000C -#define INT_MASK0_HEADER_ERR_MASK BIT(5) -#define INT_MASK0_LOSS_OF_FRAME_ERR_MASK BIT(4) -#define INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK BIT(3) -#define INT_MASK0_TX_PROTOCOL_ERR_MASK BIT(0) - -/* PHY Clause 22 registers base address and mask */ -#define OA_TC6_PHY_STD_REG_ADDR_BASE 0xFF00 -#define OA_TC6_PHY_STD_REG_ADDR_MASK 0x1F - /* Control command header */ #define OA_TC6_CTRL_HEADER_DATA_NOT_CTRL BIT(31) #define OA_TC6_CTRL_HEADER_WRITE_NOT_READ BIT(29) @@ -78,23 +41,17 @@ #define OA_TC6_DATA_FOOTER_END_BYTE_OFFSET GENMASK(13, 8) #define OA_TC6_DATA_FOOTER_TX_CREDITS GENMASK(5, 1) -/* PHY – Clause 45 registers memory map selector (MMS) as per table 6 in the - * OPEN Alliance specification. - */ -#define OA_TC6_PHY_C45_PCS_MMS2 2 /* MMD 3 */ -#define OA_TC6_PHY_C45_PMA_PMD_MMS3 3 /* MMD 1 */ -#define OA_TC6_PHY_C45_VS_PLCA_MMS4 4 /* MMD 31 */ -#define OA_TC6_PHY_C45_AUTO_NEG_MMS5 5 /* MMD 7 */ -#define OA_TC6_PHY_C45_POWER_UNIT_MMS6 6 /* MMD 13 */ - +#define OA_TC6_CTRL_PROT_REPLY_SIZE 4 #define OA_TC6_CTRL_HEADER_SIZE 4 #define OA_TC6_CTRL_REG_VALUE_SIZE 4 #define OA_TC6_CTRL_IGNORED_SIZE 4 #define OA_TC6_CTRL_MAX_REGISTERS 128 -#define OA_TC6_CTRL_SPI_BUF_SIZE (OA_TC6_CTRL_HEADER_SIZE +\ - (OA_TC6_CTRL_MAX_REGISTERS *\ - OA_TC6_CTRL_REG_VALUE_SIZE) +\ - OA_TC6_CTRL_IGNORED_SIZE) +#define OA_TC6_CTRL_SPI_BUF_SIZE (OA_TC6_CTRL_HEADER_SIZE +\ + (OA_TC6_CTRL_MAX_REGISTERS *\ + (OA_TC6_CTRL_REG_VALUE_SIZE +\ + OA_TC6_CTRL_PROT_REPLY_SIZE)) +\ + OA_TC6_CTRL_IGNORED_SIZE) + #define OA_TC6_CHUNK_PAYLOAD_SIZE 64 #define OA_TC6_DATA_HEADER_SIZE 4 #define OA_TC6_CHUNK_SIZE (OA_TC6_DATA_HEADER_SIZE +\ @@ -121,14 +78,14 @@ struct oa_tc6 { struct sk_buff *ongoing_tx_skb; struct sk_buff *waiting_tx_skb; struct sk_buff *rx_skb; - struct task_struct *spi_thread; - wait_queue_head_t spi_wq; u16 tx_skb_offset; u16 spi_data_tx_buf_offset; u16 tx_credits; u8 rx_chunks_available; - bool rx_buf_overflow; + bool wait_until_start_valid; bool int_flag; + bool disable_traffic; + bool prot_ctrl; }; enum oa_tc6_header_type { @@ -212,25 +169,36 @@ static void oa_tc6_update_ctrl_write_data(struct oa_tc6 *tc6, u32 value[], { __be32 *tx_buf = tc6->spi_ctrl_tx_buf + OA_TC6_CTRL_HEADER_SIZE; - for (int i = 0; i < length; i++) + for (int i = 0; i < length; i++) { *tx_buf++ = cpu_to_be32(value[i]); + if (tc6->prot_ctrl) + *tx_buf++ = cpu_to_be32(~value[i]); + } } -static u16 oa_tc6_calculate_ctrl_buf_size(u8 length) +static u16 oa_tc6_calculate_ctrl_buf_size(u8 length, bool ctrl_prot) { + u32 reply_size = OA_TC6_CTRL_REG_VALUE_SIZE; + + if (ctrl_prot) + reply_size += OA_TC6_CTRL_PROT_REPLY_SIZE; + /* Control command consists 4 bytes header + 4 bytes register value for - * each register + 4 bytes ignored value. + * each register (+ 4 bytes for the register value complement in case + * protected mode is used) + 4 bytes ignored value. */ - return OA_TC6_CTRL_HEADER_SIZE + OA_TC6_CTRL_REG_VALUE_SIZE * length + + return OA_TC6_CTRL_HEADER_SIZE + reply_size * length + OA_TC6_CTRL_IGNORED_SIZE; } static void oa_tc6_prepare_ctrl_spi_buf(struct oa_tc6 *tc6, u32 address, u32 value[], u8 length, - enum oa_tc6_register_op reg_op) + enum oa_tc6_register_op reg_op, + u16 buf_size) { __be32 *tx_buf = tc6->spi_ctrl_tx_buf; + memset(tx_buf, 0, buf_size); *tx_buf = oa_tc6_prepare_ctrl_header(address, length, reg_op); if (reg_op == OA_TC6_CTRL_REG_WRITE) @@ -253,10 +221,12 @@ static int oa_tc6_check_ctrl_write_reply(struct oa_tc6 *tc6, u8 size) return 0; } -static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 size) +static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 length) { - u32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE; - u32 *tx_buf = tc6->spi_ctrl_tx_buf; + __be32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE; + __be32 *tx_buf = tc6->spi_ctrl_tx_buf; + u32 complement; + u32 reply; /* The echoed control read header must match with the one that was * transmitted. @@ -264,6 +234,20 @@ static int oa_tc6_check_ctrl_read_reply(struct oa_tc6 *tc6, u8 size) if (*tx_buf != *rx_buf) return -EPROTO; + if (tc6->prot_ctrl) { + /* Skip past the echoed header to the value/complement pairs */ + rx_buf += 1; + for (int i = 0; i < length; i++) { + reply = be32_to_cpu(rx_buf[0]); + complement = be32_to_cpu(rx_buf[1]); + + if (complement != ~reply) + return -EPROTO; + + rx_buf += 2; + } + } + return 0; } @@ -273,8 +257,13 @@ static void oa_tc6_copy_ctrl_read_data(struct oa_tc6 *tc6, u32 value[], __be32 *rx_buf = tc6->spi_ctrl_rx_buf + OA_TC6_CTRL_IGNORED_SIZE + OA_TC6_CTRL_HEADER_SIZE; - for (int i = 0; i < length; i++) + for (int i = 0; i < length; i++) { value[i] = be32_to_cpu(*rx_buf++); + + /* skip complement word */ + if (tc6->prot_ctrl) + rx_buf++; + } } static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], @@ -283,10 +272,10 @@ static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], u16 size; int ret; - /* Prepare control command and copy to SPI control buffer */ - oa_tc6_prepare_ctrl_spi_buf(tc6, address, value, length, reg_op); + size = oa_tc6_calculate_ctrl_buf_size(length, tc6->prot_ctrl); - size = oa_tc6_calculate_ctrl_buf_size(length); + /* Prepare control command and copy to SPI control buffer */ + oa_tc6_prepare_ctrl_spi_buf(tc6, address, value, length, reg_op, size); /* Perform SPI transfer */ ret = oa_tc6_spi_transfer(tc6, OA_TC6_CTRL_HEADER, size); @@ -301,7 +290,7 @@ static int oa_tc6_perform_ctrl(struct oa_tc6 *tc6, u32 address, u32 value[], return oa_tc6_check_ctrl_write_reply(tc6, size); /* Check echoed/received control read command reply for errors */ - ret = oa_tc6_check_ctrl_read_reply(tc6, size); + ret = oa_tc6_check_ctrl_read_reply(tc6, length); if (ret) return ret; @@ -407,7 +396,7 @@ static int oa_tc6_check_phy_reg_direct_access_capability(struct oa_tc6 *tc6) if (ret) return ret; - if (!(regval & STDCAP_DIRECT_PHY_REG_ACCESS)) + if (!(regval & OA_TC6_STDCAP_DIRECT_PHY_REG_ACCESS)) return -ENODEV; return 0; @@ -422,7 +411,7 @@ static int oa_tc6_mdiobus_read(struct mii_bus *bus, int addr, int regnum) { struct oa_tc6 *tc6 = bus->priv; u32 regval; - bool ret; + int ret; ret = oa_tc6_read_register(tc6, OA_TC6_PHY_STD_REG_ADDR_BASE | (regnum & OA_TC6_PHY_STD_REG_ADDR_MASK), @@ -600,7 +589,7 @@ static int oa_tc6_read_status0(struct oa_tc6 *tc6) static int oa_tc6_sw_reset_macphy(struct oa_tc6 *tc6) { - u32 regval = RESET_SWRESET; + u32 regval = OA_TC6_RESET_SWRESET; int ret; ret = oa_tc6_write_register(tc6, OA_TC6_REG_RESET, regval); @@ -609,7 +598,7 @@ static int oa_tc6_sw_reset_macphy(struct oa_tc6 *tc6) /* Poll for soft reset complete for every 1ms until 1s timeout */ ret = readx_poll_timeout(oa_tc6_read_status0, tc6, regval, - regval & STATUS0_RESETC, + regval & OA_TC6_STATUS0_RESETC, STATUS0_RESETC_POLL_DELAY, STATUS0_RESETC_POLL_TIMEOUT); if (ret) @@ -628,10 +617,10 @@ static int oa_tc6_unmask_macphy_error_interrupts(struct oa_tc6 *tc6) if (ret) return ret; - regval &= ~(INT_MASK0_TX_PROTOCOL_ERR_MASK | - INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK | - INT_MASK0_LOSS_OF_FRAME_ERR_MASK | - INT_MASK0_HEADER_ERR_MASK); + regval &= ~(OA_TC6_INT_MASK0_TX_PROTOCOL_ERR_MASK | + OA_TC6_INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK | + OA_TC6_INT_MASK0_LOSS_OF_FRAME_ERR_MASK | + OA_TC6_INT_MASK0_HEADER_ERR_MASK); return oa_tc6_write_register(tc6, OA_TC6_REG_INT_MASK0, regval); } @@ -646,11 +635,31 @@ static int oa_tc6_enable_data_transfer(struct oa_tc6 *tc6) return ret; /* Enable configuration synchronization for data transfer */ - value |= CONFIG0_SYNC; + value |= OA_TC6_CONFIG0_SYNC; return oa_tc6_write_register(tc6, OA_TC6_REG_CONFIG0, value); } +/* Called when a frame that is meant to be transmitted, is dropped. */ +static void oa_tc6_drop_tx_skb(struct oa_tc6 *tc6, struct sk_buff *skb) +{ + if (skb) { + tc6->netdev->stats.tx_dropped++; + dev_kfree_skb_any(skb); + } +} + +static struct sk_buff *oa_tc6_detach_waiting_tx_skb(struct oa_tc6 *tc6) +{ + struct sk_buff *skb; + + lockdep_assert_held(&tc6->tx_skb_lock); + skb = tc6->waiting_tx_skb; + tc6->waiting_tx_skb = NULL; + + return skb; +} + static void oa_tc6_cleanup_ongoing_rx_skb(struct oa_tc6 *tc6) { if (tc6->rx_skb) { @@ -662,11 +671,63 @@ static void oa_tc6_cleanup_ongoing_rx_skb(struct oa_tc6 *tc6) static void oa_tc6_cleanup_ongoing_tx_skb(struct oa_tc6 *tc6) { - if (tc6->ongoing_tx_skb) { - tc6->netdev->stats.tx_dropped++; - kfree_skb(tc6->ongoing_tx_skb); - tc6->ongoing_tx_skb = NULL; - } + oa_tc6_drop_tx_skb(tc6, tc6->ongoing_tx_skb); + tc6->ongoing_tx_skb = NULL; +} + +static void oa_tc6_cleanup_waiting_tx_skb(struct oa_tc6 *tc6) +{ + struct sk_buff *skb; + + spin_lock_bh(&tc6->tx_skb_lock); + skb = oa_tc6_detach_waiting_tx_skb(tc6); + spin_unlock_bh(&tc6->tx_skb_lock); + + oa_tc6_drop_tx_skb(tc6, skb); +} + +static void oa_tc6_free_ongoing_skbs(struct oa_tc6 *tc6) +{ + oa_tc6_cleanup_ongoing_tx_skb(tc6); + oa_tc6_cleanup_ongoing_rx_skb(tc6); +} + +static void oa_tc6_free_pending_skbs(struct oa_tc6 *tc6) +{ + oa_tc6_free_ongoing_skbs(tc6); + oa_tc6_cleanup_waiting_tx_skb(tc6); +} + +static void oa_tc6_look_for_new_frame(struct oa_tc6 *tc6) +{ + tc6->wait_until_start_valid = true; + oa_tc6_cleanup_ongoing_rx_skb(tc6); +} + +/* If the failure is at SPI interface level, masking and clearing + * the interrupt of the device won't work. Since SPI interrupt is + * disabled, it should stop the repeated interrupts. + */ +static void oa_tc6_disable_traffic(struct oa_tc6 *tc6) +{ + u32 regval = OA_TC6_INT_MASK0_ALL_INTERRUPTS; + struct sk_buff *skb; + + spin_lock_bh(&tc6->tx_skb_lock); + tc6->disable_traffic = true; + skb = oa_tc6_detach_waiting_tx_skb(tc6); + spin_unlock_bh(&tc6->tx_skb_lock); + + /* disable_traffic, when set, is a point of no return to + * working state. Keeping the TX queues disabled. + */ + netif_tx_disable(tc6->netdev); + oa_tc6_drop_tx_skb(tc6, skb); + oa_tc6_free_ongoing_skbs(tc6); + oa_tc6_write_register(tc6, OA_TC6_REG_INT_MASK0, regval); + oa_tc6_read_register(tc6, OA_TC6_REG_STATUS0, ®val); + oa_tc6_write_register(tc6, OA_TC6_REG_STATUS0, regval); + dev_err(&tc6->spi->dev, "Device interrupt disabled to avoid interrupt storm"); } static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) @@ -681,6 +742,13 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) return ret; } + /* This function is called for each chunk received in a given SPI + * transaction. In case, extended status bit is set in more than + * one chunk, skip the write, if status0 is already cleared. + */ + if (!value) + return 0; + /* Clear the error interrupts status */ ret = oa_tc6_write_register(tc6, OA_TC6_REG_STATUS0, value); if (ret) { @@ -689,25 +757,24 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) return ret; } - if (FIELD_GET(STATUS0_RX_BUFFER_OVERFLOW_ERROR, value)) { - tc6->rx_buf_overflow = true; - oa_tc6_cleanup_ongoing_rx_skb(tc6); + if (FIELD_GET(OA_TC6_STATUS0_RX_BUFFER_OVERFLOW_ERROR, value)) { + oa_tc6_look_for_new_frame(tc6); net_err_ratelimited("%s: Receive buffer overflow error\n", tc6->netdev->name); return -EAGAIN; } - if (FIELD_GET(STATUS0_TX_PROTOCOL_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_TX_PROTOCOL_ERROR, value)) { netdev_err(tc6->netdev, "Transmit protocol error\n"); return -ENODEV; } /* TODO: Currently loss of frame and header errors are treated as * non-recoverable errors. They will be handled in the next version. */ - if (FIELD_GET(STATUS0_LOSS_OF_FRAME_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_LOSS_OF_FRAME_ERROR, value)) { netdev_err(tc6->netdev, "Loss of frame error\n"); return -ENODEV; } - if (FIELD_GET(STATUS0_HEADER_ERROR, value)) { + if (FIELD_GET(OA_TC6_STATUS0_HEADER_ERROR, value)) { netdev_err(tc6->netdev, "Header error\n"); return -ENODEV; } @@ -717,6 +784,8 @@ static int oa_tc6_process_extended_status(struct oa_tc6 *tc6) static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) { + int ret = 0; + /* Process rx chunk footer for the following, * 1. tx credits * 2. errors if any from MAC-PHY @@ -727,9 +796,11 @@ static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) footer); if (FIELD_GET(OA_TC6_DATA_FOOTER_EXTENDED_STS, footer)) { - int ret = oa_tc6_process_extended_status(tc6); - - if (ret) + ret = oa_tc6_process_extended_status(tc6); + /* EAGAIN error is recoverable. Move on to check + * HEADER and SYNC errors before returning. + */ + if (ret && ret != -EAGAIN) return ret; } @@ -747,7 +818,7 @@ static int oa_tc6_process_rx_chunk_footer(struct oa_tc6 *tc6, u32 footer) return -ENODEV; } - return 0; + return ret; } static void oa_tc6_submit_rx_skb(struct oa_tc6 *tc6) @@ -772,13 +843,35 @@ static void oa_tc6_submit_rx_skb(struct oa_tc6 *tc6) tc6->rx_skb = NULL; } -static void oa_tc6_update_rx_skb(struct oa_tc6 *tc6, u8 *payload, u8 length) +/* On oversubscribed traffic condition, particularly with overwhelming rx + * buffer overflow errors, there could be data chunk loss. If tail + length + * goes beyond end pointer, that is an indication that the data chunk with + * end_valid bit is lost. Time to look for a data chunk with start_valid bit. + * + * If rx_skb is NULL, it is time to start looking for data chunk with + * start_bit. + */ +static int oa_tc6_update_rx_skb(struct oa_tc6 *tc6, u8 *payload, u8 length) { + if (!tc6->rx_skb || + skb_tailroom(tc6->rx_skb) < length) { + oa_tc6_look_for_new_frame(tc6); + return -EAGAIN; + } + memcpy(skb_put(tc6->rx_skb, length), payload, length); + return 0; } +/* On overwhelming rx buffer overflow errors, due to data chunk loss, it is + * possible that we get two data chunks with start_valid bit set, without + * end_valid bit set in between. In this case, rx_skb would have a valid + * buffer pointer. We should release, if a valid pointer is found before + * allocating a new one. + */ static int oa_tc6_allocate_rx_skb(struct oa_tc6 *tc6) { + oa_tc6_cleanup_ongoing_rx_skb(tc6); tc6->rx_skb = netdev_alloc_skb_ip_align(tc6->netdev, tc6->netdev->mtu + ETH_HLEN + ETH_FCS_LEN); if (!tc6->rx_skb) { @@ -798,7 +891,9 @@ static int oa_tc6_prcs_complete_rx_frame(struct oa_tc6 *tc6, u8 *payload, if (ret) return ret; - oa_tc6_update_rx_skb(tc6, payload, size); + ret = oa_tc6_update_rx_skb(tc6, payload, size); + if (ret) + return ret; oa_tc6_submit_rx_skb(tc6); @@ -813,22 +908,24 @@ static int oa_tc6_prcs_rx_frame_start(struct oa_tc6 *tc6, u8 *payload, u16 size) if (ret) return ret; - oa_tc6_update_rx_skb(tc6, payload, size); - - return 0; + return oa_tc6_update_rx_skb(tc6, payload, size); } -static void oa_tc6_prcs_rx_frame_end(struct oa_tc6 *tc6, u8 *payload, u16 size) +static int oa_tc6_prcs_rx_frame_end(struct oa_tc6 *tc6, u8 *payload, u16 size) { - oa_tc6_update_rx_skb(tc6, payload, size); + int ret; - oa_tc6_submit_rx_skb(tc6); + ret = oa_tc6_update_rx_skb(tc6, payload, size); + if (!ret) + oa_tc6_submit_rx_skb(tc6); + return ret; } -static void oa_tc6_prcs_ongoing_rx_frame(struct oa_tc6 *tc6, u8 *payload, - u32 footer) +static int oa_tc6_prcs_ongoing_rx_frame(struct oa_tc6 *tc6, u8 *payload, + u32 footer) { - oa_tc6_update_rx_skb(tc6, payload, OA_TC6_CHUNK_PAYLOAD_SIZE); + return oa_tc6_update_rx_skb(tc6, payload, + OA_TC6_CHUNK_PAYLOAD_SIZE); } static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, @@ -843,10 +940,10 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, u16 size; /* Restart the new rx frame after receiving rx buffer overflow error */ - if (start_valid && tc6->rx_buf_overflow) - tc6->rx_buf_overflow = false; + if (start_valid && tc6->wait_until_start_valid) + tc6->wait_until_start_valid = false; - if (tc6->rx_buf_overflow) + if (tc6->wait_until_start_valid) return 0; /* Process the chunk with complete rx frame */ @@ -868,8 +965,7 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, /* Process the chunk with only rx frame end */ if (end_valid && !start_valid) { size = end_byte_offset + 1; - oa_tc6_prcs_rx_frame_end(tc6, data, size); - return 0; + return oa_tc6_prcs_rx_frame_end(tc6, data, size); } /* Process the chunk with previous rx frame end and next rx frame @@ -883,6 +979,15 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, if (tc6->rx_skb) { size = end_byte_offset + 1; oa_tc6_prcs_rx_frame_end(tc6, data, size); + + /* Return value from oa_tc6_prcs_rx_frame_end is not + * checked. If it returned an error, it is to make + * the code to look for new frame. At this stage, + * code below is going to process a new frame. So, + * error condition is set to false, in case it is + * set before proceeding. + */ + tc6->wait_until_start_valid = false; } size = OA_TC6_CHUNK_PAYLOAD_SIZE - start_byte_offset; return oa_tc6_prcs_rx_frame_start(tc6, @@ -891,9 +996,7 @@ static int oa_tc6_prcs_rx_chunk_payload(struct oa_tc6 *tc6, u8 *data, } /* Process the chunk with ongoing rx frame data */ - oa_tc6_prcs_ongoing_rx_frame(tc6, data, footer); - - return 0; + return oa_tc6_prcs_ongoing_rx_frame(tc6, data, footer); } static u32 oa_tc6_get_rx_chunk_footer(struct oa_tc6 *tc6, u16 footer_offset) @@ -909,8 +1012,9 @@ static u32 oa_tc6_get_rx_chunk_footer(struct oa_tc6 *tc6, u16 footer_offset) static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) { u16 no_of_rx_chunks = length / OA_TC6_CHUNK_SIZE; + bool retry = false; + int ret = 0; u32 footer; - int ret; /* All the rx chunks in the receive SPI data buffer are examined here */ for (int i = 0; i < no_of_rx_chunks; i++) { @@ -919,8 +1023,11 @@ static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) OA_TC6_CHUNK_PAYLOAD_SIZE); ret = oa_tc6_process_rx_chunk_footer(tc6, footer); - if (ret) - return ret; + if (ret) { + if (ret != -EAGAIN) + return ret; + retry = true; + } /* If there is a data valid chunks then process it for the * information needed to determine the validity and the location @@ -932,12 +1039,35 @@ static int oa_tc6_process_spi_data_rx_buf(struct oa_tc6 *tc6, u16 length) ret = oa_tc6_prcs_rx_chunk_payload(tc6, payload, footer); - if (ret) - return ret; + if (ret) { + if (ret != -ENOMEM && ret != -EAGAIN) + return ret; + retry = true; + } } } - return 0; + /* Not bailing out on recoverable error codes, -EAGAIN and + * -ENOMEM. If subsequent loop iterations, if any, succeeds, + * error code would be overwritten. retry flag helps to + * make the caller to continue and retry. Since recovery + * action for -ENOMEM and -EAGAIN are same, we are returning + * one of the error codes, that is -EAGAIN. + * + * Successful recovery depends on how small the frames are, + * how many chunks, among the received chunks triggered the + * error, whether data is intact even with error conditions. + * As a result, there is no single, best method to recover + * most data when error conditions hit. We do our best by + * processing all the chunks with good "footer header" and + * "data valid" bit set. + */ + if (retry) { + ret = -EAGAIN; + oa_tc6_look_for_new_frame(tc6); + } + + return ret; } static __be32 oa_tc6_prepare_data_header(bool data_valid, bool start_valid, @@ -1099,12 +1229,9 @@ static int oa_tc6_try_spi_transfer(struct oa_tc6 *tc6) } ret = oa_tc6_process_spi_data_rx_buf(tc6, spi_len); - if (ret) { - if (ret == -EAGAIN) - continue; - oa_tc6_cleanup_ongoing_tx_skb(tc6); - oa_tc6_cleanup_ongoing_rx_skb(tc6); + if (ret && ret != -EAGAIN) { + oa_tc6_free_ongoing_skbs(tc6); netdev_err(tc6->netdev, "Device error: %d\n", ret); return ret; } @@ -1116,29 +1243,34 @@ static int oa_tc6_try_spi_transfer(struct oa_tc6 *tc6) return 0; } -static int oa_tc6_spi_thread_handler(void *data) +static irqreturn_t oa_tc6_macphy_threaded_irq(int irq, void *data) { struct oa_tc6 *tc6 = data; - int ret; - - while (likely(!kthread_should_stop())) { - /* This kthread will be waken up if there is a tx skb or mac-phy - * interrupt to perform spi transfer with tx chunks. - */ - wait_event_interruptible(tc6->spi_wq, tc6->int_flag || - (tc6->waiting_tx_skb && - tc6->tx_credits) || - kthread_should_stop()); + int ret = 0; - if (kthread_should_stop()) - break; + /* It is possible that interrupt woke the thread before it is + * disabled. Until we come up with good recovery mechanism, + * no need to attempt spi transfer, once it fails. Pending skbs + * are already freed. + */ + spin_lock_bh(&tc6->tx_skb_lock); + if (tc6->disable_traffic) { + spin_unlock_bh(&tc6->tx_skb_lock); + return IRQ_HANDLED; + } + spin_unlock_bh(&tc6->tx_skb_lock); + while (tc6->int_flag || + (tc6->waiting_tx_skb && tc6->tx_credits)) { ret = oa_tc6_try_spi_transfer(tc6); - if (ret) - return ret; + if (ret) { + disable_irq_nosync(tc6->spi->irq); + oa_tc6_disable_traffic(tc6); + break; + } } - return 0; + return IRQ_HANDLED; } static int oa_tc6_update_buffer_status_from_register(struct oa_tc6 *tc6) @@ -1154,9 +1286,10 @@ static int oa_tc6_update_buffer_status_from_register(struct oa_tc6 *tc6) if (ret) return ret; - tc6->tx_credits = FIELD_GET(BUFFER_STATUS_TX_CREDITS_AVAILABLE, value); - tc6->rx_chunks_available = FIELD_GET(BUFFER_STATUS_RX_CHUNKS_AVAILABLE, - value); + tc6->tx_credits = FIELD_GET(OA_TC6_BUFFER_STATUS_TX_CREDITS_AVAILABLE, + value); + tc6->rx_chunks_available = + FIELD_GET(OA_TC6_BUFFER_STATUS_RX_CHUNKS_AVAILABLE, value); return 0; } @@ -1172,11 +1305,15 @@ static irqreturn_t oa_tc6_macphy_isr(int irq, void *data) * the previous rx footer. * - extended status event not reported in the previous rx footer. */ - tc6->int_flag = true; - /* Wake spi kthread to perform spi transfer */ - wake_up_interruptible(&tc6->spi_wq); - - return IRQ_HANDLED; + if (tc6->disable_traffic) + disable_irq_nosync(tc6->spi->irq); + else + tc6->int_flag = true; + /* Wake IRQ thread to perform spi transfer . In case + * disable_traffic is set, threaded irq may run again + * one more time. + */ + return IRQ_WAKE_THREAD; } /** @@ -1196,7 +1333,7 @@ int oa_tc6_zero_align_receive_frame_enable(struct oa_tc6 *tc6) return ret; /* Set Zero-Align Receive Frame Enable */ - regval |= CONFIG0_ZARFE_ENABLE; + regval |= OA_TC6_CONFIG0_ZARFE_ENABLE; return oa_tc6_write_register(tc6, OA_TC6_REG_CONFIG0, regval); } @@ -1208,33 +1345,54 @@ EXPORT_SYMBOL_GPL(oa_tc6_zero_align_receive_frame_enable); * @tc6: oa_tc6 struct. * @skb: socket buffer in which the ethernet frame is stored. * - * Return: NETDEV_TX_OK if the transmit ethernet frame skb added in the tx_skb_q - * otherwise returns NETDEV_TX_BUSY. + * Return: NETDEV_TX_OK either on successful queueing of the packet for + * transmission, or on packet getting dropped. Packet can be dropped due to + * failure in linearizing the buffer or disable_traffic is set due to + * earlier fatal error. Returns NETDEV_TX_BUSY when there is no room + * to queue the packet. */ netdev_tx_t oa_tc6_start_xmit(struct oa_tc6 *tc6, struct sk_buff *skb) { + if (skb_linearize(skb)) { + oa_tc6_drop_tx_skb(tc6, skb); + return NETDEV_TX_OK; + } + + spin_lock_bh(&tc6->tx_skb_lock); if (tc6->waiting_tx_skb) { netif_stop_queue(tc6->netdev); + spin_unlock_bh(&tc6->tx_skb_lock); return NETDEV_TX_BUSY; } - - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - tc6->netdev->stats.tx_dropped++; + if (tc6->disable_traffic) { + spin_unlock_bh(&tc6->tx_skb_lock); + oa_tc6_drop_tx_skb(tc6, skb); return NETDEV_TX_OK; } - - spin_lock_bh(&tc6->tx_skb_lock); tc6->waiting_tx_skb = skb; spin_unlock_bh(&tc6->tx_skb_lock); - /* Wake spi kthread to perform spi transfer */ - wake_up_interruptible(&tc6->spi_wq); + /* Wake the threaded IRQ to perform spi transfer. */ + irq_wake_thread(tc6->spi->irq, tc6); return NETDEV_TX_OK; } EXPORT_SYMBOL_GPL(oa_tc6_start_xmit); +static int oa_tc6_check_ctrl_protection(struct oa_tc6 *tc6) +{ + u32 regval; + int ret; + + ret = oa_tc6_read_register(tc6, OA_TC6_REG_CONFIG0, ®val); + if (ret) + return ret; + + tc6->prot_ctrl = FIELD_GET(OA_TC6_CONFIG0_PROTE, regval); + + return 0; +} + /** * oa_tc6_init - allocates and initializes oa_tc6 structure. * @spi: device with which data will be exchanged. @@ -1287,6 +1445,14 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev) if (!tc6->spi_data_rx_buf) return NULL; + /* Check the PROTE bit status so that we can reset the device */ + ret = oa_tc6_check_ctrl_protection(tc6); + if (ret) { + dev_err(&tc6->spi->dev, + "Failed to check the protection mode: %d\n", ret); + return NULL; + } + ret = oa_tc6_sw_reset_macphy(tc6); if (ret) { dev_err(&tc6->spi->dev, @@ -1322,24 +1488,15 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev) goto phy_exit; } - init_waitqueue_head(&tc6->spi_wq); - - tc6->spi_thread = kthread_run(oa_tc6_spi_thread_handler, tc6, - "oa-tc6-spi-thread"); - if (IS_ERR(tc6->spi_thread)) { - dev_err(&tc6->spi->dev, "Failed to create SPI thread\n"); - goto phy_exit; - } - - sched_set_fifo(tc6->spi_thread); - - ret = devm_request_irq(&tc6->spi->dev, tc6->spi->irq, oa_tc6_macphy_isr, - IRQF_TRIGGER_FALLING, dev_name(&tc6->spi->dev), - tc6); + ret = devm_request_threaded_irq(&tc6->spi->dev, tc6->spi->irq, + oa_tc6_macphy_isr, + oa_tc6_macphy_threaded_irq, + IRQF_TRIGGER_LOW | IRQF_ONESHOT, + dev_name(&tc6->spi->dev), tc6); if (ret) { dev_err(&tc6->spi->dev, "Failed to request macphy isr %d\n", ret); - goto kthread_stop; + goto phy_exit; } /* oa_tc6_sw_reset_macphy() function resets and clears the MAC-PHY reset @@ -1349,12 +1506,10 @@ struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev) * 7.7 and 9.2.8.8 in the OPEN Alliance specification for more details. */ tc6->int_flag = true; - wake_up_interruptible(&tc6->spi_wq); + irq_wake_thread(tc6->spi->irq, tc6); return tc6; -kthread_stop: - kthread_stop(tc6->spi_thread); phy_exit: oa_tc6_phy_exit(tc6); return NULL; @@ -1367,11 +1522,12 @@ EXPORT_SYMBOL_GPL(oa_tc6_init); */ void oa_tc6_exit(struct oa_tc6 *tc6) { + disable_irq(tc6->spi->irq); + spin_lock_bh(&tc6->tx_skb_lock); + tc6->disable_traffic = true; + spin_unlock_bh(&tc6->tx_skb_lock); oa_tc6_phy_exit(tc6); - kthread_stop(tc6->spi_thread); - dev_kfree_skb_any(tc6->ongoing_tx_skb); - dev_kfree_skb_any(tc6->waiting_tx_skb); - dev_kfree_skb_any(tc6->rx_skb); + oa_tc6_free_pending_skbs(tc6); } EXPORT_SYMBOL_GPL(oa_tc6_exit); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index 29a8a25a3ed08..ae7a689390327 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -918,8 +918,21 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) }; int err; - q->partner = &lif->txqcqs[q->index]->q; - q->partner->partner = q; + q->partner = NULL; + + /* Only normal RX queues have matching TX queue partners. */ + if (q->index < lif->nxqs) { + if (!lif->txqcqs || + q->index >= lif->ionic->ntxqs_per_lif || + !lif->txqcqs[q->index]) { + dev_err(dev, "missing TX queue partner for RX queue %u\n", + q->index); + return -ENXIO; + } + + q->partner = &lif->txqcqs[q->index]->q; + q->partner->partner = q; + } if (!lif->xdp_prog || (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags)) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index 301ebee2fdc50..c8ecb0ee231ef 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -545,13 +545,18 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, break; case XDP_TX: + txq = rxq->partner; + if (unlikely(!txq)) { + err = -EIO; + break; + } + xdpf = xdp_convert_buff_to_frame(&xdp_buf); if (!xdpf) { err = -ENOSPC; break; } - txq = rxq->partner; nq = netdev_get_tx_queue(netdev, txq->index); __netif_tx_lock(nq, smp_processor_id()); txq_trans_cond_update(nq); @@ -701,11 +706,7 @@ static void ionic_rx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_rxq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -729,7 +730,12 @@ static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog) struct ionic_queue *q = cq->bound_q; struct ionic_rxq_comp *comp; - comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_rxq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->pkt_type_color, cq->done_color)) return false; @@ -1180,7 +1186,6 @@ static void ionic_tx_clean(struct ionic_queue *q, bool in_napi) { struct ionic_tx_stats *stats = q_to_tx_stats(q); - struct ionic_qcq *qcq = q_to_qcq(q); struct sk_buff *skb; if (desc_info->xdpf) { @@ -1205,11 +1210,7 @@ static void ionic_tx_clean(struct ionic_queue *q, __le64 *cq_desc_hwstamp; u64 hwstamp; - cq_desc_hwstamp = - (void *)comp + - qcq->cq.desc_size - - sizeof(struct ionic_txq_comp) - - IONIC_HWSTAMP_CQ_NEGOFFSET; + cq_desc_hwstamp = (void *)comp - IONIC_HWSTAMP_CQ_NEGOFFSET; hwstamp = le64_to_cpu(*cq_desc_hwstamp); @@ -1244,7 +1245,12 @@ static bool ionic_tx_service(struct ionic_cq *cq, unsigned int pkts = 0; u16 index; - comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + if (likely(cq->desc_size == sizeof(*comp))) + comp = &((struct ionic_txq_comp *)cq->base)[cq->tail_idx]; + else + comp = cq->base + + cq->desc_size * cq->tail_idx + + cq->desc_size - sizeof(*comp); if (!color_match(comp->color, cq->done_color)) return false; @@ -1662,15 +1668,22 @@ static int ionic_tx(struct net_device *netdev, struct ionic_queue *q, stats->pkts++; stats->bytes += skb->len; + ionic_txq_post(q, false); + if (likely(!ionic_txq_hwstamp_enabled(q))) { struct netdev_queue *ndq = q_to_ndq(netdev, q); - if (unlikely(!ionic_q_has_space(q, MAX_SKB_FRAGS + 1))) - netif_tx_stop_queue(ndq); + netif_txq_maybe_stop(ndq, ionic_q_space_avail(q), + MAX_SKB_FRAGS + 1, MAX_SKB_FRAGS + 1); ring_dbell = __netdev_tx_sent_queue(ndq, skb->len, netdev_xmit_more()); } - ionic_txq_post(q, ring_dbell); + + if (ring_dbell) { + ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type, + q->dbval | q->head_idx); + q->dbell_jiffies = jiffies; + } return 0; } diff --git a/drivers/net/ethernet/qlogic/qede/qede.h b/drivers/net/ethernet/qlogic/qede/qede.h index 042a75f340608..0e7a0c2c17651 100644 --- a/drivers/net/ethernet/qlogic/qede/qede.h +++ b/drivers/net/ethernet/qlogic/qede/qede.h @@ -303,10 +303,10 @@ enum qede_agg_state { }; struct qede_agg_info { - /* rx_buf is a data buffer that can be placed / consumed from rx bd - * chain. It has two purposes: We will preallocate the data buffer - * for each aggregation when we open the interface and will place this - * buffer on the rx-bd-ring when we receive TPA_START. We don't want + /* buffer is used to retain the Rx consumer descriptor when a TPA + * session starts. If the SKB allocation fails during TPA_START, + * we use this saved buffer to safely recycle the physical page + * back into the rx-bd-ring via qede_reuse_page(). We don't want * to be in a state where allocation fails, as we can't reuse the * consumer buffer in the rx-chain since FW may still be writing to it * (since header needs to be modified for TPA). diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index c11e0d8f98aa5..a02624f6d6d59 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -850,6 +850,7 @@ static void qede_tpa_start(struct qede_dev *edev, pad, false); tpa_info->buffer.page_offset = sw_rx_data_cons->page_offset; tpa_info->buffer.mapping = sw_rx_data_cons->mapping; + tpa_info->buffer.data = sw_rx_data_cons->data; if (unlikely(!tpa_info->skb)) { DP_NOTICE(edev, "Failed to allocate SKB for gro\n"); diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index b5d744d2586f7..59a80f7193723 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -107,7 +107,7 @@ static void qede_remove(struct pci_dev *pdev); static void qede_shutdown(struct pci_dev *pdev); static void qede_link_update(void *dev, struct qed_link_output *link); static void qede_schedule_recovery_handler(void *dev); -static void qede_recovery_handler(struct qede_dev *edev); +static bool qede_recovery_handler(struct qede_dev *edev); static void qede_schedule_hw_err_handler(void *dev, enum qed_hw_err_type err_type); static void qede_get_eth_tlv_data(void *edev, void *data); @@ -1065,21 +1065,6 @@ void __qede_unlock(struct qede_dev *edev) mutex_unlock(&edev->qede_lock); } -/* This version of the lock should be used when acquiring the RTNL lock is also - * needed in addition to the internal qede lock. - */ -static void qede_lock(struct qede_dev *edev) -{ - rtnl_lock(); - __qede_lock(edev); -} - -static void qede_unlock(struct qede_dev *edev) -{ - __qede_unlock(edev); - rtnl_unlock(); -} - static void qede_periodic_task(struct work_struct *work) { struct qede_dev *edev = container_of(work, struct qede_dev, @@ -1116,6 +1101,8 @@ static void qede_sp_task(struct work_struct *work) */ if (test_and_clear_bit(QEDE_SP_RECOVERY, &edev->sp_flags)) { + bool reloaded; + cancel_delayed_work_sync(&edev->periodic_task); #ifdef CONFIG_QED_SRIOV /* SRIOV must be disabled outside the lock to avoid a deadlock. @@ -1124,9 +1111,17 @@ static void qede_sp_task(struct work_struct *work) if (pci_num_vf(edev->pdev)) qede_sriov_configure(edev->pdev, 0); #endif - qede_lock(edev); - qede_recovery_handler(edev); - qede_unlock(edev); + rtnl_lock(); + __qede_lock(edev); + reloaded = qede_recovery_handler(edev); + __qede_unlock(edev); + + /* The udp_tunnel core synchronously calls back into + * qede_udp_tunnel_sync(), which takes the qede lock. + */ + if (reloaded) + udp_tunnel_nic_reset_ntf(edev->ndev); + rtnl_unlock(); } __qede_lock(edev); @@ -2667,9 +2662,13 @@ static void qede_recovery_failed(struct qede_dev *edev) edev->ops->common->set_power_state(edev->cdev, PCI_D3hot); } -static void qede_recovery_handler(struct qede_dev *edev) +/* Returns true if an open device was successfully reloaded and its + * udp_tunnel ports need to be re-synced by the caller. + */ +static bool qede_recovery_handler(struct qede_dev *edev) { u32 curr_state = edev->state; + bool reloaded = false; int rc; DP_NOTICE(edev, "Starting a recovery process\n"); @@ -2699,17 +2698,18 @@ static void qede_recovery_handler(struct qede_dev *edev) goto err; qede_config_rx_mode(edev->ndev); - udp_tunnel_nic_reset_ntf(edev->ndev); + reloaded = true; } edev->state = curr_state; DP_NOTICE(edev, "Recovery handling is done\n"); - return; + return reloaded; err: qede_recovery_failed(edev); + return false; } static void qede_atomic_hw_err_handler(struct qede_dev *edev) diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c index 09f20c7947545..d26c26c21cbe8 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_init.c @@ -6,6 +6,7 @@ #include "qlcnic.h" #include "qlcnic_hw.h" +#include struct crb_addr_pair { u32 addr; @@ -740,159 +741,212 @@ qlcnic_has_mn(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_table_desc *qlcnic_get_table_desc(const u8 *unirom, int section) -{ - u32 i, entries; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - entries = le32_to_cpu(directory->num_entries); - - for (i = 0; i < entries; i++) { +#define FILEHEADER_SIZE (14 * 4) +#define QLCNIC_UNI_DIR_TYPE_OFF (8 * sizeof(__le32)) +#define QLCNIC_UNI_DIR_ENTRY_MIN_SIZE (9 * sizeof(__le32)) +#define QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE \ + ((QLCNIC_UNI_FIRMWARE_IDX_OFF + 1) * sizeof(__le32)) +#define QLCNIC_UNI_VERSION_TAIL_SIZE 17 +#define QLCNIC_UNI_BOOTLD_SIZE \ + (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) + +struct qlcnic_uni_data { + u32 offset; + u32 size; +}; - u32 offs = le32_to_cpu(directory->findex) + - i * le32_to_cpu(directory->entry_size); - u32 tab_type = le32_to_cpu(*((__le32 *)&unirom[offs] + 8)); +static bool qlcnic_rom_range_valid(size_t size, size_t offset, size_t len) +{ + return offset <= size && len <= size - offset; +} - if (tab_type == section) - return (struct uni_table_desc *) &unirom[offs]; - } +static bool qlcnic_rom_table_valid(size_t size, u32 offset, u32 entries, + u32 entry_size, u32 min_entry_size) +{ + if (entry_size < min_entry_size || offset > size) + return false; - return NULL; + return entries <= (size - offset) / entry_size; } -#define FILEHEADER_SIZE (14 * 4) - -static int -qlcnic_validate_header(struct qlcnic_adapter *adapter) +static int qlcnic_get_directory(struct qlcnic_adapter *adapter, + size_t *offset, u32 *entries, u32 *entry_size) { - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *directory = (struct uni_table_desc *) &unirom[0]; - u32 entries, entry_size, tab_size, fw_file_size; - - fw_file_size = adapter->fw->size; + const struct firmware *fw = adapter->fw; + const u8 *directory = fw->data; - if (fw_file_size < FILEHEADER_SIZE) + if (fw->size < FILEHEADER_SIZE) return -EINVAL; - entries = le32_to_cpu(directory->num_entries); - entry_size = le32_to_cpu(directory->entry_size); - tab_size = le32_to_cpu(directory->findex) + (entries * entry_size); + *offset = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, findex)); + *entries = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, num_entries)); + *entry_size = get_unaligned_le32(directory + + offsetof(struct uni_table_desc, entry_size)); - if (fw_file_size < tab_size) + if (!qlcnic_rom_table_valid(fw->size, *offset, *entries, *entry_size, + QLCNIC_UNI_DIR_ENTRY_MIN_SIZE)) return -EINVAL; return 0; } -static int -qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +static int qlcnic_get_table_desc(struct qlcnic_adapter *adapter, int section, + size_t *desc_offset) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - u32 offs, tab_size, data_size, idx; const u8 *unirom = adapter->fw->data; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_BOOTLD_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_BOOTLD); - - if (!tab_desc) - return -EINVAL; - - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); - - if (adapter->fw->size < tab_size) - return -EINVAL; + size_t directory_offset; + u32 entries, entry_size; + size_t i; + int ret; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; + ret = qlcnic_get_directory(adapter, &directory_offset, &entries, + &entry_size); + if (ret) + return ret; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + for (i = 0; i < entries; i++) { + size_t offset = directory_offset + i * entry_size; + u32 table_type; - if (adapter->fw->size < data_size) - return -EINVAL; + table_type = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_DIR_TYPE_OFF); + if (table_type == section) { + *desc_offset = offset; + return 0; + } + } - return 0; + return -ENOENT; } static int -qlcnic_validate_fw(struct qlcnic_adapter *adapter) +qlcnic_validate_header(struct qlcnic_adapter *adapter) { - struct uni_table_desc *tab_desc; - struct uni_data_desc *descr; - const u8 *unirom = adapter->fw->data; - u32 offs, tab_size, data_size, idx; - __le32 temp; + u32 entries, entry_size; + size_t offset; + + return qlcnic_get_directory(adapter, &offset, &entries, &entry_size); +} - temp = *((__le32 *)&unirom[adapter->file_prd_off] + - QLCNIC_UNI_FIRMWARE_IDX_OFF); - idx = le32_to_cpu(temp); - tab_desc = qlcnic_get_table_desc(unirom, QLCNIC_UNI_DIR_SECT_FW); +static int qlcnic_get_data_desc(struct qlcnic_adapter *adapter, u32 section, + u32 index_offset, struct qlcnic_uni_data *data) +{ + size_t table_desc_offset, table_offset, desc_offset; + const struct firmware *fw = adapter->fw; + const u8 *unirom = fw->data; + size_t product_index_offset; + u32 entries, entry_size, idx; + int ret; - if (!tab_desc) + product_index_offset = adapter->file_prd_off + + (size_t)index_offset * sizeof(__le32); + if (!qlcnic_rom_range_valid(fw->size, product_index_offset, + sizeof(__le32))) return -EINVAL; - tab_size = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * (idx + 1); + idx = get_unaligned_le32(unirom + product_index_offset); + ret = qlcnic_get_table_desc(adapter, section, &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(fw->size, table_offset, entries, + entry_size, sizeof(struct uni_data_desc)) || + idx >= entries) return -EINVAL; - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - descr = (struct uni_data_desc *)&unirom[offs]; - data_size = le32_to_cpu(descr->findex) + le32_to_cpu(descr->size); + desc_offset = table_offset + (size_t)idx * entry_size; + data->offset = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, findex)); + data->size = get_unaligned_le32(unirom + desc_offset + + offsetof(struct uni_data_desc, size)); - if (adapter->fw->size < data_size) + if (!qlcnic_rom_range_valid(fw->size, data->offset, data->size)) return -EINVAL; return 0; } +static int qlcnic_validate_bootld(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_BOOTLD_SIZE ? -EINVAL : 0; +} + +static int qlcnic_validate_fw(struct qlcnic_adapter *adapter) +{ + struct qlcnic_uni_data data; + int ret; + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + + return data.size < QLCNIC_UNI_VERSION_TAIL_SIZE ? -EINVAL : 0; +} + static int qlcnic_validate_product_offs(struct qlcnic_adapter *adapter) { - struct uni_table_desc *ptab_descr; + size_t table_desc_offset, table_offset; const u8 *unirom = adapter->fw->data; int mn_present = qlcnic_has_mn(adapter); - u32 entries, entry_size, tab_size, i; - __le32 temp; - - ptab_descr = qlcnic_get_table_desc(unirom, - QLCNIC_UNI_DIR_SECT_PRODUCT_TBL); - if (!ptab_descr) - return -EINVAL; + u32 entries, entry_size; + size_t i; + int ret; - entries = le32_to_cpu(ptab_descr->num_entries); - entry_size = le32_to_cpu(ptab_descr->entry_size); - tab_size = le32_to_cpu(ptab_descr->findex) + (entries * entry_size); + ret = qlcnic_get_table_desc(adapter, QLCNIC_UNI_DIR_SECT_PRODUCT_TBL, + &table_desc_offset); + if (ret) + return ret; - if (adapter->fw->size < tab_size) + table_offset = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, findex)); + entries = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, num_entries)); + entry_size = get_unaligned_le32(unirom + table_desc_offset + + offsetof(struct uni_table_desc, entry_size)); + if (!qlcnic_rom_table_valid(adapter->fw->size, table_offset, entries, + entry_size, + QLCNIC_UNI_PRODUCT_ENTRY_MIN_SIZE)) return -EINVAL; nomn: for (i = 0; i < entries; i++) { - - u32 flags, file_chiprev, offs; + size_t offset = table_offset + i * entry_size; u8 chiprev = adapter->ahw->revision_id; + u32 flags, file_chiprev; u32 flagbit; - offs = le32_to_cpu(ptab_descr->findex) + - i * le32_to_cpu(ptab_descr->entry_size); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_FLAGS_OFF); - flags = le32_to_cpu(temp); - temp = *((__le32 *)&unirom[offs] + QLCNIC_UNI_CHIP_REV_OFF); - file_chiprev = le32_to_cpu(temp); + flags = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_FLAGS_OFF * sizeof(__le32)); + file_chiprev = get_unaligned_le32(unirom + offset + + QLCNIC_UNI_CHIP_REV_OFF * + sizeof(__le32)); flagbit = mn_present ? 1 : 2; if ((chiprev == file_chiprev) && ((1ULL << flagbit) & flags)) { - adapter->file_prd_off = offs; + if (offset > U32_MAX) + return -EINVAL; + + adapter->file_prd_off = offset; return 0; } } @@ -933,93 +987,81 @@ qlcnic_validate_unified_romimage(struct qlcnic_adapter *adapter) return 0; } -static -struct uni_data_desc *qlcnic_get_data_desc(struct qlcnic_adapter *adapter, - u32 section, u32 idx_offset) -{ - const u8 *unirom = adapter->fw->data; - struct uni_table_desc *tab_desc; - u32 offs, idx; - __le32 temp; - - temp = *((__le32 *)&unirom[adapter->file_prd_off] + idx_offset); - idx = le32_to_cpu(temp); - - tab_desc = qlcnic_get_table_desc(unirom, section); - - if (tab_desc == NULL) - return NULL; - - offs = le32_to_cpu(tab_desc->findex) + - le32_to_cpu(tab_desc->entry_size) * idx; - - return (struct uni_data_desc *)&unirom[offs]; -} - -static u8 * -qlcnic_get_bootld_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_bootld_data(struct qlcnic_adapter *adapter, + const u8 **bootld) { u32 offs = QLCNIC_BOOTLD_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, - QLCNIC_UNI_BOOTLD_IDX_OFF); + struct qlcnic_uni_data data; + int ret; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_BOOTLD, + QLCNIC_UNI_BOOTLD_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_BOOTLD_SIZE) + return -EINVAL; + offs = data.offset; + } else if (!qlcnic_rom_range_valid(adapter->fw->size, offs, + QLCNIC_UNI_BOOTLD_SIZE)) { + return -EINVAL; + } - return (u8 *)&adapter->fw->data[offs]; + *bootld = adapter->fw->data + offs; + return 0; } -static u8 * -qlcnic_get_fw_offs(struct qlcnic_adapter *adapter) +static int qlcnic_get_fw_data(struct qlcnic_adapter *adapter, + const u8 **image, u32 *image_size) { u32 offs = QLCNIC_IMAGE_START; - struct uni_data_desc *data_desc; - - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - offs = le32_to_cpu(data_desc->findex); - - return (u8 *)&adapter->fw->data[offs]; -} + struct qlcnic_uni_data data; + int ret; -static u32 qlcnic_get_fw_size(struct qlcnic_adapter *adapter) -{ - struct uni_data_desc *data_desc; - const u8 *unirom = adapter->fw->data; + if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) { + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret) + return ret; + offs = data.offset; + *image_size = data.size; + } else { + if (!qlcnic_rom_range_valid(adapter->fw->size, + QLCNIC_FW_SIZE_OFFSET, + sizeof(__le32))) + return -EINVAL; + *image_size = get_unaligned_le32(adapter->fw->data + + QLCNIC_FW_SIZE_OFFSET); + } - data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); + if (!qlcnic_rom_range_valid(adapter->fw->size, offs, *image_size)) + return -EINVAL; - if (adapter->ahw->fw_type == QLCNIC_UNIFIED_ROMIMAGE) - return le32_to_cpu(data_desc->size); - else - return le32_to_cpu(*(__le32 *)&unirom[QLCNIC_FW_SIZE_OFFSET]); + *image = adapter->fw->data + offs; + return 0; } static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) { - struct uni_data_desc *fw_data_desc; + char ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE + 1]; const struct firmware *fw = adapter->fw; + struct qlcnic_uni_data data; u32 major, minor, sub; - __le32 version_offset; - const u8 *ver_str; int i, ret; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = *(__le32 *)&fw->data[QLCNIC_FW_VERSION_OFFSET]; - return le32_to_cpu(version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_FW_VERSION_OFFSET); + + ret = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, + QLCNIC_UNI_FIRMWARE_IDX_OFF, &data); + if (ret || data.size < QLCNIC_UNI_VERSION_TAIL_SIZE) + return 0; - fw_data_desc = qlcnic_get_data_desc(adapter, QLCNIC_UNI_DIR_SECT_FW, - QLCNIC_UNI_FIRMWARE_IDX_OFF); - ver_str = fw->data + le32_to_cpu(fw_data_desc->findex) + - le32_to_cpu(fw_data_desc->size) - 17; + memcpy(ver_str, fw->data + data.offset + data.size - + QLCNIC_UNI_VERSION_TAIL_SIZE, + QLCNIC_UNI_VERSION_TAIL_SIZE); + ver_str[QLCNIC_UNI_VERSION_TAIL_SIZE] = '\0'; for (i = 0; i < 12; i++) { - if (!strncmp(&ver_str[i], "REV=", 4)) { + if (!strncmp(ver_str + i, "REV=", 4)) { ret = sscanf(&ver_str[i+4], "%u.%u.%u ", &major, &minor, &sub); if (ret != 3) @@ -1034,18 +1076,15 @@ static u32 qlcnic_get_fw_version(struct qlcnic_adapter *adapter) static u32 qlcnic_get_bios_version(struct qlcnic_adapter *adapter) { - const struct firmware *fw = adapter->fw; u32 bios_ver, prd_off = adapter->file_prd_off; - u8 *version_offset; - __le32 temp; + const struct firmware *fw = adapter->fw; - if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) { - version_offset = (u8 *)&fw->data[QLCNIC_BIOS_VERSION_OFFSET]; - return le32_to_cpu(*(__le32 *)version_offset); - } + if (adapter->ahw->fw_type != QLCNIC_UNIFIED_ROMIMAGE) + return get_unaligned_le32(fw->data + QLCNIC_BIOS_VERSION_OFFSET); - temp = *((__le32 *)(&fw->data[prd_off]) + QLCNIC_UNI_BIOS_VERSION_OFF); - bios_ver = le32_to_cpu(temp); + bios_ver = get_unaligned_le32(fw->data + prd_off + + QLCNIC_UNI_BIOS_VERSION_OFF * + sizeof(__le32)); return (bios_ver << 16) + ((bios_ver >> 8) & 0xff00) + (bios_ver >> 24); } @@ -1106,24 +1145,26 @@ static const char *fw_name[] = { int qlcnic_load_firmware(struct qlcnic_adapter *adapter) { - __le64 *ptr64; - u32 i, flashaddr, size; const struct firmware *fw = adapter->fw; struct pci_dev *pdev = adapter->pdev; + const u8 *bootld, *image; + u32 i, flashaddr, image_size; + int ret; dev_info(&pdev->dev, "loading firmware from %s\n", fw_name[adapter->ahw->fw_type]); if (fw) { + u32 words, remainder; u64 data; - size = (QLCNIC_IMAGE_START - QLCNIC_BOOTLD_START) / 8; - - ptr64 = (__le64 *)qlcnic_get_bootld_offs(adapter); + ret = qlcnic_get_bootld_data(adapter, &bootld); + if (ret) + return ret; flashaddr = QLCNIC_BOOTLD_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < QLCNIC_UNI_BOOTLD_SIZE / sizeof(u64); i++) { + data = get_unaligned_le64(bootld + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) return -EIO; @@ -1131,13 +1172,15 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) / 8; - - ptr64 = (__le64 *)qlcnic_get_fw_offs(adapter); + ret = qlcnic_get_fw_data(adapter, &image, &image_size); + if (ret) + return ret; + words = image_size / sizeof(u64); + remainder = image_size % sizeof(u64); flashaddr = QLCNIC_IMAGE_START; - for (i = 0; i < size; i++) { - data = le64_to_cpu(ptr64[i]); + for (i = 0; i < words; i++) { + data = get_unaligned_le64(image + i * sizeof(u64)); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1146,9 +1189,11 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) flashaddr += 8; } - size = qlcnic_get_fw_size(adapter) % 8; - if (size) { - data = le64_to_cpu(ptr64[i]); + if (remainder) { + __le64 tail = 0; + + memcpy(&tail, image + words * sizeof(u64), remainder); + data = le64_to_cpu(tail); if (qlcnic_pci_mem_write_2M(adapter, flashaddr, data)) @@ -1156,10 +1201,9 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) } } else { - u64 data; - u32 hi, lo; - int ret; struct qlcnic_flt_entry bootld_entry; + u32 hi, lo, size; + u64 data; ret = qlcnic_get_flt_entry(adapter, QLCNIC_BOOTLD_REGION, &bootld_entry); @@ -1200,26 +1244,33 @@ qlcnic_load_firmware(struct qlcnic_adapter *adapter) static int qlcnic_validate_firmware(struct qlcnic_adapter *adapter) { - u32 val; - u32 ver, bios, min_size; - struct pci_dev *pdev = adapter->pdev; const struct firmware *fw = adapter->fw; + struct pci_dev *pdev = adapter->pdev; u8 fw_type = adapter->ahw->fw_type; + u32 ver, bios, min_size; + const u8 *data; + u32 data_size; + u32 val; + + if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) + min_size = QLCNIC_UNI_FW_MIN_SIZE; + else + min_size = QLCNIC_FW_MIN_SIZE; + + if (fw->size < min_size) + return -EINVAL; if (fw_type == QLCNIC_UNIFIED_ROMIMAGE) { if (qlcnic_validate_unified_romimage(adapter)) return -EINVAL; - - min_size = QLCNIC_UNI_FW_MIN_SIZE; } else { - val = le32_to_cpu(*(__le32 *)&fw->data[QLCNIC_FW_MAGIC_OFFSET]); + val = get_unaligned_le32(fw->data + QLCNIC_FW_MAGIC_OFFSET); if (val != QLCNIC_BDINFO_MAGIC) return -EINVAL; - - min_size = QLCNIC_FW_MIN_SIZE; } - if (fw->size < min_size) + if (qlcnic_get_bootld_data(adapter, &data) || + qlcnic_get_fw_data(adapter, &data, &data_size)) return -EINVAL; val = qlcnic_get_fw_version(adapter); diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c index 305ae15ae8f39..e6f48dc9fe64d 100644 --- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c +++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map_data.c @@ -394,6 +394,7 @@ struct sk_buff *rmnet_map_deaggregate(struct sk_buff *skb, if (!skbn) return NULL; + skbn->dev = skb->dev; skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); skb_put(skbn, packet_len); memcpy(skbn->data, skb->data, packet_len); diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 5652da8a178c0..9016527e229ac 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -2066,7 +2066,7 @@ static int __maybe_unused cp_suspend(struct device *device) /* Disable Rx and Tx */ cpw16 (IntrMask, 0); - cpw8 (Cmd, cpr8 (Cmd) & (~RxOn | ~TxOn)); + cpw8 (Cmd, cpr8 (Cmd) & ~(RxOn | TxOn)); spin_unlock_irqrestore (&cp->lock, flags); diff --git a/drivers/net/ethernet/realtek/rtase/rtase.h b/drivers/net/ethernet/realtek/rtase/rtase.h index b9209eb6ea735..03b12d83f6e9d 100644 --- a/drivers/net/ethernet/realtek/rtase/rtase.h +++ b/drivers/net/ethernet/realtek/rtase/rtase.h @@ -153,6 +153,10 @@ enum rtase_registers { #define RTASE_FORCE_TXFLOW_EN BIT(10) #define RTASE_RX_CHKSUM BIT(5) + RTASE_GPHY_STD_00 = 0x6024, +#define RTASE_RXFLOW_EN BIT(7) +#define RTASE_TXFLOW_EN BIT(6) + RTASE_Q0_RX_DESC_ADDR0 = 0x00E4, RTASE_Q0_RX_DESC_ADDR4 = 0x00E8, RTASE_Q1_RX_DESC_ADDR0 = 0x4000, @@ -188,6 +192,12 @@ enum rtase_sw_flag_content { RTASE_SWF_MSIX_ENABLED = BIT(2), }; +enum rtase_parse_result { + RTASE_PARSE_OK, + RTASE_PARSE_SKIP, + RTASE_PARSE_DROP, +}; + #define RSVD_MASK 0x3FFFC000 struct rtase_tx_desc { @@ -359,4 +369,6 @@ struct rtase_private { #define RTASE_MSS_MASK GENMASK(28, 18) +#define RTASE_MIN_PAD_LEN 47 + #endif /* RTASE_H */ diff --git a/drivers/net/ethernet/realtek/rtase/rtase_main.c b/drivers/net/ethernet/realtek/rtase/rtase_main.c index 55105d34bc797..e3cd4f7c13804 100644 --- a/drivers/net/ethernet/realtek/rtase/rtase_main.c +++ b/drivers/net/ethernet/realtek/rtase/rtase_main.c @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -976,6 +977,9 @@ static void rtase_hw_config(struct net_device *dev) rtase_hw_set_features(dev, dev->features); /* enable flow control */ + reg_data16 = rtase_r16(tp, RTASE_GPHY_STD_00); + reg_data16 &= ~(RTASE_TXFLOW_EN | RTASE_RXFLOW_EN); + rtase_w16(tp, RTASE_GPHY_STD_00, reg_data16); reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD); reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN); rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16); @@ -1249,6 +1253,199 @@ static u32 rtase_tx_csum(struct sk_buff *skb, const struct net_device *dev) return csum_cmd; } +static enum rtase_parse_result rtase_get_l3_proto(struct sk_buff *skb, + __be16 *proto, + u32 *network_offset) +{ + struct vlan_hdr *vh, _vh; + struct ethhdr *eh, _eh; + u32 offset = ETH_HLEN; + + eh = skb_header_pointer(skb, 0, sizeof(_eh), &_eh); + if (!eh) + return RTASE_PARSE_DROP; + + *proto = eh->h_proto; + + while (eth_type_vlan(*proto)) { + vh = skb_header_pointer(skb, offset, sizeof(_vh), &_vh); + if (!vh) + return RTASE_PARSE_DROP; + + *proto = vh->h_vlan_encapsulated_proto; + offset += VLAN_HLEN; + } + + *network_offset = offset; + + return RTASE_PARSE_OK; +} + +static bool rtase_pad_to_transport_len(struct sk_buff *skb, + u32 transport_offset, + u32 pad_to_len) +{ + u32 trans_data_len; + u32 pad_len; + + trans_data_len = skb->len - transport_offset; + if (trans_data_len >= pad_to_len) + return true; + + if (skb_is_nonlinear(skb)) { + if (skb_linearize(skb)) + return false; + } + + pad_len = pad_to_len - trans_data_len; + if (__skb_put_padto(skb, skb->len + pad_len, false)) + return false; + + return true; +} + +static enum rtase_parse_result rtase_get_transport_offset(struct sk_buff *skb, + u32 *transport_offset, + u8 *transport_proto, + u32 *pad_to_len) +{ + enum rtase_parse_result ret; + struct ipv6hdr *i6h, _i6h; + struct iphdr *ih, _ih; + bool non_first_frag; + __be16 proto; + u32 offset; + u32 no; + + ret = rtase_get_l3_proto(skb, &proto, &no); + if (ret != RTASE_PARSE_OK) + return ret; + + switch (proto) { + case htons(ETH_P_IP): + ih = skb_header_pointer(skb, no, sizeof(_ih), &_ih); + if (!ih) + return RTASE_PARSE_DROP; + + if (ih->ihl < 5) + return RTASE_PARSE_DROP; + + offset = no + ih->ihl * 4; + if (offset > skb->len) + return RTASE_PARSE_DROP; + + non_first_frag = ntohs(ih->frag_off) & IP_OFFSET; + + if (ih->protocol == IPPROTO_TCP) { + if (skb->len - offset < sizeof(struct tcphdr)) { + if (non_first_frag) { + *transport_offset = offset; + *transport_proto = IPPROTO_TCP; + *pad_to_len = sizeof(struct tcphdr); + + return RTASE_PARSE_OK; + } + + return RTASE_PARSE_DROP; + } + + return RTASE_PARSE_SKIP; + } + + if (ih->protocol != IPPROTO_UDP) + return RTASE_PARSE_SKIP; + + *transport_offset = offset; + *transport_proto = IPPROTO_UDP; + + if (skb->len - offset < sizeof(struct udphdr)) { + if (non_first_frag) { + *pad_to_len = sizeof(struct udphdr); + + return RTASE_PARSE_OK; + } + + return RTASE_PARSE_DROP; + } + + return RTASE_PARSE_OK; + + case htons(ETH_P_IPV6): + i6h = skb_header_pointer(skb, no, sizeof(_i6h), &_i6h); + if (!i6h) + return RTASE_PARSE_DROP; + + offset = no + sizeof(*i6h); + + if (i6h->nexthdr == IPPROTO_TCP) { + if (skb->len - offset < sizeof(struct tcphdr)) + return RTASE_PARSE_DROP; + + return RTASE_PARSE_SKIP; + } + + if (i6h->nexthdr != IPPROTO_UDP) + return RTASE_PARSE_SKIP; + + if (skb->len - offset < sizeof(struct udphdr)) + return RTASE_PARSE_DROP; + + *transport_offset = offset; + *transport_proto = IPPROTO_UDP; + + return RTASE_PARSE_OK; + + default: + return RTASE_PARSE_SKIP; + } +} + +static bool rtase_skb_pad(struct sk_buff *skb) +{ + enum rtase_parse_result ret; + u32 transport_offset; + __be16 *dest, _dest; + u32 trans_data_len; + u32 pad_to_len = 0; + u8 transport_proto; + u16 dest_port; + + ret = rtase_get_transport_offset(skb, &transport_offset, + &transport_proto, &pad_to_len); + if (ret == RTASE_PARSE_SKIP) { + return true; + } else if (ret == RTASE_PARSE_DROP) { + netdev_dbg(skb->dev, "drop malformed packet\n"); + return false; + } + + if (pad_to_len && + !rtase_pad_to_transport_len(skb, transport_offset, pad_to_len)) + return false; + + if (transport_proto != IPPROTO_UDP) + return true; + + trans_data_len = skb->len - transport_offset; + if (trans_data_len < offsetof(struct udphdr, len) || + trans_data_len >= RTASE_MIN_PAD_LEN) + return true; + + dest = skb_header_pointer(skb, + transport_offset + + offsetof(struct udphdr, dest), + sizeof(_dest), &_dest); + if (!dest) + return true; + + dest_port = ntohs(*dest); + if (dest_port != PTP_EV_PORT && dest_port != PTP_GEN_PORT) + return true; + + return rtase_pad_to_transport_len(skb, transport_offset, + RTASE_MIN_PAD_LEN); +} + static int rtase_xmit_frags(struct rtase_ring *ring, struct sk_buff *skb, u32 opts1, u32 opts2) { @@ -1362,6 +1559,9 @@ static netdev_tx_t rtase_start_xmit(struct sk_buff *skb, opts2 |= rtase_tx_csum(skb, dev); } + if (!rtase_skb_pad(skb)) + goto err_dma_0; + frags = rtase_xmit_frags(ring, skb, opts1, opts2); if (unlikely(frags < 0)) goto err_dma_0; @@ -1423,6 +1623,9 @@ static netdev_tx_t rtase_start_xmit(struct sk_buff *skb, err_dma_1: ring->skbuff[entry] = NULL; rtase_tx_clear_range(ring, ring->cur_idx + 1, frags); + if (frags) + /* the frags were cleared above, along with the skb */ + return NETDEV_TX_OK; err_dma_0: tp->stats.tx_dropped++; diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h index 7b48060c250b4..330379c4f405e 100644 --- a/drivers/net/ethernet/renesas/ravb.h +++ b/drivers/net/ethernet/renesas/ravb.h @@ -1039,6 +1039,7 @@ struct ravb_ptp_perout { struct ravb_ptp { struct ptp_clock *clock; struct ptp_clock_info info; + int phc_index; u32 default_addend; u32 current_addend; int extts[N_EXT_TS]; @@ -1133,6 +1134,8 @@ struct ravb_private { int msg_enable; int speed; int emac_irq; + int err_irq; + int mgmt_irq; unsigned no_avb_link:1; unsigned avb_link_active_low:1; diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index e2d7ce1a85e84..cd0d65cd699a4 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -1757,7 +1757,7 @@ static int ravb_get_ts_info(struct net_device *ndev, (1 << HWTSTAMP_FILTER_NONE) | (1 << HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | (1 << HWTSTAMP_FILTER_ALL); - info->phc_index = ptp_clock_index(priv->ptp.clock); + info->phc_index = READ_ONCE(priv->ptp.phc_index); } return 0; @@ -2896,11 +2896,13 @@ static int ravb_setup_irqs(struct ravb_private *priv) return error; if (info->err_mgmt_irqs) { - error = ravb_setup_irq(priv, "err_a", "err_a", NULL, ravb_multi_interrupt); + error = ravb_setup_irq(priv, "err_a", "err_a", &priv->err_irq, + ravb_multi_interrupt); if (error) return error; - error = ravb_setup_irq(priv, "mgmt_a", "mgmt_a", NULL, ravb_multi_interrupt); + error = ravb_setup_irq(priv, "mgmt_a", "mgmt_a", &priv->mgmt_irq, + ravb_multi_interrupt); if (error) return error; } @@ -2963,6 +2965,7 @@ static int ravb_probe(struct platform_device *pdev) priv->rstc = rstc; priv->ndev = ndev; priv->pdev = pdev; + priv->ptp.phc_index = -1; priv->num_tx_ring[RAVB_BE] = BE_TX_RING_SIZE; priv->num_rx_ring[RAVB_BE] = BE_RX_RING_SIZE; if (info->nc_queues) { diff --git a/drivers/net/ethernet/renesas/ravb_ptp.c b/drivers/net/ethernet/renesas/ravb_ptp.c index 226c6c0ab945b..43218bc15b151 100644 --- a/drivers/net/ethernet/renesas/ravb_ptp.c +++ b/drivers/net/ethernet/renesas/ravb_ptp.c @@ -289,16 +289,17 @@ static const struct ptp_clock_info ravb_ptp_info = { void ravb_ptp_interrupt(struct net_device *ndev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock = READ_ONCE(priv->ptp.clock); u32 gis = ravb_read(ndev, GIS); gis &= ravb_read(ndev, GIC); - if (gis & GIS_PTCF) { + if ((gis & GIS_PTCF) && clock) { struct ptp_clock_event event; event.type = PTP_CLOCK_EXTTS; event.index = 0; event.timestamp = ravb_read(ndev, GCPT); - ptp_clock_event(priv->ptp.clock, &event); + ptp_clock_event(clock, &event); } if (gis & GIS_PTMF) { struct ravb_ptp_perout *perout = priv->ptp.perout; @@ -315,6 +316,7 @@ void ravb_ptp_interrupt(struct net_device *ndev) void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock; unsigned long flags; priv->ptp.info = ravb_ptp_info; @@ -327,15 +329,45 @@ void ravb_ptp_init(struct net_device *ndev, struct platform_device *pdev) ravb_modify(ndev, GCCR, GCCR_TCSS, GCCR_TCSS_ADJGPTP); spin_unlock_irqrestore(&priv->lock, flags); - priv->ptp.clock = ptp_clock_register(&priv->ptp.info, &pdev->dev); + clock = ptp_clock_register(&priv->ptp.info, &pdev->dev); + if (IS_ERR(clock)) { + netdev_err(ndev, "failed to register PTP clock: %pe\n", clock); + clock = NULL; + } + + WRITE_ONCE(priv->ptp.clock, clock); + if (clock) + WRITE_ONCE(priv->ptp.phc_index, ptp_clock_index(clock)); +} + +static void ravb_ptp_disable(struct net_device *ndev) +{ + ravb_write(ndev, 0, GIC); + ravb_write(ndev, 0, GIS); +} + +static void ravb_ptp_sync_irqs(struct net_device *ndev) +{ + struct ravb_private *priv = netdev_priv(ndev); + + synchronize_irq(ndev->irq); + if (priv->info->err_mgmt_irqs) { + synchronize_irq(priv->err_irq); + synchronize_irq(priv->mgmt_irq); + } } void ravb_ptp_stop(struct net_device *ndev) { struct ravb_private *priv = netdev_priv(ndev); + struct ptp_clock *clock; - ravb_write(ndev, 0, GIC); - ravb_write(ndev, 0, GIS); + WRITE_ONCE(priv->ptp.phc_index, -1); + clock = xchg(&priv->ptp.clock, NULL); + + ravb_ptp_disable(ndev); + ravb_ptp_sync_irqs(ndev); - ptp_clock_unregister(priv->ptp.clock); + if (clock) + ptp_clock_unregister(clock); } diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c index 849c5a6c2af1e..848a0578cea1f 100644 --- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c +++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c @@ -599,14 +599,13 @@ static int init_dma_desc_rings(struct net_device *netd) return 0; -txalloc_err: - while (queue_num--) - free_tx_ring(priv->device, priv->txq[queue_num], tx_rsize); - return ret; - rxalloc_err: while (queue_num--) free_rx_ring(priv->device, priv->rxq[queue_num], rx_rsize); + queue_num = SXGBE_TX_QUEUES; +txalloc_err: + while (queue_num--) + free_tx_ring(priv->device, priv->txq[queue_num], tx_rsize); return ret; } @@ -1081,7 +1080,9 @@ static int sxgbe_open(struct net_device *dev) priv->dma_buf_sz = SXGBE_ALIGN(DMA_BUFFER_SIZE); priv->tx_tc = TC_DEFAULT; priv->rx_tc = TC_DEFAULT; - init_dma_desc_rings(dev); + ret = init_dma_desc_rings(dev); + if (ret) + goto init_phy_error; /* DMA initialization and SW reset */ ret = sxgbe_init_dma_engine(priv); @@ -1190,6 +1191,7 @@ static int sxgbe_open(struct net_device *dev) init_error: free_dma_desc_resources(priv); +init_phy_error: if (dev->phydev) phy_disconnect(dev->phydev); phy_error: diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index acd7719506b61..83929477b058b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -197,9 +197,6 @@ struct stmmac_extra_stats { unsigned long irq_pcs_ane_n; unsigned long irq_pcs_link_n; unsigned long irq_rgmii_n; - unsigned long pcs_link; - unsigned long pcs_duplex; - unsigned long pcs_speed; /* debug register */ unsigned long mtl_tx_status_fifo_full; unsigned long mtl_tx_fifo_not_empty; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 47fda982d6b15..a54bd924032c1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -524,6 +524,32 @@ static int intel_set_reg_access(const struct pmc_serdes_regs *regs, int max_regs return ret; } +/* + * Return true if the SerDes lane rate must change to serve @interface. + * If the current rate cannot be determined, reconfigure as before. + */ +static bool intel_serdes_needs_reconfig(struct stmmac_priv *priv, + struct intel_priv_data *intel_priv, + phy_interface_t interface) +{ + u32 cur_rate, want_rate; + int data; + + if (!intel_priv->mdio_adhoc_addr) + return true; + + data = mdiobus_read(priv->mii, intel_priv->mdio_adhoc_addr, + SERDES_GCR0); + if (data < 0) + return true; + + cur_rate = (data & SERDES_RATE_MASK) >> SERDES_RATE_PCIE_SHIFT; + want_rate = interface == PHY_INTERFACE_MODE_2500BASEX ? + SERDES_RATE_PCIE_GEN2 : SERDES_RATE_PCIE_GEN1; + + return cur_rate != want_rate; +} + static int intel_mac_finish(struct net_device *ndev, void *intel_data, unsigned int mode, @@ -535,6 +561,11 @@ static int intel_mac_finish(struct net_device *ndev, int max_regs = 0; int ret = 0; + if (!intel_serdes_needs_reconfig(priv, intel_priv, interface)) { + priv->plat->phy_interface = interface; + return 0; + } + ret = intel_tsn_lane_is_available(ndev, intel_priv); if (ret < 0) { netdev_info(priv->dev, "No TSN lane available to set the registers.\n"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index ab431bf9b25f2..e6bce3a433a78 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -209,9 +209,8 @@ static void loongson_dwmac_dma_init_channel(struct stmmac_priv *priv, value |= DMA_BUS_MODE_MAXPBL; value |= DMA_BUS_MODE_USP; - value &= ~(DMA_BUS_MODE_PBL_MASK | DMA_BUS_MODE_RPBL_MASK); - value |= (txpbl << DMA_BUS_MODE_PBL_SHIFT); - value |= (rxpbl << DMA_BUS_MODE_RPBL_SHIFT); + value = u32_replace_bits(value, txpbl, DMA_BUS_MODE_PBL_MASK); + value = u32_replace_bits(value, rxpbl, DMA_BUS_MODE_RPBL_MASK); /* Set the Fixed burst mode */ if (dma_cfg->fixed_burst) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index 2ff5db6d41ca0..e40d0d6963037 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -15,8 +16,10 @@ #include #include +#include "dwxgmac2.h" #include "stmmac.h" #include "stmmac_platform.h" +#include "stmmac_ptp.h" #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_GMII_MII 0x0 #define SYSMGR_EMACGRP_CTRL_PHYSEL_ENUM_RGMII 0x1 @@ -41,9 +44,17 @@ #define SGMII_ADAPTER_ENABLE 0x0000 #define SGMII_ADAPTER_DISABLE 0x0001 +#define SMTG_MDIO_ADDR 0x15 +#define SMTG_TSC_WORD0 0xC +#define SMTG_TSC_WORD1 0xD +#define SMTG_TSC_WORD2 0xE +#define SMTG_TSC_WORD3 0xF +#define SMTG_TSC_SHIFT 16 + struct socfpga_dwmac; struct socfpga_dwmac_ops { int (*set_phy_mode)(struct socfpga_dwmac *dwmac_priv); + void (*setup_plat_dat)(struct socfpga_dwmac *dwmac_priv); }; struct socfpga_dwmac { @@ -268,6 +279,111 @@ static int socfpga_set_phy_mode_common(int phymode, u32 *val) return 0; } +static void get_smtgtime(struct mii_bus *mii, int smtg_addr, u64 *smtg_time) +{ + u64 ns; + + ns = mdiobus_read(mii, smtg_addr, SMTG_TSC_WORD3); + ns <<= SMTG_TSC_SHIFT; + ns |= mdiobus_read(mii, smtg_addr, SMTG_TSC_WORD2); + ns <<= SMTG_TSC_SHIFT; + ns |= mdiobus_read(mii, smtg_addr, SMTG_TSC_WORD1); + ns <<= SMTG_TSC_SHIFT; + ns |= mdiobus_read(mii, smtg_addr, SMTG_TSC_WORD0); + + *smtg_time = ns; +} + +static int smtg_crosststamp(ktime_t *device, struct system_counterval_t *system, + void *ctx) +{ + struct stmmac_priv *priv = (struct stmmac_priv *)ctx; + u32 num_snapshot, gpio_value, acr_value; + void __iomem *ptpaddr = priv->ptpaddr; + void __iomem *ioaddr = priv->hw->pcsr; + unsigned long flags; + u64 smtg_time = 0; + u64 ptp_time = 0; + int i, ret; + u32 v; + + /* Both internal crosstimestamping and external triggered event + * timestamping cannot be run concurrently. + */ + if (priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN) + return -EBUSY; + + mutex_lock(&priv->aux_ts_lock); + /* Enable Internal snapshot trigger */ + acr_value = readl(ptpaddr + PTP_ACR); + acr_value &= ~PTP_ACR_MASK; + switch (priv->plat->int_snapshot_num) { + case AUX_SNAPSHOT0: + acr_value |= PTP_ACR_ATSEN0; + break; + case AUX_SNAPSHOT1: + acr_value |= PTP_ACR_ATSEN1; + break; + case AUX_SNAPSHOT2: + acr_value |= PTP_ACR_ATSEN2; + break; + case AUX_SNAPSHOT3: + acr_value |= PTP_ACR_ATSEN3; + break; + default: + mutex_unlock(&priv->aux_ts_lock); + return -EINVAL; + } + writel(acr_value, ptpaddr + PTP_ACR); + + /* Clear FIFO */ + acr_value = readl(ptpaddr + PTP_ACR); + acr_value |= PTP_ACR_ATSFC; + writel(acr_value, ptpaddr + PTP_ACR); + /* Release the mutex */ + mutex_unlock(&priv->aux_ts_lock); + + /* Trigger Internal snapshot signal. Create a rising edge by just toggle + * the GPO0 to low and back to high. + */ + gpio_value = readl(ioaddr + XGMAC_GPIO_STATUS); + gpio_value &= ~XGMAC_GPIO_GPO0; + writel(gpio_value, ioaddr + XGMAC_GPIO_STATUS); + gpio_value |= XGMAC_GPIO_GPO0; + writel(gpio_value, ioaddr + XGMAC_GPIO_STATUS); + + /* Poll for time sync operation done */ + ret = readl_poll_timeout(priv->ioaddr + XGMAC_INT_STATUS, v, + (v & XGMAC_INT_TSIS), 100, 10000); + if (ret) { + netdev_err(priv->dev, "%s: Wait for time sync operation timeout\n", + __func__); + return ret; + } + + *system = (struct system_counterval_t) { + .cycles = 0, + .cs_id = CSID_ARM_ARCH_COUNTER, + .use_nsecs = false, + }; + + num_snapshot = FIELD_GET(XGMAC_TIMESTAMP_ATSNS_MASK, + readl(ioaddr + XGMAC_TIMESTAMP_STATUS)); + + /* Repeat until the timestamps are from the FIFO last segment */ + for (i = 0; i < num_snapshot; i++) { + read_lock_irqsave(&priv->ptp_lock, flags); + stmmac_get_ptptime(priv, ptpaddr, &ptp_time); + *device = ns_to_ktime(ptp_time); + read_unlock_irqrestore(&priv->ptp_lock, flags); + } + + get_smtgtime(priv->mii, SMTG_MDIO_ADDR, &smtg_time); + system->cycles = smtg_time; + + return 0; +} + static int socfpga_gen5_set_phy_mode(struct socfpga_dwmac *dwmac) { struct regmap *sys_mgr_base_addr = dwmac->sys_mgr_base_addr; @@ -441,6 +557,40 @@ static int socfpga_dwmac_init(struct platform_device *pdev, void *bsp_priv) return dwmac->ops->set_phy_mode(dwmac); } +static void socfpga_gen5_setup_plat_dat(struct socfpga_dwmac *dwmac) +{ + struct plat_stmmacenet_data *plat_dat = dwmac->plat_dat; + + plat_dat->core_type = DWMAC_CORE_GMAC; + + /* Rx watchdog timer in dwmac is buggy in this hw */ + plat_dat->riwt_off = 1; +} + +static void socfpga_agilex5_setup_plat_dat(struct socfpga_dwmac *dwmac) +{ + struct plat_stmmacenet_data *plat_dat = dwmac->plat_dat; + + plat_dat->core_type = DWMAC_CORE_XGMAC; + + /* Enable TBS */ + switch (plat_dat->tx_queues_to_use) { + case 8: + plat_dat->tx_queues_cfg[7].tbs_en = true; + fallthrough; + case 7: + plat_dat->tx_queues_cfg[6].tbs_en = true; + break; + default: + /* Tx Queues 0 - 5 doesn't support TBS on Agilex5 */ + break; + } + + /* Hw supported cross-timestamp */ + plat_dat->int_snapshot_num = AUX_SNAPSHOT0; + plat_dat->crosststamp = smtg_crosststamp; +} + static int socfpga_dwmac_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat_dat; @@ -497,25 +647,31 @@ static int socfpga_dwmac_probe(struct platform_device *pdev) plat_dat->pcs_init = socfpga_dwmac_pcs_init; plat_dat->pcs_exit = socfpga_dwmac_pcs_exit; plat_dat->select_pcs = socfpga_dwmac_select_pcs; - plat_dat->core_type = DWMAC_CORE_GMAC; - plat_dat->riwt_off = 1; + ops->setup_plat_dat(dwmac); return devm_stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res); } static const struct socfpga_dwmac_ops socfpga_gen5_ops = { .set_phy_mode = socfpga_gen5_set_phy_mode, + .setup_plat_dat = socfpga_gen5_setup_plat_dat, }; static const struct socfpga_dwmac_ops socfpga_gen10_ops = { .set_phy_mode = socfpga_gen10_set_phy_mode, + .setup_plat_dat = socfpga_gen5_setup_plat_dat, +}; + +static const struct socfpga_dwmac_ops socfpga_agilex5_ops = { + .set_phy_mode = socfpga_gen10_set_phy_mode, + .setup_plat_dat = socfpga_agilex5_setup_plat_dat, }; static const struct of_device_id socfpga_dwmac_match[] = { { .compatible = "altr,socfpga-stmmac", .data = &socfpga_gen5_ops }, { .compatible = "altr,socfpga-stmmac-a10-s10", .data = &socfpga_gen10_ops }, - { .compatible = "altr,socfpga-stmmac-agilex5", .data = &socfpga_gen10_ops }, + { .compatible = "altr,socfpga-stmmac-agilex5", .data = &socfpga_agilex5_ops }, { } }; MODULE_DEVICE_TABLE(of, socfpga_dwmac_match); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h index 7ab791c8d355f..eae929955ad78 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h @@ -59,8 +59,7 @@ #define MAC_CORE_INIT (MAC_CONTROL_HBD) /* MAC FLOW CTRL defines */ -#define MAC_FLOW_CTRL_PT_MASK 0xffff0000 /* Pause Time Mask */ -#define MAC_FLOW_CTRL_PT_SHIFT 16 +#define MAC_FLOW_CTRL_PT_MASK GENMASK(31, 16) /* Pause Time Mask */ #define MAC_FLOW_CTRL_PASS 0x00000004 /* Pass Control Frames */ #define MAC_FLOW_CTRL_ENABLE 0x00000002 /* Flow Control Enable */ #define MAC_FLOW_CTRL_PAUSE 0x00000001 /* Flow Control Busy ... */ @@ -76,10 +75,8 @@ /* DMA Bus Mode register defines */ #define DMA_BUS_MODE_DBO 0x00100000 /* Descriptor Byte Ordering */ #define DMA_BUS_MODE_BLE 0x00000080 /* Big Endian/Little Endian */ -#define DMA_BUS_MODE_PBL_MASK 0x00003f00 /* Programmable Burst Len */ -#define DMA_BUS_MODE_PBL_SHIFT 8 -#define DMA_BUS_MODE_DSL_MASK 0x0000007c /* Descriptor Skip Length */ -#define DMA_BUS_MODE_DSL_SHIFT 2 /* (in DWORDS) */ +#define DMA_BUS_MODE_PBL_MASK GENMASK(13, 8) /* Programmable Burst Len */ +#define DMA_BUS_MODE_DSL_MASK GENMASK(6, 2) /* Descriptor Skip Length */ #define DMA_BUS_MODE_BAR_BUS 0x00000002 /* Bar-Bus Arbitration */ #define DMA_BUS_MODE_DEFAULT 0x00000000 diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index 0c011a47d5a3e..12c82235c1d7e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -77,7 +77,6 @@ enum power_event { /* SGMII/RGMII status register */ #define GMAC_RGSMIIIS_LNKMODE BIT(0) #define GMAC_RGSMIIIS_SPEED GENMASK(2, 1) -#define GMAC_RGSMIIIS_SPEED_SHIFT 1 #define GMAC_RGSMIIIS_LNKSTS BIT(3) #define GMAC_RGSMIIIS_JABTO BIT(4) #define GMAC_RGSMIIIS_FALSECARDET BIT(5) @@ -134,8 +133,7 @@ enum inter_frame_gap { #define GMAC_MII_ADDR_WRITE 0x00000002 /* MII Write */ #define GMAC_MII_ADDR_BUSY 0x00000001 /* MII Busy */ /* GMAC FLOW CTRL defines */ -#define GMAC_FLOW_CTRL_PT_MASK 0xffff0000 /* Pause Time Mask */ -#define GMAC_FLOW_CTRL_PT_SHIFT 16 +#define GMAC_FLOW_CTRL_PT_MASK GENMASK(31, 16) /* Pause Time Mask */ #define GMAC_FLOW_CTRL_UP 0x00000008 /* Unicast pause frame enable */ #define GMAC_FLOW_CTRL_RFE 0x00000004 /* Rx Flow Control Enable */ #define GMAC_FLOW_CTRL_TFE 0x00000002 /* Tx Flow Control Enable */ @@ -148,7 +146,6 @@ enum inter_frame_gap { #define GMAC_DEBUG_TWCSTS BIT(22) /* MTL Tx FIFO Write Controller */ /* MTL Tx FIFO Read Controller Status */ #define GMAC_DEBUG_TRCSTS_MASK GENMASK(21, 20) -#define GMAC_DEBUG_TRCSTS_SHIFT 20 #define GMAC_DEBUG_TRCSTS_IDLE 0 #define GMAC_DEBUG_TRCSTS_READ 1 #define GMAC_DEBUG_TRCSTS_TXW 2 @@ -156,7 +153,6 @@ enum inter_frame_gap { #define GMAC_DEBUG_TXPAUSED BIT(19) /* MAC Transmitter in PAUSE */ /* MAC Transmit Frame Controller Status */ #define GMAC_DEBUG_TFCSTS_MASK GENMASK(18, 17) -#define GMAC_DEBUG_TFCSTS_SHIFT 17 #define GMAC_DEBUG_TFCSTS_IDLE 0 #define GMAC_DEBUG_TFCSTS_WAIT 1 #define GMAC_DEBUG_TFCSTS_GEN_PAUSE 2 @@ -164,13 +160,11 @@ enum inter_frame_gap { /* MAC GMII or MII Transmit Protocol Engine Status */ #define GMAC_DEBUG_TPESTS BIT(16) #define GMAC_DEBUG_RXFSTS_MASK GENMASK(9, 8) /* MTL Rx FIFO Fill-level */ -#define GMAC_DEBUG_RXFSTS_SHIFT 8 #define GMAC_DEBUG_RXFSTS_EMPTY 0 #define GMAC_DEBUG_RXFSTS_BT 1 #define GMAC_DEBUG_RXFSTS_AT 2 #define GMAC_DEBUG_RXFSTS_FULL 3 #define GMAC_DEBUG_RRCSTS_MASK GENMASK(6, 5) /* MTL Rx FIFO Read Controller */ -#define GMAC_DEBUG_RRCSTS_SHIFT 5 #define GMAC_DEBUG_RRCSTS_IDLE 0 #define GMAC_DEBUG_RRCSTS_RDATA 1 #define GMAC_DEBUG_RRCSTS_RSTAT 2 @@ -178,7 +172,6 @@ enum inter_frame_gap { #define GMAC_DEBUG_RWCSTS BIT(4) /* MTL Rx FIFO Write Controller Active */ /* MAC Receive Frame Controller FIFO Status */ #define GMAC_DEBUG_RFCFCSTS_MASK GENMASK(2, 1) -#define GMAC_DEBUG_RFCFCSTS_SHIFT 1 /* MAC GMII or MII Receive Protocol Engine Status */ #define GMAC_DEBUG_RPESTS BIT(0) @@ -188,8 +181,7 @@ enum inter_frame_gap { #define DMA_BUS_MODE_DSL_MASK 0x0000007c /* Descriptor Skip Length */ #define DMA_BUS_MODE_DSL_SHIFT 2 /* (in DWORDS) */ /* Programmable burst length (passed thorugh platform)*/ -#define DMA_BUS_MODE_PBL_MASK 0x00003f00 /* Programmable Burst Len */ -#define DMA_BUS_MODE_PBL_SHIFT 8 +#define DMA_BUS_MODE_PBL_MASK GENMASK(13, 8) /* Programmable Burst Len */ #define DMA_BUS_MODE_ATDS 0x00000080 /* Alternate Descriptor Size */ enum rx_tx_priority_ratio { @@ -200,8 +192,7 @@ enum rx_tx_priority_ratio { #define DMA_BUS_MODE_FB 0x00010000 /* Fixed burst */ #define DMA_BUS_MODE_MB 0x04000000 /* Mixed burst */ -#define DMA_BUS_MODE_RPBL_MASK 0x007e0000 /* Rx-Programmable Burst Len */ -#define DMA_BUS_MODE_RPBL_SHIFT 17 +#define DMA_BUS_MODE_RPBL_MASK GENMASK(22, 17) /* Rx-Programmable Burst Len */ #define DMA_BUS_MODE_USP 0x00800000 #define DMA_BUS_MODE_MAXPBL 0x01000000 #define DMA_BUS_MODE_AAL 0x02000000 @@ -321,7 +312,6 @@ enum rtc_control { /* PTP and timestamping registers */ #define GMAC3_X_ATSNS GENMASK(29, 25) -#define GMAC3_X_ATSNS_SHIFT 25 #define GMAC_PTP_TCR_ATSFC BIT(24) #define GMAC_PTP_TCR_ATSEN0 BIT(25) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index fe776ddf68895..8a786bececdad 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -187,7 +187,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, dwmac1000_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses (perfect filtering) */ - if (netdev_uc_count(dev) > perfect_addr_number) + if (netdev_uc_count(dev) + 1 > perfect_addr_number) /* Switch to promiscuous mode if more than unicast * addresses are requested than supported by hardware. */ @@ -240,7 +240,7 @@ static void dwmac1000_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, if (duplex) { pr_debug("\tduplex mode: PAUSE %d\n", pause_time); - flow |= (pause_time << GMAC_FLOW_CTRL_PT_SHIFT); + flow |= FIELD_PREP(GMAC_FLOW_CTRL_PT_MASK, pause_time); } writel(flow, ioaddr + GMAC_FLOW_CTRL); @@ -266,34 +266,8 @@ static void dwmac1000_pmt(struct mac_device_info *hw, unsigned long mode) /* RGMII or SMII interface */ static void dwmac1000_rgsmii(void __iomem *ioaddr, struct stmmac_extra_stats *x) { - u32 status; - - status = readl(ioaddr + GMAC_RGSMIIIS); + readl(ioaddr + GMAC_RGSMIIIS); x->irq_rgmii_n++; - - /* Check the link status */ - if (status & GMAC_RGSMIIIS_LNKSTS) { - int speed_value; - - x->pcs_link = 1; - - speed_value = ((status & GMAC_RGSMIIIS_SPEED) >> - GMAC_RGSMIIIS_SPEED_SHIFT); - if (speed_value == GMAC_RGSMIIIS_SPEED_125) - x->pcs_speed = SPEED_1000; - else if (speed_value == GMAC_RGSMIIIS_SPEED_25) - x->pcs_speed = SPEED_100; - else - x->pcs_speed = SPEED_10; - - x->pcs_duplex = (status & GMAC_RGSMIIIS_LNKMOD_MASK); - - pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed, - x->pcs_duplex ? "Full" : "Half"); - } else { - x->pcs_link = 0; - pr_info("Link is Down\n"); - } } static int dwmac1000_irq_status(struct mac_device_info *hw, @@ -412,8 +386,8 @@ static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & GMAC_DEBUG_TWCSTS) x->mmtl_fifo_ctrl++; if (value & GMAC_DEBUG_TRCSTS_MASK) { - u32 trcsts = (value & GMAC_DEBUG_TRCSTS_MASK) - >> GMAC_DEBUG_TRCSTS_SHIFT; + u32 trcsts = FIELD_GET(GMAC_DEBUG_TRCSTS_MASK, value); + if (trcsts == GMAC_DEBUG_TRCSTS_WRITE) x->mtl_tx_fifo_read_ctrl_write++; else if (trcsts == GMAC_DEBUG_TRCSTS_TXW) @@ -426,8 +400,7 @@ static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & GMAC_DEBUG_TXPAUSED) x->mac_tx_in_pause++; if (value & GMAC_DEBUG_TFCSTS_MASK) { - u32 tfcsts = (value & GMAC_DEBUG_TFCSTS_MASK) - >> GMAC_DEBUG_TFCSTS_SHIFT; + u32 tfcsts = FIELD_GET(GMAC_DEBUG_TFCSTS_MASK, value); if (tfcsts == GMAC_DEBUG_TFCSTS_XFER) x->mac_tx_frame_ctrl_xfer++; @@ -441,8 +414,7 @@ static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & GMAC_DEBUG_TPESTS) x->mac_gmii_tx_proto_engine++; if (value & GMAC_DEBUG_RXFSTS_MASK) { - u32 rxfsts = (value & GMAC_DEBUG_RXFSTS_MASK) - >> GMAC_DEBUG_RRCSTS_SHIFT; + u32 rxfsts = FIELD_GET(GMAC_DEBUG_RXFSTS_MASK, value); if (rxfsts == GMAC_DEBUG_RXFSTS_FULL) x->mtl_rx_fifo_fill_level_full++; @@ -454,8 +426,7 @@ static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr, x->mtl_rx_fifo_fill_level_empty++; } if (value & GMAC_DEBUG_RRCSTS_MASK) { - u32 rrcsts = (value & GMAC_DEBUG_RRCSTS_MASK) >> - GMAC_DEBUG_RRCSTS_SHIFT; + u32 rrcsts = FIELD_GET(GMAC_DEBUG_RRCSTS_MASK, value); if (rrcsts == GMAC_DEBUG_RRCSTS_FLUSH) x->mtl_rx_fifo_read_ctrl_flush++; @@ -469,8 +440,8 @@ static void dwmac1000_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & GMAC_DEBUG_RWCSTS) x->mtl_rx_fifo_ctrl_active++; if (value & GMAC_DEBUG_RFCFCSTS_MASK) - x->mac_rx_frame_ctrl_fifo = (value & GMAC_DEBUG_RFCFCSTS_MASK) - >> GMAC_DEBUG_RFCFCSTS_SHIFT; + x->mac_rx_frame_ctrl_fifo = FIELD_GET(GMAC_DEBUG_RFCFCSTS_MASK, + value); if (value & GMAC_DEBUG_RPESTS) x->mac_gmii_rx_proto_engine++; } @@ -566,7 +537,7 @@ void dwmac1000_timestamp_interrupt(struct stmmac_priv *priv) if (!(priv->plat->flags & STMMAC_FLAG_EXT_SNAPSHOT_EN)) return; - num_snapshot = (ts_status & GMAC3_X_ATSNS) >> GMAC3_X_ATSNS_SHIFT; + num_snapshot = FIELD_GET(GMAC3_X_ATSNS, ts_status); for (i = 0; i < num_snapshot; i++) { read_lock_irqsave(&priv->ptp_lock, flags); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c index 118a22406a2e9..cacbf6d4365c7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_dma.c @@ -29,13 +29,10 @@ static void dwmac1000_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi) if (axi->axi_xit_frm) value |= DMA_AXI_LPI_XIT_FRM; - value &= ~DMA_AXI_WR_OSR_LMT; - value |= (axi->axi_wr_osr_lmt & DMA_AXI_WR_OSR_LMT_MASK) << - DMA_AXI_WR_OSR_LMT_SHIFT; - - value &= ~DMA_AXI_RD_OSR_LMT; - value |= (axi->axi_rd_osr_lmt & DMA_AXI_RD_OSR_LMT_MASK) << - DMA_AXI_RD_OSR_LMT_SHIFT; + value = u32_replace_bits(value, axi->axi_wr_osr_lmt, + DMA_AXI_WR_OSR_LMT); + value = u32_replace_bits(value, axi->axi_rd_osr_lmt, + DMA_AXI_RD_OSR_LMT); /* Depending on the UNDEF bit the Master AXI will perform any burst * length according to the BLEN programmed (by default all BLEN are @@ -88,9 +85,8 @@ static void dwmac1000_dma_init_channel(struct stmmac_priv *priv, if (dma_cfg->pblx8) value |= DMA_BUS_MODE_MAXPBL; value |= DMA_BUS_MODE_USP; - value &= ~(DMA_BUS_MODE_PBL_MASK | DMA_BUS_MODE_RPBL_MASK); - value |= (txpbl << DMA_BUS_MODE_PBL_SHIFT); - value |= (rxpbl << DMA_BUS_MODE_RPBL_SHIFT); + value = u32_replace_bits(value, txpbl, DMA_BUS_MODE_PBL_MASK); + value = u32_replace_bits(value, rxpbl, DMA_BUS_MODE_RPBL_MASK); /* Set the Fixed burst mode */ if (dma_cfg->fixed_burst) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c index 14e847c0e1a91..dbc0c1019ed5e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c @@ -132,7 +132,7 @@ static void dwmac100_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int flow = MAC_FLOW_CTRL_ENABLE; if (duplex) - flow |= (pause_time << MAC_FLOW_CTRL_PT_SHIFT); + flow |= FIELD_PREP(MAC_FLOW_CTRL_PT_MASK, pause_time); writel(flow, ioaddr + MAC_FLOW_CTRL); } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c index 82957db47c991..12b2bf2d739ab 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_dma.c @@ -22,7 +22,8 @@ static void dwmac100_dma_init(void __iomem *ioaddr, struct stmmac_dma_cfg *dma_cfg) { /* Enable Application Access by writing to DMA CSR0 */ - writel(DMA_BUS_MODE_DEFAULT | (dma_cfg->pbl << DMA_BUS_MODE_PBL_SHIFT), + writel(DMA_BUS_MODE_DEFAULT | + FIELD_PREP(DMA_BUS_MODE_PBL_MASK, dma_cfg->pbl), ioaddr + DMA_BUS_MODE); /* Mask interrupts by writing to CSR7 */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h index 3dec1a264cf60..7a6609185fd0c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h @@ -95,7 +95,7 @@ /* MAC Flow Control TX */ #define GMAC_TX_FLOW_CTRL_TFE BIT(1) -#define GMAC_TX_FLOW_CTRL_PT_SHIFT 16 +#define GMAC_TX_FLOW_CTRL_PT_MASK GENMASK(31, 16) /* MAC Interrupt bitmap*/ #define GMAC_INT_RGSMIIS BIT(0) @@ -145,23 +145,19 @@ enum power_event { /* MAC Debug bitmap */ #define GMAC_DEBUG_TFCSTS_MASK GENMASK(18, 17) -#define GMAC_DEBUG_TFCSTS_SHIFT 17 #define GMAC_DEBUG_TFCSTS_IDLE 0 #define GMAC_DEBUG_TFCSTS_WAIT 1 #define GMAC_DEBUG_TFCSTS_GEN_PAUSE 2 #define GMAC_DEBUG_TFCSTS_XFER 3 #define GMAC_DEBUG_TPESTS BIT(16) #define GMAC_DEBUG_RFCFCSTS_MASK GENMASK(2, 1) -#define GMAC_DEBUG_RFCFCSTS_SHIFT 1 #define GMAC_DEBUG_RPESTS BIT(0) /* MAC config */ #define GMAC_CONFIG_ARPEN BIT(31) #define GMAC_CONFIG_SARC GENMASK(30, 28) -#define GMAC_CONFIG_SARC_SHIFT 28 #define GMAC_CONFIG_IPC BIT(27) #define GMAC_CONFIG_IPG GENMASK(26, 24) -#define GMAC_CONFIG_IPG_SHIFT 24 #define GMAC_CONFIG_2K BIT(22) #define GMAC_CONFIG_ACS BIT(20) #define GMAC_CONFIG_BE BIT(18) @@ -169,7 +165,6 @@ enum power_event { #define GMAC_CONFIG_JE BIT(16) #define GMAC_CONFIG_PS BIT(15) #define GMAC_CONFIG_FES BIT(14) -#define GMAC_CONFIG_FES_SHIFT 14 #define GMAC_CONFIG_DM BIT(13) #define GMAC_CONFIG_LM BIT(12) #define GMAC_CONFIG_DCRS BIT(9) @@ -178,11 +173,9 @@ enum power_event { /* MAC extended config */ #define GMAC_CONFIG_EIPG GENMASK(29, 25) -#define GMAC_CONFIG_EIPG_SHIFT 25 #define GMAC_CONFIG_EIPG_EN BIT(24) #define GMAC_CONFIG_HDSMS GENMASK(22, 20) -#define GMAC_CONFIG_HDSMS_SHIFT 20 -#define GMAC_CONFIG_HDSMS_256 (0x2 << GMAC_CONFIG_HDSMS_SHIFT) +#define GMAC_CONFIG_HDSMS_256 FIELD_PREP_CONST(GMAC_CONFIG_HDSMS, 0x2) /* MAC HW features0 bitmap */ #define GMAC_HW_FEAT_SAVLANINS BIT(27) @@ -245,7 +238,6 @@ enum power_event { /* MAC HW ADDR regs */ #define GMAC_HI_DCS GENMASK(18, 16) -#define GMAC_HI_DCS_SHIFT 16 #define GMAC_HI_REG_AE BIT(31) /* L3/L4 Filters regs */ @@ -260,7 +252,6 @@ enum power_event { #define GMAC_L3SAM0 BIT(2) #define GMAC_L3PEN0 BIT(0) #define GMAC_L4DP0 GENMASK(31, 16) -#define GMAC_L4DP0_SHIFT 16 #define GMAC_L4SP0 GENMASK(15, 0) /* MAC Timestamp Status */ @@ -317,39 +308,32 @@ static inline u32 mtl_chanx_base_addr(const struct dwmac4_addrs *addrs, #define MTL_OP_MODE_TSF BIT(1) #define MTL_OP_MODE_TQS_MASK GENMASK(24, 16) -#define MTL_OP_MODE_TQS_SHIFT 16 -#define MTL_OP_MODE_TTC_MASK 0x70 -#define MTL_OP_MODE_TTC_SHIFT 4 - -#define MTL_OP_MODE_TTC_32 0 -#define MTL_OP_MODE_TTC_64 (1 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_96 (2 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_128 (3 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_192 (4 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_256 (5 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_384 (6 << MTL_OP_MODE_TTC_SHIFT) -#define MTL_OP_MODE_TTC_512 (7 << MTL_OP_MODE_TTC_SHIFT) +#define MTL_OP_MODE_TTC_MASK GENMASK(6, 4) +#define MTL_OP_MODE_TTC_32 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 0) +#define MTL_OP_MODE_TTC_64 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 1) +#define MTL_OP_MODE_TTC_96 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 2) +#define MTL_OP_MODE_TTC_128 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 3) +#define MTL_OP_MODE_TTC_192 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 4) +#define MTL_OP_MODE_TTC_256 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 5) +#define MTL_OP_MODE_TTC_384 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 6) +#define MTL_OP_MODE_TTC_512 FIELD_PREP(MTL_OP_MODE_TTC_MASK, 7) #define MTL_OP_MODE_RQS_MASK GENMASK(29, 20) -#define MTL_OP_MODE_RQS_SHIFT 20 #define MTL_OP_MODE_RFD_MASK GENMASK(19, 14) -#define MTL_OP_MODE_RFD_SHIFT 14 #define MTL_OP_MODE_RFA_MASK GENMASK(13, 8) -#define MTL_OP_MODE_RFA_SHIFT 8 #define MTL_OP_MODE_EHFC BIT(7) #define MTL_OP_MODE_DIS_TCP_EF BIT(6) #define MTL_OP_MODE_RTC_MASK GENMASK(1, 0) -#define MTL_OP_MODE_RTC_SHIFT 0 -#define MTL_OP_MODE_RTC_32 (1 << MTL_OP_MODE_RTC_SHIFT) -#define MTL_OP_MODE_RTC_64 0 -#define MTL_OP_MODE_RTC_96 (2 << MTL_OP_MODE_RTC_SHIFT) -#define MTL_OP_MODE_RTC_128 (3 << MTL_OP_MODE_RTC_SHIFT) +#define MTL_OP_MODE_RTC_32 FIELD_PREP(MTL_OP_MODE_RTC_MASK, 1) +#define MTL_OP_MODE_RTC_64 FIELD_PREP(MTL_OP_MODE_RTC_MASK, 0) +#define MTL_OP_MODE_RTC_96 FIELD_PREP(MTL_OP_MODE_RTC_MASK, 2) +#define MTL_OP_MODE_RTC_128 FIELD_PREP(MTL_OP_MODE_RTC_MASK, 3) /* MTL ETS Control register */ #define MTL_ETS_CTRL_BASE_ADDR 0x00000d10 @@ -454,7 +438,6 @@ static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs, /* MTL debug: Tx FIFO Read Controller Status */ #define MTL_DEBUG_TRCSTS_MASK GENMASK(2, 1) -#define MTL_DEBUG_TRCSTS_SHIFT 1 #define MTL_DEBUG_TRCSTS_IDLE 0 #define MTL_DEBUG_TRCSTS_READ 1 #define MTL_DEBUG_TRCSTS_TXW 2 @@ -469,7 +452,6 @@ static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs, #define MTL_DEBUG_RXFSTS_AT 2 #define MTL_DEBUG_RXFSTS_FULL 3 #define MTL_DEBUG_RRCSTS_MASK GENMASK(2, 1) -#define MTL_DEBUG_RRCSTS_SHIFT 1 #define MTL_DEBUG_RRCSTS_IDLE 0 #define MTL_DEBUG_RRCSTS_RDATA 1 #define MTL_DEBUG_RRCSTS_RSTAT 2 @@ -523,7 +505,6 @@ static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs, #define GMAC_PHYIF_CTRLSTATUS_SMIDRXS BIT(4) #define GMAC_PHYIF_CTRLSTATUS_LNKMOD BIT(16) #define GMAC_PHYIF_CTRLSTATUS_SPEED GENMASK(18, 17) -#define GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT 17 #define GMAC_PHYIF_CTRLSTATUS_LNKSTS BIT(19) #define GMAC_PHYIF_CTRLSTATUS_JABTO BIT(20) #define GMAC_PHYIF_CTRLSTATUS_FALSECARDET BIT(21) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index d85bc0bb5c3c0..666aed0906b5a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -515,7 +515,7 @@ static void dwmac4_set_filter(struct mac_device_info *hw, value |= GMAC_PACKET_FILTER_HPF; /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { /* Switch to promiscuous mode if more than 128 addrs * are required */ @@ -572,8 +572,8 @@ static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, flow = GMAC_TX_FLOW_CTRL_TFE; if (duplex) - flow |= - (pause_time << GMAC_TX_FLOW_CTRL_PT_SHIFT); + flow |= FIELD_PREP(GMAC_TX_FLOW_CTRL_PT_MASK, + pause_time); writel(flow, ioaddr + GMAC_QX_TX_FLOW_CTRL(queue)); } @@ -592,34 +592,8 @@ static void dwmac4_ctrl_ane(struct stmmac_priv *priv, bool ane, bool srgmi_ral, /* RGMII or SMII interface */ static void dwmac4_phystatus(void __iomem *ioaddr, struct stmmac_extra_stats *x) { - u32 status; - - status = readl(ioaddr + GMAC_PHYIF_CONTROL_STATUS); + readl(ioaddr + GMAC_PHYIF_CONTROL_STATUS); x->irq_rgmii_n++; - - /* Check the link status */ - if (status & GMAC_PHYIF_CTRLSTATUS_LNKSTS) { - int speed_value; - - x->pcs_link = 1; - - speed_value = ((status & GMAC_PHYIF_CTRLSTATUS_SPEED) >> - GMAC_PHYIF_CTRLSTATUS_SPEED_SHIFT); - if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_125) - x->pcs_speed = SPEED_1000; - else if (speed_value == GMAC_PHYIF_CTRLSTATUS_SPEED_25) - x->pcs_speed = SPEED_100; - else - x->pcs_speed = SPEED_10; - - x->pcs_duplex = (status & GMAC_PHYIF_CTRLSTATUS_LNKMOD); - - pr_info("Link is Up - %d/%s\n", (int)x->pcs_speed, - x->pcs_duplex ? "Full" : "Half"); - } else { - x->pcs_link = 0; - pr_info("Link is Down\n"); - } } static int dwmac4_irq_mtl_status(struct stmmac_priv *priv, @@ -717,8 +691,8 @@ static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & MTL_DEBUG_TWCSTS) x->mmtl_fifo_ctrl++; if (value & MTL_DEBUG_TRCSTS_MASK) { - u32 trcsts = (value & MTL_DEBUG_TRCSTS_MASK) - >> MTL_DEBUG_TRCSTS_SHIFT; + u32 trcsts = FIELD_GET(MTL_DEBUG_TRCSTS_MASK, value); + if (trcsts == MTL_DEBUG_TRCSTS_WRITE) x->mtl_tx_fifo_read_ctrl_write++; else if (trcsts == MTL_DEBUG_TRCSTS_TXW) @@ -749,8 +723,7 @@ static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr, x->mtl_rx_fifo_fill_level_empty++; } if (value & MTL_DEBUG_RRCSTS_MASK) { - u32 rrcsts = (value & MTL_DEBUG_RRCSTS_MASK) >> - MTL_DEBUG_RRCSTS_SHIFT; + u32 rrcsts = FIELD_GET(MTL_DEBUG_RRCSTS_MASK, value); if (rrcsts == MTL_DEBUG_RRCSTS_FLUSH) x->mtl_rx_fifo_read_ctrl_flush++; @@ -769,8 +742,7 @@ static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr, value = readl(ioaddr + GMAC_DEBUG); if (value & GMAC_DEBUG_TFCSTS_MASK) { - u32 tfcsts = (value & GMAC_DEBUG_TFCSTS_MASK) - >> GMAC_DEBUG_TFCSTS_SHIFT; + u32 tfcsts = FIELD_GET(GMAC_DEBUG_TFCSTS_MASK, value); if (tfcsts == GMAC_DEBUG_TFCSTS_XFER) x->mac_tx_frame_ctrl_xfer++; @@ -784,8 +756,8 @@ static void dwmac4_debug(struct stmmac_priv *priv, void __iomem *ioaddr, if (value & GMAC_DEBUG_TPESTS) x->mac_gmii_tx_proto_engine++; if (value & GMAC_DEBUG_RFCFCSTS_MASK) - x->mac_rx_frame_ctrl_fifo = (value & GMAC_DEBUG_RFCFCSTS_MASK) - >> GMAC_DEBUG_RFCFCSTS_SHIFT; + x->mac_rx_frame_ctrl_fifo = FIELD_GET(GMAC_DEBUG_RFCFCSTS_MASK, + value); if (value & GMAC_DEBUG_RPESTS) x->mac_gmii_rx_proto_engine++; } @@ -806,8 +778,7 @@ static void dwmac4_sarc_configure(void __iomem *ioaddr, int val) { u32 value = readl(ioaddr + GMAC_CONFIG); - value &= ~GMAC_CONFIG_SARC; - value |= val << GMAC_CONFIG_SARC_SHIFT; + value = u32_replace_bits(value, val, GMAC_CONFIG_SARC); writel(value, ioaddr + GMAC_CONFIG); } @@ -915,9 +886,9 @@ static int dwmac4_config_l4_filter(struct mac_device_info *hw, u32 filter_no, writel(value, ioaddr + GMAC_L3L4_CTRL(filter_no)); if (sa) { - value = match & GMAC_L4SP0; + value = FIELD_PREP(GMAC_L4SP0, match); } else { - value = (match << GMAC_L4DP0_SHIFT) & GMAC_L4DP0; + value = FIELD_PREP(GMAC_L4DP0, match); } writel(value, ioaddr + GMAC_L4_ADDR(filter_no)); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index d87a8b595e6a3..617e55c3711ed 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -28,13 +28,10 @@ static void dwmac4_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi) if (axi->axi_xit_frm) value |= DMA_AXI_LPI_XIT_FRM; - value &= ~DMA_AXI_WR_OSR_LMT; - value |= (axi->axi_wr_osr_lmt & DMA_AXI_OSR_MAX) << - DMA_AXI_WR_OSR_LMT_SHIFT; - - value &= ~DMA_AXI_RD_OSR_LMT; - value |= (axi->axi_rd_osr_lmt & DMA_AXI_OSR_MAX) << - DMA_AXI_RD_OSR_LMT_SHIFT; + value = u32_replace_bits(value, axi->axi_wr_osr_lmt, + DMA_AXI_WR_OSR_LMT); + value = u32_replace_bits(value, axi->axi_rd_osr_lmt, + DMA_AXI_RD_OSR_LMT); /* Depending on the UNDEF bit the Master AXI will perform any burst * length according to the BLEN programmed (by default all BLEN are @@ -79,7 +76,7 @@ static void dwmac4_dma_init_rx_chan(struct stmmac_priv *priv, u32 rxpbl = dma_cfg->rxpbl ?: dma_cfg->pbl; value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan)); - value = value | (rxpbl << DMA_BUS_MODE_RPBL_SHIFT); + value = value | FIELD_PREP(DMA_CHAN_RX_CTRL_RXPBL_MASK, rxpbl); writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan)); if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && likely(dma_cfg->eame)) @@ -100,7 +97,7 @@ static void dwmac4_dma_init_tx_chan(struct stmmac_priv *priv, u32 txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; value = readl(ioaddr + DMA_CHAN_TX_CONTROL(dwmac4_addrs, chan)); - value = value | (txpbl << DMA_BUS_MODE_PBL_SHIFT); + value = value | FIELD_PREP(DMA_CHAN_TX_CTRL_TXPBL_MASK, txpbl); /* Enable OSP to get best performance */ value |= DMA_CONTROL_OSP; @@ -125,7 +122,7 @@ static void dwmac4_dma_init_channel(struct stmmac_priv *priv, /* common channel control register config */ value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan)); if (dma_cfg->pblx8) - value = value | DMA_BUS_MODE_PBL; + value = value | DMA_CHAN_CTRL_PBLX8; writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan)); /* Mask interrupts by writing to CSR7 */ @@ -143,7 +140,7 @@ static void dwmac410_dma_init_channel(struct stmmac_priv *priv, /* common channel control register config */ value = readl(ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan)); if (dma_cfg->pblx8) - value = value | DMA_BUS_MODE_PBL; + value = value | DMA_CHAN_CTRL_PBLX8; writel(value, ioaddr + DMA_CHAN_CONTROL(dwmac4_addrs, chan)); @@ -175,10 +172,9 @@ static void dwmac4_dma_init(void __iomem *ioaddr, value = readl(ioaddr + DMA_BUS_MODE); - if (dma_cfg->multi_msi_en) { - value &= ~DMA_BUS_MODE_INTM_MASK; - value |= (DMA_BUS_MODE_INTM_MODE1 << DMA_BUS_MODE_INTM_SHIFT); - } + if (dma_cfg->multi_msi_en) + value = u32_replace_bits(value, DMA_BUS_MODE_INTM_MODE1, + DMA_BUS_MODE_INTM_MASK); if (dma_cfg->dche) value |= DMA_BUS_MODE_DCHE; @@ -288,7 +284,7 @@ static void dwmac4_dma_rx_chan_op_mode(struct stmmac_priv *priv, } mtl_rx_op &= ~MTL_OP_MODE_RQS_MASK; - mtl_rx_op |= rqs << MTL_OP_MODE_RQS_SHIFT; + mtl_rx_op |= FIELD_PREP(MTL_OP_MODE_RQS_MASK, rqs); /* Enable flow control only if each channel gets 4 KiB or more FIFO and * only if channel is not an AVB channel. @@ -319,11 +315,10 @@ static void dwmac4_dma_rx_chan_op_mode(struct stmmac_priv *priv, break; } - mtl_rx_op &= ~MTL_OP_MODE_RFD_MASK; - mtl_rx_op |= rfd << MTL_OP_MODE_RFD_SHIFT; - - mtl_rx_op &= ~MTL_OP_MODE_RFA_MASK; - mtl_rx_op |= rfa << MTL_OP_MODE_RFA_SHIFT; + mtl_rx_op = u32_replace_bits(mtl_rx_op, rfd, + MTL_OP_MODE_RFD_MASK); + mtl_rx_op = u32_replace_bits(mtl_rx_op, rfa, + MTL_OP_MODE_RFA_MASK); } writel(mtl_rx_op, ioaddr + MTL_CHAN_RX_OP_MODE(dwmac4_addrs, channel)); @@ -378,8 +373,8 @@ static void dwmac4_dma_tx_chan_op_mode(struct stmmac_priv *priv, mtl_tx_op |= MTL_OP_MODE_TXQEN; else mtl_tx_op |= MTL_OP_MODE_TXQEN_AV; - mtl_tx_op &= ~MTL_OP_MODE_TQS_MASK; - mtl_tx_op |= tqs << MTL_OP_MODE_TQS_SHIFT; + + mtl_tx_op = u32_replace_bits(mtl_tx_op, tqs, MTL_OP_MODE_TQS_MASK); writel(mtl_tx_op, ioaddr + MTL_CHAN_TX_OP_MODE(dwmac4_addrs, channel)); } @@ -520,8 +515,7 @@ static void dwmac4_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr, const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs; u32 value = readl(ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan)); - value &= ~DMA_RBSZ_MASK; - value |= (bfsize << DMA_RBSZ_SHIFT) & DMA_RBSZ_MASK; + value = u32_replace_bits(value, bfsize, DMA_RBSZ_MASK); writel(value, ioaddr + DMA_CHAN_RX_CONTROL(dwmac4_addrs, chan)); } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h index 4f980dcd39582..a019eb07afcf1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h @@ -27,15 +27,11 @@ /* DMA Bus Mode bitmap */ #define DMA_BUS_MODE_DCHE BIT(19) #define DMA_BUS_MODE_INTM_MASK GENMASK(17, 16) -#define DMA_BUS_MODE_INTM_SHIFT 16 #define DMA_BUS_MODE_INTM_MODE1 0x1 #define DMA_BUS_MODE_SFT_RESET BIT(0) /* DMA SYS Bus Mode bitmap */ #define DMA_BUS_MODE_SPH BIT(24) -#define DMA_BUS_MODE_PBL BIT(16) -#define DMA_BUS_MODE_PBL_SHIFT 16 -#define DMA_BUS_MODE_RPBL_SHIFT 16 #define DMA_BUS_MODE_MB BIT(14) #define DMA_BUS_MODE_FB BIT(0) @@ -59,13 +55,7 @@ #define DMA_AXI_EN_LPI BIT(31) #define DMA_AXI_LPI_XIT_FRM BIT(30) #define DMA_AXI_WR_OSR_LMT GENMASK(27, 24) -#define DMA_AXI_WR_OSR_LMT_SHIFT 24 #define DMA_AXI_RD_OSR_LMT GENMASK(19, 16) -#define DMA_AXI_RD_OSR_LMT_SHIFT 16 - -#define DMA_AXI_OSR_MAX 0xf -#define DMA_AXI_MAX_OSR_LIMIT ((DMA_AXI_OSR_MAX << DMA_AXI_WR_OSR_LMT_SHIFT) | \ - (DMA_AXI_OSR_MAX << DMA_AXI_RD_OSR_LMT_SHIFT)) #define DMA_SYS_BUS_MB BIT(14) #define DMA_AXI_1KBBE BIT(13) @@ -134,19 +124,21 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs, #define DMA_CHAN_STATUS(addrs, x) (dma_chanx_base_addr(addrs, x) + 0x60) /* DMA Control X */ +#define DMA_CHAN_CTRL_PBLX8 BIT(16) #define DMA_CONTROL_SPH BIT(24) #define DMA_CONTROL_MSS_MASK GENMASK(13, 0) /* DMA Tx Channel X Control register defines */ #define DMA_CONTROL_EDSE BIT(28) +#define DMA_CHAN_TX_CTRL_TXPBL_MASK GENMASK(21, 16) #define DMA_CONTROL_TSE BIT(12) #define DMA_CONTROL_OSP BIT(4) #define DMA_CONTROL_ST BIT(0) /* DMA Rx Channel X Control register defines */ -#define DMA_CONTROL_SR BIT(0) +#define DMA_CHAN_RX_CTRL_RXPBL_MASK GENMASK(21, 16) #define DMA_RBSZ_MASK GENMASK(14, 1) -#define DMA_RBSZ_SHIFT 1 +#define DMA_CONTROL_SR BIT(0) /* Interrupt status per channel */ #define DMA_CHAN_STATUS_REB GENMASK(21, 19) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c index 57c03d4917744..c098047a3bff8 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c @@ -234,7 +234,7 @@ void stmmac_dwmac4_set_mac_addr(void __iomem *ioaddr, const u8 addr[6], * bit that has no effect on the High Reg 0 where the bit 31 (MO) * is RO. */ - data |= (STMMAC_CHAN0 << GMAC_HI_DCS_SHIFT); + data |= FIELD_PREP(GMAC_HI_DCS, STMMAC_CHAN0); writel(data | GMAC_HI_REG_AE, ioaddr + high); data = (addr[3] << 24) | (addr[2] << 16) | (addr[1] << 8) | addr[0]; writel(data, ioaddr + low); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h index 5d9c18f5bbf58..14cfe0de3327a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_dma.h @@ -59,11 +59,7 @@ static inline u32 dma_chan_base_addr(u32 base, u32 chan) #define DMA_AXI_EN_LPI BIT(31) #define DMA_AXI_LPI_XIT_FRM BIT(30) #define DMA_AXI_WR_OSR_LMT GENMASK(23, 20) -#define DMA_AXI_WR_OSR_LMT_SHIFT 20 -#define DMA_AXI_WR_OSR_LMT_MASK 0xf #define DMA_AXI_RD_OSR_LMT GENMASK(19, 16) -#define DMA_AXI_RD_OSR_LMT_SHIFT 16 -#define DMA_AXI_RD_OSR_LMT_MASK 0xf #define DMA_AXI_OSR_MAX 0xf #define DMA_AXI_MAX_OSR_LIMIT ((DMA_AXI_OSR_MAX << DMA_AXI_WR_OSR_LMT_SHIFT) | \ @@ -132,10 +128,8 @@ static inline u32 dma_chan_base_addr(u32 base, u32 chan) #define DMA_STATUS_EB_MASK 0x00380000 /* Error Bits Mask */ #define DMA_STATUS_EB_TX_ABORT 0x00080000 /* Error Bits - TX Abort */ #define DMA_STATUS_EB_RX_ABORT 0x00100000 /* Error Bits - RX Abort */ -#define DMA_STATUS_TS_MASK 0x00700000 /* Transmit Process State */ -#define DMA_STATUS_TS_SHIFT 20 -#define DMA_STATUS_RS_MASK 0x000e0000 /* Receive Process State */ -#define DMA_STATUS_RS_SHIFT 17 +#define DMA_STATUS_TS_MASK GENMASK(22, 20) /* Transmit Process State */ +#define DMA_STATUS_RS_MASK GENMASK(19, 17) /* Receive Process State */ #define DMA_STATUS_NIS 0x00010000 /* Normal Interrupt Summary */ #define DMA_STATUS_AIS 0x00008000 /* Abnormal Interrupt Summary */ #define DMA_STATUS_ERI 0x00004000 /* Early Receive Interrupt */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c index 467f1a05747ec..2e979f0756650 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac_lib.c @@ -92,10 +92,7 @@ void dwmac_dma_stop_rx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan) #ifdef DWMAC_DMA_DEBUG static void show_tx_process_state(unsigned int status) { - unsigned int state; - state = (status & DMA_STATUS_TS_MASK) >> DMA_STATUS_TS_SHIFT; - - switch (state) { + switch (FIELD_GET(DMA_STATUS_TS_MASK, status)) { case 0: pr_debug("- TX (Stopped): Reset or Stop command\n"); break; @@ -123,10 +120,7 @@ static void show_tx_process_state(unsigned int status) static void show_rx_process_state(unsigned int status) { - unsigned int state; - state = (status & DMA_STATUS_RS_MASK) >> DMA_STATUS_RS_SHIFT; - - switch (state) { + switch (FIELD_GET(DMA_STATUS_RS_MASK, status)) { case 0: pr_debug("- RX (Stopped): Reset or Stop command\n"); break; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h index 0d408ee17f337..301e5e8993e37 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2.h @@ -24,17 +24,15 @@ #define XGMAC_CONFIG_SS_2500 (0x6 << XGMAC_CONFIG_SS_OFF) #define XGMAC_CONFIG_SS_10_MII (0x7 << XGMAC_CONFIG_SS_OFF) #define XGMAC_CONFIG_SARC GENMASK(22, 20) -#define XGMAC_CONFIG_SARC_SHIFT 20 #define XGMAC_CONFIG_JD BIT(16) #define XGMAC_CONFIG_TE BIT(0) #define XGMAC_CORE_INIT_TX (XGMAC_CONFIG_JD) #define XGMAC_RX_CONFIG 0x00000004 #define XGMAC_CONFIG_ARPEN BIT(31) #define XGMAC_CONFIG_GPSL GENMASK(29, 16) -#define XGMAC_CONFIG_GPSL_SHIFT 16 #define XGMAC_CONFIG_HDSMS GENMASK(14, 12) #define XGMAC_CONFIG_HDSMS_SHIFT 12 -#define XGMAC_CONFIG_HDSMS_256 (0x2 << XGMAC_CONFIG_HDSMS_SHIFT) +#define XGMAC_CONFIG_HDSMS_256 FIELD_PREP(XGMAC_CONFIG_HDSMS, 0x2) #define XGMAC_CONFIG_S2KP BIT(11) #define XGMAC_CONFIG_LM BIT(10) #define XGMAC_CONFIG_IPC BIT(9) @@ -44,8 +42,10 @@ #define XGMAC_CONFIG_CST BIT(2) #define XGMAC_CONFIG_ACS BIT(1) #define XGMAC_CONFIG_RE BIT(0) -#define XGMAC_CORE_INIT_RX (XGMAC_CONFIG_GPSLCE | XGMAC_CONFIG_WD | \ - (XGMAC_JUMBO_LEN << XGMAC_CONFIG_GPSL_SHIFT)) +#define XGMAC_CORE_INIT_RX (XGMAC_CONFIG_GPSLCE | \ + XGMAC_CONFIG_WD | \ + FIELD_PREP(XGMAC_CONFIG_GPSL, \ + XGMAC_JUMBO_LEN)) #define XGMAC_PACKET_FILTER 0x00000008 #define XGMAC_FILTER_RA BIT(31) #define XGMAC_FILTER_IPFE BIT(20) @@ -79,6 +79,7 @@ #define XGMAC_PSRQ(x) GENMASK((x) * 8 + 7, (x) * 8) #define XGMAC_PSRQ_SHIFT(x) ((x) * 8) #define XGMAC_INT_STATUS 0x000000b0 +#define XGMAC_INT_TSIS BIT(12) #define XGMAC_LPIIS BIT(5) #define XGMAC_PMTIS BIT(4) #define XGMAC_INT_EN 0x000000b4 @@ -89,7 +90,6 @@ #define XGMAC_INT_DEFAULT_EN (XGMAC_LPIIE | XGMAC_PMTIE) #define XGMAC_Qx_TX_FLOW_CTRL(x) (0x00000070 + (x) * 4) #define XGMAC_PT GENMASK(31, 16) -#define XGMAC_PT_SHIFT 16 #define XGMAC_TFE BIT(1) #define XGMAC_RX_FLOW_CTRL 0x00000090 #define XGMAC_RFE BIT(0) @@ -173,16 +173,17 @@ #define XGMAC_MDIO_ADDR 0x00000200 #define XGMAC_MDIO_DATA 0x00000204 #define XGMAC_MDIO_C22P 0x00000220 +#define XGMAC_GPIO_STATUS 0x0000027c +#define XGMAC_GPIO_GPO0 BIT(16) #define XGMAC_ADDRx_HIGH(x) (0x00000300 + (x) * 0x8) #define XGMAC_ADDR_MAX 32 #define XGMAC_AE BIT(31) #define XGMAC_DCS GENMASK(19, 16) -#define XGMAC_DCS_SHIFT 16 #define XGMAC_ADDRx_LOW(x) (0x00000304 + (x) * 0x8) #define XGMAC_L3L4_ADDR_CTRL 0x00000c00 #define XGMAC_IDDR GENMASK(16, 8) -#define XGMAC_IDDR_SHIFT 8 -#define XGMAC_IDDR_FNUM 4 +#define XGMAC_IDDR_FNUM_MASK GENMASK(7, 4) /* FNUM within IDDR */ +#define XGMAC_IDDR_REG_MASK GENMASK(3, 0) /* REG within IDDR */ #define XGMAC_TT BIT(1) #define XGMAC_XB BIT(0) #define XGMAC_L3L4_DATA 0x00000c04 @@ -201,7 +202,6 @@ #define XGMAC_L3PEN0 BIT(0) #define XGMAC_L4_ADDR 0x1 #define XGMAC_L4DP0 GENMASK(31, 16) -#define XGMAC_L4DP0_SHIFT 16 #define XGMAC_L4SP0 GENMASK(15, 0) #define XGMAC_L3_ADDR0 0x4 #define XGMAC_L3_ADDR1 0x5 @@ -220,6 +220,7 @@ #define XGMAC_OB BIT(0) #define XGMAC_RSS_DATA 0x00000c8c #define XGMAC_TIMESTAMP_STATUS 0x00000d20 +#define XGMAC_TIMESTAMP_ATSNS_MASK GENMASK(29, 25) #define XGMAC_TXTSC BIT(15) #define XGMAC_TXTIMESTAMP_NSEC 0x00000d30 #define XGMAC_TXTSSTSLO GENMASK(30, 0) @@ -285,13 +286,9 @@ #define XGMAC_DPP_DISABLE BIT(0) #define XGMAC_MTL_TXQ_OPMODE(x) (0x00001100 + (0x80 * (x))) #define XGMAC_TQS GENMASK(25, 16) -#define XGMAC_TQS_SHIFT 16 #define XGMAC_Q2TCMAP GENMASK(10, 8) -#define XGMAC_Q2TCMAP_SHIFT 8 #define XGMAC_TTC GENMASK(6, 4) -#define XGMAC_TTC_SHIFT 4 #define XGMAC_TXQEN GENMASK(3, 2) -#define XGMAC_TXQEN_SHIFT 2 #define XGMAC_TSF BIT(1) #define XGMAC_MTL_TCx_ETS_CONTROL(x) (0x00001110 + (0x80 * (x))) #define XGMAC_MTL_TCx_QUANTUM_WEIGHT(x) (0x00001118 + (0x80 * (x))) @@ -305,16 +302,12 @@ #define XGMAC_ETS (0x2 << 0) #define XGMAC_MTL_RXQ_OPMODE(x) (0x00001140 + (0x80 * (x))) #define XGMAC_RQS GENMASK(25, 16) -#define XGMAC_RQS_SHIFT 16 #define XGMAC_EHFC BIT(7) #define XGMAC_RSF BIT(5) #define XGMAC_RTC GENMASK(1, 0) -#define XGMAC_RTC_SHIFT 0 #define XGMAC_MTL_RXQ_FLOW_CONTROL(x) (0x00001150 + (0x80 * (x))) #define XGMAC_RFD GENMASK(31, 17) -#define XGMAC_RFD_SHIFT 17 #define XGMAC_RFA GENMASK(15, 1) -#define XGMAC_RFA_SHIFT 1 #define XGMAC_MTL_QINTEN(x) (0x00001170 + (0x80 * (x))) #define XGMAC_RXOIE BIT(16) #define XGMAC_MTL_QINT_STATUS(x) (0x00001174 + (0x80 * (x))) @@ -328,9 +321,7 @@ #define XGMAC_SWR BIT(0) #define XGMAC_DMA_SYSBUS_MODE 0x00003004 #define XGMAC_WR_OSR_LMT GENMASK(29, 24) -#define XGMAC_WR_OSR_LMT_SHIFT 24 #define XGMAC_RD_OSR_LMT GENMASK(21, 16) -#define XGMAC_RD_OSR_LMT_SHIFT 16 #define XGMAC_EN_LPI BIT(15) #define XGMAC_LPI_XIT_PKT BIT(14) #define XGMAC_AAL BIT(12) @@ -372,15 +363,12 @@ #define XGMAC_DMA_CH_TX_CONTROL(x) (0x00003104 + (0x80 * (x))) #define XGMAC_EDSE BIT(28) #define XGMAC_TxPBL GENMASK(21, 16) -#define XGMAC_TxPBL_SHIFT 16 #define XGMAC_TSE BIT(12) #define XGMAC_OSP BIT(4) #define XGMAC_TXST BIT(0) #define XGMAC_DMA_CH_RX_CONTROL(x) (0x00003108 + (0x80 * (x))) #define XGMAC_RxPBL GENMASK(21, 16) -#define XGMAC_RxPBL_SHIFT 16 #define XGMAC_RBSZ GENMASK(14, 1) -#define XGMAC_RBSZ_SHIFT 1 #define XGMAC_RXST BIT(0) #define XGMAC_DMA_CH_TxDESC_HADDR(x) (0x00003110 + (0x80 * (x))) #define XGMAC_DMA_CH_TxDESC_LADDR(x) (0x00003114 + (0x80 * (x))) @@ -397,9 +385,9 @@ #define XGMAC_RIE BIT(6) #define XGMAC_TBUE BIT(2) #define XGMAC_TIE BIT(0) -#define XGMAC_DMA_INT_DEFAULT_EN (XGMAC_NIE | XGMAC_AIE | XGMAC_RBUE | \ +#define XGMAC_DMA_INT_DEFAULT_EN (XGMAC_NIE | XGMAC_AIE | \ XGMAC_RIE | XGMAC_TIE) -#define XGMAC_DMA_INT_DEFAULT_RX (XGMAC_RBUE | XGMAC_RIE) +#define XGMAC_DMA_INT_DEFAULT_RX (XGMAC_RIE) #define XGMAC_DMA_INT_DEFAULT_TX (XGMAC_TIE) #define XGMAC_DMA_CH_Rx_WATCHDOG(x) (0x0000313c + (0x80 * (x))) #define XGMAC_RWT GENMASK(7, 0) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 00e929bf280ba..42cd0a01a9cdc 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -376,7 +376,7 @@ static void dwxgmac2_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, u32 value = XGMAC_TFE; if (duplex) - value |= pause_time << XGMAC_PT_SHIFT; + value |= FIELD_PREP(XGMAC_PT, pause_time); writel(value, ioaddr + XGMAC_Qx_TX_FLOW_CTRL(i)); } @@ -541,7 +541,7 @@ static void dwxgmac2_set_filter(struct mac_device_info *hw, dwxgmac2_set_mchash(ioaddr, mc_filter, mcbitslog2); /* Handle multiple unicast addresses */ - if (netdev_uc_count(dev) > hw->unicast_filter_entries) { + if (netdev_uc_count(dev) + 1 > hw->unicast_filter_entries) { value |= XGMAC_FILTER_PR; } else { struct netdev_hw_addr *ha; @@ -1233,8 +1233,7 @@ static void dwxgmac2_sarc_configure(void __iomem *ioaddr, int val) { u32 value = readl(ioaddr + XGMAC_TX_CONFIG); - value &= ~XGMAC_CONFIG_SARC; - value |= val << XGMAC_CONFIG_SARC_SHIFT; + value = u32_replace_bits(value, val, XGMAC_CONFIG_SARC); writel(value, ioaddr + XGMAC_TX_CONFIG); } @@ -1254,14 +1253,16 @@ static int dwxgmac2_filter_read(struct mac_device_info *hw, u32 filter_no, u8 reg, u32 *data) { void __iomem *ioaddr = hw->pcsr; - u32 value; + u32 value, iddr; int ret; ret = dwxgmac2_filter_wait(hw); if (ret) return ret; - value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT; + iddr = FIELD_PREP(XGMAC_IDDR_FNUM_MASK, filter_no) | + FIELD_PREP(XGMAC_IDDR_REG_MASK, reg); + value = FIELD_PREP(XGMAC_IDDR, iddr); value |= XGMAC_TT | XGMAC_XB; writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL); @@ -1277,7 +1278,7 @@ static int dwxgmac2_filter_write(struct mac_device_info *hw, u32 filter_no, u8 reg, u32 data) { void __iomem *ioaddr = hw->pcsr; - u32 value; + u32 value, iddr; int ret; ret = dwxgmac2_filter_wait(hw); @@ -1286,7 +1287,9 @@ static int dwxgmac2_filter_write(struct mac_device_info *hw, u32 filter_no, writel(data, ioaddr + XGMAC_L3L4_DATA); - value = ((filter_no << XGMAC_IDDR_FNUM) | reg) << XGMAC_IDDR_SHIFT; + iddr = FIELD_PREP(XGMAC_IDDR_FNUM_MASK, filter_no) | + FIELD_PREP(XGMAC_IDDR_REG_MASK, reg); + value = FIELD_PREP(XGMAC_IDDR, iddr); value |= XGMAC_XB; writel(value, ioaddr + XGMAC_L3L4_ADDR_CTRL); @@ -1378,36 +1381,40 @@ static int dwxgmac2_config_l4_filter(struct mac_device_info *hw, u32 filter_no, value &= ~XGMAC_L4PEN0; } - value &= ~(XGMAC_L4SPM0 | XGMAC_L4SPIM0); - value &= ~(XGMAC_L4DPM0 | XGMAC_L4DPIM0); if (sa) { value |= XGMAC_L4SPM0; if (inv) value |= XGMAC_L4SPIM0; + else + value &= ~XGMAC_L4SPIM0; } else { value |= XGMAC_L4DPM0; if (inv) value |= XGMAC_L4DPIM0; + else + value &= ~XGMAC_L4DPIM0; } ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, value); if (ret) return ret; - if (sa) { - value = match & XGMAC_L4SP0; + ret = dwxgmac2_filter_read(hw, filter_no, XGMAC_L4_ADDR, &value); + if (ret) + return ret; - ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value); - if (ret) - return ret; + if (sa) { + value &= ~XGMAC_L4SP0; + value |= FIELD_PREP(XGMAC_L4SP0, match); } else { - value = (match << XGMAC_L4DP0_SHIFT) & XGMAC_L4DP0; - - ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value); - if (ret) - return ret; + value &= ~XGMAC_L4DP0; + value |= FIELD_PREP(XGMAC_L4DP0, match); } + ret = dwxgmac2_filter_write(hw, filter_no, XGMAC_L4_ADDR, value); + if (ret) + return ret; + if (!en) return dwxgmac2_filter_write(hw, filter_no, XGMAC_L3L4_CTRL, 0); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c index 4d6bb995d8d84..964ce2d733169 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_dma.c @@ -55,8 +55,7 @@ static void dwxgmac2_dma_init_rx_chan(struct stmmac_priv *priv, u32 value; value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan)); - value &= ~XGMAC_RxPBL; - value |= (rxpbl << XGMAC_RxPBL_SHIFT) & XGMAC_RxPBL; + value = u32_replace_bits(value, rxpbl, XGMAC_RxPBL); writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan)); writel(upper_32_bits(phy), ioaddr + XGMAC_DMA_CH_RxDESC_HADDR(chan)); @@ -72,9 +71,7 @@ static void dwxgmac2_dma_init_tx_chan(struct stmmac_priv *priv, u32 value; value = readl(ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan)); - value &= ~XGMAC_TxPBL; - value |= (txpbl << XGMAC_TxPBL_SHIFT) & XGMAC_TxPBL; - value |= XGMAC_OSP; + value = u32_replace_bits(value, txpbl, XGMAC_TxPBL); writel(value, ioaddr + XGMAC_DMA_CH_TX_CONTROL(chan)); writel(upper_32_bits(phy), ioaddr + XGMAC_DMA_CH_TxDESC_HADDR(chan)); @@ -91,13 +88,8 @@ static void dwxgmac2_dma_axi(void __iomem *ioaddr, struct stmmac_axi *axi) if (axi->axi_xit_frm) value |= XGMAC_LPI_XIT_PKT; - value &= ~XGMAC_WR_OSR_LMT; - value |= (axi->axi_wr_osr_lmt << XGMAC_WR_OSR_LMT_SHIFT) & - XGMAC_WR_OSR_LMT; - - value &= ~XGMAC_RD_OSR_LMT; - value |= (axi->axi_rd_osr_lmt << XGMAC_RD_OSR_LMT_SHIFT) & - XGMAC_RD_OSR_LMT; + value = u32_replace_bits(value, axi->axi_wr_osr_lmt, XGMAC_WR_OSR_LMT); + value = u32_replace_bits(value, axi->axi_rd_osr_lmt, XGMAC_RD_OSR_LMT); if (!axi->axi_fb) value |= XGMAC_UNDEF; @@ -148,23 +140,24 @@ static void dwxgmac2_dma_rx_mode(struct stmmac_priv *priv, void __iomem *ioaddr, { u32 value = readl(ioaddr + XGMAC_MTL_RXQ_OPMODE(channel)); unsigned int rqs = fifosz / 256 - 1; + unsigned int rtc; if (mode == SF_DMA_MODE) { value |= XGMAC_RSF; } else { value &= ~XGMAC_RSF; - value &= ~XGMAC_RTC; if (mode <= 64) - value |= 0x0 << XGMAC_RTC_SHIFT; + rtc = 0x0; else if (mode <= 96) - value |= 0x2 << XGMAC_RTC_SHIFT; + rtc = 0x2; else - value |= 0x3 << XGMAC_RTC_SHIFT; + rtc = 0x3; + + value = u32_replace_bits(value, rtc, XGMAC_RTC); } - value &= ~XGMAC_RQS; - value |= (rqs << XGMAC_RQS_SHIFT) & XGMAC_RQS; + value = u32_replace_bits(value, rqs, XGMAC_RQS); if ((fifosz >= 4096) && (qmode != MTL_QUEUE_AVB)) { u32 flow = readl(ioaddr + XGMAC_MTL_RXQ_FLOW_CONTROL(channel)); @@ -193,11 +186,8 @@ static void dwxgmac2_dma_rx_mode(struct stmmac_priv *priv, void __iomem *ioaddr, break; } - flow &= ~XGMAC_RFD; - flow |= rfd << XGMAC_RFD_SHIFT; - - flow &= ~XGMAC_RFA; - flow |= rfa << XGMAC_RFA_SHIFT; + flow = u32_replace_bits(flow, rfd, XGMAC_RFD); + flow = u32_replace_bits(flow, rfa, XGMAC_RFA); writel(flow, ioaddr + XGMAC_MTL_RXQ_FLOW_CONTROL(channel)); } @@ -210,40 +200,41 @@ static void dwxgmac2_dma_tx_mode(struct stmmac_priv *priv, void __iomem *ioaddr, { u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel)); unsigned int tqs = fifosz / 256 - 1; + unsigned int ttc, txqen; if (mode == SF_DMA_MODE) { value |= XGMAC_TSF; } else { value &= ~XGMAC_TSF; - value &= ~XGMAC_TTC; if (mode <= 64) - value |= 0x0 << XGMAC_TTC_SHIFT; + ttc = 0x0; else if (mode <= 96) - value |= 0x2 << XGMAC_TTC_SHIFT; + ttc = 0x2; else if (mode <= 128) - value |= 0x3 << XGMAC_TTC_SHIFT; + ttc = 0x3; else if (mode <= 192) - value |= 0x4 << XGMAC_TTC_SHIFT; + ttc = 0x4; else if (mode <= 256) - value |= 0x5 << XGMAC_TTC_SHIFT; + ttc = 0x5; else if (mode <= 384) - value |= 0x6 << XGMAC_TTC_SHIFT; + ttc = 0x6; else - value |= 0x7 << XGMAC_TTC_SHIFT; + ttc = 0x7; + + value = u32_replace_bits(value, ttc, XGMAC_TTC); } /* Use static TC to Queue mapping */ - value |= (channel << XGMAC_Q2TCMAP_SHIFT) & XGMAC_Q2TCMAP; + value |= FIELD_PREP(XGMAC_Q2TCMAP, channel); - value &= ~XGMAC_TXQEN; if (qmode != MTL_QUEUE_AVB) - value |= 0x2 << XGMAC_TXQEN_SHIFT; + txqen = 0x2; else - value |= 0x1 << XGMAC_TXQEN_SHIFT; + txqen = 0x1; - value &= ~XGMAC_TQS; - value |= (tqs << XGMAC_TQS_SHIFT) & XGMAC_TQS; + value = u32_replace_bits(value, txqen, XGMAC_TXQEN); + value = u32_replace_bits(value, tqs, XGMAC_TQS); writel(value, ioaddr + XGMAC_MTL_TXQ_OPMODE(channel)); } @@ -547,16 +538,17 @@ static void dwxgmac2_qmode(struct stmmac_priv *priv, void __iomem *ioaddr, { u32 value = readl(ioaddr + XGMAC_MTL_TXQ_OPMODE(channel)); u32 flow = readl(ioaddr + XGMAC_RX_FLOW_CTRL); + unsigned int txqen; - value &= ~XGMAC_TXQEN; if (qmode != MTL_QUEUE_AVB) { - value |= 0x2 << XGMAC_TXQEN_SHIFT; + txqen = 0x2; writel(0, ioaddr + XGMAC_MTL_TCx_ETS_CONTROL(channel)); } else { - value |= 0x1 << XGMAC_TXQEN_SHIFT; + txqen = 0x1; writel(flow & (~XGMAC_RFE), ioaddr + XGMAC_RX_FLOW_CTRL); } + value = u32_replace_bits(value, txqen, XGMAC_TXQEN); writel(value, ioaddr + XGMAC_MTL_TXQ_OPMODE(channel)); } @@ -566,8 +558,7 @@ static void dwxgmac2_set_bfsize(struct stmmac_priv *priv, void __iomem *ioaddr, u32 value; value = readl(ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan)); - value &= ~XGMAC_RBSZ; - value |= bfsize << XGMAC_RBSZ_SHIFT; + value = u32_replace_bits(value, bfsize, XGMAC_RBSZ); writel(value, ioaddr + XGMAC_DMA_CH_RX_CONTROL(chan)); } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 1ceedd74e4290..6a6cac88f7ecd 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -892,63 +892,45 @@ static void stmmac_mac_link_up(struct phylink_config *config, old_ctrl = readl(priv->ioaddr + MAC_CTRL_REG); ctrl = old_ctrl & ~priv->hw->link.speed_mask; - if (interface == PHY_INTERFACE_MODE_USXGMII) { - switch (speed) { - case SPEED_10000: - ctrl |= priv->hw->link.xgmii.speed10000; - break; - case SPEED_5000: - ctrl |= priv->hw->link.xgmii.speed5000; - break; - case SPEED_2500: + switch (speed) { + case SPEED_100000: + ctrl |= priv->hw->link.xlgmii.speed100000; + break; + case SPEED_50000: + ctrl |= priv->hw->link.xlgmii.speed50000; + break; + case SPEED_40000: + ctrl |= priv->hw->link.xlgmii.speed40000; + break; + case SPEED_25000: + ctrl |= priv->hw->link.xlgmii.speed25000; + break; + case SPEED_10000: + ctrl |= priv->hw->link.xgmii.speed10000; + break; + case SPEED_5000: + ctrl |= priv->hw->link.xgmii.speed5000; + break; + case SPEED_2500: + if (interface == PHY_INTERFACE_MODE_USXGMII) ctrl |= priv->hw->link.xgmii.speed2500; - break; - default: - return; - } - } else if (interface == PHY_INTERFACE_MODE_XLGMII) { - switch (speed) { - case SPEED_100000: - ctrl |= priv->hw->link.xlgmii.speed100000; - break; - case SPEED_50000: - ctrl |= priv->hw->link.xlgmii.speed50000; - break; - case SPEED_40000: - ctrl |= priv->hw->link.xlgmii.speed40000; - break; - case SPEED_25000: - ctrl |= priv->hw->link.xlgmii.speed25000; - break; - case SPEED_10000: - ctrl |= priv->hw->link.xgmii.speed10000; - break; - case SPEED_2500: - ctrl |= priv->hw->link.speed2500; - break; - case SPEED_1000: - ctrl |= priv->hw->link.speed1000; - break; - default: - return; - } - } else { - switch (speed) { - case SPEED_2500: + else ctrl |= priv->hw->link.speed2500; - break; - case SPEED_1000: - ctrl |= priv->hw->link.speed1000; - break; - case SPEED_100: - ctrl |= priv->hw->link.speed100; - break; - case SPEED_10: - ctrl |= priv->hw->link.speed10; - break; - default: - return; - } + break; + case SPEED_1000: + ctrl |= priv->hw->link.speed1000; + break; + case SPEED_100: + ctrl |= priv->hw->link.speed100; + break; + case SPEED_10: + ctrl |= priv->hw->link.speed10; + break; + default: + netdev_err(priv->dev, + "unsupported speed %s on %s, leaving the MAC disabled\n", + phy_speed_to_str(speed), phy_modes(interface)); + return; } if (priv->plat->fix_mac_speed) @@ -1144,6 +1126,10 @@ static int stmmac_init_phy(struct net_device *dev) struct phy_device *phydev; if (addr < 0) { + /* If a custom PCS is in use, no PHY is needed */ + if (priv->hw->phylink_pcs) + return 0; + netdev_err(priv->dev, "no phy found\n"); return -ENODEV; } @@ -1340,9 +1326,9 @@ static void stmmac_display_rings(struct stmmac_priv *priv, static unsigned int stmmac_rx_offset(struct stmmac_priv *priv) { if (stmmac_xdp_is_enabled(priv)) - return XDP_PACKET_HEADROOM; + return XDP_PACKET_HEADROOM + NET_IP_ALIGN; - return NET_SKB_PAD; + return NET_SKB_PAD + NET_IP_ALIGN; } static int stmmac_set_bfsize(int mtu, int bufsize) @@ -2559,7 +2545,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) tx_desc = tx_q->dma_tx + entry; dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); - meta = xsk_buff_get_metadata(pool, xdp_desc.addr); + meta = xsk_buff_get_metadata(pool, xdp_desc.addr, + xdp_desc.options); xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; @@ -2592,8 +2579,8 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) meta_req.set_ic = &set_ic; meta_req.tbs = tx_q->tbs; meta_req.edesc = &tx_q->dma_entx[entry]; - xsk_tx_metadata_request(meta, &stmmac_xsk_tx_metadata_ops, - &meta_req); + xsk_tx_metadata_request(pool, &meta, + &stmmac_xsk_tx_metadata_ops, &meta_req); if (set_ic) { tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, tx_desc); @@ -3403,6 +3390,17 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) } } +/* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for + * channels on which the TSO feature is enabled." If we have a skb for a + * channel which has TBS enabled, fall back to software GSO. + */ +static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, + unsigned int chan) +{ + /* TSO and TBS cannot co-exist */ + return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); +} + /** * stmmac_hw_setup - setup mac in a usable state. * @dev : pointer to the device structure. @@ -3468,6 +3466,14 @@ static int stmmac_hw_setup(struct net_device *dev) /* Initialize MTL*/ stmmac_mtl_configuration(priv); + /* Apply the RX packet parser table */ + if (priv->tc_entries) { + ret = stmmac_rxp_config(priv, priv->hw->pcsr, priv->tc_entries, + priv->tc_entries_max); + if (ret) + return ret; + } + /* Initialize Safety Features */ stmmac_safety_feat_configuration(priv); @@ -3507,10 +3513,7 @@ static int stmmac_hw_setup(struct net_device *dev) /* Enable TSO */ if (priv->tso) { for (chan = 0; chan < tx_cnt; chan++) { - struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; - - /* TSO and TBS cannot co-exist */ - if (tx_q->tbs & STMMAC_TBS_AVAIL) + if (!stmmac_tso_channel_permitted(priv, chan)) continue; stmmac_enable_tso(priv, priv->ioaddr, 1, chan); @@ -3911,6 +3914,15 @@ static int __stmmac_open(struct net_device *dev, dma_conf->tx_queue[i].tbs = priv->dma_conf.tx_queue[i].tbs; memcpy(&priv->dma_conf, dma_conf, sizeof(*dma_conf)); + /* The PHY is suspended when the interface is reopened without + * disconnecting the PHY, e.g. on MTU change. IEEE 802.3 allows PHYs + * to stop their receive clock while powered down, but the DMA + * software reset in stmmac_hw_setup() requires a running receive + * clock, and phylink_start() below resumes the PHY only after the + * hardware setup. Resume a suspended PHY here first. + */ + phylink_prepare_resume(priv->phylink); + stmmac_reset_queues_param(priv); if (!(priv->plat->flags & STMMAC_FLAG_SERDES_UP_AFTER_PHY_LINKUP) && @@ -4152,6 +4164,52 @@ static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); } +static size_t stmmac_tso_header_size(struct sk_buff *skb) +{ + size_t size; + + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + size = skb_transport_offset(skb) + sizeof(struct udphdr); + else + size = skb_tcp_all_headers(skb); + + return size; +} + +/* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal + * (read format) descriptor that the maximum header length supported for the + * TSO feature is 1023 bytes. + * + * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 + * extension headers aren't similarly limited. + */ +static bool stmmac_tso_valid_packet(struct sk_buff *skb) +{ + size_t header_len = stmmac_tso_header_size(skb); + + return header_len <= 1023; +} + +static int stmmac_tso_get_num_desc(struct stmmac_tx_queue *tx_q, + struct sk_buff *skb, u32 pay_len) +{ + int i, ndesc = 1; + + /* head payload */ + ndesc += DIV_ROUND_UP(pay_len, TSO_MAX_BUFF_SIZE); + /* frag payload */ + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + + ndesc += DIV_ROUND_UP(skb_frag_size(frag), + TSO_MAX_BUFF_SIZE); + } + /* MSS update requires a new descriptor */ + ndesc += !!(skb_shinfo(skb)->gso_size != tx_q->mss); + + return ndesc; +} + /** * stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO) * @skb : the socket buffer @@ -4195,9 +4253,9 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) struct stmmac_priv *priv = netdev_priv(dev); unsigned int first_entry, tx_packets; struct stmmac_txq_stats *txq_stats; + int i, first_tx, nfrags, ndesc; struct stmmac_tx_queue *tx_q; u32 pay_len, mss, queue; - int i, first_tx, nfrags; u8 proto_hdr_len, hdr; dma_addr_t des; bool set_ic; @@ -4223,17 +4281,16 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) first_tx = tx_q->cur_tx; /* Compute header lengths */ - if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { - proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); + proto_hdr_len = stmmac_tso_header_size(skb); + pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ + + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) hdr = sizeof(struct udphdr); - } else { - proto_hdr_len = skb_tcp_all_headers(skb); + else hdr = tcp_hdrlen(skb); - } - /* Desc availability based on threshold should be enough safe */ - if (unlikely(stmmac_tx_avail(priv, queue) < - (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) { + ndesc = stmmac_tso_get_num_desc(tx_q, skb, pay_len); + if (unlikely(stmmac_tx_avail(priv, queue) < ndesc)) { if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) { netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); @@ -4245,8 +4302,6 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } - pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */ - mss = skb_shinfo(skb)->gso_size; /* set new MSS value if needed */ @@ -4724,6 +4779,22 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; } +static netdev_features_t stmmac_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + u16 queue; + + if (skb_is_gso(skb)) { + queue = skb_get_queue_mapping(skb); + if (!stmmac_tso_channel_permitted(netdev_priv(dev), queue) || + !stmmac_tso_valid_packet(skb)) + features &= ~NETIF_F_GSO_MASK; + } + + return vlan_features_check(skb, features); +} + static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) { struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); @@ -7093,6 +7164,7 @@ static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 static const struct net_device_ops stmmac_netdev_ops = { .ndo_open = stmmac_open, .ndo_start_xmit = stmmac_xmit, + .ndo_features_check = stmmac_features_check, .ndo_stop = stmmac_release, .ndo_change_mtu = stmmac_change_mtu, .ndo_fix_features = stmmac_fix_features, @@ -7643,6 +7715,7 @@ int stmmac_dvr_probe(struct device *device, stmmac_napi_add(ndev); mutex_init(&priv->lock); + rwlock_init(&priv->ptp_lock); stmmac_fpe_init(priv); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 3e30172fa1294..0e4481127dc09 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -362,7 +362,6 @@ void stmmac_ptp_register(struct stmmac_priv *priv) if (priv->plat->crosststamp) priv->ptp_clock_ops.getcrosststamp = stmmac_getcrosststamp; - rwlock_init(&priv->ptp_lock); mutex_init(&priv->aux_ts_lock); priv->ptp_clock = ptp_clock_register(&priv->ptp_clock_ops, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c index a01bc394d1ac4..3f215ef3c89b7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_selftests.c @@ -415,11 +415,17 @@ static int stmmac_test_mmc(struct stmmac_priv *priv) stmmac_mmc_read(priv, priv->mmcaddr, &final); /* - * The number of MMC counters available depends on HW configuration - * so we just use this one to validate the feature. I hope there is - * not a version without this counter. + * The number of MMC counters available depends on HW configuration, + * and there doesn't seem to be a way to enumerate the implemented + * counters. + * + * Let's check a hand-picked set of counters, knowing that : + * - Starfive JH7110 doesn't implement mmc_tx_framecount_g + * - Amlogic SM1 doesn't implement any mmc_tx_* + * */ - if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g) + if (final.mmc_tx_framecount_g <= initial.mmc_tx_framecount_g && + final.mmc_rx_framecount_gb <= initial.mmc_rx_framecount_gb) return -EINVAL; return 0; @@ -493,6 +499,21 @@ static int stmmac_filter_check(struct stmmac_priv *priv) return -EOPNOTSUPP; } +static int stmmac_uc_filter_check(struct stmmac_priv *priv) +{ + /* For tests involving the UC filter, we need at least one empty + * slot in the UC filter. The UC filters contains netdev_uc_count() + 1 + * entries: The dev->uc list + one entry for the HW address. + * + * Having an empty slot therefore means netdev_uc_count() + 2 entries + * can fit in the filter + */ + if (netdev_uc_count(priv->dev) + 2 > priv->hw->unicast_filter_entries) + return -EOPNOTSUPP; + + return 0; +} + static bool stmmac_hash_check(struct stmmac_priv *priv, unsigned char *addr) { int mc_offset = 32 - priv->hw->mcast_bits_log2; @@ -584,7 +605,7 @@ static int stmmac_test_pfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; while (--tries) { @@ -628,7 +649,7 @@ static int stmmac_test_mcfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -674,7 +695,7 @@ static int stmmac_test_ucfilt(struct stmmac_priv *priv) if (stmmac_filter_check(priv)) return -EOPNOTSUPP; - if (netdev_uc_count(priv->dev) >= priv->hw->unicast_filter_entries) + if (stmmac_uc_filter_check(priv)) return -EOPNOTSUPP; if (netdev_mc_count(priv->dev) >= priv->hw->multicast_filter_bins) return -EOPNOTSUPP; @@ -1452,11 +1473,11 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, struct { struct flow_dissector_key_basic bkey; struct flow_dissector_key_ports key; - } __aligned(BITS_PER_LONG / 8) keys; + } __aligned(BITS_PER_LONG / 8) keys = { }; struct { struct flow_dissector_key_basic bmask; struct flow_dissector_key_ports mask; - } __aligned(BITS_PER_LONG / 8) masks; + } __aligned(BITS_PER_LONG / 8) masks = { }; unsigned long dummy_cookie = 0xdeadbeef; struct stmmac_packet_attrs attr = { }; struct flow_dissector *dissector; @@ -1509,6 +1530,8 @@ static int __stmmac_test_l4filt(struct stmmac_priv *priv, u32 dst, u32 src, keys.bkey.ip_proto = udp ? IPPROTO_UDP : IPPROTO_TCP; keys.key.src = htons(src); keys.key.dst = htons(dst); + /* Match the full IP proto field */ + masks.bmask.ip_proto = 0xff; masks.mask.src = src_mask; masks.mask.dst = dst_mask; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c index 3b4d4696afe96..0119b4b89fc0a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_tc.c @@ -446,6 +446,7 @@ static int tc_parse_flow_actions(struct stmmac_priv *priv, } #define ETHER_TYPE_FULL_MASK cpu_to_be16(~0) +#define IP_PROTO_FULL_MASK 0xFF static int tc_add_basic_flow(struct stmmac_priv *priv, struct flow_cls_offload *cls, @@ -461,6 +462,37 @@ static int tc_add_basic_flow(struct stmmac_priv *priv, flow_rule_match_basic(rule, &match); + /* Both network proto and transport proto not present in the key */ + if (!match.mask || !(match.mask->n_proto || match.mask->ip_proto)) { + NL_SET_ERR_MSG_MOD(cls->common.extack, + "filter must specify network or transport protocol"); + return -EOPNOTSUPP; + } + + /* If the proto is present in the key and is not full mask */ + if ((match.mask->n_proto && match.mask->n_proto != ETHER_TYPE_FULL_MASK) || + (match.mask->ip_proto && match.mask->ip_proto != IP_PROTO_FULL_MASK)) { + NL_SET_ERR_MSG_MOD(cls->common.extack, + "only full protocol mask is supported"); + return -EOPNOTSUPP; + } + + /* Network proto is present in the key and is not IPv4 */ + if (match.mask->n_proto && match.key->n_proto != cpu_to_be16(ETH_P_IP)) { + NL_SET_ERR_MSG_MOD(cls->common.extack, + "only IPv4 network protocol is supported"); + return -EOPNOTSUPP; + } + + /* Transport proto is present in the key and is not TCP or UDP */ + if (match.mask->ip_proto && + match.key->ip_proto != IPPROTO_TCP && + match.key->ip_proto != IPPROTO_UDP) { + NL_SET_ERR_MSG_MOD(cls->common.extack, + "only TCP and UDP transport protocols are supported"); + return -EOPNOTSUPP; + } + entry->ip_proto = match.key->ip_proto; return 0; } @@ -598,6 +630,8 @@ static int tc_add_flow(struct stmmac_priv *priv, ret = tc_flow_parsers[i].fn(priv, cls, entry); if (!ret) entry->in_use = true; + else if (ret == -EOPNOTSUPP) + return ret; } if (!entry->in_use) @@ -627,6 +661,7 @@ static int tc_del_flow(struct stmmac_priv *priv, entry->in_use = false; entry->cookie = 0; entry->is_l4 = false; + entry->action = 0; return ret; } diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 31d436cdceb7c..5bcf1126d627e 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -1312,6 +1312,8 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma); pkt_len = cppi5_hdesc_get_pktlen(desc_rx); cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL); + /* Port ID is contained in the lower 8-bits of the 16-bit Source Tag */ + port_id &= 0xFF; dev_dbg(dev, "%s rx port_id:%d\n", __func__, port_id); port = am65_common_get_port(common, port_id); ndev = port->ndev; diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index c6cf7a0375e08..34c0e651e2da3 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -2048,7 +2048,7 @@ static int cpsw_probe(struct platform_device *pdev) ret = cpsw_register_ports(cpsw); if (ret) - goto clean_unregister_notifiers; + goto clean_unregister_devlink; dev_notice(dev, "initialized (regs %pa, pool size %d) hw_ver:%08X %d.%d (%d)\n", &ss_res->start, descs_pool_size, @@ -2060,6 +2060,8 @@ static int cpsw_probe(struct platform_device *pdev) return 0; +clean_unregister_devlink: + cpsw_unregister_devlink(cpsw); clean_unregister_notifiers: cpsw_unregister_notifiers(cpsw); clean_cpts: diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c index 06f401bd975c7..9aa3964187e1d 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c @@ -51,6 +51,11 @@ static const struct wx_stats wx_gstrings_fdir_stats[] = { WX_STAT("fdir_miss", stats.fdirmiss), }; +static const struct wx_stats wx_gstrings_rsc_stats[] = { + WX_STAT("rsc_aggregated", rsc_count), + WX_STAT("rsc_flushed", rsc_flush), +}; + /* drivers allocates num_tx_queues and num_rx_queues symmetrically so * we set the num_rx_queues to evaluate to num_tx_queues. This is * used because we do not have a good way to get the max number of @@ -64,16 +69,21 @@ static const struct wx_stats wx_gstrings_fdir_stats[] = { (sizeof(struct wx_queue_stats) / sizeof(u64))) #define WX_GLOBAL_STATS_LEN ARRAY_SIZE(wx_gstrings_stats) #define WX_FDIR_STATS_LEN ARRAY_SIZE(wx_gstrings_fdir_stats) +#define WX_RSC_STATS_LEN ARRAY_SIZE(wx_gstrings_rsc_stats) #define WX_STATS_LEN (WX_GLOBAL_STATS_LEN + WX_QUEUE_STATS_LEN) int wx_get_sset_count(struct net_device *netdev, int sset) { struct wx *wx = netdev_priv(netdev); + int len = WX_STATS_LEN; switch (sset) { case ETH_SS_STATS: - return (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)) ? - WX_STATS_LEN + WX_FDIR_STATS_LEN : WX_STATS_LEN; + if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)) + len += WX_FDIR_STATS_LEN; + if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) + len += WX_RSC_STATS_LEN; + return len; default: return -EOPNOTSUPP; } @@ -94,6 +104,10 @@ void wx_get_strings(struct net_device *netdev, u32 stringset, u8 *data) for (i = 0; i < WX_FDIR_STATS_LEN; i++) ethtool_puts(&p, wx_gstrings_fdir_stats[i].stat_string); } + if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) { + for (i = 0; i < WX_RSC_STATS_LEN; i++) + ethtool_puts(&p, wx_gstrings_rsc_stats[i].stat_string); + } for (i = 0; i < netdev->num_tx_queues; i++) { ethtool_sprintf(&p, "tx_queue_%u_packets", i); ethtool_sprintf(&p, "tx_queue_%u_bytes", i); @@ -131,6 +145,13 @@ void wx_get_ethtool_stats(struct net_device *netdev, } } + if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) { + for (k = 0; k < WX_RSC_STATS_LEN; k++) { + p = (char *)wx + wx_gstrings_rsc_stats[k].stat_offset; + data[i++] = *(u64 *)p; + } + } + for (j = 0; j < netdev->num_tx_queues; j++) { ring = wx->tx_ring[j]; if (!ring) { @@ -322,6 +343,40 @@ int wx_get_coalesce(struct net_device *netdev, } EXPORT_SYMBOL(wx_get_coalesce); +static void wx_update_rsc(struct wx *wx) +{ + struct net_device *netdev = wx->netdev; + bool need_reset = false; + + /* nothing to do if LRO or RSC are not enabled */ + if (!test_bit(WX_FLAG_RSC_CAPABLE, wx->flags) || + !(netdev->features & NETIF_F_LRO)) + return; + + /* check the feature flag value and enable RSC if necessary */ + if (wx->rx_itr_setting == 1 || + wx->rx_itr_setting > WX_MIN_RSC_ITR) { + if (!test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) { + set_bit(WX_FLAG_RSC_ENABLED, wx->flags); + dev_info(&wx->pdev->dev, + "rx-usecs value high enough to re-enable RSC\n"); + + need_reset = true; + } + /* if interrupt rate is too high then disable RSC */ + } else if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) { + clear_bit(WX_FLAG_RSC_ENABLED, wx->flags); + dev_info(&wx->pdev->dev, + "rx-usecs set too low, disabling RSC\n"); + + need_reset = true; + } + + /* reset the device to apply the new RSC setting */ + if (need_reset && wx->do_reset) + wx->do_reset(netdev); +} + int wx_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, @@ -414,6 +469,8 @@ int wx_set_coalesce(struct net_device *netdev, wx_write_eitr(q_vector); } + wx_update_rsc(wx); + return 0; } EXPORT_SYMBOL(wx_set_coalesce); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index be59a10228904..dece90d3469e9 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -1779,7 +1779,9 @@ EXPORT_SYMBOL(wx_set_rx_mode); static void wx_set_rx_buffer_len(struct wx *wx) { struct net_device *netdev = wx->netdev; + struct wx_ring *rx_ring; u32 mhadd, max_frame; + int i; max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; /* adjust max frame to be at least the size of a standard frame */ @@ -1789,6 +1791,19 @@ static void wx_set_rx_buffer_len(struct wx *wx) mhadd = rd32(wx, WX_PSR_MAX_SZ); if (max_frame != mhadd) wr32(wx, WX_PSR_MAX_SZ, max_frame); + + /* + * Setup the HW Rx Head and Tail Descriptor Pointers and + * the Base and Length of the Rx Descriptor Ring + */ + for (i = 0; i < wx->num_rx_queues; i++) { + rx_ring = wx->rx_ring[i]; + rx_ring->rx_buf_len = WX_RXBUFFER_2K; +#if (PAGE_SIZE < 8192) + if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) + rx_ring->rx_buf_len = WX_RXBUFFER_3K; +#endif + } } /** @@ -1865,11 +1880,27 @@ static void wx_configure_srrctl(struct wx *wx, srrctl |= WX_RXBUFFER_256 << WX_PX_RR_CFG_BHDRSIZE_SHIFT; /* configure the packet buffer length */ - srrctl |= WX_RX_BUFSZ >> WX_PX_RR_CFG_BSIZEPKT_SHIFT; + srrctl |= rx_ring->rx_buf_len >> WX_PX_RR_CFG_BSIZEPKT_SHIFT; wr32(wx, WX_PX_RR_CFG(reg_idx), srrctl); } +static void wx_configure_rscctl(struct wx *wx, + struct wx_ring *ring) +{ + u8 reg_idx = ring->reg_idx; + u32 rscctrl; + + if (!test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) + return; + + rscctrl = rd32(wx, WX_PX_RR_CFG(reg_idx)); + rscctrl |= WX_PX_RR_CFG_RSC; + rscctrl |= WX_PX_RR_CFG_MAX_RSCBUF_16; + + wr32(wx, WX_PX_RR_CFG(reg_idx), rscctrl); +} + static void wx_configure_tx_ring(struct wx *wx, struct wx_ring *ring) { @@ -1935,6 +1966,10 @@ static void wx_configure_rx_ring(struct wx *wx, rxdctl |= (ring->count / 128) << WX_PX_RR_CFG_RR_SIZE_SHIFT; rxdctl |= 0x1 << WX_PX_RR_CFG_RR_THER_SHIFT; + + if (test_bit(WX_FLAG_RX_MERGE_ENABLED, wx->flags)) + rxdctl |= WX_PX_RR_CFG_DESC_MERGE; + wr32(wx, WX_PX_RR_CFG(reg_idx), rxdctl); /* reset head and tail pointers */ @@ -1943,6 +1978,7 @@ static void wx_configure_rx_ring(struct wx *wx, ring->tail = wx->hw_addr + WX_PX_RR_WP(reg_idx); wx_configure_srrctl(wx, ring); + wx_configure_rscctl(wx, ring); /* initialize rx_buffer_info */ memset(ring->rx_buffer_info, 0, @@ -2181,7 +2217,9 @@ void wx_configure_rx(struct wx *wx) /* RSC Setup */ psrctl = rd32(wx, WX_PSR_CTL); psrctl |= WX_PSR_CTL_RSC_ACK; /* Disable RSC for ACK packets */ - psrctl |= WX_PSR_CTL_RSC_DIS; + psrctl &= ~WX_PSR_CTL_RSC_DIS; + if (!test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) + psrctl |= WX_PSR_CTL_RSC_DIS; wr32(wx, WX_PSR_CTL, psrctl); } @@ -2190,6 +2228,12 @@ void wx_configure_rx(struct wx *wx) /* set_rx_buffer_len must be called before ring initialization */ wx_set_rx_buffer_len(wx); + if (test_bit(WX_FLAG_RX_MERGE_ENABLED, wx->flags)) { + wr32(wx, WX_RDM_DCACHE_CTL, WX_RDM_DCACHE_CTL_EN); + wr32m(wx, WX_RDM_RSC_CTL, + WX_RDM_RSC_CTL_FREE_CTL | WX_RDM_RSC_CTL_FREE_CNT_DIS, + WX_RDM_RSC_CTL_FREE_CTL); + } /* Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ @@ -2464,8 +2508,11 @@ int wx_sw_init(struct wx *wx) return -ENOMEM; } + spin_lock_init(&wx->hw_stats_lock); + mutex_init(&wx->reset_lock); bitmap_zero(wx->state, WX_STATE_NBITS); bitmap_zero(wx->flags, WX_PF_FLAGS_NBITS); + set_bit(WX_STATE_DOWN, wx->state); wx->misc_irq_domain = false; return 0; @@ -2795,6 +2842,12 @@ void wx_update_stats(struct wx *wx) u64 restart_queue = 0, tx_busy = 0; u32 i; + if (test_bit(WX_STATE_DOWN, wx->state) || + test_bit(WX_STATE_RESETTING, wx->state)) + return; + + spin_lock(&wx->hw_stats_lock); + /* gather some stats to the wx struct that are per queue */ for (i = 0; i < wx->num_rx_queues; i++) { struct wx_ring *rx_ring = wx->rx_ring[i]; @@ -2809,6 +2862,18 @@ void wx_update_stats(struct wx *wx) wx->hw_csum_rx_error = hw_csum_rx_error; wx->hw_csum_rx_good = hw_csum_rx_good; + if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) { + u64 rsc_count = 0; + u64 rsc_flush = 0; + + for (i = 0; i < wx->num_rx_queues; i++) { + rsc_count += wx->rx_ring[i]->rx_stats.rsc_count; + rsc_flush += wx->rx_ring[i]->rx_stats.rsc_flush; + } + wx->rsc_count = rsc_count; + wx->rsc_flush = rsc_flush; + } + for (i = 0; i < wx->num_tx_queues; i++) { struct wx_ring *tx_ring = wx->tx_ring[i]; @@ -2851,6 +2916,8 @@ void wx_update_stats(struct wx *wx) for (i = wx->num_vfs * wx->num_rx_queues_per_pool; i < wx->mac.max_rx_queues; i++) hwstats->qmprc += rd32(wx, WX_PX_MPRC(i)); + + spin_unlock(&wx->hw_stats_lock); } EXPORT_SYMBOL(wx_update_stats); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_lib.c index 678dce480f156..4a6851612368d 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_lib.c @@ -235,7 +235,7 @@ static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring, { unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); #if (PAGE_SIZE < 8192) - unsigned int truesize = WX_RX_BUFSZ; + unsigned int truesize = wx_rx_pg_size(rx_ring) / 2; #else unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); #endif @@ -341,7 +341,7 @@ void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count) /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, bi->page_offset, - WX_RX_BUFSZ, + rx_ring->rx_buf_len, DMA_FROM_DEVICE); rx_desc->read.pkt_addr = @@ -404,6 +404,7 @@ static bool wx_is_non_eop(struct wx_ring *rx_ring, union wx_rx_desc *rx_desc, struct sk_buff *skb) { + struct wx *wx = rx_ring->q_vector->wx; u32 ntc = rx_ring->next_to_clean + 1; /* fetch, update, and store next to clean */ @@ -412,6 +413,24 @@ static bool wx_is_non_eop(struct wx_ring *rx_ring, prefetch(WX_RX_DESC(rx_ring, ntc)); + /* update RSC append count if present */ + if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) { + __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & + cpu_to_le32(WX_RXD_RSCCNT_MASK); + + if (unlikely(rsc_enabled)) { + u32 rsc_cnt = le32_to_cpu(rsc_enabled); + + rsc_cnt >>= WX_RXD_RSCCNT_SHIFT; + WX_CB(skb)->append_cnt += rsc_cnt - 1; + + /* update ntc based on RSC value */ + ntc = le32_to_cpu(rx_desc->wb.upper.status_error); + ntc &= WX_RXD_NEXTP_MASK; + ntc >>= WX_RXD_NEXTP_SHIFT; + } + } + /* if we are the last buffer then there is nothing else to do */ if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))) return false; @@ -582,6 +601,33 @@ static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc, } } +static void wx_set_rsc_gso_size(struct wx_ring *ring, + struct sk_buff *skb) +{ + u16 hdr_len = skb_headlen(skb); + + /* set gso_size to avoid messing up TCP MSS */ + skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len), + WX_CB(skb)->append_cnt); + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; +} + +static void wx_update_rsc_stats(struct wx_ring *rx_ring, + struct sk_buff *skb) +{ + /* if append_cnt is 0 then frame is not RSC */ + if (!WX_CB(skb)->append_cnt) + return; + + rx_ring->rx_stats.rsc_count += WX_CB(skb)->append_cnt; + rx_ring->rx_stats.rsc_flush++; + + wx_set_rsc_gso_size(rx_ring, skb); + + /* gso_size is computed using append_cnt so always clear it last */ + WX_CB(skb)->append_cnt = 0; +} + /** * wx_process_skb_fields - Populate skb header fields from Rx descriptor * @rx_ring: rx descriptor ring packet is being transacted on @@ -598,6 +644,9 @@ static void wx_process_skb_fields(struct wx_ring *rx_ring, { struct wx *wx = netdev_priv(rx_ring->netdev); + if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) + wx_update_rsc_stats(rx_ring, skb); + wx_rx_hash(rx_ring, rx_desc, skb); wx_rx_checksum(rx_ring, rx_desc, skb); @@ -823,7 +872,7 @@ static bool wx_clean_tx_irq(struct wx_q_vector *q_vector, if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) && - netif_running(tx_ring->netdev)) { + !test_bit(WX_STATE_DOWN, wx->state)) { netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index); ++tx_ring->tx_stats.restart_queue; @@ -911,7 +960,7 @@ static int wx_poll(struct napi_struct *napi, int budget) if (likely(napi_complete_done(napi, work_done))) { if (wx->adaptive_itr) wx_update_dim_sample(q_vector); - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx)); } @@ -1549,6 +1598,8 @@ static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb, if (skb_vlan_tag_present(skb)) { tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT; tx_flags |= WX_TX_FLAGS_HW_VLAN; + } else if (eth_type_vlan(skb->protocol)) { + tx_flags |= WX_TX_FLAGS_SW_VLAN; } if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && @@ -2288,6 +2339,8 @@ int wx_init_interrupt_scheme(struct wx *wx) wx_cache_ring_rss(wx); + set_bit(WX_STATE_DOWN, wx->state); + return 0; } EXPORT_SYMBOL(wx_init_interrupt_scheme); @@ -2533,7 +2586,7 @@ static void wx_clean_rx_ring(struct wx_ring *rx_ring) dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - WX_RX_BUFSZ, + rx_ring->rx_buf_len, DMA_FROM_DEVICE); /* free resources associated with mapping */ @@ -2732,13 +2785,14 @@ static int wx_alloc_page_pool(struct wx_ring *rx_ring) struct page_pool_params pp_params = { .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, - .order = 0, - .pool_size = rx_ring->count, + .order = wx_rx_pg_order(rx_ring), + .pool_size = rx_ring->count * rx_ring->rx_buf_len / + wx_rx_pg_size(rx_ring), .nid = dev_to_node(rx_ring->dev), .dev = rx_ring->dev, .dma_dir = DMA_FROM_DEVICE, .offset = 0, - .max_len = PAGE_SIZE, + .max_len = wx_rx_pg_size(rx_ring), }; rx_ring->page_pool = page_pool_create(&pp_params); @@ -3027,8 +3081,25 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER)) wx_set_rx_mode(netdev); + if (test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) { + if (!(features & NETIF_F_LRO)) { + if (test_bit(WX_FLAG_RSC_ENABLED, wx->flags)) + need_reset = true; + clear_bit(WX_FLAG_RSC_ENABLED, wx->flags); + } else if (!(test_bit(WX_FLAG_RSC_ENABLED, wx->flags))) { + if (wx->rx_itr_setting == 1 || + wx->rx_itr_setting > WX_MIN_RSC_ITR) { + set_bit(WX_FLAG_RSC_ENABLED, wx->flags); + need_reset = true; + } else if (changed & NETIF_F_LRO) { + dev_info(&wx->pdev->dev, + "rx-usecs set too low, disable RSC\n"); + } + } + } + if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags))) - return 0; + goto out; /* Check if Flow Director n-tuple support was enabled or disabled. If * the state changed, we need to reset. @@ -3054,6 +3125,7 @@ int wx_set_features(struct net_device *netdev, netdev_features_t features) break; } +out: if (need_reset && wx->do_reset) wx->do_reset(netdev); @@ -3103,6 +3175,14 @@ netdev_features_t wx_fix_features(struct net_device *netdev, } } + /* If Rx checksum is disabled, then RSC/LRO should also be disabled */ + if (!(features & NETIF_F_RXCSUM)) + features &= ~NETIF_F_LRO; + + /* Turn off LRO if not RSC capable */ + if (!test_bit(WX_FLAG_RSC_CAPABLE, wx->flags)) + features &= ~NETIF_F_LRO; + return features; } EXPORT_SYMBOL(wx_fix_features); @@ -3202,7 +3282,8 @@ EXPORT_SYMBOL(wx_set_ring); void wx_service_event_schedule(struct wx *wx) { - if (!test_and_set_bit(WX_STATE_SERVICE_SCHED, wx->state)) + if (!test_bit(WX_STATE_DOWN, wx->state) && + !test_and_set_bit(WX_STATE_SERVICE_SCHED, wx->state)) queue_work(system_power_efficient_wq, &wx->service_task); } EXPORT_SYMBOL(wx_service_event_schedule); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c index 44f3e65052468..1165518d55225 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ptp.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ptp.c @@ -555,13 +555,11 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, { u32 tsync_tx_ctl = WX_TSC_1588_CTL_ENABLED; u32 tsync_rx_ctl = WX_PSR_1588_CTL_ENABLED; - DECLARE_BITMAP(flags, WX_PF_FLAGS_NBITS); u32 tsync_rx_mtrl = PTP_EV_PORT << 16; + bool rx_tstamp = false; bool is_l2 = false; u32 regval; - memcpy(flags, wx->flags, sizeof(wx->flags)); - switch (config->tx_type) { case HWTSTAMP_TX_OFF: tsync_tx_ctl = 0; @@ -576,20 +574,16 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_NONE: tsync_rx_ctl = 0; tsync_rx_mtrl = 0; - clear_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - clear_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_SYNC; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_L4_V1; tsync_rx_mtrl |= WX_PSR_1588_MSG_V1_DELAY_REQ; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); + rx_tstamp = true; break; case HWTSTAMP_FILTER_PTP_V2_EVENT: case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: @@ -602,9 +596,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: tsync_rx_ctl |= WX_PSR_1588_CTL_TYPE_EVENT_V2; is_l2 = true; + rx_tstamp = true; config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; - set_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, flags); - set_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, flags); break; default: /* register PSR_1588_MSG must be set in order to do V1 packets, @@ -643,7 +636,8 @@ static int wx_ptp_set_timestamp_mode(struct wx *wx, WX_WRITE_FLUSH(wx); /* configure adapter flags only when HW is actually configured */ - memcpy(wx->flags, flags, sizeof(wx->flags)); + assign_bit(WX_FLAG_RX_HWTSTAMP_ENABLED, wx->flags, rx_tstamp); + assign_bit(WX_FLAG_RX_HWTSTAMP_IN_REGISTER, wx->flags, rx_tstamp); /* clear TX/RX timestamp state, just to be sure */ wx_ptp_clear_tx_timestamp(wx); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_sriov.c b/drivers/net/ethernet/wangxun/libwx/wx_sriov.c index c6d158cd70da3..6ce5d5936d588 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_sriov.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_sriov.c @@ -122,6 +122,10 @@ static int __wx_enable_sriov(struct wx *wx, u8 num_vfs) WX_CFG_PORT_CTL_NUM_VT_MASK, value); + /* Disable RSC when in SR-IOV mode */ + clear_bit(WX_FLAG_RSC_CAPABLE, wx->flags); + clear_bit(WX_FLAG_RSC_ENABLED, wx->flags); + return ret; } @@ -896,7 +900,7 @@ static void wx_set_vf_link_state(struct wx *wx, int vf, int state) wx->vfinfo[vf].link_state = state; switch (state) { case IFLA_VF_LINK_STATE_AUTO: - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) wx->vfinfo[vf].link_enable = true; else wx->vfinfo[vf].link_enable = false; diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 12bb1e19d88d4..d66ea92aceade 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -83,8 +83,13 @@ /*********************** Receive DMA registers **************************/ #define WX_RDM_VF_RE(_i) (0x12004 + ((_i) * 4)) +#define WX_RDM_RSC_CTL 0x1200C +#define WX_RDM_RSC_CTL_FREE_CNT_DIS BIT(8) +#define WX_RDM_RSC_CTL_FREE_CTL BIT(7) #define WX_RDM_PF_QDE(_i) (0x12080 + ((_i) * 4)) #define WX_RDM_VFRE_CLR(_i) (0x120A0 + ((_i) * 4)) +#define WX_RDM_DCACHE_CTL 0x120A8 +#define WX_RDM_DCACHE_CTL_EN BIT(0) #define WX_RDM_DRP_PKT 0x12500 #define WX_RDM_PKT_CNT 0x12504 #define WX_RDM_BYTE_CNT_LSB 0x12508 @@ -421,6 +426,7 @@ enum WX_MSCA_CMD_value { #define WX_7K_ITR 595 #define WX_12K_ITR 336 #define WX_20K_ITR 200 +#define WX_MIN_RSC_ITR 24 #define WX_SP_MAX_EITR 0x00000FF8U #define WX_AML_MAX_EITR 0x00000FFFU #define WX_EM_MAX_EITR 0x00007FFCU @@ -448,7 +454,10 @@ enum WX_MSCA_CMD_value { /* PX_RR_CFG bit definitions */ #define WX_PX_RR_CFG_VLAN BIT(31) #define WX_PX_RR_CFG_DROP_EN BIT(30) +#define WX_PX_RR_CFG_RSC BIT(29) #define WX_PX_RR_CFG_SPLIT_MODE BIT(26) +#define WX_PX_RR_CFG_MAX_RSCBUF_16 FIELD_PREP(GENMASK(24, 23), 3) +#define WX_PX_RR_CFG_DESC_MERGE BIT(19) #define WX_PX_RR_CFG_RR_THER_SHIFT 16 #define WX_PX_RR_CFG_RR_HDR_SZ GENMASK(15, 12) #define WX_PX_RR_CFG_RR_BUF_SZ GENMASK(11, 8) @@ -545,14 +554,9 @@ enum WX_MSCA_CMD_value { /* Supported Rx Buffer Sizes */ #define WX_RXBUFFER_256 256 /* Used for skb receive header */ #define WX_RXBUFFER_2K 2048 +#define WX_RXBUFFER_3K 3072 #define WX_MAX_RXBUFFER 16384 /* largest size for single descriptor */ -#if MAX_SKB_FRAGS < 8 -#define WX_RX_BUFSZ ALIGN(WX_MAX_RXBUFFER / MAX_SKB_FRAGS, 1024) -#else -#define WX_RX_BUFSZ WX_RXBUFFER_2K -#endif - #define WX_RX_BUFFER_WRITE 16 /* Must be power of 2 */ #define WX_MAX_DATA_PER_TXD BIT(14) @@ -644,6 +648,12 @@ enum wx_l2_ptypes { #define WX_RXD_PKTTYPE(_rxd) \ ((le32_to_cpu((_rxd)->wb.lower.lo_dword.data) >> 9) & 0xFF) + +#define WX_RXD_RSCCNT_MASK GENMASK(20, 17) +#define WX_RXD_RSCCNT_SHIFT 17 +#define WX_RXD_NEXTP_MASK GENMASK(19, 4) +#define WX_RXD_NEXTP_SHIFT 4 + /*********************** Transmit Descriptor Config Masks ****************/ #define WX_TXD_STAT_DD BIT(0) /* Descriptor Done */ #define WX_TXD_DTYP_DATA 0 /* Adv Data Descriptor */ @@ -1030,6 +1040,8 @@ struct wx_rx_queue_stats { u64 csum_good_cnt; u64 csum_err; u64 alloc_rx_buff_failed; + u64 rsc_count; + u64 rsc_flush; }; /* iterator for handling rings in ring container */ @@ -1070,6 +1082,7 @@ struct wx_ring { */ u16 next_to_use; u16 next_to_clean; + u16 rx_buf_len; union { u16 next_to_alloc; struct { @@ -1165,6 +1178,7 @@ struct wx_hw_stats { }; enum wx_state { + WX_STATE_DOWN, WX_STATE_RESETTING, WX_STATE_SWFW_BUSY, WX_STATE_PTP_RUNNING, @@ -1226,6 +1240,7 @@ enum wx_pf_flags { WX_FLAG_FDIR_HASH, WX_FLAG_FDIR_PERFECT, WX_FLAG_RSC_CAPABLE, + WX_FLAG_RSC_ENABLED, WX_FLAG_RX_HWTSTAMP_ENABLED, WX_FLAG_RX_HWTSTAMP_IN_REGISTER, WX_FLAG_PTP_PPS_ENABLED, @@ -1233,6 +1248,7 @@ enum wx_pf_flags { WX_FLAG_NEED_MODULE_RESET, WX_FLAG_NEED_UPDATE_LINK, WX_FLAG_NEED_DO_RESET, + WX_FLAG_RX_MERGE_ENABLED, WX_PF_FLAGS_NBITS /* must be last */ }; @@ -1333,12 +1349,15 @@ struct wx { bool default_up; struct wx_hw_stats stats; + spinlock_t hw_stats_lock; /* spinlock for accessing to hw stats */ u64 tx_busy; u64 non_eop_descs; u64 restart_queue; u64 hw_csum_rx_good; u64 hw_csum_rx_error; u64 alloc_rx_buff_failed; + u64 rsc_count; + u64 rsc_flush; unsigned int num_vfs; struct vf_data_storage *vfinfo; struct vf_macvlans vf_mvs; @@ -1377,10 +1396,11 @@ struct wx { struct timer_list service_timer; struct work_struct service_task; + struct mutex reset_lock; /* mutex for reset */ }; #define WX_INTR_ALL (~0ULL) -#define WX_INTR_Q(i) BIT((i)) +#define WX_INTR_Q(i) BIT_ULL((i)) /* register operations */ #define wr32(a, reg, value) writel((value), ((a)->hw_addr + (reg))) @@ -1455,19 +1475,15 @@ static inline struct wx *phylink_to_wx(struct phylink_config *config) return container_of(config, struct wx, phylink_config); } -static inline int wx_set_state_reset(struct wx *wx) +static inline unsigned int wx_rx_pg_order(struct wx_ring *ring) { - u8 timeout = 50; - - while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) { - timeout--; - if (!timeout) - return -EBUSY; - - usleep_range(1000, 2000); - } - +#if (PAGE_SIZE < 8192) + if (ring->rx_buf_len == WX_RXBUFFER_3K) + return 1; +#endif return 0; } +#define wx_rx_pg_size(_ring) (PAGE_SIZE << wx_rx_pg_order(_ring)) + #endif /* _WX_TYPE_H_ */ diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf.h b/drivers/net/ethernet/wangxun/libwx/wx_vf.h index 3f16de0fa4272..ecb1985923936 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_vf.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_vf.h @@ -74,6 +74,7 @@ #define WX_VXRXDCTL_BUFSZ(f) FIELD_PREP(GENMASK(11, 8), f) #define WX_VXRXDCTL_HDRSZ_MASK GENMASK(15, 12) #define WX_VXRXDCTL_HDRSZ(f) FIELD_PREP(GENMASK(15, 12), f) +#define WX_VXRXDCTL_DESC_MERGE BIT(19) #define WX_VXRXDCTL_RSCMAX_MASK GENMASK(24, 23) #define WX_VXRXDCTL_RSCMAX(f) FIELD_PREP(GENMASK(24, 23), f) #define WX_VXRXDCTL_RSCEN BIT(29) diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c index 5478f2fdfce88..11ee2436b9c94 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c @@ -67,7 +67,7 @@ static irqreturn_t wx_msix_misc_vf(int __always_unused irq, void *data) set_bit(WX_FLAG_NEED_UPDATE_LINK, wx->flags); /* Clear the interrupt */ - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) wr32(wx, WX_VXIMC, wx->eims_other); return IRQ_HANDLED; @@ -277,6 +277,7 @@ static void wxvf_up_complete(struct wx *wx) wx_configure_msix_vf(wx); smp_mb__before_atomic(); + clear_bit(WX_STATE_DOWN, wx->state); wx_napi_enable_all(wx); /* clear any pending interrupts, may auto mask */ @@ -326,6 +327,9 @@ static void wxvf_down(struct wx *wx) { struct net_device *netdev = wx->netdev; + if (test_and_set_bit(WX_STATE_DOWN, wx->state)) + return; + timer_delete_sync(&wx->service_timer); netif_tx_stop_all_queues(netdev); netif_tx_disable(netdev); @@ -339,14 +343,16 @@ static void wxvf_down(struct wx *wx) static void wxvf_reinit_locked(struct wx *wx) { - while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) - usleep_range(1000, 2000); + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); + wxvf_down(wx); wx_free_irq(wx); wx_configure_vf(wx); wx_request_msix_irqs_vf(wx); wxvf_up_complete(wx); clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); } static void wxvf_reset_subtask(struct wx *wx) @@ -357,7 +363,7 @@ static void wxvf_reset_subtask(struct wx *wx) rtnl_lock(); if (test_bit(WX_STATE_RESETTING, wx->state) || - !(netif_running(wx->netdev))) { + test_bit(WX_STATE_DOWN, wx->state)) { rtnl_unlock(); return; } diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf_lib.c b/drivers/net/ethernet/wangxun/libwx/wx_vf_lib.c index a87887b9f8ee3..f54107f3c6d7f 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_vf_lib.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_vf_lib.c @@ -272,6 +272,9 @@ void wx_configure_rx_ring_vf(struct wx *wx, struct wx_ring *ring) rxdctl |= WX_VXRXDCTL_RSCMAX(0); rxdctl |= WX_VXRXDCTL_RSCEN; + if (test_bit(WX_FLAG_RX_MERGE_ENABLED, wx->flags)) + rxdctl |= WX_VXRXDCTL_DESC_MERGE; + wr32(wx, WX_VXRXDCTL(reg_idx), rxdctl); /* pf/vf reuse */ diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c index 662f28bdde8a4..646977521d39a 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c @@ -64,9 +64,8 @@ static int ngbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; - err = wx_set_state_reset(wx); - if (err) - return err; + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) @@ -97,6 +96,7 @@ static int ngbe_set_ringparam(struct net_device *netdev, clear_reset: clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); return err; } diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c index 58488e138beb4..35d7a0c76ca07 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -138,6 +138,26 @@ static int ngbe_sw_init(struct wx *wx) return 0; } +/** + * ngbe_service_task - manages and runs subtasks + * @work: pointer to work_struct containing our data + **/ +static void ngbe_service_task(struct work_struct *work) +{ + struct wx *wx = container_of(work, struct wx, service_task); + + wx_update_stats(wx); + + wx_service_event_complete(wx); +} + +static void ngbe_init_service(struct wx *wx) +{ + timer_setup(&wx->service_timer, wx_service_timer, 0); + INIT_WORK(&wx->service_task, ngbe_service_task); + clear_bit(WX_STATE_SERVICE_SCHED, wx->state); +} + /** * ngbe_irq_enable - Enable default interrupt generation settings * @wx: board private structure @@ -160,8 +180,10 @@ static void ngbe_irq_enable(struct wx *wx, bool queues) /* mask interrupt */ if (queues) wx_intr_enable(wx, NGBE_INTR_ALL); - else + else if (wx->pdev->msix_enabled) wx_intr_enable(wx, NGBE_INTR_MISC(wx)); + else + wx_intr_enable(wx, BIT(0)); } /** @@ -184,7 +206,7 @@ static irqreturn_t ngbe_intr(int __always_unused irq, void *data) /* shared interrupt alert! * the interrupt that we masked before the EICR read. */ - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) ngbe_irq_enable(wx, true); return IRQ_NONE; /* Not our interrupt */ } @@ -200,7 +222,7 @@ static irqreturn_t ngbe_intr(int __always_unused irq, void *data) /* would disable interrupts here but it is auto disabled */ napi_schedule_irqoff(&q_vector->napi); - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) ngbe_irq_enable(wx, false); return IRQ_HANDLED; @@ -215,7 +237,7 @@ static irqreturn_t __ngbe_msix_misc(struct wx *wx, u32 eicr) wx_ptp_check_pps_event(wx); /* re-enable the original interrupt state, no lsc, no queues */ - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) ngbe_irq_enable(wx, false); return IRQ_HANDLED; @@ -242,7 +264,7 @@ static irqreturn_t ngbe_misc_and_queue(int __always_unused irq, void *data) /* queue */ q_vector = wx->q_vector[0]; napi_schedule_irqoff(&q_vector->napi); - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) ngbe_irq_enable(wx, true); return IRQ_HANDLED; } @@ -343,6 +365,9 @@ static void ngbe_disable_device(struct wx *wx) struct net_device *netdev = wx->netdev; u32 i; + if (test_and_set_bit(WX_STATE_DOWN, wx->state)) + return; + if (wx->num_vfs) { /* Clear EITR Select mapping */ wr32(wx, WX_PX_ITRSEL, 0); @@ -368,6 +393,10 @@ static void ngbe_disable_device(struct wx *wx) wx_napi_disable_all(wx); netif_tx_stop_all_queues(netdev); netif_tx_disable(netdev); + + timer_delete_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); + if (wx->gpio_ctrl) ngbe_sfp_modules_txrx_powerctl(wx, false); wx_irq_disable(wx); @@ -404,9 +433,11 @@ void ngbe_up(struct wx *wx) /* make sure to complete pre-operations */ smp_mb__before_atomic(); + clear_bit(WX_STATE_DOWN, wx->state); wx_napi_enable_all(wx); /* enable transmits */ netif_tx_start_all_queues(wx->netdev); + mod_timer(&wx->service_timer, jiffies); /* clear any pending interrupts, may auto mask */ rd32(wx, WX_PX_IC(0)); @@ -770,9 +801,11 @@ static int ngbe_probe(struct pci_dev *pdev, eth_hw_addr_set(netdev, wx->mac.perm_addr); wx_mac_set_default_filter(wx, wx->mac.perm_addr); + ngbe_init_service(wx); + err = wx_init_interrupt_scheme(wx); if (err) - goto err_free_mac_table; + goto err_cancel_service; /* phy Interface Configuration */ err = ngbe_mdio_init(wx); @@ -792,6 +825,9 @@ static int ngbe_probe(struct pci_dev *pdev, wx_control_hw(wx, false); err_clear_interrupt_scheme: wx_clear_interrupt_scheme(wx); +err_cancel_service: + timer_delete_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); err_free_mac_table: kfree(wx->rss_key); kfree(wx->mac_table); @@ -820,6 +856,10 @@ static void ngbe_remove(struct pci_dev *pdev) netdev = wx->netdev; wx_disable_sriov(wx); unregister_netdev(netdev); + + timer_shutdown_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); + phylink_destroy(wx->phylink); pci_release_selected_regions(pdev, pci_select_bars(pdev, IORESOURCE_MEM)); diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c index 07ae491e3bc9b..a919962b71605 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_aml.c @@ -381,6 +381,7 @@ int txgbe_phylink_init_aml(struct txgbe *txgbe) err = phylink_set_fixed_link(phylink, &state); if (err) { wx_err(wx, "Failed to set fixed link\n"); + phylink_destroy(phylink); return err; } diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c index d7f9053594588..659006b4b8282 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c @@ -58,9 +58,8 @@ static int txgbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; - err = wx_set_state_reset(wx); - if (err) - return err; + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) @@ -90,6 +89,7 @@ static int txgbe_set_ringparam(struct net_device *netdev, clear_reset: clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); return err; } diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c index 3885283681ec7..5ceca13eaa5bc 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_irq.c @@ -141,7 +141,7 @@ static irqreturn_t txgbe_misc_irq_handle(int irq, void *data) /* shared interrupt alert! * the interrupt that we masked before the ICR read. */ - if (netif_running(wx->netdev)) + if (!test_bit(WX_STATE_DOWN, wx->state)) txgbe_irq_enable(wx, true); return IRQ_NONE; /* Not our interrupt */ } @@ -164,6 +164,7 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) struct wx *wx = txgbe->wx; unsigned int nhandled = 0; unsigned int sub_irq; + u64 misc_mask; u32 eicr; eicr = txgbe->eicr; @@ -183,7 +184,9 @@ static irqreturn_t txgbe_misc_irq_thread_fn(int irq, void *data) nhandled++; } - wx_intr_enable(wx, TXGBE_INTR_MISC(wx)); + misc_mask = wx->pdev->msix_enabled ? TXGBE_INTR_MISC(wx) : BIT(0); + if (!test_bit(WX_STATE_DOWN, wx->state)) + wx_intr_enable(wx, misc_mask); return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE); } diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index 4d20b178af236..9d32c4a7f1e97 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -130,6 +130,7 @@ static void txgbe_service_task(struct work_struct *work) txgbe_module_detection_subtask(wx); txgbe_link_config_subtask(wx); + wx_update_stats(wx); wx_service_event_complete(wx); } @@ -151,6 +152,7 @@ static void txgbe_up_complete(struct wx *wx) /* make sure to complete pre-operations */ smp_mb__before_atomic(); + clear_bit(WX_STATE_DOWN, wx->state); wx_napi_enable_all(wx); switch (wx->mac.type) { @@ -217,6 +219,9 @@ static void txgbe_disable_device(struct wx *wx) struct net_device *netdev = wx->netdev; u32 i; + if (test_and_set_bit(WX_STATE_DOWN, wx->state)) + return; + wx_disable_pcie_master(wx); /* disable receives */ wx_disable_rx(wx); @@ -398,6 +403,7 @@ static int txgbe_sw_init(struct wx *wx) wx->configure_fdir = txgbe_configure_fdir; set_bit(WX_FLAG_RSC_CAPABLE, wx->flags); + set_bit(WX_FLAG_RSC_ENABLED, wx->flags); set_bit(WX_FLAG_MULTI_64_FUNC, wx->flags); /* enable itr by default in dynamic mode */ @@ -423,6 +429,7 @@ static int txgbe_sw_init(struct wx *wx) break; case wx_mac_aml: case wx_mac_aml40: + set_bit(WX_FLAG_RX_MERGE_ENABLED, wx->flags); set_bit(WX_FLAG_SWFW_RING, wx->flags); wx->swfw_index = 0; break; @@ -600,20 +607,16 @@ int txgbe_setup_tc(struct net_device *dev, u8 tc) static void txgbe_reinit_locked(struct wx *wx) { - int err = 0; - netif_trans_update(wx->netdev); - err = wx_set_state_reset(wx); - if (err) { - wx_err(wx, "wait device reset timeout\n"); - return; - } + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); txgbe_down(wx); txgbe_up(wx); clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); } void txgbe_do_reset(struct net_device *netdev) @@ -801,6 +804,8 @@ static int txgbe_probe(struct pci_dev *pdev, netdev->features |= NETIF_F_HIGHDMA; netdev->hw_features |= NETIF_F_GRO; netdev->features |= NETIF_F_GRO; + netdev->hw_features |= NETIF_F_LRO; + netdev->features |= NETIF_F_LRO; netdev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; netdev->priv_flags |= IFF_UNICAST_FLT; @@ -956,6 +961,7 @@ static void txgbe_remove(struct pci_dev *pdev) netdev = wx->netdev; wx_disable_sriov(wx); unregister_netdev(netdev); + txgbe_fdir_filter_exit(wx); txgbe_remove_phy(txgbe); wx_free_isb_resources(wx); diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h index 4d77da720eba1..70f89a42b0b30 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_type.h @@ -302,7 +302,7 @@ struct txgbe_fdir_filter { #define TXGBE_DEFAULT_RX_WORK 128 #endif -#define TXGBE_INTR_MISC(A) BIT((A)->num_q_vectors) +#define TXGBE_INTR_MISC(A) BIT_ULL((A)->num_q_vectors) #define TXGBE_INTR_QALL(A) (TXGBE_INTR_MISC(A) - 1) #define TXGBE_MAX_EITR GENMASK(11, 3) diff --git a/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c b/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c index 72663e3c4205a..52c1e223bbd78 100644 --- a/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c +++ b/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c @@ -157,6 +157,17 @@ static int txgbevf_sw_init(struct wx *wx) wx->set_num_queues = txgbevf_set_num_queues; + switch (wx->mac.type) { + case wx_mac_sp: + break; + case wx_mac_aml: + case wx_mac_aml40: + set_bit(WX_FLAG_RX_MERGE_ENABLED, wx->flags); + break; + default: + break; + } + return 0; err_reset_hw: kfree(wx->vfinfo); diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 77b0c3d520414..73e020270b989 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -1736,6 +1736,9 @@ static int geneve_changelink(struct net_device *dev, struct nlattr *tb[], struct geneve_config cfg; int err; + if (!rtnl_dev_link_net_capable(dev, geneve->net)) + return -EPERM; + /* If the geneve device is configured for metadata (or externally * controlled, for example, OVS), then nothing can be changed. */ diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c index 2f8626c3824d6..184cb367bbe13 100644 --- a/drivers/net/gtp.c +++ b/drivers/net/gtp.c @@ -12,6 +12,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include #include #include #include @@ -108,6 +109,7 @@ struct gtp_net { }; static u32 gtp_h_initval; +static DEFINE_MUTEX(gtp_pdp_lock); static struct genl_family gtp_genl_family; @@ -151,7 +153,8 @@ static struct pdp_ctx *gtp0_pdp_find(struct gtp_dev *gtp, u64 tid, u16 family) head = >p->tid_hash[gtp0_hashfn(tid) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_tid) { + hlist_for_each_entry_rcu(pdp, head, hlist_tid, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == family && pdp->gtp_version == GTP_V0 && pdp->u.v0.tid == tid) @@ -168,7 +171,8 @@ static struct pdp_ctx *gtp1_pdp_find(struct gtp_dev *gtp, u32 tid, u16 family) head = >p->tid_hash[gtp1u_hashfn(tid) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_tid) { + hlist_for_each_entry_rcu(pdp, head, hlist_tid, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == family && pdp->gtp_version == GTP_V1 && pdp->u.v1.i_tei == tid) @@ -185,7 +189,8 @@ static struct pdp_ctx *ipv4_pdp_find(struct gtp_dev *gtp, __be32 ms_addr) head = >p->addr_hash[ipv4_hashfn(ms_addr) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_addr) { + hlist_for_each_entry_rcu(pdp, head, hlist_addr, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == AF_INET && pdp->ms.addr.s_addr == ms_addr) return pdp; @@ -220,7 +225,8 @@ static struct pdp_ctx *ipv6_pdp_find(struct gtp_dev *gtp, head = >p->addr_hash[ipv6_hashfn(ms_addr) % gtp->hash_size]; - hlist_for_each_entry_rcu(pdp, head, hlist_addr) { + hlist_for_each_entry_rcu(pdp, head, hlist_addr, + lockdep_is_held(>p_pdp_lock)) { if (pdp->af == AF_INET6 && ipv6_pdp_addr_equal(&pdp->ms.addr6, ms_addr)) return pdp; @@ -312,6 +318,11 @@ static int gtp_inner_proto(struct sk_buff *skb, unsigned int hdrlen, static int gtp_rx(struct pdp_ctx *pctx, struct sk_buff *skb, unsigned int hdrlen, unsigned int role, __u16 inner_proto) { + if (skb_is_gso(skb)) { + netdev_dbg(pctx->dev, "GSO is not supported in GTP\n"); + goto err; + } + if (!gtp_check_ms(skb, pctx, hdrlen, role, inner_proto)) { netdev_dbg(pctx->dev, "No PDP ctx for this MS\n"); return 1; @@ -669,8 +680,9 @@ static int gtp1u_send_echo_resp(struct gtp_dev *gtp, struct sk_buff *skb) return -1; /* pull GTP and UDP headers */ - skb_pull_data(skb, - sizeof(struct gtp1_header_long) + sizeof(struct udphdr)); + if (!skb_pull_data(skb, sizeof(struct gtp1_header_long) + + sizeof(struct udphdr))) + return -1; gtp_pkt = skb_push(skb, sizeof(struct gtp1u_packet)); memset(gtp_pkt, 0, sizeof(struct gtp1u_packet)); @@ -826,13 +838,17 @@ static int gtp1u_udp_encap_recv(struct gtp_dev *gtp, struct sk_buff *skb) if (!pskb_may_pull(skb, hdrlen)) return -1; + gtp1 = (struct gtp1_header *)(skb->data + sizeof(struct udphdr)); + + if (gtp1->flags & GTP1_F_EXTHDR && + gtp_parse_exthdrs(skb, &hdrlen) < 0) + return -1; + if (gtp_inner_proto(skb, hdrlen, &inner_proto) < 0) { netdev_dbg(gtp->dev, "GTP packet does not encapsulate an IP packet\n"); return -1; } - gtp1 = (struct gtp1_header *)(skb->data + sizeof(struct udphdr)); - pctx = gtp1_pdp_find(gtp, ntohl(gtp1->tid), gtp_proto_to_family(inner_proto)); if (!pctx) { @@ -840,10 +856,6 @@ static int gtp1u_udp_encap_recv(struct gtp_dev *gtp, struct sk_buff *skb) return 1; } - if (gtp1->flags & GTP1_F_EXTHDR && - gtp_parse_exthdrs(skb, &hdrlen) < 0) - return -1; - return gtp_rx(pctx, skb, hdrlen, gtp->role, inner_proto); } @@ -1542,6 +1554,8 @@ static int gtp_newlink(struct net_device *dev, out_encap: gtp_encap_disable(gtp); out_hashtable: + /* Wait for RCU readers that may still reference this gtp_dev. */ + synchronize_net(); kfree(gtp->addr_hash); kfree(gtp->tid_hash); return err; @@ -1554,9 +1568,11 @@ static void gtp_dellink(struct net_device *dev, struct list_head *head) struct pdp_ctx *pctx; int i; + mutex_lock(>p_pdp_lock); for (i = 0; i < gtp->hash_size; i++) hlist_for_each_entry_safe(pctx, next, >p->tid_hash[i], hlist_tid) pdp_context_delete(pctx); + mutex_unlock(>p_pdp_lock); list_del(>p->list); unregister_netdevice_queue(dev, head); @@ -2052,6 +2068,7 @@ static int gtp_genl_new_pdp(struct sk_buff *skb, struct genl_info *info) goto out_unlock; } + mutex_lock(>p_pdp_lock); pctx = gtp_pdp_add(gtp, sk, info); if (IS_ERR(pctx)) { err = PTR_ERR(pctx); @@ -2059,6 +2076,7 @@ static int gtp_genl_new_pdp(struct sk_buff *skb, struct genl_info *info) gtp_tunnel_notify(pctx, GTP_CMD_NEWPDP, GFP_KERNEL); err = 0; } + mutex_unlock(>p_pdp_lock); out_unlock: rtnl_unlock(); @@ -2133,6 +2151,8 @@ static int gtp_genl_del_pdp(struct sk_buff *skb, struct genl_info *info) if (!info->attrs[GTPA_VERSION]) return -EINVAL; + mutex_lock(>p_pdp_lock); + rcu_read_lock(); pctx = gtp_find_pdp(sock_net(skb->sk), info->attrs); @@ -2153,6 +2173,7 @@ static int gtp_genl_del_pdp(struct sk_buff *skb, struct genl_info *info) out_unlock: rcu_read_unlock(); + mutex_unlock(>p_pdp_lock); return err; } diff --git a/drivers/net/ieee802154/cc2520.c b/drivers/net/ieee802154/cc2520.c index 2b7034193a00b..abfcfe07246a7 100644 --- a/drivers/net/ieee802154/cc2520.c +++ b/drivers/net/ieee802154/cc2520.c @@ -1156,11 +1156,10 @@ static void cc2520_remove(struct spi_device *spi) { struct cc2520_private *priv = spi_get_drvdata(spi); - mutex_destroy(&priv->buffer_mutex); - flush_work(&priv->fifop_irqwork); - + disable_work_sync(&priv->fifop_irqwork); ieee802154_unregister_hw(priv->hw); ieee802154_free_hw(priv->hw); + mutex_destroy(&priv->buffer_mutex); } static const struct spi_device_id cc2520_ids[] = { diff --git a/drivers/net/ieee802154/mac802154_hwsim.c b/drivers/net/ieee802154/mac802154_hwsim.c index 1cab20b5a885c..7d6fa93a89804 100644 --- a/drivers/net/ieee802154/mac802154_hwsim.c +++ b/drivers/net/ieee802154/mac802154_hwsim.c @@ -72,6 +72,8 @@ struct hwsim_phy { struct ieee802154_hw *hw; u32 idx; + /* Serializes phy->pib_updates. */ + spinlock_t pib_lock; struct hwsim_pib __rcu *pib; bool suspended; @@ -102,8 +104,6 @@ static int hwsim_update_pib(struct ieee802154_hw *hw, u8 page, u8 channel, if (!pib) return -ENOMEM; - pib_old = rtnl_dereference(phy->pib); - pib->page = page; pib->channel = channel; pib->filt.short_addr = filt->short_addr; @@ -112,7 +112,10 @@ static int hwsim_update_pib(struct ieee802154_hw *hw, u8 page, u8 channel, pib->filt.pan_coord = filt->pan_coord; pib->filt_level = filt_level; - rcu_assign_pointer(phy->pib, pib); + spin_lock_bh(&phy->pib_lock); + pib_old = rcu_replace_pointer(phy->pib, pib, + lockdep_is_held(&phy->pib_lock)); + spin_unlock_bh(&phy->pib_lock); kfree_rcu(pib_old, rcu); return 0; } @@ -952,6 +955,7 @@ static int hwsim_add_one(struct genl_info *info, struct device *dev, goto err_pib; } + spin_lock_init(&phy->pib_lock); pib->channel = 13; pib->filt.short_addr = cpu_to_le16(IEEE802154_ADDR_BROADCAST); pib->filt.pan_id = cpu_to_le16(IEEE802154_PANID_BROADCAST); diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index 30fe12e3582b2..eadd389134579 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -966,12 +966,12 @@ static void ipa_remove(struct platform_device *pdev) } if (ret) { /* - * Not cleaning up here properly might also yield a - * crash later on. As the device is still unregistered - * in this case, this might even yield a crash later on. + * Continuing teardown after failing to stop the modem + * could crash, so leave the remaining resources allocated. */ dev_err(dev, "Failed to stop modem (%pe), leaking resources\n", ERR_PTR(ret)); + pm_runtime_put_noidle(dev); return; } diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c index 8fe0d0e1a00fd..bd54fd6bd3805 100644 --- a/drivers/net/ipa/ipa_modem.c +++ b/drivers/net/ipa/ipa_modem.c @@ -269,13 +269,29 @@ void ipa_modem_suspend(struct net_device *netdev) * the modem. We can't enable the queue directly in ipa_modem_resume() * because transmits restart the instant the queue is awakened; but the * device power state won't be ACTIVE until *after* ipa_modem_resume() - * returns. + * returns. A transmit restarted before that would stop the queue + * again and get -EINPROGRESS from pm_runtime_get(), and with this + * work having already run, nothing would ever wake the queue again. + * So wait for the resume to complete before waking the queue. */ static void ipa_modem_wake_queue_work(struct work_struct *work) { struct ipa_priv *priv = container_of(work, struct ipa_priv, work); + struct device *dev = priv->ipa->dev; + int ret; + + ret = pm_runtime_get_sync(dev); + /* Wake the queue even if the device could not be resumed, so + * that pending packets are dropped by the transmit path rather + * than stranded behind a stopped queue. + */ netif_wake_queue(priv->tx->netdev); + + if (ret < 0) + pm_runtime_put_noidle(dev); + else + (void)pm_runtime_put_autosuspend(dev); } /** ipa_modem_resume() - resume callback for runtime_pm diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c index fcaadd111a8a3..0c184f6f91c6f 100644 --- a/drivers/net/ipa/ipa_smp2p.c +++ b/drivers/net/ipa/ipa_smp2p.c @@ -234,19 +234,27 @@ ipa_smp2p_init(struct ipa *ipa, struct platform_device *pdev, bool modem_init) &valid_bit); if (IS_ERR(valid_state)) return PTR_ERR(valid_state); - if (valid_bit >= 32) /* BITS_PER_U32 */ - return -EINVAL; + if (valid_bit >= 32) { /* BITS_PER_U32 */ + ret = -EINVAL; + goto err_valid_state_put; + } enabled_state = qcom_smem_state_get(dev, "ipa-clock-enabled", &enabled_bit); - if (IS_ERR(enabled_state)) - return PTR_ERR(enabled_state); - if (enabled_bit >= 32) /* BITS_PER_U32 */ - return -EINVAL; + if (IS_ERR(enabled_state)) { + ret = PTR_ERR(enabled_state); + goto err_valid_state_put; + } + if (enabled_bit >= 32) { /* BITS_PER_U32 */ + ret = -EINVAL; + goto err_enabled_state_put; + } smp2p = kzalloc(sizeof(*smp2p), GFP_KERNEL); - if (!smp2p) - return -ENOMEM; + if (!smp2p) { + ret = -ENOMEM; + goto err_enabled_state_put; + } smp2p->ipa = ipa; @@ -291,6 +299,10 @@ ipa_smp2p_init(struct ipa *ipa, struct platform_device *pdev, bool modem_init) ipa->smp2p = NULL; mutex_destroy(&smp2p->mutex); kfree(smp2p); +err_enabled_state_put: + qcom_smem_state_put(enabled_state); +err_valid_state_put: + qcom_smem_state_put(valid_state); return ret; } @@ -307,6 +319,8 @@ void ipa_smp2p_exit(struct ipa *ipa) ipa_smp2p_power_release(ipa); ipa->smp2p = NULL; mutex_destroy(&smp2p->mutex); + qcom_smem_state_put(smp2p->enabled_state); + qcom_smem_state_put(smp2p->valid_state); kfree(smp2p); } diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index baccdad695fda..93a8a7c91fb0e 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -146,6 +146,8 @@ static int ipvlan_init(struct net_device *dev) dev->lltx = true; netif_inherit_tso_max(dev, phy_dev); dev->hard_header_len = phy_dev->hard_header_len; + dev->needed_headroom = phy_dev->needed_headroom; + dev->needed_tailroom = phy_dev->needed_tailroom; netdev_lockdep_set_classes(dev); @@ -773,6 +775,8 @@ static int ipvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(ipvlan, &port->ipvlans, pnode) { netif_inherit_tso_max(ipvlan->dev, dev); + ipvlan->dev->needed_headroom = dev->needed_headroom; + ipvlan->dev->needed_tailroom = dev->needed_tailroom; netdev_update_features(ipvlan->dev); } break; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 35dcaa985cfdc..7151ae883136b 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -943,6 +943,8 @@ static int macvlan_init(struct net_device *dev) dev->lltx = true; netif_inherit_tso_max(dev, lowerdev); dev->hard_header_len = lowerdev->hard_header_len; + dev->needed_headroom = lowerdev->needed_headroom; + dev->needed_tailroom = lowerdev->needed_tailroom; macvlan_set_lockdep_class(dev); vlan->pcpu_stats = netdev_alloc_pcpu_stats(struct vlan_pcpu_stats); @@ -1817,6 +1819,8 @@ static int macvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(vlan, &port->vlans, list) { netif_inherit_tso_max(vlan->dev, dev); + vlan->dev->needed_headroom = dev->needed_headroom; + vlan->dev->needed_tailroom = dev->needed_tailroom; netdev_update_features(vlan->dev); } break; diff --git a/drivers/net/mctp/mctp-i3c.c b/drivers/net/mctp/mctp-i3c.c index c678f79aa3561..1d9d0c1418ebd 100644 --- a/drivers/net/mctp/mctp-i3c.c +++ b/drivers/net/mctp/mctp-i3c.c @@ -288,6 +288,7 @@ __must_hold(&busdevs_lock) static int mctp_i3c_probe(struct i3c_device *i3c) { struct mctp_i3c_bus *b = NULL, *mbus = NULL; + int rc; /* Look for a known bus */ mutex_lock(&busdevs_lock); @@ -296,14 +297,16 @@ static int mctp_i3c_probe(struct i3c_device *i3c) mbus = b; break; } - mutex_unlock(&busdevs_lock); if (!mbus) { /* probably no "mctp-controller" property on the i3c bus */ - return -ENODEV; + rc = -ENODEV; + } else { + rc = mctp_i3c_add_device(mbus, i3c); } + mutex_unlock(&busdevs_lock); - return mctp_i3c_add_device(mbus, i3c); + return rc; } static void mctp_i3c_remove_device(struct mctp_i3c_device *mi) @@ -731,18 +734,21 @@ static __init int mctp_i3c_mod_init(void) int rc; rc = i3c_register_notifier(&mctp_i3c_notifier); - if (rc < 0) { - i3c_driver_unregister(&mctp_i3c_driver); + if (rc < 0) return rc; - } i3c_for_each_bus_locked(mctp_i3c_bus_add_new, NULL); rc = i3c_driver_register(&mctp_i3c_driver); if (rc < 0) - return rc; + goto err_unregister_notifier; return 0; + +err_unregister_notifier: + i3c_unregister_notifier(&mctp_i3c_notifier); + mctp_i3c_bus_remove_all(); + return rc; } static __exit void mctp_i3c_mod_exit(void) diff --git a/drivers/net/mctp/mctp-serial.c b/drivers/net/mctp/mctp-serial.c index 26c9a33fd6364..a5070ffa9a955 100644 --- a/drivers/net/mctp/mctp-serial.c +++ b/drivers/net/mctp/mctp-serial.c @@ -318,7 +318,7 @@ static void mctp_serial_push_header(struct mctp_serial *dev, u8 c) } else { dev->rxlen = c; dev->rxpos = 0; - dev->rxstate = STATE_DATA; + dev->rxstate = c > 0 ? STATE_DATA : STATE_TRAILER; dev->rxfcs = crc_ccitt_byte(dev->rxfcs, c); } break; diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c index cd16b3dd7720a..ee9bfc96f0b3e 100644 --- a/drivers/net/net_failover.c +++ b/drivers/net/net_failover.c @@ -675,7 +675,7 @@ static int net_failover_slave_name_change(struct net_device *slave_dev, /* We need to bring up the slave after the rename by udev in case * open failed with EBUSY when it was registered. */ - dev_open(slave_dev, NULL); + netif_open(slave_dev, NULL); return 0; } diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index 06345487d6aac..fec111d64f611 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -1326,9 +1326,11 @@ static void drop_netconsole_target(struct config_group *group, unsigned long flags; struct netconsole_target *nt = to_target(item); + mutex_lock(&target_cleanup_list_lock); spin_lock_irqsave(&target_list_lock, flags); list_del(&nt->list); spin_unlock_irqrestore(&target_list_lock, flags); + mutex_unlock(&target_cleanup_list_lock); /* * The target may have never been enabled, or was manually disabled diff --git a/drivers/net/netdevsim/netdev.c b/drivers/net/netdevsim/netdev.c index 06446b03cd9bc..37260e488b9e8 100644 --- a/drivers/net/netdevsim/netdev.c +++ b/drivers/net/netdevsim/netdev.c @@ -576,6 +576,7 @@ static void nsim_del_napi(struct netdevsim *ns) for (i = 0; i < dev->num_rx_queues; i++) { struct nsim_rq *rq = ns->rq[i]; + netif_queue_set_napi(dev, i, NETDEV_QUEUE_TYPE_RX, NULL); napi_disable_locked(&rq->napi); __netif_napi_del_locked(&rq->napi); } @@ -760,7 +761,9 @@ struct nsim_queue_mem { }; static int -nsim_queue_mem_alloc(struct net_device *dev, void *per_queue_mem, int idx) +nsim_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_queue_mem, int idx) { struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev); @@ -809,7 +812,8 @@ static void nsim_queue_mem_free(struct net_device *dev, void *per_queue_mem) } static int -nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) +nsim_queue_start(struct net_device *dev, struct netdev_queue_config *qcfg, + void *per_queue_mem, int idx) { struct nsim_queue_mem *qmem = per_queue_mem; struct netdevsim *ns = netdev_priv(dev); @@ -836,6 +840,7 @@ nsim_queue_start(struct net_device *dev, void *per_queue_mem, int idx) } ns->rq[idx] = qmem->rq; + netif_queue_set_napi(dev, idx, NETDEV_QUEUE_TYPE_RX, &ns->rq[idx]->napi); napi_enable_locked(&ns->rq[idx]->napi); return 0; diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c index fbeae05817e9f..6b107e0f2cce2 100644 --- a/drivers/net/ntb_netdev.c +++ b/drivers/net/ntb_netdev.c @@ -53,6 +53,7 @@ #include #include #include +#include #define NTB_NETDEV_VER "0.7" @@ -70,11 +71,24 @@ static unsigned int tx_start = 10; /* Number of descriptors still available before stop upper layer tx */ static unsigned int tx_stop = 5; +#define NTB_NETDEV_MAX_QUEUES 64 +#define NTB_NETDEV_DEFAULT_QUEUES 1 + +struct ntb_netdev; + +struct ntb_netdev_queue { + struct ntb_netdev *ntdev; + struct ntb_transport_qp *qp; + struct timer_list tx_timer; + u16 qid; +}; + struct ntb_netdev { struct pci_dev *pdev; + struct device *client_dev; struct net_device *ndev; - struct ntb_transport_qp *qp; - struct timer_list tx_timer; + unsigned int num_queues; + struct ntb_netdev_queue *queues; }; #define NTB_TX_TIMEOUT_MS 1000 @@ -82,14 +96,17 @@ struct ntb_netdev { static void ntb_netdev_event_handler(void *data, int link_is_up) { - struct net_device *ndev = data; - struct ntb_netdev *dev = netdev_priv(ndev); + struct ntb_netdev_queue *q = data; + struct ntb_netdev *dev = q->ntdev; + struct net_device *ndev; + + ndev = dev->ndev; - netdev_dbg(ndev, "Event %x, Link %x\n", link_is_up, - ntb_transport_link_query(dev->qp)); + netdev_dbg(ndev, "Event %x, Link %x, qp %u\n", link_is_up, + ntb_transport_link_query(q->qp), q->qid); if (link_is_up) { - if (ntb_transport_link_query(dev->qp)) + if (ntb_transport_link_query(q->qp)) netif_carrier_on(ndev); } else { netif_carrier_off(ndev); @@ -99,10 +116,13 @@ static void ntb_netdev_event_handler(void *data, int link_is_up) static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, int len) { - struct net_device *ndev = qp_data; - struct sk_buff *skb; + struct ntb_netdev_queue *q = qp_data; + struct ntb_netdev *dev = q->ntdev; + struct sk_buff *skb, *new_skb; + struct net_device *ndev; int rc; + ndev = dev->ndev; skb = data; if (!skb) return; @@ -115,24 +135,23 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, goto enqueue_again; } + ndev->stats.rx_packets++; + ndev->stats.rx_bytes += len; + + new_skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); + if (!new_skb) { + ndev->stats.rx_dropped++; + goto enqueue_again; + } + skb_put(skb, len); skb->protocol = eth_type_trans(skb, ndev); skb->ip_summed = CHECKSUM_NONE; + skb_record_rx_queue(skb, q->qid); - if (netif_rx(skb) == NET_RX_DROP) { - ndev->stats.rx_errors++; - ndev->stats.rx_dropped++; - } else { - ndev->stats.rx_packets++; - ndev->stats.rx_bytes += len; - } + netif_rx(skb); - skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); - if (!skb) { - ndev->stats.rx_errors++; - ndev->stats.rx_frame_errors++; - return; - } + skb = new_skb; enqueue_again: rc = ntb_transport_rx_enqueue(qp, skb, skb->data, ndev->mtu + ETH_HLEN); @@ -144,42 +163,45 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data, } static int __ntb_netdev_maybe_stop_tx(struct net_device *netdev, - struct ntb_transport_qp *qp, int size) + struct ntb_netdev_queue *q, int size) { - struct ntb_netdev *dev = netdev_priv(netdev); + netif_stop_subqueue(netdev, q->qid); - netif_stop_queue(netdev); /* Make sure to see the latest value of ntb_transport_tx_free_entry() * since the queue was last started. */ smp_mb(); - if (likely(ntb_transport_tx_free_entry(qp) < size)) { - mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time)); + if (likely(ntb_transport_tx_free_entry(q->qp) < size)) { + mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); return -EBUSY; } - netif_start_queue(netdev); + netif_start_subqueue(netdev, q->qid); return 0; } static int ntb_netdev_maybe_stop_tx(struct net_device *ndev, - struct ntb_transport_qp *qp, int size) + struct ntb_netdev_queue *q, int size) { - if (netif_queue_stopped(ndev) || - (ntb_transport_tx_free_entry(qp) >= size)) + if (__netif_subqueue_stopped(ndev, q->qid)) + return -EBUSY; + + if (ntb_transport_tx_free_entry(q->qp) >= size) return 0; - return __ntb_netdev_maybe_stop_tx(ndev, qp, size); + return __ntb_netdev_maybe_stop_tx(ndev, q, size); } static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, int len) { - struct net_device *ndev = qp_data; + struct ntb_netdev_queue *q = qp_data; + struct ntb_netdev *dev = q->ntdev; + struct net_device *ndev; struct sk_buff *skb; - struct ntb_netdev *dev = netdev_priv(ndev); + ndev = dev->ndev; skb = data; if (!skb || !ndev) return; @@ -194,13 +216,13 @@ static void ntb_netdev_tx_handler(struct ntb_transport_qp *qp, void *qp_data, dev_kfree_skb_any(skb); - if (ntb_transport_tx_free_entry(dev->qp) >= tx_start) { + if (ntb_transport_tx_free_entry(qp) >= tx_start) { /* Make sure anybody stopping the queue after this sees the new * value of ntb_transport_tx_free_entry() */ smp_mb(); - if (netif_queue_stopped(ndev)) - netif_wake_queue(ndev); + if (__netif_subqueue_stopped(ndev, q->qid)) + netif_wake_subqueue(ndev, q->qid); } } @@ -208,101 +230,138 @@ static netdev_tx_t ntb_netdev_start_xmit(struct sk_buff *skb, struct net_device *ndev) { struct ntb_netdev *dev = netdev_priv(ndev); + u16 qid = skb_get_queue_mapping(skb); + struct ntb_netdev_queue *q; int rc; - ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop); + q = &dev->queues[qid]; - rc = ntb_transport_tx_enqueue(dev->qp, skb, skb->data, skb->len); - if (rc) - goto err; + if (unlikely(ntb_netdev_maybe_stop_tx(ndev, q, tx_stop))) + return NETDEV_TX_BUSY; + + rc = ntb_transport_tx_enqueue(q->qp, skb, skb->data, skb->len); + if (rc) { + if (rc == -EAGAIN || rc == -EBUSY) { + netif_stop_subqueue(ndev, q->qid); + mod_timer(&q->tx_timer, + jiffies + usecs_to_jiffies(tx_time)); + return NETDEV_TX_BUSY; + } + + goto drop; + } /* check for next submit */ - ntb_netdev_maybe_stop_tx(ndev, dev->qp, tx_stop); + ntb_netdev_maybe_stop_tx(ndev, q, tx_stop); return NETDEV_TX_OK; -err: +drop: + dev_kfree_skb_any(skb); ndev->stats.tx_dropped++; - ndev->stats.tx_errors++; - return NETDEV_TX_BUSY; + return NETDEV_TX_OK; } static void ntb_netdev_tx_timer(struct timer_list *t) { - struct ntb_netdev *dev = timer_container_of(dev, t, tx_timer); - struct net_device *ndev = dev->ndev; + struct ntb_netdev_queue *q = timer_container_of(q, t, tx_timer); + struct ntb_netdev *dev = q->ntdev; + struct net_device *ndev; - if (ntb_transport_tx_free_entry(dev->qp) < tx_stop) { - mod_timer(&dev->tx_timer, jiffies + usecs_to_jiffies(tx_time)); + ndev = dev->ndev; + + if (ntb_transport_tx_free_entry(q->qp) < tx_stop) { + mod_timer(&q->tx_timer, jiffies + usecs_to_jiffies(tx_time)); } else { /* Make sure anybody stopping the queue after this sees the new * value of ntb_transport_tx_free_entry() */ smp_mb(); - if (netif_queue_stopped(ndev)) - netif_wake_queue(ndev); + if (__netif_subqueue_stopped(ndev, q->qid)) + netif_wake_subqueue(ndev, q->qid); } } static int ntb_netdev_open(struct net_device *ndev) { struct ntb_netdev *dev = netdev_priv(ndev); + struct ntb_netdev_queue *queue; struct sk_buff *skb; - int rc, i, len; - - /* Add some empty rx bufs */ - for (i = 0; i < NTB_RXQ_SIZE; i++) { - skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); - if (!skb) { - rc = -ENOMEM; - goto err; - } + int rc = 0, i, len; + unsigned int q; - rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data, - ndev->mtu + ETH_HLEN); - if (rc) { - dev_kfree_skb(skb); - goto err; + /* Add some empty rx bufs for each queue */ + for (q = 0; q < dev->num_queues; q++) { + queue = &dev->queues[q]; + + for (i = 0; i < NTB_RXQ_SIZE; i++) { + skb = netdev_alloc_skb(ndev, ndev->mtu + ETH_HLEN); + if (!skb) { + rc = -ENOMEM; + goto err; + } + + rc = ntb_transport_rx_enqueue(queue->qp, skb, skb->data, + ndev->mtu + ETH_HLEN); + if (rc) { + dev_kfree_skb(skb); + goto err; + } } - } - timer_setup(&dev->tx_timer, ntb_netdev_tx_timer, 0); + timer_setup(&queue->tx_timer, ntb_netdev_tx_timer, 0); + } netif_carrier_off(ndev); - ntb_transport_link_up(dev->qp); + + for (q = 0; q < dev->num_queues; q++) + ntb_transport_link_up(dev->queues[q].qp); + netif_start_queue(ndev); return 0; err: - while ((skb = ntb_transport_rx_remove(dev->qp, &len))) - dev_kfree_skb(skb); + for (q = 0; q < dev->num_queues; q++) { + queue = &dev->queues[q]; + + while ((skb = ntb_transport_rx_remove(queue->qp, &len))) + dev_kfree_skb(skb); + } return rc; } static int ntb_netdev_close(struct net_device *ndev) { struct ntb_netdev *dev = netdev_priv(ndev); + struct ntb_netdev_queue *queue; struct sk_buff *skb; + unsigned int q; int len; - ntb_transport_link_down(dev->qp); - while ((skb = ntb_transport_rx_remove(dev->qp, &len))) - dev_kfree_skb(skb); + for (q = 0; q < dev->num_queues; q++) { + queue = &dev->queues[q]; + + ntb_transport_link_down(queue->qp); - timer_delete_sync(&dev->tx_timer); + while ((skb = ntb_transport_rx_remove(queue->qp, &len))) + dev_kfree_skb(skb); + timer_delete_sync(&queue->tx_timer); + } return 0; } static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu) { struct ntb_netdev *dev = netdev_priv(ndev); + struct ntb_netdev_queue *queue; struct sk_buff *skb; - int len, rc; + unsigned int q, i; + int len, rc = 0; - if (new_mtu > ntb_transport_max_size(dev->qp) - ETH_HLEN) + if (new_mtu > ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN) return -EINVAL; if (!netif_running(ndev)) { @@ -311,41 +370,54 @@ static int ntb_netdev_change_mtu(struct net_device *ndev, int new_mtu) } /* Bring down the link and dispose of posted rx entries */ - ntb_transport_link_down(dev->qp); + for (q = 0; q < dev->num_queues; q++) + ntb_transport_link_down(dev->queues[q].qp); if (ndev->mtu < new_mtu) { - int i; - - for (i = 0; (skb = ntb_transport_rx_remove(dev->qp, &len)); i++) - dev_kfree_skb(skb); + for (q = 0; q < dev->num_queues; q++) { + queue = &dev->queues[q]; - for (; i; i--) { - skb = netdev_alloc_skb(ndev, new_mtu + ETH_HLEN); - if (!skb) { - rc = -ENOMEM; - goto err; - } - - rc = ntb_transport_rx_enqueue(dev->qp, skb, skb->data, - new_mtu + ETH_HLEN); - if (rc) { + for (i = 0; + (skb = ntb_transport_rx_remove(queue->qp, &len)); + i++) dev_kfree_skb(skb); - goto err; + + for (; i; i--) { + skb = netdev_alloc_skb(ndev, + new_mtu + ETH_HLEN); + if (!skb) { + rc = -ENOMEM; + goto err; + } + + rc = ntb_transport_rx_enqueue(queue->qp, skb, + skb->data, + new_mtu + + ETH_HLEN); + if (rc) { + dev_kfree_skb(skb); + goto err; + } } } } WRITE_ONCE(ndev->mtu, new_mtu); - ntb_transport_link_up(dev->qp); + for (q = 0; q < dev->num_queues; q++) + ntb_transport_link_up(dev->queues[q].qp); return 0; err: - ntb_transport_link_down(dev->qp); + for (q = 0; q < dev->num_queues; q++) { + struct ntb_netdev_queue *queue = &dev->queues[q]; - while ((skb = ntb_transport_rx_remove(dev->qp, &len))) - dev_kfree_skb(skb); + ntb_transport_link_down(queue->qp); + + while ((skb = ntb_transport_rx_remove(queue->qp, &len))) + dev_kfree_skb(skb); + } netdev_err(ndev, "Error changing MTU, device inoperable\n"); return rc; @@ -404,6 +476,7 @@ static int ntb_netdev_probe(struct device *client_dev) struct net_device *ndev; struct pci_dev *pdev; struct ntb_netdev *dev; + unsigned int q; int rc; ntb = dev_ntb(client_dev->parent); @@ -411,7 +484,7 @@ static int ntb_netdev_probe(struct device *client_dev) if (!pdev) return -ENODEV; - ndev = alloc_etherdev(sizeof(*dev)); + ndev = alloc_etherdev_mq(sizeof(*dev), NTB_NETDEV_MAX_QUEUES); if (!ndev) return -ENOMEM; @@ -420,6 +493,16 @@ static int ntb_netdev_probe(struct device *client_dev) dev = netdev_priv(ndev); dev->ndev = ndev; dev->pdev = pdev; + dev->client_dev = client_dev; + dev->num_queues = 0; + + dev->queues = kzalloc_objs(*dev->queues, NTB_NETDEV_MAX_QUEUES, + GFP_KERNEL); + if (!dev->queues) { + rc = -ENOMEM; + goto err_free_netdev; + } + ndev->features = NETIF_F_HIGHDMA; ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE; @@ -436,26 +519,47 @@ static int ntb_netdev_probe(struct device *client_dev) ndev->min_mtu = 0; ndev->max_mtu = ETH_MAX_MTU; - dev->qp = ntb_transport_create_queue(ndev, client_dev, - &ntb_netdev_handlers); - if (!dev->qp) { + for (q = 0; q < NTB_NETDEV_DEFAULT_QUEUES; q++) { + struct ntb_netdev_queue *queue = &dev->queues[q]; + + queue->ntdev = dev; + queue->qid = q; + queue->qp = ntb_transport_create_queue(queue, client_dev, + &ntb_netdev_handlers); + if (!queue->qp) + break; + + dev->num_queues++; + } + + if (!dev->num_queues) { rc = -EIO; - goto err; + goto err_free_queues; } - ndev->mtu = ntb_transport_max_size(dev->qp) - ETH_HLEN; + rc = netif_set_real_num_queues(ndev, dev->num_queues, dev->num_queues); + if (rc) + goto err_free_qps; + + ndev->mtu = ntb_transport_max_size(dev->queues[0].qp) - ETH_HLEN; rc = register_netdev(ndev); if (rc) - goto err1; + goto err_free_qps; dev_set_drvdata(client_dev, ndev); - dev_info(&pdev->dev, "%s created\n", ndev->name); + dev_info(&pdev->dev, "%s created with %u queue pairs\n", + ndev->name, dev->num_queues); return 0; -err1: - ntb_transport_free_queue(dev->qp); -err: +err_free_qps: + for (q = 0; q < dev->num_queues; q++) + ntb_transport_free_queue(dev->queues[q].qp); + +err_free_queues: + kfree(dev->queues); + +err_free_netdev: free_netdev(ndev); return rc; } @@ -464,9 +568,13 @@ static void ntb_netdev_remove(struct device *client_dev) { struct net_device *ndev = dev_get_drvdata(client_dev); struct ntb_netdev *dev = netdev_priv(ndev); + unsigned int q; unregister_netdev(ndev); - ntb_transport_free_queue(dev->qp); + for (q = 0; q < dev->num_queues; q++) + ntb_transport_free_queue(dev->queues[q].qp); + + kfree(dev->queues); free_netdev(ndev); } diff --git a/drivers/net/ovpn/crypto.c b/drivers/net/ovpn/crypto.c index 90580e32052fb..2e95f29514fc4 100644 --- a/drivers/net/ovpn/crypto.c +++ b/drivers/net/ovpn/crypto.c @@ -58,15 +58,19 @@ void ovpn_crypto_state_release(struct ovpn_crypto_state *cs) bool ovpn_crypto_kill_key(struct ovpn_crypto_state *cs, u8 key_id) { struct ovpn_crypto_key_slot *ks = NULL; + struct ovpn_crypto_key_slot *tmp; + int slot = 0; spin_lock_bh(&cs->lock); - if (rcu_access_pointer(cs->slots[0])->key_id == key_id) { - ks = rcu_replace_pointer(cs->slots[0], NULL, - lockdep_is_held(&cs->lock)); - } else if (rcu_access_pointer(cs->slots[1])->key_id == key_id) { - ks = rcu_replace_pointer(cs->slots[1], NULL, - lockdep_is_held(&cs->lock)); + tmp = rcu_access_pointer(cs->slots[slot]); + if (!tmp || tmp->key_id != key_id) { + slot = 1; + tmp = rcu_access_pointer(cs->slots[slot]); } + + if (tmp && tmp->key_id == key_id) + ks = rcu_replace_pointer(cs->slots[slot], NULL, + lockdep_is_held(&cs->lock)); spin_unlock_bh(&cs->lock); if (ks) diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index c03e58e28a860..65a904c64e056 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -137,7 +137,7 @@ void ovpn_decrypt_post(void *data, int ret) } /* keep track of last received authenticated packet for keepalive */ - WRITE_ONCE(peer->last_recv, ktime_get_real_seconds()); + WRITE_ONCE(peer->last_recv, ktime_get_boottime_seconds()); rcu_read_lock(); sock = rcu_dereference(peer->sock); @@ -199,10 +199,10 @@ void ovpn_decrypt_post(void *data, int ret) ovpn_dev_dstats_rx_dropped(peer->ovpn->dev); kfree_skb(skb); drop_nocount: - if (likely(peer)) - ovpn_peer_put(peer); if (likely(ks)) ovpn_crypto_key_slot_put(ks); + if (likely(peer)) + ovpn_peer_put(peer); } /* RX path entry point: decrypt packet and forward it to the device */ @@ -291,7 +291,7 @@ void ovpn_encrypt_post(void *data, int ret) ovpn_peer_stats_increment_tx(&peer->link_stats, orig_len); /* keep track of last sent packet for keepalive */ - WRITE_ONCE(peer->last_sent, ktime_get_real_seconds()); + WRITE_ONCE(peer->last_sent, ktime_get_boottime_seconds()); /* skb passed down the stack - don't free it */ skb = NULL; err_unlock: @@ -299,11 +299,11 @@ void ovpn_encrypt_post(void *data, int ret) err: if (unlikely(skb)) ovpn_dev_dstats_tx_dropped(peer->ovpn->dev); - if (likely(peer)) - ovpn_peer_put(peer); + kfree_skb(skb); if (likely(ks)) ovpn_crypto_key_slot_put(ks); - kfree_skb(skb); + if (likely(peer)) + ovpn_peer_put(peer); } static bool ovpn_encrypt_one(struct ovpn_peer *peer, struct sk_buff *skb) diff --git a/drivers/net/ovpn/main.c b/drivers/net/ovpn/main.c index 3f76b1b0e5f60..c18ed8690baff 100644 --- a/drivers/net/ovpn/main.c +++ b/drivers/net/ovpn/main.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,9 @@ #include "tcp.h" #include "udp.h" +/* module-owned workqueue on which all ovpn-specific work is queued */ +struct workqueue_struct *ovpn_wq; + static void ovpn_priv_free(struct net_device *net) { struct ovpn_priv *ovpn = netdev_priv(net); @@ -35,25 +39,11 @@ static void ovpn_priv_free(struct net_device *net) static int ovpn_mp_alloc(struct ovpn_priv *ovpn) { - struct in_device *dev_v4; int i; if (ovpn->mode != OVPN_MODE_MP) return 0; - dev_v4 = __in_dev_get_rtnl(ovpn->dev); - if (dev_v4) { - /* disable redirects as Linux gets confused by ovpn - * handling same-LAN routing. - * This happens because a multipeer interface is used as - * relay point between hosts in the same subnet, while - * in a classic LAN this would not be needed because the - * two hosts would be able to talk directly. - */ - IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); - IPV4_DEVCONF_ALL(dev_net(ovpn->dev), SEND_REDIRECTS) = false; - } - /* the peer container is fairly large, therefore we allocate it only in * MP mode */ @@ -97,9 +87,38 @@ static void ovpn_net_uninit(struct net_device *dev) gro_cells_destroy(&ovpn->gro_cells); } +static int ovpn_net_open(struct net_device *dev) +{ + struct ovpn_priv *ovpn = netdev_priv(dev); + struct in_device *dev_v4; + + /* the IPv4 in_device (and thus its config) is recreated whenever the + * interface is moved to a new netns, so redirects must be disabled on + * every bring-up rather than once at creation time, otherwise the + * setting is silently lost after such a move + */ + if (ovpn->mode == OVPN_MODE_MP) { + dev_v4 = __in_dev_get_rtnl(dev); + if (dev_v4) { + /* disable redirects as Linux gets confused by ovpn + * handling same-LAN routing. + * This happens because a multipeer interface is used as + * relay point between hosts in the same subnet, while + * in a classic LAN this would not be needed because the + * two hosts would be able to talk directly. + */ + IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false); + IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false; + } + } + + return 0; +} + static const struct net_device_ops ovpn_netdev_ops = { .ndo_init = ovpn_net_init, .ndo_uninit = ovpn_net_uninit, + .ndo_open = ovpn_net_open, .ndo_start_xmit = ovpn_net_xmit, }; @@ -183,6 +202,7 @@ static int ovpn_newlink(struct net_device *dev, struct ovpn_priv *ovpn = netdev_priv(dev); struct nlattr **data = params->data; enum ovpn_mode mode = OVPN_MODE_P2P; + int ret; if (data && data[IFLA_OVPN_MODE]) { mode = nla_get_u8(data[IFLA_OVPN_MODE]); @@ -207,7 +227,17 @@ static int ovpn_newlink(struct net_device *dev, else netif_carrier_off(dev); - return register_netdevice(dev); + ret = register_netdevice(dev); + if (ret < 0) + return ret; + + return 0; +} + +static size_t ovpn_get_size(const struct net_device *dev) +{ + /* IFLA_OVPN_MODE */ + return nla_total_size(sizeof(u8)); } static int ovpn_fill_info(struct sk_buff *skb, const struct net_device *dev) @@ -228,16 +258,26 @@ static struct rtnl_link_ops ovpn_link_ops = { .policy = ovpn_policy, .maxtype = IFLA_OVPN_MAX, .newlink = ovpn_newlink, + .get_size = ovpn_get_size, .fill_info = ovpn_fill_info, }; static int __init ovpn_init(void) { - int err = rtnl_link_register(&ovpn_link_ops); + int err; + + ovpn_tcp_init(); + ovpn_wq = alloc_workqueue("ovpn", WQ_PERCPU, 0); + if (!ovpn_wq) { + pr_err("ovpn: cannot allocate workqueue\n"); + return -ENOMEM; + } + + err = rtnl_link_register(&ovpn_link_ops); if (err) { pr_err("ovpn: can't register rtnl link ops: %d\n", err); - return err; + goto destroy_wq; } err = ovpn_nl_register(); @@ -246,12 +286,13 @@ static int __init ovpn_init(void) goto unreg_rtnl; } - ovpn_tcp_init(); - return 0; unreg_rtnl: rtnl_link_unregister(&ovpn_link_ops); +destroy_wq: + destroy_workqueue(ovpn_wq); + ovpn_wq = NULL; return err; } @@ -260,7 +301,11 @@ static __exit void ovpn_cleanup(void) ovpn_nl_unregister(); rtnl_link_unregister(&ovpn_link_ops); + flush_workqueue(ovpn_wq); rcu_barrier(); + + destroy_workqueue(ovpn_wq); + ovpn_wq = NULL; } module_init(ovpn_init); diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c index bdb56ef0c9040..0051166c59c75 100644 --- a/drivers/net/ovpn/netlink.c +++ b/drivers/net/ovpn/netlink.c @@ -527,6 +527,12 @@ int ovpn_nl_peer_set_doit(struct sk_buff *skb, struct genl_info *info) */ if (ret > 0) ovpn_peer_hash_vpn_ip(peer); + /* if the remote endpoint was updated, the by_transp_addr hash bucket + * also needs to be refreshed, otherwise incoming packets from the new + * remote address would fail the lockless lookup + */ + if (attrs[OVPN_A_PEER_REMOTE_IPV4] || attrs[OVPN_A_PEER_REMOTE_IPV6]) + ovpn_peer_hash_transp_addr(peer); spin_unlock_bh(&ovpn->lock); ovpn_peer_put(peer); diff --git a/drivers/net/ovpn/ovpnpriv.h b/drivers/net/ovpn/ovpnpriv.h index 5898f6adada7f..84499140e4bd9 100644 --- a/drivers/net/ovpn/ovpnpriv.h +++ b/drivers/net/ovpn/ovpnpriv.h @@ -15,6 +15,10 @@ #include #include +struct workqueue_struct; + +extern struct workqueue_struct *ovpn_wq; + /** * struct ovpn_peer_collection - container of peers for MultiPeer mode * @by_id: table of peers index by ID diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c index 87a83321f1dd5..6b9f5f12fc5a8 100644 --- a/drivers/net/ovpn/peer.c +++ b/drivers/net/ovpn/peer.c @@ -26,11 +26,12 @@ static void unlock_ovpn(struct ovpn_priv *ovpn, struct llist_head *release_list) __releases(&ovpn->lock) { - struct ovpn_peer *peer; + struct ovpn_peer *peer, *next; spin_unlock_bh(&ovpn->lock); - llist_for_each_entry(peer, release_list->first, release_entry) { + llist_for_each_entry_safe(peer, next, release_list->first, + release_entry) { ovpn_socket_release(peer); ovpn_peer_put(peer); } @@ -44,7 +45,7 @@ static void unlock_ovpn(struct ovpn_priv *ovpn, */ void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) { - time64_t now = ktime_get_real_seconds(); + time64_t now = ktime_get_boottime_seconds(); netdev_dbg(peer->ovpn->dev, "scheduling keepalive for peer %u: interval=%u timeout=%u\n", @@ -61,7 +62,7 @@ void ovpn_peer_keepalive_set(struct ovpn_peer *peer, u32 interval, u32 timeout) /* now that interval and timeout have been changed, kick * off the worker so that the next delay can be recomputed */ - mod_delayed_work(system_wq, &peer->ovpn->keepalive_work, 0); + mod_delayed_work(ovpn_wq, &peer->ovpn->keepalive_work, 0); } /** @@ -184,6 +185,9 @@ int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, &(*__tbl1)[ovpn_get_hash_slot(*__tbl1, _key, _key_len)];\ }) +static void __ovpn_peer_hash_transp_addr(struct ovpn_peer *peer, + const struct ovpn_bind *bind); + /** * ovpn_peer_endpoints_update - update remote or local endpoint for peer * @peer: peer to update the remote endpoint for @@ -191,7 +195,6 @@ int ovpn_peer_reset_sockaddr(struct ovpn_peer *peer, */ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) { - struct hlist_nulls_head *nhead; struct sockaddr_storage ss; struct sockaddr_in6 *sa6; bool reset_cache = false; @@ -215,9 +218,16 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) */ local_ip = &ip_hdr(skb)->daddr; sa = (struct sockaddr_in *)&ss; - sa->sin_family = AF_INET; - sa->sin_addr.s_addr = ip_hdr(skb)->saddr; - sa->sin_port = udp_hdr(skb)->source; + /* use a designated initializer so the sin_zero padding + * is zeroed (it ends up in the by_transp_addr hash key) + * without memset-ing the whole sockaddr_storage on the + * RX fast path + */ + *sa = (struct sockaddr_in) { + .sin_family = AF_INET, + .sin_addr.s_addr = ip_hdr(skb)->saddr, + .sin_port = udp_hdr(skb)->source, + }; salen = sizeof(*sa); reset_cache = true; break; @@ -243,11 +253,19 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) */ local_ip = &ipv6_hdr(skb)->daddr; sa6 = (struct sockaddr_in6 *)&ss; - sa6->sin6_family = AF_INET6; - sa6->sin6_addr = ipv6_hdr(skb)->saddr; - sa6->sin6_port = udp_hdr(skb)->source; - sa6->sin6_scope_id = ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, - skb->skb_iif); + /* use a designated initializer so the sin6_flowinfo + * padding is zeroed (it ends up in the by_transp_addr + * hash key) without memset-ing the whole + * sockaddr_storage on the RX fast path + */ + *sa6 = (struct sockaddr_in6) { + .sin6_family = AF_INET6, + .sin6_addr = ipv6_hdr(skb)->saddr, + .sin6_port = udp_hdr(skb)->source, + .sin6_scope_id = + ipv6_iface_scope_id(&ipv6_hdr(skb)->saddr, + skb->skb_iif), + }; salen = sizeof(*sa6); reset_cache = true; break; @@ -288,42 +306,25 @@ void ovpn_peer_endpoints_update(struct ovpn_peer *peer, struct sk_buff *skb) spin_unlock_bh(&peer->lock); /* rehashing is required only in MP mode as P2P has one peer - * only and thus there is no hashtable + * only and thus there is no hashtable. + * + * This function may be invoked concurrently, so re-read peer->bind + * under the proper locks and rehash against its current value. */ - if (peer->ovpn->mode == OVPN_MODE_MP) { - spin_lock_bh(&peer->ovpn->lock); - spin_lock_bh(&peer->lock); - bind = rcu_dereference_protected(peer->bind, - lockdep_is_held(&peer->lock)); - if (unlikely(!bind)) { - spin_unlock_bh(&peer->lock); - spin_unlock_bh(&peer->ovpn->lock); - return; - } - - /* This function may be invoked concurrently, therefore another - * float may have happened in parallel: perform rehashing - * using the peer->bind->remote directly as key - */ - - switch (bind->remote.in4.sin_family) { - case AF_INET: - salen = sizeof(*sa); - break; - case AF_INET6: - salen = sizeof(*sa6); - break; - } + if (peer->ovpn->mode != OVPN_MODE_MP) + return; - /* remove old hashing */ - hlist_nulls_del_init_rcu(&peer->hash_entry_transp_addr); - /* re-add with new transport address */ - nhead = ovpn_get_hash_head(peer->ovpn->peers->by_transp_addr, - &bind->remote, salen); - hlist_nulls_add_head_rcu(&peer->hash_entry_transp_addr, nhead); - spin_unlock_bh(&peer->lock); - spin_unlock_bh(&peer->ovpn->lock); - } + /* This function may be invoked concurrently, therefore another + * float may have happened in parallel: re-acquire the locks and + * rehash using the peer->bind->remote directly as key + */ + spin_lock_bh(&peer->ovpn->lock); + spin_lock_bh(&peer->lock); + bind = rcu_dereference_protected(peer->bind, + lockdep_is_held(&peer->lock)); + __ovpn_peer_hash_transp_addr(peer, bind); + spin_unlock_bh(&peer->lock); + spin_unlock_bh(&peer->ovpn->lock); return; unlock: spin_unlock_bh(&peer->lock); @@ -890,6 +891,83 @@ bool ovpn_peer_check_by_src(struct ovpn_priv *ovpn, struct sk_buff *skb, return match; } +/* Move @peer to the by_transp_addr bucket matching its current bind. + * + * Caller must hold both peer->ovpn->lock and peer->lock, and must have + * already dereferenced a valid (non-NULL) peer->bind, passed in as @bind. + */ +static void __ovpn_peer_hash_transp_addr(struct ovpn_peer *peer, + const struct ovpn_bind *bind) +{ + struct sockaddr_storage sa = {}; + struct hlist_nulls_head *nhead; + struct sockaddr_in6 *sa6; + struct sockaddr_in *sa4; + size_t salen; + + lockdep_assert_held(&peer->ovpn->lock); + lockdep_assert_held(&peer->lock); + + if (WARN_ON_ONCE(!bind)) + return; + + /* peer may have been concurrently removed between the caller's + * initial lookup and our acquisition of ovpn->lock; skip the + * rehash so we don't re-insert a removed peer + */ + if (unlikely(hlist_unhashed(&peer->hash_entry_id))) + return; + + /* Build the hash key from the transport identity only + * (family/address/port), matching ovpn_peer_add_mp() and the lookup + * in ovpn_peer_get_by_transp_addr(). Hashing bind->remote directly + * would fold in sin6_scope_id (set on the float path but never by the + * lookup), scattering the peer into a bucket lookups cannot reach. + */ + switch (bind->remote.in4.sin_family) { + case AF_INET: + sa4 = (struct sockaddr_in *)&sa; + sa4->sin_family = AF_INET; + sa4->sin_addr.s_addr = bind->remote.in4.sin_addr.s_addr; + sa4->sin_port = bind->remote.in4.sin_port; + salen = sizeof(*sa4); + break; + case AF_INET6: + sa6 = (struct sockaddr_in6 *)&sa; + sa6->sin6_family = AF_INET6; + sa6->sin6_addr = bind->remote.in6.sin6_addr; + sa6->sin6_port = bind->remote.in6.sin6_port; + salen = sizeof(*sa6); + break; + default: + return; + } + + /* remove old hashing (no-op if entry is not currently linked) */ + hlist_nulls_del_init_rcu(&peer->hash_entry_transp_addr); + /* re-add with current transport address */ + nhead = ovpn_get_hash_head(peer->ovpn->peers->by_transp_addr, &sa, + salen); + hlist_nulls_add_head_rcu(&peer->hash_entry_transp_addr, nhead); +} + +void ovpn_peer_hash_transp_addr(struct ovpn_peer *peer) +{ + struct ovpn_bind *bind; + + lockdep_assert_held(&peer->ovpn->lock); + + /* rehashing makes sense only in multipeer mode */ + if (peer->ovpn->mode != OVPN_MODE_MP) + return; + + spin_lock_bh(&peer->lock); + bind = rcu_dereference_protected(peer->bind, + lockdep_is_held(&peer->lock)); + __ovpn_peer_hash_transp_addr(peer, bind); + spin_unlock_bh(&peer->lock); +} + void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) { struct hlist_nulls_head *nhead; @@ -900,6 +978,13 @@ void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer) if (peer->ovpn->mode != OVPN_MODE_MP) return; + /* peer may have been concurrently removed between the caller's + * initial lookup and our acquisition of ovpn->lock; skip the + * rehash so we don't re-insert a removed peer + */ + if (hlist_unhashed(&peer->hash_entry_id)) + return; + if (peer->vpn_addrs.ipv4.s_addr != htonl(INADDR_ANY)) { /* remove potential old hashing */ hlist_nulls_del_init_rcu(&peer->hash_entry_addr4); @@ -1159,10 +1244,9 @@ static void ovpn_peer_release_p2p(struct ovpn_priv *ovpn, struct sock *sk, } if (sk) { - ovpn_sock = rcu_access_pointer(peer->sock); + ovpn_sock = rcu_dereference_bh(peer->sock); if (!ovpn_sock || ovpn_sock->sk != sk) { spin_unlock_bh(&ovpn->lock); - ovpn_peer_put(peer); return; } } @@ -1280,8 +1364,10 @@ static time64_t ovpn_peer_keepalive_work_single(struct ovpn_peer *peer, netdev_dbg(peer->ovpn->dev, "sending keepalive to peer %u\n", peer->id); - if (schedule_work(&peer->keepalive_work)) - ovpn_peer_hold(peer); + if (WARN_ON(!ovpn_peer_hold(peer))) + return 0; + if (!queue_work(ovpn_wq, &peer->keepalive_work)) + ovpn_peer_put(peer); } if (next_run1 < next_run2) @@ -1352,7 +1438,7 @@ void ovpn_peer_keepalive_work(struct work_struct *work) { struct ovpn_priv *ovpn = container_of(work, struct ovpn_priv, keepalive_work.work); - time64_t next_run = 0, now = ktime_get_real_seconds(); + time64_t next_run = 0, now = ktime_get_boottime_seconds(); LLIST_HEAD(release_list); spin_lock_bh(&ovpn->lock); @@ -1372,8 +1458,8 @@ void ovpn_peer_keepalive_work(struct work_struct *work) netdev_dbg(ovpn->dev, "scheduling keepalive work: now=%llu next_run=%llu delta=%llu\n", next_run, now, next_run - now); - schedule_delayed_work(&ovpn->keepalive_work, - (next_run - now) * HZ); + queue_delayed_work(ovpn_wq, &ovpn->keepalive_work, + (next_run - now) * HZ); } unlock_ovpn(ovpn, &release_list); } diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h index 4de5aeae33f7d..9a05a278e1cd0 100644 --- a/drivers/net/ovpn/peer.h +++ b/drivers/net/ovpn/peer.h @@ -148,6 +148,7 @@ struct ovpn_peer *ovpn_peer_get_by_id(struct ovpn_priv *ovpn, u32 peer_id); struct ovpn_peer *ovpn_peer_get_by_dst(struct ovpn_priv *ovpn, struct sk_buff *skb); void ovpn_peer_hash_vpn_ip(struct ovpn_peer *peer); +void ovpn_peer_hash_transp_addr(struct ovpn_peer *peer); bool ovpn_peer_check_by_src(struct ovpn_priv *ovpn, struct sk_buff *skb, struct ovpn_peer *peer); diff --git a/drivers/net/ovpn/socket.c b/drivers/net/ovpn/socket.c index 448cee3b3f9fa..e40e320a6abf5 100644 --- a/drivers/net/ovpn/socket.c +++ b/drivers/net/ovpn/socket.c @@ -162,6 +162,15 @@ struct ovpn_socket *ovpn_socket_new(struct socket *sock, struct ovpn_peer *peer) rcu_read_lock(); ovpn_sock = rcu_dereference_sk_user_data(sk); if (ovpn_sock) { + /* something else filled the sk_user_data without + * setting the encap_type. Reject the socket. + */ + if (!type) { + ovpn_sock = ERR_PTR(-EBUSY); + rcu_read_unlock(); + goto sock_release; + } + /* socket owned by another ovpn instance, we can't use it */ if (ovpn_sock->ovpn != peer->ovpn) { ovpn_sock = ERR_PTR(-EBUSY); diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c index 433bd07a4f1be..8fe8a8e750a40 100644 --- a/drivers/net/ovpn/tcp.c +++ b/drivers/net/ovpn/tcp.c @@ -151,7 +151,8 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb) /* take reference for deferred peer deletion. should never fail */ if (WARN_ON(!ovpn_peer_hold(peer))) goto err_nopeer; - schedule_work(&peer->tcp.defer_del_work); + if (!queue_work(ovpn_wq, &peer->tcp.defer_del_work)) + ovpn_peer_put(peer); ovpn_dev_dstats_rx_dropped(peer->ovpn->dev); err_nopeer: kfree_skb(skb); @@ -283,12 +284,12 @@ static void ovpn_tcp_send_sock(struct ovpn_peer *peer, struct sock *sk) * stream therefore we abort the connection */ ovpn_peer_hold(peer); - schedule_work(&peer->tcp.defer_del_work); + if (!queue_work(ovpn_wq, &peer->tcp.defer_del_work)) + ovpn_peer_put(peer); /* we bail out immediately and keep tx_in_progress set * to true. This way we prevent more TX attempts - * which would lead to more invocations of - * schedule_work() + * which would lead to more invocations of queue_work() */ return; } @@ -485,7 +486,7 @@ static void ovpn_tcp_write_space(struct sock *sk) rcu_read_lock(); sock = rcu_dereference_sk_user_data(sk); if (likely(sock && sock->peer)) { - schedule_work(&sock->tcp_tx_work); + queue_work(ovpn_wq, &sock->tcp_tx_work); sock->peer->tcp.sk_cb.sk_write_space(sk); } rcu_read_unlock(); diff --git a/drivers/net/pcs/pcs-xpcs.c b/drivers/net/pcs/pcs-xpcs.c index 3d1bd5aac0937..14c4b10fee74c 100644 --- a/drivers/net/pcs/pcs-xpcs.c +++ b/drivers/net/pcs/pcs-xpcs.c @@ -958,6 +958,7 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs, /* Reset link_state */ state->link = false; + state->an_complete = false; state->speed = SPEED_UNKNOWN; state->duplex = DUPLEX_UNKNOWN; state->pause = 0; @@ -969,6 +970,8 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs, if (ret < 0) return ret; + state->an_complete = ret & DW_VR_MII_AN_STS_C37_ANCMPLT_INTR; + if (ret & DW_VR_MII_C37_ANSGM_SP_LNKSTS) { int speed_value; @@ -986,34 +989,13 @@ static int xpcs_get_state_c37_sgmii(struct dw_xpcs *xpcs, state->duplex = DUPLEX_FULL; else state->duplex = DUPLEX_HALF; - } else if (ret == DW_VR_MII_AN_STS_C37_ANCMPLT_INTR) { - int speed, duplex; - - state->link = true; - - speed = xpcs_read(xpcs, MDIO_MMD_VEND2, MII_BMCR); - if (speed < 0) - return speed; - - speed &= BMCR_SPEED100 | BMCR_SPEED1000; - if (speed == BMCR_SPEED1000) - state->speed = SPEED_1000; - else if (speed == BMCR_SPEED100) - state->speed = SPEED_100; - else if (speed == 0) - state->speed = SPEED_10; - - duplex = xpcs_read(xpcs, MDIO_MMD_VEND2, MII_ADVERTISE); - if (duplex < 0) - return duplex; - if (duplex & ADVERTISE_1000XFULL) - state->duplex = DUPLEX_FULL; - else if (duplex & ADVERTISE_1000XHALF) - state->duplex = DUPLEX_HALF; + return 0; + } + /* Clear AN complete status or interrupt */ + if (state->an_complete) xpcs_write(xpcs, MDIO_MMD_VEND2, DW_VR_MII_AN_INTR_STS, 0); - } return 0; } diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c index 36a0c1b7f59c7..411fd65722743 100644 --- a/drivers/net/phy/dp83867.c +++ b/drivers/net/phy/dp83867.c @@ -1108,6 +1108,9 @@ static int dp83867_led_polarity_set(struct phy_device *phydev, int index, case PHY_LED_ACTIVE_LOW: polarity = 0; break; + case PHY_LED_ACTIVE_HIGH: + polarity = DP83867_LED_POLARITY(index); + break; default: return -EINVAL; } diff --git a/drivers/net/phy/dp83td510.c b/drivers/net/phy/dp83td510.c index 23af1ac194fa9..b9adee1039293 100644 --- a/drivers/net/phy/dp83td510.c +++ b/drivers/net/phy/dp83td510.c @@ -380,6 +380,9 @@ static int dp83td510_led_polarity_set(struct phy_device *phydev, int index, case PHY_LED_ACTIVE_LOW: polarity = 0; break; + case PHY_LED_ACTIVE_HIGH: + polarity = DP83TD510E_LED_POLARITY(index); + break; default: return -EINVAL; } diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c index c248c90510ae5..f191b8c432188 100644 --- a/drivers/net/phy/marvell.c +++ b/drivers/net/phy/marvell.c @@ -753,7 +753,7 @@ static int m88e1111_config_inband(struct phy_device *phydev, unsigned int modes) err = phy_modify(phydev, MII_M1111_PHY_EXT_SR, MII_M1111_HWCFG_SERIAL_AN_BYPASS, extsr); if (err < 0) - return extsr; + return err; return phy_modify_paged(phydev, MII_MARVELL_FIBER_PAGE, MII_BMCR, BMCR_ANENABLE, bmcr); diff --git a/drivers/net/phy/mediatek/mtk-ge.c b/drivers/net/phy/mediatek/mtk-ge.c index 73d9b72f9d9e2..96d8ac5154e5e 100644 --- a/drivers/net/phy/mediatek/mtk-ge.c +++ b/drivers/net/phy/mediatek/mtk-ge.c @@ -62,10 +62,38 @@ static void mtk_gephy_config_init(struct phy_device *phydev) FIELD_PREP(MTK_MCC_NEARECHO_OFFSET_MASK, 0x3)); } +static int mt7530_phy_probe(struct phy_device *phydev) +{ + /* The MT7530 internal GE PHY has broken EEE: with EEE advertised, + * some link partners fail to establish a stable link (on a 2-pair + * cable, 1000BASE-T training fails and the port loops instead of + * falling back). MediaTek recommends disabling EEE on this PHY. + * Clear the advertisement as early as possible, before anything + * can negotiate EEE with the link partner. + */ + return phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, 0); +} + static int mt7530_phy_config_init(struct phy_device *phydev) { + int ret; + mtk_gephy_config_init(phydev); + /* The probe() clear alone is not durable: phy_init_hw() replays only + * ->config_init after a PHY reset, with the register back at its + * EEE-advertising hardware default, and phy_probe() zeroes + * eee_disabled_modes (of_set_phy_eee_broken()) after ->probe already + * ran. Clear the advertisement again and mark EEE disabled, so that + * neither phylib nor userspace can re-enable it; dp83867 disables + * broken EEE from config_init() the same way. + */ + ret = phy_write_mmd(phydev, MDIO_MMD_AN, MDIO_AN_EEE_ADV, 0); + if (ret) + return ret; + + phy_disable_eee(phydev); + /* Increase post_update_timer */ phy_write_paged(phydev, MTK_PHY_PAGE_EXTENDED_3, MTK_PHY_RG_LPI_PCS_DSP_CTRL_REG11, 0x4b); @@ -100,6 +128,7 @@ static struct phy_driver mtk_gephy_driver[] = { { PHY_ID_MATCH_EXACT(MTK_GPHY_ID_MT7530), .name = "MediaTek MT7530 PHY", + .probe = mt7530_phy_probe, .config_init = mt7530_phy_config_init, /* Interrupts are handled by the switch, not the PHY * itself. diff --git a/drivers/net/phy/mediatek/mtk.h b/drivers/net/phy/mediatek/mtk.h index 320f76ffa81fa..79e09727b81bc 100644 --- a/drivers/net/phy/mediatek/mtk.h +++ b/drivers/net/phy/mediatek/mtk.h @@ -53,15 +53,15 @@ #define MTK_GPHY_LED_RX_BLINK_SET (MTK_PHY_LED_BLINK_1000RX | \ MTK_PHY_LED_BLINK_100RX | \ MTK_PHY_LED_BLINK_10RX) -#define MTK_GPHY_LED_TX_BLINK_SET (MTK_PHY_LED_BLINK_1000RX | \ - MTK_PHY_LED_BLINK_100RX | \ - MTK_PHY_LED_BLINK_10RX) +#define MTK_GPHY_LED_TX_BLINK_SET (MTK_PHY_LED_BLINK_1000TX | \ + MTK_PHY_LED_BLINK_100TX | \ + MTK_PHY_LED_BLINK_10TX) #define MTK_2P5GPHY_LED_ON_SET (MTK_PHY_LED_ON_LINK2500 | \ MTK_GPHY_LED_ON_SET) #define MTK_2P5GPHY_LED_RX_BLINK_SET (MTK_PHY_LED_BLINK_2500RX | \ MTK_GPHY_LED_RX_BLINK_SET) -#define MTK_2P5GPHY_LED_TX_BLINK_SET (MTK_PHY_LED_BLINK_2500RX | \ +#define MTK_2P5GPHY_LED_TX_BLINK_SET (MTK_PHY_LED_BLINK_2500TX | \ MTK_GPHY_LED_TX_BLINK_SET) #define MTK_PHY_LED_STATE_FORCE_ON 0 diff --git a/drivers/net/phy/motorcomm.c b/drivers/net/phy/motorcomm.c index b49897500a592..b76011b227c46 100644 --- a/drivers/net/phy/motorcomm.c +++ b/drivers/net/phy/motorcomm.c @@ -10,7 +10,7 @@ #include #include #include -#include +#include #define PHY_ID_YT8511 0x0000010a #define PHY_ID_YT8521 0x0000011a @@ -843,12 +843,12 @@ static u32 ytphy_get_delay_reg_value(struct phy_device *phydev, u16 *rxc_dly_en, u32 dflt) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; int tb_size_half = tb_size / 2; u32 val; int i; - if (of_property_read_u32(node, prop_name, &val)) + if (device_property_read_u32(dev, prop_name, &val)) goto err_dts_val; /* when rxc_dly_en is NULL, it is get the delay for tx, only half of @@ -992,12 +992,12 @@ static int yt8531_get_ds_map(struct phy_device *phydev, u32 cur) static int yt8531_set_ds(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; u32 ds_field_low, ds_field_hi, val; int ret, ds; /* set rgmii rx clk driver strength */ - if (!of_property_read_u32(node, "motorcomm,rx-clk-drv-microamp", &val)) { + if (!device_property_read_u32(dev, "motorcomm,rx-clk-drv-microamp", &val)) { ds = yt8531_get_ds_map(phydev, val); if (ds < 0) return dev_err_probe(&phydev->mdio.dev, ds, @@ -1014,7 +1014,7 @@ static int yt8531_set_ds(struct phy_device *phydev) return ret; /* set rgmii rx data driver strength */ - if (!of_property_read_u32(node, "motorcomm,rx-data-drv-microamp", &val)) { + if (!device_property_read_u32(dev, "motorcomm,rx-data-drv-microamp", &val)) { ds = yt8531_get_ds_map(phydev, val); if (ds < 0) return dev_err_probe(&phydev->mdio.dev, ds, @@ -1047,7 +1047,6 @@ static int yt8531_set_ds(struct phy_device *phydev) */ static int yt8521_probe(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; struct device *dev = &phydev->mdio.dev; struct yt8521_priv *priv; int chip_config; @@ -1097,7 +1096,7 @@ static int yt8521_probe(struct phy_device *phydev) return ret; } - if (of_property_read_u32(node, "motorcomm,clk-out-frequency-hz", &freq)) + if (device_property_read_u32(dev, "motorcomm,clk-out-frequency-hz", &freq)) freq = YTPHY_DTS_OUTPUT_CLK_DIS; if (phydev->drv->phy_id == PHY_ID_YT8521) { @@ -1165,11 +1164,11 @@ static int yt8521_probe(struct phy_device *phydev) static int yt8531_probe(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; u16 mask, val; u32 freq; - if (of_property_read_u32(node, "motorcomm,clk-out-frequency-hz", &freq)) + if (device_property_read_u32(dev, "motorcomm,clk-out-frequency-hz", &freq)) freq = YTPHY_DTS_OUTPUT_CLK_DIS; switch (freq) { @@ -1661,7 +1660,7 @@ static int yt8521_resume(struct phy_device *phydev) */ static int yt8521_config_init(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; int old_page; int ret = 0; @@ -1676,7 +1675,7 @@ static int yt8521_config_init(struct phy_device *phydev) goto err_restore_page; } - if (of_property_read_bool(node, "motorcomm,auto-sleep-disabled")) { + if (device_property_read_bool(dev, "motorcomm,auto-sleep-disabled")) { /* disable auto sleep */ ret = ytphy_modify_ext(phydev, YT8521_EXTREG_SLEEP_CONTROL1_REG, YT8521_ESC1R_SLEEP_SW, 0); @@ -1684,7 +1683,7 @@ static int yt8521_config_init(struct phy_device *phydev) goto err_restore_page; } - if (of_property_read_bool(node, "motorcomm,keep-pll-enabled")) { + if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled")) { /* enable RXC clock when no wire plug */ ret = ytphy_modify_ext(phydev, YT8521_CLOCK_GATING_REG, YT8521_CGR_RX_CLK_EN, 0); @@ -1797,14 +1796,14 @@ static int yt8521_led_hw_control_get(struct phy_device *phydev, u8 index, static int yt8531_config_init(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; int ret; ret = ytphy_rgmii_clk_delay_config_with_lock(phydev); if (ret < 0) return ret; - if (of_property_read_bool(node, "motorcomm,auto-sleep-disabled")) { + if (device_property_read_bool(dev, "motorcomm,auto-sleep-disabled")) { /* disable auto sleep */ ret = ytphy_modify_ext_with_lock(phydev, YT8521_EXTREG_SLEEP_CONTROL1_REG, @@ -1813,7 +1812,7 @@ static int yt8531_config_init(struct phy_device *phydev) return ret; } - if (of_property_read_bool(node, "motorcomm,keep-pll-enabled")) { + if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled")) { /* enable RXC clock when no wire plug */ ret = ytphy_modify_ext_with_lock(phydev, YT8521_CLOCK_GATING_REG, @@ -1840,7 +1839,7 @@ static int yt8531_config_init(struct phy_device *phydev) */ static void yt8531_link_change_notify(struct phy_device *phydev) { - struct device_node *node = phydev->mdio.dev.of_node; + struct device *dev = &phydev->mdio.dev; bool tx_clk_1000_inverted = false; bool tx_clk_100_inverted = false; bool tx_clk_10_inverted = false; @@ -1848,17 +1847,17 @@ static void yt8531_link_change_notify(struct phy_device *phydev) u16 val = 0; int ret; - if (of_property_read_bool(node, "motorcomm,tx-clk-adj-enabled")) + if (device_property_read_bool(dev, "motorcomm,tx-clk-adj-enabled")) tx_clk_adj_enabled = true; if (!tx_clk_adj_enabled) return; - if (of_property_read_bool(node, "motorcomm,tx-clk-10-inverted")) + if (device_property_read_bool(dev, "motorcomm,tx-clk-10-inverted")) tx_clk_10_inverted = true; - if (of_property_read_bool(node, "motorcomm,tx-clk-100-inverted")) + if (device_property_read_bool(dev, "motorcomm,tx-clk-100-inverted")) tx_clk_100_inverted = true; - if (of_property_read_bool(node, "motorcomm,tx-clk-1000-inverted")) + if (device_property_read_bool(dev, "motorcomm,tx-clk-1000-inverted")) tx_clk_1000_inverted = true; if (phydev->speed < 0) diff --git a/drivers/net/phy/mxl-86110.c b/drivers/net/phy/mxl-86110.c index 42a5fe3f115f4..20ea4ed636f05 100644 --- a/drivers/net/phy/mxl-86110.c +++ b/drivers/net/phy/mxl-86110.c @@ -7,6 +7,7 @@ */ #include +#include // For compatible LED definitions #include #include #include @@ -651,6 +652,67 @@ static int mxl86110_enable_led_activity_blink(struct phy_device *phydev) return ret; } +/** + * mxl86110_emulate_brcm_leds - Support LED configuration from Device Tree, in a + * a manner similar to the Broadcom Ethernet PHYs. + * @phydev: Pointer to the PHY device structure + * + * If the PHY DT node contains an led-modes property, it is interpreted as 2 + * integers - one for each LED - where each value is one of the following: + * 0=Link/activity + * 1=Speed + * 3=FDX + * 4=Off + * 5=On + * 8=Link + * 9=Activity + * Other values are undefined. + */ +static int mxl86110_emulate_brcm_leds(struct phy_device *phydev) +{ + int i, ret = 0; + static const u16 brcm_to_mxl[16] = { + [BCM_LED_MULTICOLOR_LINK_ACT] = 0x670, + [BCM_LED_MULTICOLOR_SPEED] = 0x62, + [BCM_LED_MULTICOLOR_FDX] = 0x1000, + [BCM_LED_MULTICOLOR_OFF] = 0x8000, + [BCM_LED_MULTICOLOR_ON] = 0xc000, + [BCM_LED_MULTICOLOR_LINK] = 0x70, + [BCM_LED_MULTICOLOR_ACT] = 0x2600, + }; + u32 led_modes[] = { BCM_LED_MULTICOLOR_LINK_ACT, BCM_LED_MULTICOLOR_LINK }; + struct device_node *np = phydev->mdio.dev.of_node; + + if (!of_property_read_u32_array(np, "led-modes", led_modes, 2)) { + int led_swap = of_property_read_bool(np, "led-swap") ? 1 : 0; + + for (i = 0; i < 2; i++) { + u16 led_mode = brcm_to_mxl[led_modes[i] & 0xf]; + int led_num = 1 + (i ^ led_swap); + + if (led_mode & 0x8000) { + // This is a static on/off + u16 set = MXL86110_COM_EXT_LED_GEN_CFG_LFE(led_num); + + if (led_mode & 0x4000) + set |= MXL86110_COM_EXT_LED_GEN_CFG_LFME(led_num); + ret = __mxl86110_modify_extended_reg(phydev, + MXL86110_COM_EXT_LED_GEN_CFG, + MXL86110_COM_EXT_LED_GEN_CFG_LFM(led_num), + set); + } else { + ret = __mxl86110_write_extended_reg(phydev, + MXL86110_LED0_CFG_REG + led_num, + led_mode); + } + if (ret < 0) + break; + } + } + + return ret; +} + /** * mxl86110_config_rgmii_delay() - configure RGMII delays * @phydev: pointer to the phy_device @@ -731,6 +793,10 @@ static int mxl86110_config_init(struct phy_device *phydev) if (ret < 0) goto out; + ret = mxl86110_emulate_brcm_leds(phydev); + if (ret < 0) + goto out; + ret = mxl86110_broadcast_cfg(phydev); out: diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index 9182443082158..848b6ec9bd85b 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -959,7 +959,7 @@ static unsigned int phylink_inband_caps(struct phylink *pl, return 0; pcs = pl->mac_ops->mac_select_pcs(pl->config, interface); - if (!pcs) + if (IS_ERR_OR_NULL(pcs)) return 0; return phylink_pcs_inband_caps(pcs, interface); @@ -1861,8 +1861,8 @@ struct phylink *phylink_create(struct phylink_config *config, } else if (config->type == PHYLINK_DEV) { pl->dev = config->dev; } else { - kfree(pl); - return ERR_PTR(-EINVAL); + ret = -EINVAL; + goto free_pl; } pl->mac_supports_eee_ops = phylink_mac_implements_lpi(mac_ops); @@ -1895,28 +1895,29 @@ struct phylink *phylink_create(struct phylink_config *config, phylink_validate(pl, pl->supported, &pl->link_config); ret = phylink_parse_mode(pl, fwnode); - if (ret < 0) { - kfree(pl); - return ERR_PTR(ret); - } + if (ret < 0) + goto free_pl; if (pl->cfg_link_an_mode == MLO_AN_FIXED) { ret = phylink_parse_fixedlink(pl, fwnode); - if (ret < 0) { - kfree(pl); - return ERR_PTR(ret); - } + if (ret < 0) + goto release_link_gpio; } pl->req_link_an_mode = pl->cfg_link_an_mode; ret = phylink_register_sfp(pl, fwnode); - if (ret < 0) { - kfree(pl); - return ERR_PTR(ret); - } + if (ret < 0) + goto release_link_gpio; return pl; + +release_link_gpio: + if (pl->link_gpio) + gpiod_put(pl->link_gpio); +free_pl: + kfree(pl); + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(phylink_create); diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c index 05df1a33b0e20..ef8c3b23bee88 100644 --- a/drivers/net/phy/realtek/realtek_main.c +++ b/drivers/net/phy/realtek/realtek_main.c @@ -1164,7 +1164,7 @@ static int rtlgen_write_mmd(struct phy_device *phydev, int devnum, u16 regnum, if (devnum == MDIO_MMD_VEND2) ret = rtlgen_write_vend2(phydev, regnum, val); else if (devnum == MDIO_MMD_AN && regnum == MDIO_AN_EEE_ADV) - ret = rtlgen_write_vend2(phydev, regnum, RTL_MDIO_AN_EEE_ADV); + ret = rtlgen_write_vend2(phydev, RTL_MDIO_AN_EEE_ADV, val); else ret = -EOPNOTSUPP; diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c index 9683f9c2cd882..af43530ffb2c3 100644 --- a/drivers/net/phy/sfp.c +++ b/drivers/net/phy/sfp.c @@ -583,6 +583,8 @@ static const struct sfp_quirk sfp_quirks[] = { SFP_QUIRK_S("OEM", "SFP-2.5G-T", sfp_quirk_oem_2_5g), SFP_QUIRK_S("OEM", "SFP-2.5G-BX10-D", sfp_quirk_2500basex), SFP_QUIRK_S("OEM", "SFP-2.5G-BX10-U", sfp_quirk_2500basex), + SFP_QUIRK_S("OEM", "SFP-2.5G-LH03-B", sfp_quirk_2500basex), + SFP_QUIRK_S("OEM", "SFP-2.5G-LH20-A", sfp_quirk_2500basex), SFP_QUIRK_F("OEM", "RTSFP-10", sfp_fixup_rollball_cc), SFP_QUIRK_F("OEM", "RTSFP-10G", sfp_fixup_rollball_cc), SFP_QUIRK_F("Turris", "RTSFP-2.5G", sfp_fixup_rollball), diff --git a/drivers/net/ppp/ppp_async.c b/drivers/net/ppp/ppp_async.c index c97406c6004d4..cde8bd78c70be 100644 --- a/drivers/net/ppp/ppp_async.c +++ b/drivers/net/ppp/ppp_async.c @@ -65,8 +65,6 @@ struct asyncppp { struct tasklet_struct tsk; - refcount_t refcnt; - struct completion dead; struct ppp_channel chan; /* interface to generic ppp layer */ unsigned char obuf[OBUFSIZE]; }; @@ -116,38 +114,6 @@ static const struct ppp_channel_ops async_ops = { * Routines implementing the PPP line discipline. */ -/* - * We have a potential race on dereferencing tty->disc_data, - * because the tty layer provides no locking at all - thus one - * cpu could be running ppp_asynctty_receive while another - * calls ppp_asynctty_close, which zeroes tty->disc_data and - * frees the memory that ppp_asynctty_receive is using. The best - * way to fix this is to use a rwlock in the tty struct, but for now - * we use a single global rwlock for all ttys in ppp line discipline. - * - * FIXME: this is no longer true. The _close path for the ldisc is - * now guaranteed to be sane. - */ -static DEFINE_RWLOCK(disc_data_lock); - -static struct asyncppp *ap_get(struct tty_struct *tty) -{ - struct asyncppp *ap; - - read_lock(&disc_data_lock); - ap = tty->disc_data; - if (ap != NULL) - refcount_inc(&ap->refcnt); - read_unlock(&disc_data_lock); - return ap; -} - -static void ap_put(struct asyncppp *ap) -{ - if (refcount_dec_and_test(&ap->refcnt)) - complete(&ap->dead); -} - /* * Called when a tty is put into PPP line discipline. Called in process * context. @@ -182,9 +148,6 @@ ppp_asynctty_open(struct tty_struct *tty) skb_queue_head_init(&ap->rqueue); tasklet_setup(&ap->tsk, ppp_async_process); - refcount_set(&ap->refcnt, 1); - init_completion(&ap->dead); - ap->chan.private = ap; ap->chan.ops = &async_ops; ap->chan.mtu = PPP_MRU; @@ -205,34 +168,18 @@ ppp_asynctty_open(struct tty_struct *tty) } /* - * Called when the tty is put into another line discipline - * or it hangs up. We have to wait for any cpu currently - * executing in any of the other ppp_asynctty_* routines to - * finish before we can call ppp_unregister_channel and free - * the asyncppp struct. This routine must be called from - * process context, not interrupt or softirq context. + * Called when the tty is put into another line discipline or it hangs up. + * This call is serialized against other ldisc functions. */ static void ppp_asynctty_close(struct tty_struct *tty) { - struct asyncppp *ap; + struct asyncppp *ap = tty->disc_data; - write_lock_irq(&disc_data_lock); - ap = tty->disc_data; - tty->disc_data = NULL; - write_unlock_irq(&disc_data_lock); if (!ap) return; - /* - * We have now ensured that nobody can start using ap from now - * on, but we have to wait for all existing users to finish. - * Note that ppp_unregister_channel ensures that no calls to - * our channel ops (i.e. ppp_async_send/ioctl) are in progress - * by the time it returns. - */ - if (!refcount_dec_and_test(&ap->refcnt)) - wait_for_completion(&ap->dead); + tty->disc_data = NULL; tasklet_kill(&ap->tsk); ppp_unregister_channel(&ap->chan); @@ -242,17 +189,6 @@ ppp_asynctty_close(struct tty_struct *tty) kfree(ap); } -/* - * Called on tty hangup in process context. - * - * Wait for I/O to driver to complete and unregister PPP channel. - * This is already done by the close routine, so just call that. - */ -static void ppp_asynctty_hangup(struct tty_struct *tty) -{ - ppp_asynctty_close(tty); -} - /* * Read does nothing - no data is ever available this way. * Pppd reads and writes packets via /dev/ppp instead. @@ -283,7 +219,7 @@ ppp_asynctty_write(struct tty_struct *tty, struct file *file, const u8 *buf, static int ppp_asynctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; int err, val; int __user *p = (int __user *)arg; @@ -324,7 +260,6 @@ ppp_asynctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) err = tty_mode_ioctl(tty, cmd, arg); } - ap_put(ap); return err; } @@ -333,7 +268,7 @@ static void ppp_asynctty_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, size_t count) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; unsigned long flags; if (!ap) @@ -343,21 +278,19 @@ ppp_asynctty_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, spin_unlock_irqrestore(&ap->recv_lock, flags); if (!skb_queue_empty(&ap->rqueue)) tasklet_schedule(&ap->tsk); - ap_put(ap); tty_unthrottle(tty); } static void ppp_asynctty_wakeup(struct tty_struct *tty) { - struct asyncppp *ap = ap_get(tty); + struct asyncppp *ap = tty->disc_data; clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); if (!ap) return; set_bit(XMIT_WAKEUP, &ap->xmit_flags); tasklet_schedule(&ap->tsk); - ap_put(ap); } @@ -367,7 +300,6 @@ static struct tty_ldisc_ops ppp_ldisc = { .name = "ppp", .open = ppp_asynctty_open, .close = ppp_asynctty_close, - .hangup = ppp_asynctty_hangup, .read = ppp_asynctty_read, .write = ppp_asynctty_write, .ioctl = ppp_asynctty_ioctl, @@ -812,11 +744,8 @@ process_input_packet(struct asyncppp *ap) err: /* frame had an error, remember that, reset SC_TOSS & SC_ESCAPE */ ap->state = SC_PREV_ERROR; - if (skb) { - /* make skb appear as freshly allocated */ - skb_trim(skb, 0); - skb_reserve(skb, - skb_headroom(skb)); - } + kfree_skb(skb); + ap->rpkt = NULL; } /* Called when the tty driver has data for us. Runs parallel with the diff --git a/drivers/net/ppp/ppp_generic.c b/drivers/net/ppp/ppp_generic.c index 507d216256c0d..14fa87a8e0834 100644 --- a/drivers/net/ppp/ppp_generic.c +++ b/drivers/net/ppp/ppp_generic.c @@ -134,7 +134,6 @@ struct ppp { int debug; /* debug flags 70 */ struct slcompress *vj; /* state for VJ header compression */ enum NPmode npmode[NUM_NP]; /* what to do with each net proto 78 */ - struct sk_buff *xmit_pending; /* a packet ready to go out 88 */ struct compressor *xcomp; /* transmit packet compressor 8c */ void *xc_state; /* its internal state 90 */ struct compressor *rcomp; /* receive decompressor 94 */ @@ -185,6 +184,7 @@ struct channel { struct list_head clist; /* link in list of channels per unit */ spinlock_t upl; /* protects `ppp' and 'bridge' */ struct channel __rcu *bridge; /* "bridged" ppp channel */ + struct rcu_head rcu; /* for RCU-deferred free of the channel */ #ifdef CONFIG_PPP_MULTILINK u8 avail; /* flag used in multilink stuff */ u8 had_frag; /* >= 1 fragments have been sent */ @@ -264,8 +264,8 @@ struct ppp_net { static int ppp_unattached_ioctl(struct net *net, struct ppp_file *pf, struct file *file, unsigned int cmd, unsigned long arg); static void ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb); -static void ppp_send_frame(struct ppp *ppp, struct sk_buff *skb); -static void ppp_push(struct ppp *ppp); +static int ppp_prepare_tx_skb(struct ppp *ppp, struct sk_buff **pskb); +static int ppp_push(struct ppp *ppp, struct sk_buff *skb); static void ppp_channel_push(struct channel *pch); static void ppp_receive_frame(struct ppp *ppp, struct sk_buff *skb, struct channel *pch); @@ -818,7 +818,9 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case PPPIOCSMRU: if (get_user(val, p)) break; + ppp_recv_lock(ppp); ppp->mru = val; + ppp_recv_unlock(ppp); err = 0; break; @@ -839,7 +841,9 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; case PPPIOCGFLAGS: + ppp_lock(ppp); val = ppp->flags | ppp->xstate | ppp->rstate; + ppp_unlock(ppp); if (put_user(val, p)) break; err = 0; @@ -863,7 +867,7 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) case PPPIOCSDEBUG: if (get_user(val, p)) break; - ppp->debug = val; + WRITE_ONCE(ppp->debug, val); err = 0; break; @@ -874,16 +878,16 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) break; case PPPIOCGIDLE32: - idle32.xmit_idle = (jiffies - ppp->last_xmit) / HZ; - idle32.recv_idle = (jiffies - ppp->last_recv) / HZ; - if (copy_to_user(argp, &idle32, sizeof(idle32))) + idle32.xmit_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_xmit))) / HZ; + idle32.recv_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_recv))) / HZ; + if (copy_to_user(argp, &idle32, sizeof(idle32))) break; err = 0; break; case PPPIOCGIDLE64: - idle64.xmit_idle = (jiffies - ppp->last_xmit) / HZ; - idle64.recv_idle = (jiffies - ppp->last_recv) / HZ; + idle64.xmit_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_xmit))) / HZ; + idle64.recv_idle = max(0L, (long)(jiffies - READ_ONCE(ppp->last_recv))) / HZ; if (copy_to_user(argp, &idle64, sizeof(idle64))) break; err = 0; @@ -924,7 +928,7 @@ static long ppp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (copy_to_user(argp, &npi, sizeof(npi))) break; } else { - ppp->npmode[i] = npi.mode; + WRITE_ONCE(ppp->npmode[i], npi.mode); /* we may be able to transmit more packets now (??) */ netif_wake_queue(ppp->dev); } @@ -1462,7 +1466,7 @@ ppp_start_xmit(struct sk_buff *skb, struct net_device *dev) goto outf; /* Drop, accept or reject the packet */ - switch (ppp->npmode[npi]) { + switch (READ_ONCE(ppp->npmode[npi])) { case NPMODE_PASS: break; case NPMODE_QUEUE: @@ -1490,7 +1494,7 @@ ppp_start_xmit(struct sk_buff *skb, struct net_device *dev) outf: kfree_skb(skb); - ++dev->stats.tx_dropped; + DEV_STATS_INC(dev, tx_dropped); return NETDEV_TX_OK; } @@ -1540,11 +1544,11 @@ ppp_net_siocdevprivate(struct net_device *dev, struct ifreq *ifr, static void ppp_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats64) { - stats64->rx_errors = dev->stats.rx_errors; - stats64->tx_errors = dev->stats.tx_errors; - stats64->rx_dropped = dev->stats.rx_dropped; - stats64->tx_dropped = dev->stats.tx_dropped; - stats64->rx_length_errors = dev->stats.rx_length_errors; + stats64->rx_errors = DEV_STATS_READ(dev, rx_errors); + stats64->tx_errors = DEV_STATS_READ(dev, tx_errors); + stats64->rx_dropped = DEV_STATS_READ(dev, rx_dropped); + stats64->tx_dropped = DEV_STATS_READ(dev, tx_dropped); + stats64->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors); dev_fetch_sw_netstats(stats64, dev->tstats); } @@ -1644,6 +1648,8 @@ static void ppp_setup(struct net_device *dev) dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; dev->priv_destructor = ppp_dev_priv_destructor; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; + dev->features = NETIF_F_SG | NETIF_F_FRAGLIST; + dev->hw_features = dev->features; netif_keep_dst(dev); } @@ -1652,26 +1658,44 @@ static void ppp_setup(struct net_device *dev) */ /* Called to do any work queued up on the transmit side that can now be done */ +static void ppp_xmit_flush(struct ppp *ppp) +{ + struct sk_buff *skb; + + while ((skb = skb_dequeue(&ppp->file.xq))) { + if (unlikely(!ppp_push(ppp, skb))) { + skb_queue_head(&ppp->file.xq, skb); + return; + } + } + /* If there's no work left to do, tell the core net code that we can + * accept some more. + */ + netif_wake_queue(ppp->dev); +} + static void __ppp_xmit_process(struct ppp *ppp, struct sk_buff *skb) { ppp_xmit_lock(ppp); - if (!ppp->closing) { - ppp_push(ppp); - - if (skb) + if (unlikely(ppp->closing)) { + kfree_skb(skb); + goto out; + } + if (unlikely(ppp_prepare_tx_skb(ppp, &skb))) + goto out; + /* Fastpath: No backlog, just send the new skb. */ + if (likely(skb_queue_empty(&ppp->file.xq))) { + if (unlikely(!ppp_push(ppp, skb))) { skb_queue_tail(&ppp->file.xq, skb); - while (!ppp->xmit_pending && - (skb = skb_dequeue(&ppp->file.xq))) - ppp_send_frame(ppp, skb); - /* If there's no work left to do, tell the core net - code that we can accept some more. */ - if (!ppp->xmit_pending && !skb_peek(&ppp->file.xq)) - netif_wake_queue(ppp->dev); - else netif_stop_queue(ppp->dev); - } else { - kfree_skb(skb); + } + goto out; } + + /* Slowpath: Enqueue the new skb and process backlog */ + skb_queue_tail(&ppp->file.xq, skb); + ppp_xmit_flush(ppp); +out: ppp_xmit_unlock(ppp); } @@ -1713,6 +1737,10 @@ pad_compress_skb(struct ppp *ppp, struct sk_buff *skb) ppp->xcomp->comp_extra + ppp->dev->hard_header_len; int compressor_skb_size = ppp->dev->mtu + ppp->xcomp->comp_extra + PPP_HDRLEN; + + if (skb_linearize(skb)) + return NULL; + new_skb = alloc_skb(new_skb_size, GFP_ATOMIC); if (!new_skb) { if (net_ratelimit()) @@ -1754,13 +1782,15 @@ pad_compress_skb(struct ppp *ppp, struct sk_buff *skb) } /* - * Compress and send a frame. - * The caller should have locked the xmit path, - * and xmit_pending should be 0. + * Compress and prepare to send a frame. + * The caller should have locked the xmit path. + * Returns 1 if the skb was consumed, 0 if it can be passed to ppp_push(). + * @pskb is updated if a compressor is in use. */ -static void -ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) +static int +ppp_prepare_tx_skb(struct ppp *ppp, struct sk_buff **pskb) { + struct sk_buff *skb = *pskb; int proto = PPP_PROTO(skb); struct sk_buff *new_skb; int len; @@ -1776,21 +1806,21 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) *(__be16 *)skb_push(skb, 2) = htons(PPP_FILTER_OUTBOUND_TAG); if (ppp->pass_filter && bpf_prog_run(ppp->pass_filter, skb) == 0) { - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, "PPP: outbound frame " "not passed\n"); kfree_skb(skb); - return; + return 1; } /* if this packet passes the active filter, record the time */ if (!(ppp->active_filter && bpf_prog_run(ppp->active_filter, skb) == 0)) - ppp->last_xmit = jiffies; + WRITE_ONCE(ppp->last_xmit, jiffies); skb_pull(skb, 2); #else /* for data packets, record the time */ - ppp->last_xmit = jiffies; + WRITE_ONCE(ppp->last_xmit, jiffies); #endif /* CONFIG_PPP_FILTER */ } @@ -1800,6 +1830,10 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) case PPP_IP: if (!ppp->vj || (ppp->flags & SC_COMP_TCP) == 0) break; + + if (skb_linearize(skb)) + goto drop; + /* try to do VJ TCP header compression */ new_skb = alloc_skb(skb->len + ppp->dev->hard_header_len - 2, GFP_ATOMIC); @@ -1825,6 +1859,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) } consume_skb(skb); skb = new_skb; + *pskb = skb; cp = skb_put(skb, len + 2); cp[0] = 0; cp[1] = proto; @@ -1851,6 +1886,7 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) if (!new_skb) goto drop; skb = new_skb; + *pskb = skb; } /* @@ -1862,67 +1898,69 @@ ppp_send_frame(struct ppp *ppp, struct sk_buff *skb) goto drop; skb_queue_tail(&ppp->file.rq, skb); wake_up_interruptible(&ppp->file.rwait); - return; + return 1; } - ppp->xmit_pending = skb; - ppp_push(ppp); - return; + return 0; drop: kfree_skb(skb); - ++ppp->dev->stats.tx_errors; + DEV_STATS_INC(ppp->dev, tx_errors); + return 1; } /* - * Try to send the frame in xmit_pending. + * Try to send the frame. * The caller should have the xmit path locked. + * Returns 1 if the skb was consumed, 0 if not. */ -static void -ppp_push(struct ppp *ppp) +static int +ppp_push(struct ppp *ppp, struct sk_buff *skb) { struct list_head *list; struct channel *pch; - struct sk_buff *skb = ppp->xmit_pending; - - if (!skb) - return; list = &ppp->channels; if (list_empty(list)) { /* nowhere to send the packet, just drop it */ - ppp->xmit_pending = NULL; kfree_skb(skb); - return; + return 1; } if ((ppp->flags & SC_MULTILINK) == 0) { + struct ppp_channel *chan; + int ret; /* not doing multilink: send it down the first channel */ list = list->next; pch = list_entry(list, struct channel, clist); spin_lock(&pch->downl); - if (pch->chan) { - if (pch->chan->ops->start_xmit(pch->chan, skb)) - ppp->xmit_pending = NULL; - } else { - /* channel got unregistered */ + chan = pch->chan; + if (unlikely(!chan || (!chan->direct_xmit && skb_linearize(skb)))) { + /* channel got unregistered, or it requires a linear + * skb but linearization failed + */ kfree_skb(skb); - ppp->xmit_pending = NULL; + ret = 1; + goto out; } + + ret = chan->ops->start_xmit(chan, skb); + +out: spin_unlock(&pch->downl); - return; + return ret; } #ifdef CONFIG_PPP_MULTILINK /* Multilink: fragment the packet over as many links as can take the packet at the moment. */ if (!ppp_mp_explode(ppp, skb)) - return; + return 0; #endif /* CONFIG_PPP_MULTILINK */ - ppp->xmit_pending = NULL; kfree_skb(skb); + return 1; } #ifdef CONFIG_PPP_MULTILINK @@ -1991,9 +2029,11 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb) * performance if we have a lot of channels. */ if (nfree == 0 || nfree < navail / 2) - return 0; /* can't take now, leave it in xmit_pending */ + return 0; /* can't take now, leave it in transmit queue */ /* Do protocol field compression */ + if (skb_linearize(skb)) + goto err_linearize; p = skb->data; len = skb->len; if (*p == 0 && mp_protocol_compress) { @@ -2152,9 +2192,10 @@ static int ppp_mp_explode(struct ppp *ppp, struct sk_buff *skb) noskb: spin_unlock(&pch->downl); - if (ppp->debug & 1) + err_linearize: + if (READ_ONCE(ppp->debug) & 1) netdev_err(ppp->dev, "PPP: no memory (fragment)\n"); - ++ppp->dev->stats.tx_errors; + DEV_STATS_INC(ppp->dev, tx_errors); ++ppp->nxseq; return 1; /* abandon the frame */ } @@ -2182,8 +2223,12 @@ static void __ppp_channel_push(struct channel *pch, struct ppp *ppp) spin_unlock(&pch->downl); /* see if there is anything from the attached unit to be sent */ if (skb_queue_empty(&pch->file.xq)) { - if (ppp) - __ppp_xmit_process(ppp, NULL); + if (ppp) { + ppp_xmit_lock(ppp); + if (!ppp->closing) + ppp_xmit_flush(ppp); + ppp_xmit_unlock(ppp); + } } } @@ -2323,7 +2368,7 @@ ppp_input(struct ppp_channel *chan, struct sk_buff *skb) if (!ppp_decompress_proto(skb)) { kfree_skb(skb); if (ppp) { - ++ppp->dev->stats.rx_length_errors; + DEV_STATS_INC(ppp->dev, rx_length_errors); ppp_receive_error(ppp); } goto done; @@ -2396,7 +2441,7 @@ ppp_receive_frame(struct ppp *ppp, struct sk_buff *skb, struct channel *pch) static void ppp_receive_error(struct ppp *ppp) { - ++ppp->dev->stats.rx_errors; + DEV_STATS_INC(ppp->dev, rx_errors); if (ppp->vj) slhc_toss(ppp->vj); } @@ -2507,7 +2552,7 @@ ppp_receive_nonmp_frame(struct ppp *ppp, struct sk_buff *skb) *(__be16 *)skb_push(skb, 2) = htons(PPP_FILTER_INBOUND_TAG); if (ppp->pass_filter && bpf_prog_run(ppp->pass_filter, skb) == 0) { - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, "PPP: inbound frame " "not passed\n"); @@ -2516,14 +2561,14 @@ ppp_receive_nonmp_frame(struct ppp *ppp, struct sk_buff *skb) } if (!(ppp->active_filter && bpf_prog_run(ppp->active_filter, skb) == 0)) - ppp->last_recv = jiffies; + WRITE_ONCE(ppp->last_recv, jiffies); __skb_pull(skb, 2); } else #endif /* CONFIG_PPP_FILTER */ - ppp->last_recv = jiffies; + WRITE_ONCE(ppp->last_recv, jiffies); if ((ppp->dev->flags & IFF_UP) == 0 || - ppp->npmode[npi] != NPMODE_PASS) { + READ_ONCE(ppp->npmode[npi]) != NPMODE_PASS) { kfree_skb(skb); } else { /* chop off protocol */ @@ -2663,7 +2708,7 @@ ppp_receive_mp_frame(struct ppp *ppp, struct sk_buff *skb, struct channel *pch) */ if (seq_before(seq, ppp->nextseq)) { kfree_skb(skb); - ++ppp->dev->stats.rx_dropped; + DEV_STATS_INC(ppp->dev, rx_dropped); ppp_receive_error(ppp); return; } @@ -2699,7 +2744,7 @@ ppp_receive_mp_frame(struct ppp *ppp, struct sk_buff *skb, struct channel *pch) if (pskb_may_pull(skb, 2)) ppp_receive_nonmp_frame(ppp, skb); else { - ++ppp->dev->stats.rx_length_errors; + DEV_STATS_INC(ppp->dev, rx_length_errors); kfree_skb(skb); ppp_receive_error(ppp); } @@ -2776,7 +2821,7 @@ ppp_mp_reconstruct(struct ppp *ppp) seq = seq_before(minseq, PPP_MP_CB(p)->sequence)? minseq + 1: PPP_MP_CB(p)->sequence; - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, "lost frag %u..%u\n", oldseq, seq-1); @@ -2805,7 +2850,7 @@ ppp_mp_reconstruct(struct ppp *ppp) if (lost == 0 && (PPP_MP_CB(p)->BEbits & E) && (PPP_MP_CB(head)->BEbits & B)) { if (len > ppp->mrru + 2) { - ++ppp->dev->stats.rx_length_errors; + DEV_STATS_INC(ppp->dev, rx_length_errors); netdev_printk(KERN_DEBUG, ppp->dev, "PPP: reconstructed packet" " is too long (%d)\n", len); @@ -2825,7 +2870,7 @@ ppp_mp_reconstruct(struct ppp *ppp) struct sk_buff *tmp2; skb_queue_reverse_walk_from_safe(list, p, tmp2) { - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, "discarding frag %u\n", PPP_MP_CB(p)->sequence); @@ -2847,7 +2892,7 @@ ppp_mp_reconstruct(struct ppp *ppp) skb_queue_walk_safe(list, p, tmp) { if (p == head) break; - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, "discarding frag %u\n", PPP_MP_CB(p)->sequence); @@ -2855,12 +2900,12 @@ ppp_mp_reconstruct(struct ppp *ppp) kfree_skb(p); } - if (ppp->debug & 1) + if (READ_ONCE(ppp->debug) & 1) netdev_printk(KERN_DEBUG, ppp->dev, " missed pkts %u..%u\n", ppp->nextseq, PPP_MP_CB(head)->sequence-1); - ++ppp->dev->stats.rx_dropped; + DEV_STATS_INC(ppp->dev, rx_dropped); ppp_receive_error(ppp); } @@ -3168,7 +3213,8 @@ ppp_ccp_peek(struct ppp *ppp, struct sk_buff *skb, int inbound) if (!ppp->rc_state) break; if (ppp->rcomp->decomp_init(ppp->rc_state, dp, len, - ppp->file.index, 0, ppp->mru, ppp->debug)) { + ppp->file.index, 0, ppp->mru, + READ_ONCE(ppp->debug))) { ppp->rstate |= SC_DECOMP_RUN; ppp->rstate &= ~(SC_DC_ERROR | SC_DC_FERROR); } @@ -3177,7 +3223,8 @@ ppp_ccp_peek(struct ppp *ppp, struct sk_buff *skb, int inbound) if (!ppp->xc_state) break; if (ppp->xcomp->comp_init(ppp->xc_state, dp, len, - ppp->file.index, 0, ppp->debug)) + ppp->file.index, 0, + READ_ONCE(ppp->debug))) ppp->xstate |= SC_COMP_RUN; } break; @@ -3329,8 +3376,8 @@ ppp_get_stats(struct ppp *ppp, struct ppp_stats *st) st->p.ppp_opackets += tx_packets; st->p.ppp_obytes += tx_bytes; } - st->p.ppp_ierrors = ppp->dev->stats.rx_errors; - st->p.ppp_oerrors = ppp->dev->stats.tx_errors; + st->p.ppp_ierrors = DEV_STATS_READ(ppp->dev, rx_errors); + st->p.ppp_oerrors = DEV_STATS_READ(ppp->dev, tx_errors); if (!vj) return; st->vj.vjs_packets = vj->sls_o_compressed + vj->sls_o_uncompressed; @@ -3443,7 +3490,6 @@ static void ppp_destroy_interface(struct ppp *ppp) } #endif /* CONFIG_PPP_FILTER */ - kfree_skb(ppp->xmit_pending); free_percpu(ppp->xmit_recursion); free_netdev(ppp->dev); @@ -3570,6 +3616,18 @@ ppp_disconnect_channel(struct channel *pch) return err; } +/* Purge after the grace period: a late ppp_input() may still queue an + * skb on pch->file.rq before the last RCU reader drains. + */ +static void ppp_release_channel_free(struct rcu_head *rcu) +{ + struct channel *pch = container_of(rcu, struct channel, rcu); + + skb_queue_purge(&pch->file.xq); + skb_queue_purge(&pch->file.rq); + kfree(pch); +} + /* * Free up the resources used by a ppp channel. */ @@ -3585,9 +3643,7 @@ static void ppp_destroy_channel(struct channel *pch) pr_err("ppp: destroying undead channel %p !\n", pch); return; } - skb_queue_purge(&pch->file.xq); - skb_queue_purge(&pch->file.rq); - kfree(pch); + call_rcu(&pch->rcu, ppp_release_channel_free); } static void __exit ppp_cleanup(void) @@ -3600,6 +3656,7 @@ static void __exit ppp_cleanup(void) device_destroy(&ppp_class, MKDEV(PPP_MAJOR, 0)); class_unregister(&ppp_class); unregister_pernet_device(&ppp_net_ops); + rcu_barrier(); /* wait for RCU callbacks before module unload */ } /* diff --git a/drivers/net/ppp/ppp_synctty.c b/drivers/net/ppp/ppp_synctty.c index 9c4932198931f..4514bcadd6eef 100644 --- a/drivers/net/ppp/ppp_synctty.c +++ b/drivers/net/ppp/ppp_synctty.c @@ -38,11 +38,9 @@ #include #include #include -#include #include #include #include -#include #include #include @@ -69,8 +67,6 @@ struct syncppp { struct tasklet_struct tsk; - refcount_t refcnt; - struct completion dead_cmp; struct ppp_channel chan; /* interface to generic ppp layer */ }; @@ -118,37 +114,6 @@ ppp_print_buffer (const char *name, const __u8 *buf, int count) * Routines implementing the synchronous PPP line discipline. */ -/* - * We have a potential race on dereferencing tty->disc_data, - * because the tty layer provides no locking at all - thus one - * cpu could be running ppp_synctty_receive while another - * calls ppp_synctty_close, which zeroes tty->disc_data and - * frees the memory that ppp_synctty_receive is using. The best - * way to fix this is to use a rwlock in the tty struct, but for now - * we use a single global rwlock for all ttys in ppp line discipline. - * - * FIXME: Fixed in tty_io nowadays. - */ -static DEFINE_RWLOCK(disc_data_lock); - -static struct syncppp *sp_get(struct tty_struct *tty) -{ - struct syncppp *ap; - - read_lock(&disc_data_lock); - ap = tty->disc_data; - if (ap != NULL) - refcount_inc(&ap->refcnt); - read_unlock(&disc_data_lock); - return ap; -} - -static void sp_put(struct syncppp *ap) -{ - if (refcount_dec_and_test(&ap->refcnt)) - complete(&ap->dead_cmp); -} - /* * Called when a tty is put into sync-PPP line discipline. */ @@ -179,9 +144,6 @@ ppp_sync_open(struct tty_struct *tty) skb_queue_head_init(&ap->rqueue); tasklet_setup(&ap->tsk, ppp_sync_process); - refcount_set(&ap->refcnt, 1); - init_completion(&ap->dead_cmp); - ap->chan.private = ap; ap->chan.ops = &sync_ops; ap->chan.mtu = PPP_MRU; @@ -203,34 +165,18 @@ ppp_sync_open(struct tty_struct *tty) } /* - * Called when the tty is put into another line discipline - * or it hangs up. We have to wait for any cpu currently - * executing in any of the other ppp_synctty_* routines to - * finish before we can call ppp_unregister_channel and free - * the syncppp struct. This routine must be called from - * process context, not interrupt or softirq context. + * Called when the tty is put into another line discipline or it hangs up. + * This call is serialized against other ldisc functions. */ static void ppp_sync_close(struct tty_struct *tty) { - struct syncppp *ap; + struct syncppp *ap = tty->disc_data; - write_lock_irq(&disc_data_lock); - ap = tty->disc_data; - tty->disc_data = NULL; - write_unlock_irq(&disc_data_lock); if (!ap) return; - /* - * We have now ensured that nobody can start using ap from now - * on, but we have to wait for all existing users to finish. - * Note that ppp_unregister_channel ensures that no calls to - * our channel ops (i.e. ppp_sync_send/ioctl) are in progress - * by the time it returns. - */ - if (!refcount_dec_and_test(&ap->refcnt)) - wait_for_completion(&ap->dead_cmp); + tty->disc_data = NULL; tasklet_kill(&ap->tsk); ppp_unregister_channel(&ap->chan); @@ -239,17 +185,6 @@ ppp_sync_close(struct tty_struct *tty) kfree(ap); } -/* - * Called on tty hangup in process context. - * - * Wait for I/O to driver to complete and unregister PPP channel. - * This is already done by the close routine, so just call that. - */ -static void ppp_sync_hangup(struct tty_struct *tty) -{ - ppp_sync_close(tty); -} - /* * Read does nothing - no data is ever available this way. * Pppd reads and writes packets via /dev/ppp instead. @@ -275,7 +210,7 @@ ppp_sync_write(struct tty_struct *tty, struct file *file, const u8 *buf, static int ppp_synctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; int __user *p = (int __user *)arg; int err, val; @@ -316,7 +251,6 @@ ppp_synctty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) break; } - sp_put(ap); return err; } @@ -325,7 +259,7 @@ static void ppp_sync_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, size_t count) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; unsigned long flags; if (!ap) @@ -335,21 +269,19 @@ ppp_sync_receive(struct tty_struct *tty, const u8 *buf, const u8 *cflags, spin_unlock_irqrestore(&ap->recv_lock, flags); if (!skb_queue_empty(&ap->rqueue)) tasklet_schedule(&ap->tsk); - sp_put(ap); tty_unthrottle(tty); } static void ppp_sync_wakeup(struct tty_struct *tty) { - struct syncppp *ap = sp_get(tty); + struct syncppp *ap = tty->disc_data; clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); if (!ap) return; set_bit(XMIT_WAKEUP, &ap->xmit_flags); tasklet_schedule(&ap->tsk); - sp_put(ap); } @@ -359,7 +291,6 @@ static struct tty_ldisc_ops ppp_sync_ldisc = { .name = "pppsync", .open = ppp_sync_open, .close = ppp_sync_close, - .hangup = ppp_sync_hangup, .read = ppp_sync_read, .write = ppp_sync_write, .ioctl = ppp_synctty_ioctl, @@ -526,17 +457,9 @@ ppp_sync_txmunge(struct syncppp *ap, struct sk_buff *skb) /* prepend address/control fields if necessary */ if ((ap->flags & SC_COMP_AC) == 0 || islcp) { - if (skb_headroom(skb) < 2) { - struct sk_buff *npkt = dev_alloc_skb(skb->len + 2); - if (npkt == NULL) { - kfree_skb(skb); - return NULL; - } - skb_reserve(npkt,2); - skb_copy_from_linear_data(skb, - skb_put(npkt, skb->len), skb->len); - consume_skb(skb); - skb = npkt; + if (skb_cow_head(skb, 2)) { + kfree_skb(skb); + return NULL; } skb_push(skb,2); skb->data[0] = PPP_ALLSTATIONS; diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c index dc6d139432387..7ee63051a1619 100644 --- a/drivers/net/ppp/pppoe.c +++ b/drivers/net/ppp/pppoe.c @@ -903,6 +903,7 @@ static int pppoe_sendmsg(struct socket *sock, struct msghdr *m, dev_hard_header(skb, dev, ETH_P_PPP_SES, po->pppoe_pa.remote, NULL, total_len); + ph = pppoe_hdr(skb); memcpy(ph, &hdr, sizeof(struct pppoe_hdr)); ph->length = htons(total_len); diff --git a/drivers/net/ppp/pppox.c b/drivers/net/ppp/pppox.c index 08364f10a43fa..61b3453469dce 100644 --- a/drivers/net/ppp/pppox.c +++ b/drivers/net/ppp/pppox.c @@ -74,7 +74,9 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) switch (cmd) { case PPPIOCGCHAN: { + struct sk_buff *skb; int index; + rc = -ENOTCONN; if (!(sk->sk_state & PPPOX_CONNECTED)) break; @@ -85,7 +87,22 @@ int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) break; rc = 0; + /* PPPIOCGCHAN historically marks the userspace handoff to + * generic PPP; pppd then attaches the returned channel to + * /dev/ppp. + */ sk->sk_state |= PPPOX_BOUND; + /* Let lockless receive paths finish queueing against the old + * state. + */ + synchronize_net(); + /* Drain packets queued before the handoff because a bound + * socket is no longer readable. + */ + while ((skb = skb_dequeue(&sk->sk_receive_queue))) { + skb_orphan(skb); + ppp_input(&po->chan, skb); + } break; } default: diff --git a/drivers/net/slip/slip.c b/drivers/net/slip/slip.c index c889fb3747030..d16897686d1b5 100644 --- a/drivers/net/slip/slip.c +++ b/drivers/net/slip/slip.c @@ -628,9 +628,15 @@ static void sl_uninit(struct net_device *dev) struct slip *sl = netdev_priv(dev); sl_free_bufs(sl); + /* Drop the slip_devs[] entry here rather than from the destructor: + * ndo_uninit runs under RTNL, so it cannot race sl_sync(). + */ + slip_devs[dev->base_addr] = NULL; } -/* Hook the destructor so we can free slip devices at the right point in time */ +/* Only for the slip_open() error path: register_netdevice() can fail before + * ndo_init, and then ndo_uninit is not called either. + */ static void sl_free_netdev(struct net_device *dev) { int i = dev->base_addr; @@ -657,7 +663,6 @@ static void sl_setup(struct net_device *dev) { dev->netdev_ops = &sl_netdev_ops; dev->needs_free_netdev = true; - dev->priv_destructor = sl_free_netdev; dev->hard_header_len = 0; dev->addr_len = 0; @@ -693,6 +698,8 @@ static void slip_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, if (!sl || sl->magic != SLIP_MAGIC || !netif_running(sl->dev)) return; + spin_lock_bh(&sl->lock); + /* Read the characters out of the buffer */ while (count--) { if (fp && *fp++) { @@ -708,6 +715,8 @@ static void slip_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, #endif slip_unesc(sl, *cp++); } + + spin_unlock_bh(&sl->lock); } /************************************ @@ -877,8 +886,6 @@ static int slip_open(struct tty_struct *tty) * Close down a SLIP channel. * This means flushing out any pending queues, and then returning. This * call is serialized against other ldisc functions. - * - * We also use this method fo a hangup event */ static void slip_close(struct tty_struct *tty) @@ -904,12 +911,7 @@ static void slip_close(struct tty_struct *tty) #endif /* Flush network side */ unregister_netdev(sl->dev); - /* This will complete via sl_free_netdev */ -} - -static void slip_hangup(struct tty_struct *tty) -{ - slip_close(tty); + /* sl_uninit() has dropped the slip_devs[] entry by now */ } /************************************************************************ * STANDARD SLIP ENCAPSULATION * @@ -1271,7 +1273,6 @@ static struct tty_ldisc_ops sl_ldisc = { .name = "slip", .open = slip_open, .close = slip_close, - .hangup = slip_hangup, .ioctl = slip_ioctl, .receive_buf = slip_receive_buf, .write_wakeup = slip_write_wakeup, diff --git a/drivers/net/tap.c b/drivers/net/tap.c index b51ce7af1b20f..1aeaecd4030fa 100644 --- a/drivers/net/tap.c +++ b/drivers/net/tap.c @@ -1074,26 +1074,31 @@ static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp) skb_reset_mac_header(skb); skb->protocol = eth_hdr(skb)->h_proto; + rcu_read_lock(); + tap = rcu_dereference(q->tap); + if (!tap) { + kfree_skb(skb); + rcu_read_unlock(); + return 0; + } + skb->dev = tap->dev; + if (vnet_hdr_len) { err = tun_vnet_hdr_to_skb(q->flags, skb, gso); - if (err) + if (err) { + rcu_read_unlock(); goto err_kfree; + } } + skb_probe_transport_header(skb); + /* Move network header to the right position for VLAN tagged packets */ if (eth_type_vlan(skb->protocol) && vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) skb_set_network_header(skb, depth); - rcu_read_lock(); - tap = rcu_dereference(q->tap); - if (tap) { - skb->dev = tap->dev; - skb_probe_transport_header(skb); - dev_queue_xmit(skb); - } else { - kfree_skb(skb); - } + dev_queue_xmit(skb); rcu_read_unlock(); return 0; diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c index 5d7d11d75b0b8..096f3ff36a437 100644 --- a/drivers/net/thunderbolt/main.c +++ b/drivers/net/thunderbolt/main.c @@ -389,11 +389,16 @@ static void tbnet_tear_down(struct tbnet *net, bool send_logout) break; } - tb_ring_stop(net->rx_ring.ring); - tb_ring_stop(net->tx_ring.ring); - tbnet_free_buffers(&net->rx_ring); - tbnet_free_buffers(&net->tx_ring); - + /* Tear the paths down before stopping the rings. This mirrors + * tbnet_connected_work(), which enables the paths last so the + * Rx ring is primed before packets can arrive. Stopping a + * ring zeroes its descriptor base and tbnet_free_buffers() + * unmaps and frees the frame buffers, leaving anything still + * in flight with nowhere to drain to; + * __tb_path_deactivate_hop() then waits for the hop's + * 'pending' bit, which on some host routers never clears in + * that state. + */ ret = tb_xdomain_disable_paths(net->xd, net->local_transmit_path, net->tx_ring.ring->hop, @@ -402,6 +407,11 @@ static void tbnet_tear_down(struct tbnet *net, bool send_logout) if (ret) netdev_warn(net->dev, "failed to disable DMA paths\n"); + tb_ring_stop(net->rx_ring.ring); + tb_ring_stop(net->tx_ring.ring); + tbnet_free_buffers(&net->rx_ring); + tbnet_free_buffers(&net->tx_ring); + tb_xdomain_release_in_hopid(net->xd, net->remote_transmit_path); net->remote_transmit_path = 0; } @@ -619,6 +629,14 @@ static int tbnet_alloc_tx_buffers(struct tbnet *net) return 0; } +static void tbnet_connect_failed(struct tbnet *net) +{ + /* Leave login_received set: only the peer can make it true again. */ + mutex_lock(&net->connection_lock); + net->login_sent = false; + mutex_unlock(&net->connection_lock); +} + static void tbnet_connected_work(struct work_struct *work) { struct tbnet *net = container_of(work, typeof(*net), connected_work); @@ -640,6 +658,9 @@ static void tbnet_connected_work(struct work_struct *work) ret = tb_xdomain_alloc_in_hopid(net->xd, net->remote_transmit_path); if (ret != net->remote_transmit_path) { netdev_err(net->dev, "failed to allocate Rx HopID\n"); + if (ret >= 0) + tb_xdomain_release_in_hopid(net->xd, ret); + tbnet_connect_failed(net); return; } @@ -684,6 +705,7 @@ static void tbnet_connected_work(struct work_struct *work) tb_ring_stop(net->rx_ring.ring); tb_ring_stop(net->tx_ring.ring); tb_xdomain_release_in_hopid(net->xd, net->remote_transmit_path); + tbnet_connect_failed(net); } static void tbnet_login_work(struct work_struct *work) @@ -885,9 +907,11 @@ static int tbnet_poll(struct napi_struct *napi, int budget) le32_to_cpu(net->rx_hdr.frame_count) - 1; rx_packets++; - net->stats.rx_bytes += frame_size; if (last) { + /* Before eth_type_trans() pulls the Ethernet header. */ + net->stats.rx_packets++; + net->stats.rx_bytes += skb->len; skb->protocol = eth_type_trans(skb, net->dev); trace_tbnet_rx_skb(skb); napi_gro_receive(&net->napi, skb); @@ -895,8 +919,6 @@ static int tbnet_poll(struct napi_struct *napi, int budget) } } - net->stats.rx_packets += rx_packets; - if (cleaned_count) tbnet_alloc_rx_buffers(net, cleaned_count); @@ -928,12 +950,8 @@ static int tbnet_open(struct net_device *dev) netif_carrier_off(dev); - flags = RING_FLAG_FRAME; - /* Only enable full E2E if the other end supports it too */ - if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E) - flags |= RING_FLAG_E2E; - - ring = tb_ring_alloc_tx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags); + ring = tb_ring_alloc_tx(xd->tb->nhi, -1, TBNET_RING_SIZE, + RING_FLAG_FRAME); if (!ring) { netdev_err(dev, "failed to allocate Tx ring\n"); return -ENOMEM; @@ -952,6 +970,11 @@ static int tbnet_open(struct net_device *dev) sof_mask = BIT(TBIP_PDF_FRAME_START); eof_mask = BIT(TBIP_PDF_FRAME_END); + flags = RING_FLAG_FRAME; + /* Only enable full E2E if the other end supports it too */ + if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E) + flags |= RING_FLAG_E2E; + ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags, net->tx_ring.ring->hop, sof_mask, eof_mask, tbnet_start_poll, net); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index d27c3229465ad..172273ae65754 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -1105,11 +1105,16 @@ static netdev_features_t tun_net_fix_features(struct net_device *dev, static void tun_set_headroom(struct net_device *dev, int new_hr) { struct tun_struct *tun = netdev_priv(dev); + size_t max_headroom; - if (new_hr < NET_SKB_PAD) - new_hr = NET_SKB_PAD; + max_headroom = min_t(size_t, SKB_MAX_HEAD(0), U16_MAX - 1); - tun->align = new_hr; + if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) + max_headroom -= ETH_HLEN + NET_IP_ALIGN; + else + max_headroom -= 1; + + tun->align = clamp_t(int, new_hr, NET_SKB_PAD, max_headroom); } static void @@ -1820,7 +1825,13 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, switch (tun->flags & TUN_TYPE_MASK) { case IFF_TUN: if (tun->flags & IFF_NO_PI) { - u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0; + u8 ip_version; + + if (!pskb_may_pull(skb, 1)) { + err = -EINVAL; + goto drop; + } + ip_version = skb->data[0] >> 4; switch (ip_version) { case 4: @@ -1840,7 +1851,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, skb->dev = tun->dev; break; case IFF_TAP: - if (frags && !pskb_may_pull(skb, ETH_HLEN)) { + if (!pskb_may_pull(skb, ETH_HLEN)) { err = -ENOMEM; drop_reason = SKB_DROP_REASON_HDR_TRUNC; goto drop; diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index b034ef8a73ea7..cf40b604f135d 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -1493,8 +1493,10 @@ ax88179_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) headroom = skb_headroom(skb) - 8; - if ((dev->net->features & NETIF_F_SG) && skb_linearize(skb)) + if ((dev->net->features & NETIF_F_SG) && skb_linearize(skb)) { + dev_kfree_skb_any(skb); return NULL; + } if ((skb_header_cloned(skb) || headroom < 0) && pskb_expand_head(skb, headroom < 0 ? 8 : 0, 0, GFP_ATOMIC)) { diff --git a/drivers/net/usb/cx82310_eth.c b/drivers/net/usb/cx82310_eth.c index 79a47e2fd4378..840d014ac1aa8 100644 --- a/drivers/net/usb/cx82310_eth.c +++ b/drivers/net/usb/cx82310_eth.c @@ -282,6 +282,7 @@ static int cx82310_rx_fixup(struct usbnet *dev, struct sk_buff *skb) if (len == 0xffff) { netdev_info(dev->net, "router was rebooted, re-enabling ethernet mode"); schedule_work(&priv->reenable_work); + return 0; } else if (len > CX82310_MTU) { netdev_err(dev->net, "RX packet too long: %d B\n", len); return 0; diff --git a/drivers/net/usb/hso.c b/drivers/net/usb/hso.c index 1aeb36119d3f4..9876152bae4f9 100644 --- a/drivers/net/usb/hso.c +++ b/drivers/net/usb/hso.c @@ -1549,10 +1549,10 @@ hso_wait_modem_status(struct hso_serial *serial, unsigned long arg) spin_unlock_irq(&serial->serial_lock); add_wait_queue(&tiocmget->waitq, &wait); for (;;) { + set_current_state(TASK_INTERRUPTIBLE); spin_lock_irq(&serial->serial_lock); memcpy(&cnow, &tiocmget->icount, sizeof(struct uart_icount)); spin_unlock_irq(&serial->serial_lock); - set_current_state(TASK_INTERRUPTIBLE); if (((arg & TIOCM_RNG) && (cnow.rng != cprev.rng)) || ((arg & TIOCM_DSR) && (cnow.dsr != cprev.dsr)) || ((arg & TIOCM_CD) && (cnow.dcd != cprev.dcd))) { diff --git a/drivers/net/usb/ipheth.c b/drivers/net/usb/ipheth.c index a19789b571905..a86d0a4dbd285 100644 --- a/drivers/net/usb/ipheth.c +++ b/drivers/net/usb/ipheth.c @@ -490,6 +490,7 @@ static int ipheth_open(struct net_device *net) if (retval) return retval; + enable_delayed_work(&dev->carrier_work); schedule_delayed_work(&dev->carrier_work, IPHETH_CARRIER_CHECK_TIMEOUT); return retval; } @@ -499,7 +500,11 @@ static int ipheth_close(struct net_device *net) struct ipheth_device *dev = netdev_priv(net); netif_stop_queue(net); - cancel_delayed_work_sync(&dev->carrier_work); + /* A TX URB can still complete with an error after this point and + * try to re-arm the carrier work. Disable it instead of cancelling + * it, so that such a schedule_delayed_work() is a no-op. + */ + disable_delayed_work_sync(&dev->carrier_work); return 0; } @@ -633,6 +638,10 @@ static int ipheth_probe(struct usb_interface *intf, } INIT_DELAYED_WORK(&dev->carrier_work, ipheth_carrier_check_work); + /* Armed only between ipheth_open() and ipheth_close(). Start out + * disabled so the enable/disable counts balance from the first open. + */ + disable_delayed_work(&dev->carrier_work); retval = ipheth_alloc_urbs(dev); if (retval) { diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c index 475b066081c7f..8ec798f97f99b 100644 --- a/drivers/net/usb/pegasus.c +++ b/drivers/net/usb/pegasus.c @@ -46,21 +46,12 @@ static bool loopback; static bool mii_mode; static char *devid; -static struct usb_eth_dev usb_dev_id[] = { -#define PEGASUS_DEV(pn, vid, pid, flags) \ - {.name = pn, .vendor = vid, .device = pid, .private = flags}, -#define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \ - PEGASUS_DEV(pn, vid, pid, flags) -#include "pegasus.h" -#undef PEGASUS_DEV -#undef PEGASUS_DEV_CLASS - {NULL, 0, 0, 0}, - {NULL, 0, 0, 0} -}; +static struct usb_eth_dev dynamic_id_info = {}; static struct usb_device_id pegasus_ids[] = { #define PEGASUS_DEV(pn, vid, pid, flags) \ - {.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid}, + {.match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = vid, .idProduct = pid, \ + .driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}}, /* * The Belkin F8T012xx1 bluetooth adaptor has the same vendor and product * IDs as the Belkin F5D5050, so we need to teach the pegasus driver to @@ -69,7 +60,8 @@ static struct usb_device_id pegasus_ids[] = { */ #define PEGASUS_DEV_CLASS(pn, vid, pid, dclass, flags) \ {.match_flags = (USB_DEVICE_ID_MATCH_DEVICE | USB_DEVICE_ID_MATCH_DEV_CLASS), \ - .idVendor = vid, .idProduct = pid, .bDeviceClass = dclass}, + .idVendor = vid, .idProduct = pid, .bDeviceClass = dclass, \ + .driver_info = (kernel_ulong_t)&(const struct usb_eth_dev) {.name = pn, .private = flags}}, #include "pegasus.h" #undef PEGASUS_DEV #undef PEGASUS_DEV_CLASS @@ -405,12 +397,12 @@ static inline int reset_mac(pegasus_t *pegasus) if (i == REG_TIMEOUT) return -ETIMEDOUT; - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) { set_register(pegasus, Gpio0, 0x24); set_register(pegasus, Gpio0, 0x26); } - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_ELCON) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_ELCON) { __u16 auxmode; ret = read_mii_word(pegasus, 3, 0x1b, &auxmode); if (ret < 0) @@ -448,9 +440,9 @@ static int enable_net_traffic(struct net_device *dev, struct usb_device *usb) memcpy(pegasus->eth_regs, data, sizeof(data)); ret = set_registers(pegasus, EthCtrl0, 3, data); - if (usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_LINKSYS2 || - usb_dev_id[pegasus->dev_index].vendor == VENDOR_DLINK) { + if (le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_LINKSYS2 || + le16_to_cpu(pegasus->usb->descriptor.idVendor) == VENDOR_DLINK) { u16 auxmode; ret = read_mii_word(pegasus, 0, 0x1b, &auxmode); if (ret < 0) @@ -1156,7 +1148,7 @@ static int pegasus_probe(struct usb_interface *intf, struct usb_device *dev = interface_to_usbdev(intf); struct net_device *net; pegasus_t *pegasus; - int dev_index = id - pegasus_ids; + const struct usb_eth_dev *info = (const struct usb_eth_dev *)id->driver_info; int res = -ENOMEM; static const u8 bulk_ep_addr[] = { PEGASUS_USB_EP_BULK_IN | USB_DIR_IN, @@ -1181,7 +1173,6 @@ static int pegasus_probe(struct usb_interface *intf, goto out; pegasus = netdev_priv(net); - pegasus->dev_index = dev_index; pegasus->intf = intf; res = alloc_urbs(pegasus); @@ -1209,7 +1200,7 @@ static int pegasus_probe(struct usb_interface *intf, pegasus->msg_enable = netif_msg_init(msg_level, NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK); - pegasus->features = usb_dev_id[dev_index].private; + pegasus->features = info ? info->private : DEFAULT_GPIO_RESET; res = get_interrupt_interval(pegasus); if (res) goto out2; @@ -1238,7 +1229,7 @@ static int pegasus_probe(struct usb_interface *intf, queue_delayed_work(system_long_wq, &pegasus->carrier_check, CARRIER_CHECK_DELAY); dev_info(&intf->dev, "%s, %s, %pM\n", net->name, - usb_dev_id[dev_index].name, net->dev_addr); + info ? info->name : "(unknown)", net->dev_addr); return 0; out3: @@ -1328,8 +1319,9 @@ static struct usb_driver pegasus_driver = { static void __init parse_id(char *id) { - unsigned int vendor_id = 0, device_id = 0, flags = 0, i = 0; + unsigned int vendor_id = 0, device_id = 0, flags = 0; char *token, *name = NULL; + int dyn_id_index = ARRAY_SIZE(pegasus_ids) - 2; if ((token = strsep(&id, ":")) != NULL) name = token; @@ -1347,14 +1339,12 @@ static void __init parse_id(char *id) if (device_id > 0x10000 || device_id == 0) return; - for (i = 0; usb_dev_id[i].name; i++); - usb_dev_id[i].name = name; - usb_dev_id[i].vendor = vendor_id; - usb_dev_id[i].device = device_id; - usb_dev_id[i].private = flags; - pegasus_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE; - pegasus_ids[i].idVendor = vendor_id; - pegasus_ids[i].idProduct = device_id; + dynamic_id_info.name = name; + dynamic_id_info.private = flags; + pegasus_ids[dyn_id_index].match_flags = USB_DEVICE_ID_MATCH_DEVICE; + pegasus_ids[dyn_id_index].idVendor = vendor_id; + pegasus_ids[dyn_id_index].idProduct = device_id; + pegasus_ids[dyn_id_index].driver_info = (kernel_ulong_t)&dynamic_id_info; } static int __init pegasus_init(void) diff --git a/drivers/net/usb/pegasus.h b/drivers/net/usb/pegasus.h index a05b143155ba8..ccdedcef52e76 100644 --- a/drivers/net/usb/pegasus.h +++ b/drivers/net/usb/pegasus.h @@ -85,7 +85,6 @@ typedef struct pegasus { unsigned features; u32 msg_enable; u32 wolopts; - int dev_index; int intr_interval; struct tasklet_struct rx_tl; struct delayed_work carrier_check; @@ -102,8 +101,6 @@ typedef struct pegasus { struct usb_eth_dev { char *name; - __u16 vendor; - __u16 device; __u32 private; /* LSB is gpio reset value */ }; diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c index 05acac10cd2ba..8178a8758cd3f 100644 --- a/drivers/net/usb/qmi_wwan.c +++ b/drivers/net/usb/qmi_wwan.c @@ -1360,6 +1360,7 @@ static const struct usb_device_id products[] = { {QMI_FIXED_INTF(0x1bbb, 0x0203, 2)}, /* Alcatel L800MA */ {QMI_FIXED_INTF(0x2357, 0x0201, 4)}, /* TP-LINK HSUPA Modem MA180 */ {QMI_FIXED_INTF(0x2357, 0x9000, 4)}, /* TP-LINK MA260 */ + {QMI_QUIRK_SET_DTR(0x1bc7, 0x0991, 0)}, /* Telit FE990D50 */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1031, 3)}, /* Telit LE910C1-EUX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1034, 2)}, /* Telit LE910C4-WWX */ {QMI_QUIRK_SET_DTR(0x1bc7, 0x1037, 4)}, /* Telit LE910C4-WWX */ @@ -1452,6 +1453,8 @@ static const struct usb_device_id products[] = { {QMI_QUIRK_SET_DTR(0x1546, 0x1342, 4)}, /* u-blox LARA-L6 */ {QMI_QUIRK_SET_DTR(0x33f8, 0x0104, 4)}, /* Rolling RW101 RMNET */ {QMI_FIXED_INTF(0x2dee, 0x4d22, 5)}, /* MeiG Smart SRM825L */ + {QMI_QUIRK_SET_DTR(0x2dee, 0x4d63, 0)}, /* MeiG SRM813Q w/ Modem(PPP) */ + {QMI_QUIRK_SET_DTR(0x2dee, 0x4d64, 0)}, /* MeiG SRM813Q w/ NMEA */ /* 4. Gobi 1000 devices */ {QMI_GOBI1K_DEVICE(0x05c6, 0x9212)}, /* Acer Gobi Modem Device */ diff --git a/drivers/net/usb/rndis_host.c b/drivers/net/usb/rndis_host.c index 7b3739b29c8f7..1d7e9a1a3cf1d 100644 --- a/drivers/net/usb/rndis_host.c +++ b/drivers/net/usb/rndis_host.c @@ -14,6 +14,7 @@ #include #include #include +#include /* @@ -506,6 +507,7 @@ int rndis_rx_fixup(struct usbnet *dev, struct sk_buff *skb) struct rndis_data_hdr *hdr = (void *)skb->data; struct sk_buff *skb2; u32 msg_type, msg_len, data_offset, data_len; + u32 overflow_check; msg_type = le32_to_cpu(hdr->msg_type); msg_len = le32_to_cpu(hdr->msg_len); @@ -514,7 +516,9 @@ int rndis_rx_fixup(struct usbnet *dev, struct sk_buff *skb) /* don't choke if we see oob, per-packet data, etc */ if (unlikely(msg_type != RNDIS_MSG_PACKET || skb->len < msg_len - || (data_offset + data_len + 8) > msg_len)) { + || (data_offset + data_len + 8) > msg_len + || check_add_overflow(data_offset, data_len, &overflow_check) + || check_add_overflow(overflow_check, 8, &overflow_check))) { dev->net->stats.rx_frame_errors++; netdev_dbg(dev->net, "bad rndis message %d/%d/%d/%d, len %d\n", le32_to_cpu(hdr->msg_type), diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 4595c61c0790b..56519b646cacd 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -756,7 +756,7 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, u32 frame_sz; if (skb_shared(skb) || skb_head_is_locked(skb) || - skb_shinfo(skb)->nr_frags || + skb_is_nonlinear(skb) || skb_headroom(skb) < XDP_PACKET_HEADROOM) { if (skb_pp_cow_data(rq->page_pool, pskb, XDP_PACKET_HEADROOM)) goto drop; @@ -771,7 +771,7 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, xdp_prepare_buff(xdp, skb->head, skb_headroom(skb), skb_headlen(skb), true); - if (skb_is_nonlinear(skb)) { + if (skb_shinfo(skb)->nr_frags) { skb_shinfo(skb)->xdp_frags_size = skb->data_len; xdp_buff_set_frags_flag(xdp); } else { @@ -865,18 +865,24 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, skb_reset_mac_header(skb); - /* check if bpf_xdp_adjust_tail was used */ - off = xdp->data_end - orig_data_end; - if (off != 0) - __skb_put(skb, off); /* positive on grow, negative on shrink */ - /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers - * (e.g. bpf_xdp_adjust_tail), we need to update data_len here. + * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution + * from skb->len before updating data_len, then add the new one back. */ - if (xdp_buff_has_frags(xdp)) + skb->len -= skb->data_len; + if (xdp_buff_has_frags(xdp)) { skb->data_len = skb_shinfo(skb)->xdp_frags_size; - else + skb->len += skb->data_len; + } else { skb->data_len = 0; + } + + /* Synchronize the skb tail with XDP's updated linear area. */ + off = xdp->data_end - orig_data_end; + if (off != 0) { + skb_set_tail_pointer(skb, xdp->data_end - xdp->data); + skb->len += off; /* positive on grow, negative on shrink */ + } skb->protocol = eth_type_trans(skb, rq->dev); @@ -961,7 +967,7 @@ static int veth_poll(struct napi_struct *napi, int budget) struct veth_rq *rq = container_of(napi, struct veth_rq, xdp_napi); struct veth_priv *priv = netdev_priv(rq->dev); - int queue_idx = rq->xdp_rxq.queue_index; + int queue_idx = rq - priv->rq; struct netdev_queue *peer_txq; struct veth_stats stats = {}; struct net_device *peer_dev; @@ -972,7 +978,8 @@ static int veth_poll(struct napi_struct *napi, int budget) /* NAPI functions as RCU section */ peer_dev = rcu_dereference_check(priv->peer, rcu_read_lock_bh_held()); - peer_txq = peer_dev ? netdev_get_tx_queue(peer_dev, queue_idx) : NULL; + peer_txq = (peer_dev && queue_idx < peer_dev->real_num_tx_queues) ? + netdev_get_tx_queue(peer_dev, queue_idx) : NULL; xdp_set_return_frame_no_direct(); done = veth_xdp_rcv(rq, budget, &bq, &stats); diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c index 0572f6a9bdb62..679208d587f85 100644 --- a/drivers/net/vmxnet3/vmxnet3_drv.c +++ b/drivers/net/vmxnet3/vmxnet3_drv.c @@ -1530,7 +1530,11 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, struct ipv6hdr *ipv6; struct tcphdr *tcp; } hdr; - BUG_ON(gdesc->rcd.tcp == 0); + + /* v4/v6/tcp then describe the inner header, which we can't locate. */ + if ((le32_to_cpu(gdesc->dword[0]) & (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)) || + gdesc->rcd.tcp == 0) + return 0; maplen = skb_headlen(skb); if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) @@ -1544,15 +1548,21 @@ vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, hdr.eth = eth_hdr(skb); if (gdesc->rcd.v4) { - BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && - hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); + if (hdr.eth->h_proto != htons(ETH_P_IP) && + hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)) + return 0; + hdr.ptr += hlen; - BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); + if (hdr.ipv4->protocol != IPPROTO_TCP) + return 0; + hlen = hdr.ipv4->ihl << 2; hdr.ptr += hdr.ipv4->ihl << 2; } else if (gdesc->rcd.v6) { - BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && - hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); + if (hdr.eth->h_proto != htons(ETH_P_IPV6) && + hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)) + return 0; + hdr.ptr += hlen; /* Use an estimated value, since we also need to handle * TSO case. diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c index b706e1777354c..1e411a57d1fe7 100644 --- a/drivers/net/vxlan/vxlan_core.c +++ b/drivers/net/vxlan/vxlan_core.c @@ -997,6 +997,12 @@ static int vxlan_fdb_update_existing(struct vxlan_dev *vxlan, return -EOPNOTSUPP; } + if (rcu_access_pointer(f->nh) && + !(state & (NUD_PERMANENT | NUD_NOARP))) { + NL_SET_ERR_MSG(extack, "Cannot make a nexthop fdb dynamic"); + return -EOPNOTSUPP; + } + /* Do not allow an externally learned entry to take over an entry added * by the user. */ @@ -1258,6 +1264,11 @@ static int vxlan_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], if (err) return err; + if (nhid && !(ndm->ndm_state & (NUD_PERMANENT | NUD_NOARP))) { + NL_SET_ERR_MSG(extack, "A nexthop fdb cannot be dynamic"); + return -EINVAL; + } + if (vxlan->default_dst.remote_ip.sa.sa_family != ip.sa.sa_family) return -EAFNOSUPPORT; @@ -1852,7 +1863,7 @@ static int arp_reduce(struct net_device *dev, struct sk_buff *skb, __be32 vni) if (dev->flags & IFF_NOARP) goto out; - if (!pskb_may_pull(skb, arp_hdr_len(dev))) { + if (!pskb_network_may_pull(skb, arp_hdr_len(dev))) { dev_dstats_tx_dropped(dev); vxlan_vnifilter_count(vxlan, vni, NULL, VXLAN_VNI_STATS_TX_DROPS, 0); @@ -2115,7 +2126,7 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) { struct iphdr *pip; - if (!pskb_may_pull(skb, sizeof(struct iphdr))) + if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) return false; pip = ip_hdr(skb); n = neigh_lookup(&arp_tbl, &pip->daddr, dev); @@ -2141,7 +2152,7 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) */ if (!ipv6_stub->nd_tbl) return false; - if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) + if (!pskb_network_may_pull(skb, sizeof(struct ipv6hdr))) return false; pip6 = ipv6_hdr(skb); n = neigh_lookup(ipv6_stub->nd_tbl, &pip6->daddr, dev); @@ -2163,13 +2174,19 @@ static bool route_shortcircuit(struct net_device *dev, struct sk_buff *skb) } if (n) { + u8 haddr[ETH_ALEN]; bool diff; - diff = !ether_addr_equal(eth_hdr(skb)->h_dest, n->ha); + neigh_ha_snapshot(haddr, n, dev); + diff = !ether_addr_equal_unaligned(eth_hdr(skb)->h_dest, haddr); if (diff) { + if (skb_cow_head(skb, 0)) { + neigh_release(n); + return false; + } memcpy(eth_hdr(skb)->h_source, eth_hdr(skb)->h_dest, dev->addr_len); - memcpy(eth_hdr(skb)->h_dest, n->ha, dev->addr_len); + memcpy(eth_hdr(skb)->h_dest, haddr, dev->addr_len); } neigh_release(n); return diff; @@ -2352,7 +2369,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev, struct ip_tunnel_key key; struct vxlan_dev *vxlan = netdev_priv(dev); const struct iphdr *old_iph; - struct vxlan_metadata _md; + struct vxlan_metadata _md = {}; struct vxlan_metadata *md = &_md; unsigned int pkt_len = skb->len; __be16 src_port = 0, dst_port; @@ -2759,8 +2776,8 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev) return arp_reduce(dev, skb, vni); #if IS_ENABLED(CONFIG_IPV6) else if (ntohs(eth->h_proto) == ETH_P_IPV6 && - pskb_may_pull(skb, sizeof(struct ipv6hdr) + - sizeof(struct nd_msg)) && + pskb_network_may_pull(skb, sizeof(struct ipv6hdr) + + sizeof(struct nd_msg)) && ipv6_hdr(skb)->nexthdr == IPPROTO_ICMPV6) { struct nd_msg *m = (struct nd_msg *)(ipv6_hdr(skb) + 1); @@ -2798,6 +2815,7 @@ static netdev_tx_t vxlan_xmit(struct sk_buff *skb, struct net_device *dev) (ntohs(eth->h_proto) == ETH_P_IP || ntohs(eth->h_proto) == ETH_P_IPV6)) { did_rsc = route_shortcircuit(dev, skb); + eth = eth_hdr(skb); if (did_rsc) f = vxlan_find_mac_tx(vxlan, eth->h_dest, vni); } @@ -3053,18 +3071,19 @@ vxlan_fdb_flush_match_remotes(struct vxlan_fdb *f, struct vxlan_dev *vxlan, const struct vxlan_fdb_flush_desc *desc, bool *p_destroy_fdb) { - bool remotes_flushed = false; struct vxlan_rdst *rd, *tmp; list_for_each_entry_safe(rd, tmp, &f->remotes, list) { if (!vxlan_fdb_flush_remote_matches(desc, rd)) continue; + if (list_is_singular(&f->remotes)) { + *p_destroy_fdb = true; + return; + } + vxlan_fdb_dst_destroy(vxlan, f, rd, true); - remotes_flushed = true; } - - *p_destroy_fdb = remotes_flushed && list_empty(&f->remotes); } /* Purge the forwarding table */ @@ -4415,6 +4434,9 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], struct vxlan_rdst *dst; int err; + if (!rtnl_dev_link_net_capable(dev, vxlan->net)) + return -EPERM; + dst = &vxlan->default_dst; err = vxlan_nl2conf(tb, data, dev, &conf, true, extack); if (err) @@ -4484,7 +4506,7 @@ static int vxlan_changelink(struct net_device *dev, struct nlattr *tb[], if (change_igmp && vxlan_addr_multicast(&dst->remote_ip)) err = vxlan_multicast_leave(vxlan); - if (conf.age_interval != vxlan->cfg.age_interval) + if (netif_running(dev) && conf.age_interval != vxlan->cfg.age_interval) mod_timer(&vxlan->age_timer, jiffies); netdev_adjacent_change_commit(dst->remote_dev, lowerdev, dev); diff --git a/drivers/net/vxlan/vxlan_mdb.c b/drivers/net/vxlan/vxlan_mdb.c index 816ab1aa05262..167e921828c15 100644 --- a/drivers/net/vxlan/vxlan_mdb.c +++ b/drivers/net/vxlan/vxlan_mdb.c @@ -42,6 +42,7 @@ struct vxlan_mdb_remote { }; #define VXLAN_SGRP_F_DELETE BIT(0) +#define VXLAN_SGRP_F_NEW BIT(1) struct vxlan_mdb_src_entry { struct hlist_node node; @@ -353,6 +354,10 @@ static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto, NL_SET_ERR_MSG_MOD(extack, "IPv4 multicast source address is not allowed"); return false; } + if (ipv4_is_zeronet(nla_get_in_addr(attr))) { + NL_SET_ERR_MSG_MOD(extack, "IPv4 all-zeros source address is not allowed"); + return false; + } break; #if IS_ENABLED(CONFIG_IPV6) case htons(ETH_P_IPV6): { @@ -367,6 +372,10 @@ static bool vxlan_mdb_is_valid_source(const struct nlattr *attr, __be16 proto, NL_SET_ERR_MSG_MOD(extack, "IPv6 multicast source address is not allowed"); return false; } + if (ipv6_addr_any(&src)) { + NL_SET_ERR_MSG_MOD(extack, "IPv6 all-zeros source address is not allowed"); + return false; + } break; } #endif @@ -844,6 +853,7 @@ vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg, ent = vxlan_mdb_remote_src_entry_add(remote, &src->addr); if (!ent) return -ENOMEM; + ent->flags |= VXLAN_SGRP_F_NEW; } else if (!(cfg->nlflags & NLM_F_REPLACE)) { NL_SET_ERR_MSG_MOD(extack, "Source entry already exists"); return -EEXIST; @@ -853,15 +863,16 @@ vxlan_mdb_remote_src_add(const struct vxlan_mdb_config *cfg, if (err) goto err_src_del; - /* Clear flags in case source entry was marked for deletion as part of - * replace flow. + /* Clear the deletion mark so the entry survives the replace sweep. + * The new mark is retained until the whole operation succeeds. */ - ent->flags = 0; + ent->flags &= ~VXLAN_SGRP_F_DELETE; return 0; err_src_del: - vxlan_mdb_remote_src_entry_del(ent); + if (ent->flags & VXLAN_SGRP_F_NEW) + vxlan_mdb_remote_src_entry_del(ent); return err; } @@ -889,11 +900,19 @@ static int vxlan_mdb_remote_srcs_add(const struct vxlan_mdb_config *cfg, goto err_src_del; } + hlist_for_each_entry(ent, &remote->src_list, node) + ent->flags &= ~VXLAN_SGRP_F_NEW; + return 0; err_src_del: - hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) - vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote, ent); + hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) { + if (ent->flags & VXLAN_SGRP_F_NEW) + vxlan_mdb_remote_src_del(cfg->vxlan, &cfg->group, remote, + ent); + else + ent->flags &= ~VXLAN_SGRP_F_DELETE; + } return err; } @@ -1069,7 +1088,7 @@ vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg, err = vxlan_mdb_remote_srcs_add(cfg, remote, extack); if (err) - goto err_clear_delete; + return err; hlist_for_each_entry_safe(ent, tmp, &remote->src_list, node) { if (ent->flags & VXLAN_SGRP_F_DELETE) @@ -1078,11 +1097,6 @@ vxlan_mdb_remote_srcs_replace(const struct vxlan_mdb_config *cfg, } return 0; - -err_clear_delete: - hlist_for_each_entry(ent, &remote->src_list, node) - ent->flags &= ~VXLAN_SGRP_F_DELETE; - return err; } static int vxlan_mdb_remote_replace(const struct vxlan_mdb_config *cfg, @@ -1422,14 +1436,17 @@ static void vxlan_mdb_flush(struct vxlan_dev *vxlan, struct vxlan_mdb_entry *mdb_entry; struct hlist_node *tmp; - /* The removal of an entry cannot trigger the removal of another entry - * since entries are always added to the head of the list. - */ hlist_for_each_entry_safe(mdb_entry, tmp, &vxlan->mdb_list, mdb_node) { if (desc->src_vni && desc->src_vni != mdb_entry->key.vni) continue; vxlan_mdb_remotes_flush(vxlan, mdb_entry, desc); + /* The flush can remove the (S, G) entries created for the + * source list of this entry, including the one saved by + * hlist_for_each_entry_safe(), so re-read it while this entry + * is still linked. + */ + tmp = mdb_entry->mdb_node.next; /* Entry will only be removed if its remotes list is empty. */ vxlan_mdb_entry_put(vxlan, mdb_entry); } @@ -1625,7 +1642,7 @@ struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan, switch (skb->protocol) { case htons(ETH_P_IP): - if (!pskb_may_pull(skb, sizeof(struct iphdr))) + if (!pskb_network_may_pull(skb, sizeof(struct iphdr))) return NULL; group.dst.sa.sa_family = AF_INET; group.dst.sin.sin_addr.s_addr = ip_hdr(skb)->daddr; @@ -1634,7 +1651,7 @@ struct vxlan_mdb_entry *vxlan_mdb_entry_skb_get(struct vxlan_dev *vxlan, break; #if IS_ENABLED(CONFIG_IPV6) case htons(ETH_P_IPV6): - if (!pskb_may_pull(skb, sizeof(struct ipv6hdr))) + if (!pskb_network_may_pull(skb, sizeof(struct ipv6hdr))) return NULL; group.dst.sa.sa_family = AF_INET6; group.dst.sin6.sin6_addr = ipv6_hdr(skb)->daddr; diff --git a/drivers/net/vxlan/vxlan_vnifilter.c b/drivers/net/vxlan/vxlan_vnifilter.c index 215e82876662d..3f2d2396243a7 100644 --- a/drivers/net/vxlan/vxlan_vnifilter.c +++ b/drivers/net/vxlan/vxlan_vnifilter.c @@ -462,10 +462,8 @@ static int vxlan_vnifilter_dump(struct sk_buff *skb, struct netlink_callback *cb static const struct nla_policy vni_filter_entry_policy[VXLAN_VNIFILTER_ENTRY_MAX + 1] = { [VXLAN_VNIFILTER_ENTRY_START] = { .type = NLA_U32 }, [VXLAN_VNIFILTER_ENTRY_END] = { .type = NLA_U32 }, - [VXLAN_VNIFILTER_ENTRY_GROUP] = { .type = NLA_BINARY, - .len = sizeof_field(struct iphdr, daddr) }, - [VXLAN_VNIFILTER_ENTRY_GROUP6] = { .type = NLA_BINARY, - .len = sizeof(struct in6_addr) }, + [VXLAN_VNIFILTER_ENTRY_GROUP] = NLA_POLICY_EXACT_LEN(sizeof_field(struct iphdr, daddr)), + [VXLAN_VNIFILTER_ENTRY_GROUP6] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)), }; static const struct nla_policy vni_filter_policy[VXLAN_VNIFILTER_MAX + 1] = { diff --git a/drivers/net/wan/wanxl.c b/drivers/net/wan/wanxl.c index 5a9e262188efe..c38dd741401e1 100644 --- a/drivers/net/wan/wanxl.c +++ b/drivers/net/wan/wanxl.c @@ -514,7 +514,8 @@ static void wanxl_pci_remove_one(struct pci_dev *pdev) if (card->irq) free_irq(card->irq, card); - wanxl_reset(card); + if (card->plx) + wanxl_reset(card); for (i = 0; i < RX_QUEUE_LENGTH; i++) if (card->rx_skbs[i]) { diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c index b3f6424c17d36..0785ea2b8270f 100644 --- a/drivers/net/wireless/ath/ath10k/snoc.c +++ b/drivers/net/wireless/ath/ath10k/snoc.c @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -1456,11 +1457,15 @@ static void ath10k_msa_dump_memory(struct ath10k *ar, hdr->length = cpu_to_le32(ar->msa.mem_size); if (current_region->len < ar->msa.mem_size) { - memcpy(buf, ar->msa.vaddr, current_region->len); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + current_region->len); ath10k_warn(ar, "msa dump length is less than msa size %x, %x\n", current_region->len, ar->msa.mem_size); } else { - memcpy(buf, ar->msa.vaddr, ar->msa.mem_size); + memcpy_fromio(buf, + (const void __iomem __force *)ar->msa.vaddr, + ar->msa.mem_size); } } diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c index 8dfe9b40c1262..b8437e23e5192 100644 --- a/drivers/net/wireless/ath/ath11k/ahb.c +++ b/drivers/net/wireless/ath/ath11k/ahb.c @@ -998,6 +998,7 @@ static int ath11k_ahb_fw_resources_init(struct ath11k_base *ab) ret = ath11k_ahb_setup_msa_resources(ab); if (ret) { ath11k_err(ab, "failed to setup msa resources\n"); + of_node_put(node); return ret; } diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index 5666f66474455..51e4170e7662b 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -2332,10 +2332,10 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, case RX_MSDU_START_PKT_TYPE_11N: rx_status->encoding = RX_ENC_HT; if (rate_mcs > ATH11K_HT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HT_MCS_MAX); + rate_mcs = ATH11K_HT_MCS_MAX; } rx_status->rate_idx = rate_mcs + (8 * (nss - 1)); if (sgi) @@ -2344,13 +2344,13 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, break; case RX_MSDU_START_PKT_TYPE_11AC: rx_status->encoding = RX_ENC_VHT; - rx_status->rate_idx = rate_mcs; if (rate_mcs > ATH11K_VHT_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in VHT mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received VHT frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_VHT_MCS_MAX); + rate_mcs = ATH11K_VHT_MCS_MAX; } + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; if (sgi) rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI; @@ -2360,14 +2360,14 @@ static void ath11k_dp_rx_h_rate(struct ath11k *ar, struct hal_rx_desc *rx_desc, rx_status->enc_flags |= RX_ENC_FLAG_LDPC; break; case RX_MSDU_START_PKT_TYPE_11AX: - rx_status->rate_idx = rate_mcs; + rx_status->encoding = RX_ENC_HE; if (rate_mcs > ATH11K_HE_MCS_MAX) { - ath11k_warn(ar->ab, - "Received with invalid mcs in HE mode %d\n", - rate_mcs); - break; + ath11k_dbg(ar->ab, ATH11K_DBG_DP_RX, + "Received HE frame with out-of-range mcs %d, capping to %d\n", + rate_mcs, ATH11K_HE_MCS_MAX); + rate_mcs = ATH11K_HE_MCS_MAX; } - rx_status->encoding = RX_ENC_HE; + rx_status->rate_idx = rate_mcs; rx_status->nss = nss; rx_status->he_gi = ath11k_mac_he_gi_to_nl80211_he_gi(sgi); rx_status->bw = ath11k_mac_bw_to_mac80211_bw(bw); @@ -2501,6 +2501,29 @@ static void ath11k_dp_rx_deliver_msdu(struct ath11k *ar, struct napi_struct *nap ieee80211_rx_napi(ar->hw, pubsta, msdu, napi); } +static bool ath11k_dp_rx_check_nwifi_hdr_len_valid(struct ath11k_base *ab, + struct hal_rx_desc *rx_desc, + struct sk_buff *msdu) +{ + struct ieee80211_hdr *hdr; + u8 decap_type; + u32 hdr_len; + + decap_type = ath11k_dp_rx_h_msdu_start_decap_type(ab, rx_desc); + if (decap_type != DP_RX_DECAP_TYPE_NATIVE_WIFI) + return true; + + hdr = (struct ieee80211_hdr *)msdu->data; + hdr_len = ieee80211_hdrlen(hdr->frame_control); + + if (likely(hdr_len <= DP_MAX_NWIFI_HDR_LEN)) + return true; + + ab->soc_stats.invalid_rbm++; + WARN_ON_ONCE(1); + return false; +} + static int ath11k_dp_rx_process_msdu(struct ath11k *ar, struct sk_buff *msdu, struct sk_buff_head *msdu_list, @@ -2571,6 +2594,11 @@ static int ath11k_dp_rx_process_msdu(struct ath11k *ar, } } + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ab, rx_desc, msdu))) { + ret = -EINVAL; + goto free_out; + } + ath11k_dp_rx_h_ppdu(ar, rx_desc, rx_status); ath11k_dp_rx_h_mpdu(ar, msdu, rx_desc, rx_status); @@ -3306,6 +3334,12 @@ static int ath11k_dp_rx_h_verify_tkip_mic(struct ath11k *ar, struct ath11k_peer RX_FLAG_IV_STRIPPED | RX_FLAG_DECRYPTED; skb_pull(msdu, hal_rx_desc_sz); + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ar->ab, rx_desc, + msdu))) { + dev_kfree_skb_any(msdu); + return -EINVAL; + } + ath11k_dp_rx_h_ppdu(ar, rx_desc, rxs); ath11k_dp_rx_h_undecap(ar, msdu, rx_desc, HAL_ENCRYPT_TYPE_TKIP_MIC, rxs, true); @@ -3998,6 +4032,10 @@ static int ath11k_dp_rx_h_null_q_desc(struct ath11k *ar, struct sk_buff *msdu, skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); } + + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ar->ab, desc, msdu))) + return -EINVAL; + ath11k_dp_rx_h_ppdu(ar, desc, status); ath11k_dp_rx_h_mpdu(ar, msdu, desc, status); @@ -4042,7 +4080,7 @@ static bool ath11k_dp_rx_h_reo_err(struct ath11k *ar, struct sk_buff *msdu, return drop; } -static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, +static bool ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, struct ieee80211_rx_status *status) { u16 msdu_len; @@ -4050,6 +4088,7 @@ static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, u8 l3pad_bytes; struct ath11k_skb_rxcb *rxcb = ATH11K_SKB_RXCB(msdu); u32 hal_rx_desc_sz = ar->ab->hw_params.hal_desc_sz; + struct ath11k_base *ab = ar->ab; rxcb->is_first_msdu = ath11k_dp_rx_h_msdu_end_first_msdu(ar->ab, desc); rxcb->is_last_msdu = ath11k_dp_rx_h_msdu_end_last_msdu(ar->ab, desc); @@ -4059,6 +4098,9 @@ static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, skb_put(msdu, hal_rx_desc_sz + l3pad_bytes + msdu_len); skb_pull(msdu, hal_rx_desc_sz + l3pad_bytes); + if (unlikely(!ath11k_dp_rx_check_nwifi_hdr_len_valid(ab, desc, msdu))) + return true; + ath11k_dp_rx_h_ppdu(ar, desc, status); status->flag |= (RX_FLAG_MMIC_STRIPPED | RX_FLAG_MMIC_ERROR | @@ -4066,19 +4108,21 @@ static void ath11k_dp_rx_h_tkip_mic_err(struct ath11k *ar, struct sk_buff *msdu, ath11k_dp_rx_h_undecap(ar, msdu, desc, HAL_ENCRYPT_TYPE_TKIP_MIC, status, false); + + return false; } static bool ath11k_dp_rx_h_rxdma_err(struct ath11k *ar, struct sk_buff *msdu, struct ieee80211_rx_status *status) { struct ath11k_skb_rxcb *rxcb = ATH11K_SKB_RXCB(msdu); - bool drop = false; + bool drop; ar->ab->soc_stats.rxdma_error[rxcb->err_code]++; switch (rxcb->err_code) { case HAL_REO_ENTR_RING_RXDMA_ECODE_TKIP_MIC_ERR: - ath11k_dp_rx_h_tkip_mic_err(ar, msdu, status); + drop = ath11k_dp_rx_h_tkip_mic_err(ar, msdu, status); break; default: /* TODO: Review other rxdma error code to check if anything is @@ -4609,6 +4653,9 @@ static void ath11k_hal_rx_msdu_list_get(struct ath11k *ar, msdu_details = &msdu_link->msdu_link[0]; for (i = 0; i < HAL_RX_NUM_MSDU_DESC; i++) { + if (!i && FIELD_GET(BUFFER_ADDR_INFO0_ADDR, + msdu_details[i].buf_addr_info.info0) == 0) + break; if (FIELD_GET(BUFFER_ADDR_INFO0_ADDR, msdu_details[i].buf_addr_info.info0) == 0) { msdu_desc_info = &msdu_details[i - 1].rx_msdu_info; diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index f142c17aa9aa7..4794caa9f7eaa 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -1557,12 +1557,15 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif, if (!beacons || !beacons->cnt) { ath11k_warn(arvif->ar->ab, "failed to get ema beacon templates from mac80211\n"); - return -EPERM; + ret = -EPERM; + goto free; } if (tx_arvif == arvif) { - if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb)) - return -EINVAL; + if (ath11k_mac_set_vif_params(tx_arvif, beacons->bcn[0].skb)) { + ret = -EINVAL; + goto free; + } } else { arvif->wpaie_present = tx_arvif->wpaie_present; } @@ -1589,11 +1592,11 @@ static int ath11k_mac_setup_bcn_tmpl_ema(struct ath11k_vif *arvif, } } - ieee80211_beacon_free_ema_list(beacons); - if (tx_arvif != arvif && !nontx_vif_params_set) - return -EINVAL; /* Profile not found in the beacons */ + ret = -EINVAL; /* Profile not found in the beacons */ +free: + ieee80211_beacon_free_ema_list(beacons); return ret; } @@ -1622,19 +1625,22 @@ static int ath11k_mac_setup_bcn_tmpl_mbssid(struct ath11k_vif *arvif, } if (tx_arvif == arvif) { - if (ath11k_mac_set_vif_params(tx_arvif, bcn)) - return -EINVAL; + if (ath11k_mac_set_vif_params(tx_arvif, bcn)) { + ret = -EINVAL; + goto free; + } } else if (!ath11k_mac_set_nontx_vif_params(tx_arvif, arvif, bcn)) { - return -EINVAL; + ret = -EINVAL; + goto free; } ret = ath11k_wmi_bcn_tmpl(ar, arvif->vdev_id, &offs, bcn, 0); - kfree_skb(bcn); - if (ret) ath11k_warn(ab, "failed to submit beacon template command: %d\n", ret); +free: + kfree_skb(bcn); return ret; } diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c index d2c44f7f9b622..5c08cd73d743d 100644 --- a/drivers/net/wireless/ath/ath11k/mhi.c +++ b/drivers/net/wireless/ath/ath11k/mhi.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: BSD-3-Clause-Clear /* * Copyright (c) 2020 The Linux Foundation. All rights reserved. - * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include @@ -282,8 +282,10 @@ static void ath11k_mhi_op_status_cb(struct mhi_controller *mhi_cntrl, break; } + spin_lock_bh(&ab->base_lock); if (!(test_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags))) queue_work(ab->workqueue_aux, &ab->reset_work); + spin_unlock_bh(&ab->base_lock); break; default: diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index 7114eca8810db..a163168f36176 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -199,6 +199,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab) val |= PCIE_SOC_GLOBAL_RESET_V; ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val); + /* Flush the posted write to the device */ + ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET); /* TODO: exact time to sleep is uncertain */ delay = 10; @@ -208,6 +210,8 @@ static void ath11k_pci_soc_global_reset(struct ath11k_base *ab) val &= ~PCIE_SOC_GLOBAL_RESET_V; ath11k_pcic_write32(ab, PCIE_SOC_GLOBAL_RESET, val); + /* Flush the posted write to the device */ + ath11k_pcic_read32(ab, PCIE_SOC_GLOBAL_RESET); mdelay(delay); @@ -1210,6 +1214,14 @@ static void ath11k_pci_shutdown(struct pci_dev *pdev) struct ath11k_pci *ab_pci = ath11k_pci_priv(ab); ath11k_pci_set_irq_affinity_hint(ab_pci, NULL); + + spin_lock_bh(&ab->base_lock); + set_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags); + spin_unlock_bh(&ab->base_lock); + + cancel_work_sync(&ab->reset_work); + cancel_work_sync(&ab->dump_work); + ath11k_pci_power_down(ab, false); } diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index aea56c38bf8f3..f23d75c8ad679 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -3295,9 +3295,14 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work) clear_bit(ATH11K_FLAG_CRASH_FLUSH, &ab->dev_flags); clear_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags); - ath11k_core_qmi_firmware_ready(ab); - set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags); - + if (!test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) { + ret = ath11k_core_qmi_firmware_ready(ab); + if (ret) { + set_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags); + break; + } + set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags); + } break; case ATH11K_QMI_EVENT_COLD_BOOT_CAL_DONE: break; diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index e1b00dc811e7b..3305ec66ddbf2 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -159,6 +159,8 @@ static const struct wmi_tlv_policy wmi_tlv_policies[] = { .min_len = sizeof(struct ath11k_wmi_p2p_noa_info) }, [WMI_TAG_P2P_NOA_EVENT] = { .min_len = sizeof(struct wmi_p2p_noa_event) }, + [WMI_TAG_PDEV_CSA_SWITCH_COUNT_STATUS_EVENT] = { + .min_len = sizeof(struct wmi_pdev_csa_switch_ev) }, }; #define PRIMAP(_hw_mode_) \ @@ -262,6 +264,13 @@ const void **ath11k_wmi_tlv_parse_alloc(struct ath11k_base *ab, return tb; } +static u32 ath11k_wmi_tlv_data_len(const void *data) +{ + const struct wmi_tlv *tlv = (const struct wmi_tlv *)data - 1; + + return FIELD_GET(WMI_TLV_LEN, tlv->header); +} + static int ath11k_wmi_cmd_send_nowait(struct ath11k_pdev_wmi *wmi, struct sk_buff *skb, u32 cmd_id) { @@ -2411,8 +2420,8 @@ int ath11k_wmi_send_scan_start_cmd(struct ath11k *ar, for (i = 0; i < params->num_hint_bssid; ++i) { hint_bssid->freq_flags = params->hint_bssid[i].freq_flags; - ether_addr_copy(¶ms->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + ¶ms->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4747,14 +4756,16 @@ static int ath11k_wmi_tlv_mac_phy_caps_parse(struct ath11k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kcalloc(svc_rdy_ext->tot_phy_id, - len, GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct wmi_mac_phy_capabilities)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -5066,6 +5077,7 @@ static int ath11k_service_ready_ext_event(struct ath11k_base *ab, return 0; err: + kfree(svc_rdy_ext.mac_phy_caps); ath11k_wmi_free_dbring_caps(ab); return ret; } @@ -8302,15 +8314,23 @@ ath11k_wmi_process_csa_switch_count_event(struct ath11k_base *ab, const struct wmi_pdev_csa_switch_ev *ev, const u32 *vdev_ids) { - int i; + u32 vdev_ids_len = ath11k_wmi_tlv_data_len(vdev_ids); + u32 num_vdevs = ev->num_vdevs; struct ath11k_vif *arvif; + int i; /* Finish CSA once the switch count becomes NULL */ if (ev->current_switch_count) return; + if (num_vdevs > vdev_ids_len / sizeof(*vdev_ids)) { + ath11k_warn(ab, "csa switch count num_vdevs %u exceeds tlv array length %u\n", + num_vdevs, vdev_ids_len); + return; + } + rcu_read_lock(); - for (i = 0; i < ev->num_vdevs; i++) { + for (i = 0; i < num_vdevs; i++) { arvif = ath11k_mac_get_arvif_by_vdev_id(ab, vdev_ids[i]); if (!arvif) { @@ -8757,13 +8777,15 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; + if (skb->len < sizeof(*cmd_hdr)) + goto out; + cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = FIELD_GET(WMI_CMD_HDR_CMD_ID, (cmd_hdr->cmd_id)); trace_ath11k_wmi_event(ab, id, skb->data, skb->len); - if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) - goto out; + skb_pull(skb, sizeof(*cmd_hdr)); switch (id) { /* Process all the WMI events here */ diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c index b30527c402f6c..af68d5cd6a9d7 100644 --- a/drivers/net/wireless/ath/ath12k/ahb.c +++ b/drivers/net/wireless/ath/ath12k/ahb.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "ahb.h" #include "debug.h" #include "hif.h" @@ -345,24 +346,25 @@ static int ath12k_ahb_power_up(struct ath12k_base *ab) char fw2_name[ATH12K_USERPD_FW_NAME_LEN]; struct device *dev = ab->dev; const struct firmware *fw, *fw2; - struct reserved_mem *rmem = NULL; unsigned long time_left; phys_addr_t mem_phys; + struct resource res; void *mem_region; size_t mem_size; u32 pasid; int ret; - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) - return -ENODEV; + ret = of_reserved_mem_region_to_resource_byname(dev->of_node, "q6-region", + &res); + if (ret) + return ret; - mem_phys = rmem->base; - mem_size = rmem->size; + mem_phys = res.start; + mem_size = resource_size(&res); mem_region = devm_memremap(dev, mem_phys, mem_size, MEMREMAP_WC); if (IS_ERR(mem_region)) { - ath12k_err(ab, "unable to map memory region: %pa+%pa\n", - &rmem->base, &rmem->size); + ath12k_err(ab, "unable to map memory region: %pa+%zx\n", + &res.start, mem_size); return PTR_ERR(mem_region); } diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c index cc352eef19399..a02db6fd23549 100644 --- a/drivers/net/wireless/ath/ath12k/core.c +++ b/drivers/net/wireless/ath/ath12k/core.c @@ -46,7 +46,7 @@ ath12k_mem_profile_based_param ath12k_mem_profile_based_param[] = { .dp_params = { .tx_comp_ring_size = 32768, .rxdma_monitor_buf_ring_size = 4096, - .rxdma_monitor_dst_ring_size = 8092, + .rxdma_monitor_dst_ring_size = 8192, .num_pool_tx_desc = 32768, .rx_desc_count = 12288, }, @@ -633,31 +633,6 @@ u32 ath12k_core_get_max_peers_per_radio(struct ath12k_base *ab) return ath12k_core_get_max_station_per_radio(ab) + TARGET_NUM_VDEVS(ab); } -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index) -{ - struct device *dev = ab->dev; - struct reserved_mem *rmem; - struct device_node *node; - - node = of_parse_phandle(dev->of_node, "memory-region", index); - if (!node) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "failed to parse memory-region for index %d\n", index); - return NULL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { - ath12k_dbg(ab, ATH12K_DBG_BOOT, - "unable to get memory-region for index %d\n", index); - return NULL; - } - - return rmem; -} - static inline void ath12k_core_to_group_ref_get(struct ath12k_base *ab) { diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h index d7688b383f62c..93cdfcaf95bd2 100644 --- a/drivers/net/wireless/ath/ath12k/core.h +++ b/drivers/net/wireless/ath/ath12k/core.h @@ -1369,8 +1369,6 @@ void ath12k_fw_stats_init(struct ath12k *ar); void ath12k_fw_stats_bcn_free(struct list_head *head); void ath12k_fw_stats_free(struct ath12k_fw_stats *stats); void ath12k_fw_stats_reset(struct ath12k *ar); -struct reserved_mem *ath12k_core_get_reserved_mem(struct ath12k_base *ab, - int index); enum ath12k_qmi_mem_mode ath12k_core_get_memory_mode(struct ath12k_base *ab); static inline const char *ath12k_scan_state_str(enum ath12k_scan_state state) diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c index 4bb4482e27343..2f47054efa44e 100644 --- a/drivers/net/wireless/ath/ath12k/mac.c +++ b/drivers/net/wireless/ath/ath12k/mac.c @@ -10888,6 +10888,9 @@ ath12k_mac_update_vif_chan(struct ath12k *ar, continue; } + if (WARN_ON(!arvif)) + continue; + ath12k_dbg(ab, ATH12K_DBG_MAC, "mac chanctx switch vdev_id %i freq %u->%u width %d->%d\n", arvif->vdev_id, @@ -11628,23 +11631,85 @@ ath12k_mac_op_switch_vif_chanctx(struct ieee80211_hw *hw, int n_vifs, enum ieee80211_chanctx_switch_mode mode) { - struct ath12k *ar; + struct ath12k *curr_ar, *new_ar, *group_ar; + struct ieee80211_vif_chanctx_switch *v; + int i, j, count = 0; lockdep_assert_wiphy(hw->wiphy); - ar = ath12k_get_ar_by_ctx(hw, vifs->old_ctx); - if (!ar) - return -EINVAL; + if (n_vifs == 0) + return 0; - /* Switching channels across radio is not allowed */ - if (ar != ath12k_get_ar_by_ctx(hw, vifs->new_ctx)) - return -EINVAL; + struct ath12k **ar_map __free(kfree) = kzalloc_objs(*ar_map, n_vifs); - ath12k_dbg(ar->ab, ATH12K_DBG_MAC, - "mac chanctx switch n_vifs %d mode %d\n", - n_vifs, mode); - ath12k_mac_update_vif_chan(ar, vifs, n_vifs); + if (!ar_map) + return -ENOMEM; + + for (i = 0; i < n_vifs; i++) { + v = &vifs[i]; + + if (v->old_ctx->def.chan->band != v->new_ctx->def.chan->band) { + ath12k_generic_dbg(ATH12K_DBG_MAC, + "mac chanctx switch band change not supported\n"); + return -EOPNOTSUPP; + } + + curr_ar = ath12k_get_ar_by_ctx(hw, v->old_ctx); + new_ar = ath12k_get_ar_by_ctx(hw, v->new_ctx); + + if (!curr_ar || !new_ar) { + ath12k_generic_dbg(ATH12K_DBG_MAC, + "unable to determine device for the passed channel ctx\n"); + ath12k_generic_dbg(ATH12K_DBG_MAC, + "Old freq %d MHz (device %s) to new freq %d MHz (device %s)\n", + v->old_ctx->def.chan->center_freq, + curr_ar ? "valid" : "invalid", + v->new_ctx->def.chan->center_freq, + new_ar ? "valid" : "invalid"); + return -EINVAL; + } + /* Switching a vif between two radios is not allowed */ + if (curr_ar != new_ar) { + ath12k_dbg(curr_ar->ab, ATH12K_DBG_MAC, + "mac chanctx switch to another radio not supported\n"); + return -EOPNOTSUPP; + } + + ar_map[i] = curr_ar; + } + + /* Group vifs by radio (ar) and process each group independently. */ + bool *processed __free(kfree) = kzalloc_objs(*processed, n_vifs); + + if (!processed) + return -ENOMEM; + + struct ieee80211_vif_chanctx_switch *group_vifs __free(kfree) = + kzalloc_objs(*group_vifs, n_vifs); + + if (!group_vifs) + return -ENOMEM; + + for (i = 0; i < n_vifs; i++) { + if (processed[i]) + continue; + + group_ar = ar_map[i]; + + count = 0; + for (j = 0; j < n_vifs; j++) { + if (!processed[j] && ar_map[j] == group_ar) { + group_vifs[count++] = vifs[j]; + processed[j] = true; + } + } + + ath12k_dbg(group_ar->ab, ATH12K_DBG_MAC, + "mac chanctx switch n_vifs %d mode %d\n", + count, mode); + ath12k_mac_update_vif_chan(group_ar, group_vifs, count); + } return 0; } diff --git a/drivers/net/wireless/ath/ath12k/pci.c b/drivers/net/wireless/ath/ath12k/pci.c index 60b8f7361b7f6..37f41635d3b00 100644 --- a/drivers/net/wireless/ath/ath12k/pci.c +++ b/drivers/net/wireless/ath/ath12k/pci.c @@ -240,6 +240,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab) val |= PCIE_SOC_GLOBAL_RESET_V; ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val); + /* Flush the posted write to the device */ + ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET); /* TODO: exact time to sleep is uncertain */ delay = 10; @@ -249,6 +251,8 @@ static void ath12k_pci_soc_global_reset(struct ath12k_base *ab) val &= ~PCIE_SOC_GLOBAL_RESET_V; ath12k_pci_write32(ab, PCIE_SOC_GLOBAL_RESET, val); + /* Flush the posted write to the device */ + ath12k_pci_read32(ab, PCIE_SOC_GLOBAL_RESET); mdelay(delay); diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c index 8de9aee2498ec..b66cff3c70e22 100644 --- a/drivers/net/wireless/ath/ath12k/qmi.c +++ b/drivers/net/wireless/ath/ath12k/qmi.c @@ -13,6 +13,7 @@ #include #include #include +#include #define SLEEP_CLOCK_SELECT_INTERNAL_BIT 0x02 #define HOST_CSTATE_BIT 0x04 @@ -2683,121 +2684,96 @@ static int ath12k_qmi_alloc_target_mem_chunk(struct ath12k_base *ab) return ret; } +static const char *ath12k_qmi_get_mem_reg_name(int mem_type) +{ + switch (mem_type) { + case HOST_DDR_REGION_TYPE: + case BDF_MEM_REGION_TYPE: + return "q6-region"; + case M3_DUMP_REGION_TYPE: + return "m3-dump"; + case CALDB_MEM_REGION_TYPE: + return "q6-caldb"; + case MLO_GLOBAL_MEM_REGION_TYPE: + return "mlo-global-mem"; + default: + return NULL; + } +} + static int ath12k_qmi_assign_target_mem_chunk(struct ath12k_base *ab) { - struct reserved_mem *rmem; - size_t avail_rmem_size; + struct device_node *np = ab->dev->of_node; + size_t avail_rmem_size, offset = 0; + struct target_mem_chunk *chunk; + struct resource res; + const char *rname; int i, idx, ret; for (i = 0, idx = 0; i < ab->qmi.mem_seg_count; i++) { - switch (ab->qmi.target_mem[i].type) { - case HOST_DDR_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case BDF_MEM_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 0); - if (!rmem) { - ret = -ENODEV; - goto out; - } - - avail_rmem_size = rmem->size - ab->hw_params->bdf_addr_offset; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, - avail_rmem_size); - ret = -EINVAL; - goto out; - } - ab->qmi.target_mem[idx].paddr = - rmem->base + ab->hw_params->bdf_addr_offset; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - case CALDB_MEM_REGION_TYPE: - /* Cold boot calibration is not enabled in Ath12k. Hence, + chunk = &ab->qmi.target_mem[i]; + if (chunk->type == CALDB_MEM_REGION_TYPE) { + /* + * Cold boot calibration is not enabled in Ath12k. Hence, * assign paddr = 0. * Once cold boot calibration is enabled add support to * assign reserved memory from DT. */ ab->qmi.target_mem[idx].paddr = 0; ab->qmi.target_mem[idx].v.ioaddr = NULL; - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; idx++; - break; - case M3_DUMP_REGION_TYPE: - rmem = ath12k_core_get_reserved_mem(ab, 1); - if (!rmem) { - ret = -EINVAL; - goto out; - } + continue; + } - avail_rmem_size = rmem->size; - if (avail_rmem_size < ab->qmi.target_mem[i].size) { - ath12k_dbg(ab, ATH12K_DBG_QMI, - "failed to assign mem type %u req size %u avail size %zu\n", - ab->qmi.target_mem[i].type, - ab->qmi.target_mem[i].size, + rname = ath12k_qmi_get_mem_reg_name(chunk->type); + if (!rname) { + ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", + chunk->type); + continue; + } + + ret = of_reserved_mem_region_to_resource_byname(np, rname, &res); + if (ret) + goto out; + + avail_rmem_size = resource_size(&res); + if (chunk->type == BDF_MEM_REGION_TYPE || + chunk->type == HOST_DDR_REGION_TYPE) { + if (ab->hw_params->bdf_addr_offset > avail_rmem_size || + offset > avail_rmem_size - ab->hw_params->bdf_addr_offset) { + ath12k_err(ab, "qmi mem offset overflow: bdf_offset=%u offset=%zu size=%zu\n", + ab->hw_params->bdf_addr_offset, offset, avail_rmem_size); ret = -EINVAL; goto out; } - ab->qmi.target_mem[idx].paddr = rmem->base; - ab->qmi.target_mem[idx].v.ioaddr = - ioremap(ab->qmi.target_mem[idx].paddr, - ab->qmi.target_mem[i].size); - if (!ab->qmi.target_mem[idx].v.ioaddr) { - ret = -EIO; - goto out; - } - ab->qmi.target_mem[idx].size = ab->qmi.target_mem[i].size; - ab->qmi.target_mem[idx].type = ab->qmi.target_mem[i].type; - idx++; - break; - default: - ath12k_warn(ab, "qmi ignore invalid mem req type %u\n", - ab->qmi.target_mem[i].type); - break; + avail_rmem_size -= ab->hw_params->bdf_addr_offset + offset; + res.start += ab->hw_params->bdf_addr_offset + offset; + offset += chunk->size; } + + if (avail_rmem_size < chunk->size) { + ath12k_dbg(ab, ATH12K_DBG_QMI, + "failed to assign mem type %u req size %u avail size %zu\n", + chunk->type, chunk->size, avail_rmem_size); + ret = -EINVAL; + goto out; + } + + ab->qmi.target_mem[idx].paddr = res.start; + ab->qmi.target_mem[idx].v.ioaddr = ioremap(ab->qmi.target_mem[idx].paddr, + chunk->size); + if (!ab->qmi.target_mem[idx].v.ioaddr) { + ret = -EIO; + goto out; + } + + ab->qmi.target_mem[idx].size = chunk->size; + ab->qmi.target_mem[idx].type = chunk->type; + idx++; } ab->qmi.mem_seg_count = idx; diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c index f3474a13e32aa..f4392462ae3a2 100644 --- a/drivers/net/wireless/ath/ath12k/wmi.c +++ b/drivers/net/wireless/ath/ath12k/wmi.c @@ -2816,8 +2816,8 @@ int ath12k_wmi_send_scan_start_cmd(struct ath12k *ar, for (i = 0; i < arg->num_hint_bssid; ++i) { hint_bssid->freq_flags = arg->hint_bssid[i].freq_flags; - ether_addr_copy(&arg->hint_bssid[i].bssid.addr[0], - &hint_bssid->bssid.addr[0]); + ether_addr_copy(&hint_bssid->bssid.addr[0], + &arg->hint_bssid[i].bssid.addr[0]); hint_bssid++; } } @@ -4525,14 +4525,16 @@ static int ath12k_wmi_mac_phy_caps_parse(struct ath12k_base *soc, if (svc_rdy_ext->n_mac_phy_caps >= svc_rdy_ext->tot_phy_id) return -ENOBUFS; - len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); if (!svc_rdy_ext->n_mac_phy_caps) { - svc_rdy_ext->mac_phy_caps = kzalloc((svc_rdy_ext->tot_phy_id) * len, - GFP_ATOMIC); + svc_rdy_ext->mac_phy_caps = + kzalloc_objs(*svc_rdy_ext->mac_phy_caps, + svc_rdy_ext->tot_phy_id, + GFP_ATOMIC); if (!svc_rdy_ext->mac_phy_caps) return -ENOMEM; } + len = min_t(u16, len, sizeof(struct ath12k_wmi_mac_phy_caps_params)); memcpy(svc_rdy_ext->mac_phy_caps + svc_rdy_ext->n_mac_phy_caps, ptr, len); svc_rdy_ext->n_mac_phy_caps++; return 0; @@ -9410,6 +9412,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, void *ptr = skb->data; struct ath12k *ar; u16 tlv_tag; + u16 tlv_len; u32 event_count; int ret; @@ -9425,6 +9428,7 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, tlv = (struct wmi_tlv *)ptr; tlv_tag = le32_get_bits(tlv->header, WMI_TLV_TAG); + tlv_len = le32_get_bits(tlv->header, WMI_TLV_LEN); ptr += sizeof(*tlv); if (tlv_tag != WMI_TAG_HALPHY_CTRL_PATH_EVENT_FIXED_PARAM) { @@ -9432,6 +9436,12 @@ static void ath12k_wmi_process_tpc_stats(struct ath12k_base *ab, return; } + if (tlv_len < sizeof(*fixed_param)) { + ath12k_warn(ab, "TPC stats fixed param tlv len %u too short\n", + tlv_len); + return; + } + fixed_param = (struct ath12k_wmi_pdev_tpc_stats_event_fixed_params *)ptr; rcu_read_lock(); ar = ath12k_mac_get_ar_by_pdev_id(ab, le32_to_cpu(fixed_param->pdev_id) + 1); @@ -9739,12 +9749,12 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb) struct wmi_cmd_hdr *cmd_hdr; enum wmi_tlv_event_id id; - cmd_hdr = (struct wmi_cmd_hdr *)skb->data; - id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); - - if (!skb_pull(skb, sizeof(struct wmi_cmd_hdr))) + cmd_hdr = skb_pull_data(skb, sizeof(*cmd_hdr)); + if (!cmd_hdr) goto out; + id = le32_get_bits(cmd_hdr->cmd_id, WMI_CMD_HDR_CMD_ID); + switch (id) { /* Process all the WMI events here */ case WMI_SERVICE_READY_EVENTID: diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c index 88f0197fc041a..e4f87018f1642 100644 --- a/drivers/net/wireless/ath/ath6kl/cfg80211.c +++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c @@ -753,6 +753,11 @@ void ath6kl_cfg80211_connect_event(struct ath6kl_vif *vif, u16 channel, u8 *assoc_resp_ie = assoc_info + beacon_ie_len + assoc_req_len + assoc_resp_ie_offset; + if (assoc_req_len < assoc_req_ie_offset) + assoc_req_len = assoc_req_ie_offset; + if (assoc_resp_len < assoc_resp_ie_offset) + assoc_resp_len = assoc_resp_ie_offset; + assoc_req_len -= assoc_req_ie_offset; assoc_resp_len -= assoc_resp_ie_offset; diff --git a/drivers/net/wireless/ath/ath6kl/txrx.c b/drivers/net/wireless/ath/ath6kl/txrx.c index c3b06b515c4f4..5fd9cfdb56717 100644 --- a/drivers/net/wireless/ath/ath6kl/txrx.c +++ b/drivers/net/wireless/ath/ath6kl/txrx.c @@ -1723,13 +1723,15 @@ void aggr_recv_addba_req_evt(struct ath6kl_vif *vif, u8 tid_mux, u16 seq_no, rxtid = &aggr_conn->rx_tid[tid]; - if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX) - ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n", - __func__, win_sz, tid); - if (rxtid->aggr) aggr_delete_tid_state(aggr_conn, tid); + if (win_sz < AGGR_WIN_SZ_MIN || win_sz > AGGR_WIN_SZ_MAX) { + ath6kl_dbg(ATH6KL_DBG_WLAN_RX, "%s: win_sz %d, tid %d\n", + __func__, win_sz, tid); + return; + } + rxtid->seq_next = seq_no; hold_q_size = TID_WINDOW_SZ(win_sz) * sizeof(struct skb_hold_q); rxtid->hold_q = kzalloc(hold_q_size, GFP_KERNEL); @@ -1828,7 +1830,7 @@ void aggr_reset_state(struct aggr_info_conn *aggr_conn) return; if (aggr_conn->timer_scheduled) { - timer_delete(&aggr_conn->timer); + timer_delete_sync(&aggr_conn->timer); aggr_conn->timer_scheduled = false; } diff --git a/drivers/net/wireless/ath/ath6kl/wmi.c b/drivers/net/wireless/ath/ath6kl/wmi.c index 08a154bce1396..2353b6d18c6c4 100644 --- a/drivers/net/wireless/ath/ath6kl/wmi.c +++ b/drivers/net/wireless/ath/ath6kl/wmi.c @@ -484,6 +484,18 @@ static int ath6kl_wmi_tx_complete_event_rx(u8 *datap, int len) evt = (struct wmi_tx_complete_event *) datap; + if (len < sizeof(*evt)) { + ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d\n", + len); + return -EINVAL; + } + + if (len < sizeof(*evt) + evt->num_msg * sizeof(struct tx_complete_msg_v1)) { + ath6kl_dbg(ATH6KL_DBG_WMI, "tx complete: invalid len %d for %u msgs\n", + len, evt->num_msg); + return -EINVAL; + } + ath6kl_dbg(ATH6KL_DBG_WMI, "comp: %d %d %d\n", evt->num_msg, evt->msg_len, evt->msg_type); @@ -862,6 +874,14 @@ static int ath6kl_wmi_connect_event_rx(struct wmi *wmi, u8 *datap, int len, ev = (struct wmi_connect_event *) datap; + if (len < sizeof(*ev) + ev->beacon_ie_len + + ev->assoc_req_len + ev->assoc_resp_len) { + ath6kl_dbg(ATH6KL_DBG_WMI, + "connect event: IE lengths %u+%u+%u exceed buffer %d\n", + ev->beacon_ie_len, ev->assoc_req_len, + ev->assoc_resp_len, len); + return -EINVAL; + } if (vif->nw_type == AP_NETWORK) { /* AP mode start/STA connected event */ struct net_device *dev = vif->ndev; @@ -1276,6 +1296,9 @@ static int ath6kl_wmi_scan_complete_rx(struct wmi *wmi, u8 *datap, int len, { struct wmi_scan_complete_event *ev; + if (len < sizeof(*ev)) + return -EINVAL; + ev = (struct wmi_scan_complete_event *) datap; ath6kl_scan_complete_evt(vif, a_sle32_to_cpu(ev->status)); @@ -3352,7 +3375,12 @@ static int ath6kl_wmi_get_pmkid_list_event_rx(struct wmi *wmi, u8 *datap, static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_addba_req_event *cmd = (struct wmi_addba_req_event *) datap; + struct wmi_addba_req_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_addba_req_event *)datap; aggr_recv_addba_req_evt(vif, cmd->tid, le16_to_cpu(cmd->st_seq_no), cmd->win_sz); @@ -3363,7 +3391,12 @@ static int ath6kl_wmi_addba_req_event_rx(struct wmi *wmi, u8 *datap, int len, static int ath6kl_wmi_delba_req_event_rx(struct wmi *wmi, u8 *datap, int len, struct ath6kl_vif *vif) { - struct wmi_delba_event *cmd = (struct wmi_delba_event *) datap; + struct wmi_delba_event *cmd; + + if (len < sizeof(*cmd)) + return -EINVAL; + + cmd = (struct wmi_delba_event *)datap; aggr_recv_delba_req_evt(vif, cmd->tid); diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.c b/drivers/net/wireless/ath/ath9k/hif_usb.c index fe9abe8cd268f..c15219257499f 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.c +++ b/drivers/net/wireless/ath/ath9k/hif_usb.c @@ -1097,7 +1097,7 @@ static int ath9k_hif_usb_download_fw(struct hif_device_usb *hif_dev) } kfree(buf); - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) firm_offset = AR7010_FIRMWARE_TEXT; else firm_offset = AR9271_FIRMWARE_TEXT; @@ -1192,7 +1192,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev, if (MAJOR_VERSION_REQ == 1 && hif_dev->fw_minor_index == 3) { const char *filename; - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) filename = FIRMWARE_AR7010_1_1; else filename = FIRMWARE_AR9271; @@ -1208,7 +1208,7 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev, return -ENOENT; } else { - if (IS_AR7010_DEVICE(hif_dev->usb_device_id->driver_info)) + if (IS_AR7010_DEVICE(hif_dev->id_info)) chip = "7010"; else chip = "9271"; @@ -1225,15 +1225,10 @@ static int ath9k_hif_request_firmware(struct hif_device_usb *hif_dev, ret = request_firmware_nowait(THIS_MODULE, true, hif_dev->fw_name, &hif_dev->udev->dev, GFP_KERNEL, hif_dev, ath9k_hif_usb_firmware_cb); - if (ret) { + if (ret) dev_err(&hif_dev->udev->dev, "ath9k_htc: Async request for firmware %s failed\n", hif_dev->fw_name); - return ret; - } - - dev_info(&hif_dev->udev->dev, "ath9k_htc: Firmware %s requested\n", - hif_dev->fw_name); return ret; } @@ -1270,9 +1265,9 @@ static void ath9k_hif_usb_firmware_cb(const struct firmware *fw, void *context) ret = ath9k_htc_hw_init(hif_dev->htc_handle, &hif_dev->interface->dev, - hif_dev->usb_device_id->idProduct, + le16_to_cpu(hif_dev->udev->descriptor.idProduct), hif_dev->udev->product, - hif_dev->usb_device_id->driver_info); + hif_dev->id_info); if (ret) { ret = -EINVAL; goto err_htc_hw_init; @@ -1386,7 +1381,7 @@ static int ath9k_hif_usb_probe(struct usb_interface *interface, hif_dev->udev = udev; hif_dev->interface = interface; - hif_dev->usb_device_id = id; + hif_dev->id_info = id->driver_info; #ifdef CONFIG_PM udev->reset_resume = 1; #endif diff --git a/drivers/net/wireless/ath/ath9k/hif_usb.h b/drivers/net/wireless/ath/ath9k/hif_usb.h index b3e66b0485a56..02655ed4e66ff 100644 --- a/drivers/net/wireless/ath/ath9k/hif_usb.h +++ b/drivers/net/wireless/ath/ath9k/hif_usb.h @@ -115,7 +115,7 @@ struct cmd_buf { struct hif_device_usb { struct usb_device *udev; struct usb_interface *interface; - const struct usb_device_id *usb_device_id; + int id_info; const void *fw_data; size_t fw_size; struct completion fw_done; diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c index 14de62c1a32bd..9a32cf683c4fd 100644 --- a/drivers/net/wireless/ath/ath9k/hw.c +++ b/drivers/net/wireless/ath/ath9k/hw.c @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include "hw.h" @@ -2719,19 +2719,28 @@ static void ath9k_hw_gpio_cfg_output_mux(struct ath_hw *ah, u32 gpio, u32 type) static void ath9k_hw_gpio_cfg_soc(struct ath_hw *ah, u32 gpio, bool out, const char *label) { + enum gpiod_flags flags = out ? GPIOD_OUT_LOW : GPIOD_IN; + struct gpio_desc *gpiod; int err; - if (ah->caps.gpio_requested & BIT(gpio)) + if (ah->gpiods[gpio]) return; - err = devm_gpio_request_one(ah->dev, gpio, out ? GPIOF_OUT_INIT_LOW : GPIOF_IN, label); + /* + * Obtains a system specific GPIO descriptor from another GPIO controller. + * Ideally this should come from the device tree, this is a legacy code + * path. + */ + gpiod = gpiod_get_index(NULL, "ath9k", gpio, flags); + err = PTR_ERR_OR_ZERO(gpiod); if (err) { ath_err(ath9k_hw_common(ah), "request GPIO%d failed:%d\n", gpio, err); return; } - ah->caps.gpio_requested |= BIT(gpio); + gpiod_set_consumer_name(gpiod, label); + ah->gpiods[gpio] = gpiod; } static void ath9k_hw_gpio_cfg_wmac(struct ath_hw *ah, u32 gpio, bool out, @@ -2791,10 +2800,12 @@ void ath9k_hw_gpio_free(struct ath_hw *ah, u32 gpio) if (!AR_SREV_SOC(ah)) return; - WARN_ON(gpio >= ah->caps.num_gpio_pins); + if (ah->gpiods[gpio]) { + gpiod_put(ah->gpiods[gpio]); + ah->gpiods[gpio] = NULL; + } - if (ah->caps.gpio_requested & BIT(gpio)) - ah->caps.gpio_requested &= ~BIT(gpio); + WARN_ON(gpio >= ah->caps.num_gpio_pins); } EXPORT_SYMBOL(ath9k_hw_gpio_free); @@ -2822,8 +2833,8 @@ u32 ath9k_hw_gpio_get(struct ath_hw *ah, u32 gpio) val = REG_READ(ah, AR_GPIO_IN(ah)) & BIT(gpio); else val = MS_REG_READ(AR, gpio); - } else if (BIT(gpio) & ah->caps.gpio_requested) { - val = gpio_get_value(gpio) & BIT(gpio); + } else if (ah->gpiods[gpio]) { + val = gpiod_get_value(ah->gpiods[gpio]); } else { WARN_ON(1); } @@ -2846,8 +2857,8 @@ void ath9k_hw_set_gpio(struct ath_hw *ah, u32 gpio, u32 val) AR7010_GPIO_OUT : AR_GPIO_IN_OUT(ah); REG_RMW(ah, out_addr, val << gpio, BIT(gpio)); - } else if (BIT(gpio) & ah->caps.gpio_requested) { - gpio_set_value(gpio, val); + } else if (ah->gpiods[gpio]) { + gpiod_set_value(ah->gpiods[gpio], val); } else { WARN_ON(1); } diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h index eaa07d6dbde00..d9d2f64c55700 100644 --- a/drivers/net/wireless/ath/ath9k/hw.h +++ b/drivers/net/wireless/ath/ath9k/hw.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -302,7 +303,6 @@ struct ath9k_hw_capabilities { u8 max_rxchains; u8 num_gpio_pins; u32 gpio_mask; - u32 gpio_requested; u8 rx_hp_qdepth; u8 rx_lp_qdepth; u8 rx_status_len; @@ -783,6 +783,7 @@ struct ath_hw { struct ath9k_hw_capabilities caps; struct ath9k_channel channels[ATH9K_NUM_CHANNELS]; struct ath9k_channel *curchan; + struct gpio_desc *gpiods[32]; union { struct ar5416_eeprom_def def; diff --git a/drivers/net/wireless/ath/carl9170/rx.c b/drivers/net/wireless/ath/carl9170/rx.c index 6833430130f4c..0383d5c9698bf 100644 --- a/drivers/net/wireless/ath/carl9170/rx.c +++ b/drivers/net/wireless/ath/carl9170/rx.c @@ -150,7 +150,8 @@ static void carl9170_cmd_callback(struct ar9170 *ar, u32 len, void *buffer) spin_lock(&ar->cmd_lock); if (ar->readbuf) { if (len >= 4) - memcpy(ar->readbuf, buffer + 4, len - 4); + memcpy(ar->readbuf, buffer + 4, + min_t(u32, len - 4, ar->readlen)); ar->readbuf = NULL; } @@ -917,7 +918,9 @@ static void carl9170_rx_stream(struct ar9170 *ar, void *buf, unsigned int len) } } - skb_put_data(ar->rx_failover, tbuf, tlen); + skb_put_data(ar->rx_failover, tbuf, + min_t(unsigned int, tlen, + ar->rx_failover_missing)); ar->rx_failover_missing -= tlen; if (ar->rx_failover_missing <= 0) { diff --git a/drivers/net/wireless/ath/carl9170/tx.c b/drivers/net/wireless/ath/carl9170/tx.c index b7717f9e1e9b9..b0eee8f892af7 100644 --- a/drivers/net/wireless/ath/carl9170/tx.c +++ b/drivers/net/wireless/ath/carl9170/tx.c @@ -692,7 +692,7 @@ void carl9170_tx_process_status(struct ar9170 *ar, unsigned int i; for (i = 0; i < cmd->hdr.ext; i++) { - if (WARN_ON(i > ((cmd->hdr.len / 2) + 1))) { + if (WARN_ON(i >= (cmd->hdr.len / 2))) { print_hex_dump_bytes("UU:", DUMP_PREFIX_NONE, (void *) cmd, cmd->hdr.len + 4); break; diff --git a/drivers/net/wireless/atmel/at76c50x-usb.c b/drivers/net/wireless/atmel/at76c50x-usb.c index aa683eacaf38e..b983b51ffee71 100644 --- a/drivers/net/wireless/atmel/at76c50x-usb.c +++ b/drivers/net/wireless/atmel/at76c50x-usb.c @@ -1521,13 +1521,16 @@ static inline int at76_guess_freq(struct at76_priv *priv) if (ieee80211_is_probe_resp(hdr->frame_control)) { el_off = offsetof(struct ieee80211_mgmt, u.probe_resp.variable); - el = ((struct ieee80211_mgmt *)hdr)->u.probe_resp.variable; } else if (ieee80211_is_beacon(hdr->frame_control)) { el_off = offsetof(struct ieee80211_mgmt, u.beacon.variable); - el = ((struct ieee80211_mgmt *)hdr)->u.beacon.variable; } else { goto exit; } + + if (len < el_off) + goto exit; + + el = priv->rx_skb->data + el_off; len -= el_off; el = cfg80211_find_ie(WLAN_EID_DS_PARAMS, el, len); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c index e8767423aec84..070b296315f9e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c @@ -911,6 +911,7 @@ int brcmf_sdiod_probe(struct brcmf_sdio_dev *sdiodev) return ret; } switch (sdiodev->func2->device) { + case SDIO_DEVICE_ID_BROADCOM_43752: case SDIO_DEVICE_ID_BROADCOM_CYPRESS_4373: f2_blksz = SDIO_4373_FUNC2_BLOCKSIZE; break; @@ -1069,6 +1070,7 @@ static int brcmf_ops_sdio_probe(struct sdio_func *func, bus_if = kzalloc(sizeof(*bus_if), GFP_KERNEL); if (!bus_if) return -ENOMEM; + mutex_init(&bus_if->bus_reset_lock); sdiodev = kzalloc(sizeof(*sdiodev), GFP_KERNEL); if (!sdiodev) { kfree(bus_if); @@ -1130,6 +1132,14 @@ static void brcmf_ops_sdio_remove(struct sdio_func *func) if (func->num != 1) return; + /* Drain bus_reset before the shared brcmf_sdiod_remove() + * teardown, which the SDIO reset callback also reaches. The + * data worker can arm bus_reset via brcmf_fw_crashed(); cancel + * it first. + */ + brcmf_sdio_cancel_datawork(sdiodev->bus); + brcmf_bus_cancel_reset_work(bus_if); + /* only proceed with rest of cleanup if func 1 */ brcmf_sdiod_remove(sdiodev); @@ -1204,6 +1214,8 @@ static int brcmf_ops_sdio_suspend(struct device *dev) } else { /* power will be cut so remove device, probe again in resume */ brcmf_sdiod_intr_unregister(sdiodev); + brcmf_sdio_cancel_datawork(sdiodev->bus); + brcmf_bus_cancel_reset_work(bus_if); ret = brcmf_sdiod_remove(sdiodev); if (ret) brcmf_err("Failed to remove device on suspend\n"); @@ -1229,6 +1241,8 @@ static int brcmf_ops_sdio_resume(struct device *dev) ret = brcmf_sdiod_probe(sdiodev); if (ret) brcmf_err("Failed to probe device on resume\n"); + else + brcmf_bus_allow_reset_work(bus_if); } else { if (sdiodev->wowl_enabled && sdiodev->settings->bus.sdio.oob_irq_supported) disable_irq_wake(sdiodev->settings->bus.sdio.oob_irq_nr); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h index 8c3613ff23f5f..0ed145c3d46c7 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "debug.h" /* IDs of the 6 default common rings of msgbuf protocol */ @@ -179,6 +180,8 @@ struct brcmf_bus { enum brcmf_fwvendor fwvid; bool always_use_fws_queue; bool wowl_supported; + bool removing; /* device removal in progress; quiesce async work */ + struct mutex bus_reset_lock; const struct brcmf_bus_ops *ops; struct brcmf_bus_msgbuf *msgbuf; @@ -186,6 +189,9 @@ struct brcmf_bus { struct list_head list; }; +void brcmf_bus_cancel_reset_work(struct brcmf_bus *bus_if); +void brcmf_bus_allow_reset_work(struct brcmf_bus *bus_if); + /* * callback wrappers */ diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index d32b35ce0978d..bc93437f49a59 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -2147,7 +2147,7 @@ brcmf_set_key_mgmt(struct net_device *ndev, struct cfg80211_connect_params *sme) sme->crypto.akm_suites[0]); return -EINVAL; } - } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED)) { + } else if (val & (WPA2_AUTH_PSK | WPA2_AUTH_UNSPECIFIED | WPA2_AUTH_1X_SHA256)) { switch (sme->crypto.akm_suites[0]) { case WLAN_AKM_SUITE_8021X: val = WPA2_AUTH_UNSPECIFIED; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c index 89907f76cb19c..55e21f1079814 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c @@ -1167,6 +1167,35 @@ static int brcmf_revinfo_read(struct seq_file *s, void *data) return 0; } +/* + * Serialize arming from debugfs reset and brcmf_fw_crashed() against + * teardown. The remove path sets ->removing and drains the work while + * holding bus_reset_lock, so a racing armer is either drained or skips it. + */ +static void brcmf_bus_schedule_reset(struct brcmf_bus *bus_if) +{ + mutex_lock(&bus_if->bus_reset_lock); + if (bus_if->drvr && bus_if->drvr->bus_reset.func && !bus_if->removing) + schedule_work(&bus_if->drvr->bus_reset); + mutex_unlock(&bus_if->bus_reset_lock); +} + +void brcmf_bus_cancel_reset_work(struct brcmf_bus *bus_if) +{ + mutex_lock(&bus_if->bus_reset_lock); + bus_if->removing = true; + if (bus_if->drvr) + cancel_work_sync(&bus_if->drvr->bus_reset); + mutex_unlock(&bus_if->bus_reset_lock); +} + +void brcmf_bus_allow_reset_work(struct brcmf_bus *bus_if) +{ + mutex_lock(&bus_if->bus_reset_lock); + bus_if->removing = false; + mutex_unlock(&bus_if->bus_reset_lock); +} + static void brcmf_core_bus_reset(struct work_struct *work) { struct brcmf_pub *drvr = container_of(work, struct brcmf_pub, @@ -1187,7 +1216,7 @@ static ssize_t bus_reset_write(struct file *file, const char __user *user_buf, if (value != 1) return -EINVAL; - schedule_work(&drvr->bus_reset); + brcmf_bus_schedule_reset(drvr->bus_if); return count; } @@ -1419,14 +1448,23 @@ void brcmf_dev_coredump(struct device *dev) void brcmf_fw_crashed(struct device *dev) { struct brcmf_bus *bus_if = dev_get_drvdata(dev); - struct brcmf_pub *drvr = bus_if->drvr; + struct brcmf_pub *drvr; + + /* May fire before brcmf_attach() wires up drvr, or after removal + * has cleared it; guard the derefs below (and the arming gate in + * brcmf_bus_schedule_reset() already checks drvr/->removing). + */ + if (!bus_if) + return; + drvr = bus_if->drvr; + if (!drvr) + return; bphy_err(drvr, "Firmware has halted or crashed\n"); brcmf_dev_coredump(dev); - if (drvr->bus_reset.func) - schedule_work(&drvr->bus_reset); + brcmf_bus_schedule_reset(bus_if); } void brcmf_detach(struct device *dev) diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c index 88c76e18ea4e7..511ac39c01b5b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c @@ -1383,16 +1383,20 @@ static int brcmf_pcie_init_ringbuffers(struct brcmf_pciedev_info *devinfo) static void brcmf_pcie_release_scratchbuffers(struct brcmf_pciedev_info *devinfo) { - if (devinfo->shared.scratch) + if (devinfo->shared.scratch) { dma_free_coherent(&devinfo->pdev->dev, BRCMF_DMA_D2H_SCRATCH_BUF_LEN, devinfo->shared.scratch, devinfo->shared.scratch_dmahandle); - if (devinfo->shared.ringupd) + devinfo->shared.scratch = NULL; + } + if (devinfo->shared.ringupd) { dma_free_coherent(&devinfo->pdev->dev, BRCMF_DMA_D2H_RINGUPD_BUF_LEN, devinfo->shared.ringupd, devinfo->shared.ringupd_dmahandle); + devinfo->shared.ringupd = NULL; + } } static int brcmf_pcie_init_scratchbuffers(struct brcmf_pciedev_info *devinfo) @@ -2500,6 +2504,7 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = -ENOMEM; goto fail; } + mutex_init(&bus->bus_reset_lock); bus->msgbuf = kzalloc(sizeof(*bus->msgbuf), GFP_KERNEL); if (!bus->msgbuf) { ret = -ENOMEM; @@ -2595,6 +2600,11 @@ brcmf_pcie_remove(struct pci_dev *pdev) if (devinfo->ci) brcmf_pcie_intr_disable(devinfo); + if (devinfo->irq_allocated) + synchronize_irq(pdev->irq); + + brcmf_bus_cancel_reset_work(bus); + brcmf_detach(&pdev->dev); brcmf_free(&pdev->dev); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c index a0ed101392223..80bcbb4a08abd 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c @@ -1843,17 +1843,18 @@ brcmf_sdio_read_control(struct brcmf_sdio *bus, u8 *hdr, uint len, uint doff) if (bus->rxctl) { brcmf_err("last control frame is being processed.\n"); spin_unlock_bh(&bus->rxctl_lock); - vfree(buf); goto done; } bus->rxctl = buf + doff; bus->rxctl_orig = buf; bus->rxlen = len - doff; spin_unlock_bh(&bus->rxctl_lock); + buf = NULL; done: /* Awake any waiters */ brcmf_sdio_dcmd_resp_wake(bus); + vfree(buf); } /* Pad read to blocksize for efficiency */ @@ -4677,6 +4678,7 @@ int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) bus->sdiodev = sdiodev; sdiodev->bus = bus; skb_queue_head_init(&bus->glom); + INIT_WORK(&bus->datawork, brcmf_sdio_dataworker); bus->txbound = BRCMF_TXBOUND; bus->rxbound = BRCMF_RXBOUND; bus->txminmax = BRCMF_TXMINMAX; @@ -4691,7 +4693,6 @@ int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) goto fail; } brcmf_sdiod_freezer_count(sdiodev); - INIT_WORK(&bus->datawork, brcmf_sdio_dataworker); bus->brcmf_wq = wq; /* attempt to attach to the dongle */ @@ -4772,6 +4773,12 @@ int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev) return ret; } +void brcmf_sdio_cancel_datawork(struct brcmf_sdio *bus) +{ + if (bus) + cancel_work_sync(&bus->datawork); +} + /* Detach and free everything */ void brcmf_sdio_remove(struct brcmf_sdio *bus) { diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h index ed35b56ee4278..ef42172daf2bf 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h @@ -392,6 +392,7 @@ int brcmf_sdiod_remove(struct brcmf_sdio_dev *sdiodev); int brcmf_sdio_probe(struct brcmf_sdio_dev *sdiodev); void brcmf_sdio_remove(struct brcmf_sdio *bus); void brcmf_sdio_isr(struct brcmf_sdio *bus, bool in_isr); +void brcmf_sdio_cancel_datawork(struct brcmf_sdio *bus); void brcmf_sdio_wd_timer(struct brcmf_sdio *bus, bool active); void brcmf_sdio_wowl_config(struct device *dev, bool enabled); diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c index 4b28883f98270..9833a9dd4307e 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c @@ -1260,6 +1260,7 @@ static int brcmf_usb_probe_cb(struct brcmf_usbdev_info *devinfo, ret = -ENOMEM; goto fail; } + mutex_init(&bus->bus_reset_lock); bus->dev = dev; bus_pub->bus = bus; @@ -1329,6 +1330,8 @@ brcmf_usb_disconnect_cb(struct brcmf_usbdev_info *devinfo) return; brcmf_dbg(USB, "Enter, bus_pub %p\n", devinfo); + brcmf_bus_cancel_reset_work(devinfo->bus_pub.bus); + brcmf_detach(devinfo->dev); brcmf_free(devinfo->dev); kfree(devinfo->bus_pub.bus); diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c index 215814861cbda..0808e69c3c178 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c @@ -6162,6 +6162,8 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev, if (err) { printk(KERN_WARNING DRV_NAME "Error calling pci_enable_device.\n"); + free_libipw(dev, 0); + pci_iounmap(pci_dev, ioaddr); return err; } @@ -6174,16 +6176,14 @@ static int ipw2100_pci_init_one(struct pci_dev *pci_dev, if (err) { printk(KERN_WARNING DRV_NAME "Error calling pci_set_dma_mask.\n"); - pci_disable_device(pci_dev); - return err; + goto fail; } err = pci_request_regions(pci_dev, DRV_NAME); if (err) { printk(KERN_WARNING DRV_NAME "Error calling pci_request_regions.\n"); - pci_disable_device(pci_dev); - return err; + goto fail; } /* We disable the RETRY_TIMEOUT register (0x41) to keep diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c index b7bc94f7abd8a..c8841f9b9ad91 100644 --- a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c +++ b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c @@ -414,7 +414,7 @@ int libipw_rx(struct libipw_device *ieee, struct sk_buff *skb, ieee->host_mc_decrypt : ieee->host_decrypt; if (can_be_decrypted) { - if (skb->len >= hdrlen + 3) { + if (skb->len >= hdrlen + 4) { /* Top two-bits of byte 3 are the key index */ keyidx = skb->data[hdrlen + 3] >> 6; } @@ -660,7 +660,7 @@ int libipw_rx(struct libipw_device *ieee, struct sk_buff *skb, int trimlen = 0; /* Top two-bits of byte 3 are the key index */ - if (skb->len >= hdrlen + 3) + if (skb->len >= hdrlen + 4) keyidx = skb->data[hdrlen + 3] >> 6; /* To strip off any security data which appears before the diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c index c16cda087a7c3..307b557dce997 100644 --- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c +++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright (C) 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include "iwl-config.h" @@ -71,6 +71,7 @@ const char iwl_ax411_killer_1690i_name[] = const char iwl_ax210_name[] = "Intel(R) Wi-Fi 6E AX210 160MHz"; const char iwl_ax211_name[] = "Intel(R) Wi-Fi 6E AX211 160MHz"; +const char iwl_ax231_name[] = "Intel(R) Wi-Fi 6 AX231"; const char iwl_ax411_name[] = "Intel(R) Wi-Fi 6E AX411 160MHz"; IWL_FW_AND_PNVM(IWL_SO_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX); diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/main.c b/drivers/net/wireless/intel/iwlwifi/dvm/main.c index 2b4dbebc71c2e..72dc0d8dc9b95 100644 --- a/drivers/net/wireless/intel/iwlwifi/dvm/main.c +++ b/drivers/net/wireless/intel/iwlwifi/dvm/main.c @@ -1511,10 +1511,10 @@ static struct iwl_op_mode *iwl_op_mode_dvm_start(struct iwl_trans *trans, priv->workqueue = NULL; out_uninit_drv: iwl_uninit_drv(priv); -out_free_eeprom_blob: - kfree(priv->eeprom_blob); out_free_eeprom: kfree(priv->nvm_data); +out_free_eeprom_blob: + kfree(priv->eeprom_blob); out_leave_trans: iwl_trans_op_mode_leave(priv->trans); out_free_hw: diff --git a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c index 52edc19d8cdd5..852c5ac36c221 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/acpi.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/acpi.c @@ -761,6 +761,11 @@ int iwl_acpi_get_wgds_table(struct iwl_fw_runtime *fwrt) min_size, max_size, &tbl_rev); if (!IS_ERR(wifi_pkg)) { + if (tbl_rev < 0 || + tbl_rev >= BITS_PER_BYTE * + sizeof(rev_data[idx].revisions)) + continue; + if (!(BIT(tbl_rev) & rev_data[idx].revisions)) continue; diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h index 535864e226260..9eeb3600ed1f4 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h +++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2025 Intel Corporation + * Copyright (C) 2012-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015-2017 Intel Deutschland GmbH */ @@ -85,12 +85,13 @@ struct iwl_ltr_config_cmd { * '1' PM could sleep over DTIM till listen Interval. * @POWER_FLAGS_SNOOZE_ENA_MSK: Enable snoozing only if uAPSD is enabled and all * access categories are both delivery and trigger enabled. + * (Not supported since version 3) * @POWER_FLAGS_BT_SCO_ENA: Enable BT SCO coex only if uAPSD and * PBW Snoozing enabled * @POWER_FLAGS_ADVANCE_PM_ENA_MSK: Advanced PM (uAPSD) enable mask * @POWER_FLAGS_LPRX_ENA_MSK: Low Power RX enable. * @POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK: AP/GO's uAPSD misbehaving - * detection enablement + * detection enablement (Not supported since version 3) * @POWER_FLAGS_ENABLE_SMPS_MSK: SMPS is allowed for this vif */ enum iwl_power_flags { @@ -175,9 +176,9 @@ struct iwl_device_power_cmd { } __packed; /** - * struct iwl_mac_power_cmd - New power command containing uAPSD support + * struct iwl_mac_power_cmd_v2 - power command V2 containing uAPSD support * MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response) - * @id_and_color: MAC contex identifier, &enum iwl_ctxt_id_and_color + * @id_and_color: MAC context identifier, &enum iwl_ctxt_id_and_color * @flags: Power table command flags from POWER_FLAGS_* * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. * Minimum allowed:- 3 * DTIM. Keep alive period must be @@ -216,7 +217,7 @@ struct iwl_device_power_cmd { * @limited_ps_threshold: (unused) * @reserved: reserved (padding) */ -struct iwl_mac_power_cmd { +struct iwl_mac_power_cmd_v2 { /* CONTEXT_DESC_API_T_VER_1 */ __le32 id_and_color; @@ -242,6 +243,43 @@ struct iwl_mac_power_cmd { u8 reserved; } __packed; /* CLIENT_PM_POWER_TABLE_S_VER_1, VER_2 */ +/** + * struct iwl_mac_power_cmd - power command + * MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response) + * @id_and_color: MAC context identifier, &enum iwl_ctxt_id_and_color + * @flags: Power table command flags from POWER_FLAGS_* + * @keep_alive_seconds: Keep alive period in seconds. Default - 25 sec. + * Minimum allowed:- 3 * DTIM. Keep alive period must be + * set regardless of power scheme or current power state. + * FW use this value also when PM is disabled. + * @rx_data_timeout: Minimum time (usec) from last Rx packet for AM to + * PSM transition - legacy PM + * @tx_data_timeout: Minimum time (usec) from last Tx packet for AM to + * PSM transition - legacy PM + * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled. + * Default: 80dbm + * @skip_dtim_periods: Number of DTIM periods to skip if Skip over DTIM flag + * is set. For example, if it is required to skip over + * one DTIM, this value need to be set to 2 (DTIM periods). + * @qndp_tid: TID client shall use for uAPSD QNDP triggers + * @uapsd_ac_flags: Set trigger-enabled and delivery-enabled indication for + * each corresponding AC. + * Use IEEE80211_WMM_IE_STA_QOSINFO_AC* for correct values. + */ +struct iwl_mac_power_cmd { + /* CONTEXT_DESC_API_T_VER_1 */ + __le32 id_and_color; + + __le16 flags; + __le16 keep_alive_seconds; + __le32 rx_data_timeout; + __le32 tx_data_timeout; + u8 lprx_rssi_threshold; + u8 skip_dtim_periods; + u8 qndp_tid; + u8 uapsd_ac_flags; +} __packed; /* CLIENT_PM_POWER_TABLE_S_VER_3 */ + /* * struct iwl_uapsd_misbehaving_ap_notif - FW sends this notification when * associated AP is identified as improperly implementing uAPSD protocol. diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dump.c b/drivers/net/wireless/intel/iwlwifi/fw/dump.c index ddd714cff2f4d..87acbb482435a 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/dump.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/dump.c @@ -436,7 +436,7 @@ static void iwl_fwrt_dump_fseq_regs(struct iwl_fw_runtime *fwrt) void iwl_fwrt_dump_error_logs(struct iwl_fw_runtime *fwrt) { struct iwl_pc_data *pc_data; - u8 count; + u32 count; if (!iwl_trans_device_enabled(fwrt->trans)) { IWL_ERR(fwrt, diff --git a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c index f297e82d63d23..5de3a9c0b3ef0 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/pnvm.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright(c) 2020-2025 Intel Corporation + * Copyright(c) 2020-2026 Intel Corporation */ #include "iwl-drv.h" @@ -12,6 +12,7 @@ #include "fw/api/alive.h" #include "fw/uefi.h" #include "fw/img.h" +#include "fw/dbg.h" #define IWL_PNVM_REDUCED_CAP_BIT BIT(25) @@ -26,6 +27,12 @@ static bool iwl_pnvm_complete_fn(struct iwl_notif_wait_data *notif_wait, struct iwl_trans *trans = (struct iwl_trans *)data; struct iwl_pnvm_init_complete_ntfy *pnvm_ntf = (void *)pkt->data; + if (IWL_FW_CHECK(trans, + iwl_rx_packet_payload_len(pkt) < sizeof(*pnvm_ntf), + "Bad notif len: %d\n", + iwl_rx_packet_payload_len(pkt))) + return true; + IWL_DEBUG_FW(trans, "PNVM complete notification received with status 0x%0x\n", le32_to_cpu(pnvm_ntf->status)); diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c index e1f28b0532530..818eb1c4b158f 100644 --- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c +++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c @@ -483,7 +483,7 @@ bool iwl_add_mcc_to_tas_block_list(u16 *list, u8 *size, u16 mcc) if (*size >= IWL_WTAS_BLACK_LIST_MAX) return false; - list[*size++] = mcc; + list[(*size)++] = mcc; return true; } IWL_EXPORT_SYMBOL(iwl_add_mcc_to_tas_block_list); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h index 3b4f990a8d0bb..cfdea6afe3bf6 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h @@ -2,7 +2,7 @@ /* * Copyright (C) 2005-2014, 2018-2021 Intel Corporation * Copyright (C) 2016-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #ifndef __IWL_CONFIG_H__ #define __IWL_CONFIG_H__ @@ -670,6 +670,7 @@ extern const char iwl_ax411_killer_1690s_name[]; extern const char iwl_ax411_killer_1690i_name[]; extern const char iwl_ax210_name[]; extern const char iwl_ax211_name[]; +extern const char iwl_ax231_name[]; extern const char iwl_ax411_name[]; extern const char iwl_killer_be1750s_name[]; extern const char iwl_killer_be1750i_name[]; @@ -739,6 +740,7 @@ extern const struct iwl_rf_cfg iwl_rf_hr; extern const struct iwl_rf_cfg iwl_rf_hr_80mhz; extern const struct iwl_rf_cfg iwl_rf_gf; +#define iwl_rf_ot iwl_rf_hr_80mhz #endif /* CONFIG_IWLMVM */ #if IS_ENABLED(CONFIG_IWLMLD) diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c index 5240dacf13607..fe37c8f75c71d 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-dbg-tlv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include #include "iwl-drv.h" @@ -602,6 +602,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, cpu_to_le32(IWL_FW_INI_LOCATION_DRAM_PATH)) return 0; + if (!fw_mon_cfg->req_size) + return -EIO; + num_frags = le32_to_cpu(fw_mon_cfg->max_frags_num); if (fwrt->trans->mac_cfg->device_family < IWL_DEVICE_FAMILY_AX210) { if (alloc_id != IWL_FW_INI_ALLOCATION_ID_DBGC1) @@ -612,6 +615,9 @@ static int iwl_dbg_tlv_alloc_fragments(struct iwl_fw_runtime *fwrt, return -EIO; } + if (!num_frags) + return -EIO; + remain_pages = DIV_ROUND_UP(le32_to_cpu(fw_mon_cfg->req_size), PAGE_SIZE); num_frags = min_t(u32, num_frags, BUF_ALLOC_MAX_NUM_FRAGS); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c index 0f002ef261fcc..3a4bcfd60905c 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c @@ -803,6 +803,7 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv, u32 build, paging_mem_size; int num_of_cpus; bool usniffer_req = false; + size_t aligned_tlv_len; if (len < sizeof(*ucode)) { IWL_ERR(drv, "uCode has invalid length: %zd\n", len); @@ -851,8 +852,16 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv, len, tlv_len); return -EINVAL; } - len -= ALIGN(tlv_len, 4); - data += sizeof(*tlv) + ALIGN(tlv_len, 4); + + aligned_tlv_len = ALIGN(tlv_len, 4); + if (len < aligned_tlv_len) { + IWL_ERR(drv, "invalid aligned TLV len: %zd/%zu\n", + len, aligned_tlv_len); + return -EINVAL; + } + + len -= aligned_tlv_len; + data += sizeof(*tlv) + aligned_tlv_len; switch (tlv_type) { case IWL_UCODE_TLV_INST: diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c index b1944584c6931..c4ccfffdf6af9 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2003-2014, 2018-2022, 2024-2025 Intel Corporation + * Copyright (C) 2003-2014, 2018-2022, 2024-2026 Intel Corporation * Copyright (C) 2015-2016 Intel Deutschland GmbH */ #include @@ -168,6 +168,22 @@ int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr, return -ETIMEDOUT; } +int iwl_poll_umac_prph_bits_no_grab(struct iwl_trans *trans, u32 addr, + u32 bits, u32 mask, int timeout) +{ + int t = 0; + + do { + if ((iwl_read_umac_prph_no_grab(trans, addr) & mask) == + (bits & mask)) + return 0; + udelay(IWL_POLL_INTERVAL); + t += IWL_POLL_INTERVAL; + } while (t < timeout); + + return -ETIMEDOUT; +} + void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask) { if (iwl_trans_grab_nic_access(trans)) { diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.h b/drivers/net/wireless/intel/iwlwifi/iwl-io.h index 5bcec239ffc4a..d920a32fc173c 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-io.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2018-2021, 2025 Intel Corporation + * Copyright (C) 2018-2021, 2025-2026 Intel Corporation */ #ifndef __iwl_io_h__ #define __iwl_io_h__ @@ -51,6 +51,8 @@ static inline void iwl_write_prph(struct iwl_trans *trans, u32 ofs, u32 val) int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr, u32 bits, u32 mask, int timeout); +int iwl_poll_umac_prph_bits_no_grab(struct iwl_trans *trans, u32 addr, + u32 bits, u32 mask, int timeout); void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask); void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs, u32 bits, u32 mask); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h index a7214ddcfaf56..6ca1f51b69a1c 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2005-2014, 2018-2025 Intel Corporation + * Copyright (C) 2005-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016 Intel Deutschland GmbH */ @@ -411,6 +411,11 @@ enum { #define HPM_SECONDARY_DEVICE_STATE 0xa03404 #define WFPM_MAC_OTP_CFG7_ADDR 0xa03338 #define WFPM_MAC_OTP_CFG7_DATA 0xa0333c +#define WFPM_RSRCS_4PHS_REQ_STTS 0xa033f8 +#define WFPM_RSRCS_4PHS_ACK_STTS 0xa033fc + +#define RSRC_REQ_CNVR_TOP BIT(6) +#define RSRC_ACK_CNVR_TOP BIT(6) /* For UMAG_GEN_HW_STATUS reg check */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c index dce0b7cf7b265..1486702d37e75 100644 --- a/drivers/net/wireless/intel/iwlwifi/mei/main.c +++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2021-2024 Intel Corporation + * Copyright (C) 2026 Intel Corporation */ #include @@ -457,7 +458,7 @@ static int iwl_mei_send_sap_msg_payload(struct mei_cl_device *cldev, notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; q_sz = mei->shared_mem.q_size[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; - ret = iwl_mei_write_cyclic_buf(q_head, notif_q, q_head, hdr, q_sz); + ret = iwl_mei_write_cyclic_buf(cldev, notif_q, q_head, hdr, q_sz); if (ret < 0) return ret; @@ -1147,6 +1148,11 @@ static void iwl_mei_handle_sap_rx_cmd(struct mei_cl_device *cldev, iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr, sizeof(*hdr)); valid_rx_sz -= sizeof(*hdr); len = le16_to_cpu(hdr->len); + if (len + sizeof(*hdr) > PAGE_SIZE) { + dev_err(&cldev->dev, + "SAP message is too big: %u\n", len); + break; + } if (valid_rx_sz < len) break; diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ap.c b/drivers/net/wireless/intel/iwlwifi/mld/ap.c index 5c59acc8c4c5a..c29e4a77be058 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/ap.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/ap.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024, 2026 Intel Corporation */ #include @@ -239,6 +239,25 @@ int iwl_mld_store_ap_early_key(struct iwl_mld *mld, return -ENOSPC; } +void iwl_mld_stop_beacon(struct iwl_mld *mld, struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link) +{ + struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link); + struct iwl_mac_beacon_cmd cmd = {}; + int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, BEACON_TEMPLATE_CMD, 14); + + if (WARN_ON(!mld_link)) + return; + + if (cmd_ver < 15) + return; + + /* leave byte_cnt 0 */ + cmd.link_id = cpu_to_le32(mld_link->fw_id); + + iwl_mld_send_cmd_pdu(mld, BEACON_TEMPLATE_CMD, &cmd); +} + static int iwl_mld_send_ap_early_keys(struct iwl_mld *mld, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link) @@ -276,10 +295,6 @@ int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw, if (vif->type == NL80211_IFTYPE_AP) iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link); - ret = iwl_mld_update_beacon_template(mld, vif, link); - if (ret) - return ret; - /* the link should be already activated when assigning chan context, * and LINK_CONTEXT_MODIFY_EHT_PARAMS is deprecated */ diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ap.h b/drivers/net/wireless/intel/iwlwifi/mld/ap.h index 4a6f52b9552d7..a3b6bed814adf 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/ap.h +++ b/drivers/net/wireless/intel/iwlwifi/mld/ap.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2024 Intel Corporation + * Copyright (C) 2024, 2026 Intel Corporation */ #ifndef __iwl_ap_h__ #define __iwl_ap_h__ @@ -14,6 +14,10 @@ int iwl_mld_update_beacon_template(struct iwl_mld *mld, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link_conf); +void iwl_mld_stop_beacon(struct iwl_mld *mld, + struct ieee80211_vif *vif, + struct ieee80211_bss_conf *link_conf); + int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c index dd85be94433cc..0cb7bfcd6420e 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c @@ -1943,8 +1943,11 @@ int iwl_mld_wowlan_suspend(struct iwl_mld *mld, struct cfg80211_wowlan *wowlan) if (!bss_vif->cfg.assoc) { int ret; - /* If we're not associated, this must be netdetect */ - if (WARN_ON(!wowlan->nd_config)) + /* + * If not associated we can only do netdetect, if + * that's not enabled then just suspend normally. + */ + if (!wowlan->nd_config) return 1; ret = iwl_mld_netdetect_config(mld, bss_vif, wowlan); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c index 80bcd18930c57..b07e9e34a711a 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c @@ -588,24 +588,6 @@ void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld, kfree_rcu(old_data, rcu_head); } -void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld, - struct iwl_rx_packet *pkt) -{ - struct iwl_uapsd_misbehaving_ap_notif *notif = (void *)pkt->data; - struct ieee80211_vif *vif; - - if (IWL_FW_CHECK(mld, notif->mac_id >= ARRAY_SIZE(mld->fw_id_to_vif), - "mac id is invalid: %d\n", notif->mac_id)) - return; - - vif = wiphy_dereference(mld->wiphy, mld->fw_id_to_vif[notif->mac_id]); - - if (WARN_ON(!vif) || ieee80211_vif_is_mld(vif)) - return; - - IWL_WARN(mld, "uapsd misbehaving AP: %pM\n", vif->bss_conf.bssid); -} - void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt) { diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.h b/drivers/net/wireless/intel/iwlwifi/mld/iface.h index a3573d20f214a..b5e4852cb0374 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/iface.h +++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.h @@ -234,9 +234,6 @@ void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld, void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld, struct iwl_rx_packet *pkt); -void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld, - struct iwl_rx_packet *pkt); - void iwl_mld_reset_cca_40mhz_workaround(struct iwl_mld *mld, struct ieee80211_vif *vif); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/link.c b/drivers/net/wireless/intel/iwlwifi/mld/link.c index e67ba3a24d025..49e3d97f286f9 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/link.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/link.c @@ -600,15 +600,20 @@ void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld, * OR more than IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED * on current link and the link's bss_param_ch_count has changed on * the other link's beacon. + * + * When both links lose beacons, keep the primary (symmetric failure). + * When only the current link is sick, keep the other link. */ - if ((missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS && - scnd_lnk_bcn_lost >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) || - missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH || - (bss_param_ch_cnt_link_id != link_id && - missed_bcon >= - IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED)) { + if (missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS && + scnd_lnk_bcn_lost >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) { iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_MISSED_BEACON, iwl_mld_get_primary_link(vif)); + } else if (missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH || + (bss_param_ch_cnt_link_id != link_id && + missed_bcon >= + IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED)) { + iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_MISSED_BEACON, + iwl_mld_get_other_link(vif, link_id)); } } EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_handle_missed_beacon_notif); diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c index 67b61765adf39..27ae06e208402 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c @@ -1154,8 +1154,14 @@ iwl_mld_link_info_changed_ap_ibss(struct iwl_mld *mld, if (link_changes) iwl_mld_change_link_in_fw(mld, link, link_changes); - if (changes & BSS_CHANGED_BEACON) + if (changes & BSS_CHANGED_BEACON) { + WARN_ON(!link->enable_beacon); iwl_mld_update_beacon_template(mld, vif, link); + } + + /* Enabling beacons was already covered above */ + if ((changes & BSS_CHANGED_BEACON_ENABLED) && !link->enable_beacon) + iwl_mld_stop_beacon(mld, vif, link); } static @@ -2158,7 +2164,9 @@ static void iwl_mld_set_key_remove(struct iwl_mld *mld, } /* if this key was stored to be added later to the FW - free it here */ - if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) + if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE) && + (vif->type == NL80211_IFTYPE_AP || + vif->type == NL80211_IFTYPE_ADHOC)) iwl_mld_free_ap_early_key(mld, key, mld_vif); /* We already removed it */ diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c index 16bb1b4904f99..8502129abe493 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2024-2025 Intel Corporation + * Copyright (C) 2024-2026 Intel Corporation */ #include @@ -129,7 +129,7 @@ iwl_mld_get_regdomain(struct iwl_mld *mld, mld->mcc_src = resp->source_id; - /* FM is the earliest supported and later always do puncturing */ + /* FM follows BIOS/MCC policy, WH disallows puncturing only in US/CA. */ if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_FM) { if (!iwl_puncturing_is_allowed_in_bios(mld->bios_enable_puncturing, le16_to_cpu(resp->mcc))) @@ -137,6 +137,15 @@ iwl_mld_get_regdomain(struct iwl_mld *mld, else __clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING, mld->hw->flags); + } else if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) == + IWL_CFG_RF_TYPE_WH) { + u16 mcc = le16_to_cpu(resp->mcc); + + if (mcc == IWL_MCC_US || mcc == IWL_MCC_CANADA) + ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING); + else + __clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING, + mld->hw->flags); } out: diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c index 3cfe1bcb7d4e5..49e46babf8221 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c @@ -162,7 +162,6 @@ static const struct iwl_hcmd_names iwl_mld_legacy_names[] = { HCMD_NAME(PHY_CONFIGURATION_CMD), HCMD_NAME(SCAN_OFFLOAD_UPDATE_PROFILES_CMD), HCMD_NAME(POWER_TABLE_CMD), - HCMD_NAME(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION), HCMD_NAME(BEACON_NOTIFICATION), HCMD_NAME(BEACON_TEMPLATE_CMD), HCMD_NAME(TX_ANT_CONFIGURATION_CMD), @@ -671,6 +670,15 @@ iwl_mld_nic_error(struct iwl_op_mode *op_mode, if (type != IWL_ERR_TYPE_RESET_HS_TIMEOUT && mld->fw_status.running) mld->fw_status.in_hw_restart = true; + + /* FW is dead. We don't want to process its notifications. + * Right, we cancel them also in iwl_mld_stop_fw, but + * iwl_mld_async_handlers_wk might be executed before + * ieee80211_restart_work. + * In addition, in case of an error during recovery, + * iwl_mld_stop_fw might be too late. + */ + iwl_mld_cancel_async_notifications(mld); } static void iwl_mld_dump_error(struct iwl_op_mode *op_mode, diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c index a3fd0b7387d69..5f2216f8c58f5 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c @@ -338,8 +338,6 @@ CMD_VERSIONS(emlsr_mode_notif, CMD_VER_ENTRY(2, iwl_esr_mode_notif)) CMD_VERSIONS(emlsr_trans_fail_notif, CMD_VER_ENTRY(1, iwl_esr_trans_fail_notif)) -CMD_VERSIONS(uapsd_misbehaving_ap_notif, - CMD_VER_ENTRY(1, iwl_uapsd_misbehaving_ap_notif)) CMD_VERSIONS(time_msmt_notif, CMD_VER_ENTRY(1, iwl_time_msmt_notify)) CMD_VERSIONS(time_sync_confirm_notif, @@ -360,8 +358,6 @@ DEFINE_SIMPLE_CANCELLATION(scan_complete, iwl_umac_scan_complete, uid) DEFINE_SIMPLE_CANCELLATION(scan_start, iwl_umac_scan_start, uid) DEFINE_SIMPLE_CANCELLATION(probe_resp_data, iwl_probe_resp_data_notif, mac_id) -DEFINE_SIMPLE_CANCELLATION(uapsd_misbehaving_ap, iwl_uapsd_misbehaving_ap_notif, - mac_id) DEFINE_SIMPLE_CANCELLATION(ftm_resp, iwl_tof_range_rsp_ntfy, request_id) DEFINE_SIMPLE_CANCELLATION(beacon_filter, iwl_beacon_filter_notif, link_id) @@ -452,8 +448,6 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = { emlsr_mode_notif, RX_HANDLER_ASYNC) RX_HANDLER_NO_OBJECT(MAC_CONF_GROUP, EMLSR_TRANS_FAIL_NOTIF, emlsr_trans_fail_notif, RX_HANDLER_ASYNC) - RX_HANDLER_OF_VIF(LEGACY_GROUP, PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION, - uapsd_misbehaving_ap_notif) RX_HANDLER_NO_OBJECT(LEGACY_GROUP, WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION, time_msmt_notif, RX_HANDLER_SYNC) diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c index f664b277adf7d..38c77a33871d1 100644 --- a/drivers/net/wireless/intel/iwlwifi/mld/power.c +++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c @@ -113,10 +113,10 @@ static bool iwl_mld_power_is_radar(struct iwl_mld *mld, return chanctx_conf->def.chan->flags & IEEE80211_CHAN_RADAR; } -static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld, - struct iwl_mld_link *link, - struct iwl_mac_power_cmd *cmd, - bool ps_poll) +static void iwl_mld_power_configure_uapsd_v2(struct iwl_mld *mld, + struct iwl_mld_link *link, + struct iwl_mac_power_cmd_v2 *cmd, + bool ps_poll) { bool tid_found = false; @@ -175,10 +175,54 @@ static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld, cmd->uapsd_max_sp = mld->hw->uapsd_max_sp_len; } +static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld, + struct iwl_mld_link *link, + struct iwl_mac_power_cmd *cmd, + bool ps_poll) +{ + bool tid_found = false; + + /* set advanced pm flag with no uapsd ACs to enable ps-poll */ + if (ps_poll) { + cmd->flags |= cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK); + return; + } + + for (enum ieee80211_ac_numbers ac = IEEE80211_AC_VO; + ac <= IEEE80211_AC_BK; + ac++) { + if (!link->queue_params[ac].uapsd) + continue; + + cmd->flags |= + cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK); + cmd->uapsd_ac_flags |= BIT(ac); + + /* QNDP TID - the highest TID with no admission control */ + if (!tid_found && !link->queue_params[ac].acm) { + tid_found = true; + switch (ac) { + case IEEE80211_AC_VO: + cmd->qndp_tid = 6; + break; + case IEEE80211_AC_VI: + cmd->qndp_tid = 5; + break; + case IEEE80211_AC_BE: + cmd->qndp_tid = 0; + break; + case IEEE80211_AC_BK: + cmd->qndp_tid = 1; + break; + } + } + } +} + static void iwl_mld_power_config_skip_dtim(struct iwl_mld *mld, const struct ieee80211_bss_conf *link_conf, - struct iwl_mac_power_cmd *cmd) + u8 *skip_dtim_periods, __le16 *flags) { unsigned int dtimper_tu; unsigned int dtimper; @@ -196,15 +240,15 @@ iwl_mld_power_config_skip_dtim(struct iwl_mld *mld, /* configure skip over dtim up to 900 TU DTIM interval */ skip = max_t(int, 1, 900 / dtimper_tu); - cmd->skip_dtim_periods = skip; - cmd->flags |= cpu_to_le16(POWER_FLAGS_SKIP_OVER_DTIM_MSK); + *skip_dtim_periods = skip; + *flags |= cpu_to_le16(POWER_FLAGS_SKIP_OVER_DTIM_MSK); } #define POWER_KEEP_ALIVE_PERIOD_SEC 25 -static void iwl_mld_power_build_cmd(struct iwl_mld *mld, - struct ieee80211_vif *vif, - struct iwl_mac_power_cmd *cmd, - bool d3) +static void iwl_mld_power_build_cmd_v2(struct iwl_mld *mld, + struct ieee80211_vif *vif, + struct iwl_mac_power_cmd_v2 *cmd, + bool d3) { int dtimper, bi; int keep_alive; @@ -265,7 +309,9 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld, } if (d3) { - iwl_mld_power_config_skip_dtim(mld, link_conf, cmd); + iwl_mld_power_config_skip_dtim(mld, link_conf, + &cmd->skip_dtim_periods, + &cmd->flags); cmd->rx_data_timeout = cpu_to_le32(IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT); cmd->tx_data_timeout = @@ -286,6 +332,95 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld, * mac80211 will allow uAPSD. Always call iwl_mld_power_configure_uapsd * which will look at what mac80211 is saying. */ +#ifdef CONFIG_IWLWIFI_DEBUGFS + ps_poll = mld_vif->use_ps_poll; +#endif + iwl_mld_power_configure_uapsd_v2(mld, link, cmd, ps_poll); +} + +static void iwl_mld_power_build_cmd(struct iwl_mld *mld, + struct ieee80211_vif *vif, + struct iwl_mac_power_cmd *cmd, + bool d3) +{ + int dtimper, bi; + int keep_alive; + struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif); + struct ieee80211_bss_conf *link_conf = &vif->bss_conf; + struct iwl_mld_link *link = &mld_vif->deflink; + bool ps_poll = false; + __le32 fw_id = cpu_to_le32(mld_vif->fw_id); + + if (ieee80211_vif_is_mld(vif)) { + int link_id; + + if (WARN_ON(!vif->active_links)) + return; + + /* The firmware consumes one single configuration for the vif + * and can't differentiate between links, just pick the lowest + * link_id's configuration and use that. + */ + link_id = __ffs(vif->active_links); + link_conf = link_conf_dereference_check(vif, link_id); + link = iwl_mld_link_dereference_check(mld_vif, link_id); + + if (WARN_ON(!link_conf || !link)) + return; + } + dtimper = link_conf->dtim_period; + bi = link_conf->beacon_int; + + /* Regardless of power management state the driver must set + * keep alive period. FW will use it for sending keep alive NDPs + * immediately after association. Check that keep alive period + * is at least 3 * DTIM + */ + keep_alive = DIV_ROUND_UP(ieee80211_tu_to_usec(3 * dtimper * bi), + USEC_PER_SEC); + keep_alive = max(keep_alive, POWER_KEEP_ALIVE_PERIOD_SEC); + + cmd->id_and_color = fw_id; + cmd->keep_alive_seconds = cpu_to_le16(keep_alive); + + if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM) + cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_SAVE_ENA_MSK); + + if (vif->cfg.ps && iwl_mld_tdls_sta_count(mld) == 0) { + cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK); + cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK); + + /* firmware supports LPRX for beacons at rate 1 Mbps or + * 6 Mbps only + */ + if (link_conf->beacon_rate && + (link_conf->beacon_rate->bitrate == 10 || + link_conf->beacon_rate->bitrate == 60)) { + cmd->flags |= cpu_to_le16(POWER_FLAGS_LPRX_ENA_MSK); + cmd->lprx_rssi_threshold = POWER_LPRX_RSSI_THRESHOLD; + } + } + + if (d3) { + iwl_mld_power_config_skip_dtim(mld, link_conf, + &cmd->skip_dtim_periods, + &cmd->flags); + cmd->rx_data_timeout = + cpu_to_le32(IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT); + cmd->tx_data_timeout = + cpu_to_le32(IWL_MLD_WOWLAN_PS_TX_DATA_TIMEOUT); + } else if (iwl_mld_vif_low_latency(mld_vif) && vif->p2p) { + cmd->tx_data_timeout = + cpu_to_le32(IWL_MLD_SHORT_PS_TX_DATA_TIMEOUT); + cmd->rx_data_timeout = + cpu_to_le32(IWL_MLD_SHORT_PS_RX_DATA_TIMEOUT); + } else { + cmd->rx_data_timeout = + cpu_to_le32(IWL_MLD_DEFAULT_PS_RX_DATA_TIMEOUT); + cmd->tx_data_timeout = + cpu_to_le32(IWL_MLD_DEFAULT_PS_TX_DATA_TIMEOUT); + } + #ifdef CONFIG_IWLWIFI_DEBUGFS ps_poll = mld_vif->use_ps_poll; #endif @@ -295,11 +430,23 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld, int iwl_mld_update_mac_power(struct iwl_mld *mld, struct ieee80211_vif *vif, bool d3) { - struct iwl_mac_power_cmd cmd = {}; + int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0); - iwl_mld_power_build_cmd(mld, vif, &cmd, d3); + if (cmd_ver >= 3) { + struct iwl_mac_power_cmd cmd = {}; - return iwl_mld_send_cmd_pdu(mld, MAC_PM_POWER_TABLE, &cmd); + iwl_mld_power_build_cmd(mld, vif, &cmd, d3); + return iwl_mld_send_cmd_with_flags_pdu(mld, + MAC_PM_POWER_TABLE, 0, + &cmd, sizeof(cmd)); + } else { + struct iwl_mac_power_cmd_v2 cmd = {}; + + iwl_mld_power_build_cmd_v2(mld, vif, &cmd, d3); + return iwl_mld_send_cmd_with_flags_pdu(mld, + MAC_PM_POWER_TABLE, 0, + &cmd, sizeof(cmd)); + } } static void diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/binding.c b/drivers/net/wireless/intel/iwlwifi/mvm/binding.c index 58e9a940024db..0812522edea0d 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/binding.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/binding.c @@ -2,7 +2,7 @@ /* * Copyright (C) 2012-2014, 2020 Intel Corporation * Copyright (C) 2016 Intel Deutschland GmbH - * Copyright (C) 2022, 2024 Intel Corporation + * Copyright (C) 2022, 2024, 2026 Intel Corporation */ #include #include "fw-api.h" @@ -76,6 +76,9 @@ static void iwl_mvm_iface_iterator(void *_data, u8 *mac, if (vif == data->ignore_vif) return; + if (vif->type == NL80211_IFTYPE_P2P_DEVICE && !mvmvif->p2p_in_binding) + return; + if (mvmvif->deflink.phy_ctxt != data->phyctxt) return; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c index 11afe373961f3..163f124149aa5 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c @@ -1516,6 +1516,10 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm, if (WARN_ON_ONCE(truncated < 0)) truncated = 0; + /* this would be a firmware bug */ + if (WARN_ON_ONCE(pktsize < sizeof(*hdr))) + return; + if (ieee80211_is_data(hdr->frame_control)) { int hdrlen = ieee80211_hdrlen(hdr->frame_control); int ivlen = 0, icvlen = 4; /* also FCS */ @@ -1524,10 +1528,6 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm, if (!pkt) goto report; - skb_put_data(pkt, pktdata, hdrlen); - pktdata += hdrlen; - pktsize -= hdrlen; - if (ieee80211_has_protected(hdr->frame_control)) { /* * This is unlocked and using gtk_i(c)vlen, @@ -1553,6 +1553,17 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm, truncated = 0; } + if (IWL_FW_CHECK(mvm, + pktsize <= hdrlen + ivlen + icvlen, + "pktsize is too small %d\n", + pktsize)) { + kfree_skb(pkt); + return; + } + + skb_put_data(pkt, pktdata, hdrlen); + pktdata += hdrlen; + pktsize -= hdrlen; pktsize -= ivlen + icvlen; pktdata += ivlen; @@ -2783,7 +2794,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm, struct iwl_wowlan_status_data *status, u32 len) { - u32 data_size, packet_len = le32_to_cpu(notif->wake_packet_length); + u32 data_size, packet_len; if (len < sizeof(*notif)) { IWL_ERR(mvm, "Invalid WoWLAN wake packet notification!\n"); @@ -2802,6 +2813,7 @@ static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm, return -EIO; } + packet_len = le32_to_cpu(notif->wake_packet_length); data_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif, wake_packet); /* data_size got the padding from the notification, remove it. */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index aa517978fc7a3..cfca5a3ee78ec 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -955,12 +955,22 @@ int iwl_mvm_get_sar_geo_profile(struct iwl_mvm *mvm) return ret; } + if (IWL_FW_CHECK(mvm, + iwl_rx_packet_payload_len(cmd.resp_pkt) != + sizeof(*resp), + "Wrong size for iwl_geo_tx_power_profiles_resp: %d\n", + iwl_rx_packet_payload_len(cmd.resp_pkt))) { + ret = -EIO; + goto out; + } + resp = (void *)cmd.resp_pkt->data; ret = le32_to_cpu(resp->profile_idx); if (WARN_ON(ret > BIOS_GEO_MAX_PROFILE_NUM)) ret = -EIO; +out: iwl_free_resp(&cmd); return ret; } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c index 44380ebfe09d0..cee6bde6ec822 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c @@ -1504,49 +1504,54 @@ void iwl_mvm_rx_beacon_notif(struct iwl_mvm *mvm, { struct iwl_rx_packet *pkt = rxb_addr(rxb); unsigned int pkt_len = iwl_rx_packet_payload_len(pkt); - struct iwl_extended_beacon_notif *beacon = (void *)pkt->data; - struct iwl_extended_beacon_notif_v5 *beacon_v5 = (void *)pkt->data; struct ieee80211_vif *csa_vif; struct ieee80211_vif *tx_blocked_vif; struct agg_tx_status *agg_status; + u32 beacon_gp2; u16 status; lockdep_assert_held(&mvm->mutex); - mvm->ap_last_beacon_gp2 = le32_to_cpu(beacon->gp2); - if (!iwl_mvm_is_short_beacon_notif_supported(mvm)) { + struct iwl_extended_beacon_notif_v5 *beacon = (void *)pkt->data; struct iwl_tx_resp *beacon_notify_hdr = - &beacon_v5->beacon_notify_hdr; + &beacon->beacon_notify_hdr; - if (unlikely(pkt_len < sizeof(*beacon_v5))) + if (unlikely(pkt_len < sizeof(*beacon))) return; - mvm->ibss_manager = beacon_v5->ibss_mgr_status != 0; + beacon_gp2 = le32_to_cpu(beacon->gp2); + + mvm->ibss_manager = beacon->ibss_mgr_status != 0; agg_status = iwl_mvm_get_agg_status(mvm, beacon_notify_hdr); status = le16_to_cpu(agg_status->status) & TX_STATUS_MSK; IWL_DEBUG_RX(mvm, "beacon status %#x retries:%d tsf:0x%016llX gp2:0x%X rate:%d\n", status, beacon_notify_hdr->failure_frame, le64_to_cpu(beacon->tsf), - mvm->ap_last_beacon_gp2, + beacon_gp2, le32_to_cpu(beacon_notify_hdr->initial_rate)); } else { + const struct iwl_extended_beacon_notif *beacon = + (void *)pkt->data; + if (unlikely(pkt_len < sizeof(*beacon))) return; + beacon_gp2 = le32_to_cpu(beacon->gp2); + mvm->ibss_manager = beacon->ibss_mgr_status != 0; status = le32_to_cpu(beacon->status) & TX_STATUS_MSK; IWL_DEBUG_RX(mvm, "beacon status %#x tsf:0x%016llX gp2:0x%X\n", status, le64_to_cpu(beacon->tsf), - mvm->ap_last_beacon_gp2); + beacon_gp2); } csa_vif = rcu_dereference_protected(mvm->csa_vif, lockdep_is_held(&mvm->mutex)); if (unlikely(csa_vif && csa_vif->bss_conf.csa_active)) - iwl_mvm_csa_count_down(mvm, csa_vif, mvm->ap_last_beacon_gp2, + iwl_mvm_csa_count_down(mvm, csa_vif, beacon_gp2, (status == TX_STATUS_SUCCESS)); tx_blocked_vif = rcu_dereference_protected(mvm->csa_tx_blocked_vif, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c index 44029ceb8f779..a0bf9815aef6b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c @@ -625,9 +625,7 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm) hw->wiphy->max_sched_scan_reqs = 1; hw->wiphy->max_sched_scan_ssids = PROBE_OPTION_MAX; hw->wiphy->max_match_sets = iwl_umac_scan_get_max_profiles(mvm->fw); - /* we create the 802.11 header and zero length SSID IE. */ - hw->wiphy->max_sched_scan_ie_len = - SCAN_OFFLOAD_PROBE_REQ_SIZE - 24 - 2; + hw->wiphy->max_sched_scan_ie_len = iwl_mvm_max_scan_ie_len(mvm); hw->wiphy->max_sched_scan_plans = IWL_MAX_SCHED_SCAN_PLANS; hw->wiphy->max_sched_scan_plan_interval = U16_MAX; @@ -1104,6 +1102,7 @@ static void iwl_mvm_cleanup_iterator(void *data, u8 *mac, spin_unlock_bh(&mvm->time_event_lock); mvmvif->roc_activity = ROC_NUM_ACTIVITIES; + mvmvif->p2p_in_binding = false; mvmvif->bf_enabled = false; mvmvif->ba_enabled = false; @@ -3099,7 +3098,6 @@ void iwl_mvm_stop_ap_ibss_common(struct iwl_mvm *mvm, } mvmvif->ap_ibss_active = false; - mvm->ap_last_beacon_gp2 = 0; if (vif->type == NL80211_IFTYPE_AP && !vif->p2p) { iwl_mvm_vif_set_low_latency(mvmvif, false, @@ -3528,7 +3526,7 @@ static void iwl_mvm_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy, elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data, ies->len); - if (!elem || elem->datalen < 10 || + if (!elem || elem->datalen < 11 || !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) { data->tolerated = false; @@ -4681,6 +4679,7 @@ static int iwl_mvm_add_aux_sta_for_hs20(struct iwl_mvm *mvm, u32 lmac_id) static int iwl_mvm_roc_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif) { + struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); int ret; lockdep_assert_held(&mvm->mutex); @@ -4689,10 +4688,18 @@ static int iwl_mvm_roc_link(struct iwl_mvm *mvm, struct ieee80211_vif *vif) if (WARN(ret, "Failed binding P2P_DEVICE\n")) return ret; + mvmvif->p2p_in_binding = true; + /* The station and queue allocation must be done only after the binding * is done, as otherwise the FW might incorrectly configure its state. */ - return iwl_mvm_add_p2p_bcast_sta(mvm, vif); + ret = iwl_mvm_add_p2p_bcast_sta(mvm, vif); + if (ret) { + iwl_mvm_binding_remove_vif(mvm, vif); + mvmvif->p2p_in_binding = false; + } + + return ret; } static int iwl_mvm_roc(struct ieee80211_hw *hw, diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index b515028adc8f5..319301504b7eb 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2025 Intel Corporation + * Copyright (C) 2012-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -390,6 +390,8 @@ struct iwl_mvm_vif_link_info { * and in eSR mode. Valid only for a STA. * @roc_activity: currently running ROC activity for this vif (or * ROC_NUM_ACTIVITIES if no activity is running). + * @p2p_in_binding: indicates that this P2P-Device interface should be + * added to the binding, i.e. is running ROC right now * @session_prot_connection_loss: the connection was lost due to session * protection ending without receiving a beacon, so we need to now * protect the deauth separately @@ -477,7 +479,7 @@ struct iwl_mvm_vif { struct dentry *dbgfs_slink; struct iwl_dbgfs_pm dbgfs_pm; struct iwl_dbgfs_bf dbgfs_bf; - struct iwl_mac_power_cmd mac_pwr_cmd; + struct iwl_mac_power_cmd_v2 mac_pwr_cmd; int dbgfs_quota_min; bool ftm_unprotected; #endif @@ -500,6 +502,7 @@ struct iwl_mvm_vif { struct iwl_mvm_time_event_data time_event_data; struct iwl_mvm_time_event_data hs_time_event_data; enum iwl_roc_activity roc_activity; + bool p2p_in_binding; /* TCP Checksum Offload */ netdev_features_t features; @@ -1177,9 +1180,6 @@ struct iwl_mvm { struct ieee80211_vif __rcu *csa_tx_blocked_vif; u8 csa_tx_block_bcn_timeout; - /* system time of last beacon (for AP/GO interface) */ - u32 ap_last_beacon_gp2; - /* indicates that we transmitted the last beacon */ bool ibss_manager; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index 8e6913c7712f0..d6a13174b5617 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -954,7 +954,7 @@ static void iwl_mvm_frob_txf_key_iter(struct ieee80211_hw *hw, } match++; if (match == keylen) { - memset(txf->buf + i - keylen, 0xAA, keylen); + memset(txf->buf + i + 1 - keylen, 0xAA, keylen); match = 0; } } diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c index 610de29b7be0d..46792c5087532 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation + * Copyright (C) 2012-2014, 2018-2019, 2021-2026 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015-2017 Intel Deutschland GmbH */ @@ -83,7 +83,7 @@ void iwl_mvm_beacon_filter_set_cqm_params(struct iwl_mvm *mvm, } static void iwl_mvm_power_log(struct iwl_mvm *mvm, - struct iwl_mac_power_cmd *cmd) + struct iwl_mac_power_cmd_v2 *cmd) { IWL_DEBUG_POWER(mvm, "Sending power table command on mac id 0x%X for power level %d, flags = 0x%X\n", @@ -121,7 +121,7 @@ static void iwl_mvm_power_log(struct iwl_mvm *mvm, static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm, struct ieee80211_vif *vif, - struct iwl_mac_power_cmd *cmd) + struct iwl_mac_power_cmd_v2 *cmd) { struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); enum ieee80211_ac_numbers ac; @@ -296,7 +296,7 @@ static bool iwl_mvm_power_is_radar(struct ieee80211_bss_conf *link_conf) static void iwl_mvm_power_config_skip_dtim(struct iwl_mvm *mvm, struct ieee80211_vif *vif, - struct iwl_mac_power_cmd *cmd) + struct iwl_mac_power_cmd_v2 *cmd) { struct ieee80211_bss_conf *link_conf; unsigned int min_link_skip = ~0; @@ -344,7 +344,7 @@ static void iwl_mvm_power_config_skip_dtim(struct iwl_mvm *mvm, static void iwl_mvm_power_build_cmd(struct iwl_mvm *mvm, struct ieee80211_vif *vif, - struct iwl_mac_power_cmd *cmd) + struct iwl_mac_power_cmd_v2 *cmd) { int dtimper, bi; int keep_alive; @@ -466,7 +466,7 @@ static void iwl_mvm_power_build_cmd(struct iwl_mvm *mvm, static int iwl_mvm_power_send_cmd(struct iwl_mvm *mvm, struct ieee80211_vif *vif) { - struct iwl_mac_power_cmd cmd = {}; + struct iwl_mac_power_cmd_v2 cmd = {}; iwl_mvm_power_build_cmd(mvm, vif, &cmd); iwl_mvm_power_log(mvm, &cmd); @@ -717,7 +717,7 @@ int iwl_mvm_power_mac_dbgfs_read(struct iwl_mvm *mvm, int bufsz) { struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); - struct iwl_mac_power_cmd cmd = {}; + struct iwl_mac_power_cmd_v2 cmd = {}; int pos = 0; mutex_lock(&mvm->mutex); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c index 8c1bb3a7ffcaa..1e6d06bb674c1 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c @@ -1224,6 +1224,11 @@ void iwl_mvm_window_status_notif(struct iwl_mvm *mvm, /* get the station */ sta_id = (ratid & BA_WINDOW_STATUS_STA_ID_MSK) >> BA_WINDOW_STATUS_STA_ID_POS; + if (IWL_FW_CHECK(mvm, + sta_id >= mvm->fw->ucode_capa.num_stations, + "Invalid sta id (%d) in BA window status notification\n", + sta_id)) + continue; sta = rcu_dereference(mvm->fw_id_to_mac_id[sta_id]); if (IS_ERR_OR_NULL(sta)) continue; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c index b588f1dcf20d5..bc2c240160b2c 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c @@ -3148,8 +3148,6 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm, if (ret) return ret; - iwl_mvm_build_scan_probe(mvm, vif, ies, ¶ms); - /* for 6 GHZ band only PSC channels need to be added */ for (i = 0; i < params.n_channels; i++) { struct ieee80211_channel *channel = params.channels[i]; @@ -3183,6 +3181,8 @@ int iwl_mvm_sched_scan_start(struct iwl_mvm *mvm, goto out; } + iwl_mvm_build_scan_probe(mvm, vif, ies, ¶ms); + uid = iwl_mvm_build_scan_cmd(mvm, vif, &hcmd, ¶ms, type); if (uid < 0) { ret = uid; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c index 0b12ee8ad6180..356f68f5e6ac3 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/time-event.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2025 Intel Corporation + * Copyright (C) 2012-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2017 Intel Deutschland GmbH */ @@ -89,6 +89,7 @@ static void iwl_mvm_cleanup_roc(struct iwl_mvm *mvm) } else { iwl_mvm_rm_p2p_bcast_sta(mvm, vif); iwl_mvm_binding_remove_vif(mvm, vif); + mvmvif->p2p_in_binding = false; } /* Do not remove the PHY context as removing and adding @@ -959,6 +960,11 @@ void iwl_mvm_rx_session_protect_notif(struct iwl_mvm *mvm, struct ieee80211_vif *vif; struct iwl_mvm_vif *mvmvif; + if (IWL_FW_CHECK(mvm, id >= ARRAY_SIZE(mvm->vif_id_to_mac), + "Invalid mac_link_id (%d) in session protect notif\n", + id)) + return; + rcu_read_lock(); /* note we use link ID == MAC ID */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c index bb97837baedaf..eab3f7184b49b 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c @@ -1581,6 +1581,17 @@ static inline u32 iwl_mvm_get_scd_ssn(struct iwl_mvm *mvm, return val & 0xFFF; } +static inline size_t iwl_mvm_tx_resp_min_len(struct iwl_mvm *mvm, + struct iwl_tx_resp *tx_resp) +{ + struct agg_tx_status *agg_status = + iwl_mvm_get_agg_status(mvm, tx_resp); + + /* The aggregate response ends with a trailing SCD SSN __le32 word. */ + return (u8 *)(agg_status + tx_resp->frame_count) - (u8 *)tx_resp + + sizeof(__le32); +} + static void iwl_mvm_rx_tx_cmd_single(struct iwl_mvm *mvm, struct iwl_rx_packet *pkt) { @@ -1884,6 +1895,9 @@ static void iwl_mvm_rx_tx_cmd_agg(struct iwl_mvm *mvm, int queue = SEQ_TO_QUEUE(sequence); struct ieee80211_sta *sta; + if (WARN_ON_ONCE(iwl_mvm_has_new_tx_api(mvm))) + return; + if (WARN_ON_ONCE(queue < IWL_MVM_DQA_MIN_DATA_QUEUE && (queue != IWL_MVM_DQA_BSS_CLIENT_QUEUE))) return; @@ -1918,6 +1932,26 @@ void iwl_mvm_rx_tx_cmd(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb) { struct iwl_rx_packet *pkt = rxb_addr(rxb); struct iwl_tx_resp *tx_resp = (void *)pkt->data; + size_t min_len; + + if (IWL_FW_CHECK(mvm, !tx_resp->frame_count, + "invalid TX_CMD frame_count %u\n", + tx_resp->frame_count)) + return; + + if (IWL_FW_CHECK(mvm, + iwl_mvm_has_new_tx_api(mvm) && + tx_resp->frame_count != 1, + "invalid TX_CMD frame_count %u for new TX API\n", + tx_resp->frame_count)) + return; + + min_len = iwl_mvm_tx_resp_min_len(mvm, tx_resp); + if (IWL_FW_CHECK(mvm, iwl_rx_packet_payload_len(pkt) < min_len, + "invalid TX_CMD len %u (frame_count %u, min %zu)\n", + iwl_rx_packet_payload_len(pkt), tx_resp->frame_count, + min_len)) + return; if (tx_resp->frame_count == 1) iwl_mvm_rx_tx_cmd_single(mvm, pkt); @@ -2137,8 +2171,14 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb) if (tid == IWL_MGMT_TID) tid = IWL_MAX_TID_COUNT; + if (IWL_FW_CHECK(mvm, tid >= + ARRAY_SIZE(mvmsta->tid_data), + "invalid TID %d in compressed BA\n", + tid)) + continue; + if (mvmsta) - mvmsta->tid_data[i].lq_color = lq_color; + mvmsta->tid_data[tid].lq_color = lq_color; iwl_mvm_tx_reclaim(mvm, sta_id, tid, (int)(le16_to_cpu(ba_tfd->q_num)), @@ -2158,6 +2198,9 @@ void iwl_mvm_rx_ba_notif(struct iwl_mvm *mvm, struct iwl_rx_cmd_buffer *rxb) ba_notif = (void *)pkt->data; sta_id = ba_notif->sta_id; tid = ba_notif->tid; + if (IWL_FW_CHECK(mvm, tid >= ARRAY_SIZE(mvmsta->tid_data), + "invalid TID %d in BA notif\n", tid)) + return; /* "flow" corresponds to Tx queue */ txq = le16_to_cpu(ba_notif->scd_flow); /* "ssn" is start of block-ack Tx window, corresponds to index diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 706dc7bb9a18d..0e8481219839b 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2005-2014, 2018-2025 Intel Corporation + * Copyright (C) 2005-2014, 2018-2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -537,6 +537,8 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct pci_device_id iwl_hw_card_ids[] = { {IWL_PCI_DEVICE(0xA840, 0x4314, iwl_bz_mac_cfg)}, {IWL_PCI_DEVICE(0xA840, 0x1775, iwl_bz_mac_cfg)}, {IWL_PCI_DEVICE(0xA840, 0x1776, iwl_bz_mac_cfg)}, + {IWL_PCI_DEVICE(0xA840, 0x1735, iwl_bz_mac_cfg)}, + {IWL_PCI_DEVICE(0xA840, 0x1736, iwl_bz_mac_cfg)}, {IWL_PCI_DEVICE(0x7740, PCI_ANY_ID, iwl_bz_mac_cfg)}, {IWL_PCI_DEVICE(0x4D40, PCI_ANY_ID, iwl_bz_mac_cfg)}, @@ -1014,6 +1016,7 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = { /* GF RF */ IWL_DEV_INFO(iwl_rf_gf, iwl_ax211_name, RF_TYPE(GF)), + IWL_DEV_INFO(iwl_rf_ot, iwl_ax231_name, RF_TYPE(GF), SUBDEV(0x0294)), IWL_DEV_INFO(iwl_rf_gf, iwl_ax411_name, RF_TYPE(GF), CDB), IWL_DEV_INFO(iwl_rf_gf, iwl_ax210_name, DEVICE(0x2725)), diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c index 619a9505e6d9d..c723839288f2e 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/rx.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2003-2014, 2018-2024 Intel Corporation + * Copyright (C) 2003-2014, 2018-2024, 2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -1241,11 +1241,16 @@ void iwl_pcie_rx_free(struct iwl_trans *trans) } } kfree(trans_pcie->rx_pool); + trans_pcie->rx_pool = NULL; kfree(trans_pcie->global_table); + trans_pcie->global_table = NULL; kfree(trans_pcie->rxq); + trans_pcie->rxq = NULL; - if (trans_pcie->alloc_page) + if (trans_pcie->alloc_page) { __free_pages(trans_pcie->alloc_page, trans_pcie->rx_page_order); + trans_pcie->alloc_page = NULL; + } } static void iwl_pcie_rx_move_to_allocator(struct iwl_rxq *rxq, diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c index 59307b5df4417..0c920e3d6c5e0 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2007-2015, 2018-2024 Intel Corporation + * Copyright (C) 2007-2015, 2018-2024, 2026 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -2044,7 +2044,7 @@ iwl_trans_pcie_call_prod_reset_dsm(struct pci_dev *pdev, u16 cmd, u16 value) 0xDD, 0x26, 0xB5, 0xFD); if (!acpi_check_dsm(ACPI_HANDLE(&pdev->dev), &dsm_guid, ACPI_DSM_REV, - DSM_INTERNAL_FUNC_PRODUCT_RESET)) + BIT(DSM_INTERNAL_FUNC_PRODUCT_RESET))) return ERR_PTR(-ENODEV); return iwl_acpi_get_dsm_object(&pdev->dev, ACPI_DSM_REV, @@ -4000,6 +4000,30 @@ static void get_crf_id(struct iwl_trans *iwl_trans, else sd_reg_ver_addr = SD_REG_VER; + /* wait until the device is ready to access the prph registers */ + if (iwl_trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_DR || + iwl_trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_SC) { + u32 req = iwl_read_umac_prph_no_grab(iwl_trans, + WFPM_RSRCS_4PHS_REQ_STTS); + int ret; + + if (!(req & RSRC_REQ_CNVR_TOP)) { + IWL_ERR(iwl_trans, + "WFPM_RSRCS_4PHS_REQ_STTS bit 6 is clear 0x%x\n", + req); + return; + } + + ret = iwl_poll_umac_prph_bits_no_grab(iwl_trans, + WFPM_RSRCS_4PHS_ACK_STTS, + RSRC_ACK_CNVR_TOP, + RSRC_ACK_CNVR_TOP, + 50 * 1000); + if (ret < 0) + IWL_ERR(iwl_trans, + "WFPM_RSRCS_4PHS_ACK_STTS bit 6 is clear\n"); + } + /* Enable access to peripheral registers */ val = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG); val |= WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK; diff --git a/drivers/net/wireless/intersil/p54/txrx.c b/drivers/net/wireless/intersil/p54/txrx.c index 1294a1d6528e2..9f491334c8d04 100644 --- a/drivers/net/wireless/intersil/p54/txrx.c +++ b/drivers/net/wireless/intersil/p54/txrx.c @@ -499,11 +499,19 @@ static void p54_rx_eeprom_readback(struct p54_common *priv, if (le16_to_cpu(eeprom->v2.len) != priv->eeprom_slice_size) return; + if (eeprom->v2.data + priv->eeprom_slice_size > + skb_tail_pointer(skb)) + return; + memcpy(priv->eeprom, eeprom->v2.data, priv->eeprom_slice_size); } else { if (le16_to_cpu(eeprom->v1.len) != priv->eeprom_slice_size) return; + if (eeprom->v1.data + priv->eeprom_slice_size > + skb_tail_pointer(skb)) + return; + memcpy(priv->eeprom, eeprom->v1.data, priv->eeprom_slice_size); } diff --git a/drivers/net/wireless/marvell/libertas/firmware.c b/drivers/net/wireless/marvell/libertas/firmware.c index f124110944b7e..9bf7d4c207b9e 100644 --- a/drivers/net/wireless/marvell/libertas/firmware.c +++ b/drivers/net/wireless/marvell/libertas/firmware.c @@ -78,6 +78,7 @@ static void helper_firmware_cb(const struct firmware *firmware, void *context) } else { /* No main firmware needed for this helper --> success! */ lbs_fw_loaded(priv, 0, firmware, NULL); + release_firmware(firmware); } } diff --git a/drivers/net/wireless/marvell/libertas/tx.c b/drivers/net/wireless/marvell/libertas/tx.c index 27304a98787d6..13d08022e4141 100644 --- a/drivers/net/wireless/marvell/libertas/tx.c +++ b/drivers/net/wireless/marvell/libertas/tx.c @@ -117,6 +117,13 @@ netdev_tx_t lbs_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) if (priv->wdev->iftype == NL80211_IFTYPE_MONITOR) { struct tx_radiotap_hdr *rtap_hdr = (void *)skb->data; + if (skb->len < sizeof(*rtap_hdr) + 4 + ETH_ALEN) { + lbs_deb_tx("tx err: short monitor frame %u\n", skb->len); + dev->stats.tx_dropped++; + dev->stats.tx_errors++; + goto free; + } + /* set txpd fields from the radiotap header */ txpd->tx_control = cpu_to_le32(convert_radiotap_rate_to_mv(rtap_hdr->rate)); diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c index f3397dc6c422e..c27814c1ccb1b 100644 --- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c +++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c @@ -44,7 +44,7 @@ static int mwifiex_11n_dispatch_amsdu_pkt(struct mwifiex_private *priv, ntohs(rx_hdr->eth803_hdr.h_proto) == ETH_P_TDLS) { mwifiex_process_tdls_action_frame(priv, (u8 *)rx_hdr, - skb->len); + rx_skb->len); } if (priv->bss_role == MWIFIEX_BSS_ROLE_UAP) diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h index e9e8966069121..93561116959ac 100644 --- a/drivers/net/wireless/marvell/mwifiex/fw.h +++ b/drivers/net/wireless/marvell/mwifiex/fw.h @@ -823,7 +823,7 @@ struct chan_band_param_set { struct mwifiex_ie_types_chan_band_list_param_set { struct mwifiex_ie_types_header header; - struct chan_band_param_set chan_band_param[1]; + struct chan_band_param_set chan_band_param[]; } __packed; struct mwifiex_ie_types_rates_param_set { @@ -886,7 +886,7 @@ struct mwifiex_ie_types_wildcard_ssid_params { #define TSF_DATA_SIZE 8 struct mwifiex_ie_types_tsf_timestamp { struct mwifiex_ie_types_header header; - u8 tsf_data[1]; + u8 tsf_data[]; } __packed; struct mwifiex_cf_param_set { @@ -903,8 +903,8 @@ struct mwifiex_ibss_param_set { struct mwifiex_ie_types_ss_param_set { struct mwifiex_ie_types_header header; union { - struct mwifiex_cf_param_set cf_param_set[1]; - struct mwifiex_ibss_param_set ibss_param_set[1]; + DECLARE_FLEX_ARRAY(struct mwifiex_cf_param_set, cf_param_set); + DECLARE_FLEX_ARRAY(struct mwifiex_ibss_param_set, ibss_param_set); } cf_ibss; } __packed; @@ -922,8 +922,8 @@ struct mwifiex_ds_param_set { struct mwifiex_ie_types_phy_param_set { struct mwifiex_ie_types_header header; union { - struct mwifiex_fh_param_set fh_param_set[1]; - struct mwifiex_ds_param_set ds_param_set[1]; + DECLARE_FLEX_ARRAY(struct mwifiex_fh_param_set, fh_param_set); + DECLARE_FLEX_ARRAY(struct mwifiex_ds_param_set, ds_param_set); } fh_ds; } __packed; @@ -1383,7 +1383,7 @@ struct host_cmd_ds_802_11_snmp_mib { __le16 query_type; __le16 oid; __le16 buf_size; - u8 value[1]; + u8 value[]; } __packed; struct mwifiex_rate_scope { @@ -1551,7 +1551,7 @@ struct mwifiex_scan_cmd_config { * TLV_TYPE_CHANLIST, mwifiex_ie_types_chan_list_param_set * WLAN_EID_SSID, mwifiex_ie_types_ssid_param_set */ - u8 tlv_buf[1]; /* SSID TLV(s) and ChanList TLVs are stored + u8 tlv_buf[]; /* SSID TLV(s) and ChanList TLVs are stored here */ } __packed; @@ -1683,7 +1683,7 @@ struct host_cmd_ds_802_11_bg_scan_query_rsp { struct mwifiex_ietypes_domain_param_set { struct mwifiex_ie_types_header header; u8 country_code[IEEE80211_COUNTRY_STRING_LEN]; - struct ieee80211_country_ie_triplet triplet[1]; + struct ieee80211_country_ie_triplet triplet[]; } __packed; struct host_cmd_ds_802_11d_domain_info { diff --git a/drivers/net/wireless/marvell/mwifiex/join.c b/drivers/net/wireless/marvell/mwifiex/join.c index b48f7febaf03f..259140395d353 100644 --- a/drivers/net/wireless/marvell/mwifiex/join.c +++ b/drivers/net/wireless/marvell/mwifiex/join.c @@ -421,15 +421,15 @@ int mwifiex_cmd_802_11_associate(struct mwifiex_private *priv, phy_tlv = (struct mwifiex_ie_types_phy_param_set *) pos; phy_tlv->header.type = cpu_to_le16(WLAN_EID_DS_PARAMS); - phy_tlv->header.len = cpu_to_le16(sizeof(phy_tlv->fh_ds.ds_param_set)); - memcpy(&phy_tlv->fh_ds.ds_param_set, + phy_tlv->header.len = cpu_to_le16(sizeof(*phy_tlv->fh_ds.ds_param_set)); + memcpy(phy_tlv->fh_ds.ds_param_set, &bss_desc->phy_param_set.ds_param_set.current_chan, - sizeof(phy_tlv->fh_ds.ds_param_set)); + sizeof(*phy_tlv->fh_ds.ds_param_set)); pos += sizeof(phy_tlv->header) + le16_to_cpu(phy_tlv->header.len); ss_tlv = (struct mwifiex_ie_types_ss_param_set *) pos; ss_tlv->header.type = cpu_to_le16(WLAN_EID_CF_PARAMS); - ss_tlv->header.len = cpu_to_le16(sizeof(ss_tlv->cf_ibss.cf_param_set)); + ss_tlv->header.len = cpu_to_le16(sizeof(*ss_tlv->cf_ibss.cf_param_set)); pos += sizeof(ss_tlv->header) + le16_to_cpu(ss_tlv->header.len); /* Get the common rates supported between the driver and the BSS Desc */ diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c index dcca71158fc68..0a0458e15e289 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c @@ -108,7 +108,7 @@ static int mwifiex_cmd_802_11_snmp_mib(struct mwifiex_private *priv, "cmd: SNMP_CMD: cmd_oid = 0x%x\n", cmd_oid); cmd->command = cpu_to_le16(HostCmd_CMD_802_11_SNMP_MIB); cmd->size = cpu_to_le16(sizeof(struct host_cmd_ds_802_11_snmp_mib) - - 1 + S_DS_GEN); + + S_DS_GEN); snmp_mib->oid = cpu_to_le16((u16)cmd_oid); if (cmd_action == HostCmd_ACT_GEN_GET) { diff --git a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c index ef6722ffdc74d..b8267a5ebcfb0 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_ioctl.c @@ -57,6 +57,18 @@ int mwifiex_wait_queue_complete(struct mwifiex_adapter *adapter, mwifiex_dbg(adapter, ERROR, "cmd_wait_q terminated: %d\n", status); mwifiex_cancel_all_pending_cmd(adapter); + + /* The command response path writes through cmd_node->data_buf. + * On an interrupted wait, the caller can return and release a + * stack-allocated data_buf before a late firmware response is + * processed. Detach the caller-owned buffer from the current + * command so a late response cannot corrupt freed stack memory. + */ + spin_lock_bh(&adapter->mwifiex_cmd_lock); + if (adapter->curr_cmd == cmd_queued) + adapter->curr_cmd->data_buf = NULL; + spin_unlock_bh(&adapter->mwifiex_cmd_lock); + return status; } @@ -196,6 +208,7 @@ static int mwifiex_request_rgpower_table(struct mwifiex_private *priv) struct mwifiex_adapter *adapter = priv->adapter; char rgpower_table_name[30]; char country_code[3]; + int ret; strscpy(country_code, domain_info->country_code, sizeof(country_code)); @@ -214,16 +227,17 @@ static int mwifiex_request_rgpower_table(struct mwifiex_private *priv) adapter->rgpower_data = NULL; } - if ((request_firmware(&adapter->rgpower_data, rgpower_table_name, - adapter->dev))) { + ret = request_firmware_direct(&adapter->rgpower_data, rgpower_table_name, + adapter->dev); + + if (ret) { mwifiex_dbg( adapter, INFO, - "info: %s: failed to request regulatory power table\n", - __func__); - return -EIO; + "info: %s: failed to request regulatory power table: %d\n", + __func__, ret); } - return 0; + return ret; } static int mwifiex_dnld_rgpower_table(struct mwifiex_private *priv) diff --git a/drivers/net/wireless/marvell/mwifiex/tdls.c b/drivers/net/wireless/marvell/mwifiex/tdls.c index 77a9a6de636d1..b94ca13086d67 100644 --- a/drivers/net/wireless/marvell/mwifiex/tdls.c +++ b/drivers/net/wireless/marvell/mwifiex/tdls.c @@ -215,7 +215,7 @@ mwifiex_tdls_add_ht_oper(struct mwifiex_private *priv, const u8 *mac, /* follow AP's channel bandwidth */ if (ISSUPP_CHANWIDTH40(priv->adapter->hw_dot_11n_dev_cap) && - bss_desc->bcn_ht_cap && + bss_desc->bcn_ht_oper && ISALLOWED_CHANWIDTH40(bss_desc->bcn_ht_oper->ht_param)) ht_oper->ht_param = bss_desc->bcn_ht_oper->ht_param; diff --git a/drivers/net/wireless/marvell/mwifiex/uap_event.c b/drivers/net/wireless/marvell/mwifiex/uap_event.c index 245cb99a3daad..fc9da9006a782 100644 --- a/drivers/net/wireless/marvell/mwifiex/uap_event.c +++ b/drivers/net/wireless/marvell/mwifiex/uap_event.c @@ -123,11 +123,31 @@ int mwifiex_process_uap_event(struct mwifiex_private *priv) len = ETH_ALEN; if (len != -1) { + u16 evt_len = le16_to_cpu(event->len); + sinfo->assoc_req_ies = &event->data[len]; len = (u8 *)sinfo->assoc_req_ies - (u8 *)&event->frame_control; - sinfo->assoc_req_ies_len = - le16_to_cpu(event->len) - (u16)len; + + /* + * event->len is reported by the device firmware + * and is not otherwise validated. Reject a + * length that underflows the header, or that + * would place the association request IEs + * outside the fixed-size event_body[] buffer the + * event was copied into; otherwise the IE walk + * in mwifiex_set_sta_ht_cap() reads past + * event_body and out of the adapter slab object. + */ + if (evt_len < len || + (u8 *)&event->frame_control + evt_len > + adapter->event_body + MAX_EVENT_SIZE) { + mwifiex_dbg(adapter, ERROR, + "invalid STA assoc event length\n"); + kfree(sinfo); + return -1; + } + sinfo->assoc_req_ies_len = evt_len - (u16)len; } } cfg80211_new_sta(priv->netdev, event->sta_addr, sinfo, diff --git a/drivers/net/wireless/mediatek/mt76/Kconfig b/drivers/net/wireless/mediatek/mt76/Kconfig index a86f800b8bf54..71e58689e12c4 100644 --- a/drivers/net/wireless/mediatek/mt76/Kconfig +++ b/drivers/net/wireless/mediatek/mt76/Kconfig @@ -37,6 +37,10 @@ config MT792x_USB tristate select MT76_USB +config MT76_NPU + tristate + depends on NET_AIROHA_NPU=y || MT76_CORE=NET_AIROHA_NPU + source "drivers/net/wireless/mediatek/mt76/mt76x0/Kconfig" source "drivers/net/wireless/mediatek/mt76/mt76x2/Kconfig" source "drivers/net/wireless/mediatek/mt76/mt7603/Kconfig" diff --git a/drivers/net/wireless/mediatek/mt76/Makefile b/drivers/net/wireless/mediatek/mt76/Makefile index 87512d101a919..df2ac99c88729 100644 --- a/drivers/net/wireless/mediatek/mt76/Makefile +++ b/drivers/net/wireless/mediatek/mt76/Makefile @@ -12,6 +12,11 @@ mt76-y := \ mmio.o util.o trace.o dma.o mac80211.o debugfs.o eeprom.o \ tx.o agg-rx.o mcu.o wed.o scan.o channel.o +ifdef CONFIG_MT76_NPU +# CONFIG_MT76_NPU is tristate to simplify dependency tracking, +# but it behaves as a bool symbol here. +mt76-y += npu.o +endif mt76-$(CONFIG_PCI) += pci.o mt76-$(CONFIG_NL80211_TESTMODE) += testmode.o diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c index 9ef073c27f309..a487887ef5444 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.c +++ b/drivers/net/wireless/mediatek/mt76/dma.c @@ -6,37 +6,7 @@ #include #include "mt76.h" #include "dma.h" - -#if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) - -#define Q_READ(_q, _field) ({ \ - u32 _offset = offsetof(struct mt76_queue_regs, _field); \ - u32 _val; \ - if ((_q)->flags & MT_QFLAG_WED) \ - _val = mtk_wed_device_reg_read((_q)->wed, \ - ((_q)->wed_regs + \ - _offset)); \ - else \ - _val = readl(&(_q)->regs->_field); \ - _val; \ -}) - -#define Q_WRITE(_q, _field, _val) do { \ - u32 _offset = offsetof(struct mt76_queue_regs, _field); \ - if ((_q)->flags & MT_QFLAG_WED) \ - mtk_wed_device_reg_write((_q)->wed, \ - ((_q)->wed_regs + _offset), \ - _val); \ - else \ - writel(_val, &(_q)->regs->_field); \ -} while (0) - -#else - -#define Q_READ(_q, _field) readl(&(_q)->regs->_field) -#define Q_WRITE(_q, _field, _val) writel(_val, &(_q)->regs->_field) - -#endif +#include "mt76_connac.h" static struct mt76_txwi_cache * mt76_alloc_txwi(struct mt76_dev *dev) @@ -216,15 +186,35 @@ mt76_dma_queue_magic_cnt_init(struct mt76_dev *dev, struct mt76_queue *q) } } +/* A hung bus (e.g. after a PCIe AER error) reads 0xffffffff from every + * register, so clamp an out-of-range index to the fallback to keep it from + * corrupting q->head/q->tail. + */ +static int +mt76_dma_read_dma_idx(struct mt76_queue *q, int fallback) +{ + u32 idx = Q_READ(q, dma_idx); + + return idx < q->ndesc ? idx : fallback; +} + static void mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q) { - Q_WRITE(q, desc_base, q->desc_dma); - if (q->flags & MT_QFLAG_WED_RRO_EN) + if ((q->flags & MT_QFLAG_WED_RRO_EN) && + (!is_mt7992(dev) || !mt76_npu_device_active(dev))) Q_WRITE(q, ring_size, MT_DMA_RRO_EN | q->ndesc); else Q_WRITE(q, ring_size, q->ndesc); - q->head = Q_READ(q, dma_idx); + + if (mt76_queue_is_npu_tx(q)) { + writel(q->ndesc, &q->regs->ring_size); + writel(q->desc_dma, &q->regs->desc_base); + } + + Q_WRITE(q, desc_base, q->desc_dma); + + q->head = mt76_dma_read_dma_idx(q, 0); q->tail = q->head; } @@ -235,7 +225,7 @@ void mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q, return; if (!mt76_queue_is_wed_rro_ind(q) && - !mt76_queue_is_wed_rro_rxdmad_c(q)) { + !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { int i; /* clear descriptors */ @@ -442,10 +432,11 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) if (flush) last = -1; else - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); while (q->queued > 0 && q->tail != last) { mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry); + mt76_npu_txdesc_cleanup(q, q->tail); mt76_queue_tx_complete(dev, q, &entry); if (entry.txwi) { @@ -454,7 +445,7 @@ mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush) } if (!flush && q->tail == last) - last = Q_READ(q, dma_idx); + last = mt76_dma_read_dma_idx(q, -1); } spin_unlock_bh(&q->cleanup_lock); @@ -556,8 +547,13 @@ mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx, t->ptr = NULL; mt76_put_rxwi(dev, t); - if (drop) +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* the WO MCU owns the RX path only on WED v2, on newer + * versions this buf1 bit carries no drop information + */ + if (drop && dev->mmio.wed.version == 2) *drop |= !!(buf1 & MT_DMA_CTL_WO_DROP); +#endif } else { dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], SKB_WITH_OVERHEAD(q->buf_size), @@ -617,6 +613,15 @@ mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush, q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE); else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE))) return NULL; +#ifdef CONFIG_NET_MEDIATEK_SOC_WED + /* on WED v3 the M_DONE bit signals that WED is done reading + * the txfree descriptor; WED v2 does not set it + */ + else if (dev->mmio.wed.version > 2 && + mt76_queue_is_wed_tx_free(q) && + !(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_M_DONE))) + return NULL; +#endif } done: q->tail = (q->tail + 1) % q->ndesc; @@ -647,8 +652,8 @@ mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q, buf.len = skb->len; spin_lock_bh(&q->lock); - mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL); - mt76_dma_kick_queue(dev, q); + if (mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL) >= 0) + mt76_dma_kick_queue(dev, q); spin_unlock_bh(&q->lock); return 0; @@ -680,6 +685,10 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, if (test_bit(MT76_RESET, &phy->state)) goto free_skb; + /* TODO: Take into account unlinear skbs */ + if (mt76_npu_device_active(dev) && skb_linearize(skb)) + goto free_skb; + t = mt76_get_txwi(dev); if (!t) goto free_skb; @@ -727,6 +736,9 @@ mt76_dma_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, if (ret < 0) goto unmap; + if (mt76_npu_device_active(dev)) + return mt76_npu_dma_add_buf(phy, q, skb, &tx_info.buf[1], txwi); + return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf, tx_info.info, tx_info.skb, t); @@ -826,8 +838,15 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, q->buf_size = bufsize; q->hw_idx = idx; - size = mt76_queue_is_wed_rro_ind(q) ? sizeof(struct mt76_wed_rro_desc) - : sizeof(struct mt76_desc); + if (mt76_queue_is_wed_rro_ind(q)) + size = sizeof(struct mt76_wed_rro_desc); + else if (mt76_queue_is_npu_tx(q)) + size = sizeof(struct airoha_npu_tx_dma_desc); + else if (mt76_queue_is_npu_rx(q)) + size = sizeof(struct airoha_npu_rx_dma_desc); + else + size = sizeof(struct mt76_desc); + q->desc = dmam_alloc_coherent(dev->dma_dev, q->ndesc * size, &q->desc_dma, GFP_KERNEL); if (!q->desc) @@ -843,6 +862,7 @@ mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q, if (ret) return ret; + mt76_npu_queue_setup(dev, q); ret = mt76_wed_dma_setup(dev, q, false); if (ret) return ret; @@ -870,6 +890,11 @@ mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q) if (!q->ndesc) return; + if (mt76_queue_is_npu(q)) { + mt76_npu_queue_cleanup(dev, q); + return; + } + do { spin_lock_bh(&q->lock); buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more, NULL); @@ -905,7 +930,7 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) return; if (!mt76_queue_is_wed_rro_ind(q) && - !mt76_queue_is_wed_rro_rxdmad_c(q)) { + !mt76_queue_is_wed_rro_rxdmad_c(q) && !mt76_queue_is_npu(q)) { int i; for (i = 0; i < q->ndesc; i++) @@ -925,7 +950,10 @@ mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid) return; mt76_dma_sync_idx(dev, q); - mt76_dma_rx_fill_buf(dev, q, false); + if (mt76_queue_is_npu(q)) + mt76_npu_fill_rx_queue(dev, q); + else + mt76_dma_rx_fill(dev, q, false); } static void @@ -962,23 +990,24 @@ mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget) struct sk_buff *skb; unsigned char *data; bool check_ddone = false; - bool allow_direct = !mt76_queue_is_wed_rx(q); + bool allow_direct = !mt76_queue_is_wed_rx(q) && + !mt76_queue_is_wed_rro_rxdmad_c(q); bool more; if ((q->flags & MT_QFLAG_WED_RRO_EN) || (IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) && mt76_queue_is_wed_tx_free(q))) { - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); check_ddone = true; } while (done < budget) { bool drop = false; - u32 info; + u32 info = 0; if (check_ddone) { if (q->tail == dma_idx) - dma_idx = Q_READ(q, dma_idx); + dma_idx = mt76_dma_read_dma_idx(q, q->tail); if (q->tail == dma_idx) break; diff --git a/drivers/net/wireless/mediatek/mt76/dma.h b/drivers/net/wireless/mediatek/mt76/dma.h index 17a80e1757fcc..1c556c8c86347 100644 --- a/drivers/net/wireless/mediatek/mt76/dma.h +++ b/drivers/net/wireless/mediatek/mt76/dma.h @@ -11,7 +11,7 @@ #define MT_DMA_CTL_SD_LEN1 GENMASK(13, 0) #define MT_DMA_CTL_LAST_SEC1 BIT(14) -#define MT_DMA_CTL_BURST BIT(15) +#define MT_DMA_CTL_M_DONE BIT(15) #define MT_DMA_CTL_SD_LEN0 GENMASK(29, 16) #define MT_DMA_CTL_LAST_SEC0 BIT(30) #define MT_DMA_CTL_DMA_DONE BIT(31) @@ -46,6 +46,73 @@ #define MT_FCE_INFO_LEN 4 #define MT_RX_RXWI_LEN 32 +#if IS_ENABLED(CONFIG_NET_MEDIATEK_SOC_WED) + +#define Q_READ(_q, _field) ({ \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + u32 _val; \ + if ((_q)->flags & MT_QFLAG_WED) \ + _val = mtk_wed_device_reg_read((_q)->wed, \ + ((_q)->wed_regs + \ + _offset)); \ + else \ + _val = readl(&(_q)->regs->_field); \ + _val; \ +}) + +#define Q_WRITE(_q, _field, _val) do { \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + if ((_q)->flags & MT_QFLAG_WED) \ + mtk_wed_device_reg_write((_q)->wed, \ + ((_q)->wed_regs + _offset), \ + _val); \ + else \ + writel(_val, &(_q)->regs->_field); \ +} while (0) + +#elif IS_ENABLED(CONFIG_MT76_NPU) + +#define Q_READ(_q, _field) ({ \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + u32 _val = 0; \ + if ((_q)->flags & MT_QFLAG_NPU) { \ + struct airoha_npu *npu; \ + \ + rcu_read_lock(); \ + npu = rcu_dereference(q->dev->mmio.npu); \ + if (npu) \ + regmap_read(npu->regmap, \ + ((_q)->wed_regs + _offset), &_val); \ + rcu_read_unlock(); \ + } else { \ + _val = readl(&(_q)->regs->_field); \ + } \ + _val; \ +}) + +#define Q_WRITE(_q, _field, _val) do { \ + u32 _offset = offsetof(struct mt76_queue_regs, _field); \ + if ((_q)->flags & MT_QFLAG_NPU) { \ + struct airoha_npu *npu; \ + \ + rcu_read_lock(); \ + npu = rcu_dereference(q->dev->mmio.npu); \ + if (npu) \ + regmap_write(npu->regmap, \ + ((_q)->wed_regs + _offset), _val); \ + rcu_read_unlock(); \ + } else { \ + writel(_val, &(_q)->regs->_field); \ + } \ +} while (0) + +#else + +#define Q_READ(_q, _field) readl(&(_q)->regs->_field) +#define Q_WRITE(_q, _field, _val) writel(_val, &(_q)->regs->_field) + +#endif + struct mt76_desc { __le32 buf0; __le32 ctrl; diff --git a/drivers/net/wireless/mediatek/mt76/mac80211.c b/drivers/net/wireless/mediatek/mt76/mac80211.c index 7348ac5cf7e60..2d8cd627ef4c6 100644 --- a/drivers/net/wireless/mediatek/mt76/mac80211.c +++ b/drivers/net/wireless/mediatek/mt76/mac80211.c @@ -632,6 +632,8 @@ int mt76_create_page_pool(struct mt76_dev *dev, struct mt76_queue *q) case MT_RXQ_MAIN: case MT_RXQ_BAND1: case MT_RXQ_BAND2: + case MT_RXQ_NPU0: + case MT_RXQ_NPU1: pp_params.pool_size = 256; break; default: @@ -817,6 +819,7 @@ void mt76_free_device(struct mt76_dev *dev) destroy_workqueue(dev->wq); dev->wq = NULL; } + mt76_npu_deinit(dev); ieee80211_free_hw(dev->hw); } EXPORT_SYMBOL_GPL(mt76_free_device); @@ -883,6 +886,7 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) struct sk_buff *skb = phy->rx_amsdu[q].head; struct mt76_rx_status *status = (struct mt76_rx_status *)skb->cb; struct mt76_dev *dev = phy->dev; + struct mt76_queue *rxq = &dev->q_rx[q]; phy->rx_amsdu[q].head = NULL; phy->rx_amsdu[q].tail = NULL; @@ -911,6 +915,13 @@ static void mt76_rx_release_amsdu(struct mt76_phy *phy, enum mt76_rxq_id q) return; } } + + /* RRO 3.0 data queue skbs are processed and completed in the context + * of the indicator queue NAPI, which only polls its own skb list + */ + if (mt76_queue_is_wed_rro_data(rxq) && dev->hwrro_mode == MT76_HWRRO_V3) + q = MT_RXQ_RRO_IND; + __skb_queue_tail(&dev->rx_skb[q], skb); } @@ -1554,7 +1565,8 @@ void mt76_rx_poll_complete(struct mt76_dev *dev, enum mt76_rxq_id q, while ((skb = __skb_dequeue(&dev->rx_skb[q])) != NULL) { mt76_check_sta(dev, skb); - if (mtk_wed_device_active(&dev->mmio.wed)) + if (mtk_wed_device_active(&dev->mmio.wed) || + mt76_npu_device_active(dev)) __skb_queue_tail(&frames, skb); else mt76_rx_aggr_reorder(skb, &frames); diff --git a/drivers/net/wireless/mediatek/mt76/mcu.c b/drivers/net/wireless/mediatek/mt76/mcu.c index 65d4c2adb5386..951781dcf0ade 100644 --- a/drivers/net/wireless/mediatek/mt76/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mcu.c @@ -78,15 +78,19 @@ int mt76_mcu_skb_send_and_get_msg(struct mt76_dev *dev, struct sk_buff *skb, unsigned long expires; int ret, seq; - if (mt76_is_sdio(dev)) - if (test_bit(MT76_RESET, &dev->phy.state) && atomic_read(&dev->bus_hung)) - return -EIO; - if (ret_skb) *ret_skb = NULL; mutex_lock(&dev->mcu.mutex); + if ((mt76_is_mmio(dev) && atomic_read(&dev->bus_hung)) || + (mt76_is_sdio(dev) && test_bit(MT76_RESET, &dev->phy.state) && + atomic_read(&dev->bus_hung))) { + orig_skb = skb; + ret = -EIO; + goto out; + } + if (dev->mcu_ops->mcu_skb_prepare_msg) { orig_skb = skb; ret = dev->mcu_ops->mcu_skb_prepare_msg(dev, skb, cmd, &seq); diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h index 125ac1eb2d541..2a186b47659b6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76.h +++ b/drivers/net/wireless/mediatek/mt76/mt76.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #define MT_QFLAG_WED_RRO BIT(6) #define MT_QFLAG_WED_RRO_EN BIT(7) #define MT_QFLAG_EMI_EN BIT(8) +#define MT_QFLAG_NPU BIT(9) #define __MT_WED_Q(_type, _n) (MT_QFLAG_WED | \ FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ @@ -48,6 +50,12 @@ #define MT_WED_RRO_Q_IND __MT_WED_RRO_Q(MT76_WED_RRO_Q_IND, 0) #define MT_WED_RRO_Q_RXDMAD_C __MT_WED_RRO_Q(MT76_WED_RRO_Q_RXDMAD_C, 0) +#define __MT_NPU_Q(_type, _n) (MT_QFLAG_NPU | \ + FIELD_PREP(MT_QFLAG_WED_TYPE, _type) | \ + FIELD_PREP(MT_QFLAG_WED_RING, _n)) +#define MT_NPU_Q_TX(_n) __MT_NPU_Q(MT76_WED_Q_TX, _n) +#define MT_NPU_Q_RX(_n) __MT_NPU_Q(MT76_WED_Q_RX, _n) + struct mt76_dev; struct mt76_phy; struct mt76_wcid; @@ -139,6 +147,8 @@ enum mt76_rxq_id { MT_RXQ_TXFREE_BAND2, MT_RXQ_RRO_IND, MT_RXQ_RRO_RXDMAD_C, + MT_RXQ_NPU0, + MT_RXQ_NPU1, __MT_RXQ_MAX }; @@ -706,6 +716,11 @@ struct mt76_mmio { struct mtk_wed_device wed_hif2; struct completion wed_reset; struct completion wed_reset_complete; + + struct airoha_ppe_dev __rcu *ppe_dev; + struct airoha_npu __rcu *npu; + phys_addr_t phy_addr; + int npu_type; }; struct mt76_rx_status { @@ -1611,6 +1626,109 @@ int mt76_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *skb, int mt76_testmode_set_state(struct mt76_phy *phy, enum mt76_testmode_state state); int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len); +#if IS_ENABLED(CONFIG_MT76_NPU) +void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, + u32 info); +int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, + struct sk_buff *skb, struct mt76_queue_buf *buf, + void *txwi_ptr); +int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q); +int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_disable_irqs(struct mt76_dev *dev); +int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type); +void mt76_npu_deinit(struct mt76_dev *dev); +void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q); +void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index); +int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct net_device *dev, enum tc_setup_type type, + void *type_data); +#else +static inline void mt76_npu_check_ppe(struct mt76_dev *dev, + struct sk_buff *skb, u32 info) +{ +} + +static inline int mt76_npu_dma_add_buf(struct mt76_phy *phy, + struct mt76_queue *q, + struct sk_buff *skb, + struct mt76_queue_buf *buf, + void *txwi_ptr) +{ + return -EOPNOTSUPP; +} + +static inline int mt76_npu_fill_rx_queue(struct mt76_dev *dev, + struct mt76_queue *q) +{ + return 0; +} + +static inline void mt76_npu_queue_cleanup(struct mt76_dev *dev, + struct mt76_queue *q) +{ +} + +static inline void mt76_npu_disable_irqs(struct mt76_dev *dev) +{ +} + +static inline int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, + int type) +{ + return 0; +} + +static inline void mt76_npu_deinit(struct mt76_dev *dev) +{ +} + +static inline void mt76_npu_queue_setup(struct mt76_dev *dev, + struct mt76_queue *q) +{ +} + +static inline void mt76_npu_txdesc_cleanup(struct mt76_queue *q, + int index) +{ +} + +static inline int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, + struct ieee80211_vif *vif, + struct net_device *dev, + enum tc_setup_type type, + void *type_data) +{ + return -EOPNOTSUPP; +} +#endif /* CONFIG_MT76_NPU */ + +static inline bool mt76_npu_device_active(struct mt76_dev *dev) +{ + return mt76_is_mmio(dev) && !!rcu_access_pointer(dev->mmio.npu); +} + +static inline bool mt76_ppe_device_active(struct mt76_dev *dev) +{ + return mt76_is_mmio(dev) && !!rcu_access_pointer(dev->mmio.ppe_dev); +} + +static inline int mt76_npu_send_msg(struct airoha_npu *npu, int ifindex, + enum airoha_npu_wlan_set_cmd cmd, + u32 val, gfp_t gfp) +{ + return airoha_npu_wlan_send_msg(npu, ifindex, cmd, &val, sizeof(val), + gfp); +} + +static inline int mt76_npu_get_msg(struct airoha_npu *npu, int ifindex, + enum airoha_npu_wlan_get_cmd cmd, + u32 *val, gfp_t gfp) +{ + return airoha_npu_wlan_get_msg(npu, ifindex, cmd, val, sizeof(*val), + gfp); +} + static inline void mt76_testmode_reset(struct mt76_phy *phy, bool disable) { #ifdef CONFIG_NL80211_TESTMODE @@ -1852,6 +1970,23 @@ static inline bool mt76_queue_is_emi(struct mt76_queue *q) return q->flags & MT_QFLAG_EMI_EN; } +static inline bool mt76_queue_is_npu(struct mt76_queue *q) +{ + return q->flags & MT_QFLAG_NPU; +} + +static inline bool mt76_queue_is_npu_tx(struct mt76_queue *q) +{ + return mt76_queue_is_npu(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_TX; +} + +static inline bool mt76_queue_is_npu_rx(struct mt76_queue *q) +{ + return mt76_queue_is_npu(q) && + FIELD_GET(MT_QFLAG_WED_TYPE, q->flags) == MT76_WED_Q_RX; +} + struct mt76_txwi_cache * mt76_token_release(struct mt76_dev *dev, int token, bool *wake); int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi); @@ -1936,6 +2071,9 @@ mt76_vif_link(struct mt76_dev *dev, struct ieee80211_vif *vif, int link_id) if (!link_id) return mlink; + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + return NULL; + return mt76_dereference(mvif->link[link_id], dev); } diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c index 13c9dbeaee84e..6c7debe439b6c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/mac.c @@ -1601,6 +1601,8 @@ bool mt7615_rx_check(struct mt76_dev *mdev, void *data, int len) switch (type) { case PKT_TYPE_TXRX_NOTIFY: + if (!mt76_is_mmio(mdev)) + return false; mt7615_mac_tx_free(dev, data, len); return false; case PKT_TYPE_TXS: @@ -1634,6 +1636,10 @@ void mt7615_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, dev_kfree_skb(skb); break; case PKT_TYPE_TXRX_NOTIFY: + if (!mt76_is_mmio(mdev)) { + dev_kfree_skb(skb); + break; + } mt7615_mac_tx_free(dev, skb->data, skb->len); dev_kfree_skb(skb); break; diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c index 87a2e5163699c..25936fb309fbd 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c @@ -1239,11 +1239,12 @@ static int mt7615_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&dev->pm.ps_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); + cancel_delayed_work_sync(&phy->scan_work); + cancel_delayed_work_sync(&phy->mt76->mac_work); + mt7615_mutex_acquire(dev); clear_bit(MT76_STATE_RUNNING, &phy->mt76->state); - cancel_delayed_work_sync(&phy->scan_work); - cancel_delayed_work_sync(&phy->mt76->mac_work); set_bit(MT76_STATE_SUSPEND, &phy->mt76->state); ieee80211_iterate_active_interfaces(hw, diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h index 192dcc374a642..ba7eaf3ebe988 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h @@ -48,6 +48,11 @@ enum rx_pkt_type { #define MT_TXD_LEN_MSDU_LAST BIT(14) #define MT_TXD_LEN_AMSDU_LAST BIT(15) +/* PCIE part */ +#define PCIE_AER_UNC_STATUS_OFFSET 0x204 +#define PCIE_AER_UNC_MASK_OFFSET 0x208 +#define PCIE_AER_CO_STATUS_OFFSET 0x210 + enum { CMD_CBW_20MHZ = IEEE80211_STA_RX_BW_20, CMD_CBW_40MHZ = IEEE80211_STA_RX_BW_40, @@ -177,6 +182,11 @@ static inline bool is_mt7925(struct mt76_dev *dev) return mt76_chip(dev) == 0x7925; } +static inline bool is_320mhz_supported(struct mt76_dev *dev) +{ + return mt76_chip(dev) == 0x7927; +} + static inline bool is_mt7920(struct mt76_dev *dev) { return mt76_chip(dev) == 0x7920; @@ -224,7 +234,8 @@ static inline bool is_mt798x(struct mt76_dev *dev) static inline bool is_mt7996(struct mt76_dev *dev) { - return mt76_chip(dev) == 0x7990; + u16 chip = mt76_chip(dev); + return chip == 0x7990 || chip == 0x7991; } static inline bool is_mt7992(struct mt76_dev *dev) diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c index fc3e6728fcfbf..46e337307c3d8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c @@ -1295,8 +1295,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif, wtbl_hdr); ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } ret = mt76_mcu_skb_send_msg(dev, skb, cmd, true); if (ret) @@ -1309,8 +1311,10 @@ int mt76_connac_mcu_sta_ba(struct mt76_dev *dev, struct mt76_vif_link *mvif, mt76_connac_mcu_sta_ba_tlv(skb, params, enable, tx); ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, cmd, true); } @@ -1441,6 +1445,8 @@ mt76_connac_mcu_uni_bss_he_tlv(struct mt76_phy *phy, struct ieee80211_vif *vif, struct bss_info_uni_he *he; cap = mt76_connac_get_he_phy_cap(phy, vif); + if (!cap) + return; he = (struct bss_info_uni_he *)tlv; he->he_pe_duration = vif->bss_conf.htc_trig_based_pkt_ext; @@ -2764,12 +2770,16 @@ int mt76_connac_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, return PTR_ERR(skb); ret = mt76_connac_mcu_sta_key_tlv(sta_key_conf, skb, key, cmd); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } ret = mt76_connac_mcu_sta_wed_update(dev, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c index 83488b2d6efb9..06f17da2c0106 100644 --- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c @@ -848,7 +848,7 @@ int mt76x02_mac_process_rx(struct mt76x02_dev *dev, struct sk_buff *skb, } } - if (WARN_ON_ONCE(len > skb->len)) + if (len > skb->len) return -EINVAL; if (pskb_trim(skb, len)) diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c index 38dfd5de365ca..14c0770229bc5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/eeprom.c @@ -257,7 +257,7 @@ void mt7915_eeprom_parse_hw_cap(struct mt7915_dev *dev, nss = min_t(u8, min_t(u8, nss_max, nss), path); mphy->chainmask = BIT(path) - 1; - if (band) + if (band && dev->dbdc_support) mphy->chainmask <<= dev->chainshift; mphy->antenna_mask = BIT(nss) - 1; dev->chainmask |= mphy->chainmask; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/init.c b/drivers/net/wireless/mediatek/mt76/mt7915/init.c index d2b163a5fce5b..f247846520138 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/init.c @@ -1265,14 +1265,19 @@ int mt7915_register_device(struct mt7915_dev *dev) ret = mt7915_init_debugfs(&dev->phy); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; ret = mt7915_coredump_register(dev); if (ret) - goto unreg_thermal; + goto unreg_ext_phy; return 0; +unreg_ext_phy: + if (phy2) { + mt7915_unregister_ext_phy(dev); + phy2 = NULL; + } unreg_thermal: mt7915_unregister_thermal(&dev->phy); unreg_dev: diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c index 50af0d9f240b6..dda4e7d179536 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c @@ -437,7 +437,7 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb, if (v0 & MT_PRXV_HT_AD_CODE) status->enc_flags |= RX_ENC_FLAG_LDPC; - status->chains = mphy->antenna_mask; + status->chains = mt7915_band_chainmask(phy); status->chain_signal[0] = to_rssi(MT_PRXV_RCPI0, v1); status->chain_signal[1] = to_rssi(MT_PRXV_RCPI1, v1); status->chain_signal[2] = to_rssi(MT_PRXV_RCPI2, v1); @@ -2345,8 +2345,10 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw, } flow->tsf = le64_to_cpu(twt_agrt->twt); - if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) + if (mt7915_mcu_twt_agrt_update(dev, msta->vif, flow, MCU_TWT_AGRT_ADD)) { + list_del(&flow->list); goto unlock; + } setup_cmd = TWT_SETUP_CMD_ACCEPT; dev->twt.table_mask |= BIT(table_id); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c index 6f594677474b0..9d546eb8022ac 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c @@ -249,7 +249,7 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, idx = mt76_wcid_alloc(dev->mt76.wcid_mask, mt7915_wtbl_size(dev)); if (idx < 0) { ret = -ENOSPC; - goto out; + goto err; } INIT_LIST_HEAD(&mvif->sta.rc_list); @@ -277,7 +277,17 @@ static int mt7915_add_interface(struct ieee80211_hw *hw, mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &mvif->sta.wcid); + mutex_unlock(&dev->mt76.mutex); + + return 0; + +err: + dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); + phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt7915_mcu_add_dev_info(phy, vif, false); out: + if (phy->monitor_vif == vif) + phy->monitor_vif = NULL; mutex_unlock(&dev->mt76.mutex); return ret; @@ -294,7 +304,6 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mt7915_mcu_add_bss_info(phy, vif, false); mt7915_mcu_add_sta(dev, vif, NULL, CONN_STATE_DISCONNECT, false); - mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_lock(&dev->mt76.mutex); mt76_testmode_reset(phy->mt76, true); @@ -310,6 +319,7 @@ static void mt7915_remove_interface(struct ieee80211_hw *hw, mutex_lock(&dev->mt76.mutex); dev->mt76.vif_mask &= ~BIT_ULL(mvif->mt76.idx); phy->omac_mask &= ~BIT_ULL(mvif->mt76.omac_idx); + mt76_wcid_mask_clear(dev->mt76.wcid_mask, mvif->sta.wcid.idx); mutex_unlock(&dev->mt76.mutex); spin_lock_bh(&dev->mt76.sta_poll_lock); @@ -493,7 +503,7 @@ static int mt7915_config(struct ieee80211_hw *hw, int radio_idx, mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_DMA_DCR0_RXD_G5_EN, enabled); - mt76_rmw_field(dev, MT_DMA_DCR0(band), MT_MDP_DCR0_RX_HDR_TRANS_EN, + mt76_rmw_field(dev, MT_MDP_DCR0, MT_MDP_DCR0_RX_HDR_TRANS_EN, !dev->monitor_mask); mt76_testmode_reset(phy->mt76, true); mt76_wr(dev, MT_WF_RFCR(band), rxfilter); @@ -1123,7 +1133,7 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an struct mt7915_dev *dev = mt7915_hw_dev(hw); struct mt7915_phy *phy = mt7915_hw_phy(hw); int max_nss = hweight8(hw->wiphy->available_antennas_tx); - u8 chainshift = dev->chainshift; + u8 shift = mt7915_band_chainshift(phy); u8 band = phy->mt76->band_idx; if (!tx_ant || tx_ant != rx_ant || ffs(tx_ant) > max_nss) @@ -1136,9 +1146,9 @@ mt7915_set_antenna(struct ieee80211_hw *hw, int radio_idx, u32 tx_ant, u32 rx_an /* handle a variant of mt7916/mt7981 which has 3T3R but nss2 on 5 GHz band */ if ((is_mt7916(&dev->mt76) || is_mt7981(&dev->mt76)) && band && hweight8(tx_ant) == max_nss) - phy->mt76->chainmask = (dev->chainmask >> chainshift) << chainshift; + phy->mt76->chainmask = (dev->chainmask >> shift) << shift; else - phy->mt76->chainmask = tx_ant << (chainshift * band); + phy->mt76->chainmask = tx_ant << shift; mt76_set_stream_caps(phy->mt76, true); mt7915_set_stream_vht_txbf_caps(phy); @@ -1738,7 +1748,7 @@ mt7915_net_fill_forward_path(struct ieee80211_hw *hw, path->mtk_wdma.wdma_idx = wed->wdma_idx; path->mtk_wdma.bss = mvif->mt76.idx; path->mtk_wdma.wcid = is_mt7915(&dev->mt76) ? msta->wcid.idx : 0x3ff; - path->mtk_wdma.queue = phy != &dev->phy; + path->mtk_wdma.queue = phy->mt76->band_idx; ctx->dev = NULL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c index 79e021ac0bdbc..29b3f0cda1ce7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c @@ -51,7 +51,7 @@ mt7915_mcu_get_sta_nss(u16 mcs_map) break; } - return nss - 1; + return nss ? nss - 1 : 0; } static void @@ -576,9 +576,9 @@ mt7915_mcu_bss_ra_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, ra->rx_streams = max_nss; ra->algo = 4; ra->train_up_rule = 2; - ra->train_up_high_thres = 110; - ra->train_up_rule_rssi = -70; - ra->low_traffic_thres = 2; + ra->train_up_high_thres = cpu_to_le16(110); + ra->train_up_rule_rssi = cpu_to_le16(-70); + ra->low_traffic_thres = cpu_to_le16(2); ra->phy_cap = cpu_to_le32(0xfdf); ra->interval = cpu_to_le32(500); ra->fast_interval = cpu_to_le32(100); @@ -595,6 +595,8 @@ mt7915_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, struct tlv *tlv; cap = mt76_connac_get_he_phy_cap(phy->mt76, vif); + if (!cap) + return; tlv = mt76_connac_mcu_add_tlv(skb, BSS_INFO_HE_BASIC, sizeof(*he)); @@ -1177,13 +1179,12 @@ mt7915_mcu_sta_bfer_vht(struct ieee80211_sta *sta, struct mt7915_phy *phy, } static void -mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, struct ieee80211_vif *vif, - struct mt7915_phy *phy, struct sta_rec_bf *bf) +mt7915_mcu_sta_bfer_he(struct ieee80211_sta *sta, + const struct ieee80211_sta_he_cap *vc, + struct sta_rec_bf *bf) { struct ieee80211_sta_he_cap *pc = &sta->deflink.he_cap; struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem; - const struct ieee80211_sta_he_cap *vc = - mt76_connac_get_he_phy_cap(phy->mt76, vif); const struct ieee80211_he_cap_elem *ve = &vc->he_cap_elem; u16 mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80); u8 nss_mcs = mt7915_mcu_get_sta_nss(mcs_map); @@ -1242,6 +1243,7 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb, { struct mt7915_vif *mvif = (struct mt7915_vif *)vif->drv_priv; struct mt7915_phy *phy = mvif->phy; + const struct ieee80211_sta_he_cap *vc = NULL; int tx_ant = hweight8(phy->mt76->chainmask) - 1; struct sta_rec_bf *bf; struct tlv *tlv; @@ -1260,6 +1262,12 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb, if (!ebf && !dev->ibf) return; + if (sta->deflink.he_cap.has_he && ebf) { + vc = mt76_connac_get_he_phy_cap(phy->mt76, vif); + if (!vc) + return; + } + tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_BF, sizeof(*bf)); bf = (struct sta_rec_bf *)tlv; @@ -1268,7 +1276,7 @@ mt7915_mcu_sta_bfer_tlv(struct mt7915_dev *dev, struct sk_buff *skb, * ht: iBF only, since mac80211 lacks of eBF support */ if (sta->deflink.he_cap.has_he && ebf) - mt7915_mcu_sta_bfer_he(sta, vif, phy, bf); + mt7915_mcu_sta_bfer_he(sta, vc, bf); else if (sta->deflink.vht_cap.vht_supported) mt7915_mcu_sta_bfer_vht(sta, phy, bf, ebf); else if (sta->deflink.ht_cap.ht_supported) @@ -1765,8 +1773,10 @@ int mt7915_mcu_add_sta(struct mt7915_dev *dev, struct ieee80211_vif *vif, } out: ret = mt76_connac_mcu_sta_wed_update(&dev->mt76, skb); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(&dev->mt76, skb, MCU_EXT_CMD(STA_REC_UPDATE), true); @@ -2794,7 +2804,7 @@ int mt7915_mcu_set_chan_info(struct mt7915_phy *phy, int cmd) .center_ch = ieee80211_frequency_to_channel(freq1), .bw = mt76_connac_chan_bw(chandef), .tx_path_num = hweight16(phy->mt76->chainmask), - .rx_path = phy->mt76->chainmask >> (dev->chainshift * band), + .rx_path = mt7915_band_chainmask(phy), .band_idx = band, .channel_band = ch_band[chandef->chan->band], }; @@ -2907,8 +2917,15 @@ int mt7915_mcu_get_eeprom(struct mt7915_dev *dev, u32 offset, u8 *read_buf) return ret; res = (struct mt7915_mcu_eeprom_info *)skb->data; - if (!buf) - buf = dev->mt76.eeprom.data + le32_to_cpu(res->addr); + if (!buf) { + u32 addr = le32_to_cpu(res->addr); + + if (addr > dev->mt76.eeprom.size - MT7915_EEPROM_BLOCK_SIZE) { + dev_kfree_skb(skb); + return -EINVAL; + } + buf = dev->mt76.eeprom.data + addr; + } memcpy(buf, res->data, MT7915_EEPROM_BLOCK_SIZE); dev_kfree_skb(skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h index 4049ed864003d..4c25abb0297de 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.h @@ -318,9 +318,9 @@ struct bss_info_ra { u8 antenna_idx; u8 train_up_rule; u8 rsv[3]; - unsigned short train_up_high_thres; - short train_up_rule_rssi; - unsigned short low_traffic_thres; + __le16 train_up_high_thres; + __le16 train_up_rule_rssi; + __le16 low_traffic_thres; __le16 max_phyrate; __le32 phy_cap; __le32 interval; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h index f1194d147dc89..27857347f05f4 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h +++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h @@ -393,6 +393,24 @@ mt7915_ext_phy(struct mt7915_dev *dev) return phy->priv; } +/* without dbdc, the chainmask is stored unshifted, even if the phy is + * bound to band 1 + */ +static inline u8 mt7915_band_chainshift(struct mt7915_phy *phy) +{ + struct mt7915_dev *dev = phy->dev; + + if (!dev->dbdc_support) + return 0; + + return phy->mt76->band_idx * dev->chainshift; +} + +static inline u16 mt7915_band_chainmask(struct mt7915_phy *phy) +{ + return phy->mt76->chainmask >> mt7915_band_chainshift(phy); +} + static inline u32 mt7915_check_adie(struct mt7915_dev *dev, bool sku) { u32 mask = sku ? MT_CONNINFRA_SKU_MASK : MT_ADIE_TYPE_MASK; diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c index 07b0a5766eab7..c24f1b12f064f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c @@ -135,7 +135,6 @@ static int mt7915_pci_probe(struct pci_dev *pdev, mdev = &dev->mt76; mt7915_wfsys_reset(dev); - hif2 = mt7915_pci_init_hif2(pdev); ret = mt7915_mmio_wed_init(dev, pdev, true, &irq); if (ret < 0) @@ -145,16 +144,20 @@ static int mt7915_pci_probe(struct pci_dev *pdev, hif2 = mt7915_pci_init_hif2(pdev); ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); - if (ret < 0) + if (ret < 0) { + mt7915_put_hif2(hif2); goto free_device; + } irq = pdev->irq; } ret = devm_request_irq(mdev->dev, irq, mt7915_irq_handler, IRQF_SHARED, KBUILD_MODNAME, dev); - if (ret) + if (ret) { + mt7915_put_hif2(hif2); goto free_wed_or_irq_vector; + } /* master switch of PCIe tnterrupt enable */ mt76_wr(dev, MT_PCIE_MAC_INT_ENABLE, 0xff); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c index c823a7554a3ac..df641c1334809 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/soc.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/soc.c @@ -913,7 +913,7 @@ static void mt7986_wmac_clock_enable(struct mt7915_dev *dev, u32 adie_type) read_poll_timeout(mt76_rr, cur, !(cur & MT_SLP_CTRL_BSY_MASK), USEC_PER_MSEC, 50 * USEC_PER_MSEC, false, - dev, MT_ADIE_SLP_CTRL_CK0(0)); + dev, MT_ADIE_SLP_CTRL_CK0(1)); } mt76_wmac_spi_unlock(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c index d534fff5c952b..2a9e5d9ff101c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c +++ b/drivers/net/wireless/mediatek/mt76/mt7915/testmode.c @@ -694,9 +694,7 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, { struct mt76_testmode_data *td = &mphy->test; struct mt7915_phy *phy = mphy->priv; - struct mt7915_dev *dev = phy->dev; - u32 chainmask = mphy->chainmask, changed = 0; - bool ext_phy = phy != &dev->phy; + u32 chainmask = mt7915_band_chainmask(phy), changed = 0; int i; BUILD_BUG_ON(NUM_TM_CHANGED >= 32); @@ -705,7 +703,6 @@ mt7915_tm_set_params(struct mt76_phy *mphy, struct nlattr **tb, td->state == MT76_TM_STATE_OFF) return 0; - chainmask = ext_phy ? chainmask >> dev->chainshift : chainmask; if (td->tx_antenna_mask > chainmask) return -EINVAL; diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c index d2731e9953bf8..0d48dbb2354f3 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c @@ -569,8 +569,9 @@ bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len) switch (type) { case PKT_TYPE_TXRX_NOTIFY: - /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ - mt7921_mac_tx_free(dev, data, len); /* mmio */ + if (!mt76_is_mmio(mdev)) + return false; + mt7921_mac_tx_free(dev, data, len); return false; case PKT_TYPE_TXS: for (rxd += 2; rxd + 8 <= end; rxd += 8) @@ -599,7 +600,10 @@ void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, switch (type) { case PKT_TYPE_TXRX_NOTIFY: - /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ + if (!mt76_is_mmio(mdev)) { + napi_consume_skb(skb, 1); + break; + } mt7921_mac_tx_free(dev, skb->data, skb->len); napi_consume_skb(skb, 1); break; @@ -669,6 +673,9 @@ void mt7921_mac_reset_work(struct work_struct *work) cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { + if (atomic_read(&dev->mt76.bus_hung)) + return; + mutex_lock(&dev->mt76.mutex); ret = mt792x_dev_reset(dev); mutex_unlock(&dev->mt76.mutex); diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c index 663b245f2891f..8c8b35cba856c 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c @@ -418,7 +418,8 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) struct mt76_dev *mdev = &dev->mt76; struct mt792x_phy *phy = &dev->phy; const struct firmware *fw; - int ret, i, len, offset = 0; + size_t clc_len, fw_data_len, len, offset = 0; + int ret, i; u8 *clc_base = NULL, hw_encap = 0; dev->phy.clc_chan_conf = 0xff; @@ -444,13 +445,21 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) } hdr = (const void *)(fw->data + fw->size - sizeof(*hdr)); + if (hdr->n_region > (fw->size - sizeof(*hdr)) / sizeof(*region)) { + dev_err(mdev->dev, "Invalid firmware region table\n"); + ret = -EINVAL; + goto out; + } + fw_data_len = fw->size - sizeof(*hdr) - + hdr->n_region * sizeof(*region); + for (i = 0; i < hdr->n_region; i++) { region = (const void *)((const u8 *)hdr - (hdr->n_region - i) * sizeof(*region)); len = le32_to_cpu(region->len); /* check if we have valid buffer size */ - if (offset + len > fw->size) { + if (len > fw_data_len - offset) { dev_err(mdev->dev, "Invalid firmware region\n"); ret = -EINVAL; goto out; @@ -467,8 +476,24 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) if (!clc_base) goto out; - for (offset = 0; offset < len; offset += le32_to_cpu(clc->len)) { + for (offset = 0; offset < len; offset += clc_len) { + if (len - offset < sizeof(*clc)) { + dev_err(mdev->dev, "Invalid CLC record\n"); + ret = -EINVAL; + goto out; + } + clc = (const struct mt7921_clc *)(clc_base + offset); + clc_len = le32_to_cpu(clc->len); + if (clc_len < sizeof(*clc) || clc_len > len - offset) { + dev_err(mdev->dev, "Invalid CLC record\n"); + ret = -EINVAL; + goto out; + } + + /* Newer firmware may add records this driver does not use yet */ + if (clc->idx >= ARRAY_SIZE(phy->clc)) + continue; /* do not init buf again if chip reset triggered */ if (phy->clc[clc->idx]) @@ -480,7 +505,7 @@ static int mt7921_load_clc(struct mt792x_dev *dev, const char *fw_name) continue; phy->clc[clc->idx] = devm_kmemdup(mdev->dev, clc, - le32_to_cpu(clc->len), + clc_len, GFP_KERNEL); if (!phy->clc[clc->idx]) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c index a0c9df3c2cc75..85caa2fbb78f0 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c @@ -554,6 +554,108 @@ static int mt7921_pci_resume(struct device *device) return err; } +static u32 mt7921_aer_rr(struct mt76_dev *mdev, u32 offset) +{ + return 0; +} + +static void mt7921_aer_wr(struct mt76_dev *mdev, u32 offset, u32 val) +{ + ; +} + +static u32 mt791_aer_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) +{ + return 0; +} + +static const struct mt76_bus_ops mt7921_aer_bus_hung_ops = { + .rr = mt7921_aer_rr, + .wr = mt7921_aer_wr, + .rmw = mt791_aer_rmw, + .type = MT76_BUS_MMIO +}; + +static void mt7921_pci_set_aer_bus_hung_ops(struct mt792x_dev *dev) +{ + if (READ_ONCE(dev->mt76.bus) == &mt7921_aer_bus_hung_ops) + return; + + atomic_set(&dev->mt76.bus_hung, true); + WRITE_ONCE(dev->mt76.bus, &mt7921_aer_bus_hung_ops); +} + +static pci_ers_result_t mt7921_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + struct mt792x_dev *dev = container_of(mdev, struct mt792x_dev, mt76); + u32 aer_unc_val = 0, aer_co_val = 0; + + dev_err(mdev->dev, "PCIE error detect state: %d\n", state); + + /* Clear SW IRQ tasklet first */ + tasklet_kill(&mdev->irq_tasklet); + + if (state == pci_channel_io_perm_failure) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_read_config_dword(pdev, PCIE_AER_UNC_STATUS_OFFSET, &aer_unc_val); + pci_read_config_dword(pdev, PCIE_AER_CO_STATUS_OFFSET, &aer_co_val); + + dev_warn(mdev->dev, "PCIE_AER_UNC_STATUS_OFFSET: 0x%x\n", aer_unc_val); + dev_warn(mdev->dev, "PCIE_AER_CO_STATUS_OFFSET: 0x%x\n", aer_co_val); + + /** + * Due to this error is from link error and this AER is un-correctable, + * so can't covered by device + **/ + if (aer_unc_val != 0) { + mt7921_pci_set_aer_bus_hung_ops(dev); + return PCI_ERS_RESULT_DISCONNECT; + } + + /** + * Try to recover it when state is pci_channel_io_frozen or + * AER is correctable error + **/ + if (state == pci_channel_io_frozen || aer_co_val != 0) { + /* Disable PCIE activity first. */ + pci_disable_device(pdev); + return PCI_ERS_RESULT_NEED_RESET; + } + + return PCI_ERS_RESULT_NONE; +} + +static pci_ers_result_t mt7921_slot_reset(struct pci_dev *pdev) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + int ret = 0; + + ret = pci_enable_device_mem(pdev); + + if (ret) { + dev_err(mdev->dev, "pci_enable_device_mem failed: %d\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_set_master(pdev); + pci_restore_state(pdev); + pci_save_state(pdev); + /* Also try do the vendor reset to let it more clear. */ + mt792x_reset(mdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static const struct pci_error_handlers mt7921_err_handler = { + .error_detected = mt7921_error_detected, + .slot_reset = mt7921_slot_reset, +}; + static void mt7921_pci_shutdown(struct pci_dev *pdev) { mt7921_pci_remove(pdev); @@ -568,6 +670,7 @@ static struct pci_driver mt7921_pci_driver = { .remove = mt7921_pci_remove, .shutdown = mt7921_pci_shutdown, .driver.pm = pm_sleep_ptr(&mt7921_pm_ops), + .err_handler = &mt7921_err_handler, }; module_pci_driver(mt7921_pci_driver); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c index d951a46e9d48c..a8572fdba634a 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mac.c @@ -338,6 +338,11 @@ mt7925_mac_fill_rx_rate(struct mt792x_dev *dev, case IEEE80211_STA_RX_BW_160: status->bw = RATE_INFO_BW_160; break; + /* RXV can report 320 in two positions */ + case IEEE80211_STA_RX_BW_320: + case IEEE80211_STA_RX_BW_320 + 1: + status->bw = RATE_INFO_BW_320; + break; default: return -EINVAL; } @@ -995,6 +1000,10 @@ mt7925_mac_add_txs_skb(struct mt792x_dev *dev, struct mt76_wcid *wcid, stats->tx_mode[mode]++; switch (FIELD_GET(MT_TXS0_BW, txs)) { + case IEEE80211_STA_RX_BW_320: + rate.bw = RATE_INFO_BW_320; + stats->tx_bw[4]++; + break; case IEEE80211_STA_RX_BW_160: rate.bw = RATE_INFO_BW_160; stats->tx_bw[3]++; @@ -1193,8 +1202,9 @@ bool mt7925_rx_check(struct mt76_dev *mdev, void *data, int len) switch (type) { case PKT_TYPE_TXRX_NOTIFY: - /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ - mt7925_mac_tx_free(dev, data, len); /* mmio */ + if (!mt76_is_mmio(mdev)) + return false; + mt7925_mac_tx_free(dev, data, len); return false; case PKT_TYPE_TXS: for (rxd += 4; rxd + 12 <= end; rxd += 12) @@ -1230,7 +1240,10 @@ void mt7925_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q, switch (type) { case PKT_TYPE_TXRX_NOTIFY: - /* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */ + if (!mt76_is_mmio(mdev)) { + napi_consume_skb(skb, 1); + break; + } mt7925_mac_tx_free(dev, skb->data, skb->len); napi_consume_skb(skb, 1); break; @@ -1274,6 +1287,9 @@ mt7925_vif_connect_iter(void *priv, u8 *mac, for_each_set_bit(i, &valid, IEEE80211_MLD_MAX_NUM_LINKS) { bss_conf = mt792x_vif_to_bss_conf(vif, i); + if (!bss_conf) + continue; + mconf = mt792x_vif_to_link(mvif, i); mt76_connac_mcu_uni_add_dev(&dev->mphy, bss_conf, &mconf->mt76, @@ -1305,6 +1321,7 @@ void mt7925_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&dev->mphy.mac_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); for (i = 0; i < 10; i++) { @@ -1418,6 +1435,10 @@ int mt7925_usb_sdio_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, if (!wcid) wcid = &dev->mt76.global_wcid; + err = skb_cow_head(skb, MT_SDIO_TXD_SIZE + MT_SDIO_HDR_SIZE); + if (err) + return err; + if (sta) { struct mt792x_sta *msta = (struct mt792x_sta *)sta->drv_priv; diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/main.c b/drivers/net/wireless/mediatek/mt76/mt7925/main.c index 6c0dc72efc392..1eeb33981c743 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/main.c @@ -178,20 +178,36 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER | IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[0] |= + IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ; + + val = (sts > 3) ? sts - 1 : 3; + eht_cap_elem->phy_cap_info[0] |= - u8_encode_bits(u8_get_bits(sts - 1, BIT(0)), + u8_encode_bits(u8_get_bits(val, BIT(0)), IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK); eht_cap_elem->phy_cap_info[1] = - u8_encode_bits(u8_get_bits(sts - 1, GENMASK(2, 1)), + u8_encode_bits(u8_get_bits(val, GENMASK(2, 1)), IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK) | - u8_encode_bits(sts - 1, + u8_encode_bits(val, IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[1] |= + u8_encode_bits(val, + IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK); + eht_cap_elem->phy_cap_info[2] = u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK) | u8_encode_bits(sts - 1, IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK); + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) + eht_cap_elem->phy_cap_info[2] |= + u8_encode_bits(sts - 1, + IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK); + eht_cap_elem->phy_cap_info[3] = IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK | IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK | @@ -212,7 +228,8 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(0x11, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_160 ? 0x7 : + val = width == NL80211_CHAN_WIDTH_320 ? 0xf : + width == NL80211_CHAN_WIDTH_160 ? 0x7 : width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; eht_cap_elem->phy_cap_info[6] = u8_encode_bits(u8_get_bits(0x11, GENMASK(4, 2)), @@ -234,6 +251,11 @@ mt7925_init_eht_caps(struct mt792x_phy *phy, enum nl80211_band band, eht_nss->bw._160.rx_tx_mcs9_max_nss = val; eht_nss->bw._160.rx_tx_mcs11_max_nss = val; eht_nss->bw._160.rx_tx_mcs13_max_nss = val; + if (band == NL80211_BAND_6GHZ && is_320mhz_supported(&phy->dev->mt76)) { + eht_nss->bw._320.rx_tx_mcs9_max_nss = val; + eht_nss->bw._320.rx_tx_mcs11_max_nss = val; + eht_nss->bw._320.rx_tx_mcs13_max_nss = val; + } } int mt7925_init_mlo_caps(struct mt792x_phy *phy) @@ -1502,6 +1524,7 @@ static int mt7925_suspend(struct ieee80211_hw *hw, cancel_delayed_work_sync(&phy->mt76->mac_work); cancel_delayed_work_sync(&dev->pm.ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_free_pending_tx_skbs(&dev->pm, NULL); mt792x_mutex_acquire(dev); @@ -1573,6 +1596,9 @@ static void mt7925_sta_set_decap_offload(struct ieee80211_hw *hw, mlink = mt792x_sta_to_link(msta, i); + if (!mlink) + continue; + if (enabled) set_bit(MT_WCID_FLAG_HDR_TRANS, &mlink->wcid.flags); else @@ -1684,9 +1710,15 @@ static int mt7925_set_sar_specs(struct ieee80211_hw *hw, int err; mt792x_mutex_acquire(dev); + err = mt7925_mcu_set_clc(dev, dev->mt76.alpha2, + dev->country_ie_env); + if (err < 0) + goto out; + err = mt7925_set_tx_sar_pwr(hw, sar); - mt792x_mutex_release(dev); +out: + mt792x_mutex_release(dev); return err; } @@ -2177,10 +2209,19 @@ static void mt7925_rfkill_poll(struct ieee80211_hw *hw) wiphy_rfkill_set_hw_state(hw->wiphy, ret == 0); } +static void mt7925_stop(struct ieee80211_hw *hw, bool suspend) +{ + struct mt792x_dev *dev = mt792x_hw_dev(hw); + + cancel_delayed_work_sync(&dev->mlo_pm_work); + + mt792x_stop(hw, suspend); +} + const struct ieee80211_ops mt7925_ops = { .tx = mt792x_tx, .start = mt7925_start, - .stop = mt792x_stop, + .stop = mt7925_stop, .add_interface = mt7925_add_interface, .remove_interface = mt792x_remove_interface, .config = mt7925_config, diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c index d887aa9a3dff7..8b1e1d2ed2546 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c @@ -1290,8 +1290,10 @@ int mt7925_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif, return PTR_ERR(skb); ret = mt7925_mcu_sta_key_tlv(wcid, sta_key_conf, skb, key, cmd, msta); - if (ret) + if (ret) { + dev_kfree_skb(skb); return ret; + } return mt76_mcu_skb_send_msg(dev, skb, mcu_cmd, true); } @@ -1688,6 +1690,7 @@ mt7925_mcu_sta_eht_tlv(struct sk_buff *skb, struct ieee80211_link_sta *link_sta) memcpy(eht->mcs_map_bw20, &mcs_map->only_20mhz, sizeof(eht->mcs_map_bw20)); memcpy(eht->mcs_map_bw80, &mcs_map->bw._80, sizeof(eht->mcs_map_bw80)); memcpy(eht->mcs_map_bw160, &mcs_map->bw._160, sizeof(eht->mcs_map_bw160)); + memcpy(eht->mcs_map_bw320, &mcs_map->bw._320, sizeof(eht->mcs_map_bw320)); } static void @@ -2336,6 +2339,10 @@ void mt7925_mcu_bss_rlm_tlv(struct sk_buff *skb, struct mt76_phy *phy, case NL80211_CHAN_WIDTH_160: req->bw = CMD_CBW_160MHZ; break; + case NL80211_CHAN_WIDTH_320: + req->bw = CMD_CBW_320MHZ; + req->center_chan2 = ieee80211_frequency_to_channel(freq2); + break; case NL80211_CHAN_WIDTH_5: req->bw = CMD_CBW_5MHZ; break; @@ -2706,6 +2713,8 @@ mt7925_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_bss_conf *link_conf, struct tlv *tlv; cap = mt76_connac_get_he_phy_cap(phy->mt76, link_conf->vif); + if (!cap) + return; tlv = mt76_connac_mcu_add_tlv(skb, UNI_BSS_INFO_HE_BASIC, sizeof(*he)); @@ -3729,7 +3738,7 @@ mt7925_mcu_rate_txpower_band(struct mt76_phy *phy, memcpy(tx_power_tlv->alpha2, dev->alpha2, sizeof(dev->alpha2)); tx_power_tlv->n_chan = num_ch; tx_power_tlv->tag = cpu_to_le16(0x1); - tx_power_tlv->len = cpu_to_le16(msg_len); + tx_power_tlv->len = cpu_to_le16(msg_len - 4); switch (band) { case NL80211_BAND_2GHZ: diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c index d6732f50784cf..7972111d53a44 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/pci.c @@ -41,6 +41,7 @@ static void mt7925e_unregister_device(struct mt792x_dev *dev) mt76_for_each_q_rx(&dev->mt76, i) napi_disable(&dev->mt76.napi[i]); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); cancel_work_sync(&dev->reset_work); @@ -454,6 +455,7 @@ static int mt7925_pci_suspend(struct device *device) dev->hif_resumed = false; flush_work(&dev->reset_work); cancel_delayed_work_sync(&pm->ps_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); cancel_work_sync(&pm->wake_work); mt7925_roc_abort_sync(dev); diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c index bf040f34e4b9f..95047ee48a425 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7925/usb.c +++ b/drivers/net/wireless/mediatek/mt76/mt7925/usb.c @@ -12,6 +12,9 @@ static const struct usb_device_id mt7925u_device_table[] = { { USB_DEVICE_AND_INTERFACE_INFO(0x0e8d, 0x7925, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7925_FIRMWARE_WM }, + /* Netgear, Inc. A8500 */ + { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9050, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)MT7925_FIRMWARE_WM }, /* Netgear, Inc. A9000 */ { USB_DEVICE_AND_INTERFACE_INFO(0x0846, 0x9072, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)MT7925_FIRMWARE_WM }, @@ -251,6 +254,7 @@ static int mt7925u_suspend(struct usb_interface *intf, pm_message_t state) pm->suspended = true; dev->hif_resumed = false; flush_work(&dev->reset_work); + cancel_delayed_work_sync(&dev->mlo_pm_work); mt76_connac_mcu_set_hif_suspend(&dev->mt76, true, false); ret = wait_event_timeout(dev->wait, diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c index 0ab827f52fd7d..1d8b8bec8e0e5 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/debugfs.c @@ -645,7 +645,7 @@ mt7996_sta_hw_queue_read(void *data, struct ieee80211_sta *sta) if (!mlink) continue; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (!msta_link) continue; @@ -953,16 +953,34 @@ bool mt7996_debugfs_rx_log(struct mt7996_dev *dev, const void *data, int len) #ifdef CONFIG_MAC80211_DEBUGFS /** per-station debugfs **/ -static ssize_t mt7996_sta_fixed_rate_set(struct file *file, - const char __user *user_buf, - size_t count, loff_t *ppos) +static int +mt7996_queues_show(struct seq_file *s, void *data) +{ + struct ieee80211_sta *sta = s->private; + + mt7996_sta_hw_queue_read(s, sta); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(mt7996_queues); + +void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_sta *sta, struct dentry *dir) +{ + debugfs_create_file("hw-queues", 0400, dir, sta, &mt7996_queues_fops); +} + +static ssize_t mt7996_link_sta_fixed_rate_set(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) { #define SHORT_PREAMBLE 0 #define LONG_PREAMBLE 1 - struct ieee80211_sta *sta = file->private_data; - struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct ieee80211_link_sta *link_sta = file->private_data; + struct mt7996_sta *msta = (struct mt7996_sta *)link_sta->sta->drv_priv; struct mt7996_dev *dev = msta->vif->deflink.phy->dev; - struct mt7996_sta_link *msta_link = &msta->deflink; + struct mt7996_sta_link *msta_link; struct ra_rate phy = {}; char buf[100]; int ret; @@ -981,12 +999,13 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, /* mode - cck: 0, ofdm: 1, ht: 2, gf: 3, vht: 4, he_su: 8, he_er: 9 EHT: 15 * bw - bw20: 0, bw40: 1, bw80: 2, bw160: 3, BW320: 4 - * nss - vht: 1~4, he: 1~4, eht: 1~4, others: ignore * mcs - cck: 0~4, ofdm: 0~7, ht: 0~32, vht: 0~9, he_su: 0~11, he_er: 0~2, eht: 0~13 + * nss - vht: 1~4, he: 1~4, eht: 1~4, others: ignore * gi - (ht/vht) lgi: 0, sgi: 1; (he) 0.8us: 0, 1.6us: 1, 3.2us: 2 * preamble - short: 1, long: 0 - * ldpc - off: 0, on: 1 * stbc - off: 0, on: 1 + * ldpc - off: 0, on: 1 + * spe - off: 0, on: 1 * ltf - 1xltf: 0, 2xltf: 1, 4xltf: 2 */ if (sscanf(buf, "%hhu %hhu %hhu %hhu %hu %hhu %hhu %hhu %hhu %hu", @@ -994,9 +1013,16 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, &phy.preamble, &phy.stbc, &phy.ldpc, &phy.spe, <f) != 10) { dev_warn(dev->mt76.dev, "format: Mode BW MCS NSS GI Preamble STBC LDPC SPE ltf\n"); - goto out; + return -EINVAL; } + mutex_lock(&dev->mt76.mutex); + + msta_link = mt7996_sta_link_protected(dev, msta, link_sta->link_id); + if (!msta_link) { + ret = -EINVAL; + goto out; + } phy.wlan_idx = cpu_to_le16(msta_link->wcid.idx); phy.gi = cpu_to_le16(gi); phy.ltf = cpu_to_le16(ltf); @@ -1005,36 +1031,26 @@ static ssize_t mt7996_sta_fixed_rate_set(struct file *file, ret = mt7996_mcu_set_fixed_rate_ctrl(dev, &phy, 0); if (ret) - return -EFAULT; + goto out; + ret = count; out: - return count; + mutex_unlock(&dev->mt76.mutex); + return ret; } static const struct file_operations fops_fixed_rate = { - .write = mt7996_sta_fixed_rate_set, + .write = mt7996_link_sta_fixed_rate_set, .open = simple_open, .owner = THIS_MODULE, .llseek = default_llseek, }; -static int -mt7996_queues_show(struct seq_file *s, void *data) -{ - struct ieee80211_sta *sta = s->private; - - mt7996_sta_hw_queue_read(s, sta); - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(mt7996_queues); - -void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, - struct ieee80211_sta *sta, struct dentry *dir) +void mt7996_link_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct dentry *dir) { - debugfs_create_file("fixed_rate", 0600, dir, sta, &fops_fixed_rate); - debugfs_create_file("hw-queues", 0400, dir, sta, &mt7996_queues_fops); + debugfs_create_file("fixed_rate", 0600, dir, link_sta, &fops_fixed_rate); } #endif diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c index da3231c9aa119..11e0a28113a2f 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/eeprom.c @@ -148,6 +148,12 @@ mt7996_eeprom_check_or_use_default(struct mt7996_dev *dev, bool use_default) goto out; } + if (fw->size < MT7996_EEPROM_SIZE) { + dev_err(dev->mt76.dev, "Invalid default bin size\n"); + ret = -EINVAL; + goto out; + } + if (!use_default && mt7996_eeprom_variant_valid(dev, fw->data)) goto out; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/init.c b/drivers/net/wireless/mediatek/mt76/mt7996/init.c index efbd46d649017..d0a4530db1cf7 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/init.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/init.c @@ -34,7 +34,6 @@ static const struct ieee80211_iface_combination if_comb_global = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_combination if_comb_global_7992 = { @@ -47,7 +46,6 @@ static const struct ieee80211_iface_combination if_comb_global_7992 = { BIT(NL80211_CHAN_WIDTH_40) | BIT(NL80211_CHAN_WIDTH_80) | BIT(NL80211_CHAN_WIDTH_160), - .beacon_int_min_gcd = 100, }; static const struct ieee80211_iface_limit if_limits[] = { @@ -1502,7 +1500,6 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, struct ieee80211_sta_eht_cap *eht_cap = &data->eht_cap; struct ieee80211_eht_cap_elem_fixed *eht_cap_elem = &eht_cap->eht_cap_elem; struct ieee80211_eht_mcs_nss_supp *eht_nss = &eht_cap->eht_mcs_nss_supp; - enum nl80211_chan_width width = phy->mt76->chandef.width; int nss = hweight8(phy->mt76->antenna_mask); int sts = hweight16(phy->mt76->chainmask); u8 val; @@ -1578,11 +1575,16 @@ mt7996_init_eht_caps(struct mt7996_phy *phy, enum nl80211_band band, u8_encode_bits(u8_get_bits(1, GENMASK(1, 0)), IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK); - val = width == NL80211_CHAN_WIDTH_320 ? 0xf : - width == NL80211_CHAN_WIDTH_160 ? 0x7 : - width == NL80211_CHAN_WIDTH_80 ? 0x3 : 0x1; - eht_cap_elem->phy_cap_info[6] = - u8_encode_bits(val, IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK); + eht_cap_elem->phy_cap_info[6] = IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK; + if (band != NL80211_BAND_6GHZ) { + eht_cap_elem->phy_cap_info[6] &= + ~IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ; + + if (band != NL80211_BAND_5GHZ) + eht_cap_elem->phy_cap_info[6] &= + ~(IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ | + IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ); + } val = u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_RX) | u8_encode_bits(nss, IEEE80211_EHT_MCS_NSS_TX); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 29f599a7ad017..dc1a3e91218ad 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -87,7 +87,7 @@ static struct mt76_wcid *mt7996_rx_get_wcid(struct mt7996_dev *dev, if (mlink->band_idx != band_idx) continue; - msta_link = rcu_dereference(msta->link[i]); + msta_link = mt7996_sta_link(msta, i); break; } @@ -480,7 +480,13 @@ mt7996_mac_fill_rx(struct mt7996_dev *dev, enum mt76_rxq_id q, memset(status, 0, sizeof(*status)); band_idx = FIELD_GET(MT_RXD1_NORMAL_BAND_IDX, rxd1); + if (!mt7996_band_valid(dev, band_idx)) + return -EINVAL; + mphy = dev->mt76.phys[band_idx]; + if (!mphy) + return -EINVAL; + phy = mphy->priv; status->phy_idx = mphy->band_idx; @@ -893,6 +899,33 @@ mt7996_mac_write_txwi_80211(struct mt7996_dev *dev, __le32 *txwi, txwi[6] |= cpu_to_le32(MT_TXD6_DIS_MAT); } +/* The WLAN_IDX in the TXD and TXP must belong to the primary or secondary + * link of an MLD station; any other link id can make the firmware spin when + * that link is in powersave. Completion events carry the same index, so the + * wcid used for status tracking and accounting must match it + */ +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid) +{ + struct mt7996_sta_link *msta_link; + struct mt7996_sta *msta; + + if (!wcid->sta) + return wcid; + + msta_link = container_of(wcid, struct mt7996_sta_link, wcid); + msta = msta_link->sta; + + if (!msta || wcid->link_id == msta->seclink_id || + wcid->link_id == msta->deflink_id) + return wcid; + + msta_link = mt7996_sta_link(msta, msta->deflink_id); + if (msta_link) + return &msta_link->wcid; + + return wcid; +} + void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, @@ -1069,10 +1102,15 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, IEEE80211_TX_CTRL_MLO_LINK); } + /* non-MLD frames are LINK_UNSPECIFIED; use the wcid's own link */ + if (link_id == IEEE80211_LINK_UNSPECIFIED && + wcid != &dev->mt76.global_wcid) + link_id = wcid->link_id; + if (link_id != wcid->link_id && link_id != IEEE80211_LINK_UNSPECIFIED) { if (msta) { struct mt7996_sta_link *msta_link = - rcu_dereference(msta->link[link_id]); + mt7996_sta_link(msta, link_id); if (msta_link) wcid = &msta_link->wcid; @@ -1125,6 +1163,8 @@ int mt7996_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr, tx_info->buf[1].len, DMA_TO_DEVICE); } + wcid = mt7996_get_tx_wcid(wcid); + pid = mt76_tx_status_skb_add(mdev, wcid, tx_info->skb); memset(txwi_ptr, 0, MT_TXD_SIZE); /* Transmit non qos data by 802.11 header and need to fill txd by host*/ @@ -1278,6 +1318,30 @@ mt7996_tx_check_aggr(struct ieee80211_link_sta *link_sta, clear_bit(tid, &wcid->ampdu_state); } +static void +mt7996_txp_skb_unmap(struct mt76_dev *mdev, struct mt76_txwi_cache *t) +{ + u8 *txwi_ptr = mt76_get_txwi_ptr(mdev, t); + __le32 *txwi = (__le32 *)txwi_ptr; + __le32 *txp; + dma_addr_t addr; + u32 val; + + if (!(le32_to_cpu(txwi[7]) & MT_TXD7_MAC_TXD)) { + mt76_connac_txp_skb_unmap(mdev, t); + return; + } + + txp = (__le32 *)(txwi_ptr + MT_TXD_SIZE); + val = le32_to_cpu(txp[3]); + addr = le32_to_cpu(txp[2]); +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + addr |= (dma_addr_t)FIELD_GET(MT_TXP3_DMA_ADDR_H, val) << 32; +#endif + dma_unmap_single(mdev->dma_dev, addr, FIELD_GET(MT_TXP_BUF_LEN, val), + DMA_TO_DEVICE); +} + static void mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, struct ieee80211_link_sta *link_sta, @@ -1287,7 +1351,7 @@ mt7996_txwi_free(struct mt7996_dev *dev, struct mt76_txwi_cache *t, __le32 *txwi; u16 wcid_idx; - mt76_connac_txp_skb_unmap(mdev, t); + mt7996_txp_skb_unmap(mdev, t); if (!t->skb) goto out; @@ -1384,7 +1448,7 @@ mt7996_mac_tx_free(struct mt7996_dev *dev, void *data, int len) IEEE80211_MLD_MAX_NUM_LINKS) { struct mt7996_sta_link *msta_link; - msta_link = rcu_dereference(msta->link[id]); + msta_link = mt7996_sta_link(msta, id); if (!msta_link) continue; @@ -2452,6 +2516,7 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) dev->recovery.hw_full_reset = true; + set_bit(MT76_MCU_RESET, &dev->mphy.state); wake_up(&dev->mt76.mcu.wait); ieee80211_stop_queues(hw); @@ -2474,10 +2539,10 @@ mt7996_mac_full_reset(struct mt7996_dev *dev) phy->omac_mask = 0; ieee80211_iterate_stations_atomic(hw, mt7996_mac_reset_sta_iter, dev); + mt76_reset_device(&dev->mt76); ieee80211_iterate_active_interfaces_atomic(hw, IEEE80211_IFACE_SKIP_SDATA_NOT_IN_DRIVER, mt7996_mac_reset_vif_iter, dev); - mt76_reset_device(&dev->mt76); INIT_LIST_HEAD(&dev->sta_rc_list); INIT_LIST_HEAD(&dev->twt_list); @@ -2555,8 +2620,8 @@ void mt7996_mac_reset_work(struct work_struct *work) set_bit(MT76_RESET, &dev->mphy.state); set_bit(MT76_MCU_RESET, &dev->mphy.state); - mt76_abort_scan(&dev->mt76); wake_up(&dev->mt76.mcu.wait); + mt76_abort_scan(&dev->mt76); cancel_work_sync(&dev->wed_rro.work); mt7996_for_each_phy(dev, phy) { @@ -2565,6 +2630,8 @@ void mt7996_mac_reset_work(struct work_struct *work) cancel_delayed_work_sync(&phy->mt76->mac_work); } + mutex_lock(&dev->mt76.mutex); + mt76_worker_disable(&dev->mt76.tx_worker); mt76_for_each_q_rx(&dev->mt76, i) { if (mtk_wed_device_active(&dev->mt76.mmio.wed) && @@ -2575,8 +2642,6 @@ void mt7996_mac_reset_work(struct work_struct *work) } napi_disable(&dev->mt76.tx_napi); - mutex_lock(&dev->mt76.mutex); - mt76_wr(dev, MT_MCU_INT_EVENT, MT_MCU_INT_EVENT_DMA_STOPPED); if (mt7996_wait_reset_state(dev, MT_MCU_CMD_RESET_DONE)) { diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/main.c b/drivers/net/wireless/mediatek/mt76/mt7996/main.c index 20da0c10669a9..c6d7088b15cab 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/main.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/main.c @@ -207,8 +207,7 @@ mt7996_set_hw_key(struct ieee80211_hw *hw, enum set_key_cmd cmd, struct mt7996_sta *msta; msta = (struct mt7996_sta *)sta->drv_priv; - msta_link = mt76_dereference(msta->link[link_id], - &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) return 0; @@ -363,7 +362,8 @@ int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif, CONN_STATE_PORT_SECURE, true); rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid); - ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); + if (!mlink->wcid->offchannel) + ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it); if (vif->txq && !mlink->wcid->offchannel && mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) { @@ -950,12 +950,24 @@ mt7996_post_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct cfg80211_chan_def *chandef = &link_conf->chanreq.oper; struct mt7996_dev *dev = mt7996_hw_dev(hw); struct mt7996_phy *phy = mt7996_band_phy(dev, chandef->chan->band); - int ret; + struct mt7996_vif_link *link; + int ret = -EINVAL; mutex_lock(&dev->mt76.mutex); + link = mt7996_vif_conf_link(dev, vif, link_conf); + if (!link) + goto out; + + ret = mt7996_mcu_update_bss_rfch(phy, link); + if (ret) + goto out; + + ieee80211_iterate_stations_mtx(hw, mt7996_mcu_update_sta_rec_bw, link); + ret = mt7996_mcu_rdd_resume_tx(phy); +out: mutex_unlock(&dev->mt76.mutex); return ret; @@ -1224,7 +1236,7 @@ mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif, if (!link) continue; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -1349,20 +1361,26 @@ static void mt7996_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif = info->control.vif; struct mt7996_vif *mvif = vif ? (void *)vif->drv_priv : NULL; struct mt76_wcid *wcid = &dev->mt76.global_wcid; + u8 deflink_id = IEEE80211_LINK_UNSPECIFIED; u8 link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK); rcu_read_lock(); + if (msta) + deflink_id = msta->deflink_id; + else if (mvif) + deflink_id = mvif->mt76.deflink_id; + + /* the primary link is unset until the first link has been added */ + if (deflink_id >= IEEE80211_MLD_MAX_NUM_LINKS) + deflink_id = 0; + /* Use primary link_id if the value from mac80211 is set to * IEEE80211_LINK_UNSPECIFIED. */ - if (link_id == IEEE80211_LINK_UNSPECIFIED) { - if (msta) - link_id = msta->deflink_id; - else if (mvif) - link_id = mvif->mt76.deflink_id; - } + if (link_id == IEEE80211_LINK_UNSPECIFIED) + link_id = deflink_id; if (vif && ieee80211_vif_is_mld(vif)) { struct ieee80211_bss_conf *link_conf; @@ -1372,7 +1390,7 @@ static void mt7996_tx(struct ieee80211_hw *hw, link_sta = rcu_dereference(sta->link[link_id]); if (!link_sta) - link_sta = rcu_dereference(sta->link[msta->deflink_id]); + link_sta = rcu_dereference(sta->link[deflink_id]); if (link_sta) { memcpy(hdr->addr1, link_sta->addr, ETH_ALEN); @@ -1414,11 +1432,11 @@ static void mt7996_tx(struct ieee80211_hw *hw, if (msta) { struct mt7996_sta_link *msta_link; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (msta_link) wcid = &msta_link->wcid; } - mt76_tx(mphy, control->sta, wcid, skb); + mt76_tx(mphy, control->sta, mt7996_get_tx_wcid(wcid), skb); unlock: rcu_read_unlock(); } @@ -1722,8 +1740,6 @@ static void mt7996_sta_statistics(struct ieee80211_hw *hw, sinfo->txrate.flags = txrate->flags; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } - sinfo->txrate.flags = txrate->flags; - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); sinfo->tx_failed = msta_link->wcid.stats.tx_failed; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED); @@ -1780,7 +1796,7 @@ static void mt7996_link_sta_rc_update(struct ieee80211_hw *hw, rcu_read_lock(); - msta_link = rcu_dereference(msta->link[link_sta->link_id]); + msta_link = mt7996_sta_link(msta, link_sta->link_id); if (msta_link) { struct mt7996_dev *dev = mt7996_hw_dev(hw); @@ -1801,7 +1817,7 @@ static void mt7996_sta_rate_ctrl_update(void *data, struct ieee80211_sta *sta) if (msta->vif != mvif) return; - msta_link = rcu_dereference(msta->link[msta->deflink_id]); + msta_link = mt7996_sta_link(msta, msta->deflink_id); if (msta_link) mt7996_link_rate_ctrl_update(&changed, msta_link); } @@ -1850,7 +1866,7 @@ static void mt7996_sta_set_4addr(struct ieee80211_hw *hw, if (!link) continue; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -1888,7 +1904,7 @@ static void mt7996_sta_set_decap_offload(struct ieee80211_hw *hw, if (!link) continue; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -2230,7 +2246,7 @@ mt7996_net_fill_forward_path(struct ieee80211_hw *hw, if (!mlink) return -EIO; - msta_link = rcu_dereference(msta->link[msta->deflink_id]); + msta_link = mt7996_sta_link(msta, msta->deflink_id); if (!msta_link) return -EIO; @@ -2288,6 +2304,7 @@ mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif, idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1; if (idx < 0) { + dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx); ret = -ENOSPC; goto out; } @@ -2371,6 +2388,7 @@ const struct ieee80211_ops mt7996_ops = { .twt_teardown_request = mt7996_twt_teardown_request, #ifdef CONFIG_MAC80211_DEBUGFS .sta_add_debugfs = mt7996_sta_add_debugfs, + .link_sta_add_debugfs = mt7996_link_sta_add_debugfs, #endif .set_radar_background = mt7996_set_radar_background, #ifdef CONFIG_NET_MEDIATEK_SOC_WED diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c index a3f813be107df..92f8a28cd4a51 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.c @@ -860,6 +860,8 @@ mt7996_mcu_bss_he_tlv(struct sk_buff *skb, struct ieee80211_vif *vif, struct tlv *tlv; cap = mt76_connac_get_he_phy_cap(phy->mt76, vif); + if (!cap) + return; tlv = mt7996_mcu_add_uni_tlv(skb, UNI_BSS_INFO_HE_BASIC, sizeof(*he)); @@ -1061,7 +1063,7 @@ mt7996_mcu_bss_basic_tlv(struct sk_buff *skb, struct mt7996_sta_link *msta_link; int link_id = link_conf->link_id; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (msta_link) sta_wlan_idx = msta_link->wcid.idx; } @@ -1175,6 +1177,22 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); } +int mt7996_mcu_update_bss_rfch(struct mt7996_phy *phy, struct mt7996_vif_link *link) +{ + struct mt7996_dev *dev = phy->dev; + struct sk_buff *skb; + + skb = __mt7996_mcu_alloc_bss_req(&dev->mt76, &link->mt76, + MT7996_BSS_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + mt7996_mcu_bss_rfch_tlv(skb, phy); + + return mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(BSS_INFO_UPDATE), true); +} + int mt7996_mcu_set_timing(struct mt7996_phy *phy, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link_conf) { @@ -1237,7 +1255,7 @@ int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, struct mt7996_sta_link *msta_link; struct mt7996_vif_link *link; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -1271,7 +1289,7 @@ int mt7996_mcu_add_rx_ba(struct mt7996_dev *dev, struct mt7996_sta_link *msta_link; struct mt7996_vif_link *link; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -1659,17 +1677,18 @@ mt7996_mcu_sta_bfer_he(struct ieee80211_link_sta *link_sta, { struct ieee80211_sta_he_cap *pc = &link_sta->he_cap; struct ieee80211_he_cap_elem *pe = &pc->he_cap_elem; - const struct ieee80211_sta_he_cap *vc = - mt76_connac_get_he_phy_cap(phy->mt76, vif); - const struct ieee80211_he_cap_elem *ve = &vc->he_cap_elem; u16 mcs_map = le16_to_cpu(pc->he_mcs_nss_supp.rx_mcs_80); u8 nss_mcs = mt7996_mcu_get_sta_nss(mcs_map); + const struct ieee80211_he_cap_elem *ve; + const struct ieee80211_sta_he_cap *vc; u8 snd_dim, sts; + vc = mt76_connac_get_he_phy_cap(phy->mt76, vif); if (!vc) return; bf->tx_mode = MT_PHY_TYPE_HE_SU; + ve = &vc->he_cap_elem; mt7996_mcu_sta_sounding_rate(bf, phy); @@ -1725,14 +1744,18 @@ mt7996_mcu_sta_bfer_eht(struct ieee80211_link_sta *link_sta, struct ieee80211_sta_eht_cap *pc = &link_sta->eht_cap; struct ieee80211_eht_cap_elem_fixed *pe = &pc->eht_cap_elem; struct ieee80211_eht_mcs_nss_supp *eht_nss = &pc->eht_mcs_nss_supp; - const struct ieee80211_sta_eht_cap *vc = - mt76_connac_get_eht_phy_cap(phy->mt76, vif); - const struct ieee80211_eht_cap_elem_fixed *ve = &vc->eht_cap_elem; u8 nss_mcs = u8_get_bits(eht_nss->bw._80.rx_tx_mcs9_max_nss, IEEE80211_EHT_MCS_NSS_RX) - 1; + const struct ieee80211_eht_cap_elem_fixed *ve; + const struct ieee80211_sta_eht_cap *vc; u8 snd_dim, sts; + vc = mt76_connac_get_eht_phy_cap(phy->mt76, vif); + if (!vc) + return; + bf->tx_mode = MT_PHY_TYPE_EHT_MU; + ve = &vc->eht_cap_elem; mt7996_mcu_sta_sounding_rate(bf, phy); @@ -2000,7 +2023,7 @@ int mt7996_mcu_set_fixed_field(struct mt7996_dev *dev, struct mt7996_sta *msta, if (!mlink) goto error_unlock; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (!msta_link) goto error_unlock; @@ -2089,7 +2112,7 @@ mt7996_mcu_add_rate_ctrl_fixed(struct mt7996_dev *dev, struct mt7996_sta *msta, if (!link) goto error_unlock; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (!msta_link) goto error_unlock; @@ -2296,7 +2319,7 @@ int mt7996_mcu_add_rate_ctrl(struct mt7996_dev *dev, struct mt7996_sta *msta, if (!link) goto error_unlock; - msta_link = rcu_dereference(msta->link[link_id]); + msta_link = mt7996_sta_link(msta, link_id); if (!msta_link) goto error_unlock; @@ -2385,7 +2408,7 @@ mt7996_mcu_sta_mld_setup_tlv(struct mt7996_dev *dev, struct sk_buff *skb, unsigned int link_id; struct tlv *tlv; - msta_link = mt76_dereference(msta->link[msta->deflink_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, msta->deflink_id); if (!msta_link) return; @@ -2399,8 +2422,8 @@ mt7996_mcu_sta_mld_setup_tlv(struct mt7996_dev *dev, struct sk_buff *skb, mld_setup->primary_id = cpu_to_le16(msta_link->wcid.idx); if (nlinks > 1) { - msta_link = mt76_dereference(msta->link[msta->seclink_id], - &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, + msta->seclink_id); if (!msta_link) return; } @@ -2411,7 +2434,7 @@ mt7996_mcu_sta_mld_setup_tlv(struct mt7996_dev *dev, struct sk_buff *skb, for_each_sta_active_link(vif, sta, link_sta, link_id) { struct mt7996_vif_link *link; - msta_link = mt76_dereference(msta->link[link_id], &dev->mt76); + msta_link = mt7996_sta_link_protected(dev, msta, link_id); if (!msta_link) continue; @@ -2534,6 +2557,49 @@ int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev, MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); } +void mt7996_mcu_update_sta_rec_bw(void *data, struct ieee80211_sta *sta) +{ + struct mt7996_vif_link *link = (struct mt7996_vif_link *)data; + struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv; + struct mt7996_sta_link *msta_link; + struct mt7996_dev *dev; + struct ieee80211_bss_conf *link_conf; + struct ieee80211_link_sta *link_sta; + struct ieee80211_vif *vif; + struct sk_buff *skb; + int link_id; + + if (link->mt76.mvif != &msta->vif->mt76) + return; + + dev = link->phy->dev; + link_id = link->msta_link.wcid.link_id; + link_sta = link_sta_dereference_protected(sta, link_id); + if (!link_sta) + return; + + msta_link = mt7996_sta_link_protected(dev, msta, link_id); + if (!msta_link) + return; + + vif = container_of((void *)msta->vif, struct ieee80211_vif, drv_priv); + link_conf = link_conf_dereference_protected(vif, link_id); + if (!link_conf) + return; + + skb = __mt76_connac_mcu_alloc_sta_req(&dev->mt76, &link->mt76, + &msta_link->wcid, + MT7996_STA_UPDATE_MAX_SIZE); + if (IS_ERR(skb)) + return; + + mt7996_mcu_sta_bfer_tlv(dev, skb, link_conf, link_sta, link); + mt7996_mcu_sta_rate_ctrl_tlv(skb, dev, vif, link_conf, link_sta, link); + + mt76_mcu_skb_send_msg(&dev->mt76, skb, + MCU_WMWA_UNI_CMD(STA_REC_UPDATE), true); +} + static int mt7996_mcu_sta_key_tlv(struct mt76_dev *dev, struct mt76_wcid *wcid, struct sk_buff *skb, @@ -3669,6 +3735,9 @@ int mt7996_mcu_rdd_background_enable(struct mt7996_phy *phy, struct mt7996_dev *dev = phy->dev; int err, region, rdd_idx = mt7996_get_rdd_idx(phy, true); + if (rdd_idx < 0) + return -EINVAL; + if (!chandef) { /* disable offchain */ err = mt7996_mcu_rdd_cmd(dev, RDD_STOP, rdd_idx, 0); if (err) @@ -3859,8 +3928,14 @@ int mt7996_mcu_get_eeprom(struct mt7996_dev *dev, u32 offset, u8 *buf, u32 buf_l if (valid) { u32 addr = le32_to_cpu(*(__le32 *)(skb->data + 12)); - if (!buf) + if (!buf) { + if (addr > dev->mt76.eeprom.size - + MT7996_EEPROM_BLOCK_SIZE) { + dev_kfree_skb(skb); + return -EINVAL; + } buf = (u8 *)dev->mt76.eeprom.data + addr; + } if (!buf_len || buf_len > MT7996_EEPROM_BLOCK_SIZE) buf_len = MT7996_EEPROM_BLOCK_SIZE; @@ -3929,21 +4004,31 @@ int mt7996_mcu_get_chip_config(struct mt7996_dev *dev, u32 *cap) return ret; /* fixed field */ + if (skb->len < 4) { + dev_kfree_skb(skb); + return -EINVAL; + } skb_pull(skb, 4); buf = skb->data; - while (buf - skb->data < skb->len) { + while (buf - skb->data + sizeof(struct tlv) <= skb->len) { struct tlv *tlv = (struct tlv *)buf; + u16 tlv_len = le16_to_cpu(tlv->len); + + if (tlv_len < sizeof(*tlv) || + tlv_len > skb->len - (buf - skb->data)) + break; switch (le16_to_cpu(tlv->tag)) { case UNI_EVENT_CHIP_CONFIG_EFUSE_VERSION: - *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); + if (tlv_len >= sizeof(*tlv) + sizeof(__le32)) + *cap = le32_to_cpu(*(__le32 *)(buf + sizeof(*tlv))); break; default: break; } - buf += le16_to_cpu(tlv->len); + buf += tlv_len; } dev_kfree_skb(skb); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h index 7b51d7346bcaa..848c6c5751155 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mcu.h @@ -828,7 +828,7 @@ enum { #define MT7996_BEACON_UPDATE_SIZE (sizeof(struct bss_req_hdr) + \ sizeof(struct bss_bcn_content_tlv) + \ 4 + MT_TXD_SIZE + \ - sizeof(struct bss_bcn_cntdwn_tlv) + \ + sizeof(struct bss_bcn_cntdwn_tlv) * 2 + \ sizeof(struct bss_bcn_mbss_tlv)) #define MT7996_MAX_BSS_OFFLOAD_SIZE 2048 #define MT7996_MAX_BEACON_SIZE (MT7996_MAX_BSS_OFFLOAD_SIZE - \ diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c index 80db102ed809c..e37b95f5b00b8 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mmio.c @@ -17,6 +17,8 @@ static bool wed_enable; module_param(wed_enable, bool, 0644); +#define INVALID_REG_ADDR 0xffffffff + static const struct __base mt7996_reg_base[] = { [WF_AGG_BASE] = { { 0x820e2000, 0x820f2000, 0x830e2000 } }, [WF_ARB_BASE] = { { 0x820e3000, 0x820f3000, 0x830e3000 } }, @@ -54,6 +56,14 @@ static const u32 mt7996_offs[] = { [MIB_BSCR7] = 0x9e8, [MIB_BSCR17] = 0xa10, [MIB_TRDR1] = 0xa28, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x24, [HIF_REMAP_BASE_L1] = 0x130000, [HIF_REMAP_L2] = 0x1b4, @@ -91,6 +101,14 @@ static const u32 mt7992_offs[] = { [MIB_BSCR7] = 0xae4, [MIB_BSCR17] = 0xb0c, [MIB_TRDR1] = 0xb24, + [MIB_TSCR0] = 0x6b0, + [MIB_TSCR1] = 0x6b4, + [MIB_TSCR2] = 0x6b8, + [MIB_TSCR3] = 0x6bc, + [MIB_TSCR4] = 0x6c0, + [MIB_TSCR5] = 0x6c4, + [MIB_TSCR6] = 0x6c8, + [MIB_TSCR7] = 0x6d0, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b4, @@ -128,6 +146,14 @@ static const u32 mt7990_offs[] = { [MIB_BSCR7] = 0xbd4, [MIB_BSCR17] = 0xbfc, [MIB_TRDR1] = 0xc14, + [MIB_TSCR0] = 0x750, + [MIB_TSCR1] = 0x754, + [MIB_TSCR2] = 0x758, + [MIB_TSCR3] = 0x75c, + [MIB_TSCR4] = 0x760, + [MIB_TSCR5] = 0x764, + [MIB_TSCR6] = 0x768, + [MIB_TSCR7] = 0x770, [HIF_REMAP_L1] = 0x8, [HIF_REMAP_BASE_L1] = 0x40000, [HIF_REMAP_L2] = 0x1b8, @@ -334,7 +360,7 @@ static u32 __mt7996_reg_addr(struct mt7996_dev *dev, u32 addr) return dev->reg.map[i].mapped + ofs; } - return 0; + return INVALID_REG_ADDR; } static u32 __mt7996_reg_remap_addr(struct mt7996_dev *dev, u32 addr) @@ -366,7 +392,7 @@ void mt7996_memcpy_fromio(struct mt7996_dev *dev, void *buf, u32 offset, { u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { memcpy_fromio(buf, dev->mt76.mmio.regs + addr, len); return; } @@ -382,7 +408,7 @@ static u32 mt7996_rr(struct mt76_dev *mdev, u32 offset) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset), val; - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rr(mdev, addr); spin_lock_bh(&dev->reg_lock); @@ -397,7 +423,7 @@ static void mt7996_wr(struct mt76_dev *mdev, u32 offset, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) { + if (addr != INVALID_REG_ADDR) { dev->bus_ops->wr(mdev, addr, val); return; } @@ -412,7 +438,7 @@ static u32 mt7996_rmw(struct mt76_dev *mdev, u32 offset, u32 mask, u32 val) struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); u32 addr = __mt7996_reg_addr(dev, offset); - if (addr) + if (addr != INVALID_REG_ADDR) return dev->bus_ops->rmw(mdev, addr, mask, val); spin_lock_bh(&dev->reg_lock); @@ -464,6 +490,9 @@ int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, if (!wed_enable) return 0; + if (hif2 && !mtk_wed_device_active(&dev->mt76.mmio.wed)) + return 0; + dev->mt76.hwrro_mode = is_mt7996(&dev->mt76) ? MT76_HWRRO_V3 : MT76_HWRRO_V3_1; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index b8ffa42c5a1da..e371b2f174e58 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -618,6 +618,25 @@ mt7996_vif_conf_link(struct mt7996_dev *dev, struct ieee80211_vif *vif, link_conf); } +static inline struct mt7996_sta_link * +mt7996_sta_link(struct mt7996_sta *msta, u8 link_id) +{ + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + return NULL; + + return rcu_dereference(msta->link[link_id]); +} + +static inline struct mt7996_sta_link * +mt7996_sta_link_protected(struct mt7996_dev *dev, struct mt7996_sta *msta, + u8 link_id) +{ + if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS) + return NULL; + + return mt76_dereference(msta->link[link_id], &dev->mt76); +} + #define mt7996_for_each_phy(dev, phy) \ for (int __i = 0; __i < ARRAY_SIZE((dev)->radio_phy); __i++) \ if (((phy) = (dev)->radio_phy[__i]) != NULL) @@ -670,6 +689,8 @@ int mt7996_mcu_add_bss_info(struct mt7996_phy *phy, struct ieee80211_vif *vif, struct ieee80211_bss_conf *link_conf, struct mt76_vif_link *mlink, struct mt7996_sta_link *msta_link, int enable); +int mt7996_mcu_update_bss_rfch(struct mt7996_phy *phy, + struct mt7996_vif_link *link); int mt7996_mcu_add_sta(struct mt7996_dev *dev, struct ieee80211_bss_conf *link_conf, struct ieee80211_link_sta *link_sta, @@ -679,6 +700,7 @@ int mt7996_mcu_add_sta(struct mt7996_dev *dev, int mt7996_mcu_teardown_mld_sta(struct mt7996_dev *dev, struct mt7996_vif_link *link, struct mt7996_sta_link *msta_link); +void mt7996_mcu_update_sta_rec_bw(void *data, struct ieee80211_sta *sta); int mt7996_mcu_add_tx_ba(struct mt7996_dev *dev, struct ieee80211_ampdu_params *params, struct ieee80211_vif *vif, bool enable); @@ -810,6 +832,7 @@ bool mt7996_mac_wtbl_update(struct mt7996_dev *dev, int idx, u32 mask); void mt7996_mac_reset_counters(struct mt7996_phy *phy); void mt7996_mac_cca_stats_reset(struct mt7996_phy *phy); void mt7996_mac_enable_nf(struct mt7996_dev *dev, u8 band); +struct mt76_wcid *mt7996_get_tx_wcid(struct mt76_wcid *wcid); void mt7996_mac_write_txwi(struct mt7996_dev *dev, __le32 *txwi, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_key_conf *key, int pid, @@ -867,6 +890,9 @@ int mt7996_mcu_cp_support(struct mt7996_dev *dev, u8 mode); #ifdef CONFIG_MAC80211_DEBUGFS void mt7996_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, struct dentry *dir); +void mt7996_link_sta_add_debugfs(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct ieee80211_link_sta *link_sta, + struct dentry *dir); #endif int mt7996_mmio_wed_init(struct mt7996_dev *dev, void *pdev_ptr, bool hif2, int *irq); diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h index 0fa325f87fcd9..a46c790550c62 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/regs.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/regs.h @@ -64,6 +64,14 @@ enum offs_rev { MIB_BSCR7, MIB_BSCR17, MIB_TRDR1, + MIB_TSCR0, + MIB_TSCR1, + MIB_TSCR2, + MIB_TSCR3, + MIB_TSCR4, + MIB_TSCR5, + MIB_TSCR6, + MIB_TSCR7, HIF_REMAP_L1, HIF_REMAP_BASE_L1, HIF_REMAP_L2, @@ -247,9 +255,9 @@ enum offs_rev { #define MT_MIB_BSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR7)) #define MT_MIB_BSCR17(_band) MT_WF_MIB(_band, __OFFS(MIB_BSCR17)) -#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, 0x6c4) -#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, 0x6c8) -#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, 0x6d0) +#define MT_MIB_TSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR5)) +#define MT_MIB_TSCR6(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR6)) +#define MT_MIB_TSCR7(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR7)) #define MT_MIB_RSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR1)) /* rx mpdu counter, full 32 bits */ @@ -265,14 +273,14 @@ enum offs_rev { #define MT_MIB_RSCR36(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR36)) /* tx ampdu cnt, full 32 bits */ -#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, 0x6b0) -#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, 0x6b8) +#define MT_MIB_TSCR0(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR0)) +#define MT_MIB_TSCR2(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR2)) /* counts all mpdus in ampdu, regardless of success */ -#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, 0x6bc) +#define MT_MIB_TSCR3(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR3)) /* counts all successfully tx'd mpdus in ampdu */ -#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, 0x6c0) +#define MT_MIB_TSCR4(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR4)) /* rx ampdu count, 32-bit */ #define MT_MIB_RSCR27(_band) MT_WF_MIB(_band, __OFFS(MIB_RSCR27)) @@ -296,7 +304,7 @@ enum offs_rev { #define MT_MIB_RVSR1(_band) MT_WF_MIB(_band, __OFFS(MIB_RVSR1)) /* rx blockack count, 32 bits */ -#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, 0x6b4) +#define MT_MIB_TSCR1(_band) MT_WF_MIB(_band, __OFFS(MIB_TSCR1)) #define MT_MIB_BTSCR0(_band) MT_WF_MIB(_band, 0x5e0) #define MT_MIB_BTSCR5(_band) MT_WF_MIB(_band, __OFFS(MIB_BTSCR5)) diff --git a/drivers/net/wireless/mediatek/mt76/npu.c b/drivers/net/wireless/mediatek/mt76/npu.c new file mode 100644 index 0000000000000..9679237f73984 --- /dev/null +++ b/drivers/net/wireless/mediatek/mt76/npu.c @@ -0,0 +1,502 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 AIROHA Inc + * Author: Lorenzo Bianconi + */ +#include +#include +#include + +#include "mt76.h" +#include "dma.h" +#include "mt76_connac.h" + +#define MT76_NPU_RX_BUF_SIZE (1800 + \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) + +int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) +{ + int nframes = 0; + + while (q->queued < q->ndesc - 1) { + struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; + struct mt76_queue_entry *e = &q->entry[q->head]; + struct page *page; + int offset; + + e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size); + if (!e->buf) + break; + + e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size); + page = virt_to_head_page(e->buf); + e->dma_addr[0] = page_pool_get_dma_addr(page) + offset; + + memset(&desc[q->head], 0, sizeof(*desc)); + desc[q->head].addr = e->dma_addr[0]; + + q->head = (q->head + 1) % q->ndesc; + q->queued++; + nframes++; + } + + return nframes; +} + +void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q) +{ + spin_lock_bh(&q->lock); + while (q->queued > 0) { + struct mt76_queue_entry *e = &q->entry[q->tail]; + + dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], + e->dma_len[0], + page_pool_get_dma_dir(q->page_pool)); + mt76_put_page_pool_buf(e->buf, false); + q->tail = (q->tail + 1) % q->ndesc; + q->queued--; + } + spin_unlock_bh(&q->lock); +} + +static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev, + struct mt76_queue *q, + u32 *info) +{ + struct airoha_npu_rx_dma_desc *desc = (void *)q->desc; + int i, nframes, index = q->tail; + struct sk_buff *skb = NULL; + + nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info); + nframes = max_t(int, nframes, 1); + + for (i = 0; i < nframes; i++) { + struct mt76_queue_entry *e = &q->entry[index]; + int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK, + desc[index].ctrl); + + if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) { + dev_kfree_skb(skb); + return NULL; + } + + dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0], + e->dma_len[0], + page_pool_get_dma_dir(q->page_pool)); + + if (!skb) { + skb = napi_build_skb(e->buf, q->buf_size); + if (!skb) + return NULL; + + __skb_put(skb, len); + skb_reset_mac_header(skb); + skb_mark_for_recycle(skb); + } else { + struct skb_shared_info *shinfo = skb_shinfo(skb); + struct page *page = virt_to_head_page(e->buf); + int nr_frags = shinfo->nr_frags; + + if (nr_frags < ARRAY_SIZE(shinfo->frags)) + skb_add_rx_frag(skb, nr_frags, page, + e->buf - page_address(page), + len, q->buf_size); + } + + *info = desc[index].info; + index = (index + 1) % q->ndesc; + } + q->tail = index; + q->queued -= i; + Q_WRITE(q, dma_idx, q->tail); + + return skb; +} + +void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb, + u32 info) +{ + struct airoha_ppe_dev *ppe_dev; + u16 reason, hash; + + if (!mt76_npu_device_active(dev)) + return; + + rcu_read_lock(); + + ppe_dev = rcu_dereference(dev->mmio.ppe_dev); + if (!ppe_dev) + goto out; + + hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info); + skb_set_hash(skb, hash, PKT_HASH_TYPE_L4); + + reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info); + if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) { + skb_set_mac_header(skb, 0); + airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true); + } +out: + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(mt76_npu_check_ppe); + +static int mt76_npu_rx_poll(struct napi_struct *napi, int budget) +{ + struct mt76_dev *dev = mt76_priv(napi->dev); + enum mt76_rxq_id qid = napi - dev->napi; + struct airoha_npu *npu; + int done = 0; + + rcu_read_lock(); + + npu = rcu_dereference(dev->mmio.npu); + if (!npu) + goto out; + + while (done < budget) { + struct sk_buff *skb; + u32 info = 0; + + skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info); + if (!skb) + break; + + dev->drv->rx_skb(dev, qid, skb, &info); + mt76_rx_poll_complete(dev, qid, napi); + done++; + } + + mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]); +out: + if (done < budget && napi_complete(napi)) + dev->drv->rx_poll_complete(dev, qid); + + rcu_read_unlock(); + + return done; +} + +static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance) +{ + struct mt76_queue *q = q_instance; + struct mt76_dev *dev = q->dev; + int qid = q - &dev->q_rx[0]; + int index = qid - MT_RXQ_NPU0; + struct airoha_npu *npu; + u32 status; + + rcu_read_lock(); + + npu = rcu_dereference(dev->mmio.npu); + if (!npu) + goto out; + + status = airoha_npu_wlan_get_irq_status(npu, index); + airoha_npu_wlan_set_irq_status(npu, status); + + airoha_npu_wlan_disable_irq(npu, index); + napi_schedule(&dev->napi[qid]); +out: + rcu_read_unlock(); + + return IRQ_HANDLED; +} + +int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q, + struct sk_buff *skb, struct mt76_queue_buf *buf, + void *txwi_ptr) +{ + u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN); + struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; + int ret; + + /* TODO: Take into account unlinear skbs */ + memcpy(desc[q->head].txwi, txwi_ptr, txwi_len); + desc[q->head].addr = buf->addr; + desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) | + FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) | + NPU_TX_DMA_DESC_DONE_MASK; + + ret = q->head; + q->entry[q->head].skip_buf0 = true; + q->entry[q->head].skip_buf1 = true; + q->entry[q->head].txwi = NULL; + q->entry[q->head].skb = NULL; + q->entry[q->head].wcid = 0xffff; + + q->head = (q->head + 1) % q->ndesc; + q->queued++; + + return ret; +} + +void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index) +{ + struct airoha_npu_tx_dma_desc *desc = (void *)q->desc; + + if (!mt76_queue_is_npu_tx(q)) + return; + + desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK; +} + +void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q) +{ + int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags); + bool xmit = mt76_queue_is_npu_tx(q); + struct airoha_npu *npu; + + if (!mt76_queue_is_npu(q)) + return; + + npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); + if (npu) + q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit); +} + +int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q) +{ + int err, irq, qid = q - &dev->q_rx[0]; + int size, index = qid - MT_RXQ_NPU0; + struct airoha_npu *npu; + const char *name; + + mutex_lock(&dev->mutex); + + npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex); + irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index] + : -EINVAL; + if (irq < 0) { + err = irq; + goto out; + } + + q->flags = MT_NPU_Q_RX(index); + size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM; + err = dev->queue_ops->alloc(dev, q, 0, size, + MT76_NPU_RX_BUF_SIZE, 0); + if (err) + goto out; + + name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index); + if (!name) { + err = -ENOMEM; + goto out; + } + + err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler, + IRQF_SHARED, name, q); + if (err) + goto out; + + netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll); + mt76_npu_fill_rx_queue(dev, q); + napi_enable(&dev->napi[qid]); +out: + mutex_unlock(&dev->mutex); + + return err; +} +EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init); + +static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type, + void *type_data, void *cb_priv) +{ + struct mt76_phy *phy = cb_priv; + struct mt76_dev *dev = phy->dev; + struct airoha_ppe_dev *ppe_dev; + int err = -EOPNOTSUPP; + + if (type != TC_SETUP_CLSFLOWER) + return -EOPNOTSUPP; + + mutex_lock(&dev->mutex); + + ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex); + if (ppe_dev) + err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data); + + mutex_unlock(&dev->mutex); + + return err; +} + +static int mt76_npu_setup_tc_block(struct mt76_phy *phy, + struct net_device *dev, + struct flow_block_offload *f) +{ + flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb; + static LIST_HEAD(block_cb_list); + struct flow_block_cb *block_cb; + + if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) + return -EOPNOTSUPP; + + if (!tc_can_offload(dev)) + return -EOPNOTSUPP; + + f->driver_block_list = &block_cb_list; + switch (f->command) { + case FLOW_BLOCK_BIND: + block_cb = flow_block_cb_lookup(f->block, cb, dev); + if (block_cb) { + flow_block_cb_incref(block_cb); + return 0; + } + + block_cb = flow_block_cb_alloc(cb, dev, phy, NULL); + if (IS_ERR(block_cb)) + return PTR_ERR(block_cb); + + flow_block_cb_incref(block_cb); + flow_block_cb_add(block_cb, f); + list_add_tail(&block_cb->driver_list, &block_cb_list); + return 0; + case FLOW_BLOCK_UNBIND: + block_cb = flow_block_cb_lookup(f->block, cb, dev); + if (!block_cb) + return -ENOENT; + + if (!flow_block_cb_decref(block_cb)) { + flow_block_cb_remove(block_cb, f); + list_del(&block_cb->driver_list); + } + return 0; + default: + return -EOPNOTSUPP; + } +} + +int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + struct net_device *dev, enum tc_setup_type type, + void *type_data) +{ + struct mt76_phy *phy = hw->priv; + + if (!tc_can_offload(dev)) + return -EOPNOTSUPP; + + if (!mt76_npu_device_active(phy->dev)) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_BLOCK: + case TC_SETUP_FT: + return mt76_npu_setup_tc_block(phy, dev, type_data); + default: + return -EOPNOTSUPP; + } +} +EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc); + +void mt76_npu_disable_irqs(struct mt76_dev *dev) +{ + struct airoha_npu *npu; + int i; + + rcu_read_lock(); + + npu = rcu_dereference(dev->mmio.npu); + if (!npu) + goto unlock; + + for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) { + int qid = i - MT_RXQ_NPU0; + u32 status; + + status = airoha_npu_wlan_get_irq_status(npu, qid); + airoha_npu_wlan_set_irq_status(npu, status); + airoha_npu_wlan_disable_irq(npu, qid); + } +unlock: + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs); + +int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type) +{ + struct airoha_ppe_dev *ppe_dev; + struct airoha_npu *npu; + int err = 0; + + /* NPU offloading is only supported by MT7992 */ + if (!is_mt7992(dev)) + return 0; + + mutex_lock(&dev->mutex); + + npu = airoha_npu_get(dev->dev); + if (IS_ERR(npu)) { + request_module("airoha-npu"); + npu = airoha_npu_get(dev->dev); + } + + if (IS_ERR(npu)) { + err = PTR_ERR(npu); + goto error_unlock; + } + + ppe_dev = airoha_ppe_get_dev(dev->dev); + if (IS_ERR(ppe_dev)) { + request_module("airoha-eth"); + ppe_dev = airoha_ppe_get_dev(dev->dev); + } + + if (IS_ERR(ppe_dev)) { + err = PTR_ERR(ppe_dev); + goto error_npu_put; + } + + err = airoha_npu_wlan_init_reserved_memory(npu); + if (err) + goto error_ppe_put; + + dev->dma_dev = npu->dev; + dev->mmio.phy_addr = phy_addr; + dev->mmio.npu_type = type; + /* NPU offloading requires HW-RRO for RX packet reordering. */ + dev->hwrro_mode = MT76_HWRRO_V3_1; + dev->rx_token_size = 32768; + + rcu_assign_pointer(dev->mmio.npu, npu); + rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev); + synchronize_rcu(); + + mutex_unlock(&dev->mutex); + + return 0; + +error_ppe_put: + airoha_ppe_put_dev(ppe_dev); +error_npu_put: + airoha_npu_put(npu); +error_unlock: + mutex_unlock(&dev->mutex); + + return err; +} +EXPORT_SYMBOL_GPL(mt76_npu_init); + +void mt76_npu_deinit(struct mt76_dev *dev) +{ + struct airoha_ppe_dev *ppe_dev; + struct airoha_npu *npu; + + mutex_lock(&dev->mutex); + + npu = rcu_replace_pointer(dev->mmio.npu, NULL, + lockdep_is_held(&dev->mutex)); + if (npu) + airoha_npu_put(npu); + + ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL, + lockdep_is_held(&dev->mutex)); + if (ppe_dev) + airoha_ppe_put_dev(ppe_dev); + + mutex_unlock(&dev->mutex); + + mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]); + mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]); +} diff --git a/drivers/net/wireless/mediatek/mt76/pci.c b/drivers/net/wireless/mediatek/mt76/pci.c index b5031ca7f73fb..4c864afefe2f2 100644 --- a/drivers/net/wireless/mediatek/mt76/pci.c +++ b/drivers/net/wireless/mediatek/mt76/pci.c @@ -30,8 +30,14 @@ void mt76_pci_disable_aspm(struct pci_dev *pdev) if (IS_ENABLED(CONFIG_PCIEASPM)) { int err; + int state = 0; - err = pci_disable_link_state(pdev, aspm_conf); + if (aspm_conf & PCI_EXP_LNKCTL_ASPM_L0S) + state |= PCIE_LINK_STATE_L0S; + if (aspm_conf & PCI_EXP_LNKCTL_ASPM_L1) + state |= PCIE_LINK_STATE_L1; + + err = pci_disable_link_state(pdev, state); if (!err) return; } diff --git a/drivers/net/wireless/mediatek/mt76/scan.c b/drivers/net/wireless/mediatek/mt76/scan.c index 6cfca5108bc7e..2faa84c516e7b 100644 --- a/drivers/net/wireless/mediatek/mt76/scan.c +++ b/drivers/net/wireless/mediatek/mt76/scan.c @@ -46,6 +46,7 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) struct mt76_phy *phy = dev->scan.phy; struct ieee80211_tx_info *info; struct sk_buff *skb; + u8 link_id; skb = ieee80211_probereq_get(phy->hw, vif->addr, ssid->ssid, ssid->ssid_len, req->ie_len); @@ -75,6 +76,10 @@ mt76_scan_send_probe(struct mt76_dev *dev, struct cfg80211_ssid *ssid) info->flags |= IEEE80211_TX_CTL_NO_CCK_RATE; info->control.flags |= IEEE80211_TX_CTRL_DONT_USE_RATE_MASK; + link_id = mvif->wcid ? mvif->wcid->link_id : IEEE80211_LINK_UNSPECIFIED; + info->control.flags &= ~IEEE80211_TX_CTRL_MLO_LINK; + info->control.flags |= u32_encode_bits(link_id, IEEE80211_TX_CTRL_MLO_LINK); + mt76_tx(phy, NULL, mvif->wcid, skb); out: diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c index 8e9576747052d..e4c3dcc34f68a 100644 --- a/drivers/net/wireless/mediatek/mt76/sdio.c +++ b/drivers/net/wireless/mediatek/mt76/sdio.c @@ -519,6 +519,10 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, enum mt76_txq_id qid, struct sk_buff *skb, struct mt76_wcid *wcid, struct ieee80211_sta *sta) { + struct ieee80211_tx_status status = { + .sta = sta, + }; + struct mt76_tx_info tx_info = { .skb = skb, }; @@ -531,8 +535,13 @@ mt76s_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, skb->prev = skb->next = NULL; err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info); - if (err < 0) + if (err < 0) { + status.skb = tx_info.skb; + spin_lock_bh(&dev->rx_lock); + ieee80211_tx_status_ext(dev->hw, &status); + spin_unlock_bh(&dev->rx_lock); return err; + } q->entry[q->head].skb = tx_info.skb; q->entry[q->head].buf_sz = len; diff --git a/drivers/net/wireless/mediatek/mt76/tx.c b/drivers/net/wireless/mediatek/mt76/tx.c index 9fb0cca5524a4..28630d75bdb21 100644 --- a/drivers/net/wireless/mediatek/mt76/tx.c +++ b/drivers/net/wireless/mediatek/mt76/tx.c @@ -311,10 +311,6 @@ __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, if (idx < 0 || !sta) return idx; - wcid = (struct mt76_wcid *)sta->drv_priv; - if (!wcid->sta) - return idx; - q->entry[idx].wcid = wcid->idx; if (!non_aql) @@ -619,6 +615,7 @@ mt76_txq_schedule_pending_wcid(struct mt76_phy *phy, struct mt76_wcid *wcid, !ieee80211_is_data(hdr->frame_control) && (!ieee80211_is_bufferable_mmpdu(skb) || ieee80211_is_deauth(hdr->frame_control) || + ieee80211_is_disassoc(hdr->frame_control) || head == &wcid->tx_offchannel)) qid = MT_TXQ_PSD; @@ -670,8 +667,8 @@ static void mt76_txq_schedule_pending(struct mt76_phy *phy) ret = mt76_txq_schedule_pending_wcid(phy, wcid, &wcid->tx_pending); spin_lock(&phy->tx_lock); - if (!skb_queue_empty(&wcid->tx_pending) && - !skb_queue_empty(&wcid->tx_offchannel) && + if ((!skb_queue_empty(&wcid->tx_pending) || + !skb_queue_empty(&wcid->tx_offchannel)) && list_empty(&wcid->tx_list)) list_add_tail(&wcid->tx_list, &phy->tx_list); } diff --git a/drivers/net/wireless/microchip/wilc1000/hif.c b/drivers/net/wireless/microchip/wilc1000/hif.c index d795ca05729ab..f08426ca51135 100644 --- a/drivers/net/wireless/microchip/wilc1000/hif.c +++ b/drivers/net/wireless/microchip/wilc1000/hif.c @@ -600,6 +600,11 @@ static s32 wilc_parse_assoc_resp_info(u8 *buffer, u32 buffer_len, u16 ies_len; struct wilc_assoc_resp *res = (struct wilc_assoc_resp *)buffer; + if (buffer_len < sizeof(*res)) { + ret_conn_info->status = WLAN_STATUS_UNSPECIFIED_FAILURE; + return -EINVAL; + } + ret_conn_info->status = le16_to_cpu(res->status_code); if (ret_conn_info->status == WLAN_STATUS_SUCCESS) { ies = &buffer[sizeof(*res)]; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c index 42e21e9f303b2..15073b856adf9 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2400pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2400pci.c @@ -1429,7 +1429,7 @@ static irqreturn_t rt2400pci_interrupt(int irq, void *dev_instance) */ static int rt2400pci_validate_eeprom(struct rt2x00_dev *rt2x00dev) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; u32 reg; u16 word; u8 *mac; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c index 36ddc5a69fa40..3536a0f311171 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2500pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2500pci.c @@ -1555,7 +1555,7 @@ static irqreturn_t rt2500pci_interrupt(int irq, void *dev_instance) */ static int rt2500pci_validate_eeprom(struct rt2x00_dev *rt2x00dev) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; u32 reg; u16 word; u8 *mac; diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c index 14c45aba836f2..853fd31d13629 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt2800pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt2800pci.c @@ -108,7 +108,7 @@ static void rt2800pci_eepromregister_write(struct eeprom_93cx6 *eeprom) static int rt2800pci_read_eeprom_pci(struct rt2x00_dev *rt2x00dev) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; u32 reg; reg = rt2x00mmio_register_read(rt2x00dev, E2PROM_CSR); diff --git a/drivers/net/wireless/ralink/rt2x00/rt61pci.c b/drivers/net/wireless/ralink/rt2x00/rt61pci.c index d1cd5694e3c73..5dca2afb4b96c 100644 --- a/drivers/net/wireless/ralink/rt2x00/rt61pci.c +++ b/drivers/net/wireless/ralink/rt2x00/rt61pci.c @@ -2298,7 +2298,7 @@ static irqreturn_t rt61pci_interrupt(int irq, void *dev_instance) */ static int rt61pci_validate_eeprom(struct rt2x00_dev *rt2x00dev) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; u32 reg; u16 word; u8 *mac; diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c index 070c0431c4821..4a5989172a5e3 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8180/dev.c @@ -1652,7 +1652,7 @@ static void rtl8180_eeprom_register_write(struct eeprom_93cx6 *eeprom) static void rtl8180_eeprom_read(struct rtl8180_priv *priv) { - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; int eeprom_cck_table_adr; u16 eeprom_val; int i; diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c index 7aa2da0cd63cc..98174bb3cf2ae 100644 --- a/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c +++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/dev.c @@ -1445,7 +1445,7 @@ static int rtl8187_probe(struct usb_interface *intf, struct usb_device *udev = interface_to_usbdev(intf); struct ieee80211_hw *dev; struct rtl8187_priv *priv; - struct eeprom_93cx6 eeprom; + struct eeprom_93cx6 eeprom = {}; struct ieee80211_channel *channel; const char *chip_name; u16 txpwr, reg; diff --git a/drivers/net/wireless/realtek/rtl8xxxu/core.c b/drivers/net/wireless/realtek/rtl8xxxu/core.c index 25e5fd48b8e28..f8bd9bfe74ba3 100644 --- a/drivers/net/wireless/realtek/rtl8xxxu/core.c +++ b/drivers/net/wireless/realtek/rtl8xxxu/core.c @@ -5885,14 +5885,19 @@ static void rtl8xxxu_queue_rx_urb(struct rtl8xxxu_priv *priv, { struct sk_buff *skb; unsigned long flags; - int pending = 0; spin_lock_irqsave(&priv->rx_urb_lock, flags); if (!priv->shutdown) { list_add_tail(&rx_urb->list, &priv->rx_urb_pending_list); priv->rx_urb_pending_count++; - pending = priv->rx_urb_pending_count; + /* + * Arm the worker under rx_urb_lock so this is atomic with the + * shutdown check: moving it out of the lock would let a + * completion arm the work after rtl8xxxu_stop() canceled it. + */ + if (priv->rx_urb_pending_count > RTL8XXXU_RX_URB_PENDING_WATER) + schedule_work(&priv->rx_urb_wq); } else { skb = (struct sk_buff *)rx_urb->urb.context; dev_kfree_skb_irq(skb); @@ -5900,9 +5905,6 @@ static void rtl8xxxu_queue_rx_urb(struct rtl8xxxu_priv *priv, } spin_unlock_irqrestore(&priv->rx_urb_lock, flags); - - if (pending > RTL8XXXU_RX_URB_PENDING_WATER) - schedule_work(&priv->rx_urb_wq); } static void rtl8xxxu_rx_urb_work(struct work_struct *work) @@ -7550,6 +7552,13 @@ static void rtl8xxxu_stop(struct ieee80211_hw *hw, bool suspend) priv->shutdown = true; spin_unlock_irqrestore(&priv->rx_urb_lock, flags); + /* + * Cancel before killing rx_anchor: the worker re-anchors every URB + * it drained via rtl8xxxu_submit_rx_urb(), so a worker still running + * after the kill could submit a URB that escapes it. + */ + cancel_work_sync(&priv->rx_urb_wq); + usb_kill_anchored_urbs(&priv->rx_anchor); usb_kill_anchored_urbs(&priv->tx_anchor); if (priv->usb_interrupts) diff --git a/drivers/net/wireless/realtek/rtlwifi/pci.c b/drivers/net/wireless/realtek/rtlwifi/pci.c index f0010336e78c1..8fb9604c3c533 100644 --- a/drivers/net/wireless/realtek/rtlwifi/pci.c +++ b/drivers/net/wireless/realtek/rtlwifi/pci.c @@ -2226,13 +2226,17 @@ int rtl_pci_probe(struct pci_dev *pdev, rtl_dbg(rtlpriv, COMP_INIT, DBG_DMESG, "%s: failed to register IRQ handler\n", wiphy_name(hw->wiphy)); - goto fail3; + goto fail6; } rtlpci->irq_alloc = 1; set_bit(RTL_STATUS_INTERFACE_START, &rtlpriv->status); return 0; +fail6: + rtl_deinit_rfkill(hw); + rtl_debug_remove_one(hw); + ieee80211_unregister_hw(hw); fail5: rtl_pci_deinit(hw); fail4: diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c index cc699efa9c793..a7824ba346e60 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/sw.c @@ -147,8 +147,10 @@ static int rtl92du_init_sw_vars(struct ieee80211_hw *hw) /* for firmware buf */ rtlpriv->rtlhal.pfirmware = kmalloc(0x8000, GFP_KERNEL); - if (!rtlpriv->rtlhal.pfirmware) - return -ENOMEM; + if (!rtlpriv->rtlhal.pfirmware) { + err = -ENOMEM; + goto error; + } rtlpriv->max_fw_size = 0x8000; pr_info("Driver for Realtek RTL8192DU WLAN interface\n"); @@ -162,10 +164,14 @@ static int rtl92du_init_sw_vars(struct ieee80211_hw *hw) pr_err("Failed to request firmware!\n"); kfree(rtlpriv->rtlhal.pfirmware); rtlpriv->rtlhal.pfirmware = NULL; - return err; + goto error; } return 0; + +error: + rtl92du_deinit_shared_data(hw); + return err; } static void rtl92du_deinit_sw_vars(struct ieee80211_hw *hw) diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c index 743ce0cfffe6d..c608c51f1b785 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192du/trx.c @@ -106,7 +106,8 @@ void rtl92du_tx_fill_desc(struct ieee80211_hw *hw, if (sta) { sta_entry = (struct rtl_sta_info *)sta->drv_priv; tid = ieee80211_get_tid(hdr); - agg_state = sta_entry->tids[tid].agg.agg_state; + if (tid < MAX_TID_COUNT) + agg_state = sta_entry->tids[tid].agg.agg_state; ampdu_density = sta->deflink.ht_cap.ampdu_density; } diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c index c68a9fff68082..7ea8390f4d5a8 100644 --- a/drivers/net/wireless/realtek/rtw88/fw.c +++ b/drivers/net/wireless/realtek/rtw88/fw.c @@ -279,6 +279,9 @@ static void rtw_fw_adaptivity_result(struct rtw_dev *rtwdev, u8 *payload, const struct rtw_hw_reg_offset *edcca_th = rtwdev->chip->edcca_th; struct rtw_c2h_adaptivity *result = (struct rtw_c2h_adaptivity *)payload; + if (!edcca_th) + return; + rtw_dbg(rtwdev, RTW_DBG_ADAPTIVITY, "Adaptivity: density %x igi %x l2h_th_init %x l2h %x h2l %x option %x\n", result->density, result->igi, result->l2h_th_init, result->l2h, diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index eae54f50485c2..8bdac965d6ffa 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1799,7 +1799,7 @@ int rtw_pci_probe(struct pci_dev *pdev, ret = rtw_pci_napi_init(rtwdev); if (ret) { rtw_err(rtwdev, "failed to setup NAPI\n"); - goto err_pci_declaim; + goto err_destroy_rsrc; } ret = rtw_chip_info_setup(rtwdev); @@ -1831,6 +1831,8 @@ int rtw_pci_probe(struct pci_dev *pdev, err_destroy_pci: rtw_pci_napi_deinit(rtwdev); + +err_destroy_rsrc: rtw_pci_destroy(rtwdev, pdev); err_pci_declaim: diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c index 9e24741b2cb15..611dade1b1eb9 100644 --- a/drivers/net/wireless/realtek/rtw88/tx.c +++ b/drivers/net/wireless/realtek/rtw88/tx.c @@ -619,6 +619,7 @@ static int rtw_txq_push_skb(struct rtw_dev *rtwdev, ret = rtw_hci_tx_write(rtwdev, &pkt_info, skb); if (ret) { rtw_err(rtwdev, "failed to write TX skb to HCI\n"); + ieee80211_free_txskb(rtwdev->hw, skb); return ret; } return 0; diff --git a/drivers/net/wireless/realtek/rtw89/cam.c b/drivers/net/wireless/realtek/rtw89/cam.c index 385a238fe5cc2..93a0294396351 100644 --- a/drivers/net/wireless/realtek/rtw89/cam.c +++ b/drivers/net/wireless/realtek/rtw89/cam.c @@ -236,7 +236,8 @@ static int __rtw89_cam_detach_sec_cam(struct rtw89_dev *rtwdev, if (ret) rtw89_err(rtwdev, "failed to update dctl cam del key: %d\n", ret); - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_INFO_CHANGE); if (ret) rtw89_err(rtwdev, "failed to update cam del key: %d\n", ret); } @@ -276,7 +277,8 @@ static int __rtw89_cam_attach_sec_cam(struct rtw89_dev *rtwdev, ret); return ret; } - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_INFO_CHANGE); if (ret) { rtw89_err(rtwdev, "failed to update addr cam sec entry: %d\n", ret); @@ -760,7 +762,8 @@ int rtw89_cam_init(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link) int rtw89_cam_fill_bssid_cam_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, - struct rtw89_sta_link *rtwsta_link, u8 *cmd) + struct rtw89_sta_link *rtwsta_link, + struct rtw89_h2c_addr_cam_v0 *h2c) { struct rtw89_bssid_cam_entry *bssid_cam = rtw89_get_bssid_cam_of(rtwvif_link, rtwsta_link); @@ -780,20 +783,19 @@ int rtw89_cam_fill_bssid_cam_info(struct rtw89_dev *rtwdev, rcu_read_unlock(); - FWCMD_SET_ADDR_BSSID_IDX(cmd, bssid_cam->bssid_cam_idx); - FWCMD_SET_ADDR_BSSID_OFFSET(cmd, bssid_cam->offset); - FWCMD_SET_ADDR_BSSID_LEN(cmd, bssid_cam->len); - FWCMD_SET_ADDR_BSSID_VALID(cmd, bssid_cam->valid); - FWCMD_SET_ADDR_BSSID_MASK(cmd, bss_mask); - FWCMD_SET_ADDR_BSSID_BB_SEL(cmd, bssid_cam->phy_idx); - FWCMD_SET_ADDR_BSSID_BSS_COLOR(cmd, bss_color); - - FWCMD_SET_ADDR_BSSID_BSSID0(cmd, bssid_cam->bssid[0]); - FWCMD_SET_ADDR_BSSID_BSSID1(cmd, bssid_cam->bssid[1]); - FWCMD_SET_ADDR_BSSID_BSSID2(cmd, bssid_cam->bssid[2]); - FWCMD_SET_ADDR_BSSID_BSSID3(cmd, bssid_cam->bssid[3]); - FWCMD_SET_ADDR_BSSID_BSSID4(cmd, bssid_cam->bssid[4]); - FWCMD_SET_ADDR_BSSID_BSSID5(cmd, bssid_cam->bssid[5]); + h2c->w12 = le32_encode_bits(bssid_cam->bssid_cam_idx, ADDR_CAM_W12_BSSID_IDX) | + le32_encode_bits(bssid_cam->offset, ADDR_CAM_W12_BSSID_OFFSET) | + le32_encode_bits(bssid_cam->len, ADDR_CAM_W12_BSSID_LEN); + h2c->w13 = le32_encode_bits(bssid_cam->valid, ADDR_CAM_W13_BSSID_VALID) | + le32_encode_bits(bss_mask, ADDR_CAM_W13_BSSID_MASK) | + le32_encode_bits(bssid_cam->phy_idx, ADDR_CAM_W13_BSSID_BB_SEL) | + le32_encode_bits(bss_color, ADDR_CAM_W13_BSSID_BSS_COLOR) | + le32_encode_bits(bssid_cam->bssid[0], ADDR_CAM_W13_BSSID_BSSID0) | + le32_encode_bits(bssid_cam->bssid[1], ADDR_CAM_W13_BSSID_BSSID1); + h2c->w14 = le32_encode_bits(bssid_cam->bssid[2], ADDR_CAM_W14_BSSID_BSSID2) | + le32_encode_bits(bssid_cam->bssid[3], ADDR_CAM_W14_BSSID_BSSID3) | + le32_encode_bits(bssid_cam->bssid[4], ADDR_CAM_W14_BSSID_BSSID4) | + le32_encode_bits(bssid_cam->bssid[5], ADDR_CAM_W14_BSSID_BSSID5); return 0; } @@ -813,7 +815,7 @@ void rtw89_cam_fill_addr_cam_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, struct rtw89_sta_link *rtwsta_link, const u8 *scan_mac_addr, - u8 *cmd) + struct rtw89_h2c_addr_cam_v0 *h2c) { struct ieee80211_vif *vif = rtwvif_link_to_vif(rtwvif_link); struct rtw89_addr_cam_entry *addr_cam = @@ -845,69 +847,65 @@ void rtw89_cam_fill_addr_cam_info(struct rtw89_dev *rtwdev, sma_hash = rtw89_cam_addr_hash(sma_start, sma); tma_hash = rtw89_cam_addr_hash(tma_start, tma); - FWCMD_SET_ADDR_IDX(cmd, addr_cam->addr_cam_idx); - FWCMD_SET_ADDR_OFFSET(cmd, addr_cam->offset); - FWCMD_SET_ADDR_LEN(cmd, addr_cam->len); - - FWCMD_SET_ADDR_VALID(cmd, addr_cam->valid); - FWCMD_SET_ADDR_NET_TYPE(cmd, rtwvif_link->net_type); - FWCMD_SET_ADDR_BCN_HIT_COND(cmd, rtwvif_link->bcn_hit_cond); - FWCMD_SET_ADDR_HIT_RULE(cmd, rtwvif_link->hit_rule); - FWCMD_SET_ADDR_BB_SEL(cmd, rtwvif_link->phy_idx); - FWCMD_SET_ADDR_ADDR_MASK(cmd, addr_cam->addr_mask); - FWCMD_SET_ADDR_MASK_SEL(cmd, addr_cam->mask_sel); - FWCMD_SET_ADDR_SMA_HASH(cmd, sma_hash); - FWCMD_SET_ADDR_TMA_HASH(cmd, tma_hash); - - FWCMD_SET_ADDR_BSSID_CAM_IDX(cmd, addr_cam->bssid_cam_idx); - - FWCMD_SET_ADDR_SMA0(cmd, sma[0]); - FWCMD_SET_ADDR_SMA1(cmd, sma[1]); - FWCMD_SET_ADDR_SMA2(cmd, sma[2]); - FWCMD_SET_ADDR_SMA3(cmd, sma[3]); - FWCMD_SET_ADDR_SMA4(cmd, sma[4]); - FWCMD_SET_ADDR_SMA5(cmd, sma[5]); - - FWCMD_SET_ADDR_TMA0(cmd, tma[0]); - FWCMD_SET_ADDR_TMA1(cmd, tma[1]); - FWCMD_SET_ADDR_TMA2(cmd, tma[2]); - FWCMD_SET_ADDR_TMA3(cmd, tma[3]); - FWCMD_SET_ADDR_TMA4(cmd, tma[4]); - FWCMD_SET_ADDR_TMA5(cmd, tma[5]); - - FWCMD_SET_ADDR_PORT_INT(cmd, rtwvif_link->port); - FWCMD_SET_ADDR_TSF_SYNC(cmd, rtwvif_link->port); - FWCMD_SET_ADDR_TF_TRS(cmd, rtwvif_link->trigger); - FWCMD_SET_ADDR_LSIG_TXOP(cmd, rtwvif_link->lsig_txop); - FWCMD_SET_ADDR_TGT_IND(cmd, rtwvif_link->tgt_ind); - FWCMD_SET_ADDR_FRM_TGT_IND(cmd, rtwvif_link->frm_tgt_ind); - FWCMD_SET_ADDR_MACID(cmd, rtwsta_link ? rtwsta_link->mac_id : - rtwvif_link->mac_id); + h2c->w1 = le32_encode_bits(addr_cam->addr_cam_idx, ADDR_CAM_W1_IDX) | + le32_encode_bits(addr_cam->offset, ADDR_CAM_W1_OFFSET) | + le32_encode_bits(addr_cam->len, ADDR_CAM_W1_LEN); + h2c->w2 = le32_encode_bits(addr_cam->valid, ADDR_CAM_W2_VALID) | + le32_encode_bits(rtwvif_link->net_type, ADDR_CAM_W2_NET_TYPE) | + le32_encode_bits(rtwvif_link->bcn_hit_cond, ADDR_CAM_W2_BCN_HIT_COND) | + le32_encode_bits(rtwvif_link->hit_rule, ADDR_CAM_W2_HIT_RULE) | + le32_encode_bits(rtwvif_link->phy_idx, ADDR_CAM_W2_BB_SEL) | + le32_encode_bits(addr_cam->addr_mask, ADDR_CAM_W2_ADDR_MASK) | + le32_encode_bits(addr_cam->mask_sel, ADDR_CAM_W2_MASK_SEL) | + le32_encode_bits(sma_hash, ADDR_CAM_W2_SMA_HASH) | + le32_encode_bits(tma_hash, ADDR_CAM_W2_TMA_HASH); + h2c->w3 = le32_encode_bits(addr_cam->bssid_cam_idx, ADDR_CAM_W3_BSSID_CAM_IDX); + h2c->w4 = le32_encode_bits(sma[0], ADDR_CAM_W4_SMA0) | + le32_encode_bits(sma[1], ADDR_CAM_W4_SMA1) | + le32_encode_bits(sma[2], ADDR_CAM_W4_SMA2) | + le32_encode_bits(sma[3], ADDR_CAM_W4_SMA3); + h2c->w5 = le32_encode_bits(sma[4], ADDR_CAM_W5_SMA4) | + le32_encode_bits(sma[5], ADDR_CAM_W5_SMA5) | + le32_encode_bits(tma[0], ADDR_CAM_W5_TMA0) | + le32_encode_bits(tma[1], ADDR_CAM_W5_TMA1); + h2c->w6 = le32_encode_bits(tma[2], ADDR_CAM_W6_TMA2) | + le32_encode_bits(tma[3], ADDR_CAM_W6_TMA3) | + le32_encode_bits(tma[4], ADDR_CAM_W6_TMA4) | + le32_encode_bits(tma[5], ADDR_CAM_W6_TMA5); + h2c->w8 = le32_encode_bits(rtwvif_link->port, ADDR_CAM_W8_PORT_INT) | + le32_encode_bits(rtwvif_link->port, ADDR_CAM_W8_TSF_SYNC) | + le32_encode_bits(rtwvif_link->trigger, ADDR_CAM_W8_TF_TRS) | + le32_encode_bits(rtwvif_link->lsig_txop, ADDR_CAM_W8_LSIG_TXOP) | + le32_encode_bits(rtwvif_link->tgt_ind, ADDR_CAM_W8_TGT_IND) | + le32_encode_bits(rtwvif_link->frm_tgt_ind, ADDR_CAM_W8_FRM_TGT_IND) | + le32_encode_bits(rtwsta_link ? rtwsta_link->mac_id : + rtwvif_link->mac_id, ADDR_CAM_W8_MACID); + if (rtwvif_link->net_type == RTW89_NET_TYPE_INFRA) - FWCMD_SET_ADDR_AID12(cmd, vif->cfg.aid & 0xfff); + h2c->w9 = le32_encode_bits(vif->cfg.aid & 0xfff, ADDR_CAM_W9_AID12); else if (rtwvif_link->net_type == RTW89_NET_TYPE_AP_MODE) - FWCMD_SET_ADDR_AID12(cmd, sta ? sta->aid & 0xfff : 0); - FWCMD_SET_ADDR_WOL_PATTERN(cmd, rtwvif_link->wowlan_pattern); - FWCMD_SET_ADDR_WOL_UC(cmd, rtwvif_link->wowlan_uc); - FWCMD_SET_ADDR_WOL_MAGIC(cmd, rtwvif_link->wowlan_magic); - FWCMD_SET_ADDR_WAPI(cmd, addr_cam->wapi); - FWCMD_SET_ADDR_SEC_ENT_MODE(cmd, addr_cam->sec_ent_mode); - FWCMD_SET_ADDR_SEC_ENT0_KEYID(cmd, addr_cam->sec_ent_keyid[0]); - FWCMD_SET_ADDR_SEC_ENT1_KEYID(cmd, addr_cam->sec_ent_keyid[1]); - FWCMD_SET_ADDR_SEC_ENT2_KEYID(cmd, addr_cam->sec_ent_keyid[2]); - FWCMD_SET_ADDR_SEC_ENT3_KEYID(cmd, addr_cam->sec_ent_keyid[3]); - FWCMD_SET_ADDR_SEC_ENT4_KEYID(cmd, addr_cam->sec_ent_keyid[4]); - FWCMD_SET_ADDR_SEC_ENT5_KEYID(cmd, addr_cam->sec_ent_keyid[5]); - FWCMD_SET_ADDR_SEC_ENT6_KEYID(cmd, addr_cam->sec_ent_keyid[6]); - - FWCMD_SET_ADDR_SEC_ENT_VALID(cmd, addr_cam->sec_cam_map[0] & 0xff); - FWCMD_SET_ADDR_SEC_ENT0(cmd, addr_cam->sec_ent[0]); - FWCMD_SET_ADDR_SEC_ENT1(cmd, addr_cam->sec_ent[1]); - FWCMD_SET_ADDR_SEC_ENT2(cmd, addr_cam->sec_ent[2]); - FWCMD_SET_ADDR_SEC_ENT3(cmd, addr_cam->sec_ent[3]); - FWCMD_SET_ADDR_SEC_ENT4(cmd, addr_cam->sec_ent[4]); - FWCMD_SET_ADDR_SEC_ENT5(cmd, addr_cam->sec_ent[5]); - FWCMD_SET_ADDR_SEC_ENT6(cmd, addr_cam->sec_ent[6]); + h2c->w9 = le32_encode_bits(sta ? sta->aid & 0xfff : 0, ADDR_CAM_W9_AID12); + + h2c->w9 |= le32_encode_bits(rtwvif_link->wowlan_pattern, ADDR_CAM_W9_WOL_PATTERN) | + le32_encode_bits(rtwvif_link->wowlan_uc, ADDR_CAM_W9_WOL_UC) | + le32_encode_bits(rtwvif_link->wowlan_magic, ADDR_CAM_W9_WOL_MAGIC) | + le32_encode_bits(addr_cam->wapi, ADDR_CAM_W9_WAPI) | + le32_encode_bits(addr_cam->sec_ent_mode, ADDR_CAM_W9_SEC_ENT_MODE) | + le32_encode_bits(addr_cam->sec_ent_keyid[0], ADDR_CAM_W9_SEC_ENT0_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[1], ADDR_CAM_W9_SEC_ENT1_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[2], ADDR_CAM_W9_SEC_ENT2_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[3], ADDR_CAM_W9_SEC_ENT3_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[4], ADDR_CAM_W9_SEC_ENT4_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[5], ADDR_CAM_W9_SEC_ENT5_KEYID) | + le32_encode_bits(addr_cam->sec_ent_keyid[6], ADDR_CAM_W9_SEC_ENT6_KEYID); + h2c->w10 = le32_encode_bits(addr_cam->sec_cam_map[0] & 0xff, ADDR_CAM_W10_SEC_ENT_VALID) | + le32_encode_bits(addr_cam->sec_ent[0], ADDR_CAM_W10_SEC_ENT0) | + le32_encode_bits(addr_cam->sec_ent[1], ADDR_CAM_W10_SEC_ENT1) | + le32_encode_bits(addr_cam->sec_ent[2], ADDR_CAM_W10_SEC_ENT2); + h2c->w11 = le32_encode_bits(addr_cam->sec_ent[3], ADDR_CAM_W11_SEC_ENT3) | + le32_encode_bits(addr_cam->sec_ent[4], ADDR_CAM_W11_SEC_ENT4) | + le32_encode_bits(addr_cam->sec_ent[5], ADDR_CAM_W11_SEC_ENT5) | + le32_encode_bits(addr_cam->sec_ent[6], ADDR_CAM_W11_SEC_ENT6); rcu_read_unlock(); } diff --git a/drivers/net/wireless/realtek/rtw89/cam.h b/drivers/net/wireless/realtek/rtw89/cam.h index 8fd2d776408ea..4436bacf0348e 100644 --- a/drivers/net/wireless/realtek/rtw89/cam.h +++ b/drivers/net/wireless/realtek/rtw89/cam.h @@ -12,345 +12,98 @@ #define RTW89_BSSID_MATCH_ALL GENMASK(5, 0) #define RTW89_BSSID_MATCH_5_BYTES GENMASK(4, 0) -static inline void FWCMD_SET_ADDR_IDX(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 1, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_OFFSET(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 1, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_LEN(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 1, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_VALID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, BIT(0)); -} - -static inline void FWCMD_SET_ADDR_NET_TYPE(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(2, 1)); -} - -static inline void FWCMD_SET_ADDR_BCN_HIT_COND(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(4, 3)); -} - -static inline void FWCMD_SET_ADDR_HIT_RULE(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(6, 5)); -} - -static inline void FWCMD_SET_ADDR_BB_SEL(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, BIT(7)); -} - -static inline void FWCMD_SET_ADDR_ADDR_MASK(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(13, 8)); -} - -static inline void FWCMD_SET_ADDR_MASK_SEL(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(15, 14)); -} - -static inline void FWCMD_SET_ADDR_SMA_HASH(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_TMA_HASH(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 2, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_BSSID_CAM_IDX(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 3, value, GENMASK(5, 0)); -} - -static inline void FWCMD_SET_ADDR_SMA0(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 4, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_SMA1(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 4, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_SMA2(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 4, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_SMA3(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 4, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_SMA4(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 5, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_SMA5(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 5, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_TMA0(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 5, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_TMA1(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 5, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_TMA2(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 6, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_TMA3(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 6, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_TMA4(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 6, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_TMA5(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 6, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_MACID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_PORT_INT(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, GENMASK(10, 8)); -} - -static inline void FWCMD_SET_ADDR_TSF_SYNC(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, GENMASK(13, 11)); -} - -static inline void FWCMD_SET_ADDR_TF_TRS(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, BIT(14)); -} - -static inline void FWCMD_SET_ADDR_LSIG_TXOP(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, BIT(15)); -} - -static inline void FWCMD_SET_ADDR_TGT_IND(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, GENMASK(26, 24)); -} - -static inline void FWCMD_SET_ADDR_FRM_TGT_IND(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 8, value, GENMASK(29, 27)); -} - -static inline void FWCMD_SET_ADDR_AID12(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(11, 0)); -} - -static inline void FWCMD_SET_ADDR_AID12_0(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_AID12_1(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(11, 8)); -} - -static inline void FWCMD_SET_ADDR_WOL_PATTERN(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, BIT(12)); -} - -static inline void FWCMD_SET_ADDR_WOL_UC(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, BIT(13)); -} - -static inline void FWCMD_SET_ADDR_WOL_MAGIC(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, BIT(14)); -} - -static inline void FWCMD_SET_ADDR_WAPI(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, BIT(15)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT_MODE(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(17, 16)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT0_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(19, 18)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT1_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(21, 20)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT2_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(23, 22)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT3_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(25, 24)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT4_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(27, 26)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT5_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(29, 28)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT6_KEYID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 9, value, GENMASK(31, 30)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT_VALID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 10, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT0(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 10, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT1(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 10, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT2(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 10, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT3(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 11, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT4(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 11, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT5(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 11, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_SEC_ENT6(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 11, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_BSSID_IDX(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 12, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_BSSID_OFFSET(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 12, value, GENMASK(15, 8)); -} - -static inline void FWCMD_SET_ADDR_BSSID_LEN(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 12, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_BSSID_VALID(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, BIT(0)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BB_SEL(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, BIT(1)); -} - -static inline void FWCMD_SET_ADDR_BSSID_MASK(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, GENMASK(7, 2)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BSS_COLOR(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, GENMASK(13, 8)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BSSID0(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, GENMASK(23, 16)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BSSID1(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 13, value, GENMASK(31, 24)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BSSID2(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 14, value, GENMASK(7, 0)); -} - -static inline void FWCMD_SET_ADDR_BSSID_BSSID3(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 14, value, GENMASK(15, 8)); -} +struct rtw89_h2c_addr_cam_v0 { + __le32 w0; + __le32 w1; + __le32 w2; + __le32 w3; + __le32 w4; + __le32 w5; + __le32 w6; + __le32 w7; + __le32 w8; + __le32 w9; + __le32 w10; + __le32 w11; + __le32 w12; + __le32 w13; + __le32 w14; +} __packed; -static inline void FWCMD_SET_ADDR_BSSID_BSSID4(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 14, value, GENMASK(23, 16)); -} +struct rtw89_h2c_addr_cam { + struct rtw89_h2c_addr_cam_v0 v0; + __le32 w15; +} __packed; -static inline void FWCMD_SET_ADDR_BSSID_BSSID5(void *cmd, u32 value) -{ - le32p_replace_bits((__le32 *)(cmd) + 14, value, GENMASK(31, 24)); -} +#define ADDR_CAM_W1_IDX GENMASK(7, 0) +#define ADDR_CAM_W1_OFFSET GENMASK(15, 8) +#define ADDR_CAM_W1_LEN GENMASK(23, 16) +#define ADDR_CAM_W2_VALID BIT(0) +#define ADDR_CAM_W2_NET_TYPE GENMASK(2, 1) +#define ADDR_CAM_W2_BCN_HIT_COND GENMASK(4, 3) +#define ADDR_CAM_W2_HIT_RULE GENMASK(6, 5) +#define ADDR_CAM_W2_BB_SEL BIT(7) +#define ADDR_CAM_W2_ADDR_MASK GENMASK(13, 8) +#define ADDR_CAM_W2_MASK_SEL GENMASK(15, 14) +#define ADDR_CAM_W2_SMA_HASH GENMASK(23, 16) +#define ADDR_CAM_W2_TMA_HASH GENMASK(31, 24) +#define ADDR_CAM_W3_BSSID_CAM_IDX GENMASK(5, 0) +#define ADDR_CAM_W4_SMA0 GENMASK(7, 0) +#define ADDR_CAM_W4_SMA1 GENMASK(15, 8) +#define ADDR_CAM_W4_SMA2 GENMASK(23, 16) +#define ADDR_CAM_W4_SMA3 GENMASK(31, 24) +#define ADDR_CAM_W5_SMA4 GENMASK(7, 0) +#define ADDR_CAM_W5_SMA5 GENMASK(15, 8) +#define ADDR_CAM_W5_TMA0 GENMASK(23, 16) +#define ADDR_CAM_W5_TMA1 GENMASK(31, 24) +#define ADDR_CAM_W6_TMA2 GENMASK(7, 0) +#define ADDR_CAM_W6_TMA3 GENMASK(15, 8) +#define ADDR_CAM_W6_TMA4 GENMASK(23, 16) +#define ADDR_CAM_W6_TMA5 GENMASK(31, 24) +#define ADDR_CAM_W8_MACID GENMASK(7, 0) +#define ADDR_CAM_W8_PORT_INT GENMASK(10, 8) +#define ADDR_CAM_W8_TSF_SYNC GENMASK(13, 11) +#define ADDR_CAM_W8_TF_TRS BIT(14) +#define ADDR_CAM_W8_LSIG_TXOP BIT(15) +#define ADDR_CAM_W8_TGT_IND GENMASK(26, 24) +#define ADDR_CAM_W8_FRM_TGT_IND GENMASK(29, 27) +#define ADDR_CAM_W9_AID12 GENMASK(11, 0) +#define ADDR_CAM_W9_AID12_0 GENMASK(7, 0) +#define ADDR_CAM_W9_AID12_1 GENMASK(11, 8) +#define ADDR_CAM_W9_WOL_PATTERN BIT(12) +#define ADDR_CAM_W9_WOL_UC BIT(13) +#define ADDR_CAM_W9_WOL_MAGIC BIT(14) +#define ADDR_CAM_W9_WAPI BIT(15) +#define ADDR_CAM_W9_SEC_ENT_MODE GENMASK(17, 16) +#define ADDR_CAM_W9_SEC_ENT0_KEYID GENMASK(19, 18) +#define ADDR_CAM_W9_SEC_ENT1_KEYID GENMASK(21, 20) +#define ADDR_CAM_W9_SEC_ENT2_KEYID GENMASK(23, 22) +#define ADDR_CAM_W9_SEC_ENT3_KEYID GENMASK(25, 24) +#define ADDR_CAM_W9_SEC_ENT4_KEYID GENMASK(27, 26) +#define ADDR_CAM_W9_SEC_ENT5_KEYID GENMASK(29, 28) +#define ADDR_CAM_W9_SEC_ENT6_KEYID GENMASK(31, 30) +#define ADDR_CAM_W10_SEC_ENT_VALID GENMASK(7, 0) +#define ADDR_CAM_W10_SEC_ENT0 GENMASK(15, 8) +#define ADDR_CAM_W10_SEC_ENT1 GENMASK(23, 16) +#define ADDR_CAM_W10_SEC_ENT2 GENMASK(31, 24) +#define ADDR_CAM_W11_SEC_ENT3 GENMASK(7, 0) +#define ADDR_CAM_W11_SEC_ENT4 GENMASK(15, 8) +#define ADDR_CAM_W11_SEC_ENT5 GENMASK(23, 16) +#define ADDR_CAM_W11_SEC_ENT6 GENMASK(31, 24) +#define ADDR_CAM_W12_BSSID_IDX GENMASK(7, 0) +#define ADDR_CAM_W12_BSSID_OFFSET GENMASK(15, 8) +#define ADDR_CAM_W12_BSSID_LEN GENMASK(23, 16) +#define ADDR_CAM_W13_BSSID_VALID BIT(0) +#define ADDR_CAM_W13_BSSID_BB_SEL BIT(1) +#define ADDR_CAM_W13_BSSID_MASK GENMASK(7, 2) +#define ADDR_CAM_W13_BSSID_BSS_COLOR GENMASK(13, 8) +#define ADDR_CAM_W13_BSSID_BSSID0 GENMASK(23, 16) +#define ADDR_CAM_W13_BSSID_BSSID1 GENMASK(31, 24) +#define ADDR_CAM_W14_BSSID_BSSID2 GENMASK(7, 0) +#define ADDR_CAM_W14_BSSID_BSSID3 GENMASK(15, 8) +#define ADDR_CAM_W14_BSSID_BSSID4 GENMASK(23, 16) +#define ADDR_CAM_W14_BSSID_BSSID5 GENMASK(31, 24) +#define ADDR_CAM_W15_UPD_MODE GENMASK(2, 0) struct rtw89_h2c_dctlinfo_ud_v1 { __le32 c0; @@ -552,9 +305,10 @@ int rtw89_cam_init_bssid_cam(struct rtw89_dev *rtwdev, void rtw89_cam_deinit_bssid_cam(struct rtw89_dev *rtwdev, struct rtw89_bssid_cam_entry *bssid_cam); void rtw89_cam_fill_addr_cam_info(struct rtw89_dev *rtwdev, - struct rtw89_vif_link *vif, + struct rtw89_vif_link *rtwvif_link, struct rtw89_sta_link *rtwsta_link, - const u8 *scan_mac_addr, u8 *cmd); + const u8 *scan_mac_addr, + struct rtw89_h2c_addr_cam_v0 *h2c); void rtw89_cam_fill_dctl_sec_cam_info_v1(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, struct rtw89_sta_link *rtwsta_link, @@ -565,7 +319,8 @@ void rtw89_cam_fill_dctl_sec_cam_info_v2(struct rtw89_dev *rtwdev, struct rtw89_h2c_dctlinfo_ud_v2 *h2c); int rtw89_cam_fill_bssid_cam_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, - struct rtw89_sta_link *rtwsta_link, u8 *cmd); + struct rtw89_sta_link *rtwsta_link, + struct rtw89_h2c_addr_cam_v0 *h2c); int rtw89_cam_sec_key_add(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif, struct ieee80211_sta *sta, diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c index 018857d3569a8..def596703c371 100644 --- a/drivers/net/wireless/realtek/rtw89/core.c +++ b/drivers/net/wireless/realtek/rtw89/core.c @@ -847,6 +847,7 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev, enum btc_pkt_type pkt_type) { struct rtw89_sta_link *rtwsta_link = tx_req->rtwsta_link; + struct ieee80211_vif *vif = tx_req->vif; struct sk_buff *skb = tx_req->skb; struct ieee80211_hdr *hdr = (void *)skb->data; struct ieee80211_link_sta *link_sta; @@ -878,6 +879,9 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev, if (rtwsta_link && rtwsta_link->ra_report.might_fallback_legacy) return false; + if (vif->type == NL80211_IFTYPE_AP) + return false; + return true; } @@ -2022,13 +2026,15 @@ static int rtw89_core_rx_parse_phy_sts(struct rtw89_dev *rtwdev, const struct rtw89_phy_sts_iehdr *iehdr = pos; ie_len = rtw89_core_get_phy_status_ie_len(rtwdev, iehdr); - rtw89_core_process_phy_status_ie(rtwdev, iehdr, phy_ppdu); pos += ie_len; if (pos > end || ie_len == 0) { rtw89_debug(rtwdev, RTW89_DBG_TXRX, "phy status parse failed\n"); + return -EINVAL; } + + rtw89_core_process_phy_status_ie(rtwdev, iehdr, phy_ppdu); } rtw89_chip_convert_rpl_to_rssi(rtwdev, phy_ppdu); @@ -2499,12 +2505,14 @@ static u16 rtw89_bcn_get_histogram_bound(struct rtw89_dev *rtwdev, u8 target) } static u16 rtw89_bcn_get_rx_time(struct rtw89_dev *rtwdev, - const struct rtw89_chan *chan) + struct rtw89_vif_link *rtwvif_link) { #define RTW89_SYMBOL_TIME_2GHZ 192 #define RTW89_SYMBOL_TIME_5GHZ 20 #define RTW89_SYMBOL_TIME_6GHZ 20 - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat; + const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); + struct rtw89_bb_ctx *bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx); + struct rtw89_pkt_stat *pkt_stat = &bb->cur_pkt_stat; u16 bitrate, val; if (!rtw89_legacy_rate_to_bitrate(rtwdev, pkt_stat->beacon_rate, &bitrate)) @@ -2535,15 +2543,15 @@ static void rtw89_bcn_calc_timeout(struct rtw89_dev *rtwdev, #define RTW89_BCN_TRACK_EXTEND_TIMEOUT 5 #define RTW89_BCN_TRACK_COVERAGE_TH 0 /* unit: TU */ #define RTW89_BCN_TRACK_STRONG_RSSI 80 - const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat; + struct rtw89_bb_ctx *bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx); struct rtw89_beacon_stat *bcn_stat = &rtwdev->phystat.bcn_stat; struct rtw89_beacon_track_info *bcn_track = &rtwdev->bcn_track; + struct rtw89_pkt_stat *pkt_stat = &bb->cur_pkt_stat; struct rtw89_beacon_dist *bcn_dist = &bcn_stat->bcn_dist; u16 outlier_high_bcn_th = bcn_track->outlier_high_bcn_th; u16 outlier_low_bcn_th = bcn_track->outlier_low_bcn_th; - u8 rssi = ewma_rssi_read(&rtwdev->phystat.bcn_rssi); u16 target_bcn_th = bcn_track->target_bcn_th; + u8 rssi = ewma_rssi_read(&bb->bcn_rssi); u16 low_bcn_th = bcn_track->low_bcn_th; u16 med_bcn_th = bcn_track->med_bcn_th; u16 beacon_int = bcn_track->beacon_int; @@ -2589,7 +2597,7 @@ static void rtw89_bcn_calc_timeout(struct rtw89_dev *rtwdev, bcn_timeout = bcn_stat->drift[target_bcn_th]; out: - bcn_track->bcn_timeout = bcn_timeout + rtw89_bcn_get_rx_time(rtwdev, chan); + bcn_track->bcn_timeout = bcn_timeout + rtw89_bcn_get_rx_time(rtwdev, rtwvif_link); } static void rtw89_bcn_update_timeout(struct rtw89_dev *rtwdev, @@ -2733,7 +2741,6 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, struct rtw89_vif_rx_stats_iter_data *iter_data = data; struct rtw89_dev *rtwdev = iter_data->rtwdev; struct rtw89_vif *rtwvif = vif_to_rtwvif(vif); - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat; struct rtw89_rx_desc_info *desc_info = iter_data->desc_info; struct sk_buff *skb = iter_data->skb; struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb); @@ -2743,6 +2750,8 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, struct ieee80211_bss_conf *bss_conf; struct rtw89_vif_link *rtwvif_link; const u8 *bssid = iter_data->bssid; + struct rtw89_pkt_stat *pkt_stat; + struct rtw89_bb_ctx *bb; const u8 *target_bssid; if (rtwdev->scanning && @@ -2776,6 +2785,9 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, rx_status->link_id = rtwvif_link->link_id; } + bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx); + pkt_stat = &bb->cur_pkt_stat; + if (ieee80211_is_beacon(hdr->frame_control)) { if (vif->type == NL80211_IFTYPE_STATION && !test_bit(RTW89_FLAG_WOWLAN, rtwdev->flags)) { @@ -2784,7 +2796,7 @@ static void rtw89_vif_rx_stats_iter(void *data, u8 *mac, } if (phy_ppdu) { - ewma_rssi_add(&rtwdev->phystat.bcn_rssi, phy_ppdu->rssi_avg); + ewma_rssi_add(&bb->bcn_rssi, phy_ppdu->rssi_avg); if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags)) rtwvif_link->bcn_bw_idx = phy_ppdu->bw_idx; } @@ -4041,6 +4053,7 @@ void rtw89_roc_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) reg = rtw89_mac_reg_by_idx(rtwdev, mac->rx_fltr, rtwvif_link->mac_idx); rtw89_write32_clr(rtwdev, reg, B_AX_A_UC_CAM_MATCH | B_AX_A_BC_CAM_MATCH); + rtw89_phy_dig_suspend(rtwdev); ieee80211_ready_on_channel(hw); wiphy_delayed_work_cancel(hw->wiphy, &rtwvif->roc.roc_work); wiphy_delayed_work_queue(hw->wiphy, &rtwvif->roc.roc_work, @@ -4088,6 +4101,7 @@ void rtw89_roc_end(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) rtw89_core_handle_sta_pending_tx(rtwdev, rtwvif_link); queue_work(rtwdev->txq_wq, &rtwdev->txq_work); + rtw89_phy_dig_resume(rtwdev, true); if (hw->conf.flags & IEEE80211_CONF_IDLE) wiphy_delayed_work_queue(hw->wiphy, &roc->roc_work, @@ -4270,14 +4284,22 @@ static void rtw89_core_mlsr_link_decision(struct rtw89_dev *rtwdev, { unsigned int sel_link_id = IEEE80211_MLD_MAX_NUM_LINKS; struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif); + u8 decided_bands = BIT(RTW89_BAND_NUM) - 1; struct rtw89_vif_link *rtwvif_link; const struct rtw89_chan *chan; unsigned long usable_links; + struct rtw89_bb_ctx *bb; unsigned int link_id; - u8 decided_bands; u8 rssi; - rssi = ewma_rssi_read(&rtwdev->phystat.bcn_rssi); + usable_links = ieee80211_vif_usable_links(vif); + + rtwvif_link = rtw89_get_designated_link(rtwvif); + if (unlikely(!rtwvif_link)) + goto select; + + bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx); + rssi = ewma_rssi_read(&bb->bcn_rssi); if (unlikely(!rssi)) return; @@ -4288,12 +4310,6 @@ static void rtw89_core_mlsr_link_decision(struct rtw89_dev *rtwdev, else return; - usable_links = ieee80211_vif_usable_links(vif); - - rtwvif_link = rtw89_get_designated_link(rtwvif); - if (unlikely(!rtwvif_link)) - goto select; - chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); if (decided_bands & BIT(chan->band_type)) return; @@ -4703,7 +4719,8 @@ int rtw89_core_sta_link_disconnect(struct rtw89_dev *rtwdev, } /* update cam aid mac_id net_type */ - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_CON_DISCONN); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; @@ -4777,7 +4794,8 @@ int rtw89_core_sta_link_assoc(struct rtw89_dev *rtwdev, } /* update cam aid mac_id net_type */ - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_CON_DISCONN); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; @@ -5895,7 +5913,8 @@ void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwv rtw89_phy_config_edcca(rtwdev, bb, true); rtw89_tas_scan(rtwdev, true); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, mac_addr); + rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, mac_addr, + RTW89_ROLE_INFO_CHANGE); } void rtw89_core_scan_complete(struct rtw89_dev *rtwdev, @@ -5915,7 +5934,8 @@ void rtw89_core_scan_complete(struct rtw89_dev *rtwdev, rcu_read_unlock(); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL); + rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, + RTW89_ROLE_INFO_CHANGE); rtw89_chip_rfk_scan(rtwdev, rtwvif_link, false); rtw89_btc_ntfy_scan_finish(rtwdev, rtwvif_link->phy_idx); diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index e146fd082f21a..4f2a9ac1ffd63 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -38,6 +38,8 @@ extern const struct ieee80211_ops rtw89_ops; #define RFREG_MASK 0xfffff #define INV_RF_DATA 0xffffffff #define BYPASS_CR_DATA 0xbabecafe +#define RTW89_R32_EA 0xEAEAEAEA +#define RTW89_R32_DEAD 0xDEADBEEF #define RTW89_TRACK_WORK_PERIOD round_jiffies_relative(HZ * 2) #define RTW89_TRACK_PS_WORK_PERIOD msecs_to_jiffies(100) @@ -3652,6 +3654,8 @@ struct rtw89_hci_ops { void (*write16)(struct rtw89_dev *rtwdev, u32 addr, u16 data); void (*write32)(struct rtw89_dev *rtwdev, u32 addr, u32 data); + u32 (*read32_pci_cfg)(struct rtw89_dev *rtwdev, u32 addr); + int (*mac_pre_init)(struct rtw89_dev *rtwdev); int (*mac_pre_deinit)(struct rtw89_dev *rtwdev); int (*mac_post_init)(struct rtw89_dev *rtwdev); @@ -4640,6 +4644,7 @@ enum rtw89_fw_feature { RTW89_FW_FEATURE_RFK_NTFY_MCC_V0, RTW89_FW_FEATURE_LPS_DACK_BY_C2H_REG, RTW89_FW_FEATURE_BEACON_TRACKING, + RTW89_FW_FEATURE_ADDR_CAM_V0, }; struct rtw89_fw_suit { @@ -4700,6 +4705,7 @@ struct rtw89_fw_elm_info { struct rtw89_phy_rfk_log_fmt *rfk_log_fmt; const struct rtw89_regd_data *regd; const struct rtw89_fw_element_hdr *afe; + const struct rtw89_fw_element_hdr *diag_mac; }; enum rtw89_fw_mss_dev_type { @@ -5042,6 +5048,7 @@ enum rtw89_flags { RTW89_FLAG_CHANGING_INTERFACE, RTW89_FLAG_HW_RFKILL_STATE, RTW89_FLAG_UNPLUGGED, + RTW89_FLAG_SHUTDOWN, NUM_OF_RTW89_FLAGS, }; @@ -5129,9 +5136,6 @@ DECLARE_EWMA(thermal, 4, 4); struct rtw89_phy_stat { struct ewma_thermal avg_thermal[RF_PATH_MAX]; u8 last_thermal_max; - struct ewma_rssi bcn_rssi; - struct rtw89_pkt_stat cur_pkt_stat; - struct rtw89_pkt_stat last_pkt_stat; struct rtw89_beacon_stat bcn_stat; }; @@ -6056,6 +6060,9 @@ struct rtw89_dev { struct rtw89_dig_info dig; struct rtw89_phy_ch_info ch_info; struct rtw89_edcca_bak edcca_bak; + struct ewma_rssi bcn_rssi; + struct rtw89_pkt_stat cur_pkt_stat; + struct rtw89_pkt_stat last_pkt_stat; } bbs[RTW89_PHY_NUM]; struct wiphy_delayed_work track_work; @@ -6624,6 +6631,15 @@ rtw89_write_rf(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path, mutex_unlock(&rtwdev->rf_mutex); } +static inline u32 rtw89_read32_pci_cfg(struct rtw89_dev *rtwdev, u32 addr) +{ + if (rtwdev->hci.type != RTW89_HCI_TYPE_PCIE || + !rtwdev->hci.ops->read32_pci_cfg) + return RTW89_R32_EA; + + return rtwdev->hci.ops->read32_pci_cfg(rtwdev, addr); +} + static inline struct ieee80211_txq *rtw89_txq_to_txq(struct rtw89_txq *rtwtxq) { void *p = rtwtxq; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index a82df3814069c..531b8b90cb75e 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -87,6 +87,7 @@ struct rtw89_debugfs { struct rtw89_debugfs_priv disable_dm; struct rtw89_debugfs_priv mlo_mode; struct rtw89_debugfs_priv beacon_info; + struct rtw89_debugfs_priv diag_mac; }; struct rtw89_debugfs_iter_data { @@ -3914,38 +3915,20 @@ static const struct rtw89_rx_rate_cnt_info { {FIRST_RATE_GEV1(EHT_NSS2_MCS0), 14, 0, "EHT 2SS:"}, }; -static ssize_t rtw89_debug_priv_phy_info_get(struct rtw89_dev *rtwdev, - struct rtw89_debugfs_priv *debugfs_priv, - char *buf, size_t bufsz) +static int rtw89_get_rx_pkt_stat(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb, + char *buf, size_t bufsz) { - struct rtw89_traffic_stats *stats = &rtwdev->stats; - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.last_pkt_stat; + struct rtw89_pkt_stat *pkt_stat = &bb->last_pkt_stat; const struct rtw89_chip_info *chip = rtwdev->chip; - struct rtw89_debugfs_iter_data iter_data; const struct rtw89_rx_rate_cnt_info *info; - struct rtw89_hal *hal = &rtwdev->hal; + u8 rssi = ewma_rssi_read(&bb->bcn_rssi); char *p = buf, *end = buf + bufsz; enum rtw89_hw_rate first_rate; - u8 rssi; int i; - rssi = ewma_rssi_read(&rtwdev->phystat.bcn_rssi); - - p += scnprintf(p, end - p, "TP TX: %u [%u] Mbps (lv: %d", - stats->tx_throughput, stats->tx_throughput_raw, - stats->tx_tfc_lv); - if (hal->thermal_prot_lv) - p += scnprintf(p, end - p, ", duty: %d%%", - 100 - hal->thermal_prot_lv * RTW89_THERMAL_PROT_STEP); - p += scnprintf(p, end - p, "), RX: %u [%u] Mbps (lv: %d)\n", - stats->rx_throughput, stats->rx_throughput_raw, - stats->rx_tfc_lv); - p += scnprintf(p, end - p, "Beacon: %u (%d dBm), TF: %u\n", + p += scnprintf(p, end - p, "Beacon: %u (%d dBm)\n", pkt_stat->beacon_nr, - RTW89_RSSI_RAW_TO_DBM(rssi), stats->rx_tf_periodic); - p += scnprintf(p, end - p, "Avg packet length: TX=%u, RX=%u\n", - stats->tx_avg_len, - stats->rx_avg_len); + RTW89_RSSI_RAW_TO_DBM(rssi)); p += scnprintf(p, end - p, "RX count:\n"); @@ -3966,6 +3949,39 @@ static ssize_t rtw89_debug_priv_phy_info_get(struct rtw89_dev *rtwdev, p += scnprintf(p, end - p, "]\n"); } + return p - buf; +} + +static ssize_t rtw89_debug_priv_phy_info_get(struct rtw89_dev *rtwdev, + struct rtw89_debugfs_priv *debugfs_priv, + char *buf, size_t bufsz) +{ + struct rtw89_traffic_stats *stats = &rtwdev->stats; + struct rtw89_debugfs_iter_data iter_data; + struct rtw89_hal *hal = &rtwdev->hal; + char *p = buf, *end = buf + bufsz; + struct rtw89_bb_ctx *bb; + + p += scnprintf(p, end - p, "TP TX: %u [%u] Mbps (lv: %d", + stats->tx_throughput, stats->tx_throughput_raw, + stats->tx_tfc_lv); + if (hal->thermal_prot_lv) + p += scnprintf(p, end - p, ", duty: %d%%", + 100 - hal->thermal_prot_lv * RTW89_THERMAL_PROT_STEP); + p += scnprintf(p, end - p, "), RX: %u [%u] Mbps (lv: %d)\n", + stats->rx_throughput, stats->rx_throughput_raw, + stats->rx_tfc_lv); + p += scnprintf(p, end - p, "Avg packet length: TX=%u, RX=%u\n", + stats->tx_avg_len, + stats->rx_avg_len); + p += scnprintf(p, end - p, "TF: %u\n", stats->rx_tf_periodic); + + rtw89_for_each_active_bb(rtwdev, bb) { + p += scnprintf(p, end - p, "\n[PHY %u]\n", bb->phy_idx); + p += rtw89_get_rx_pkt_stat(rtwdev, bb, p, end - p); + } + p += scnprintf(p, end - p, "\n"); + rtw89_debugfs_iter_data_setup(&iter_data, p, end - p); ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_info_get_iter, &iter_data); p += iter_data.written_sz; @@ -4361,12 +4377,322 @@ rtw89_debug_priv_mlo_mode_set(struct rtw89_dev *rtwdev, return count; } +enum __diag_mac_cmd { + __CMD_EQUALV, + __CMD_EQUALO, + __CMD_NEQUALV, + __CMD_NEQUALO, + __CMD_SETEQUALV, + __CMD_SETEQUALO, + __CMD_CMPWCR, + __CMD_CMPWWD, + __CMD_NEQ_CMPWCR, + __CMD_NEQ_CMPWWD, + __CMD_INCREMENT, + __CMD_MESSAGE, +}; + +enum __diag_mac_io { + __IO_NORMAL, + __IO_NORMAL_PCIE, + __IO_NORMAL_USB, + __IO_NORMAL_SDIO, + __IO_PCIE_CFG, + __IO_SDIO_CCCR, +}; + +struct __diag_mac_rule_header { + u8 sheet; + u8 cmd; + u8 seq_major; + u8 seq_minor; + u8 io_band; + #define __DIAG_MAC_IO GENMASK(3, 0) + #define __DIAG_MAC_N_BAND BIT(4) + #define __DIAG_MAC_HAS_BAND BIT(5) + u8 len; /* include header. Unit: 4 bytes */ + u8 rsvd[2]; +} __packed; + +struct __diag_mac_rule_equal { + struct __diag_mac_rule_header header; + __le32 addr; + __le32 addr_name_offset; + __le32 mask; + __le32 val; + __le32 msg_offset; + u8 rsvd[4]; +} __packed; + +struct __diag_mac_rule_increment { + struct __diag_mac_rule_header header; + __le32 addr; + __le32 addr_name_offset; + __le32 mask; + __le16 sel; + __le16 delay; + __le32 msg_offset; + u8 rsvd[4]; +} __packed; + +struct __diag_mac_msg_buf { + __le16 len; + char string[]; +} __packed; + +static ssize_t rtw89_mac_diag_do_equalv(struct rtw89_dev *rtwdev, + char *buf, size_t bufsz, + const struct __diag_mac_rule_equal *r, + const void *msg_start, + u64 *positive_bmp) +{ + const struct __diag_mac_msg_buf *name = msg_start + + le32_to_cpu(r->addr_name_offset); + const struct __diag_mac_msg_buf *msg = msg_start + + le32_to_cpu(r->msg_offset); + bool want_eq = r->header.cmd == __CMD_EQUALV; + char *p = buf, *end = buf + bufsz; + bool equal = false; + u32 val; + + *positive_bmp <<= 1; + + if (u8_get_bits(r->header.io_band, __DIAG_MAC_IO) == __IO_PCIE_CFG) + val = rtw89_read32_pci_cfg(rtwdev, le32_to_cpu(r->addr)); + else + val = rtw89_read32(rtwdev, le32_to_cpu(r->addr)); + + if ((val & le32_to_cpu(r->mask)) == le32_to_cpu(r->val)) + equal = true; + + if (want_eq == equal) { + *positive_bmp |= BIT(0); + return p - buf; + } + + p += scnprintf(p, end - p, "sheet: %d, cmd: %d, Reg: %.*s => %x, %.*s\n", + r->header.sheet, r->header.cmd, le16_to_cpu(name->len), + name->string, val, le16_to_cpu(msg->len), msg->string); + + return p - buf; +} + +static ssize_t rtw89_mac_diag_do_increment(struct rtw89_dev *rtwdev, + char *buf, size_t bufsz, + const struct __diag_mac_rule_increment *r, + const void *msg_start, + u64 *positive_bmp) +{ + const struct __diag_mac_msg_buf *name = msg_start + + le32_to_cpu(r->addr_name_offset); + const struct __diag_mac_msg_buf *msg = msg_start + + le32_to_cpu(r->msg_offset); + char *p = buf, *end = buf + bufsz; + u32 addr = le32_to_cpu(r->addr); + u32 mask = le32_to_cpu(r->mask); + u16 sel = le16_to_cpu(r->sel); + u32 val1, val2; + + *positive_bmp <<= 1; + + rtw89_write32(rtwdev, addr, sel); + + if (u8_get_bits(r->header.io_band, __DIAG_MAC_IO) == __IO_PCIE_CFG) + val1 = rtw89_read32_pci_cfg(rtwdev, addr); + else + val1 = rtw89_read32(rtwdev, addr); + + mdelay(le16_to_cpu(r->delay)); + + if (u8_get_bits(r->header.io_band, __DIAG_MAC_IO) == __IO_PCIE_CFG) + val2 = rtw89_read32_pci_cfg(rtwdev, addr); + else + val2 = rtw89_read32(rtwdev, addr); + + if ((val2 & mask) > (val1 & mask)) { + *positive_bmp |= BIT(0); + return p - buf; + } + + p += scnprintf(p, end - p, "sheet: %d, cmd: %d, Reg: %.*s [%d]=> %x, %.*s\n", + r->header.sheet, r->header.cmd, le16_to_cpu(name->len), + name->string, le16_to_cpu(r->sel), val1, + le16_to_cpu(msg->len), msg->string); + + return p - buf; +} + +static bool rtw89_mac_diag_match_hci(struct rtw89_dev *rtwdev, + const struct __diag_mac_rule_header *rh) +{ + switch (u8_get_bits(rh->io_band, __DIAG_MAC_IO)) { + case __IO_NORMAL: + default: + return true; + case __IO_NORMAL_PCIE: + case __IO_PCIE_CFG: + if (rtwdev->hci.type == RTW89_HCI_TYPE_PCIE) + return true; + break; + case __IO_NORMAL_USB: + if (rtwdev->hci.type == RTW89_HCI_TYPE_USB) + return true; + break; + case __IO_NORMAL_SDIO: + case __IO_SDIO_CCCR: + if (rtwdev->hci.type == RTW89_HCI_TYPE_SDIO) + return true; + break; + } + + return false; +} + +static bool rtw89_mac_diag_match_band(struct rtw89_dev *rtwdev, + const struct __diag_mac_rule_header *rh) +{ + u8 active_bands; + bool has_band; + u8 band; + + has_band = u8_get_bits(rh->io_band, __DIAG_MAC_HAS_BAND); + if (!has_band) + return true; + + band = u8_get_bits(rh->io_band, __DIAG_MAC_N_BAND); + active_bands = rtw89_get_active_phy_bitmap(rtwdev); + + if (active_bands & BIT(band)) + return true; + + return false; +} + +static ssize_t rtw89_mac_diag_iter_all(struct rtw89_dev *rtwdev, + char *buf, size_t bufsz) +{ + const struct rtw89_fw_element_hdr *elm = rtwdev->fw.elm_info.diag_mac; + u32 n_plains = 0, n_rules = 0, n_positive = 0, n_ignore = 0; + char *p = buf, *end = buf + bufsz, *p_rewind; + const void *rule, *rule_end; + u32 elm_size, rule_size; + const void *msg_start; + u64 positive_bmp = 0; + u8 prev_sheet = 0; + u8 prev_seq = 0; + int limit; + + if (!elm) { + p += scnprintf(p, end - p, "No diag_mac entry\n"); + goto out; + } + + rule_size = le32_to_cpu(elm->u.diag_mac.rule_size); + elm_size = le32_to_cpu(elm->size); + + if (ALIGN(rule_size, 16) > elm_size) { + p += scnprintf(p, end - p, "rule size (%u) exceed elm_size (%u)\n", + ALIGN(rule_size, 16), elm_size); + goto out; + } + + rule = &elm->u.diag_mac.rules_and_msgs[0]; + rule_end = &elm->u.diag_mac.rules_and_msgs[rule_size]; + msg_start = &elm->u.diag_mac.rules_and_msgs[ALIGN(rule_size, 16)]; + + for (limit = 0; limit < 5000 && rule < rule_end; limit++) { + const struct __diag_mac_rule_header *rh = rule; + u8 sheet = rh->sheet; + u8 seq = rh->seq_major; + + if (!rtw89_mac_diag_match_hci(rtwdev, rh) || + !rtw89_mac_diag_match_band(rtwdev, rh)) { + n_ignore++; + goto next; + } + + if (!seq || prev_sheet != sheet || prev_seq != seq) { + if (positive_bmp) { + n_positive++; + /* + * discard output for negative results if one in + * a sequence set is positive. + */ + if (p_rewind) + p = p_rewind; + } + p_rewind = seq ? p : NULL; + positive_bmp = 0; + n_rules++; + } + + switch (rh->cmd) { + case __CMD_EQUALV: + case __CMD_NEQUALV: + p += rtw89_mac_diag_do_equalv(rtwdev, p, end - p, rule, + msg_start, &positive_bmp); + break; + case __CMD_INCREMENT: + p += rtw89_mac_diag_do_increment(rtwdev, p, end - p, rule, + msg_start, &positive_bmp); + break; + default: + p += scnprintf(p, end - p, "unknown rule cmd %u\n", rh->cmd); + break; + } + +next: + n_plains++; + rule += rh->len * 4; + prev_seq = seq; + prev_sheet = sheet; + } + + if (positive_bmp) { + n_positive++; + if (p_rewind) + p = p_rewind; + } + + p += scnprintf(p, end - p, "\nPlain(Ignore)/Rules/Positive: %u(%u)/%u/%u\n", + n_plains, n_ignore, n_rules, n_positive); + +out: + return p - buf; +} + +static ssize_t +rtw89_debug_priv_diag_mac_get(struct rtw89_dev *rtwdev, + struct rtw89_debugfs_priv *debugfs_priv, + char *buf, size_t bufsz) +{ + lockdep_assert_wiphy(rtwdev->hw->wiphy); + + rtw89_leave_lps(rtwdev); + + return rtw89_mac_diag_iter_all(rtwdev, buf, bufsz); +} + +static int rtw89_get_beacon_info(struct rtw89_dev *rtwdev, struct rtw89_bb_ctx *bb, + char *buf, size_t bufsz) +{ + struct rtw89_pkt_stat *pkt_stat = &bb->last_pkt_stat; + char *p = buf, *end = buf + bufsz; + + p += scnprintf(p, end - p, "[PHY %u]\n", bb->phy_idx); + p += scnprintf(p, end - p, "Beacon: %u\n", pkt_stat->beacon_nr); + p += scnprintf(p, end - p, "raw rssi: %lu\n", ewma_rssi_read(&bb->bcn_rssi)); + p += scnprintf(p, end - p, "hw rate: %u\n", pkt_stat->beacon_rate); + p += scnprintf(p, end - p, "length: %u\n\n", pkt_stat->beacon_len); + + return p - buf; +} + static ssize_t rtw89_debug_priv_beacon_info_get(struct rtw89_dev *rtwdev, struct rtw89_debugfs_priv *debugfs_priv, char *buf, size_t bufsz) { - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.last_pkt_stat; struct rtw89_beacon_track_info *bcn_track = &rtwdev->bcn_track; struct rtw89_beacon_stat *bcn_stat = &rtwdev->phystat.bcn_stat; struct rtw89_beacon_dist *bcn_dist = &bcn_stat->bcn_dist; @@ -4374,17 +4700,16 @@ rtw89_debug_priv_beacon_info_get(struct rtw89_dev *rtwdev, char *p = buf, *end = buf + bufsz; u16 *drift = bcn_stat->drift; u8 bcn_num = bcn_stat->num; + struct rtw89_bb_ctx *bb; u8 count; u8 i; + rtw89_for_each_active_bb(rtwdev, bb) + p += rtw89_get_beacon_info(rtwdev, bb, p, end - p); + p += scnprintf(p, end - p, "[Beacon info]\n"); - p += scnprintf(p, end - p, "count: %u\n", pkt_stat->beacon_nr); p += scnprintf(p, end - p, "interval: %u\n", bcn_track->beacon_int); p += scnprintf(p, end - p, "dtim: %u\n", bcn_track->dtim); - p += scnprintf(p, end - p, "raw rssi: %lu\n", - ewma_rssi_read(&rtwdev->phystat.bcn_rssi)); - p += scnprintf(p, end - p, "hw rate: %u\n", pkt_stat->beacon_rate); - p += scnprintf(p, end - p, "length: %u\n", pkt_stat->beacon_len); p += scnprintf(p, end - p, "\n[Distribution]\n"); p += scnprintf(p, end - p, "tbtt\n"); @@ -4478,6 +4803,7 @@ static const struct rtw89_debugfs rtw89_debugfs_templ = { .disable_dm = rtw89_debug_priv_set_and_get(disable_dm, RWLOCK), .mlo_mode = rtw89_debug_priv_set_and_get(mlo_mode, RWLOCK), .beacon_info = rtw89_debug_priv_get(beacon_info), + .diag_mac = rtw89_debug_priv_get(diag_mac, RSIZE_16K, RLOCK), }; #define rtw89_debugfs_add(name, mode, fopname, parent) \ @@ -4524,6 +4850,7 @@ void rtw89_debugfs_add_sec1(struct rtw89_dev *rtwdev, struct dentry *debugfs_top rtw89_debugfs_add_rw(disable_dm); rtw89_debugfs_add_rw(mlo_mode); rtw89_debugfs_add_r(beacon_info); + rtw89_debugfs_add_r(diag_mac); } void rtw89_debugfs_init(struct rtw89_dev *rtwdev) diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c index 15ac357bbdbac..c6533a085c7fe 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.c +++ b/drivers/net/wireless/realtek/rtw89/fw.c @@ -870,6 +870,7 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = { __CFG_FW_FEAT(RTL8922A, ge, 0, 35, 76, 0, LPS_DACK_BY_C2H_REG), __CFG_FW_FEAT(RTL8922A, ge, 0, 35, 79, 0, CRASH_TRIGGER_TYPE_1), __CFG_FW_FEAT(RTL8922A, ge, 0, 35, 80, 0, BEACON_TRACKING), + __CFG_FW_FEAT(RTL8922A, lt, 0, 35, 84, 0, ADDR_CAM_V0), }; static void rtw89_fw_iterate_feature_cfg(struct rtw89_fw_info *fw, @@ -1298,6 +1299,18 @@ int rtw89_build_afe_pwr_seq_from_elm(struct rtw89_dev *rtwdev, return 0; } +static +int rtw89_recognize_diag_mac_from_elm(struct rtw89_dev *rtwdev, + const struct rtw89_fw_element_hdr *elm, + const union rtw89_fw_element_arg arg) +{ + struct rtw89_fw_elm_info *elm_info = &rtwdev->fw.elm_info; + + elm_info->diag_mac = elm; + + return 0; +} + static const struct rtw89_fw_element_handler __fw_element_handlers[] = { [RTW89_FW_ELEMENT_ID_BBMCU0] = {__rtw89_fw_recognize_from_elm, { .fw_type = RTW89_FW_BBMCU0 }, NULL}, @@ -1386,6 +1399,9 @@ static const struct rtw89_fw_element_handler __fw_element_handlers[] = { [RTW89_FW_ELEMENT_ID_AFE_PWR_SEQ] = { rtw89_build_afe_pwr_seq_from_elm, {}, "AFE", }, + [RTW89_FW_ELEMENT_ID_DIAG_MAC] = { + rtw89_recognize_diag_mac_from_elm, {}, NULL, + }, }; int rtw89_fw_recognize_elements(struct rtw89_dev *rtwdev) @@ -2110,28 +2126,48 @@ void rtw89_fw_log_dump(struct rtw89_dev *rtwdev, u8 *buf, u32 len) } -#define H2C_CAM_LEN 60 int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, - struct rtw89_sta_link *rtwsta_link, const u8 *scan_mac_addr) + struct rtw89_sta_link *rtwsta_link, const u8 *scan_mac_addr, + enum rtw89_upd_mode upd_mode) { + const struct rtw89_chip_info *chip = rtwdev->chip; + struct rtw89_h2c_addr_cam_v0 *h2c_v0; + struct rtw89_h2c_addr_cam *h2c; + u32 len = sizeof(*h2c); struct sk_buff *skb; + u8 ver = U8_MAX; int ret; - skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CAM_LEN); + if (RTW89_CHK_FW_FEATURE(ADDR_CAM_V0, &rtwdev->fw) || + chip->chip_gen == RTW89_CHIP_AX) { + len = sizeof(*h2c_v0); + ver = 0; + } + + skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len); if (!skb) { rtw89_err(rtwdev, "failed to alloc skb for fw dl\n"); return -ENOMEM; } - skb_put(skb, H2C_CAM_LEN); - rtw89_cam_fill_addr_cam_info(rtwdev, rtwvif_link, rtwsta_link, scan_mac_addr, - skb->data); - rtw89_cam_fill_bssid_cam_info(rtwdev, rtwvif_link, rtwsta_link, skb->data); + skb_put(skb, len); + h2c_v0 = (struct rtw89_h2c_addr_cam_v0 *)skb->data; + + rtw89_cam_fill_addr_cam_info(rtwdev, rtwvif_link, rtwsta_link, + scan_mac_addr, h2c_v0); + rtw89_cam_fill_bssid_cam_info(rtwdev, rtwvif_link, rtwsta_link, h2c_v0); + if (ver == 0) + goto hdr; + + h2c = (struct rtw89_h2c_addr_cam *)skb->data; + h2c->w15 = le32_encode_bits(upd_mode, ADDR_CAM_W15_UPD_MODE); + +hdr: rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C, H2C_CAT_MAC, H2C_CL_MAC_ADDR_CAM_UPDATE, H2C_FUNC_MAC_ADDR_CAM_UPD, 0, 1, - H2C_CAM_LEN); + len); ret = rtw89_h2c_tx(rtwdev, skb, false); if (ret) { @@ -2972,14 +3008,15 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif) { const struct rtw89_phy_bb_gain_info_be *gain = &rtwdev->bb_gain.be; - struct rtw89_pkt_stat *pkt_stat = &rtwdev->phystat.cur_pkt_stat; static const u8 bcn_bw_ofst[] = {0, 0, 0, 3, 6, 9, 0, 12}; const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_efuse *efuse = &rtwdev->efuse; struct rtw89_h2c_lps_ml_cmn_info *h2c; struct rtw89_vif_link *rtwvif_link; + struct rtw89_pkt_stat *pkt_stat; const struct rtw89_chan *chan; u8 bw_idx = RTW89_BB_BW_20_40; + struct rtw89_bb_ctx *bb; u32 len = sizeof(*h2c); unsigned int link_id; struct sk_buff *skb; @@ -3010,11 +3047,14 @@ int rtw89_fw_h2c_lps_ml_cmn_info(struct rtw89_dev *rtwdev, path = rtwvif_link->phy_idx == RTW89_PHY_1 ? RF_PATH_B : RF_PATH_A; chan = rtw89_chan_get(rtwdev, rtwvif_link->chanctx_idx); gain_band = rtw89_subband_to_gain_band_be(chan->subband_type); + bb = rtw89_get_bb_ctx(rtwdev, rtwvif_link->phy_idx); h2c->central_ch[rtwvif_link->phy_idx] = chan->channel; h2c->pri_ch[rtwvif_link->phy_idx] = chan->primary_channel; h2c->band[rtwvif_link->phy_idx] = chan->band_type; h2c->bw[rtwvif_link->phy_idx] = chan->band_width; + + pkt_stat = &bb->cur_pkt_stat; if (pkt_stat->beacon_rate < RTW89_HW_RATE_OFDM6) h2c->bcn_rate_type[rtwvif_link->phy_idx] = 0x1; else diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h index 47e5cbec306d2..98078a3ef89e6 100644 --- a/drivers/net/wireless/realtek/rtw89/fw.h +++ b/drivers/net/wireless/realtek/rtw89/fw.h @@ -3989,6 +3989,7 @@ enum rtw89_fw_element_id { RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_5GHZ = 25, RTW89_FW_ELEMENT_ID_TXPWR_DA_LMT_RU_6GHZ = 26, RTW89_FW_ELEMENT_ID_AFE_PWR_SEQ = 27, + RTW89_FW_ELEMENT_ID_DIAG_MAC = 28, RTW89_FW_ELEMENT_ID_NUM, }; @@ -4166,6 +4167,11 @@ struct rtw89_fw_element_hdr { __le32 val; } __packed infos[]; } __packed afe; + struct { + __le32 rule_size; + u8 rsvd[4]; + u8 rules_and_msgs[]; + } __packed diag_mac; struct __rtw89_fw_txpwr_element txpwr; struct __rtw89_fw_regd_element regd; } __packed u; @@ -4827,7 +4833,8 @@ int rtw89_fw_h2c_tbtt_tuning(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, u32 offset); int rtw89_fw_h2c_pwr_lvl(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link); int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif_link *vif, - struct rtw89_sta_link *rtwsta_link, const u8 *scan_mac_addr); + struct rtw89_sta_link *rtwsta_link, const u8 *scan_mac_addr, + enum rtw89_upd_mode upd_mode); int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_link, struct rtw89_sta_link *rtwsta_link); diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index 71194ea68bcee..f0773f7c3749a 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -4786,7 +4786,7 @@ int rtw89_mac_vif_init(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif_l if (ret) return ret; - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_CREATE); if (ret) return ret; @@ -4811,7 +4811,7 @@ int rtw89_mac_vif_deinit(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwvif rtw89_cam_deinit(rtwdev, rtwvif_link); - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_REMOVE); if (ret) return ret; @@ -4885,7 +4885,7 @@ static void rtw89_mac_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy, elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data, ies->len); - if (!elem || elem->datalen < 10 || + if (!elem || elem->datalen < 11 || !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT)) *tolerated = false; rcu_read_unlock(); diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 51e37c183a35e..1ac1d0cd79594 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -574,8 +574,6 @@ enum rtw89_mac_bf_rrsc_rate { RTW89_MAC_BF_RRSC_MAX = 32 }; -#define RTW89_R32_EA 0xEAEAEAEA -#define RTW89_R32_DEAD 0xDEADBEEF #define MAC_REG_POOL_COUNT 10 #define ACCESS_CMAC(_addr) \ ({typeof(_addr) __addr = (_addr); \ diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c index 474be7a5e49c7..fd818d4b35314 100644 --- a/drivers/net/wireless/realtek/rtw89/mac80211.c +++ b/drivers/net/wireless/realtek/rtw89/mac80211.c @@ -745,7 +745,7 @@ static void rtw89_ops_link_info_changed(struct ieee80211_hw *hw, if (changed & BSS_CHANGED_BSSID) { ether_addr_copy(rtwvif_link->bssid, conf->bssid); rtw89_cam_bssid_changed(rtwdev, rtwvif_link); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL); + rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_INFO_CHANGE); WRITE_ONCE(rtwvif_link->sync_bcn_tsf, 0); } @@ -800,11 +800,36 @@ static int rtw89_ops_start_ap(struct ieee80211_hw *hw, ether_addr_copy(rtwvif_link->bssid, link_conf->bssid); rtw89_cam_bssid_changed(rtwdev, rtwvif_link); - rtw89_mac_port_update(rtwdev, rtwvif_link); - rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); - rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); - rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); - rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL); + ret = rtw89_mac_port_update(rtwdev, rtwvif_link); + if (ret) { + rtw89_warn(rtwdev, "failed to update mac port\n"); + return ret; + } + + ret = rtw89_chip_h2c_assoc_cmac_tbl(rtwdev, rtwvif_link, NULL); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cmac table\n"); + return ret; + } + + ret = rtw89_fw_h2c_role_maintain(rtwdev, rtwvif_link, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c role info\n"); + return ret; + } + + ret = rtw89_fw_h2c_join_info(rtwdev, rtwvif_link, NULL, true); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c join info\n"); + return ret; + } + + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, NULL, NULL, RTW89_ROLE_TYPE_CHANGE); + if (ret) { + rtw89_warn(rtwdev, "failed to send h2c cam\n"); + return ret; + } + rtw89_chip_rfk_channel(rtwdev, rtwvif_link); if (RTW89_CHK_FW_FEATURE(NOTIFY_AP_INFO, &rtwdev->fw)) { @@ -1879,7 +1904,8 @@ static void rtw89_ops_rfkill_poll(struct ieee80211_hw *hw) lockdep_assert_wiphy(hw->wiphy); /* wl_disable GPIO get floating when entering LPS */ - if (test_bit(RTW89_FLAG_RUNNING, rtwdev->flags)) + if (test_bit(RTW89_FLAG_RUNNING, rtwdev->flags) || + test_bit(RTW89_FLAG_SHUTDOWN, rtwdev->flags)) return; rtw89_core_rfkill_poll(rtwdev, false); diff --git a/drivers/net/wireless/realtek/rtw89/mac_be.c b/drivers/net/wireless/realtek/rtw89/mac_be.c index e5a61c628b731..b03392612f252 100644 --- a/drivers/net/wireless/realtek/rtw89/mac_be.c +++ b/drivers/net/wireless/realtek/rtw89/mac_be.c @@ -2147,6 +2147,10 @@ static int rtw89_mac_set_csi_para_reg_be(struct rtw89_dev *rtwdev, sound_dim = min(sound_dim, t); } + /* Disable stbc_en in VHT 160MHz to avoid IOT issues with certain APs */ + if (link_sta->vht_cap.cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ) + stbc_en = 0; + nc = min(nc, sound_dim); nr = min(nr, sound_dim); diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 17a199ee1d4a4..364278a71e87d 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -2071,6 +2071,20 @@ static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data) writel(data, rtwpci->mmap + addr); } +static u32 rtw89_pci_ops_read32_pci_cfg(struct rtw89_dev *rtwdev, u32 addr) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + u32 value; + int ret; + + ret = pci_read_config_dword(pdev, addr, &value); + if (ret) + return RTW89_R32_EA; + + return value; +} + static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable) { const struct rtw89_pci_info *info = rtwdev->pci_info; @@ -3022,6 +3036,17 @@ static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev) return 0; } +static bool rtw89_pci_dev_ltr_enabled(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + u16 cap; + + pcie_capability_read_word(pdev, PCI_EXP_DEVCTL2, &cap); + + return !!(cap & PCI_EXP_DEVCTL2_LTR_EN); +} + static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; @@ -3126,7 +3151,7 @@ int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en) { u32 val; - if (!en) + if (!en || !rtw89_pci_dev_ltr_enabled(rtwdev)) return 0; val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0); @@ -3162,6 +3187,9 @@ int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en) u32 dec_ctrl; u32 val32; + if (!rtw89_pci_dev_ltr_enabled(rtwdev)) + return 0; + val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0); if (rtw89_pci_ltr_is_err_reg_val(val32)) return -EINVAL; @@ -4691,6 +4719,8 @@ static const struct rtw89_hci_ops rtw89_pci_ops = { .write16 = rtw89_pci_ops_write16, .write32 = rtw89_pci_ops_write32, + .read32_pci_cfg = rtw89_pci_ops_read32_pci_cfg, + .mac_pre_init = rtw89_pci_ops_mac_pre_init, .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit, .mac_post_init = rtw89_pci_ops_mac_post_init, @@ -4829,6 +4859,19 @@ void rtw89_pci_remove(struct pci_dev *pdev) } EXPORT_SYMBOL(rtw89_pci_remove); +void rtw89_pci_shutdown(struct pci_dev *pdev) +{ + struct ieee80211_hw *hw = pci_get_drvdata(pdev); + struct rtw89_dev *rtwdev; + + if (!hw) + return; + + rtwdev = hw->priv; + set_bit(RTW89_FLAG_SHUTDOWN, rtwdev->flags); +} +EXPORT_SYMBOL(rtw89_pci_shutdown); + MODULE_AUTHOR("Realtek Corporation"); MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver"); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h index cb05c83dfd567..09fca0e46cd1b 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.h +++ b/drivers/net/wireless/realtek/rtw89/pci.h @@ -1737,6 +1737,7 @@ struct pci_device_id; int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id); void rtw89_pci_remove(struct pci_dev *pdev); +void rtw89_pci_shutdown(struct pci_dev *pdev); void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume); void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev); int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en); diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index 36dee482bc34b..4424dae1c7aa1 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -5392,6 +5392,7 @@ static void rtw89_phy_stat_rssi_update(struct rtw89_dev *rtwdev) static void rtw89_phy_stat_init(struct rtw89_dev *rtwdev) { struct rtw89_phy_stat *phystat = &rtwdev->phystat; + struct rtw89_bb_ctx *bb; int i; for (i = 0; i < rtwdev->chip->rf_path_num; i++) @@ -5399,24 +5400,28 @@ static void rtw89_phy_stat_init(struct rtw89_dev *rtwdev) rtw89_phy_stat_thermal_update(rtwdev); - memset(&phystat->cur_pkt_stat, 0, sizeof(phystat->cur_pkt_stat)); - memset(&phystat->last_pkt_stat, 0, sizeof(phystat->last_pkt_stat)); + rtw89_for_each_capab_bb(rtwdev, bb) { + memset(&bb->cur_pkt_stat, 0, sizeof(bb->cur_pkt_stat)); + memset(&bb->last_pkt_stat, 0, sizeof(bb->last_pkt_stat)); - ewma_rssi_init(&phystat->bcn_rssi); + ewma_rssi_init(&bb->bcn_rssi); + } rtwdev->hal.thermal_prot_lv = 0; } void rtw89_phy_stat_track(struct rtw89_dev *rtwdev) { - struct rtw89_phy_stat *phystat = &rtwdev->phystat; + struct rtw89_bb_ctx *bb; rtw89_phy_stat_thermal_update(rtwdev); rtw89_phy_thermal_protect(rtwdev); rtw89_phy_stat_rssi_update(rtwdev); - phystat->last_pkt_stat = phystat->cur_pkt_stat; - memset(&phystat->cur_pkt_stat, 0, sizeof(phystat->cur_pkt_stat)); + rtw89_for_each_active_bb(rtwdev, bb) { + bb->last_pkt_stat = bb->cur_pkt_stat; + memset(&bb->cur_pkt_stat, 0, sizeof(bb->cur_pkt_stat)); + } } static u16 rtw89_phy_ccx_us_to_idx(struct rtw89_dev *rtwdev, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851be.c b/drivers/net/wireless/realtek/rtw89/rtw8851be.c index ce59ac9f56ba8..79ea3ccb79572 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8851be.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8851be.c @@ -92,6 +92,7 @@ static struct pci_driver rtw89_8851be_driver = { .id_table = rtw89_8851be_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8851bu.c b/drivers/net/wireless/realtek/rtw89/rtw8851bu.c index 04e1ab13b7535..10fcfade6151e 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8851bu.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8851bu.c @@ -19,6 +19,9 @@ static const struct usb_device_id rtw_8851bu_id_table[] = { /* D-Link AX9U rev. A1 */ { USB_DEVICE_AND_INTERFACE_INFO(0x2001, 0x332a, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8851bu_info }, + /* Mercusys MA60XNB */ + { USB_DEVICE_AND_INTERFACE_INFO(0x2C4E, 0x0128, 0xff, 0xff, 0xff), + .driver_info = (kernel_ulong_t)&rtw89_8851bu_info }, /* TP-Link Archer TX10UB Nano */ { USB_DEVICE_AND_INTERFACE_INFO(0x3625, 0x010b, 0xff, 0xff, 0xff), .driver_info = (kernel_ulong_t)&rtw89_8851bu_info }, diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 232f4c1bee1ba..3defb38223e8b 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -2107,13 +2107,14 @@ static void rtw8852a_query_ppdu(struct rtw89_dev *rtwdev, struct rtw89_rx_phy_ppdu *phy_ppdu, struct ieee80211_rx_status *status) { - u8 path; + struct rtw89_bb_ctx *bb = rtw89_get_bb_ctx(rtwdev, phy_ppdu->phy_idx); u8 *rx_power = phy_ppdu->rssi; + u8 path; u8 raw; if (!status->signal) { - if (phy_ppdu->to_self) - raw = ewma_rssi_read(&rtwdev->phystat.bcn_rssi); + if (phy_ppdu->to_self && ewma_rssi_read(&bb->bcn_rssi)) + raw = ewma_rssi_read(&bb->bcn_rssi); else raw = max(rx_power[RF_PATH_A], rx_power[RF_PATH_B]); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c index 9e05e831569d2..4f1624740dd2b 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c @@ -94,6 +94,7 @@ static struct pci_driver rtw89_8852ae_driver = { .id_table = rtw89_8852ae_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852be.c b/drivers/net/wireless/realtek/rtw89/rtw8852be.c index 12db0d0be5479..72bbaf22893fc 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852be.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852be.c @@ -96,6 +96,7 @@ static struct pci_driver rtw89_8852be_driver = { .id_table = rtw89_8852be_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852bte.c b/drivers/net/wireless/realtek/rtw89/rtw8852bte.c index 8c995aa953251..b10587fa5f324 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852bte.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852bte.c @@ -98,6 +98,7 @@ static struct pci_driver rtw89_8852bte_driver = { .id_table = rtw89_8852bte_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c index 150fed189414d..281f46455de42 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c @@ -121,6 +121,7 @@ static struct pci_driver rtw89_8852ce_driver = { .id_table = rtw89_8852ce_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/rtw8922ae.c b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c index 90c62b757c57b..1ad7d44d03522 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8922ae.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8922ae.c @@ -109,6 +109,7 @@ static struct pci_driver rtw89_8922ae_driver = { .id_table = rtw89_8922ae_id_table, .probe = rtw89_pci_probe, .remove = rtw89_pci_remove, + .shutdown = rtw89_pci_shutdown, .driver.pm = &rtw89_pm_ops_be, .err_handler = &rtw89_pci_err_handler, }; diff --git a/drivers/net/wireless/realtek/rtw89/wow.c b/drivers/net/wireless/realtek/rtw89/wow.c index f34cd863d1009..f6cdbc15dcc4a 100644 --- a/drivers/net/wireless/realtek/rtw89/wow.c +++ b/drivers/net/wireless/realtek/rtw89/wow.c @@ -1225,7 +1225,8 @@ static int rtw89_wow_cfg_wake(struct rtw89_dev *rtwdev, bool wow) } } - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_INFO_CHANGE); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; @@ -1322,7 +1323,8 @@ static int rtw89_wow_swap_fw(struct rtw89_dev *rtwdev, bool wow) return ret; } - ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL); + ret = rtw89_fw_h2c_cam(rtwdev, rtwvif_link, rtwsta_link, NULL, + RTW89_ROLE_FW_RESTORE); if (ret) { rtw89_warn(rtwdev, "failed to send h2c cam\n"); return ret; diff --git a/drivers/net/wireless/rsi/rsi_91x_hal.c b/drivers/net/wireless/rsi/rsi_91x_hal.c index 7d26314a3e76a..50f77f3eee92c 100644 --- a/drivers/net/wireless/rsi/rsi_91x_hal.c +++ b/drivers/net/wireless/rsi/rsi_91x_hal.c @@ -431,6 +431,7 @@ int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb) struct ieee80211_vif *vif; struct sk_buff *mac_bcn; u8 vap_id = 0, i; + unsigned int tailroom; u16 tim_offset = 0; for (i = 0; i < RSI_MAX_VIFS; i++) { @@ -480,6 +481,13 @@ int rsi_prepare_beacon(struct rsi_common *common, struct sk_buff *skb) if (mac_bcn->data[tim_offset + 2] == 0) bcn_frm->frame_info |= cpu_to_le16(RSI_DATA_DESC_DTIM_BEACON); + tailroom = skb_tailroom(skb); + if (tailroom < FRAME_DESC_SZ || + mac_bcn->len > tailroom - FRAME_DESC_SZ) { + dev_kfree_skb(mac_bcn); + return -EMSGSIZE; + } + memcpy(&skb->data[FRAME_DESC_SZ], mac_bcn->data, mac_bcn->len); skb_put(skb, mac_bcn->len + FRAME_DESC_SZ); diff --git a/drivers/net/wireless/rsi/rsi_91x_mgmt.c b/drivers/net/wireless/rsi/rsi_91x_mgmt.c index 7f2c1608f2ce3..2ddf4d158bfe3 100644 --- a/drivers/net/wireless/rsi/rsi_91x_mgmt.c +++ b/drivers/net/wireless/rsi/rsi_91x_mgmt.c @@ -848,8 +848,10 @@ int rsi_hal_load_key(struct rsi_common *common, } else { memcpy(&set_key->key[0][0], data, key_len); } - memcpy(set_key->tx_mic_key, &data[16], 8); - memcpy(set_key->rx_mic_key, &data[24], 8); + if (cipher == WLAN_CIPHER_SUITE_TKIP) { + memcpy(set_key->tx_mic_key, &data[16], 8); + memcpy(set_key->rx_mic_key, &data[24], 8); + } } else { memset(&set_key[FRAME_DESC_SZ], 0, frame_len - FRAME_DESC_SZ); } diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c index 20815fdc9d376..90caa4290dc45 100644 --- a/drivers/net/wireless/virtual/mac80211_hwsim.c +++ b/drivers/net/wireless/virtual/mac80211_hwsim.c @@ -912,9 +912,9 @@ static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = { [HWSIM_ATTR_FLAGS] = { .type = NLA_U32 }, [HWSIM_ATTR_RX_RATE] = { .type = NLA_U32 }, [HWSIM_ATTR_SIGNAL] = { .type = NLA_U32 }, - [HWSIM_ATTR_TX_INFO] = { .type = NLA_BINARY, - .len = IEEE80211_TX_MAX_RATES * - sizeof(struct hwsim_tx_rate)}, + [HWSIM_ATTR_TX_INFO] = + NLA_POLICY_EXACT_LEN(IEEE80211_TX_MAX_RATES * + sizeof(struct hwsim_tx_rate)), [HWSIM_ATTR_COOKIE] = { .type = NLA_U64 }, [HWSIM_ATTR_CHANNELS] = { .type = NLA_U32 }, [HWSIM_ATTR_RADIO_ID] = { .type = NLA_U32 }, @@ -2159,6 +2159,7 @@ static int mac80211_hwsim_start(struct ieee80211_hw *hw) static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend) { struct mac80211_hwsim_data *data = hw->priv; + struct sk_buff *skb; int i; data->started = false; @@ -2166,8 +2167,8 @@ static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend) for (i = 0; i < ARRAY_SIZE(data->link_data); i++) hrtimer_cancel(&data->link_data[i].beacon_timer); - while (!skb_queue_empty(&data->pending)) - ieee80211_free_txskb(hw, skb_dequeue(&data->pending)); + while ((skb = skb_dequeue(&data->pending))) + ieee80211_free_txskb(hw, skb); wiphy_dbg(hw->wiphy, "%s\n", __func__); } @@ -6898,6 +6899,7 @@ static void hwsim_virtio_rx_work(struct work_struct *work) skb->data = skb->head; skb_reset_tail_pointer(skb); + len = min(len, skb_end_offset(skb)); skb_put(skb, len); hwsim_virtio_handle_cmd(skb); diff --git a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c index 8ee15a15f4ca2..ebf4e29c566c8 100644 --- a/drivers/net/wireless/zydas/zd1211rw/zd_usb.c +++ b/drivers/net/wireless/zydas/zd1211rw/zd_usb.c @@ -1353,6 +1353,14 @@ static int probe(struct usb_interface *intf, const struct usb_device_id *id) struct zd_usb *usb; struct ieee80211_hw *hw = NULL; + /* + * ZD1211 devices are single-function. Reject secondary interfaces + * to prevent multiple instances from conflicting on hardcoded endpoints + * and triggering recursive locking warnings. + */ + if (intf->cur_altsetting->desc.bInterfaceNumber != 0) + return -ENODEV; + print_id(udev); if (id->driver_info & DEVICE_INSTALLER) diff --git a/drivers/net/wwan/t7xx/t7xx_pci.c b/drivers/net/wwan/t7xx/t7xx_pci.c index eb137e0784232..46613bba1f411 100644 --- a/drivers/net/wwan/t7xx/t7xx_pci.c +++ b/drivers/net/wwan/t7xx/t7xx_pci.c @@ -447,6 +447,9 @@ static int __t7xx_pci_pm_suspend(struct pci_dev *pdev) goto abort_suspend; } + /* Delay to prevent SAP suspend timeout */ + msleep(50); + ret = t7xx_send_pm_request(t7xx_dev, H2D_CH_SUSPEND_REQ_AP); if (ret) { t7xx_send_pm_request(t7xx_dev, H2D_CH_RESUME_REQ); diff --git a/drivers/nfc/fdp/i2c.c b/drivers/nfc/fdp/i2c.c index c1896a1d978cd..f292e7f374568 100644 --- a/drivers/nfc/fdp/i2c.c +++ b/drivers/nfc/fdp/i2c.c @@ -166,9 +166,36 @@ static int fdp_nci_i2c_read(struct fdp_i2c_phy *phy, struct sk_buff **skb) /* Packet that contains a length */ if (tmp[0] == 0 && tmp[1] == 0) { phy->next_read_size = (tmp[2] << 8) + tmp[3] + 3; + + /* + * next_read_size is taken from the device and is used + * as the i2c_master_recv() count for the next packet + * and as the data skb size. A value above the receive + * buffer overflows tmp[]; one below the minimum frame + * size runs the header/LRC strip and the length-field + * read past a short receive. Either way the packet is + * corrupt: drop it and force resynchronization. + */ + if (phy->next_read_size < FDP_NCI_I2C_MIN_PAYLOAD || + phy->next_read_size > FDP_NCI_I2C_MAX_PAYLOAD) { + dev_dbg(&client->dev, "%s: corrupted packet\n", + __func__); + phy->next_read_size = FDP_NCI_I2C_MIN_PAYLOAD; + goto flush; + } } else { phy->next_read_size = FDP_NCI_I2C_MIN_PAYLOAD; + /* + * Only one data packet is delivered per call; if the + * device sends another, do not overwrite and leak the + * skb allocated for the previous one. + */ + if (*skb) { + kfree_skb(*skb); + *skb = NULL; + } + *skb = alloc_skb(len, GFP_KERNEL); if (*skb == NULL) { r = -ENOMEM; diff --git a/drivers/nfc/microread/microread.c b/drivers/nfc/microread/microread.c index bb4d029bb8885..b0e483ee7eec4 100644 --- a/drivers/nfc/microread/microread.c +++ b/drivers/nfc/microread/microread.c @@ -483,13 +483,19 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, switch (gate) { case MICROREAD_GATE_ID_MREAD_ISO_A: + if (skb->len <= MICROREAD_EMCF_A_LEN) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = nfc_hci_sak_to_protocol(skb->data[MICROREAD_EMCF_A_SAK]); targets->sens_res = be16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_A_ATQA]); targets->sel_res = skb->data[MICROREAD_EMCF_A_SAK]; targets->nfcid1_len = skb->data[MICROREAD_EMCF_A_LEN]; - if (targets->nfcid1_len > sizeof(targets->nfcid1)) { + if (targets->nfcid1_len > sizeof(targets->nfcid1) || + targets->nfcid1_len > skb->len - MICROREAD_EMCF_A_UID) { r = -EINVAL; goto exit_free; } @@ -497,13 +503,19 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len); break; case MICROREAD_GATE_ID_MREAD_ISO_A_3: + if (skb->len <= MICROREAD_EMCF_A3_LEN) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = nfc_hci_sak_to_protocol(skb->data[MICROREAD_EMCF_A3_SAK]); targets->sens_res = be16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_A3_ATQA]); targets->sel_res = skb->data[MICROREAD_EMCF_A3_SAK]; targets->nfcid1_len = skb->data[MICROREAD_EMCF_A3_LEN]; - if (targets->nfcid1_len > sizeof(targets->nfcid1)) { + if (targets->nfcid1_len > sizeof(targets->nfcid1) || + targets->nfcid1_len > skb->len - MICROREAD_EMCF_A3_UID) { r = -EINVAL; goto exit_free; } @@ -511,11 +523,21 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len); break; case MICROREAD_GATE_ID_MREAD_ISO_B: + if (skb->len < MICROREAD_EMCF_B_UID + 4) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_ISO14443_B_MASK; memcpy(targets->nfcid1, &skb->data[MICROREAD_EMCF_B_UID], 4); targets->nfcid1_len = 4; break; case MICROREAD_GATE_ID_MREAD_NFC_T1: + if (skb->len < MICROREAD_EMCF_T1_UID + 4) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_JEWEL_MASK; targets->sens_res = le16_to_cpu(*(u16 *)&skb->data[MICROREAD_EMCF_T1_ATQA]); @@ -523,6 +545,11 @@ static void microread_target_discovered(struct nfc_hci_dev *hdev, u8 gate, targets->nfcid1_len = 4; break; case MICROREAD_GATE_ID_MREAD_NFC_T3: + if (skb->len < MICROREAD_EMCF_T3_UID + 8) { + r = -EINVAL; + goto exit_free; + } + targets->supported_protocols = NFC_PROTO_FELICA_MASK; memcpy(targets->nfcid1, &skb->data[MICROREAD_EMCF_T3_UID], 8); targets->nfcid1_len = 8; diff --git a/drivers/nfc/pn533/pn533.c b/drivers/nfc/pn533/pn533.c index 2b043a9f9533e..9db9e559aeae6 100644 --- a/drivers/nfc/pn533/pn533.c +++ b/drivers/nfc/pn533/pn533.c @@ -434,6 +434,18 @@ static int pn533_send_async_complete(struct pn533 *dev) return rc; } +static int pn533_send_cmd_frame(struct pn533 *dev, struct pn533_cmd *cmd) +{ + struct sk_buff *req = cmd->req; + int rc; + + skb_get(req); + dev->cmd = cmd; + rc = dev->phy_ops->send_frame(dev, req); + dev_kfree_skb(req); + return rc; +} + static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, struct sk_buff *req, pn533_send_async_complete_t complete_cb, @@ -458,8 +470,7 @@ static int __pn533_send_async(struct pn533 *dev, u8 cmd_code, mutex_lock(&dev->cmd_lock); if (!dev->cmd_pending) { - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc) { dev->cmd = NULL; goto error; @@ -529,8 +540,7 @@ static int pn533_send_cmd_direct_async(struct pn533 *dev, u8 cmd_code, pn533_build_cmd_frame(dev, cmd_code, req); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; kfree(cmd); @@ -569,8 +579,7 @@ static void pn533_wq_cmd(struct work_struct *work) mutex_unlock(&dev->cmd_lock); - dev->cmd = cmd; - rc = dev->phy_ops->send_frame(dev, cmd->req); + rc = pn533_send_cmd_frame(dev, cmd); if (rc < 0) { dev->cmd = NULL; dev_kfree_skb(cmd->req); @@ -2799,6 +2808,7 @@ void pn53x_common_clean(struct pn533 *priv) destroy_workqueue(priv->wq); skb_queue_purge(&priv->resp_q); + skb_queue_purge(&priv->fragment_skb); list_for_each_entry_safe(cmd, n, &priv->cmd_queue, queue) { list_del(&cmd->queue); diff --git a/drivers/nfc/st21nfca/dep.c b/drivers/nfc/st21nfca/dep.c index 3425b68f0ddcf..a5fab4fd5129d 100644 --- a/drivers/nfc/st21nfca/dep.c +++ b/drivers/nfc/st21nfca/dep.c @@ -205,6 +205,9 @@ static int st21nfca_tm_recv_atr_req(struct nfc_hci_dev *hdev, if (atr_req->length < sizeof(struct st21nfca_atr_req)) return -EPROTO; + if (atr_req->length > skb->len) + return -EPROTO; + r = st21nfca_tm_send_atr_res(hdev, atr_req); if (r) return r; diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 2cee3c1729c35..aea63fab2ea37 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -1755,9 +1755,16 @@ static void ntb_transport_rxc_db(unsigned long data) static void ntb_tx_copy_callback(void *data, const struct dmaengine_result *res) { + struct ntb_payload_header __iomem *hdr; struct ntb_queue_entry *entry = data; - struct ntb_transport_qp *qp = entry->qp; - struct ntb_payload_header __iomem *hdr = entry->tx_hdr; + struct ntb_transport_qp *qp; + unsigned int len; + void *cb_data; + + qp = entry->qp; + hdr = entry->tx_hdr; + cb_data = entry->cb_data; + len = entry->len; /* we need to check DMA results if we are using DMA */ if (res) { @@ -1797,15 +1804,13 @@ static void ntb_tx_copy_callback(void *data, * "link down" or similar. Since no payload is being sent in these * cases, there is nothing to add to the completion queue. */ - if (entry->len > 0) { - qp->tx_bytes += entry->len; - - if (qp->tx_handler) - qp->tx_handler(qp, qp->cb_data, entry->cb_data, - entry->len); - } + if (len > 0) + qp->tx_bytes += len; ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, &qp->tx_free_q); + + if (len > 0 && qp->tx_handler) + qp->tx_handler(qp, qp->cb_data, cb_data, len); } static void ntb_memcpy_tx(struct ntb_queue_entry *entry, void __iomem *offset) @@ -1930,15 +1935,6 @@ static int ntb_process_tx(struct ntb_transport_qp *qp, return -EAGAIN; } - if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { - if (qp->tx_handler) - qp->tx_handler(qp, qp->cb_data, NULL, -EIO); - - ntb_list_add(&qp->ntb_tx_free_q_lock, &entry->entry, - &qp->tx_free_q); - return 0; - } - ntb_async_tx(qp, entry); qp->tx_index++; @@ -2310,9 +2306,11 @@ int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, if (!qp || !len) return -EINVAL; - /* If the qp link is down already, just ignore. */ if (!qp->link_is_up) - return 0; + return -ENOLINK; + + if (len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) + return -EMSGSIZE; entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); if (!entry) { diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index eed4f15f42578..236fa1a323537 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -885,6 +885,14 @@ static int discover_arenas(struct btt *btt) arena->external_lba_start = cur_nlba; parse_arena_meta(arena, super, cur_off); + if (arena->nfree < btt->nd_region->num_lanes) { + dev_err(to_dev(arena), + "nfree %u smaller than lane count %d\n", + arena->nfree, btt->nd_region->num_lanes); + ret = -ENODEV; + goto out; + } + ret = log_set_indices(arena); if (ret) { dev_err(to_dev(arena), diff --git a/drivers/nvdimm/label.c b/drivers/nvdimm/label.c index 04f4a049599a1..93ff219d91fc5 100644 --- a/drivers/nvdimm/label.c +++ b/drivers/nvdimm/label.c @@ -145,10 +145,21 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) /* label sizes larger than 128 arrived with v1.2 */ version = __le16_to_cpu(nsindex[i]->major) * 100 + __le16_to_cpu(nsindex[i]->minor); - if (version >= 102) + if (version >= 102) { + /* + * labelsize feeds the shift below; only 0 (128-byte) + * and 1 (256-byte) are valid -- a larger value would + * overflow or exceed the width of int. + */ + if (nsindex[i]->labelsize > 1) { + dev_dbg(dev, "nsindex%d labelsize: %d invalid\n", + i, nsindex[i]->labelsize); + continue; + } labelsize = 1 << (7 + nsindex[i]->labelsize); - else + } else { labelsize = 128; + } if (labelsize != sizeof_namespace_label(ndd)) { dev_dbg(dev, "nsindex%d labelsize %d invalid\n", @@ -202,7 +213,7 @@ static int __nd_label_validate(struct nvdimm_drvdata *ndd) } nslot = __le32_to_cpu(nsindex[i]->nslot); - if (nslot * sizeof_namespace_label(ndd) + if ((u64)nslot * sizeof_namespace_label(ndd) + 2 * sizeof_namespace_index(ndd) > ndd->nsarea.config_size) { dev_dbg(dev, "nsindex%d nslot: %u invalid, config_size: %#x\n", diff --git a/drivers/nvdimm/nd_virtio.c b/drivers/nvdimm/nd_virtio.c index af82385be7c6a..942ded0a27be4 100644 --- a/drivers/nvdimm/nd_virtio.c +++ b/drivers/nvdimm/nd_virtio.c @@ -9,26 +9,48 @@ #include "virtio_pmem.h" #include "nd.h" +struct virtio_pmem_flush_work { + struct work_struct work; + struct nd_region *nd_region; + struct bio *bio; +}; + +static void virtio_pmem_req_release(struct kref *kref) +{ + struct virtio_pmem_request *req; + + req = container_of(kref, struct virtio_pmem_request, kref); + kfree(req); +} + +static void virtio_pmem_wake_one_waiter(struct virtio_pmem *vpmem) +{ + struct virtio_pmem_request *req_buf; + + if (list_empty(&vpmem->req_list)) + return; + + req_buf = list_first_entry(&vpmem->req_list, + struct virtio_pmem_request, list); + list_del_init(&req_buf->list); + WRITE_ONCE(req_buf->wq_buf_avail, true); + wake_up(&req_buf->wq_buf); +} + /* The interrupt handler */ void virtio_pmem_host_ack(struct virtqueue *vq) { struct virtio_pmem *vpmem = vq->vdev->priv; - struct virtio_pmem_request *req_data, *req_buf; + struct virtio_pmem_request *req_data; unsigned long flags; unsigned int len; spin_lock_irqsave(&vpmem->pmem_lock, flags); while ((req_data = virtqueue_get_buf(vq, &len)) != NULL) { - req_data->done = true; + virtio_pmem_wake_one_waiter(vpmem); + WRITE_ONCE(req_data->done, true); wake_up(&req_data->host_acked); - - if (!list_empty(&vpmem->req_list)) { - req_buf = list_first_entry(&vpmem->req_list, - struct virtio_pmem_request, list); - req_buf->wq_buf_avail = true; - wake_up(&req_buf->wq_buf); - list_del(&req_buf->list); - } + kref_put(&req_data->kref, virtio_pmem_req_release); } spin_unlock_irqrestore(&vpmem->pmem_lock, flags); } @@ -59,7 +81,8 @@ static int virtio_pmem_flush(struct nd_region *nd_region) if (!req_data) return -ENOMEM; - req_data->done = false; + kref_init(&req_data->kref); + WRITE_ONCE(req_data->done, false); init_waitqueue_head(&req_data->host_acked); init_waitqueue_head(&req_data->wq_buf); INIT_LIST_HEAD(&req_data->list); @@ -76,18 +99,32 @@ static int virtio_pmem_flush(struct nd_region *nd_region) * to req_list and wait for host_ack to wake us up when free * slots are available. */ - while ((err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, - GFP_ATOMIC)) == -ENOSPC) { + for (;;) { + err = virtqueue_add_sgs(vpmem->req_vq, sgs, 1, 1, req_data, + GFP_ATOMIC); + if (!err) { + /* + * Take the virtqueue reference while @pmem_lock is + * held so completion cannot run concurrently. + */ + kref_get(&req_data->kref); + break; + } - dev_info(&vdev->dev, "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); - req_data->wq_buf_avail = false; + if (err != -ENOSPC) + break; + + dev_info_ratelimited(&vdev->dev, + "failed to send command to virtio pmem device, no free slots in the virtqueue\n"); + WRITE_ONCE(req_data->wq_buf_avail, false); list_add_tail(&req_data->list, &vpmem->req_list); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* A host response results in "host_ack" getting called */ - wait_event(req_data->wq_buf, req_data->wq_buf_avail); + wait_event(req_data->wq_buf, READ_ONCE(req_data->wq_buf_avail)); spin_lock_irqsave(&vpmem->pmem_lock, flags); } + err1 = virtqueue_kick(vpmem->req_vq); spin_unlock_irqrestore(&vpmem->pmem_lock, flags); /* @@ -99,38 +136,54 @@ static int virtio_pmem_flush(struct nd_region *nd_region) err = -EIO; } else { /* A host response results in "host_ack" getting called */ - wait_event(req_data->host_acked, req_data->done); + wait_event(req_data->host_acked, READ_ONCE(req_data->done)); err = le32_to_cpu(req_data->resp.ret); } - kfree(req_data); + kref_put(&req_data->kref, virtio_pmem_req_release); return err; }; +static void virtio_pmem_flush_work(struct work_struct *work) +{ + struct virtio_pmem_flush_work *flush; + int err; + + flush = container_of(work, struct virtio_pmem_flush_work, work); + err = virtio_pmem_flush(flush->nd_region); + if (err > 0) + err = -EIO; + if (err) + flush->bio->bi_status = errno_to_blk_status(err); + bio_endio(flush->bio); + kfree(flush); +} + /* The asynchronous flush callback function */ int async_pmem_flush(struct nd_region *nd_region, struct bio *bio) { - /* - * Create child bio for asynchronous flush and chain with - * parent bio. Otherwise directly call nd_region flush. - */ - if (bio && bio->bi_iter.bi_sector != -1) { - struct bio *child = bio_alloc(bio->bi_bdev, 0, - REQ_OP_WRITE | REQ_PREFLUSH, - GFP_ATOMIC); + struct virtio_device *vdev = nd_region->provider_data; + struct virtio_pmem *vpmem = vdev->priv; + struct virtio_pmem_flush_work *flush; + int err; - if (!child) + if (bio && bio->bi_iter.bi_sector != -1) { + flush = kmalloc_obj(*flush, GFP_NOIO); + if (!flush) return -ENOMEM; - bio_clone_blkg_association(child, bio); - child->bi_iter.bi_sector = -1; - bio_chain(child, bio); - submit_bio(child); - return 0; + + INIT_WORK(&flush->work, virtio_pmem_flush_work); + flush->nd_region = nd_region; + flush->bio = bio; + queue_work(vpmem->flush_wq, &flush->work); + return NVDIMM_FLUSH_ASYNC; } - if (virtio_pmem_flush(nd_region)) + + err = virtio_pmem_flush(nd_region); + if (err > 0) return -EIO; - return 0; + return err; }; EXPORT_SYMBOL_GPL(async_pmem_flush); MODULE_DESCRIPTION("Virtio Persistent Memory Driver"); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index 05785ff21a8bd..614701a539ab8 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -208,8 +208,14 @@ static void pmem_submit_bio(struct bio *bio) struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data; struct nd_region *nd_region = to_region(pmem); - if (bio->bi_opf & REQ_PREFLUSH) - ret = nvdimm_flush(nd_region, bio); + if (bio->bi_opf & REQ_PREFLUSH) { + ret = nvdimm_flush(nd_region, NULL); + if (ret) { + bio->bi_status = errno_to_blk_status(ret); + bio_endio(bio); + return; + } + } do_acct = blk_queue_io_stat(bio->bi_bdev->bd_disk->queue); if (do_acct) @@ -229,8 +235,11 @@ static void pmem_submit_bio(struct bio *bio) if (do_acct) bio_end_io_acct(bio, start); - if (bio->bi_opf & REQ_FUA) + if ((bio->bi_opf & REQ_FUA) && !bio->bi_status) { ret = nvdimm_flush(nd_region, bio); + if (ret == NVDIMM_FLUSH_ASYNC) + return; + } if (ret) bio->bi_status = errno_to_blk_status(ret); diff --git a/drivers/nvdimm/region_devs.c b/drivers/nvdimm/region_devs.c index 5a1c68c701cfc..bb0ee554097ba 100644 --- a/drivers/nvdimm/region_devs.c +++ b/drivers/nvdimm/region_devs.c @@ -1094,7 +1094,10 @@ int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) if (!nd_region->flush) rc = generic_nvdimm_flush(nd_region); else { - if (nd_region->flush(nd_region, bio)) + rc = nd_region->flush(nd_region, bio); + if (rc > 0) + return rc; + if (rc && rc != -ENOMEM) rc = -EIO; } diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 77b1966619059..9cf822a6c0c38 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -67,10 +67,17 @@ static int virtio_pmem_probe(struct virtio_device *vdev) mutex_init(&vpmem->flush_lock); vpmem->vdev = vdev; vdev->priv = vpmem; + vpmem->flush_wq = alloc_ordered_workqueue("virtio-pmem-flush", + WQ_MEM_RECLAIM); + if (!vpmem->flush_wq) { + err = -ENOMEM; + goto out_err; + } + err = init_vq(vpmem); if (err) { dev_err(&vdev->dev, "failed to initialize virtio pmem vq's\n"); - goto out_err; + goto out_wq; } if (virtio_has_feature(vdev, VIRTIO_PMEM_F_SHMEM_REGION)) { @@ -131,6 +138,8 @@ static int virtio_pmem_probe(struct virtio_device *vdev) nvdimm_bus_unregister(vpmem->nvdimm_bus); out_vq: vdev->config->del_vqs(vdev); +out_wq: + destroy_workqueue(vpmem->flush_wq); out_err: return err; } @@ -138,14 +147,20 @@ static int virtio_pmem_probe(struct virtio_device *vdev) static void virtio_pmem_remove(struct virtio_device *vdev) { struct nvdimm_bus *nvdimm_bus = dev_get_drvdata(&vdev->dev); + struct virtio_pmem *vpmem = vdev->priv; nvdimm_bus_unregister(nvdimm_bus); + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); + destroy_workqueue(vpmem->flush_wq); } static int virtio_pmem_freeze(struct virtio_device *vdev) { + struct virtio_pmem *vpmem = vdev->priv; + + drain_workqueue(vpmem->flush_wq); vdev->config->del_vqs(vdev); virtio_reset_device(vdev); diff --git a/drivers/nvdimm/virtio_pmem.h b/drivers/nvdimm/virtio_pmem.h index f72cf17f9518f..3af92588bd9d1 100644 --- a/drivers/nvdimm/virtio_pmem.h +++ b/drivers/nvdimm/virtio_pmem.h @@ -12,11 +12,14 @@ #include #include +#include #include #include #include +#include struct virtio_pmem_request { + struct kref kref; struct virtio_pmem_req req; struct virtio_pmem_resp resp; @@ -39,6 +42,9 @@ struct virtio_pmem { /* Serialize flush requests to the device. */ struct mutex flush_lock; + /* Complete asynchronous FUA flushes outside the submit path. */ + struct workqueue_struct *flush_wq; + /* nvdimm bus registers virtio pmem device */ struct nvdimm_bus *nvdimm_bus; struct nvdimm_bus_descriptor nd_desc; diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 9cc614227bc70..e3cd538d950a7 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -47,9 +47,6 @@ #define APPLE_ANS_BOOT_STATUS 0x1300 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55 -#define APPLE_ANS_UNKNOWN_CTRL 0x24008 -#define APPLE_ANS_PRP_NULL_CHECK BIT(11) - #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908 #define APPLE_ANS_LINEAR_SQ_EN BIT(0) @@ -151,6 +148,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -318,13 +332,15 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; - tcb->opcode = cmd->common.opcode; + tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; tcb->length = cmd->rw.length; tcb->command_id = tag; - if (nvme_is_write(cmd)) + if (!cmd->common.dptr.prp1) + tcb->dma_flags = 0; + else if (nvme_is_write(cmd)) tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; @@ -677,7 +693,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -780,7 +796,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -863,7 +879,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -887,8 +903,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1016,8 +1032,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) @@ -1125,17 +1140,6 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); - - /* - * This is probably a chicken bit: without it all commands - * where any PRP is set to zero (including those that don't use - * that field) fail and the co-processor complains about - * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the - * syslog - */ - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & - ~APPLE_ANS_PRP_NULL_CHECK, - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); } /* Setup the admin queue */ @@ -1277,6 +1281,7 @@ static const struct nvme_ctrl_ops nvme_ctrl_ops = { .reg_read64 = apple_nvme_reg_read64, .free_ctrl = apple_nvme_free_ctrl, .get_address = apple_nvme_get_address, + .get_virt_boundary = nvme_get_virt_boundary, }; static void apple_nvme_async_probe(void *data, async_cookie_t cookie) @@ -1580,7 +1585,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, - NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS); + NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto put_dev; @@ -1635,6 +1641,15 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); + if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { + /* + * If the controller was reset during removal, it's possible + * user requests may be waiting on a stopped queue. Start the + * queue to flush these to completion. + */ + nvme_unquiesce_admin_queue(&anv->ctrl); + blk_mq_destroy_queue(anv->ctrl.admin_q); + } nvme_uninit_ctrl(&anv->ctrl); if (apple_rtkit_is_running(anv->rtk)) { diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 24a5ca35aeb25..d5fb6612fa229 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -148,8 +148,6 @@ static const struct class nvme_ns_chr_class = { }; static void nvme_put_subsystem(struct nvme_subsystem *subsys); -static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, - unsigned nsid); static void nvme_update_keep_alive(struct nvme_ctrl *ctrl, struct nvme_command *cmd); static int nvme_get_log_lsi(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, @@ -1882,26 +1880,6 @@ static bool nvme_init_integrity(struct nvme_ns_head *head, return true; } -static void nvme_config_discard(struct nvme_ns *ns, struct queue_limits *lim) -{ - struct nvme_ctrl *ctrl = ns->ctrl; - - if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns->head, UINT_MAX)) - lim->max_hw_discard_sectors = - nvme_lba_to_sect(ns->head, ctrl->dmrsl); - else if (ctrl->oncs & NVME_CTRL_ONCS_DSM) - lim->max_hw_discard_sectors = UINT_MAX; - else - lim->max_hw_discard_sectors = 0; - - lim->discard_granularity = lim->logical_block_size; - - if (ctrl->dmrl) - lim->max_discard_segments = ctrl->dmrl; - else - lim->max_discard_segments = NVME_DSM_MAX_RANGES; -} - static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b) { return uuid_equal(&a->uuid, &b->uuid) && @@ -2069,21 +2047,25 @@ static u32 nvme_max_drv_segments(struct nvme_ctrl *ctrl) } static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, - struct queue_limits *lim) + struct queue_limits *lim, bool is_admin) { lim->max_hw_sectors = ctrl->max_hw_sectors; lim->max_segments = min_t(u32, USHRT_MAX, min_not_zero(nvme_max_drv_segments(ctrl), ctrl->max_segments)); lim->max_integrity_segments = ctrl->max_integrity_segments; - lim->virt_boundary_mask = NVME_CTRL_PAGE_SIZE - 1; + lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; - lim->dma_alignment = 3; + if (is_admin && (ctrl->quirks & NVME_QUIRK_ADMIN_PAGE_ALIGN)) + lim->dma_alignment = NVME_CTRL_PAGE_SIZE - 1; + else + lim->dma_alignment = 3; } static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, struct queue_limits *lim) { struct nvme_ns_head *head = ns->head; + struct nvme_ctrl *ctrl = ns->ctrl; u32 bs = 1U << head->lba_shift; u32 atomic_bs, phys_bs, io_opt = 0; bool valid = true; @@ -2118,11 +2100,26 @@ static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, lim->physical_block_size = min(phys_bs, atomic_bs); lim->io_min = phys_bs; lim->io_opt = io_opt; - if ((ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) && - (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)) + if ((ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES) && + (ctrl->oncs & NVME_CTRL_ONCS_DSM)) lim->max_write_zeroes_sectors = UINT_MAX; else - lim->max_write_zeroes_sectors = ns->ctrl->max_zeroes_sectors; + lim->max_write_zeroes_sectors = ctrl->max_zeroes_sectors; + + if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns->head, UINT_MAX)) + lim->max_hw_discard_sectors = + nvme_lba_to_sect(ns->head, ctrl->dmrsl); + else if (ctrl->oncs & NVME_CTRL_ONCS_DSM) + lim->max_hw_discard_sectors = UINT_MAX; + else + lim->max_hw_discard_sectors = 0; + + lim->discard_granularity = lim->logical_block_size; + + if (ctrl->dmrl) + lim->max_discard_segments = ctrl->dmrl; + else + lim->max_discard_segments = NVME_DSM_MAX_RANGES; return valid; } @@ -2177,7 +2174,7 @@ static int nvme_update_ns_info_generic(struct nvme_ns *ns, int ret; lim = queue_limits_start_update(ns->disk->queue); - nvme_set_ctrl_limits(ns->ctrl, &lim); + nvme_set_ctrl_limits(ns->ctrl, &lim, false); memflags = blk_mq_freeze_queue(ns->disk->queue); ret = queue_limits_commit_update(ns->disk->queue, &lim); @@ -2241,14 +2238,16 @@ static int nvme_query_fdp_granularity(struct nvme_ctrl *ctrl, desc = log; end = log + size - sizeof(*h); for (i = 0; i < fdp_idx; i++) { - log += le16_to_cpu(desc->dsze); - desc = log; - if (log >= end) { + u16 dsze = le16_to_cpu(desc->dsze); + + if (!dsze || log + dsze > end) { dev_warn(ctrl->device, - "FDP invalid config descriptor list\n"); + "FDP invalid config descriptor at index %d\n", i); ret = 0; goto out; } + log += dsze; + desc = log; } if (le32_to_cpu(desc->nrg) > 1) { @@ -2381,16 +2380,32 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, ns->head->lba_shift = id->lbaf[lbaf].ds; ns->head->nuse = le64_to_cpu(id->nuse); capacity = nvme_lba_to_sect(ns->head, le64_to_cpu(id->nsze)); - nvme_set_ctrl_limits(ns->ctrl, &lim); + nvme_set_ctrl_limits(ns->ctrl, &lim, false); nvme_configure_metadata(ns->ctrl, ns->head, id, nvm, info); nvme_set_chunk_sectors(ns, id, &lim); if (!nvme_update_disk_info(ns, id, &lim)) capacity = 0; - nvme_config_discard(ns, &lim); + /* + * A failed zone info query leaves zi zero-initialized, so skip the + * zoned limits update instead of configuring the queue from it. + * During a revalidation that keeps the zone geometry the queue was + * last validated with; on a first scan the namespace is registered + * without zoned limits, so that it is still available as a handle + * for admin commands. + */ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) && - ns->head->ids.csi == NVME_CSI_ZNS) - nvme_update_zone_info(ns, &lim, &zi); + ns->head->ids.csi == NVME_CSI_ZNS) { + if (zi.zone_size) + nvme_update_zone_info(ns, &lim, &zi); + else + dev_warn(ns->ctrl->device, + "zone info query failed for nsid %u, %s\n", + ns->head->ns_id, + blk_queue_is_zoned(ns->disk->queue) ? + "keeping the previous zone limits" : + "not enabling zoned mode"); + } if ((ns->ctrl->vwc & NVME_CTRL_VWC_PRESENT) && !info->no_vwc) lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; @@ -2450,6 +2465,14 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, return ret; } +static void nvme_stack_zone_resources(struct queue_limits *t, + const struct queue_limits *b) +{ + t->max_open_zones = min_not_zero(t->max_open_zones, b->max_open_zones); + t->max_active_zones = + min_not_zero(t->max_active_zones, b->max_active_zones); +} + static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info) { bool unsupported = false; @@ -2516,6 +2539,8 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info) lim.io_opt = ns_lim->io_opt; queue_limits_stack_bdev(&lim, ns->disk->part0, 0, ns->head->disk->disk_name); + if (lim.features & BLK_FEAT_ZONED) + nvme_stack_zone_resources(&lim, ns_lim); if (unsupported) ns->head->disk->flags |= GENHD_FL_HIDDEN; else @@ -3589,7 +3614,7 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl) min_not_zero(ctrl->max_hw_sectors, max_hw_sectors); lim = queue_limits_start_update(ctrl->admin_q); - nvme_set_ctrl_limits(ctrl, &lim); + nvme_set_ctrl_limits(ctrl, &lim, true); ret = queue_limits_commit_update(ctrl->admin_q, &lim); if (ret) goto out_free; @@ -4218,6 +4243,9 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) last_path = true; } mutex_unlock(&ctrl->subsys->lock); + + /* guarantee not available in head->list */ + synchronize_srcu(&ns->head->srcu); if (last_path) nvme_put_ns_head(ns->head); nvme_put_ns_head(ns->head); @@ -4384,15 +4412,16 @@ static void nvme_scan_ns_async(void *data, async_cookie_t cookie) nvme_scan_ns(scan_info->ctrl, nsid); } -static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, - unsigned nsid) +static void nvme_remove_nsid_range(struct nvme_ctrl *ctrl, u32 start, u32 end) { struct nvme_ns *ns, *next; LIST_HEAD(rm_list); mutex_lock(&ctrl->namespaces_lock); list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) { - if (ns->head->ns_id > nsid) { + if (ns->head->ns_id >= end) + break; + if (ns->head->ns_id > start) { list_del_rcu(&ns->list); synchronize_srcu(&ctrl->srcu); list_add_tail_rcu(&ns->list, &rm_list); @@ -4442,13 +4471,14 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) goto out; async_schedule_domain(nvme_scan_ns_async, &scan_info, &domain); - while (++prev < nsid) - nvme_ns_remove_by_nsid(ctrl, prev); + if (prev + 1 < nsid) + nvme_remove_nsid_range(ctrl, prev, nsid); + prev = max(prev + 1, nsid); } async_synchronize_full_domain(&domain); } out: - nvme_remove_invalid_namespaces(ctrl, prev); + nvme_remove_nsid_range(ctrl, prev, UINT_MAX); free: async_synchronize_full_domain(&domain); kfree(ns_list); @@ -4468,7 +4498,7 @@ static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl) for (i = 1; i <= nn; i++) nvme_scan_ns(ctrl, i); - nvme_remove_invalid_namespaces(ctrl, nn); + nvme_remove_nsid_range(ctrl, nn, UINT_MAX); } static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl) @@ -4908,10 +4938,8 @@ void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl) */ nvme_stop_keep_alive(ctrl); blk_mq_destroy_queue(ctrl->admin_q); - if (ctrl->ops->flags & NVME_F_FABRICS) { + if (ctrl->fabrics_q) blk_mq_destroy_queue(ctrl->fabrics_q); - blk_put_queue(ctrl->fabrics_q); - } blk_mq_free_tag_set(ctrl->admin_tagset); } EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set); @@ -5053,6 +5081,8 @@ static void nvme_free_ctrl(struct device *dev) if (ctrl->admin_q) blk_put_queue(ctrl->admin_q); + if (ctrl->fabrics_q) + blk_put_queue(ctrl->fabrics_q); if (!subsys || ctrl->instance != subsys->instance) ida_free(&nvme_instance_ida, ctrl->instance); nvme_free_cels(ctrl); @@ -5120,7 +5150,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > PAGE_SIZE); - ctrl->discard_page = alloc_page(GFP_KERNEL); + ctrl->discard_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!ctrl->discard_page) { ret = -ENOMEM; goto out; diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index d37cb140d8323..b13b455ee8299 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -1028,6 +1028,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) { pr_err("Invalid DH-CHAP secret %s\n", p); + kfree_sensitive(p); ret = -EINVAL; goto out; } @@ -1042,6 +1043,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts, } if (strlen(p) < 11 || strncmp(p, "DHHC-1:", 7)) { pr_err("Invalid DH-CHAP secret %s\n", p); + kfree_sensitive(p); ret = -EINVAL; goto out; } diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index 1b58ee7d0dcee..caf5503d08332 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -217,6 +217,12 @@ static inline unsigned int nvmf_nr_io_queues(struct nvmf_ctrl_options *opts) min(opts->nr_poll_queues, num_online_cpus()); } +static inline unsigned long nvmf_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + return 0; +} + int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val); int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val); int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index bf78faf1a4ffa..fc7aa3fc6c2b5 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2098,9 +2098,15 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, dev_err(ctrl->dev, "FCP Op failed - rspiu dma mapping failed.\n"); ret = -EFAULT; + goto out_unmap; } atomic_set(&op->state, FCPOP_STATE_IDLE); + return 0; + +out_unmap: + fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, + sizeof(op->cmd_iu), DMA_TO_DEVICE); out_on_error: return ret; } @@ -2316,7 +2322,7 @@ nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) return 0; delete_queues: - for (; i > 0; i--) + for (--i; i > 0; i--) __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); return ret; } @@ -2356,9 +2362,15 @@ nvme_fc_ctrl_free(struct kref *ref) struct nvme_fc_ctrl *ctrl = container_of(ref, struct nvme_fc_ctrl, ref); unsigned long flags; + bool owns_opts; - /* remove from rport list */ + /* + * Presence on the rport list means nvme_fc_init_ctrl() completed, + * and with it ownership of the fabrics options passed to it. If it + * failed instead, the options still belong to nvmf_create_ctrl(). + */ spin_lock_irqsave(&ctrl->rport->lock, flags); + owns_opts = !list_empty(&ctrl->ctrl_list); list_del(&ctrl->ctrl_list); spin_unlock_irqrestore(&ctrl->rport->lock, flags); @@ -2368,7 +2380,7 @@ nvme_fc_ctrl_free(struct kref *ref) nvme_fc_rport_put(ctrl->rport); ida_free(&nvme_fc_ctrl_cnt, ctrl->cnum); - if (ctrl->ctrl.opts) + if (owns_opts) nvmf_free_options(ctrl->ctrl.opts); kfree(ctrl); } @@ -2459,7 +2471,7 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues) * io requests back to the block layer as part of normal completions * (but with error status). */ - if (ctrl->ctrl.queue_count > 1) { + if (ctrl->ctrl.queue_count > 1 && ctrl->ctrl.tagset) { nvme_quiesce_io_queues(&ctrl->ctrl); nvme_sync_io_queues(&ctrl->ctrl); blk_mq_tagset_busy_iter(&ctrl->tag_set, @@ -2898,6 +2910,11 @@ nvme_fc_create_io_queues(struct nvme_fc_ctrl *ctrl) out_delete_hw_queues: nvme_fc_delete_hw_io_queues(ctrl); out_cleanup_tagset: + /* + * In CONNECTING state ctrl->ioerr_work will abort both admin + * and io tagsets. Cancel it first before removing io tagset. + */ + cancel_work_sync(&ctrl->ioerr_work); nvme_remove_io_tag_set(&ctrl->ctrl); nvme_fc_free_io_queues(ctrl); @@ -3363,6 +3380,7 @@ static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = { .submit_async_event = nvme_fc_submit_async_event, .delete_ctrl = nvme_fc_delete_ctrl, .get_address = nvmf_get_address, + .get_virt_boundary = nvmf_get_virt_boundary, }; static void @@ -3558,14 +3576,14 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { dev_err(ctrl->ctrl.device, "NVME-FC{%d}: failed to init ctrl state\n", ctrl->cnum); - goto fail_ctrl; + goto fail_unlist; } if (!queue_delayed_work(nvme_wq, &ctrl->connect_work, 0)) { dev_err(ctrl->ctrl.device, "NVME-FC{%d}: failed to schedule initial connect\n", ctrl->cnum); - goto fail_ctrl; + goto fail_unlist; } flush_delayed_work(&ctrl->connect_work); @@ -3576,14 +3594,22 @@ nvme_fc_init_ctrl(struct device *dev, struct nvmf_ctrl_options *opts, return &ctrl->ctrl; +fail_unlist: + /* + * Leaving the list hands the options back to nvmf_create_ctrl(); + * see nvme_fc_ctrl_free(). Re-init so that list_empty() there + * reports the controller as unlisted. + */ + spin_lock_irqsave(&rport->lock, flags); + list_del_init(&ctrl->ctrl_list); + spin_unlock_irqrestore(&rport->lock, flags); + fail_ctrl: nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING); cancel_work_sync(&ctrl->ioerr_work); cancel_work_sync(&ctrl->ctrl.reset_work); cancel_delayed_work_sync(&ctrl->connect_work); - ctrl->ctrl.opts = NULL; - if (ctrl->ctrl.admin_tagset) nvme_remove_admin_tag_set(&ctrl->ctrl); /* initiate nvme ctrl ref counting teardown */ diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 5bbaf257fd6c5..fb8533127116a 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -14,45 +14,54 @@ enum { NVME_IOCTL_PARTITION = (1 << 1), }; -static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, - unsigned int flags, bool open_for_write) +static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, + struct nvme_command *c) { - u32 effects; - - /* - * Do not allow unprivileged passthrough on partitions, as that allows an - * escape from the containment of the partition. - */ - if (flags & NVME_IOCTL_PARTITION) - goto admin; - - /* - * Do not allow unprivileged processes to send vendor specific or fabrics - * commands as we can't be sure about their effects. - */ - if (c->common.opcode >= nvme_cmd_vendor_start || - c->common.opcode == nvme_fabrics_command) - goto admin; - /* * Do not allow unprivileged passthrough of admin commands except * for a subset of identify commands that contain information required * to form proper I/O commands in userspace and do not expose any * potentially sensitive information. */ - if (!ns) { - if (c->common.opcode == nvme_admin_identify) { - switch (c->identify.cns) { - case NVME_ID_CNS_NS: - case NVME_ID_CNS_CS_NS: - case NVME_ID_CNS_NS_CS_INDEP: - case NVME_ID_CNS_CS_CTRL: - case NVME_ID_CNS_CTRL: - return true; - } + switch (c->common.opcode) { + case nvme_admin_identify: + switch (c->identify.cns) { + case NVME_ID_CNS_NS: + case NVME_ID_CNS_CS_NS: + case NVME_ID_CNS_NS_CS_INDEP: + case NVME_ID_CNS_CS_CTRL: + case NVME_ID_CNS_CTRL: + return true; + } + break; + case nvme_admin_set_features: + /* + * Reject Set Features that change controller state the driver + * manages itself; setting them behind the driver's back from + * userspace leaves it unable to react correctly. Keep Alive is + * only armed for fabrics - on other transports it has no + * reserved tag and harms idle power states. + */ + switch (le32_to_cpu(c->features.fid) & 0xff) { + case NVME_FEAT_KATO: + if (ctrl->ops->flags & NVME_F_FABRICS) + break; + fallthrough; + case NVME_FEAT_HOST_BEHAVIOR: + case NVME_FEAT_HOST_MEM_BUF: + case NVME_FEAT_NUM_QUEUES: + case NVME_FEAT_AUTO_PST: + return false; } - goto admin; + break; } + return capable(CAP_SYS_ADMIN); +} + +static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, + bool open_for_write) +{ + u32 effects; /* * Check if the controller provides a Commands Supported and Effects log @@ -61,7 +70,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); if (!(effects & NVME_CMD_EFFECTS_CSUPP)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Don't allow passthrough for command that have intrusive (or unknown) @@ -70,7 +79,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_UUID_SEL | NVME_CMD_EFFECTS_SCOPE_MASK)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Only allow I/O commands that transfer data to the controller or that @@ -79,11 +88,34 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && !open_for_write) - goto admin; + return capable(CAP_SYS_ADMIN); return true; -admin: - return capable(CAP_SYS_ADMIN); +} + +static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + struct nvme_command *c, unsigned int flags, + bool open_for_write) +{ + /* + * Do not allow unprivileged passthrough on partitions, as that + * allows an escape from the containment of the partition. + */ + if (flags & NVME_IOCTL_PARTITION) + return capable(CAP_SYS_ADMIN); + + /* + * Do not allow unprivileged processes to send vendor specific or + * fabrics commands as we can't be sure about their effects. + */ + if (c->common.opcode >= nvme_cmd_vendor_start || + c->common.opcode == nvme_fabrics_command) + return capable(CAP_SYS_ADMIN); + + if (!ns) + return nvme_admin_cmd_allowed(ctrl, c); + + return nvme_ns_cmd_allowed(ns, c, open_for_write); } /* @@ -202,7 +234,8 @@ static int nvme_submit_user_cmd(struct request_queue *q, return ret; } -static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, + unsigned int flags, bool open_for_write) { struct nvme_user_io io; struct nvme_command c; @@ -260,6 +293,9 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); + if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) + return -EACCES; + return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, meta_len, NULL, 0, 0); } @@ -307,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, 0, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -354,7 +390,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -440,6 +476,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; struct nvme_uring_data d; struct nvme_command c; struct iov_iter iter; @@ -470,7 +507,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); - if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; d.metadata = READ_ONCE(cmd->metadata); @@ -586,7 +623,7 @@ static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, case NVME_IOCTL_SUBMIT_IO32: #endif case NVME_IOCTL_SUBMIT_IO: - return nvme_submit_io(ns, argp); + return nvme_submit_io(ns, argp, flags, open_for_write); case NVME_IOCTL_IO64_CMD_VEC: flags |= NVME_IOCTL_VEC; fallthrough; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 102fae6a231c5..ec03a07617af1 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -178,8 +178,69 @@ enum nvme_quirks { * Align dma pool segment size to 512 bytes */ NVME_QUIRK_DMAPOOL_ALIGN_512 = (1 << 22), + + /* + * Admin queue DMA buffers must be page aligned + */ + NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), }; +static inline char *nvme_quirk_name(enum nvme_quirks q) +{ + switch (q) { + case NVME_QUIRK_STRIPE_SIZE: + return "stripe_size"; + case NVME_QUIRK_IDENTIFY_CNS: + return "identify_cns"; + case NVME_QUIRK_DEALLOCATE_ZEROES: + return "deallocate_zeroes"; + case NVME_QUIRK_DELAY_BEFORE_CHK_RDY: + return "delay_before_chk_rdy"; + case NVME_QUIRK_NO_APST: + return "no_apst"; + case NVME_QUIRK_NO_DEEPEST_PS: + return "no_deepest_ps"; + case NVME_QUIRK_QDEPTH_ONE: + return "qdepth_one"; + case NVME_QUIRK_MEDIUM_PRIO_SQ: + return "medium_prio_sq"; + case NVME_QUIRK_IGNORE_DEV_SUBNQN: + return "ignore_dev_subnqn"; + case NVME_QUIRK_DISABLE_WRITE_ZEROES: + return "disable_write_zeroes"; + case NVME_QUIRK_SIMPLE_SUSPEND: + return "simple_suspend"; + case NVME_QUIRK_SINGLE_VECTOR: + return "single_vector"; + case NVME_QUIRK_128_BYTES_SQES: + return "128_bytes_sqes"; + case NVME_QUIRK_SHARED_TAGS: + return "shared_tags"; + case NVME_QUIRK_NO_TEMP_THRESH_CHANGE: + return "no_temp_thresh_change"; + case NVME_QUIRK_NO_NS_DESC_LIST: + return "no_ns_desc_list"; + case NVME_QUIRK_DMA_ADDRESS_BITS_48: + return "dma_address_bits_48"; + case NVME_QUIRK_SKIP_CID_GEN: + return "skip_cid_gen"; + case NVME_QUIRK_BOGUS_NID: + return "bogus_nid"; + case NVME_QUIRK_NO_SECONDARY_TEMP_THRESH: + return "no_secondary_temp_thresh"; + case NVME_QUIRK_FORCE_NO_SIMPLE_SUSPEND: + return "force_no_simple_suspend"; + case NVME_QUIRK_BROKEN_MSI: + return "broken_msi"; + case NVME_QUIRK_DMAPOOL_ALIGN_512: + return "dmapool_align_512"; + case NVME_QUIRK_ADMIN_PAGE_ALIGN: + return "admin_page_align"; + } + + return "unknown"; +} + /* * Common request structure for NVMe passthrough. All drivers must have * this structure as the first member of their request-private data. @@ -558,6 +619,12 @@ static inline bool nvme_ns_has_pi(struct nvme_ns_head *head) return head->pi_type && head->ms == head->pi_size; } +static inline unsigned long nvme_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + return NVME_CTRL_PAGE_SIZE - 1; +} + struct nvme_ctrl_ops { const char *name; struct module *module; @@ -578,6 +645,7 @@ struct nvme_ctrl_ops { int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); void (*print_device_info)(struct nvme_ctrl *ctrl); bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl); + unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool is_admin); }; /* diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 4aecf88c44bbe..794561183545f 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -620,9 +620,22 @@ static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev, struct nvme_queue *nvmeq = req->mq_hctx->driver_data; if (nvmeq->qid && nvme_ctrl_sgl_supported(&dev->ctrl)) { - if (nvme_req(req)->flags & NVME_REQ_USERCMD) - return SGL_FORCED; - if (req->nr_integrity_segments > 1) + /* + * When the controller is capable of using SGL, there are + * several conditions that we force to use it: + * + * 1. A request containing page gaps within the controller's + * mask can not use the PRP format. + * + * 2. User commands use SGL because that lets the device + * validate the requested transfer lengths. + * + * 3. Multiple integrity segments must use SGL as that's the + * only way to describe such a command in NVMe. + */ + if (req_phys_gap_mask(req) & (NVME_CTRL_PAGE_SIZE - 1) || + nvme_req(req)->flags & NVME_REQ_USERCMD || + req->nr_integrity_segments > 1) return SGL_FORCED; return SGL_SUPPORTED; } @@ -2163,6 +2176,7 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) result = queue_request_irq(nvmeq); if (result) { dev->online_queues--; + nvme_disable_ctrl(&dev->ctrl, false); return result; } @@ -3383,6 +3397,14 @@ static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl) return dma_pci_p2pdma_supported(dev->dev); } +static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + if (!nvme_ctrl_sgl_supported(ctrl) || is_admin) + return NVME_CTRL_PAGE_SIZE - 1; + return 0; +} + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .name = "pcie", .module = THIS_MODULE, @@ -3397,6 +3419,7 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .get_address = nvme_pci_get_address, .print_device_info = nvme_pci_print_device_info, .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma, + .get_virt_boundary = nvme_pci_get_virt_boundary, }; static int nvme_dev_map(struct nvme_dev *dev) @@ -3654,6 +3677,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) nvme_dev_remove_admin(dev); nvme_dbbuf_dma_free(dev); nvme_free_queues(dev, 0); + nvme_release_descriptor_pools(dev); out_release_iod_mempool: mempool_destroy(dev->dmavec_mempool); out_dev_unmap: diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 190a4cfa8a5ee..19a9dd7caed8d 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -2001,7 +2001,7 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, struct ib_device *dev; bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags); blk_status_t ret; - int err; + int err = 0; WARN_ON_ONCE(rq->tag < 0); @@ -2057,16 +2057,18 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, err_unmap: nvme_rdma_unmap_data(queue, rq); err: - if (err == -EIO) - ret = nvme_host_path_error(rq); - else if (err == -ENOMEM || err == -EAGAIN) - ret = BLK_STS_RESOURCE; - else - ret = BLK_STS_IOERR; - nvme_cleanup_cmd(rq); + if (err != -EIO) { + nvme_cleanup_cmd(rq); + if (err == -ENOMEM || err == -EAGAIN) + ret = BLK_STS_RESOURCE; + else + ret = BLK_STS_IOERR; + } unmap_qe: ib_dma_unmap_single(dev, req->sqe.dma, sizeof(struct nvme_command), DMA_TO_DEVICE); + if (err == -EIO) + return nvme_host_path_error(rq); return ret; } @@ -2202,6 +2204,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .delete_ctrl = nvme_rdma_delete_ctrl, .get_address = nvmf_get_address, .stop_ctrl = nvme_rdma_stop_ctrl, + .get_virt_boundary = nvme_get_virt_boundary, }; /* diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 29430949ce2f0..16c6fea4b2db6 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -601,6 +601,28 @@ static ssize_t dctype_show(struct device *dev, } static DEVICE_ATTR_RO(dctype); +static ssize_t quirks_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + int count = 0, i; + struct nvme_ctrl *ctrl = dev_get_drvdata(dev); + unsigned long quirks = ctrl->quirks; + + if (!quirks) + return sysfs_emit(buf, "none\n"); + + for (i = 0; quirks; ++i) { + if (quirks & 1) { + count += sysfs_emit_at(buf, count, "%s\n", + nvme_quirk_name(BIT(i))); + } + quirks >>= 1; + } + + return count; +} +static DEVICE_ATTR_RO(quirks); + #ifdef CONFIG_NVME_HOST_AUTH static ssize_t nvme_ctrl_dhchap_secret_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -742,6 +764,7 @@ static struct attribute *nvme_dev_attrs[] = { &dev_attr_kato.attr, &dev_attr_cntrltype.attr, &dev_attr_dctype.attr, + &dev_attr_quirks.attr, #ifdef CONFIG_NVME_HOST_AUTH &dev_attr_dhchap_secret.attr, &dev_attr_dhchap_ctrl_secret.attr, diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index afdbcff3d4821..86e064d058d35 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -117,6 +117,7 @@ struct nvme_tcp_request { struct bio *curr_bio; struct iov_iter iter; + u32 data_recvd; /* send state */ size_t offset; @@ -145,6 +146,7 @@ struct nvme_tcp_queue { struct mutex queue_lock; struct mutex send_mutex; + struct mutex pf_cache_lock; struct llist_head req_list; struct list_head send_list; @@ -415,8 +417,13 @@ static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, * if we're the first on the send_list and we can try to send * directly, otherwise queue io_work. Also, only do that if we * are on the same cpu, so we don't introduce contention. + * + * TLS kTLS send takes ctx->tx_lock while blk_mq holds set->srcu. + * lockdep reports circular locking via elevator_lock. Defer TLS + * sends to the io workqueue instead of inline from this path. */ if (queue->io_cpu == raw_smp_processor_id() && + !nvme_tcp_queue_tls(queue) && empty && mutex_trylock(&queue->send_mutex)) { nvme_tcp_send_all(queue); mutex_unlock(&queue->send_mutex); @@ -556,9 +563,11 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); req->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!req->pdu) return -ENOMEM; @@ -618,6 +627,29 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); } +/* + * NVMe has no short read: a read that completes successfully must + * have transferred everything it asked for. + */ +static bool nvme_tcp_data_in_short(struct nvme_tcp_queue *queue, + struct request *rq) +{ + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + + if (le16_to_cpu(req->status) >> 1) + return false; + if (req_op(rq) != REQ_OP_READ || !req->data_len) + return false; + if (likely(req->data_recvd == req->data_len)) + return false; + + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x short data-in: got %u of %u\n", + nvme_tcp_queue_id(queue), rq->tag, + req->data_recvd, req->data_len); + return true; +} + static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, struct nvme_completion *cqe) { @@ -637,6 +669,9 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) req->status = cqe->status; + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + if (!nvme_try_complete_req(rq, req->status, cqe->result)) nvme_complete_rq(rq); queue->nr_cqe++; @@ -647,6 +682,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, struct nvme_tcp_data_pdu *pdu) { + struct nvme_tcp_request *req; struct request *rq; rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); @@ -657,7 +693,15 @@ static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, return -ENOENT; } - if (!blk_rq_payload_bytes(rq)) { + if (rq_data_dir(rq) != READ) { + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x unexpected data for a write\n", + nvme_tcp_queue_id(queue), rq->tag); + return -EIO; + } + + req = blk_mq_rq_to_pdu(rq); + if (!blk_rq_payload_bytes(rq) || !req->curr_bio || !req->data_len) { dev_err(queue->ctrl->ctrl.device, "queue %d tag %#x unexpected data\n", nvme_tcp_queue_id(queue), rq->tag); @@ -751,6 +795,13 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, } req = blk_mq_rq_to_pdu(rq); + if (unlikely(rq_data_dir(rq) != WRITE)) { + dev_err(queue->ctrl->ctrl.device, + "req %d unexpected r2t for a non-write command\n", + rq->tag); + return -EPROTO; + } + if (unlikely(!r2t_length)) { dev_err(queue->ctrl->ctrl.device, "req %d r2t len is %u, probably a bug...\n", @@ -959,6 +1010,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, *len -= recv_len; *offset += recv_len; queue->data_remaining -= recv_len; + req->data_recvd += recv_len; } if (!queue->data_remaining) { @@ -967,6 +1019,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; } else { if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; @@ -1015,6 +1069,9 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, pdu->command_id); struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; } @@ -1423,9 +1480,11 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) struct nvme_tcp_request *async = &ctrl->async_req; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); async->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!async->pdu) return -ENOMEM; @@ -1453,6 +1512,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) kfree(queue->pdu); mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); } static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) @@ -1775,6 +1835,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, INIT_LIST_HEAD(&queue->send_list); mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); + mutex_init(&queue->pf_cache_lock); if (qid > 0) queue->cmnd_capsule_len = nctrl->ioccsz * 16; @@ -1906,6 +1967,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); return ret; } @@ -2712,6 +2774,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, req->status = cpu_to_le16(NVME_SC_SUCCESS); req->offset = 0; req->data_sent = 0; + req->data_recvd = 0; req->pdu_len = 0; req->pdu_sent = 0; req->h2cdata_left = 0; @@ -2865,6 +2928,7 @@ static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { .delete_ctrl = nvme_tcp_delete_ctrl, .get_address = nvme_tcp_get_address, .stop_ctrl = nvme_tcp_stop_ctrl, + .get_virt_boundary = nvmf_get_virt_boundary, }; static bool diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 950b7f8e8ad56..fe7b6f8639f0a 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -963,7 +963,7 @@ static void nvmet_execute_identify_nslist(struct nvmet_req *req, bool match_css) nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { if (ns->nsid <= min_nsid) continue; - if (match_css && req->ns->csi != req->cmd->identify.csi) + if (match_css && ns->csi != req->cmd->identify.csi) continue; list[i++] = cpu_to_le32(ns->nsid); if (i == buf_size / sizeof(__le32)) diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c index 300d5e032f6d4..95e14f3f03d78 100644 --- a/drivers/nvme/target/auth.c +++ b/drivers/nvme/target/auth.c @@ -250,6 +250,12 @@ void nvmet_auth_sq_free(struct nvmet_sq *sq) sq->dhchap_skey = NULL; } +void nvmet_auth_sq_destroy(struct nvmet_sq *sq) +{ + cancel_delayed_work_sync(&sq->auth_expired_work); + nvmet_auth_sq_free(sq); +} + void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) { ctrl->shash_id = 0; diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 39f41416d3a7f..1f145b88120ba 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -601,7 +601,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) goto out_dev_put; } - if (percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL)) + ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); + if (ret) goto out_pr_exit; nvmet_ns_changed(subsys, ns->nsid); @@ -968,7 +969,7 @@ void nvmet_sq_destroy(struct nvmet_sq *sq) wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->free_done); percpu_ref_exit(&sq->ref); - nvmet_auth_sq_free(sq); + nvmet_auth_sq_destroy(sq); nvmet_cq_put(sq->cq); /* diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 2400d70233987..9642f13601656 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -31,12 +31,16 @@ void nvmet_auth_sq_init(struct nvmet_sq *sq) sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } -static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) +static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_negotiate_data *data = d; int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid; + if (tl < sizeof(*data) + + sizeof(struct nvmf_auth_dhchap_protocol_descriptor)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n", __func__, ctrl->cntlid, req->sq->qid, data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid, @@ -72,6 +76,10 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) NVME_AUTH_DHCHAP_AUTH_ID) return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + if (data->auth_protocol[0].dhchap.dhlen > NVME_AUTH_DHCHAP_MAX_DH_IDS || + data->auth_protocol[0].dhchap.halen > NVME_AUTH_DHCHAP_MAX_HASH_IDS) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) { u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i]; @@ -317,7 +325,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } else if (data->auth_id != req->sq->dhchap_step) goto done_failure1; /* Validate negotiation parameters */ - dhchap_status = nvmet_auth_negotiate(req, d); + dhchap_status = nvmet_auth_negotiate(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE; @@ -558,7 +566,7 @@ void nvmet_execute_auth_receive(struct nvmet_req *req) return; } - d = kmalloc(al, GFP_KERNEL); + d = kzalloc(al, GFP_KERNEL); if (!d) { status = NVME_SC_INTERNAL; goto done; diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 7d84527d5a43e..a61c3c93b3820 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -569,7 +569,7 @@ nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport) list_del(&iod->ls_rcv_list); } - kfree(iod); + kfree(tgtport->iod); return -EFAULT; } diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index f85a8441bcc6e..fc8e7c9ad8588 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -511,6 +511,7 @@ static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = { .submit_async_event = nvme_loop_submit_async_event, .delete_ctrl = nvme_loop_delete_ctrl_host, .get_address = nvmf_get_address, + .get_virt_boundary = nvme_get_virt_boundary, }; static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 059fd9f356c45..f2ea7993eabc7 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -901,6 +901,7 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq); void nvmet_auth_sq_init(struct nvmet_sq *sq); void nvmet_destroy_auth(struct nvmet_ctrl *ctrl); void nvmet_auth_sq_free(struct nvmet_sq *sq); +void nvmet_auth_sq_destroy(struct nvmet_sq *sq); int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id); bool nvmet_check_auth_status(struct nvmet_req *req); int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, @@ -927,6 +928,7 @@ static inline void nvmet_auth_sq_init(struct nvmet_sq *sq) } static inline void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) {}; static inline void nvmet_auth_sq_free(struct nvmet_sq *sq) {}; +static inline void nvmet_auth_sq_destroy(struct nvmet_sq *sq) {}; static inline bool nvmet_check_auth_status(struct nvmet_req *req) { return true; diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 9c5b0f78ce8df..9b1df0ad07a29 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -1338,6 +1338,7 @@ static u16 nvmet_pci_epf_create_cq(struct nvmet_ctrl *tctrl, nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); err_internal: status = NVME_SC_INTERNAL | NVME_STATUS_DNR; + nvmet_cq_put(&cq->nvme_cq); err: if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags)) nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); @@ -1595,6 +1596,7 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) struct nvmet_pci_epf_iod *iod = container_of(work, struct nvmet_pci_epf_iod, work); struct nvmet_req *req = &iod->req; + bool no_wait; int ret; if (!iod->ctrl->link_up) { @@ -1639,14 +1641,16 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) } } - req->execute(req); - /* * If we do not have data to transfer after the command execution * finishes, nvmet_pci_epf_queue_response() will complete the command * directly. No need to wait for the completion in this case. */ - if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE) + no_wait = !iod->data_len || iod->dma_dir != DMA_TO_DEVICE; + + req->execute(req); + + if (no_wait) return; wait_for_completion(&iod->done); diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index cd22d8333314e..783e6a5ec3e04 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -355,9 +355,15 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_pr *pr = &req->ns->pr; - struct nvmet_pr_registrant *reg; + struct nvmet_pr_registrant *reg, *new = NULL; u64 nrkey = le64_to_cpu(d->nrkey); + if (ignore_key && nrkey) { + new = kzalloc_obj(*new); + if (!new) + return NVME_SC_INTERNAL; + } + down(&pr->pr_sem); list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { if (uuid_equal(®->hostid, &ctrl->hostid)) { @@ -365,9 +371,26 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, status = nvmet_pr_update_reg_attr(pr, reg, nvmet_pr_update_reg_rkey, &nrkey); - break; + goto free_data; + } + } + + if (ignore_key) { + if (!nrkey) { + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto free_data; } + INIT_LIST_HEAD(&new->entry); + new->rkey = nrkey; + uuid_copy(&new->hostid, &ctrl->hostid); + list_add_tail_rcu(&new->entry, &pr->registrant_list); + status = NVME_SC_SUCCESS; + goto out; } + +free_data: + kfree(new); +out: up(&pr->pr_sem); return status; } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 4b493226d07c1..2b03bce27a88c 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -658,18 +658,25 @@ static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) req->sg, req->sg_cnt, nvmet_data_dir(req)); } -static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) { struct nvmet_rdma_queue *queue = rsp->queue; - atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg < rsp->cmd->inline_sg || rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); +} + +static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +{ + struct nvmet_rdma_queue *queue = rsp->queue; + + atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); + + nvmet_rdma_free_rsp_resources(rsp); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); @@ -1339,9 +1346,27 @@ static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) goto out; } +static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, + void *data) +{ + struct nvmet_rdma_queue *queue = data; + + nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); + + return true; +} + static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) { ib_drain_qp(queue->qp); + + /* + * Reclaim resources of a response that is still in-flight when the + * queue is being torn down. This happens when the connection was + * forcefully disconnected while an I/O is in flight. + */ + sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); + if (queue->cm_id) rdma_destroy_id(queue->cm_id); ib_destroy_qp(queue->qp); @@ -1603,19 +1628,13 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, mutex_unlock(&nvmet_rdma_queue_mutex); if (pending > NVMET_RDMA_BACKLOG) { ret = NVME_SC_CONNECT_CTRL_BUSY; - goto put_device; + goto free_queue; } } ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); - if (ret) { - /* - * Don't destroy the cm_id in free path, as we implicitly - * destroy the cm_id here with non-zero ret code. - */ - queue->cm_id = NULL; + if (ret) goto free_queue; - } mutex_lock(&nvmet_rdma_queue_mutex); list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list); @@ -1624,6 +1643,11 @@ static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, return 0; free_queue: + /* + * Don't destroy the cm_id in free path, as we implicitly + * destroy the cm_id here with non-zero ret code. + */ + queue->cm_id = NULL; nvmet_rdma_free_queue(queue); put_device: kref_put(&ndev->ref, nvmet_rdma_free_dev); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 0b8cfa3dbab12..b9a904f184892 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -103,6 +103,7 @@ enum nvmet_tcp_recv_state { enum { NVMET_TCP_F_INIT_FAILED = (1 << 0), + NVMET_TCP_F_R2T_SENT = (1 << 1), }; struct nvmet_tcp_cmd { @@ -433,6 +434,19 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) if (!len) return 0; + /* + * inline_data_size only bounds the in-capsule (type 0x01) SGL + * descriptor below. A non-inline transport SGL data-block + * descriptor skips that check entirely and would otherwise reach + * sgl_alloc() with an attacker-controlled len of up to 4 GiB, + * pinning that much kernel memory for a command that may never + * complete. Bound every descriptor type here, before allocating + * anything, using the same ceiling this file already applies to + * per-PDU H2C data. + */ + if (len > NVMET_TCP_MAXH2CDATA) + return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; + if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET)) { if (!nvme_is_write(cmd->req.cmd)) @@ -444,14 +458,15 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) } cmd->req.transfer_len += len; - cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt); + cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, + &cmd->req.sg_cnt); if (!cmd->req.sg) return NVME_SC_INTERNAL; cmd->cur_sg = cmd->req.sg; if (nvmet_tcp_has_data_in(cmd)) { cmd->iov = kmalloc_array(cmd->req.sg_cnt, - sizeof(*cmd->iov), GFP_KERNEL); + sizeof(*cmd->iov), GFP_KERNEL | __GFP_NOWARN); if (!cmd->iov) goto err; } @@ -773,6 +788,7 @@ static int nvmet_try_send_r2t(struct nvmet_tcp_cmd *cmd, bool last_in_batch) return -EAGAIN; cmd->queue->snd_cmd = NULL; + cmd->flags |= NVMET_TCP_F_R2T_SENT; return 1; } @@ -1008,6 +1024,12 @@ static int nvmet_tcp_handle_h2c_data_pdu(struct nvmet_tcp_queue *queue) cmd = &queue->connect; } + if (unlikely(!(cmd->flags & NVMET_TCP_F_R2T_SENT))) { + pr_err("queue %d: unsolicited H2CData (ttag %u)\n", + queue->idx, data->ttag); + goto err_proto; + } + if (le32_to_cpu(data->data_offset) != cmd->rbytes_done) { pr_err("ttag %u unexpected data offset %u (expected %u)\n", data->ttag, le32_to_cpu(data->data_offset), @@ -1248,6 +1270,8 @@ static int nvmet_tcp_try_recv_pdu(struct nvmet_tcp_queue *queue) } queue->left = hdr->hlen - queue->offset + hdgst; + if (queue->left > sizeof(queue->pdu) - queue->offset) + return -EPROTO; goto recv; } diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index 15a579cf528c1..95af4b67108c7 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -120,7 +120,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req) mutex_unlock(&req->ns->subsys->lock); } - if (!bdev_is_zoned(req->ns->bdev)) { + if (!req->ns->bdev || !bdev_is_zoned(req->ns->bdev)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvme_identify, nsid); goto out; diff --git a/drivers/nvmem/apple-spmi-nvmem.c b/drivers/nvmem/apple-spmi-nvmem.c index 88614005d5ce1..7acb0c07d6abe 100644 --- a/drivers/nvmem/apple-spmi-nvmem.c +++ b/drivers/nvmem/apple-spmi-nvmem.c @@ -18,6 +18,22 @@ static const struct regmap_config apple_spmi_regmap_config = { .max_register = 0xffff, }; +static int apple_spmi_nvmem_read(void *priv, unsigned int offset, void *val, + size_t bytes) +{ + struct regmap *map = priv; + + return regmap_bulk_read(map, offset, val, bytes); +} + +static int apple_spmi_nvmem_write(void *priv, unsigned int offset, void *val, + size_t bytes) +{ + struct regmap *map = priv; + + return regmap_bulk_write(map, offset, val, bytes); +} + static int apple_spmi_nvmem_probe(struct spmi_device *sdev) { struct regmap *regmap; @@ -28,8 +44,8 @@ static int apple_spmi_nvmem_probe(struct spmi_device *sdev) .word_size = 1, .stride = 1, .size = 0xffff, - .reg_read = (void *)regmap_bulk_read, - .reg_write = (void *)regmap_bulk_write, + .reg_read = apple_spmi_nvmem_read, + .reg_write = apple_spmi_nvmem_write, }; regmap = devm_regmap_init_spmi_ext(sdev, &apple_spmi_regmap_config); diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index d2eb5c7dbdd9d..b61028ae0651b 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -786,7 +786,7 @@ static int nvmem_validate_keepouts(struct nvmem_device *nvmem) return 0; } -static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np) +int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np) { struct device *dev = &nvmem->dev; struct device_node *child; @@ -838,29 +838,13 @@ static int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_nod return 0; } +EXPORT_SYMBOL_GPL(nvmem_add_cells_from_dt); static int nvmem_add_cells_from_legacy_of(struct nvmem_device *nvmem) { return nvmem_add_cells_from_dt(nvmem, nvmem->dev.of_node); } -static int nvmem_add_cells_from_fixed_layout(struct nvmem_device *nvmem) -{ - struct device_node *layout_np; - int err = 0; - - layout_np = of_nvmem_layout_get_container(nvmem); - if (!layout_np) - return 0; - - if (of_device_is_compatible(layout_np, "fixed-layout")) - err = nvmem_add_cells_from_dt(nvmem, layout_np); - - of_node_put(layout_np); - - return err; -} - int nvmem_layout_register(struct nvmem_layout *layout) { int ret; @@ -1009,10 +993,6 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) goto err_remove_cells; } - rval = nvmem_add_cells_from_fixed_layout(nvmem); - if (rval) - goto err_remove_cells; - dev_dbg(&nvmem->dev, "Registering nvmem device %s\n", config->name); rval = device_add(&nvmem->dev); diff --git a/drivers/nvmem/internals.h b/drivers/nvmem/internals.h index 18fed57270e5e..f6d452b6a28f1 100644 --- a/drivers/nvmem/internals.h +++ b/drivers/nvmem/internals.h @@ -35,6 +35,8 @@ struct nvmem_device { bool sysfs_cells_populated; }; +int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, struct device_node *np); + #if IS_ENABLED(CONFIG_OF) int nvmem_layout_bus_register(void); void nvmem_layout_bus_unregister(void); diff --git a/drivers/nvmem/layouts.c b/drivers/nvmem/layouts.c index 7ebe53249035e..0e84d65b42d4c 100644 --- a/drivers/nvmem/layouts.c +++ b/drivers/nvmem/layouts.c @@ -125,11 +125,6 @@ static int nvmem_layout_create_device(struct nvmem_device *nvmem, return 0; } -static const struct of_device_id of_nvmem_layout_skip_table[] = { - { .compatible = "fixed-layout", }, - {} -}; - static int nvmem_layout_bus_populate(struct nvmem_device *nvmem, struct device_node *layout_dn) { @@ -142,12 +137,6 @@ static int nvmem_layout_bus_populate(struct nvmem_device *nvmem, return 0; } - /* Fixed layouts are parsed manually somewhere else for now */ - if (of_match_node(of_nvmem_layout_skip_table, layout_dn)) { - pr_debug("%s() - skipping %pOF node\n", __func__, layout_dn); - return 0; - } - if (of_node_check_flag(layout_dn, OF_POPULATED_BUS)) { pr_debug("%s() - skipping %pOF, already populated\n", __func__, layout_dn); diff --git a/drivers/nvmem/layouts/Makefile b/drivers/nvmem/layouts/Makefile index 4940c9db06656..dd6c6c70b1a92 100644 --- a/drivers/nvmem/layouts/Makefile +++ b/drivers/nvmem/layouts/Makefile @@ -3,6 +3,7 @@ # Makefile for nvmem layouts. # +obj-$(CONFIG_NVMEM_LAYOUTS) += fixed-layout.o obj-$(CONFIG_NVMEM_LAYOUT_SL28_VPD) += sl28vpd.o obj-$(CONFIG_NVMEM_LAYOUT_ONIE_TLV) += onie-tlv.o obj-$(CONFIG_NVMEM_LAYOUT_U_BOOT_ENV) += u-boot-env.o diff --git a/drivers/nvmem/layouts/fixed-layout.c b/drivers/nvmem/layouts/fixed-layout.c new file mode 100644 index 0000000000000..635d448b3dd25 --- /dev/null +++ b/drivers/nvmem/layouts/fixed-layout.c @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2026 Bootlin + * + * Authors: Mathieu Dubois-Briand + */ + +#include +#include + +#include "../internals.h" + +static int fixed_layout_add_cells(struct nvmem_layout *layout) +{ + struct device_node *np; + int ret; + + np = of_nvmem_layout_get_container(layout->nvmem); + if (!np) + return -ENOENT; + + ret = nvmem_add_cells_from_dt(layout->nvmem, np); + of_node_put(np); + + return ret; +} + +static int fixed_layout_probe(struct nvmem_layout *layout) +{ + layout->add_cells = fixed_layout_add_cells; + + return nvmem_layout_register(layout); +} + +static void fixed_layout_remove(struct nvmem_layout *layout) +{ + nvmem_layout_unregister(layout); +} + +static const struct of_device_id fixed_layout_of_match_table[] = { + { .compatible = "fixed-layout", }, + {}, +}; + +static struct nvmem_layout_driver fixed_layout_layout = { + .driver = { + .name = "fixed-layout", + .of_match_table = fixed_layout_of_match_table, + }, + .probe = fixed_layout_probe, + .remove = fixed_layout_remove, +}; +module_nvmem_layout_driver(fixed_layout_layout); + +MODULE_AUTHOR("Mathieu Dubois-Briand"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, fixed_layout_of_match_table); +MODULE_DESCRIPTION("NVMEM fixed-layout driver"); diff --git a/drivers/of/address.c b/drivers/of/address.c index f0f8f0dd191cf..07d90831dec08 100644 --- a/drivers/of/address.c +++ b/drivers/of/address.c @@ -757,6 +757,7 @@ EXPORT_SYMBOL(of_property_read_reg); static int parser_init(struct of_pci_range_parser *parser, struct device_node *node, const char *name) { + const __be32 *range; int rlen; parser->node = node; @@ -765,12 +766,20 @@ static int parser_init(struct of_pci_range_parser *parser, parser->ns = of_bus_n_size_cells(node); parser->dma = !strcmp(name, "dma-ranges"); parser->bus = of_match_bus(node); + parser->range = NULL; + parser->end = NULL; - parser->range = of_get_property(node, name, &rlen); - if (parser->range == NULL) + range = of_get_property(node, name, &rlen); + if (!range) return -ENOENT; - parser->end = parser->range + rlen / sizeof(__be32); + if (!parser->bus || + !OF_CHECK_COUNTS(parser->na, parser->ns) || + !OF_CHECK_ADDR_COUNT(parser->pna)) + return -EINVAL; + + parser->range = range; + parser->end = range + rlen / sizeof(__be32); return 0; } @@ -796,7 +805,7 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser, int na = parser->na; int ns = parser->ns; int np = parser->pna + na + ns; - int busflag_na = parser->bus->flag_cells; + int busflag_na; if (!range) return NULL; @@ -804,6 +813,8 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser, if (!parser->range || parser->range + np > parser->end) return NULL; + busflag_na = parser->bus->flag_cells; + range->flags = parser->bus->get_flags(parser->range); range->bus_addr = of_read_number(parser->range + busflag_na, na - busflag_na); @@ -980,8 +991,7 @@ phys_addr_t __init of_dma_get_max_cpu_address(struct device_node *np) np = of_root; ranges = of_get_property(np, "dma-ranges", &len); - if (ranges && len) { - of_dma_range_parser_init(&parser, np); + if (ranges && len && !of_dma_range_parser_init(&parser, np)) { for_each_of_range(&parser, &range) if (range.cpu_addr + range.size > cpu_end) cpu_end = range.cpu_addr + range.size - 1; diff --git a/drivers/of/base.c b/drivers/of/base.c index 6620bf07b79b8..ffd1d51dff38b 100644 --- a/drivers/of/base.c +++ b/drivers/of/base.c @@ -82,9 +82,17 @@ EXPORT_SYMBOL(of_node_name_prefix); static bool __of_node_is_type(const struct device_node *np, const char *type) { - const char *match = __of_get_property(np, "device_type", NULL); + const char *match; + int len; + + if (!np || !type) + return false; + + match = __of_get_property(np, "device_type", &len); + if (!match || len <= 0 || strnlen(match, len) >= len) + return false; - return np && match && type && !strcmp(match, type); + return !strcmp(match, type); } #define EXCLUDED_DEFAULT_CELLS_PLATFORMS ( \ @@ -444,22 +452,22 @@ static bool __of_device_is_status(const struct device_node *device, return false; status = __of_get_property(device, "status", &statlen); - if (status == NULL) + if (!status || statlen <= 0) + return false; + if (strnlen(status, statlen) >= statlen) return false; - if (statlen > 0) { - while (*strings) { - unsigned int len = strlen(*strings); + while (*strings) { + unsigned int len = strlen(*strings); - if ((*strings)[len - 1] == '-') { - if (!strncmp(status, *strings, len)) - return true; - } else { - if (!strcmp(status, *strings)) - return true; - } - strings++; + if ((*strings)[len - 1] == '-') { + if (!strncmp(status, *strings, len)) + return true; + } else { + if (!strcmp(status, *strings)) + return true; } + strings++; } return false; @@ -1170,10 +1178,11 @@ EXPORT_SYMBOL(of_find_matching_node_and_match); int of_alias_from_compatible(const struct device_node *node, char *alias, int len) { const char *compatible, *p; - int cplen; + int ret; - compatible = of_get_property(node, "compatible", &cplen); - if (!compatible || strlen(compatible) > cplen) + ret = of_property_read_string_index(node, "compatible", 0, + &compatible); + if (ret) return -ENODEV; p = strchr(compatible, ','); strscpy(alias, p ? p + 1 : compatible, len); @@ -1891,7 +1900,7 @@ void of_alias_scan(void * (*dt_alloc)(u64 size, u64 align)) /* walk the alias backwards to extract the id and work out * the 'stem' string */ - while (isdigit(*(end-1)) && end > start) + while (end > start && isdigit(*(end - 1))) end--; len = end - start; diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c index b5be7484fb36d..3e08484d351b4 100644 --- a/drivers/of/dynamic.c +++ b/drivers/of/dynamic.c @@ -74,6 +74,20 @@ static const char *action_names[] = { [OF_RECONFIG_UPDATE_PROPERTY] = "UPDATE_PROPERTY", }; +static bool of_property_status_ok(const struct property *prop) +{ + const char *status; + + if (!prop || !prop->value || prop->length <= 0) + return false; + + status = prop->value; + if (strnlen(status, prop->length) >= prop->length) + return false; + + return !strcmp(status, "okay") || !strcmp(status, "ok"); +} + #define _do_print(func, prefix, action, node, prop, ...) ({ \ func("changeset: " prefix "%-15s %pOF%s%s\n", \ ##__VA_ARGS__, action_names[action], node, \ @@ -135,11 +149,9 @@ int of_reconfig_get_state_change(unsigned long action, struct of_reconfig_data * if (prop && !strcmp(prop->name, "status")) { is_status = 1; - status_state = !strcmp(prop->value, "okay") || - !strcmp(prop->value, "ok"); + status_state = of_property_status_ok(prop); if (old_prop) - old_status_state = !strcmp(old_prop->value, "okay") || - !strcmp(old_prop->value, "ok"); + old_status_state = of_property_status_ok(old_prop); } switch (action) { diff --git a/drivers/of/of_reserved_mem.c b/drivers/of/of_reserved_mem.c index 215fb723504a9..16d188a17f2c1 100644 --- a/drivers/of/of_reserved_mem.c +++ b/drivers/of/of_reserved_mem.c @@ -353,6 +353,7 @@ int __init fdt_scan_reserved_mem(void) err = __reserved_mem_reserve_reg(child, uname); if (!err) count++; + /* * Save the nodes for the dynamically-placed regions * into an array which will be used for allocation right @@ -360,10 +361,17 @@ int __init fdt_scan_reserved_mem(void) * or marked as no-map. This is done to avoid dynamically * allocating from one of the statically-placed regions. */ - if (err == -ENOENT && of_get_flat_dt_prop(child, "size", NULL)) { - dynamic_nodes[dynamic_nodes_cnt] = child; - dynamic_nodes_cnt++; + if (err != -ENOENT || !of_get_flat_dt_prop(child, "size", NULL)) + continue; + + if (dynamic_nodes_cnt == MAX_RESERVED_REGIONS) { + pr_err("too many defined dynamic regions, skip '%s'\n", + uname); + continue; } + + dynamic_nodes[dynamic_nodes_cnt] = child; + dynamic_nodes_cnt++; } for (int i = 0; i < dynamic_nodes_cnt; i++) { const char *uname; diff --git a/drivers/of/property.c b/drivers/of/property.c index c1feb631e3831..71322b5bda267 100644 --- a/drivers/of/property.c +++ b/drivers/of/property.c @@ -614,16 +614,31 @@ EXPORT_SYMBOL_GPL(of_prop_next_u32); const char *of_prop_next_string(const struct property *prop, const char *cur) { - const void *curv = cur; + const char *curv; + const char *end; + size_t len; - if (!prop) + if (!prop || !prop->value || !prop->length) return NULL; - if (!cur) - return prop->value; + curv = cur ? cur : prop->value; + end = prop->value + prop->length; - curv += strlen(cur) + 1; - if (curv >= prop->value + prop->length) + if (curv < (const char *)prop->value || curv >= end) + return NULL; + + if (cur) { + len = strnlen(curv, end - curv); + if (len >= end - curv) + return NULL; + + curv += len + 1; + if (curv >= end) + return NULL; + } + + len = strnlen(curv, end - curv); + if (len >= end - curv) return NULL; return curv; diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c index 02b780b6e8e25..729813d54c22d 100644 --- a/drivers/of/unittest.c +++ b/drivers/of/unittest.c @@ -713,6 +713,7 @@ static void __init of_unittest_parse_phandle_with_args_map(void) static void __init of_unittest_property_string(void) { const char *strings[4]; + const struct property *prop; struct device_node *np; int rc; @@ -789,6 +790,37 @@ static void __init of_unittest_property_string(void) strings[1] = NULL; rc = of_property_read_string_array(np, "phandle-list-names", strings, 1); unittest(rc == 1 && strings[1] == NULL, "Overwrote end of string array; rc=%i, str='%s'\n", rc, strings[1]); + + /* of_prop_next_string() tests */ + prop = of_find_property(np, "phandle-list-names", NULL); + strings[0] = of_prop_next_string(prop, NULL); + unittest(strings[0] && !strcmp(strings[0], "first"), + "of_prop_next_string() failure; got '%s'\n", strings[0]); + strings[0] = of_prop_next_string(prop, strings[0]); + unittest(strings[0] && !strcmp(strings[0], "second"), + "of_prop_next_string() failure; got '%s'\n", strings[0]); + strings[0] = of_prop_next_string(prop, strings[0]); + unittest(strings[0] && !strcmp(strings[0], "third"), + "of_prop_next_string() failure; got '%s'\n", strings[0]); + strings[0] = of_prop_next_string(prop, strings[0]); + unittest(!strings[0], + "of_prop_next_string() should return NULL at end of list\n"); + + prop = of_find_property(np, "unterminated-string", NULL); + strings[0] = of_prop_next_string(prop, NULL); + unittest(!strings[0], + "of_prop_next_string() should reject unterminated first string\n"); + + prop = of_find_property(np, "unterminated-string-list", NULL); + strings[0] = of_prop_next_string(prop, NULL); + unittest(strings[0] && !strcmp(strings[0], "first"), + "of_prop_next_string() failure; got '%s'\n", strings[0]); + strings[0] = of_prop_next_string(prop, strings[0]); + unittest(strings[0] && !strcmp(strings[0], "second"), + "of_prop_next_string() failure; got '%s'\n", strings[0]); + strings[0] = of_prop_next_string(prop, strings[0]); + unittest(!strings[0], + "of_prop_next_string() should reject unterminated trailing string\n"); } #define propcmp(p1, p2) (((p1)->length == (p2)->length) && \ diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 7b4b7deb5ac6f..e2f4d89bfad33 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -1037,7 +1037,7 @@ static int _of_add_opp_table_v1(struct device *dev, struct opp_table *opp_table) val = prop->value; while (nr) { - unsigned long freq = be32_to_cpup(val++) * 1000; + unsigned long freq = (unsigned long)be32_to_cpup(val++) * 1000; unsigned long volt = be32_to_cpup(val++); struct dev_pm_opp_data data = { .freq = freq, diff --git a/drivers/parisc/eisa.c b/drivers/parisc/eisa.c index 9eab974e6baf6..b5b5413bfd1e9 100644 --- a/drivers/parisc/eisa.c +++ b/drivers/parisc/eisa.c @@ -442,6 +442,11 @@ static int __init eisa_irq_setup(char *str) val = (int) simple_strtoul(cur, &pe, 0); if (val > 15 || val < 0) { printk(KERN_ERR "eisa: EISA irq value are 0-15\n"); + cur = strchr(cur, ','); + if (cur) + cur++; + else + break; continue; } if (val == 2) { diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c index 0413d163cfea0..e2ec684cc54ca 100644 --- a/drivers/pci/controller/cadence/pci-j721e.c +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -381,7 +381,7 @@ static const struct j721e_pcie_data j7200_pcie_rc_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .byte_access_allowed = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data j7200_pcie_ep_data = { @@ -389,7 +389,7 @@ static const struct j721e_pcie_data j7200_pcie_ep_data = { .quirk_detect_quiet_flag = true, .linkdown_irq_regfield = J7200_LINK_DOWN, .quirk_disable_flr = true, - .max_lanes = 2, + .max_lanes = 4, }; static const struct j721e_pcie_data am64_pcie_rc_data = { diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c index 225d887cd0a3e..e3fe61ef862b3 100644 --- a/drivers/pci/controller/dwc/pci-meson.c +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -402,7 +402,7 @@ static int meson_pcie_probe(struct platform_device *pdev) return PTR_ERR(mp->phy); } - mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); if (IS_ERR(mp->reset_gpio)) { dev_err(dev, "get reset gpio failed\n"); return PTR_ERR(mp->reset_gpio); diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c index c21906eced618..8b51eb30112d9 100644 --- a/drivers/pci/controller/dwc/pcie-intel-gw.c +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -285,13 +285,9 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) intel_pcie_core_rst_assert(pcie); intel_pcie_device_rst_assert(pcie); - - ret = phy_init(pcie->phy); - if (ret) - return ret; - intel_pcie_core_rst_deassert(pcie); + /* Controller clock must be provided earlier than PHY */ ret = clk_prepare_enable(pcie->core_clk); if (ret) { dev_err(pcie->pci.dev, "Core clock enable failed: %d\n", ret); @@ -300,13 +296,17 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) pci->atu_base = pci->dbi_base + 0xC0000; + ret = phy_init(pcie->phy); + if (ret) + goto phy_err; + intel_pcie_ltssm_disable(pcie); intel_pcie_link_setup(pcie); intel_pcie_init_n_fts(pci); ret = dw_pcie_setup_rc(&pci->pp); if (ret) - goto app_init_err; + goto err; dw_pcie_upconfig_setup(pci); @@ -315,7 +315,7 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) ret = dw_pcie_wait_for_link(pci); if (ret) - goto app_init_err; + goto err; /* Enable integrated interrupts */ pcie_app_wr_mask(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT, @@ -323,11 +323,12 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie) return 0; -app_init_err: +err: + phy_exit(pcie->phy); +phy_err: clk_disable_unprepare(pcie->core_clk); clk_err: intel_pcie_core_rst_assert(pcie); - phy_exit(pcie->phy); return ret; } diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c index c1bc0d34348f4..9e85c6e9b4254 100644 --- a/drivers/pci/controller/pci-host-generic.c +++ b/drivers/pci/controller/pci-host-generic.c @@ -16,15 +16,6 @@ #include "pci-host-common.h" -static const struct pci_ecam_ops gen_pci_cfg_cam_bus_ops = { - .bus_shift = 16, - .pci_ops = { - .map_bus = pci_ecam_map_bus, - .read = pci_generic_config_read, - .write = pci_generic_config_write, - } -}; - static bool pci_dw_valid_device(struct pci_bus *bus, unsigned int devfn) { struct pci_config_window *cfg = bus->sysdata; @@ -60,7 +51,7 @@ static const struct pci_ecam_ops pci_dw_ecam_bus_ops = { static const struct of_device_id gen_pci_of_match[] = { { .compatible = "pci-host-cam-generic", - .data = &gen_pci_cfg_cam_bus_ops }, + .data = &pci_generic_cam_ops }, { .compatible = "pci-host-ecam-generic", .data = &pci_generic_ecam_ops }, diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c index 28b1572974879..fff5df771d5bd 100644 --- a/drivers/pci/controller/pci-hyperv.c +++ b/drivers/pci/controller/pci-hyperv.c @@ -2120,6 +2120,7 @@ static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *dom info->ops->msi_prepare = hv_msi_prepare; chip->irq_set_affinity = irq_chip_set_affinity_parent; + chip->irq_retrigger = irq_chip_retrigger_hierarchy; if (IS_ENABLED(CONFIG_X86)) chip->flags |= IRQCHIP_MOVE_DEFERRED; diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c index b95afa35201d0..83c9a2930eeca 100644 --- a/drivers/pci/controller/pci-xgene.c +++ b/drivers/pci/controller/pci-xgene.c @@ -58,7 +58,6 @@ #define XGENE_PCIE_IP_VER_2 2 struct xgene_pcie { - struct device_node *node; struct device *dev; struct clk *clk; void __iomem *csr_base; @@ -526,7 +525,7 @@ static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port, static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port) { - struct device_node *np = port->node; + struct device_node *np = port->dev->of_node; struct of_pci_range range; struct of_pci_range_parser parser; struct device *dev = port->dev; @@ -612,7 +611,6 @@ static bool xgene_check_pcie_msi_ready(void) static int xgene_pcie_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *dn = dev->of_node; struct xgene_pcie *port; struct pci_host_bridge *bridge; int ret; @@ -627,7 +625,6 @@ static int xgene_pcie_probe(struct platform_device *pdev) port = pci_host_bridge_priv(bridge); - port->node = of_node_get(dn); port->dev = dev; port->version = XGENE_PCIE_IP_VER_1; diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c index 76f3823d96139..8ef56e2fabcfb 100644 --- a/drivers/pci/controller/pcie-altera.c +++ b/drivers/pci/controller/pcie-altera.c @@ -1075,8 +1075,10 @@ static void altera_pcie_remove(struct platform_device *pdev) struct altera_pcie *pcie = platform_get_drvdata(pdev); struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + pci_lock_rescan_remove(); pci_stop_root_bus(bridge->bus); pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); altera_pcie_irq_teardown(pcie); } diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 5aa677f81e4fa..61dcb232757fb 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -1540,8 +1540,10 @@ void iproc_pcie_remove(struct iproc_pcie *pcie) { struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + pci_lock_rescan_remove(); pci_stop_root_bus(host->bus); pci_remove_root_bus(host->bus); + pci_unlock_rescan_remove(); iproc_pcie_msi_disable(pcie); diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c index 6afce031523bd..ce2a3e04a953e 100644 --- a/drivers/pci/controller/pcie-mediatek.c +++ b/drivers/pci/controller/pcie-mediatek.c @@ -1154,8 +1154,10 @@ static void mtk_pcie_remove(struct platform_device *pdev) struct mtk_pcie *pcie = platform_get_drvdata(pdev); struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + pci_lock_rescan_remove(); pci_stop_root_bus(host->bus); pci_remove_root_bus(host->bus); + pci_unlock_rescan_remove(); mtk_pcie_free_resources(pcie); mtk_pcie_irq_teardown(pcie); diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c index ee1822ca01db3..d203c4876d304 100644 --- a/drivers/pci/controller/pcie-rockchip-host.c +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -1012,8 +1012,10 @@ static void rockchip_pcie_remove(struct platform_device *pdev) struct rockchip_pcie *rockchip = dev_get_drvdata(dev); struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); + pci_lock_rescan_remove(); pci_stop_root_bus(bridge->bus); pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); irq_domain_remove(rockchip->irq_domain); rockchip_pcie_deinit_phys(rockchip); diff --git a/drivers/pci/controller/plda/pcie-plda-host.c b/drivers/pci/controller/plda/pcie-plda-host.c index 3c2f683830107..fd1a11b4c8eb8 100644 --- a/drivers/pci/controller/plda/pcie-plda-host.c +++ b/drivers/pci/controller/plda/pcie-plda-host.c @@ -419,6 +419,8 @@ static int plda_pcie_init_irq_domains(struct plda_pcie_rp *port) return plda_allocate_msi_domains(port); } +static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie); + int plda_init_interrupts(struct platform_device *pdev, struct plda_pcie_rp *port, const struct plda_event *event) @@ -440,14 +442,17 @@ int plda_init_interrupts(struct platform_device *pdev, } port->irq = platform_get_irq(pdev, 0); - if (port->irq < 0) - return -ENODEV; + if (port->irq < 0) { + ret = -ENODEV; + goto err_irq_domain_deinit; + } for_each_set_bit(i, &port->events_bitmap, port->num_events) { event_irq = irq_create_mapping(port->event_domain, i); if (!event_irq) { dev_err(dev, "failed to map hwirq %d\n", i); - return -ENXIO; + ret = -ENXIO; + goto err_irq_domain_deinit; } if (event->request_event_irq) @@ -459,7 +464,7 @@ int plda_init_interrupts(struct platform_device *pdev, if (ret) { dev_err(dev, "failed to request IRQ %d\n", event_irq); - return ret; + goto err_irq_domain_deinit; } } @@ -467,7 +472,8 @@ int plda_init_interrupts(struct platform_device *pdev, event->intx_event); if (!port->intx_irq) { dev_err(dev, "failed to map INTx interrupt\n"); - return -ENXIO; + ret = -ENXIO; + goto err_irq_domain_deinit; } /* Plug the INTx chained handler */ @@ -475,8 +481,11 @@ int plda_init_interrupts(struct platform_device *pdev, port->msi_irq = irq_create_mapping(port->event_domain, event->msi_event); - if (!port->msi_irq) - return -ENXIO; + if (!port->msi_irq) { + dev_err(dev, "failed to map MSI interrupt\n"); + ret = -ENXIO; + goto err_irq_domain_deinit; + } /* Plug the MSI chained handler */ irq_set_chained_handler_and_data(port->msi_irq, plda_handle_msi, port); @@ -485,6 +494,11 @@ int plda_init_interrupts(struct platform_device *pdev, irq_set_chained_handler_and_data(port->irq, plda_handle_event, port); return 0; + +err_irq_domain_deinit: + plda_pcie_irq_domain_deinit(port); + + return ret; } EXPORT_SYMBOL_GPL(plda_init_interrupts); @@ -559,9 +573,27 @@ EXPORT_SYMBOL_GPL(plda_pcie_setup_iomems); static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie) { - irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); - irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL); - irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL); + u32 i, event_irq; + + if (pcie->irq > 0) + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + if (pcie->msi_irq > 0) + irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL); + if (pcie->intx_irq > 0) + irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL); + + for_each_set_bit(i, &pcie->events_bitmap, pcie->num_events) { + event_irq = irq_find_mapping(pcie->event_domain, i); + if (event_irq) { + devm_free_irq(pcie->dev, event_irq, pcie); + irq_dispose_mapping(event_irq); + } + } + + if (pcie->intx_irq) + irq_dispose_mapping(pcie->intx_irq); + if (pcie->msi_irq) + irq_dispose_mapping(pcie->msi_irq); irq_domain_remove(pcie->msi.dev_domain); @@ -640,8 +672,10 @@ EXPORT_SYMBOL_GPL(plda_pcie_host_init); void plda_pcie_host_deinit(struct plda_pcie_rp *port) { + pci_lock_rescan_remove(); pci_stop_root_bus(port->bridge->bus); pci_remove_root_bus(port->bridge->bus); + pci_unlock_rescan_remove(); plda_pcie_irq_domain_deinit(port); diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c index 3caf53c6c0823..fab44054a5de2 100644 --- a/drivers/pci/controller/plda/pcie-starfive.c +++ b/drivers/pci/controller/plda/pcie-starfive.c @@ -55,7 +55,7 @@ struct starfive_jh7110_pcie { struct reset_control *resets; struct clk_bulk_data *clks; struct regmap *reg_syscon; - struct gpio_desc *power_gpio; + struct regulator *vpcie3v3; struct gpio_desc *reset_gpio; struct phy *phy; @@ -153,11 +153,13 @@ static int starfive_pcie_parse_dt(struct starfive_jh7110_pcie *pcie, return dev_err_probe(dev, PTR_ERR(pcie->reset_gpio), "failed to get perst-gpio\n"); - pcie->power_gpio = devm_gpiod_get_optional(dev, "enable", - GPIOD_OUT_LOW); - if (IS_ERR(pcie->power_gpio)) - return dev_err_probe(dev, PTR_ERR(pcie->power_gpio), - "failed to get power-gpio\n"); + pcie->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); + if (IS_ERR(pcie->vpcie3v3)) { + if (PTR_ERR(pcie->vpcie3v3) != -ENODEV) + return dev_err_probe(dev, PTR_ERR(pcie->vpcie3v3), + "failed to get vpcie3v3 regulator\n"); + pcie->vpcie3v3 = NULL; + } return 0; } @@ -270,8 +272,8 @@ static void starfive_pcie_host_deinit(struct plda_pcie_rp *plda) container_of(plda, struct starfive_jh7110_pcie, plda); starfive_pcie_clk_rst_deinit(pcie); - if (pcie->power_gpio) - gpiod_set_value_cansleep(pcie->power_gpio, 0); + if (pcie->vpcie3v3) + regulator_disable(pcie->vpcie3v3); starfive_pcie_disable_phy(pcie); } @@ -302,10 +304,15 @@ static int starfive_pcie_host_init(struct plda_pcie_rp *plda) ret = starfive_pcie_clk_rst_init(pcie); if (ret) - return ret; + goto err_disable_phy; - if (pcie->power_gpio) - gpiod_set_value_cansleep(pcie->power_gpio, 1); + if (pcie->vpcie3v3) { + ret = regulator_enable(pcie->vpcie3v3); + if (ret) { + dev_err_probe(dev, ret, "failed to enable vpcie3v3 regulator\n"); + goto err_clk_rst; + } + } if (pcie->reset_gpio) gpiod_set_value_cansleep(pcie->reset_gpio, 1); @@ -374,6 +381,13 @@ static int starfive_pcie_host_init(struct plda_pcie_rp *plda) dev_info(dev, "port link down\n"); return 0; + +err_clk_rst: + starfive_pcie_clk_rst_deinit(pcie); +err_disable_phy: + starfive_pcie_disable_phy(pcie); + + return ret; } static const struct plda_pcie_host_ops sf_host_ops = { @@ -405,7 +419,11 @@ static int starfive_pcie_probe(struct platform_device *pdev) return ret; pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + pm_runtime_disable(&pdev->dev); + return dev_err_probe(dev, ret, "failed to resume device\n"); + } plda->host_ops = &sf_host_ops; plda->num_events = PLDA_MAX_EVENT_NUM; @@ -431,9 +449,9 @@ static void starfive_pcie_remove(struct platform_device *pdev) { struct starfive_jh7110_pcie *pcie = platform_get_drvdata(pdev); - pm_runtime_put(&pdev->dev); - pm_runtime_disable(&pdev->dev); plda_pcie_host_deinit(&pcie->plda); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); platform_set_drvdata(pdev, NULL); } diff --git a/drivers/pci/ecam.c b/drivers/pci/ecam.c index 260b7de2dbd57..0becd957b236a 100644 --- a/drivers/pci/ecam.c +++ b/drivers/pci/ecam.c @@ -208,6 +208,19 @@ const struct pci_ecam_ops pci_generic_ecam_ops = { }; EXPORT_SYMBOL_GPL(pci_generic_ecam_ops); +/* CAM ops */ +const struct pci_ecam_ops pci_generic_cam_ops = { + .bus_shift = 16, + .pci_ops = { + .add_bus = pci_ecam_add_bus, + .remove_bus = pci_ecam_remove_bus, + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; +EXPORT_SYMBOL_GPL(pci_generic_cam_ops); + #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) /* ECAM ops for 32-bit access only (non-compliant) */ const struct pci_ecam_ops pci_32b_ops = { diff --git a/drivers/pci/hotplug/pnv_php.c b/drivers/pci/hotplug/pnv_php.c index c5345bff9a553..e9c8f7fb22134 100644 --- a/drivers/pci/hotplug/pnv_php.c +++ b/drivers/pci/hotplug/pnv_php.c @@ -813,7 +813,7 @@ static struct pnv_php_slot *pnv_php_alloc_slot(struct device_node *dn) if (dn->child && PCI_DN(dn->child)) php_slot->slot_no = PCI_SLOT(PCI_DN(dn->child)->devfn); else - php_slot->slot_no = -1; /* Placeholder slot */ + php_slot->slot_no = PCI_SLOT_PLACEHOLDER; /* Placeholder slot */ kref_init(&php_slot->kref); php_slot->state = PNV_PHP_STATE_INITIALIZED; diff --git a/drivers/pci/hotplug/rpaphp_slot.c b/drivers/pci/hotplug/rpaphp_slot.c index 779eab12e9810..ac8c512190df4 100644 --- a/drivers/pci/hotplug/rpaphp_slot.c +++ b/drivers/pci/hotplug/rpaphp_slot.c @@ -85,7 +85,7 @@ int rpaphp_register_slot(struct slot *slot) struct device_node *child; u32 my_index; int retval; - int slotno = -1; + int slotno = PCI_SLOT_PLACEHOLDER; dbg("%s registering slot:path[%pOF] index[%x], name[%s] pdomain[%x] type[%d]\n", __func__, slot->dn, slot->index, slot->name, diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 34d664139f48f..7e23328693065 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -860,6 +860,7 @@ void __pci_restore_msix_state(struct pci_dev *dev) { struct msi_desc *entry; bool write_msg; + u16 cmd; if (!dev->msix_enabled) return; @@ -869,6 +870,14 @@ void __pci_restore_msix_state(struct pci_dev *dev) pci_msix_clear_and_set_ctrl(dev, 0, PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); + /* + * The restored device state may not have Memory Space enabled. + * Since the MSI-X Table and PBA are in Memory Space, enable it + * while restoring them. + */ + pci_read_config_word(dev, PCI_COMMAND, &cmd); + pci_write_config_word(dev, PCI_COMMAND, cmd | PCI_COMMAND_MEMORY); + write_msg = arch_restore_msi_irqs(dev); scoped_guard (msi_descs_lock, &dev->dev) { @@ -879,6 +888,7 @@ void __pci_restore_msix_state(struct pci_dev *dev) } } + pci_write_config_word(dev, PCI_COMMAND, cmd); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index f9ead94d5ebb6..1219f39b1c69b 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -490,13 +490,6 @@ static void pci_device_remove(struct device *dev) /* Undo the runtime PM settings in local_pci_probe() */ pm_runtime_put_sync(dev); - /* - * If the device is still on, set the power state as "unknown", - * since it might change by the next time we load the driver. - */ - if (pci_dev->current_state == PCI_D0) - pci_dev->current_state = PCI_UNKNOWN; - /* * We would love to complain here if pci_dev->is_enabled is set, that * the driver should have called pci_disable_device(), but the @@ -871,7 +864,7 @@ static int pci_pm_suspend_noirq(struct device *dev) if (!pm) { pci_save_state(pci_dev); - goto Fixup; + goto set_unknown; } if (pm->suspend_noirq) { @@ -923,6 +916,7 @@ static int pci_pm_suspend_noirq(struct device *dev) goto Fixup; } +set_unknown: pci_pm_set_unknown_state(pci_dev); /* diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index edb9bc34c0c58..76966343e0186 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -723,7 +723,7 @@ static ssize_t pci_read_config(struct file *filp, struct kobject *kobj, else if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) size = 128; - if (off > size) + if (off >= size) return 0; if (off + count > size) { size -= off; @@ -804,7 +804,7 @@ static ssize_t pci_write_config(struct file *filp, struct kobject *kobj, add_taint(TAINT_USER, LOCKDEP_STILL_OK); } - if (off > dev->cfg_size) + if (off >= dev->cfg_size) return 0; if (off + count > dev->cfg_size) { size = dev->cfg_size - off; @@ -906,12 +906,30 @@ static ssize_t pci_read_legacy_io(struct file *filp, struct kobject *kobj, char *buf, loff_t off, size_t count) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + u32 val = 0; + int ret; /* Only support 1, 2 or 4 byte accesses */ if (count != 1 && count != 2 && count != 4) return -EINVAL; - return pci_legacy_read(bus, off, (u32 *)buf, count); + ret = pci_legacy_read(bus, off, &val, count); + if (ret < 0) + return ret; + + switch (count) { + case 1: + buf[0] = *(u8 *)&val; + break; + case 2: + put_unaligned_le16(*(u16 *)&val, buf); + break; + case 4: + put_unaligned_le32(val, buf); + break; + } + + return ret; } /** @@ -931,12 +949,24 @@ static ssize_t pci_write_legacy_io(struct file *filp, struct kobject *kobj, char *buf, loff_t off, size_t count) { struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj)); + u32 val; - /* Only support 1, 2 or 4 byte accesses */ - if (count != 1 && count != 2 && count != 4) + /* Only support 1, 2 or 4 byte accesses. */ + switch (count) { + case 1: + val = *(u8 *)buf; + break; + case 2: + val = get_unaligned_le16(buf); + break; + case 4: + val = get_unaligned_le32(buf); + break; + default: return -EINVAL; + } - return pci_legacy_write(bus, off, *(u32 *)buf, count); + return pci_legacy_write(bus, off, val, count); } /** @@ -1308,18 +1338,19 @@ void __weak pci_remove_resource_files(struct pci_dev *dev) { return; } * @off: file offset * @count: number of byte in input * - * writing anything except 0 enables it + * Writing a boolean value enables or disables the ROM display. */ static ssize_t pci_write_rom(struct file *filp, struct kobject *kobj, const struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count) { struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + bool enable; - if ((off == 0) && (*buf == '0') && (count == 2)) - pdev->rom_attr_enabled = 0; - else - pdev->rom_attr_enabled = 1; + if (kstrtobool(buf, &enable)) + return -EINVAL; + + pdev->rom_attr_enabled = enable; return count; } @@ -1575,6 +1606,9 @@ static ssize_t __resource_resize_store(struct device *dev, int n, int ret; u16 cmd; + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + if (kstrtoul(buf, 0, &size) < 0) return -EINVAL; diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 439045ab69319..6aa41fbf251c1 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -1305,7 +1305,18 @@ int pci_power_up(struct pci_dev *dev) bool need_restore; pci_power_t state; u16 pmcsr; + int ret; + /* + * When setting power state to D0, platform_pci_set_power_state() + * ensures main power is on. If it puts the device in D0, it also + * completes any required delays after the transition; if it leaves + * the device in D1, D2, or D3hot, we use the PM Capability to + * transition to D0. + * + * In all cases, the device is either Configuration-Ready or + * inaccessible upon return. + */ platform_pci_set_power_state(dev, PCI_D0); if (!dev->pm_cap) { @@ -1346,10 +1357,19 @@ int pci_power_up(struct pci_dev *dev) pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0); /* Mandatory transition delays; see PCI PM 1.2. */ - if (state == PCI_D3hot) + if (state == PCI_D3hot) { pci_dev_d3_sleep(dev); - else if (state == PCI_D2) + if (!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET)) { + ret = pci_dev_wait(dev, "power up D3hot->D0uninitialized", + PCIE_RESET_READY_POLL_MS); + if (ret) { + dev->current_state = PCI_D3cold; + return -EIO; + } + } + } else if (state == PCI_D2) { udelay(PCI_PM_D2_DELAY); + } end: dev->current_state = PCI_D0; @@ -4828,8 +4848,9 @@ static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe) static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe) { - if (dev->multifunction || dev->subordinate || !dev->slot || - dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) + if (dev->subordinate || !dev->slot || + dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || + (dev->multifunction && !dev->slot->per_func_slot)) return -ENOTTY; return pci_reset_hotplug_slot(dev->slot->hotplug, probe); diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 85f1f0cf62d61..9f90abe3e88ed 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -427,23 +427,32 @@ void pci_aer_exit(struct pci_dev *dev) #define AER_AGENT_REQUESTER 1 #define AER_AGENT_COMPLETER 2 #define AER_AGENT_TRANSMITTER 3 +#define AER_AGENT_COMPONENT 4 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ - 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) + 0 : PCI_ERR_UNC_COMP_TIME) #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 0 : PCI_ERR_UNC_COMP_ABORT) #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ - (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) + (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : \ + (PCI_ERR_UNC_POISON_BLK|PCI_ERR_UNC_ATOMEG| \ + PCI_ERR_UNC_DMWR_BLK|PCI_ERR_UNC_XLAT_BLK| \ + PCI_ERR_UNC_TLPPRE)) +#define AER_AGENT_COMPONENT_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_INTERNAL|PCI_ERR_COR_LOG_OVER) : \ + (PCI_ERR_UNC_INTN|PCI_ERR_UNC_SURPDN)) #define AER_GET_AGENT(t, e) \ ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ + (e & AER_AGENT_COMPONENT_MASK(t)) ? AER_AGENT_COMPONENT : \ AER_AGENT_RECEIVER) #define AER_PHYSICAL_LAYER_ERROR 0 #define AER_DATA_LINK_LAYER_ERROR 1 #define AER_TRANSACTION_LAYER_ERROR 2 +#define AER_GENERAL_ERROR 3 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ PCI_ERR_COR_RCVR : 0) @@ -451,11 +460,14 @@ void pci_aer_exit(struct pci_dev *dev) (PCI_ERR_COR_BAD_TLP| \ PCI_ERR_COR_BAD_DLLP| \ PCI_ERR_COR_REP_ROLL| \ - PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) + PCI_ERR_COR_REP_TIMER) : (PCI_ERR_UNC_DLP|PCI_ERR_UNC_SURPDN)) +#define AER_GENERAL_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ + (PCI_ERR_COR_INTERNAL|PCI_ERR_COR_LOG_OVER) : PCI_ERR_UNC_INTN) #define AER_GET_LAYER_ERROR(t, e) \ ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ + (e & AER_GENERAL_ERROR_MASK(t)) ? AER_GENERAL_ERROR : \ AER_TRANSACTION_LAYER_ERROR) /* @@ -470,7 +482,8 @@ static const char * const aer_error_severity_string[] = { static const char *aer_error_layer[] = { "Physical Layer", "Data Link Layer", - "Transaction Layer" + "Transaction Layer", + "General", }; static const char *aer_correctable_error_string[] = { @@ -547,7 +560,8 @@ static const char *aer_agent_string[] = { "Receiver ID", "Requester ID", "Completer ID", - "Transmitter ID" + "Transmitter ID", + "Component ID", }; #define aer_stats_dev_attr(name, stats_array, strings_array, \ @@ -938,7 +952,8 @@ void pci_print_aer(struct pci_dev *dev, int aer_severity, status = aer->uncor_status; mask = aer->uncor_mask; info.level = KERN_ERR; - tlp_header_valid = tlp_header_logged(status, aer->cap_control); + tlp_header_valid = tlp_header_logged(status & ~mask, + aer->cap_control); } info.status = status; @@ -1427,7 +1442,7 @@ int aer_get_device_error_info(struct aer_err_info *info, int i) pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); info->first_error = PCI_ERR_CAP_FEP(aercc); - if (tlp_header_logged(info->status, aercc)) { + if (tlp_header_logged(info->status & ~info->mask, aercc)) { info->tlp_header_valid = 1; pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, aer + PCI_ERR_PREFIX_LOG, diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c index a1f275259f7dd..dac81130edd3d 100644 --- a/drivers/pci/pcie/aspm.c +++ b/drivers/pci/pcie/aspm.c @@ -821,6 +821,7 @@ static void pcie_aspm_override_default_link_state(struct pcie_link_state *link) static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) { struct pci_dev *child = link->downstream, *parent = link->pdev; + struct pci_dev *fn; u16 parent_lnkctl, child_lnkctl; struct pci_bus *linkbus = parent->subordinate; @@ -854,10 +855,11 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Disable L0s/L1 before updating L1SS config */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(child, PCI_EXP_LNKCTL, - child_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, - parent_lnkctl & ~PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0); } /* @@ -887,8 +889,13 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) /* Restore L0s/L1 if they were enabled */ if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, child_lnkctl) || FIELD_GET(PCI_EXP_LNKCTL_ASPMC, parent_lnkctl)) { - pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_lnkctl); - pcie_capability_write_word(child, PCI_EXP_LNKCTL, child_lnkctl); + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + parent_lnkctl & PCI_EXP_LNKCTL_ASPMC); + list_for_each_entry(fn, &linkbus->devices, bus_list) + pcie_capability_clear_and_set_word(fn, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + child_lnkctl & PCI_EXP_LNKCTL_ASPMC); } /* Save default state */ diff --git a/drivers/pci/proc.c b/drivers/pci/proc.c index 9348a0fb80847..ef19c7f049af6 100644 --- a/drivers/pci/proc.c +++ b/drivers/pci/proc.c @@ -14,10 +14,13 @@ #include #include #include +#include +#include #include #include "pci.h" static int proc_initialized; /* = 0 */ +static DEFINE_MUTEX(pci_proc_lock); static loff_t proc_bus_pci_lseek(struct file *file, loff_t off, int whence) { @@ -38,13 +41,16 @@ static ssize_t proc_bus_pci_read(struct file *file, char __user *buf, * undefined locations (think of Intel PIIX4 as a typical example). */ - if (capable(CAP_SYS_ADMIN)) + if (file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN)) size = dev->cfg_size; else if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS) size = 128; else size = 64; + if (!nbytes) + return 0; + if (pos >= size) return 0; if (nbytes >= size) @@ -121,6 +127,15 @@ static ssize_t proc_bus_pci_write(struct file *file, const char __user *buf, if (ret) return ret; + if (!nbytes) + return 0; + + if (resource_is_exclusive(&dev->driver_exclusive_resource, pos, nbytes)) { + pci_warn_once(dev, "%s: Unexpected write to kernel-exclusive config offset %x", + current->comm, pos); + add_taint(TAINT_USER, LOCKDEP_STILL_OK); + } + if (pos >= size) return 0; if (nbytes >= size) @@ -416,40 +431,64 @@ static const struct seq_operations proc_bus_pci_devices_op = { static struct proc_dir_entry *proc_bus_pci_dir; -int pci_proc_attach_device(struct pci_dev *dev) +static int __pci_proc_attach_bus(struct pci_bus *bus) { - struct pci_bus *bus = dev->bus; - struct proc_dir_entry *e; + struct proc_dir_entry *dir; char name[16]; + lockdep_assert_held(&pci_proc_lock); + if (!proc_initialized) return -EACCES; - if (!bus->procdir) { - if (pci_proc_domain(bus)) { - sprintf(name, "%04x:%02x", pci_domain_nr(bus), - bus->number); - } else { - sprintf(name, "%02x", bus->number); - } - bus->procdir = proc_mkdir(name, proc_bus_pci_dir); - if (!bus->procdir) - return -ENOMEM; - } + if (bus->procdir) + return 0; + + if (pci_proc_domain(bus)) + sprintf(name, "%04x:%02x", pci_domain_nr(bus), bus->number); + else + sprintf(name, "%02x", bus->number); + + dir = proc_mkdir(name, proc_bus_pci_dir); + if (!dir) + return -ENOMEM; + + bus->procdir = dir; + + return 0; +} + +int pci_proc_attach_device(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct proc_dir_entry *entry; + char name[16]; + int ret; + + guard(mutex)(&pci_proc_lock); + + if (dev->procent) + return 0; + + ret = __pci_proc_attach_bus(bus); + if (ret) + return ret; sprintf(name, "%02x.%x", PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn)); - e = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUSR, bus->procdir, - &proc_bus_pci_ops, dev); - if (!e) + entry = proc_create_data(name, S_IFREG | S_IRUGO | S_IWUSR, + bus->procdir, &proc_bus_pci_ops, dev); + if (!entry) return -ENOMEM; - proc_set_size(e, dev->cfg_size); - dev->procent = e; + + proc_set_size(entry, dev->cfg_size); + dev->procent = entry; return 0; } int pci_proc_detach_device(struct pci_dev *dev) { + guard(mutex)(&pci_proc_lock); proc_remove(dev->procent); dev->procent = NULL; return 0; @@ -457,19 +496,27 @@ int pci_proc_detach_device(struct pci_dev *dev) int pci_proc_detach_bus(struct pci_bus *bus) { + guard(mutex)(&pci_proc_lock); proc_remove(bus->procdir); + bus->procdir = NULL; return 0; } static int __init pci_proc_init(void) { struct pci_dev *dev = NULL; - proc_bus_pci_dir = proc_mkdir("bus/pci", NULL); - proc_create_seq("devices", 0, proc_bus_pci_dir, - &proc_bus_pci_devices_op); - proc_initialized = 1; + + scoped_guard(mutex, &pci_proc_lock) { + proc_bus_pci_dir = proc_mkdir("bus/pci", NULL); + proc_create_seq("devices", 0, proc_bus_pci_dir, + &proc_bus_pci_devices_op); + proc_initialized = 1; + } + + pci_lock_rescan_remove(); for_each_pci_dev(dev) pci_proc_attach_device(dev); + pci_unlock_rescan_remove(); return 0; } diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c index 5fa2c388c35b4..dc62f3b50df3e 100644 --- a/drivers/pci/quirks.c +++ b/drivers/pci/quirks.c @@ -2506,6 +2506,9 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10f1, quirk_disable_aspm_l0s); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x10f4, quirk_disable_aspm_l0s); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1508, quirk_disable_aspm_l0s); +/* Realtek RTS525A generates a Replay Timer Timeout storm when L0s is enabled. */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_REALTEK, 0x525a, quirk_disable_aspm_l0s); + static void quirk_disable_aspm_l0s_l1(struct pci_dev *dev) { pcie_aspm_remove_cap(dev, @@ -3776,6 +3779,9 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003c, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0033, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x0034, quirk_no_bus_reset); DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ATHEROS, 0x003e, quirk_no_bus_reset); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_QCOM, 0x1103, quirk_no_bus_reset); /* WCN6855 */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_QCOM, 0x1107, quirk_no_bus_reset); /* WCN7850 */ +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_QCOM, 0x0308, quirk_no_bus_reset); /* SDX62/SDX65 */ /* * Root port on some Cavium CN8xxx chips do not successfully complete a bus @@ -5331,7 +5337,7 @@ static void pci_quirk_enable_intel_rp_mpc_acs(struct pci_dev *dev) if (!(mpc & INTEL_MPC_REG_IRBNCE)) { pci_info(dev, "Enabling MPC IRBNCE\n"); mpc |= INTEL_MPC_REG_IRBNCE; - pci_write_config_word(dev, INTEL_MPC_REG, mpc); + pci_write_config_dword(dev, INTEL_MPC_REG, mpc); } } @@ -5564,6 +5570,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x443, quirk_intel_qat_vf_cap); * Intel 82579LM Gigabit Ethernet Controller 0x1502 * Intel 82579V Gigabit Ethernet Controller 0x1503 * Mediatek MT7922 802.11ax PCI Express Wireless Network Adapter + * Mediatek MT7925 802.11be PCI Express Wireless Network Adapter */ static void quirk_no_flr(struct pci_dev *dev) { @@ -5576,6 +5583,7 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_AMD, 0x7901, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1502, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_INTEL, 0x1503, quirk_no_flr); DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_MEDIATEK, 0x0616, quirk_no_flr); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_MEDIATEK, 0x7925, quirk_no_flr); /* FLR may cause the SolidRun SNET DPU (rev 0x1) to hang */ static void quirk_no_flr_snet(struct pci_dev *dev) @@ -6210,6 +6218,10 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0xb404, pci_fixup_pericom_acs_store_forward); DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0xb404, pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_PERICOM, 0x2608, + pci_fixup_pericom_acs_store_forward); +DECLARE_PCI_FIXUP_RESUME(PCI_VENDOR_ID_PERICOM, 0x2608, + pci_fixup_pericom_acs_store_forward); static void nvidia_ion_ahci_fixup(struct pci_dev *pdev) { diff --git a/drivers/pci/slot.c b/drivers/pci/slot.c index 50fb3eb595fe6..179aa68612539 100644 --- a/drivers/pci/slot.c +++ b/drivers/pci/slot.c @@ -37,11 +37,20 @@ static const struct sysfs_ops pci_slot_sysfs_ops = { static ssize_t address_read_file(struct pci_slot *slot, char *buf) { - if (slot->number == 0xff) + if (slot->number == PCI_SLOT_PLACEHOLDER) return sysfs_emit(buf, "%04x:%02x\n", pci_domain_nr(slot->bus), slot->bus->number); + /* + * Preserve legacy ABI expectations that hotplug drivers that manage + * multiple devices per slot emit 0 for the device number. + */ + if (slot->number == PCI_SLOT_ALL_DEVICES) + return sysfs_emit(buf, "%04x:%02x:00\n", + pci_domain_nr(slot->bus), + slot->bus->number); + return sysfs_emit(buf, "%04x:%02x:%02x\n", pci_domain_nr(slot->bus), slot->bus->number, @@ -63,6 +72,23 @@ static ssize_t cur_speed_read_file(struct pci_slot *slot, char *buf) return bus_speed_read(slot->bus->cur_bus_speed, buf); } +static bool pci_dev_matches_slot(struct pci_dev *dev, struct pci_slot *slot) +{ + if (slot->per_func_slot) + return dev->devfn == slot->number; + + return slot->number == PCI_SLOT_ALL_DEVICES || + PCI_SLOT(dev->devfn) == slot->number; +} + +static bool pci_slot_enabled_per_func(void) +{ + if (IS_ENABLED(CONFIG_S390)) + return true; + + return false; +} + static void pci_slot_release(struct kobject *kobj) { struct pci_dev *dev; @@ -73,7 +99,7 @@ static void pci_slot_release(struct kobject *kobj) down_read(&pci_bus_sem); list_for_each_entry(dev, &slot->bus->devices, bus_list) - if (PCI_SLOT(dev->devfn) == slot->number) + if (pci_dev_matches_slot(dev, slot)) dev->slot = NULL; up_read(&pci_bus_sem); @@ -166,7 +192,7 @@ void pci_dev_assign_slot(struct pci_dev *dev) mutex_lock(&pci_slot_mutex); list_for_each_entry(slot, &dev->bus->slots, list) - if (PCI_SLOT(dev->devfn) == slot->number) + if (pci_dev_matches_slot(dev, slot)) dev->slot = slot; mutex_unlock(&pci_slot_mutex); } @@ -188,7 +214,8 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr) /** * pci_create_slot - create or increment refcount for physical PCI slot * @parent: struct pci_bus of parent bridge - * @slot_nr: PCI_SLOT(pci_dev->devfn) or -1 for placeholder + * @slot_nr: PCI_SLOT(pci_dev->devfn), PCI_SLOT_PLACEHOLDER for placeholder, or + * PCI_SLOT_ALL_DEVICES * @name: user visible string presented in /sys/bus/pci/slots/ * @hotplug: set if caller is hotplug driver, NULL otherwise * @@ -213,15 +240,26 @@ static struct pci_slot *get_slot(struct pci_bus *parent, int slot_nr) * In most cases, @pci_bus, @slot_nr will be sufficient to uniquely identify * a slot. There is one notable exception - pSeries (rpaphp), where the * @slot_nr cannot be determined until a device is actually inserted into - * the slot. In this scenario, the caller may pass -1 for @slot_nr. + * the slot. In this scenario, the caller may pass PCI_SLOT_PLACEHOLDER for @slot_nr. * * The following semantics are imposed when the caller passes @slot_nr == - * -1. First, we no longer check for an existing %struct pci_slot, as there - * may be many slots with @slot_nr of -1. The other change in semantics is - * user-visible, which is the 'address' parameter presented in sysfs will - * consist solely of a dddd:bb tuple, where dddd is the PCI domain of the - * %struct pci_bus and bb is the bus number. In other words, the devfn of - * the 'placeholder' slot will not be displayed. + * PCI_SLOT_PLACEHOLDER. First, we no longer check for an existing %struct + * pci_slot, as there may be many slots with @slot_nr of + * PCI_SLOT_PLACEHOLDER. The other change in semantics is user-visible, + * which is the 'address' parameter presented in sysfs will consist solely + * of a dddd:bb tuple, where dddd is the PCI domain of the %struct pci_bus + * and bb is the bus number. In other words, the devfn of the 'placeholder' + * slot will not be displayed. + * + * Bus-wide slots: + * For PCIe hotplug, the physical slot encompasses the entire secondary + * bus, not just a single device number. If the device supports ARI and ARI + * Forwarding is enabled in the upstream bridge, a multi-function device + * may include functions that appear to have several different device + * numbers, i.e., PCI_SLOT() values. Pass @slot_nr == PCI_SLOT_ALL_DEVICES + * to create a slot that matches all devices on the bus. Unlike placeholder + * slots, bus-wide slots go through normal slot lookup and reuse existing + * slots if present. */ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, const char *name, @@ -234,7 +272,7 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, mutex_lock(&pci_slot_mutex); - if (slot_nr == -1) + if (slot_nr == PCI_SLOT_PLACEHOLDER) goto placeholder; /* @@ -265,6 +303,9 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, slot->bus = pci_bus_get(parent); slot->number = slot_nr; + if (pci_slot_enabled_per_func()) + slot->per_func_slot = 1; + slot->kobj.kset = pci_slots_kset; slot_name = make_slot_name(name); @@ -285,7 +326,7 @@ struct pci_slot *pci_create_slot(struct pci_bus *parent, int slot_nr, down_read(&pci_bus_sem); list_for_each_entry(dev, &parent->devices, bus_list) - if (PCI_SLOT(dev->devfn) == slot_nr) + if (pci_dev_matches_slot(dev, slot)) dev->slot = slot; up_read(&pci_bus_sem); diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c index 5ff84fb8fb0f4..f32ddfa79da99 100644 --- a/drivers/pci/switch/switchtec.c +++ b/drivers/pci/switch/switchtec.c @@ -1852,6 +1852,22 @@ static const struct pci_device_id switchtec_pci_tbl[] = { SWITCHTEC_PCI_DEVICE(0x5552, SWITCHTEC_GEN5), /* PAXA 52XG5 */ SWITCHTEC_PCI_DEVICE(0x5536, SWITCHTEC_GEN5), /* PAXA 36XG5 */ SWITCHTEC_PCI_DEVICE(0x5528, SWITCHTEC_GEN5), /* PAXA 28XG5 */ + SWITCHTEC_PCI_DEVICE(0x6048, SWITCHTEC_GEN6), /* PFXs 48XG6 */ + SWITCHTEC_PCI_DEVICE(0x6064, SWITCHTEC_GEN6), /* PFXs 64XG6 */ + SWITCHTEC_PCI_DEVICE(0x6044, SWITCHTEC_GEN6), /* PFXs 144XG6 */ + SWITCHTEC_PCI_DEVICE(0x6060, SWITCHTEC_GEN6), /* PFXs 160XG6 */ + SWITCHTEC_PCI_DEVICE(0x6148, SWITCHTEC_GEN6), /* PSXs 48XG6 */ + SWITCHTEC_PCI_DEVICE(0x6164, SWITCHTEC_GEN6), /* PSXs 64XG6 */ + SWITCHTEC_PCI_DEVICE(0x6144, SWITCHTEC_GEN6), /* PSXs 144XG6 */ + SWITCHTEC_PCI_DEVICE(0x6160, SWITCHTEC_GEN6), /* PSXs 160XG6 */ + SWITCHTEC_PCI_DEVICE(0x6248, SWITCHTEC_GEN6), /* PFX 48XG6 */ + SWITCHTEC_PCI_DEVICE(0x6264, SWITCHTEC_GEN6), /* PFX 64XG6 */ + SWITCHTEC_PCI_DEVICE(0x6244, SWITCHTEC_GEN6), /* PFX 144XG6 */ + SWITCHTEC_PCI_DEVICE(0x6260, SWITCHTEC_GEN6), /* PFX 160XG6 */ + SWITCHTEC_PCI_DEVICE(0x6348, SWITCHTEC_GEN6), /* PSX 48XG6 */ + SWITCHTEC_PCI_DEVICE(0x6364, SWITCHTEC_GEN6), /* PSX 64XG6 */ + SWITCHTEC_PCI_DEVICE(0x6344, SWITCHTEC_GEN6), /* PSX 144XG6 */ + SWITCHTEC_PCI_DEVICE(0x6360, SWITCHTEC_GEN6), /* PSX 160XG6 */ SWITCHTEC_PCI100X_DEVICE(0x1001, SWITCHTEC_GEN4), /* PCI1001 16XG4 */ SWITCHTEC_PCI100X_DEVICE(0x1002, SWITCHTEC_GEN4), /* PCI1002 12XG4 */ SWITCHTEC_PCI100X_DEVICE(0x1003, SWITCHTEC_GEN4), /* PCI1003 16XG4 */ diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 69c5cc8f56067..265c10f2c8b07 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -1351,7 +1351,7 @@ static int branch_records_alloc(struct arm_pmu *armpmu) struct pmu_hw_events *events_cpu; events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); - events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); + events_cpu->branch_stack = kzalloc(size, GFP_KERNEL); if (!events_cpu->branch_stack) return -ENOMEM; } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index e4e4e63c64c42..7bc24db50d1ff 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -552,7 +552,7 @@ static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) * the page boundary following it. Keep the tail boundary if * that's lower. */ - if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); if (limit > head) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 68a54d97d2a8a..39b46550a5109 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -635,7 +635,7 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) cfg = cxl_pmu_config2_get_hdm_decoder(event); else cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ - writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); } cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); diff --git a/drivers/perf/riscv_pmu_legacy.c b/drivers/perf/riscv_pmu_legacy.c index 93c8e0fdb5898..3af1d694b1f41 100644 --- a/drivers/perf/riscv_pmu_legacy.c +++ b/drivers/perf/riscv_pmu_legacy.c @@ -111,8 +111,9 @@ static void pmu_legacy_init(struct riscv_pmu *pmu) { pr_info("Legacy PMU implementation is available\n"); - pmu->cmask = BIT(RISCV_PMU_LEGACY_CYCLE) | - BIT(RISCV_PMU_LEGACY_INSTRET); + bitmap_zero(pmu->cmask, RISCV_MAX_COUNTERS); + set_bit(RISCV_PMU_LEGACY_CYCLE, pmu->cmask); + set_bit(RISCV_PMU_LEGACY_INSTRET, pmu->cmask); pmu->ctr_start = pmu_legacy_ctr_start; pmu->ctr_stop = NULL; pmu->event_map = pmu_legacy_event_map; diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 7dd282da67cef..25f3a45dd7869 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -98,7 +98,7 @@ static unsigned int riscv_pmu_irq_mask; static unsigned int riscv_pmu_irq; /* Cache the available counters in a bitmask */ -static unsigned long cmask; +static DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); static int pmu_event_find_cache(u64 config); struct sbi_pmu_event_data { @@ -360,16 +360,38 @@ static int pmu_sbi_check_event_info(void) return result; } +static struct sbiret pmu_sbi_ctr_cfg_match(unsigned long cbase, + unsigned long ctr_mask, + unsigned long cflags, + unsigned long event_idx, + u64 config) +{ +#if defined(CONFIG_32BIT) + return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + ctr_mask, cflags, event_idx, config, config >> 32); +#else + return sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, + ctr_mask, cflags, event_idx, config, 0); +#endif +} + static void pmu_sbi_check_event(struct sbi_pmu_event_data *edata) { - struct sbiret ret; + struct sbiret ret = { .error = SBI_ERR_NOT_SUPPORTED }; + int i; - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, - 0, cmask, 0, edata->event_idx, 0, 0); - if (!ret.error) { - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); - } else if (ret.error == SBI_ERR_NOT_SUPPORTED) { + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!cmask[i]) + continue; + ret = pmu_sbi_ctr_cfg_match(i * BITS_PER_LONG, cmask[i], 0, + edata->event_idx, 0); + if (!ret.error) { + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, + ret.value, 0x1, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + return; + } + } + if (ret.error == SBI_ERR_NOT_SUPPORTED) { /* This event cannot be monitored by any counter */ edata->event_idx = -ENOENT; } @@ -489,10 +511,10 @@ int riscv_pmu_get_hpm_info(u32 *hw_ctr_width, u32 *num_hw_ctr) union sbi_pmu_ctr_info *info; u32 hpm_width = 0, hpm_count = 0; - if (!cmask) + if (bitmap_empty(cmask, RISCV_MAX_COUNTERS)) return -EINVAL; - for_each_set_bit(i, &cmask, RISCV_MAX_COUNTERS) { + for_each_set_bit(i, cmask, RISCV_MAX_COUNTERS) { info = &pmu_ctr_list[i]; if (!info) continue; @@ -541,8 +563,8 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); struct sbiret ret; - int idx; - uint64_t cbase = 0, cmask = rvpmu->cmask; + int idx, i; + u64 cbase = 0, cmask = 0; unsigned long cflags = 0; cflags = pmu_sbi_get_filter_flags(event); @@ -563,14 +585,21 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) } /* retrieve the available counter index */ -#if defined(CONFIG_32BIT) - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, - cmask, cflags, hwc->event_base, hwc->config, - hwc->config >> 32); -#else - ret = sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_CFG_MATCH, cbase, - cmask, cflags, hwc->event_base, hwc->config, 0); -#endif + if (cmask) { + ret = pmu_sbi_ctr_cfg_match(cbase, cmask, cflags, hwc->event_base, + hwc->config); + } else { + ret.error = SBI_ERR_NOT_SUPPORTED; + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!rvpmu->cmask[i]) + continue; + cbase = i * BITS_PER_LONG; + ret = pmu_sbi_ctr_cfg_match(cbase, rvpmu->cmask[i], cflags, + hwc->event_base, hwc->config); + if (!ret.error) + break; + } + } if (ret.error) { pr_debug("Not able to find a counter for event %lx config %llx\n", hwc->event_base, hwc->config); @@ -578,7 +607,7 @@ static int pmu_sbi_ctr_get_idx(struct perf_event *event) } idx = ret.value; - if (!test_bit(idx, &rvpmu->cmask) || !pmu_ctr_list[idx].value) + if (!test_bit(idx, rvpmu->cmask) || !pmu_ctr_list[idx].value) return -ENOENT; /* Additional sanity check for the counter id */ @@ -882,7 +911,7 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) /* The logical counter ids are not expected to be contiguous */ continue; - *mask |= BIT(i); + set_bit(i, mask); cinfo.value = ret.value; if (cinfo.type == SBI_PMU_CTR_TYPE_FW) @@ -899,12 +928,19 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) { + int i; + /* * No need to check the error because we are disabling all the counters * which may include counters that are not enabled yet. */ - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - 0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + if (!pmu->cmask[i]) + continue; + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, + i * BITS_PER_LONG, pmu->cmask[i], + SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + } } static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) @@ -1003,7 +1039,7 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { - if (ctr_ovf_mask & BIT(idx)) { + if (ctr_ovf_mask & BIT_ULL(idx)) { event = cpu_hw_evt->events[idx]; hwc = &event->hw; max_period = riscv_pmu_ctr_get_width_mask(event); @@ -1051,11 +1087,13 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) u64 overflowed_ctrs = 0; struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); - struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; + struct riscv_pmu_snapshot_data *sdata; if (WARN_ON_ONCE(!cpu_hw_evt)) return IRQ_NONE; + sdata = cpu_hw_evt->snapshot_addr; + /* Firmware counter don't support overflow yet */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { @@ -1110,14 +1148,14 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hidx = info->csr - CSR_CYCLE; /* check if the corresponding bit is set in scountovf or overflow mask in shmem */ - if (!(overflow & BIT(hidx))) + if (!(overflow & BIT_ULL(hidx))) continue; /* * Keep a track of overflowed counters so that they can be started * with updated initial value. */ - overflowed_ctrs |= BIT(lidx); + overflowed_ctrs |= BIT_ULL(lidx); hw_evt = &event->hw; /* Update the event states here so that we know the state while reading */ hw_evt->state |= PERF_HES_STOPPED; @@ -1443,7 +1481,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } /* cache all the information about counters now */ - if (pmu_sbi_get_ctrinfo(num_counters, &cmask)) + if (pmu_sbi_get_ctrinfo(num_counters, cmask)) goto out_free; ret = pmu_sbi_setup_irqs(pmu, pdev); @@ -1455,7 +1493,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) pmu->pmu.attr_groups = riscv_pmu_attr_groups; pmu->pmu.parent = &pdev->dev; - pmu->cmask = cmask; + bitmap_copy(pmu->cmask, cmask, RISCV_MAX_COUNTERS); pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; pmu->event_map = pmu_sbi_event_map; diff --git a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c index 8e7b6e10e1f0b..b4f602965c994 100644 --- a/drivers/phy/freescale/phy-fsl-imx8mq-usb.c +++ b/drivers/phy/freescale/phy-fsl-imx8mq-usb.c @@ -166,9 +166,9 @@ static struct typec_switch_dev *tca_blk_get_typec_switch(struct platform_device return sw; } -static void tca_blk_put_typec_switch(struct typec_switch_dev *sw) +static void tca_blk_put_typec_switch(void *data) { - typec_switch_unregister(sw); + typec_switch_unregister(data); } static void tca_blk_orientation_set(struct tca_blk *tca, @@ -241,6 +241,7 @@ static struct tca_blk *imx95_usb_phy_get_tca(struct platform_device *pdev, struct device *dev = &pdev->dev; struct resource *res; struct tca_blk *tca; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (!res) @@ -259,17 +260,11 @@ static struct tca_blk *imx95_usb_phy_get_tca(struct platform_device *pdev, tca->orientation = TYPEC_ORIENTATION_NORMAL; tca->sw = tca_blk_get_typec_switch(pdev, imx_phy); - return tca; -} - -static void imx95_usb_phy_put_tca(struct imx8mq_usb_phy *imx_phy) -{ - struct tca_blk *tca = imx_phy->tca; - - if (!tca) - return; + ret = devm_add_action_or_reset(&pdev->dev, tca_blk_put_typec_switch, tca->sw); + if (ret) + return ERR_PTR(ret); - tca_blk_put_typec_switch(tca->sw); + return tca; } static u32 phy_tx_vref_tune_from_property(u32 percent) @@ -702,9 +697,7 @@ static int imx8mq_usb_phy_probe(struct platform_device *pdev) static void imx8mq_usb_phy_remove(struct platform_device *pdev) { - struct imx8mq_usb_phy *imx_phy = platform_get_drvdata(pdev); - imx95_usb_phy_put_tca(imx_phy); } static struct platform_driver imx8mq_usb_phy_driver = { diff --git a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c index 68f1ba8fec4ad..275df38256de3 100644 --- a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c +++ b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c @@ -221,7 +221,7 @@ static int m31eusb2_phy_init(struct phy *uphy) disable_vreg: regulator_bulk_disable(M31_EUSB_NUM_VREGS, phy->vregs); - return 0; + return ret; } static int m31eusb2_phy_exit(struct phy *uphy) @@ -285,7 +285,7 @@ static int m31eusb2_phy_probe(struct platform_device *pdev) phy_set_drvdata(phy->phy, phy); - phy->repeater = devm_of_phy_get_by_index(dev, dev->of_node, 0); + phy->repeater = devm_phy_optional_get(dev, NULL); if (IS_ERR(phy->repeater)) return dev_err_probe(dev, PTR_ERR(phy->repeater), "failed to get repeater\n"); diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c index 7b5af30f1d028..3ac29c6eaa68d 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c @@ -29,7 +29,12 @@ #include "phy-qcom-qmp-common.h" #include "phy-qcom-qmp.h" +#include "phy-qcom-qmp-pcs-aon-v6.h" +#include "phy-qcom-qmp-pcs-aon-v8.h" #include "phy-qcom-qmp-pcs-misc-v3.h" +#include "phy-qcom-qmp-pcs-misc-v4.h" +#include "phy-qcom-qmp-pcs-misc-v5.h" +#include "phy-qcom-qmp-pcs-misc-v8.h" #include "phy-qcom-qmp-pcs-usb-v4.h" #include "phy-qcom-qmp-pcs-usb-v5.h" #include "phy-qcom-qmp-pcs-usb-v6.h" @@ -42,6 +47,9 @@ #include "phy-qcom-qmp-dp-phy-v4.h" #include "phy-qcom-qmp-dp-phy-v5.h" #include "phy-qcom-qmp-dp-phy-v6.h" +#include "phy-qcom-qmp-dp-phy-v8.h" + +#include "phy-qcom-qmp-usb43-pcs-v8.h" /* QPHY_V3_DP_COM_RESET_OVRD_CTRL register bits */ /* DP PHY soft reset */ @@ -78,6 +86,7 @@ enum qphy_reg_layout { QPHY_PCS_AUTONOMOUS_MODE_CTRL, QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR, QPHY_PCS_POWER_DOWN_CONTROL, + QPHY_PCS_CLAMP_ENABLE, QPHY_COM_RESETSM_CNTRL, QPHY_COM_C_READY_STATUS, @@ -93,6 +102,8 @@ enum qphy_reg_layout { QPHY_TX_HIGHZ_DRVR_EN, QPHY_TX_TRANSCEIVER_BIAS_EN, + QPHY_AON_TOGGLE_ENABLE, + QPHY_DP_AON_TOGGLE_ENABLE, /* Keep last to ensure regs_layout arrays are properly initialized */ QPHY_LAYOUT_SIZE }; @@ -105,6 +116,8 @@ static const unsigned int qmp_v3_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V3_PCS_AUTONOMOUS_MODE_CTRL, [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V3_PCS_LFPS_RXTERM_IRQ_CLEAR, + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V3_PCS_MISC_CLAMP_ENABLE, + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V3_COM_RESETSM_CNTRL, [QPHY_COM_C_READY_STATUS] = QSERDES_V3_COM_C_READY_STATUS, [QPHY_COM_CMN_STATUS] = QSERDES_V3_COM_CMN_STATUS, @@ -130,6 +143,8 @@ static const unsigned int qmp_v45_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V4_PCS_USB3_AUTONOMOUS_MODE_CTRL, [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V4_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR, + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V4_PCS_MISC_CLAMP_ENABLE, + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V4_COM_RESETSM_CNTRL, [QPHY_COM_C_READY_STATUS] = QSERDES_V4_COM_C_READY_STATUS, [QPHY_COM_CMN_STATUS] = QSERDES_V4_COM_CMN_STATUS, @@ -155,6 +170,8 @@ static const unsigned int qmp_v5_5nm_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V5_PCS_USB3_AUTONOMOUS_MODE_CTRL, [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V5_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR, + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V5_PCS_MISC_CLAMP_ENABLE, + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V5_COM_RESETSM_CNTRL, [QPHY_COM_C_READY_STATUS] = QSERDES_V5_COM_C_READY_STATUS, [QPHY_COM_CMN_STATUS] = QSERDES_V5_COM_CMN_STATUS, @@ -180,6 +197,8 @@ static const unsigned int qmp_v6_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V6_PCS_USB3_AUTONOMOUS_MODE_CTRL, [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V6_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR, + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V6_PCS_AON_CLAMP_ENABLE, + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V6_COM_RESETSM_CNTRL, [QPHY_COM_C_READY_STATUS] = QSERDES_V6_COM_C_READY_STATUS, [QPHY_COM_CMN_STATUS] = QSERDES_V6_COM_CMN_STATUS, @@ -205,6 +224,8 @@ static const unsigned int qmp_v6_n4_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V6_PCS_USB3_AUTONOMOUS_MODE_CTRL, [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V6_PCS_USB3_LFPS_RXTERM_IRQ_CLEAR, + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V6_PCS_AON_CLAMP_ENABLE, + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V6_COM_RESETSM_CNTRL, [QPHY_COM_C_READY_STATUS] = QSERDES_V6_COM_C_READY_STATUS, [QPHY_COM_CMN_STATUS] = QSERDES_V6_COM_CMN_STATUS, @@ -245,6 +266,237 @@ static const unsigned int qmp_v8_usb3phy_regs_layout[QPHY_LAYOUT_SIZE] = { [QPHY_TX_TRANSCEIVER_BIAS_EN] = QSERDES_V8_TX_TRANSCEIVER_BIAS_EN, }; +static const unsigned int qmp_v8_n3_usb43dpphy_regs_layout[QPHY_LAYOUT_SIZE] = { + [QPHY_SW_RESET] = QPHY_V8_USB43_PCS_SW_RESET, + [QPHY_START_CTRL] = QPHY_V8_USB43_PCS_START_CONTROL, + [QPHY_PCS_STATUS] = QPHY_V8_USB43_PCS_PCS_STATUS1, + [QPHY_PCS_POWER_DOWN_CONTROL] = QPHY_V8_USB43_PCS_POWER_DOWN_CONTROL, + + /* In PCS_USB */ + [QPHY_PCS_AUTONOMOUS_MODE_CTRL] = QPHY_V8_PCS_USB_AUTONOMOUS_MODE_CTRL, + [QPHY_PCS_LFPS_RXTERM_IRQ_CLEAR] = QPHY_V8_PCS_USB_LFPS_RXTERM_IRQ_CLEAR, + + [QPHY_PCS_CLAMP_ENABLE] = QPHY_V8_PCS_AON_USB3_AON_CLAMP_ENABLE, + [QPHY_AON_TOGGLE_ENABLE] = QPHY_V8_PCS_AON_USB3_AON_TOGGLE_ENABLE, + [QPHY_DP_AON_TOGGLE_ENABLE] = QPHY_V8_PCS_AON_DP_AON_TOGGLE_ENABLE, + + [QPHY_COM_RESETSM_CNTRL] = QSERDES_V8_COM_RESETSM_CNTRL, + [QPHY_COM_C_READY_STATUS] = QSERDES_V8_COM_C_READY_STATUS, + [QPHY_COM_CMN_STATUS] = QSERDES_V8_COM_CMN_STATUS, + [QPHY_COM_BIAS_EN_CLKBUFLR_EN] = QSERDES_V8_COM_BIAS_EN_CLKBUFLR_EN, + + [QPHY_DP_PHY_STATUS] = QSERDES_V8_DP_PHY_STATUS, + [QPHY_DP_PHY_VCO_DIV] = QSERDES_V8_DP_PHY_VCO_DIV, + + [QPHY_TX_TX_DRV_LVL] = QSERDES_V8_LALB_TX0_DRV_LVL, + [QPHY_TX_TX_EMP_POST1_LVL] = QSERDES_V8_LALB_TX0_EMP_POST1_LVL, + [QPHY_TX_HIGHZ_DRVR_EN] = QSERDES_V8_LALB_HIGHZ_DRVR_EN, + [QPHY_TX_TRANSCEIVER_BIAS_EN] = QSERDES_V8_LALB_TRANSMITTER_EN_CTRL, +}; + +static const struct qmp_phy_init_tbl glymur_usb43dp_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE1, 0xe1), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CP_CTRL_MODE1, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_RCTRL_MODE1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_CCTRL_MODE1, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORECLK_DIV_MODE1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE1, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE1, 0x41), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE1, 0x41), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MSB_MODE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE1, 0xab), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE1, 0xaa), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_SEL_1, 0x13), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE1, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE1, 0x4d), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE1, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE1, 0x95), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE1, 0x1e), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0x4b), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0, 0xe1), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE0, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CP_CTRL_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORECLK_DIV_MODE0, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE0, 0x41), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MSB_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE0, 0xab), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0, 0xaa), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_HS_SWITCH_SEL_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE0, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0, 0x4d), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BG_TIMER, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_EN_CENTER, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER1, 0x62), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER2, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_BUF_ENABLE, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_IVCO, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_IVCO_MODE1, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_EN_SEL, 0x1a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP_EN, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP_CFG, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_MAP, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORE_CLK_EN, 0xa0), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_CONFIG_1, 0x76), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SVS_MODE_CLK_SEL, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_SPARE_FOR_ECO, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DCC_CAL_1, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DCC_CAL_2, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DCC_CAL_3, 0x60), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PSM_CAL_EN, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1, 0x33), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL, 0xaf), +}; + +static const struct qmp_phy_init_tbl glymur_usb43dp_pcs_misc_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V8_PCS_MISC_PCS_MISC_CONFIG1, 0x01), +}; + +static const struct qmp_phy_init_tbl glymur_usb43dp_pcs_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG1, 0xc4), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG2, 0x89), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG3, 0x20), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG6, 0x13), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_REFGEN_REQ_CONFIG1, 0x21), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_RX_SIGDET_LVL, 0x55), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_RCVR_DTCT_DLY_P1U2_L, 0xe7), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_RCVR_DTCT_DLY_P1U2_H, 0x03), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_TSYNC_RSYNC_TIME, 0xa4), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_RX_CONFIG, 0x0a), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_TSYNC_DLY_TIME, 0x04), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_ALIGN_DETECT_CONFIG1, 0xd4), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_ALIGN_DETECT_CONFIG2, 0x30), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_PCS_TX_RX_CONFIG, 0x0c), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_EQ_CONFIG1, 0x4b), + QMP_PHY_INIT_CFG(QPHY_V8_USB43_PCS_EQ_CONFIG5, 0x10), +}; + +static const struct qmp_phy_init_tbl glymur_usb43dp_pcs_usb_tbl[] = { + QMP_PHY_INIT_CFG(QPHY_V8_PCS_USB_LFPS_DET_HIGH_COUNT_VAL, 0xf8), + QMP_PHY_INIT_CFG(QPHY_V8_PCS_USB_RXEQTRAINING_DFE_TIME_S2, 0x07), +}; + +static const struct qmp_phy_init_tbl glymur_usb43dp_lalb_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CLKBUF_ENABLE, 0x81), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX_LVL_UPDATE_CTRL, 0x0d), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL3, 0x80), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL4, 0x8D), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TRANSMITTER_EN_CTRL, 0x13), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_3, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_4, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_RESTRIM_CAL_CTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_RESTRIM_CAL_CTRL, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_RESTRIM_POST_CAL_OFFSET, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_RESTRIM_VREF_SEL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_RESTRIM_VREF_SEL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_ANA_INTERFACE_SELECT2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCS_INTERFACE_SELECT1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B0, 0xa4), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B1, 0xa2), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B2, 0x6e), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B3, 0x51), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B4, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B5, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B6, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE_0_1_B7, 0x2a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B0, 0x4c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B1, 0xc4), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B2, 0x38), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B3, 0x64), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B4, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B5, 0x4b), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B6, 0x12), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_MODE_RATE2_B7, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX_DCC_ANA_CTRL2, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE1, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE1, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE2, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE2, 0x26), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_INIT_RATE_0_1, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_INIT_RATE_2_3, 0x11), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE1, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE2, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE1, 0x15), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE2, 0x22), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CODE_OVRD_RATE_2_3, 0x22), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE1, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE2, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE1, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE2, 0x09), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_SUMMER_CAL_SPD_MODE_RATE_0123, 0x2f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_CAL_CTRL2, 0x85), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_CAL_CTRL3, 0x45), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_ENABLES, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_CNTRL, 0xa3), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_LVL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_DEGLITCH_CNTRL, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_CAL_CTRL1, 0x14), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_CAL_CTRL2_AND_CDR_LOCK_EDGE, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_SIGDET_CAL_TRIM, 0x66), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE1, 0xff), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE2, 0x32), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE1, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE2, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE2, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_CCODE_RATE_01, 0x76), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_CCODE_RATE_23, 0x67), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_FAST_RATE_0_1, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_FAST_RATE_2_3, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_FLL_RATE_0_1, 0x33), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_FLL_RATE_2_3, 0x43), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_PLL_RATE_0_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_RCODE_PLL_RATE_2_3, 0x51), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_FLL_DIV_RATIO_RATE_0123, 0xe5), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CAP_CODE_RATE_0123, 0xf5), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_TYPE_CONFIG, 0x1f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_EN_LOWFREQ, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_LOOP_FUNC_CTRL, 0xd0), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_GM_CAL_EN, 0x1f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_GM_CAL_RES_RATE0_1, 0x88), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_GM_CAL_RES_RATE2_3, 0x88), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_AUX_CLK_CTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_EOM_CTRL1, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL2, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL3, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL4, 0xaa), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CTLE_POST_CAL_OFFSET_RATE_0_1_2, 0x77), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_VGA_CAL_CNTRL1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_VGA_CAL_MAN_VAL_RATE0_1, 0xdd), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_VGA_CAL_MAN_VAL_RATE2_3, 0xd8), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_DFE_TAP1_DAC_ENABLE, 0x1c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_DFE_TAP2_DAC_ENABLE, 0x1c), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_DFE_TAP345_DAC_ENABLE, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_DFE_TAP67_DAC_ENABLE, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_IQTUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_IQTUNE_MAN_INDEX, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_IQTUNE_DIV2_CTRL_RATE0123, 0x1C), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CDR_VCO_CAP_CODE_OVRD_MUXES, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_DIG_BKUP_CTRL16, 0x37), +}; + static const struct qmp_phy_init_tbl qmp_v3_usb3_serdes_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V3_COM_PLL_IVCO, 0x07), QMP_PHY_INIT_CFG(QSERDES_V3_COM_SYSCLK_EN_SEL, 0x14), @@ -1131,6 +1383,38 @@ static const struct qmp_phy_init_tbl qmp_v6_n4_dp_serdes_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V6_COM_CORE_CLK_EN, 0x0f), }; +static const struct qmp_phy_init_tbl qmp_v8_dp_serdes_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CP_CTRL_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_RCTRL_MODE1, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_CCTRL_MODE1, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORECLK_DIV_MODE0, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE0, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BG_TIMER, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_EN_CENTER, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_ADJ_PER1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_ENABLE1, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYS_CLK_CTRL, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_BUF_ENABLE, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_IVCO, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_EN_SEL, 0x3b), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP_EN, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_MAP, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_SELECT, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORE_CLK_EN, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_CONFIG_1, 0x56), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SVS_MODE_CLK_SEL, 0x15), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD1, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DCC_CAL_1, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DCC_CAL_3, 0x60), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PSM_CAL_EN, 0x01), +}; + static const struct qmp_phy_init_tbl qmp_v6_dp_tx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V6_TX_VMODE_CTRL1, 0x40), QMP_PHY_INIT_CFG(QSERDES_V6_TX_PRE_STALL_LDO_BOOST_EN, 0x30), @@ -1158,6 +1442,33 @@ static const struct qmp_phy_init_tbl qmp_v6_n4_dp_tx_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V6_N4_TX_TX_BAND, 0x1), }; +static const struct qmp_phy_init_tbl qmp_v8_n3p_dp_tx_tbl[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TRANSMITTER_EN_CTRL, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_VMODE_CTRL1, 0x40), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_ANA_INTERFACE_SELECT1, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_ANA_INTERFACE_SELECT2, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_PCS_INTERFACE_SELECT1, 0x50), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_1, 0x0d), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_CLKBUF_ENABLE, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_RESET_TSYNC_EN_CTRL, 0x0a), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX_LVL_UPDATE_CTRL, 0x0f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TRAN_DRVR_EMP_EN, 0x5f), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_EMP_POST1_LVL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_EMP_POST1_LVL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_PRE1_EMPH, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_PRE1_EMPH, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_DRV_LVL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_DRV_LVL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_HIGHZ_DRVR_EN, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_2, 0x50), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_LANE_MODE_3, 0x51), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX_DCC_ANA_CTRL2, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_RESTRIM_CAL_CTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_RESTRIM_CAL_CTRL, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX0_RESTRIM_POST_CAL_OFFSET, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_LALB_TX1_RESTRIM_POST_CAL_OFFSET, 0x10), +}; + static const struct qmp_phy_init_tbl qmp_v6_dp_serdes_tbl_rbr[] = { QMP_PHY_INIT_CFG(QSERDES_V6_COM_HSCLK_SEL_1, 0x05), QMP_PHY_INIT_CFG(QSERDES_V6_COM_DEC_START_MODE0, 0x34), @@ -1274,6 +1585,109 @@ static const struct qmp_phy_init_tbl qmp_v6_n4_dp_serdes_tbl_hbr3[] = { QMP_PHY_INIT_CFG(QSERDES_V6_COM_SSC_STEP_SIZE2_MODE0, 0x01), }; +static const struct qmp_phy_init_tbl qmp_v8_dp_serdes_tbl_rbr[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_SEL_1, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0x7a), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0, 0x83), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0, 0x37), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE0, 0x54), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0, 0xfe), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD3, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL, 0xa4), +}; + +static const struct qmp_phy_init_tbl qmp_v8_dp_serdes_tbl_hbr[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_SEL_1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0x21), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0, 0x18), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE0, 0x46), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0, 0xae), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD3, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL, 0xa3), +}; + +static const struct qmp_phy_init_tbl qmp_v8_dp_serdes_tbl_hbr2[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_SEL_1, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0xf6), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0, 0x0), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0, 0x0e), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE0, 0x46), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0, 0xae), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL, 0xbf), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIAS_EN_CLKBUFLR_EN, 0x1c), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_RESETSM_CNTRL, 0x20), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD3, 0x03), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1, 0x3f), +}; + +static const struct qmp_phy_init_tbl qmp_v8_dp_serdes_tbl_hbr3[] = { + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_HSCLK_SEL_1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0, 0x63), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0, 0x0c), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0, 0x5b), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE0, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CP_CTRL_MODE0, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_RCTRL_MODE0, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_CCTRL_MODE0, 0x36), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORECLK_DIV_MODE0, 0x0a), + + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0, 0x17), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0, 0x15), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DEC_START_MODE0, 0x4f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE0, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0, 0xa0), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE0, 0x3f), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_INTEGLOOP_GAIN1_MODE0, 0x00), + + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0, 0xa0), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0, 0x01), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_ADJ_PER1, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER1, 0x6b), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SSC_PER2, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_ENABLE1, 0x0c), + + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYS_CLK_CTRL, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_BUF_ENABLE, 0x06), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_PLL_IVCO, 0x07), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SYSCLK_EN_SEL, 0x04), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_CTRL, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_VCO_TUNE_MAP, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_SELECT, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CORE_CLK_EN, 0x00), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_CONFIG_1, 0x16), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_SVS_MODE_CLK_SEL, 0x15), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1, 0x30), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIAS_EN_CLKBUFLR_EN, 0x10), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD3, 0x05), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_CMN_MODE_CONTD1, 0x24), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1, 0x02), + QMP_PHY_INIT_CFG(QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL, 0x84), +}; + static const struct qmp_phy_init_tbl sc8280xp_usb43dp_serdes_tbl[] = { QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_EN_CENTER, 0x01), QMP_PHY_INIT_CFG(QSERDES_V5_COM_SSC_PER1, 0x31), @@ -1643,14 +2057,15 @@ static const struct qmp_phy_init_tbl x1e80100_usb43dp_pcs_usb_tbl[] = { }; /* list of regulators */ -struct qmp_regulator_data { - const char *name; - unsigned int enable_load; +static struct regulator_bulk_data qmp_phy_vreg_l[] = { + { .supply = "vdda-phy", .init_load_uA = 21800, }, + { .supply = "vdda-pll", .init_load_uA = 36000, }, }; -static struct qmp_regulator_data qmp_phy_vreg_l[] = { - { .name = "vdda-phy", .enable_load = 21800 }, - { .name = "vdda-pll", .enable_load = 36000 }, +static struct regulator_bulk_data qmp_phy_vreg_refgen[] = { + { .supply = "vdda-phy", .init_load_uA = 21800 }, + { .supply = "vdda-pll", .init_load_uA = 36000 }, + { .supply = "refgen", .init_load_uA = 3270 }, }; static const u8 qmp_dp_v3_pre_emphasis_hbr3_hbr2[4][4] = { @@ -1691,7 +2106,7 @@ static const u8 qmp_dp_v4_pre_emphasis_hbr3_hbr2[4][4] = { static const u8 qmp_dp_v4_pre_emphasis_hbr_rbr[4][4] = { { 0x00, 0x0d, 0x14, 0x1a }, { 0x00, 0x0e, 0x15, 0xff }, - { 0x00, 0x0d, 0xff, 0xff }, + { 0x00, 0x0e, 0xff, 0xff }, { 0x03, 0xff, 0xff, 0xff } }; @@ -1755,6 +2170,7 @@ struct qmp_combo_offsets { u16 usb3_serdes; u16 usb3_pcs_misc; u16 usb3_pcs; + u16 usb3_pcs_aon; u16 usb3_pcs_usb; u16 dp_serdes; u16 dp_txa; @@ -1776,6 +2192,8 @@ struct qmp_phy_cfg { int pcs_tbl_num; const struct qmp_phy_init_tbl *pcs_usb_tbl; int pcs_usb_tbl_num; + const struct qmp_phy_init_tbl *pcs_misc_tbl; + int pcs_misc_tbl_num; const struct qmp_phy_init_tbl *dp_serdes_tbl; int dp_serdes_tbl_num; @@ -1799,6 +2217,7 @@ struct qmp_phy_cfg { const u8 (*pre_emphasis_hbr3_hbr2)[4][4]; /* DP PHY callbacks */ + int (*configure_dp_clocks)(struct qmp_combo *qmp); int (*configure_dp_phy)(struct qmp_combo *qmp); void (*configure_dp_tx)(struct qmp_combo *qmp); int (*calibrate_dp_phy)(struct qmp_combo *qmp); @@ -1808,7 +2227,7 @@ struct qmp_phy_cfg { const char * const *reset_list; int num_resets; /* regulators to be requested */ - const struct qmp_regulator_data *vreg_list; + const struct regulator_bulk_data *vreg_list; int num_vregs; /* array of registers with different offsets */ @@ -1836,6 +2255,7 @@ struct qmp_combo { void __iomem *tx2; void __iomem *rx2; void __iomem *pcs_misc; + void __iomem *pcs_aon; void __iomem *pcs_usb; void __iomem *dp_serdes; @@ -1875,13 +2295,17 @@ struct qmp_combo { static void qmp_v3_dp_aux_init(struct qmp_combo *qmp); static void qmp_v3_configure_dp_tx(struct qmp_combo *qmp); +static int qmp_v3_configure_dp_clocks(struct qmp_combo *qmp); static int qmp_v3_configure_dp_phy(struct qmp_combo *qmp); static int qmp_v3_calibrate_dp_phy(struct qmp_combo *qmp); static void qmp_v4_dp_aux_init(struct qmp_combo *qmp); static void qmp_v4_configure_dp_tx(struct qmp_combo *qmp); static int qmp_v4_configure_dp_phy(struct qmp_combo *qmp); -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp); + +static void qmp_v8_dp_aux_init(struct qmp_combo *qmp); +static int qmp_v8_configure_dp_clocks(struct qmp_combo *qmp); +static int qmp_v8_configure_dp_phy(struct qmp_combo *qmp); static inline void qphy_setbits(void __iomem *base, u32 offset, u32 val) { @@ -1960,6 +2384,7 @@ static const struct qmp_combo_offsets qmp_combo_offsets_v8 = { .usb3_serdes = 0x1000, .usb3_pcs_misc = 0x1c00, .usb3_pcs = 0x1e00, + .usb3_pcs_aon = 0x2000, .usb3_pcs_usb = 0x2100, .dp_serdes = 0x3000, .dp_txa = 0x3400, @@ -1967,6 +2392,19 @@ static const struct qmp_combo_offsets qmp_combo_offsets_v8 = { .dp_dp_phy = 0x3c00, }; +static const struct qmp_combo_offsets qmp_combo_usb43dp_offsets_v8 = { + .com = 0x0000, + .usb3_pcs_aon = 0x0100, + .usb3_serdes = 0x1000, + .usb3_pcs_misc = 0x1400, + .usb3_pcs = 0x1600, + .usb3_pcs_usb = 0x1900, + .dp_serdes = 0x2000, + .dp_dp_phy = 0x2400, + .txa = 0x4000, + .txb = 0x5000, +}; + static const struct qmp_phy_cfg sar2130p_usb3dpphy_cfg = { .offsets = &qmp_combo_offsets_v3, @@ -2002,8 +2440,8 @@ static const struct qmp_phy_cfg sar2130p_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2137,8 +2575,8 @@ static const struct qmp_phy_cfg sc8180x_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2183,8 +2621,8 @@ static const struct qmp_phy_cfg sc8280xp_usb43dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2228,8 +2666,8 @@ static const struct qmp_phy_cfg x1e80100_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2316,8 +2754,8 @@ static const struct qmp_phy_cfg sm8250_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2364,8 +2802,8 @@ static const struct qmp_phy_cfg sm8350_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .reset_list = msm8996_usb3phy_reset_l, .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), @@ -2411,8 +2849,8 @@ static const struct qmp_phy_cfg sm8550_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2456,8 +2894,8 @@ static const struct qmp_phy_cfg sm8650_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v6_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2501,8 +2939,8 @@ static const struct qmp_phy_cfg sm8750_usb3dpphy_cfg = { .dp_aux_init = qmp_v4_dp_aux_init, .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v3_configure_dp_clocks, .configure_dp_phy = qmp_v4_configure_dp_phy, - .calibrate_dp_phy = qmp_v4_calibrate_dp_phy, .regs = qmp_v8_usb3phy_regs_layout, .reset_list = msm8996_usb3phy_reset_l, @@ -2511,6 +2949,51 @@ static const struct qmp_phy_cfg sm8750_usb3dpphy_cfg = { .num_vregs = ARRAY_SIZE(qmp_phy_vreg_l), }; +static const struct qmp_phy_cfg glymur_usb3dpphy_cfg = { + .offsets = &qmp_combo_usb43dp_offsets_v8, + + .serdes_tbl = glymur_usb43dp_serdes_tbl, + .serdes_tbl_num = ARRAY_SIZE(glymur_usb43dp_serdes_tbl), + .tx_tbl = glymur_usb43dp_lalb_tbl, + .tx_tbl_num = ARRAY_SIZE(glymur_usb43dp_lalb_tbl), + .pcs_tbl = glymur_usb43dp_pcs_tbl, + .pcs_tbl_num = ARRAY_SIZE(glymur_usb43dp_pcs_tbl), + .pcs_usb_tbl = glymur_usb43dp_pcs_usb_tbl, + .pcs_usb_tbl_num = ARRAY_SIZE(glymur_usb43dp_pcs_usb_tbl), + .pcs_misc_tbl = glymur_usb43dp_pcs_misc_tbl, + .pcs_misc_tbl_num = ARRAY_SIZE(glymur_usb43dp_pcs_misc_tbl), + + .dp_serdes_tbl = qmp_v8_dp_serdes_tbl, + .dp_serdes_tbl_num = ARRAY_SIZE(qmp_v8_dp_serdes_tbl), + .dp_tx_tbl = qmp_v8_n3p_dp_tx_tbl, + .dp_tx_tbl_num = ARRAY_SIZE(qmp_v8_n3p_dp_tx_tbl), + + .serdes_tbl_rbr = qmp_v8_dp_serdes_tbl_rbr, + .serdes_tbl_rbr_num = ARRAY_SIZE(qmp_v8_dp_serdes_tbl_rbr), + .serdes_tbl_hbr = qmp_v8_dp_serdes_tbl_hbr, + .serdes_tbl_hbr_num = ARRAY_SIZE(qmp_v8_dp_serdes_tbl_hbr), + .serdes_tbl_hbr2 = qmp_v8_dp_serdes_tbl_hbr2, + .serdes_tbl_hbr2_num = ARRAY_SIZE(qmp_v8_dp_serdes_tbl_hbr2), + .serdes_tbl_hbr3 = qmp_v8_dp_serdes_tbl_hbr3, + .serdes_tbl_hbr3_num = ARRAY_SIZE(qmp_v8_dp_serdes_tbl_hbr3), + + .swing_hbr_rbr = &qmp_dp_v6_voltage_swing_hbr_rbr, + .pre_emphasis_hbr_rbr = &qmp_dp_v6_pre_emphasis_hbr_rbr, + .swing_hbr3_hbr2 = &qmp_dp_v5_voltage_swing_hbr3_hbr2, + .pre_emphasis_hbr3_hbr2 = &qmp_dp_v5_pre_emphasis_hbr3_hbr2, + + .dp_aux_init = qmp_v8_dp_aux_init, + .configure_dp_tx = qmp_v4_configure_dp_tx, + .configure_dp_clocks = qmp_v8_configure_dp_clocks, + .configure_dp_phy = qmp_v8_configure_dp_phy, + + .regs = qmp_v8_n3_usb43dpphy_regs_layout, + .reset_list = msm8996_usb3phy_reset_l, + .num_resets = ARRAY_SIZE(msm8996_usb3phy_reset_l), + .vreg_list = qmp_phy_vreg_refgen, + .num_vregs = ARRAY_SIZE(qmp_phy_vreg_refgen), +}; + static int qmp_combo_dp_serdes_init(struct qmp_combo *qmp) { const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -2673,7 +3156,7 @@ static bool qmp_combo_configure_dp_mode(struct qmp_combo *qmp) return reverse; } -static int qmp_combo_configure_dp_clocks(struct qmp_combo *qmp) +static int qmp_v3_configure_dp_clocks(struct qmp_combo *qmp) { const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; u32 phy_vco_div; @@ -2720,7 +3203,7 @@ static int qmp_v3_configure_dp_phy(struct qmp_combo *qmp) writel(0x05, qmp->dp_dp_phy + QSERDES_V3_DP_PHY_TX0_TX1_LANE_CTL); writel(0x05, qmp->dp_dp_phy + QSERDES_V3_DP_PHY_TX2_TX3_LANE_CTL); - ret = qmp_combo_configure_dp_clocks(qmp); + ret = qmp_v3_configure_dp_clocks(qmp); if (ret) return ret; @@ -2798,7 +3281,34 @@ static void qmp_v4_dp_aux_init(struct qmp_combo *qmp) writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); - qmp->dp_aux_cfg = 0; + + writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | + PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | + PHY_AUX_REQ_ERR_MASK, + qmp->dp_dp_phy + QSERDES_V4_DP_PHY_AUX_INTERRUPT_MASK); +} + +static void qmp_v8_dp_aux_init(struct qmp_combo *qmp) +{ + const struct qmp_phy_cfg *cfg = qmp->cfg; + + writel(DP_PHY_PD_CTL_PWRDN | DP_PHY_PD_CTL_PSR_PWRDN | DP_PHY_PD_CTL_AUX_PWRDN | + DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN, + qmp->dp_dp_phy + QSERDES_DP_PHY_PD_CTL); + + /* Turn on BIAS current for PHY/PLL */ + writel(0x1c, qmp->dp_serdes + cfg->regs[QPHY_COM_BIAS_EN_CLKBUFLR_EN]); + + writel(0x00, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG0); + writel(0x13, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); + writel(0x06, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG2); + writel(0x00, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG3); + writel(0x0a, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG4); + writel(0x26, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG5); + writel(0x0a, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG6); + writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG7); + writel(0xb7, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG8); + writel(0x03, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG9); writel(PHY_AUX_STOP_ERR_MASK | PHY_AUX_DEC_ERR_MASK | PHY_AUX_SYNC_ERR_MASK | PHY_AUX_ALIGN_ERR_MASK | @@ -2820,6 +3330,58 @@ static void qmp_v4_configure_dp_tx(struct qmp_combo *qmp) qmp_combo_configure_dp_swing(qmp); } +static int qmp_v8_configure_dp_clocks(struct qmp_combo *qmp) +{ + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + u32 phy_vco_div; + unsigned long pixel_freq; + const struct qmp_phy_cfg *cfg = qmp->cfg; + + switch (dp_opts->link_rate) { + case 1620: + phy_vco_div = 0x4; + pixel_freq = 1620000000UL / 2; + break; + case 2700: + phy_vco_div = 0x2; + pixel_freq = 2700000000UL / 2; + break; + case 5400: + phy_vco_div = 0x4; + pixel_freq = 5400000000UL / 4; + break; + case 8100: + phy_vco_div = 0x3; + pixel_freq = 8100000000UL / 6; + break; + default: + /* Other link rates aren't supported */ + return -EINVAL; + } + writel(phy_vco_div, qmp->dp_dp_phy + cfg->regs[QPHY_DP_PHY_VCO_DIV]); + + /* disable core reset tsync */ + writel(0x09, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + + writel(0x04, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_AUXLESS_SETUP_CYC); + writel(0x08, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_AUXLESS_SILENCE_CYC); + writel(0x08, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_LFPS_CYC); + writel(0x11, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_LFPS_PERIOD); + + writel(0x3e, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_TSYNC_OVRD); + writel(0x05, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_TX2_TX3_LANE_CTL); + writel(0x05, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_TX0_TX1_LANE_CTL); + writel(0x01, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_AUXLESS_CFG1); + writel(0x11, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_LFPS_PERIOD); + writel(0x1f, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_LN0_DRV_LVL); + writel(0x1f, qmp->dp_dp_phy + QSERDES_V8_DP_PHY_LN1_DRV_LVL); + + clk_set_rate(qmp->dp_link_hw.clk, dp_opts->link_rate * 100000); + clk_set_rate(qmp->dp_pixel_hw.clk, pixel_freq); + + return 0; +} + static int qmp_v456_configure_dp_phy(struct qmp_combo *qmp) { const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -2836,7 +3398,7 @@ static int qmp_v456_configure_dp_phy(struct qmp_combo *qmp) writel(0x05, qmp->dp_dp_phy + QSERDES_V4_DP_PHY_TX0_TX1_LANE_CTL); writel(0x05, qmp->dp_dp_phy + QSERDES_V4_DP_PHY_TX2_TX3_LANE_CTL); - ret = qmp_combo_configure_dp_clocks(qmp); + ret = qmp->cfg->configure_dp_clocks(qmp); if (ret) return ret; @@ -2950,20 +3512,58 @@ static int qmp_v4_configure_dp_phy(struct qmp_combo *qmp) return 0; } -/* - * We need to calibrate the aux setting here as many times - * as the caller tries - */ -static int qmp_v4_calibrate_dp_phy(struct qmp_combo *qmp) +static int qmp_v8_configure_dp_phy(struct qmp_combo *qmp) { - static const u8 cfg1_settings[] = { 0x20, 0x13, 0x23, 0x1d }; - u8 val; + const struct qmp_phy_cfg *cfg = qmp->cfg; + bool reverse = (qmp->orientation == TYPEC_ORIENTATION_REVERSE); + const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts; + u32 bias0_en, drvr0_en, bias1_en, drvr1_en; + u32 status; + int ret; - qmp->dp_aux_cfg++; - qmp->dp_aux_cfg %= ARRAY_SIZE(cfg1_settings); - val = cfg1_settings[qmp->dp_aux_cfg]; + ret = qmp_v456_configure_dp_phy(qmp); + if (ret < 0) + return ret; - writel(val, qmp->dp_dp_phy + QSERDES_DP_PHY_AUX_CFG1); + if (dp_opts->lanes == 1) { + bias0_en = reverse ? 0x3e : 0x15; + bias1_en = reverse ? 0x15 : 0x3e; + drvr0_en = reverse ? 0x13 : 0x10; + drvr1_en = reverse ? 0x10 : 0x13; + } else if (dp_opts->lanes == 2) { + bias0_en = reverse ? 0x3f : 0x15; + bias1_en = reverse ? 0x15 : 0x3f; + drvr0_en = 0x10; + drvr1_en = 0x10; + } else { + bias0_en = 0x3f; + bias1_en = 0x3f; + drvr0_en = 0x34; + drvr1_en = 0x34; + } + + writel(drvr0_en, qmp->dp_tx + cfg->regs[QPHY_TX_HIGHZ_DRVR_EN]); + writel(bias0_en, qmp->dp_tx + cfg->regs[QPHY_TX_TRANSCEIVER_BIAS_EN]); + writel(drvr1_en, qmp->dp_tx2 + cfg->regs[QPHY_TX_HIGHZ_DRVR_EN]); + writel(bias1_en, qmp->dp_tx2 + cfg->regs[QPHY_TX_TRANSCEIVER_BIAS_EN]); + + writel(0x08, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + udelay(100); + writel(0x09, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG); + udelay(500); + + if (readl_poll_timeout(qmp->dp_dp_phy + cfg->regs[QPHY_DP_PHY_STATUS], + status, + ((status & BIT(1)) > 0), + 500, + 10000)) + return -ETIMEDOUT; + + writel(0x00, qmp->dp_tx + cfg->regs[QPHY_TX_TX_DRV_LVL]); + writel(0x00, qmp->dp_tx2 + cfg->regs[QPHY_TX_TX_DRV_LVL]); + + writel(0x2b, qmp->dp_tx + cfg->regs[QPHY_TX_TX_EMP_POST1_LVL]); + writel(0x2b, qmp->dp_tx2 + cfg->regs[QPHY_TX_TX_EMP_POST1_LVL]); return 0; } @@ -3007,6 +3607,7 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force) { const struct qmp_phy_cfg *cfg = qmp->cfg; void __iomem *com = qmp->com; + void __iomem *pcs_aon = qmp->pcs_aon; int ret; u32 val; @@ -3042,6 +3643,12 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force) SW_DPPHY_RESET_MUX | SW_DPPHY_RESET | SW_USB3PHY_RESET_MUX | SW_USB3PHY_RESET); + /* override hardware control for reset of qmp phy */ + if (pcs_aon && cfg->regs[QPHY_AON_TOGGLE_ENABLE]) { + qphy_clrbits(pcs_aon, cfg->regs[QPHY_AON_TOGGLE_ENABLE], 0x1); + qphy_clrbits(pcs_aon, cfg->regs[QPHY_DP_AON_TOGGLE_ENABLE], 0x1); + } + /* Use software based port select and switch on typec orientation */ val = SW_PORTSELECT_MUX; if (qmp->orientation == TYPEC_ORIENTATION_REVERSE) @@ -3219,6 +3826,8 @@ static int qmp_combo_usb_power_on(struct phy *phy) qmp_configure_lane(qmp->dev, rx2, cfg->rx_tbl, cfg->rx_tbl_num, 2); qmp_configure(qmp->dev, pcs, cfg->pcs_tbl, cfg->pcs_tbl_num); + qmp_configure(qmp->dev, qmp->pcs_misc, cfg->pcs_misc_tbl, cfg->pcs_misc_tbl_num); + if (pcs_usb) qmp_configure(qmp->dev, pcs_usb, cfg->pcs_usb_tbl, @@ -3345,6 +3954,7 @@ static void qmp_combo_enable_autonomous_mode(struct qmp_combo *qmp) const struct qmp_phy_cfg *cfg = qmp->cfg; void __iomem *pcs_usb = qmp->pcs_usb ?: qmp->pcs; void __iomem *pcs_misc = qmp->pcs_misc; + void __iomem *pcs_aon = qmp->pcs_aon; u32 intr_mask; if (qmp->phy_mode == PHY_MODE_USB_HOST_SS || @@ -3364,9 +3974,14 @@ static void qmp_combo_enable_autonomous_mode(struct qmp_combo *qmp) /* Enable required PHY autonomous mode interrupts */ qphy_setbits(pcs_usb, cfg->regs[QPHY_PCS_AUTONOMOUS_MODE_CTRL], intr_mask); - /* Enable i/o clamp_n for autonomous mode */ - if (pcs_misc) - qphy_clrbits(pcs_misc, QPHY_V3_PCS_MISC_CLAMP_ENABLE, CLAMP_EN); + /* + * Enable i/o clamp_n for autonomous mode + * V6 and later versions use pcs aon clamp register + */ + if (pcs_aon) + qphy_clrbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN); + else if (pcs_misc) + qphy_clrbits(pcs_misc, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN); } static void qmp_combo_disable_autonomous_mode(struct qmp_combo *qmp) @@ -3374,10 +3989,13 @@ static void qmp_combo_disable_autonomous_mode(struct qmp_combo *qmp) const struct qmp_phy_cfg *cfg = qmp->cfg; void __iomem *pcs_usb = qmp->pcs_usb ?: qmp->pcs; void __iomem *pcs_misc = qmp->pcs_misc; + void __iomem *pcs_aon = qmp->pcs_aon; /* Disable i/o clamp_n on resume for normal mode */ - if (pcs_misc) - qphy_setbits(pcs_misc, QPHY_V3_PCS_MISC_CLAMP_ENABLE, CLAMP_EN); + if (pcs_aon) + qphy_setbits(pcs_aon, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN); + else if (pcs_misc) + qphy_setbits(pcs_misc, cfg->regs[QPHY_PCS_CLAMP_ENABLE], CLAMP_EN); qphy_clrbits(pcs_usb, cfg->regs[QPHY_PCS_AUTONOMOUS_MODE_CTRL], ARCVR_DTCT_EN | ARCVR_DTCT_EVENT_SEL | ALFPS_DTCT_EN); @@ -3439,39 +4057,6 @@ static const struct dev_pm_ops qmp_combo_pm_ops = { qmp_combo_runtime_resume, NULL) }; -static int qmp_combo_vreg_init(struct qmp_combo *qmp) -{ - const struct qmp_phy_cfg *cfg = qmp->cfg; - struct device *dev = qmp->dev; - int num = cfg->num_vregs; - int ret, i; - - qmp->vregs = devm_kcalloc(dev, num, sizeof(*qmp->vregs), GFP_KERNEL); - if (!qmp->vregs) - return -ENOMEM; - - for (i = 0; i < num; i++) - qmp->vregs[i].supply = cfg->vreg_list[i].name; - - ret = devm_regulator_bulk_get(dev, num, qmp->vregs); - if (ret) { - dev_err(dev, "failed at devm_regulator_bulk_get\n"); - return ret; - } - - for (i = 0; i < num; i++) { - ret = regulator_set_load(qmp->vregs[i].consumer, - cfg->vreg_list[i].enable_load); - if (ret) { - dev_err(dev, "failed to set load at %s\n", - qmp->vregs[i].supply); - return ret; - } - } - - return 0; -} - static int qmp_combo_reset_init(struct qmp_combo *qmp) { const struct qmp_phy_cfg *cfg = qmp->cfg; @@ -4075,6 +4660,8 @@ static int qmp_combo_parse_dt(struct qmp_combo *qmp) qmp->serdes = base + offs->usb3_serdes; qmp->pcs_misc = base + offs->usb3_pcs_misc; qmp->pcs = base + offs->usb3_pcs; + if (offs->usb3_pcs_aon) + qmp->pcs_aon = base + offs->usb3_pcs_aon; qmp->pcs_usb = base + offs->usb3_pcs_usb; qmp->dp_serdes = base + offs->dp_serdes; @@ -4144,7 +4731,8 @@ static int qmp_combo_probe(struct platform_device *pdev) if (ret) return ret; - ret = qmp_combo_vreg_init(qmp); + ret = devm_regulator_bulk_get_const(dev, qmp->cfg->num_vregs, + qmp->cfg->vreg_list, &qmp->vregs); if (ret) return ret; @@ -4230,6 +4818,10 @@ static int qmp_combo_probe(struct platform_device *pdev) } static const struct of_device_id qmp_combo_of_match_table[] = { + { + .compatible = "qcom,glymur-qmp-usb3-dp-phy", + .data = &glymur_usb3dpphy_cfg, + }, { .compatible = "qcom,sar2130p-qmp-usb3-dp-phy", .data = &sar2130p_usb3dpphy_cfg, diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v8.h new file mode 100644 index 0000000000000..b6a8ab59c2ff4 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-dp-phy-v8.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, The Linux Foundation. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_DP_PHY_V8_H_ +#define QCOM_PHY_QMP_DP_PHY_V8_H_ + +/* Only for QMP V8 PHY - DP PHY registers */ +#define QSERDES_V8_DP_PHY_VCO_DIV 0x070 +#define QSERDES_V8_DP_PHY_AUX_INTERRUPT_STATUS 0x0e0 +#define QSERDES_V8_DP_PHY_TSYNC_OVRD 0x074 +#define QSERDES_V8_DP_PHY_TX0_TX1_LANE_CTL 0x078 +#define QSERDES_V8_DP_PHY_TX2_TX3_LANE_CTL 0x0bc +#define QSERDES_V8_DP_PHY_AUXLESS_CFG1 0x0c8 +#define QSERDES_V8_DP_PHY_LFPS_PERIOD 0x0d0 +#define QSERDES_V8_DP_PHY_LFPS_CYC 0x0d4 +#define QSERDES_V8_DP_PHY_AUXLESS_SETUP_CYC 0x0d8 +#define QSERDES_V8_DP_PHY_AUXLESS_SILENCE_CYC 0x0d8 +#define QSERDES_V8_DP_PHY_LN0_DRV_LVL 0x0e0 +#define QSERDES_V8_DP_PHY_LN1_DRV_LVL 0x0e4 +#define QSERDES_V8_DP_PHY_STATUS 0x114 + + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-dp-qserdes-com-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-dp-qserdes-com-v8.h new file mode 100644 index 0000000000000..2bef1eecdc56a --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-dp-qserdes-com-v8.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2025 Linaro Ltd. + */ + +#ifndef QCOM_PHY_QMP_DP_QSERDES_COM_V8_H_ +#define QCOM_PHY_QMP_DP_QSERDES_COM_V8_H_ + +/* Only for DP QMP V8 PHY - QSERDES COM registers */ +#define DP_QSERDES_V8_COM_HSCLK_SEL_1 0x03c +#define DP_QSERDES_V8_COM_BIN_VCOCAL_CMP_CODE1_MODE0 0x058 +#define DP_QSERDES_V8_COM_BIN_VCOCAL_CMP_CODE2_MODE0 0x05c +#define DP_QSERDES_V8_COM_SSC_STEP_SIZE1_MODE0 0x060 +#define DP_QSERDES_V8_COM_SSC_STEP_SIZE2_MODE0 0x064 +#define DP_QSERDES_V8_COM_CP_CTRL_MODE0 0x070 +#define DP_QSERDES_V8_COM_PLL_RCTRL_MODE0 0x074 +#define DP_QSERDES_V8_COM_PLL_CCTRL_MODE0 0x078 +#define DP_QSERDES_V8_COM_CORECLK_DIV_MODE0 0x07c +#define DP_QSERDES_V8_COM_LOCK_CMP1_MODE0 0x080 +#define DP_QSERDES_V8_COM_LOCK_CMP2_MODE0 0x084 +#define DP_QSERDES_V8_COM_DEC_START_MODE0 0x088 +#define DP_QSERDES_V8_COM_DIV_FRAC_START1_MODE0 0x090 +#define DP_QSERDES_V8_COM_DIV_FRAC_START2_MODE0 0x094 +#define DP_QSERDES_V8_COM_DIV_FRAC_START3_MODE0 0x098 +#define DP_QSERDES_V8_COM_INTEGLOOP_GAIN0_MODE0 0x0a0 +#define DP_QSERDES_V8_COM_VCO_TUNE1_MODE0 0x0a8 +#define DP_QSERDES_V8_COM_INTEGLOOP_GAIN1_MODE0 0x0a4 +#define DP_QSERDES_V8_COM_VCO_TUNE2_MODE0 0x0ac +#define DP_QSERDES_V8_COM_BG_TIMER 0x0bc +#define DP_QSERDES_V8_COM_SSC_EN_CENTER 0x0c0 +#define DP_QSERDES_V8_COM_SSC_ADJ_PER1 0x0c4 +#define DP_QSERDES_V8_COM_SSC_PER1 0x0cc +#define DP_QSERDES_V8_COM_SSC_PER2 0x0d0 +#define DP_QSERDES_V8_COM_BIAS_EN_CLKBUFLR_EN 0x0dc +#define DP_QSERDES_V8_COM_CLK_ENABLE1 0x0e0 +#define DP_QSERDES_V8_COM_SYS_CLK_CTRL 0x0e4 +#define DP_QSERDES_V8_COM_SYSCLK_BUF_ENABLE 0x0e8 +#define DP_QSERDES_V8_COM_PLL_IVCO 0x0f4 +#define DP_QSERDES_V8_COM_SYSCLK_EN_SEL 0x110 +#define DP_QSERDES_V8_COM_RESETSM_CNTRL 0x118 +#define DP_QSERDES_V8_COM_LOCK_CMP_EN 0x120 +#define DP_QSERDES_V8_COM_VCO_TUNE_CTRL 0x13c +#define DP_QSERDES_V8_COM_VCO_TUNE_MAP 0x140 +#define DP_QSERDES_V8_COM_CLK_SELECT 0x164 +#define DP_QSERDES_V8_COM_CORE_CLK_EN 0x170 +#define DP_QSERDES_V8_COM_CMN_CONFIG_1 0x174 +#define DP_QSERDES_V8_COM_SVS_MODE_CLK_SEL 0x180 +#define DP_QSERDES_V8_COM_CLK_FWD_CONFIG_1 0x2f4 +#define DP_QSERDES_V8_COM_CMN_STATUS 0x314 +#define DP_QSERDES_V8_COM_C_READY_STATUS 0x33c + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c index 62b1c845b6275..456cdb6e7213c 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c @@ -3288,6 +3288,7 @@ static const struct qmp_pcie_offsets qmp_pcie_offsets_v5 = { .pcs_misc = 0x0600, .tx = 0x0e00, .rx = 0x1000, + .pcs_lane1 = 0x1400, .tx2 = 0x1600, .rx2 = 0x1800, }; diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v6.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v6.h new file mode 100644 index 0000000000000..52db31a7cf22e --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v6.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_PCS_AON_V6_H_ +#define QCOM_PHY_QMP_PCS_AON_V6_H_ + +/* Only for QMP V6 PHY - PCS_AON registers */ +#define QPHY_V6_PCS_AON_CLAMP_ENABLE 0x00 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v8.h new file mode 100644 index 0000000000000..f6a275c0938ff --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-aon-v8.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025, Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_PCS_AON_V8_H_ +#define QCOM_PHY_QMP_PCS_AON_V8_H_ + +/* Only for QMP V8 PHY - PCS_AON registers */ +#define QPHY_V8_PCS_AON_USB3_AON_CLAMP_ENABLE 0x00 +#define QPHY_V8_PCS_AON_USB4_AON_CLAMP_ENABLE 0x04 +#define QPHY_V8_PCS_AON_USB3_AON_TOGGLE_ENABLE 0x08 +#define QPHY_V8_PCS_AON_USB4_AON_TOGGLE_ENABLE 0x0c +#define QPHY_V8_PCS_AON_DP_AON_TOGGLE_ENABLE 0x10 +#define QPHY_V8_PCS_AON_DUMMY_STATUS 0x14 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v5.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v5.h new file mode 100644 index 0000000000000..77d04c6a1644c --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v5.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_PCS_MISC_V5_H_ +#define QCOM_PHY_QMP_PCS_MISC_V5_H_ + +/* Only for QMP V5 PHY - PCS_MISC registers */ +#define QPHY_V5_PCS_MISC_CLAMP_ENABLE 0x0c + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v8.h new file mode 100644 index 0000000000000..a93ef2faa894a --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcs-misc-v8.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_PCS_MISC_V8_H_ +#define QCOM_PHY_QMP_PCS_MISC_V8_H_ + +/* Only for QMP V8 PHY - PCS_MISC registers */ +#define QPHY_V8_PCS_MISC_PCS_MISC_CONFIG1 0x08 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-lalb-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-lalb-v8.h new file mode 100644 index 0000000000000..60ba730620f82 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-lalb-v8.h @@ -0,0 +1,639 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_QSERDES_V8_LALBH_ +#define QCOM_PHY_QMP_QSERDES_V8_LALBH_ + +#define QSERDES_V8_LALB_BIST_MODE_LANENO 0x0 +#define QSERDES_V8_LALB_BIST_INVERT 0x4 +#define QSERDES_V8_LALB_PERL_LENGTH1 0x8 +#define QSERDES_V8_LALB_PERL_LENGTH2 0xc +#define QSERDES_V8_LALB_BIST_PATTERN1 0x10 +#define QSERDES_V8_LALB_BIST_PATTERN2 0x14 +#define QSERDES_V8_LALB_BIST_PATTERN3 0x18 +#define QSERDES_V8_LALB_BIST_PATTERN4 0x1c +#define QSERDES_V8_LALB_BIST_PATTERN5 0x20 +#define QSERDES_V8_LALB_BIST_PATTERN6 0x24 +#define QSERDES_V8_LALB_BIST_PATTERN7 0x28 +#define QSERDES_V8_LALB_BIST_PATTERN8 0x2c +#define QSERDES_V8_LALB_PRBS_SEED1 0x30 +#define QSERDES_V8_LALB_PRBS_SEED2 0x34 +#define QSERDES_V8_LALB_PRBS_SEED3 0x38 +#define QSERDES_V8_LALB_PRBS_SEED4 0x3c +#define QSERDES_V8_LALB_PRBS_SEED5 0x40 +#define QSERDES_V8_LALB_PRBS_SEED6 0x44 +#define QSERDES_V8_LALB_PRBS_SEED7 0x48 +#define QSERDES_V8_LALB_SW_RESET_PWRDNB 0x4c +#define QSERDES_V8_LALB_RESET_GEN 0x50 +#define QSERDES_V8_LALB_RESET_TSYNC_EN_CTRL 0x54 +#define QSERDES_V8_LALB_CDR_EN_RXEQ_RESET 0x58 +#define QSERDES_V8_LALB_CLKBUF_ENABLE 0x5c +#define QSERDES_V8_LALB_TX0_EMP_POST1_LVL 0x60 +#define QSERDES_V8_LALB_TX1_EMP_POST1_LVL 0x64 +#define QSERDES_V8_LALB_TX0_IDLE_CTRL 0x68 +#define QSERDES_V8_LALB_TX1_IDLE_CTRL 0x6c +#define QSERDES_V8_LALB_TX0_DRV_LVL 0x70 +#define QSERDES_V8_LALB_TX0_DRV_LVL_OFFSET 0x74 +#define QSERDES_V8_LALB_TX1_DRV_LVL 0x78 +#define QSERDES_V8_LALB_TX1_DRV_LVL_OFFSET 0x7c +#define QSERDES_V8_LALB_TRAN_DRVR_EMP_EN 0x80 +#define QSERDES_V8_LALB_TX_LVL_UPDATE_CTRL 0x84 +#define QSERDES_V8_LALB_TX0_PRE1_EMPH 0x88 +#define QSERDES_V8_LALB_TX1_PRE1_EMPH 0x8c +#define QSERDES_V8_LALB_TX0_PRE2_EMPH 0x90 +#define QSERDES_V8_LALB_TX1_PRE2_EMPH 0x94 +#define QSERDES_V8_LALB_STALL_LDO_BOOST_EN 0x98 +#define QSERDES_V8_LALB_PRE_EMPH_EN_CTRL 0x9c +#define QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL1 0xa0 +#define QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL2 0xa4 +#define QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL3 0xa8 +#define QSERDES_V8_LALB_PCIE5_TOP_LDO_CODE_CTRL4 0xac +#define QSERDES_V8_LALB_TRANSMITTER_EN_CTRL 0xb0 +#define QSERDES_V8_LALB_HIGHZ_DRVR_EN 0xb4 +#define QSERDES_V8_LALB_TX_MISC_CTRL1 0xb8 +#define QSERDES_V8_LALB_LPB_EN_CTRL1 0xbc +#define QSERDES_V8_LALB_LBP_EN_CTRL2 0xc0 +#define QSERDES_V8_LALB_TX0_SERDES_BYP_CTRL 0xc4 +#define QSERDES_V8_LALB_TX1_SERDES_BYP_CTRL 0xc8 +#define QSERDES_V8_LALB_LANE_MODE_1 0xcc +#define QSERDES_V8_LALB_LANE_MODE_2 0xd0 +#define QSERDES_V8_LALB_LANE_MODE_3 0xd4 +#define QSERDES_V8_LALB_LANE_MODE_4 0xd8 +#define QSERDES_V8_LALB_ATB_SEL1 0xdc +#define QSERDES_V8_LALB_ATB_SEL2 0xe0 +#define QSERDES_V8_LALB_TX0_RES_CODE_LANE 0xe4 +#define QSERDES_V8_LALB_TX0_RESTRIM_ICAL_OVRD 0xe8 +#define QSERDES_V8_LALB_TX0_RESTRIM_CAL_CTRL 0xec +#define QSERDES_V8_LALB_TX0_RESTRIM_INIT_CODE 0xf0 +#define QSERDES_V8_LALB_TX0_RESTRIM_POST_CAL_OFFSET 0xf4 +#define QSERDES_V8_LALB_TX1_RES_CODE_LANE 0xf8 +#define QSERDES_V8_LALB_TX1_RESTRIM_ICAL_OVRD 0xfc +#define QSERDES_V8_LALB_TX1_RESTRIM_CAL_CTRL 0x100 +#define QSERDES_V8_LALB_TX1_RESTRIM_INIT_CODE 0x104 +#define QSERDES_V8_LALB_TX1_RESTRIM_POST_CAL_OFFSET 0x108 +#define QSERDES_V8_LALB_TX0_RESTRIM_VREF_SEL 0x10c +#define QSERDES_V8_LALB_TX1_RESTRIM_VREF_SEL 0x110 +#define QSERDES_V8_LALB_VMODE_CTRL1 0x114 +#define QSERDES_V8_LALB_SLEW_CNTL_RATE01 0x118 +#define QSERDES_V8_LALB_SLEW_CNTL_RATE23 0x11c +#define QSERDES_V8_LALB_SLEW_CNTL_RATE4 0x120 +#define QSERDES_V8_LALB_ANA_INTERFACE_SELECT1 0x124 +#define QSERDES_V8_LALB_ANA_INTERFACE_SELECT2 0x128 +#define QSERDES_V8_LALB_ANA_INTERFACE_SELECT3 0x12c +#define QSERDES_V8_LALB_PCS_INTERFACE_SELECT1 0x130 +#define QSERDES_V8_LALB_PCS_INTERFACE_SELECT2 0x134 +#define QSERDES_V8_LALB_LDO_TIMER_CTRL 0x138 +#define QSERDES_V8_LALB_AC_JTAG_ENABLE 0x13c +#define QSERDES_V8_LALB_AC_JTAG_INITP 0x140 +#define QSERDES_V8_LALB_AC_JTAG_INITN 0x144 +#define QSERDES_V8_LALB_AC_JTAG_LVL 0x148 +#define QSERDES_V8_LALB_AC_JTAG_MODE 0x14c +#define QSERDES_V8_LALB_AC_JTAG_RESET 0x150 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B0 0x154 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B1 0x158 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B2 0x15c +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B3 0x160 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B4 0x164 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B5 0x168 +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B6 0x16c +#define QSERDES_V8_LALB_RX_MODE_RATE_0_1_B7 0x170 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B0 0x174 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B1 0x178 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B2 0x17c +#define QSERDES_V8_LALB_RX_MODE_RATE2_B3 0x180 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B4 0x184 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B5 0x188 +#define QSERDES_V8_LALB_RX_MODE_RATE2_B6 0x18c +#define QSERDES_V8_LALB_RX_MODE_RATE2_B7 0x190 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B0 0x194 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B1 0x198 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B2 0x19c +#define QSERDES_V8_LALB_RX_MODE_RATE3_B3 0x1a0 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B4 0x1a4 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B5 0x1a8 +#define QSERDES_V8_LALB_RX_MODE_RATE3_B6 0x1ac +#define QSERDES_V8_LALB_RX_MODE_RATE3_B7 0x1b0 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B0 0x1b4 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B1 0x1b8 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B2 0x1bc +#define QSERDES_V8_LALB_RX_MODE_RATE4_B3 0x1c0 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B4 0x1c4 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B5 0x1c8 +#define QSERDES_V8_LALB_RX_MODE_RATE4_B6 0x1cc +#define QSERDES_V8_LALB_RX_MODE_RATE4_B7 0x1d0 +#define QSERDES_V8_LALB_TX_DCC_ANA_CTRL1 0x1d4 +#define QSERDES_V8_LALB_TX_DCC_ANA_CTRL2 0x1d8 +#define QSERDES_V8_LALB_CMUX_DCC_CTRL1 0x1dc +#define QSERDES_V8_LALB_CMUX_DCC_POSTCAL_OFFSET 0x1e0 +#define QSERDES_V8_LALB_CMUX_DCC_OVRD 0x1e4 +#define QSERDES_V8_LALB_TX_DCC_CTRL 0x1e8 +#define QSERDES_V8_LALB_TX0_CTUNE_DCC_CONFIG 0x1ec +#define QSERDES_V8_LALB_TX0_CTUNE_DCC_POSTCAL_OFFSET 0x1f0 +#define QSERDES_V8_LALB_TX0_CTUNE_DCC_OVRD 0x1f4 +#define QSERDES_V8_LALB_TX0_FTUNE_MSB_DCC_CONFIG 0x1f8 +#define QSERDES_V8_LALB_TX0_FTUNE_MSB_DCC_OFFSET_AND_OVRD 0x1fc +#define QSERDES_V8_LALB_TX0_FTUNE_LSB_DCC_CONFIG 0x200 +#define QSERDES_V8_LALB_TX0_FTUNE_LSB_DCC_OFFSET_AND_OVRD 0x204 +#define QSERDES_V8_LALB_TX1_CTUNE_DCC_CONFIG 0x208 +#define QSERDES_V8_LALB_TX1_CTUNE_DCC_POSTCAL_OFFSET 0x20c +#define QSERDES_V8_LALB_TX1_CTUNE_DCC_OVRD 0x210 +#define QSERDES_V8_LALB_TX1_FTUNE_MSB_DCC_CONFIG 0x214 +#define QSERDES_V8_LALB_TX1_FTUNE_MSB_DCC_OFFSET_AND_OVRD 0x218 +#define QSERDES_V8_LALB_TX1_FTUNE_LSB_DCC_CONFIG 0x21c +#define QSERDES_V8_LALB_TX1_FTUNE_LSB_DCC_OFFSET_AND_OVRD 0x220 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CTRL 0x224 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CODE_OVRD_RATE0 0x228 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CODE_OVRD_RATE1 0x22c +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CODE_OVRD_RATE2 0x230 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CODE_OVRD_RATE3 0x234 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_CODE_OVRD_RATE4 0x238 +#define QSERDES_V8_LALB_CDR_VCO_CAL_CTRL 0x23c +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE0 0x240 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE0 0x244 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE1 0x248 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE1 0x24c +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE2 0x250 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE2 0x254 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE3 0x258 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE3 0x25c +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT1_RATE4 0x260 +#define QSERDES_V8_LALB_CDR_VCO_CTUNE_MEAS_CNT2_RATE4 0x264 +#define QSERDES_V8_LALB_CDR_VCTRL_RATE_0_1 0x268 +#define QSERDES_V8_LALB_CDR_VCTRL_RATE_2_3 0x26c +#define QSERDES_V8_LALB_CDR_VCTRL_RATE_4 0x270 +#define QSERDES_V8_LALB_KVCO_INIT_RATE_0_1 0x274 +#define QSERDES_V8_LALB_KVCO_INIT_RATE_2_3 0x278 +#define QSERDES_V8_LALB_KVCO_INIT_RATE_4 0x27c +#define QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE0 0x280 +#define QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE1 0x284 +#define QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE2 0x288 +#define QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE3 0x28c +#define QSERDES_V8_LALB_KVCO_CODE_OVRD_RATE4 0x290 +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_HIGH_RATE_0_1 0x294 +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_HIGH_RATE_2_3 0x298 +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_HIGH_RATE_4 0x29c +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_LOW_RATE_0_1 0x2a0 +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_LOW_RATE_2_3 0x2a4 +#define QSERDES_V8_LALB_KVCO_CAL_VCTRL_LOW_RATE_4 0x2a8 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE0 0x2ac +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE0 0x2b0 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE1 0x2b4 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE1 0x2b8 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE2 0x2bc +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE2 0x2c0 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE3 0x2c4 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE3 0x2c8 +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF1_RATE4 0x2cc +#define QSERDES_V8_LALB_KVCO_IDEAL_FREQ_DIFF2_RATE4 0x2d0 +#define QSERDES_V8_LALB_KP_CDR_UP_DN 0x2d4 +#define QSERDES_V8_LALB_KP_CODE_OVRD_RATE_0_1 0x2d8 +#define QSERDES_V8_LALB_KP_CODE_OVRD_RATE_2_3 0x2dc +#define QSERDES_V8_LALB_KP_CODE_OVRD_RATE4 0x2e0 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE0 0x2e4 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE0 0x2e8 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE1 0x2ec +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE1 0x2f0 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE2 0x2f4 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE2 0x2f8 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE3 0x2fc +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE3 0x300 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND1_RATE4 0x304 +#define QSERDES_V8_LALB_KP_CAL_UPPER_FREQ_DIFF_BND2_RATE4 0x308 +#define QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE0 0x30c +#define QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE1 0x310 +#define QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE2 0x314 +#define QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE3 0x318 +#define QSERDES_V8_LALB_KP_CAL_LOWER_FREQ_DIFF_BND_RATE4 0x31c +#define QSERDES_V8_LALB_CDR_KVCO_KP_CAL_FREQ_MEAS_CTRL 0x320 +#define QSERDES_V8_LALB_PLLLOCK_CMP_DEBUG_CTRL 0x324 +#define QSERDES_V8_LALB_PLLLOCK_CMP_DEBUG_CNT1 0x328 +#define QSERDES_V8_LALB_PLLLOCK_CMP_DEBUG_CNT2 0x32c +#define QSERDES_V8_LALB_PLLLOCK_CMP_DEBUG_CNT3 0x330 +#define QSERDES_V8_LALB_RX_SUMMER_CAL_SPD_MODE_RATE_0123 0x334 +#define QSERDES_V8_LALB_RX_SUMMER_CAL_SPD_MODE_RATE_4 0x338 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE0 0x33c +#define QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE1 0x340 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE2 0x344 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE3 0x348 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CODE_OVERRIDE_RATE4 0x34c +#define QSERDES_V8_LALB_RX_IVCM_CAL_CTRL1 0x350 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CTRL2 0x354 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CTRL3 0x358 +#define QSERDES_V8_LALB_RX_IVCM_CAL_CTRL4 0x35c +#define QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE0 0x360 +#define QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE1 0x364 +#define QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE2 0x368 +#define QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE3 0x36c +#define QSERDES_V8_LALB_RX_IVCM_POSTCAL_OFFSET_RATE4 0x370 +#define QSERDES_V8_LALB_RX_IDAC_I0_DC_OFFSETS 0x374 +#define QSERDES_V8_LALB_RX_IDAC_I0BAR_DC_OFFSETS 0x378 +#define QSERDES_V8_LALB_RX_IDAC_I1_DC_OFFSETS 0x37c +#define QSERDES_V8_LALB_RX_IDAC_I1BAR_DC_OFFSETS 0x380 +#define QSERDES_V8_LALB_RX_IDAC_Q_DC_OFFSETS 0x384 +#define QSERDES_V8_LALB_RX_IDAC_QBAR_DC_OFFSETS 0x388 +#define QSERDES_V8_LALB_RX_IDAC_A_DC_OFFSETS 0x38c +#define QSERDES_V8_LALB_RX_IDAC_ABAR_DC_OFFSETS 0x390 +#define QSERDES_V8_LALB_RX_IDAC_EN 0x394 +#define QSERDES_V8_LALB_DATA_SLICER_INIT_TIMER_CTRL 0x398 +#define QSERDES_V8_LALB_RX_IDAC_ENABLES 0x39c +#define QSERDES_V8_LALB_RX_IDAC_SIGN 0x3a0 +#define QSERDES_V8_LALB_RX_IDAC_TSETTLE 0x3a4 +#define QSERDES_V8_LALB_SIGDET_ENABLES 0x3a8 +#define QSERDES_V8_LALB_SIGDET_CNTRL 0x3ac +#define QSERDES_V8_LALB_SIGDET_LVL 0x3b0 +#define QSERDES_V8_LALB_SIGDET_DEGLITCH_CNTRL 0x3b4 +#define QSERDES_V8_LALB_SIGDET_CAL_CTRL1 0x3b8 +#define QSERDES_V8_LALB_SIGDET_CAL_CTRL2_AND_CDR_LOCK_EDGE 0x3bc +#define QSERDES_V8_LALB_SIGDET_CAL_TRIM 0x3c0 +#define QSERDES_V8_LALB_IA_OFFSET_CENTER_CAL_CTRL 0x3c4 +#define QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE0 0x3c8 +#define QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE1 0x3cc +#define QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE2 0x3d0 +#define QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE3 0x3d4 +#define QSERDES_V8_LALB_FREQ_LOCK_DET_DLY_RATE4 0x3d8 +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_RATE0 0x3dc +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_RATE1 0x3e0 +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_RATE2 0x3e4 +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_RATE3 0x3e8 +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_RATE4 0x3ec +#define QSERDES_V8_LALB_CDR_LOCK_CTRL 0x3f0 +#define QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE0 0x3f4 +#define QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE1 0x3f8 +#define QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE2 0x3fc +#define QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE3 0x400 +#define QSERDES_V8_LALB_CDR_CP_CUR_FLL_RATE4 0x404 +#define QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE0 0x408 +#define QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE1 0x40c +#define QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE2 0x410 +#define QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE3 0x414 +#define QSERDES_V8_LALB_CDR_CP_CUR_PLL_RATE4 0x418 +#define QSERDES_V8_LALB_CDR_FLL_DIV_RATIO_RATE_0123 0x41c +#define QSERDES_V8_LALB_CDR_FLL_DIV_RATIO_RATE4 0x420 +#define QSERDES_V8_LALB_CDR_LOOP_CCODE_RATE_01 0x424 +#define QSERDES_V8_LALB_CDR_LOOP_CCODE_RATE_23 0x428 +#define QSERDES_V8_LALB_CDR_LOOP_CCODE_RATE4 0x42c +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FAST_RATE_0_1 0x430 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FAST_RATE_2_3 0x434 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FAST_RATE4 0x438 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FLL_RATE_0_1 0x43c +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FLL_RATE_2_3 0x440 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_FLL_RATE4 0x444 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_PLL_RATE_0_1 0x448 +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_PLL_RATE_2_3 0x44c +#define QSERDES_V8_LALB_CDR_LOOP_RCODE_PLL_RATE4 0x450 +#define QSERDES_V8_LALB_CDR_VCO_CAP_CODE_RATE_0123 0x454 +#define QSERDES_V8_LALB_CDR_VCO_CAP_CODE_RATE4 0x458 +#define QSERDES_V8_LALB_CDR_VCO_TYPE_CONFIG 0x45c +#define QSERDES_V8_LALB_CDR_VCO_EN_LOWFREQ 0x460 +#define QSERDES_V8_LALB_CDR_FAST_SLOW_VCO_OVRD 0x464 +#define QSERDES_V8_LALB_CDR_LOOP_FUNC_CTRL 0x468 +#define QSERDES_V8_LALB_CDR_FAST_LOCK_EN_CTRL 0x46c +#define QSERDES_V8_LALB_RX_RCVR_EN 0x470 +#define QSERDES_V8_LALB_LANE_RATE_CTRL 0x474 +#define QSERDES_V8_LALB_RX_TERM_RCVR_CTRL 0x478 +#define QSERDES_V8_LALB_REC_DETECT_CTRL 0x47c +#define QSERDES_V8_LALB_RCV_DETECT_LVL 0x480 +#define QSERDES_V8_LALB_GM_CAL_EN 0x484 +#define QSERDES_V8_LALB_GM_CAL_RES_RATE0_1 0x488 +#define QSERDES_V8_LALB_GM_CAL_RES_RATE2_3 0x48c +#define QSERDES_V8_LALB_GM_CAL_RES_RATE4 0x490 +#define QSERDES_V8_LALB_RX_TERM_BW_RATE_0123 0x494 +#define QSERDES_V8_LALB_RX_TERM_BW_RATE4 0x498 +#define QSERDES_V8_LALB_AUX_CLK_CTRL 0x49c +#define QSERDES_V8_LALB_AUX_OFFSET_CONTROL 0x4a0 +#define QSERDES_V8_LALB_AUXDATA_TB 0x4a4 +#define QSERDES_V8_LALB_EOM_CTRL1 0x4a8 +#define QSERDES_V8_LALB_EOM_CTRL2 0x4ac +#define QSERDES_V8_LALB_EOM_CTRL3 0x4b0 +#define QSERDES_V8_LALB_EOM_CTRL4 0x4b4 +#define QSERDES_V8_LALB_DFE_EN_TIMER 0x4b8 +#define QSERDES_V8_LALB_RX_EQ_OFFSET_LSB 0x4bc +#define QSERDES_V8_LALB_RX_EQ_OFFSET_MSB 0x4c0 +#define QSERDES_V8_LALB_RX_EQ_OFFSET_ADAPTOR_CNTRL1 0x4c4 +#define QSERDES_V8_LALB_RX_OFFSET_ADAPTOR_CNTRL2 0x4c8 +#define QSERDES_V8_LALB_RX_OFFSET_ADAPTOR_CNTRL3 0x4cc +#define QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL1 0x4d0 +#define QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL2 0x4d4 +#define QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL3 0x4d8 +#define QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL4 0x4dc +#define QSERDES_V8_LALB_RX_EQU_ADAPTOR_CNTRL5 0x4e0 +#define QSERDES_V8_LALB_RX_EQU_KEQ_UP_LSB 0x4e4 +#define QSERDES_V8_LALB_RX_EQU_KEQ_UP_MSB 0x4e8 +#define QSERDES_V8_LALB_RX_EQU_KEQ_DN_LSB 0x4ec +#define QSERDES_V8_LALB_RX_EQU_KEQ_DN_MSB 0x4f0 +#define QSERDES_V8_LALB_CTLE_ADP_RESET_INIT_CODE_RATE_0_1_2 0x4f4 +#define QSERDES_V8_LALB_CTLE_ADP_RESET_INIT_CODE_RATE_3_4 0x4f8 +#define QSERDES_V8_LALB_CTLE_POST_CAL_OFFSET_RATE_0_1_2 0x4fc +#define QSERDES_V8_LALB_CTLE_POST_CAL_OFFSET_RATE_3_4 0x500 +#define QSERDES_V8_LALB_RX_VGA_GAIN2_BLK1 0x504 +#define QSERDES_V8_LALB_RX_VGA_GAIN2_BLK2 0x508 +#define QSERDES_V8_LALB_VGA_CAL_CNTRL1 0x50c +#define QSERDES_V8_LALB_VGA_CAL_CNTRL2 0x510 +#define QSERDES_V8_LALB_VGA_CAL_MAN_VAL_RATE0_1 0x514 +#define QSERDES_V8_LALB_VGA_CAL_MAN_VAL_RATE2_3 0x518 +#define QSERDES_V8_LALB_VGA_CAL_MAN_VAL_RATE4 0x51c +#define QSERDES_V8_LALB_KVGA_CTRL1 0x520 +#define QSERDES_V8_LALB_KVGA_CTRL2 0x524 +#define QSERDES_V8_LALB_VTHRESH_CAL_CNTRL1 0x528 +#define QSERDES_V8_LALB_VTHRESH_CAL_CNTRL2 0x52c +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_VAL_RATE0 0x530 +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_VAL_RATE1 0x534 +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_VAL_RATE2 0x538 +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_VAL_RATE3 0x53c +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_VAL_RATE4 0x540 +#define QSERDES_V8_LALB_VTHRESH_CAL_MAN_CAL_PAM3 0x544 +#define QSERDES_V8_LALB_VTH_POST_CAL_OFFSET_RATE_0_1 0x548 +#define QSERDES_V8_LALB_VTH_POST_CAL_OFFSET_RATE_2_3 0x54c +#define QSERDES_V8_LALB_VTH_POST_CAL_OFFSET_RATE4 0x550 +#define QSERDES_V8_LALB_DFE_TAP1_CTRL 0x554 +#define QSERDES_V8_LALB_DFE_TAP1_MANVAL_KTAP 0x558 +#define QSERDES_V8_LALB_DFE_TAP1_POST_CAL_OFFSET_RATE_0_1_2 0x55c +#define QSERDES_V8_LALB_DFE_TAP1_POST_CAL_OFFSET_RATE_3_4 0x560 +#define QSERDES_V8_LALB_DFE_TAP2_CTRL 0x564 +#define QSERDES_V8_LALB_DFE_TAP2_MANVAL_KTAP 0x568 +#define QSERDES_V8_LALB_DFE_TAP3_CTRL 0x56c +#define QSERDES_V8_LALB_DFE_TAP3_MANVAL_KTAP 0x570 +#define QSERDES_V8_LALB_DFE_TAP4_CTRL 0x574 +#define QSERDES_V8_LALB_DFE_TAP4_MANVAL_KTAP 0x578 +#define QSERDES_V8_LALB_DFE_TAP5_CTRL 0x57c +#define QSERDES_V8_LALB_DFE_TAP5_MANVAL_KTAP 0x580 +#define QSERDES_V8_LALB_DFE_TAP6_CTRL 0x584 +#define QSERDES_V8_LALB_DFE_TAP6_MANVAL_KTAP 0x588 +#define QSERDES_V8_LALB_DFE_TAP7_CTRL 0x58c +#define QSERDES_V8_LALB_DFE_TAP7_MANVAL_KTAP 0x590 +#define QSERDES_V8_LALB_DFE_TAP1_DAC_ENABLE 0x594 +#define QSERDES_V8_LALB_DFE_TAP2_DAC_ENABLE 0x598 +#define QSERDES_V8_LALB_DFE_TAP345_DAC_ENABLE 0x59c +#define QSERDES_V8_LALB_DFE_TAP67_DAC_ENABLE 0x5a0 +#define QSERDES_V8_LALB_CDR_IQTUNE_CTRL 0x5a4 +#define QSERDES_V8_LALB_CDR_IQTUNE_GAIN 0x5a8 +#define QSERDES_V8_LALB_CDR_IQTUNE_MAN_INDEX 0x5ac +#define QSERDES_V8_LALB_CDR_IQTUNE_FILTER_CAL_CTRL1 0x5b0 +#define QSERDES_V8_LALB_CDR_IQTUNE_FILTER_CAL_CTRL2 0x5b4 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK0_CAL_CODE_RATE0 0x5b8 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK0_CAL_CODE_RATE1 0x5bc +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK0_CAL_CODE_RATE2 0x5c0 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK0_CAL_CODE_RATE3 0x5c4 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK0_CAL_CODE_RATE4 0x5c8 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK90_CAL_CODE_RATE0 0x5cc +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK90_CAL_CODE_RATE1 0x5d0 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK90_CAL_CODE_RATE2 0x5d4 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK90_CAL_CODE_RATE3 0x5d8 +#define QSERDES_V8_LALB_CDR_IQTUNE_CLK90_CAL_CODE_RATE4 0x5dc +#define QSERDES_V8_LALB_CDR_IQTUNE_ANA_CTRL 0x5e0 +#define QSERDES_V8_LALB_CDR_IQTUNE_VDCC_CTRL 0x5e4 +#define QSERDES_V8_LALB_CDR_IQTUNE_DIV2_CTRL_RATE0123 0x5e8 +#define QSERDES_V8_LALB_CDR_IQTUNE_DIV2_CTRL_RATE4 0x5ec +#define QSERDES_V8_LALB_BLW_CTRL 0x5f0 +#define QSERDES_V8_LALB_BLW_ANA_VER_CTRL 0x5f4 +#define QSERDES_V8_LALB_BLW_GAIN_CAL_CTRL 0x5f8 +#define QSERDES_V8_LALB_BLW_GAIN_FORCE_CODE 0x5fc +#define QSERDES_V8_LALB_BLW_MAN_VAL_RATE3 0x600 +#define QSERDES_V8_LALB_BLW_MAN_VAL_RATE4 0x604 +#define QSERDES_V8_LALB_IVTH_CAL_CTRL1 0x608 +#define QSERDES_V8_LALB_IVTH_CAL_CTRL2 0x60c +#define QSERDES_V8_LALB_IVTH_CAL_CTRL3 0x610 +#define QSERDES_V8_LALB_VTH_I_UP_CNTRL_VAL 0x614 +#define QSERDES_V8_LALB_VTH_I_DN_CNTRL_VAL 0x618 +#define QSERDES_V8_LALB_NRZ_EYE_HEIGHT_SEL_VAL 0x61c +#define QSERDES_V8_LALB_IVTH_CAL_VAL_OVRD_MUX 0x620 +#define QSERDES_V8_LALB_CDR_VCO_CAP_CODE_OVRD_MUXES 0x624 +#define QSERDES_V8_LALB_VCO_CTUNE_LOWER_BND_RATE0 0x628 +#define QSERDES_V8_LALB_VCO_CTUNE_LOWER_BND_RATE1 0x62c +#define QSERDES_V8_LALB_VCO_CTUNE_LOWER_BND_RATE2 0x630 +#define QSERDES_V8_LALB_VCO_CTUNE_LOWER_BND_RATE3 0x634 +#define QSERDES_V8_LALB_VCO_CTUNE_LOWER_BND_RATE4 0x638 +#define QSERDES_V8_LALB_VCO_CTUNE_UPPER_BND_RATE0 0x63c +#define QSERDES_V8_LALB_VCO_CTUNE_UPPER_BND_RATE1 0x640 +#define QSERDES_V8_LALB_VCO_CTUNE_UPPER_BND_RATE2 0x644 +#define QSERDES_V8_LALB_VCO_CTUNE_UPPER_BND_RATE3 0x648 +#define QSERDES_V8_LALB_VCO_CTUNE_UPPER_BND_RATE4 0x64c +#define QSERDES_V8_LALB_CDR_LOCK_KVCO_OFFSET_RATE0 0x650 +#define QSERDES_V8_LALB_CDR_LOCK_KVCO_OFFSET_RATE1 0x654 +#define QSERDES_V8_LALB_CDR_LOCK_KVCO_OFFSET_RATE2 0x658 +#define QSERDES_V8_LALB_CDR_LOCK_KVCO_OFFSET_RATE3 0x65c +#define QSERDES_V8_LALB_CDR_LOCK_KVCO_OFFSET_RATE4 0x660 +#define QSERDES_V8_LALB_CDR_LOCK_KP_OFFSET_RATE0 0x664 +#define QSERDES_V8_LALB_CDR_LOCK_KP_OFFSET_RATE1 0x668 +#define QSERDES_V8_LALB_CDR_LOCK_KP_OFFSET_RATE2 0x66c +#define QSERDES_V8_LALB_CDR_LOCK_KP_OFFSET_RATE3 0x670 +#define QSERDES_V8_LALB_CDR_LOCK_KP_OFFSET_RATE4 0x674 +#define QSERDES_V8_LALB_CDR_FASTLOCK_CP_CUR_PLL_RATE0 0x678 +#define QSERDES_V8_LALB_CDR_FASTLOCK_CP_CUR_PLL_RATE1 0x67c +#define QSERDES_V8_LALB_CDR_FASTLOCK_CP_CUR_PLL_RATE2 0x680 +#define QSERDES_V8_LALB_CDR_FASTLOCK_CP_CUR_PLL_RATE3 0x684 +#define QSERDES_V8_LALB_CDR_FASTLOCK_CP_CUR_PLL_RATE4 0x688 +#define QSERDES_V8_LALB_DEBUG_BUS_SEL 0x68c +#define QSERDES_V8_LALB_BIST_STATUS 0x690 +#define QSERDES_V8_LALB_BIST_ERROR_COUNT1 0x694 +#define QSERDES_V8_LALB_BIST_ERROR_COUNT2 0x698 +#define QSERDES_V8_LALB_AC_JTAG_OUTP 0x69c +#define QSERDES_V8_LALB_AC_JTAG_OUTN 0x6a0 +#define QSERDES_V8_LALB_DATA_SLICER_DEBUG_STATUS 0x6a4 +#define QSERDES_V8_LALB_DATA_SLICER_TIMER1_STATUS 0x6a8 +#define QSERDES_V8_LALB_DATA_SLICER_TIMER2_STATUS 0x6ac +#define QSERDES_V8_LALB_TX0_RESTRIM_CODE_STATUS 0x6b0 +#define QSERDES_V8_LALB_TX0_RESTRIM_ICAL_CODE_STATUS 0x6b4 +#define QSERDES_V8_LALB_TX0_RESTRIM_CAL_STATUS 0x6b8 +#define QSERDES_V8_LALB_TX1_RESTRIM_CODE_STATUS 0x6bc +#define QSERDES_V8_LALB_TX1_RESTRIM_ICAL_CODE_STATUS 0x6c0 +#define QSERDES_V8_LALB_TX1_RESTRIM_CAL_STATUS 0x6c4 +#define QSERDES_V8_LALB_CMUX_DCC_CAL_FSM_STATUS 0x6c8 +#define QSERDES_V8_LALB_CMUX_DCC_READCODE_STATUS 0x6cc +#define QSERDES_V8_LALB_TX_DCC_CAL_ANA_STATUS 0x6d0 +#define QSERDES_V8_LALB_TX0_CTUNE_DCC_FSM_DEBUG_STATUS 0x6d4 +#define QSERDES_V8_LALB_TX0_COARSE_DCC_READCODE_STATUS 0x6d8 +#define QSERDES_V8_LALB_TX0_FTUNE_MSB_DCC_FSM_DEBUG_STATUS 0x6dc +#define QSERDES_V8_LALB_TX0_FTUNE_LSB_DCC_FSM_DEBUG_STATUS 0x6e0 +#define QSERDES_V8_LALB_TX0_FINE_DCC_READCODE_STATUS 0x6e4 +#define QSERDES_V8_LALB_TX1_CTUNE_DCC_FSM_DEBUG_STATUS 0x6e8 +#define QSERDES_V8_LALB_TX1_COARSE_DCC_READCODE_STATUS 0x6ec +#define QSERDES_V8_LALB_TX1_FTUNE_MSB_DCC_FSM_DEBUG_STATUS 0x6f0 +#define QSERDES_V8_LALB_TX1_FTUNE_LSB_DCC_FSM_DEBUG_STATUS 0x6f4 +#define QSERDES_V8_LALB_TX1_FINE_DCC_READCODE_STATUS 0x6f8 +#define QSERDES_V8_LALB_CDR_VCO_CAL_STATUS 0x6fc +#define QSERDES_V8_LALB_CDR_VCTRL_STATUS 0x700 +#define QSERDES_V8_LALB_CDR_VCO_CAP_CODE_STATUS 0x704 +#define QSERDES_V8_LALB_KVCO_CAL_DEBUG1_STATUS 0x708 +#define QSERDES_V8_LALB_KVCO_CAL_DEBUG2_STATUS 0x70c +#define QSERDES_V8_LALB_KP_CAL_DEBUG1_STATUS 0x710 +#define QSERDES_V8_LALB_KP_CAL_DEBUG2_STATUS 0x714 +#define QSERDES_V8_LALB_CDR_VCO_FREQ_DEBUG1_STATUS 0x718 +#define QSERDES_V8_LALB_CDR_VCO_FREQ_DEBUG2_STATUS 0x71c +#define QSERDES_V8_LALB_CDR_VCO_FREQ_DEBUG3_STATUS 0x720 +#define QSERDES_V8_LALB_CDR_VCO_FREQ_DEBUG4_STATUS 0x724 +#define QSERDES_V8_LALB_IVCM_CAL_STATUS 0x728 +#define QSERDES_V8_LALB_IVCM_CAL_DEBUG_STATUS 0x72c +#define QSERDES_V8_LALB_IDAC_STATUS_I0 0x730 +#define QSERDES_V8_LALB_IDAC_STATUS_I0BAR 0x734 +#define QSERDES_V8_LALB_IDAC_STATUS_I1 0x738 +#define QSERDES_V8_LALB_IDAC_STATUS_I1BAR 0x73c +#define QSERDES_V8_LALB_IDAC_STATUS_Q 0x740 +#define QSERDES_V8_LALB_IDAC_STATUS_QBAR 0x744 +#define QSERDES_V8_LALB_IDAC_STATUS_A 0x748 +#define QSERDES_V8_LALB_IDAC_STATUS_ABAR 0x74c +#define QSERDES_V8_LALB_IDAC_STATUS_SM_ON 0x750 +#define QSERDES_V8_LALB_IDAC_STATUS_SIGNERROR 0x754 +#define QSERDES_V8_LALB_RX_SIGDET_STATUS 0x758 +#define QSERDES_V8_LALB_SIGDET_CAL_CODE_STATUS 0x75c +#define QSERDES_V8_LALB_SIGDET_CAL_FSM_DEBUG_STATUS 0x760 +#define QSERDES_V8_LALB_CDR_FREQ_LOCK_CNT_STATUS 0x764 +#define QSERDES_V8_LALB_CDR_PHASE_LOCK_CNT_STATUS 0x768 +#define QSERDES_V8_LALB_CDR_LOCK_DEBUG_STATUS 0x76c +#define QSERDES_V8_LALB_IDATA_HIGH_STATUS1 0x770 +#define QSERDES_V8_LALB_IDATA_HIGH_STATUS2 0x774 +#define QSERDES_V8_LALB_IDATA_HIGH_STATUS3 0x778 +#define QSERDES_V8_LALB_IDATA_HIGH_STATUS4 0x77c +#define QSERDES_V8_LALB_IDATA_LOW_STATUS1 0x780 +#define QSERDES_V8_LALB_IDATA_LOW_STATUS2 0x784 +#define QSERDES_V8_LALB_IDATA_LOW_STATUS3 0x788 +#define QSERDES_V8_LALB_IDATA_LOW_STATUS4 0x78c +#define QSERDES_V8_LALB_QDATA_STATUS1 0x790 +#define QSERDES_V8_LALB_QDATA_STATUS2 0x794 +#define QSERDES_V8_LALB_QDATA_STATUS3 0x798 +#define QSERDES_V8_LALB_QDATA_STATUS4 0x79c +#define QSERDES_V8_LALB_IA_ERROR_COUNTER_LOW 0x7a0 +#define QSERDES_V8_LALB_IA_ERROR_COUNTER_HIGH 0x7a4 +#define QSERDES_V8_LALB_EOM_ERR_CNT_LSB_STATUS 0x7a8 +#define QSERDES_V8_LALB_EOM_ERR_CNT_MSB_STATUS 0x7ac +#define QSERDES_V8_LALB_EOM_OP_STATUS 0x7b0 +#define QSERDES_V8_LALB_AUX_MIXER_INDEX_STATUS 0x7b4 +#define QSERDES_V8_LALB_AUX_OFFSET_STATUS 0x7b8 +#define QSERDES_V8_LALB_AUXDATA_TB_STATUS 0x7bc +#define QSERDES_V8_LALB_AUX_MIXER_CTRL_0_STATUS 0x7c0 +#define QSERDES_V8_LALB_AUX_MIXER_CTRL_90_STATUS 0x7c4 +#define QSERDES_V8_LALB_AUX_MIXER_CTRL_180_STATUS 0x7c8 +#define QSERDES_V8_LALB_IQ_MIXER_INDEX_STATUS 0x7cc +#define QSERDES_V8_LALB_IQTUNE_FLTR_INDEX_STATUS 0x7d0 +#define QSERDES_V8_LALB_IQ_MIXER_CTRL_0_STATUS 0x7d4 +#define QSERDES_V8_LALB_IQ_MIXER_CTRL_90_STATUS 0x7d8 +#define QSERDES_V8_LALB_IQ_MIXER_CTRL_180_STATUS 0x7dc +#define QSERDES_V8_LALB_READ_EQCODE 0x7e0 +#define QSERDES_V8_LALB_READ_OFFSETCODE 0x7e4 +#define QSERDES_V8_LALB_VGA_READ_CODE 0x7e8 +#define QSERDES_V8_LALB_VTHRESH_READ_CODE 0x7ec +#define QSERDES_V8_LALB_DFE_TAP1_READ_CODE 0x7f0 +#define QSERDES_V8_LALB_DFE_TAP2_READ_CODE 0x7f4 +#define QSERDES_V8_LALB_DFE_TAP3_READ_CODE 0x7f8 +#define QSERDES_V8_LALB_DFE_TAP4_READ_CODE 0x7fc +#define QSERDES_V8_LALB_DFE_TAP5_READ_CODE 0x800 +#define QSERDES_V8_LALB_DFE_TAP6_READ_CODE 0x804 +#define QSERDES_V8_LALB_DFE_TAP7_READ_CODE 0x808 +#define QSERDES_V8_LALB_CDR_IQTUNE_FILTER_BIN_CODE 0x80c +#define QSERDES_V8_LALB_CDR_IQTUNE_FILTER_CLK0_CODE 0x810 +#define QSERDES_V8_LALB_CDR_IQTUNE_FILTER_CLK90_CODE 0x814 +#define QSERDES_V8_LALB_BLW_READ_CODE 0x818 +#define QSERDES_V8_LALB_IA_OFFSET_CAL_DEBUG_STATUS 0x81c +#define QSERDES_V8_LALB_IA_OFFSET_CAL_STATUS 0x820 +#define QSERDES_V8_LALB_IVTH_CAL_STATUS 0x824 +#define QSERDES_V8_LALB_IVTH_NRZ_EYE_HEIGHT_STATUS 0x828 +#define QSERDES_V8_LALB_IVTH_UPPER_EYE_MAX_STATUS 0x82c +#define QSERDES_V8_LALB_IVTH_UPPER_EYE_MIN_STATUS 0x830 +#define QSERDES_V8_LALB_IVTH_LOWER_EYE_MAX_STATUS 0x834 +#define QSERDES_V8_LALB_IVTH_LOWER_EYE_MIN_STATUS 0x838 +#define QSERDES_V8_LALB_IVTH_UP_INIT_CTR_STATUS 0x83c +#define QSERDES_V8_LALB_VTH_I_UP_CNTRL_STATUS 0x840 +#define QSERDES_V8_LALB_VTH_I_DN_CNTRL_STATUS 0x844 +#define QSERDES_V8_LALB_NRZ_EYE_HEIGHT_SEL_STATUS 0x848 +#define QSERDES_V8_LALB_DEBUG_BUS0 0x84c +#define QSERDES_V8_LALB_DEBUG_BUS1 0x850 +#define QSERDES_V8_LALB_DEBUG_BUS2 0x854 +#define QSERDES_V8_LALB_DEBUG_BUS3 0x858 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL1 0x85c +#define QSERDES_V8_LALB_DIG_BKUP_CTRL2 0x860 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL3 0x864 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL4 0x868 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL5 0x86c +#define QSERDES_V8_LALB_DIG_BKUP_CTRL6 0x870 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL7 0x874 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL8 0x878 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL9 0x87c +#define QSERDES_V8_LALB_DIG_BKUP_CTRL10 0x880 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL11 0x884 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL12 0x888 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL13 0x88c +#define QSERDES_V8_LALB_DIG_BKUP_CTRL14 0x890 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL15 0x894 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL16 0x898 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL17 0x89c +#define QSERDES_V8_LALB_DIG_BKUP_CTRL18 0x8a0 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL19 0x8a4 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL20 0x8a8 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL21 0x8ac +#define QSERDES_V8_LALB_DIG_BKUP_CTRL22 0x8b0 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL23 0x8b4 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL24 0x8b8 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL25 0x8bc +#define QSERDES_V8_LALB_DIG_BKUP_CTRL26 0x8c0 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL27 0x8c4 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL28 0x8c8 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL29 0x8cc +#define QSERDES_V8_LALB_DIG_BKUP_CTRL30 0x8d0 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL31 0x8d4 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL32 0x8d8 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL_V2_1 0x8dc +#define QSERDES_V8_LALB_DIG_BKUP_CTRL_V2_2 0x8e0 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL_V2_3 0x8e4 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL_V2_4 0x8e8 +#define QSERDES_V8_LALB_DIG_BKUP_CTRL_V2_5 0x8ec +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS1 0x8f0 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS2 0x8f4 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS3 0x8f8 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS4 0x8fc +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS5 0x900 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS6 0x904 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS7 0x908 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS8 0x90c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS9 0x910 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS10 0x914 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS11 0x918 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS12 0x91c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS13 0x920 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS14 0x924 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS15 0x928 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS16 0x92c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS17 0x930 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS18 0x934 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS19 0x938 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS20 0x93c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS21 0x940 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS22 0x944 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS23 0x948 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS24 0x94c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS25 0x950 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS26 0x954 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS27 0x958 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS28 0x95c +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS29 0x960 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS30 0x964 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS31 0x968 +#define QSERDES_V8_LALB_DIG_BKUP_RO_BUS32 0x96c +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS1 0x970 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS2 0x974 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS3 0x978 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS4 0x97c +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS5 0x980 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS6 0x984 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS7 0x988 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS8 0x98c +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS9 0x990 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS10 0x994 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS11 0x998 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS12 0x99c +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS13 0x9a0 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS14 0x9a4 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS15 0x9a8 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS16 0x9ac +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS17 0x9b0 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS18 0x9b4 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS19 0x9b8 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS20 0x9bc +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS21 0x9c0 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS22 0x9c4 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS23 0x9c8 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS24 0x9cc +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS25 0x9d0 +#define QSERDES_V8_LALB_DIG_BKUP_RO_V2_BUS26 0x9d4 +#endif /* QCOM_PHY_QMP_QSERDES_V8_LALBH_ */ diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c index 8bf951b0490cf..fc490589c8e48 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb-legacy.c @@ -542,6 +542,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -895,6 +897,7 @@ static int qmp_usb_legacy_power_off(struct phy *phy) static int qmp_usb_legacy_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_legacy_init(phy); @@ -904,14 +907,19 @@ static int qmp_usb_legacy_enable(struct phy *phy) ret = qmp_usb_legacy_power_on(phy); if (ret) qmp_usb_legacy_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_legacy_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_legacy_power_off(phy); if (ret) return ret; @@ -988,7 +996,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -1009,7 +1017,7 @@ static int __maybe_unused qmp_usb_legacy_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c index ed646a7e705ba..e7dd140ad29e7 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c @@ -1300,6 +1300,8 @@ struct qmp_usb { enum phy_mode mode; + bool phy_initialized; + struct phy *phy; struct clk_fixed_rate pipe_clk_fixed; @@ -1848,6 +1850,7 @@ static int qmp_usb_power_off(struct phy *phy) static int qmp_usb_enable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; ret = qmp_usb_init(phy); @@ -1857,14 +1860,19 @@ static int qmp_usb_enable(struct phy *phy) ret = qmp_usb_power_on(phy); if (ret) qmp_usb_exit(phy); + else + qmp->phy_initialized = true; return ret; } static int qmp_usb_disable(struct phy *phy) { + struct qmp_usb *qmp = phy_get_drvdata(phy); int ret; + qmp->phy_initialized = false; + ret = qmp_usb_power_off(phy); if (ret) return ret; @@ -1940,7 +1948,7 @@ static int __maybe_unused qmp_usb_runtime_suspend(struct device *dev) dev_vdbg(dev, "Suspending QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } @@ -1960,7 +1968,7 @@ static int __maybe_unused qmp_usb_runtime_resume(struct device *dev) dev_vdbg(dev, "Resuming QMP phy, mode:%d\n", qmp->mode); - if (!qmp->phy->init_count) { + if (!qmp->phy_initialized) { dev_vdbg(dev, "PHY not initialized, bailing out\n"); return 0; } diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb43-pcs-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-usb43-pcs-v8.h new file mode 100644 index 0000000000000..4f387c8ed9e55 --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb43-pcs-v8.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_USB43_PCS_V8_H_ +#define QCOM_PHY_QMP_USB43_PCS_V8_H_ + +#define QPHY_V8_USB43_PCS_SW_RESET 0x000 +#define QPHY_V8_USB43_PCS_PCS_STATUS1 0x014 +#define QPHY_V8_USB43_PCS_POWER_DOWN_CONTROL 0x040 +#define QPHY_V8_USB43_PCS_START_CONTROL 0x044 +#define QPHY_V8_USB43_PCS_POWER_STATE_CONFIG1 0x090 +#define QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG1 0x0c4 +#define QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG2 0x0c8 +#define QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG3 0x0cc +#define QPHY_V8_USB43_PCS_LOCK_DETECT_CONFIG6 0x0d8 +#define QPHY_V8_USB43_PCS_REFGEN_REQ_CONFIG1 0x0dc +#define QPHY_V8_USB43_PCS_RX_SIGDET_LVL 0x188 +#define QPHY_V8_USB43_PCS_RCVR_DTCT_DLY_P1U2_L 0x190 +#define QPHY_V8_USB43_PCS_RCVR_DTCT_DLY_P1U2_H 0x194 +#define QPHY_V8_USB43_PCS_RATE_SLEW_CNTRL1 0x198 +#define QPHY_V8_USB43_PCS_TSYNC_RSYNC_TIME 0x1ac +#define QPHY_V8_USB43_PCS_RX_CONFIG 0x1b0 +#define QPHY_V8_USB43_PCS_TSYNC_DLY_TIME 0x1b4 +#define QPHY_V8_USB43_PCS_ALIGN_DETECT_CONFIG1 0x1c0 +#define QPHY_V8_USB43_PCS_ALIGN_DETECT_CONFIG2 0x1c4 +#define QPHY_V8_USB43_PCS_PCS_TX_RX_CONFIG 0x1d0 +#define QPHY_V8_USB43_PCS_EQ_CONFIG1 0x1dc +#define QPHY_V8_USB43_PCS_EQ_CONFIG2 0x1e0 +#define QPHY_V8_USB43_PCS_EQ_CONFIG5 0x1ec + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb43-qserdes-com-v8.h b/drivers/phy/qualcomm/phy-qcom-qmp-usb43-qserdes-com-v8.h new file mode 100644 index 0000000000000..e9c743fce9d1b --- /dev/null +++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb43-qserdes-com-v8.h @@ -0,0 +1,224 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2025 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef QCOM_PHY_QMP_USB43_QSERDES_COM_V8_H_ +#define QCOM_PHY_QMP_USB43_QSERDES_COM_V8_H_ + +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE1 0x000 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE1 0x004 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE3_MODE1 0x008 +#define QSERDES_V8_USB43_COM_CLK_EP_DIV_MODE1 0x00c +#define QSERDES_V8_USB43_COM_CP_CTRL_MODE1 0x010 +#define QSERDES_V8_USB43_COM_PLL_RCTRL_MODE1 0x014 +#define QSERDES_V8_USB43_COM_PLL_CCTRL_MODE1 0x018 +#define QSERDES_V8_USB43_COM_CORECLK_DIV_MODE1 0x01c +#define QSERDES_V8_USB43_COM_LOCK_CMP1_MODE1 0x020 +#define QSERDES_V8_USB43_COM_LOCK_CMP2_MODE1 0x024 +#define QSERDES_V8_USB43_COM_DEC_START_MODE1 0x028 +#define QSERDES_V8_USB43_COM_DEC_START_MSB_MODE1 0x02c +#define QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE1 0x030 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE1 0x034 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE1 0x038 +#define QSERDES_V8_USB43_COM_HSCLK_SEL_1 0x03c +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE1 0x040 +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN1_MODE1 0x044 +#define QSERDES_V8_USB43_COM_VCO_TUNE1_MODE1 0x048 +#define QSERDES_V8_USB43_COM_VCO_TUNE2_MODE1 0x04c +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE1 0x050 +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE1 0x054 +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE0 0x058 +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE0 0x05c +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE0 0x060 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE0 0x064 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE3_MODE0 0x068 +#define QSERDES_V8_USB43_COM_CLK_EP_DIV_MODE0 0x06c +#define QSERDES_V8_USB43_COM_CP_CTRL_MODE0 0x070 +#define QSERDES_V8_USB43_COM_PLL_RCTRL_MODE0 0x074 +#define QSERDES_V8_USB43_COM_PLL_CCTRL_MODE0 0x078 +#define QSERDES_V8_USB43_COM_CORECLK_DIV_MODE0 0x07c +#define QSERDES_V8_USB43_COM_LOCK_CMP1_MODE0 0x080 +#define QSERDES_V8_USB43_COM_LOCK_CMP2_MODE0 0x084 +#define QSERDES_V8_USB43_COM_DEC_START_MODE0 0x088 +#define QSERDES_V8_USB43_COM_DEC_START_MSB_MODE0 0x08c +#define QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE0 0x090 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE0 0x094 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE0 0x098 +#define QSERDES_V8_USB43_COM_HSCLK_HS_SWITCH_SEL_1 0x09c +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE0 0x0a0 +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN1_MODE0 0x0a4 +#define QSERDES_V8_USB43_COM_VCO_TUNE1_MODE0 0x0a8 +#define QSERDES_V8_USB43_COM_VCO_TUNE2_MODE0 0x0ac +#define QSERDES_V8_USB43_COM_ATB_SEL1 0x0b0 +#define QSERDES_V8_USB43_COM_ATB_SEL2 0x0b4 +#define QSERDES_V8_USB43_COM_FREQ_UPDATE 0x0b8 +#define QSERDES_V8_USB43_COM_BG_TIMER 0x0bc +#define QSERDES_V8_USB43_COM_SSC_EN_CENTER 0x0c0 +#define QSERDES_V8_USB43_COM_SSC_ADJ_PER1 0x0c4 +#define QSERDES_V8_USB43_COM_SSC_ADJ_PER2 0x0c8 +#define QSERDES_V8_USB43_COM_SSC_PER1 0x0cc +#define QSERDES_V8_USB43_COM_SSC_PER2 0x0d0 +#define QSERDES_V8_USB43_COM_POST_DIV 0x0d4 +#define QSERDES_V8_USB43_COM_POST_DIV_MUX 0x0d8 +#define QSERDES_V8_USB43_COM_BIAS_EN_CLKBUFLR_EN 0x0dc +#define QSERDES_V8_USB43_COM_CLK_ENABLE1 0x0e0 +#define QSERDES_V8_USB43_COM_SYS_CLK_CTRL 0x0e4 +#define QSERDES_V8_USB43_COM_SYSCLK_BUF_ENABLE 0x0e8 +#define QSERDES_V8_USB43_COM_PLL_EN 0x0ec +#define QSERDES_V8_USB43_COM_DEBUG_BUS_OVRD 0x0f0 +#define QSERDES_V8_USB43_COM_PLL_IVCO 0x0f4 +#define QSERDES_V8_USB43_COM_PLL_IVCO_MODE1 0x0f8 +#define QSERDES_V8_USB43_COM_CMN_IETRIM 0x0fc +#define QSERDES_V8_USB43_COM_CMN_IPTRIM 0x100 +#define QSERDES_V8_USB43_COM_EP_CLOCK_DETECT_CTRL 0x104 +#define QSERDES_V8_USB43_COM_PLL_CNTRL 0x108 +#define QSERDES_V8_USB43_COM_BIAS_EN_CTRL_BY_PSM 0x10c +#define QSERDES_V8_USB43_COM_SYSCLK_EN_SEL 0x110 +#define QSERDES_V8_USB43_COM_CML_SYSCLK_SEL 0x114 +#define QSERDES_V8_USB43_COM_RESETSM_CNTRL 0x118 +#define QSERDES_V8_USB43_COM_RESETSM_CNTRL2 0x11c +#define QSERDES_V8_USB43_COM_LOCK_CMP_EN 0x120 +#define QSERDES_V8_USB43_COM_LOCK_CMP_CFG 0x124 +#define QSERDES_V8_USB43_COM_INTEGLOOP_INITVAL 0x128 +#define QSERDES_V8_USB43_COM_INTEGLOOP_EN 0x12c +#define QSERDES_V8_USB43_COM_INTEGLOOP_P_PATH_GAIN0 0x130 +#define QSERDES_V8_USB43_COM_INTEGLOOP_P_PATH_GAIN1 0x134 +#define QSERDES_V8_USB43_COM_VCOCAL_DEADMAN_CTRL 0x138 +#define QSERDES_V8_USB43_COM_VCO_TUNE_CTRL 0x13c +#define QSERDES_V8_USB43_COM_VCO_TUNE_MAP 0x140 +#define QSERDES_V8_USB43_COM_VCO_TUNE_INITVAL1 0x144 +#define QSERDES_V8_USB43_COM_VCO_TUNE_INITVAL2 0x148 +#define QSERDES_V8_USB43_COM_VCO_TUNE_MINVAL1 0x14c +#define QSERDES_V8_USB43_COM_VCO_TUNE_MINVAL2 0x150 +#define QSERDES_V8_USB43_COM_VCO_TUNE_MAXVAL1 0x154 +#define QSERDES_V8_USB43_COM_VCO_TUNE_MAXVAL2 0x158 +#define QSERDES_V8_USB43_COM_VCO_TUNE_TIMER1 0x15c +#define QSERDES_V8_USB43_COM_VCO_TUNE_TIMER2 0x160 +#define QSERDES_V8_USB43_COM_CLK_SELECT 0x164 +#define QSERDES_V8_USB43_COM_PLL_ANALOG 0x168 +#define QSERDES_V8_USB43_COM_SW_RESET 0x16c +#define QSERDES_V8_USB43_COM_CORE_CLK_EN 0x170 +#define QSERDES_V8_USB43_COM_CMN_CONFIG_1 0x174 +#define QSERDES_V8_USB43_COM_CMN_CONFIG_3 0x178 +#define QSERDES_V8_USB43_COM_CMN_RATE_OVERRIDE 0x17c +#define QSERDES_V8_USB43_COM_SVS_MODE_CLK_SEL 0x180 +#define QSERDES_V8_USB43_COM_DEBUG_BUS_SEL 0x184 +#define QSERDES_V8_USB43_COM_CMN_MISC1 0x188 +#define QSERDES_V8_USB43_COM_CMN_MODE 0x18c +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD 0x190 +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD1 0x194 +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD2 0x198 +#define QSERDES_V8_USB43_COM_VCO_DC_LEVEL_CTRL 0x19c +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_1 0x1a0 +#define QSERDES_V8_USB43_COM_ADDITIONAL_CTRL_1 0x1a4 +#define QSERDES_V8_USB43_COM_AUTO_GAIN_ADJ_CTRL_1 0x1a8 +#define QSERDES_V8_USB43_COM_AUTO_GAIN_ADJ_CTRL_2 0x1ac +#define QSERDES_V8_USB43_COM_AUTO_GAIN_ADJ_CTRL_3 0x1b0 +#define QSERDES_V8_USB43_COM_AUTO_GAIN_ADJ_CTRL_4 0x1b4 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC 0x1b8 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_2 0x1bc +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_3 0x1c0 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_4 0x1c4 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_5 0x1c8 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE1_MODE2 0x1cc +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE2_MODE2 0x1d0 +#define QSERDES_V8_USB43_COM_SSC_STEP_SIZE3_MODE2 0x1d4 +#define QSERDES_V8_USB43_COM_CLK_EP_DIV_MODE2 0x1d8 +#define QSERDES_V8_USB43_COM_CP_CTRL_MODE2 0x1dc +#define QSERDES_V8_USB43_COM_PLL_RCTRL_MODE2 0x1e0 +#define QSERDES_V8_USB43_COM_PLL_CCTRL_MODE2 0x1e4 +#define QSERDES_V8_USB43_COM_CORECLK_DIV_MODE2 0x1e8 +#define QSERDES_V8_USB43_COM_LOCK_CMP1_MODE2 0x1ec +#define QSERDES_V8_USB43_COM_LOCK_CMP2_MODE2 0x1f0 +#define QSERDES_V8_USB43_COM_DEC_START_MODE2 0x1f4 +#define QSERDES_V8_USB43_COM_DEC_START_MSB_MODE2 0x1f8 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START1_MODE2 0x1fc +#define QSERDES_V8_USB43_COM_DIV_FRAC_START2_MODE2 0x200 +#define QSERDES_V8_USB43_COM_DIV_FRAC_START3_MODE2 0x204 +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN0_MODE2 0x208 +#define QSERDES_V8_USB43_COM_INTEGLOOP_GAIN1_MODE2 0x20c +#define QSERDES_V8_USB43_COM_VCO_TUNE1_MODE2 0x210 +#define QSERDES_V8_USB43_COM_VCO_TUNE2_MODE2 0x214 +#define QSERDES_V8_USB43_COM_PLL_IVCO_MODE2 0x218 +#define QSERDES_V8_USB43_COM_HSCLK_SEL_2 0x21c +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE1_MODE2 0x220 +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_CMP_CODE2_MODE2 0x224 +#define QSERDES_V8_USB43_COM_HSCLK_HS_SWITCH_SEL_2 0x228 +#define QSERDES_V8_USB43_COM_CMN_CONFIG_2 0x22c +#define QSERDES_V8_USB43_COM_BIN_VCOCAL_HSCLK_SEL_2 0x230 +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_0 0x234 +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_1 0x238 +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_2 0x23c +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_3 0x240 +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_4 0x244 +#define QSERDES_V8_USB43_COM_IVCOCAL_CONFIG_5 0x248 +#define QSERDES_V8_USB43_COM_LOCK_CMP1_EARLY_MODE0 0x24c +#define QSERDES_V8_USB43_COM_LOCK_CMP2_EARLY_MODE0 0x250 +#define QSERDES_V8_USB43_COM_LOCK_CMP1_EARLY_MODE1 0x254 +#define QSERDES_V8_USB43_COM_LOCK_CMP2_EARLY_MODE1 0x258 +#define QSERDES_V8_USB43_COM_LOCK_CMP1_EARLY_MODE2 0x25c +#define QSERDES_V8_USB43_COM_LOCK_CMP2_EARLY_MODE2 0x260 +#define QSERDES_V8_USB43_COM_EARLY_LOCK_CONFIG_0 0x264 +#define QSERDES_V8_USB43_COM_EARLY_LOCK_CONFIG_1 0x268 +#define QSERDES_V8_USB43_COM_ADAPTIVE_ANALOG_CONFIG 0x26c +#define QSERDES_V8_USB43_COM_CP_CTRL_ADAPTIVE_MODE0 0x270 +#define QSERDES_V8_USB43_COM_PLL_RCCTRL_ADAPTIVE_MODE0 0x274 +#define QSERDES_V8_USB43_COM_PLL_CCTRL_ADAPTIVE_MODE0 0x278 +#define QSERDES_V8_USB43_COM_CP_CTRL_ADAPTIVE_MODE1 0x27c +#define QSERDES_V8_USB43_COM_PLL_RCCTRL_ADAPTIVE_MODE1 0x280 +#define QSERDES_V8_USB43_COM_PLL_CCTRL_ADAPTIVE_MODE1 0x284 +#define QSERDES_V8_USB43_COM_CP_CTRL_ADAPTIVE_MODE2 0x288 +#define QSERDES_V8_USB43_COM_PLL_RCCTRL_ADAPTIVE_MODE2 0x28c +#define QSERDES_V8_USB43_COM_PLL_CCTRL_ADAPTIVE_MODE2 0x290 +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD3 0x294 +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD4 0x298 +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD5 0x29c +#define QSERDES_V8_USB43_COM_CMN_MODE_CONTD6 0x2a0 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_6 0x2a4 +#define QSERDES_V8_USB43_COM_ADDITIONAL_MISC_7 0x2a8 +#define QSERDES_V8_USB43_COM_VCO_WAIT_CYCLES 0x2ac +#define QSERDES_V8_USB43_COM_BIAS_WAIT_CYCLES 0x2b0 +#define QSERDES_V8_USB43_COM_AUX_CLK_PSM_ENABLE 0x2b4 +#define QSERDES_V8_USB43_COM_PLL_SPARE_FOR_ECO 0x2b8 +#define QSERDES_V8_USB43_COM_PLL_SPARE_FOR_ECO_1 0x2bc +#define QSERDES_V8_USB43_COM_PLL_SPARE_FOR_ECO_2 0x2c0 +#define QSERDES_V8_USB43_COM_LDO_CAL_1 0x2c4 +#define QSERDES_V8_USB43_COM_LDO_CAL_2 0x2c8 +#define QSERDES_V8_USB43_COM_LDO_CAL_3 0x2cc +#define QSERDES_V8_USB43_COM_LDO_CAL_4 0x2d0 +#define QSERDES_V8_USB43_COM_LDO_CAL_5 0x2d4 +#define QSERDES_V8_USB43_COM_DCC_CAL_1 0x2d8 +#define QSERDES_V8_USB43_COM_DCC_CAL_2 0x2dc +#define QSERDES_V8_USB43_COM_DCC_CAL_3 0x2e0 +#define QSERDES_V8_USB43_COM_DCC_CAL_4 0x2e4 +#define QSERDES_V8_USB43_COM_DCC_CAL_5 0x2e8 +#define QSERDES_V8_USB43_COM_DCC_CAL_6 0x2ec +#define QSERDES_V8_USB43_COM_PSM_CAL_EN 0x2f0 +#define QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_1 0x2f4 +#define QSERDES_V8_USB43_COM_CLK_FWD_CONFIG_2 0x2f8 +#define QSERDES_V8_USB43_COM_IP_CTRL_AND_DP_SEL 0x2fc +#define QSERDES_V8_USB43_COM_DCC_CAL_7 0x300 +#define QSERDES_V8_USB43_COM_DCC_CAL_8 0x304 +#define QSERDES_V8_USB43_COM_DCC_CAL_9 0x308 +#define QSERDES_V8_USB43_COM_MODE_OPERATION_STATUS 0x30c +#define QSERDES_V8_USB43_COM_SYSCLK_DET_COMP_STATUS 0x310 +#define QSERDES_V8_USB43_COM_CMN_STATUS 0x314 +#define QSERDES_V8_USB43_COM_RESET_SM_STATUS 0x318 +#define QSERDES_V8_USB43_COM_RESTRIM_CODE_STATUS 0x31c +#define QSERDES_V8_USB43_COM_PLLCAL_CODE1_STATUS 0x320 +#define QSERDES_V8_USB43_COM_PLLCAL_CODE2_STATUS 0x324 +#define QSERDES_V8_USB43_COM_INTEGLOOP_BINCODE_STATUS 0x328 +#define QSERDES_V8_USB43_COM_DEBUG_BUS0 0x32c +#define QSERDES_V8_USB43_COM_DEBUG_BUS1 0x330 +#define QSERDES_V8_USB43_COM_DEBUG_BUS2 0x334 +#define QSERDES_V8_USB43_COM_DEBUG_BUS3 0x338 +#define QSERDES_V8_USB43_COM_C_READY_STATUS 0x33c +#define QSERDES_V8_USB43_COM_READ_DUMMY_1 0x340 +#define QSERDES_V8_USB43_COM_READ_DUMMY_2 0x344 +#define QSERDES_V8_USB43_COM_READ_DUMMY_3 0x348 +#define QSERDES_V8_USB43_COM_IVCO_CAL_CODE_STATUS 0x34c +#define QSERDES_V8_USB43_COM_PLL_LDO_CAL_STATUS_2 0x350 +#define QSERDES_V8_USB43_COM_PLL_LDO_CAL_STATUS_3 0x354 + +#endif diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h index f58c82b2dd23e..aa34b6ca6b56c 100644 --- a/drivers/phy/qualcomm/phy-qcom-qmp.h +++ b/drivers/phy/qualcomm/phy-qcom-qmp.h @@ -32,7 +32,9 @@ #include "phy-qcom-qmp-qserdes-txrx-v7.h" #include "phy-qcom-qmp-qserdes-com-v8.h" +#include "phy-qcom-qmp-usb43-qserdes-com-v8.h" #include "phy-qcom-qmp-qserdes-txrx-v8.h" +#include "phy-qcom-qmp-qserdes-lalb-v8.h" #include "phy-qcom-qmp-qserdes-pll.h" diff --git a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c index 5b1c82459c126..8de0fba25ecff 100644 --- a/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c +++ b/drivers/phy/qualcomm/phy-qcom-sgmii-eth.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "phy-qcom-qmp-pcs-sgmii.h" #include "phy-qcom-qmp-qserdes-com-v5.h" @@ -25,7 +26,15 @@ #define QSERDES_PCS_SGMIIPHY_READY BIT(7) #define QSERDES_COM_C_PLL_LOCKED BIT(1) +static const struct regulator_bulk_data qcom_dwmac_sgmii_phy_vregs[] = { + { .supply = "vdda-0p9", .init_load_uA = 46000 }, + { .supply = "vdda-1p2", .init_load_uA = 15000 }, +}; + +#define QCOM_SGMII_NUM_SUPPLIES ARRAY_SIZE(qcom_dwmac_sgmii_phy_vregs) + struct qcom_dwmac_sgmii_phy_data { + struct regulator_bulk_data *vregs; struct regmap *regmap; struct clk *refclk; int speed; @@ -267,8 +276,28 @@ static int qcom_dwmac_sgmii_phy_calibrate(struct phy *phy) static int qcom_dwmac_sgmii_phy_power_on(struct phy *phy) { struct qcom_dwmac_sgmii_phy_data *data = phy_get_drvdata(phy); + int ret; + + ret = regulator_bulk_enable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + if (ret) + return ret; - return clk_prepare_enable(data->refclk); + ret = clk_prepare_enable(data->refclk); + if (ret) + goto err_disable_regulators; + + ret = qcom_dwmac_sgmii_phy_calibrate(phy); + if (ret) + goto err_disable_clk; + + return 0; + +err_disable_clk: + clk_disable_unprepare(data->refclk); +err_disable_regulators: + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + + return ret; } static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) @@ -283,6 +312,8 @@ static int qcom_dwmac_sgmii_phy_power_off(struct phy *phy) clk_disable_unprepare(data->refclk); + regulator_bulk_disable(QCOM_SGMII_NUM_SUPPLIES, data->vregs); + return 0; } @@ -293,6 +324,9 @@ static int qcom_dwmac_sgmii_phy_set_speed(struct phy *phy, int speed) if (speed != data->speed) data->speed = speed; + if (phy->power_count == 0) + return 0; + return qcom_dwmac_sgmii_phy_calibrate(phy); } @@ -319,6 +353,7 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) struct phy_provider *provider; void __iomem *base; struct phy *phy; + int ret; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) @@ -343,6 +378,12 @@ static int qcom_dwmac_sgmii_phy_probe(struct platform_device *pdev) if (IS_ERR(data->refclk)) return PTR_ERR(data->refclk); + ret = devm_regulator_bulk_get_const(dev, QCOM_SGMII_NUM_SUPPLIES, + qcom_dwmac_sgmii_phy_vregs, + &data->vregs); + if (ret) + return ret; + provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); if (IS_ERR(provider)) return PTR_ERR(provider); diff --git a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c index eb0b0f61d98e0..980ad1fb1e2e3 100644 --- a/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c +++ b/drivers/phy/qualcomm/phy-qcom-snps-femto-v2.c @@ -599,8 +599,18 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "failed to get regulator supplies\n"); + /* + * Enable runtime PM before creating the PHY, phy_create() only enables + * it on the PHY device if already enabled on the parent. Hold a usage + * reference so callbacks cannot run before the PHY is ready. + */ + pm_runtime_get_noresume(dev); pm_runtime_set_active(dev); - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) { + pm_runtime_put_noidle(dev); + return ret; + } /* * Prevent runtime pm from being ON by default. Users can enable * it using power/control in sysfs. @@ -611,6 +621,7 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) if (IS_ERR(generic_phy)) { ret = PTR_ERR(generic_phy); dev_err(dev, "failed to create phy, %d\n", ret); + pm_runtime_put_noidle(dev); return ret; } hsphy->phy = generic_phy; @@ -620,12 +631,15 @@ static int qcom_snps_hsphy_probe(struct platform_device *pdev) qcom_snps_hsphy_read_override_param_seq(dev); phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate); - if (!IS_ERR(phy_provider)) - dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); - else - pm_runtime_disable(dev); + if (IS_ERR(phy_provider)) { + pm_runtime_put_noidle(dev); + return PTR_ERR(phy_provider); + } - return PTR_ERR_OR_ZERO(phy_provider); + dev_dbg(dev, "Registered Qcom-SNPS HS phy\n"); + pm_runtime_put(dev); + + return 0; } static struct platform_driver qcom_snps_hsphy_driver = { diff --git a/drivers/phy/renesas/Kconfig b/drivers/phy/renesas/Kconfig index e342eef0640b7..e128b9c177a51 100644 --- a/drivers/phy/renesas/Kconfig +++ b/drivers/phy/renesas/Kconfig @@ -29,6 +29,7 @@ config PHY_RCAR_GEN3_USB2 depends on ARCH_RENESAS depends on EXTCON || !EXTCON # if EXTCON=m, this cannot be built-in depends on USB_SUPPORT + depends on REGULATOR select GENERIC_PHY select USB_COMMON help diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c index a38ead7c8055d..4739de1e99fb8 100644 --- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c +++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -140,6 +141,7 @@ struct rcar_gen3_chan { bool extcon_host; bool is_otg_channel; bool uses_otg_pins; + bool otg_internal_reg; }; struct rcar_gen3_phy_drv_data { @@ -203,28 +205,43 @@ static void rcar_gen3_set_linectrl(struct rcar_gen3_chan *ch, int dp, int dm) writel(val, usb2_base + USB2_LINECTRL1); } -static void rcar_gen3_enable_vbus_ctrl(struct rcar_gen3_chan *ch, int vbus) +static void rcar_gen3_phy_usb2_set_vbus(struct rcar_gen3_chan *ch, + u32 vbus_ctrl_reg, + u32 vbus_ctrl_val, + bool enable) { void __iomem *usb2_base = ch->base; - u32 vbus_ctrl_reg = USB2_ADPCTRL; - u32 vbus_ctrl_val = USB2_ADPCTRL_DRVVBUS; u32 val; + val = readl(usb2_base + vbus_ctrl_reg); + if (enable) + val |= vbus_ctrl_val; + else + val &= ~vbus_ctrl_val; + writel(val, usb2_base + vbus_ctrl_reg); + + dev_vdbg(ch->dev, "%s: reg=0x%08x, val=%08x, enable=%d\n", + __func__, vbus_ctrl_reg, val, enable); +} + +static void rcar_gen3_enable_vbus_ctrl(struct rcar_gen3_chan *ch, int vbus) +{ + if (ch->otg_internal_reg) { + regulator_hardware_enable(ch->vbus, vbus); + return; + } + if (ch->phy_data->no_adp_ctrl || ch->phy_data->vblvl_ctrl) { if (ch->vbus) regulator_hardware_enable(ch->vbus, vbus); - vbus_ctrl_reg = USB2_VBCTRL; - vbus_ctrl_val = USB2_VBCTRL_VBOUT; + rcar_gen3_phy_usb2_set_vbus(ch, USB2_VBCTRL, + USB2_VBCTRL_VBOUT, vbus); + return; } - val = readl(usb2_base + vbus_ctrl_reg); - if (vbus) - val |= vbus_ctrl_val; - else - val &= ~vbus_ctrl_val; - dev_vdbg(ch->dev, "%s: %08x, %d\n", __func__, val, vbus); - writel(val, usb2_base + vbus_ctrl_reg); + rcar_gen3_phy_usb2_set_vbus(ch, USB2_ADPCTRL, + USB2_ADPCTRL_DRVVBUS, vbus); } static void rcar_gen3_control_otg_irq(struct rcar_gen3_chan *ch, int enable) @@ -582,7 +599,7 @@ static int rcar_gen3_phy_usb2_power_on(struct phy *p) u32 val; int ret = 0; - if (channel->vbus) { + if (channel->vbus && !channel->otg_internal_reg) { ret = regulator_enable(channel->vbus); if (ret) return ret; @@ -623,7 +640,7 @@ static int rcar_gen3_phy_usb2_power_off(struct phy *p) } } - if (channel->vbus) + if (channel->vbus && !channel->otg_internal_reg) ret = regulator_disable(channel->vbus); return ret; @@ -809,6 +826,139 @@ static int rcar_gen3_phy_usb2_init_bus(struct rcar_gen3_chan *channel) return ret; } +static int rcar_gen3_phy_usb2_regulator_endisable(struct regulator_dev *rdev, + bool enable) +{ + struct rcar_gen3_chan *channel = rdev_get_drvdata(rdev); + struct device *dev = channel->dev; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_warn(dev, "pm_runtime_get failed: %i\n", ret); + return ret; + } + + rcar_gen3_phy_usb2_set_vbus(channel, USB2_VBCTRL, + USB2_VBCTRL_VBOUT, enable); + pm_runtime_put_noidle(dev); + + return ret; +} + +static int rcar_gen3_phy_usb2_regulator_enable(struct regulator_dev *rdev) +{ + return rcar_gen3_phy_usb2_regulator_endisable(rdev, true); +} + +static int rcar_gen3_phy_usb2_regulator_disable(struct regulator_dev *rdev) +{ + return rcar_gen3_phy_usb2_regulator_endisable(rdev, false); +} + +static int rcar_gen3_phy_usb2_regulator_is_enabled(struct regulator_dev *rdev) +{ + struct rcar_gen3_chan *channel = rdev_get_drvdata(rdev); + void __iomem *usb2_base = channel->base; + struct device *dev = channel->dev; + u32 vbus_ctrl_reg = USB2_VBCTRL; + u32 val; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_warn(dev, "pm_runtime_get failed: %i\n", ret); + return ret; + } + + val = readl(usb2_base + vbus_ctrl_reg); + + pm_runtime_put_noidle(dev); + dev_dbg(channel->dev, "%s: %08x\n", __func__, val); + + return (val & USB2_VBCTRL_VBOUT) ? 1 : 0; +} + +static const struct regulator_ops rcar_gen3_phy_usb2_regulator_ops = { + .enable = rcar_gen3_phy_usb2_regulator_enable, + .disable = rcar_gen3_phy_usb2_regulator_disable, + .is_enabled = rcar_gen3_phy_usb2_regulator_is_enabled, +}; + +static const struct regulator_desc rcar_gen3_phy_usb2_regulator = { + .name = "otg-vbus-regulator", + .of_match = of_match_ptr("vbus-regulator"), + .ops = &rcar_gen3_phy_usb2_regulator_ops, + .type = REGULATOR_VOLTAGE, + .owner = THIS_MODULE, + .fixed_uV = 5000000, + .n_voltages = 1, +}; + +static void rcar_gen3_phy_usb2_vbus_disable_action(void *data) +{ + struct regulator *vbus = data; + + regulator_disable(vbus); +} + +static int rcar_gen3_phy_usb2_vbus_regulator_get_exclusive_enable(struct rcar_gen3_chan *channel, + bool enable) +{ + struct device *dev = channel->dev; + int ret; + + channel->vbus = devm_regulator_get_exclusive(dev, "vbus"); + if (IS_ERR(channel->vbus)) { + ret = PTR_ERR(channel->vbus); + /* If vbus-regulator node was present vbus regulator should be available */ + if (channel->otg_internal_reg) + return ret; + + if (ret == -EPROBE_DEFER) + return ret; + + return 0; + } + + if (enable) { + ret = regulator_enable(channel->vbus); + if (ret) + return ret; + } + + if (regulator_is_enabled(channel->vbus)) + return devm_add_action_or_reset(dev, rcar_gen3_phy_usb2_vbus_disable_action, + channel->vbus); + + return 0; +} + +static int rcar_gen3_phy_usb2_vbus_regulator_register(struct rcar_gen3_chan *channel) +{ + struct device *dev = channel->dev; + struct regulator_config rcfg = { .dev = dev, }; + struct regulator_dev *rdev; + bool enable = false; + + rcfg.of_node = of_get_available_child_by_name(dev->of_node, + "vbus-regulator"); + if (rcfg.of_node) { + rcfg.driver_data = channel; + rdev = devm_regulator_register(dev, &rcar_gen3_phy_usb2_regulator, + &rcfg); + of_node_put(rcfg.of_node); + if (IS_ERR(rdev)) + return dev_err_probe(dev, PTR_ERR(rdev), + "Failed to create vbus-regulator\n"); + + channel->otg_internal_reg = true; + enable = true; + } + + return rcar_gen3_phy_usb2_vbus_regulator_get_exclusive_enable(channel, enable); +} + static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -881,10 +1031,13 @@ static int rcar_gen3_phy_usb2_probe(struct platform_device *pdev) phy_set_drvdata(channel->rphys[i].phy, &channel->rphys[i]); } - if (channel->phy_data->no_adp_ctrl && channel->is_otg_channel) - channel->vbus = devm_regulator_get_exclusive(dev, "vbus"); - else + if (channel->phy_data->no_adp_ctrl && channel->is_otg_channel) { + ret = rcar_gen3_phy_usb2_vbus_regulator_register(channel); + if (ret) + return ret; + } else { channel->vbus = devm_regulator_get_optional(dev, "vbus"); + } if (IS_ERR(channel->vbus)) { if (PTR_ERR(channel->vbus) == -EPROBE_DEFER) { ret = PTR_ERR(channel->vbus); diff --git a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c index c79fb53d8ee5c..5281f8dea0ad3 100644 --- a/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c +++ b/drivers/phy/rockchip/phy-rockchip-inno-csidphy.c @@ -170,7 +170,7 @@ static const struct hsfreq_range rk1808_mipidphy_hsfreq_ranges[] = { { 299, 0x06}, { 399, 0x08}, { 499, 0x0b}, { 599, 0x0e}, { 699, 0x10}, { 799, 0x12}, { 999, 0x16}, {1199, 0x1e}, {1399, 0x23}, {1599, 0x2d}, {1799, 0x32}, {1999, 0x37}, - {2199, 0x3c}, {2399, 0x41}, {2499, 0x46} + {2199, 0x3c}, {2399, 0x41}, {2500, 0x46} }; static const struct hsfreq_range rk3326_mipidphy_hsfreq_ranges[] = { diff --git a/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c b/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c index 4508a31472723..d4e49db6feeda 100644 --- a/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c +++ b/drivers/phy/rockchip/phy-rockchip-samsung-dcphy.c @@ -1525,7 +1525,7 @@ static const struct regmap_config samsung_mipi_dcphy_regmap_config = { .reg_bits = 32, .val_bits = 32, .reg_stride = 4, - .max_register = 0x10000, + .max_register = 0xfffc, }; static struct phy *samsung_mipi_dcphy_xlate(struct device *dev, diff --git a/drivers/phy/starfive/phy-jh7110-dphy-rx.c b/drivers/phy/starfive/phy-jh7110-dphy-rx.c index 0b039e1f71c55..d06f21ad63325 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-rx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-rx.c @@ -150,6 +150,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -190,7 +191,9 @@ static int stf_dphy_probe(struct platform_device *pdev) return PTR_ERR(dphy->phy); } - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; phy_set_drvdata(dphy->phy, dphy); phy_provider = devm_of_phy_provider_register(&pdev->dev, diff --git a/drivers/phy/starfive/phy-jh7110-dphy-tx.c b/drivers/phy/starfive/phy-jh7110-dphy-tx.c index c64d1c91b1307..181491a938079 100644 --- a/drivers/phy/starfive/phy-jh7110-dphy-tx.c +++ b/drivers/phy/starfive/phy-jh7110-dphy-tx.c @@ -392,6 +392,7 @@ static int stf_dphy_probe(struct platform_device *pdev) { struct phy_provider *phy_provider; struct stf_dphy *dphy; + int ret; dphy = devm_kzalloc(&pdev->dev, sizeof(*dphy), GFP_KERNEL); if (!dphy) @@ -406,7 +407,9 @@ static int stf_dphy_probe(struct platform_device *pdev) if (IS_ERR(dphy->topsys)) return PTR_ERR(dphy->topsys); - pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; dphy->txesc_clk = devm_clk_get(&pdev->dev, "txesc"); if (IS_ERR(dphy->txesc_clk)) diff --git a/drivers/phy/sunplus/phy-sunplus-usb2.c b/drivers/phy/sunplus/phy-sunplus-usb2.c index 637a5fbae6d9a..0ad4c7160d17c 100644 --- a/drivers/phy/sunplus/phy-sunplus-usb2.c +++ b/drivers/phy/sunplus/phy-sunplus-usb2.c @@ -116,11 +116,11 @@ static int sp_uphy_init(struct phy *phy) ret = clk_prepare_enable(usbphy->phy_clk); if (ret) - goto err_clk; + return ret; ret = reset_control_deassert(usbphy->rstc); if (ret) - goto err_reset; + goto err_clk; /* Default value modification */ writel(HIGH_MASK_BITS | 0x4002, usbphy->moon4_regs + UPHY_CONTROL0); @@ -129,7 +129,7 @@ static int sp_uphy_init(struct phy *phy) /* disconnect voltage */ ret = update_disc_vol(usbphy); if (ret < 0) - return ret; + goto err_reset; /* board uphy 0 internal register modification for tid certification */ val = readl(usbphy->phy_regs + CONFIG9); diff --git a/drivers/phy/xilinx/phy-zynqmp.c b/drivers/phy/xilinx/phy-zynqmp.c index fe6b4925d1662..240626b554750 100644 --- a/drivers/phy/xilinx/phy-zynqmp.c +++ b/drivers/phy/xilinx/phy-zynqmp.c @@ -53,7 +53,7 @@ #define L0_TM_DIG_6 0x106c #define L0_TM_DIS_DESCRAMBLE_DECODER 0x0f #define L0_TX_DIG_61 0x00f4 -#define L0_TM_DISABLE_SCRAMBLE_ENCODER 0x0f +#define L0_TM_DISABLE_SCRAMBLE_ENCODER (BIT(3) | GENMASK(1, 0)) /* PLL Test Mode register parameters */ #define L0_TM_PLL_DIG_37 0x2094 @@ -502,11 +502,30 @@ static void xpsgtr_lane_set_protocol(struct xpsgtr_phy *gtr_phy) } } -/* Bypass (de)scrambler and 8b/10b decoder and encoder. */ -static void xpsgtr_bypass_scrambler_8b10b(struct xpsgtr_phy *gtr_phy) +/** + * xpsgtr_bypass_scrambler_8b10b - Configure scrambler/encoder behavior + * @gtr_phy: pointer to lane context + * @bypass: true to enable scrambler/encoder bypass (SATA/SGMII), + * false to disable scrambler/encoder bypass (USB3) + * + * Uses RMW to preserve reserved and unrelated register fields. + */ +static void xpsgtr_bypass_scrambler_8b10b(struct xpsgtr_phy *gtr_phy, + bool bypass) { - xpsgtr_write_phy(gtr_phy, L0_TM_DIG_6, L0_TM_DIS_DESCRAMBLE_DECODER); - xpsgtr_write_phy(gtr_phy, L0_TX_DIG_61, L0_TM_DISABLE_SCRAMBLE_ENCODER); + if (bypass) { + xpsgtr_clr_set_phy(gtr_phy, L0_TM_DIG_6, + L0_TM_DIS_DESCRAMBLE_DECODER, + L0_TM_DIS_DESCRAMBLE_DECODER); + xpsgtr_clr_set_phy(gtr_phy, L0_TX_DIG_61, + L0_TM_DISABLE_SCRAMBLE_ENCODER, + L0_TM_DISABLE_SCRAMBLE_ENCODER); + } else { + xpsgtr_clr_set_phy(gtr_phy, L0_TM_DIG_6, + L0_TM_DIS_DESCRAMBLE_DECODER, 0); + xpsgtr_clr_set_phy(gtr_phy, L0_TX_DIG_61, + L0_TM_DISABLE_SCRAMBLE_ENCODER, 0); + } } /* DP-specific initialization. */ @@ -527,7 +546,7 @@ static void xpsgtr_phy_init_sata(struct xpsgtr_phy *gtr_phy) { struct xpsgtr_dev *gtr_dev = gtr_phy->dev; - xpsgtr_bypass_scrambler_8b10b(gtr_phy); + xpsgtr_bypass_scrambler_8b10b(gtr_phy, true); writel(gtr_phy->lane, gtr_dev->siou + SATA_CONTROL_OFFSET); } @@ -543,7 +562,7 @@ static void xpsgtr_phy_init_sgmii(struct xpsgtr_phy *gtr_phy) xpsgtr_clr_set(gtr_dev, TX_PROT_BUS_WIDTH, mask, val); xpsgtr_clr_set(gtr_dev, RX_PROT_BUS_WIDTH, mask, val); - xpsgtr_bypass_scrambler_8b10b(gtr_phy); + xpsgtr_bypass_scrambler_8b10b(gtr_phy, true); } /* Configure TX de-emphasis and margining for DP. */ @@ -658,12 +677,13 @@ static int xpsgtr_phy_init(struct phy *phy) { struct xpsgtr_phy *gtr_phy = phy_get_drvdata(phy); struct xpsgtr_dev *gtr_dev = gtr_phy->dev; - int ret = 0; + int ret; mutex_lock(>r_dev->gtr_mutex); /* Configure and enable the clock when peripheral phy_init call */ - if (clk_prepare_enable(gtr_dev->clk[gtr_phy->refclk])) + ret = clk_prepare_enable(gtr_dev->clk[gtr_phy->refclk]); + if (ret) goto out; /* Skip initialization if not required. */ @@ -673,7 +693,7 @@ static int xpsgtr_phy_init(struct phy *phy) if (gtr_dev->tx_term_fix) { ret = xpsgtr_phy_tx_term_fix(gtr_phy); if (ret < 0) - goto out; + goto out_disable_clk; gtr_dev->tx_term_fix = false; } @@ -687,7 +707,7 @@ static int xpsgtr_phy_init(struct phy *phy) */ ret = xpsgtr_configure_pll(gtr_phy); if (ret) - goto out; + goto out_disable_clk; xpsgtr_lane_set_protocol(gtr_phy); @@ -703,8 +723,16 @@ static int xpsgtr_phy_init(struct phy *phy) case ICM_PROTOCOL_SGMII: xpsgtr_phy_init_sgmii(gtr_phy); break; + + case ICM_PROTOCOL_USB: + xpsgtr_bypass_scrambler_8b10b(gtr_phy, false); + break; } + goto out; + +out_disable_clk: + clk_disable_unprepare(gtr_dev->clk[gtr_phy->refclk]); out: mutex_unlock(>r_dev->gtr_mutex); return ret; @@ -1039,6 +1067,12 @@ static int xpsgtr_probe(struct platform_device *pdev) return PTR_ERR(provider); } + gtr_dev->saved_regs = devm_kmalloc(gtr_dev->dev, + sizeof(save_reg_address), + GFP_KERNEL); + if (!gtr_dev->saved_regs) + return -ENOMEM; + pm_runtime_set_active(gtr_dev->dev); pm_runtime_enable(gtr_dev->dev); @@ -1048,12 +1082,6 @@ static int xpsgtr_probe(struct platform_device *pdev) return ret; } - gtr_dev->saved_regs = devm_kmalloc(gtr_dev->dev, - sizeof(save_reg_address), - GFP_KERNEL); - if (!gtr_dev->saved_regs) - return -ENOMEM; - return 0; } diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig index 582d79cc5b437..575cff0f56617 100644 --- a/drivers/pinctrl/Kconfig +++ b/drivers/pinctrl/Kconfig @@ -162,6 +162,7 @@ config PINCTRL_BM1880 depends on OF && (ARCH_BITMAIN || COMPILE_TEST) default ARCH_BITMAIN select PINMUX + select GENERIC_PINCONF help Pinctrl driver for Bitmain BM1880 SoC. @@ -414,6 +415,7 @@ config PINCTRL_MICROCHIP_SGPIO select GENERIC_PINCTRL_GROUPS select GENERIC_PINMUX_FUNCTIONS select OF_GPIO + select REGMAP_MMIO help Support for the serial GPIO interface used on Microsemi and Microchip SoCs. By using a serial interface, the SIO diff --git a/drivers/pinctrl/bcm/pinctrl-bcm2835.c b/drivers/pinctrl/bcm/pinctrl-bcm2835.c index d8da3fb071d62..2b6739bfd216c 100644 --- a/drivers/pinctrl/bcm/pinctrl-bcm2835.c +++ b/drivers/pinctrl/bcm/pinctrl-bcm2835.c @@ -1392,7 +1392,6 @@ static int bcm2835_pinctrl_probe(struct platform_device *pdev) pc->pctl_desc = *pdata->pctl_desc; pc->pctl_dev = devm_pinctrl_register(dev, &pc->pctl_desc, pc); if (IS_ERR(pc->pctl_dev)) { - gpiochip_remove(&pc->gpio_chip); return PTR_ERR(pc->pctl_dev); } diff --git a/drivers/pinctrl/devicetree.c b/drivers/pinctrl/devicetree.c index 0b7f74beb6a6a..fdb5ff0e4b0be 100644 --- a/drivers/pinctrl/devicetree.c +++ b/drivers/pinctrl/devicetree.c @@ -69,6 +69,10 @@ static int dt_remember_or_free_map(struct pinctrl *p, const char *statename, int i; struct pinctrl_dt_map *dt_map; + /* Initialize dev_name before any allocation can fail */ + for (i = 0; i < num_maps; i++) + map[i].dev_name = NULL; + /* Initialize common mapping table entry fields */ for (i = 0; i < num_maps; i++) { const char *devname; diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c index 5f12af59a91b5..df99401c0b7d7 100644 --- a/drivers/pinctrl/mediatek/mtk-eint.c +++ b/drivers/pinctrl/mediatek/mtk-eint.c @@ -12,8 +12,10 @@ */ #include +#include #include #include +#include #include #include #include @@ -504,6 +506,27 @@ int mtk_eint_find_irq(struct mtk_eint *eint, unsigned long eint_n) } EXPORT_SYMBOL_GPL(mtk_eint_find_irq); +static void mtk_eint_teardown(void *data) +{ + struct mtk_eint *eint = data; + unsigned int i, virq; + + /* Detach the demux handler so it can no longer reference freed data. */ + irq_set_chained_handler_and_data(eint->irq, NULL, NULL); + + /* Wait for any in-flight handler to finish before tearing down. */ + synchronize_irq(eint->irq); + + /* Dispose of all child mappings before the domain is removed. */ + for (i = 0; i < eint->hw->ap_num; i++) { + virq = irq_find_mapping(eint->domain, i); + if (virq) + irq_dispose_mapping(virq); + } + + irq_domain_remove(eint->domain); +} + int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) { unsigned int size, i, port, virq, inst = 0; @@ -596,7 +619,7 @@ int mtk_eint_do_init(struct mtk_eint *eint, struct mtk_eint_pin *eint_pin) irq_set_chained_handler_and_data(eint->irq, mtk_eint_irq_handler, eint); - return 0; + return devm_add_action_or_reset(eint->dev, mtk_eint_teardown, eint); err_eint: for (i = 0; i < eint->nbase; i++) { diff --git a/drivers/pinctrl/mediatek/pinctrl-airoha.c b/drivers/pinctrl/mediatek/pinctrl-airoha.c index 9f8e05e3671d6..bdb5e7012d46b 100644 --- a/drivers/pinctrl/mediatek/pinctrl-airoha.c +++ b/drivers/pinctrl/mediatek/pinctrl-airoha.c @@ -49,7 +49,7 @@ /* MUX */ #define REG_GPIO_2ND_I2C_MODE 0x0214 -#define GPIO_MDC_IO_MASTER_MODE_MODE BIT(14) +#define GPIO_MDC_IO_MASTER_MODE_MASK BIT(14) #define GPIO_I2C_MASTER_MODE_MODE BIT(13) #define GPIO_I2S_MODE_MASK BIT(12) #define GPIO_I2C_SLAVE_MODE_MODE BIT(11) @@ -66,10 +66,12 @@ #define GPIO_2ND_I2C_MODE_MASK BIT(0) #define REG_GPIO_SPI_CS1_MODE 0x0218 +#define AN7583_GPIO_MDC_IO_MASTER_MODE_MASK BIT(22) #define GPIO_PCM_SPI_CS4_MODE_MASK BIT(21) #define GPIO_PCM_SPI_CS3_MODE_MASK BIT(20) #define GPIO_PCM_SPI_CS2_MODE_P156_MASK BIT(19) #define GPIO_PCM_SPI_CS2_MODE_P128_MASK BIT(18) +#define AN7583_GPIO_PCM_SPI_CS2_MODE_MASK BIT(18) #define GPIO_PCM_SPI_CS1_MODE_MASK BIT(17) #define GPIO_PCM_SPI_MODE_MASK BIT(16) #define GPIO_PCM2_MODE_MASK BIT(13) @@ -83,6 +85,18 @@ #define GPIO_SPI_CS1_MODE_MASK BIT(0) #define REG_GPIO_PON_MODE 0x021c +#define AN7583_MDIO_0_GPIO_MODE_MASK BIT(26) +#define AN7583_MDC_0_GPIO_MODE_MASK BIT(25) +#define AN7583_UART_RXD_GPIO_MODE_MASK BIT(24) +#define AN7583_UART_TXD_GPIO_MODE_MASK BIT(23) +#define AN7583_SPI_MISO_GPIO_MODE_MASK BIT(22) +#define AN7583_SPI_MOSI_GPIO_MODE_MASK BIT(21) +#define AN7583_SPI_CS_GPIO_MODE_MASK BIT(20) +#define AN7583_SPI_CLK_GPIO_MODE_MASK BIT(19) +#define AN7583_I2C1_SDA_GPIO_MODE_MASK BIT(18) +#define AN7583_I2C1_SCL_GPIO_MODE_MASK BIT(17) +#define AN7583_I2C0_SDA_GPIO_MODE_MASK BIT(16) +#define AN7583_I2C0_SCL_GPIO_MODE_MASK BIT(15) #define GPIO_PARALLEL_NAND_MODE_MASK BIT(14) #define GPIO_SGMII_MDIO_MODE_MASK BIT(13) #define GPIO_PCIE_RESET2_MASK BIT(12) @@ -127,6 +141,8 @@ /* CONF */ #define REG_I2C_SDA_E2 0x001c +#define AN7583_I2C1_SCL_E2_MASK BIT(16) +#define AN7583_I2C1_SDA_E2_MASK BIT(15) #define SPI_MISO_E2_MASK BIT(14) #define SPI_MOSI_E2_MASK BIT(13) #define SPI_CLK_E2_MASK BIT(12) @@ -134,12 +150,16 @@ #define PCIE2_RESET_E2_MASK BIT(10) #define PCIE1_RESET_E2_MASK BIT(9) #define PCIE0_RESET_E2_MASK BIT(8) +#define AN7583_MDIO_0_E2_MASK BIT(5) +#define AN7583_MDC_0_E2_MASK BIT(4) #define UART1_RXD_E2_MASK BIT(3) #define UART1_TXD_E2_MASK BIT(2) #define I2C_SCL_E2_MASK BIT(1) #define I2C_SDA_E2_MASK BIT(0) #define REG_I2C_SDA_E4 0x0020 +#define AN7583_I2C1_SCL_E4_MASK BIT(16) +#define AN7583_I2C1_SDA_E4_MASK BIT(15) #define SPI_MISO_E4_MASK BIT(14) #define SPI_MOSI_E4_MASK BIT(13) #define SPI_CLK_E4_MASK BIT(12) @@ -147,6 +167,8 @@ #define PCIE2_RESET_E4_MASK BIT(10) #define PCIE1_RESET_E4_MASK BIT(9) #define PCIE0_RESET_E4_MASK BIT(8) +#define AN7583_MDIO_0_E4_MASK BIT(5) +#define AN7583_MDC_0_E4_MASK BIT(4) #define UART1_RXD_E4_MASK BIT(3) #define UART1_TXD_E4_MASK BIT(2) #define I2C_SCL_E4_MASK BIT(1) @@ -158,6 +180,8 @@ #define REG_GPIO_H_E4 0x0030 #define REG_I2C_SDA_PU 0x0044 +#define AN7583_I2C1_SCL_PU_MASK BIT(16) +#define AN7583_I2C1_SDA_PU_MASK BIT(15) #define SPI_MISO_PU_MASK BIT(14) #define SPI_MOSI_PU_MASK BIT(13) #define SPI_CLK_PU_MASK BIT(12) @@ -165,12 +189,16 @@ #define PCIE2_RESET_PU_MASK BIT(10) #define PCIE1_RESET_PU_MASK BIT(9) #define PCIE0_RESET_PU_MASK BIT(8) +#define AN7583_MDIO_0_PU_MASK BIT(5) +#define AN7583_MDC_0_PU_MASK BIT(4) #define UART1_RXD_PU_MASK BIT(3) #define UART1_TXD_PU_MASK BIT(2) #define I2C_SCL_PU_MASK BIT(1) #define I2C_SDA_PU_MASK BIT(0) #define REG_I2C_SDA_PD 0x0048 +#define AN7583_I2C1_SCL_PD_MASK BIT(16) +#define AN7583_I2C1_SDA_PD_MASK BIT(15) #define SPI_MISO_PD_MASK BIT(14) #define SPI_MOSI_PD_MASK BIT(13) #define SPI_CLK_PD_MASK BIT(12) @@ -178,6 +206,8 @@ #define PCIE2_RESET_PD_MASK BIT(10) #define PCIE1_RESET_PD_MASK BIT(9) #define PCIE0_RESET_PD_MASK BIT(8) +#define AN7583_MDIO_0_PD_MASK BIT(5) +#define AN7583_MDC_0_PD_MASK BIT(4) #define UART1_RXD_PD_MASK BIT(3) #define UART1_TXD_PD_MASK BIT(2) #define I2C_SCL_PD_MASK BIT(1) @@ -452,9 +482,9 @@ static struct pinctrl_pin_desc en7581_pinctrl_pins[] = { PINCTRL_PIN(57, "gpio44"), PINCTRL_PIN(58, "gpio45"), PINCTRL_PIN(59, "gpio46"), - PINCTRL_PIN(61, "pcie_reset0"), - PINCTRL_PIN(62, "pcie_reset1"), - PINCTRL_PIN(63, "pcie_reset2"), + PINCTRL_PIN(60, "pcie_reset0"), + PINCTRL_PIN(61, "pcie_reset1"), + PINCTRL_PIN(62, "pcie_reset2"), }; static const int en7581_pon_pins[] = { 49, 50, 51, 52, 53, 54 }; @@ -537,9 +567,12 @@ static const int en7581_gpio43_pins[] = { 56 }; static const int en7581_gpio44_pins[] = { 57 }; static const int en7581_gpio45_pins[] = { 58 }; static const int en7581_gpio46_pins[] = { 59 }; -static const int en7581_pcie_reset0_pins[] = { 61 }; -static const int en7581_pcie_reset1_pins[] = { 62 }; -static const int en7581_pcie_reset2_pins[] = { 63 }; +static const int en7581_gpio47_pins[] = { 60 }; +static const int en7581_gpio48_pins[] = { 61 }; +static const int en7581_gpio49_pins[] = { 62 }; +static const int en7581_pcie_reset0_pins[] = { 60 }; +static const int en7581_pcie_reset1_pins[] = { 61 }; +static const int en7581_pcie_reset2_pins[] = { 62 }; static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("pon", en7581_pon), @@ -622,15 +655,243 @@ static const struct pingroup en7581_pinctrl_groups[] = { PINCTRL_PIN_GROUP("gpio44", en7581_gpio44), PINCTRL_PIN_GROUP("gpio45", en7581_gpio45), PINCTRL_PIN_GROUP("gpio46", en7581_gpio46), + PINCTRL_PIN_GROUP("gpio47", en7581_gpio47), + PINCTRL_PIN_GROUP("gpio48", en7581_gpio48), + PINCTRL_PIN_GROUP("gpio49", en7581_gpio49), PINCTRL_PIN_GROUP("pcie_reset0", en7581_pcie_reset0), PINCTRL_PIN_GROUP("pcie_reset1", en7581_pcie_reset1), PINCTRL_PIN_GROUP("pcie_reset2", en7581_pcie_reset2), }; +static struct pinctrl_pin_desc an7583_pinctrl_pins[] = { + PINCTRL_PIN(2, "gpio0"), + PINCTRL_PIN(3, "gpio1"), + PINCTRL_PIN(4, "gpio2"), + PINCTRL_PIN(5, "gpio3"), + PINCTRL_PIN(6, "gpio4"), + PINCTRL_PIN(7, "gpio5"), + PINCTRL_PIN(8, "gpio6"), + PINCTRL_PIN(9, "gpio7"), + PINCTRL_PIN(10, "gpio8"), + PINCTRL_PIN(11, "gpio9"), + PINCTRL_PIN(12, "gpio10"), + PINCTRL_PIN(13, "gpio11"), + PINCTRL_PIN(14, "gpio12"), + PINCTRL_PIN(15, "gpio13"), + PINCTRL_PIN(16, "gpio14"), + PINCTRL_PIN(17, "gpio15"), + PINCTRL_PIN(18, "gpio16"), + PINCTRL_PIN(19, "gpio17"), + PINCTRL_PIN(20, "gpio18"), + PINCTRL_PIN(21, "gpio19"), + PINCTRL_PIN(22, "gpio20"), + PINCTRL_PIN(23, "gpio21"), + PINCTRL_PIN(24, "gpio22"), + PINCTRL_PIN(25, "gpio23"), + PINCTRL_PIN(26, "gpio24"), + PINCTRL_PIN(27, "gpio25"), + PINCTRL_PIN(28, "gpio26"), + PINCTRL_PIN(29, "gpio27"), + PINCTRL_PIN(30, "gpio28"), + PINCTRL_PIN(31, "gpio29"), + PINCTRL_PIN(32, "gpio30"), + PINCTRL_PIN(33, "gpio31"), + PINCTRL_PIN(34, "gpio32"), + PINCTRL_PIN(35, "gpio33"), + PINCTRL_PIN(36, "gpio34"), + PINCTRL_PIN(37, "gpio35"), + PINCTRL_PIN(38, "gpio36"), + PINCTRL_PIN(39, "gpio37"), + PINCTRL_PIN(40, "gpio38"), + PINCTRL_PIN(41, "i2c0_scl"), + PINCTRL_PIN(42, "i2c0_sda"), + PINCTRL_PIN(43, "i2c1_scl"), + PINCTRL_PIN(44, "i2c1_sda"), + PINCTRL_PIN(45, "spi_clk"), + PINCTRL_PIN(46, "spi_cs"), + PINCTRL_PIN(47, "spi_mosi"), + PINCTRL_PIN(48, "spi_miso"), + PINCTRL_PIN(49, "uart_txd"), + PINCTRL_PIN(50, "uart_rxd"), + PINCTRL_PIN(51, "pcie_reset0"), + PINCTRL_PIN(52, "pcie_reset1"), + PINCTRL_PIN(53, "mdc_0"), + PINCTRL_PIN(54, "mdio_0"), +}; + +static const int an7583_pon_pins[] = { 15, 16, 17, 18, 19, 20 }; +static const int an7583_pon_tod_1pps_pins[] = { 32 }; +static const int an7583_gsw_tod_1pps_pins[] = { 32 }; +static const int an7583_sipo_pins[] = { 34, 35 }; +static const int an7583_sipo_rclk_pins[] = { 34, 35, 33 }; +static const int an7583_mdio_pins[] = { 43, 44 }; +static const int an7583_uart2_pins[] = { 34, 35 }; +static const int an7583_uart2_cts_rts_pins[] = { 32, 33 }; +static const int an7583_hsuart_pins[] = { 30, 31 }; +static const int an7583_hsuart_cts_rts_pins[] = { 28, 29 }; +static const int an7583_npu_uart_pins[] = { 7, 8 }; +static const int an7583_uart4_pins[] = { 7, 8 }; +static const int an7583_uart5_pins[] = { 23, 24 }; +static const int an7583_i2c0_pins[] = { 41, 42 }; +static const int an7583_i2c1_pins[] = { 43, 44 }; +static const int an7583_jtag_udi_pins[] = { 23, 24, 22, 25, 26 }; +static const int an7583_jtag_dfd_pins[] = { 23, 24, 22, 25, 26 }; +static const int an7583_pcm1_pins[] = { 10, 11, 12, 13, 14 }; +static const int an7583_pcm2_pins[] = { 28, 29, 30, 31, 24 }; +static const int an7583_spi_pins[] = { 45, 46, 47, 48 }; +static const int an7583_spi_quad_pins[] = { 25, 26 }; +static const int an7583_spi_cs1_pins[] = { 27 }; +static const int an7583_pcm_spi_pins[] = { 28, 29, 30, 31, 10, 11, 12, 13 }; +static const int an7583_pcm_spi_rst_pins[] = { 14 }; +static const int an7583_pcm_spi_cs1_pins[] = { 24 }; +static const int an7583_emmc_pins[] = { 7, 8, 9, 22, 23, 24, 25, 26, 45, 46, 47 }; +static const int an7583_pnand_pins[] = { 7, 8, 9, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 45, 46, 47, 48 }; +static const int an7583_gpio0_pins[] = { 2 }; +static const int an7583_gpio1_pins[] = { 3 }; +static const int an7583_gpio2_pins[] = { 4 }; +static const int an7583_gpio3_pins[] = { 5 }; +static const int an7583_gpio4_pins[] = { 6 }; +static const int an7583_gpio5_pins[] = { 7 }; +static const int an7583_gpio6_pins[] = { 8 }; +static const int an7583_gpio7_pins[] = { 9 }; +static const int an7583_gpio8_pins[] = { 10 }; +static const int an7583_gpio9_pins[] = { 11 }; +static const int an7583_gpio10_pins[] = { 12 }; +static const int an7583_gpio11_pins[] = { 13 }; +static const int an7583_gpio12_pins[] = { 14 }; +static const int an7583_gpio13_pins[] = { 15 }; +static const int an7583_gpio14_pins[] = { 16 }; +static const int an7583_gpio15_pins[] = { 17 }; +static const int an7583_gpio16_pins[] = { 18 }; +static const int an7583_gpio17_pins[] = { 19 }; +static const int an7583_gpio18_pins[] = { 20 }; +static const int an7583_gpio19_pins[] = { 21 }; +static const int an7583_gpio20_pins[] = { 22 }; +static const int an7583_gpio21_pins[] = { 23 }; +static const int an7583_gpio22_pins[] = { 24 }; +static const int an7583_gpio23_pins[] = { 25 }; +static const int an7583_gpio24_pins[] = { 26 }; +static const int an7583_gpio25_pins[] = { 27 }; +static const int an7583_gpio26_pins[] = { 28 }; +static const int an7583_gpio27_pins[] = { 29 }; +static const int an7583_gpio28_pins[] = { 30 }; +static const int an7583_gpio29_pins[] = { 31 }; +static const int an7583_gpio30_pins[] = { 32 }; +static const int an7583_gpio31_pins[] = { 33 }; +static const int an7583_gpio32_pins[] = { 34 }; +static const int an7583_gpio33_pins[] = { 35 }; +static const int an7583_gpio34_pins[] = { 36 }; +static const int an7583_gpio35_pins[] = { 37 }; +static const int an7583_gpio36_pins[] = { 38 }; +static const int an7583_gpio37_pins[] = { 39 }; +static const int an7583_gpio38_pins[] = { 40 }; +static const int an7583_gpio39_pins[] = { 41 }; +static const int an7583_gpio40_pins[] = { 42 }; +static const int an7583_gpio41_pins[] = { 43 }; +static const int an7583_gpio42_pins[] = { 44 }; +static const int an7583_gpio43_pins[] = { 45 }; +static const int an7583_gpio44_pins[] = { 46 }; +static const int an7583_gpio45_pins[] = { 47 }; +static const int an7583_gpio46_pins[] = { 48 }; +static const int an7583_gpio47_pins[] = { 49 }; +static const int an7583_gpio48_pins[] = { 50 }; +static const int an7583_gpio49_pins[] = { 51 }; +static const int an7583_gpio50_pins[] = { 52 }; +static const int an7583_gpio51_pins[] = { 53 }; +static const int an7583_gpio52_pins[] = { 54 }; +static const int an7583_pcie_reset0_pins[] = { 51 }; +static const int an7583_pcie_reset1_pins[] = { 52 }; + +static const struct pingroup an7583_pinctrl_groups[] = { + PINCTRL_PIN_GROUP("pon", an7583_pon), + PINCTRL_PIN_GROUP("pon_tod_1pps", an7583_pon_tod_1pps), + PINCTRL_PIN_GROUP("gsw_tod_1pps", an7583_gsw_tod_1pps), + PINCTRL_PIN_GROUP("sipo", an7583_sipo), + PINCTRL_PIN_GROUP("sipo_rclk", an7583_sipo_rclk), + PINCTRL_PIN_GROUP("mdio", an7583_mdio), + PINCTRL_PIN_GROUP("uart2", an7583_uart2), + PINCTRL_PIN_GROUP("uart2_cts_rts", an7583_uart2_cts_rts), + PINCTRL_PIN_GROUP("hsuart", an7583_hsuart), + PINCTRL_PIN_GROUP("hsuart_cts_rts", an7583_hsuart_cts_rts), + PINCTRL_PIN_GROUP("npu_uart", an7583_npu_uart), + PINCTRL_PIN_GROUP("uart4", an7583_uart4), + PINCTRL_PIN_GROUP("uart5", an7583_uart5), + PINCTRL_PIN_GROUP("i2c0", an7583_i2c0), + PINCTRL_PIN_GROUP("i2c1", an7583_i2c1), + PINCTRL_PIN_GROUP("jtag_udi", an7583_jtag_udi), + PINCTRL_PIN_GROUP("jtag_dfd", an7583_jtag_dfd), + PINCTRL_PIN_GROUP("pcm1", an7583_pcm1), + PINCTRL_PIN_GROUP("pcm2", an7583_pcm2), + PINCTRL_PIN_GROUP("spi", an7583_spi), + PINCTRL_PIN_GROUP("spi_quad", an7583_spi_quad), + PINCTRL_PIN_GROUP("spi_cs1", an7583_spi_cs1), + PINCTRL_PIN_GROUP("pcm_spi", an7583_pcm_spi), + PINCTRL_PIN_GROUP("pcm_spi_rst", an7583_pcm_spi_rst), + PINCTRL_PIN_GROUP("pcm_spi_cs1", an7583_pcm_spi_cs1), + PINCTRL_PIN_GROUP("emmc", an7583_emmc), + PINCTRL_PIN_GROUP("pnand", an7583_pnand), + PINCTRL_PIN_GROUP("gpio0", an7583_gpio0), + PINCTRL_PIN_GROUP("gpio1", an7583_gpio1), + PINCTRL_PIN_GROUP("gpio2", an7583_gpio2), + PINCTRL_PIN_GROUP("gpio3", an7583_gpio3), + PINCTRL_PIN_GROUP("gpio4", an7583_gpio4), + PINCTRL_PIN_GROUP("gpio5", an7583_gpio5), + PINCTRL_PIN_GROUP("gpio6", an7583_gpio6), + PINCTRL_PIN_GROUP("gpio7", an7583_gpio7), + PINCTRL_PIN_GROUP("gpio8", an7583_gpio8), + PINCTRL_PIN_GROUP("gpio9", an7583_gpio9), + PINCTRL_PIN_GROUP("gpio10", an7583_gpio10), + PINCTRL_PIN_GROUP("gpio11", an7583_gpio11), + PINCTRL_PIN_GROUP("gpio12", an7583_gpio12), + PINCTRL_PIN_GROUP("gpio13", an7583_gpio13), + PINCTRL_PIN_GROUP("gpio14", an7583_gpio14), + PINCTRL_PIN_GROUP("gpio15", an7583_gpio15), + PINCTRL_PIN_GROUP("gpio16", an7583_gpio16), + PINCTRL_PIN_GROUP("gpio17", an7583_gpio17), + PINCTRL_PIN_GROUP("gpio18", an7583_gpio18), + PINCTRL_PIN_GROUP("gpio19", an7583_gpio19), + PINCTRL_PIN_GROUP("gpio20", an7583_gpio20), + PINCTRL_PIN_GROUP("gpio21", an7583_gpio21), + PINCTRL_PIN_GROUP("gpio22", an7583_gpio22), + PINCTRL_PIN_GROUP("gpio23", an7583_gpio23), + PINCTRL_PIN_GROUP("gpio24", an7583_gpio24), + PINCTRL_PIN_GROUP("gpio25", an7583_gpio25), + PINCTRL_PIN_GROUP("gpio26", an7583_gpio26), + PINCTRL_PIN_GROUP("gpio27", an7583_gpio27), + PINCTRL_PIN_GROUP("gpio28", an7583_gpio28), + PINCTRL_PIN_GROUP("gpio29", an7583_gpio29), + PINCTRL_PIN_GROUP("gpio30", an7583_gpio30), + PINCTRL_PIN_GROUP("gpio31", an7583_gpio31), + PINCTRL_PIN_GROUP("gpio32", an7583_gpio32), + PINCTRL_PIN_GROUP("gpio33", an7583_gpio33), + PINCTRL_PIN_GROUP("gpio34", an7583_gpio34), + PINCTRL_PIN_GROUP("gpio35", an7583_gpio35), + PINCTRL_PIN_GROUP("gpio36", an7583_gpio36), + PINCTRL_PIN_GROUP("gpio37", an7583_gpio37), + PINCTRL_PIN_GROUP("gpio38", an7583_gpio38), + PINCTRL_PIN_GROUP("gpio39", an7583_gpio39), + PINCTRL_PIN_GROUP("gpio40", an7583_gpio40), + PINCTRL_PIN_GROUP("gpio41", an7583_gpio41), + PINCTRL_PIN_GROUP("gpio42", an7583_gpio42), + PINCTRL_PIN_GROUP("gpio43", an7583_gpio43), + PINCTRL_PIN_GROUP("gpio44", an7583_gpio44), + PINCTRL_PIN_GROUP("gpio45", an7583_gpio45), + PINCTRL_PIN_GROUP("gpio46", an7583_gpio46), + PINCTRL_PIN_GROUP("gpio47", an7583_gpio47), + PINCTRL_PIN_GROUP("gpio48", an7583_gpio48), + PINCTRL_PIN_GROUP("gpio49", an7583_gpio49), + PINCTRL_PIN_GROUP("gpio50", an7583_gpio50), + PINCTRL_PIN_GROUP("gpio51", an7583_gpio51), + PINCTRL_PIN_GROUP("gpio52", an7583_gpio52), + PINCTRL_PIN_GROUP("pcie_reset0", an7583_pcie_reset0), + PINCTRL_PIN_GROUP("pcie_reset1", an7583_pcie_reset1), +}; + static const char *const pon_groups[] = { "pon" }; static const char *const tod_1pps_groups[] = { "pon_tod_1pps", "gsw_tod_1pps" }; static const char *const sipo_groups[] = { "sipo", "sipo_rclk" }; static const char *const mdio_groups[] = { "mdio" }; +static const char *const an7583_mdio_groups[] = { "mdio" }; static const char *const uart_groups[] = { "uart2", "uart2_cts_rts", "hsuart", "hsuart_cts_rts", "uart4", "uart5" }; @@ -643,11 +904,20 @@ static const char *const pcm_spi_groups[] = { "pcm_spi", "pcm_spi_int", "pcm_spi_cs2_p156", "pcm_spi_cs2_p128", "pcm_spi_cs3", "pcm_spi_cs4" }; +static const char *const an7583_pcm_spi_groups[] = { "pcm_spi", + "pcm_spi_rst", "pcm_spi_cs1" }; static const char *const i2s_groups[] = { "i2s" }; static const char *const emmc_groups[] = { "emmc" }; static const char *const pnand_groups[] = { "pnand" }; +static const char *const gpio_groups[] = { "gpio47", "gpio48", "gpio49" }; static const char *const pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1", "pcie_reset2" }; +static const char *const an7583_gpio_groups[] = { "gpio39", "gpio40", "gpio41", + "gpio42", "gpio43", "gpio44", + "gpio45", "gpio46", "gpio47", + "gpio48", "gpio49", "gpio50", + "gpio51", "gpio52" }; +static const char *const an7583_pcie_reset_groups[] = { "pcie_reset0", "pcie_reset1" }; static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio2", "gpio3", "gpio4", "gpio5", @@ -669,7 +939,32 @@ static const char *const pwm_groups[] = { "gpio0", "gpio1", "gpio40", "gpio41", "gpio42", "gpio43", "gpio44", "gpio45", - "gpio46", "gpio47" }; + "gpio46", "gpio47", + "gpio48", "gpio49" }; +static const char *const an7583_pwm_groups[] = { "gpio0", "gpio1", + "gpio2", "gpio3", + "gpio4", "gpio5", + "gpio6", "gpio7", + "gpio8", "gpio9", + "gpio10", "gpio11", + "gpio12", "gpio13", + "gpio14", "gpio15", + "gpio16", "gpio17", + "gpio18", "gpio19", + "gpio20", "gpio21", + "gpio22", "gpio23", + "gpio24", "gpio25", + "gpio26", "gpio27", + "gpio28", "gpio29", + "gpio30", "gpio31", + "gpio36", "gpio37", + "gpio38", "gpio39", + "gpio40", "gpio41", + "gpio42", "gpio43", + "gpio44", "gpio45", + "gpio46", "gpio47", + "gpio48", "gpio49", + "gpio50", "gpio51" }; static const char *const phy1_led0_groups[] = { "gpio33", "gpio34", "gpio35", "gpio42" }; static const char *const phy2_led0_groups[] = { "gpio33", "gpio34", @@ -686,6 +981,22 @@ static const char *const phy3_led1_groups[] = { "gpio43", "gpio44", "gpio45", "gpio46" }; static const char *const phy4_led1_groups[] = { "gpio43", "gpio44", "gpio45", "gpio46" }; +static const char *const an7583_phy1_led0_groups[] = { "gpio1", "gpio2", + "gpio3", "gpio4" }; +static const char *const an7583_phy2_led0_groups[] = { "gpio1", "gpio2", + "gpio3", "gpio4" }; +static const char *const an7583_phy3_led0_groups[] = { "gpio1", "gpio2", + "gpio3", "gpio4" }; +static const char *const an7583_phy4_led0_groups[] = { "gpio1", "gpio2", + "gpio3", "gpio4" }; +static const char *const an7583_phy1_led1_groups[] = { "gpio8", "gpio9", + "gpio10", "gpio11" }; +static const char *const an7583_phy2_led1_groups[] = { "gpio8", "gpio9", + "gpio10", "gpio11" }; +static const char *const an7583_phy3_led1_groups[] = { "gpio8", "gpio9", + "gpio10", "gpio11" }; +static const char *const an7583_phy4_led1_groups[] = { "gpio8", "gpio9", + "gpio10", "gpio11" }; static const struct airoha_pinctrl_func_group pon_func_group[] = { { @@ -750,8 +1061,8 @@ static const struct airoha_pinctrl_func_group mdio_func_group[] = { .regmap[0] = { AIROHA_FUNC_MUX, REG_GPIO_2ND_I2C_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE, - GPIO_MDC_IO_MASTER_MODE_MODE + GPIO_MDC_IO_MASTER_MODE_MASK, + GPIO_MDC_IO_MASTER_MODE_MASK }, .regmap[1] = { AIROHA_FUNC_MUX, @@ -763,6 +1074,25 @@ static const struct airoha_pinctrl_func_group mdio_func_group[] = { }, }; +static const struct airoha_pinctrl_func_group an7583_mdio_func_group[] = { + { + .name = "mdio", + .regmap[0] = { + AIROHA_FUNC_MUX, + REG_GPIO_PON_MODE, + GPIO_SGMII_MDIO_MODE_MASK, + GPIO_SGMII_MDIO_MODE_MASK + }, + .regmap[1] = { + AIROHA_FUNC_MUX, + REG_GPIO_SPI_CS1_MODE, + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK, + AN7583_GPIO_MDC_IO_MASTER_MODE_MASK + }, + .regmap_size = 2, + }, +}; + static const struct airoha_pinctrl_func_group uart_func_group[] = { { .name = "uart2", @@ -1004,989 +1334,605 @@ static const struct airoha_pinctrl_func_group pcm_spi_func_group[] = { }, }; -static const struct airoha_pinctrl_func_group i2s_func_group[] = { - { - .name = "i2s", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_I2S_MODE_MASK, - GPIO_I2S_MODE_MASK - }, - .regmap_size = 1, - }, -}; - -static const struct airoha_pinctrl_func_group emmc_func_group[] = { - { - .name = "emmc", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_PON_MODE, - GPIO_EMMC_MODE_MASK, - GPIO_EMMC_MODE_MASK - }, - .regmap_size = 1, - }, -}; - -static const struct airoha_pinctrl_func_group pnand_func_group[] = { - { - .name = "pnand", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_PON_MODE, - GPIO_PARALLEL_NAND_MODE_MASK, - GPIO_PARALLEL_NAND_MODE_MASK - }, - .regmap_size = 1, - }, -}; - -static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { +static const struct airoha_pinctrl_func_group an7583_pcm_spi_func_group[] = { { - .name = "pcie_reset0", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_PON_MODE, - GPIO_PCIE_RESET0_MASK, - GPIO_PCIE_RESET0_MASK - }, - .regmap_size = 1, - }, { - .name = "pcie_reset1", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_PON_MODE, - GPIO_PCIE_RESET1_MASK, - GPIO_PCIE_RESET1_MASK - }, - .regmap_size = 1, - }, { - .name = "pcie_reset2", + .name = "pcm_spi", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_PON_MODE, - GPIO_PCIE_RESET2_MASK, - GPIO_PCIE_RESET2_MASK - }, - .regmap_size = 1, - }, -}; - -/* PWM */ -static const struct airoha_pinctrl_func_group pwm_func_group[] = { - { - .name = "gpio0", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO0_FLASH_MODE_CFG, - GPIO0_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio1", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO1_FLASH_MODE_CFG, - GPIO1_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio2", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO2_FLASH_MODE_CFG, - GPIO2_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio3", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO3_FLASH_MODE_CFG, - GPIO3_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio4", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO4_FLASH_MODE_CFG, - GPIO4_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio5", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO5_FLASH_MODE_CFG, - GPIO5_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio6", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO6_FLASH_MODE_CFG, - GPIO6_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio7", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO7_FLASH_MODE_CFG, - GPIO7_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio8", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO8_FLASH_MODE_CFG, - GPIO8_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio9", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO9_FLASH_MODE_CFG, - GPIO9_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio10", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO10_FLASH_MODE_CFG, - GPIO10_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio11", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO11_FLASH_MODE_CFG, - GPIO11_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio12", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO12_FLASH_MODE_CFG, - GPIO12_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio13", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO13_FLASH_MODE_CFG, - GPIO13_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio14", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO14_FLASH_MODE_CFG, - GPIO14_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio15", - .regmap[0] = { - AIROHA_FUNC_PWM_MUX, - REG_GPIO_FLASH_MODE_CFG, - GPIO15_FLASH_MODE_CFG, - GPIO15_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio16", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO16_FLASH_MODE_CFG, - GPIO16_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio17", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO17_FLASH_MODE_CFG, - GPIO17_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio18", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO18_FLASH_MODE_CFG, - GPIO18_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio19", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO19_FLASH_MODE_CFG, - GPIO19_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio20", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO20_FLASH_MODE_CFG, - GPIO20_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio21", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO21_FLASH_MODE_CFG, - GPIO21_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio22", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO22_FLASH_MODE_CFG, - GPIO22_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio23", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO23_FLASH_MODE_CFG, - GPIO23_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio24", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO24_FLASH_MODE_CFG, - GPIO24_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio25", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO25_FLASH_MODE_CFG, - GPIO25_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio26", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO26_FLASH_MODE_CFG, - GPIO26_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio27", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO27_FLASH_MODE_CFG, - GPIO27_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio28", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO28_FLASH_MODE_CFG, - GPIO28_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio29", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO29_FLASH_MODE_CFG, - GPIO29_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio30", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO30_FLASH_MODE_CFG, - GPIO30_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio31", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO31_FLASH_MODE_CFG, - GPIO31_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio36", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO36_FLASH_MODE_CFG, - GPIO36_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio37", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO37_FLASH_MODE_CFG, - GPIO37_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio38", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO38_FLASH_MODE_CFG, - GPIO38_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio39", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO39_FLASH_MODE_CFG, - GPIO39_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio40", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO40_FLASH_MODE_CFG, - GPIO40_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio41", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO41_FLASH_MODE_CFG, - GPIO41_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio42", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO42_FLASH_MODE_CFG, - GPIO42_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio43", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO43_FLASH_MODE_CFG, - GPIO43_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio44", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO44_FLASH_MODE_CFG, - GPIO44_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio45", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO45_FLASH_MODE_CFG, - GPIO45_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio46", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO46_FLASH_MODE_CFG, - GPIO46_FLASH_MODE_CFG - }, - .regmap_size = 1, - }, { - .name = "gpio47", - .regmap[0] = { - AIROHA_FUNC_PWM_EXT_MUX, - REG_GPIO_FLASH_MODE_CFG_EXT, - GPIO47_FLASH_MODE_CFG, - GPIO47_FLASH_MODE_CFG + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_SPI_MODE_MASK, + GPIO_PCM_SPI_MODE_MASK }, .regmap_size = 1, - }, -}; - -static const struct airoha_pinctrl_func_group phy1_led0_func_group[] = { - { - .name = "gpio33", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED0_MODE_MASK, - GPIO_LAN0_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio34", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED0_MODE_MASK, - GPIO_LAN1_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio35", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED0_MODE_MASK, - GPIO_LAN2_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio42", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED0_MODE_MASK, - GPIO_LAN3_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy2_led0_func_group[] = { - { - .name = "gpio33", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED0_MODE_MASK, - GPIO_LAN0_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(1) - }, - .regmap_size = 2, }, { - .name = "gpio34", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED0_MODE_MASK, - GPIO_LAN1_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, { - .name = "gpio35", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED0_MODE_MASK, - GPIO_LAN2_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, { - .name = "gpio42", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED0_MODE_MASK, - GPIO_LAN3_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy3_led0_func_group[] = { - { - .name = "gpio33", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED0_MODE_MASK, - GPIO_LAN0_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(2) - }, - .regmap_size = 2, - }, { - .name = "gpio34", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED0_MODE_MASK, - GPIO_LAN1_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(2) - }, - .regmap_size = 2, - }, { - .name = "gpio35", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED0_MODE_MASK, - GPIO_LAN2_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(2) - }, - .regmap_size = 2, - }, { - .name = "gpio42", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED0_MODE_MASK, - GPIO_LAN3_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(2) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy4_led0_func_group[] = { - { - .name = "gpio33", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED0_MODE_MASK, - GPIO_LAN0_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(3) - }, - .regmap_size = 2, - }, { - .name = "gpio34", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED0_MODE_MASK, - GPIO_LAN1_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(3) - }, - .regmap_size = 2, - }, { - .name = "gpio35", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED0_MODE_MASK, - GPIO_LAN2_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(3) - }, - .regmap_size = 2, - }, { - .name = "gpio42", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED0_MODE_MASK, - GPIO_LAN3_LED0_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED0_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(3) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy1_led1_func_group[] = { - { - .name = "gpio43", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED1_MODE_MASK, - GPIO_LAN0_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio44", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED1_MODE_MASK, - GPIO_LAN1_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio45", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED1_MODE_MASK, - GPIO_LAN2_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, { - .name = "gpio46", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED1_MODE_MASK, - GPIO_LAN3_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(0) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy2_led1_func_group[] = { - { - .name = "gpio43", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED1_MODE_MASK, - GPIO_LAN0_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, { - .name = "gpio44", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED1_MODE_MASK, - GPIO_LAN1_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, { - .name = "gpio45", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED1_MODE_MASK, - GPIO_LAN2_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, { - .name = "gpio46", - .regmap[0] = { - AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED1_MODE_MASK, - GPIO_LAN3_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(1) - }, - .regmap_size = 2, - }, -}; - -static const struct airoha_pinctrl_func_group phy3_led1_func_group[] = { - { - .name = "gpio43", + .name = "pcm_spi_int", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED1_MODE_MASK, - GPIO_LAN0_LED1_MODE_MASK + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_INT_MODE_MASK, + GPIO_PCM_INT_MODE_MASK }, - .regmap[1] = { + .regmap_size = 1, + }, { + .name = "pcm_spi_rst", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(2) + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_RESET_MODE_MASK, + GPIO_PCM_RESET_MODE_MASK }, - .regmap_size = 2, + .regmap_size = 1, }, { - .name = "gpio44", + .name = "pcm_spi_cs1", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED1_MODE_MASK, - GPIO_LAN1_LED1_MODE_MASK + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_SPI_CS1_MODE_MASK, + GPIO_PCM_SPI_CS1_MODE_MASK }, - .regmap[1] = { + .regmap_size = 1, + }, { + .name = "pcm_spi_cs2", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(2) + REG_GPIO_SPI_CS1_MODE, + AN7583_GPIO_PCM_SPI_CS2_MODE_MASK, + AN7583_GPIO_PCM_SPI_CS2_MODE_MASK }, - .regmap_size = 2, + .regmap_size = 1, }, { - .name = "gpio45", + .name = "pcm_spi_cs3", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED1_MODE_MASK, - GPIO_LAN2_LED1_MODE_MASK + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_SPI_CS3_MODE_MASK, + GPIO_PCM_SPI_CS3_MODE_MASK }, - .regmap[1] = { + .regmap_size = 1, + }, { + .name = "pcm_spi_cs4", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(2) + REG_GPIO_SPI_CS1_MODE, + GPIO_PCM_SPI_CS4_MODE_MASK, + GPIO_PCM_SPI_CS4_MODE_MASK }, - .regmap_size = 2, - }, { - .name = "gpio46", + .regmap_size = 1, + }, +}; + +static const struct airoha_pinctrl_func_group i2s_func_group[] = { + { + .name = "i2s", .regmap[0] = { AIROHA_FUNC_MUX, REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED1_MODE_MASK, - GPIO_LAN3_LED1_MODE_MASK + GPIO_I2S_MODE_MASK, + GPIO_I2S_MODE_MASK }, - .regmap[1] = { + .regmap_size = 1, + }, +}; + +static const struct airoha_pinctrl_func_group emmc_func_group[] = { + { + .name = "emmc", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(2) + REG_GPIO_PON_MODE, + GPIO_EMMC_MODE_MASK, + GPIO_EMMC_MODE_MASK }, - .regmap_size = 2, + .regmap_size = 1, }, }; -static const struct airoha_pinctrl_func_group phy4_led1_func_group[] = { +static const struct airoha_pinctrl_func_group pnand_func_group[] = { { - .name = "gpio43", + .name = "pnand", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN0_LED1_MODE_MASK, - GPIO_LAN0_LED1_MODE_MASK + REG_GPIO_PON_MODE, + GPIO_PARALLEL_NAND_MODE_MASK, + GPIO_PARALLEL_NAND_MODE_MASK }, - .regmap[1] = { + .regmap_size = 1, + }, +}; + +#define AIROHA_PINCTRL_GPIO(gpio, mux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap_size = 1, \ + } + +#define AIROHA_PINCTRL_GPIO_EXT(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + 0 \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + +static const struct airoha_pinctrl_func_group gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), +}; + +static const struct airoha_pinctrl_func_group pcie_reset_func_group[] = { + { + .name = "pcie_reset0", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN0_LED_MAPPING_MASK, - LAN0_PHY_LED_MAP(3) + REG_GPIO_PON_MODE, + GPIO_PCIE_RESET0_MASK, + 0 }, - .regmap_size = 2, + .regmap_size = 1, }, { - .name = "gpio44", + .name = "pcie_reset1", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN1_LED1_MODE_MASK, - GPIO_LAN1_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN1_LED_MAPPING_MASK, - LAN1_PHY_LED_MAP(3) + REG_GPIO_PON_MODE, + GPIO_PCIE_RESET1_MASK, + 0 }, - .regmap_size = 2, + .regmap_size = 1, }, { - .name = "gpio45", + .name = "pcie_reset2", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN2_LED1_MODE_MASK, - GPIO_LAN2_LED1_MODE_MASK + REG_GPIO_PON_MODE, + GPIO_PCIE_RESET2_MASK, + 0 }, - .regmap[1] = { + .regmap_size = 1, + }, +}; + +static const struct airoha_pinctrl_func_group an7583_gpio_func_group[] = { + AIROHA_PINCTRL_GPIO_EXT("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_GPIO_EXT("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), + AIROHA_PINCTRL_GPIO("gpio52", AN7583_MDIO_0_GPIO_MODE_MASK), +}; + +static const struct airoha_pinctrl_func_group an7583_pcie_reset_func_group[] = { + { + .name = "pcie_reset0", + .regmap[0] = { AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN2_LED_MAPPING_MASK, - LAN2_PHY_LED_MAP(3) + REG_GPIO_PON_MODE, + GPIO_PCIE_RESET0_MASK, + 0 }, - .regmap_size = 2, + .regmap_size = 1, }, { - .name = "gpio46", + .name = "pcie_reset1", .regmap[0] = { AIROHA_FUNC_MUX, - REG_GPIO_2ND_I2C_MODE, - GPIO_LAN3_LED1_MODE_MASK, - GPIO_LAN3_LED1_MODE_MASK - }, - .regmap[1] = { - AIROHA_FUNC_MUX, - REG_LAN_LED1_MAPPING, - LAN3_LED_MAPPING_MASK, - LAN3_PHY_LED_MAP(3) + REG_GPIO_PON_MODE, + GPIO_PCIE_RESET1_MASK, + 0 }, - .regmap_size = 2, + .regmap_size = 1, }, }; +/* PWM */ +#define AIROHA_PINCTRL_PWM(gpio, mux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_MUX, \ + REG_GPIO_FLASH_MODE_CFG, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap_size = 1, \ + } \ + +#define AIROHA_PINCTRL_PWM_EXT(gpio, mux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap_size = 1, \ + } \ + +#define AIROHA_PINCTRL_PWM_EXT_SEC(gpio, mux_val, smux_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_PWM_EXT_MUX, \ + REG_GPIO_FLASH_MODE_CFG_EXT, \ + (mux_val), \ + (mux_val) \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_PON_MODE, \ + (smux_val), \ + (smux_val) \ + }, \ + .regmap_size = 2, \ + } + +static const struct airoha_pinctrl_func_group pwm_func_group[] = { + AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio2", GPIO2_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio3", GPIO3_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio4", GPIO4_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio5", GPIO5_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio6", GPIO6_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio7", GPIO7_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio8", GPIO8_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio9", GPIO9_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio10", GPIO10_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio11", GPIO11_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio12", GPIO12_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio13", GPIO13_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio14", GPIO14_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio15", GPIO15_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio16", GPIO16_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio17", GPIO17_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio18", GPIO18_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio19", GPIO19_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio20", GPIO20_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio21", GPIO21_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio22", GPIO22_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio23", GPIO23_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio24", GPIO24_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio25", GPIO25_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio26", GPIO26_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio27", GPIO27_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio28", GPIO28_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio29", GPIO29_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio30", GPIO30_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio31", GPIO31_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio36", GPIO36_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio39", GPIO39_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio40", GPIO40_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio41", GPIO41_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio42", GPIO42_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio43", GPIO43_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio44", GPIO44_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio45", GPIO45_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio46", GPIO46_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET2_MASK), +}; + +static const struct airoha_pinctrl_func_group an7583_pwm_func_group[] = { + AIROHA_PINCTRL_PWM("gpio0", GPIO0_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio1", GPIO1_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio2", GPIO2_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio3", GPIO3_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio4", GPIO4_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio5", GPIO5_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio6", GPIO6_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio7", GPIO7_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio8", GPIO8_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio9", GPIO9_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio10", GPIO10_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio11", GPIO11_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio12", GPIO12_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio13", GPIO13_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio14", GPIO14_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM("gpio15", GPIO15_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio16", GPIO16_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio17", GPIO17_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio18", GPIO18_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio19", GPIO19_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio20", GPIO20_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio21", GPIO21_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio22", GPIO22_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio23", GPIO23_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio24", GPIO24_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio25", GPIO25_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio26", GPIO26_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio27", GPIO27_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio28", GPIO28_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio29", GPIO29_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio30", GPIO30_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio31", GPIO31_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio36", GPIO36_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio37", GPIO37_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT("gpio38", GPIO38_FLASH_MODE_CFG), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio39", GPIO39_FLASH_MODE_CFG, + AN7583_I2C0_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio40", GPIO40_FLASH_MODE_CFG, + AN7583_I2C0_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio41", GPIO41_FLASH_MODE_CFG, + AN7583_I2C1_SCL_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio42", GPIO42_FLASH_MODE_CFG, + AN7583_I2C1_SDA_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio43", GPIO43_FLASH_MODE_CFG, + AN7583_SPI_CLK_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio44", GPIO44_FLASH_MODE_CFG, + AN7583_SPI_CS_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio45", GPIO45_FLASH_MODE_CFG, + AN7583_SPI_MOSI_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio46", GPIO46_FLASH_MODE_CFG, + AN7583_SPI_MISO_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio47", GPIO47_FLASH_MODE_CFG, + AN7583_UART_TXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio48", GPIO48_FLASH_MODE_CFG, + AN7583_UART_RXD_GPIO_MODE_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio49", GPIO49_FLASH_MODE_CFG, + GPIO_PCIE_RESET0_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio50", GPIO50_FLASH_MODE_CFG, + GPIO_PCIE_RESET1_MASK), + AIROHA_PINCTRL_PWM_EXT_SEC("gpio51", GPIO51_FLASH_MODE_CFG, + AN7583_MDC_0_GPIO_MODE_MASK), +}; + +#define AIROHA_PINCTRL_PHY_LED0(gpio, mux_val, map_mask, map_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_2ND_I2C_MODE, \ + (mux_val), \ + (mux_val), \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_LAN_LED0_MAPPING, \ + (map_mask), \ + (map_val), \ + }, \ + .regmap_size = 2, \ + } + +#define AIROHA_PINCTRL_PHY_LED1(gpio, mux_val, map_mask, map_val) \ + { \ + .name = (gpio), \ + .regmap[0] = { \ + AIROHA_FUNC_MUX, \ + REG_GPIO_2ND_I2C_MODE, \ + (mux_val), \ + (mux_val), \ + }, \ + .regmap[1] = { \ + AIROHA_FUNC_MUX, \ + REG_LAN_LED1_MAPPING, \ + (map_mask), \ + (map_val), \ + }, \ + .regmap_size = 2, \ + } + +static const struct airoha_pinctrl_func_group phy1_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio33", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio34", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio35", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio42", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)), +}; + +static const struct airoha_pinctrl_func_group phy2_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio33", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio34", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio35", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio42", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)), +}; + +static const struct airoha_pinctrl_func_group phy3_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio33", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio34", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio35", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio42", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), +}; + +static const struct airoha_pinctrl_func_group phy4_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio33", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio34", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio35", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio42", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), +}; + +static const struct airoha_pinctrl_func_group phy1_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio43", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio44", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio45", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio46", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)), +}; + +static const struct airoha_pinctrl_func_group phy2_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio43", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio44", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio45", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio46", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)), +}; + +static const struct airoha_pinctrl_func_group phy3_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio43", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio44", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio45", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio46", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), +}; + +static const struct airoha_pinctrl_func_group phy4_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio43", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio44", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio45", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio46", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy1_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio1", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio2", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio3", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED0("gpio4", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy2_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio1", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio2", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio3", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED0("gpio4", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy3_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio1", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio2", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio3", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED0("gpio4", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy4_led0_func_group[] = { + AIROHA_PINCTRL_PHY_LED0("gpio1", GPIO_LAN0_LED0_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio2", GPIO_LAN1_LED0_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio3", GPIO_LAN2_LED0_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED0("gpio4", GPIO_LAN3_LED0_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy1_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio8", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio9", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(0)), + AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(0)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy2_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio8", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio9", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(1)), + AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(1)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy3_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio8", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio9", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(2)), + AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(2)), +}; + +static const struct airoha_pinctrl_func_group an7583_phy4_led1_func_group[] = { + AIROHA_PINCTRL_PHY_LED1("gpio8", GPIO_LAN0_LED1_MODE_MASK, + LAN0_LED_MAPPING_MASK, LAN0_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio9", GPIO_LAN1_LED1_MODE_MASK, + LAN1_LED_MAPPING_MASK, LAN1_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio10", GPIO_LAN2_LED1_MODE_MASK, + LAN2_LED_MAPPING_MASK, LAN2_PHY_LED_MAP(3)), + AIROHA_PINCTRL_PHY_LED1("gpio11", GPIO_LAN3_LED1_MODE_MASK, + LAN3_LED_MAPPING_MASK, LAN3_PHY_LED_MAP(3)), +}; + static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("pon", pon), PINCTRL_FUNC_DESC("tod_1pps", tod_1pps), @@ -2001,6 +1947,7 @@ static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("i2s", i2s), PINCTRL_FUNC_DESC("emmc", emmc), PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", gpio), PINCTRL_FUNC_DESC("pcie_reset", pcie_reset), PINCTRL_FUNC_DESC("pwm", pwm), PINCTRL_FUNC_DESC("phy1_led0", phy1_led0), @@ -2013,11 +1960,37 @@ static const struct airoha_pinctrl_func en7581_pinctrl_funcs[] = { PINCTRL_FUNC_DESC("phy4_led1", phy4_led1), }; +static const struct airoha_pinctrl_func an7583_pinctrl_funcs[] = { + PINCTRL_FUNC_DESC("pon", pon), + PINCTRL_FUNC_DESC("tod_1pps", tod_1pps), + PINCTRL_FUNC_DESC("sipo", sipo), + PINCTRL_FUNC_DESC("mdio", an7583_mdio), + PINCTRL_FUNC_DESC("uart", uart), + PINCTRL_FUNC_DESC("i2c", i2c), + PINCTRL_FUNC_DESC("jtag", jtag), + PINCTRL_FUNC_DESC("pcm", pcm), + PINCTRL_FUNC_DESC("spi", spi), + PINCTRL_FUNC_DESC("pcm_spi", an7583_pcm_spi), + PINCTRL_FUNC_DESC("emmc", emmc), + PINCTRL_FUNC_DESC("pnand", pnand), + PINCTRL_FUNC_DESC("gpio", an7583_gpio), + PINCTRL_FUNC_DESC("pcie_reset", an7583_pcie_reset), + PINCTRL_FUNC_DESC("pwm", an7583_pwm), + PINCTRL_FUNC_DESC("phy1_led0", an7583_phy1_led0), + PINCTRL_FUNC_DESC("phy2_led0", an7583_phy2_led0), + PINCTRL_FUNC_DESC("phy3_led0", an7583_phy3_led0), + PINCTRL_FUNC_DESC("phy4_led0", an7583_phy4_led0), + PINCTRL_FUNC_DESC("phy1_led1", an7583_phy1_led1), + PINCTRL_FUNC_DESC("phy2_led1", an7583_phy2_led1), + PINCTRL_FUNC_DESC("phy3_led1", an7583_phy3_led1), + PINCTRL_FUNC_DESC("phy4_led1", an7583_phy4_led1), +}; + static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PU, UART1_TXD_PU_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PU, UART1_RXD_PU_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PU, SPI_CS0_PU_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PU, SPI_CLK_PU_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PU, SPI_MOSI_PU_MASK), @@ -2069,16 +2042,72 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pullup_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PU, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PU, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PU, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PU, PCIE2_RESET_PU_MASK), +}; + +static const struct airoha_pinctrl_conf an7583_pinctrl_pullup_conf[] = { + PINCTRL_CONF_DESC(2, REG_GPIO_L_PU, BIT(0)), + PINCTRL_CONF_DESC(3, REG_GPIO_L_PU, BIT(1)), + PINCTRL_CONF_DESC(4, REG_GPIO_L_PU, BIT(2)), + PINCTRL_CONF_DESC(5, REG_GPIO_L_PU, BIT(3)), + PINCTRL_CONF_DESC(6, REG_GPIO_L_PU, BIT(4)), + PINCTRL_CONF_DESC(7, REG_GPIO_L_PU, BIT(5)), + PINCTRL_CONF_DESC(8, REG_GPIO_L_PU, BIT(6)), + PINCTRL_CONF_DESC(9, REG_GPIO_L_PU, BIT(7)), + PINCTRL_CONF_DESC(10, REG_GPIO_L_PU, BIT(8)), + PINCTRL_CONF_DESC(11, REG_GPIO_L_PU, BIT(9)), + PINCTRL_CONF_DESC(12, REG_GPIO_L_PU, BIT(10)), + PINCTRL_CONF_DESC(13, REG_GPIO_L_PU, BIT(11)), + PINCTRL_CONF_DESC(14, REG_GPIO_L_PU, BIT(12)), + PINCTRL_CONF_DESC(15, REG_GPIO_L_PU, BIT(13)), + PINCTRL_CONF_DESC(16, REG_GPIO_L_PU, BIT(14)), + PINCTRL_CONF_DESC(17, REG_GPIO_L_PU, BIT(15)), + PINCTRL_CONF_DESC(18, REG_GPIO_L_PU, BIT(16)), + PINCTRL_CONF_DESC(19, REG_GPIO_L_PU, BIT(17)), + PINCTRL_CONF_DESC(20, REG_GPIO_L_PU, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_PU, BIT(19)), + PINCTRL_CONF_DESC(22, REG_GPIO_L_PU, BIT(20)), + PINCTRL_CONF_DESC(23, REG_GPIO_L_PU, BIT(21)), + PINCTRL_CONF_DESC(24, REG_GPIO_L_PU, BIT(22)), + PINCTRL_CONF_DESC(25, REG_GPIO_L_PU, BIT(23)), + PINCTRL_CONF_DESC(26, REG_GPIO_L_PU, BIT(24)), + PINCTRL_CONF_DESC(27, REG_GPIO_L_PU, BIT(25)), + PINCTRL_CONF_DESC(28, REG_GPIO_L_PU, BIT(26)), + PINCTRL_CONF_DESC(29, REG_GPIO_L_PU, BIT(27)), + PINCTRL_CONF_DESC(30, REG_GPIO_L_PU, BIT(28)), + PINCTRL_CONF_DESC(31, REG_GPIO_L_PU, BIT(29)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_PU, BIT(30)), + PINCTRL_CONF_DESC(33, REG_GPIO_L_PU, BIT(31)), + PINCTRL_CONF_DESC(34, REG_GPIO_H_PU, BIT(0)), + PINCTRL_CONF_DESC(35, REG_GPIO_H_PU, BIT(1)), + PINCTRL_CONF_DESC(36, REG_GPIO_H_PU, BIT(2)), + PINCTRL_CONF_DESC(37, REG_GPIO_H_PU, BIT(3)), + PINCTRL_CONF_DESC(38, REG_GPIO_H_PU, BIT(4)), + PINCTRL_CONF_DESC(39, REG_GPIO_H_PU, BIT(5)), + PINCTRL_CONF_DESC(40, REG_GPIO_H_PU, BIT(6)), + PINCTRL_CONF_DESC(41, REG_I2C_SDA_PU, I2C_SCL_PU_MASK), + PINCTRL_CONF_DESC(42, REG_I2C_SDA_PU, I2C_SDA_PU_MASK), + PINCTRL_CONF_DESC(43, REG_I2C_SDA_PU, AN7583_I2C1_SCL_PU_MASK), + PINCTRL_CONF_DESC(44, REG_I2C_SDA_PU, AN7583_I2C1_SDA_PU_MASK), + PINCTRL_CONF_DESC(45, REG_I2C_SDA_PU, SPI_CLK_PU_MASK), + PINCTRL_CONF_DESC(46, REG_I2C_SDA_PU, SPI_CS0_PU_MASK), + PINCTRL_CONF_DESC(47, REG_I2C_SDA_PU, SPI_MOSI_PU_MASK), + PINCTRL_CONF_DESC(48, REG_I2C_SDA_PU, SPI_MISO_PU_MASK), + PINCTRL_CONF_DESC(49, REG_I2C_SDA_PU, UART1_TXD_PU_MASK), + PINCTRL_CONF_DESC(50, REG_I2C_SDA_PU, UART1_RXD_PU_MASK), + PINCTRL_CONF_DESC(51, REG_I2C_SDA_PU, PCIE0_RESET_PU_MASK), + PINCTRL_CONF_DESC(52, REG_I2C_SDA_PU, PCIE1_RESET_PU_MASK), + PINCTRL_CONF_DESC(53, REG_I2C_SDA_PU, AN7583_MDC_0_PU_MASK), + PINCTRL_CONF_DESC(54, REG_I2C_SDA_PU, AN7583_MDIO_0_PU_MASK), }; static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_PD, UART1_TXD_PD_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_PD, UART1_RXD_PD_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_PD, SPI_CS0_PD_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_PD, SPI_CLK_PD_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_PD, SPI_MOSI_PD_MASK), @@ -2130,16 +2159,72 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_pulldown_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_PD, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_PD, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_PD, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_PD, PCIE2_RESET_PD_MASK), +}; + +static const struct airoha_pinctrl_conf an7583_pinctrl_pulldown_conf[] = { + PINCTRL_CONF_DESC(2, REG_GPIO_L_PD, BIT(0)), + PINCTRL_CONF_DESC(3, REG_GPIO_L_PD, BIT(1)), + PINCTRL_CONF_DESC(4, REG_GPIO_L_PD, BIT(2)), + PINCTRL_CONF_DESC(5, REG_GPIO_L_PD, BIT(3)), + PINCTRL_CONF_DESC(6, REG_GPIO_L_PD, BIT(4)), + PINCTRL_CONF_DESC(7, REG_GPIO_L_PD, BIT(5)), + PINCTRL_CONF_DESC(8, REG_GPIO_L_PD, BIT(6)), + PINCTRL_CONF_DESC(9, REG_GPIO_L_PD, BIT(7)), + PINCTRL_CONF_DESC(10, REG_GPIO_L_PD, BIT(8)), + PINCTRL_CONF_DESC(11, REG_GPIO_L_PD, BIT(9)), + PINCTRL_CONF_DESC(12, REG_GPIO_L_PD, BIT(10)), + PINCTRL_CONF_DESC(13, REG_GPIO_L_PD, BIT(11)), + PINCTRL_CONF_DESC(14, REG_GPIO_L_PD, BIT(12)), + PINCTRL_CONF_DESC(15, REG_GPIO_L_PD, BIT(13)), + PINCTRL_CONF_DESC(16, REG_GPIO_L_PD, BIT(14)), + PINCTRL_CONF_DESC(17, REG_GPIO_L_PD, BIT(15)), + PINCTRL_CONF_DESC(18, REG_GPIO_L_PD, BIT(16)), + PINCTRL_CONF_DESC(19, REG_GPIO_L_PD, BIT(17)), + PINCTRL_CONF_DESC(20, REG_GPIO_L_PD, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_PD, BIT(19)), + PINCTRL_CONF_DESC(22, REG_GPIO_L_PD, BIT(20)), + PINCTRL_CONF_DESC(23, REG_GPIO_L_PD, BIT(21)), + PINCTRL_CONF_DESC(24, REG_GPIO_L_PD, BIT(22)), + PINCTRL_CONF_DESC(25, REG_GPIO_L_PD, BIT(23)), + PINCTRL_CONF_DESC(26, REG_GPIO_L_PD, BIT(24)), + PINCTRL_CONF_DESC(27, REG_GPIO_L_PD, BIT(25)), + PINCTRL_CONF_DESC(28, REG_GPIO_L_PD, BIT(26)), + PINCTRL_CONF_DESC(29, REG_GPIO_L_PD, BIT(27)), + PINCTRL_CONF_DESC(30, REG_GPIO_L_PD, BIT(28)), + PINCTRL_CONF_DESC(31, REG_GPIO_L_PD, BIT(29)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_PD, BIT(30)), + PINCTRL_CONF_DESC(33, REG_GPIO_L_PD, BIT(31)), + PINCTRL_CONF_DESC(34, REG_GPIO_H_PD, BIT(0)), + PINCTRL_CONF_DESC(35, REG_GPIO_H_PD, BIT(1)), + PINCTRL_CONF_DESC(36, REG_GPIO_H_PD, BIT(2)), + PINCTRL_CONF_DESC(37, REG_GPIO_H_PD, BIT(3)), + PINCTRL_CONF_DESC(38, REG_GPIO_H_PD, BIT(4)), + PINCTRL_CONF_DESC(39, REG_GPIO_H_PD, BIT(5)), + PINCTRL_CONF_DESC(40, REG_GPIO_H_PD, BIT(6)), + PINCTRL_CONF_DESC(41, REG_I2C_SDA_PD, I2C_SCL_PD_MASK), + PINCTRL_CONF_DESC(42, REG_I2C_SDA_PD, I2C_SDA_PD_MASK), + PINCTRL_CONF_DESC(43, REG_I2C_SDA_PD, AN7583_I2C1_SCL_PD_MASK), + PINCTRL_CONF_DESC(44, REG_I2C_SDA_PD, AN7583_I2C1_SDA_PD_MASK), + PINCTRL_CONF_DESC(45, REG_I2C_SDA_PD, SPI_CLK_PD_MASK), + PINCTRL_CONF_DESC(46, REG_I2C_SDA_PD, SPI_CS0_PD_MASK), + PINCTRL_CONF_DESC(47, REG_I2C_SDA_PD, SPI_MOSI_PD_MASK), + PINCTRL_CONF_DESC(48, REG_I2C_SDA_PD, SPI_MISO_PD_MASK), + PINCTRL_CONF_DESC(49, REG_I2C_SDA_PD, UART1_TXD_PD_MASK), + PINCTRL_CONF_DESC(50, REG_I2C_SDA_PD, UART1_RXD_PD_MASK), + PINCTRL_CONF_DESC(51, REG_I2C_SDA_PD, PCIE0_RESET_PD_MASK), + PINCTRL_CONF_DESC(52, REG_I2C_SDA_PD, PCIE1_RESET_PD_MASK), + PINCTRL_CONF_DESC(53, REG_I2C_SDA_PD, AN7583_MDC_0_PD_MASK), + PINCTRL_CONF_DESC(54, REG_I2C_SDA_PD, AN7583_MDIO_0_PD_MASK), }; static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E2, UART1_TXD_E2_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E2, UART1_RXD_E2_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E2, SPI_CS0_E2_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E2, SPI_CLK_E2_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E2, SPI_MOSI_E2_MASK), @@ -2191,16 +2276,72 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e2_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E2, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E2, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E2, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E2, PCIE2_RESET_E2_MASK), +}; + +static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e2_conf[] = { + PINCTRL_CONF_DESC(2, REG_GPIO_L_E2, BIT(0)), + PINCTRL_CONF_DESC(3, REG_GPIO_L_E2, BIT(1)), + PINCTRL_CONF_DESC(4, REG_GPIO_L_E2, BIT(2)), + PINCTRL_CONF_DESC(5, REG_GPIO_L_E2, BIT(3)), + PINCTRL_CONF_DESC(6, REG_GPIO_L_E2, BIT(4)), + PINCTRL_CONF_DESC(7, REG_GPIO_L_E2, BIT(5)), + PINCTRL_CONF_DESC(8, REG_GPIO_L_E2, BIT(6)), + PINCTRL_CONF_DESC(9, REG_GPIO_L_E2, BIT(7)), + PINCTRL_CONF_DESC(10, REG_GPIO_L_E2, BIT(8)), + PINCTRL_CONF_DESC(11, REG_GPIO_L_E2, BIT(9)), + PINCTRL_CONF_DESC(12, REG_GPIO_L_E2, BIT(10)), + PINCTRL_CONF_DESC(13, REG_GPIO_L_E2, BIT(11)), + PINCTRL_CONF_DESC(14, REG_GPIO_L_E2, BIT(12)), + PINCTRL_CONF_DESC(15, REG_GPIO_L_E2, BIT(13)), + PINCTRL_CONF_DESC(16, REG_GPIO_L_E2, BIT(14)), + PINCTRL_CONF_DESC(17, REG_GPIO_L_E2, BIT(15)), + PINCTRL_CONF_DESC(18, REG_GPIO_L_E2, BIT(16)), + PINCTRL_CONF_DESC(19, REG_GPIO_L_E2, BIT(17)), + PINCTRL_CONF_DESC(20, REG_GPIO_L_E2, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_E2, BIT(19)), + PINCTRL_CONF_DESC(22, REG_GPIO_L_E2, BIT(20)), + PINCTRL_CONF_DESC(23, REG_GPIO_L_E2, BIT(21)), + PINCTRL_CONF_DESC(24, REG_GPIO_L_E2, BIT(22)), + PINCTRL_CONF_DESC(25, REG_GPIO_L_E2, BIT(23)), + PINCTRL_CONF_DESC(26, REG_GPIO_L_E2, BIT(24)), + PINCTRL_CONF_DESC(27, REG_GPIO_L_E2, BIT(25)), + PINCTRL_CONF_DESC(28, REG_GPIO_L_E2, BIT(26)), + PINCTRL_CONF_DESC(29, REG_GPIO_L_E2, BIT(27)), + PINCTRL_CONF_DESC(30, REG_GPIO_L_E2, BIT(28)), + PINCTRL_CONF_DESC(31, REG_GPIO_L_E2, BIT(29)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_E2, BIT(30)), + PINCTRL_CONF_DESC(33, REG_GPIO_L_E2, BIT(31)), + PINCTRL_CONF_DESC(34, REG_GPIO_H_E2, BIT(0)), + PINCTRL_CONF_DESC(35, REG_GPIO_H_E2, BIT(1)), + PINCTRL_CONF_DESC(36, REG_GPIO_H_E2, BIT(2)), + PINCTRL_CONF_DESC(37, REG_GPIO_H_E2, BIT(3)), + PINCTRL_CONF_DESC(38, REG_GPIO_H_E2, BIT(4)), + PINCTRL_CONF_DESC(39, REG_GPIO_H_E2, BIT(5)), + PINCTRL_CONF_DESC(40, REG_GPIO_H_E2, BIT(6)), + PINCTRL_CONF_DESC(41, REG_I2C_SDA_E2, I2C_SCL_E2_MASK), + PINCTRL_CONF_DESC(42, REG_I2C_SDA_E2, I2C_SDA_E2_MASK), + PINCTRL_CONF_DESC(43, REG_I2C_SDA_E2, AN7583_I2C1_SCL_E2_MASK), + PINCTRL_CONF_DESC(44, REG_I2C_SDA_E2, AN7583_I2C1_SDA_E2_MASK), + PINCTRL_CONF_DESC(45, REG_I2C_SDA_E2, SPI_CLK_E2_MASK), + PINCTRL_CONF_DESC(46, REG_I2C_SDA_E2, SPI_CS0_E2_MASK), + PINCTRL_CONF_DESC(47, REG_I2C_SDA_E2, SPI_MOSI_E2_MASK), + PINCTRL_CONF_DESC(48, REG_I2C_SDA_E2, SPI_MISO_E2_MASK), + PINCTRL_CONF_DESC(49, REG_I2C_SDA_E2, UART1_TXD_E2_MASK), + PINCTRL_CONF_DESC(50, REG_I2C_SDA_E2, UART1_RXD_E2_MASK), + PINCTRL_CONF_DESC(51, REG_I2C_SDA_E2, PCIE0_RESET_E2_MASK), + PINCTRL_CONF_DESC(52, REG_I2C_SDA_E2, PCIE1_RESET_E2_MASK), + PINCTRL_CONF_DESC(53, REG_I2C_SDA_E2, AN7583_MDC_0_E2_MASK), + PINCTRL_CONF_DESC(54, REG_I2C_SDA_E2, AN7583_MDIO_0_E2_MASK), }; static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(0, REG_I2C_SDA_E4, UART1_TXD_E4_MASK), PINCTRL_CONF_DESC(1, REG_I2C_SDA_E4, UART1_RXD_E4_MASK), - PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), - PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(2, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(3, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), PINCTRL_CONF_DESC(4, REG_I2C_SDA_E4, SPI_CS0_E4_MASK), PINCTRL_CONF_DESC(5, REG_I2C_SDA_E4, SPI_CLK_E4_MASK), PINCTRL_CONF_DESC(6, REG_I2C_SDA_E4, SPI_MOSI_E4_MASK), @@ -2252,15 +2393,76 @@ static const struct airoha_pinctrl_conf en7581_pinctrl_drive_e4_conf[] = { PINCTRL_CONF_DESC(57, REG_GPIO_H_E4, BIT(12)), PINCTRL_CONF_DESC(58, REG_GPIO_H_E4, BIT(13)), PINCTRL_CONF_DESC(59, REG_GPIO_H_E4, BIT(14)), - PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), - PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), - PINCTRL_CONF_DESC(63, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), + PINCTRL_CONF_DESC(60, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), + PINCTRL_CONF_DESC(61, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), + PINCTRL_CONF_DESC(62, REG_I2C_SDA_E4, PCIE2_RESET_E4_MASK), +}; + +static const struct airoha_pinctrl_conf an7583_pinctrl_drive_e4_conf[] = { + PINCTRL_CONF_DESC(2, REG_GPIO_L_E4, BIT(0)), + PINCTRL_CONF_DESC(3, REG_GPIO_L_E4, BIT(1)), + PINCTRL_CONF_DESC(4, REG_GPIO_L_E4, BIT(2)), + PINCTRL_CONF_DESC(5, REG_GPIO_L_E4, BIT(3)), + PINCTRL_CONF_DESC(6, REG_GPIO_L_E4, BIT(4)), + PINCTRL_CONF_DESC(7, REG_GPIO_L_E4, BIT(5)), + PINCTRL_CONF_DESC(8, REG_GPIO_L_E4, BIT(6)), + PINCTRL_CONF_DESC(9, REG_GPIO_L_E4, BIT(7)), + PINCTRL_CONF_DESC(10, REG_GPIO_L_E4, BIT(8)), + PINCTRL_CONF_DESC(11, REG_GPIO_L_E4, BIT(9)), + PINCTRL_CONF_DESC(12, REG_GPIO_L_E4, BIT(10)), + PINCTRL_CONF_DESC(13, REG_GPIO_L_E4, BIT(11)), + PINCTRL_CONF_DESC(14, REG_GPIO_L_E4, BIT(12)), + PINCTRL_CONF_DESC(15, REG_GPIO_L_E4, BIT(13)), + PINCTRL_CONF_DESC(16, REG_GPIO_L_E4, BIT(14)), + PINCTRL_CONF_DESC(17, REG_GPIO_L_E4, BIT(15)), + PINCTRL_CONF_DESC(18, REG_GPIO_L_E4, BIT(16)), + PINCTRL_CONF_DESC(19, REG_GPIO_L_E4, BIT(17)), + PINCTRL_CONF_DESC(20, REG_GPIO_L_E4, BIT(18)), + PINCTRL_CONF_DESC(21, REG_GPIO_L_E4, BIT(19)), + PINCTRL_CONF_DESC(22, REG_GPIO_L_E4, BIT(20)), + PINCTRL_CONF_DESC(23, REG_GPIO_L_E4, BIT(21)), + PINCTRL_CONF_DESC(24, REG_GPIO_L_E4, BIT(22)), + PINCTRL_CONF_DESC(25, REG_GPIO_L_E4, BIT(23)), + PINCTRL_CONF_DESC(26, REG_GPIO_L_E4, BIT(24)), + PINCTRL_CONF_DESC(27, REG_GPIO_L_E4, BIT(25)), + PINCTRL_CONF_DESC(28, REG_GPIO_L_E4, BIT(26)), + PINCTRL_CONF_DESC(29, REG_GPIO_L_E4, BIT(27)), + PINCTRL_CONF_DESC(30, REG_GPIO_L_E4, BIT(28)), + PINCTRL_CONF_DESC(31, REG_GPIO_L_E4, BIT(29)), + PINCTRL_CONF_DESC(32, REG_GPIO_L_E4, BIT(30)), + PINCTRL_CONF_DESC(33, REG_GPIO_L_E4, BIT(31)), + PINCTRL_CONF_DESC(34, REG_GPIO_H_E4, BIT(0)), + PINCTRL_CONF_DESC(35, REG_GPIO_H_E4, BIT(1)), + PINCTRL_CONF_DESC(36, REG_GPIO_H_E4, BIT(2)), + PINCTRL_CONF_DESC(37, REG_GPIO_H_E4, BIT(3)), + PINCTRL_CONF_DESC(38, REG_GPIO_H_E4, BIT(4)), + PINCTRL_CONF_DESC(39, REG_GPIO_H_E4, BIT(5)), + PINCTRL_CONF_DESC(40, REG_GPIO_H_E4, BIT(6)), + PINCTRL_CONF_DESC(41, REG_I2C_SDA_E4, I2C_SCL_E4_MASK), + PINCTRL_CONF_DESC(42, REG_I2C_SDA_E4, I2C_SDA_E4_MASK), + PINCTRL_CONF_DESC(43, REG_I2C_SDA_E4, AN7583_I2C1_SCL_E4_MASK), + PINCTRL_CONF_DESC(44, REG_I2C_SDA_E4, AN7583_I2C1_SDA_E4_MASK), + PINCTRL_CONF_DESC(45, REG_I2C_SDA_E4, SPI_CLK_E4_MASK), + PINCTRL_CONF_DESC(46, REG_I2C_SDA_E4, SPI_CS0_E4_MASK), + PINCTRL_CONF_DESC(47, REG_I2C_SDA_E4, SPI_MOSI_E4_MASK), + PINCTRL_CONF_DESC(48, REG_I2C_SDA_E4, SPI_MISO_E4_MASK), + PINCTRL_CONF_DESC(49, REG_I2C_SDA_E4, UART1_TXD_E4_MASK), + PINCTRL_CONF_DESC(50, REG_I2C_SDA_E4, UART1_RXD_E4_MASK), + PINCTRL_CONF_DESC(51, REG_I2C_SDA_E4, PCIE0_RESET_E4_MASK), + PINCTRL_CONF_DESC(52, REG_I2C_SDA_E4, PCIE1_RESET_E4_MASK), + PINCTRL_CONF_DESC(53, REG_I2C_SDA_E4, AN7583_MDC_0_E4_MASK), + PINCTRL_CONF_DESC(54, REG_I2C_SDA_E4, AN7583_MDIO_0_E4_MASK), }; static const struct airoha_pinctrl_conf en7581_pinctrl_pcie_rst_od_conf[] = { - PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), - PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), - PINCTRL_CONF_DESC(63, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), + PINCTRL_CONF_DESC(60, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), + PINCTRL_CONF_DESC(61, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), + PINCTRL_CONF_DESC(62, REG_PCIE_RESET_OD, PCIE2_RESET_OD_MASK), +}; + +static const struct airoha_pinctrl_conf an7583_pinctrl_pcie_rst_od_conf[] = { + PINCTRL_CONF_DESC(51, REG_PCIE_RESET_OD, PCIE0_RESET_OD_MASK), + PINCTRL_CONF_DESC(52, REG_PCIE_RESET_OD, PCIE1_RESET_OD_MASK), }; static int airoha_convert_pin_to_reg_offset(struct pinctrl_dev *pctrl_dev, @@ -2302,6 +2504,22 @@ static int airoha_gpio_get(struct gpio_chip *chip, unsigned int gpio) return err ? err : !!(val & BIT(pin)); } +static int airoha_gpio_get_direction(struct gpio_chip *chip, unsigned int gpio) +{ + struct airoha_pinctrl *pinctrl = gpiochip_get_data(chip); + u32 val, mask; + u8 index; + int err; + + index = gpio / AIROHA_REG_GPIOCTRL_NUM_PIN; + err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); + if (err) + return err; + + mask = BIT(2 * (gpio % AIROHA_REG_GPIOCTRL_NUM_PIN)); + return val & mask ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int airoha_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, int value) { @@ -2317,18 +2535,18 @@ static int airoha_gpio_direction_output(struct gpio_chip *chip, /* irq callbacks */ static void airoha_irq_unmask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; u32 val = BIT(2 * offset); - gpiochip = irq_data_get_irq_chip_data(data); if (WARN_ON_ONCE(data->hwirq >= ARRAY_SIZE(gpiochip->irq_type))) return; - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + gpiochip_enable_irq(gc, irqd_to_hwirq(data)); switch (gpiochip->irq_type[data->hwirq]) { case IRQ_TYPE_LEVEL_LOW: val = val << 1; @@ -2354,34 +2572,64 @@ static void airoha_irq_unmask(struct irq_data *data) static void airoha_irq_mask(struct irq_data *data) { + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; u8 offset = data->hwirq % AIROHA_REG_GPIOCTRL_NUM_PIN; u8 index = data->hwirq / AIROHA_REG_GPIOCTRL_NUM_PIN; u32 mask = GENMASK(2 * offset + 1, 2 * offset); - struct airoha_pinctrl_gpiochip *gpiochip; - struct airoha_pinctrl *pinctrl; - gpiochip = irq_data_get_irq_chip_data(data); - pinctrl = container_of(gpiochip, struct airoha_pinctrl, gpiochip); + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; regmap_clear_bits(pinctrl->regmap, gpiochip->level[index], mask); regmap_clear_bits(pinctrl->regmap, gpiochip->edge[index], mask); + gpiochip_disable_irq(gc, irqd_to_hwirq(data)); +} + +static void airoha_irq_ack(struct irq_data *data) +{ + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; + u8 offset = data->hwirq % AIROHA_PIN_BANK_SIZE; + u8 index = data->hwirq / AIROHA_PIN_BANK_SIZE; + + if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) + return; + + regmap_write(pinctrl->regmap, gpiochip->status[index], BIT(offset)); } static int airoha_irq_type(struct irq_data *data, unsigned int type) { - struct airoha_pinctrl_gpiochip *gpiochip; + struct gpio_chip *gc = irq_data_get_irq_chip_data(data); + struct airoha_pinctrl *pinctrl = gpiochip_get_data(gc); + struct airoha_pinctrl_gpiochip *gpiochip = &pinctrl->gpiochip; - gpiochip = irq_data_get_irq_chip_data(data); if (data->hwirq >= ARRAY_SIZE(gpiochip->irq_type)) return -EINVAL; + if (type == IRQ_TYPE_NONE) { + gpiochip->irq_type[data->hwirq] = IRQ_TYPE_NONE; + irq_set_handler_locked(data, handle_bad_irq); + + return 0; + } + if (type == IRQ_TYPE_PROBE) { if (gpiochip->irq_type[data->hwirq]) return 0; type = IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING; } + gpiochip->irq_type[data->hwirq] = type & IRQ_TYPE_SENSE_MASK; + if (type & IRQ_TYPE_EDGE_BOTH) + irq_set_handler_locked(data, handle_edge_irq); + else + irq_set_handler_locked(data, handle_level_irq); + return 0; } @@ -2406,8 +2654,7 @@ static irqreturn_t airoha_irq_handler(int irq, void *data) for_each_set_bit(irq, &status, AIROHA_PIN_BANK_SIZE) { u32 offset = irq + i * AIROHA_PIN_BANK_SIZE; - generic_handle_irq(irq_find_mapping(girq->domain, - offset)); + generic_handle_domain_irq(girq->domain, offset); regmap_write(pinctrl->regmap, pinctrl->gpiochip.status[i], BIT(irq)); } @@ -2421,13 +2668,14 @@ static const struct irq_chip airoha_gpio_irq_chip = { .name = "airoha-gpio-irq", .irq_unmask = airoha_irq_unmask, .irq_mask = airoha_irq_mask, - .irq_mask_ack = airoha_irq_mask, + .irq_ack = airoha_irq_ack, .irq_set_type = airoha_irq_type, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_IMMUTABLE, + GPIOCHIP_IRQ_RESOURCE_HELPERS, }; static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, - struct platform_device *pdev) + struct platform_device *pdev) { struct airoha_pinctrl_gpiochip *chip = &pinctrl->gpiochip; struct gpio_chip *gc = &chip->chip; @@ -2435,26 +2683,20 @@ static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, struct device *dev = &pdev->dev; int irq, err; - chip->data = gpio_data_regs; - chip->dir = gpio_dir_regs; - chip->out = gpio_out_regs; - chip->status = irq_status_regs; - chip->level = irq_level_regs; - chip->edge = irq_edge_regs; - gc->parent = dev; gc->label = dev_name(dev); gc->request = gpiochip_generic_request; gc->free = gpiochip_generic_free; gc->direction_input = pinctrl_gpio_direction_input; gc->direction_output = airoha_gpio_direction_output; + gc->get_direction = airoha_gpio_get_direction; gc->set = airoha_gpio_set; gc->get = airoha_gpio_get; gc->base = -1; gc->ngpio = AIROHA_NUM_PINS; girq->default_type = IRQ_TYPE_NONE; - girq->handler = handle_simple_irq; + girq->handler = handle_bad_irq; gpio_irq_chip_set_chip(girq, &airoha_gpio_irq_chip); irq = platform_get_irq(pdev, 0); @@ -2462,7 +2704,7 @@ static int airoha_pinctrl_add_gpiochip(struct airoha_pinctrl *pinctrl, return irq; err = devm_request_irq(dev, irq, airoha_irq_handler, IRQF_SHARED, - dev_name(dev), pinctrl); + dev_name(dev), pinctrl); if (err) { dev_err(dev, "error requesting irq %d: %d\n", irq, err); return err; @@ -2526,8 +2768,8 @@ static int airoha_pinmux_set_mux(struct pinctrl_dev *pctrl_dev, } static int airoha_pinmux_set_direction(struct pinctrl_dev *pctrl_dev, - struct pinctrl_gpio_range *range, - unsigned int p, bool input) + struct pinctrl_gpio_range *range, + unsigned int p, bool input) { struct airoha_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctrl_dev); u32 mask, index; @@ -2617,7 +2859,7 @@ static int airoha_pinctrl_set_conf(struct airoha_pinctrl *pinctrl, if (regmap_update_bits(pinctrl->chip_scu, reg->offset, reg->mask, - val << __ffs(reg->mask))) + val << __ffs(reg->mask))) return -EINVAL; return 0; @@ -2657,21 +2899,18 @@ static int airoha_pinctrl_set_conf(struct airoha_pinctrl *pinctrl, static int airoha_pinconf_get_direction(struct pinctrl_dev *pctrl_dev, u32 p) { struct airoha_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctrl_dev); - u32 val, mask; - int err, pin; - u8 index; + int ret, pin; pin = airoha_convert_pin_to_reg_offset(pctrl_dev, NULL, p); if (pin < 0) return pin; - index = pin / AIROHA_REG_GPIOCTRL_NUM_PIN; - err = regmap_read(pinctrl->regmap, pinctrl->gpiochip.dir[index], &val); - if (err) - return err; + ret = airoha_gpio_get_direction(&pinctrl->gpiochip.chip, pin); + if (ret < 0) + return ret; - mask = BIT(2 * (pin % AIROHA_REG_GPIOCTRL_NUM_PIN)); - return val & mask ? PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; + return ret == GPIO_LINE_DIRECTION_OUT ? + PIN_CONFIG_OUTPUT_ENABLE : PIN_CONFIG_INPUT_ENABLE; } static int airoha_pinconf_get(struct pinctrl_dev *pctrl_dev, @@ -2834,8 +3073,8 @@ static int airoha_pinconf_group_get(struct pinctrl_dev *pctrl_dev, for (i = 0; i < pinctrl->grps[group].npins; i++) { if (airoha_pinconf_get(pctrl_dev, - pinctrl->grps[group].pins[i], - config)) + pinctrl->grps[group].pins[i], + config)) return -ENOTSUPP; if (i && cur_config != *config) @@ -2917,6 +3156,17 @@ static int airoha_pinctrl_probe(struct platform_device *pdev) pinctrl->desc.pins = data->pins, pinctrl->desc.npins = data->num_pins, + /* + * some pinctrl operations (ex: get_direction) might use gpio registers + * before gpio chip abstraction will be completely initialized. + */ + pinctrl->gpiochip.data = gpio_data_regs; + pinctrl->gpiochip.dir = gpio_dir_regs; + pinctrl->gpiochip.out = gpio_out_regs; + pinctrl->gpiochip.status = irq_status_regs; + pinctrl->gpiochip.level = irq_level_regs; + pinctrl->gpiochip.edge = irq_edge_regs; + err = devm_pinctrl_register_and_init(dev, &pinctrl->desc, pinctrl, &pinctrl->ctrl); if (err) @@ -2994,8 +3244,40 @@ static const struct airoha_pinctrl_match_data en7581_pinctrl_match_data = { }, }; +static const struct airoha_pinctrl_match_data an7583_pinctrl_match_data = { + .pins = an7583_pinctrl_pins, + .num_pins = ARRAY_SIZE(an7583_pinctrl_pins), + .grps = an7583_pinctrl_groups, + .num_grps = ARRAY_SIZE(an7583_pinctrl_groups), + .funcs = an7583_pinctrl_funcs, + .num_funcs = ARRAY_SIZE(an7583_pinctrl_funcs), + .confs_info = { + [AIROHA_PINCTRL_CONFS_PULLUP] = { + .confs = an7583_pinctrl_pullup_conf, + .num_confs = ARRAY_SIZE(an7583_pinctrl_pullup_conf), + }, + [AIROHA_PINCTRL_CONFS_PULLDOWN] = { + .confs = an7583_pinctrl_pulldown_conf, + .num_confs = ARRAY_SIZE(an7583_pinctrl_pulldown_conf), + }, + [AIROHA_PINCTRL_CONFS_DRIVE_E2] = { + .confs = an7583_pinctrl_drive_e2_conf, + .num_confs = ARRAY_SIZE(an7583_pinctrl_drive_e2_conf), + }, + [AIROHA_PINCTRL_CONFS_DRIVE_E4] = { + .confs = an7583_pinctrl_drive_e4_conf, + .num_confs = ARRAY_SIZE(an7583_pinctrl_drive_e4_conf), + }, + [AIROHA_PINCTRL_CONFS_PCIE_RST_OD] = { + .confs = an7583_pinctrl_pcie_rst_od_conf, + .num_confs = ARRAY_SIZE(an7583_pinctrl_pcie_rst_od_conf), + }, + }, +}; + static const struct of_device_id airoha_pinctrl_of_match[] = { { .compatible = "airoha,en7581-pinctrl", .data = &en7581_pinctrl_match_data }, + { .compatible = "airoha,an7583-pinctrl", .data = &an7583_pinctrl_match_data }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, airoha_pinctrl_of_match); diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c index 3f518dce6d23f..dd2c8aa039385 100644 --- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c +++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c @@ -802,20 +802,27 @@ static const struct pinmux_ops mtk_pmx_ops = { .get_function_name = mtk_pmx_get_func_name, .get_function_groups = mtk_pmx_get_func_groups, .set_mux = mtk_pmx_set_mux, - .gpio_set_direction = mtk_pmx_gpio_set_direction, .gpio_request_enable = mtk_pmx_gpio_request_enable, }; +static int mtk_gpio_direction_input(struct gpio_chip *chip, unsigned offset) +{ + struct mtk_pinctrl *pctl = gpiochip_get_data(chip); + + return mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, offset, true); +} + static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned offset, int value) { + struct mtk_pinctrl *pctl = gpiochip_get_data(chip); int ret; ret = mtk_gpio_set(chip, offset, value); if (ret) return ret; - return pinctrl_gpio_direction_output(chip, offset); + return mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, offset, false); } static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned offset) @@ -895,7 +902,7 @@ static const struct gpio_chip mtk_gpio_chip = { .request = gpiochip_generic_request, .free = gpiochip_generic_free, .get_direction = mtk_gpio_get_direction, - .direction_input = pinctrl_gpio_direction_input, + .direction_input = mtk_gpio_direction_input, .direction_output = mtk_gpio_direction_output, .get = mtk_gpio_get, .set = mtk_gpio_set, diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c index 6bf37d8085fae..23f04b24fd65e 100644 --- a/drivers/pinctrl/mediatek/pinctrl-paris.c +++ b/drivers/pinctrl/mediatek/pinctrl-paris.c @@ -771,7 +771,6 @@ static const struct pinmux_ops mtk_pmxops = { .get_function_name = mtk_pmx_get_func_name, .get_function_groups = mtk_pmx_get_func_groups, .set_mux = mtk_pmx_set_mux, - .gpio_set_direction = mtk_pinmux_gpio_set_direction, .gpio_request_enable = mtk_pinmux_gpio_request_enable, }; @@ -886,19 +885,22 @@ static int mtk_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value) static int mtk_gpio_direction_input(struct gpio_chip *chip, unsigned int gpio) { - return pinctrl_gpio_direction_input(chip, gpio); + struct mtk_pinctrl *hw = gpiochip_get_data(chip); + + return mtk_pinmux_gpio_set_direction(hw->pctrl, NULL, gpio, true); } static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio, int value) { + struct mtk_pinctrl *hw = gpiochip_get_data(chip); int ret; ret = mtk_gpio_set(chip, gpio, value); if (ret) return ret; - return pinctrl_gpio_direction_output(chip, gpio); + return mtk_pinmux_gpio_set_direction(hw->pctrl, NULL, gpio, false); } static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned int offset) diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c index 2dac5c71eb008..d962f2e1d2acd 100644 --- a/drivers/pinctrl/pinctrl-amd.c +++ b/drivers/pinctrl/pinctrl-amd.c @@ -886,8 +886,7 @@ static void amd_gpio_irq_init(struct amd_gpio *gpio_dev) u32 pin_reg, mask; int i; - mask = BIT(WAKE_CNTRL_OFF_S0I3) | BIT(WAKE_CNTRL_OFF_S3) | - BIT(WAKE_CNTRL_OFF_S4); + mask = BIT(WAKE_CNTRL_OFF_S0I3) | BIT(WAKE_CNTRL_OFF_S3); for (i = 0; i < desc->npins; i++) { int pin = desc->pins[i].number; diff --git a/drivers/pinctrl/pinctrl-eic7700.c b/drivers/pinctrl/pinctrl-eic7700.c index ffcd0ec5c2dc6..1e72931feca56 100644 --- a/drivers/pinctrl/pinctrl-eic7700.c +++ b/drivers/pinctrl/pinctrl-eic7700.c @@ -423,6 +423,9 @@ static int eic7700_pin_config_set(struct pinctrl_dev *pctldev, unsigned int pin, else value &= ~EIC7700_ST; break; + /* All pins are persistent over suspend */ + case PIN_CONFIG_PERSIST_STATE: + return 0; default: return -EOPNOTSUPP; } diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index f1cba3d2367a3..834229831de99 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -3675,6 +3675,16 @@ static struct rockchip_pin_ctrl *rockchip_pinctrl_get_soc_data( pmu_offs = ctrl->pmu_mux_offset; drv_pmu_offs = ctrl->pmu_drv_offset; drv_grf_offs = ctrl->grf_drv_offset; + + /* + * This function mutates the static per-SoC data. Most of it is + * idempotent: recalculated iomux and drv offsets anchor at the + * values calculated by a previous run. The pin count is not, so + * reset it here; otherwise it accumulates when the probe runs + * again after a probe deferral, shifting every bank's pin_base. + */ + ctrl->nr_pins = 0; + bank = ctrl->pin_banks; for (i = 0; i < ctrl->nr_banks; ++i, ++bank) { int bank_pins = 0; diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8064.c b/drivers/pinctrl/qcom/pinctrl-ipq8064.c index 0a9e357e64c60..f35dbe07ab6fe 100644 --- a/drivers/pinctrl/qcom/pinctrl-ipq8064.c +++ b/drivers/pinctrl/qcom/pinctrl-ipq8064.c @@ -480,7 +480,7 @@ static const char * const ps_hold_groups[] = { }; static const struct pinfunction ipq8064_functions[] = { - IPQ_PIN_FUNCTION(gpio), + IPQ_GPIO_PIN_FUNCTION(gpio), IPQ_PIN_FUNCTION(mdio), IPQ_PIN_FUNCTION(ssbi), IPQ_PIN_FUNCTION(spmi), @@ -507,19 +507,19 @@ static const struct pinfunction ipq8064_functions[] = { IPQ_PIN_FUNCTION(usb2_hsic), IPQ_PIN_FUNCTION(rgmii2), IPQ_PIN_FUNCTION(sata), - IPQ_PIN_FUNCTION(pcie1_rst), + IPQ_GPIO_PIN_FUNCTION(pcie1_rst), IPQ_PIN_FUNCTION(pcie1_prsnt), IPQ_PIN_FUNCTION(pcie1_pwren_n), IPQ_PIN_FUNCTION(pcie1_pwren), IPQ_PIN_FUNCTION(pcie1_pwrflt), IPQ_PIN_FUNCTION(pcie1_clk_req), - IPQ_PIN_FUNCTION(pcie2_rst), + IPQ_GPIO_PIN_FUNCTION(pcie2_rst), IPQ_PIN_FUNCTION(pcie2_prsnt), IPQ_PIN_FUNCTION(pcie2_pwren_n), IPQ_PIN_FUNCTION(pcie2_pwren), IPQ_PIN_FUNCTION(pcie2_pwrflt), IPQ_PIN_FUNCTION(pcie2_clk_req), - IPQ_PIN_FUNCTION(pcie3_rst), + IPQ_GPIO_PIN_FUNCTION(pcie3_rst), IPQ_PIN_FUNCTION(pcie3_prsnt), IPQ_PIN_FUNCTION(pcie3_pwren_n), IPQ_PIN_FUNCTION(pcie3_pwren), diff --git a/drivers/pinctrl/qcom/pinctrl-msm.c b/drivers/pinctrl/qcom/pinctrl-msm.c index e99871b90ab9c..424b14bc29e96 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.c +++ b/drivers/pinctrl/qcom/pinctrl-msm.c @@ -1226,12 +1226,12 @@ static int msm_gpio_irq_reqres(struct irq_data *d) /* * If the wakeup_enable bit is present and marked as available for the * requested GPIO, it should be enabled when the GPIO is marked as - * wake irq in order to allow the interrupt event to be transfered to - * the PDC HW. + * wake irq in order to allow the interrupt event to be transferred to + * the PDC/MPM HW. * While the name implies only the wakeup event, it's also required for * the interrupt event. */ - if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) { + if (g->intr_wakeup_present_bit) { u32 intr_cfg; raw_spin_lock_irqsave(&pctrl->lock, flags); @@ -1259,7 +1259,7 @@ static void msm_gpio_irq_relres(struct irq_data *d) unsigned long flags; /* Disable the wakeup_enable bit if it has been set in msm_gpio_irq_reqres() */ - if (test_bit(d->hwirq, pctrl->skip_wake_irqs) && g->intr_wakeup_present_bit) { + if (g->intr_wakeup_present_bit) { u32 intr_cfg; raw_spin_lock_irqsave(&pctrl->lock, flags); diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h index 120217012a9f6..33d8b17f2e559 100644 --- a/drivers/pinctrl/qcom/pinctrl-msm.h +++ b/drivers/pinctrl/qcom/pinctrl-msm.h @@ -24,6 +24,11 @@ struct pinctrl_pin_desc; fname##_groups, \ ARRAY_SIZE(fname##_groups)) +#define IPQ_GPIO_PIN_FUNCTION(fname) \ + [IPQ_MUX_##fname] = PINCTRL_GPIO_PINFUNCTION(#fname, \ + fname##_groups, \ + ARRAY_SIZE(fname##_groups)) + #define MSM_PIN_FUNCTION(fname) \ [msm_mux_##fname] = PINCTRL_PINFUNCTION(#fname, \ fname##_groups, \ diff --git a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c index cf8297e8b8f8c..d6e31b00e1c5e 100644 --- a/drivers/pinctrl/qcom/pinctrl-sc8280xp.c +++ b/drivers/pinctrl/qcom/pinctrl-sc8280xp.c @@ -1884,16 +1884,17 @@ static const struct msm_gpio_wakeirq_map sc8280xp_pdc_map[] = { { 126, 200 }, { 127, 225 }, { 128, 262 }, { 129, 201 }, { 130, 209 }, { 131, 173 }, { 132, 202 }, { 136, 210 }, { 138, 171 }, { 139, 226 }, { 140, 227 }, { 142, 228 }, - { 144, 229 }, { 145, 230 }, { 146, 231 }, { 148, 232 }, - { 149, 233 }, { 150, 234 }, { 152, 235 }, { 154, 212 }, - { 157, 213 }, { 161, 219 }, { 170, 236 }, { 171, 221 }, - { 174, 222 }, { 175, 237 }, { 176, 223 }, { 177, 170 }, - { 180, 238 }, { 181, 239 }, { 182, 240 }, { 183, 241 }, - { 184, 242 }, { 185, 243 }, { 190, 178 }, { 193, 184 }, - { 196, 185 }, { 198, 186 }, { 200, 174 }, { 201, 175 }, - { 205, 176 }, { 206, 177 }, { 208, 187 }, { 210, 198 }, - { 211, 199 }, { 212, 204 }, { 215, 205 }, { 220, 188 }, - { 221, 194 }, { 223, 195 }, { 225, 196 }, { 227, 197 }, + { 143, 261 }, { 144, 229 }, { 145, 230 }, { 146, 231 }, + { 148, 232 }, { 149, 233 }, { 150, 234 }, { 151, 264 }, + { 152, 235 }, { 154, 212 }, { 157, 213 }, { 161, 219 }, + { 170, 236 }, { 171, 221 }, { 174, 222 }, { 175, 237 }, + { 176, 223 }, { 177, 170 }, { 180, 238 }, { 181, 239 }, + { 182, 240 }, { 183, 241 }, { 184, 242 }, { 185, 243 }, + { 190, 178 }, { 193, 184 }, { 196, 185 }, { 198, 186 }, + { 200, 174 }, { 201, 175 }, { 205, 176 }, { 206, 177 }, + { 208, 187 }, { 210, 198 }, { 211, 199 }, { 212, 204 }, + { 215, 205 }, { 220, 188 }, { 221, 194 }, { 223, 195 }, + { 225, 196 }, { 227, 197 }, }; static struct msm_pinctrl_soc_data sc8280xp_pinctrl = { diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c index aaa783c79e6dd..b4d7e80dd6468 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c +++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c @@ -322,6 +322,7 @@ struct rzg2l_pinctrl_pin_settings { * @pupd: PUPD registers cache * @ien: IEN registers cache * @smt: SMT registers cache + * @sr: SR registers cache * @sd_ch: SD_CH registers cache * @eth_poc: ET_POC registers cache * @oen: Output Enable register cache @@ -336,6 +337,7 @@ struct rzg2l_pinctrl_reg_cache { u32 *ien[2]; u32 *pupd[2]; u32 *smt[2]; + u32 *sr[2]; u8 sd_ch[2]; u8 eth_poc[2]; u8 oen; @@ -1097,7 +1099,7 @@ static int rzg2l_read_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin) int bit; if (!pctrl->data->pin_to_oen_bit) - return -EOPNOTSUPP; + return -ENOTSUPP; bit = pctrl->data->pin_to_oen_bit(pctrl, _pin); if (bit < 0) @@ -1115,7 +1117,7 @@ static int rzg2l_write_oen(struct rzg2l_pinctrl *pctrl, unsigned int _pin, u8 oe int bit; if (!pctrl->data->pin_to_oen_bit) - return -EOPNOTSUPP; + return -ENOTSUPP; bit = pctrl->data->pin_to_oen_bit(pctrl, _pin); if (bit < 0) @@ -1552,7 +1554,7 @@ static int rzg2l_pinctrl_pinconf_set(struct pinctrl_dev *pctldev, break; default: - return -EOPNOTSUPP; + return -ENOTSUPP; } } @@ -1634,7 +1636,7 @@ static int rzg2l_pinctrl_pinconf_group_get(struct pinctrl_dev *pctldev, /* Check config matching between to pin */ if (i && prev_config != *config) - return -EOPNOTSUPP; + return -ENOTSUPP; prev_config = *config; } @@ -2746,6 +2748,11 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl) if (!cache->smt[i]) return -ENOMEM; + cache->sr[i] = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->sr[i]), + GFP_KERNEL); + if (!cache->sr[i]) + return -ENOMEM; + /* Allocate dedicated cache. */ dedicated_cache->iolh[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins, sizeof(*dedicated_cache->iolh[i]), @@ -2758,6 +2765,12 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl) GFP_KERNEL); if (!dedicated_cache->ien[i]) return -ENOMEM; + + dedicated_cache->sr[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins, + sizeof(*dedicated_cache->sr[i]), + GFP_KERNEL); + if (!dedicated_cache->sr[i]) + return -ENOMEM; } pctrl->cache = cache; @@ -2989,7 +3002,7 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen struct rzg2l_pinctrl_reg_cache *cache = pctrl->cache; for (u32 port = 0; port < nports; port++) { - bool has_iolh, has_ien, has_pupd, has_smt; + bool has_iolh, has_ien, has_pupd, has_smt, has_sr; u32 off, caps; u8 pincnt; u64 cfg; @@ -3010,6 +3023,7 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen has_ien = !!(caps & PIN_CFG_IEN); has_pupd = !!(caps & PIN_CFG_PUPD); has_smt = !!(caps & PIN_CFG_SMT); + has_sr = !!(caps & PIN_CFG_SR); if (suspend) RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PFC(off), cache->pfc[port]); @@ -3061,6 +3075,15 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen cache->smt[1][port]); } } + + if (has_sr) { + RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off), + cache->sr[0][port]); + if (pincnt >= 4) { + RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off) + 4, + cache->sr[1][port]); + } + } } } @@ -3075,7 +3098,7 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b * port offset are close together. */ for (i = 0, caps = 0; i < pctrl->data->n_dedicated_pins; i++) { - bool has_iolh, has_ien; + bool has_iolh, has_ien, has_sr; u32 off, next_off = 0; u64 cfg, next_cfg; u8 pincnt; @@ -3094,8 +3117,10 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b } /* And apply them in a single shot. */ - has_iolh = !!(caps & (PIN_CFG_IOLH_A | PIN_CFG_IOLH_B | PIN_CFG_IOLH_C)); + has_iolh = !!(caps & (PIN_CFG_IOLH_A | PIN_CFG_IOLH_B | + PIN_CFG_IOLH_C | PIN_CFG_IOLH_RZV2H)); has_ien = !!(caps & PIN_CFG_IEN); + has_sr = !!(caps & PIN_CFG_SR); pincnt = hweight8(FIELD_GET(RZG2L_SINGLE_PIN_BITS_MASK, cfg)); if (has_iolh) { @@ -3106,7 +3131,10 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + IEN(off), cache->ien[0][i]); } - + if (has_sr) { + RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off), + cache->sr[0][i]); + } if (pincnt >= 4) { if (has_iolh) { RZG2L_PCTRL_REG_ACCESS32(suspend, @@ -3118,6 +3146,11 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b pctrl->base + IEN(off) + 4, cache->ien[1][i]); } + if (has_sr) { + RZG2L_PCTRL_REG_ACCESS32(suspend, + pctrl->base + SR(off) + 4, + cache->sr[1][i]); + } } caps = 0; } diff --git a/drivers/pinctrl/renesas/pinctrl-rzv2m.c b/drivers/pinctrl/renesas/pinctrl-rzv2m.c index dce68f93d2d57..e08f24d5a89f5 100644 --- a/drivers/pinctrl/renesas/pinctrl-rzv2m.c +++ b/drivers/pinctrl/renesas/pinctrl-rzv2m.c @@ -661,7 +661,7 @@ static int rzv2m_pinctrl_pinconf_set(struct pinctrl_dev *pctldev, } default: - return -EOPNOTSUPP; + return -ENOTSUPP; } } @@ -710,7 +710,7 @@ static int rzv2m_pinctrl_pinconf_group_get(struct pinctrl_dev *pctldev, /* Check config matches previous pins */ if (i && prev_config != *config) - return -EOPNOTSUPP; + return -ENOTSUPP; prev_config = *config; } diff --git a/drivers/pinctrl/spacemit/pinctrl-k1.c b/drivers/pinctrl/spacemit/pinctrl-k1.c index 8d797bf24b778..813f86865dcd6 100644 --- a/drivers/pinctrl/spacemit/pinctrl-k1.c +++ b/drivers/pinctrl/spacemit/pinctrl-k1.c @@ -475,13 +475,14 @@ static int spacemit_pinconf_get(struct pinctrl_dev *pctldev, unsigned int pin, unsigned long *config) { struct spacemit_pinctrl *pctrl = pinctrl_dev_get_drvdata(pctldev); + const struct spacemit_pin *spin = spacemit_get_pin(pctrl, pin); int param = pinconf_to_config_param(*config); u32 value, arg = 0; - if (!pin) + if (!spin) return -EINVAL; - value = readl(spacemit_pin_to_reg(pctrl, pin)); + value = readl(spacemit_pin_to_reg(pctrl, spin->pin)); switch (param) { case PIN_CONFIG_SLEW_RATE: @@ -654,6 +655,11 @@ static int spacemit_pinconf_group_set(struct pinctrl_dev *pctldev, if (spacemit_pinconf_generate_config(spin, configs, num_configs, &value)) return -EINVAL; + for (i = 0; i < group->grp.npins; i++) { + if (!spacemit_get_pin(pctrl, group->grp.pins[i])) + return -EINVAL; + } + for (i = 0; i < group->grp.npins; i++) spacemit_pin_set_config(pctrl, group->grp.pins[i], value); diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c index d10f9561990c8..e482b23b9f635 100644 --- a/drivers/platform/chrome/cros_ec_debugfs.c +++ b/drivers/platform/chrome/cros_ec_debugfs.c @@ -513,7 +513,7 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) ret = blocking_notifier_chain_register(&ec->ec_dev->panic_notifier, &debug_info->notifier_panic); if (ret) - goto remove_debugfs; + goto cleanup_console_log; ec->debug_info = debug_info; @@ -521,6 +521,8 @@ static int cros_ec_debugfs_probe(struct platform_device *pd) return 0; +cleanup_console_log: + cros_ec_cleanup_console_log(debug_info); remove_debugfs: debugfs_remove_recursive(debug_info->dir); return ret; @@ -530,6 +532,8 @@ static void cros_ec_debugfs_remove(struct platform_device *pd) { struct cros_ec_dev *ec = dev_get_drvdata(pd->dev.parent); + blocking_notifier_chain_unregister(&ec->ec_dev->panic_notifier, + &ec->debug_info->notifier_panic); debugfs_remove_recursive(ec->debug_info->dir); cros_ec_cleanup_console_log(ec->debug_info); } diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c index 9bad8f72680ea..f938c3fc84e4f 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub.c +++ b/drivers/platform/chrome/cros_ec_sensorhub.c @@ -122,8 +122,12 @@ static int cros_ec_sensorhub_register(struct device *dev, sensor_type[sensorhub->resp->info.type]++; } - if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2) + if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2) { ec->has_kb_wake_angle = true; + if (ec->group && sysfs_update_group(&ec->class_dev.kobj, + ec->group)) + dev_warn(dev, "Unable to update sysfs"); + } if (cros_ec_check_features(ec, EC_FEATURE_REFINED_TABLET_MODE_HYSTERESIS)) { diff --git a/drivers/platform/chrome/cros_ec_sensorhub_ring.c b/drivers/platform/chrome/cros_ec_sensorhub_ring.c index 1205219515d67..b5970e8620bf0 100644 --- a/drivers/platform/chrome/cros_ec_sensorhub_ring.c +++ b/drivers/platform/chrome/cros_ec_sensorhub_ring.c @@ -464,6 +464,21 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, fifo_timestamp, *current_timestamp, now); + + /* + * A standalone timestamp event typically has a sensor_num of + * 0xff. Return early here to prevent it from hitting the + * bounds check below and spamming the logs. + */ + return false; + } + + /* Skip event if sensor_num from EC is out of bounds. */ + if (in->sensor_num >= sensorhub->sensor_num) { + dev_warn_ratelimited(sensorhub->dev, + "Invalid sensor number %u from EC\n", + in->sensor_num); + return false; } if (in->flags & MOTIONSENSE_SENSOR_FLAG_ODR) { @@ -491,10 +506,6 @@ cros_ec_sensor_ring_process_event(struct cros_ec_sensorhub *sensorhub, return true; } - if (in->flags & MOTIONSENSE_SENSOR_FLAG_TIMESTAMP) - /* If we just have a timestamp, skip this entry. */ - return false; - /* Regular sample */ out->sensor_id = in->sensor_num; trace_cros_ec_sensorhub_data(in->sensor_num, @@ -814,8 +825,15 @@ static void cros_ec_sensorhub_ring_handler(struct cros_ec_sensorhub *sensorhub) sensorhub->msg->outsize = 1; sensorhub->msg->insize = fifo_info_length; - if (cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg) < 0) + ret = cros_ec_cmd_xfer_status(ec->ec_dev, sensorhub->msg); + if (ret < 0) goto error; + if (ret != fifo_info_length) { + dev_warn_ratelimited(sensorhub->dev, + "Mismatch read length: size %d - expected %d\n", + ret, fifo_info_length); + goto error; + } memcpy(fifo_info, &sensorhub->resp->fifo_info, fifo_info_length); diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c index f22e9523da3e8..9d3767ab15480 100644 --- a/drivers/platform/chrome/cros_ec_sysfs.c +++ b/drivers/platform/chrome/cros_ec_sysfs.c @@ -405,7 +405,8 @@ static int cros_ec_sysfs_probe(struct platform_device *pd) struct device *dev = &pd->dev; int ret; - ret = sysfs_create_group(&ec_dev->class_dev.kobj, &cros_ec_attr_group); + ec_dev->group = &cros_ec_attr_group; + ret = sysfs_create_group(&ec_dev->class_dev.kobj, ec_dev->group); if (ret < 0) dev_err(dev, "failed to create attributes. err=%d\n", ret); @@ -416,7 +417,7 @@ static void cros_ec_sysfs_remove(struct platform_device *pd) { struct cros_ec_dev *ec_dev = dev_get_drvdata(pd->dev.parent); - sysfs_remove_group(&ec_dev->class_dev.kobj, &cros_ec_attr_group); + sysfs_remove_group(&ec_dev->class_dev.kobj, ec_dev->group); } static const struct platform_device_id cros_ec_sysfs_id[] = { diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c index b712bcff6fb26..03ca776a837fb 100644 --- a/drivers/platform/chrome/cros_ec_typec.c +++ b/drivers/platform/chrome/cros_ec_typec.c @@ -1118,6 +1118,12 @@ static void cros_typec_register_partner_pdos(struct cros_typec_data *typec, if (!resp->source_cap_count && !resp->sink_cap_count) return; + if (resp->source_cap_count > PDO_MAX_OBJECTS || + resp->sink_cap_count > PDO_MAX_OBJECTS) { + dev_warn(typec->dev, "Invalid PDO count from EC, port: %d\n", port_num); + return; + } + port->partner_pd = typec_partner_usb_power_delivery_register(port->partner, &desc); if (IS_ERR(port->partner_pd)) { dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num); diff --git a/drivers/platform/loongarch/loongson-laptop.c b/drivers/platform/loongarch/loongson-laptop.c index 61b18ac206c9e..dc8142f4fa954 100644 --- a/drivers/platform/loongarch/loongson-laptop.c +++ b/drivers/platform/loongarch/loongson-laptop.c @@ -189,6 +189,7 @@ static int __init setup_acpi_notify(struct generic_sub_driver *sub_driver) static int loongson_hotkey_suspend(struct device *dev) { + bl_powered = false; return 0; } diff --git a/drivers/platform/mellanox/mlxbf-bootctl.c b/drivers/platform/mellanox/mlxbf-bootctl.c index f67c7f56ab2b7..9ddc7fa1a9732 100644 --- a/drivers/platform/mellanox/mlxbf-bootctl.c +++ b/drivers/platform/mellanox/mlxbf-bootctl.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include diff --git a/drivers/platform/mellanox/mlxbf-pmc.c b/drivers/platform/mellanox/mlxbf-pmc.c index 5ec1ad4716967..2ad9e2b0493c4 100644 --- a/drivers/platform/mellanox/mlxbf-pmc.c +++ b/drivers/platform/mellanox/mlxbf-pmc.c @@ -2262,13 +2262,19 @@ static int mlxbf_pmc_map_counters(struct device *dev) static int mlxbf_pmc_probe(struct platform_device *pdev) { - struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev); - const char *hid = acpi_device_hid(acpi_dev); struct device *dev = &pdev->dev; + struct acpi_device *acpi_dev; struct arm_smccc_res res; + const char *hid; guid_t guid; int ret; + acpi_dev = ACPI_COMPANION(&pdev->dev); + if (!acpi_dev) + return -ENODEV; + + hid = acpi_device_hid(acpi_dev); + /* Ensure we have the UUID we expect for this service. */ arm_smccc_smc(MLXBF_PMC_SIP_SVC_UID, 0, 0, 0, 0, 0, 0, 0, &res); guid_parse(mlxbf_pmc_svc_uuid_str, &guid); diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c index 3b30cfe3466b8..3bad4e689f5ce 100644 --- a/drivers/platform/surface/surface_acpi_notify.c +++ b/drivers/platform/surface/surface_acpi_notify.c @@ -777,12 +777,16 @@ static int san_consumer_links_setup(struct platform_device *pdev) static int san_probe(struct platform_device *pdev) { - struct acpi_device *san = ACPI_COMPANION(&pdev->dev); struct ssam_controller *ctrl; + struct acpi_device *san; struct san_data *data; acpi_status astatus; int status; + san = ACPI_COMPANION(&pdev->dev); + if (!san) + return -ENODEV; + ctrl = ssam_client_bind(&pdev->dev); if (IS_ERR(ctrl)) return PTR_ERR(ctrl) == -ENODEV ? -EPROBE_DEFER : PTR_ERR(ctrl); diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c index d848afc91f87d..55b1980313542 100644 --- a/drivers/platform/x86/acer-wmi.c +++ b/drivers/platform/x86/acer-wmi.c @@ -1530,7 +1530,9 @@ static int WMI_gaming_execute_u32_u64(u32 method_id, u32 in, u64 *out) return -EIO; obj = result.pointer; - if (obj && out) { + if (!obj && out) { + ret = -ENOMSG; + } else if (obj && out) { switch (obj->type) { case ACPI_TYPE_INTEGER: *out = obj->integer.value; diff --git a/drivers/platform/x86/amd/hsmp/hwmon.c b/drivers/platform/x86/amd/hsmp/hwmon.c index 0cc9a742497fc..c8314eee06f4f 100644 --- a/drivers/platform/x86/amd/hsmp/hwmon.c +++ b/drivers/platform/x86/amd/hsmp/hwmon.c @@ -31,6 +31,9 @@ static int hsmp_hwmon_write(struct device *dev, enum hwmon_sensor_types type, if (attr != hwmon_power_cap) return -EOPNOTSUPP; + if (val < 0) + return -EINVAL; + msg.num_args = 1; msg.args[0] = val / MICROWATT_PER_MILLIWATT; msg.msg_id = HSMP_SET_SOCKET_POWER_LIMIT; diff --git a/drivers/platform/x86/amd/pmc/mp1_stb.c b/drivers/platform/x86/amd/pmc/mp1_stb.c index 3b9b9f30faa3e..c2f4e6caf58b6 100644 --- a/drivers/platform/x86/amd/pmc/mp1_stb.c +++ b/drivers/platform/x86/amd/pmc/mp1_stb.c @@ -157,7 +157,7 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) struct amd_pmc_dev *dev = filp->f_inode->i_private; u32 fsize, num_samples, val, stb_rdptr_offset = 0; struct amd_stb_v2_data *stb_data_arr; - int ret; + int ret = 0; /* Write dummy postcode while reading the STB buffer */ ret = amd_stb_write(dev, AMD_PMC_STB_DUMMY_PC); @@ -176,22 +176,24 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) * the enhanced dram size. Note that we land here only for the * platforms that support enhanced dram size reporting. */ - if (dump_custom_stb) - return amd_stb_handle_efr(filp); + if (dump_custom_stb) { + ret = amd_stb_handle_efr(filp); + goto out; + } /* Get the num_samples to calculate the last push location */ ret = amd_pmc_send_cmd(dev, S2D_NUM_SAMPLES, &num_samples, dev->stb_arg.s2d_msg_id, true); - /* Clear msg_port for other SMU operation */ - dev->msg_port = MSG_PORT_PMC; if (ret) { dev_err(dev->dev, "error: S2D_NUM_SAMPLES not supported : %d\n", ret); - return ret; + goto out; } fsize = min(num_samples, S2D_TELEMETRY_BYTES_MAX); stb_data_arr = kmalloc(struct_size(stb_data_arr, data, fsize), GFP_KERNEL); - if (!stb_data_arr) - return -ENOMEM; + if (!stb_data_arr) { + ret = -ENOMEM; + goto out; + } stb_data_arr->size = fsize; @@ -214,7 +216,10 @@ static int amd_stb_debugfs_open_v2(struct inode *inode, struct file *filp) filp->private_data = stb_data_arr; - return 0; +out: + /* Restore the default message port for subsequent SMU operations */ + dev->msg_port = MSG_PORT_PMC; + return ret; } static ssize_t amd_stb_debugfs_read_v2(struct file *filp, char __user *buf, size_t size, @@ -289,7 +294,7 @@ int amd_stb_s2d_init(struct amd_pmc_dev *dev) u32 phys_addr_low, phys_addr_hi; u64 stb_phys_addr; u32 size = 0; - int ret; + int ret = 0; if (!enable_stb) return 0; @@ -306,27 +311,42 @@ int amd_stb_s2d_init(struct amd_pmc_dev *dev) /* Spill to DRAM feature uses separate SMU message port */ dev->msg_port = MSG_PORT_S2D; - amd_pmc_send_cmd(dev, S2D_TELEMETRY_SIZE, &size, dev->stb_arg.s2d_msg_id, true); - if (size != S2D_TELEMETRY_BYTES_MAX) - return -EIO; + ret = amd_pmc_send_cmd(dev, S2D_TELEMETRY_SIZE, &size, dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; + if (size != S2D_TELEMETRY_BYTES_MAX) { + ret = -EIO; + goto out; + } - /* Get DRAM size */ - ret = amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->stb_arg.s2d_msg_id, true); - if (ret || !dev->dram_size) + /* Get DRAM size; fall back to the default if the query fails */ + if (amd_pmc_send_cmd(dev, S2D_DRAM_SIZE, &dev->dram_size, dev->stb_arg.s2d_msg_id, true) || + !dev->dram_size) dev->dram_size = S2D_TELEMETRY_DRAMBYTES_MAX; /* Get STB DRAM address */ - amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_LOW, &phys_addr_low, dev->stb_arg.s2d_msg_id, true); - amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_HIGH, &phys_addr_hi, dev->stb_arg.s2d_msg_id, true); + ret = amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_LOW, &phys_addr_low, + dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; + ret = amd_pmc_send_cmd(dev, S2D_PHYS_ADDR_HIGH, &phys_addr_hi, + dev->stb_arg.s2d_msg_id, true); + if (ret) + goto out; stb_phys_addr = ((u64)phys_addr_hi << 32 | phys_addr_low); - - /* Clear msg_port for other SMU operation */ - dev->msg_port = MSG_PORT_PMC; + if (!stb_phys_addr) { + dev_err(dev->dev, "S2D phys addr query returned invalid address\n"); + ret = -ENXIO; + goto out; + } dev->stb_virt_addr = devm_ioremap(dev->dev, stb_phys_addr, dev->dram_size); if (!dev->stb_virt_addr) - return -ENOMEM; + ret = -ENOMEM; - return 0; +out: + /* Restore the default message port for subsequent SMU operations */ + dev->msg_port = MSG_PORT_PMC; + return ret; } diff --git a/drivers/platform/x86/amd/pmc/pmc.c b/drivers/platform/x86/amd/pmc/pmc.c index 2cd77a44c7e7c..000f876e27ebb 100644 --- a/drivers/platform/x86/amd/pmc/pmc.c +++ b/drivers/platform/x86/amd/pmc/pmc.c @@ -870,13 +870,17 @@ static int amd_pmc_probe(struct platform_device *pdev) amd_pmc_dbgfs_register(dev); err = amd_stb_s2d_init(dev); if (err) - goto err_pci_dev_put; + goto err_dbgfs_unregister; if (IS_ENABLED(CONFIG_AMD_MP2_STB)) amd_mp2_stb_init(dev); pm_report_max_hw_sleep(U64_MAX); return 0; +err_dbgfs_unregister: + amd_pmc_dbgfs_unregister(dev); + if (IS_ENABLED(CONFIG_SUSPEND)) + acpi_unregister_lps0_dev(&amd_pmc_s2idle_dev_ops); err_pci_dev_put: pci_dev_put(rdev); return err; diff --git a/drivers/platform/x86/dell/dell-laptop.c b/drivers/platform/x86/dell/dell-laptop.c index 053f40572bf6e..89e85c7f7132a 100644 --- a/drivers/platform/x86/dell/dell-laptop.c +++ b/drivers/platform/x86/dell/dell-laptop.c @@ -222,6 +222,15 @@ static const struct dmi_system_id dell_quirks[] __initconst = { }, .driver_data = &quirk_dell_vostro_v130, }, + { + .callback = dmi_matched, + .ident = "Dell Inspiron N5110", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron N5110"), + }, + .driver_data = &quirk_dell_vostro_v130, + }, { .callback = dmi_matched, .ident = "Dell Vostro 3360", diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c index 907f1da01c8db..ba6c88e2a5ec0 100644 --- a/drivers/platform/x86/dell/dell-wmi-base.c +++ b/drivers/platform/x86/dell/dell-wmi-base.c @@ -851,9 +851,22 @@ static int __init dell_wmi_init(void) err = dell_privacy_register_driver(); if (err) - return err; + goto out_smbios; - return wmi_driver_register(&dell_wmi_driver); + err = wmi_driver_register(&dell_wmi_driver); + if (err) + goto out_privacy; + + return 0; + +out_privacy: + dell_privacy_unregister_driver(); + +out_smbios: + if (wmi_requires_smbios_request) + dell_wmi_events_set_enabled(false); + + return err; } late_initcall(dell_wmi_init); diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c index 4b65e1655d42a..7c2b6fae96c52 100644 --- a/drivers/platform/x86/dell/dell-wmi-privacy.c +++ b/drivers/platform/x86/dell/dell-wmi-privacy.c @@ -69,11 +69,11 @@ bool dell_privacy_has_mic_mute(void) { struct privacy_wmi_data *priv; - mutex_lock(&list_mutex); + guard(mutex)(&list_mutex); + priv = list_first_entry_or_null(&wmi_list, struct privacy_wmi_data, list); - mutex_unlock(&list_mutex); return priv && (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO)); } diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c index c2dd2de6bc203..fea97d6c3bf02 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c +++ b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c @@ -84,7 +84,6 @@ int set_attribute(const char *a_name, const char *a_value) if (ret < 0) goto out; - print_hex_dump_bytes("set attribute data: ", DUMP_PREFIX_NONE, buffer, buffer_size); ret = call_biosattributes_interface(wmi_priv.bios_attr_wdev, buffer, buffer_size, SETATTRIBUTE_METHOD_ID); diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h index 817ee7ba07ca0..b0810a02e6dac 100644 --- a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h +++ b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h @@ -107,7 +107,7 @@ enum { static int get_##type##_instance_id(struct kobject *kobj) \ { \ int i; \ - for (i = 0; i <= wmi_priv.type##_instances_count; i++) { \ + for (i = 0; i < wmi_priv.type##_instances_count; i++) { \ if (!(strcmp(kobj->name, wmi_priv.type##_data[i].attribute_name)))\ return i; \ } \ diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c index 51e8977d3eb4a..274bdecce6852 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.c @@ -85,7 +85,7 @@ int hp_get_string_from_buffer(u8 **buffer, u32 *buffer_size, char *dst, u32 dst_ * bytes. */ conv_dst_size = size; - if (size > dst_size) + if (size >= dst_size) conv_dst_size = dst_size - 1; /* @@ -661,12 +661,17 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type, int ret = 0; /* Take action appropriate to each ACPI TYPE */ - if (obj->package.count < min_elements) { - pr_err("ACPI-package does not have enough elements: %d < %d\n", - obj->package.count, min_elements); + if (obj->package.count < COMMON_ELEM_CNT) { + pr_err("ACPI-package is missing common elements: %d < %d\n", + obj->package.count, COMMON_ELEM_CNT); goto pack_attr_exit; } + if (obj->package.count < min_elements) { + pr_warn("ACPI-package has fewer elements than expected: %d < %d, parsing available elements\n", + obj->package.count, min_elements); + } + elements = obj->package.elements; /* sanity checking */ @@ -731,26 +736,31 @@ static int hp_init_bios_package_attribute(enum hp_wmi_data_type attr_type, switch (attr_type) { case HPWMI_STRING_TYPE: ret = hp_populate_string_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_INTEGER_TYPE: ret = hp_populate_integer_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_ENUMERATION_TYPE: ret = hp_populate_enumeration_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_ORDERED_LIST_TYPE: ret = hp_populate_ordered_list_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; case HPWMI_PASSWORD_TYPE: ret = hp_populate_password_package_data(elements, + obj->package.count, instance_id, attr_name_kobj); break; diff --git a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h index f1eec0e4ba075..ac57d6eab4c35 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h +++ b/drivers/platform/x86/hp/hp-bioscfg/bioscfg.h @@ -279,6 +279,9 @@ enum hp_wmi_data_elements { PSWD_ENCODINGS = 13, PSWD_IS_SET = 14, PSWD_ELEM_CNT = 15, + + /* Minimum elements shared by all attribute types (NAME..SECURITY_LEVEL) */ + COMMON_ELEM_CNT = SECURITY_LEVEL + 1, }; #define GET_INSTANCE_ID(type) \ @@ -401,6 +404,7 @@ int hp_populate_string_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_string_data(void); void hp_exit_string_attributes(void); int hp_populate_string_package_data(union acpi_object *str_obj, + int str_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -411,6 +415,7 @@ int hp_populate_integer_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_integer_data(void); void hp_exit_integer_attributes(void); int hp_populate_integer_package_data(union acpi_object *integer_obj, + int integer_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -421,6 +426,7 @@ int hp_populate_enumeration_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int hp_alloc_enumeration_data(void); void hp_exit_enumeration_attributes(void); int hp_populate_enumeration_package_data(union acpi_object *enum_obj, + int enum_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -432,6 +438,7 @@ int hp_populate_ordered_list_buffer_data(u8 *buffer_ptr, int hp_alloc_ordered_list_data(void); void hp_exit_ordered_list_attributes(void); int hp_populate_ordered_list_package_data(union acpi_object *order_obj, + int order_obj_count, int instance_id, struct kobject *attr_name_kobj); @@ -440,6 +447,7 @@ int hp_populate_password_buffer_data(u8 *buffer_ptr, u32 *buffer_size, int instance_id, struct kobject *attr_name_kobj); int hp_populate_password_package_data(union acpi_object *password_obj, + int password_obj_count, int instance_id, struct kobject *attr_name_kobj); int hp_alloc_password_data(void); diff --git a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c index af4d1920d4880..446dd18d2cee4 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/enum-attributes.c @@ -163,10 +163,11 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum /* Check that both expected and read object type match */ if (expected_enum_types[eloc] != enum_obj[elem].type) { - pr_err("Error expected type %d for elem %d, but got type %d instead\n", - expected_enum_types[eloc], elem, enum_obj[elem].type); + pr_warn("Unexpected element type at elem %d: expected %d, got %d, skipping\n", + elem, expected_enum_types[eloc], enum_obj[elem].type); kfree(str_value); - return -EIO; + str_value = NULL; + continue; } /* Assign appropriate element value to corresponding field */ @@ -227,6 +228,8 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -280,6 +283,8 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum kfree(str_value); str_value = NULL; } + if (size) + elem += (size < MAX_VALUES_SIZE ? size : MAX_VALUES_SIZE) - 1; break; default: pr_warn("Invalid element: %d found in Enumeration attribute or data may be malformed\n", elem); @@ -300,10 +305,12 @@ static int hp_populate_enumeration_elements_from_package(union acpi_object *enum * Populate all properties of an instance under enumeration attribute * * @enum_obj: ACPI object with enumeration data + * @enum_obj_count: Number of elements in @enum_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_enumeration_package_data(union acpi_object *enum_obj, + int enum_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -312,7 +319,7 @@ int hp_populate_enumeration_package_data(union acpi_object *enum_obj, enum_data->attr_name_kobj = attr_name_kobj; hp_populate_enumeration_elements_from_package(enum_obj, - enum_obj->package.count, + enum_obj_count, instance_id); hp_update_attribute_permissions(enum_data->common.is_readonly, &enumeration_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c index 63b1fda2be4e2..5577f571d7c06 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/int-attributes.c @@ -243,6 +243,8 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_ kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -275,10 +277,12 @@ static int hp_populate_integer_elements_from_package(union acpi_object *integer_ * Populate all properties of an instance under integer attribute * * @integer_obj: ACPI object with integer data + * @integer_obj_count: Number of elements in @integer_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_integer_package_data(union acpi_object *integer_obj, + int integer_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -286,7 +290,7 @@ int hp_populate_integer_package_data(union acpi_object *integer_obj, integer_data->attr_name_kobj = attr_name_kobj; hp_populate_integer_elements_from_package(integer_obj, - integer_obj->package.count, + integer_obj_count, instance_id); hp_update_attribute_permissions(integer_data->common.is_readonly, &integer_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c index 6a31f47ce3f5b..db5513d86e001 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/order-list-attributes.c @@ -146,7 +146,7 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord if (!order_obj) return -EINVAL; - for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT; elem++, eloc++) { + for (elem = 1, eloc = 1; eloc < ORD_ELEM_CNT && elem < order_obj_count; elem++, eloc++) { switch (order_obj[elem].type) { case ACPI_TYPE_STRING: @@ -233,6 +233,8 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -262,7 +264,9 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord * Ordered list data is stored in hex and comma separated format * Convert the data and split it to show each element */ - ret = hp_convert_hexstr_to_str(str_value, value_len, &tmpstr, &tmp_len); + ret = hp_convert_hexstr_to_str(order_obj[elem].string.pointer, + order_obj[elem].string.length, + &tmpstr, &tmp_len); if (ret) goto exit_list; @@ -299,10 +303,12 @@ static int hp_populate_ordered_list_elements_from_package(union acpi_object *ord * Populate all properties of an instance under ordered_list attribute * * @order_obj: ACPI object with ordered_list data + * @order_obj_count: Number of elements in @order_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ -int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int instance_id, +int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int order_obj_count, + int instance_id, struct kobject *attr_name_kobj) { struct ordered_list_data *ordered_list_data = &bioscfg_drv.ordered_list_data[instance_id]; @@ -310,7 +316,7 @@ int hp_populate_ordered_list_package_data(union acpi_object *order_obj, int inst ordered_list_data->attr_name_kobj = attr_name_kobj; hp_populate_ordered_list_elements_from_package(order_obj, - order_obj->package.count, + order_obj_count, instance_id); hp_update_attribute_permissions(ordered_list_data->common.is_readonly, &ordered_list_current_val); diff --git a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c index ec79d9d50377a..6b94f90bea07a 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/passwdobj-attributes.c @@ -66,7 +66,7 @@ static int validate_password_input(int instance_id, const char *buf) struct password_data *password_data = &bioscfg_drv.password_data[instance_id]; length = strlen(buf); - if (buf[length - 1] == '\n') + if (length > 0 && buf[length - 1] == '\n') length--; if (length > MAX_PASSWD_SIZE) @@ -123,7 +123,7 @@ static ssize_t new_password_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - return store_password_instance(kobj, buf, count, true); + return store_password_instance(kobj, buf, count, false); } static struct kobj_attribute password_new_password = __ATTR_WO(new_password); @@ -321,6 +321,8 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: password_data->common.security_level = int_value; @@ -351,6 +353,11 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor case PSWD_ENCODINGS: size = min_t(u32, password_data->encodings_size, MAX_ENCODINGS_SIZE); for (pos_values = 0; pos_values < size; pos_values++) { + if (elem + pos_values >= password_obj_count) { + pr_err("Error elem-objects package is too small\n"); + return -EINVAL; + } + ret = hp_convert_hexstr_to_str(password_obj[elem + pos_values].string.pointer, password_obj[elem + pos_values].string.length, &str_value, &value_len); @@ -362,6 +369,8 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor str_value = NULL; } + if (size) + elem += size - 1; break; case PSWD_IS_SET: password_data->is_enabled = int_value; @@ -385,10 +394,12 @@ static int hp_populate_password_elements_from_package(union acpi_object *passwor * Populate all properties for an instance under password attribute * * @password_obj: ACPI object with password data + * @password_obj_count: Number of elements in @password_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ -int hp_populate_password_package_data(union acpi_object *password_obj, int instance_id, +int hp_populate_password_package_data(union acpi_object *password_obj, int password_obj_count, + int instance_id, struct kobject *attr_name_kobj) { struct password_data *password_data = &bioscfg_drv.password_data[instance_id]; @@ -396,7 +407,7 @@ int hp_populate_password_package_data(union acpi_object *password_obj, int insta password_data->attr_name_kobj = attr_name_kobj; hp_populate_password_elements_from_package(password_obj, - password_obj->package.count, + password_obj_count, instance_id); hp_friendly_user_name_update(password_data->common.path, diff --git a/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c index 2b00a14792e92..4d94e48c1a4c8 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/spmobj-attributes.c @@ -238,7 +238,7 @@ static ssize_t sk_store(struct kobject *kobj, ret = hp_wmi_perform_query(HPWMI_SECUREPLATFORM_SET_SK, HPWMI_SECUREPLATFORM, (void *)bioscfg_drv.spm_data.signing_key, - count, 0); + length, 0); if (!ret) { bioscfg_drv.spm_data.mechanism = SIGNING_KEY; @@ -274,7 +274,7 @@ static ssize_t kek_store(struct kobject *kobj, ret = hp_wmi_perform_query(HPWMI_SECUREPLATFORM_SET_KEK, HPWMI_SECUREPLATFORM, (void *)bioscfg_drv.spm_data.endorsement_key, - count, 0); + length, 0); if (!ret) { bioscfg_drv.spm_data.mechanism = ENDORSEMENT_KEY; diff --git a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c index 7b885d25650c5..4e3d1fe90310d 100644 --- a/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c +++ b/drivers/platform/x86/hp/hp-bioscfg/string-attributes.c @@ -233,6 +233,8 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob kfree(str_value); str_value = NULL; } + if (size) + elem += size - 1; break; case SECURITY_LEVEL: @@ -263,10 +265,12 @@ static int hp_populate_string_elements_from_package(union acpi_object *string_ob * Populate all properties of an instance under string attribute * * @string_obj: ACPI object with string data + * @string_obj_count: Number of elements in @string_obj * @instance_id: The instance to enumerate * @attr_name_kobj: The parent kernel object */ int hp_populate_string_package_data(union acpi_object *string_obj, + int string_obj_count, int instance_id, struct kobject *attr_name_kobj) { @@ -275,7 +279,7 @@ int hp_populate_string_package_data(union acpi_object *string_obj, string_data->attr_name_kobj = attr_name_kobj; hp_populate_string_elements_from_package(string_obj, - string_obj->package.count, + string_obj_count, instance_id); hp_update_attribute_permissions(string_data->common.is_readonly, diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c index 550d4b39a92af..5eb2aade44cca 100644 --- a/drivers/platform/x86/hp/hp-wmi.c +++ b/drivers/platform/x86/hp/hp-wmi.c @@ -194,6 +194,10 @@ static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst .matches = { DMI_MATCH(DMI_BOARD_NAME, "8D41") }, .driver_data = (void *)&victus_s_thermal_params, }, + { + .matches = { DMI_MATCH(DMI_BOARD_NAME, "8DD6") }, + .driver_data = (void *)&omen_v1_thermal_params, + }, {}, }; diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c index 0dcf3a68a3355..f97f7f111319e 100644 --- a/drivers/platform/x86/intel/hid.c +++ b/drivers/platform/x86/intel/hid.c @@ -158,6 +158,13 @@ static const struct dmi_system_id button_array_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Surface Go 4"), }, }, + { + .ident = "HP ProBook x360 440 G1", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP ProBook x360 440 G1"), + }, + }, { } }; diff --git a/drivers/platform/x86/intel/int1092/intel_sar.c b/drivers/platform/x86/intel/int1092/intel_sar.c index e526841aff603..7263114f0b3d0 100644 --- a/drivers/platform/x86/intel/int1092/intel_sar.c +++ b/drivers/platform/x86/intel/int1092/intel_sar.c @@ -91,8 +91,10 @@ static acpi_status parse_package(struct wwan_sar_context *context, union acpi_ob item->package.count <= data->total_dev_mode) return AE_ERROR; - data->device_mode_info = kmalloc_array(data->total_dev_mode, - sizeof(struct wwan_device_mode_info), GFP_KERNEL); + data->device_mode_info = devm_kmalloc_array(&context->sar_device->dev, + data->total_dev_mode, + sizeof(*data->device_mode_info), + GFP_KERNEL); if (!data->device_mode_info) return AE_ERROR; @@ -245,21 +247,26 @@ static void sar_get_data(int reg, struct wwan_sar_context *context) static int sar_probe(struct platform_device *device) { struct wwan_sar_context *context; + acpi_handle handle; int reg; int result; - context = kzalloc(sizeof(*context), GFP_KERNEL); + handle = ACPI_HANDLE(&device->dev); + if (!handle) + return -ENODEV; + + context = devm_kzalloc(&device->dev, sizeof(*context), GFP_KERNEL); if (!context) return -ENOMEM; context->sar_device = device; - context->handle = ACPI_HANDLE(&device->dev); + context->handle = handle; dev_set_drvdata(&device->dev, context); result = guid_parse(SAR_DSM_UUID, &context->guid); if (result) { dev_err(&device->dev, "SAR UUID parse error: %d\n", result); - goto r_free; + return result; } for (reg = 0; reg < MAX_REGULATORY; reg++) @@ -267,43 +274,29 @@ static int sar_probe(struct platform_device *device) if (sar_get_device_mode(device) != AE_OK) { dev_err(&device->dev, "Failed to get device mode\n"); - result = -EIO; - goto r_free; + return -EIO; } result = sysfs_create_group(&device->dev.kobj, &intcsar_group); if (result) { dev_err(&device->dev, "sysfs creation failed\n"); - goto r_free; + return result; } if (acpi_install_notify_handler(ACPI_HANDLE(&device->dev), ACPI_DEVICE_NOTIFY, sar_notify, (void *)device) != AE_OK) { dev_err(&device->dev, "Failed acpi_install_notify_handler\n"); - result = -EIO; - goto r_sys; + sysfs_remove_group(&device->dev.kobj, &intcsar_group); + return -EIO; } return 0; - -r_sys: - sysfs_remove_group(&device->dev.kobj, &intcsar_group); -r_free: - kfree(context); - return result; } static void sar_remove(struct platform_device *device) { - struct wwan_sar_context *context = dev_get_drvdata(&device->dev); - int reg; - acpi_remove_notify_handler(ACPI_HANDLE(&device->dev), ACPI_DEVICE_NOTIFY, sar_notify); sysfs_remove_group(&device->dev.kobj, &intcsar_group); - for (reg = 0; reg < MAX_REGULATORY; reg++) - kfree(context->config_data[reg].device_mode_info); - - kfree(context); } static struct platform_driver sar_driver = { diff --git a/drivers/platform/x86/intel/ishtp_eclite.c b/drivers/platform/x86/intel/ishtp_eclite.c index 93ac8b2dbf387..bca7e217878b4 100644 --- a/drivers/platform/x86/intel/ishtp_eclite.c +++ b/drivers/platform/x86/intel/ishtp_eclite.c @@ -600,13 +600,16 @@ static int ecl_ishtp_cl_probe(struct ishtp_cl_device *cl_device) rv = acpi_opregion_init(opr_dev); if (rv) { dev_err(cl_data_to_dev(opr_dev), "ACPI opregion init failed\n"); - goto err_exit; + goto err_put; } /* Reprobe devices depending on ECLite - battery, fan, etc. */ acpi_dev_clear_dependencies(opr_dev->adev); return 0; + +err_put: + acpi_dev_put(opr_dev->adev); err_exit: ishtp_set_connection_state(ecl_ishtp_cl, ISHTP_CL_DISCONNECTING); ishtp_cl_disconnect(ecl_ishtp_cl); diff --git a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c index b11d62c934b37..7c45b05190f3c 100644 --- a/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c +++ b/drivers/platform/x86/intel/speed_select_if/isst_tpmi_core.c @@ -336,8 +336,11 @@ static int sst_add_perf_profiles(struct auxiliary_device *auxdev, int i; pd_info->perf_levels = devm_kcalloc(dev, levels, sizeof(struct perf_level), GFP_KERNEL); - if (!pd_info->perf_levels) - return 0; + if (!pd_info->perf_levels) { + pd_info->pp_header.allowed_level_mask = 0; + pd_info->pp_header.level_en_mask = 0; + return -ENOMEM; + } pd_info->ratio_unit = pd_info->pp_header.ratio_unit; pd_info->avx_levels = SST_MAX_AVX_LEVELS; @@ -367,7 +370,7 @@ static int sst_add_perf_profiles(struct auxiliary_device *auxdev, static int sst_main(struct auxiliary_device *auxdev, struct tpmi_per_power_domain_info *pd_info) { struct device *dev = &auxdev->dev; - int i, mask, levels; + int i, ret, mask, levels; *((u64 *)&pd_info->sst_header) = readq(pd_info->sst_base); pd_info->sst_header.cp_offset *= 8; @@ -399,8 +402,12 @@ static int sst_main(struct auxiliary_device *auxdev, struct tpmi_per_power_domai levels = i; mask <<= 1; } + + ret = sst_add_perf_profiles(auxdev, pd_info, levels + 1); + if (ret) + return ret; + pd_info->max_level = levels; - sst_add_perf_profiles(auxdev, pd_info, levels + 1); return 0; } @@ -599,6 +606,9 @@ static bool disable_dynamic_sst_features(void) #define SST_CP_PRIORITY_TYPE_START 1 #define SST_CP_PRIORITY_TYPE_WIDTH 1 +#define SST_CP_MAX_ENABLE 1 +#define SST_CP_MAX_PRIORITY_TYPE 1 + static long isst_if_core_power_state(void __user *argp) { struct tpmi_per_power_domain_info *power_domain_info; @@ -615,9 +625,13 @@ static long isst_if_core_power_state(void __user *argp) return -EINVAL; if (core_power.get_set) { - if (power_domain_info->write_blocked) + if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (core_power.enable > SST_CP_MAX_ENABLE || + core_power.priority_type > SST_CP_MAX_PRIORITY_TYPE) + return -EINVAL; + _write_cp_info("cp_enable", core_power.enable, SST_CP_CONTROL_OFFSET, SST_CP_ENABLE_START, SST_CP_ENABLE_WIDTH, SST_MUL_FACTOR_NONE) _write_cp_info("cp_prio_type", core_power.priority_type, SST_CP_CONTROL_OFFSET, @@ -649,6 +663,11 @@ static long isst_if_core_power_state(void __user *argp) #define SST_CLOS_CONFIG_MAX_START 16 #define SST_CLOS_CONFIG_MAX_WIDTH 8 +#define SST_MAX_CLOS 3 + +#define SST_MAX_FREQ 0xff +#define SST_CLOS_MAX_PRIORITY 0x0f + static long isst_if_clos_param(void __user *argp) { struct tpmi_per_power_domain_info *power_domain_info; @@ -657,14 +676,26 @@ static long isst_if_clos_param(void __user *argp) if (copy_from_user(&clos_param, argp, sizeof(clos_param))) return -EFAULT; + if (clos_param.clos > SST_MAX_CLOS) + return -EINVAL; + power_domain_info = get_instance(clos_param.socket_id, clos_param.power_domain_id); if (!power_domain_info) return -EINVAL; if (clos_param.get_set) { - if (power_domain_info->write_blocked) + if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (!in_range(clos_param.min_freq_mhz / SST_MUL_FACTOR_FREQ, 0, SST_MAX_FREQ + 1)) + return -EINVAL; + + if (!in_range(clos_param.max_freq_mhz / SST_MUL_FACTOR_FREQ, 0, SST_MAX_FREQ + 1)) + return -EINVAL; + + if (!in_range(clos_param.prop_prio, 0, SST_CLOS_MAX_PRIORITY + 1)) + return -EINVAL; + _write_cp_info("clos.min_freq", clos_param.min_freq_mhz, (SST_CLOS_CONFIG_0_OFFSET + clos_param.clos * SST_REG_SIZE), SST_CLOS_CONFIG_MIN_START, SST_CLOS_CONFIG_MIN_WIDTH, @@ -703,6 +734,8 @@ static long isst_if_clos_param(void __user *argp) #define SST_CLOS_ASSOC_CPUS_PER_REG 16 #define SST_CLOS_ASSOC_BITS_PER_CPU 4 +#define SST_CLOS_ASSOC_MAX_LOGICAL_CPU 63 + static long isst_if_clos_assoc(void __user *argp) { struct isst_if_clos_assoc_cmds assoc_cmds; @@ -729,7 +762,13 @@ static long isst_if_clos_assoc(void __user *argp) if (copy_from_user(&clos_assoc, ptr, sizeof(clos_assoc))) return -EFAULT; - if (clos_assoc.socket_id > topology_max_packages()) + if (clos_assoc.clos > SST_MAX_CLOS) + return -EINVAL; + + if (clos_assoc.socket_id >= topology_max_packages()) + return -EINVAL; + + if (clos_assoc.logical_cpu > SST_CLOS_ASSOC_MAX_LOGICAL_CPU) return -EINVAL; cpu = clos_assoc.logical_cpu; @@ -747,6 +786,8 @@ static long isst_if_clos_assoc(void __user *argp) pkg_id = clos_assoc.socket_id; sst_inst = isst_common.sst_inst[pkg_id]; + if (!sst_inst) + return -EINVAL; punit_id = map_partition_power_domain_id(sst_inst, punit_id, &part); if (punit_id < 0) @@ -754,7 +795,8 @@ static long isst_if_clos_assoc(void __user *argp) power_domain_info = &sst_inst->power_domain_info[part][punit_id]; - if (assoc_cmds.get_set && power_domain_info->write_blocked) + if (assoc_cmds.get_set && (power_domain_info->write_blocked || + !capable(CAP_SYS_ADMIN))) return -EPERM; offset = SST_CLOS_ASSOC_0_OFFSET + @@ -844,6 +886,7 @@ static long isst_if_clos_assoc(void __user *argp) #define SST_PP_FEATURE_STATE_START 8 #define SST_PP_FEATURE_STATE_WIDTH 8 +#define SST_PP_FEATURE_STATE_VALID_MASK GENMASK(1, 0) #define SST_BF_FEATURE_SUPPORTED_START 12 #define SST_BF_FEATURE_SUPPORTED_WIDTH 1 @@ -876,7 +919,7 @@ static int isst_if_get_perf_level(void __user *argp) SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) perf_level.enabled = !!(power_domain_info->sst_header.cap_mask & BIT(1)); - level_mask = perf_level.level_mask; + level_mask = perf_level.level_mask & power_domain_info->pp_header.level_en_mask; perf_level.sst_bf_support = 0; for_each_set_bit(level, &level_mask, BITS_PER_BYTE) { /* @@ -931,7 +974,10 @@ static int isst_if_set_perf_level(void __user *argp) if (!power_domain_info) return -EINVAL; - if (power_domain_info->write_blocked) + if (perf_level.level > power_domain_info->max_level) + return -EINVAL; + + if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; if (!(power_domain_info->pp_header.allowed_level_mask & BIT(perf_level.level))) @@ -991,9 +1037,12 @@ static int isst_if_set_perf_feature(void __user *argp) if (!power_domain_info) return -EINVAL; - if (power_domain_info->write_blocked) + if (power_domain_info->write_blocked || !capable(CAP_SYS_ADMIN)) return -EPERM; + if (perf_feature.feature & ~SST_PP_FEATURE_STATE_VALID_MASK) + return -EINVAL; + _write_pp_info("perf_feature", perf_feature.feature, SST_PP_CONTROL_OFFSET, SST_PP_FEATURE_STATE_START, SST_PP_FEATURE_STATE_WIDTH, SST_MUL_FACTOR_NONE) @@ -1260,6 +1309,12 @@ static int isst_if_get_perf_level_mask(void __user *argp) if (!power_domain_info) return -EINVAL; + if (cpumask.level > power_domain_info->max_level) + return -EINVAL; + + if (!(power_domain_info->pp_header.level_en_mask & BIT(cpumask.level))) + return -EINVAL; + _read_pp_level_info("mask", mask, cpumask.level, SST_PP_INFO_2_OFFSET, SST_PP_RSLVD_CORE_MASK_START, SST_PP_RSLVD_CORE_MASK_WIDTH, SST_MUL_FACTOR_NONE) @@ -1305,6 +1360,9 @@ static int isst_if_get_base_freq_info(void __user *argp) if (base_freq.level > power_domain_info->max_level) return -EINVAL; + if (!(power_domain_info->pp_header.level_en_mask & BIT(base_freq.level))) + return -EINVAL; + _read_bf_level_info("p1_high", base_freq.high_base_freq_mhz, base_freq.level, SST_BF_INFO_0_OFFSET, SST_BF_P1_HIGH_START, SST_BF_P1_HIGH_WIDTH, SST_MUL_FACTOR_FREQ) @@ -1341,6 +1399,12 @@ static int isst_if_get_base_freq_mask(void __user *argp) if (!power_domain_info) return -EINVAL; + if (cpumask.level > power_domain_info->max_level) + return -EINVAL; + + if (!(power_domain_info->pp_header.level_en_mask & BIT(cpumask.level))) + return -EINVAL; + _read_bf_level_info("BF-cpumask", mask, cpumask.level, SST_BF_INFO_1_OFFSET, P1_HI_CORE_MASK_START, P1_HI_CORE_MASK_WIDTH, SST_MUL_FACTOR_NONE) @@ -1369,6 +1433,8 @@ static int isst_if_get_tpmi_instance_count(void __user *argp) return -EINVAL; sst_inst = isst_common.sst_inst[tpmi_inst.socket_id]; + if (!sst_inst) + return -EINVAL; tpmi_inst.count = isst_instance_count(sst_inst); @@ -1436,6 +1502,9 @@ static int isst_if_get_turbo_freq_info(void __user *argp) if (turbo_freq.level > power_domain_info->max_level) return -EINVAL; + if (!(power_domain_info->pp_header.level_en_mask & BIT(turbo_freq.level))) + return -EINVAL; + turbo_freq.max_buckets = TRL_MAX_BUCKETS; turbo_freq.max_trl_levels = TRL_MAX_LEVELS; turbo_freq.max_clip_freqs = SST_TF_MAX_LP_CLIP_RATIOS; diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c index 65897fae17dfb..6091b617480ff 100644 --- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c +++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c @@ -274,15 +274,20 @@ int uncore_freq_add_entry(struct uncore_data *data, int cpu) sprintf(data->name, "package_%02d_die_%02d", data->package_id, data->die_id); } + /* + * Set the control CPU before any read path so entry recreation after CPU + * hotplug can populate read-only attributes from the new online CPU. + */ + data->control_cpu = cpu; uncore_read(data, &data->initial_min_freq_khz, UNCORE_INDEX_MIN_FREQ); uncore_read(data, &data->initial_max_freq_khz, UNCORE_INDEX_MAX_FREQ); ret = create_attr_group(data, data->name); if (ret) { + data->control_cpu = -1; if (data->domain_id != UNCORE_DOMAIN_ID_INVALID) ida_free(&intel_uncore_ida, data->instance_id); } else { - data->control_cpu = cpu; data->valid = true; } diff --git a/drivers/platform/x86/lenovo/think-lmi.c b/drivers/platform/x86/lenovo/think-lmi.c index 540b472b1bf35..a56cca9d65989 100644 --- a/drivers/platform/x86/lenovo/think-lmi.c +++ b/drivers/platform/x86/lenovo/think-lmi.c @@ -438,14 +438,13 @@ static ssize_t current_password_store(struct kobject *kobj, struct tlmi_pwd_setting *setting = to_tlmi_pwd_setting(kobj); size_t pwdlen; - pwdlen = strlen(buf); + /* Strip newline; setting password won't work if one is present. */ + pwdlen = strchrnul(buf, '\n') - buf; /* pwdlen == 0 is allowed to clear the password */ if (pwdlen && ((pwdlen < setting->minlen) || (pwdlen > setting->maxlen))) return -EINVAL; - strscpy(setting->password, buf, setting->maxlen); - /* Strip out CR if one is present, setting password won't work if it is present */ - strreplace(setting->password, '\n', '\0'); + strscpy(setting->password, buf, pwdlen + 1); return count; } @@ -741,6 +740,8 @@ static ssize_t certificate_thumbprint_show(struct kobject *kobj, struct kobj_att return -EOPNOTSUPP; for (i = 0; i < ARRAY_SIZE(thumbtypes); i++) { + ssize_t ret; + if (tlmi_priv.pwdcfg.core.password_mode >= TLMI_PWDCFG_MODE_MULTICERT) { /* Format: 'SVC | SMC, Thumbtype' */ wmistr = kasprintf(GFP_KERNEL, "%s,%s", @@ -752,8 +753,12 @@ static ssize_t certificate_thumbprint_show(struct kobject *kobj, struct kobj_att } if (!wmistr) return -ENOMEM; - count += cert_thumbprint(buf, wmistr, count); + + ret = cert_thumbprint(buf, wmistr, count); kfree(wmistr); + if (ret < 0) + return ret; + count = ret; } return count; @@ -1452,6 +1457,10 @@ static void tlmi_release_attr(void) /* Free up any saved signatures */ kfree(tlmi_priv.pwd_admin->signature); kfree(tlmi_priv.pwd_admin->save_signature); + if (tlmi_priv.pwd_system) { + kfree(tlmi_priv.pwd_system->signature); + kfree(tlmi_priv.pwd_system->save_signature); + } /* Authentication structures */ list_for_each_entry_safe(pos, n, &tlmi_priv.authentication_kset->list, entry) diff --git a/drivers/platform/x86/lenovo/wmi-helpers.c b/drivers/platform/x86/lenovo/wmi-helpers.c index f6fef6296251e..910e024a5c828 100644 --- a/drivers/platform/x86/lenovo/wmi-helpers.c +++ b/drivers/platform/x86/lenovo/wmi-helpers.c @@ -45,7 +45,6 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id, unsigned char *buf, size_t size, u32 *retval) { struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL }; - union acpi_object *ret_obj __free(kfree) = NULL; struct acpi_buffer input = { size, buf }; acpi_status status; @@ -54,8 +53,9 @@ int lwmi_dev_evaluate_int(struct wmi_device *wdev, u8 instance, u32 method_id, if (ACPI_FAILURE(status)) return -EIO; + union acpi_object *ret_obj __free(kfree) = output.pointer; + if (retval) { - ret_obj = output.pointer; if (!ret_obj) return -ENODATA; diff --git a/drivers/platform/x86/lenovo/ymc.c b/drivers/platform/x86/lenovo/ymc.c index 470d53e3c9d29..c71d92f896ec6 100644 --- a/drivers/platform/x86/lenovo/ymc.c +++ b/drivers/platform/x86/lenovo/ymc.c @@ -8,6 +8,8 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include +#include +#include #include #include #include @@ -20,6 +22,8 @@ #define LENOVO_YMC_QUERY_INSTANCE 0 #define LENOVO_YMC_QUERY_METHOD 0x01 +#define LENOVO_YMC_STATE_MASK GENMASK(7, 0) + static bool force; module_param(force, bool, 0444); MODULE_PARM_DESC(force, "Force loading on boards without a convertible DMI chassis-type"); @@ -85,7 +89,9 @@ static void lenovo_ymc_notify(struct wmi_device *wdev, union acpi_object *data) "WMI event data is not an integer\n"); goto free_obj; } - code = obj->integer.value; + + /* strip upper bits (e.g. 0x50000) on newer devices */ + code = FIELD_GET(LENOVO_YMC_STATE_MASK, obj->integer.value); if (!sparse_keymap_report_event(priv->input_dev, code, 1, true)) dev_warn(&wdev->dev, "Unknown key %d pressed\n", code); diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c index 6af6cf477c5b5..f4e43e09cbe4e 100644 --- a/drivers/platform/x86/lg-laptop.c +++ b/drivers/platform/x86/lg-laptop.c @@ -98,6 +98,7 @@ static u32 inited; #define INIT_SPARSE_KEYMAP 0x80 static int battery_limit_use_wmbb; +static bool kbd_backlight_available; static struct led_classdev kbd_backlight; static enum led_brightness get_kbd_backlight_level(struct device *dev); @@ -212,6 +213,7 @@ static union acpi_object *lg_wmbb(struct device *dev, u32 method_id, u32 arg1, u static void wmi_notify(union acpi_object *obj, void *context) { long data = (long)context; + unsigned int brightness; pr_debug("event guid %li\n", data); if (!obj) @@ -222,8 +224,11 @@ static void wmi_notify(union acpi_object *obj, void *context) struct key_entry *key; if (eventcode == 0x10000000) { - led_classdev_notify_brightness_hw_changed( - &kbd_backlight, get_kbd_backlight_level(kbd_backlight.dev->parent)); + if (kbd_backlight_available) { + brightness = get_kbd_backlight_level(kbd_backlight.dev->parent); + led_classdev_notify_brightness_hw_changed(&kbd_backlight, + brightness); + } } else { key = sparse_keymap_entry_from_scancode( wmi_input_dev, eventcode); @@ -269,11 +274,6 @@ static void wmi_input_setup(void) } } -static void acpi_notify(struct acpi_device *device, u32 event) -{ - acpi_handle_debug(device->handle, "notify: %d\n", event); -} - static ssize_t fan_mode_store(struct device *dev, struct device_attribute *attr, const char *buffer, size_t count) @@ -753,13 +753,13 @@ static void lg_laptop_remove_address_space_handler(void *data) &lg_laptop_address_space_handler); } -static int acpi_add(struct acpi_device *device) +static int acpi_probe(struct platform_device *pdev) { struct platform_device_info pdev_info = { - .fwnode = acpi_fwnode_handle(device), .name = PLATFORM_NAME, .id = PLATFORM_DEVID_NONE, }; + struct acpi_device *device; acpi_status status; int ret; const char *product; @@ -768,13 +768,19 @@ static int acpi_add(struct acpi_device *device) if (pf_device) return 0; + device = ACPI_COMPANION(&pdev->dev); + if (!device) + return -ENODEV; + + pdev_info.fwnode = acpi_fwnode_handle(device), + status = acpi_install_address_space_handler(device->handle, LG_ADDRESS_SPACE_ID, &lg_laptop_address_space_handler, - NULL, &device->dev); + NULL, &pdev->dev); if (ACPI_FAILURE(status)) return -ENODEV; - ret = devm_add_action_or_reset(&device->dev, lg_laptop_remove_address_space_handler, + ret = devm_add_action_or_reset(&pdev->dev, lg_laptop_remove_address_space_handler, device); if (ret < 0) return ret; @@ -844,8 +850,13 @@ static int acpi_add(struct acpi_device *device) goto out_platform_device; /* LEDs are optional */ - led_classdev_register(&pf_device->dev, &kbd_backlight); - led_classdev_register(&pf_device->dev, &tpad_led); + ret = devm_led_classdev_register(&pdev->dev, &kbd_backlight); + if (ret < 0) + kbd_backlight_available = false; + else + kbd_backlight_available = true; + + devm_led_classdev_register(&pdev->dev, &tpad_led); wmi_input_setup(); battery_hook_register(&battery_hook); @@ -859,13 +870,10 @@ static int acpi_add(struct acpi_device *device) return ret; } -static void acpi_remove(struct acpi_device *device) +static void acpi_remove(struct platform_device *pdev) { sysfs_remove_group(&pf_device->dev.kobj, &dev_attribute_group); - led_classdev_unregister(&tpad_led); - led_classdev_unregister(&kbd_backlight); - battery_hook_unregister(&battery_hook); wmi_input_destroy(); platform_device_unregister(pf_device); @@ -879,34 +887,13 @@ static const struct acpi_device_id device_ids[] = { }; MODULE_DEVICE_TABLE(acpi, device_ids); -static struct acpi_driver acpi_driver = { - .name = "LG Gram Laptop Support", - .class = "lg-laptop", - .ids = device_ids, - .ops = { - .add = acpi_add, - .remove = acpi_remove, - .notify = acpi_notify, - }, +static struct platform_driver acpi_driver = { + .probe = acpi_probe, + .remove = acpi_remove, + .driver = { + .name = "LG Gram Laptop Support", + .acpi_match_table = device_ids, + }, }; -static int __init acpi_init(void) -{ - int result; - - result = acpi_bus_register_driver(&acpi_driver); - if (result < 0) { - pr_debug("Error registering driver\n"); - return -ENODEV; - } - - return 0; -} - -static void __exit acpi_exit(void) -{ - acpi_bus_unregister_driver(&acpi_driver); -} - -module_init(acpi_init); -module_exit(acpi_exit); +module_platform_driver(acpi_driver); diff --git a/drivers/platform/x86/msi-ec.c b/drivers/platform/x86/msi-ec.c index f19504dbf164c..0157e233e4300 100644 --- a/drivers/platform/x86/msi-ec.c +++ b/drivers/platform/x86/msi-ec.c @@ -823,6 +823,7 @@ static struct msi_ec_conf CONF9 __initdata = { static const char * const ALLOWED_FW_10[] __initconst = { "1582EMS1.107", // GF66 11UC + "1583EMS1.109", // Pulse GL66 12UEK NULL }; diff --git a/drivers/platform/x86/sel3350-platform.c b/drivers/platform/x86/sel3350-platform.c index 02e2081e2333b..f3a3142356325 100644 --- a/drivers/platform/x86/sel3350-platform.c +++ b/drivers/platform/x86/sel3350-platform.c @@ -9,6 +9,8 @@ */ #include +#include +#include #include #include #include @@ -30,19 +32,82 @@ #define SEL_PS_B_DETECT "sel_ps_b_detect" #define SEL_PS_B_GOOD "sel_ps_b_good" +#define AUX_LED_GRN1 "sel_aux_led_grn1" +#define AUX_LED_GRN2 "sel_aux_led_grn2" +#define AUX_LED_GRN3 "sel_aux_led_grn3" +#define AUX_LED_GRN4 "sel_aux_led_grn4" +#define ALARM_STATE_USER "sel_alarm_state_user" +#define ENABLE_STATE_USER "sel_enable_state_user" +#define AUX_LED_RED1 "sel_aux_led_red1" +#define AUX_LED_RED2 "sel_aux_led_red2" +#define AUX_LED_RED3 "sel_aux_led_red3" +#define AUX_LED_RED4 "sel_aux_led_red4" + +static const char *const sel3350_leds_gpio_names[] = { + AUX_LED_GRN1, + AUX_LED_GRN2, + AUX_LED_GRN3, + AUX_LED_GRN4, + ALARM_STATE_USER, + ENABLE_STATE_USER, + AUX_LED_RED1, + AUX_LED_RED2, + AUX_LED_RED3, + AUX_LED_RED4, +}; + /* LEDs */ -static const struct gpio_led sel3350_leds[] = { - { .name = "sel:green:aux1" }, - { .name = "sel:green:aux2" }, - { .name = "sel:green:aux3" }, - { .name = "sel:green:aux4" }, - { .name = "sel:red:alarm" }, +static struct gpio_led sel3350_leds[] = { + { .name = "sel:green:aux1", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:green:aux2", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:green:aux3", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:green:aux4", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:red:alarm", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, { .name = "sel:green:enabled", - .default_state = LEDS_GPIO_DEFSTATE_ON }, - { .name = "sel:red:aux1" }, - { .name = "sel:red:aux2" }, - { .name = "sel:red:aux3" }, - { .name = "sel:red:aux4" }, + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:red:aux1", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:red:aux2", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:red:aux3", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, + { .name = "sel:red:aux4", + .default_state = LEDS_GPIO_DEFSTATE_KEEP, + .retain_state_suspended = 1, + .retain_state_shutdown = 1, + }, }; static const struct gpio_led_platform_data sel3350_leds_pdata = { @@ -50,25 +115,6 @@ static const struct gpio_led_platform_data sel3350_leds_pdata = { .leds = sel3350_leds, }; -/* Map GPIOs to LEDs */ -static struct gpiod_lookup_table sel3350_leds_table = { - .dev_id = "leds-gpio", - .table = { - GPIO_LOOKUP_IDX(BXT_NW, 49, NULL, 0, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_NW, 50, NULL, 1, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_NW, 51, NULL, 2, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_NW, 52, NULL, 3, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_W, 20, NULL, 4, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_W, 21, NULL, 5, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_SW, 37, NULL, 6, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_SW, 38, NULL, 7, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_SW, 39, NULL, 8, GPIO_ACTIVE_HIGH), - GPIO_LOOKUP_IDX(BXT_SW, 40, NULL, 9, GPIO_ACTIVE_HIGH), - {}, - } -}; - -/* Map GPIOs to power supplies */ static struct gpiod_lookup_table sel3350_gpios_table = { .dev_id = B2093_GPIO_ACPI_ID ":00", .table = { @@ -76,6 +122,16 @@ static struct gpiod_lookup_table sel3350_gpios_table = { GPIO_LOOKUP(BXT_NW, 45, SEL_PS_A_GOOD, GPIO_ACTIVE_LOW), GPIO_LOOKUP(BXT_NW, 46, SEL_PS_B_DETECT, GPIO_ACTIVE_LOW), GPIO_LOOKUP(BXT_NW, 47, SEL_PS_B_GOOD, GPIO_ACTIVE_LOW), + GPIO_LOOKUP(BXT_NW, 49, AUX_LED_GRN1, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_NW, 50, AUX_LED_GRN2, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_NW, 51, AUX_LED_GRN3, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_NW, 52, AUX_LED_GRN4, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_W, 20, ALARM_STATE_USER, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_W, 21, ENABLE_STATE_USER, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_SW, 37, AUX_LED_RED1, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_SW, 38, AUX_LED_RED2, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_SW, 39, AUX_LED_RED3, GPIO_ACTIVE_HIGH), + GPIO_LOOKUP(BXT_SW, 40, AUX_LED_RED4, GPIO_ACTIVE_HIGH), {}, } }; @@ -149,6 +205,7 @@ struct sel3350_data { static int sel3350_probe(struct platform_device *pdev) { int rs; + int i; struct sel3350_data *sel3350; struct power_supply_config ps_cfg = {}; @@ -158,9 +215,19 @@ static int sel3350_probe(struct platform_device *pdev) platform_set_drvdata(pdev, sel3350); - gpiod_add_lookup_table(&sel3350_leds_table); gpiod_add_lookup_table(&sel3350_gpios_table); + for (i = 0; i < ARRAY_SIZE(sel3350_leds); ++i) { + sel3350_leds[i].gpiod = devm_gpiod_get(&pdev->dev, + sel3350_leds_gpio_names[i], + GPIOD_ASIS); + if (IS_ERR_OR_NULL(sel3350_leds[i].gpiod)) { + rs = -EPROBE_DEFER; + goto err_gpio_loop; + } + gpiod_set_consumer_name(sel3350_leds[i].gpiod, sel3350_leds[i].name); + } + sel3350->leds_pdev = platform_device_register_data( NULL, "leds-gpio", @@ -209,11 +276,15 @@ static int sel3350_probe(struct platform_device *pdev) return 0; +err_gpio_loop: + while (i--) + devm_gpiod_put(&pdev->dev, sel3350_leds[i].gpiod); + goto err_platform; + err_ps: platform_device_unregister(sel3350->leds_pdev); err_platform: gpiod_remove_lookup_table(&sel3350_gpios_table); - gpiod_remove_lookup_table(&sel3350_leds_table); return rs; } @@ -224,7 +295,6 @@ static void sel3350_remove(struct platform_device *pdev) platform_device_unregister(sel3350->leds_pdev); gpiod_remove_lookup_table(&sel3350_gpios_table); - gpiod_remove_lookup_table(&sel3350_leds_table); } static const struct acpi_device_id sel3350_device_ids[] = { diff --git a/drivers/pmdomain/arm/scmi_perf_domain.c b/drivers/pmdomain/arm/scmi_perf_domain.c index 3693423459c9c..e390f902a4442 100644 --- a/drivers/pmdomain/arm/scmi_perf_domain.c +++ b/drivers/pmdomain/arm/scmi_perf_domain.c @@ -33,7 +33,7 @@ scmi_pd_set_perf_state(struct generic_pm_domain *genpd, unsigned int state) return 0; if (!state) - return -EINVAL; + return 0; ret = pd->perf_ops->level_set(pd->ph, pd->domain_id, state, false); if (ret) diff --git a/drivers/pmdomain/bcm/bcm2835-power.c b/drivers/pmdomain/bcm/bcm2835-power.c index b008b47a8ce49..b29914ba034dc 100644 --- a/drivers/pmdomain/bcm/bcm2835-power.c +++ b/drivers/pmdomain/bcm/bcm2835-power.c @@ -684,7 +684,12 @@ static int bcm2835_power_probe(struct platform_device *pdev) if (ret) goto fail; - of_genpd_add_provider_onecell(dev->parent->of_node, &power->pd_xlate); + ret = of_genpd_add_provider_onecell(dev->parent->of_node, + &power->pd_xlate); + if (ret) { + dev_err_probe(dev, ret, "failed to add genpd provider\n"); + goto fail; + } dev_info(dev, "Broadcom BCM2835 power domains driver"); return 0; diff --git a/drivers/pmdomain/imx/imx93-blk-ctrl.c b/drivers/pmdomain/imx/imx93-blk-ctrl.c index e094fe5a42bf6..28688ca46517a 100644 --- a/drivers/pmdomain/imx/imx93-blk-ctrl.c +++ b/drivers/pmdomain/imx/imx93-blk-ctrl.c @@ -47,6 +47,8 @@ #define PRIO(X) (X) +#define BLK_CTRL_NO_PARENT UINT_MAX + struct imx93_blk_ctrl_domain; struct imx93_blk_ctrl { @@ -67,12 +69,18 @@ struct imx93_blk_ctrl_qos { u32 cfg_prio; }; +struct imx93_blk_ctrl_subdomain_link { + struct generic_pm_domain *parent; + struct generic_pm_domain *subdomain; +}; + struct imx93_blk_ctrl_domain_data { const char *name; const char * const *clk_names; int num_clks; u32 rst_mask; u32 clk_mask; + u32 parent; int num_qos; struct imx93_blk_ctrl_qos qos[DOMAIN_MAX_QOS]; }; @@ -188,6 +196,27 @@ static int imx93_blk_ctrl_power_off(struct generic_pm_domain *genpd) return 0; } +static void imx93_release_genpd_provider(void *data) +{ + struct device_node *of_node = data; + + of_genpd_del_provider(of_node); +} + +static void imx93_release_pm_genpd(void *data) +{ + struct generic_pm_domain *genpd = data; + + pm_genpd_remove(genpd); +} + +static void imx93_release_subdomain(void *data) +{ + struct imx93_blk_ctrl_subdomain_link *link = data; + + pm_genpd_remove_subdomain(link->parent, link->subdomain); +} + static struct lock_class_key blk_ctrl_genpd_lock_class; static int imx93_blk_ctrl_probe(struct platform_device *pdev) @@ -258,10 +287,8 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev) domain->clks[j].id = data->clk_names[j]; ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks); - if (ret) { - dev_err_probe(dev, ret, "failed to get clock\n"); - goto cleanup_pds; - } + if (ret) + return dev_err_probe(dev, ret, "failed to get clock\n"); domain->genpd.name = data->name; domain->genpd.power_on = imx93_blk_ctrl_power_on; @@ -269,11 +296,12 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev) domain->bc = bc; ret = pm_genpd_init(&domain->genpd, NULL, true); - if (ret) { - dev_err_probe(dev, ret, "failed to init power domain\n"); - goto cleanup_pds; - } + if (ret) + return dev_err_probe(dev, ret, "failed to init power domain\n"); + ret = devm_add_action_or_reset(dev, imx93_release_pm_genpd, &domain->genpd); + if (ret) + return dev_err_probe(dev, ret, "failed to add pm_genpd release callback\n"); /* * We use runtime PM to trigger power on/off of the upstream GPC * domain, as a strict hierarchical parent/child power domain @@ -290,39 +318,47 @@ static int imx93_blk_ctrl_probe(struct platform_device *pdev) bc->onecell_data.domains[i] = &domain->genpd; } - pm_runtime_enable(dev); - - ret = of_genpd_add_provider_onecell(dev->of_node, &bc->onecell_data); - if (ret) { - dev_err_probe(dev, ret, "failed to add power domain provider\n"); - goto cleanup_pds; - } + for (i = 0; i < bc_data->num_domains; i++) { + struct imx93_blk_ctrl_domain *domain = &bc->domains[i]; + const struct imx93_blk_ctrl_domain_data *data = domain->data; + struct imx93_blk_ctrl_subdomain_link *link; - dev_set_drvdata(dev, bc); + if (bc_data->skip_mask & BIT(i) || + data->parent == BLK_CTRL_NO_PARENT) + continue; - return 0; + link = devm_kzalloc(dev, sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; -cleanup_pds: - for (i--; i >= 0; i--) - pm_genpd_remove(&bc->domains[i].genpd); + link->parent = &bc->domains[data->parent].genpd; + link->subdomain = &domain->genpd; - return ret; -} + ret = pm_genpd_add_subdomain(&bc->domains[data->parent].genpd, + &domain->genpd); + if (ret) + return dev_err_probe(dev, ret, "failed to add subdomain %s\n", + domain->genpd.name); -static void imx93_blk_ctrl_remove(struct platform_device *pdev) -{ - struct imx93_blk_ctrl *bc = dev_get_drvdata(&pdev->dev); - int i; + ret = devm_add_action_or_reset(dev, imx93_release_subdomain, link); + if (ret) + return dev_err_probe(dev, ret, + "failed to add subdomain release callback\n"); + } - of_genpd_del_provider(pdev->dev.of_node); + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "failed to enable pm-runtime\n"); - pm_runtime_disable(&pdev->dev); + ret = of_genpd_add_provider_onecell(dev->of_node, &bc->onecell_data); + if (ret) + return dev_err_probe(dev, ret, "failed to add power domain provider\n"); - for (i = 0; i < bc->onecell_data.num_domains; i++) { - struct imx93_blk_ctrl_domain *domain = &bc->domains[i]; + ret = devm_add_action_or_reset(dev, imx93_release_genpd_provider, dev->of_node); + if (ret) + return dev_err_probe(dev, ret, "failed to add genpd_provider release callback\n"); - pm_genpd_remove(&domain->genpd); - } + return 0; } static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] = { @@ -330,8 +366,9 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .name = "mediablk-mipi-dsi", .clk_names = (const char *[]){ "dsi" }, .num_clks = 1, - .rst_mask = BIT(11) | BIT(12), - .clk_mask = BIT(11) | BIT(12), + .rst_mask = BIT(11), + .clk_mask = BIT(11), + .parent = IMX93_MEDIABLK_PD_MIPI_PHY, }, [IMX93_MEDIABLK_PD_MIPI_CSI] = { .name = "mediablk-mipi-csi", @@ -339,6 +376,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 2, .rst_mask = BIT(9) | BIT(10), .clk_mask = BIT(9) | BIT(10), + .parent = IMX93_MEDIABLK_PD_MIPI_PHY, }, [IMX93_MEDIABLK_PD_PXP] = { .name = "mediablk-pxp", @@ -346,6 +384,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 1, .rst_mask = BIT(7) | BIT(8), .clk_mask = BIT(7) | BIT(8), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 2, .qos = { { @@ -367,6 +406,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 2, .rst_mask = BIT(4) | BIT(5) | BIT(6), .clk_mask = BIT(4) | BIT(5) | BIT(6), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 1, .qos = { { @@ -383,6 +423,7 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] .num_clks = 1, .rst_mask = BIT(2) | BIT(3), .clk_mask = BIT(2) | BIT(3), + .parent = BLK_CTRL_NO_PARENT, .num_qos = 4, .qos = { { @@ -408,6 +449,14 @@ static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] } } }, + [IMX93_MEDIABLK_PD_MIPI_PHY] = { + .name = "mediablk-mipi-phy", + .clk_names = NULL, + .num_clks = 0, + .rst_mask = BIT(12), + .clk_mask = BIT(12), + .parent = BLK_CTRL_NO_PARENT, + }, }; static const struct regmap_range imx93_media_blk_ctl_yes_ranges[] = { @@ -457,7 +506,6 @@ MODULE_DEVICE_TABLE(of, imx93_blk_ctrl_of_match); static struct platform_driver imx93_blk_ctrl_driver = { .probe = imx93_blk_ctrl_probe, - .remove = imx93_blk_ctrl_remove, .driver = { .name = "imx93-blk-ctrl", .of_match_table = imx93_blk_ctrl_of_match, diff --git a/drivers/pmdomain/mediatek/Kconfig b/drivers/pmdomain/mediatek/Kconfig index 0e34a517ab7d5..0610b77641ea8 100644 --- a/drivers/pmdomain/mediatek/Kconfig +++ b/drivers/pmdomain/mediatek/Kconfig @@ -26,9 +26,12 @@ config MTK_SCPSYS_PM_DOMAINS Control Processor System (SCPSYS) has several power management related tasks in the system. +endmenu + config AIROHA_CPU_PM_DOMAIN tristate "Airoha CPU power domain" default ARCH_AIROHA + depends on ARCH_AIROHA || COMPILE_TEST depends on HAVE_ARM_SMCCC depends on PM select PM_GENERIC_DOMAINS @@ -37,5 +40,3 @@ config AIROHA_CPU_PM_DOMAIN CPU frequency and power is controlled by ATF with SMC command to set performance states. - -endmenu diff --git a/drivers/pmdomain/mediatek/mt8183-pm-domains.h b/drivers/pmdomain/mediatek/mt8183-pm-domains.h index 3742782a2702e..5e33b8628e852 100644 --- a/drivers/pmdomain/mediatek/mt8183-pm-domains.h +++ b/drivers/pmdomain/mediatek/mt8183-pm-domains.h @@ -47,7 +47,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .pwr_sta2nd_offs = 0x0184, .sram_pdn_bits = 0, .sram_pdn_ack_bits = 0, - .caps = MTK_SCPD_DOMAIN_SUPPLY, + .caps = MTK_SCPD_DOMAIN_SUPPLY | MTK_SCPD_KEEP_DEFAULT_OFF, }, [MT8183_POWER_DOMAIN_MFG] = { .name = "mfg", @@ -57,7 +57,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .pwr_sta2nd_offs = 0x0184, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), - .caps = MTK_SCPD_DOMAIN_SUPPLY, + .caps = MTK_SCPD_DOMAIN_SUPPLY | MTK_SCPD_KEEP_DEFAULT_OFF, }, [MT8183_POWER_DOMAIN_MFG_CORE0] = { .name = "mfg_core0", @@ -67,6 +67,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .pwr_sta2nd_offs = 0x0184, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), + .caps = MTK_SCPD_KEEP_DEFAULT_OFF, }, [MT8183_POWER_DOMAIN_MFG_CORE1] = { .name = "mfg_core1", @@ -76,6 +77,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .pwr_sta2nd_offs = 0x0184, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), + .caps = MTK_SCPD_KEEP_DEFAULT_OFF, }, [MT8183_POWER_DOMAIN_MFG_2D] = { .name = "mfg_2d", @@ -85,6 +87,7 @@ static const struct scpsys_domain_data scpsys_domain_data_mt8183[] = { .pwr_sta2nd_offs = 0x0184, .sram_pdn_bits = GENMASK(8, 8), .sram_pdn_ack_bits = GENMASK(12, 12), + .caps = MTK_SCPD_KEEP_DEFAULT_OFF, .bp_cfg = { BUS_PROT_WR(INFRA, MT8183_TOP_AXI_PROT_EN_1_MFG, diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index 1716d726b8cca..5b3752043d8a5 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -266,16 +266,8 @@ static int scpsys_ctl_pwrseq_on(struct scpsys_domain *pd) if (ret < 0) return ret; - if (pd->data->rtff_type == SCPSYS_RTFF_TYPE_PCIE_PHY) - regmap_set_bits(scpsys->base, pd->data->ctl_offs, PWR_RTFF_CLK_DIS); - regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_CLK_DIS_BIT); regmap_clear_bits(scpsys->base, pd->data->ctl_offs, PWR_ISO_BIT); - - /* Wait for RTFF HW to sync buck isolation state if this is PCIe PHY RTFF */ - if (pd->data->rtff_type == SCPSYS_RTFF_TYPE_PCIE_PHY) - udelay(5); - regmap_set_bits(scpsys->base, pd->data->ctl_offs, PWR_RST_B_BIT); /* @@ -774,12 +766,15 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s node = of_find_node_with_property(np, "mediatek,infracfg"); if (node) { regmap[0] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg"); - of_node_put(node); num_regmaps++; - if (IS_ERR(regmap[0])) - return dev_err_probe(dev, PTR_ERR(regmap[0]), + if (IS_ERR(regmap[0])) { + ret = dev_err_probe(dev, PTR_ERR(regmap[0]), "%pOF: failed to get infracfg regmap\n", node); + of_node_put(node); + return ret; + } + of_node_put(node); } else { regmap[0] = NULL; } @@ -788,17 +783,22 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s node = of_find_node_with_property(np, "mediatek,smi"); if (node) { smi_np = of_parse_phandle(node, "mediatek,smi", 0); - of_node_put(node); - if (!smi_np) + if (!smi_np) { + of_node_put(node); return -ENODEV; + } regmap[1] = device_node_to_regmap(smi_np); num_regmaps++; of_node_put(smi_np); - if (IS_ERR(regmap[1])) - return dev_err_probe(dev, PTR_ERR(regmap[1]), + if (IS_ERR(regmap[1])) { + ret = dev_err_probe(dev, PTR_ERR(regmap[1]), "%pOF: failed to get SMI regmap\n", node); + of_node_put(node); + return ret; + } + of_node_put(node); } else { regmap[1] = NULL; } diff --git a/drivers/power/reset/sc27xx-poweroff.c b/drivers/power/reset/sc27xx-poweroff.c index 90287c31992c4..393bd1c33b73b 100644 --- a/drivers/power/reset/sc27xx-poweroff.c +++ b/drivers/power/reset/sc27xx-poweroff.c @@ -28,7 +28,7 @@ static struct regmap *regmap; * taking cpus down to avoid racing regmap or spi mutex lock when poweroff * system through PMIC. */ -static void sc27xx_poweroff_shutdown(void) +static void sc27xx_poweroff_shutdown(void *data) { #ifdef CONFIG_HOTPLUG_CPU int cpu; @@ -40,10 +40,14 @@ static void sc27xx_poweroff_shutdown(void) #endif } -static struct syscore_ops poweroff_syscore_ops = { +static const struct syscore_ops poweroff_syscore_ops = { .shutdown = sc27xx_poweroff_shutdown, }; +static struct syscore poweroff_syscore = { + .ops = &poweroff_syscore_ops, +}; + static void sc27xx_poweroff_do_poweroff(void) { /* Disable the external subsys connection's power firstly */ @@ -62,7 +66,7 @@ static int sc27xx_poweroff_probe(struct platform_device *pdev) return -ENODEV; pm_power_off = sc27xx_poweroff_do_poweroff; - register_syscore_ops(&poweroff_syscore_ops); + register_syscore(&poweroff_syscore); return 0; } diff --git a/drivers/power/supply/ab8500_fg.c b/drivers/power/supply/ab8500_fg.c index 9dd99722667ae..969a75f4c016d 100644 --- a/drivers/power/supply/ab8500_fg.c +++ b/drivers/power/supply/ab8500_fg.c @@ -3054,6 +3054,20 @@ static void ab8500_fg_unbind(struct device *dev, struct device *master, flush_workqueue(di->fg_wq); } +/* Disable, not cancel: works stay disabled so nothing can re-arm them. */ +static void ab8500_fg_destroy_workqueue(void *data) +{ + struct ab8500_fg *di = data; + + disable_work_sync(&di->fg_acc_cur_work); + disable_work_sync(&di->fg_work); + disable_delayed_work_sync(&di->fg_reinit_work); + disable_delayed_work_sync(&di->fg_low_bat_work); + disable_delayed_work_sync(&di->fg_check_hw_failure_work); + disable_delayed_work_sync(&di->fg_periodic_work); + destroy_workqueue(di->fg_wq); +} + static const struct component_ops ab8500_fg_component_ops = { .bind = ab8500_fg_bind, .unbind = ab8500_fg_unbind, @@ -3155,6 +3169,11 @@ static int ab8500_fg_probe(struct platform_device *pdev) return PTR_ERR(di->fg_psy); } + /* Registered after fg_psy, before the IRQs: devm frees IRQ -> workqueue -> fg_psy. */ + ret = devm_add_action_or_reset(dev, ab8500_fg_destroy_workqueue, di); + if (ret) + return ret; + di->fg_samples = SEC_TO_SAMPLE(di->bm->fg_params->init_timer); /* @@ -3167,22 +3186,16 @@ static int ab8500_fg_probe(struct platform_device *pdev) /* Register primary interrupt handlers */ for (i = 0; i < ARRAY_SIZE(ab8500_fg_irq); i++) { irq = platform_get_irq_byname(pdev, ab8500_fg_irq[i].name); - if (irq < 0) { - destroy_workqueue(di->fg_wq); + if (irq < 0) return irq; - } ret = devm_request_threaded_irq(dev, irq, NULL, ab8500_fg_irq[i].isr, IRQF_SHARED | IRQF_NO_SUSPEND | IRQF_ONESHOT, ab8500_fg_irq[i].name, di); - if (ret != 0) { - dev_err(dev, "failed to request %s IRQ %d: %d\n", - ab8500_fg_irq[i].name, irq, ret); - destroy_workqueue(di->fg_wq); + if (ret != 0) return ret; - } dev_dbg(dev, "Requested %s IRQ %d: %d\n", ab8500_fg_irq[i].name, irq, ret); } @@ -3196,7 +3209,6 @@ static int ab8500_fg_probe(struct platform_device *pdev) ret = ab8500_fg_sysfs_init(di); if (ret) { dev_err(dev, "failed to create sysfs entry\n"); - destroy_workqueue(di->fg_wq); return ret; } @@ -3204,7 +3216,6 @@ static int ab8500_fg_probe(struct platform_device *pdev) if (ret) { dev_err(dev, "failed to create FG psy\n"); ab8500_fg_sysfs_exit(di); - destroy_workqueue(di->fg_wq); return ret; } @@ -3224,7 +3235,6 @@ static void ab8500_fg_remove(struct platform_device *pdev) { struct ab8500_fg *di = platform_get_drvdata(pdev); - destroy_workqueue(di->fg_wq); component_del(&pdev->dev, &ab8500_fg_component_ops); list_del(&di->node); ab8500_fg_sysfs_exit(di); diff --git a/drivers/power/supply/bd99954-charger.h b/drivers/power/supply/bd99954-charger.h index f588979253830..d053ac901274c 100644 --- a/drivers/power/supply/bd99954-charger.h +++ b/drivers/power/supply/bd99954-charger.h @@ -279,17 +279,7 @@ enum bd9995x_fields { F_VBUS_EXTID, F_VBUS_IDRDET, F_VBUS_INDO, - F_VBUS_UCDSWEN, - F_VBUS_RREF_EN, - F_VBUS_DPPU_EN, - F_VBUS_DPREF_EN, - F_VBUS_DMREF_EN, - F_VBUS_DPDET_EN, - F_VBUS_DMDET_EN, - F_VBUS_DPSINK_EN, - F_VBUS_DMSINK_EN, - F_VBUS_DP_BUFF_EN, - F_VBUS_DM_BUFF_EN, + F_VBUS_EXTCLKENBL, F_VBUS_PLSTESTEN, F_VBUS_UCDSWEN_TSTENB, @@ -626,17 +616,6 @@ static const struct reg_field bd9995x_reg_fields[] = { [F_VBUS_EXTID] = REG_FIELD(VBUS_IDD_STATUS, 5, 5), [F_VBUS_IDRDET] = REG_FIELD(VBUS_IDD_STATUS, 4, 4), [F_VBUS_INDO] = REG_FIELD(VBUS_IDD_STATUS, 0, 3), - [F_VBUS_UCDSWEN] = REG_FIELD(VCC_UCD_FCTRL_SET, 10, 10), - [F_VBUS_RREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 9, 9), - [F_VBUS_DPPU_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 8, 8), - [F_VBUS_DPREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 7, 7), - [F_VBUS_DMREF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 6, 6), - [F_VBUS_DPDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 5, 5), - [F_VBUS_DMDET_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 4, 4), - [F_VBUS_DPSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 3, 3), - [F_VBUS_DMSINK_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 2, 2), - [F_VBUS_DP_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 1, 1), - [F_VBUS_DM_BUFF_EN] = REG_FIELD(VCC_UCD_FCTRL_SET, 0, 0), [F_VBUS_EXTCLKENBL] = REG_FIELD(VBUS_UCD_FCTRL_EN, 15, 15), [F_VBUS_PLSTESTEN] = REG_FIELD(VBUS_UCD_FCTRL_EN, 14, 14), diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c index 766eecb356941..097b51271ba92 100644 --- a/drivers/power/supply/bq24257_charger.c +++ b/drivers/power/supply/bq24257_charger.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -1003,10 +1004,6 @@ static int bq24257_probe(struct i2c_client *client) if (bq->info->chip == BQ24250) bq->iilimit_autoset_enable = false; - if (bq->iilimit_autoset_enable) - INIT_DELAYED_WORK(&bq->iilimit_setup_work, - bq24257_iilimit_setup_work); - /* * The BQ24250 doesn't have a dedicated Power Good (PG) pin so let's * not probe for it and instead use a SW-based approach to determine @@ -1047,6 +1044,14 @@ static int bq24257_probe(struct i2c_client *client) return ret; } + if (bq->iilimit_autoset_enable) { + ret = devm_delayed_work_autocancel(dev, + &bq->iilimit_setup_work, + bq24257_iilimit_setup_work); + if (ret) + return ret; + } + ret = devm_request_threaded_irq(dev, client->irq, NULL, bq24257_irq_handler_thread, IRQF_TRIGGER_FALLING | @@ -1064,9 +1069,6 @@ static void bq24257_remove(struct i2c_client *client) { struct bq24257_device *bq = i2c_get_clientdata(client); - if (bq->iilimit_autoset_enable) - cancel_delayed_work_sync(&bq->iilimit_setup_work); - bq24257_field_write(bq, F_RESET, 1); /* reset to defaults */ } diff --git a/drivers/power/supply/bq256xx_charger.c b/drivers/power/supply/bq256xx_charger.c index d3de4f8b80db1..8cdd45ddd05b7 100644 --- a/drivers/power/supply/bq256xx_charger.c +++ b/drivers/power/supply/bq256xx_charger.c @@ -897,6 +897,8 @@ static void bq256xx_charger_reset(void *data) if (!IS_ERR_OR_NULL(bq->usb3_phy)) usb_unregister_notifier(bq->usb3_phy, &bq->usb_nb); + + cancel_work_sync(&bq->usb_work); } static int bq256xx_set_charger_property(struct power_supply *psy, @@ -1722,24 +1724,12 @@ static int bq256xx_probe(struct i2c_client *client) return ret; } - ret = devm_add_action_or_reset(dev, bq256xx_charger_reset, bq); - if (ret) - return ret; + INIT_WORK(&bq->usb_work, bq256xx_usb_work); + bq->usb_nb.notifier_call = bq256xx_usb_notifier; /* OTG reporting */ bq->usb2_phy = devm_usb_get_phy(dev, USB_PHY_TYPE_USB2); - if (!IS_ERR_OR_NULL(bq->usb2_phy)) { - INIT_WORK(&bq->usb_work, bq256xx_usb_work); - bq->usb_nb.notifier_call = bq256xx_usb_notifier; - usb_register_notifier(bq->usb2_phy, &bq->usb_nb); - } - bq->usb3_phy = devm_usb_get_phy(dev, USB_PHY_TYPE_USB3); - if (!IS_ERR_OR_NULL(bq->usb3_phy)) { - INIT_WORK(&bq->usb_work, bq256xx_usb_work); - bq->usb_nb.notifier_call = bq256xx_usb_notifier; - usb_register_notifier(bq->usb3_phy, &bq->usb_nb); - } ret = bq256xx_power_supply_init(bq, &psy_cfg, dev); if (ret) { @@ -1747,6 +1737,17 @@ static int bq256xx_probe(struct i2c_client *client) return ret; } + /* Register after the power supplies so devm runs it first. */ + ret = devm_add_action_or_reset(dev, bq256xx_charger_reset, bq); + if (ret) + return ret; + + if (!IS_ERR_OR_NULL(bq->usb2_phy)) + usb_register_notifier(bq->usb2_phy, &bq->usb_nb); + + if (!IS_ERR_OR_NULL(bq->usb3_phy)) + usb_register_notifier(bq->usb3_phy, &bq->usb_nb); + if (client->irq) { ret = devm_request_threaded_irq(dev, client->irq, NULL, bq256xx_irq_handler_thread, diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c index 868e86e1749b7..305318b21e5c9 100644 --- a/drivers/power/supply/bq25890_charger.c +++ b/drivers/power/supply/bq25890_charger.c @@ -320,7 +320,7 @@ static const u32 bq25890_tspct_tbl[] = { 145, 140, 130, 120, 115, 110, 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 0, -10, -20, -30, -40, -60, -70, -80, - -90, -10, -120, -140, -150, -170, -190, -210, + -90, -100, -120, -140, -150, -170, -190, -210, }; #define BQ25890_TSPCT_TBL_SIZE ARRAY_SIZE(bq25890_tspct_tbl) @@ -1389,6 +1389,14 @@ static int bq25890_fw_read_u32_props(struct bq25890_device *bq) return 0; } +static void bq25890_release_secondary_chrg(void *data) +{ + struct bq25890_device *bq = data; + + power_supply_put(bq->secondary_chrg); + bq->secondary_chrg = NULL; +} + static int bq25890_fw_probe(struct bq25890_device *bq) { int ret; @@ -1401,6 +1409,10 @@ static int bq25890_fw_probe(struct bq25890_device *bq) bq->secondary_chrg = power_supply_get_by_name(str); if (!bq->secondary_chrg) return -EPROBE_DEFER; + + ret = devm_add_action_or_reset(bq->dev, bq25890_release_secondary_chrg, bq); + if (ret) + return ret; } /* Optional, left at 0 if property is not present */ diff --git a/drivers/power/supply/bq27xxx_battery.c b/drivers/power/supply/bq27xxx_battery.c index 45f0e39b8c2dd..01c9ec1116fa0 100644 --- a/drivers/power/supply/bq27xxx_battery.c +++ b/drivers/power/supply/bq27xxx_battery.c @@ -330,7 +330,7 @@ static u8 [BQ27XXX_REG_FLAGS] = 0x0a, [BQ27XXX_REG_TTE] = 0x16, [BQ27XXX_REG_TTF] = INVALID_REG_ADDR, - [BQ27XXX_REG_TTES] = 0x1c, + [BQ27XXX_REG_TTES] = 0x1a, [BQ27XXX_REG_TTECP] = INVALID_REG_ADDR, [BQ27XXX_REG_NAC] = 0x0c, [BQ27XXX_REG_RC] = 0x10, @@ -495,7 +495,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, @@ -516,7 +516,7 @@ static u8 [BQ27XXX_REG_RC] = 0x10, [BQ27XXX_REG_FCC] = 0x12, [BQ27XXX_REG_CYCT] = 0x2a, - [BQ27XXX_REG_AE] = 0x22, + [BQ27XXX_REG_AE] = INVALID_REG_ADDR, [BQ27XXX_REG_SOC] = 0x2c, [BQ27XXX_REG_DCAP] = 0x3c, [BQ27XXX_REG_AP] = 0x22, diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c index c3644018b6bb4..70af1a4ce1102 100644 --- a/drivers/power/supply/charger-manager.c +++ b/drivers/power/supply/charger-manager.c @@ -1018,6 +1018,29 @@ static int charger_extcon_init(struct charger_manager *cm, return 0; } +static int charger_manager_get_regulators(struct charger_manager *cm) +{ + struct charger_desc *desc = cm->desc; + struct charger_regulator *charger; + int i, ret; + + for (i = 0; i < desc->num_charger_regulators; i++) { + charger = &desc->charger_regulators[i]; + charger->consumer = regulator_get(cm->dev, + charger->regulator_name); + if (IS_ERR(charger->consumer)) { + dev_err(cm->dev, "Cannot find charger(%s)\n", + charger->regulator_name); + ret = PTR_ERR(charger->consumer); + while (i-- > 0) + regulator_put(desc->charger_regulators[i].consumer); + return ret; + } + charger->cm = cm; + } + return 0; +} + /** * charger_manager_register_extcon - Register extcon device to receive state * of charger cable. @@ -1040,15 +1063,6 @@ static int charger_manager_register_extcon(struct charger_manager *cm) for (i = 0; i < desc->num_charger_regulators; i++) { charger = &desc->charger_regulators[i]; - charger->consumer = regulator_get(cm->dev, - charger->regulator_name); - if (IS_ERR(charger->consumer)) { - dev_err(cm->dev, "Cannot find charger(%s)\n", - charger->regulator_name); - return PTR_ERR(charger->consumer); - } - charger->cm = cm; - for (j = 0; j < charger->num_cables; j++) { struct charger_cable *cable = &charger->cables[j]; @@ -1584,13 +1598,23 @@ static int charger_manager_probe(struct platform_device *pdev) } psy_cfg.attr_grp = desc->sysfs_groups; + /* + * Acquire charger regulators before exposing the sysfs entries, so + * userspace cannot reach externally_control before the regulators + * (and charger->cm) are available. Mirrors the order in remove(). + */ + ret = charger_manager_get_regulators(cm); + if (ret < 0) + return ret; + cm->charger_psy = power_supply_register(&pdev->dev, &cm->charger_psy_desc, &psy_cfg); if (IS_ERR(cm->charger_psy)) { dev_err(&pdev->dev, "Cannot register charger-manager with name \"%s\"\n", cm->charger_psy_desc.name); - return PTR_ERR(cm->charger_psy); + ret = PTR_ERR(cm->charger_psy); + goto err_regulator; } /* Register extcon device for charger cable */ @@ -1624,11 +1648,11 @@ static int charger_manager_probe(struct platform_device *pdev) return 0; err_reg_extcon: + power_supply_unregister(cm->charger_psy); +err_regulator: for (i = 0; i < desc->num_charger_regulators; i++) regulator_put(desc->charger_regulators[i].consumer); - power_supply_unregister(cm->charger_psy); - return ret; } @@ -1646,12 +1670,12 @@ static void charger_manager_remove(struct platform_device *pdev) cancel_work_sync(&setup_polling); cancel_delayed_work_sync(&cm_monitor_work); - for (i = 0 ; i < desc->num_charger_regulators ; i++) - regulator_put(desc->charger_regulators[i].consumer); + try_charger_enable(cm, false); power_supply_unregister(cm->charger_psy); - try_charger_enable(cm, false); + for (i = 0 ; i < desc->num_charger_regulators ; i++) + regulator_put(desc->charger_regulators[i].consumer); } static const struct platform_device_id charger_manager_id[] = { diff --git a/drivers/power/supply/cros_usbpd-charger.c b/drivers/power/supply/cros_usbpd-charger.c index 47d3f58aa15ce..93f61da8d5962 100644 --- a/drivers/power/supply/cros_usbpd-charger.c +++ b/drivers/power/supply/cros_usbpd-charger.c @@ -125,6 +125,11 @@ static int cros_usbpd_charger_get_num_ports(struct charger_data *charger) if (ret < 0) return ret; + if (resp.port_count > EC_USB_PD_MAX_PORTS) { + dev_warn(charger->dev, "Charge port count out of bounds\n"); + return EC_USB_PD_MAX_PORTS; + } + return resp.port_count; } @@ -138,6 +143,11 @@ static int cros_usbpd_charger_get_usbpd_num_ports(struct charger_data *charger) if (ret < 0) return ret; + if (resp.num_ports > EC_USB_PD_MAX_PORTS) { + dev_warn(charger->dev, "USB PD port count out of bounds\n"); + return EC_USB_PD_MAX_PORTS; + } + return resp.num_ports; } @@ -589,10 +599,13 @@ static int cros_usbpd_charger_probe(struct platform_device *pd) /* * Sanity checks on the number of ports: - * there should be at most 1 dedicated port + * there should be at most 1 dedicated port, and the count must + * not exceed the maximum number of supported ports + * (EC_USB_PD_MAX_PORTS). */ if (charger->num_charger_ports < charger->num_usbpd_ports || - charger->num_charger_ports > (charger->num_usbpd_ports + 1)) { + charger->num_charger_ports > (charger->num_usbpd_ports + 1) || + charger->num_charger_ports > EC_USB_PD_MAX_PORTS) { dev_err(dev, "Unexpected number of charge port count\n"); ret = -EPROTO; goto fail_nowarn; diff --git a/drivers/power/supply/isp1704_charger.c b/drivers/power/supply/isp1704_charger.c index 237912a922724..e329321d06dbd 100644 --- a/drivers/power/supply/isp1704_charger.c +++ b/drivers/power/supply/isp1704_charger.c @@ -482,6 +482,7 @@ static void isp1704_charger_remove(struct platform_device *pdev) struct isp1704_charger *isp = platform_get_drvdata(pdev); usb_unregister_notifier(isp->phy, &isp->nb); + cancel_work_sync(&isp->work); power_supply_unregister(isp->psy); isp1704_charger_set_power(isp, 0); } diff --git a/drivers/power/supply/lp8727_charger.c b/drivers/power/supply/lp8727_charger.c index 4186fcd375127..e81bb393340cb 100644 --- a/drivers/power/supply/lp8727_charger.c +++ b/drivers/power/supply/lp8727_charger.c @@ -280,10 +280,10 @@ static int lp8727_setup_irq(struct lp8727_chg *pchg) static void lp8727_release_irq(struct lp8727_chg *pchg) { - cancel_delayed_work_sync(&pchg->work); - if (pchg->irq) free_irq(pchg->irq, pchg); + + cancel_delayed_work_sync(&pchg->work); } static enum power_supply_property lp8727_charger_prop[] = { diff --git a/drivers/power/supply/lp8788-charger.c b/drivers/power/supply/lp8788-charger.c index f0a680c155c48..1bc2c150fb912 100644 --- a/drivers/power/supply/lp8788-charger.c +++ b/drivers/power/supply/lp8788-charger.c @@ -710,8 +710,8 @@ static void lp8788_charger_remove(struct platform_device *pdev) { struct lp8788_charger *pchg = platform_get_drvdata(pdev); - flush_work(&pchg->charger_work); lp8788_irq_unregister(pdev, pchg); + cancel_work_sync(&pchg->charger_work); } static struct platform_driver lp8788_charger_driver = { diff --git a/drivers/power/supply/max17040_battery.c b/drivers/power/supply/max17040_battery.c index 48453508688a4..0d95ff5cc8927 100644 --- a/drivers/power/supply/max17040_battery.c +++ b/drivers/power/supply/max17040_battery.c @@ -192,8 +192,11 @@ static int max17040_raw_vcell_to_uvolts(struct max17040_chip *chip, u16 vcell) static int max17040_get_vcell(struct max17040_chip *chip) { u32 vcell; + int ret; - regmap_read(chip->regmap, MAX17040_VCELL, &vcell); + ret = regmap_read(chip->regmap, MAX17040_VCELL, &vcell); + if (ret) + return ret; return max17040_raw_vcell_to_uvolts(chip, vcell); } @@ -201,8 +204,11 @@ static int max17040_get_vcell(struct max17040_chip *chip) static int max17040_get_soc(struct max17040_chip *chip) { u32 soc; + int ret; - regmap_read(chip->regmap, MAX17040_SOC, &soc); + ret = regmap_read(chip->regmap, MAX17040_SOC, &soc); + if (ret) + return ret; return soc >> (chip->quirk_double_soc ? 9 : 8); } @@ -261,7 +267,11 @@ static int max17040_get_of_data(struct max17040_chip *chip) static void max17040_check_changes(struct max17040_chip *chip) { - chip->soc = max17040_get_soc(chip); + int soc; + + soc = max17040_get_soc(chip); + if (soc >= 0) + chip->soc = soc; } static void max17040_queue_work(struct max17040_chip *chip) @@ -396,16 +406,26 @@ static int max17040_get_property(struct power_supply *psy, val->intval = max17040_get_online(chip); break; case POWER_SUPPLY_PROP_VOLTAGE_NOW: - val->intval = max17040_get_vcell(chip); + ret = max17040_get_vcell(chip); + if (ret < 0) + return ret; + val->intval = ret; break; case POWER_SUPPLY_PROP_CAPACITY: - val->intval = max17040_get_soc(chip); + ret = max17040_get_soc(chip); + if (ret < 0) + return ret; + val->intval = ret; break; case POWER_SUPPLY_PROP_CAPACITY_ALERT_MIN: val->intval = chip->low_soc_alert; break; case POWER_SUPPLY_PROP_STATUS: - power_supply_get_property_from_supplier(psy, psp, val); + ret = power_supply_get_property_from_supplier(psy, psp, val); + if (ret == -ENODEV) + val->intval = POWER_SUPPLY_STATUS_UNKNOWN; + else if (ret) + return ret; break; case POWER_SUPPLY_PROP_TEMP: if (!chip->channel_temp) @@ -454,16 +474,12 @@ static const struct power_supply_desc max17040_battery_desc = { static int max17040_probe(struct i2c_client *client) { const struct i2c_device_id *id = i2c_client_get_device_id(client); - struct i2c_adapter *adapter = client->adapter; struct power_supply_config psy_cfg = {}; struct max17040_chip *chip; enum chip_id chip_id; bool enable_irq = false; int ret; - if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE)) - return -EIO; - chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL); if (!chip) return -ENOMEM; @@ -564,7 +580,7 @@ static int max17040_suspend(struct device *dev) // disable soc alert to prevent wakeup max17040_set_soc_alert(chip, 0); else - cancel_delayed_work(&chip->work); + cancel_delayed_work_sync(&chip->work); if (client->irq && device_may_wakeup(dev)) enable_irq_wake(client->irq); diff --git a/drivers/power/supply/qcom_battmgr.c b/drivers/power/supply/qcom_battmgr.c index ff77dba29a3ef..7454ec0e99633 100644 --- a/drivers/power/supply/qcom_battmgr.c +++ b/drivers/power/supply/qcom_battmgr.c @@ -5,6 +5,7 @@ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include +#include #include #include #include @@ -1235,7 +1236,7 @@ static void qcom_battmgr_sc8280xp_strcpy(char *dest, const char *src) memcpy(dest, src + 1, len); dest[len] = '\0'; } else { - memcpy(dest, src, BATTMGR_STRING_LEN); + strscpy(dest, src, BATTMGR_STRING_LEN); } } @@ -1650,7 +1651,6 @@ static int qcom_battmgr_probe(struct auxiliary_device *adev, psy_cfg_supply.supplied_to = qcom_battmgr_battery; psy_cfg_supply.num_supplicants = 1; - INIT_WORK(&battmgr->enable_work, qcom_battmgr_enable_worker); mutex_init(&battmgr->lock); init_completion(&battmgr->ack); @@ -1713,6 +1713,11 @@ static int qcom_battmgr_probe(struct auxiliary_device *adev, "failed to register wireless charing power supply\n"); } + ret = devm_work_autocancel(dev, &battmgr->enable_work, + qcom_battmgr_enable_worker); + if (ret) + return ret; + battmgr->client = devm_pmic_glink_client_alloc(dev, PMIC_GLINK_OWNER_BATTMGR, qcom_battmgr_callback, qcom_battmgr_pdr_notify, diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c index 5130d2395e88f..710d510f9809f 100644 --- a/drivers/power/supply/rt9455_charger.c +++ b/drivers/power/supply/rt9455_charger.c @@ -1582,6 +1582,19 @@ static const struct regmap_config rt9455_regmap_config = { .cache_type = REGCACHE_MAPLE, }; +static void rt9455_cancel_all_delayed_works(void *data) +{ + struct rt9455_info *info = data; + + /* + * Both pwr_rdy_work and batt_presence_work can queue + * max_charging_time_work, so cancel them first. + */ + cancel_delayed_work_sync(&info->pwr_rdy_work); + cancel_delayed_work_sync(&info->batt_presence_work); + cancel_delayed_work_sync(&info->max_charging_time_work); +} + static int rt9455_probe(struct i2c_client *client) { struct i2c_adapter *adapter = client->adapter; @@ -1672,6 +1685,10 @@ static int rt9455_probe(struct i2c_client *client) goto put_usb_notifier; } + ret = devm_add_action_or_reset(dev, rt9455_cancel_all_delayed_works, info); + if (ret) + goto put_usb_notifier; + ret = devm_request_threaded_irq(dev, client->irq, NULL, rt9455_irq_handler_thread, IRQF_TRIGGER_LOW | IRQF_ONESHOT, @@ -1712,10 +1729,6 @@ static void rt9455_remove(struct i2c_client *client) if (info->nb.notifier_call) usb_unregister_notifier(info->usb_phy, &info->nb); #endif - - cancel_delayed_work_sync(&info->pwr_rdy_work); - cancel_delayed_work_sync(&info->max_charging_time_work); - cancel_delayed_work_sync(&info->batt_presence_work); } static const struct i2c_device_id rt9455_i2c_id_table[] = { diff --git a/drivers/power/supply/sbs-battery.c b/drivers/power/supply/sbs-battery.c index 43c48196c1674..3947429f6f819 100644 --- a/drivers/power/supply/sbs-battery.c +++ b/drivers/power/supply/sbs-battery.c @@ -217,6 +217,7 @@ struct sbs_info { u32 flags; int technology; char strings[NR_STRING_BUFFERS][I2C_SMBUS_BLOCK_MAX + 1]; + char serial[5]; }; static char *sbs_get_string_buf(struct sbs_info *chip, @@ -821,18 +822,18 @@ static int sbs_get_battery_capacity(struct i2c_client *client, return 0; } -static char sbs_serial[5]; static int sbs_get_battery_serial_number(struct i2c_client *client, union power_supply_propval *val) { + struct sbs_info *chip = i2c_get_clientdata(client); int ret; ret = sbs_read_word_data(client, sbs_data[REG_SERIAL_NUMBER].addr); if (ret < 0) return ret; - sprintf(sbs_serial, "%04x", ret); - val->strval = sbs_serial; + sprintf(chip->serial, "%04x", ret); + val->strval = chip->serial; return 0; } diff --git a/drivers/power/supply/sc2731_charger.c b/drivers/power/supply/sc2731_charger.c index 58b86fd787713..2b25e44da7978 100644 --- a/drivers/power/supply/sc2731_charger.c +++ b/drivers/power/supply/sc2731_charger.c @@ -466,6 +466,7 @@ static int sc2731_charger_probe(struct platform_device *pdev) mutex_init(&info->lock); info->dev = &pdev->dev; INIT_WORK(&info->work, sc2731_charger_work); + platform_set_drvdata(pdev, info); info->regmap = dev_get_regmap(pdev->dev.parent, NULL); if (!info->regmap) { @@ -516,6 +517,7 @@ static void sc2731_charger_remove(struct platform_device *pdev) struct sc2731_charger_info *info = platform_get_drvdata(pdev); usb_unregister_notifier(info->usb_phy, &info->usb_notify); + cancel_work_sync(&info->work); } static const struct of_device_id sc2731_charger_of_match[] = { diff --git a/drivers/power/supply/twl4030_charger.c b/drivers/power/supply/twl4030_charger.c index 04216b2bfb6c3..e5b4d72720de5 100644 --- a/drivers/power/supply/twl4030_charger.c +++ b/drivers/power/supply/twl4030_charger.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1002,8 +1003,15 @@ static int twl4030_bci_probe(struct platform_device *pdev) platform_set_drvdata(pdev, bci); - INIT_WORK(&bci->work, twl4030_bci_usb_work); - INIT_DELAYED_WORK(&bci->current_worker, twl4030_current_worker); + ret = devm_delayed_work_autocancel(&pdev->dev, &bci->current_worker, + twl4030_current_worker); + if (ret) + return ret; + + ret = devm_work_autocancel(&pdev->dev, &bci->work, + twl4030_bci_usb_work); + if (ret) + return ret; bci->channel_vac = devm_iio_channel_get(&pdev->dev, "vac"); if (IS_ERR(bci->channel_vac)) { diff --git a/drivers/power/supply/ucs1002_power.c b/drivers/power/supply/ucs1002_power.c index fe94435340de6..da49de82c0824 100644 --- a/drivers/power/supply/ucs1002_power.c +++ b/drivers/power/supply/ucs1002_power.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -641,7 +642,10 @@ static int ucs1002_probe(struct i2c_client *client) } info->health = POWER_SUPPLY_HEALTH_GOOD; - INIT_DELAYED_WORK(&info->health_poll, ucs1002_health_poll); + ret = devm_delayed_work_autocancel(dev, &info->health_poll, + ucs1002_health_poll); + if (ret) + return ret; if (irq_a_det > 0) { ret = devm_request_threaded_irq(dev, irq_a_det, NULL, diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index 57bebd07c7d0d..f08ac99f5e55e 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -2216,7 +2216,8 @@ struct rapl_package *rapl_add_package_cpuslocked(int id, struct rapl_if_priv *pr topology_physical_package_id(id) : topology_logical_die_id(id); if ((int)(rp->id) < 0) { pr_err("topology_logical_(package/die)_id() returned a negative value"); - return ERR_PTR(-EINVAL); + ret = -EINVAL; + goto err_free_package; } rp->lead_cpu = id; if (!rapl_msrs_are_pkg_scope() && topology_max_dies_per_package() > 1) diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 34c0bd1edd61a..ee2a5e8397871 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -314,7 +314,10 @@ static int intel_rapl_tpmi_probe(struct auxiliary_device *auxdev, goto err; } - rapl_package_add_pmu(trp->rp); + ret = rapl_package_add_pmu(trp->rp); + if (ret) + dev_info(&auxdev->dev, "Failed to add RAPL PMU for Package%d, %d\n", + info->package_id, ret); auxiliary_set_drvdata(auxdev, trp); diff --git a/drivers/pps/clients/pps-gpio.c b/drivers/pps/clients/pps-gpio.c index 65d17781ddb7d..1eba112afeb80 100644 --- a/drivers/pps/clients/pps-gpio.c +++ b/drivers/pps/clients/pps-gpio.c @@ -32,38 +32,37 @@ struct pps_gpio_device_data { struct gpio_desc *echo_pin; struct timer_list echo_timer; /* timer to reset echo active state */ bool assert_falling_edge; - bool capture_clear; unsigned int echo_active_ms; /* PPS echo active duration */ unsigned long echo_timeout; /* timer timeout value in jiffies */ + struct pps_event_time ts; /* timestamp captured in hardirq */ }; /* * Report the PPS event */ -static irqreturn_t pps_gpio_irq_handler(int irq, void *data) +/* + * Primary hardirq handler -- runs in hardirq context even on PREEMPT_RT. + * Only captures the timestamp; all other work is deferred to the thread. + */ +static irqreturn_t pps_gpio_irq_hardirq(int irq, void *data) { - const struct pps_gpio_device_data *info; - struct pps_event_time ts; - int rising_edge; - - /* Get the time stamp first */ - pps_get_ts(&ts); - - info = data; - - /* Small trick to bypass the check on edge's direction when capture_clear is unset */ - rising_edge = info->capture_clear ? - gpiod_get_value(info->gpio_pin) : !info->assert_falling_edge; - if ((rising_edge && !info->assert_falling_edge) || - (!rising_edge && info->assert_falling_edge)) - pps_event(info->pps, &ts, PPS_CAPTUREASSERT, data); - else if (info->capture_clear && - ((rising_edge && info->assert_falling_edge) || - (!rising_edge && !info->assert_falling_edge))) - pps_event(info->pps, &ts, PPS_CAPTURECLEAR, data); - else - dev_warn_ratelimited(&info->pps->dev, "IRQ did not trigger any PPS event\n"); + struct pps_gpio_device_data *info = data; + + pps_get_ts(&info->ts); + + return IRQ_WAKE_THREAD; +} + +/* + * Threaded handler -- processes the PPS event using the timestamp + * captured in hardirq context above. + */ +static irqreturn_t pps_gpio_irq_thread(int irq, void *data) +{ + struct pps_gpio_device_data *info = data; + + pps_event(info->pps, &info->ts, PPS_CAPTUREASSERT, data); return IRQ_HANDLED; } @@ -79,11 +78,6 @@ static void pps_gpio_echo(struct pps_device *pps, int event, void *data) if (pps->params.mode & PPS_ECHOASSERT) gpiod_set_value(info->echo_pin, 1); break; - - case PPS_CAPTURECLEAR: - if (pps->params.mode & PPS_ECHOCLEAR) - gpiod_set_value(info->echo_pin, 1); - break; } /* fire the timer */ @@ -117,9 +111,6 @@ static int pps_gpio_setup(struct device *dev) data->assert_falling_edge = device_property_read_bool(dev, "assert-falling-edge"); - data->capture_clear = - device_property_read_bool(dev, "capture-clear"); - data->echo_pin = devm_gpiod_get_optional(dev, "echo", GPIOD_OUT_LOW); if (IS_ERR(data->echo_pin)) return dev_err_probe(dev, PTR_ERR(data->echo_pin), @@ -148,15 +139,8 @@ static int pps_gpio_setup(struct device *dev) static unsigned long get_irqf_trigger_flags(const struct pps_gpio_device_data *data) { - unsigned long flags = data->assert_falling_edge ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING; - - if (data->capture_clear) { - flags |= ((flags & IRQF_TRIGGER_RISING) ? - IRQF_TRIGGER_FALLING : IRQF_TRIGGER_RISING); - } - - return flags; + return data->assert_falling_edge ? IRQF_TRIGGER_FALLING : + IRQF_TRIGGER_RISING; } static int pps_gpio_probe(struct platform_device *pdev) @@ -189,9 +173,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* initialize PPS specific parts of the bookkeeping data structure. */ data->info.mode = PPS_CAPTUREASSERT | PPS_OFFSETASSERT | PPS_ECHOASSERT | PPS_CANWAIT | PPS_TSFMT_TSPEC; - if (data->capture_clear) - data->info.mode |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR | - PPS_ECHOCLEAR; data->info.owner = THIS_MODULE; snprintf(data->info.name, PPS_MAX_NAME_LEN - 1, "%s.%d", pdev->name, pdev->id); @@ -203,8 +184,6 @@ static int pps_gpio_probe(struct platform_device *pdev) /* register PPS source */ pps_default_params = PPS_CAPTUREASSERT | PPS_OFFSETASSERT; - if (data->capture_clear) - pps_default_params |= PPS_CAPTURECLEAR | PPS_OFFSETCLEAR; data->pps = pps_register_source(&data->info, pps_default_params); if (IS_ERR(data->pps)) { dev_err(dev, "failed to register IRQ %d as PPS source\n", @@ -213,8 +192,10 @@ static int pps_gpio_probe(struct platform_device *pdev) } /* register IRQ interrupt handler */ - ret = request_irq(data->irq, pps_gpio_irq_handler, - get_irqf_trigger_flags(data), data->info.name, data); + ret = request_threaded_irq(data->irq, + pps_gpio_irq_hardirq, pps_gpio_irq_thread, + get_irqf_trigger_flags(data) | IRQF_ONESHOT, + data->info.name, data); if (ret) { pps_unregister_source(data->pps); dev_err(dev, "failed to acquire IRQ %d\n", data->irq); diff --git a/drivers/pps/pps.c b/drivers/pps/pps.c index 442d7e8f6fb95..7f3b23247dce9 100644 --- a/drivers/pps/pps.c +++ b/drivers/pps/pps.c @@ -63,13 +63,19 @@ static int pps_cdev_pps_fetch(struct pps_device *pps, struct pps_fdata *fdata) err = wait_event_interruptible(pps->queue, ev != pps->last_ev); else { + struct timespec64 ts; unsigned long ticks; dev_dbg(&pps->dev, "timeout %lld.%09d\n", (long long) fdata->timeout.sec, fdata->timeout.nsec); - ticks = fdata->timeout.sec * HZ; - ticks += fdata->timeout.nsec / (NSEC_PER_SEC / HZ); + + if (fdata->timeout.sec < 0) + return -ETIMEDOUT; + + ts.tv_sec = fdata->timeout.sec; + ts.tv_nsec = fdata->timeout.nsec; + ticks = timespec64_to_jiffies(&ts); if (ticks != 0) { err = wait_event_interruptible_timeout( diff --git a/drivers/ptp/ptp_netc.c b/drivers/ptp/ptp_netc.c index 94e952ee69902..19ca99d80e95b 100644 --- a/drivers/ptp/ptp_netc.c +++ b/drivers/ptp/ptp_netc.c @@ -440,7 +440,7 @@ static int net_timer_enable_perout(struct netc_timer *priv, } if (on) { - u64 period_ns, gclk_period, max_period, min_period; + u64 period_ns, gclk_period, min_period; struct timespec64 period, stime; u32 integral_period; int alarm_id; @@ -450,12 +450,12 @@ static int net_timer_enable_perout(struct netc_timer *priv, period_ns = timespec64_to_ns(&period); integral_period = netc_timer_get_integral_period(priv); - max_period = (u64)NETC_TMR_DEFAULT_FIPER + integral_period; gclk_period = netc_timer_get_gclk_period(priv); min_period = gclk_period * 4 + integral_period; - if (period_ns > max_period || period_ns < min_period) { - dev_err(dev, "The period range is %llu ~ %llu\n", - min_period, max_period); + if (period_ns > NETC_TMR_DEFAULT_FIPER || + period_ns < min_period) { + dev_err(dev, "The period range is %llu ~ %lu\n", + min_period, NETC_TMR_DEFAULT_FIPER); err = -EINVAL; goto unlock_spinlock; } @@ -482,6 +482,9 @@ static int net_timer_enable_perout(struct netc_timer *priv, netc_timer_enable_periodic_pulse(priv, channel); } else { + if (!pp->enabled) + goto unlock_spinlock; + netc_timer_disable_periodic_pulse(priv, channel); priv->fs_alarm_bitmap &= ~BIT(pp->alarm_id); memset(pp, 0, sizeof(*pp)); @@ -769,6 +772,7 @@ static void netc_timer_init(struct netc_timer *priv) TMR_CTRL_TE | TMR_CTRL_FS; netc_timer_wr(priv, NETC_TMR_CTRL, tmr_ctrl); netc_timer_wr(priv, NETC_TMR_PRSC, priv->oclk_prsc); + netc_timer_wr(priv, NETC_TMR_TEMASK, 0); /* Disable FIPER by default */ fiper_ctrl = netc_timer_rd(priv, NETC_TMR_FIPER_CTRL); @@ -779,6 +783,7 @@ static void netc_timer_init(struct netc_timer *priv) netc_timer_wr(priv, NETC_TMR_FIPER_CTRL, fiper_ctrl); netc_timer_wr(priv, NETC_TMR_ECTRL, NETC_TMR_DEFAULT_ETTF_THR); + netc_timer_offset_write(priv, 0); ktime_get_real_ts64(&now); ns = timespec64_to_ns(&now); netc_timer_cnt_write(priv, ns); @@ -900,6 +905,11 @@ static irqreturn_t netc_timer_isr(int irq, void *data) /* Clear interrupts status */ netc_timer_wr(priv, NETC_TMR_TEVENT, tmr_event); + if (!tmr_event) { + spin_unlock(&priv->lock); + return IRQ_NONE; + } + if (tmr_event & TMR_TEVENT_ALMEN(0)) netc_timer_alarm_write(priv, NETC_TMR_DEFAULT_ALARM, 0); @@ -935,7 +945,8 @@ static int netc_timer_init_msix_irq(struct netc_timer *priv) } priv->irq = pci_irq_vector(pdev, 0); - err = request_irq(priv->irq, netc_timer_isr, 0, priv->irq_name, priv); + err = request_irq(priv->irq, netc_timer_isr, IRQF_NO_AUTOEN, + priv->irq_name, priv); if (err) { dev_err(&pdev->dev, "request_irq() failed\n"); pci_free_irq_vectors(pdev); @@ -950,7 +961,6 @@ static void netc_timer_free_msix_irq(struct netc_timer *priv) { struct pci_dev *pdev = priv->pdev; - disable_irq(priv->irq); free_irq(priv->irq, priv); pci_free_irq_vectors(pdev); } @@ -1004,6 +1014,8 @@ static int netc_timer_probe(struct pci_dev *pdev, goto free_msix_irq; } + enable_irq(priv->irq); + return 0; free_msix_irq: @@ -1018,9 +1030,10 @@ static void netc_timer_remove(struct pci_dev *pdev) { struct netc_timer *priv = pci_get_drvdata(pdev); + disable_irq(priv->irq); + ptp_clock_unregister(priv->clock); netc_timer_wr(priv, NETC_TMR_TEMASK, 0); netc_timer_wr(priv, NETC_TMR_CTRL, 0); - ptp_clock_unregister(priv->clock); netc_timer_free_msix_irq(priv); netc_timer_pci_remove(pdev); } diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index a5c3632529862..249bb3877f673 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -1916,9 +1916,11 @@ ptp_ocp_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req, if (err) return err; + snprintf(buf, sizeof(buf), "%.*s", OCP_BOARD_ID_LEN, + (const char *)bp->board_id); err = devlink_info_version_fixed_put(req, DEVLINK_INFO_VERSION_GENERIC_BOARD_ID, - bp->board_id); + buf); if (err) return err; @@ -4866,6 +4868,7 @@ static struct pci_driver ptp_ocp_driver = { .id_table = ptp_ocp_pcidev_id, .probe = ptp_ocp_probe, .remove = ptp_ocp_remove, + .shutdown = ptp_ocp_remove, }; static int diff --git a/drivers/ptp/ptp_s390.c b/drivers/ptp/ptp_s390.c index 29618eb9bf442..02d624d89a0ab 100644 --- a/drivers/ptp/ptp_s390.c +++ b/drivers/ptp/ptp_s390.c @@ -107,6 +107,9 @@ static __init int ptp_s390_init(void) if (IS_ERR(ptp_stcke_clock)) return PTR_ERR(ptp_stcke_clock); + if (!test_facility(28) || !ptff_query(PTFF_QPT)) + return 0; + ptp_qpt_clock = ptp_clock_register(&ptp_s390_qpt_info, NULL); if (IS_ERR(ptp_qpt_clock)) { ptp_clock_unregister(ptp_stcke_clock); @@ -117,7 +120,8 @@ static __init int ptp_s390_init(void) static __exit void ptp_s390_exit(void) { - ptp_clock_unregister(ptp_qpt_clock); + if (ptp_qpt_clock) + ptp_clock_unregister(ptp_qpt_clock); ptp_clock_unregister(ptp_stcke_clock); } diff --git a/drivers/ptp/ptp_vmclock.c b/drivers/ptp/ptp_vmclock.c index cbbfc494680c7..8edc2880c609b 100644 --- a/drivers/ptp/ptp_vmclock.c +++ b/drivers/ptp/ptp_vmclock.c @@ -365,6 +365,18 @@ static int vmclock_miscdev_mmap(struct file *fp, struct vm_area_struct *vma) if ((vma->vm_flags & (VM_READ|VM_WRITE)) != VM_READ) return -EROFS; + /* + * Restrict the read-only mapping so it cannot be upgraded to + * writable later with mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + + /* + * Restrict the read-only mapping so it cannot be upgraded to + * writable later with mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + if (vma->vm_end - vma->vm_start != PAGE_SIZE || vma->vm_pgoff) return -EINVAL; diff --git a/drivers/pwm/pwm-mediatek.c b/drivers/pwm/pwm-mediatek.c index 4291072a13a7f..e87feda2e7831 100644 --- a/drivers/pwm/pwm-mediatek.c +++ b/drivers/pwm/pwm-mediatek.c @@ -458,7 +458,7 @@ static const struct pwm_mediatek_of_data mt7623_pwm_data = { static const struct pwm_mediatek_of_data mt7628_pwm_data = { .num_pwms = 4, - .pwm45_fixup = true, + .pwm45_fixup = false, .chanreg_base = 0x10, .chanreg_width = 0x40, }; diff --git a/drivers/rapidio/devices/rio_mport_cdev.c b/drivers/rapidio/devices/rio_mport_cdev.c index 995cfeca972b1..c3fa45b26c1be 100644 --- a/drivers/rapidio/devices/rio_mport_cdev.c +++ b/drivers/rapidio/devices/rio_mport_cdev.c @@ -564,9 +564,13 @@ static void dma_req_free(struct kref *ref) } if (req->map) { - mutex_lock(&req->map->md->buf_mutex); - kref_put(&req->map->ref, mport_release_mapping); - mutex_unlock(&req->map->md->buf_mutex); + struct rio_mport_mapping *map = req->map; + struct mport_dev *md = map->md; + + mutex_lock(&md->buf_mutex); + req->map = NULL; + kref_put(&map->ref, mport_release_mapping); + mutex_unlock(&md->buf_mutex); } kref_put(&priv->dma_ref, mport_release_dma); diff --git a/drivers/rapidio/rio-scan.c b/drivers/rapidio/rio-scan.c index dcd6619a4b027..3cc25d0534513 100644 --- a/drivers/rapidio/rio-scan.c +++ b/drivers/rapidio/rio-scan.c @@ -874,6 +874,7 @@ static struct rio_net *rio_scan_alloc_net(struct rio_mport *mport, net->dev.release = rio_scan_release_dev; if (rio_add_net(net)) { put_device(&net->dev); + mport->net = NULL; net = NULL; } } diff --git a/drivers/ras/amd/atl/core.c b/drivers/ras/amd/atl/core.c index 4197e10993aca..81ebde74cb107 100644 --- a/drivers/ras/amd/atl/core.c +++ b/drivers/ras/amd/atl/core.c @@ -190,7 +190,6 @@ static const struct x86_cpu_id amd_atl_cpuids[] = { X86_MATCH_FEATURE(X86_FEATURE_ZEN, NULL), { } }; -MODULE_DEVICE_TABLE(x86cpu, amd_atl_cpuids); static int __init amd_atl_init(void) { diff --git a/drivers/regulator/adp5055-regulator.c b/drivers/regulator/adp5055-regulator.c index 4b004a6b2f84e..5b83f36351a2a 100644 --- a/drivers/regulator/adp5055-regulator.c +++ b/drivers/regulator/adp5055-regulator.c @@ -225,7 +225,7 @@ static int adp5055_of_parse_cb(struct device_node *np, adp5055->dvs_limit_upper[id] = pval; if (adp5055->dvs_limit_upper[id] > 192000 || adp5055->dvs_limit_upper[id] < 12000) - return dev_err_probe(config->dev, adp5055->dvs_limit_upper[id], + return dev_err_probe(config->dev, -EINVAL, "Out of range - dvs-limit-upper-microvolt value."); ret = of_property_read_u32(np, "adi,dvs-limit-lower-microvolt", &pval); diff --git a/drivers/regulator/as3722-regulator.c b/drivers/regulator/as3722-regulator.c index da378bfdba406..85e2aaa1a06bd 100644 --- a/drivers/regulator/as3722-regulator.c +++ b/drivers/regulator/as3722-regulator.c @@ -600,7 +600,6 @@ static int as3722_get_regulator_dt_data(struct platform_device *pdev, ret = of_regulator_match(&pdev->dev, np, as3722_regulator_matches, ARRAY_SIZE(as3722_regulator_matches)); - of_node_put(np); if (ret < 0) { dev_err(&pdev->dev, "Parsing of regulator node failed: %d\n", ret); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 019606bc36b9c..eae756c82c262 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -27,6 +27,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -1204,10 +1205,98 @@ static int machine_constraints_voltage(struct regulator_dev *rdev, { const struct regulator_ops *ops = rdev->desc->ops; int ret; + bool apply_uV; + + /* + * Decide up front, from the constraints as handed to us, whether + * apply_uV needs to run below. The clamping pass right after this + * may rewrite constraints->min_uV/max_uV (e.g. the fixed-voltage + * autoconfigure case), and we don't want that to change whether + * apply_uV fires. + */ + apply_uV = rdev->constraints->apply_uV && + rdev->constraints->min_uV && rdev->constraints->max_uV; + + /* + * Constrain machine-level voltage specs to fit the actual range + * supported by this regulator before apply_uV (below) tries to + * force hardware to a value from that range: otherwise apply_uV + * can target a constraint value that doesn't correspond to any + * real voltage selector and fail registration outright, even + * though the clamping pass would have narrowed it to a value + * the regulator can actually hit. + */ + if (ops->list_voltage && rdev->desc->n_voltages) { + int count = rdev->desc->n_voltages; + int i; + int min_uV = INT_MAX; + int max_uV = INT_MIN; + int cmin = constraints->min_uV; + int cmax = constraints->max_uV; + + /* it's safe to autoconfigure fixed-voltage supplies + * and the constraints are used by list_voltage. + */ + if (count == 1 && !cmin) { + cmin = 1; + cmax = INT_MAX; + constraints->min_uV = cmin; + constraints->max_uV = cmax; + } + + /* voltage constraints are optional */ + if ((cmin == 0) && (cmax == 0)) { + /* nothing more to do */ + + /* else require explicit machine-level constraints */ + } else if (cmin <= 0 || cmax <= 0 || cmax < cmin) { + rdev_err(rdev, "invalid voltage constraints\n"); + return -EINVAL; + + /* no need to loop voltages if range is continuous */ + } else if (rdev->desc->continuous_voltage_range) { + /* nothing more to do */ + + } else { + /* initial: [cmin..cmax] valid, [min_uV..max_uV] not */ + for (i = 0; i < count; i++) { + int value; + + value = ops->list_voltage(rdev, i); + if (value <= 0) + continue; + + /* maybe adjust [min_uV..max_uV] */ + if (value >= cmin && value < min_uV) + min_uV = value; + if (value <= cmax && value > max_uV) + max_uV = value; + } + + /* final: [min_uV..max_uV] valid iff constraints valid */ + if (max_uV < min_uV) { + rdev_err(rdev, + "unsupportable voltage constraints %u-%uuV\n", + min_uV, max_uV); + return -EINVAL; + } + + /* use regulator's subset of machine constraints */ + if (constraints->min_uV < min_uV) { + rdev_dbg(rdev, "override min_uV, %d -> %d\n", + constraints->min_uV, min_uV); + constraints->min_uV = min_uV; + } + if (constraints->max_uV > max_uV) { + rdev_dbg(rdev, "override max_uV, %d -> %d\n", + constraints->max_uV, max_uV); + constraints->max_uV = max_uV; + } + } + } /* do we need to apply the constraint voltage */ - if (rdev->constraints->apply_uV && - rdev->constraints->min_uV && rdev->constraints->max_uV) { + if (apply_uV) { int target_min, target_max; int current_uV = regulator_get_voltage_rdev(rdev); @@ -1262,77 +1351,6 @@ static int machine_constraints_voltage(struct regulator_dev *rdev, } } - /* constrain machine-level voltage specs to fit - * the actual range supported by this regulator. - */ - if (ops->list_voltage && rdev->desc->n_voltages) { - int count = rdev->desc->n_voltages; - int i; - int min_uV = INT_MAX; - int max_uV = INT_MIN; - int cmin = constraints->min_uV; - int cmax = constraints->max_uV; - - /* it's safe to autoconfigure fixed-voltage supplies - * and the constraints are used by list_voltage. - */ - if (count == 1 && !cmin) { - cmin = 1; - cmax = INT_MAX; - constraints->min_uV = cmin; - constraints->max_uV = cmax; - } - - /* voltage constraints are optional */ - if ((cmin == 0) && (cmax == 0)) - return 0; - - /* else require explicit machine-level constraints */ - if (cmin <= 0 || cmax <= 0 || cmax < cmin) { - rdev_err(rdev, "invalid voltage constraints\n"); - return -EINVAL; - } - - /* no need to loop voltages if range is continuous */ - if (rdev->desc->continuous_voltage_range) - return 0; - - /* initial: [cmin..cmax] valid, [min_uV..max_uV] not */ - for (i = 0; i < count; i++) { - int value; - - value = ops->list_voltage(rdev, i); - if (value <= 0) - continue; - - /* maybe adjust [min_uV..max_uV] */ - if (value >= cmin && value < min_uV) - min_uV = value; - if (value <= cmax && value > max_uV) - max_uV = value; - } - - /* final: [min_uV..max_uV] valid iff constraints valid */ - if (max_uV < min_uV) { - rdev_err(rdev, - "unsupportable voltage constraints %u-%uuV\n", - min_uV, max_uV); - return -EINVAL; - } - - /* use regulator's subset of machine constraints */ - if (constraints->min_uV < min_uV) { - rdev_dbg(rdev, "override min_uV, %d -> %d\n", - constraints->min_uV, min_uV); - constraints->min_uV = min_uV; - } - if (constraints->max_uV > max_uV) { - rdev_dbg(rdev, "override max_uV, %d -> %d\n", - constraints->max_uV, max_uV); - constraints->max_uV = max_uV; - } - } - return 0; } @@ -6604,8 +6622,9 @@ static int __init regulator_init_complete(void) * we'd only do this on systems that need it, and a kernel * command line option might be useful. */ - schedule_delayed_work(®ulator_init_complete_work, - msecs_to_jiffies(30000)); + queue_delayed_work(system_freezable_wq, + ®ulator_init_complete_work, + msecs_to_jiffies(30000)); return 0; } diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c index ef161eb0ca277..e4c429000b670 100644 --- a/drivers/regulator/da9121-regulator.c +++ b/drivers/regulator/da9121-regulator.c @@ -1166,15 +1166,15 @@ static void da9121_i2c_remove(struct i2c_client *i2c) } static const struct i2c_device_id da9121_i2c_id[] = { - {"da9121", DA9121_TYPE_DA9121_DA9130}, - {"da9130", DA9121_TYPE_DA9121_DA9130}, - {"da9217", DA9121_TYPE_DA9217}, - {"da9122", DA9121_TYPE_DA9122_DA9131}, - {"da9131", DA9121_TYPE_DA9122_DA9131}, - {"da9220", DA9121_TYPE_DA9220_DA9132}, - {"da9132", DA9121_TYPE_DA9220_DA9132}, - {"da9141", DA9121_TYPE_DA9141}, - {"da9142", DA9121_TYPE_DA9142}, + {"da9121", DA9121_SUBTYPE_DA9121}, + {"da9130", DA9121_SUBTYPE_DA9130}, + {"da9217", DA9121_SUBTYPE_DA9217}, + {"da9122", DA9121_SUBTYPE_DA9122}, + {"da9131", DA9121_SUBTYPE_DA9131}, + {"da9220", DA9121_SUBTYPE_DA9220}, + {"da9132", DA9121_SUBTYPE_DA9132}, + {"da9141", DA9121_SUBTYPE_DA9141}, + {"da9142", DA9121_SUBTYPE_DA9142}, {}, }; MODULE_DEVICE_TABLE(i2c, da9121_i2c_id); diff --git a/drivers/regulator/db8500-prcmu.c b/drivers/regulator/db8500-prcmu.c index 1ec2e1348891d..751fe36580faf 100644 --- a/drivers/regulator/db8500-prcmu.c +++ b/drivers/regulator/db8500-prcmu.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include #include @@ -93,13 +93,13 @@ static int enable_epod(u16 epod_id, bool ramret) if (ramret) { if (!epod_on[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); if (ret < 0) return ret; } epod_ramret[epod_id] = true; } else { - ret = prcmu_set_epod(epod_id, EPOD_STATE_ON); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_ON); if (ret < 0) return ret; epod_on[epod_id] = true; @@ -114,18 +114,18 @@ static int disable_epod(u16 epod_id, bool ramret) if (ramret) { if (!epod_on[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF); if (ret < 0) return ret; } epod_ramret[epod_id] = false; } else { if (epod_ramret[epod_id]) { - ret = prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_RAMRET); if (ret < 0) return ret; } else { - ret = prcmu_set_epod(epod_id, EPOD_STATE_OFF); + ret = db8500_prcmu_set_epod(epod_id, EPOD_STATE_OFF); if (ret < 0) return ret; } diff --git a/drivers/regulator/max8998.c b/drivers/regulator/max8998.c index 254a77887f661..52ab2208675b8 100644 --- a/drivers/regulator/max8998.c +++ b/drivers/regulator/max8998.c @@ -582,7 +582,6 @@ static int max8998_pmic_dt_parse_pdata(struct max8998_dev *iodev, } pdata->num_regulators = rdata - pdata->regulators; - of_node_put(reg_np); of_node_put(regulators_np); pdata->buck_voltage_lock = of_property_read_bool(pmic_np, "max8998,pmic-buck-voltage-lock"); diff --git a/drivers/regulator/mt6358-regulator.c b/drivers/regulator/mt6358-regulator.c index e4745f616cea3..44bef0bdfdbe3 100644 --- a/drivers/regulator/mt6358-regulator.c +++ b/drivers/regulator/mt6358-regulator.c @@ -492,7 +492,7 @@ static const struct regulator_ops mt6358_volt_fixed_ops = { .list_voltage = regulator_list_voltage_linear, .map_voltage = regulator_map_voltage_linear, .set_voltage_sel = regulator_set_voltage_sel_regmap, - .get_voltage_sel = mt6358_get_buck_voltage_sel, + .get_voltage_sel = regulator_get_voltage_sel_regmap, .set_voltage_time_sel = regulator_set_voltage_time_sel, .enable = regulator_enable_regmap, .disable = regulator_disable_regmap, diff --git a/drivers/regulator/qcom-refgen-regulator.c b/drivers/regulator/qcom-refgen-regulator.c index 299ac3c8c3bc3..6a3795469927e 100644 --- a/drivers/regulator/qcom-refgen-regulator.c +++ b/drivers/regulator/qcom-refgen-regulator.c @@ -66,7 +66,7 @@ static const struct regulator_desc sdm845_refgen_desc = { .enable_time = 5, .name = "refgen", .owner = THIS_MODULE, - .type = REGULATOR_VOLTAGE, + .type = REGULATOR_CURRENT, .ops = &(const struct regulator_ops) { .enable = qcom_sdm845_refgen_enable, .disable = qcom_sdm845_refgen_disable, @@ -82,7 +82,7 @@ static const struct regulator_desc sm8250_refgen_desc = { .enable_time = 5, .name = "refgen", .owner = THIS_MODULE, - .type = REGULATOR_VOLTAGE, + .type = REGULATOR_CURRENT, .ops = &(const struct regulator_ops) { .enable = regulator_enable_regmap, .disable = regulator_disable_regmap, diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c index 109f0aae09b1d..933eb0c28fa65 100644 --- a/drivers/regulator/qcom-rpmh-regulator.c +++ b/drivers/regulator/qcom-rpmh-regulator.c @@ -77,6 +77,7 @@ enum rpmh_regulator_type { * @hpm_min_load_uA: Minimum load current in microamps that requires * high power mode (HPM) operation. This is used * for LDO hardware type regulators only. + * @pmic_bypass_mode: The PMIC bypass mode value. * @pmic_mode_map: Array indexed by regulator framework mode * containing PMIC hardware modes. Must be large * enough to index all framework modes supported @@ -91,6 +92,7 @@ struct rpmh_vreg_hw_data { int n_linear_ranges; int n_voltages; int hpm_min_load_uA; + int pmic_bypass_mode; const int *pmic_mode_map; unsigned int (*of_map_mode)(unsigned int mode); }; @@ -277,7 +279,7 @@ static int rpmh_regulator_vrm_set_mode_bypass(struct rpmh_vreg *vreg, return pmic_mode; if (bypassed) - cmd.data = PMIC4_BOB_MODE_PASS; + cmd.data = vreg->hw_data->pmic_bypass_mode; else cmd.data = pmic_mode; @@ -692,6 +694,7 @@ static const struct rpmh_vreg_hw_data pmic4_bob = { }, .n_linear_ranges = 1, .n_voltages = 84, + .pmic_bypass_mode = PMIC4_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic4_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; @@ -900,6 +903,7 @@ static const struct rpmh_vreg_hw_data pmic5_bob = { }, .n_linear_ranges = 1, .n_voltages = 32, + .pmic_bypass_mode = PMIC5_BOB_MODE_PASS, .pmic_mode_map = pmic_mode_map_pmic5_bob, .of_map_mode = rpmh_regulator_pmic4_bob_of_map_mode, }; diff --git a/drivers/regulator/tps6594-regulator.c b/drivers/regulator/tps6594-regulator.c index 645e83462c645..31a5218d55105 100644 --- a/drivers/regulator/tps6594-regulator.c +++ b/drivers/regulator/tps6594-regulator.c @@ -669,13 +669,20 @@ static int tps6594_regulator_probe(struct platform_device *pdev) * buck_configured to avoid creating bucks for every buck in multiphase */ for (multi = 0; multi < desc->num_multi_phase_regs; multi++) { + struct device_node *np_parent; + multi_regs = &desc->multi_phase_regs[multi]; np = of_find_node_by_name(tps->dev->of_node, multi_regs->supply_name); - npname = of_node_full_name(np); - np_pmic_parent = of_get_parent(of_get_parent(np)); - if (of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name)) + if (!np) continue; - if (strcmp(npname, multi_regs->supply_name) == 0) { + + npname = of_node_full_name(np); + np_parent = of_get_parent(np); + np_pmic_parent = of_get_parent(np_parent); + + if (np_pmic_parent && + !of_node_cmp(of_node_full_name(np_pmic_parent), tps->dev->of_node->full_name) && + strcmp(npname, multi_regs->supply_name) == 0) { switch (multi) { case MULTI_BUCK12: buck_multi[0] = true; @@ -706,6 +713,10 @@ static int tps6594_regulator_probe(struct platform_device *pdev) break; } } + + of_node_put(np_pmic_parent); + of_node_put(np_parent); + of_node_put(np); } reg_irq_nb = desc->num_irq_types * (desc->num_buck_regs + desc->num_ldo_regs); diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index eb8908ea3bab0..b3613802730c1 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -35,6 +35,7 @@ struct mtk_scp *scp_get(struct platform_device *pdev) struct device *dev = &pdev->dev; struct device_node *scp_node; struct platform_device *scp_pdev; + struct mtk_scp *scp; scp_node = of_parse_phandle(dev->of_node, "mediatek,scp", 0); if (!scp_node) { @@ -50,7 +51,13 @@ struct mtk_scp *scp_get(struct platform_device *pdev) return NULL; } - return platform_get_drvdata(scp_pdev); + scp = platform_get_drvdata(scp_pdev); + if (!scp) { + put_device(&scp_pdev->dev); + return NULL; + } + + return scp; } EXPORT_SYMBOL_GPL(scp_get); diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c index 5bbbe36d09640..506678d1633e1 100644 --- a/drivers/remoteproc/qcom_common.c +++ b/drivers/remoteproc/qcom_common.c @@ -242,8 +242,10 @@ void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink, return; glink->ssr_name = kstrdup_const(ssr_name, GFP_KERNEL); - if (!glink->ssr_name) + if (!glink->ssr_name) { + of_node_put(glink->node); return; + } glink->dev = dev; glink->subdev.start = glink_subdev_start; diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index e98b7e03162c7..b7d472afd5a9b 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -355,6 +355,7 @@ static int adsp_map_carveout(struct rproc *rproc) return ret; sid = args.args[0] & SID_MASK_DEFAULT; + of_node_put(args.np); /* Add SID configuration for ADSP Firmware to SMMU */ iova = adsp->mem_phys | (sid << 32); diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 158bcd6cc85c5..8c59eac9bd973 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -248,15 +248,14 @@ static int qcom_pas_load(struct rproc *rproc, const struct firmware *fw) ret = qcom_mdt_load_no_init(pas->dev, pas->dtb_firmware, pas->dtb_firmware_name, pas->dtb_mem_region, pas->dtb_mem_phys, pas->dtb_mem_size, &pas->dtb_mem_reloc); - if (ret) - goto release_dtb_metadata; + if (ret) { + qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); + goto release_dtb_firmware; + } } return 0; -release_dtb_metadata: - qcom_scm_pas_metadata_release(&pas->dtb_pas_metadata); - release_dtb_firmware: release_firmware(pas->dtb_firmware); diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 8256721005289..bbc781e41a1b8 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -1016,60 +1016,55 @@ static rproc_handle_resource_t rproc_loading_handlers[RSC_LAST] = { [RSC_VDEV] = rproc_handle_vdev, }; -/* handle firmware resource entries before booting the remote processor */ -static int rproc_handle_resources(struct rproc *rproc, - rproc_handle_resource_t handlers[RSC_LAST]) +struct rproc_rsc_cb_data { + struct rproc *rproc; + rproc_handle_resource_t *handlers; +}; + +static int rproc_handle_rsc_entry(u32 type, void *rsc, int offset, + int avail, void *data) { + struct rproc_rsc_cb_data *d = data; + struct rproc *rproc = d->rproc; struct device *dev = &rproc->dev; rproc_handle_resource_t handler; - int ret = 0, i; - - if (!rproc->table_ptr) - return 0; - - for (i = 0; i < rproc->table_ptr->num; i++) { - int offset = rproc->table_ptr->offset[i]; - struct fw_rsc_hdr *hdr = (void *)rproc->table_ptr + offset; - int avail = rproc->table_sz - offset - sizeof(*hdr); - void *rsc = (void *)hdr + sizeof(*hdr); + int ret; - /* make sure table isn't truncated */ - if (avail < 0) { - dev_err(dev, "rsc table is truncated\n"); - return -EINVAL; - } + dev_dbg(dev, "rsc: type %d\n", type); - dev_dbg(dev, "rsc: type %d\n", hdr->type); + if (type >= RSC_VENDOR_START && type <= RSC_VENDOR_END) { + ret = rproc_handle_rsc(rproc, type, rsc, offset, avail); + if (ret == RSC_HANDLED) + return 0; + if (ret < 0) + return ret; + dev_warn(dev, "unsupported vendor resource %d\n", type); + return 0; + } - if (hdr->type >= RSC_VENDOR_START && - hdr->type <= RSC_VENDOR_END) { - ret = rproc_handle_rsc(rproc, hdr->type, rsc, - offset + sizeof(*hdr), avail); - if (ret == RSC_HANDLED) - continue; - else if (ret < 0) - break; + if (type >= RSC_LAST) { + dev_warn(dev, "unsupported resource %d\n", type); + return 0; + } - dev_warn(dev, "unsupported vendor resource %d\n", - hdr->type); - continue; - } + handler = d->handlers[type]; + if (!handler) + return 0; - if (hdr->type >= RSC_LAST) { - dev_warn(dev, "unsupported resource %d\n", hdr->type); - continue; - } + return handler(rproc, rsc, offset, avail); +} - handler = handlers[hdr->type]; - if (!handler) - continue; +/* handle firmware resource entries before booting the remote processor */ +static int rproc_handle_resources(struct rproc *rproc, + rproc_handle_resource_t handlers[RSC_LAST]) +{ + struct rproc_rsc_cb_data d = { .rproc = rproc, .handlers = handlers }; - ret = handler(rproc, rsc, offset + sizeof(*hdr), avail); - if (ret) - break; - } + if (!rproc->table_ptr) + return 0; - return ret; + return rsc_table_for_each_entry(rproc->table_ptr, rproc->table_sz, + &rproc->dev, rproc_handle_rsc_entry, &d); } static int rproc_prepare_subdevices(struct rproc *rproc) @@ -1678,18 +1673,21 @@ static void rproc_auto_boot_callback(const struct firmware *fw, void *context) release_firmware(fw); } +static void rproc_attach_work(struct work_struct *work) +{ + struct rproc *rproc = container_of(work, struct rproc, attach_work); + + rproc_boot(rproc); +} + static int rproc_trigger_auto_boot(struct rproc *rproc) { int ret; - /* - * Since the remote processor is in a detached state, it has already - * been booted by another entity. As such there is no point in waiting - * for a firmware image to be loaded, we can simply initiate the process - * of attaching to it immediately. - */ - if (rproc->state == RPROC_DETACHED) - return rproc_boot(rproc); + if (rproc->state == RPROC_DETACHED) { + schedule_work(&rproc->attach_work); + return 0; + } /* * We're initiating an asynchronous firmware loading, so we can @@ -1838,6 +1836,11 @@ int rproc_trigger_recovery(struct rproc *rproc) if (ret) return ret; + if (READ_ONCE(rproc->deleting)) { + ret = -ENODEV; + goto unlock_mutex; + } + /* State could have changed before we got the mutex */ if (rproc->state != RPROC_CRASHED) goto unlock_mutex; @@ -1870,6 +1873,11 @@ static void rproc_crash_handler_work(struct work_struct *work) mutex_lock(&rproc->lock); + if (READ_ONCE(rproc->deleting)) { + mutex_unlock(&rproc->lock); + goto out; + } + if (rproc->state == RPROC_CRASHED) { /* handle only the first crash detected */ mutex_unlock(&rproc->lock); @@ -1925,9 +1933,9 @@ int rproc_boot(struct rproc *rproc) return ret; } - if (rproc->state == RPROC_DELETED) { + if (READ_ONCE(rproc->deleting)) { ret = -ENODEV; - dev_err(dev, "can't boot deleted rproc %s\n", rproc->name); + dev_err(dev, "can't boot deleting rproc %s\n", rproc->name); goto unlock_mutex; } @@ -1989,7 +1997,8 @@ EXPORT_SYMBOL(rproc_boot); int rproc_shutdown(struct rproc *rproc) { struct device *dev = &rproc->dev; - int ret = 0; + bool crashed; + int ret; ret = mutex_lock_interruptible(&rproc->lock); if (ret) { @@ -1998,16 +2007,18 @@ int rproc_shutdown(struct rproc *rproc) } if (rproc->state != RPROC_RUNNING && - rproc->state != RPROC_ATTACHED) { + rproc->state != RPROC_ATTACHED && + rproc->state != RPROC_CRASHED) { ret = -EINVAL; goto out; } + crashed = rproc->state == RPROC_CRASHED; /* if the remote proc is still needed, bail out */ if (!atomic_dec_and_test(&rproc->power)) goto out; - ret = rproc_stop(rproc, false); + ret = rproc_stop(rproc, crashed); if (ret) { atomic_inc(&rproc->power); goto out; @@ -2516,7 +2527,9 @@ struct rproc *rproc_alloc(struct device *dev, const char *name, INIT_LIST_HEAD(&rproc->subdevs); INIT_LIST_HEAD(&rproc->dump_segments); + INIT_WORK(&rproc->attach_work, rproc_attach_work); INIT_WORK(&rproc->crash_handler, rproc_crash_handler_work); + spin_lock_init(&rproc->crash_handler_lock); rproc->state = RPROC_OFFLINE; @@ -2580,16 +2593,21 @@ EXPORT_SYMBOL(rproc_put); */ int rproc_del(struct rproc *rproc) { + unsigned long flags; + if (!rproc) return -EINVAL; + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + WRITE_ONCE(rproc->deleting, true); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + + if (cancel_work_sync(&rproc->crash_handler)) + pm_relax(rproc->dev.parent); + /* TODO: make sure this works with rproc->power > 1 */ rproc_shutdown(rproc); - mutex_lock(&rproc->lock); - rproc->state = RPROC_DELETED; - mutex_unlock(&rproc->lock); - rproc_delete_debug_dir(rproc); /* the rproc is downref'ed as soon as it's removed from the klist */ @@ -2701,18 +2719,26 @@ EXPORT_SYMBOL(rproc_get_by_child); */ void rproc_report_crash(struct rproc *rproc, enum rproc_crash_type type) { + unsigned long flags; + if (!rproc) { pr_err("NULL rproc pointer\n"); return; } + spin_lock_irqsave(&rproc->crash_handler_lock, flags); + if (READ_ONCE(rproc->deleting)) { + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); + return; + } + /* Prevent suspend while the remoteproc is being recovered */ pm_stay_awake(rproc->dev.parent); + queue_work(rproc_recovery_wq, &rproc->crash_handler); + spin_unlock_irqrestore(&rproc->crash_handler_lock, flags); dev_err(&rproc->dev, "crash detected in %s: type %s\n", rproc->name, rproc_crash_to_string(type)); - - queue_work(rproc_recovery_wq, &rproc->crash_handler); } EXPORT_SYMBOL(rproc_report_crash); diff --git a/drivers/remoteproc/remoteproc_sysfs.c b/drivers/remoteproc/remoteproc_sysfs.c index 138e752c5e4e0..925b0cdbe5778 100644 --- a/drivers/remoteproc/remoteproc_sysfs.c +++ b/drivers/remoteproc/remoteproc_sysfs.c @@ -168,7 +168,6 @@ static const char * const rproc_state_string[] = { [RPROC_SUSPENDED] = "suspended", [RPROC_RUNNING] = "running", [RPROC_CRASHED] = "crashed", - [RPROC_DELETED] = "deleted", [RPROC_ATTACHED] = "attached", [RPROC_DETACHED] = "detached", [RPROC_LAST] = "invalid", diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index 5ef586fd37bca..a238b45bf2f2c 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -959,16 +959,6 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev) goto free_rproc; } - /* - * If firmware is already available in the memory then move rproc state - * to DETACHED. Firmware can be preloaded via debugger or by any other - * agent (processors) in the system. - * If firmware isn't available in the memory and resource table isn't - * found, then rproc state remains OFFLINE. - */ - if (!zynqmp_r5_get_rsc_table_va(r5_core)) - r5_rproc->state = RPROC_DETACHED; - r5_core->rproc = r5_rproc; return r5_core; @@ -1221,6 +1211,7 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, { struct device *dev = cluster->dev; struct zynqmp_r5_core *r5_core; + u32 payload[PAYLOAD_ARG_CNT]; int ret = -EINVAL, i; r5_core = cluster->r5_cores[0]; @@ -1266,6 +1257,45 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster, ret = zynqmp_r5_get_sram_banks(r5_core); if (ret) return ret; + + /* + * It is possible that firmware is loaded into the memory, but + * RPU (remote) is not running. In such case, RPU state will be + * moved to RPROC_DETACHED wrongfully. To avoid it first make + * sure RPU is power-on and out of reset before parsing for the + * resource table. + */ + ret = zynqmp_pm_feature(PM_GET_NODE_STATUS); + if (ret < PM_API_VERSION_2) + ret = -EOPNOTSUPP; + else + ret = zynqmp_pm_invoke_fn(PM_GET_NODE_STATUS, payload, 1, + r5_core->pm_domain_id); + if (ret) { + dev_warn(r5_core->dev, + "failed to get rpu node status, err %d\n", ret); + continue; + } + + /* + * If RPU state is power on and out of reset i.e. running, then + * assign RPROC_DETACHED state. If the RPU is not out of reset + * then do not attempt to attach to the remote processor. + */ + if (payload[1] == PM_NODE_RUNNING) { + /* + * Not all the firmware that is running on the remote + * core is expected to have the resource table. The + * firmware might not use RPMsg at all, and in that case + * resource table becomes irrelevant. However, we still + * need to make sure that running core is not reported + * as offline. so do not decide remote core state based + * on the resource table availability + */ + if (zynqmp_r5_get_rsc_table_va(r5_core)) + dev_dbg(r5_core->dev, "rsc tbl not found\n"); + r5_core->rproc->state = RPROC_DETACHED; + } } return 0; diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index 833ff9cb8afe6..7605fde9e6789 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1395,26 +1395,28 @@ static int qcom_glink_announce_create(struct rpmsg_device *rpdev) return 0; } -static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) +static void qcom_glink_remove_rpmsg_device(struct qcom_glink *glink, struct glink_channel *channel) { - struct glink_channel *channel = to_glink_channel(ept); - struct qcom_glink *glink = channel->glink; struct rpmsg_channel_info chinfo; - unsigned long flags; - - spin_lock_irqsave(&channel->recv_lock, flags); - channel->ept.cb = NULL; - spin_unlock_irqrestore(&channel->recv_lock, flags); - /* Decouple the potential rpdev from the channel */ if (channel->rpdev) { strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); chinfo.src = RPMSG_ADDR_ANY; chinfo.dst = RPMSG_ADDR_ANY; - rpmsg_unregister_device(glink->dev, &chinfo); } channel->rpdev = NULL; +} + +static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct glink_channel *channel = to_glink_channel(ept); + struct qcom_glink *glink = channel->glink; + unsigned long flags; + + spin_lock_irqsave(&channel->recv_lock, flags); + channel->ept.cb = NULL; + spin_unlock_irqrestore(&channel->recv_lock, flags); qcom_glink_send_close_req(glink, channel); } @@ -1704,7 +1706,6 @@ static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) { - struct rpmsg_channel_info chinfo; struct glink_channel *channel; unsigned long flags; @@ -1720,14 +1721,7 @@ static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) /* cancel pending rx_done work */ cancel_work_sync(&channel->intent_work); - if (channel->rpdev) { - strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); - chinfo.src = RPMSG_ADDR_ANY; - chinfo.dst = RPMSG_ADDR_ANY; - - rpmsg_unregister_device(glink->dev, &chinfo); - } - channel->rpdev = NULL; + qcom_glink_remove_rpmsg_device(glink, channel); qcom_glink_send_close_ack(glink, channel); @@ -1741,7 +1735,6 @@ static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) { - struct rpmsg_channel_info chinfo; struct glink_channel *channel; unsigned long flags; @@ -1763,14 +1756,7 @@ static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) spin_unlock_irqrestore(&glink->idr_lock, flags); /* Decouple the potential rpdev from the channel */ - if (channel->rpdev) { - strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); - chinfo.src = RPMSG_ADDR_ANY; - chinfo.dst = RPMSG_ADDR_ANY; - - rpmsg_unregister_device(glink->dev, &chinfo); - } - channel->rpdev = NULL; + qcom_glink_remove_rpmsg_device(glink, channel); kref_put(&channel->refcount, qcom_glink_channel_release); } diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c index 7a982c60a8ddc..17c885f4ed501 100644 --- a/drivers/rpmsg/qcom_glink_smem.c +++ b/drivers/rpmsg/qcom_glink_smem.c @@ -103,6 +103,13 @@ static void glink_smem_rx_peek(struct qcom_glink_pipe *np, if (tail >= pipe->native.length) tail -= pipe->native.length; + /* + * Order the availability (head) read in glink_smem_rx_avail() + * against the FIFO payload read below, so APPS never consumes + * stale data the remote has not yet published. + */ + rmb(); + len = min_t(size_t, count, pipe->native.length - tail); if (len) memcpy_fromio(data, pipe->fifo + tail, len); diff --git a/drivers/rtc/rtc-aspeed.c b/drivers/rtc/rtc-aspeed.c index 0d0053b52f9b9..8f5b440f8c0a9 100644 --- a/drivers/rtc/rtc-aspeed.c +++ b/drivers/rtc/rtc-aspeed.c @@ -111,6 +111,7 @@ static const struct of_device_id aspeed_rtc_match[] = { { .compatible = "aspeed,ast2400-rtc", }, { .compatible = "aspeed,ast2500-rtc", }, { .compatible = "aspeed,ast2600-rtc", }, + { .compatible = "aspeed,ast2700-rtc", }, {} }; MODULE_DEVICE_TABLE(of, aspeed_rtc_match); diff --git a/drivers/rtc/rtc-bq32k.c b/drivers/rtc/rtc-bq32k.c index 7ad34539be4d2..245e5e45d6a16 100644 --- a/drivers/rtc/rtc-bq32k.c +++ b/drivers/rtc/rtc-bq32k.c @@ -16,6 +16,7 @@ #include #include #include +#include #define BQ32K_SECONDS 0x00 /* Seconds register address */ #define BQ32K_SECONDS_MASK 0x7F /* Mask over seconds value */ @@ -89,9 +90,17 @@ static int bq32k_write(struct device *dev, void *data, uint8_t off, uint8_t len) static int bq32k_rtc_read_time(struct device *dev, struct rtc_time *tm) { + struct i2c_client *client = to_i2c_client(dev); struct bq32k_regs regs; int error; + /* + * When the device doesn't have the interrupt connected, prevent + * userpace from polling the RTC registers too frequently. + */ + if (client->irq <= 0) + usleep_range(2000, 2500); + error = bq32k_read(dev, ®s, 0, sizeof(regs)); if (error) return error; diff --git a/drivers/rtc/rtc-cmos.c b/drivers/rtc/rtc-cmos.c index f8068c717fb85..b537f40654b72 100644 --- a/drivers/rtc/rtc-cmos.c +++ b/drivers/rtc/rtc-cmos.c @@ -27,6 +27,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#include #include #include #include @@ -1482,6 +1483,14 @@ static __init void cmos_of_init(struct platform_device *pdev) #else static inline void cmos_of_init(struct platform_device *pdev) {} #endif + +#ifdef CONFIG_ACPI +static const struct acpi_device_id acpi_cmos_rtc_ids[] = { + ACPI_CMOS_RTC_IDS +}; +MODULE_DEVICE_TABLE(acpi, acpi_cmos_rtc_ids); +#endif + /*----------------------------------------------------------------*/ /* Platform setup should have set up an RTC device, when PNP is @@ -1499,9 +1508,18 @@ static int __init cmos_platform_probe(struct platform_device *pdev) resource = platform_get_resource(pdev, IORESOURCE_IO, 0); else resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); - irq = platform_get_irq(pdev, 0); - if (irq < 0) + irq = platform_get_irq_optional(pdev, 0); + if (irq < 0) { irq = -1; +#ifdef CONFIG_X86 + /* + * On some x86 systems, the IRQ is not defined, but it should + * always be safe to hardcode it on systems with a legacy PIC. + */ + if (nr_legacy_irqs()) + irq = RTC_IRQ; +#endif + } return cmos_do_probe(&pdev->dev, resource, irq); } @@ -1536,6 +1554,7 @@ static struct platform_driver cmos_platform_driver = { .name = driver_name, .pm = &cmos_pm_ops, .of_match_table = of_match_ptr(of_cmos_match), + .acpi_match_table = ACPI_PTR(acpi_cmos_rtc_ids), } }; diff --git a/drivers/rtc/rtc-gamecube.c b/drivers/rtc/rtc-gamecube.c index 045d5d45ab4b0..889028cecf4a5 100644 --- a/drivers/rtc/rtc-gamecube.c +++ b/drivers/rtc/rtc-gamecube.c @@ -355,9 +355,7 @@ static int gamecube_rtc_probe(struct platform_device *pdev) rtc->range_max = U32_MAX; rtc->ops = &gamecube_rtc_ops; - devm_rtc_register_device(rtc); - - return 0; + return devm_rtc_register_device(rtc); } static const struct of_device_id gamecube_rtc_of_match[] = { diff --git a/drivers/rtc/rtc-mv.c b/drivers/rtc/rtc-mv.c index c27ad626d09fc..f88976fd6d5da 100644 --- a/drivers/rtc/rtc-mv.c +++ b/drivers/rtc/rtc-mv.c @@ -301,6 +301,28 @@ static const struct of_device_id rtc_mv_of_match_table[] = { MODULE_DEVICE_TABLE(of, rtc_mv_of_match_table); #endif +#ifdef CONFIG_PM_SLEEP +static int mv_rtc_suspend(struct device *dev) +{ + struct rtc_plat_data *pdata = dev_get_drvdata(dev); + + if (device_may_wakeup(dev) && pdata->irq >= 0) + enable_irq_wake(pdata->irq); + return 0; +} + +static int mv_rtc_resume(struct device *dev) +{ + struct rtc_plat_data *pdata = dev_get_drvdata(dev); + + if (device_may_wakeup(dev) && pdata->irq >= 0) + disable_irq_wake(pdata->irq); + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(mv_rtc_pm_ops, mv_rtc_suspend, mv_rtc_resume); + /* * mv_rtc_remove() lives in .exit.text. For drivers registered via * module_platform_driver_probe() this is ok because they cannot get unbound at @@ -312,6 +334,7 @@ static struct platform_driver mv_rtc_driver __refdata = { .driver = { .name = "rtc-mv", .of_match_table = of_match_ptr(rtc_mv_of_match_table), + .pm = &mv_rtc_pm_ops, }, }; diff --git a/drivers/rtc/rtc-pcf85363.c b/drivers/rtc/rtc-pcf85363.c index 540042b9eec8f..ccc7834e57599 100644 --- a/drivers/rtc/rtc-pcf85363.c +++ b/drivers/rtc/rtc-pcf85363.c @@ -426,8 +426,8 @@ static int pcf85363_probe(struct i2c_client *client) err = pcf85363_load_capacitance(pcf85363, client->dev.of_node); if (err < 0) - dev_warn(&client->dev, "failed to set xtal load capacitance: %d", - err); + return dev_err_probe(&client->dev, err, + "failed to set xtal load capacitance\n"); pcf85363->rtc->ops = &rtc_ops; pcf85363->rtc->range_min = RTC_TIMESTAMP_BEGIN_2000; @@ -436,9 +436,16 @@ static int pcf85363_probe(struct i2c_client *client) wakeup_source = device_property_read_bool(&client->dev, "wakeup-source"); if (client->irq > 0 || wakeup_source) { - regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); - regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, - PIN_IO_INTAPM, PIN_IO_INTA_OUT); + err = regmap_write(pcf85363->regmap, CTRL_FLAGS, 0); + if (err) + return dev_err_probe(&client->dev, err, + "failed to clear flags\n"); + + err = regmap_update_bits(pcf85363->regmap, CTRL_PIN_IO, + PIN_IO_INTAPM, PIN_IO_INTA_OUT); + if (err) + return dev_err_probe(&client->dev, err, + "failed to set interrupt pin mode\n"); } if (client->irq > 0) { diff --git a/drivers/rtc/rtc-pcf8563.c b/drivers/rtc/rtc-pcf8563.c index b281e9489df1d..7083d927820f9 100644 --- a/drivers/rtc/rtc-pcf8563.c +++ b/drivers/rtc/rtc-pcf8563.c @@ -449,7 +449,9 @@ static struct clk *pcf8563_clkout_register_clk(struct pcf8563 *pcf8563) clk = devm_clk_register(&pcf8563->rtc->dev, &pcf8563->clkout_hw); if (!IS_ERR(clk)) - of_clk_add_provider(node, of_clk_src_simple_get, clk); + devm_of_clk_add_hw_provider(pcf8563->rtc->dev.parent, + of_clk_hw_simple_get, + &pcf8563->clkout_hw); return clk; } diff --git a/drivers/rtc/rtc-rzn1.c b/drivers/rtc/rtc-rzn1.c index c4ed437354574..eb12957e14023 100644 --- a/drivers/rtc/rtc-rzn1.c +++ b/drivers/rtc/rtc-rzn1.c @@ -235,13 +235,24 @@ static int rzn1_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) if (ret) return ret; + ctl1 = readl(rtc->base + RZN1_RTC_CTL1); + alrm->enabled = !!(ctl1 & (RZN1_RTC_CTL1_ALME | RZN1_RTC_CTL1_1SE)); + min = readl(rtc->base + RZN1_RTC_ALM); hour = readl(rtc->base + RZN1_RTC_ALH); - wday = readl(rtc->base + RZN1_RTC_ALW); tm->tm_sec = 0; tm->tm_min = bcd2bin(min); tm->tm_hour = bcd2bin(hour); + + /* + * If wday is zero, no bit is set in RZN1_RTC_ALW. This is the + * register's power-on reset value. + */ + wday = readl(rtc->base + RZN1_RTC_ALW); + if (!wday) + return 0; + delta_days = ((fls(wday) - 1) - tm->tm_wday + 7) % 7; tm->tm_wday = fls(wday) - 1; @@ -250,9 +261,6 @@ static int rzn1_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alrm) rtc_time64_to_tm(alarm, tm); } - ctl1 = readl(rtc->base + RZN1_RTC_CTL1); - alrm->enabled = !!(ctl1 & (RZN1_RTC_CTL1_ALME | RZN1_RTC_CTL1_1SE)); - return 0; } @@ -261,7 +269,6 @@ static int rzn1_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) struct rzn1_rtc *rtc = dev_get_drvdata(dev); struct rtc_time *tm = &alrm->time, tm_now; unsigned long alarm, farest; - unsigned int days_ahead, wday; int ret; ret = rzn1_rtc_read_time(dev, &tm_now); @@ -274,13 +281,14 @@ static int rzn1_rtc_set_alarm(struct device *dev, struct rtc_wkalrm *alrm) if (time_after(alarm, farest)) return -ERANGE; - /* Convert alarm day into week day */ - days_ahead = tm->tm_mday - tm_now.tm_mday; - wday = (tm_now.tm_wday + days_ahead) % 7; + /* Disable alarm interrupts before reprogramming the alarm. */ + ret = rzn1_rtc_alarm_irq_enable(dev, 0); + if (ret) + return ret; writel(bin2bcd(tm->tm_min), rtc->base + RZN1_RTC_ALM); writel(bin2bcd(tm->tm_hour), rtc->base + RZN1_RTC_ALH); - writel(BIT(wday), rtc->base + RZN1_RTC_ALW); + writel(BIT(tm->tm_wday), rtc->base + RZN1_RTC_ALW); rtc->tm_alarm = alrm->time; @@ -465,6 +473,10 @@ static int rzn1_rtc_probe(struct platform_device *pdev) } irq = platform_get_irq_byname_optional(pdev, "pps"); + if (irq == -EPROBE_DEFER) { + ret = irq; + goto dis_runtime_pm; + } if (irq >= 0) ret = devm_request_irq(&pdev->dev, irq, rzn1_rtc_1s_irq, 0, "RZN1 RTC 1s", rtc); diff --git a/drivers/rtc/rtc-spacemit-p1.c b/drivers/rtc/rtc-spacemit-p1.c index 43ab62494bb4b..1de7bd995d296 100644 --- a/drivers/rtc/rtc-spacemit-p1.c +++ b/drivers/rtc/rtc-spacemit-p1.c @@ -57,8 +57,9 @@ static int p1_rtc_read_time(struct device *dev, struct rtc_time *t) u8 time[6]; int ret; - if (!regmap_test_bits(regmap, RTC_CTRL, RTC_EN)) - return -EINVAL; /* RTC is disabled */ + ret = regmap_test_bits(regmap, RTC_CTRL, RTC_EN); + if (ret <= 0) + return ret ?: -EINVAL; /* RTC is disabled or error */ ret = regmap_bulk_read(regmap, RTC_TIME, time, sizeof(time)); if (ret) diff --git a/drivers/rtc/rtc-zynqmp.c b/drivers/rtc/rtc-zynqmp.c index 856bc1678e7d3..bf2ecd0dfe238 100644 --- a/drivers/rtc/rtc-zynqmp.c +++ b/drivers/rtc/rtc-zynqmp.c @@ -335,10 +335,9 @@ static int xlnx_rtc_probe(struct platform_device *pdev) /* Getting the rtc info */ xrtcdev->rtc_clk = devm_clk_get_optional(&pdev->dev, "rtc"); - if (IS_ERR(xrtcdev->rtc_clk)) { - if (PTR_ERR(xrtcdev->rtc_clk) != -EPROBE_DEFER) - dev_warn(&pdev->dev, "Device clock not found.\n"); - } + if (IS_ERR(xrtcdev->rtc_clk)) + return dev_err_probe(&pdev->dev, PTR_ERR(xrtcdev->rtc_clk), + "Failed to get rtc clock\n"); xrtcdev->freq = clk_get_rate(xrtcdev->rtc_clk); if (!xrtcdev->freq) { ret = of_property_read_u32(pdev->dev.of_node, "calibration", diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 7765e40f7ceac..e7845c8ccc9ff 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -1746,8 +1746,10 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, return; } if (rq_data_dir(req) == READ) { - device->discipline->ese_read(cqr, irb); - cqr->status = DASD_CQR_SUCCESS; + if (device->discipline->ese_read(cqr, irb)) + cqr->status = DASD_CQR_ERROR; + else + cqr->status = DASD_CQR_SUCCESS; cqr->stopclk = now; dasd_device_clear_timer(device); dasd_schedule_device_bh(device); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 40bbbada67f96..8947fbd2797ab 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -1492,6 +1493,8 @@ static void dasd_eckd_reset_path(struct dasd_device *device, __u8 pm) struct dasd_eckd_private *private = device->private; unsigned long flags; + if (!private) + return; if (!private->fcx_max_data) private->fcx_max_data = get_fcx_max_data(device); spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); @@ -1647,6 +1650,9 @@ static int dasd_eckd_is_ese(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.vol_info.ese; } @@ -1654,6 +1660,9 @@ static int dasd_eckd_ext_pool_id(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.extent_pool_id; } @@ -1667,6 +1676,9 @@ static int dasd_eckd_space_configured(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_configured; @@ -1681,6 +1693,9 @@ static int dasd_eckd_space_allocated(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_allocated; @@ -1690,6 +1705,9 @@ static int dasd_eckd_logical_capacity(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.logical_capacity; } @@ -1832,7 +1850,11 @@ static int dasd_eckd_read_ext_pool_info(struct dasd_device *device) static int dasd_eckd_ext_size(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; - struct dasd_ext_pool_sum eps = private->eps; + struct dasd_ext_pool_sum eps; + + if (!private) + return 0; + eps = private->eps; if (!eps.flags.extent_size_valid) return 0; @@ -1848,6 +1870,9 @@ static int dasd_eckd_ext_pool_warn_thrshld(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.warn_thrshld; } @@ -1855,6 +1880,9 @@ static int dasd_eckd_ext_pool_cap_at_warnlevel(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.capacity_at_warnlevel; } @@ -1865,6 +1893,9 @@ static int dasd_eckd_ext_pool_oos(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.pool_oos; } @@ -3476,11 +3507,11 @@ static int dasd_eckd_check_device_format(struct dasd_device *base, { struct dasd_eckd_private *private = base->private; struct eckd_count *fmt_buffer; - struct irb irb; + size_t fmt_buffer_size; + unsigned int trkcount; int rpt_max, rpt_exp; - int fmt_buffer_size; + struct irb irb; int trk_per_cyl; - int trkcount; int tpm = 0; int rc; @@ -3491,7 +3522,9 @@ static int dasd_eckd_check_device_format(struct dasd_device *base, rpt_exp = recs_per_track(&private->rdc_data, 0, cdata->expect.blksize); trkcount = cdata->expect.stop_unit - cdata->expect.start_unit + 1; - fmt_buffer_size = trkcount * rpt_max * sizeof(struct eckd_count); + if (check_mul_overflow(trkcount, rpt_max, &fmt_buffer_size) || + check_mul_overflow(fmt_buffer_size, sizeof(struct eckd_count), &fmt_buffer_size)) + return -EINVAL; fmt_buffer = kzalloc(fmt_buffer_size, GFP_KERNEL | GFP_DMA); if (!fmt_buffer) @@ -5946,8 +5979,11 @@ static int dasd_eckd_query_host_access(struct dasd_device *device, struct ccw1 *ccw; int rc; + if (!private) + return -ENODEV; + /* not available for HYPER PAV alias devices */ - if (!device->block && private->lcu->pav == HYPER_PAV) + if (!device->block && private->lcu && private->lcu->pav == HYPER_PAV) return -EOPNOTSUPP; /* may not be supported by the storage server */ @@ -6812,6 +6848,9 @@ static int dasd_eckd_hpf_enabled(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->fcx_max_data ? 1 : 0; } diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index 89d7516b9ec86..468f0b2cc3423 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -123,6 +123,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) int success; unsigned long startclk, stopclk; struct dasd_device *startdev; + unsigned int proc_bytes; BUG_ON(cqr->refers == NULL || cqr->function == NULL); @@ -130,6 +131,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) startclk = cqr->startclk; stopclk = cqr->stopclk; startdev = cqr->startdev; + proc_bytes = cqr->proc_bytes; /* free all ERPs - but NOT the original cqr */ while (cqr->refers != NULL) { @@ -147,6 +149,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) cqr->startclk = startclk; cqr->stopclk = stopclk; cqr->startdev = startdev; + cqr->proc_bytes = proc_bytes; if (success) cqr->status = DASD_CQR_DONE; else { diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 8308046a9f8fb..a9edc0468a314 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -325,7 +325,7 @@ static int dasd_ioctl_check_format(struct block_device *bdev, void __user *argp) static int dasd_release_space(struct dasd_device *device, struct format_data_t *rdata) { - if (!device->discipline->is_ese && !device->discipline->is_ese(device)) + if (!device->discipline->is_ese || !device->discipline->is_ese(device)) return -ENOTSUPP; if (!device->discipline->release_space) return -ENOTSUPP; diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 33cebb91b9338..e74f41f201099 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -47,8 +47,6 @@ struct ipib_info { static struct debug_info *zcore_dbf; static int hsa_available; static struct dentry *zcore_dir; -static struct dentry *zcore_reipl_file; -static struct dentry *zcore_hsa_file; static struct ipl_parameter_block *zcore_ipl_block; static unsigned long os_info_flags; @@ -354,10 +352,8 @@ static int __init zcore_init(void) goto fail; zcore_dir = debugfs_create_dir("zcore" , NULL); - zcore_reipl_file = debugfs_create_file("reipl", S_IRUSR, zcore_dir, - NULL, &zcore_reipl_fops); - zcore_hsa_file = debugfs_create_file("hsa", S_IRUSR|S_IWUSR, zcore_dir, - NULL, &zcore_hsa_fops); + debugfs_create_file("reipl", S_IRUSR, zcore_dir, NULL, &zcore_reipl_fops); + debugfs_create_file("hsa", S_IRUSR|S_IWUSR, zcore_dir, NULL, &zcore_hsa_fops); register_reboot_notifier(&zcore_reboot_notifier); atomic_notifier_chain_register(&panic_notifier_list, &zcore_on_panic_notifier); diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 4b2dae6eb3760..7f8f78b1f8aea 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -1329,7 +1329,7 @@ static int purge_fn(struct subchannel *sch, void *data) cdev = sch_get_cdev(sch); if (cdev) { - if (cdev->private->state != DEV_STATE_OFFLINE) + if (cdev->online) goto unlock; if (atomic_cmpxchg(&cdev->private->onoff, 0, 1) != 0) diff --git a/drivers/s390/cio/vfio_ccw_async.c b/drivers/s390/cio/vfio_ccw_async.c index 420d89ba7f838..4aff0b58fa5d5 100644 --- a/drivers/s390/cio/vfio_ccw_async.c +++ b/drivers/s390/cio/vfio_ccw_async.c @@ -8,6 +8,7 @@ */ #include +#include #include "vfio_ccw_private.h" @@ -24,11 +25,20 @@ static ssize_t vfio_ccw_async_region_read(struct vfio_ccw_private *private, return -EINVAL; mutex_lock(&private->io_mutex); + + if (i >= private->num_regions) { + ret = -EINVAL; + goto out_unlock; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (copy_to_user(buf, (void *)region + pos, count)) ret = -EFAULT; else ret = count; + +out_unlock: mutex_unlock(&private->io_mutex); return ret; } @@ -48,6 +58,12 @@ static ssize_t vfio_ccw_async_region_write(struct vfio_ccw_private *private, if (!mutex_trylock(&private->io_mutex)) return -EAGAIN; + if (i >= private->num_regions) { + ret = -EINVAL; + goto out_unlock; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (copy_from_user((void *)region + pos, buf, count)) { ret = -EFAULT; diff --git a/drivers/s390/cio/vfio_ccw_chp.c b/drivers/s390/cio/vfio_ccw_chp.c index 38c176cf62957..7708eb4d6de0b 100644 --- a/drivers/s390/cio/vfio_ccw_chp.c +++ b/drivers/s390/cio/vfio_ccw_chp.c @@ -9,6 +9,7 @@ */ #include +#include #include #include "vfio_ccw_private.h" @@ -26,6 +27,13 @@ static ssize_t vfio_ccw_schib_region_read(struct vfio_ccw_private *private, return -EINVAL; mutex_lock(&private->io_mutex); + + if (i >= private->num_regions) { + ret = -EINVAL; + goto out; + } + + i = array_index_nospec(i, private->num_regions); region = private->region[i].data; if (cio_update_schib(sch)) { @@ -85,19 +93,30 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, loff_t pos = *ppos & VFIO_CCW_OFFSET_MASK; struct ccw_crw_region *region; struct vfio_ccw_crw *crw; + unsigned long flags; int ret; if (pos + count > sizeof(*region)) return -EINVAL; + mutex_lock(&private->io_mutex); + if (i >= private->num_regions) { + ret = -EINVAL; + goto out; + } + + i = array_index_nospec(i, private->num_regions); + region = private->region[i].data; + + spin_lock_irqsave(&private->crw_lock, flags); crw = list_first_entry_or_null(&private->crw, struct vfio_ccw_crw, next); if (crw) list_del(&crw->next); - mutex_lock(&private->io_mutex); - region = private->region[i].data; + /* Drop CRW lock while copying to userspace */ + spin_unlock_irqrestore(&private->crw_lock, flags); if (crw) memcpy(®ion->crw, &crw->crw, sizeof(region->crw)); @@ -108,14 +127,16 @@ static ssize_t vfio_ccw_crw_region_read(struct vfio_ccw_private *private, ret = count; region->crw = 0; - - mutex_unlock(&private->io_mutex); - kfree(crw); /* Notify the guest if more CRWs are on our queue */ + spin_lock_irqsave(&private->crw_lock, flags); if (!list_empty(&private->crw) && private->crw_trigger) eventfd_signal(private->crw_trigger); + spin_unlock_irqrestore(&private->crw_lock, flags); + +out: + mutex_unlock(&private->io_mutex); return ret; } diff --git a/drivers/s390/cio/vfio_ccw_cp.c b/drivers/s390/cio/vfio_ccw_cp.c index 5f6e102256276..36985aed125c5 100644 --- a/drivers/s390/cio/vfio_ccw_cp.c +++ b/drivers/s390/cio/vfio_ccw_cp.c @@ -233,6 +233,7 @@ static void convert_ccw0_to_ccw1(struct ccw1 *source, unsigned long len) } #define idal_is_2k(_cp) (!(_cp)->orb.cmd.c64 || (_cp)->orb.cmd.i2k) +#define get_idaw_size(_cp) ((_cp)->orb.cmd.c64 ? sizeof(u64) : sizeof(u32)) /* * Helpers to operate ccwchain. @@ -332,6 +333,7 @@ static struct ccwchain *ccwchain_alloc(struct channel_program *cp, int len) goto out_err; list_add_tail(&chain->next, &cp->ccwchain_list); + cp->ccwchain_count++; return chain; @@ -376,11 +378,9 @@ static void ccwchain_cda_free(struct ccwchain *chain, int idx) static int ccwchain_calc_length(u64 iova, struct channel_program *cp) { struct ccw1 *ccw = cp->guest_cp; - int cnt = 0; - - do { - cnt++; + int cnt; + for (cnt = 1; cnt <= CCWCHAIN_LEN_MAX; cnt++, ccw++) { /* * We want to keep counting if the current CCW has the * command-chaining flag enabled, or if it is a TIC CCW @@ -390,15 +390,10 @@ static int ccwchain_calc_length(u64 iova, struct channel_program *cp) * after the TIC, depending on the results of its operation. */ if (!ccw_is_chain(ccw) && !is_tic_within_range(ccw, iova, cnt)) - break; - - ccw++; - } while (cnt < CCWCHAIN_LEN_MAX + 1); - - if (cnt == CCWCHAIN_LEN_MAX + 1) - cnt = -EINVAL; + return cnt; + } - return cnt; + return -EINVAL; } static int tic_target_chain_exists(struct ccw1 *tic, struct channel_program *cp) @@ -441,6 +436,10 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp) if (len < 0) return len; + /* Limit number of chains in a single channel program */ + if (cp->ccwchain_count >= CCWCHAIN_COUNT_MAX) + return -EINVAL; + /* Need alloc a new chain for this one. */ chain = ccwchain_alloc(cp, len); if (!chain) @@ -455,9 +454,6 @@ static int ccwchain_handle_ccw(dma32_t cda, struct channel_program *cp) /* Loop for tics on this new chain. */ ret = ccwchain_loop_tic(chain, cp); - if (ret) - ccwchain_free(chain); - return ret; } @@ -486,6 +482,23 @@ static int ccwchain_loop_tic(struct ccwchain *chain, struct channel_program *cp) return 0; } +static int ccwchain_build_ccws(dma32_t cda, struct channel_program *cp) +{ + struct ccwchain *chain, *temp; + int ret; + + ret = ccwchain_handle_ccw(cda, cp); + + if (ret) { + /* Cleanup if an error occurred */ + list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) { + ccwchain_free(chain); + } + } + + return ret; +} + static int ccwchain_fetch_tic(struct ccw1 *ccw, struct channel_program *cp) { @@ -511,7 +524,8 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int &container_of(cp, struct vfio_ccw_private, cp)->vdev; dma64_t *idaws; dma32_t *idaws_f1; - int idal_len = idaw_nr * sizeof(*idaws); + u64 first_idaw; + int idal_len = idaw_nr * get_idaw_size(cp); int idaw_size = idal_is_2k(cp) ? PAGE_SIZE / 2 : PAGE_SIZE; int idaw_mask = ~(idaw_size - 1); int i, ret; @@ -527,6 +541,18 @@ static dma64_t *get_guest_idal(struct ccw1 *ccw, struct channel_program *cp, int kfree(idaws); return ERR_PTR(ret); } + + idaws_f1 = (dma32_t *)idaws; + if (cp->orb.cmd.c64) + first_idaw = dma64_to_u64(idaws[0]); + else + first_idaw = dma32_to_u32(idaws_f1[0]); + + /* Unexpected mismatch from earlier read */ + if (first_idaw != cp->guest_iova) { + kfree(idaws); + return ERR_PTR(-EINVAL); + } } else { /* Fabricate an IDAL based off CCW data address */ if (cp->orb.cmd.c64) { @@ -568,7 +594,7 @@ static int ccw_count_idaws(struct ccw1 *ccw, struct vfio_device *vdev = &container_of(cp, struct vfio_ccw_private, cp)->vdev; u64 iova; - int size = cp->orb.cmd.c64 ? sizeof(u64) : sizeof(u32); + int size = get_idaw_size(cp); int ret; int bytes = 1; @@ -592,6 +618,9 @@ static int ccw_count_idaws(struct ccw1 *ccw, iova = dma32_to_u32(ccw->cda); } + /* Save the read address for later */ + cp->guest_iova = iova; + /* Format-1 IDAWs operate on 2K each */ if (!cp->orb.cmd.c64) return idal_2k_nr_words((void *)iova, bytes); @@ -731,11 +760,12 @@ int cp_init(struct channel_program *cp, union orb *orb) vdev->dev, "Prefetching channel program even though prefetch not specified in ORB"); + cp->ccwchain_count = 0; INIT_LIST_HEAD(&cp->ccwchain_list); memcpy(&cp->orb, orb, sizeof(*orb)); /* Build a ccwchain for the first CCW segment */ - ret = ccwchain_handle_ccw(orb->cmd.cpa, cp); + ret = ccwchain_build_ccws(orb->cmd.cpa, cp); if (!ret) cp->initialized = true; @@ -947,17 +977,23 @@ void cp_update_scsw(struct channel_program *cp, union scsw *scsw) */ bool cp_iova_pinned(struct channel_program *cp, u64 iova, u64 length) { + struct vfio_ccw_private *private = + container_of(cp, struct vfio_ccw_private, cp); struct ccwchain *chain; int i; if (!cp->initialized) return false; + mutex_lock(&private->io_mutex); list_for_each_entry(chain, &cp->ccwchain_list, next) { for (i = 0; i < chain->ch_len; i++) - if (page_array_iova_pinned(&chain->ch_pa[i], iova, length)) + if (page_array_iova_pinned(&chain->ch_pa[i], iova, length)) { + mutex_unlock(&private->io_mutex); return true; + } } + mutex_unlock(&private->io_mutex); return false; } diff --git a/drivers/s390/cio/vfio_ccw_cp.h b/drivers/s390/cio/vfio_ccw_cp.h index fc31eb6998072..9af98ff12d671 100644 --- a/drivers/s390/cio/vfio_ccw_cp.h +++ b/drivers/s390/cio/vfio_ccw_cp.h @@ -23,11 +23,19 @@ */ #define CCWCHAIN_LEN_MAX 256 +/* + * Maximum number of chains + */ +#define CCWCHAIN_COUNT_MAX 16 + /** * struct channel_program - manage information for channel program * @ccwchain_list: list head of ccwchains * @orb: orb for the currently processed ssch request * @initialized: whether this instance is actually initialized + * @guest_cp: copy of guest channel program + * @ccwchain_count: number of channel program segments (linked by TIC) + * @guest_iova: first data address of a guest channel program * * @ccwchain_list is the head of a ccwchain list, that contents the * translated result of the guest channel program that pointed out by @@ -38,6 +46,8 @@ struct channel_program { union orb orb; bool initialized; struct ccw1 *guest_cp; + unsigned int ccwchain_count; + u64 guest_iova; }; int cp_init(struct channel_program *cp, union orb *orb); diff --git a/drivers/s390/cio/vfio_ccw_drv.c b/drivers/s390/cio/vfio_ccw_drv.c index 6ff5c9cfb7ed7..cf5b92a63d060 100644 --- a/drivers/s390/cio/vfio_ccw_drv.c +++ b/drivers/s390/cio/vfio_ccw_drv.c @@ -91,6 +91,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) is_final = !(scsw_actl(&irb->scsw) & (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT)); + mutex_lock(&private->io_mutex); if (scsw_is_solicited(&irb->scsw)) { cp_update_scsw(&private->cp, &irb->scsw); if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) { @@ -98,9 +99,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) cp_is_finished = true; } } - mutex_lock(&private->io_mutex); memcpy(private->io_region->irb_area, irb, sizeof(*irb)); - mutex_unlock(&private->io_mutex); /* * Reset to IDLE only if processing of a channel program @@ -110,6 +109,7 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) */ if (cp_is_finished) private->state = VFIO_CCW_STATE_IDLE; + mutex_unlock(&private->io_mutex); if (private->io_trigger) eventfd_signal(private->io_trigger); @@ -118,11 +118,25 @@ void vfio_ccw_sch_io_todo(struct work_struct *work) void vfio_ccw_crw_todo(struct work_struct *work) { struct vfio_ccw_private *private; + unsigned long flags; private = container_of(work, struct vfio_ccw_private, crw_work); + spin_lock_irqsave(&private->crw_lock, flags); if (!list_empty(&private->crw) && private->crw_trigger) eventfd_signal(private->crw_trigger); + spin_unlock_irqrestore(&private->crw_lock, flags); +} + +void vfio_ccw_notoper_todo(struct work_struct *work) +{ + struct vfio_ccw_private *private; + + private = container_of(work, struct vfio_ccw_private, notoper_work); + + mutex_lock(&private->io_mutex); + cp_free(&private->cp); + mutex_unlock(&private->io_mutex); } /* @@ -275,6 +289,7 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, unsigned int rsid) { struct vfio_ccw_crw *crw; + unsigned long flags; /* * If unable to allocate a CRW, just drop the event and @@ -292,7 +307,9 @@ static void vfio_ccw_queue_crw(struct vfio_ccw_private *private, crw->crw.erc = erc; crw->crw.rsid = rsid; + spin_lock_irqsave(&private->crw_lock, flags); list_add_tail(&crw->next, &private->crw); + spin_unlock_irqrestore(&private->crw_lock, flags); queue_work(vfio_ccw_work_q, &private->crw_work); } diff --git a/drivers/s390/cio/vfio_ccw_fsm.c b/drivers/s390/cio/vfio_ccw_fsm.c index 4d7988ea47ef0..5fd94e9d5c618 100644 --- a/drivers/s390/cio/vfio_ccw_fsm.c +++ b/drivers/s390/cio/vfio_ccw_fsm.c @@ -170,8 +170,8 @@ static void fsm_notoper(struct vfio_ccw_private *private, css_sched_sch_todo(sch, SCH_TODO_UNREG); private->state = VFIO_CCW_STATE_NOT_OPER; - /* This is usually handled during CLOSE event */ - cp_free(&private->cp); + /* This routine could be called from IRQ context, so defer */ + queue_work(vfio_ccw_work_q, &private->notoper_work); } /* @@ -410,7 +410,11 @@ static void fsm_close(struct vfio_ccw_private *private, private->state = VFIO_CCW_STATE_STANDBY; spin_unlock_irq(&sch->lock); + + mutex_lock(&private->io_mutex); cp_free(&private->cp); + mutex_unlock(&private->io_mutex); + return; err_unlock: diff --git a/drivers/s390/cio/vfio_ccw_ops.c b/drivers/s390/cio/vfio_ccw_ops.c index ea532a8a4a0c2..b7acc25320968 100644 --- a/drivers/s390/cio/vfio_ccw_ops.c +++ b/drivers/s390/cio/vfio_ccw_ops.c @@ -54,6 +54,8 @@ static int vfio_ccw_mdev_init_dev(struct vfio_device *vdev) INIT_LIST_HEAD(&private->crw); INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo); INIT_WORK(&private->crw_work, vfio_ccw_crw_todo); + INIT_WORK(&private->notoper_work, vfio_ccw_notoper_todo); + spin_lock_init(&private->crw_lock); private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1), GFP_KERNEL); @@ -131,11 +133,28 @@ static void vfio_ccw_mdev_release_dev(struct vfio_device *vdev) struct vfio_ccw_private *private = container_of(vdev, struct vfio_ccw_private, vdev); struct vfio_ccw_crw *crw, *temp; + unsigned long flags; + /* + * Ensure these work items are fully drained, so none can + * fire after being released. + * + * notoper_work should have nothing to do here, because only + * open devices could have channel_program resources in use + * and those would be released during close. Nevertheless, + * call flush here as well to be certain anything that was + * allocated is freed. + */ + cancel_work_sync(&private->io_work); + cancel_work_sync(&private->crw_work); + flush_work(&private->notoper_work); + + spin_lock_irqsave(&private->crw_lock, flags); list_for_each_entry_safe(crw, temp, &private->crw, next) { list_del(&crw->next); kfree(crw); } + spin_unlock_irqrestore(&private->crw_lock, flags); kmem_cache_free(vfio_ccw_crw_region, private->crw_region); kmem_cache_free(vfio_ccw_schib_region, private->schib_region); @@ -203,6 +222,19 @@ static void vfio_ccw_mdev_close_device(struct vfio_device *vdev) container_of(vdev, struct vfio_ccw_private, vdev); vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_CLOSE); + + /* + * Ensure these work items are drained, in the event the + * device is re-opened instead of released. + * + * notoper_work needs to be given a chance to run if it + * is queued, so any memory associated with the channel + * program can be returned. + */ + cancel_work_sync(&private->io_work); + cancel_work_sync(&private->crw_work); + flush_work(&private->notoper_work); + vfio_ccw_unregister_dev_regions(private); } @@ -244,6 +276,7 @@ static ssize_t vfio_ccw_mdev_read(struct vfio_device *vdev, return vfio_ccw_mdev_read_io_region(private, buf, count, ppos); default: index -= VFIO_CCW_NUM_REGIONS; + index = array_index_nospec(index, private->num_regions); return private->region[index].ops->read(private, buf, count, ppos); } @@ -296,6 +329,7 @@ static ssize_t vfio_ccw_mdev_write(struct vfio_device *vdev, return vfio_ccw_mdev_write_io_region(private, buf, count, ppos); default: index -= VFIO_CCW_NUM_REGIONS; + index = array_index_nospec(index, private->num_regions); return private->region[index].ops->write(private, buf, count, ppos); } @@ -338,11 +372,8 @@ static int vfio_ccw_mdev_get_region_info(struct vfio_ccw_private *private, VFIO_CCW_NUM_REGIONS + private->num_regions) return -EINVAL; - info->index = array_index_nospec(info->index, - VFIO_CCW_NUM_REGIONS + - private->num_regions); - i = info->index - VFIO_CCW_NUM_REGIONS; + i = array_index_nospec(i, private->num_regions); info->offset = VFIO_CCW_INDEX_TO_OFFSET(info->index); info->size = private->region[i].size; diff --git a/drivers/s390/cio/vfio_ccw_private.h b/drivers/s390/cio/vfio_ccw_private.h index 0501d4bbcdbd6..3bd0171d38d04 100644 --- a/drivers/s390/cio/vfio_ccw_private.h +++ b/drivers/s390/cio/vfio_ccw_private.h @@ -88,7 +88,8 @@ struct vfio_ccw_parent { * @state: internal state of the device * @completion: synchronization helper of the I/O completion * @io_region: MMIO region to input/output I/O arguments/results - * @io_mutex: protect against concurrent update of I/O regions + * @io_mutex: protect against concurrent update of I/O resources + * and @cp lifecycle * @region: additional regions for other subchannel operations * @cmd_region: MMIO region for asynchronous I/O commands other than START * @schib_region: MMIO region for SCHIB information @@ -97,11 +98,14 @@ struct vfio_ccw_parent { * @cp: channel program for the current I/O operation * @irb: irb info received from interrupt * @scsw: scsw info + * @crw_lock: serialization of CRW list information + * @crw: list of Channel Report Word elements * @io_trigger: eventfd ctx for signaling userspace I/O results * @crw_trigger: eventfd ctx for signaling userspace CRW information * @req_trigger: eventfd ctx for signaling userspace to return device * @io_work: work for deferral process of I/O handling * @crw_work: work for deferral process of CRW handling + * @notoper_work: work for deferred processing in not-operational state */ struct vfio_ccw_private { struct vfio_device vdev; @@ -118,6 +122,8 @@ struct vfio_ccw_private { struct channel_program cp; struct irb irb; union scsw scsw; + + spinlock_t crw_lock; struct list_head crw; struct eventfd_ctx *io_trigger; @@ -125,11 +131,13 @@ struct vfio_ccw_private { struct eventfd_ctx *req_trigger; struct work_struct io_work; struct work_struct crw_work; + struct work_struct notoper_work; } __aligned(8); int vfio_ccw_sch_quiesce(struct subchannel *sch); void vfio_ccw_sch_io_todo(struct work_struct *work); void vfio_ccw_crw_todo(struct work_struct *work); +void vfio_ccw_notoper_todo(struct work_struct *work); extern struct mdev_driver vfio_ccw_mdev_driver; diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c index 6c7897a93f273..d780ea60c9e1a 100644 --- a/drivers/s390/crypto/pkey_cca.c +++ b/drivers/s390/crypto/pkey_cca.c @@ -233,22 +233,16 @@ static int cca_key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns, if (hdr->type == TOKTYPE_CCA_INTERNAL && hdr->version == TOKVER_CCA_AES) { /* CCA AES data key */ - if (keylen < sizeof(struct secaeskeytoken)) - return -EINVAL; - if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0)) + if (cca_check_secaeskeytoken(pkey_dbf_info, 3, key, keylen, 0)) return -EINVAL; } else if (hdr->type == TOKTYPE_CCA_INTERNAL && hdr->version == TOKVER_CCA_VLSC) { /* CCA AES cipher key */ - if (keylen < hdr->len) - return -EINVAL; if (cca_check_secaescipherkey(pkey_dbf_info, - 3, key, 0, 1)) + 3, key, keylen, 0, 1)) return -EINVAL; } else if (hdr->type == TOKTYPE_CCA_INTERNAL_PKA) { /* CCA ECC (private) key */ - if (keylen < sizeof(struct eccprivkeytoken)) - return -EINVAL; if (cca_check_sececckeytoken(pkey_dbf_info, 3, key, keylen, 1)) return -EINVAL; } else { @@ -476,7 +470,7 @@ static int cca_verifykey(const u8 *key, u32 keylen, hdr->version == TOKVER_CCA_AES) { struct secaeskeytoken *t = (struct secaeskeytoken *)key; - rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, 0); + rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, key, keylen, 0); if (rc) goto out; *keytype = PKEY_TYPE_CCA_DATA; @@ -504,7 +498,8 @@ static int cca_verifykey(const u8 *key, u32 keylen, hdr->version == TOKVER_CCA_VLSC) { struct cipherkeytoken *t = (struct cipherkeytoken *)key; - rc = cca_check_secaescipherkey(pkey_dbf_info, 3, key, 0, 1); + rc = cca_check_secaescipherkey(pkey_dbf_info, 3, + key, keylen, 0, 1); if (rc) goto out; *keytype = PKEY_TYPE_CCA_CIPHER; diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index eb5ff49f6fe7d..7b440f4c23c34 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -48,15 +48,19 @@ static void vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q); * 1. matrix_dev->guests_lock: required to use the KVM pointer to update a KVM * guest's APCB. * 2. kvm->lock: required to update a guest's APCB - * 3. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev + * 3. kvm->arch.crypto.pqap_hook_rwsem: required to update pqap_hook and + * serialize against PQAP intercepts + * 4. matrix_dev->mdevs_lock: required to access data stored in a matrix_mdev * - * Note: If @kvm is NULL, the KVM lock will not be taken. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be taken. */ static inline void get_update_locks_for_kvm(struct kvm *kvm) { mutex_lock(&matrix_dev->guests_lock); - if (kvm) + if (kvm) { mutex_lock(&kvm->lock); + down_write(&kvm->arch.crypto.pqap_hook_rwsem); + } mutex_lock(&matrix_dev->mdevs_lock); } @@ -68,16 +72,19 @@ static inline void get_update_locks_for_kvm(struct kvm *kvm) * * The proper unlocking order is: * 1. matrix_dev->mdevs_lock - * 2. kvm->lock - * 3. matrix_dev->guests_lock + * 2. kvm->arch.crypto.pqap_hook_rwsem + * 3. kvm->lock + * 4. matrix_dev->guests_lock * - * Note: If @kvm is NULL, the KVM lock will not be released. + * Note: If @kvm is NULL, the KVM lock and pqap_hook_rwsem will not be released. */ static inline void release_update_locks_for_kvm(struct kvm *kvm) { mutex_unlock(&matrix_dev->mdevs_lock); - if (kvm) + if (kvm) { + up_write(&kvm->arch.crypto.pqap_hook_rwsem); mutex_unlock(&kvm->lock); + } mutex_unlock(&matrix_dev->guests_lock); } @@ -800,12 +807,17 @@ static int vfio_ap_mdev_probe(struct mdev_device *mdev) ret = vfio_register_emulated_iommu_dev(&matrix_mdev->vdev); if (ret) goto err_put_vdev; - matrix_mdev->req_trigger = NULL; - matrix_mdev->cfg_chg_trigger = NULL; + + /* + * Take the matrix_dev->guests_lock mutex before adding the matrix_mdev + * to the mdev_list. All functions that traverse the list must also hold + * this lock to guard against additions to or removals from the list + * while it is being traversed. + */ + mutex_lock(&matrix_dev->guests_lock); dev_set_drvdata(&mdev->dev, matrix_mdev); - mutex_lock(&matrix_dev->mdevs_lock); list_add(&matrix_mdev->node, &matrix_dev->mdev_list); - mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return 0; err_put_vdev: @@ -1821,26 +1833,17 @@ static const struct attribute_group *vfio_ap_mdev_attr_groups[] = { static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, struct kvm *kvm) { - struct ap_matrix_mdev *m; - if (kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); - - list_for_each_entry(m, &matrix_dev->mdev_list, node) { - if (m != matrix_mdev && m->kvm == kvm) { - release_update_locks_for_kvm(kvm); - return -EPERM; - } + if (kvm->arch.crypto.pqap_hook) { + release_update_locks_for_kvm(kvm); + return -EPERM; } + kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook; kvm_get_kvm(kvm); matrix_mdev->kvm = kvm; vfio_ap_mdev_update_guest_apcb(matrix_mdev); - release_update_locks_for_kvm(kvm); } @@ -1883,18 +1886,15 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev) struct kvm *kvm = matrix_mdev->kvm; if (kvm && kvm->arch.crypto.crycbd) { - down_write(&kvm->arch.crypto.pqap_hook_rwsem); - kvm->arch.crypto.pqap_hook = NULL; - up_write(&kvm->arch.crypto.pqap_hook_rwsem); - get_update_locks_for_kvm(kvm); + kvm->arch.crypto.pqap_hook = NULL; kvm_arch_crypto_clear_masks(kvm); vfio_ap_mdev_reset_queues(matrix_mdev); - kvm_put_kvm(kvm); matrix_mdev->kvm = NULL; release_update_locks_for_kvm(kvm); + kvm_put_kvm(kvm); } } @@ -2297,6 +2297,8 @@ static struct ap_matrix_mdev *vfio_ap_mdev_for_queue(struct vfio_ap_queue *q) unsigned long apid = AP_QID_CARD(q->apqn); unsigned long apqi = AP_QID_QUEUE(q->apqn); + lockdep_assert_held(&matrix_dev->guests_lock); + list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { if (test_bit_inv(apid, matrix_mdev->matrix.apm) && test_bit_inv(apqi, matrix_mdev->matrix.aqm)) @@ -2316,8 +2318,26 @@ static ssize_t status_show(struct device *dev, struct ap_matrix_mdev *matrix_mdev; struct ap_device *apdev = to_ap_dev(dev); + mutex_lock(&matrix_dev->guests_lock); mutex_lock(&matrix_dev->mdevs_lock); q = dev_get_drvdata(&apdev->device); + + /* + * Make sure the drvdata has been set before proceeding. There is a + * possibility that the drvdata was not set if the vfio_ap_queue object + * could not be allocated when the queue device was probed. In that case, + * the locks used in vfio_ap_mdev_probe_queue() are released prior to + * removing the sysfs status attribute to avoid a lockdep + * splat. That opens a very small window where the status attribute is + * still available without the vfio_ap_queue object having been + * stored in the device drvdata. In that case, indicate the queue is not + * assigned. + */ + if (!q) { + nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); + goto done; + } + matrix_mdev = vfio_ap_mdev_for_queue(q); /* If the queue is assigned to the matrix mediated device, then @@ -2342,7 +2362,9 @@ static ssize_t status_show(struct device *dev, nchars = sysfs_emit(buf, "%s\n", AP_QUEUE_UNASSIGNED); } +done: mutex_unlock(&matrix_dev->mdevs_lock); + mutex_unlock(&matrix_dev->guests_lock); return nchars; } @@ -2415,14 +2437,17 @@ void vfio_ap_mdev_unregister(void) int vfio_ap_mdev_probe_queue(struct ap_device *apdev) { - int ret; + int ret, apqn; struct vfio_ap_queue *q; DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev; + apqn = to_ap_queue(&apdev->device)->qid; + matrix_mdev = get_update_locks_by_apqn(apqn); + ret = sysfs_create_group(&apdev->device.kobj, &vfio_queue_attr_group); if (ret) - return ret; + goto err_release_locks; q = kzalloc(sizeof(*q), GFP_KERNEL); if (!q) { @@ -2430,11 +2455,10 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) goto err_remove_group; } - q->apqn = to_ap_queue(&apdev->device)->qid; + q->apqn = apqn; q->saved_isc = VFIO_AP_ISC_INVALID; memset(&q->reset_status, 0, sizeof(q->reset_status)); INIT_WORK(&q->reset_work, apq_reset_check); - matrix_mdev = get_update_locks_by_apqn(q->apqn); if (matrix_mdev) { vfio_ap_mdev_link_queue(matrix_mdev, q); @@ -2463,8 +2487,13 @@ int vfio_ap_mdev_probe_queue(struct ap_device *apdev) return ret; err_remove_group: + release_update_locks_for_mdev(matrix_mdev); sysfs_remove_group(&apdev->device.kobj, &vfio_queue_attr_group); return ret; + +err_release_locks: + release_update_locks_for_mdev(matrix_mdev); + return ret; } void vfio_ap_mdev_remove_queue(struct ap_device *apdev) @@ -2559,24 +2588,28 @@ static void vfio_ap_mdev_hot_unplug_cfg(struct ap_matrix_mdev *matrix_mdev, unsigned long *aqrem, unsigned long *cdrem) { - int do_hotplug = 0; + bool do_hotplug = false; - if (!bitmap_empty(aprem, AP_DEVICES)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.apm, - matrix_mdev->shadow_apcb.apm, - aprem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.apm, aprem, AP_DEVICES)) { + bitmap_andnot(matrix_mdev->shadow_apcb.apm, + matrix_mdev->shadow_apcb.apm, + aprem, AP_DEVICES); + do_hotplug = true; } - if (!bitmap_empty(aqrem, AP_DOMAINS)) { - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.aqm, - matrix_mdev->shadow_apcb.aqm, - aqrem, AP_DEVICES); + if (bitmap_intersects(matrix_mdev->shadow_apcb.aqm, aqrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.aqm, + matrix_mdev->shadow_apcb.aqm, + aqrem, AP_DOMAINS); + do_hotplug = true; } - if (!bitmap_empty(cdrem, AP_DOMAINS)) - do_hotplug |= bitmap_andnot(matrix_mdev->shadow_apcb.adm, - matrix_mdev->shadow_apcb.adm, - cdrem, AP_DOMAINS); + if (bitmap_intersects(matrix_mdev->shadow_apcb.adm, cdrem, AP_DOMAINS)) { + bitmap_andnot(matrix_mdev->shadow_apcb.adm, + matrix_mdev->shadow_apcb.adm, + cdrem, AP_DOMAINS); + do_hotplug = true; + } if (do_hotplug) vfio_ap_mdev_update_guest_apcb(matrix_mdev); @@ -2603,28 +2636,41 @@ static void vfio_ap_mdev_cfg_remove(unsigned long *ap_remove, DECLARE_BITMAP(aprem, AP_DEVICES); DECLARE_BITMAP(aqrem, AP_DOMAINS); DECLARE_BITMAP(cdrem, AP_DOMAINS); - int do_remove = 0; + int do_remove; + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { - mutex_lock(&matrix_mdev->kvm->lock); + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); mutex_lock(&matrix_dev->mdevs_lock); - do_remove |= bitmap_and(aprem, ap_remove, - matrix_mdev->matrix.apm, - AP_DEVICES); + do_remove = bitmap_and(aprem, ap_remove, + matrix_mdev->matrix.apm, + AP_DEVICES); do_remove |= bitmap_and(aqrem, aq_remove, matrix_mdev->matrix.aqm, AP_DOMAINS); - do_remove |= bitmap_andnot(cdrem, cd_remove, - matrix_mdev->matrix.adm, - AP_DOMAINS); + do_remove |= bitmap_and(cdrem, cd_remove, + matrix_mdev->matrix.adm, + AP_DOMAINS); if (do_remove) vfio_ap_mdev_hot_unplug_cfg(matrix_mdev, aprem, aqrem, cdrem); mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } } @@ -2748,13 +2794,27 @@ static void vfio_ap_mdev_cfg_add(unsigned long *apm_add, unsigned long *aqm_add, vfio_ap_filter_apid_by_qtype(apm_add, aqm_add); + /* + * It is safe to traverse this list here because the + * required guard - matrix_dev->guests_lock - is taken in the + * vfio_ap_on_cfg_changed function prior to this function getting + * called. + */ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held in order to access fields + * within matrix_mdev + */ + mutex_lock(&matrix_dev->mdevs_lock); + bitmap_and(matrix_mdev->apm_add, matrix_mdev->matrix.apm, apm_add, AP_DEVICES); bitmap_and(matrix_mdev->aqm_add, matrix_mdev->matrix.aqm, aqm_add, AP_DOMAINS); bitmap_and(matrix_mdev->adm_add, matrix_mdev->matrix.adm, adm_add, AP_DEVICES); + + mutex_unlock(&matrix_dev->mdevs_lock); } } @@ -2807,6 +2867,10 @@ void vfio_ap_on_cfg_changed(struct ap_config_info *cur_cfg_info, if (!cur_cfg_info || !prev_cfg_info) return; + /* + * Take the guests_lock mutex here to guard access to the + * matrix_dev->mdev_list in the two functions called below. + */ mutex_lock(&matrix_dev->guests_lock); vfio_ap_mdev_on_cfg_remove(cur_cfg_info, prev_cfg_info); @@ -2821,8 +2885,14 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) DECLARE_BITMAP(apm_filtered, AP_DEVICES); bool filter_domains, filter_adapters, filter_cdoms, do_hotplug = false; - mutex_lock(&matrix_mdev->kvm->lock); - mutex_lock(&matrix_dev->mdevs_lock); + /* + * Zero out the apm_filtered bitmap in case there are no adapters or + * domains to be added, but only control domains. In that case, + * vfio_ap_mdev_filter_matrix() - which initializes apm_filtered - will + * not get called and the reset_queues_for_apids will crash because it + * will access an uninitialized bitmap. + */ + bitmap_zero(apm_filtered, AP_DEVICES); filter_adapters = bitmap_intersects(matrix_mdev->matrix.apm, matrix_mdev->apm_add, AP_DEVICES); @@ -2841,9 +2911,6 @@ static void vfio_ap_mdev_hot_plug_cfg(struct ap_matrix_mdev *matrix_mdev) vfio_ap_mdev_update_guest_apcb(matrix_mdev); reset_queues_for_apids(matrix_mdev, apm_filtered); - - mutex_unlock(&matrix_dev->mdevs_lock); - mutex_unlock(&matrix_mdev->kvm->lock); } void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, @@ -2854,15 +2921,29 @@ void vfio_ap_on_scan_complete(struct ap_config_info *new_config_info, mutex_lock(&matrix_dev->guests_lock); list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) { + /* + * The mdevs_lock must be held to access fields within matrix_mdev, + * and kvm->lock must be taken before mdevs_lock to satisfy the lock + * ordering requirement and prevent a lockdep splat. + */ + if (matrix_mdev->kvm) + mutex_lock(&matrix_mdev->kvm->lock); + mutex_lock(&matrix_dev->mdevs_lock); + if (bitmap_empty(matrix_mdev->apm_add, AP_DEVICES) && bitmap_empty(matrix_mdev->aqm_add, AP_DOMAINS) && bitmap_empty(matrix_mdev->adm_add, AP_DOMAINS)) - continue; + goto do_unlock; vfio_ap_mdev_hot_plug_cfg(matrix_mdev); bitmap_clear(matrix_mdev->apm_add, 0, AP_DEVICES); bitmap_clear(matrix_mdev->aqm_add, 0, AP_DOMAINS); bitmap_clear(matrix_mdev->adm_add, 0, AP_DOMAINS); + +do_unlock: + mutex_unlock(&matrix_dev->mdevs_lock); + if (matrix_mdev->kvm) + mutex_unlock(&matrix_mdev->kvm->lock); } mutex_unlock(&matrix_dev->guests_lock); diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 4e6bf1cb34750..a25b58e06f923 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -1083,7 +1083,7 @@ static long _zcrypt_send_ep11_cprb(u32 xflags, struct ap_perms *perms, print_hex_dump_debug("ep11req: ", DUMP_PREFIX_ADDRESS, 16, 1, ap_msg.msg, ap_msg.len, false); - if (perms != &ap_perms && domain < AUTOSEL_DOM) { + if (perms != &ap_perms && domain < AP_DOMAINS) { if (ap_msg.flags & AP_MSG_FLAG_ADMIN) { if (!test_bit_inv(domain, perms->adm)) { rc = -ENODEV; diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index fe61df34f6146..700c2d902b791 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -63,12 +64,18 @@ static DEFINE_MUTEX(dev_status_mem_mutex); * also checked. Returns 0 on success or errno value on failure. */ int cca_check_secaeskeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, int keybitsize) + const u8 *token, u32 keysize, int keybitsize) { struct secaeskeytoken *t = (struct secaeskeytoken *)token; #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + if (keysize < sizeof(*t)) { + if (dbg) + DBF("%s keysize %u < min token size %zu\n", + __func__, keysize, sizeof(*t)); + return -EINVAL; + } if (t->type != TOKTYPE_CCA_INTERNAL) { if (dbg) DBF("%s token check failed, type 0x%02x != 0x%02x\n", @@ -102,14 +109,20 @@ EXPORT_SYMBOL(cca_check_secaeskeytoken); * Returns 0 on success or errno value on failure. */ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, - const u8 *token, int keybitsize, - int checkcpacfexport) + const u8 *token, u32 keysize, + int keybitsize, int checkcpacfexport) { struct cipherkeytoken *t = (struct cipherkeytoken *)token; bool keybitsizeok = true; #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + if (keysize < sizeof(*t)) { + if (dbg) + DBF("%s keysize %u < min token size %zu\n", + __func__, keysize, sizeof(*t)); + return -EINVAL; + } if (t->type != TOKTYPE_CCA_INTERNAL) { if (dbg) DBF("%s token check failed, type 0x%02x != 0x%02x\n", @@ -122,6 +135,18 @@ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, __func__, (int)t->version, TOKVER_CCA_VLSC); return -EINVAL; } + if (t->len > keysize) { + if (dbg) + DBF("%s token check failed, len %d > keysize %u\n", + __func__, (int)t->len, keysize); + return -EINVAL; + } + if (t->len < sizeof(*t)) { + if (dbg) + DBF("%s token check failed, len %d < min token size %zu\n", + __func__, (int)t->len, sizeof(*t)); + return -EINVAL; + } if (t->algtype != 0x02) { if (dbg) DBF("%s token check failed, algtype 0x%02x != 0x02\n", @@ -196,6 +221,12 @@ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, #define DBF(...) debug_sprintf_event(dbg, dbflvl, ##__VA_ARGS__) + if (keysize < sizeof(*t)) { + if (dbg) + DBF("%s keysize %u < min token size %zu\n", + __func__, keysize, sizeof(*t)); + return -EINVAL; + } if (t->type != TOKTYPE_CCA_INTERNAL_PKA) { if (dbg) DBF("%s token check failed, type 0x%02x != 0x%02x\n", @@ -208,6 +239,12 @@ int cca_check_sececckeytoken(debug_info_t *dbg, int dbflvl, __func__, (int)t->len, keysize); return -EINVAL; } + if (t->len < sizeof(*t)) { + if (dbg) + DBF("%s token check failed, len %d < min token size %zu\n", + __func__, (int)t->len, sizeof(*t)); + return -EINVAL; + } if (t->secid != 0x20) { if (dbg) DBF("%s token check failed, secid 0x%02x != 0x20\n", @@ -232,6 +269,10 @@ EXPORT_SYMBOL(cca_check_sececckeytoken); * block, reply CPRB and reply param block and fill in values * for the common fields. Returns 0 on success or errno value * on failure. + * It is guaranteed that request and a possible param block + * are aligned to a 4 byte boundary. Furthermore if a param + * block is used, the memory allocated for this is rounded up to + * the next multiple of 4 bytes. */ static int alloc_and_prep_cprbmem(size_t paramblen, u8 **p_cprb_mem, @@ -240,7 +281,8 @@ static int alloc_and_prep_cprbmem(size_t paramblen, u32 xflags) { u8 *cprbmem = NULL; - size_t cprbplusparamblen = sizeof(struct CPRBX) + paramblen; + size_t cprbplusparamblen = + ALIGN(sizeof(struct CPRBX), 4) + ALIGN(paramblen, 4); size_t len = 2 * cprbplusparamblen; struct CPRBX *preqcblk, *prepcblk; @@ -267,10 +309,10 @@ static int alloc_and_prep_cprbmem(size_t paramblen, memcpy(preqcblk->func_id, "T2", 2); preqcblk->rpl_msgbl = cprbplusparamblen; if (paramblen) { - preqcblk->req_parmb = - ((u8 __user *)preqcblk) + sizeof(struct CPRBX); - preqcblk->rpl_parmb = - ((u8 __user *)prepcblk) + sizeof(struct CPRBX); + preqcblk->req_parmb = ((u8 __user *)preqcblk) + + ALIGN(sizeof(struct CPRBX), 4); + preqcblk->rpl_parmb = ((u8 __user *)prepcblk) + + ALIGN(sizeof(struct CPRBX), 4); } *p_cprb_mem = cprbmem; @@ -288,8 +330,10 @@ static int alloc_and_prep_cprbmem(size_t paramblen, */ static void free_cprbmem(void *mem, size_t paramblen, bool scrub, u32 xflags) { + size_t cprblen = ALIGN(sizeof(struct CPRBX), 4) + ALIGN(paramblen, 4); + if (mem && scrub) - memzero_explicit(mem, 2 * (sizeof(struct CPRBX) + paramblen)); + memzero_explicit(mem, 2 * cprblen); if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) mempool_free(mem, cprb_mempool); @@ -444,7 +488,8 @@ int cca_genseckey(u16 cardnr, u16 domain, /* check secure key token */ rc = cca_check_secaeskeytoken(zcrypt_dbf_info, DBF_ERR, - prepparm->lv3.keyblock.tok, 8 * keysize); + prepparm->lv3.keyblock.tok, + seckeysize, 8 * keysize); if (rc) { rc = -EIO; goto out; @@ -583,7 +628,8 @@ int cca_clr2seckey(u16 cardnr, u16 domain, u32 keybitsize, /* check secure key token */ rc = cca_check_secaeskeytoken(zcrypt_dbf_info, DBF_ERR, - prepparm->lv3.keyblock.tok, 8 * keysize); + prepparm->lv3.keyblock.tok, + seckeysize, 8 * keysize); if (rc) { rc = -EIO; goto out; @@ -843,6 +889,7 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, } kb; } __packed * prepparm; struct cipherkeytoken *t; + u32 keybuflen; /* get already prepared memory for 2 cprbs with param block each */ rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, @@ -937,23 +984,28 @@ int cca_gencipherkey(u16 cardnr, u16 domain, u32 keybitsize, u32 keygenflags, } /* and some checks on the generated key */ + t = (struct cipherkeytoken *)prepparm->kb.tlv1.gen_key; + if (prepparm->kb.tlv1.len < 2 * sizeof(uint16_t) + sizeof(*t)) { + rc = -EIO; + goto out; + } + keybuflen = prepparm->kb.tlv1.len - 2 * sizeof(uint16_t); rc = cca_check_secaescipherkey(zcrypt_dbf_info, DBF_ERR, prepparm->kb.tlv1.gen_key, - keybitsize, 1); + keybuflen, keybitsize, 1); if (rc) { rc = -EIO; goto out; } /* copy the generated vlsc key token */ - t = (struct cipherkeytoken *)prepparm->kb.tlv1.gen_key; if (keybuf) { - if (*keybufsize >= t->len) - memcpy(keybuf, t, t->len); + if (*keybufsize >= keybuflen) + memcpy(keybuf, t, keybuflen); else rc = -EINVAL; } - *keybufsize = t->len; + *keybufsize = keybuflen; out: free_cprbmem(mem, PARMBSIZE, false, xflags); @@ -972,7 +1024,8 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, int clr_key_bit_size, u8 *key_token, int *key_token_size, - u32 xflags) + u32 xflags, + bool scrub) { int rc, n; u8 *mem, *ptr; @@ -1112,7 +1165,7 @@ static int _ip_cprb_helper(u16 cardnr, u16 domain, *key_token_size = t->len; out: - free_cprbmem(mem, PARMBSIZE, false, xflags); + free_cprbmem(mem, PARMBSIZE, scrub, xflags); return rc; } @@ -1163,28 +1216,32 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, * 4/4 COMPLETE the secure cipher key import */ rc = _ip_cprb_helper(card, dom, "AES ", "FIRST ", "MIN3PART", - exorbuf, keybitsize, token, &tokensize, xflags); + exorbuf, keybitsize, token, &tokensize, + xflags, true); if (rc) { ZCRYPT_DBF_ERR("%s clear key import 1/4 with CSNBKPI2 failed, rc=%d\n", __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, - clrkey, keybitsize, token, &tokensize, xflags); + clrkey, keybitsize, token, &tokensize, + xflags, true); if (rc) { ZCRYPT_DBF_ERR("%s clear key import 2/4 with CSNBKPI2 failed, rc=%d\n", __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "ADD-PART", NULL, - exorbuf, keybitsize, token, &tokensize, xflags); + exorbuf, keybitsize, token, &tokensize, + xflags, true); if (rc) { ZCRYPT_DBF_ERR("%s clear key import 3/4 with CSNBKPI2 failed, rc=%d\n", __func__, rc); goto out; } rc = _ip_cprb_helper(card, dom, "AES ", "COMPLETE", NULL, - NULL, keybitsize, token, &tokensize, xflags); + NULL, keybitsize, token, &tokensize, + xflags, true); if (rc) { ZCRYPT_DBF_ERR("%s clear key import 4/4 with CSNBKPI2 failed, rc=%d\n", __func__, rc); @@ -1201,6 +1258,8 @@ int cca_clr2cipherkey(u16 card, u16 dom, u32 keybitsize, u32 keygenflags, *keybufsize = tokensize; out: + memzero_explicit(exorbuf, sizeof(exorbuf)); + memzero_explicit(mem, CPRB_MEMPOOL_ITEM_SIZE); mempool_free(mem, cprb_mempool); return rc; } @@ -1262,6 +1321,9 @@ int cca_cipher2protkey(u16 cardnr, u16 domain, const u8 *ckey, } __packed * prepparm; int keytoklen = ((struct cipherkeytoken *)ckey)->len; + if (keytoklen > PARMBSIZE - sizeof(struct aureqparm)) + return -EINVAL; + /* get already prepared memory for 2 cprbs with param block each */ rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk, xflags); @@ -1426,6 +1488,9 @@ int cca_ecc2protkey(u16 cardnr, u16 domain, const u8 *key, } __packed * prepparm; int keylen = ((struct eccprivkeytoken *)key)->len; + if (keylen > PARMBSIZE - sizeof(struct aureqparm)) + return -EINVAL; + /* get already prepared memory for 2 cprbs with param block each */ rc = alloc_and_prep_cprbmem(PARMBSIZE, &mem, &preqcblk, &prepcblk, xflags); diff --git a/drivers/s390/crypto/zcrypt_ccamisc.h b/drivers/s390/crypto/zcrypt_ccamisc.h index 1ecc4e37e9ad3..5f5bd93e09318 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.h +++ b/drivers/s390/crypto/zcrypt_ccamisc.h @@ -136,7 +136,7 @@ struct eccprivkeytoken { * also checked. Returns 0 on success or errno value on failure. */ int cca_check_secaeskeytoken(debug_info_t *dbg, int dbflvl, - const u8 *token, int keybitsize); + const u8 *token, u32 keysize, int keybitsize); /* * Simple check if the token is a valid CCA secure AES cipher key @@ -146,8 +146,8 @@ int cca_check_secaeskeytoken(debug_info_t *dbg, int dbflvl, * Returns 0 on success or errno value on failure. */ int cca_check_secaescipherkey(debug_info_t *dbg, int dbflvl, - const u8 *token, int keybitsize, - int checkcpacfexport); + const u8 *token, u32 keysize, + int keybitsize, int checkcpacfexport); /* * Simple check if the token is a valid CCA secure ECC private diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index e92e2fd8ce5da..6d2064555e1a0 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -356,21 +357,24 @@ EXPORT_SYMBOL(ep11_check_aes_key); /* * Allocate and prepare ep11 cprb plus additional payload. + * It is guaranteed that the memory is aligned to a 4 byte boundary. + * Furthermore the memory allocation is rounded up to the next + * multiple of 4 bytes (with taking the payload_len into account). */ static void *alloc_cprbmem(size_t payload_len, u32 xflags) { - size_t len = sizeof(struct ep11_cprb) + payload_len; + size_t memlen = ALIGN(sizeof(struct ep11_cprb) + payload_len, 4); struct ep11_cprb *cprb = NULL; if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) { - if (len <= CPRB_MEMPOOL_ITEM_SIZE) + if (memlen <= CPRB_MEMPOOL_ITEM_SIZE) cprb = mempool_alloc_preallocated(cprb_mempool); } else { - cprb = kmalloc(len, GFP_KERNEL); + cprb = kmalloc(memlen, GFP_KERNEL); } if (!cprb) return NULL; - memset(cprb, 0, len); + memset(cprb, 0, memlen); cprb->cprb_len = sizeof(struct ep11_cprb); cprb->cprb_ver_id = 0x04; @@ -386,8 +390,10 @@ static void *alloc_cprbmem(size_t payload_len, u32 xflags) */ static void free_cprbmem(void *mem, size_t payload_len, bool scrub, u32 xflags) { + size_t memlen = ALIGN(sizeof(struct ep11_cprb) + payload_len, 4); + if (mem && scrub) - memzero_explicit(mem, sizeof(struct ep11_cprb) + payload_len); + memzero_explicit(mem, memlen); if (xflags & ZCRYPT_XFLAG_NOMEMALLOC) mempool_free(mem, cprb_mempool); diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index f84aa2e676e9b..da1e0baacc067 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -149,13 +149,16 @@ static int unregister_sba(struct ism_dev *ism) if (ret && ret != ISM_ERROR) return -EIO; + return 0; +} + +static void ism_free_sba(struct ism_dev *ism) +{ dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, ism->sba, ism->sba_dma_addr); ism->sba = NULL; ism->sba_dma_addr = 0; - - return 0; } static int unregister_ieq(struct ism_dev *ism) @@ -169,13 +172,16 @@ static int unregister_ieq(struct ism_dev *ism) if (ret && ret != ISM_ERROR) return -EIO; + return 0; +} + +static void ism_free_ieq(struct ism_dev *ism) +{ dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, ism->ieq, ism->ieq_dma_addr); ism->ieq = NULL; ism->ieq_dma_addr = 0; - - return 0; } static int ism_read_local_gid(struct dibs_dev *dibs) @@ -226,6 +232,7 @@ static void ism_free_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len, DMA_FROM_DEVICE); folio_put(virt_to_folio(dmb->cpu_addr)); + dmb->cpu_addr = NULL; } static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) @@ -269,7 +276,8 @@ static int ism_alloc_dmb(struct ism_dev *ism, struct dibs_dmb *dmb) return 0; out_free: - kfree(dmb->cpu_addr); + folio_put(folio); + dmb->cpu_addr = NULL; out_bit: clear_bit(dmb->idx, ism->sba_bitmap); return rc; @@ -574,6 +582,7 @@ static int ism_dev_init(struct ism_dev *ism) unreg_sba: unregister_sba(ism); + ism_free_sba(ism); free_irq: free_irq(pci_irq_vector(pdev, 0), ism); free_vectors: @@ -586,9 +595,13 @@ static void ism_dev_exit(struct ism_dev *ism) { struct pci_dev *pdev = ism->pdev; + /* ism will only generate new IRQs while ieq & sba are registered */ unregister_ieq(ism); unregister_sba(ism); + /* drain ongoing irpt handlers */ free_irq(pci_irq_vector(pdev, 0), ism); + ism_free_ieq(ism); + ism_free_sba(ism); pci_free_irq_vectors(pdev); } diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c index edc0bcd46923c..c524b4b0a328e 100644 --- a/drivers/s390/net/qeth_core_main.c +++ b/drivers/s390/net/qeth_core_main.c @@ -4712,6 +4712,9 @@ static int qeth_snmp_command(struct qeth_card *card, char __user *udata) if (req_len > QETH_BUFSIZE) return -EINVAL; + if (qinfo.udata_len < sizeof(struct qeth_snmp_ureq_hdr)) + return -EINVAL; + iob = qeth_get_adapter_cmd(card, IPA_SETADP_SET_SNMP_CONTROL, req_len); if (!iob) return -ENOMEM; @@ -6528,6 +6531,9 @@ int qeth_siocdevprivate(struct net_device *dev, struct ifreq *rq, void __user *d struct qeth_card *card = dev->ml_priv; int rc = 0; + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + switch (cmd) { case SIOC_QETH_ADP_SET_SNMP_CONTROL: rc = qeth_snmp_command(card, data); diff --git a/drivers/s390/net/qeth_l2.h b/drivers/s390/net/qeth_l2.h index 7c646e2fed7e5..f94975e970cad 100644 --- a/drivers/s390/net/qeth_l2.h +++ b/drivers/s390/net/qeth_l2.h @@ -13,7 +13,8 @@ extern const struct attribute_group *qeth_l2_attr_groups[]; int qeth_bridgeport_query_ports(struct qeth_card *card, enum qeth_sbp_roles *role, - enum qeth_sbp_states *state); + enum qeth_sbp_states *state, + bool *os_mismatch); int qeth_bridgeport_setrole(struct qeth_card *card, enum qeth_sbp_roles role); int qeth_bridgeport_an_set(struct qeth_card *card, int enable); diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index 2a3888283a943..9c7e2120cf376 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -1159,7 +1159,7 @@ static void qeth_l2_setup_bridgeport_attrs(struct qeth_card *card) qeth_bridgeport_setrole(card, card->options.sbp.role); /* Let the callback function refresh the stored role value. */ qeth_bridgeport_query_ports(card, &card->options.sbp.role, - NULL); + NULL, NULL); } if (card->options.sbp.hostnotification) { if (qeth_bridgeport_an_set(card, 1)) @@ -1546,6 +1546,7 @@ struct _qeth_sbp_cbctl { struct { enum qeth_sbp_roles *role; enum qeth_sbp_states *state; + bool *os_mismatch; } qports; } data; }; @@ -1722,10 +1723,19 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, struct qeth_ipa_cmd *cmd = (struct qeth_ipa_cmd *) data; struct _qeth_sbp_cbctl *cbctl = (struct _qeth_sbp_cbctl *)reply->param; struct qeth_sbp_port_data *qports; + u16 sbp_rc; int rc; QETH_CARD_TEXT(card, 2, "brqprtcb"); - rc = qeth_bridgeport_makerc(card, cmd); + sbp_rc = cmd->data.sbp.hdr.return_code; + + /* on OS family mismatch, query still returns valid port data; + * treat as success + */ + if (sbp_rc == IPA_RC_SBP_IQD_OS_MISMATCH && !cmd->hdr.return_code) + rc = 0; + else + rc = qeth_bridgeport_makerc(card, cmd); if (rc) return rc; @@ -1741,6 +1751,9 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, if (cbctl->data.qports.state) *cbctl->data.qports.state = qports->entry[0].state; } + if (cbctl->data.qports.os_mismatch) + *cbctl->data.qports.os_mismatch = + (sbp_rc == IPA_RC_SBP_IQD_OS_MISMATCH); return 0; } @@ -1749,13 +1762,17 @@ static int qeth_bridgeport_query_ports_cb(struct qeth_card *card, * @card: qeth_card structure pointer. * @role: Role of the port: 0-none, 1-primary, 2-secondary. * @state: State of the port: 0-inactive, 1-standby, 2-active. + * @os_mismatch: if non-NULL, set to true when firmware reports + * OS family mismatch. * * Returns negative errno-compatible error indication or 0 on success. * - * 'role' and 'state' are not updated in case of hardware operation failure. + * 'role', 'state' and 'os_mismatch' are not updated in case of + * hardware operation failure. */ int qeth_bridgeport_query_ports(struct qeth_card *card, - enum qeth_sbp_roles *role, enum qeth_sbp_states *state) + enum qeth_sbp_roles *role, enum qeth_sbp_states *state, + bool *os_mismatch) { struct qeth_cmd_buffer *iob; struct _qeth_sbp_cbctl cbctl = { @@ -1763,6 +1780,7 @@ int qeth_bridgeport_query_ports(struct qeth_card *card, .qports = { .role = role, .state = state, + .os_mismatch = os_mismatch, }, }, }; diff --git a/drivers/s390/net/qeth_l2_sys.c b/drivers/s390/net/qeth_l2_sys.c index 7f592f9125172..7101be62eb1d5 100644 --- a/drivers/s390/net/qeth_l2_sys.c +++ b/drivers/s390/net/qeth_l2_sys.c @@ -15,6 +15,7 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, { struct qeth_card *card = dev_get_drvdata(dev); enum qeth_sbp_states state = QETH_SBP_STATE_INACTIVE; + bool os_mismatch = false; int rc = 0; char *word; @@ -25,7 +26,7 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, if (qeth_card_hw_is_reachable(card) && card->options.sbp.supported_funcs) rc = qeth_bridgeport_query_ports(card, - &card->options.sbp.role, &state); + &card->options.sbp.role, &state, &os_mismatch); if (!rc) { if (show_state) switch (state) { @@ -52,6 +53,10 @@ static ssize_t qeth_bridge_port_role_state_show(struct device *dev, if (rc) QETH_CARD_TEXT_(card, 2, "SBP%02x:%02x", card->options.sbp.role, state); + else if (!show_state && + card->options.sbp.role == QETH_SBP_ROLE_NONE && + os_mismatch) + rc = sysfs_emit(buf, "%s (OS family mismatch)\n", word); else rc = sysfs_emit(buf, "%s\n", word); } diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c index 3525be8193624..2621839fb8c38 100644 --- a/drivers/s390/net/qeth_l3_main.c +++ b/drivers/s390/net/qeth_l3_main.c @@ -1416,6 +1416,11 @@ static int qeth_l3_arp_query(struct qeth_card *card, char __user *udata) rc = -EFAULT; goto out; } + + if (qinfo.udata_len < QETH_QARP_ENTRIES_OFFSET) { + rc = -EINVAL; + goto out; + } qinfo.udata = kzalloc(qinfo.udata_len, GFP_KERNEL); if (!qinfo.udata) { rc = -ENOMEM; diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c index dc2265ebb11b8..03f723042434b 100644 --- a/drivers/s390/scsi/zfcp_aux.c +++ b/drivers/s390/scsi/zfcp_aux.c @@ -254,6 +254,7 @@ static int zfcp_allocate_low_mem_buffers(struct zfcp_adapter *adapter) static void zfcp_free_low_mem_buffers(struct zfcp_adapter *adapter) { mempool_destroy(adapter->pool.erp_req); + mempool_destroy(adapter->pool.gid_pn_req); mempool_destroy(adapter->pool.scsi_req); mempool_destroy(adapter->pool.scsi_abort); mempool_destroy(adapter->pool.qtcb_pool); diff --git a/drivers/scsi/bfa/bfa_fcs_lport.c b/drivers/scsi/bfa/bfa_fcs_lport.c index 9a85f417018f9..cabe408214b3f 100644 --- a/drivers/scsi/bfa/bfa_fcs_lport.c +++ b/drivers/scsi/bfa/bfa_fcs_lport.c @@ -2627,7 +2627,7 @@ bfa_fcs_fdmi_get_hbaattr(struct bfa_fcs_lport_fdmi_s *fdmi, } -static void +static noinline_for_stack void bfa_fcs_fdmi_get_portattr(struct bfa_fcs_lport_fdmi_s *fdmi, struct bfa_fcs_fdmi_port_attr_s *port_attr) { diff --git a/drivers/scsi/fnic/fip.c b/drivers/scsi/fnic/fip.c index ce62ab1180bd7..e08a4f5ac2176 100644 --- a/drivers/scsi/fnic/fip.c +++ b/drivers/scsi/fnic/fip.c @@ -139,7 +139,7 @@ void fnic_fcoe_process_vlan_resp(struct fnic *fnic, struct fip_header *fiph) FNIC_FIP_DBG(KERN_INFO, fnic->host, fnic->fnic_num, "process_vlan_resp: FIP VLAN %d\n", vid); - vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); + vlan = kzalloc(sizeof(*vlan), GFP_ATOMIC); if (!vlan) { /* retry from timer */ diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c index f30d4a58d7ab5..16256a0529253 100644 --- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c +++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c @@ -5261,15 +5261,7 @@ static int _resume_v3_hw(struct device *device) return rc; } phys_init_v3_hw(hisi_hba); - - /* - * If a directly-attached disk is removed during suspend, a deadlock - * may occur, as the PHYE_RESUME_TIMEOUT processing will require the - * hisi_hba->device to be active, which can only happen when resume - * completes. So don't wait for the HA event workqueue to drain upon - * resume. - */ - sas_resume_ha_no_sync(sha); + sas_resume_ha(sha); clear_bit(HISI_SAS_RESETTING_BIT, &hisi_hba->flags); dev_warn(dev, "end of resuming controller\n"); diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c index 17173239301e6..a1055fe4922f6 100644 --- a/drivers/scsi/hosts.c +++ b/drivers/scsi/hosts.c @@ -343,6 +343,7 @@ static void scsi_host_dev_release(struct device *dev) /* Wait for functions invoked through call_rcu(&scmd->rcu, ...) */ rcu_barrier(); + cancel_work_sync(&shost->eh_work); if (shost->tmf_work_q) destroy_workqueue(shost->tmf_work_q); if (shost->ehandler) @@ -408,6 +409,7 @@ struct Scsi_Host *scsi_host_alloc(const struct scsi_host_template *sht, int priv INIT_LIST_HEAD(&shost->starved_list); init_waitqueue_head(&shost->host_wait); mutex_init(&shost->scan_mutex); + INIT_WORK(&shost->eh_work, scsi_rcu_eh_wakeup); index = ida_alloc(&host_index_ida, GFP_KERNEL); if (index < 0) { diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c index c9f410c509783..e2adebc80b3a9 100644 --- a/drivers/scsi/libiscsi.c +++ b/drivers/scsi/libiscsi.c @@ -918,7 +918,7 @@ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr, } senselen = get_unaligned_be16(data); - if (datalen < senselen) + if (datalen < senselen + 2) goto invalid_datalen; memcpy(sc->sense_buffer, data + 2, diff --git a/drivers/scsi/libiscsi_tcp.c b/drivers/scsi/libiscsi_tcp.c index e90805ba868fb..7223bb18b0480 100644 --- a/drivers/scsi/libiscsi_tcp.c +++ b/drivers/scsi/libiscsi_tcp.c @@ -752,13 +752,6 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr) rc = __iscsi_complete_pdu(conn, hdr, NULL, 0); spin_unlock(&conn->session->back_lock); break; - case ISCSI_OP_SCSI_CMD_RSP: - if (tcp_conn->in.datalen) { - iscsi_tcp_data_recv_prep(tcp_conn); - return 0; - } - rc = iscsi_complete_pdu(conn, hdr, NULL, 0); - break; case ISCSI_OP_R2T: if (ahslen) { rc = ISCSI_ERR_AHSLEN; @@ -766,6 +759,7 @@ iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr) } rc = iscsi_tcp_r2t_rsp(conn, hdr); break; + case ISCSI_OP_SCSI_CMD_RSP: case ISCSI_OP_LOGIN_RSP: case ISCSI_OP_TEXT_RSP: case ISCSI_OP_REJECT: diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c index 8566bb1208a05..ac157ab6a3011 100644 --- a/drivers/scsi/libsas/sas_init.c +++ b/drivers/scsi/libsas/sas_init.c @@ -409,7 +409,7 @@ static void sas_resume_insert_broadcast_ha(struct sas_ha_struct *ha) } } -static void _sas_resume_ha(struct sas_ha_struct *ha, bool drain) +void sas_resume_ha(struct sas_ha_struct *ha) { const unsigned long tmo = msecs_to_jiffies(25000); int i; @@ -425,6 +425,23 @@ static void _sas_resume_ha(struct sas_ha_struct *ha, bool drain) dev_info(ha->dev, "waiting up to 25 seconds for %d phy%s to resume\n", i, i > 1 ? "s" : ""); wait_event_timeout(ha->eh_wait_q, phys_suspended(ha) == 0, tmo); + + /* + * All phys are back up or timed out. Turn on I/O and drain + * pending work. + */ + scsi_unblock_requests(ha->shost); + sas_drain_work(ha); + + /* + * Send PHYE_RESUME_TIMEOUT after sas_drain_work(). The handler + * calls sas_deform_port() -> sas_destruct_devices(), which removes + * SCSI devices and, for LLDDs using device_link() PM sync, waits + * for the host to be runtime-active. Sending it before the drain + * would deadlock: the drain waits for the handler, the handler + * waits for host resume, and host resume waits for the drain to + * finish. + */ for (i = 0; i < ha->num_phys; i++) { struct asd_sas_phy *phy = ha->sas_phy[i]; @@ -435,12 +452,6 @@ static void _sas_resume_ha(struct sas_ha_struct *ha, bool drain) } } - /* all phys are back up or timed out, turn on i/o so we can - * flush out disks that did not return - */ - scsi_unblock_requests(ha->shost); - if (drain) - sas_drain_work(ha); clear_bit(SAS_HA_RESUMING, &ha->state); sas_queue_deferred_work(ha); @@ -449,20 +460,8 @@ static void _sas_resume_ha(struct sas_ha_struct *ha, bool drain) */ sas_resume_insert_broadcast_ha(ha); } - -void sas_resume_ha(struct sas_ha_struct *ha) -{ - _sas_resume_ha(ha, true); -} EXPORT_SYMBOL(sas_resume_ha); -/* A no-sync variant, which does not call sas_drain_ha(). */ -void sas_resume_ha_no_sync(struct sas_ha_struct *ha) -{ - _sas_resume_ha(ha, false); -} -EXPORT_SYMBOL(sas_resume_ha_no_sync); - void sas_suspend_ha(struct sas_ha_struct *ha) { int i; diff --git a/drivers/scsi/megaraid/megaraid_sas_base.c b/drivers/scsi/megaraid/megaraid_sas_base.c index abbbc4b36cd1d..d578b476d9797 100644 --- a/drivers/scsi/megaraid/megaraid_sas_base.c +++ b/drivers/scsi/megaraid/megaraid_sas_base.c @@ -1973,12 +1973,23 @@ megasas_set_nvme_device_properties(struct scsi_device *sdev, { struct megasas_instance *instance; u32 mr_nvme_pg_size; + u64 max_prp_io; instance = (struct megasas_instance *)sdev->host->hostdata; mr_nvme_pg_size = max_t(u32, instance->nvme_page_size, MR_DEFAULT_NVME_PAGE_SIZE); - lim->max_hw_sectors = max_io_size / 512; + /* + * megasas_make_prp_nvme() builds the PRP list in cmd->sg_frame without + * bounding it against that buffer, and spends one entry per page of + * it on the chain pointer. Cap the transfer at what the buffer holds, + * less one page for lists that start off a page boundary. + */ + max_prp_io = (u64)((instance->max_chain_frame_sz / sizeof(u64)) - + (instance->max_chain_frame_sz / mr_nvme_pg_size) - 1) * + mr_nvme_pg_size; + + lim->max_hw_sectors = min_t(u64, max_io_size, max_prp_io) >> SECTOR_SHIFT; lim->virt_boundary_mask = mr_nvme_pg_size - 1; } diff --git a/drivers/scsi/mpi3mr/mpi3mr_transport.c b/drivers/scsi/mpi3mr/mpi3mr_transport.c index d70f002d6487d..a34b0f2e1f5b2 100644 --- a/drivers/scsi/mpi3mr/mpi3mr_transport.c +++ b/drivers/scsi/mpi3mr/mpi3mr_transport.c @@ -1430,9 +1430,15 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc, } port = sas_port_alloc_num(mr_sas_node->parent_dev); + if (!port) { + ioc_err(mrioc, "failure at %s:%d/%s()!\n", + __FILE__, __LINE__, __func__); + goto out_fail; + } if ((sas_port_add(port))) { ioc_err(mrioc, "failure at %s:%d/%s()!\n", __FILE__, __LINE__, __func__); + sas_port_free(port); goto out_fail; } @@ -1503,6 +1509,8 @@ static struct mpi3mr_sas_port *mpi3mr_sas_port_add(struct mpi3mr_ioc *mrioc, list_for_each_entry_safe(mr_sas_phy, next, &mr_sas_port->phy_list, port_siblings) list_del(&mr_sas_phy->port_siblings); + if (tgtdev) + mpi3mr_tgtdev_put(tgtdev); kfree(mr_sas_port); return NULL; } diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index 0d652db8fe249..e5e5f0b8f34f5 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -3238,7 +3238,10 @@ _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc) * corresponding to high iops queues. */ if (ioc->high_iops_queues) { - mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev)); + int node = dev_to_node(&ioc->pdev->dev); + + mask = (node == NUMA_NO_NODE) ? + cpu_online_mask : cpumask_of_node(node); for (index = 0; index < ioc->high_iops_queues; index++) { irq = pci_irq_vector(ioc->pdev, index); @@ -5871,6 +5874,8 @@ _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc) if (ioc->pcie_sgl_dma_pool) { for (i = 0; i < ioc->scsiio_depth; i++) { + if (!ioc->pcie_sg_lookup[i].pcie_sgl) + continue; dma_pool_free(ioc->pcie_sgl_dma_pool, ioc->pcie_sg_lookup[i].pcie_sgl, ioc->pcie_sg_lookup[i].pcie_sgl_dma); diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c index c10854ec44c7b..a27f3287748ee 100644 --- a/drivers/scsi/pm8001/pm8001_ctl.c +++ b/drivers/scsi/pm8001/pm8001_ctl.c @@ -823,6 +823,14 @@ static ssize_t pm8001_store_update_fw(struct device *cdev, goto out; } + if (pm8001_ha->controller_fatal_error) { + pm8001_dbg(pm8001_ha, FAIL, + "controller in fatal error state, firmware update rejected\n"); + pm8001_ha->fw_status = FAIL_PARAMETERS; + ret = -EINVAL; + goto out; + } + for (i = 0; flash_command_table[i].code != FLASH_CMD_NONE; i++) { if (!memcmp(flash_command_table[i].command, cmd_ptr, strlen(cmd_ptr))) { diff --git a/drivers/scsi/pm8001/pm8001_init.c b/drivers/scsi/pm8001/pm8001_init.c index 8ff4b89ff81e2..10649b306912d 100644 --- a/drivers/scsi/pm8001/pm8001_init.c +++ b/drivers/scsi/pm8001/pm8001_init.c @@ -1029,8 +1029,8 @@ static u32 pm8001_request_msix(struct pm8001_hba_info *pm8001_ha) &(pm8001_ha->irq_vector[i])); if (rc) { for (j = 0; j < i; j++) { - free_irq(pci_irq_vector(pm8001_ha->pdev, i), - &(pm8001_ha->irq_vector[i])); + free_irq(pci_irq_vector(pm8001_ha->pdev, j), + &pm8001_ha->irq_vector[j]); } pci_free_irq_vectors(pm8001_ha->pdev); break; diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c index 31960b72c1e92..1ce8226fcd292 100644 --- a/drivers/scsi/pm8001/pm80xx_hwi.c +++ b/drivers/scsi/pm8001/pm80xx_hwi.c @@ -401,6 +401,13 @@ ssize_t pm80xx_get_non_fatal_dump(struct device *cdev, char *buf_copy = buf; temp = (u32 *)pm8001_ha->memoryMap.region[FORENSIC_MEM].virt_ptr; + + if (pm8001_ha->controller_fatal_error) { + pm8001_dbg(pm8001_ha, FAIL, + "non-fatal dump not available in fatal error state\n"); + return -EINVAL; + } + if (++pm8001_ha->non_fatal_count == 1) { if (pm8001_ha->chip_id == chip_8001) { snprintf(pm8001_ha->forensic_info.data_buf.direct_data, diff --git a/drivers/scsi/qla2xxx/qla_attr.c b/drivers/scsi/qla2xxx/qla_attr.c index 2e584a8bf66b2..0768b3aad50b5 100644 --- a/drivers/scsi/qla2xxx/qla_attr.c +++ b/drivers/scsi/qla2xxx/qla_attr.c @@ -580,7 +580,6 @@ qla2x00_sysfs_read_vpd(struct file *filp, struct kobject *kobj, ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); mutex_unlock(&ha->optrom_mutex); - ha->isp_ops->read_optrom(vha, ha->vpd, faddr, ha->vpd_size); skip: return memory_read_from_buffer(buf, count, &off, ha->vpd, ha->vpd_size); } @@ -816,7 +815,9 @@ qla2x00_sysfs_write_reset(struct file *filp, struct kobject *kobj, "Unable to allocate memory for VPD information update.\n"); return -ENOMEM; } + mutex_lock(&ha->optrom_mutex); ha->isp_ops->get_flash_version(vha, tmp_data); + mutex_unlock(&ha->optrom_mutex); vfree(tmp_data); break; } @@ -1660,10 +1661,8 @@ qla2x00_fw_state_show(struct device *dev, struct device_attribute *attr, rval = qla2x00_get_firmware_state(vha, state); mutex_unlock(&vha->hw->optrom_mutex); out: - if (rval != QLA_SUCCESS) { + if (rval != QLA_SUCCESS) memset(state, -1, sizeof(state)); - rval = qla2x00_get_firmware_state(vha, state); - } return scnprintf(buf, PAGE_SIZE, "0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", state[0], state[1], state[2], state[3], state[4], state[5]); diff --git a/drivers/scsi/qla2xxx/qla_bsg.c b/drivers/scsi/qla2xxx/qla_bsg.c index 0798bfd0372e4..93fd77c009006 100644 --- a/drivers/scsi/qla2xxx/qla_bsg.c +++ b/drivers/scsi/qla2xxx/qla_bsg.c @@ -158,6 +158,12 @@ qla24xx_proc_fcp_prio_cfg_cmd(struct bsg_job *bsg_job) goto exit_fcp_prio_cfg; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) { + ret = -EINVAL; + goto exit_fcp_prio_cfg; + } + /* Get the sub command */ oper = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; @@ -756,6 +762,10 @@ qla2x00_process_loopback(struct bsg_job *bsg_job) return -EIO; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 3 * sizeof(uint32_t)) + return -EINVAL; + memset(&elreq, 0, sizeof(elreq)); elreq.req_sg_cnt = dma_map_sg(&ha->pdev->dev, @@ -988,6 +998,10 @@ qla84xx_reset(struct bsg_job *bsg_job) return -EINVAL; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + flag = bsg_request->rqst_data.h_vendor.vendor_cmd[1]; rval = qla84xx_reset_chip(vha, flag == A84_ISSUE_RESET_DIAG_FW); @@ -1032,6 +1046,10 @@ qla84xx_updatefw(struct bsg_job *bsg_job) return -EINVAL; } + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + sg_cnt = dma_map_sg(&ha->pdev->dev, bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, DMA_TO_DEVICE); if (!sg_cnt) { @@ -1482,6 +1500,10 @@ qla2x00_read_optrom(struct bsg_job *bsg_job) struct qla_hw_data *ha = vha->hw; int rval = 0; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + if (ha->flags.nic_core_reset_hdlr_active) return -EBUSY; @@ -1519,6 +1541,10 @@ qla2x00_update_optrom(struct bsg_job *bsg_job) struct qla_hw_data *ha = vha->hw; int rval = 0; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + 2 * sizeof(uint32_t)) + return -EINVAL; + mutex_lock(&ha->optrom_mutex); rval = qla2x00_optrom_setup(bsg_job, vha, 1); if (rval) { @@ -1560,12 +1586,12 @@ qla2x00_update_fru_versions(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_image_version_list *list = (void *)bsg; struct qla_image_version *image; uint32_t count; dma_addr_t sfp_dma; - void *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + void *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1578,6 +1604,13 @@ qla2x00_update_fru_versions(struct bsg_job *bsg_job) image = list->version; count = list->count; + + if (struct_size(list, version, count) > sizeof(bsg)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + while (count--) { memcpy(sfp, &image->field_info, sizeof(image->field_info)); rval = qla2x00_write_sfp(vha, sfp_dma, sfp, @@ -1613,10 +1646,10 @@ qla2x00_read_fru_status(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_status_reg *sr = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1664,10 +1697,10 @@ qla2x00_write_fru_status(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_status_reg *sr = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1711,10 +1744,10 @@ qla2x00_write_i2c(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_i2c_access *i2c = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1725,6 +1758,12 @@ qla2x00_write_i2c(struct bsg_job *bsg_job) sg_copy_to_buffer(bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c)); + if (i2c->length > sizeof(i2c->buffer)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + memcpy(sfp, i2c->buffer, i2c->length); rval = qla2x00_write_sfp(vha, sfp_dma, sfp, i2c->device, i2c->offset, i2c->length, i2c->option); @@ -1757,10 +1796,10 @@ qla2x00_read_i2c(struct bsg_job *bsg_job) scsi_qla_host_t *vha = shost_priv(host); struct qla_hw_data *ha = vha->hw; int rval = 0; - uint8_t bsg[DMA_POOL_SIZE]; + uint8_t bsg[DMA_POOL_SIZE] = {}; struct qla_i2c_access *i2c = (void *)bsg; dma_addr_t sfp_dma; - uint8_t *sfp = dma_pool_alloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); + uint8_t *sfp = dma_pool_zalloc(ha->s_dma_pool, GFP_KERNEL, &sfp_dma); if (!sfp) { bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = @@ -1771,6 +1810,12 @@ qla2x00_read_i2c(struct bsg_job *bsg_job) sg_copy_to_buffer(bsg_job->request_payload.sg_list, bsg_job->request_payload.sg_cnt, i2c, sizeof(*i2c)); + if (i2c->length > sizeof(i2c->buffer)) { + bsg_reply->reply_data.vendor_reply.vendor_rsp[0] = + EXT_STATUS_INVALID_PARAM; + goto dealloc; + } + rval = qla2x00_read_sfp(vha, sfp_dma, sfp, i2c->device, i2c->offset, i2c->length, i2c->option); @@ -1991,6 +2036,11 @@ qlafx00_mgmt_cmd(struct bsg_job *bsg_job) struct fc_port *fcport; char *type = "FC_BSG_HST_FX_MGMT"; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + sizeof(uint32_t) + + sizeof(struct qla_mt_iocb_rqst_fx00)) + return -EINVAL; + /* Copy the IOCB specific information */ piocb_rqst = (struct qla_mt_iocb_rqst_fx00 *) &bsg_request->rqst_data.h_vendor.vendor_cmd[1]; @@ -2393,7 +2443,7 @@ qla2x00_do_dport_diagnostics(struct bsg_job *bsg_job) !IS_QLA28XX(vha->hw)) return -EPERM; - dd = kmalloc(sizeof(*dd), GFP_KERNEL); + dd = kzalloc(sizeof(*dd), GFP_KERNEL); if (!dd) { ql_log(ql_log_warn, vha, 0x70db, "Failed to allocate memory for dport.\n"); @@ -2880,6 +2930,13 @@ qla2x00_process_vendor_specific(struct scsi_qla_host *vha, struct bsg_job *bsg_j { struct fc_bsg_request *bsg_request = bsg_job->request; + if (bsg_job->request_len < + sizeof(struct fc_bsg_request) + sizeof(uint32_t)) { + ql_log(ql_log_warn, vha, 0x7000, + "BSG request too small for vendor cmd.\n"); + return -EINVAL; + } + ql_dbg(ql_dbg_edif, vha, 0x911b, "%s FC_BSG_HST_VENDOR cmd[0]=0x%x\n", __func__, bsg_request->rqst_data.h_vendor.vendor_cmd[0]); @@ -3005,8 +3062,11 @@ qla24xx_bsg_request(struct bsg_job *bsg_job) } /* Disable port will bring down the chip, allow enable command */ - if (bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_MANAGE_HOST_PORT || - bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_GET_HOST_STATS) + if (bsg_request->msgcode == FC_BSG_HST_VENDOR && + bsg_job->request_len >= + sizeof(struct fc_bsg_request) + sizeof(uint32_t) && + (bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_MANAGE_HOST_PORT || + bsg_request->rqst_data.h_vendor.vendor_cmd[0] == QL_VND_GET_HOST_STATS)) goto skip_chip_chk; if (vha->hw->flags.port_isolated) { diff --git a/drivers/scsi/qla2xxx/qla_dbg.c b/drivers/scsi/qla2xxx/qla_dbg.c index 5136549005e7e..8f7fbaa1556e8 100644 --- a/drivers/scsi/qla2xxx/qla_dbg.c +++ b/drivers/scsi/qla2xxx/qla_dbg.c @@ -16,7 +16,7 @@ * | | | 0x2127-0x2128 | * | Queue Command and IO tracing | 0x3074 | 0x300b | * | | | 0x3027-0x3028 | - * | | | 0x303d-0x3041 | + * | | | 0x303e-0x3041 | * | | | 0x302e,0x3033 | * | | | 0x3036,0x3038 | * | | | 0x303a | diff --git a/drivers/scsi/qla2xxx/qla_def.h b/drivers/scsi/qla2xxx/qla_def.h index b3265952c4bed..783b6c429224a 100644 --- a/drivers/scsi/qla2xxx/qla_def.h +++ b/drivers/scsi/qla2xxx/qla_def.h @@ -2570,6 +2570,8 @@ typedef struct fc_port { struct list_head list; struct scsi_qla_host *vha; struct list_head unsol_ctx_head; + /* Serializes unsol_ctx_head against ISR, DPC and NVMe transport. */ + spinlock_t unsol_ctx_lock; unsigned int conf_compl_supported:1; unsigned int deleted:2; diff --git a/drivers/scsi/qla2xxx/qla_dfs.c b/drivers/scsi/qla2xxx/qla_dfs.c index 43970caca7b39..cb4278b88e1d0 100644 --- a/drivers/scsi/qla2xxx/qla_dfs.c +++ b/drivers/scsi/qla2xxx/qla_dfs.c @@ -510,7 +510,9 @@ qla2x00_dfs_fce_write(struct file *file, const char __user *buffer, return PTR_ERR(buf); } - enable = kstrtoul(buf, 0, 0); + rc = kstrtoul(buf, 0, &enable); + if (rc) + goto out_free; rc = count; mutex_lock(&ha->fce_mutex); diff --git a/drivers/scsi/qla2xxx/qla_edif.c b/drivers/scsi/qla2xxx/qla_edif.c index ccd4485087a10..8c73956df5108 100644 --- a/drivers/scsi/qla2xxx/qla_edif.c +++ b/drivers/scsi/qla2xxx/qla_edif.c @@ -3492,6 +3492,9 @@ void qla_chk_edif_rx_sa_delete_pending(scsi_qla_host_t *vha, struct scsi_cmnd *cmd = GET_CMD_SP(sp); uint32_t handle; + if (!cmd) + return; + handle = (uint32_t)LSW(sts24->handle); /* find out if this status iosb is for a scsi read */ diff --git a/drivers/scsi/qla2xxx/qla_fw.h b/drivers/scsi/qla2xxx/qla_fw.h index f307beed9d29d..e7ded95afb9a8 100644 --- a/drivers/scsi/qla2xxx/qla_fw.h +++ b/drivers/scsi/qla2xxx/qla_fw.h @@ -1442,6 +1442,10 @@ struct vp_ctrl_entry_24xx { uint8_t reserved_5[24]; }; +/* vp_idx_map is a 128-bit (16-byte) bitmap selecting target VPs. */ +#define VP_CTRL_IDX_MAP_BITS \ + (sizeof_field(struct vp_ctrl_entry_24xx, vp_idx_map) * 8) + /* * Modify Virtual Port Configuration IOCB */ diff --git a/drivers/scsi/qla2xxx/qla_gs.c b/drivers/scsi/qla2xxx/qla_gs.c index 880cd73feaca4..ed46217ce26e1 100644 --- a/drivers/scsi/qla2xxx/qla_gs.c +++ b/drivers/scsi/qla2xxx/qla_gs.c @@ -157,8 +157,8 @@ qla2x00_chk_ms_status(scsi_qla_host_t *vha, ms_iocb_entry_t *ms_pkt, break; case CS_PORT_LOGGED_OUT: if (IS_FWI2_CAPABLE(ha)) { - if (le16_to_cpu(ms_pkt->loop_id.extended) == - NPH_SNS) + if (le16_to_cpu(((struct ct_entry_24xx *) + ms_pkt)->nport_handle) == NPH_SNS) lid_is_sns = true; } else { if (le16_to_cpu(ms_pkt->loop_id.extended) == diff --git a/drivers/scsi/qla2xxx/qla_init.c b/drivers/scsi/qla2xxx/qla_init.c index 84f89445c7470..5aa5e8f5c6d0d 100644 --- a/drivers/scsi/qla2xxx/qla_init.c +++ b/drivers/scsi/qla2xxx/qla_init.c @@ -228,7 +228,7 @@ qla2x00_async_iocb_timeout(void *data) srb_t *sp = data; fc_port_t *fcport = sp->fcport; struct srb_iocb *lio = &sp->u.iocb_cmd; - int rc, h; + int rc, h, found; unsigned long flags; if (fcport) { @@ -251,6 +251,7 @@ qla2x00_async_iocb_timeout(void *data) lio->u.logio.data[1] = lio->u.logio.flags & SRB_LOGIN_RETRIED ? QLA_LOGIO_LOGIN_RETRIED : 0; + found = 0; spin_lock_irqsave(sp->qpair->qp_lock_ptr, flags); for (h = 1; h < sp->qpair->req->num_outstanding_cmds; h++) { @@ -258,11 +259,19 @@ qla2x00_async_iocb_timeout(void *data) sp) { sp->qpair->req->outstanding_cmds[h] = NULL; + found = 1; break; } } spin_unlock_irqrestore(sp->qpair->qp_lock_ptr, flags); - sp->done(sp, QLA_FUNCTION_TIMEOUT); + /* + * Only complete the command if this path removed it + * from outstanding_cmds. Otherwise the ISR already + * completed it and a second sp->done() would race the + * submitter's freeing of the on-stack completion. + */ + if (found) + sp->done(sp, QLA_FUNCTION_TIMEOUT); } break; case SRB_LOGOUT_CMD: @@ -275,6 +284,7 @@ qla2x00_async_iocb_timeout(void *data) default: rc = qla24xx_async_abort_cmd(sp, false); if (rc) { + found = 0; spin_lock_irqsave(sp->qpair->qp_lock_ptr, flags); for (h = 1; h < sp->qpair->req->num_outstanding_cmds; h++) { @@ -282,11 +292,19 @@ qla2x00_async_iocb_timeout(void *data) sp) { sp->qpair->req->outstanding_cmds[h] = NULL; + found = 1; break; } } spin_unlock_irqrestore(sp->qpair->qp_lock_ptr, flags); - sp->done(sp, QLA_FUNCTION_TIMEOUT); + /* + * Only complete the command if this path removed it + * from outstanding_cmds. Otherwise the ISR already + * completed it and a second sp->done() would race the + * submitter's freeing of the on-stack completion. + */ + if (found) + sp->done(sp, QLA_FUNCTION_TIMEOUT); } break; } @@ -3764,11 +3782,27 @@ int qla2x00_alloc_fce_trace(scsi_qla_host_t *vha) void qla2x00_free_fce_trace(struct qla_hw_data *ha) { - if (!ha->fce) + void *fce; + dma_addr_t fce_dma; + unsigned long flags; + + /* + * Unpublish ha->fce under hardware_lock so a firmware dump in + * progress (which reads ha->fce under the same lock) cannot race + * with the buffer being freed. + */ + spin_lock_irqsave(&ha->hardware_lock, flags); + if (!ha->fce) { + spin_unlock_irqrestore(&ha->hardware_lock, flags); return; - dma_free_coherent(&ha->pdev->dev, FCE_SIZE, ha->fce, ha->fce_dma); + } + fce = ha->fce; + fce_dma = ha->fce_dma; ha->fce = NULL; ha->fce_dma = 0; + spin_unlock_irqrestore(&ha->hardware_lock, flags); + + dma_free_coherent(&ha->pdev->dev, FCE_SIZE, fce, fce_dma); } static void @@ -4374,6 +4408,19 @@ qla2x00_setup_chip(scsi_qla_host_t *vha) MIN_MULTI_ID_FABRIC)) ha->max_npiv_vports = MIN_MULTI_ID_FABRIC - 1; + + /* + * The VP_CTRL IOCB selects target VPs + * through the fixed vp_idx_map bitmap, + * so a vp_index beyond it can be enabled + * via VP_CONFIG but never disabled via + * VP_CTRL, leaking the VP. Cap the count + * to the bitmap capacity. + */ + if (ha->max_npiv_vports >= + VP_CTRL_IDX_MAP_BITS) + ha->max_npiv_vports = + VP_CTRL_IDX_MAP_BITS - 1; } qla2x00_get_resource_cnts(vha); qla_init_iocb_limit(vha); @@ -5602,6 +5649,7 @@ qla2x00_alloc_fcport(scsi_qla_host_t *vha, gfp_t flags) INIT_LIST_HEAD(&fcport->gnl_entry); INIT_LIST_HEAD(&fcport->list); INIT_LIST_HEAD(&fcport->unsol_ctx_head); + spin_lock_init(&fcport->unsol_ctx_lock); INIT_LIST_HEAD(&fcport->sess_cmd_list); spin_lock_init(&fcport->sess_cmd_lock); @@ -9833,11 +9881,28 @@ int qla2xxx_delete_qpair(struct scsi_qla_host *vha, struct qla_qpair *qpair) { int ret = QLA_FUNCTION_FAILED; struct qla_hw_data *ha = qpair->hw; + struct rsp_que *rsp = qpair->rsp; qpair->delete_in_progress = 1; qla_free_buf_pool(qpair); + /* + * The response-queue interrupt schedules qla_do_work(), which + * dereferences qpair->rsp->req. Release the interrupt and flush + * any pending work before the request queue is freed below so a + * late completion cannot touch the freed request queue. The + * firmware queue-delete order (request then response) is kept. + */ + if (rsp && rsp->msix && rsp->msix->have_irq) { + free_irq(rsp->msix->vector, rsp->msix->handle); + rsp->msix->have_irq = 0; + rsp->msix->in_use = 0; + rsp->msix->handle = NULL; + } + if (rsp && ha->wq) + cancel_work_sync(&qpair->q_work); + ret = qla25xx_delete_req_que(vha, qpair->req); if (ret != QLA_SUCCESS) goto fail; diff --git a/drivers/scsi/qla2xxx/qla_inline.h b/drivers/scsi/qla2xxx/qla_inline.h index ef4b3cc1cd77e..09ada9dfb5b86 100644 --- a/drivers/scsi/qla2xxx/qla_inline.h +++ b/drivers/scsi/qla2xxx/qla_inline.h @@ -54,6 +54,19 @@ qla2x00_debounce_register(volatile __le16 __iomem *addr) return (first); } +static inline u8 +qla_calc_queue_count(u16 msix_count) +{ + /* + * Request/response queues are bounded by the MSI-X vector count less + * the mailbox vector. These counters are u8, so a board advertising + * e.g. 257 vectors would truncate msix_count - 1 (256) to 0 and hand + * kzalloc_objs() a zero count (ZERO_SIZE_PTR), faulting on the first + * ha->req_q_map[0] store. Clamp into [1, QLA_MAX_QUEUES - 1]. + */ + return clamp_t(u16, msix_count - 1, 1, QLA_MAX_QUEUES - 1); +} + static inline void qla2x00_poll(struct rsp_que *rsp) { diff --git a/drivers/scsi/qla2xxx/qla_iocb.c b/drivers/scsi/qla2xxx/qla_iocb.c index 3224044f17753..5ad867c6f26eb 100644 --- a/drivers/scsi/qla2xxx/qla_iocb.c +++ b/drivers/scsi/qla2xxx/qla_iocb.c @@ -11,6 +11,10 @@ #include +/* All adapters supported by this tree use the 64-byte request ring. */ +#define qla_req_ring_slot(ha, req) ((req)->ring_ptr) +#define qla_req_entry_size(ha) REQUEST_ENTRY_SIZE + static int qla_start_scsi_type6(srb_t *sp); /** * qla2x00_get_cmd_direction() - Determine control_flag data direction. @@ -2286,10 +2290,9 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp) struct req_que *req = qpair->req; device_reg_t *reg = ISP_QUE_REG(ha, req->id); uint32_t handle; - request_t *pkt; uint16_t cnt, req_cnt; + request_t *pkt = NULL; - pkt = NULL; req_cnt = 1; handle = 0; @@ -2343,14 +2346,18 @@ __qla2x00_alloc_iocbs(struct qla_qpair *qpair, srb_t *sp) sp->handle = handle; } - /* Prep packet */ + /* + * Prep the current request-ring slot. + */ req->cnt -= req_cnt; - pkt = req->ring_ptr; - memset(pkt, 0, REQUEST_ENTRY_SIZE); + pkt = qla_req_ring_slot(ha, req); if (IS_QLAFX00(ha)) { + memset_io((void __iomem __force *)pkt, 0, + qla_req_entry_size(ha)); wrt_reg_byte((u8 __force __iomem *)&pkt->entry_count, req_cnt); wrt_reg_dword((__le32 __force __iomem *)&pkt->handle, handle); } else { + memset(pkt, 0, qla_req_entry_size(ha)); pkt->entry_count = req_cnt; pkt->handle = handle; } @@ -3828,6 +3835,12 @@ qla25xx_ctrlvp_iocb(srb_t *sp, struct vp_ctrl_entry_24xx *vce) */ map = (sp->u.iocb_cmd.u.ctrlvp.vp_index - 1) / 8; pos = (sp->u.iocb_cmd.u.ctrlvp.vp_index - 1) & 7; + if (map >= ARRAY_SIZE(vce->vp_idx_map)) { + ql_log(ql_log_warn, sp->vha, 0x307c, + "ctrlvp: vp_index %u exceeds vp_idx_map capacity\n", + sp->u.iocb_cmd.u.ctrlvp.vp_index); + return; + } vce->vp_idx_map[map] |= 1 << pos; } diff --git a/drivers/scsi/qla2xxx/qla_isr.c b/drivers/scsi/qla2xxx/qla_isr.c index 608d2f36e7b4f..50c14c25a38ba 100644 --- a/drivers/scsi/qla2xxx/qla_isr.c +++ b/drivers/scsi/qla2xxx/qla_isr.c @@ -205,6 +205,17 @@ void __qla_consume_iocb(struct scsi_qla_host *vha, struct purex_entry_24xx *purex = *pkt; entry_count_remaining = purex->entry_count; + + /* + * The caller already advanced ring_ptr past the head IOCB, so mark + * the head processed and account for it here, then consume only the + * continuation IOCBs that follow. + */ + ((response_t *)purex)->signature = RESPONSE_PROCESSED; + /* flush signature */ + wmb(); + --entry_count_remaining; + while (entry_count_remaining > 0) { new_pkt = rsp_q->ring_ptr; *pkt = new_pkt; @@ -910,14 +921,6 @@ qla27xx_copy_multiple_pkt(struct scsi_qla_host *vha, void **pkt, do { while ((total_bytes > 0) && (entry_count_remaining > 0)) { - if (rsp_q->ring_ptr->signature == RESPONSE_PROCESSED) { - ql_dbg(ql_dbg_async, vha, 0x5084, - "Ran out of IOCBs, partial data 0x%x\n", - buffer_copy_offset); - cpu_relax(); - continue; - } - new_pkt = (sts_cont_entry_t *)rsp_q->ring_ptr; *pkt = new_pkt; @@ -1194,14 +1197,6 @@ qla27xx_copy_fpin_pkt(struct scsi_qla_host *vha, void **pkt, do { while ((total_bytes > 0) && (entry_count_remaining > 0)) { - if (rsp_q->ring_ptr->signature == RESPONSE_PROCESSED) { - ql_dbg(ql_dbg_async, vha, 0x5084, - "Ran out of IOCBs, partial data 0x%x\n", - buffer_copy_offset); - cpu_relax(); - continue; - } - new_pkt = (sts_cont_entry_t *)rsp_q->ring_ptr; *pkt = new_pkt; @@ -3406,6 +3401,14 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt) return; } + /* Everything below is the SCSI fast path; reject other SRB types. */ + if (sp->type != SRB_SCSI_CMD) { + ql_dbg(ql_dbg_io, vha, 0x303d, + "Unexpected SRB type %x for status IOCB, sp %p.\n", + sp->type, sp); + return; + } + /* Fast path completion. */ qla_chk_edif_rx_sa_delete_pending(vha, sp, sts24); sp->qpair->cmd_completion_cnt++; @@ -3463,6 +3466,18 @@ qla2x00_status_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, void *pkt) if (scsi_status & SS_RESPONSE_INFO_LEN_VALID) { /* Sense data lies beyond any FCP RESPONSE data. */ if (IS_FWI2_CAPABLE(ha)) { + /* + * A hostile or buggy target may report an + * rsp_info_len larger than the IOCB data area. + * Clamp it so the par_sense_len subtraction cannot + * underflow and walk sense_data out of bounds. + */ + if (rsp_info_len > par_sense_len) { + ql_log(ql_log_warn, fcport->vha, 0x3107, + "Truncating bogus rsp_info_len 0x%x to 0x%x.\n", + rsp_info_len, par_sense_len); + rsp_info_len = par_sense_len; + } sense_data += rsp_info_len; par_sense_len -= rsp_info_len; } @@ -3778,10 +3793,12 @@ qla2x00_error_entry(scsi_qla_host_t *vha, struct rsp_que *rsp, sts_entry_t *pkt) "iocb type %xh with error status %xh, handle %xh, rspq id %d\n", pkt->entry_type, pkt->entry_status, pkt->handle, rsp->id); - if (que >= ha->max_req_queues || !ha->req_q_map[que]) + if (que >= ha->max_req_queues) goto fatal; req = ha->req_q_map[que]; + if (!req) + goto fatal; if (pkt->entry_status & RF_BUSY) res = DID_BUS_BUSY << 16; @@ -4109,9 +4126,24 @@ void qla24xx_process_response_queue(struct scsi_qla_host *vha, "SCM not active for this port\n"); break; } + if (qla_chk_cont_iocb_avail(vha, rsp, + (response_t *)pkt, rsp_in)) { + /* + * ring_ptr and ring_index were + * pre-incremented above. Reset them + * back to current. Wait for next + * interrupt with all IOCBs to arrive + * and re-process. + */ + rsp->ring_ptr = (response_t *)pkt; + rsp->ring_index = cur_ring_index; + + ql_dbg(ql_dbg_init, vha, 0x5095, + "Defer processing FPIN...\n"); + return; + } pure_item = qla27xx_copy_fpin_pkt(vha, (void **)&pkt, &rsp); - __update_rsp_in(is_shadow_hba, rsp, rsp_in); if (!pure_item) break; qla24xx_queue_purex_item(vha, pure_item, @@ -4548,10 +4580,10 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp) ha->msix_count = ret; /* Recalculate queue values */ if (ha->mqiobase && (ql2xmqsupport || ql2xnvmeenable)) { - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); /* ATIOQ needs 1 vector. That's 1 less QPair */ - if (QLA_TGT_MODE_ENABLED()) + if (QLA_TGT_MODE_ENABLED() && ha->max_req_queues > 1) ha->max_req_queues--; ha->max_rsp_queues = ha->max_req_queues; diff --git a/drivers/scsi/qla2xxx/qla_mbx.c b/drivers/scsi/qla2xxx/qla_mbx.c index 1f01576f044b8..5a21c3cb7731e 100644 --- a/drivers/scsi/qla2xxx/qla_mbx.c +++ b/drivers/scsi/qla2xxx/qla_mbx.c @@ -2214,6 +2214,8 @@ qla2x00_get_firmware_state(scsi_qla_host_t *vha, uint16_t *states) if (!ha->flags.fw_started) return QLA_FUNCTION_FAILED; + memset(&mc, 0, sizeof(mc)); + mcp->mb[0] = MBC_GET_FIRMWARE_STATE; mcp->out_mb = MBX_0; if (IS_FWI2_CAPABLE(vha->hw)) @@ -4145,7 +4147,10 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, spin_lock_irqsave(&ha->vport_slock, flags); list_for_each_entry(vp, &ha->vp_list, list) { if (rptid_entry->vp_idx == vp->vp_idx) { + if (test_bit(VPORT_DELETE, &vp->dpc_flags)) + break; found = 1; + atomic_inc(&vp->vref_count); break; } } @@ -4154,7 +4159,9 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, if (!found) return; + spin_lock_irqsave(&ha->vport_slock, flags); qla_update_host_map(vp, id); + spin_unlock_irqrestore(&ha->vport_slock, flags); /* * Cannot configure here as we are still sitting on the @@ -4163,6 +4170,10 @@ qla24xx_report_id_acquisition(scsi_qla_host_t *vha, set_bit(VP_IDX_ACQUIRED, &vp->vp_flags); set_bit(REGISTER_FC4_NEEDED, &vp->dpc_flags); set_bit(REGISTER_FDMI_NEEDED, &vp->dpc_flags); + + spin_lock_irqsave(&ha->vport_slock, flags); + atomic_dec(&vp->vref_count); + spin_unlock_irqrestore(&ha->vport_slock, flags); } set_bit(VP_DPC_NEEDED, &vha->dpc_flags); qla2xxx_wake_dpc(vha); @@ -4273,10 +4284,10 @@ qla24xx_modify_vp_config(scsi_qla_host_t *vha) if (rval != QLA_SUCCESS) { ql_dbg(ql_dbg_mbx, vha, 0x10bd, "Failed to issue VP config IOCB (%x).\n", rval); - } else if (vpmod->comp_status != 0) { + } else if (vpmod->entry_status != 0) { ql_dbg(ql_dbg_mbx, vha, 0x10be, "Failed to complete IOCB -- error status (%x).\n", - vpmod->comp_status); + vpmod->entry_status); rval = QLA_FUNCTION_FAILED; } else if (vpmod->comp_status != cpu_to_le16(CS_COMPLETE)) { ql_dbg(ql_dbg_mbx, vha, 0x10bf, @@ -5649,7 +5660,7 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha) ql_dbg(ql_dbg_mbx, vha, 0x1107, "Failed=%x mb[0]=%x.\n", rval, mcp->mb[0]); } else { - if (mcp->mb[1] != 0x7) + if (mcp->mb[1] != 0x7 || IS_QLA28XX(ha)) ha->link_data_rate = mcp->mb[1]; if (IS_QLA83XX(ha) || IS_QLA27XX(ha) || IS_QLA28XX(ha)) { @@ -5660,8 +5671,6 @@ qla2x00_get_data_rate(scsi_qla_host_t *vha) ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x1108, "Done %s.\n", __func__); - if (mcp->mb[1] != 0x7) - ha->link_data_rate = mcp->mb[1]; } return rval; @@ -6433,6 +6442,7 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, mbx_cmd_t mc; mbx_cmd_t *mcp = &mc; dma_addr_t dd_dma; + void *dd; if (!IS_QLA83XX(vha->hw) && !IS_QLA27XX(vha->hw) && !IS_QLA28XX(vha->hw)) @@ -6441,15 +6451,12 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, ql_dbg(ql_dbg_mbx + ql_dbg_verbose, vha, 0x119f, "Entered %s.\n", __func__); - dd_dma = dma_map_single(&vha->hw->pdev->dev, - dd_buf, size, DMA_FROM_DEVICE); - if (dma_mapping_error(&vha->hw->pdev->dev, dd_dma)) { - ql_log(ql_log_warn, vha, 0x1194, "Failed to map dma buffer.\n"); + dd = dma_alloc_coherent(&vha->hw->pdev->dev, size, &dd_dma, GFP_KERNEL); + if (!dd) { + ql_log(ql_log_warn, vha, 0x1194, "Failed to allocate dma buffer.\n"); return QLA_MEMORY_ALLOC_FAILED; } - memset(dd_buf, 0, size); - mcp->mb[0] = MBC_DPORT_DIAGNOSTICS; mcp->mb[1] = options; mcp->mb[2] = MSW(LSD(dd_dma)); @@ -6471,8 +6478,9 @@ qla26xx_dport_diagnostics(scsi_qla_host_t *vha, "Done %s.\n", __func__); } - dma_unmap_single(&vha->hw->pdev->dev, dd_dma, - size, DMA_FROM_DEVICE); + memcpy(dd_buf, dd, size); + + dma_free_coherent(&vha->hw->pdev->dev, size, dd, dd_dma); return rval; } diff --git a/drivers/scsi/qla2xxx/qla_mid.c b/drivers/scsi/qla2xxx/qla_mid.c index 0abc47e72e0bf..ce4da04ed4578 100644 --- a/drivers/scsi/qla2xxx/qla_mid.c +++ b/drivers/scsi/qla2xxx/qla_mid.c @@ -574,16 +574,19 @@ qla25xx_free_req_que(struct scsi_qla_host *vha, struct req_que *req) { struct qla_hw_data *ha = vha->hw; uint16_t que_id = req->id; + size_t req_entry_size = sizeof(request_t); - dma_free_coherent(&ha->pdev->dev, (req->length + 1) * - sizeof(request_t), req->ring, req->dma); + if (req->ring) + dma_free_coherent(&ha->pdev->dev, + (req->length + 1) * req_entry_size, + req->ring, req->dma); req->ring = NULL; req->dma = 0; if (que_id) { + mutex_lock(&ha->mq_lock); ha->req_q_map[que_id] = NULL; - mutex_lock(&ha->vport_lock); clear_bit(que_id, ha->req_qid_map); - mutex_unlock(&ha->vport_lock); + mutex_unlock(&ha->mq_lock); } kfree(req->outstanding_cmds); kfree(req); @@ -594,6 +597,7 @@ qla25xx_free_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp) { struct qla_hw_data *ha = vha->hw; uint16_t que_id = rsp->id; + size_t rsp_entry_size = sizeof(response_t); if (rsp->msix && rsp->msix->have_irq) { free_irq(rsp->msix->vector, rsp->msix->handle); @@ -601,15 +605,22 @@ qla25xx_free_rsp_que(struct scsi_qla_host *vha, struct rsp_que *rsp) rsp->msix->in_use = 0; rsp->msix->handle = NULL; } - dma_free_coherent(&ha->pdev->dev, (rsp->length + 1) * - sizeof(response_t), rsp->ring, rsp->dma); + + /* Flush any queued response work before freeing the queue/qpair. */ + if (rsp->qpair && ha->wq) + cancel_work_sync(&rsp->qpair->q_work); + + if (rsp->ring) + dma_free_coherent(&ha->pdev->dev, + (rsp->length + 1) * rsp_entry_size, + rsp->ring, rsp->dma); rsp->ring = NULL; rsp->dma = 0; if (que_id) { + mutex_lock(&ha->mq_lock); ha->rsp_q_map[que_id] = NULL; - mutex_lock(&ha->vport_lock); clear_bit(que_id, ha->rsp_qid_map); - mutex_unlock(&ha->vport_lock); + mutex_unlock(&ha->mq_lock); } kfree(rsp); } @@ -794,9 +805,6 @@ qla25xx_create_req_que(struct qla_hw_data *ha, uint16_t options, if (ret != QLA_SUCCESS) { ql_log(ql_log_fatal, base_vha, 0x00df, "%s failed.\n", __func__); - mutex_lock(&ha->mq_lock); - clear_bit(que_id, ha->req_qid_map); - mutex_unlock(&ha->mq_lock); goto que_failed; } vha->flags.qpairs_req_created = 1; @@ -908,9 +916,6 @@ qla25xx_create_rsp_que(struct qla_hw_data *ha, uint16_t options, if (ret != QLA_SUCCESS) { ql_log(ql_log_fatal, base_vha, 0x00e7, "%s failed.\n", __func__); - mutex_lock(&ha->mq_lock); - clear_bit(que_id, ha->rsp_qid_map); - mutex_unlock(&ha->mq_lock); goto que_failed; } vha->flags.qpairs_rsp_created = 1; @@ -957,6 +962,22 @@ int qla24xx_control_vp(scsi_qla_host_t *vha, int cmd) if (vp_index == 0 || vp_index >= ha->max_npiv_vports) return QLA_PARAMETER_ERROR; + /* + * The VP_CTRL IOCB selects the target VP through a fixed 128-bit + * (16-byte) vp_idx_map bitmap, so vp_index must fit within it even + * if firmware advertises more NPIV vports. + */ + if (vp_index > sizeof_field(struct vp_ctrl_entry_24xx, vp_idx_map) * 8) + return QLA_PARAMETER_ERROR; + + /* + * The VP_CTRL IOCB selects the target VP through a fixed 128-bit + * (16-byte) vp_idx_map bitmap, so vp_index must fit within it even + * if firmware advertises more NPIV vports. + */ + if (vp_index > VP_CTRL_IDX_MAP_BITS) + return QLA_PARAMETER_ERROR; + /* ref: INIT */ sp = qla2x00_get_sp(base_vha, NULL, GFP_KERNEL); if (!sp) diff --git a/drivers/scsi/qla2xxx/qla_nvme.c b/drivers/scsi/qla2xxx/qla_nvme.c index 42eb65a62f1f3..650d6be3af97d 100644 --- a/drivers/scsi/qla2xxx/qla_nvme.c +++ b/drivers/scsi/qla2xxx/qla_nvme.c @@ -257,7 +257,9 @@ static void qla_nvme_release_lsrsp_cmd_kref(struct kref *kref) fd_rsp = uctx->fd_rsp; + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); fd_rsp->done(fd_rsp); kfree(uctx); @@ -374,6 +376,7 @@ static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport, srb_t *sp; int rval = QLA_FUNCTION_FAILED; uint8_t cnt = 0; + unsigned long flags; if (!fcport || fcport->deleted) goto out; @@ -440,7 +443,14 @@ static int qla_nvme_xmt_ls_rsp(struct nvme_fc_local_port *lport, a.vp_idx = vha->vp_idx; a.nport_handle = uctx->nport_handle; a.xchg_address = uctx->exchange_address; - qla_nvme_ls_reject_iocb(vha, ha->base_qpair, &a, true); + if (ha->flags.fw_started) { + spin_lock_irqsave(ha->base_qpair->qp_lock_ptr, flags); + qla_nvme_ls_reject_iocb(vha, ha->base_qpair, &a, true); + spin_unlock_irqrestore(ha->base_qpair->qp_lock_ptr, flags); + } + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); + list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); kfree(uctx); return rval; } @@ -463,8 +473,8 @@ static void qla_nvme_ls_abort(struct nvme_fc_local_port *lport, } spin_unlock_irqrestore(&priv->cmd_lock, flags); - INIT_WORK(&priv->abort_work, qla_nvme_abort_work); - schedule_work(&priv->abort_work); + if (!schedule_work(&priv->abort_work)) + kref_put(&priv->sp->cmd_kref, priv->sp->put_fn); } static int qla_nvme_ls_req(struct nvme_fc_local_port *lport, @@ -501,6 +511,7 @@ static int qla_nvme_ls_req(struct nvme_fc_local_port *lport, priv->sp = sp; kref_init(&sp->cmd_kref); spin_lock_init(&priv->cmd_lock); + INIT_WORK(&priv->abort_work, qla_nvme_abort_work); nvme = &sp->u.iocb_cmd; priv->fd = fd; nvme->u.nvme.desc = fd; @@ -545,8 +556,8 @@ static void qla_nvme_fcp_abort(struct nvme_fc_local_port *lport, } spin_unlock_irqrestore(&priv->cmd_lock, flags); - INIT_WORK(&priv->abort_work, qla_nvme_abort_work); - schedule_work(&priv->abort_work); + if (!schedule_work(&priv->abort_work)) + kref_put(&priv->sp->cmd_kref, priv->sp->put_fn); } static inline int qla2x00_start_nvme_mq(srb_t *sp) @@ -811,6 +822,7 @@ static int qla_nvme_post_cmd(struct nvme_fc_local_port *lport, kref_init(&sp->cmd_kref); spin_lock_init(&priv->cmd_lock); + INIT_WORK(&priv->abort_work, qla_nvme_abort_work); sp->priv = priv; priv->sp = sp; sp->type = SRB_NVME_CMD; @@ -1127,6 +1139,10 @@ static void qla_nvme_lsrjt_pt_iocb(struct scsi_qla_host *vha, lsrjt_iocb->rx_byte_count = 0; } +/* + * Allocates from and advances the request ring, so the caller must hold + * qp->qp_lock_ptr (the response-queue caller already holds it). + */ static int qla_nvme_ls_reject_iocb(struct scsi_qla_host *vha, struct qla_qpair *qp, struct qla_nvme_lsrjt_pt_arg *a, bool is_xchg_terminate) @@ -1183,6 +1199,7 @@ qla2xxx_process_purls_pkt(struct scsi_qla_host *vha, struct purex_item *item) { struct qla_nvme_unsol_ctx *uctx = item->purls_context; struct qla_nvme_lsrjt_pt_arg a; + unsigned long flags; int ret = 1; #if (IS_ENABLED(CONFIG_NVME_FC)) @@ -1195,8 +1212,17 @@ qla2xxx_process_purls_pkt(struct scsi_qla_host *vha, struct purex_item *item) a.vp_idx = vha->vp_idx; a.nport_handle = uctx->nport_handle; a.xchg_address = uctx->exchange_address; - qla_nvme_ls_reject_iocb(vha, vha->hw->base_qpair, &a, true); + if (vha->hw->flags.fw_started) { + spin_lock_irqsave(vha->hw->base_qpair->qp_lock_ptr, + flags); + qla_nvme_ls_reject_iocb(vha, vha->hw->base_qpair, &a, + true); + spin_unlock_irqrestore(vha->hw->base_qpair->qp_lock_ptr, + flags); + } + spin_lock_irqsave(&uctx->fcport->unsol_ctx_lock, flags); list_del(&uctx->elem); + spin_unlock_irqrestore(&uctx->fcport->unsol_ctx_lock, flags); kfree(uctx); } } @@ -1238,6 +1264,7 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp) struct purex_item *item; port_id_t d_id = {0}; port_id_t id = {0}; + unsigned long flags; u8 *opcode; bool xmt_reject = false; @@ -1303,7 +1330,9 @@ void qla2xxx_process_purls_iocb(void **pkt, struct rsp_que **rsp) uctx->ox_id = p->ox_id; qla_rport->uctx = uctx; INIT_LIST_HEAD(&uctx->elem); + spin_lock_irqsave(&fcport->unsol_ctx_lock, flags); list_add_tail(&uctx->elem, &fcport->unsol_ctx_head); + spin_unlock_irqrestore(&fcport->unsol_ctx_lock, flags); item->purls_context = (void *)uctx; ql_dbg(ql_dbg_unsol, vha, 0x2121, diff --git a/drivers/scsi/qla2xxx/qla_nx2.c b/drivers/scsi/qla2xxx/qla_nx2.c index 41ff6fbdb9333..04d7ab6ad037a 100644 --- a/drivers/scsi/qla2xxx/qla_nx2.c +++ b/drivers/scsi/qla2xxx/qla_nx2.c @@ -3507,7 +3507,7 @@ qla8044_poll_flash_status_reg(struct scsi_qla_host *vha) msleep(QLA8044_FLASH_STATUS_REG_POLL_DELAY); } - if (!retries) + if (retries == -1) ret_val = QLA_FUNCTION_FAILED; return ret_val; diff --git a/drivers/scsi/qla2xxx/qla_os.c b/drivers/scsi/qla2xxx/qla_os.c index 59ca4adcb4315..3e148e4e94593 100644 --- a/drivers/scsi/qla2xxx/qla_os.c +++ b/drivers/scsi/qla2xxx/qla_os.c @@ -2127,7 +2127,7 @@ qla2x00_iospace_config(struct qla_hw_data *ha) ha->msix_count = msix + 1; /* Max queues are bounded by available msix vectors */ /* MB interrupt uses 1 vector */ - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); ha->max_rsp_queues = ha->max_req_queues; /* Queue pairs is the max value minus the base queue pair */ ha->max_qpairs = ha->max_rsp_queues - 1; @@ -2213,10 +2213,10 @@ qla83xx_iospace_config(struct qla_hw_data *ha) */ if (ql2xmqsupport || ql2xnvmeenable) { /* MB interrupt uses 1 vector */ - ha->max_req_queues = ha->msix_count - 1; + ha->max_req_queues = qla_calc_queue_count(ha->msix_count); /* ATIOQ needs 1 vector. That's 1 less QPair */ - if (QLA_TGT_MODE_ENABLED()) + if (QLA_TGT_MODE_ENABLED() && ha->max_req_queues > 1) ha->max_req_queues--; ha->max_rsp_queues = ha->max_req_queues; @@ -3933,8 +3933,6 @@ qla2x00_remove_one(struct pci_dev *pdev) qla2x00_dfs_remove(base_vha); - qla84xx_put_chip(base_vha); - /* Disable timer */ if (base_vha->timer_active) qla2x00_stop_timer(base_vha); @@ -3959,6 +3957,8 @@ qla2x00_remove_one(struct pci_dev *pdev) scsi_remove_host(base_vha->host); + qla84xx_put_chip(base_vha); + qla2x00_free_device(base_vha); qla2x00_clear_drv_active(ha); @@ -4486,25 +4486,40 @@ qla2x00_mem_alloc(struct qla_hw_data *ha, uint16_t req_len, uint16_t rsp_len, fail_lsrjt: dma_free_coherent(&ha->pdev->dev, ha->elsrej.size, ha->elsrej.c, ha->elsrej.cdma); + ha->elsrej.c = NULL; + ha->elsrej.cdma = 0; fail_elsrej: dma_pool_destroy(ha->purex_dma_pool); + ha->purex_dma_pool = NULL; fail_flt: dma_free_coherent(&ha->pdev->dev, sizeof(struct qla_flt_header) + FLT_REGIONS_SIZE, ha->flt, ha->flt_dma); + ha->flt = NULL; + ha->flt_dma = 0; fail_flt_buffer: dma_free_coherent(&ha->pdev->dev, SFP_DEV_SIZE, ha->sfp_data, ha->sfp_data_dma); + ha->sfp_data = NULL; + ha->sfp_data_dma = 0; fail_sfp_data: kfree(ha->loop_id_map); + ha->loop_id_map = NULL; fail_loop_id_map: dma_pool_free(ha->s_dma_pool, ha->async_pd, ha->async_pd_dma); + ha->async_pd = NULL; + ha->async_pd_dma = 0; fail_async_pd: dma_pool_free(ha->s_dma_pool, ha->sf_init_cb, ha->sf_init_cb_dma); + ha->sf_init_cb = NULL; + ha->sf_init_cb_dma = 0; fail_sf_init_cb: dma_pool_free(ha->s_dma_pool, ha->ex_init_cb, ha->ex_init_cb_dma); + ha->ex_init_cb = NULL; + ha->ex_init_cb_dma = 0; fail_ex_init_cb: kfree(ha->npiv_info); + ha->npiv_info = NULL; fail_npiv_info: dma_free_coherent(&ha->pdev->dev, ((*rsp)->length + 1) * sizeof(response_t), (*rsp)->ring, (*rsp)->dma); @@ -6830,8 +6845,6 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work) qla2x00_dfs_remove(base_vha); - qla84xx_put_chip(base_vha); - if (base_vha->timer_active) qla2x00_stop_timer(base_vha); @@ -6849,6 +6862,8 @@ qla2x00_disable_board_on_pci_error(struct work_struct *work) scsi_remove_host(base_vha->host); + qla84xx_put_chip(base_vha); + base_vha->flags.init_done = 0; qla25xx_delete_queues(base_vha); qla2x00_free_fcports(base_vha); diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c index 047d56d23beab..4bb80f9dd67f5 100644 --- a/drivers/scsi/scsi_debug.c +++ b/drivers/scsi/scsi_debug.c @@ -4322,8 +4322,8 @@ static bool comp_write_worker(struct sdeb_store_info *sip, u64 lba, u32 num, if (!res) return res; if (rest) - res = memcmp(fsp, arr + ((num - rest) * lb_size), - rest * lb_size); + res = !memcmp(fsp, arr + ((num - rest) * lb_size), + rest * lb_size); if (!res) return res; if (compare_only) @@ -5896,6 +5896,7 @@ static int resp_report_zones(struct scsi_cmnd *scp, u32 alloc_len, rep_opts, rep_len; bool partial; u64 lba, zs_lba; + u64 arr_len; u8 *arr = NULL, *desc; u8 *cmd = scp->cmnd; struct sdeb_zone_state *zsp = NULL; @@ -5917,9 +5918,12 @@ static int resp_report_zones(struct scsi_cmnd *scp, return check_condition_result; } - rep_max_zones = (alloc_len - 64) >> ilog2(RZONES_DESC_HD); + rep_max_zones = (ALIGN((u64)alloc_len, RZONES_DESC_HD) - RZONES_DESC_HD) >> + ilog2(RZONES_DESC_HD); + rep_max_zones = min_t(unsigned int, rep_max_zones, devip->nr_zones); + arr_len = (u64)RZONES_DESC_HD * (rep_max_zones + 1); - arr = kzalloc(alloc_len, GFP_ATOMIC | __GFP_NOWARN); + arr = kzalloc(arr_len, GFP_ATOMIC | __GFP_NOWARN); if (!arr) { mk_sense_buffer(scp, ILLEGAL_REQUEST, INSUFF_RES_ASC, INSUFF_RES_ASCQ); diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c index 2fd4ca96b3089..857c2acc99645 100644 --- a/drivers/scsi/scsi_error.c +++ b/drivers/scsi/scsi_error.c @@ -73,6 +73,26 @@ void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy) } } +void scsi_rcu_eh_wakeup(struct work_struct *work) +{ + struct Scsi_Host *shost = container_of(work, struct Scsi_Host, eh_work); + unsigned long flags; + unsigned int busy; + + /* + * Ensure any running scsi_dec_host_busy has completed its rcu section + * so changes to host state and host_eh_scheduled are visible to all + * future calls of scsi_dec_host_busy + */ + synchronize_rcu(); + + busy = scsi_host_busy(shost); + + spin_lock_irqsave(shost->host_lock, flags); + scsi_eh_wakeup(shost, busy); + spin_unlock_irqrestore(shost->host_lock, flags); +} + /** * scsi_schedule_eh - schedule EH for SCSI host * @shost: SCSI host to invoke error handling on. @@ -88,7 +108,7 @@ void scsi_schedule_eh(struct Scsi_Host *shost) if (scsi_host_set_state(shost, SHOST_RECOVERY) == 0 || scsi_host_set_state(shost, SHOST_CANCEL_RECOVERY) == 0) { shost->host_eh_scheduled++; - scsi_eh_wakeup(shost, scsi_host_busy(shost)); + queue_work(shost->tmf_work_q, &shost->eh_work); } spin_unlock_irqrestore(shost->host_lock, flags); @@ -2137,7 +2157,7 @@ static void scsi_eh_lock_door(struct scsi_device *sdev) struct scsi_cmnd *scmd; struct request *req; - req = scsi_alloc_request(sdev->request_queue, REQ_OP_DRV_IN, 0); + req = scsi_alloc_request(sdev->request_queue, REQ_OP_DRV_IN, BLK_MQ_REQ_NOWAIT); if (IS_ERR(req)) return; scmd = blk_mq_rq_to_pdu(req); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 3f7ba6d3987f1..31e1d74143df4 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1184,8 +1184,10 @@ blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd) if (blk_rq_bytes(rq) & rq->q->limits.dma_pad_mask) { unsigned int pad_len = (rq->q->limits.dma_pad_mask & ~blk_rq_bytes(rq)) + 1; + unsigned int data_len = last_sg->length; last_sg->length += pad_len; + sg_zero_buffer(last_sg, 1, pad_len, data_len); cmd->extra_len += pad_len; } diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h index 5b2b19f5e8ec8..68866b18a6005 100644 --- a/drivers/scsi/scsi_priv.h +++ b/drivers/scsi/scsi_priv.h @@ -91,6 +91,7 @@ extern enum blk_eh_timer_return scsi_timeout(struct request *req); extern int scsi_error_handler(void *host); extern enum scsi_disposition scsi_decide_disposition(struct scsi_cmnd *cmd); extern void scsi_eh_wakeup(struct Scsi_Host *shost, unsigned int busy); +extern void scsi_rcu_eh_wakeup(struct work_struct *work); extern void scsi_eh_scmd_add(struct scsi_cmnd *); void scsi_eh_ready_devs(struct Scsi_Host *shost, struct list_head *work_q, diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c index d9bae23cb929e..41ad0e9101b8b 100644 --- a/drivers/scsi/sd.c +++ b/drivers/scsi/sd.c @@ -2328,8 +2328,8 @@ static int sd_done(struct scsi_cmnd *SCpnt) } sdkp->medium_access_timed_out = 0; - if (!scsi_status_is_check_condition(result) && - (!sense_valid || sense_deferred)) + if (!scsi_status_is_check_condition(result) || + !sense_valid || sense_deferred) goto out; switch (sshdr.sense_key) { diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c index 090e786470b39..88cffb1fb59ce 100644 --- a/drivers/scsi/smartpqi/smartpqi_init.c +++ b/drivers/scsi/smartpqi/smartpqi_init.c @@ -66,6 +66,12 @@ static struct pqi_cmd_priv *pqi_cmd_priv(struct scsi_cmnd *cmd) return scsi_cmd_priv(cmd); } +static int pqi_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) +{ + memset(pqi_cmd_priv(cmd), 0, sizeof(struct pqi_cmd_priv)); + return 0; +} + static void pqi_verify_structures(void); static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info, enum pqi_ctrl_shutdown_reason ctrl_shutdown_reason); @@ -5943,6 +5949,17 @@ void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd) struct pqi_scsi_dev *device; struct completion *wait; + /* + * Clear the AIO-retry marker on final completion so the tag + * starts clean on its next dispatch. On DID_IMM_RETRY leave + * it intact: pqi_aio_io_complete() sets DID_IMM_RETRY and + * bumps the marker to steer the requeue onto the RAID path, + * and pqi_process_raid_io_error() consumes the non-zero + * marker to offline a misbehaving drive. + */ + if (host_byte(scmd->result) != DID_IMM_RETRY) + pqi_cmd_priv(scmd)->this_residual = 0; + if (!scmd->device) { set_host_byte(scmd, DID_NO_CONNECT); return; @@ -7596,6 +7613,7 @@ static const struct scsi_host_template pqi_driver_template = { .sdev_groups = pqi_sdev_groups, .shost_groups = pqi_shost_groups, .cmd_size = sizeof(struct pqi_cmd_priv), + .init_cmd_priv = pqi_init_cmd_priv, }; static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info) diff --git a/drivers/sh/clk/core.c b/drivers/sh/clk/core.c index 7a73f5e4a1fc7..f02e12dfa5f64 100644 --- a/drivers/sh/clk/core.c +++ b/drivers/sh/clk/core.c @@ -569,7 +569,7 @@ long clk_round_rate(struct clk *clk, unsigned long rate) EXPORT_SYMBOL_GPL(clk_round_rate); #ifdef CONFIG_PM -static void clks_core_resume(void) +static void clks_core_resume(void *data) { struct clk *clkp; @@ -588,13 +588,17 @@ static void clks_core_resume(void) } } -static struct syscore_ops clks_syscore_ops = { +static const struct syscore_ops clks_syscore_ops = { .resume = clks_core_resume, }; +static struct syscore clks_syscore = { + .ops = &clks_syscore_ops, +}; + static int __init clk_syscore_init(void) { - register_syscore_ops(&clks_syscore_ops); + register_syscore(&clks_syscore); return 0; } diff --git a/drivers/sh/intc/core.c b/drivers/sh/intc/core.c index ea571eeb30787..3dde703b77668 100644 --- a/drivers/sh/intc/core.c +++ b/drivers/sh/intc/core.c @@ -394,7 +394,7 @@ int __init register_intc_controller(struct intc_desc *desc) return -ENOMEM; } -static int intc_suspend(void) +static int intc_suspend(void *data) { struct intc_desc_int *d; @@ -420,7 +420,7 @@ static int intc_suspend(void) return 0; } -static void intc_resume(void) +static void intc_resume(void *data) { struct intc_desc_int *d; @@ -450,11 +450,15 @@ static void intc_resume(void) } } -struct syscore_ops intc_syscore_ops = { +static const struct syscore_ops intc_syscore_ops = { .suspend = intc_suspend, .resume = intc_resume, }; +static struct syscore intc_syscore = { + .ops = &intc_syscore_ops, +}; + const struct bus_type intc_subsys = { .name = "intc", .dev_name = "intc", @@ -477,7 +481,7 @@ static int __init register_intc_devs(void) struct intc_desc_int *d; int error; - register_syscore_ops(&intc_syscore_ops); + register_syscore(&intc_syscore); error = subsys_system_register(&intc_subsys, NULL); if (!error) { diff --git a/drivers/soc/bcm/brcmstb/biuctrl.c b/drivers/soc/bcm/brcmstb/biuctrl.c index 364ddbe365c24..bd830649b60dd 100644 --- a/drivers/soc/bcm/brcmstb/biuctrl.c +++ b/drivers/soc/bcm/brcmstb/biuctrl.c @@ -298,7 +298,7 @@ static int __init setup_hifcpubiuctrl_regs(struct device_node *np) #ifdef CONFIG_PM_SLEEP static u32 cpubiuctrl_reg_save[NUM_CPU_BIUCTRL_REGS]; -static int brcmstb_cpu_credit_reg_suspend(void) +static int brcmstb_cpu_credit_reg_suspend(void *data) { unsigned int i; @@ -311,7 +311,7 @@ static int brcmstb_cpu_credit_reg_suspend(void) return 0; } -static void brcmstb_cpu_credit_reg_resume(void) +static void brcmstb_cpu_credit_reg_resume(void *data) { unsigned int i; @@ -322,10 +322,14 @@ static void brcmstb_cpu_credit_reg_resume(void) cbc_writel(cpubiuctrl_reg_save[i], i); } -static struct syscore_ops brcmstb_cpu_credit_syscore_ops = { +static const struct syscore_ops brcmstb_cpu_credit_syscore_ops = { .suspend = brcmstb_cpu_credit_reg_suspend, .resume = brcmstb_cpu_credit_reg_resume, }; + +static struct syscore brcmstb_cpu_credit_syscore = { + .ops = &brcmstb_cpu_credit_syscore_ops, +}; #endif @@ -354,7 +358,7 @@ static int __init brcmstb_biuctrl_init(void) a72_b53_rac_enable_all(np); mcp_a72_b53_set(); #ifdef CONFIG_PM_SLEEP - register_syscore_ops(&brcmstb_cpu_credit_syscore_ops); + register_syscore(&brcmstb_cpu_credit_syscore); #endif ret = 0; out_put: diff --git a/drivers/soc/fsl/qe/gpio.c b/drivers/soc/fsl/qe/gpio.c index c54154b404dfd..e6ef713726549 100644 --- a/drivers/soc/fsl/qe/gpio.c +++ b/drivers/soc/fsl/qe/gpio.c @@ -135,6 +135,30 @@ static int qe_gpio_dir_out(struct gpio_chip *gc, unsigned int gpio, int val) return 0; } +static int qe_gpio_get_direction(struct gpio_chip *gc, unsigned int gpio) +{ + struct qe_gpio_chip *qe_gc = gpiochip_get_data(gc); + struct qe_pio_regs __iomem *regs = qe_gc->regs; + unsigned long flags; + u32 val, mask; + + spin_lock_irqsave(&qe_gc->lock, flags); + + if (gpio < QE_PIO_PINS / 2) + val = ioread32be(®s->cpdir1); + else + val = ioread32be(®s->cpdir2); + + spin_unlock_irqrestore(&qe_gc->lock, flags); + + mask = (u32)QE_PIO_DIR_OUT << (QE_PIO_PINS - 2 - (gpio % (QE_PIO_PINS / 2)) * 2); + + if (val & mask) + return GPIO_LINE_DIRECTION_OUT; + else + return GPIO_LINE_DIRECTION_IN; +} + struct qe_pin { /* * The qe_gpio_chip name is unfortunate, we should change that to @@ -308,6 +332,7 @@ static int qe_gpio_probe(struct platform_device *ofdev) gc->ngpio = QE_PIO_PINS; gc->direction_input = qe_gpio_dir_in; gc->direction_output = qe_gpio_dir_out; + gc->get_direction = qe_gpio_get_direction; gc->get = qe_gpio_get; gc->set = qe_gpio_set; gc->set_multiple = qe_gpio_set_multiple; diff --git a/drivers/soc/fsl/qe/qe_ic.c b/drivers/soc/fsl/qe/qe_ic.c index 943911053af68..bccc7d612009b 100644 --- a/drivers/soc/fsl/qe/qe_ic.c +++ b/drivers/soc/fsl/qe/qe_ic.c @@ -473,7 +473,6 @@ static struct platform_driver qe_ic_driver = static int __init qe_ic_of_init(void) { - platform_driver_register(&qe_ic_driver); - return 0; + return platform_driver_register(&qe_ic_driver); } subsys_initcall(qe_ic_of_init); diff --git a/drivers/soc/fsl/qe/qe_io.c b/drivers/soc/fsl/qe/qe_io.c index a5e2d0e5ab511..150913fce9818 100644 --- a/drivers/soc/fsl/qe/qe_io.c +++ b/drivers/soc/fsl/qe/qe_io.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -184,3 +185,17 @@ int par_io_of_config(struct device_node *np) return 0; } EXPORT_SYMBOL(par_io_of_config); + +static int __init par_io_populate(void) +{ + struct device_node *np = of_find_node_by_type(NULL, "par_io"); + + if (!np) + return 0; + + of_platform_default_populate(np, NULL, NULL); + of_node_put(np); + + return 0; +} +arch_initcall(par_io_populate); diff --git a/drivers/soc/qcom/qcom-geni-se.c b/drivers/soc/qcom/qcom-geni-se.c index cd1779b6a91a7..2fa6a45904e97 100644 --- a/drivers/soc/qcom/qcom-geni-se.c +++ b/drivers/soc/qcom/qcom-geni-se.c @@ -152,8 +152,6 @@ struct se_fw_hdr { /*Magic numbers*/ #define SE_MAGIC_NUM 0x57464553 -#define MAX_GENI_CFG_RAMn_CNT 455 - #define MI_PBT_NON_PAGED_SEGMENT 0x0 #define MI_PBT_HASH_SEGMENT 0x2 #define MI_PBT_NOTUSED_SEGMENT 0x3 @@ -990,24 +988,27 @@ EXPORT_SYMBOL_GPL(geni_icc_disable); /** * geni_find_protocol_fw() - Locate and validate SE firmware for a protocol. - * @dev: Pointer to the device structure. + * @se: Pointer to the serial engine structure. * @fw: Pointer to the firmware image. * @protocol: Expected serial engine protocol type. * * Identifies the appropriate firmware image or configuration required for a - * specific communication protocol instance running on a Qualcomm GENI - * controller. + * specific communication protocol instance running on a Qualcomm GENI + * controller. Validates the firmware size against the hardware PROG_RAM_DEPTH + * read from SE_HW_PARAM_2. * * Return: pointer to a valid 'struct se_fw_hdr' if found, or NULL otherwise. */ -static struct se_fw_hdr *geni_find_protocol_fw(struct device *dev, const struct firmware *fw, +static struct se_fw_hdr *geni_find_protocol_fw(struct geni_se *se, const struct firmware *fw, enum geni_se_protocol_type protocol) { + struct device *dev = se->dev; const struct elf32_hdr *ehdr; const struct elf32_phdr *phdrs; const struct elf32_phdr *phdr; struct se_fw_hdr *sefw; u32 fw_end, cfg_idx_end, cfg_val_end; + u32 prog_ram_depth; u16 fw_size; int i; @@ -1066,10 +1067,11 @@ static struct se_fw_hdr *geni_find_protocol_fw(struct device *dev, const struct sefw->fw_size_in_items = cpu_to_le16(fw_size); } - if (fw_size >= MAX_GENI_CFG_RAMn_CNT) { - dev_err(dev, - "Firmware size (%u) exceeds max allowed RAMn count (%u)\n", - fw_size, MAX_GENI_CFG_RAMn_CNT); + prog_ram_depth = FIELD_GET(PROG_RAM_DEPTH_MSK, + readl_relaxed(se->base + SE_HW_PARAM_2)); + if (fw_size >= prog_ram_depth) { + dev_err(dev, "Firmware size (%u) exceeds RAM size (%u)\n", + fw_size, prog_ram_depth); continue; } @@ -1193,7 +1195,7 @@ static int geni_load_se_fw(struct geni_se *se, const struct firmware *fw, int ret; struct se_fw_hdr *hdr; - hdr = geni_find_protocol_fw(se->dev, fw, protocol); + hdr = geni_find_protocol_fw(se, fw, protocol); if (!hdr) return -EINVAL; diff --git a/drivers/soc/qcom/rpmh-rsc.c b/drivers/soc/qcom/rpmh-rsc.c index c6f7d5c9c493d..66928ca40b9aa 100644 --- a/drivers/soc/qcom/rpmh-rsc.c +++ b/drivers/soc/qcom/rpmh-rsc.c @@ -944,17 +944,30 @@ static int rpmh_rsc_pd_callback(struct notifier_block *nfb, return NOTIFY_OK; } +static void rpmh_rsc_pd_detach(void *data) +{ + dev_pm_genpd_remove_notifier(data); +} + static int rpmh_rsc_pd_attach(struct rsc_drv *drv, struct device *dev) { int ret; - pm_runtime_enable(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + drv->genpd_nb.notifier_call = rpmh_rsc_pd_callback; ret = dev_pm_genpd_add_notifier(dev, &drv->genpd_nb); if (ret) - pm_runtime_disable(dev); + return ret; - return ret; + return devm_add_action_or_reset(dev, rpmh_rsc_pd_detach, dev); +} + +static void rpmh_rsc_cpu_pm_unregister(void *data) +{ + cpu_pm_unregister_notifier(data); } static int rpmh_probe_tcs_config(struct platform_device *pdev, struct rsc_drv *drv) @@ -1107,7 +1120,15 @@ static int rpmh_rsc_probe(struct platform_device *pdev) return ret; } else { drv->rsc_pm.notifier_call = rpmh_rsc_cpu_pm_callback; - cpu_pm_register_notifier(&drv->rsc_pm); + ret = cpu_pm_register_notifier(&drv->rsc_pm); + if (ret) + return ret; + + ret = devm_add_action_or_reset(&pdev->dev, + rpmh_rsc_cpu_pm_unregister, + &drv->rsc_pm); + if (ret) + return ret; } } @@ -1122,13 +1143,7 @@ static int rpmh_rsc_probe(struct platform_device *pdev) dev_set_drvdata(&pdev->dev, drv); drv->dev = &pdev->dev; - ret = devm_of_platform_populate(&pdev->dev); - if (ret && pdev->dev.pm_domain) { - dev_pm_genpd_remove_notifier(&pdev->dev); - pm_runtime_disable(&pdev->dev); - } - - return ret; + return devm_of_platform_populate(&pdev->dev); } static const struct of_device_id rpmh_drv_match[] = { diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig index 340a1ff7e92b4..f151b05fbb8fa 100644 --- a/drivers/soc/renesas/Kconfig +++ b/drivers/soc/renesas/Kconfig @@ -355,7 +355,6 @@ config ARCH_R8A779H0 config ARCH_R8A78000 bool "ARM64 Platform support for R8A78000 (R-Car X5H)" default y if ARCH_RENESAS - default ARCH_RENESAS select ARCH_RCAR_GEN5 help This enables support for the Renesas R-Car X5H SoC. diff --git a/drivers/soc/samsung/exynos-pmu.c b/drivers/soc/samsung/exynos-pmu.c index f8fe1a5965ab4..d910499731131 100644 --- a/drivers/soc/samsung/exynos-pmu.c +++ b/drivers/soc/samsung/exynos-pmu.c @@ -300,8 +300,8 @@ static const struct mfd_cell exynos_pmu_devs[] = { */ struct regmap *exynos_get_pmu_regmap(void) { - struct device_node *np = of_find_matching_node(NULL, - exynos_pmu_of_device_ids); + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, exynos_pmu_of_device_ids); if (np) return exynos_get_pmu_regmap_by_phandle(np, NULL); return ERR_PTR(-ENODEV); diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c index 74d2fedea71ca..972a63993e1db 100644 --- a/drivers/soc/tegra/fuse/fuse-tegra.c +++ b/drivers/soc/tegra/fuse/fuse-tegra.c @@ -182,10 +182,6 @@ static int tegra_fuse_probe(struct platform_device *pdev) } fuse->soc->init(fuse); - - err = tegra_fuse_add_lookups(fuse); - if (err) - return dev_err_probe(&pdev->dev, err, "failed to add FUSE lookups\n"); } fuse->clk = devm_clk_get_optional(&pdev->dev, "fuse"); @@ -231,6 +227,10 @@ static int tegra_fuse_probe(struct platform_device *pdev) return err; } + err = tegra_fuse_add_lookups(fuse); + if (err) + return dev_err_probe(&pdev->dev, err, "failed to add FUSE lookups\n"); + fuse->rst = devm_reset_control_get_optional(&pdev->dev, "fuse"); if (IS_ERR(fuse->rst)) return dev_err_probe(&pdev->dev, PTR_ERR(fuse->rst), "failed to get FUSE reset\n"); @@ -473,7 +473,7 @@ static int __init tegra_init_fuse(void) const struct of_device_id *match; struct device_node *np; struct resource regs; - int err; + int err = 0; tegra_init_apbmisc(); @@ -566,10 +566,6 @@ static int __init tegra_init_fuse(void) tegra_fuse_print_sku_info(&tegra_sku_info); - err = tegra_fuse_add_lookups(fuse); - if (err) - pr_err("failed to add FUSE lookups\n"); - return err; } early_initcall(tegra_init_fuse); diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c index 034a2a535a1e1..93bbebd680017 100644 --- a/drivers/soc/tegra/pmc.c +++ b/drivers/soc/tegra/pmc.c @@ -466,7 +466,7 @@ struct tegra_pmc { unsigned long *wake_type_dual_edge_map; unsigned long *wake_sw_status_map; unsigned long *wake_cntrl_level_map; - struct syscore_ops syscore; + struct syscore syscore; }; static struct tegra_pmc *pmc = &(struct tegra_pmc) { @@ -3147,7 +3147,7 @@ static void tegra186_pmc_process_wake_events(struct tegra_pmc *pmc, unsigned int } } -static void tegra186_pmc_wake_syscore_resume(void) +static void tegra186_pmc_wake_syscore_resume(void *data) { u32 status, mask; unsigned int i; @@ -3160,7 +3160,7 @@ static void tegra186_pmc_wake_syscore_resume(void) } } -static int tegra186_pmc_wake_syscore_suspend(void) +static int tegra186_pmc_wake_syscore_suspend(void *data) { wke_read_sw_wake_status(pmc); @@ -3179,6 +3179,11 @@ static int tegra186_pmc_wake_syscore_suspend(void) return 0; } +static const struct syscore_ops tegra186_pmc_wake_syscore_ops = { + .suspend = tegra186_pmc_wake_syscore_suspend, + .resume = tegra186_pmc_wake_syscore_resume, +}; + #if defined(CONFIG_PM_SLEEP) && defined(CONFIG_ARM) static int tegra_pmc_suspend(struct device *dev) { @@ -3829,10 +3834,8 @@ static const struct tegra_pmc_regs tegra186_pmc_regs = { static void tegra186_pmc_init(struct tegra_pmc *pmc) { - pmc->syscore.suspend = tegra186_pmc_wake_syscore_suspend; - pmc->syscore.resume = tegra186_pmc_wake_syscore_resume; - - register_syscore_ops(&pmc->syscore); + pmc->syscore.ops = &tegra186_pmc_wake_syscore_ops; + register_syscore(&pmc->syscore); } static void tegra186_pmc_setup_irq_polarity(struct tegra_pmc *pmc, diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c index 14e1351a3f8ae..b23a224f7a1ff 100644 --- a/drivers/soundwire/bus.c +++ b/drivers/soundwire/bus.c @@ -1372,6 +1372,37 @@ int sdw_slave_get_current_bank(struct sdw_slave *slave) } EXPORT_SYMBOL_GPL(sdw_slave_get_current_bank); +/** + * sdw_slave_wait_for_init - Wait for device initialisation + * @slave: Pointer to the SoundWire peripheral. + * @timeout_ms: Timeout in milliseconds. + * + * Wait for a peripheral device to enumerate and be initialised by the + * SoundWire core. + * + * Return: Zero on success, and a negative error code on failure. + */ +int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms) +{ + unsigned long time; + + if (!slave->unattach_request) + return 0; + + time = wait_for_completion_timeout(&slave->initialization_complete, + msecs_to_jiffies(timeout_ms)); + if (!time) { + dev_err(&slave->dev, "Initialization not complete\n"); + sdw_show_ping_status(slave->bus, true); + return -ETIMEDOUT; + } + + slave->unattach_request = 0; + + return 0; +} +EXPORT_SYMBOL_GPL(sdw_slave_wait_for_init); + static int sdw_slave_set_frequency(struct sdw_slave *slave) { int scale_index; @@ -1958,6 +1989,10 @@ int sdw_handle_slave_status(struct sdw_bus *bus, break; case SDW_SLAVE_ALERT: + if (slave->status != SDW_SLAVE_ATTACHED && + slave->status != SDW_SLAVE_ALERT) + continue; + ret = sdw_handle_slave_alerts(slave); if (ret < 0) dev_err(&slave->dev, diff --git a/drivers/soundwire/dmi-quirks.c b/drivers/soundwire/dmi-quirks.c index 5854218e1a274..32a46a2d90f7c 100644 --- a/drivers/soundwire/dmi-quirks.c +++ b/drivers/soundwire/dmi-quirks.c @@ -90,6 +90,19 @@ static const struct adr_remap intel_rooks_county[] = { {} }; +/* + * Many platforms have ghost realtek devices in the ACPI that don't physically + * exist, remove those devices. + */ +static const struct adr_remap ghost_realtek[] = { + /* rt722 on link3 */ + { + 0x000330025d072201ull, + 0x0000000000000000ull + }, + {} +}; + static const struct dmi_system_id adr_remap_quirk_table[] = { /* TGL devices */ { @@ -164,6 +177,28 @@ static const struct dmi_system_id adr_remap_quirk_table[] = { }, .driver_data = (void *)hp_omen_16, }, + /* PTL devices */ + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUS"), + DMI_MATCH(DMI_BOARD_NAME, "UX5406AA"), + }, + .driver_data = (void *)ghost_realtek, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83QK"), + }, + .driver_data = (void *)ghost_realtek, + }, + { + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "83SF"), + }, + .driver_data = (void *)ghost_realtek, + }, {} }; diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c index 9db78f3d76152..a83371233603b 100644 --- a/drivers/soundwire/intel.c +++ b/drivers/soundwire/intel.c @@ -907,19 +907,6 @@ static int intel_trigger(struct snd_pcm_substream *substream, int cmd, struct sn } switch (cmd) { - case SNDRV_PCM_TRIGGER_SUSPEND: - - /* - * The .prepare callback is used to deal with xruns and resume operations. - * In the case of xruns, the DMAs and SHIM registers cannot be touched, - * but for resume operations the DMAs and SHIM registers need to be initialized. - * the .trigger callback is used to track the suspend case only. - */ - - dai_runtime->suspended = true; - - break; - case SNDRV_PCM_TRIGGER_PAUSE_PUSH: dai_runtime->paused = true; break; @@ -956,10 +943,12 @@ static int intel_component_dais_suspend(struct snd_soc_component *component) struct snd_soc_dai *dai; /* - * In the corner case where a SUSPEND happens during a PAUSE, the ALSA core - * does not throw the TRIGGER_SUSPEND. This leaves the DAIs in an unbalanced state. - * Since the component suspend is called last, we can trap this corner case - * and force the DAIs to release their resources. + * Mark all open streams as suspended. + * Open streams at this point can be in SUSPENDED, PAUSED or STOPPED + * state and during prepare the DMAs and SHIM registers need to be + * initialized for them. + * The STOPPED state is a special corner case which can happen if audio + * experiences xrun at suspend time. */ for_each_component_dais(component, dai) { struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai); @@ -967,13 +956,7 @@ static int intel_component_dais_suspend(struct snd_soc_component *component) dai_runtime = cdns->dai_runtime_array[dai->id]; - if (!dai_runtime) - continue; - - if (dai_runtime->suspended) - continue; - - if (dai_runtime->paused) + if (dai_runtime) dai_runtime->suspended = true; } diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c index 63aef98361467..f0e0a79b30b0e 100644 --- a/drivers/soundwire/intel_ace2x.c +++ b/drivers/soundwire/intel_ace2x.c @@ -822,19 +822,6 @@ static int intel_trigger(struct snd_pcm_substream *substream, int cmd, struct sn } switch (cmd) { - case SNDRV_PCM_TRIGGER_SUSPEND: - - /* - * The .prepare callback is used to deal with xruns and resume operations. - * In the case of xruns, the DMAs and SHIM registers cannot be touched, - * but for resume operations the DMAs and SHIM registers need to be initialized. - * the .trigger callback is used to track the suspend case only. - */ - - dai_runtime->suspended = true; - - break; - case SNDRV_PCM_TRIGGER_PAUSE_PUSH: dai_runtime->paused = true; break; @@ -858,8 +845,34 @@ static const struct snd_soc_dai_ops intel_pcm_dai_ops = { .get_stream = intel_get_sdw_stream, }; +static int intel_component_dais_suspend(struct snd_soc_component *component) +{ + struct snd_soc_dai *dai; + + /* + * Mark all open streams as suspended. + * Open streams at this point can be in SUSPENDED, PAUSED or STOPPED + * state and during prepare the DMAs and SHIM registers need to be + * initialized for them. + * The STOPPED state is a special corner case which can happen if audio + * experiences xrun at suspend time. + */ + for_each_component_dais(component, dai) { + struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai); + struct sdw_cdns_dai_runtime *dai_runtime; + + dai_runtime = cdns->dai_runtime_array[dai->id]; + + if (dai_runtime) + dai_runtime->suspended = true; + } + + return 0; +} + static const struct snd_soc_component_driver dai_component = { .name = "soundwire", + .suspend = intel_component_dais_suspend, }; /* diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c index 8752b0e3ce74c..c0c8ffcecc6bf 100644 --- a/drivers/soundwire/intel_auxdevice.c +++ b/drivers/soundwire/intel_auxdevice.c @@ -51,6 +51,8 @@ struct wake_capable_part { }; static struct wake_capable_part wake_capable_list[] = { + {0x01fa, 0x2A30}, + {0x01fa, 0x2A3B}, {0x01fa, 0x4243}, {0x01fa, 0x4245}, {0x025d, 0x5682}, diff --git a/drivers/soundwire/qcom.c b/drivers/soundwire/qcom.c index 5b3078220189b..a4f108fdffddb 100644 --- a/drivers/soundwire/qcom.c +++ b/drivers/soundwire/qcom.c @@ -1201,7 +1201,7 @@ static int qcom_swrm_stream_alloc_ports(struct qcom_swrm_ctrl *ctrl, else pn = find_first_zero_bit(port_mask, maxport); - if (pn > maxport) { + if (pn >= maxport) { dev_err(ctrl->dev, "All ports busy\n"); ret = -EBUSY; goto out; diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c index 3d4d00188c26c..3c84027692460 100644 --- a/drivers/soundwire/slave.c +++ b/drivers/soundwire/slave.c @@ -240,8 +240,8 @@ int sdw_of_find_slaves(struct sdw_bus *bus) struct sdw_slave_id id; const __be32 *addr; - compat = of_get_property(node, "compatible", NULL); - if (!compat) + ret = of_property_read_string(node, "compatible", &compat); + if (ret) continue; ret = sscanf(compat, "sdw%01x%04hx%04hx%02hhx", &sdw_version, diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c index 970fddbb25857..8ddb7a1a40567 100644 --- a/drivers/spi/spi-amlogic-spisg.c +++ b/drivers/spi/spi-amlogic-spisg.c @@ -640,7 +640,7 @@ static int aml_spisg_target_abort(struct spi_controller *ctlr) static int aml_spisg_clk_init(struct spisg_device *spisg, void __iomem *base) { struct device *dev = &spisg->pdev->dev; - struct clk_init_data init; + struct clk_init_data init = {}; struct clk_divider *div; struct clk_div_table *tbl; char name[32]; diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c index 18261cbd413b4..b8bf8b5be2cf7 100644 --- a/drivers/spi/spi-bcm63xx-hsspi.c +++ b/drivers/spi/spi-bcm63xx-hsspi.c @@ -921,7 +921,13 @@ static int bcm63xx_hsspi_resume(struct device *dev) } } - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + if (bs->pll_clk) + clk_disable_unprepare(bs->pll_clk); + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c index cf7eefd39839f..8d152e906daac 100644 --- a/drivers/spi/spi-bcm63xx.c +++ b/drivers/spi/spi-bcm63xx.c @@ -661,7 +661,11 @@ static int bcm63xx_spi_resume(struct device *dev) if (ret) return ret; - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-bcmbca-hsspi.c b/drivers/spi/spi-bcmbca-hsspi.c index f0f0e6b29668e..a1ff33f6dc095 100644 --- a/drivers/spi/spi-bcmbca-hsspi.c +++ b/drivers/spi/spi-bcmbca-hsspi.c @@ -612,7 +612,13 @@ static int bcmbca_hsspi_resume(struct device *dev) } } - spi_controller_resume(host); + ret = spi_controller_resume(host); + if (ret) { + if (bs->pll_clk) + clk_disable_unprepare(bs->pll_clk); + clk_disable_unprepare(bs->clk); + return ret; + } return 0; } diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c index 7a745c2e0892c..640e8723a08ee 100644 --- a/drivers/spi/spi-cadence.c +++ b/drivers/spi/spi-cadence.c @@ -109,6 +109,7 @@ * @rxbuf: Pointer to the RX buffer * @tx_bytes: Number of bytes left to transfer * @rx_bytes: Number of bytes requested + * @n_bytes: Number of bytes per word * @dev_busy: Device busy flag * @is_decoded_cs: Flag for decoder property set or not * @tx_fifo_depth: Depth of the TX FIFO @@ -120,16 +121,24 @@ struct cdns_spi { struct clk *pclk; unsigned int clk_rate; u32 speed_hz; - const u8 *txbuf; - u8 *rxbuf; + const void *txbuf; + void *rxbuf; int tx_bytes; int rx_bytes; + u8 n_bytes; u8 dev_busy; u32 is_decoded_cs; unsigned int tx_fifo_depth; struct reset_control *rstc; }; +enum cdns_spi_frame_n_bytes { + CDNS_SPI_N_BYTES_NULL = 0, + CDNS_SPI_N_BYTES_U8 = 1, + CDNS_SPI_N_BYTES_U16 = 2, + CDNS_SPI_N_BYTES_U32 = 4 +}; + /* Macros for the SPI controller read/write */ static inline u32 cdns_spi_read(struct cdns_spi *xspi, u32 offset) { @@ -305,13 +314,87 @@ static int cdns_spi_setup_transfer(struct spi_device *spi, return 0; } +static u8 cdns_spi_n_bytes(struct spi_transfer *transfer) +{ + if (transfer->bits_per_word <= 8) + return CDNS_SPI_N_BYTES_U8; + else if (transfer->bits_per_word <= 16) + return CDNS_SPI_N_BYTES_U16; + else + return CDNS_SPI_N_BYTES_U32; +} + +static inline void cdns_spi_reader(struct cdns_spi *xspi) +{ + u32 rxw = 0; + + if (xspi->rxbuf && !IS_ALIGNED((uintptr_t)xspi->rxbuf, xspi->n_bytes)) { + pr_err("%s: rxbuf address is not aligned for %d bytes\n", + __func__, xspi->n_bytes); + return; + } + + rxw = cdns_spi_read(xspi, CDNS_SPI_RXD); + if (xspi->rxbuf) { + switch (xspi->n_bytes) { + case CDNS_SPI_N_BYTES_U8: + *(u8 *)xspi->rxbuf = rxw; + break; + case CDNS_SPI_N_BYTES_U16: + *(u16 *)xspi->rxbuf = rxw; + break; + case CDNS_SPI_N_BYTES_U32: + *(u32 *)xspi->rxbuf = rxw; + break; + default: + pr_err("%s invalid n_bytes %d\n", __func__, + xspi->n_bytes); + return; + } + xspi->rxbuf = (u8 *)xspi->rxbuf + xspi->n_bytes; + } +} + +static inline void cdns_spi_writer(struct cdns_spi *xspi) +{ + u32 txw = 0; + + if (xspi->txbuf && !IS_ALIGNED((uintptr_t)xspi->txbuf, xspi->n_bytes)) { + pr_err("%s: txbuf address is not aligned for %d bytes\n", + __func__, xspi->n_bytes); + return; + } + + if (xspi->txbuf) { + switch (xspi->n_bytes) { + case CDNS_SPI_N_BYTES_U8: + txw = *(u8 *)xspi->txbuf; + break; + case CDNS_SPI_N_BYTES_U16: + txw = *(u16 *)xspi->txbuf; + break; + case CDNS_SPI_N_BYTES_U32: + txw = *(u32 *)xspi->txbuf; + break; + default: + pr_err("%s invalid n_bytes %d\n", __func__, + xspi->n_bytes); + return; + } + cdns_spi_write(xspi, CDNS_SPI_TXD, txw); + xspi->txbuf = (u8 *)xspi->txbuf + xspi->n_bytes; + } +} + /** * cdns_spi_process_fifo - Fills the TX FIFO, and drain the RX FIFO + * @ctlr: Pointer to the spi_controller structure * @xspi: Pointer to the cdns_spi structure * @ntx: Number of bytes to pack into the TX FIFO * @nrx: Number of bytes to drain from the RX FIFO */ -static void cdns_spi_process_fifo(struct cdns_spi *xspi, int ntx, int nrx) +static void cdns_spi_process_fifo(struct spi_controller *ctlr, + struct cdns_spi *xspi, int ntx, int nrx) { ntx = clamp(ntx, 0, xspi->tx_bytes); nrx = clamp(nrx, 0, xspi->rx_bytes); @@ -321,23 +404,24 @@ static void cdns_spi_process_fifo(struct cdns_spi *xspi, int ntx, int nrx) while (ntx || nrx) { if (nrx) { - u8 data = cdns_spi_read(xspi, CDNS_SPI_RXD); - - if (xspi->rxbuf) - *xspi->rxbuf++ = data; - + cdns_spi_reader(xspi); nrx--; } if (ntx) { - if (xspi->txbuf) - cdns_spi_write(xspi, CDNS_SPI_TXD, *xspi->txbuf++); - else - cdns_spi_write(xspi, CDNS_SPI_TXD, 0); + /* When xspi in busy condition, bytes may send failed, + * then spi control didn't work thoroughly, add one byte + * delay. Only in host mode; in target mode this delay + * causes data corruption as the target fails to prepare + * data in time. + */ + if (!spi_controller_is_target(ctlr) && + (cdns_spi_read(xspi, CDNS_SPI_ISR) & CDNS_SPI_IXR_TXFULL)) + udelay(10); + cdns_spi_writer(xspi); ntx--; } - } } @@ -388,14 +472,14 @@ static irqreturn_t cdns_spi_irq(int irq, void *dev_id) cdns_spi_write(xspi, CDNS_SPI_THLD, 1); if (xspi->tx_bytes) { - cdns_spi_process_fifo(xspi, trans_cnt, trans_cnt); + cdns_spi_process_fifo(ctlr, xspi, trans_cnt, trans_cnt); } else { /* Fixed delay due to controller limitation with * RX_NEMPTY incorrect status * Xilinx AR:65885 contains more details */ udelay(10); - cdns_spi_process_fifo(xspi, 0, trans_cnt); + cdns_spi_process_fifo(ctlr, xspi, 0, trans_cnt); cdns_spi_write(xspi, CDNS_SPI_IDR, CDNS_SPI_IXR_DEFAULT); spi_finalize_current_transfer(ctlr); @@ -448,13 +532,11 @@ static int cdns_transfer_one(struct spi_controller *ctlr, cdns_spi_write(xspi, CDNS_SPI_THLD, xspi->tx_fifo_depth >> 1); } - /* When xspi in busy condition, bytes may send failed, - * then spi control didn't work thoroughly, add one byte delay - */ - if (cdns_spi_read(xspi, CDNS_SPI_ISR) & CDNS_SPI_IXR_TXFULL) - udelay(10); + xspi->n_bytes = cdns_spi_n_bytes(transfer); + xspi->tx_bytes = DIV_ROUND_UP(xspi->tx_bytes, xspi->n_bytes); + xspi->rx_bytes = DIV_ROUND_UP(xspi->rx_bytes, xspi->n_bytes); - cdns_spi_process_fifo(xspi, xspi->tx_fifo_depth, 0); + cdns_spi_process_fifo(ctlr, xspi, xspi->tx_fifo_depth, 0); cdns_spi_write(xspi, CDNS_SPI_IER, CDNS_SPI_IXR_DEFAULT); return transfer->len; @@ -653,6 +735,10 @@ static int cdns_spi_probe(struct platform_device *pdev) ctlr->unprepare_transfer_hardware = cdns_unprepare_transfer_hardware; ctlr->mode_bits = SPI_CPOL | SPI_CPHA; ctlr->bits_per_word_mask = SPI_BPW_MASK(8); + ctlr->flags = SPI_CONTROLLER_MUST_TX; + + if (of_device_is_compatible(pdev->dev.of_node, "cix,sky1-spi-r1p6")) + ctlr->bits_per_word_mask |= SPI_BPW_MASK(16) | SPI_BPW_MASK(32); if (!spi_controller_is_target(ctlr)) { ctlr->mode_bits |= SPI_CS_HIGH; @@ -812,6 +898,7 @@ static const struct dev_pm_ops cdns_spi_dev_pm_ops = { static const struct of_device_id cdns_spi_of_match[] = { { .compatible = "xlnx,zynq-spi-r1p6" }, + { .compatible = "cix,sky1-spi-r1p6" }, { .compatible = "cdns,spi-r1p6" }, { /* end of table */ } }; diff --git a/drivers/spi/spi-davinci.c b/drivers/spi/spi-davinci.c index a29934422356b..c0963483e05c8 100644 --- a/drivers/spi/spi-davinci.c +++ b/drivers/spi/spi-davinci.c @@ -868,7 +868,7 @@ static int davinci_spi_probe(struct platform_device *pdev) int ret = 0; u32 spipc0; - host = spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct davinci_spi)); if (host == NULL) { ret = -ENOMEM; goto err; @@ -998,7 +998,6 @@ static int davinci_spi_probe(struct platform_device *pdev) dma_release_channel(dspi->dma_tx); } free_host: - spi_controller_put(host); err: return ret; } @@ -1022,6 +1021,8 @@ static void davinci_spi_remove(struct platform_device *pdev) spi_bitbang_stop(&dspi->bitbang); + devm_free_irq(&pdev->dev, dspi->irq, dspi); + /* This bit needs to be cleared to disable dpsi->clk */ clear_io_bits(dspi->base + SPIGCR1, SPIGCR1_POWERDOWN_MASK); @@ -1029,8 +1030,6 @@ static void davinci_spi_remove(struct platform_device *pdev) dma_release_channel(dspi->dma_rx); dma_release_channel(dspi->dma_tx); } - - spi_controller_put(host); } static struct platform_driver davinci_spi_driver = { diff --git a/drivers/spi/spi-dw-dma.c b/drivers/spi/spi-dw-dma.c index e52fa329c2476..2c8978f8388cf 100644 --- a/drivers/spi/spi-dw-dma.c +++ b/drivers/spi/spi-dw-dma.c @@ -282,7 +282,8 @@ static int dw_spi_dma_wait(struct dw_spi *dws, unsigned int len, u32 speed) static inline bool dw_spi_dma_tx_busy(struct dw_spi *dws) { - return !(dw_readl(dws, DW_SPI_SR) & DW_SPI_SR_TF_EMPT); + return (dw_readl(dws, DW_SPI_SR) & + (DW_SPI_SR_BUSY | DW_SPI_SR_TF_EMPT)) != DW_SPI_SR_TF_EMPT; } static int dw_spi_dma_wait_tx_done(struct dw_spi *dws, diff --git a/drivers/spi/spi-dw-mmio.c b/drivers/spi/spi-dw-mmio.c index 8ab9bc488b4e6..a3b10e44004db 100644 --- a/drivers/spi/spi-dw-mmio.c +++ b/drivers/spi/spi-dw-mmio.c @@ -229,8 +229,8 @@ static int dw_spi_hssi_init(struct platform_device *pdev, return 0; } -static int dw_spi_intel_init(struct platform_device *pdev, - struct dw_spi_mmio *dwsmmio) +static int dw_spi_hssi_no_dma_init(struct platform_device *pdev, + struct dw_spi_mmio *dwsmmio) { dwsmmio->dws.ip = DW_HSSI_ID; @@ -417,7 +417,7 @@ static const struct of_device_id dw_spi_mmio_of_match[] = { { .compatible = "amazon,alpine-dw-apb-ssi", .data = dw_spi_alpine_init}, { .compatible = "renesas,rzn1-spi", .data = dw_spi_pssi_init}, { .compatible = "snps,dwc-ssi-1.01a", .data = dw_spi_hssi_init}, - { .compatible = "intel,keembay-ssi", .data = dw_spi_intel_init}, + { .compatible = "intel,keembay-ssi", .data = dw_spi_hssi_no_dma_init}, { .compatible = "intel,mountevans-imc-ssi", .data = dw_spi_mountevans_imc_init, @@ -432,6 +432,7 @@ MODULE_DEVICE_TABLE(of, dw_spi_mmio_of_match); #ifdef CONFIG_ACPI static const struct acpi_device_id dw_spi_mmio_acpi_match[] = { {"HISI0173", (kernel_ulong_t)dw_spi_pssi_init}, + {"LECA0002", (kernel_ulong_t)dw_spi_hssi_no_dma_init}, {}, }; MODULE_DEVICE_TABLE(acpi, dw_spi_mmio_acpi_match); diff --git a/drivers/spi/spi-dw.h b/drivers/spi/spi-dw.h index 2e7d8527d0a1c..59502be52ee26 100644 --- a/drivers/spi/spi-dw.h +++ b/drivers/spi/spi-dw.h @@ -286,6 +286,7 @@ static inline void dw_spi_shutdown_chip(struct dw_spi *dws) dw_spi_enable_chip(dws, 0); dw_spi_set_clk(dws, 0); dws->current_freq = 0; + dws->cur_rx_sample_dly = 0; } extern void dw_spi_set_cs(struct spi_device *spi, bool enable); diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c index 736120107184f..ec40fcb1067b7 100644 --- a/drivers/spi/spi-geni-qcom.c +++ b/drivers/spi/spi-geni-qcom.c @@ -657,25 +657,30 @@ static int spi_geni_init(struct spi_geni_master *mas) u32 spi_tx_cfg, fifo_disable; int ret = -ENXIO; - pm_runtime_get_sync(mas->dev); + PM_RUNTIME_ACQUIRE_IF_ENABLED(mas->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret < 0) { + dev_err(mas->dev, "Failed to resume and get %d\n", ret); + return ret; + } proto = geni_se_read_proto(se); if (spi->target) { if (proto != GENI_SE_SPI_SLAVE) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } spi_slv_setup(mas); } else if (proto == GENI_SE_INVALID_PROTO) { ret = geni_load_se_firmware(se, GENI_SE_SPI); if (ret) { dev_err(mas->dev, "spi master firmware load failed ret: %d\n", ret); - goto out_pm; + return ret; } } else if (proto != GENI_SE_SPI) { dev_err(mas->dev, "Invalid proto %d\n", proto); - goto out_pm; + return -EINVAL; } mas->tx_fifo_depth = geni_se_get_tx_fifo_depth(se); @@ -708,7 +713,7 @@ static int spi_geni_init(struct spi_geni_master *mas) dev_dbg(mas->dev, "Using GPI DMA mode for SPI\n"); break; } else if (ret == -EPROBE_DEFER) { - goto out_pm; + return ret; } /* * in case of failure to get gpi dma channel, we can still do the @@ -737,8 +742,6 @@ static int spi_geni_init(struct spi_geni_master *mas) writel(spi_tx_cfg, se->base + SE_SPI_TRANS_CFG); } -out_pm: - pm_runtime_put(mas->dev); return ret; } diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c index a6d15b9ca5098..4f3683a07555e 100644 --- a/drivers/spi/spi-img-spfi.c +++ b/drivers/spi/spi-img-spfi.c @@ -616,7 +616,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->tx_ch); spfi->tx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } spfi->rx_ch = dma_request_chan(spfi->dev, "rx"); @@ -624,7 +624,7 @@ static int img_spfi_probe(struct platform_device *pdev) ret = PTR_ERR(spfi->rx_ch); spfi->rx_ch = NULL; if (ret == -EPROBE_DEFER) - goto disable_pm; + goto free_dma; } if (!spfi->tx_ch || !spfi->rx_ch) { @@ -652,6 +652,7 @@ static int img_spfi_probe(struct platform_device *pdev) disable_pm: pm_runtime_disable(spfi->dev); +free_dma: if (spfi->rx_ch) dma_release_channel(spfi->rx_ch); if (spfi->tx_ch) diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index 95ccb1f7dfafa..b504a5c86692a 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -340,6 +340,18 @@ struct nxp_fspi_devtype_data { unsigned int quirks; unsigned int lut_num; bool little_endian; + /* + * The max clock rate (Hz) that FlexSPI can output to the device + * in SDR mode (RXCLKSRC=0). Defaults to 66MHz if zero. + * Some SoCs (e.g. LX2160A) support up to 100MHz in SDR mode. + */ + unsigned long max_sdr_rate; + /* + * The max clock rate (Hz) that FlexSPI can output to the device + * in DTR mode (RXCLKSRC=3). Defaults to 166MHz if zero. + * Some SoCs (e.g. i.MX95, i.MX8QM, i.MX8DXL) support up to 200MHz. + */ + unsigned long max_dtr_rate; }; static struct nxp_fspi_devtype_data lx2160a_data = { @@ -349,6 +361,10 @@ static struct nxp_fspi_devtype_data lx2160a_data = { .quirks = FSPI_QUIRK_DISABLE_DTR, .lut_num = 32, .little_endian = true, /* little-endian */ + /* + * LX2160ACEC: SDR RXCLKSRC=0 max 100MHz, DTR disabled via quirk. + */ + .max_sdr_rate = 100000000, }; static struct nxp_fspi_devtype_data imx8mm_data = { @@ -358,6 +374,21 @@ static struct nxp_fspi_devtype_data imx8mm_data = { .quirks = 0, .lut_num = 32, .little_endian = true, /* little-endian */ + /* IMX8MMCEC §3.9.10: SDR RXCLKSRC=0 max 66MHz, DDR RXCLKSRC=3 max 166MHz */ + .max_sdr_rate = 66000000, + .max_dtr_rate = 166000000, +}; + +static struct nxp_fspi_devtype_data imx8mp_data = { + .rxfifo = SZ_512, /* (64 * 64 bits) */ + .txfifo = SZ_1K, /* (128 * 64 bits) */ + .ahb_buf_size = SZ_2K, /* (256 * 64 bits) */ + .quirks = 0, + .lut_num = 32, + .little_endian = true, /* little-endian */ + /* IMX8MPCEC: SDR RXCLKSRC=0 max 66MHz, DDR RXCLKSRC=3 max 166MHz */ + .max_sdr_rate = 66000000, + .max_dtr_rate = 166000000, }; static struct nxp_fspi_devtype_data imx8qxp_data = { @@ -367,6 +398,12 @@ static struct nxp_fspi_devtype_data imx8qxp_data = { .quirks = 0, .lut_num = 32, .little_endian = true, /* little-endian */ + /* + * IMX8QXPCEC: SDR RXCLKSRC=0 max 60MHz, DDR RXCLKSRC=3 max 200MHz. + * i.MX8QM and i.MX8DXL share the same FlexSPI IP and limits. + */ + .max_sdr_rate = 60000000, + .max_dtr_rate = 200000000, }; static struct nxp_fspi_devtype_data imx8dxl_data = { @@ -376,6 +413,12 @@ static struct nxp_fspi_devtype_data imx8dxl_data = { .quirks = FSPI_QUIRK_USE_IP_ONLY, .lut_num = 32, .little_endian = true, /* little-endian */ + /* + * IMX8DXLCEC (i.MX 8XLite): SDR RXCLKSRC=0 max 60MHz, + * DDR RXCLKSRC=3 max 200MHz. + */ + .max_sdr_rate = 60000000, + .max_dtr_rate = 200000000, }; static struct nxp_fspi_devtype_data imx8ulp_data = { @@ -385,6 +428,29 @@ static struct nxp_fspi_devtype_data imx8ulp_data = { .quirks = 0, .lut_num = 16, .little_endian = true, /* little-endian */ + /* + * IMX8ULPCEC §7.3.1, Normal Drive (ND, 1.0V) mode: + * SDR RXCLKSRC=0 max 60MHz, DDR RXCLKSRC=3 max 166MHz. + * Note: Overdrive (OD, 1.05V) allows up to 180MHz DTR + * but is not the default use case. + */ + .max_sdr_rate = 60000000, + .max_dtr_rate = 166000000, +}; + +static struct nxp_fspi_devtype_data imx95_data = { + .rxfifo = SZ_512, /* (64 * 64 bits) */ + .txfifo = SZ_1K, /* (128 * 64 bits) */ + .ahb_buf_size = SZ_2K, /* (256 * 64 bits) */ + .quirks = 0, + .lut_num = 32, + .little_endian = true, /* little-endian */ + /* + * IMX95CEC Rev.8 §4.11.7: SDR RXCLKSRC=0 max 66MHz, + * DDR RXCLKSRC=3 max 200MHz (Nominal/Overdrive mode). + */ + .max_sdr_rate = 66000000, + .max_dtr_rate = 200000000, }; struct nxp_fspi { @@ -691,10 +757,20 @@ static void nxp_fspi_select_rx_sample_clk_source(struct nxp_fspi *f, reg = fspi_readl(f, f->iobase + FSPI_MCR0); if (op_is_dtr) { reg |= FSPI_MCR0_RXCLKSRC(3); - f->max_rate = 166000000; + /* + * Use the SoC-specific DTR max rate if provided, otherwise + * fall back to 166MHz (limit from IMX8MN datasheet §3.9.9). + */ + f->max_rate = f->devtype_data->max_dtr_rate ? + f->devtype_data->max_dtr_rate : 166000000; } else { /*select mode 0 */ reg &= ~FSPI_MCR0_RXCLKSRC(3); - f->max_rate = 66000000; + /* + * Use the SoC-specific SDR max rate if provided, otherwise + * fall back to 66MHz (limit from IMX8MN datasheet §3.9.9). + */ + f->max_rate = f->devtype_data->max_sdr_rate ? + f->devtype_data->max_sdr_rate : 66000000; } fspi_writel(f, reg, f->iobase + FSPI_MCR0); } @@ -1446,10 +1522,11 @@ static const struct dev_pm_ops nxp_fspi_pm_ops = { static const struct of_device_id nxp_fspi_dt_ids[] = { { .compatible = "nxp,lx2160a-fspi", .data = (void *)&lx2160a_data, }, { .compatible = "nxp,imx8mm-fspi", .data = (void *)&imx8mm_data, }, - { .compatible = "nxp,imx8mp-fspi", .data = (void *)&imx8mm_data, }, + { .compatible = "nxp,imx8mp-fspi", .data = (void *)&imx8mp_data, }, { .compatible = "nxp,imx8qxp-fspi", .data = (void *)&imx8qxp_data, }, { .compatible = "nxp,imx8dxl-fspi", .data = (void *)&imx8dxl_data, }, { .compatible = "nxp,imx8ulp-fspi", .data = (void *)&imx8ulp_data, }, + { .compatible = "nxp,imx95-fspi", .data = (void *)&imx95_data, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, nxp_fspi_dt_ids); diff --git a/drivers/spi/spi-oc-tiny.c b/drivers/spi/spi-oc-tiny.c index cba2299203573..cc5bceb1cc88b 100644 --- a/drivers/spi/spi-oc-tiny.c +++ b/drivers/spi/spi-oc-tiny.c @@ -211,11 +211,11 @@ static int tiny_spi_probe(struct platform_device *pdev) struct tiny_spi_platform_data *platp = dev_get_platdata(&pdev->dev); struct tiny_spi *hw; struct spi_controller *host; - int err = -ENODEV; + int err; - host = spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); + host = devm_spi_alloc_host(&pdev->dev, sizeof(struct tiny_spi)); if (!host) - return err; + return -ENOMEM; /* setup the host state. */ host->bus_num = pdev->id; @@ -233,10 +233,8 @@ static int tiny_spi_probe(struct platform_device *pdev) /* find and map our resources */ hw->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(hw->base)) { - err = PTR_ERR(hw->base); - goto exit; - } + if (IS_ERR(hw->base)) + return PTR_ERR(hw->base); /* irq is optional */ hw->irq = platform_get_irq(pdev, 0); if (hw->irq >= 0) { @@ -244,7 +242,7 @@ static int tiny_spi_probe(struct platform_device *pdev) err = devm_request_irq(&pdev->dev, hw->irq, tiny_spi_irq, 0, pdev->name, hw); if (err) - goto exit; + return err; } /* find platform data */ if (platp) { @@ -253,29 +251,23 @@ static int tiny_spi_probe(struct platform_device *pdev) } else { err = tiny_spi_of_probe(pdev); if (err) - goto exit; + return err; } /* register our spi controller */ err = spi_bitbang_start(&hw->bitbang); if (err) - goto exit; + return err; dev_info(&pdev->dev, "base %p, irq %d\n", hw->base, hw->irq); return 0; - -exit: - spi_controller_put(host); - return err; } static void tiny_spi_remove(struct platform_device *pdev) { struct tiny_spi *hw = platform_get_drvdata(pdev); - struct spi_controller *host = hw->bitbang.ctlr; spi_bitbang_stop(&hw->bitbang); - spi_controller_put(host); } #ifdef CONFIG_OF diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c index 3aeddada58e1c..44175367bbd00 100644 --- a/drivers/spi/spi-qcom-qspi.c +++ b/drivers/spi/spi-qcom-qspi.c @@ -759,7 +759,8 @@ static int qcom_qspi_probe(struct platform_device *pdev) return dev_err_probe(dev, ret, "could not set DMA mask\n"); host->max_speed_hz = 300000000; - host->max_dma_len = 65536; /* as per HPG */ + /* as per HPG, it is 64KB, limit to 60KB to avoid boundary condition failures */ + host->max_dma_len = 0xf000; host->dma_alignment = QSPI_ALIGN_REQ; host->num_chipselect = QSPI_NUM_CS; host->bus_num = -1; @@ -819,20 +820,34 @@ static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev) struct qcom_qspi *ctrl = spi_controller_get_devdata(host); int ret; - /* Drop the performance state vote */ - dev_pm_opp_set_rate(dev, 0); clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks); ret = icc_disable(ctrl->icc_path_cpu_to_qspi); if (ret) { dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n", __func__, ret); - return ret; + goto err_enable_clk; } - pinctrl_pm_select_sleep_state(dev); + ret = pinctrl_pm_select_sleep_state(dev); + if (ret) + goto err_enable_icc; + + /* Drop the performance state vote */ + ret = dev_pm_opp_set_rate(dev, 0); + if (ret) + goto err_select_default_state; return 0; + +err_select_default_state: + pinctrl_pm_select_default_state(dev); +err_enable_icc: + icc_enable(ctrl->icc_path_cpu_to_qspi); +err_enable_clk: + if (clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks)) + dev_err_ratelimited(ctrl->dev, "Failed to re-enable clocks\n"); + return ret; } static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) @@ -841,20 +856,34 @@ static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev) struct qcom_qspi *ctrl = spi_controller_get_devdata(host); int ret; - pinctrl_pm_select_default_state(dev); + ret = dev_pm_opp_set_rate(dev, ctrl->last_speed * 4); + if (ret) + return ret; + + ret = pinctrl_pm_select_default_state(dev); + if (ret) + goto err_opp_set_rate_zero; ret = icc_enable(ctrl->icc_path_cpu_to_qspi); if (ret) { dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n", __func__, ret); - return ret; + goto err_select_sleep_state; } ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks); if (ret) - return ret; + goto err_disable_icc; - return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4); + return 0; + +err_disable_icc: + icc_disable(ctrl->icc_path_cpu_to_qspi); +err_select_sleep_state: + pinctrl_pm_select_sleep_state(dev); +err_opp_set_rate_zero: + dev_pm_opp_set_rate(dev, 0); + return ret; } static int __maybe_unused qcom_qspi_suspend(struct device *dev) diff --git a/drivers/spi/spi-qpic-snand.c b/drivers/spi/spi-qpic-snand.c index 58ceea1ea8fb4..42f8fb272fa68 100644 --- a/drivers/spi/spi-qpic-snand.c +++ b/drivers/spi/spi-qpic-snand.c @@ -1362,6 +1362,22 @@ static int qcom_spi_send_cmdaddr(struct qcom_nand_controller *snandc, snandc->regs->addr0 = cpu_to_le32(op->addr.val); snandc->regs->addr1 = cpu_to_le32(0); + /* + * The feature value has to reach NAND_FLASH_FEATURES before the + * command is executed, otherwise the controller programs the chip + * with whatever the register happened to hold from a previous + * operation. + */ + if (opcode == SPINAND_SET_FEATURE) { + u32 ftr = 0; + + memcpy(&ftr, op->data.buf.out, + min_t(size_t, op->data.nbytes, sizeof(ftr))); + snandc->regs->flash_feature = cpu_to_le32(ftr); + qcom_write_reg_dma(snandc, &snandc->regs->flash_feature, + NAND_FLASH_FEATURES, 1, NAND_BAM_NEXT_SGL); + } + qcom_write_reg_dma(snandc, &snandc->regs->cmd, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL); qcom_write_reg_dma(snandc, &snandc->regs->exec, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL); @@ -1399,10 +1415,8 @@ static int qcom_spi_io_op(struct qcom_nand_controller *snandc, const struct spi_ copy_ftr = true; break; case SPINAND_SET_FEATURE: - snandc->regs->flash_feature = cpu_to_le32(*(u32 *)op->data.buf.out); - qcom_write_reg_dma(snandc, &snandc->regs->flash_feature, - NAND_FLASH_FEATURES, 1, NAND_BAM_NEXT_SGL); - break; + /* fully handled by qcom_spi_send_cmdaddr() */ + return 0; case SPINAND_PROGRAM_EXECUTE: case SPINAND_WRITE_EN: case SPINAND_RESET: diff --git a/drivers/spi/spi-sprd-adi.c b/drivers/spi/spi-sprd-adi.c index f25b34a91756f..72f2f30e657ed 100644 --- a/drivers/spi/spi-sprd-adi.c +++ b/drivers/spi/spi-sprd-adi.c @@ -553,6 +553,12 @@ static int sprd_adi_probe(struct platform_device *pdev) return -ENXIO; } else { switch (ret) { + case 0: + /* + * Only reachable with CONFIG_HWSPINLOCK=n, where the + * of_hwspin_lock_get_id() stub returns 0. + */ + fallthrough; case -ENOENT: dev_info(&pdev->dev, "no hardware spinlock supplied\n"); break; diff --git a/drivers/spi/spi-tegra210-quad.c b/drivers/spi/spi-tegra210-quad.c index 83def82fe48c1..543f13503df3d 100644 --- a/drivers/spi/spi-tegra210-quad.c +++ b/drivers/spi/spi-tegra210-quad.c @@ -226,11 +226,13 @@ struct tegra_qspi { struct completion xfer_completion; struct spi_transfer *curr_xfer; + struct device *rx_dma_dev; struct dma_chan *rx_dma_chan; u32 *rx_dma_buf; dma_addr_t rx_dma_phys; struct dma_async_tx_descriptor *rx_dma_desc; + struct device *tx_dma_dev; struct dma_chan *tx_dma_chan; u32 *tx_dma_buf; dma_addr_t tx_dma_phys; @@ -574,15 +576,15 @@ static int tegra_qspi_dma_map_xfer(struct tegra_qspi *tqspi, struct spi_transfer len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; if (t->tx_buf) { - t->tx_dma = dma_map_single(tqspi->dev, (void *)tx_buf, len, DMA_TO_DEVICE); - if (dma_mapping_error(tqspi->dev, t->tx_dma)) + t->tx_dma = dma_map_single(tqspi->tx_dma_dev, (void *)tx_buf, len, DMA_TO_DEVICE); + if (dma_mapping_error(tqspi->tx_dma_dev, t->tx_dma)) return -ENOMEM; } if (t->rx_buf) { - t->rx_dma = dma_map_single(tqspi->dev, (void *)rx_buf, len, DMA_FROM_DEVICE); - if (dma_mapping_error(tqspi->dev, t->rx_dma)) { - dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE); + t->rx_dma = dma_map_single(tqspi->rx_dma_dev, (void *)rx_buf, len, DMA_FROM_DEVICE); + if (dma_mapping_error(tqspi->rx_dma_dev, t->rx_dma)) { + dma_unmap_single(tqspi->tx_dma_dev, t->tx_dma, len, DMA_TO_DEVICE); return -ENOMEM; } } @@ -597,9 +599,9 @@ static void tegra_qspi_dma_unmap_xfer(struct tegra_qspi *tqspi, struct spi_trans len = DIV_ROUND_UP(tqspi->curr_dma_words * tqspi->bytes_per_word, 4) * 4; if (t->tx_buf) - dma_unmap_single(tqspi->dev, t->tx_dma, len, DMA_TO_DEVICE); + dma_unmap_single(tqspi->tx_dma_dev, t->tx_dma, len, DMA_TO_DEVICE); if (t->rx_buf) - dma_unmap_single(tqspi->dev, t->rx_dma, len, DMA_FROM_DEVICE); + dma_unmap_single(tqspi->rx_dma_dev, t->rx_dma, len, DMA_FROM_DEVICE); } static int tegra_qspi_start_dma_based_transfer(struct tegra_qspi *tqspi, struct spi_transfer *t) @@ -745,7 +747,7 @@ static int tegra_qspi_start_cpu_based_transfer(struct tegra_qspi *qspi, struct s static void tegra_qspi_deinit_dma(struct tegra_qspi *tqspi) { if (tqspi->tx_dma_buf) { - dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, + dma_free_coherent(tqspi->tx_dma_dev, tqspi->dma_buf_size, tqspi->tx_dma_buf, tqspi->tx_dma_phys); tqspi->tx_dma_buf = NULL; } @@ -756,7 +758,7 @@ static void tegra_qspi_deinit_dma(struct tegra_qspi *tqspi) } if (tqspi->rx_dma_buf) { - dma_free_coherent(tqspi->dev, tqspi->dma_buf_size, + dma_free_coherent(tqspi->rx_dma_dev, tqspi->dma_buf_size, tqspi->rx_dma_buf, tqspi->rx_dma_phys); tqspi->rx_dma_buf = NULL; } @@ -782,6 +784,7 @@ static int tegra_qspi_init_dma(struct tegra_qspi *tqspi) } tqspi->rx_dma_chan = dma_chan; + tqspi->rx_dma_dev = dmaengine_get_dma_device(tqspi->rx_dma_chan); dma_chan = dma_request_chan(tqspi->dev, "tx"); if (IS_ERR(dma_chan)) { @@ -790,15 +793,19 @@ static int tegra_qspi_init_dma(struct tegra_qspi *tqspi) } tqspi->tx_dma_chan = dma_chan; + tqspi->tx_dma_dev = dmaengine_get_dma_device(tqspi->tx_dma_chan); } else { if (!device_iommu_mapped(tqspi->dev)) { dev_warn(tqspi->dev, "IOMMU not enabled in device-tree, falling back to PIO mode\n"); return 0; } + + tqspi->rx_dma_dev = tqspi->dev; + tqspi->tx_dma_dev = tqspi->dev; } - dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); + dma_buf = dma_alloc_coherent(tqspi->rx_dma_dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); if (!dma_buf) { err = -ENOMEM; goto err_out; @@ -807,7 +814,7 @@ static int tegra_qspi_init_dma(struct tegra_qspi *tqspi) tqspi->rx_dma_buf = dma_buf; tqspi->rx_dma_phys = dma_phys; - dma_buf = dma_alloc_coherent(tqspi->dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); + dma_buf = dma_alloc_coherent(tqspi->tx_dma_dev, tqspi->dma_buf_size, &dma_phys, GFP_KERNEL); if (!dma_buf) { err = -ENOMEM; goto err_out; diff --git a/drivers/spi/spi-virtio.c b/drivers/spi/spi-virtio.c index 2acb929b2c690..fb30c9207a4b0 100644 --- a/drivers/spi/spi-virtio.c +++ b/drivers/spi/spi-virtio.c @@ -361,6 +361,8 @@ static int virtio_spi_probe(struct virtio_device *vdev) if (ret) return dev_err_probe(&vdev->dev, ret, "Cannot register virtqueue cleanup\n"); + virtio_device_ready(vdev); + /* Use devm version to register controller */ ret = devm_spi_register_controller(&vdev->dev, ctrl); if (ret) diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c index 8085571913c8b..dac6c695f0727 100644 --- a/drivers/spi/spi-xilinx.c +++ b/drivers/spi/spi-xilinx.c @@ -285,7 +285,11 @@ static int xilinx_spi_txrx_bufs(struct spi_device *spi, struct spi_transfer *t) if (use_irq) { xspi->write_fn(cr, xspi->regs + XSPI_CR_OFFSET); - wait_for_completion(&xspi->done); + if (!wait_for_completion_timeout(&xspi->done, secs_to_jiffies(1))) { + dev_err(&spi->dev, "SPI transfer timed out\n"); + xspi_init_hw(xspi); + return -ETIMEDOUT; + } /* A transmit has just completed. Process received data * and check for more data to transmit. Always inhibit * the transmitter while the Isr refills the transmit diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c index 9dfbf56ce0d31..9530be2d2f61e 100644 --- a/drivers/spi/spi.c +++ b/drivers/spi/spi.c @@ -358,12 +358,16 @@ EXPORT_SYMBOL_GPL(spi_get_device_id); const void *spi_get_device_match_data(const struct spi_device *sdev) { const void *match; + const struct spi_device_id *id; match = device_get_match_data(&sdev->dev); if (match) return match; - return (const void *)spi_get_device_id(sdev)->driver_data; + id = spi_get_device_id(sdev); + if (!id) + return NULL; + return (const void *)id->driver_data; } EXPORT_SYMBOL_GPL(spi_get_device_match_data); @@ -1216,6 +1220,8 @@ void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev, spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0); } +static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg); + static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) { struct device *tx_dev, *rx_dev; @@ -1239,7 +1245,13 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) else rx_dev = ctlr->dev.parent; - ret = -ENOMSG; + /* + * Store the devices before mapping so partial failures can be unwound + * with the device that created each mapping. + */ + ctlr->cur_tx_dma_dev = tx_dev; + ctlr->cur_rx_dma_dev = rx_dev; + list_for_each_entry(xfer, &msg->transfers, transfer_list) { /* The sync is done before each transfer. */ unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC; @@ -1252,8 +1264,8 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) (void *)xfer->tx_buf, xfer->len, DMA_TO_DEVICE, attrs); - if (ret != 0) - return ret; + if (ret) + goto unwind; xfer->tx_sg_mapped = true; } @@ -1262,25 +1274,19 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg) ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg, xfer->rx_buf, xfer->len, DMA_FROM_DEVICE, attrs); - if (ret != 0) { - spi_unmap_buf_attrs(ctlr, tx_dev, - &xfer->tx_sg, DMA_TO_DEVICE, - attrs); - - return ret; - } + if (ret) + goto unwind; xfer->rx_sg_mapped = true; } } - /* No transfer has been mapped, bail out with success */ - if (ret) - return 0; - - ctlr->cur_rx_dma_dev = rx_dev; - ctlr->cur_tx_dma_dev = tx_dev; return 0; + +unwind: + __spi_unmap_msg(ctlr, msg); + + return ret; } static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg) @@ -3497,6 +3503,9 @@ int spi_controller_suspend(struct spi_controller *ctlr) { int ret = 0; + if (ctlr->cur_msg && spi_controller_is_target(ctlr) && ctlr->target_abort) + ctlr->target_abort(ctlr); + /* Basically no-ops for non-queued controllers */ if (ctlr->queued) { ret = spi_stop_queue(ctlr); diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c index 7e0f6b14aac8c..b4f5c37d48d42 100644 --- a/drivers/staging/fbtft/fbtft-core.c +++ b/drivers/staging/fbtft/fbtft-core.c @@ -301,6 +301,7 @@ static void fbtft_mkdirty(struct fb_info *info, int y, int height) { struct fbtft_par *par = info->par; struct fb_deferred_io *fbdefio = info->fbdefio; + unsigned long flags; /* special case, needed ? */ if (y == -1) { @@ -309,12 +310,12 @@ static void fbtft_mkdirty(struct fb_info *info, int y, int height) } /* Mark display lines/area as dirty */ - spin_lock(&par->dirty_lock); + spin_lock_irqsave(&par->dirty_lock, flags); if (y < par->dirty_lines_start) par->dirty_lines_start = y; if (y + height - 1 > par->dirty_lines_end) par->dirty_lines_end = y + height - 1; - spin_unlock(&par->dirty_lock); + spin_unlock_irqrestore(&par->dirty_lock, flags); /* Schedule deferred_io to update display (no-op if already on queue)*/ schedule_delayed_work(&info->deferred_work, fbdefio->delay); @@ -327,13 +328,13 @@ static void fbtft_deferred_io(struct fb_info *info, struct list_head *pagereflis struct fb_deferred_io_pageref *pageref; unsigned int y_low = 0, y_high = 0; - spin_lock(&par->dirty_lock); + spin_lock_irq(&par->dirty_lock); dirty_lines_start = par->dirty_lines_start; dirty_lines_end = par->dirty_lines_end; /* set display line markers as clean */ par->dirty_lines_start = par->info->var.yres - 1; par->dirty_lines_end = 0; - spin_unlock(&par->dirty_lock); + spin_unlock_irq(&par->dirty_lock); /* Mark display lines as dirty */ list_for_each_entry(pageref, pagereflist, list) { diff --git a/drivers/staging/fbtft/fbtft-sysfs.c b/drivers/staging/fbtft/fbtft-sysfs.c index e45c90a03a903..7bd6cbf2f1e56 100644 --- a/drivers/staging/fbtft/fbtft-sysfs.c +++ b/drivers/staging/fbtft/fbtft-sysfs.c @@ -98,7 +98,7 @@ sprintf_gamma(struct fbtft_par *par, u32 *curves, char *buf) mutex_lock(&par->gamma.lock); for (i = 0; i < par->gamma.num_curves; i++) { for (j = 0; j < par->gamma.num_values; j++) - len += scnprintf(&buf[len], PAGE_SIZE, + len += sysfs_emit_at(buf, len, "%04x ", curves[i * par->gamma.num_values + j]); buf[len - 1] = '\n'; } diff --git a/drivers/staging/gpib/common/gpib_os.c b/drivers/staging/gpib/common/gpib_os.c index a2bed6bd757a0..b223bd23a8422 100644 --- a/drivers/staging/gpib/common/gpib_os.c +++ b/drivers/staging/gpib/common/gpib_os.c @@ -289,18 +289,19 @@ int autopoll_all_devices(struct gpib_board *board) dev_dbg(board->gpib_dev, "autopoll has board lock\n"); retval = serial_poll_all(board, serial_timeout); - if (retval < 0) { - mutex_unlock(&board->big_gpib_mutex); - mutex_unlock(&board->user_mutex); - return retval; + if (retval >= 0) { + dev_dbg(board->gpib_dev, "complete\n"); + /* + * need to wake wait queue in case someone is + * waiting on RQS + */ + wake_up_interruptible(&board->wait); } - dev_dbg(board->gpib_dev, "complete\n"); - /* - * need to wake wait queue in case someone is - * waiting on RQS - */ - wake_up_interruptible(&board->wait); + if (retval <= 0) { + atomic_set(&board->stuck_srq, 1); + set_bit(SRQI_NUM, &board->status); + } mutex_unlock(&board->big_gpib_mutex); mutex_unlock(&board->user_mutex); diff --git a/drivers/staging/gpib/common/iblib.c b/drivers/staging/gpib/common/iblib.c index b672dd6aad25f..511e1d61c1fb2 100644 --- a/drivers/staging/gpib/common/iblib.c +++ b/drivers/staging/gpib/common/iblib.c @@ -193,9 +193,6 @@ static int autospoll_thread(void *board_void) } if (retval <= 0) { dev_err(board->gpib_dev, "stuck SRQ\n"); - - atomic_set(&board->stuck_srq, 1); // XXX could be better - set_bit(SRQI_NUM, &board->status); } } return retval; diff --git a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c index b6fddb437f552..67cc3398e2831 100644 --- a/drivers/staging/gpib/ni_usb/ni_usb_gpib.c +++ b/drivers/staging/gpib/ni_usb/ni_usb_gpib.c @@ -720,7 +720,7 @@ static int ni_usb_read(struct gpib_board *board, u8 *buffer, size_t length, break; } ni_usb_soft_update_status(board, status.ibsta, 0); - if (status.ibsta & END) + if ((status.ibsta & END) && (status.error_code == NIUSB_NO_ERROR)) *end = 1; else *end = 0; diff --git a/drivers/staging/greybus/audio_gb.c b/drivers/staging/greybus/audio_gb.c index 9d8994fdb41a2..144591f1a5128 100644 --- a/drivers/staging/greybus/audio_gb.c +++ b/drivers/staging/greybus/audio_gb.c @@ -37,6 +37,19 @@ int gb_audio_gb_get_topology(struct gb_connection *connection, return ret; } + /* + * The size_* fields are supplied by the module and are used by + * gbaudio_tplg_parse_data() to compute offsets into the blob; make + * sure the sections fit within the fetched topology, so walking it + * cannot read out of bounds. + */ + if ((u64)le32_to_cpu(topo->size_dais) + le32_to_cpu(topo->size_controls) + + le32_to_cpu(topo->size_widgets) + le32_to_cpu(topo->size_routes) > + size - sizeof(*topo)) { + kfree(topo); + return -EINVAL; + } + *topology = topo; return 0; diff --git a/drivers/staging/greybus/audio_manager_sysfs.c b/drivers/staging/greybus/audio_manager_sysfs.c index fcd518f9540cd..ff323ca8154f3 100644 --- a/drivers/staging/greybus/audio_manager_sysfs.c +++ b/drivers/staging/greybus/audio_manager_sysfs.c @@ -23,7 +23,7 @@ static ssize_t manager_sysfs_add_store(struct kobject *kobj, desc.name, &desc.vid, &desc.pid, &desc.intf_id, &desc.ip_devices, &desc.op_devices); - if (num != 7) + if (num != 6) return -EINVAL; num = gb_audio_manager_add(&desc); diff --git a/drivers/staging/greybus/hid.c b/drivers/staging/greybus/hid.c index afa78c96ede89..96154d8c5f383 100644 --- a/drivers/staging/greybus/hid.c +++ b/drivers/staging/greybus/hid.c @@ -256,7 +256,7 @@ static int __gb_hid_output_raw_report(struct hid_device *hid, __u8 *buf, if (report_id && ret >= 0) ret++; /* add report_id to the number of transferred bytes */ - return 0; + return ret; } static int gb_hid_raw_request(struct hid_device *hid, unsigned char reportnum, diff --git a/drivers/staging/media/ipu7/ipu7-isys.c b/drivers/staging/media/ipu7/ipu7-isys.c index cb2f49f3e0fa3..601e5a79ef8ec 100644 --- a/drivers/staging/media/ipu7/ipu7-isys.c +++ b/drivers/staging/media/ipu7/ipu7-isys.c @@ -233,6 +233,7 @@ static int isys_notifier_init(struct ipu7_isys *isys) err_parse: fwnode_handle_put(ep); + v4l2_async_nf_cleanup(&isys->notifier); return ret; } @@ -773,6 +774,7 @@ static int isys_probe(struct auxiliary_device *auxdev, return 0; out_cleanup: + isys_notifier_cleanup(isys); isys_unregister_devices(isys); out_cleanup_fw: ipu7_fw_isys_release(isys); diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c index 8ec571c6bd07f..34fe95145c826 100644 --- a/drivers/staging/media/ipu7/ipu7.c +++ b/drivers/staging/media/ipu7/ipu7.c @@ -2343,7 +2343,7 @@ static int ipu7_init_fw_code_region_by_sys(struct ipu7_bus_device *sys, return ret; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return ret; @@ -2702,7 +2702,7 @@ static int ipu7_resume(struct device *dev) if (ret) dev_err(dev, "IPC reset protocol failed!\n"); - ret = pm_runtime_get_sync(&isp->psys->auxdev.dev); + ret = pm_runtime_resume_and_get(&isp->psys->auxdev.dev); if (ret < 0) { dev_err(dev, "Failed to get runtime PM\n"); return 0; diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c index 49e497a32973b..ccb71de22b0a9 100644 --- a/drivers/staging/media/meson/vdec/vdec.c +++ b/drivers/staging/media/meson/vdec/vdec.c @@ -504,8 +504,8 @@ vdec_try_fmt_common(struct amvdec_session *sess, u32 size, case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: fmt_out = find_format(fmts, size, pixmp->pixelformat); if (!fmt_out) { - pixmp->pixelformat = V4L2_PIX_FMT_MPEG2; - fmt_out = find_format(fmts, size, pixmp->pixelformat); + pixmp->pixelformat = fmts[0].pixfmt; + fmt_out = &fmts[0]; } break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: @@ -889,7 +889,7 @@ static int vdec_open(struct file *file) ret = vdec_init_ctrls(sess); if (ret) - goto err_m2m_release; + goto err_m2m_ctx_release; sess->pixfmt_cap = formats[0].pixfmts_cap[0]; sess->fmt_out = &formats[0]; @@ -913,6 +913,8 @@ static int vdec_open(struct file *file) return 0; +err_m2m_ctx_release: + v4l2_m2m_ctx_release(sess->m2m_ctx); err_m2m_release: v4l2_m2m_release(sess->m2m_dev); err_free_sess: diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 1c7191bb0d8fc..c3f8a0952559a 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -285,8 +285,10 @@ static int cedrus_init_ctrls(struct cedrus_dev *dev, struct cedrus_ctx *ctx) ctrl_size = sizeof(ctrl) * CEDRUS_CONTROLS_COUNT + 1; ctx->ctrls = kzalloc(ctrl_size, GFP_KERNEL); - if (!ctx->ctrls) + if (!ctx->ctrls) { + v4l2_ctrl_handler_free(hdl); return -ENOMEM; + } j = 0; for (i = 0; i < CEDRUS_CONTROLS_COUNT; i++) { @@ -391,6 +393,7 @@ static int cedrus_open(struct file *file) err_m2m_release: v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); err_free: + v4l2_fh_exit(&ctx->fh); kfree(ctx); mutex_unlock(&dev->dev_mutex); @@ -507,7 +510,7 @@ static int cedrus_probe(struct platform_device *pdev) ret = video_register_device(vfd, VFL_TYPE_VIDEO, 0); if (ret) { v4l2_err(&dev->v4l2_dev, "Failed to register video device\n"); - goto err_m2m; + goto err_media; } v4l2_info(&dev->v4l2_dev, @@ -533,7 +536,8 @@ static int cedrus_probe(struct platform_device *pdev) v4l2_m2m_unregister_media_controller(dev->m2m_dev); err_video: video_unregister_device(&dev->vfd); -err_m2m: +err_media: + media_device_cleanup(&dev->mdev); v4l2_m2m_release(dev->m2m_dev); err_v4l2: v4l2_device_unregister(&dev->v4l2_dev); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c index 3e2843ef6ccec..fc54d993b11f2 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c @@ -210,6 +210,9 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx, u8 dpb_idx; dpb_idx = ref_list[i].index; + if (dpb_idx >= V4L2_H264_NUM_DPB_ENTRIES) + continue; + dpb = &decode->dpb[dpb_idx]; if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index 14b327afe045e..f5d7ba4cdd6e0 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -80,8 +80,8 @@ static int tegra_get_format_idx_by_code(struct tegra_vi *vi, static u32 tegra_get_format_fourcc_by_idx(struct tegra_vi *vi, unsigned int index) { - if (index >= vi->soc->nformats) - return -EINVAL; + if (WARN_ON_ONCE(index >= vi->soc->nformats)) + return vi->soc->video_formats[0].fourcc; return vi->soc->video_formats[index].fourcc; } @@ -1260,6 +1260,7 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) struct device_node *parent; struct v4l2_fwnode_endpoint v4l2_ep = { .bus_type = 0 }; unsigned int lanes; + int err; int ret = 0; ports = of_get_child_by_name(node, "ports"); @@ -1270,8 +1271,8 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) if (!of_node_name_eq(port, "port")) continue; - ret = of_property_read_u32(port, "reg", &port_num); - if (ret < 0) + err = of_property_read_u32(port, "reg", &port_num); + if (err < 0) continue; if (port_num > vi->soc->vi_max_channels) { @@ -1292,10 +1293,10 @@ static int tegra_vi_channels_alloc(struct tegra_vi *vi) ep = of_graph_get_endpoint_by_regs(parent, 0, 0); of_node_put(parent); - ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), + err = v4l2_fwnode_endpoint_parse(of_fwnode_handle(ep), &v4l2_ep); of_node_put(ep); - if (ret) + if (err) continue; lanes = v4l2_ep.bus.mipi_csi2.num_data_lanes; diff --git a/drivers/staging/media/tegra-video/vip.c b/drivers/staging/media/tegra-video/vip.c index 5ec717f3afd50..eb0381c10d2bf 100644 --- a/drivers/staging/media/tegra-video/vip.c +++ b/drivers/staging/media/tegra-video/vip.c @@ -126,7 +126,7 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) if (!ep) { err = -EINVAL; dev_err_probe(dev, err, "%pOF: error getting endpoint node\n", np); - goto err_node_put; + return err; } fwh = of_fwnode_handle(ep); @@ -134,14 +134,14 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) of_node_put(ep); if (err) { dev_err_probe(dev, err, "%pOF: failed to parse v4l2 endpoint\n", np); - goto err_node_put; + return err; } num_pads = of_graph_get_endpoint_count(np); if (num_pads != TEGRA_VIP_PADS_NUM) { err = -EINVAL; dev_err_probe(dev, err, "%pOF: need 2 pads, got %d\n", np, num_pads); - goto err_node_put; + return err; } vip->chan.of_node = of_node_get(np); @@ -149,10 +149,6 @@ static int tegra_vip_channel_of_parse(struct tegra_vip *vip) vip->chan.pads[TEGRA_VIP_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE; return 0; - -err_node_put: - of_node_put(np); - return err; } static int tegra_vip_channel_init(struct tegra_vip *vip) diff --git a/drivers/staging/octeon/ethernet-mem.c b/drivers/staging/octeon/ethernet-mem.c index 532594957ebcf..68c3ef984e565 100644 --- a/drivers/staging/octeon/ethernet-mem.c +++ b/drivers/staging/octeon/ethernet-mem.c @@ -5,6 +5,7 @@ * Copyright (c) 2003-2010 Cavium Networks */ +#include #include #include #include @@ -40,11 +41,13 @@ static int cvm_oct_fill_hw_skbuff(int pool, int size, int elements) /** * cvm_oct_free_hw_skbuff- free hardware pool skbuffs + * @pdev: Platform device for logging * @pool: Pool to allocate an skbuff for * @size: Size of the buffer needed for the pool * @elements: Number of buffers to allocate */ -static void cvm_oct_free_hw_skbuff(int pool, int size, int elements) +static void cvm_oct_free_hw_skbuff(struct platform_device *pdev, + int pool, int size, int elements) { char *memory; @@ -59,22 +62,24 @@ static void cvm_oct_free_hw_skbuff(int pool, int size, int elements) } while (memory); if (elements < 0) - pr_warn("Freeing of pool %u had too many skbuffs (%d)\n", - pool, elements); + dev_warn(&pdev->dev, "Freeing of pool %u had too many skbuffs (%d)\n", + pool, elements); else if (elements > 0) - pr_warn("Freeing of pool %u is missing %d skbuffs\n", - pool, elements); + dev_warn(&pdev->dev, "Freeing of pool %u is missing %d skbuffs\n", + pool, elements); } /** * cvm_oct_fill_hw_memory - fill a hardware pool with memory. + * @pdev: Platform device for logging * @pool: Pool to populate * @size: Size of each buffer in the pool * @elements: Number of buffers to allocate * * Returns the actual number of buffers allocated. */ -static int cvm_oct_fill_hw_memory(int pool, int size, int elements) +static int cvm_oct_fill_hw_memory(struct platform_device *pdev, int pool, int size, + int elements) { char *memory; char *fpa; @@ -93,8 +98,8 @@ static int cvm_oct_fill_hw_memory(int pool, int size, int elements) */ memory = kmalloc(size + 256, GFP_ATOMIC); if (unlikely(!memory)) { - pr_warn("Unable to allocate %u bytes for FPA pool %d\n", - elements * size, pool); + dev_warn(&pdev->dev, "Unable to allocate %u bytes for FPA pool %d\n", + elements * size, pool); break; } fpa = (char *)(((unsigned long)memory + 256) & ~0x7fUL); @@ -107,11 +112,13 @@ static int cvm_oct_fill_hw_memory(int pool, int size, int elements) /** * cvm_oct_free_hw_memory - Free memory allocated by cvm_oct_fill_hw_memory + * @pdev: Platform device for logging * @pool: FPA pool to free * @size: Size of each buffer in the pool * @elements: Number of buffers that should be in the pool */ -static void cvm_oct_free_hw_memory(int pool, int size, int elements) +static void cvm_oct_free_hw_memory(struct platform_device *pdev, + int pool, int size, int elements) { char *memory; char *fpa; @@ -127,28 +134,28 @@ static void cvm_oct_free_hw_memory(int pool, int size, int elements) } while (fpa); if (elements < 0) - pr_warn("Freeing of pool %u had too many buffers (%d)\n", - pool, elements); + dev_warn(&pdev->dev, "Freeing of pool %u had too many buffers (%d)\n", + pool, elements); else if (elements > 0) - pr_warn("Warning: Freeing of pool %u is missing %d buffers\n", - pool, elements); + dev_warn(&pdev->dev, "Freeing of pool %u is missing %d buffers\n", + pool, elements); } -int cvm_oct_mem_fill_fpa(int pool, int size, int elements) +int cvm_oct_mem_fill_fpa(struct platform_device *pdev, int pool, int size, int elements) { int freed; if (pool == CVMX_FPA_PACKET_POOL) freed = cvm_oct_fill_hw_skbuff(pool, size, elements); else - freed = cvm_oct_fill_hw_memory(pool, size, elements); + freed = cvm_oct_fill_hw_memory(pdev, pool, size, elements); return freed; } -void cvm_oct_mem_empty_fpa(int pool, int size, int elements) +void cvm_oct_mem_empty_fpa(struct platform_device *pdev, int pool, int size, int elements) { if (pool == CVMX_FPA_PACKET_POOL) - cvm_oct_free_hw_skbuff(pool, size, elements); + cvm_oct_free_hw_skbuff(pdev, pool, size, elements); else - cvm_oct_free_hw_memory(pool, size, elements); + cvm_oct_free_hw_memory(pdev, pool, size, elements); } diff --git a/drivers/staging/octeon/ethernet-mem.h b/drivers/staging/octeon/ethernet-mem.h index 692dcdb7154da..9279bb0de2db4 100644 --- a/drivers/staging/octeon/ethernet-mem.h +++ b/drivers/staging/octeon/ethernet-mem.h @@ -5,5 +5,9 @@ * Copyright (c) 2003-2007 Cavium Networks */ -int cvm_oct_mem_fill_fpa(int pool, int size, int elements); -void cvm_oct_mem_empty_fpa(int pool, int size, int elements); +struct platform_device; + +int cvm_oct_mem_fill_fpa(struct platform_device *pdev, int pool, int size, + int elements); +void cvm_oct_mem_empty_fpa(struct platform_device *pdev, int pool, int size, + int elements); diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c index 965330eec80a8..3e9d58d321560 100644 --- a/drivers/staging/octeon/ethernet-rx.c +++ b/drivers/staging/octeon/ethernet-rx.c @@ -5,6 +5,7 @@ * Copyright (c) 2003-2010 Cavium Networks */ +#include #include #include #include @@ -31,12 +32,6 @@ static atomic_t oct_rx_ready = ATOMIC_INIT(0); -static struct oct_rx_group { - int irq; - int group; - struct napi_struct napi; -} oct_rx_group[16]; - /** * cvm_oct_do_interrupt - interrupt handler. * @irq: Interrupt number. @@ -397,7 +392,7 @@ static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget) /* Restore the scratch area */ cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch); } - cvm_oct_rx_refill_pool(0); + cvm_oct_rx_refill_pool(rx_group->pdev, 0); return rx_count; } @@ -434,24 +429,28 @@ static int cvm_oct_napi_poll(struct napi_struct *napi, int budget) */ void cvm_oct_poll_controller(struct net_device *dev) { + struct platform_device *pdev = to_platform_device(dev->dev.parent); + struct octeon_ethernet_platform *plat = platform_get_drvdata(pdev); int i; if (!atomic_read(&oct_rx_ready)) return; - for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { + for (i = 0; i < ARRAY_SIZE(plat->rx_group); i++) { if (!(pow_receive_groups & BIT(i))) continue; - cvm_oct_poll(&oct_rx_group[i], 16); + cvm_oct_poll(&plat->rx_group[i], 16); } } #endif -void cvm_oct_rx_initialize(void) +void cvm_oct_rx_initialize(struct platform_device *pdev) { int i; struct net_device *dev_for_napi = NULL; + struct octeon_ethernet_platform *plat = platform_get_drvdata(pdev); + struct oct_rx_group *rx_group = plat->rx_group; for (i = 0; i < TOTAL_NUMBER_OF_PORTS; i++) { if (cvm_oct_device[i]) { @@ -463,27 +462,28 @@ void cvm_oct_rx_initialize(void) if (!dev_for_napi) panic("No net_devices were allocated."); - for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { + for (i = 0; i < ARRAY_SIZE(plat->rx_group); i++) { int ret; if (!(pow_receive_groups & BIT(i))) continue; - netif_napi_add_weight(dev_for_napi, &oct_rx_group[i].napi, + netif_napi_add_weight(dev_for_napi, &rx_group[i].napi, cvm_oct_napi_poll, rx_napi_weight); - napi_enable(&oct_rx_group[i].napi); + napi_enable(&rx_group[i].napi); - oct_rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i; - oct_rx_group[i].group = i; + rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i; + rx_group[i].group = i; + rx_group[i].pdev = pdev; /* Register an IRQ handler to receive POW interrupts */ - ret = request_irq(oct_rx_group[i].irq, cvm_oct_do_interrupt, 0, - "Ethernet", &oct_rx_group[i].napi); + ret = request_irq(rx_group[i].irq, cvm_oct_do_interrupt, 0, + "Ethernet", &rx_group[i].napi); if (ret) panic("Could not acquire Ethernet IRQ %d\n", - oct_rx_group[i].irq); + rx_group[i].irq); - disable_irq_nosync(oct_rx_group[i].irq); + disable_irq_nosync(rx_group[i].irq); /* Enable POW interrupt when our port has at least one packet */ if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { @@ -515,16 +515,17 @@ void cvm_oct_rx_initialize(void) /* Schedule NAPI now. This will indirectly enable the * interrupt. */ - napi_schedule(&oct_rx_group[i].napi); + napi_schedule(&rx_group[i].napi); } atomic_inc(&oct_rx_ready); } -void cvm_oct_rx_shutdown(void) +void cvm_oct_rx_shutdown(struct platform_device *pdev) { + struct octeon_ethernet_platform *plat = platform_get_drvdata(pdev); int i; - for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { + for (i = 0; i < ARRAY_SIZE(plat->rx_group); i++) { if (!(pow_receive_groups & BIT(i))) continue; @@ -534,9 +535,11 @@ void cvm_oct_rx_shutdown(void) else cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0); + napi_disable(&plat->rx_group[i].napi); + /* Free the interrupt handler */ - free_irq(oct_rx_group[i].irq, cvm_oct_device); + free_irq(plat->rx_group[i].irq, &plat->rx_group[i].napi); - netif_napi_del(&oct_rx_group[i].napi); + netif_napi_del(&plat->rx_group[i].napi); } } diff --git a/drivers/staging/octeon/ethernet-rx.h b/drivers/staging/octeon/ethernet-rx.h index ff6482fa20d69..6093694326cb6 100644 --- a/drivers/staging/octeon/ethernet-rx.h +++ b/drivers/staging/octeon/ethernet-rx.h @@ -5,11 +5,14 @@ * Copyright (c) 2003-2007 Cavium Networks */ +struct platform_device; + void cvm_oct_poll_controller(struct net_device *dev); -void cvm_oct_rx_initialize(void); -void cvm_oct_rx_shutdown(void); +void cvm_oct_rx_initialize(struct platform_device *pdev); +void cvm_oct_rx_shutdown(struct platform_device *pdev); -static inline void cvm_oct_rx_refill_pool(int fill_threshold) +static inline void cvm_oct_rx_refill_pool(struct platform_device *pdev, + int fill_threshold) { int number_to_free; int num_freed; @@ -20,7 +23,7 @@ static inline void cvm_oct_rx_refill_pool(int fill_threshold) if (number_to_free > fill_threshold) { cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, -number_to_free); - num_freed = cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, + num_freed = cvm_oct_mem_fill_fpa(pdev, CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, number_to_free); if (num_freed != number_to_free) { diff --git a/drivers/staging/octeon/ethernet-tx.c b/drivers/staging/octeon/ethernet-tx.c index f5bbedac6a653..1b30fe1bcf43b 100644 --- a/drivers/staging/octeon/ethernet-tx.c +++ b/drivers/staging/octeon/ethernet-tx.c @@ -670,4 +670,6 @@ void cvm_oct_tx_shutdown(void) { /* Free the interrupt handler */ free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device); + + tasklet_kill(&cvm_oct_tx_cleanup_tasklet); } diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c index eadb74fc14c8d..0a24c62ae32bf 100644 --- a/drivers/staging/octeon/ethernet.c +++ b/drivers/staging/octeon/ethernet.c @@ -104,11 +104,10 @@ struct net_device *cvm_oct_device[TOTAL_NUMBER_OF_PORTS]; u64 cvm_oct_tx_poll_interval; -static void cvm_oct_rx_refill_worker(struct work_struct *work); -static DECLARE_DELAYED_WORK(cvm_oct_rx_refill_work, cvm_oct_rx_refill_worker); - static void cvm_oct_rx_refill_worker(struct work_struct *work) { + struct octeon_ethernet_platform *plat = container_of(work, + struct octeon_ethernet_platform, rx_refill_work.work); /* * FPA 0 may have been drained, try to refill it if we need * more than num_packet_buffers / 2, otherwise normal receive @@ -116,10 +115,10 @@ static void cvm_oct_rx_refill_worker(struct work_struct *work) * could be received so cvm_oct_napi_poll would never be * invoked to do the refill. */ - cvm_oct_rx_refill_pool(num_packet_buffers / 2); + cvm_oct_rx_refill_pool(plat->pdev, num_packet_buffers / 2); if (!atomic_read(&cvm_oct_poll_queue_stopping)) - schedule_delayed_work(&cvm_oct_rx_refill_work, HZ); + schedule_delayed_work(&plat->rx_refill_work, HZ); } static void cvm_oct_periodic_worker(struct work_struct *work) @@ -138,16 +137,16 @@ static void cvm_oct_periodic_worker(struct work_struct *work) schedule_delayed_work(&priv->port_periodic_work, HZ); } -static void cvm_oct_configure_common_hw(void) +static void cvm_oct_configure_common_hw(struct platform_device *pdev) { /* Setup the FPA */ cvmx_fpa_enable(); - cvm_oct_mem_fill_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, + cvm_oct_mem_fill_fpa(pdev, CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers); - cvm_oct_mem_fill_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, + cvm_oct_mem_fill_fpa(pdev, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers); if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL) - cvm_oct_mem_fill_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, + cvm_oct_mem_fill_fpa(pdev, CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 1024); #ifdef __LITTLE_ENDIAN @@ -678,6 +677,15 @@ static int cvm_oct_probe(struct platform_device *pdev) int qos; struct device_node *pip; int mtu_overhead = ETH_HLEN + ETH_FCS_LEN; + struct octeon_ethernet_platform *plat; + + plat = devm_kzalloc(&pdev->dev, sizeof(*plat), GFP_KERNEL); + if (!plat) + return -ENOMEM; + + plat->pdev = pdev; + INIT_DELAYED_WORK(&plat->rx_refill_work, cvm_oct_rx_refill_worker); + platform_set_drvdata(pdev, plat); #if IS_ENABLED(CONFIG_VLAN_8021Q) mtu_overhead += VLAN_HLEN; @@ -689,7 +697,7 @@ static int cvm_oct_probe(struct platform_device *pdev) return -EINVAL; } - cvm_oct_configure_common_hw(); + cvm_oct_configure_common_hw(pdev); cvmx_helper_initialize_packet_io_global(); @@ -912,28 +920,29 @@ static int cvm_oct_probe(struct platform_device *pdev) } cvm_oct_tx_initialize(); - cvm_oct_rx_initialize(); + cvm_oct_rx_initialize(pdev); /* * 150 uS: about 10 1500-byte packets at 1GE. */ cvm_oct_tx_poll_interval = 150 * (octeon_get_clock_rate() / 1000000); - schedule_delayed_work(&cvm_oct_rx_refill_work, HZ); + schedule_delayed_work(&plat->rx_refill_work, HZ); return 0; } static void cvm_oct_remove(struct platform_device *pdev) { + struct octeon_ethernet_platform *plat = platform_get_drvdata(pdev); int port; cvmx_ipd_disable(); atomic_inc_return(&cvm_oct_poll_queue_stopping); - cancel_delayed_work_sync(&cvm_oct_rx_refill_work); + cancel_delayed_work_sync(&plat->rx_refill_work); - cvm_oct_rx_shutdown(); + cvm_oct_rx_shutdown(pdev); cvm_oct_tx_shutdown(); cvmx_pko_disable(); @@ -958,12 +967,12 @@ static void cvm_oct_remove(struct platform_device *pdev) cvmx_ipd_free_ptr(); /* Free the HW pools */ - cvm_oct_mem_empty_fpa(CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, + cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_PACKET_POOL, CVMX_FPA_PACKET_POOL_SIZE, num_packet_buffers); - cvm_oct_mem_empty_fpa(CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, + cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_WQE_POOL, CVMX_FPA_WQE_POOL_SIZE, num_packet_buffers); if (CVMX_FPA_OUTPUT_BUFFER_POOL != CVMX_FPA_PACKET_POOL) - cvm_oct_mem_empty_fpa(CVMX_FPA_OUTPUT_BUFFER_POOL, + cvm_oct_mem_empty_fpa(pdev, CVMX_FPA_OUTPUT_BUFFER_POOL, CVMX_FPA_OUTPUT_BUFFER_POOL_SIZE, 128); } diff --git a/drivers/staging/octeon/octeon-ethernet.h b/drivers/staging/octeon/octeon-ethernet.h index a6140705706f4..0ac430db1e6eb 100644 --- a/drivers/staging/octeon/octeon-ethernet.h +++ b/drivers/staging/octeon/octeon-ethernet.h @@ -11,6 +11,7 @@ #ifndef OCTEON_ETHERNET_H #define OCTEON_ETHERNET_H +#include #include #include @@ -74,6 +75,19 @@ struct octeon_ethernet { struct device_node *of_node; }; +struct oct_rx_group { + int irq; + int group; + struct napi_struct napi; + struct platform_device *pdev; +}; + +struct octeon_ethernet_platform { + struct platform_device *pdev; + struct delayed_work rx_refill_work; + struct oct_rx_group rx_group[16]; +}; + int cvm_oct_free_work(void *work_queue_entry); int cvm_oct_rgmii_open(struct net_device *dev); diff --git a/drivers/staging/rtl8723bs/core/rtw_ap.c b/drivers/staging/rtl8723bs/core/rtw_ap.c index 0908f2234f671..98405bedf5da7 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ap.c +++ b/drivers/staging/rtl8723bs/core/rtw_ap.c @@ -1262,13 +1262,13 @@ u8 rtw_ap_set_pairwise_key(struct adapter *padapter, struct sta_info *psta) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_KERNEL); if (!ph2c) { res = _FAIL; goto exit; } - psetstakey_para = rtw_zmalloc(sizeof(struct set_stakey_parm)); + psetstakey_para = kzalloc(sizeof(*psetstakey_para), GFP_KERNEL); if (!psetstakey_para) { kfree(ph2c); res = _FAIL; @@ -1304,12 +1304,12 @@ static int rtw_ap_set_key( struct cmd_priv *pcmdpriv = &padapter->cmdpriv; int res = _SUCCESS; - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_KERNEL); if (!pcmd) { res = _FAIL; goto exit; } - psetkeyparm = rtw_zmalloc(sizeof(struct setkey_parm)); + psetkeyparm = kzalloc(sizeof(*psetkeyparm), GFP_KERNEL); if (!psetkeyparm) { kfree(pcmd); res = _FAIL; diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c index ef2d92b5588ad..2701a80ad07de 100644 --- a/drivers/staging/rtl8723bs/core/rtw_cmd.c +++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c @@ -170,15 +170,13 @@ int rtw_init_cmd_priv(struct cmd_priv *pcmdpriv) pcmdpriv->cmd_seq = 1; - pcmdpriv->cmd_allocated_buf = rtw_zmalloc(MAX_CMDSZ + CMDBUFF_ALIGN_SZ); - + pcmdpriv->cmd_allocated_buf = kzalloc(MAX_CMDSZ + CMDBUFF_ALIGN_SZ, GFP_ATOMIC); if (!pcmdpriv->cmd_allocated_buf) return -ENOMEM; pcmdpriv->cmd_buf = pcmdpriv->cmd_allocated_buf + CMDBUFF_ALIGN_SZ - ((SIZE_PTR)(pcmdpriv->cmd_allocated_buf) & (CMDBUFF_ALIGN_SZ-1)); - pcmdpriv->rsp_allocated_buf = rtw_zmalloc(MAX_RSPSZ + 4); - + pcmdpriv->rsp_allocated_buf = kzalloc(MAX_RSPSZ + 4, GFP_ATOMIC); if (!pcmdpriv->rsp_allocated_buf) { kfree(pcmdpriv->cmd_allocated_buf); return -ENOMEM; @@ -534,11 +532,11 @@ u8 rtw_sitesurvey_cmd(struct adapter *padapter, struct ndis_802_11_ssid *ssid, if (check_fwstate(pmlmepriv, _FW_LINKED)) rtw_lps_ctrl_wk_cmd(padapter, LPS_CTRL_SCAN, 1); - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) return _FAIL; - psurveyPara = rtw_zmalloc(sizeof(struct sitesurvey_parm)); + psurveyPara = kzalloc(sizeof(*psurveyPara), GFP_ATOMIC); if (!psurveyPara) { kfree(ph2c); return _FAIL; @@ -602,7 +600,7 @@ u8 rtw_createbss_cmd(struct adapter *padapter) struct wlan_bssid_ex *pdev_network = &padapter->registrypriv.dev_network; u8 res = _SUCCESS; - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_ATOMIC); if (!pcmd) { res = _FAIL; goto exit; @@ -635,7 +633,7 @@ int rtw_startbss_cmd(struct adapter *padapter, int flags) start_bss_network(padapter); } else { /* need enqueue, prepare cmd_obj and enqueue */ - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_KERNEL); if (!pcmd) { res = _FAIL; goto exit; @@ -687,7 +685,7 @@ u8 rtw_joinbss_cmd(struct adapter *padapter, struct wlan_network *pnetwork) u32 tmp_len; u8 *ptmp = NULL; - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_KERNEL); if (!pcmd) { res = _FAIL; goto exit; @@ -796,7 +794,7 @@ u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueu u8 res = _SUCCESS; /* prepare cmd parameter */ - param = rtw_zmalloc(sizeof(*param)); + param = kzalloc(sizeof(*param), GFP_KERNEL); if (!param) { res = _FAIL; goto exit; @@ -805,7 +803,7 @@ u8 rtw_disassoc_cmd(struct adapter *padapter, u32 deauth_timeout_ms, bool enqueu if (enqueue) { /* need enqueue, prepare cmd_obj and enqueue */ - cmdobj = rtw_zmalloc(sizeof(*cmdobj)); + cmdobj = kzalloc(sizeof(*cmdobj), GFP_KERNEL); if (!cmdobj) { res = _FAIL; kfree(param); @@ -832,8 +830,7 @@ u8 rtw_setopmode_cmd(struct adapter *padapter, enum ndis_802_11_network_infrast struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - psetop = rtw_zmalloc(sizeof(struct setopmode_parm)); - + psetop = kzalloc(sizeof(*psetop), GFP_KERNEL); if (!psetop) { res = _FAIL; goto exit; @@ -841,7 +838,7 @@ u8 rtw_setopmode_cmd(struct adapter *padapter, enum ndis_802_11_network_infrast psetop->mode = (u8)networktype; if (enqueue) { - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_KERNEL); if (!ph2c) { kfree(psetop); res = _FAIL; @@ -868,7 +865,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ struct security_priv *psecuritypriv = &padapter->securitypriv; u8 res = _SUCCESS; - psetstakey_para = rtw_zmalloc(sizeof(struct set_stakey_parm)); + psetstakey_para = kzalloc(sizeof(*psetstakey_para), GFP_KERNEL); if (!psetstakey_para) { res = _FAIL; goto exit; @@ -890,17 +887,17 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ padapter->securitypriv.busetkipkey = true; if (enqueue) { - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_KERNEL); if (!ph2c) { - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } - psetstakey_rsp = rtw_zmalloc(sizeof(struct set_stakey_rsp)); + psetstakey_rsp = kzalloc(sizeof(*psetstakey_rsp), GFP_KERNEL); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -911,7 +908,7 @@ u8 rtw_setstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 unicast_ res = rtw_enqueue_cmd(pcmdpriv, ph2c); } else { set_stakey_hdl(padapter, (u8 *)psetstakey_para); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); } exit: return res; @@ -935,23 +932,23 @@ u8 rtw_clearstakey_cmd(struct adapter *padapter, struct sta_info *sta, u8 enqueu rtw_camid_free(padapter, cam_id); } } else { - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_KERNEL); if (!ph2c) { res = _FAIL; goto exit; } - psetstakey_para = rtw_zmalloc(sizeof(struct set_stakey_parm)); + psetstakey_para = kzalloc(sizeof(*psetstakey_para), GFP_KERNEL); if (!psetstakey_para) { kfree(ph2c); res = _FAIL; goto exit; } - psetstakey_rsp = rtw_zmalloc(sizeof(struct set_stakey_rsp)); + psetstakey_rsp = kzalloc(sizeof(*psetstakey_rsp), GFP_KERNEL); if (!psetstakey_rsp) { kfree(ph2c); - kfree(psetstakey_para); + kfree_sensitive(psetstakey_para); res = _FAIL; goto exit; } @@ -978,13 +975,13 @@ u8 rtw_addbareq_cmd(struct adapter *padapter, u8 tid, u8 *addr) u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - paddbareq_parm = rtw_zmalloc(sizeof(struct addBaReq_parm)); + paddbareq_parm = kzalloc(sizeof(*paddbareq_parm), GFP_ATOMIC); if (!paddbareq_parm) { kfree(ph2c); res = _FAIL; @@ -1010,13 +1007,13 @@ u8 rtw_reset_securitypriv_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1043,13 +1040,13 @@ u8 rtw_free_assoc_resources_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1077,13 +1074,13 @@ u8 rtw_dynamic_chk_wk_cmd(struct adapter *padapter) u8 res = _SUCCESS; /* only primary padapter does this cmd */ - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1318,13 +1315,13 @@ u8 rtw_lps_ctrl_wk_cmd(struct adapter *padapter, u8 lps_ctrl_type, u8 enqueue) u8 res = _SUCCESS; if (enqueue) { - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1359,13 +1356,13 @@ u8 rtw_dm_in_lps_wk_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1421,13 +1418,13 @@ u8 rtw_dm_ra_mask_wk_cmd(struct adapter *padapter, u8 *psta) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1456,13 +1453,13 @@ u8 rtw_ps_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ppscmd = rtw_zmalloc(sizeof(struct cmd_obj)); + ppscmd = kzalloc(sizeof(*ppscmd), GFP_ATOMIC); if (!ppscmd) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ppscmd); res = _FAIL; @@ -1528,13 +1525,13 @@ u8 rtw_chk_hi_queue_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1620,13 +1617,13 @@ u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_ATOMIC); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; @@ -1655,13 +1652,13 @@ u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt) struct cmd_priv *pcmdpriv = &padapter->cmdpriv; u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_KERNEL); if (!ph2c) { res = _FAIL; goto exit; } - pdrvextra_cmd_parm = rtw_zmalloc(sizeof(struct drvextra_cmd_parm)); + pdrvextra_cmd_parm = kzalloc(sizeof(*pdrvextra_cmd_parm), GFP_KERNEL); if (!pdrvextra_cmd_parm) { kfree(ph2c); res = _FAIL; diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c index d5c53b614f616..62a40d4672d3f 100644 --- a/drivers/staging/rtl8723bs/core/rtw_efuse.c +++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c @@ -253,7 +253,6 @@ void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType) /* void *)&pHalData->EfuseMap[EFUSE_INIT_MAP][0], mapLen); */ } /* EFUSE_ShadowMapUpdate */ - /*----------------------------------------------------------------------------- * Function: EFUSE_ShadowRead * diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c index 01ab7fb5e5bc6..f99089214b1a2 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c +++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c @@ -372,6 +372,9 @@ unsigned char *rtw_get_wpa_ie(unsigned char *pie, int *wpa_ie_len, int limit) pbuf = rtw_get_ie(pbuf, WLAN_EID_VENDOR_SPECIFIC, &len, limit_new); if (pbuf) { + if (len < 6) + goto check_next_ie; + /* check if oui matches... */ if (memcmp((pbuf + 2), wpa_oui_type, sizeof(wpa_oui_type))) goto check_next_ie; @@ -679,7 +682,14 @@ u8 *rtw_get_wps_ie(u8 *in_ie, uint in_len, u8 *wps_ie, uint *wps_ielen) while (cnt < in_len) { eid = in_ie[cnt]; - if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (!memcmp(&in_ie[cnt + 2], wps_oui, 4))) { + if (cnt + 2 > in_len) + break; + + if (in_ie[cnt + 1] + 2 > in_len - cnt) + break; + + if ((eid == WLAN_EID_VENDOR_SPECIFIC) && (in_ie[cnt + 1] >= 4) && + (!memcmp(&in_ie[cnt + 2], wps_oui, 4))) { wpsie_ptr = &in_ie[cnt]; if (wps_ie) @@ -736,6 +746,10 @@ u8 *rtw_get_wps_attr(u8 *wps_ie, uint wps_ielen, u16 target_attr_id, u8 *buf_att u16 attr_data_len = get_unaligned_be16(attr_ptr + 2); u16 attr_len = attr_data_len + 4; + /* Reject attributes whose claimed length runs past the IE */ + if (attr_ptr + attr_len > wps_ie + wps_ielen) + break; + if (attr_id == target_attr_id) { target_attr_ptr = attr_ptr; @@ -1138,6 +1152,9 @@ int rtw_action_frame_parse(const u8 *frame, u32 frame_len, u8 *category, u8 *act u8 c; u8 a = ACT_PUBLIC_MAX; + if (frame_len < sizeof(struct ieee80211_hdr_3addr) + 2) + return false; + fc = le16_to_cpu(((struct ieee80211_hdr_3addr *)frame)->frame_control); if ((fc & (IEEE80211_FCTL_FTYPE|IEEE80211_FCTL_STYPE)) diff --git a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c index 587a87fbffeb4..df664efb21269 100644 --- a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c +++ b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c @@ -345,7 +345,6 @@ u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter, return true; } - u8 rtw_set_802_11_disassociate(struct adapter *padapter) { struct mlme_priv *pmlmepriv = &padapter->mlmepriv; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c index f2b5455591fd4..039d47f221cfc 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c @@ -612,6 +612,8 @@ int rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwor privacy = pnetwork->network.privacy; if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS)) { + if (pnetwork->network.ie_length < _FIXED_IE_LENGTH_) + return false; if (rtw_get_wps_ie(pnetwork->network.ies+_FIXED_IE_LENGTH_, pnetwork->network.ie_length-_FIXED_IE_LENGTH_, NULL, &wps_ielen)) return true; else @@ -625,11 +627,15 @@ int rtw_is_desired_network(struct adapter *adapter, struct wlan_network *pnetwor bselected = false; if (psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPA2PSK) { - p = rtw_get_ie(pnetwork->network.ies + _BEACON_IE_OFFSET_, WLAN_EID_RSN, &ie_len, (pnetwork->network.ie_length - _BEACON_IE_OFFSET_)); - if (p && ie_len > 0) - bselected = true; - else + if (pnetwork->network.ie_length < _BEACON_IE_OFFSET_) { bselected = false; + } else { + p = rtw_get_ie(pnetwork->network.ies + _BEACON_IE_OFFSET_, WLAN_EID_RSN, &ie_len, (pnetwork->network.ie_length - _BEACON_IE_OFFSET_)); + if (p && ie_len > 0) + bselected = true; + else + bselected = false; + } } } @@ -1901,13 +1907,13 @@ signed int rtw_set_auth(struct adapter *adapter, struct security_priv *psecurity struct cmd_priv *pcmdpriv = &adapter->cmdpriv; signed int res = _SUCCESS; - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_KERNEL); if (!pcmd) { res = _FAIL; /* try again */ goto exit; } - psetauthparm = rtw_zmalloc(sizeof(struct setauth_parm)); + psetauthparm = kzalloc(sizeof(*psetauthparm), GFP_KERNEL); if (!psetauthparm) { kfree(pcmd); res = _FAIL; @@ -1938,7 +1944,7 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp struct cmd_priv *pcmdpriv = &adapter->cmdpriv; signed int res = _SUCCESS; - psetkeyparm = rtw_zmalloc(sizeof(struct setkey_parm)); + psetkeyparm = kzalloc(sizeof(*psetkeyparm), GFP_KERNEL); if (!psetkeyparm) { res = _FAIL; goto exit; @@ -1980,7 +1986,7 @@ signed int rtw_set_key(struct adapter *adapter, struct security_priv *psecurityp } if (enqueue) { - pcmd = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd = kzalloc(sizeof(*pcmd), GFP_KERNEL); if (!pcmd) { kfree(psetkeyparm); res = _FAIL; /* try again */ @@ -2029,7 +2035,10 @@ int rtw_restruct_wmm_ie(struct adapter *adapter, u8 *in_ie, u8 *out_ie, uint in_ break; } - i += (in_ie[i+1]+2); /* to the next IE element */ + if (i + 1 >= in_len) + break; + + i += (in_ie[i + 1] + 2); /* to the next IE element */ } return ielength; @@ -2090,7 +2099,7 @@ static void rtw_report_sec_ie(struct adapter *adapter, u8 authmode, u8 *sec_ie) buff = NULL; if (authmode == WLAN_EID_VENDOR_SPECIFIC) { - buff = rtw_zmalloc(IW_CUSTOM_MAX); + buff = kzalloc(IW_CUSTOM_MAX, GFP_ATOMIC); if (!buff) return; diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c index 9060b63eb424d..8d33631ab8f0f 100644 --- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c +++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c @@ -536,7 +536,6 @@ unsigned int OnProbeReq(struct adapter *padapter, union recv_frame *precv_frame) p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, WLAN_EID_SSID, (int *)&ielen, len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_); - /* check (wildcard) SSID */ if (p) { if (is_valid_p2p_probereq) @@ -795,7 +794,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) if (pstat->auth_seq == 0) pstat->expire_to = pstapriv->auth_to; - if ((pstat->auth_seq + 1) != seq) { status = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION; goto auth_fail; @@ -844,7 +842,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) } } - /* Now, we are going to issue_auth... */ pstat->auth_seq = seq + 1; @@ -853,7 +850,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame) if (pstat->state & WIFI_FW_AUTH_SUCCESS) pstat->auth_seq = 0; - return _SUCCESS; auth_fail: @@ -913,7 +909,7 @@ unsigned int OnAuthClient(struct adapter *padapter, union recv_frame *precv_fram p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _AUTH_IE_OFFSET_, WLAN_EID_CHALLENGE, (int *)&len, pkt_len - WLAN_HDR_A3_LEN - _AUTH_IE_OFFSET_); - if (!p) + if (!p || len != WLAN_AUTH_CHALLENGE_LEN) goto authclnt_fail; memcpy((void *)(pmlmeinfo->chg_txt), (void *)(p + 2), len); @@ -979,7 +975,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) else /* WIFI_REASSOCREQ */ ie_offset = _REASOCREQ_IE_OFFSET_; - if (pkt_len < sizeof(struct ieee80211_hdr_3addr) + ie_offset) return _FAIL; @@ -1009,7 +1004,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) pstat->state |= WIFI_FW_ASSOC_STATE; } - pstat->capability = capab_info; /* now parse all ieee802_11 ie to point to elems */ @@ -1146,7 +1140,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) pstat->flags |= WLAN_STA_MAYBE_WPS; } - /* AP support WPA/RSN, and sta is going to do WPS, but AP is not ready */ /* that the selected registrar of AP is _FLASE */ if ((psecuritypriv->wpa_psk > 0) @@ -1181,13 +1174,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) copy_len = ((wpa_ie_len+2) > sizeof(pstat->wpa_ie)) ? (sizeof(pstat->wpa_ie)):(wpa_ie_len+2); } - if (copy_len > 0) memcpy(pstat->wpa_ie, wpa_ie-2, copy_len); } - /* check if there is WMM IE & support WWM-PS */ pstat->flags &= ~WLAN_STA_WME; pstat->qos_option = 0; @@ -1260,13 +1251,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) } else pstat->flags &= ~WLAN_STA_HT; - if ((pmlmepriv->htpriv.ht_option == false) && (pstat->flags&WLAN_STA_HT)) { status = WLAN_STATUS_CHALLENGE_FAIL; goto OnAssocReqFail; } - if ((pstat->flags & WLAN_STA_HT) && ((pstat->wpa2_pairwise_cipher&WPA_CIPHER_TKIP) || (pstat->wpa_pairwise_cipher&WPA_CIPHER_TKIP))) { @@ -1314,13 +1303,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame) goto OnAssocReqFail; - } else { pstapriv->sta_aid[pstat->aid - 1] = pstat; } } - pstat->state &= (~WIFI_FW_ASSOC_STATE); pstat->state |= WIFI_FW_ASSOC_SUCCESS; @@ -1519,7 +1506,6 @@ unsigned int OnDeAuth(struct adapter *padapter, union recv_frame *precv_frame) associated_clients_update(padapter, updated); } - return _SUCCESS; } @@ -2162,7 +2148,6 @@ void issue_beacon(struct adapter *padapter, int timeout_ms) pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET; pwlanhdr = (struct ieee80211_hdr *)pframe; - fctrl = &(pwlanhdr->frame_control); *(fctrl) = 0; @@ -2251,12 +2236,10 @@ void issue_beacon(struct adapter *padapter, int timeout_ms) pframe = rtw_set_ie(pframe, WLAN_EID_ERP_INFO, 1, &erpinfo, &pattrib->pktlen); } - /* EXTERNDED SUPPORTED RATE */ if (rate_len > 8) pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (cur_network->supported_rates + 8), &pattrib->pktlen); - /* todo:HT for adhoc */ _issue_bcn: @@ -2265,8 +2248,11 @@ void issue_beacon(struct adapter *padapter, int timeout_ms) spin_unlock_bh(&pmlmepriv->bcn_update_lock); - if ((pattrib->pktlen + TXDESC_SIZE) > 512) + if ((pattrib->pktlen + TXDESC_SIZE) > 512) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->last_txcmdsz = pattrib->pktlen; @@ -2328,9 +2314,11 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p pattrib->pktlen = pattrib->hdrlen; pframe += pattrib->hdrlen; - - if (cur_network->ie_length > MAX_IE_SZ) + if (cur_network->ie_length > MAX_IE_SZ) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } if ((pmlmeinfo->state&0x03) == WIFI_FW_AP_STATE) { pwps_ie = rtw_get_wps_ie(cur_network->ies+_FIXED_IE_LENGTH_, cur_network->ie_length-_FIXED_IE_LENGTH_, NULL, &wps_ielen); @@ -2376,9 +2364,12 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p u8 *buf; u8 *ies = pmgntframe->buf_addr+TXDESC_OFFSET+sizeof(struct ieee80211_hdr_3addr); - buf = rtw_zmalloc(MAX_IE_SZ); - if (!buf) + buf = kzalloc(MAX_IE_SZ, GFP_ATOMIC); + if (!buf) { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } ssid_ie = rtw_get_ie(ies+_FIXED_IE_LENGTH_, WLAN_EID_SSID, &ssid_ielen, (pframe-ies)-_FIXED_IE_LENGTH_); @@ -2452,19 +2443,16 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p pframe = rtw_set_ie(pframe, WLAN_EID_ERP_INFO, 1, &erpinfo, &pattrib->pktlen); } - /* EXTERNDED SUPPORTED RATE */ if (rate_len > 8) pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (cur_network->supported_rates + 8), &pattrib->pktlen); - /* todo:HT for adhoc */ } pattrib->last_txcmdsz = pattrib->pktlen; - dump_mgntframe(padapter, pmgntframe); return; @@ -2496,7 +2484,6 @@ static int _issue_probereq(struct adapter *padapter, pattrib = &pmgntframe->attrib; update_mgntframe_attrib(padapter, pattrib); - memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET); pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET; @@ -2640,7 +2627,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short pframe += sizeof(struct ieee80211_hdr_3addr); pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr); - if (psta) { /* for AP mode */ memcpy(pwlanhdr->addr1, psta->hwaddr, ETH_ALEN); memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN); @@ -2701,7 +2687,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short le_tmp = cpu_to_le16(pmlmeinfo->auth_seq); pframe = rtw_set_fixed_ie(pframe, _AUTH_SEQ_NUM_, (unsigned char *)&le_tmp, &(pattrib->pktlen)); - /* setting status code... */ le_tmp = cpu_to_le16(status); pframe = rtw_set_fixed_ie(pframe, _STATUS_CODE_, (unsigned char *)&le_tmp, &(pattrib->pktlen)); @@ -2730,7 +2715,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short dump_mgntframe(padapter, pmgntframe); } - void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_info *pstat, int pkt_type) { struct xmit_frame *pmgntframe; @@ -2755,7 +2739,6 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i pattrib = &pmgntframe->attrib; update_mgntframe_attrib(padapter, pattrib); - memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET); pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET; @@ -2768,13 +2751,15 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i memcpy((void *)GetAddr2Ptr(pwlanhdr), myid(&(padapter->eeprompriv)), ETH_ALEN); memcpy((void *)GetAddr3Ptr(pwlanhdr), get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN); - SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq); pmlmeext->mgnt_seq++; - if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) + if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP)) { SetFrameSubType(pwlanhdr, pkt_type); - else + } else { + rtw_free_xmitbuf(pxmitpriv, pmgntframe->pxmitbuf); + rtw_free_xmitframe(pxmitpriv, pmgntframe); return; + } pattrib->hdrlen = sizeof(struct ieee80211_hdr_3addr); pattrib->pktlen += pattrib->hdrlen; @@ -2927,7 +2912,6 @@ void issue_assocreq(struct adapter *padapter) if (pmlmeext->cur_channel == 14) /* for JAPAN, channel 14 can only uses B Mode(CCK) */ sta_bssrate_len = 4; - /* for (i = 0; i < sta_bssrate_len; i++) { */ /* */ @@ -2936,12 +2920,10 @@ void issue_assocreq(struct adapter *padapter) break; } - for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) { if (pmlmeinfo->network.supported_rates[i] == 0) break; - /* Check if the AP's supported rates are also supported by STA. */ for (j = 0; j < sta_bssrate_len; j++) { /* Avoid the proprietary data rate (22Mbps) of Handlink WSG-4000 AP */ @@ -2963,7 +2945,6 @@ void issue_assocreq(struct adapter *padapter) goto exit; /* don't connect to AP if no joint supported rate */ } - if (bssrate_len > 8) { pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, 8, bssrate, &(pattrib->pktlen)); pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen)); @@ -3024,7 +3005,6 @@ void issue_assocreq(struct adapter *padapter) if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK) pframe = rtw_set_ie(pframe, WLAN_EID_VENDOR_SPECIFIC, 6, REALTEK_96B_IE, &(pattrib->pktlen)); - pattrib->last_txcmdsz = pattrib->pktlen; dump_mgntframe(padapter, pmgntframe); @@ -3121,7 +3101,6 @@ int issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned int pow struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info); struct sta_info *psta; - /* da == NULL, assume it's null data for sta to ap*/ if (!da) da = get_my_bssid(&(pmlmeinfo->network)); @@ -3171,7 +3150,6 @@ s32 issue_nulldata_in_interrupt(struct adapter *padapter, u8 *da) struct mlme_ext_priv *pmlmeext; struct mlme_ext_info *pmlmeinfo; - pmlmeext = &padapter->mlmeextpriv; pmlmeinfo = &pmlmeext->mlmext_info; @@ -3340,7 +3318,6 @@ static int _issue_deauth(struct adapter *padapter, unsigned char *da, pattrib->last_txcmdsz = pattrib->pktlen; - if (wait_ack) { ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe); } else { @@ -3662,7 +3639,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter) pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen)); pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen)); - /* */ if (pmlmepriv->num_FortyMHzIntolerant > 0) { u8 iedata = 0; @@ -3673,7 +3649,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter) } - /* */ memset(ICS, 0, sizeof(ICS)); if (pmlmepriv->num_sta_no_ht > 0) { @@ -3714,7 +3689,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter) spin_unlock_bh(&(pmlmepriv->scanned_queue.lock)); - for (i = 0; i < 8; i++) { if (ICS[i][0] == 1) { int j, k = 0; @@ -3742,7 +3716,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter) } - pattrib->last_txcmdsz = pattrib->pktlen; dump_mgntframe(padapter, pmgntframe); @@ -4223,7 +4196,6 @@ void start_clnt_auth(struct adapter *padapter) pmlmeinfo->link_count = 0; pmlmeext->retry = 0; - netdev_dbg(padapter->pnetdev, "start auth\n"); issue_auth(padapter, NULL, 0); @@ -4231,7 +4203,6 @@ void start_clnt_auth(struct adapter *padapter) } - void start_clnt_assoc(struct adapter *padapter) { struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; @@ -4278,7 +4249,6 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid u8 channel; u8 i; - pregistrypriv = &padapter->registrypriv; pmlmeext = &padapter->mlmeextpriv; @@ -4435,12 +4405,12 @@ void report_survey_event(struct adapter *padapter, union recv_frame *precv_frame pmlmeext = &padapter->mlmeextpriv; pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct survey_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4488,12 +4458,12 @@ void report_surveydone_event(struct adapter *padapter) struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct surveydone_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4533,12 +4503,12 @@ void report_join_res(struct adapter *padapter, int res) struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info); struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct joinbss_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4562,10 +4532,8 @@ void report_join_res(struct adapter *padapter, int res) memcpy((unsigned char *)(&(pjoinbss_evt->network.network)), &(pmlmeinfo->network), sizeof(struct wlan_bssid_ex)); pjoinbss_evt->network.join_res = pjoinbss_evt->network.aid = res; - rtw_joinbss_event_prehandle(padapter, (u8 *)&pjoinbss_evt->network); - rtw_enqueue_cmd(pcmdpriv, pcmd_obj); return; @@ -4582,12 +4550,12 @@ void report_wmm_edca_update(struct adapter *padapter) struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct wmm_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4628,12 +4596,12 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct stadel_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4657,7 +4625,6 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi memcpy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr, ETH_ALEN); memcpy((unsigned char *)(pdel_sta_evt->rsvd), (unsigned char *)(&reason), 2); - psta = rtw_get_stainfo(&padapter->stapriv, MacAddr); if (psta) mac_id = (int)psta->mac_id; @@ -4679,12 +4646,12 @@ void report_add_sta_event(struct adapter *padapter, unsigned char *MacAddr, int struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct cmd_priv *pcmdpriv = &padapter->cmdpriv; - pcmd_obj = rtw_zmalloc(sizeof(struct cmd_obj)); + pcmd_obj = kzalloc(sizeof(*pcmd_obj), GFP_ATOMIC); if (!pcmd_obj) return; cmdsz = (sizeof(struct stassoc_event) + sizeof(struct C2HEvent_Header)); - pevtcmd = rtw_zmalloc(cmdsz); + pevtcmd = kzalloc(cmdsz, GFP_ATOMIC); if (!pevtcmd) { kfree(pcmd_obj); return; @@ -4846,7 +4813,6 @@ void mlmeext_joinbss_event_callback(struct adapter *padapter, int join_res) /* update bc/mc sta_info */ update_bmc_sta(padapter); - /* turn on dynamic functions */ Switch_DM_Func(padapter, DYNAMIC_ALL_FUNC_ENABLE, true); @@ -5130,11 +5096,11 @@ void survey_timer_hdl(struct timer_list *t) pmlmeext->scan_abort = false;/* reset */ } - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) return; - psurveyPara = rtw_zmalloc(sizeof(struct sitesurvey_parm)); + psurveyPara = kzalloc(sizeof(*psurveyPara), GFP_ATOMIC); if (!psurveyPara) { kfree(ph2c); return; @@ -5152,7 +5118,6 @@ void link_timer_hdl(struct timer_list *t) struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv; struct mlme_ext_info *pmlmeinfo = &(pmlmeext->mlmext_info); - if (pmlmeinfo->state & WIFI_FW_AUTH_NULL) { pmlmeinfo->state = WIFI_FW_NULL_STATE; report_join_res(padapter, -3); @@ -5339,7 +5304,6 @@ u8 join_cmd_hdl(struct adapter *padapter, u8 *pbuf) /* Set_MSR(padapter, _HW_STATE_NOLINK_); */ Set_MSR(padapter, _HW_STATE_STATION_); - rtw_hal_set_hwreg(padapter, HW_VAR_MLME_DISCONNECT, NULL); } @@ -5748,7 +5712,7 @@ u8 chk_bmc_sleepq_cmd(struct adapter *padapter) struct cmd_priv *pcmdpriv = &(padapter->cmdpriv); u8 res = _SUCCESS; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; @@ -5772,13 +5736,13 @@ u8 set_tx_beacon_cmd(struct adapter *padapter) u8 res = _SUCCESS; int len_diff = 0; - ph2c = rtw_zmalloc(sizeof(struct cmd_obj)); + ph2c = kzalloc(sizeof(*ph2c), GFP_ATOMIC); if (!ph2c) { res = _FAIL; goto exit; } - ptxBeacon_parm = rtw_zmalloc(sizeof(struct Tx_Beacon_param)); + ptxBeacon_parm = kzalloc(sizeof(*ptxBeacon_parm), GFP_ATOMIC); if (!ptxBeacon_parm) { kfree(ph2c); res = _FAIL; diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c index e893cb6fa2732..4727f94df4355 100644 --- a/drivers/staging/rtl8723bs/core/rtw_recv.c +++ b/drivers/staging/rtl8723bs/core/rtw_recv.c @@ -60,7 +60,6 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada precvframe = (union recv_frame *) precvpriv->precv_frame_buf; - for (i = 0; i < NR_RECVFRAME; i++) { INIT_LIST_HEAD(&(precvframe->u.list)); @@ -180,9 +179,6 @@ int rtw_free_recvframe(union recv_frame *precvframe, struct __queue *pfree_recv_ return _SUCCESS; } - - - signed int _rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *queue) { @@ -192,7 +188,6 @@ signed int _rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue * /* INIT_LIST_HEAD(&(precvframe->u.hdr.list)); */ list_del_init(&(precvframe->u.hdr.list)); - list_add_tail(&(precvframe->u.hdr.list), get_list_head(queue)); if (padapter) @@ -257,7 +252,6 @@ u32 rtw_free_uc_swdec_pending_queue(struct adapter *adapter) return cnt; } - signed int rtw_enqueue_recvbuf_to_head(struct recv_buf *precvbuf, struct __queue *queue) { spin_lock_bh(&queue->lock); @@ -405,7 +399,6 @@ static signed int recvframe_chkmic(struct adapter *adapter, union recv_frame *p bmic_err = true; } - if (bmic_err == true) { /* double check key_index for some timing issue , */ /* cannot compare with psecuritypriv->dot118021XGrpKeyid also cause timing issue */ @@ -772,8 +765,6 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame * } else ret = _FAIL; - - if (bmcast) *psta = rtw_get_bcmc_stainfo(adapter); else @@ -817,7 +808,6 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p goto exit; } - /* check BSSID */ if (is_zero_ether_addr(pattrib->bssid) || is_zero_ether_addr(mybssid) || @@ -858,14 +848,12 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p /* */ memcpy(pattrib->bssid, mybssid, ETH_ALEN); - *psta = rtw_get_stainfo(pstapriv, pattrib->bssid); /* get sta_info */ if (!*psta) { ret = _FAIL; goto exit; } - } else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) { /* Special case */ ret = RTW_RX_HANDLED; @@ -1109,7 +1097,6 @@ static union recv_frame *recvframe_defrag(struct adapter *adapter, pnextrframe = (union recv_frame *)plist; pnfhdr = &pnextrframe->u.hdr; - /* check the fragment sequence (2nd ~n fragment frame) */ if (curfragnum != pnfhdr->attrib.frag_num) { @@ -1197,7 +1184,6 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re /* free current defrag_q */ rtw_free_recvframe_queue(pdefrag_q, pfree_recv_queue); - /* Then enqueue the 0~(n-1) fragment into the defrag_q */ /* spin_lock(&pdefrag_q->lock); */ @@ -1236,7 +1222,6 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re } - if ((prtnframe) && (prtnframe->u.hdr.attrib.privacy)) { /* after defrag we must check tkip mic code */ if (recvframe_chkmic(padapter, prtnframe) == _FAIL) { @@ -1345,7 +1330,6 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f goto exit; } - if (!psta) { ret = _FAIL; goto exit; @@ -1355,7 +1339,6 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f /* psta->signal_quality = prxcmd->sq; */ precv_frame->u.hdr.psta = psta; - pattrib->amsdu = 0; pattrib->ack_policy = 0; /* parsing QC field */ @@ -1373,7 +1356,6 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f pattrib->hdrlen = pattrib->to_fr_ds == 3 ? 30 : 24; } - if (pattrib->order)/* HT-CTRL 11n */ pattrib->hdrlen += 4; @@ -1425,7 +1407,7 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame memcpy(pattrib->ta, GetAddr2Ptr(ptr), ETH_ALEN); /* actual management data frame body */ data_len = pattrib->pkt_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; - mgmt_DATA = rtw_zmalloc(data_len); + mgmt_DATA = kzalloc(data_len, GFP_ATOMIC); if (!mgmt_DATA) goto validate_80211w_fail; precv_frame = decryptor(adapter, precv_frame); @@ -1827,7 +1809,6 @@ static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, un /* spin_lock_irqsave(&ppending_recvframe_queue->lock, irql); */ /* spin_lock(&ppending_recvframe_queue->lock); */ - phead = get_list_head(ppending_recvframe_queue); plist = get_next(phead); @@ -1846,7 +1827,6 @@ static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, un } - /* spin_lock_irqsave(&ppending_recvframe_queue->lock, irql); */ /* spin_lock(&ppending_recvframe_queue->lock); */ @@ -1975,7 +1955,6 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor /* error condition; */ } - /* Update local variables. */ bPktInBuf = false; @@ -2058,7 +2037,6 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame * goto _err_exit; } - /* s4. */ /* Indication process. */ /* After Packet dropping and Sliding Window shifting as above, we can now just indicate the packets */ @@ -2086,7 +2064,6 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame * return _FAIL; } - void rtw_reordering_ctrl_timeout_handler(struct timer_list *t) { struct recv_reorder_ctrl *preorder_ctrl = @@ -2094,7 +2071,6 @@ void rtw_reordering_ctrl_timeout_handler(struct timer_list *t) struct adapter *padapter = preorder_ctrl->padapter; struct __queue *ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue; - if (padapter->bDriverStopped || padapter->bSurpriseRemoved) return; @@ -2242,7 +2218,6 @@ static int recv_func(struct adapter *padapter, union recv_frame *rframe) return ret; } - s32 rtw_recv_entry(union recv_frame *precvframe) { struct adapter *padapter; diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c index 5406f60dd31c6..6b5c9831af020 100644 --- a/drivers/staging/rtl8723bs/core/rtw_security.c +++ b/drivers/staging/rtl8723bs/core/rtw_security.c @@ -62,14 +62,14 @@ void rtw_wep_encrypt(struct adapter *padapter, u8 *pxmitframe) keylength = psecuritypriv->dot11DefKeylen[psecuritypriv->dot11PrivacyKeyIndex]; for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) { - iv = pframe+pattrib->hdrlen; + iv = pframe + pattrib->hdrlen; memcpy(&wepkey[0], iv, 3); memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[psecuritypriv->dot11PrivacyKeyIndex].skey[0], keylength); - payload = pframe+pattrib->iv_len+pattrib->hdrlen; + payload = pframe + pattrib->iv_len + pattrib->hdrlen; - if ((curfragnum+1) == pattrib->nr_frags) { /* the last fragment */ + if ((curfragnum + 1) == pattrib->nr_frags) { /* the last fragment */ - length = pattrib->last_txcmdsz-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length)); @@ -78,7 +78,7 @@ void rtw_wep_encrypt(struct adapter *padapter, u8 *pxmitframe) arc4_crypt(ctx, payload + length, crc.f1, 4); } else { - length = pxmitpriv->frag_len-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length)); arc4_setkey(ctx, wepkey, 3 + keylength); arc4_crypt(ctx, payload, payload, length); @@ -107,16 +107,16 @@ void rtw_wep_decrypt(struct adapter *padapter, u8 *precvframe) /* start to decrypt recvframe */ if ((prxattrib->encrypt == _WEP40_) || (prxattrib->encrypt == _WEP104_)) { - iv = pframe+prxattrib->hdrlen; + iv = pframe + prxattrib->hdrlen; /* keyindex =(iv[3]&0x3); */ keyindex = prxattrib->key_index; keylength = psecuritypriv->dot11DefKeylen[keyindex]; memcpy(&wepkey[0], iv, 3); /* memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[psecuritypriv->dot11PrivacyKeyIndex].skey[0], keylength); */ memcpy(&wepkey[3], &psecuritypriv->dot11DefKey[keyindex].skey[0], keylength); - length = ((union recv_frame *)precvframe)->u.hdr.len-prxattrib->hdrlen-prxattrib->iv_len; + length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len; - payload = pframe+prxattrib->iv_len+prxattrib->hdrlen; + payload = pframe + prxattrib->iv_len + prxattrib->hdrlen; /* decrypt payload include icv */ arc4_setkey(ctx, wepkey, 3 + keylength); @@ -174,7 +174,7 @@ void rtw_secmicsetkey(struct mic_data *pmicdata, u8 *key) void rtw_secmicappendbyte(struct mic_data *pmicdata, u8 b) { /* Append the byte to our word-sized buffer */ - pmicdata->M |= ((unsigned long)b) << (8*pmicdata->nBytesInM); + pmicdata->M |= ((unsigned long)b) << (8 * pmicdata->nBytesInM); pmicdata->nBytesInM++; /* Process the word if it is full. */ if (pmicdata->nBytesInM >= 4) { @@ -261,7 +261,7 @@ void rtw_seccalctkipmic(u8 *key, u8 *header, u8 *data, u32 data_len, u8 *mic_cod #define Mk16(hi, lo) ((lo) ^ (((u16)(hi)) << 8)) /* select the Nth 16-bit word of the temporal key unsigned char array TK[] */ -#define TK16(N) Mk16(tk[2*(N)+1], tk[2*(N)]) +#define TK16(N) Mk16(tk[2 * (N) + 1], tk[2 * (N)]) /* S-box lookup: 16 bits --> 16 bits */ #define _S_(v16) (Sbox1[0][Lo8(v16)] ^ Sbox1[1][Hi8(v16)]) @@ -375,11 +375,11 @@ static void phase1(u16 *p1k, const u8 *tk, const u8 *ta, u32 iv32) /* size on the 80-bit block P1K[], using the 128-bit key TK[] */ for (i = 0; i < PHASE1_LOOP_CNT; i++) { /* Each add operation here is mod 2**16 */ - p1k[0] += _S_(p1k[4] ^ TK16((i&1)+0)); - p1k[1] += _S_(p1k[0] ^ TK16((i&1)+2)); - p1k[2] += _S_(p1k[1] ^ TK16((i&1)+4)); - p1k[3] += _S_(p1k[2] ^ TK16((i&1)+6)); - p1k[4] += _S_(p1k[3] ^ TK16((i&1)+0)); + p1k[0] += _S_(p1k[4] ^ TK16((i & 1) + 0)); + p1k[1] += _S_(p1k[0] ^ TK16((i & 1) + 2)); + p1k[2] += _S_(p1k[1] ^ TK16((i & 1) + 4)); + p1k[3] += _S_(p1k[2] ^ TK16((i & 1) + 6)); + p1k[4] += _S_(p1k[3] ^ TK16((i & 1) + 0)); p1k[4] += (unsigned short)i; /* avoid "slide attacks" */ } } @@ -417,7 +417,7 @@ static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16) for (i = 0; i < 5; i++) PPK[i] = p1k[i]; /* first, copy P1K to PPK */ - PPK[5] = p1k[4]+iv16; /* next, add in IV16 */ + PPK[5] = p1k[4] + iv16; /* next, add in IV16 */ /* Bijective non-linear mixing of the 96 bits of PPK[0..5] */ PPK[0] += _S_(PPK[5] ^ TK16(0)); /* Mix key in each "round" */ @@ -448,8 +448,8 @@ static void phase2(u8 *rc4key, const u8 *tk, const u16 *p1k, u16 iv16) /* Copy 96 bits of PPK[0..5] to RC4KEY[4..15] (little-endian) */ for (i = 0; i < 6; i++) { - rc4key[4+2*i] = Lo8(PPK[i]); - rc4key[5+2*i] = Hi8(PPK[i]); + rc4key[4 + 2 * i] = Lo8(PPK[i]); + rc4key[5 + 2 * i] = Hi8(PPK[i]); } } @@ -492,20 +492,20 @@ u32 rtw_tkip_encrypt(struct adapter *padapter, u8 *pxmitframe) prwskey = pattrib->dot118021x_UncstKey.skey; for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) { - iv = pframe+pattrib->hdrlen; - payload = pframe+pattrib->iv_len+pattrib->hdrlen; + iv = pframe + pattrib->hdrlen; + payload = pframe + pattrib->iv_len + pattrib->hdrlen; GET_TKIP_PN(iv, dot11txpn); pnl = (u16)(dot11txpn.val); - pnh = (u32)(dot11txpn.val>>16); + pnh = (u32)(dot11txpn.val >> 16); phase1((u16 *)&ttkey[0], prwskey, &pattrib->ta[0], pnh); phase2(&rc4key[0], prwskey, (u16 *)&ttkey[0], pnl); - if ((curfragnum+1) == pattrib->nr_frags) { /* 4 the last fragment */ - length = pattrib->last_txcmdsz-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + if ((curfragnum + 1) == pattrib->nr_frags) { /* 4 the last fragment */ + length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length)); arc4_setkey(ctx, rc4key, 16); @@ -513,7 +513,7 @@ u32 rtw_tkip_encrypt(struct adapter *padapter, u8 *pxmitframe) arc4_crypt(ctx, payload + length, crc.f1, 4); } else { - length = pxmitpriv->frag_len-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length)); arc4_setkey(ctx, rc4key, 16); @@ -601,14 +601,14 @@ u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe) prwskey = &stainfo->dot118021x_UncstKey.skey[0]; } - iv = pframe+prxattrib->hdrlen; - payload = pframe+prxattrib->iv_len+prxattrib->hdrlen; - length = ((union recv_frame *)precvframe)->u.hdr.len-prxattrib->hdrlen-prxattrib->iv_len; + iv = pframe + prxattrib->hdrlen; + payload = pframe + prxattrib->iv_len + prxattrib->hdrlen; + length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len; GET_TKIP_PN(iv, dot11txpn); pnl = (u16)(dot11txpn.val); - pnh = (u32)(dot11txpn.val>>16); + pnh = (u32)(dot11txpn.val >> 16); phase1((u16 *)&ttkey[0], prwskey, &prxattrib->ta[0], pnh); phase2(&rc4key[0], prwskey, (unsigned short *)&ttkey[0], pnl); @@ -758,7 +758,7 @@ static void construct_mic_header2(u8 *mic_header2, if (!qc_exists && a4_exists) { for (i = 0; i < 6; i++) - mic_header2[8+i] = mpdu[24+i]; /* A4 */ + mic_header2[8 + i] = mpdu[24 + i]; /* A4 */ } if (qc_exists && !a4_exists) { @@ -768,7 +768,7 @@ static void construct_mic_header2(u8 *mic_header2, if (qc_exists && a4_exists) { for (i = 0; i < 6; i++) - mic_header2[8+i] = mpdu[24+i]; /* A4 */ + mic_header2[8 + i] = mpdu[24 + i]; /* A4 */ mic_header2[14] = mpdu[30] & 0x0f; mic_header2[15] = mpdu[31] & 0x00; @@ -839,16 +839,16 @@ static signed int aes_cipher(u8 *key, uint hdrlen, uint frtype = GetFrameType(pframe); uint frsubtype = GetFrameSubType(pframe); - frsubtype = frsubtype>>4; + frsubtype = frsubtype >> 4; if ((hdrlen == WLAN_HDR_A3_LEN) || (hdrlen == WLAN_HDR_A3_QOS_LEN)) a4_exists = 0; else a4_exists = 1; - if (((frtype|frsubtype) == WIFI_DATA_CFACK) || - ((frtype|frsubtype) == WIFI_DATA_CFPOLL) || - ((frtype|frsubtype) == WIFI_DATA_CFACKPOLL)) { + if (((frtype | frsubtype) == WIFI_DATA_CFACK) || + ((frtype | frsubtype) == WIFI_DATA_CFPOLL) || + ((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) { qc_exists = 1; if (hdrlen != WLAN_HDR_A3_QOS_LEN) hdrlen += 2; @@ -867,11 +867,11 @@ static signed int aes_cipher(u8 *key, uint hdrlen, } pn_vector[0] = pframe[hdrlen]; - pn_vector[1] = pframe[hdrlen+1]; - pn_vector[2] = pframe[hdrlen+4]; - pn_vector[3] = pframe[hdrlen+5]; - pn_vector[4] = pframe[hdrlen+6]; - pn_vector[5] = pframe[hdrlen+7]; + pn_vector[1] = pframe[hdrlen + 1]; + pn_vector[2] = pframe[hdrlen + 4]; + pn_vector[3] = pframe[hdrlen + 5]; + pn_vector[4] = pframe[hdrlen + 6]; + pn_vector[5] = pframe[hdrlen + 7]; construct_mic_iv(mic_iv, qc_exists, @@ -927,12 +927,12 @@ static signed int aes_cipher(u8 *key, uint hdrlen, /* Insert MIC into payload */ for (j = 0; j < 8; j++) - pframe[payload_index+j] = mic[j]; + pframe[payload_index + j] = mic[j]; payload_index = hdrlen + 8; for (i = 0; i < num_blocks; i++) { construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, /* message, */ - pn_vector, i+1, frtype); + pn_vector, i + 1, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, &pframe[payload_index], 16); @@ -944,13 +944,13 @@ static signed int aes_cipher(u8 *key, uint hdrlen, /* If there is a short final block, then pad it,*/ /* encrypt it and copy the unpadded part back */ construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, /* message, */ - pn_vector, num_blocks+1, frtype); + pn_vector, num_blocks + 1, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ for (j = 0; j < 16; j++) padded_buffer[j] = 0x00; for (j = 0; j < payload_remainder; j++) - padded_buffer[j] = pframe[payload_index+j]; + padded_buffer[j] = pframe[payload_index + j]; aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, padded_buffer, 16); @@ -966,7 +966,7 @@ static signed int aes_cipher(u8 *key, uint hdrlen, for (j = 0; j < 16; j++) padded_buffer[j] = 0x00; for (j = 0; j < 8; j++) - padded_buffer[j] = pframe[j+hdrlen+8+plen]; + padded_buffer[j] = pframe[j + hdrlen + 8 + plen]; aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, padded_buffer, 16); @@ -1006,12 +1006,12 @@ u32 rtw_aes_encrypt(struct adapter *padapter, u8 *pxmitframe) prwskey = pattrib->dot118021x_UncstKey.skey; for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) { - if ((curfragnum+1) == pattrib->nr_frags) { /* 4 the last fragment */ - length = pattrib->last_txcmdsz-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + if ((curfragnum + 1) == pattrib->nr_frags) { /* 4 the last fragment */ + length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; aes_cipher(prwskey, pattrib->hdrlen, pframe, length); } else { - length = pxmitpriv->frag_len-pattrib->hdrlen-pattrib->iv_len-pattrib->icv_len; + length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len; aes_cipher(prwskey, pattrib->hdrlen, pframe, length); pframe += pxmitpriv->frag_len; @@ -1044,13 +1044,13 @@ static signed int aes_decipher(u8 *key, uint hdrlen, uint frtype = GetFrameType(pframe); uint frsubtype = GetFrameSubType(pframe); - frsubtype = frsubtype>>4; + frsubtype = frsubtype >> 4; /* start to decrypt the payload */ - num_blocks = (plen-8) / 16; /* plen including LLC, payload_length and mic) */ + num_blocks = (plen - 8) / 16; /* plen including LLC, payload_length and mic) */ - payload_remainder = (plen-8) % 16; + payload_remainder = (plen - 8) % 16; pn_vector[0] = pframe[hdrlen]; pn_vector[1] = pframe[hdrlen + 1]; @@ -1064,9 +1064,9 @@ static signed int aes_decipher(u8 *key, uint hdrlen, else a4_exists = 1; - if (((frtype|frsubtype) == WIFI_DATA_CFACK) || - ((frtype|frsubtype) == WIFI_DATA_CFPOLL) || - ((frtype|frsubtype) == WIFI_DATA_CFACKPOLL)) { + if (((frtype | frsubtype) == WIFI_DATA_CFACK) || + ((frtype | frsubtype) == WIFI_DATA_CFPOLL) || + ((frtype | frsubtype) == WIFI_DATA_CFACKPOLL)) { qc_exists = 1; if (hdrlen != WLAN_HDR_A3_QOS_LEN) hdrlen += 2; @@ -1105,13 +1105,13 @@ static signed int aes_decipher(u8 *key, uint hdrlen, /* If there is a short final block, then pad it,*/ /* encrypt it and copy the unpadded part back */ construct_ctr_preload(ctr_preload, a4_exists, qc_exists, pframe, pn_vector, - num_blocks+1, frtype); + num_blocks + 1, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ for (j = 0; j < 16; j++) padded_buffer[j] = 0x00; for (j = 0; j < payload_remainder; j++) - padded_buffer[j] = pframe[payload_index+j]; + padded_buffer[j] = pframe[payload_index + j]; aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, padded_buffer, 16); @@ -1120,25 +1120,25 @@ static signed int aes_decipher(u8 *key, uint hdrlen, } /* start to calculate the mic */ - if ((hdrlen + plen+8) <= MAX_MSG_SIZE) - memcpy((void *)message, pframe, (hdrlen + plen+8)); /* 8 is for ext iv len */ + if ((hdrlen + plen + 8) <= MAX_MSG_SIZE) + memcpy((void *)message, pframe, (hdrlen + plen + 8)); /* 8 is for ext iv len */ pn_vector[0] = pframe[hdrlen]; - pn_vector[1] = pframe[hdrlen+1]; - pn_vector[2] = pframe[hdrlen+4]; - pn_vector[3] = pframe[hdrlen+5]; - pn_vector[4] = pframe[hdrlen+6]; - pn_vector[5] = pframe[hdrlen+7]; + pn_vector[1] = pframe[hdrlen + 1]; + pn_vector[2] = pframe[hdrlen + 4]; + pn_vector[3] = pframe[hdrlen + 5]; + pn_vector[4] = pframe[hdrlen + 6]; + pn_vector[5] = pframe[hdrlen + 7]; - construct_mic_iv(mic_iv, qc_exists, a4_exists, message, plen-8, pn_vector, frtype); + construct_mic_iv(mic_iv, qc_exists, a4_exists, message, plen - 8, pn_vector, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ construct_mic_header1(mic_header1, hdrlen, message, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ construct_mic_header2(mic_header2, message, a4_exists, qc_exists); - payload_remainder = (plen-8) % 16; - num_blocks = (plen-8) / 16; + payload_remainder = (plen - 8) % 16; + num_blocks = (plen - 8) / 16; /* Find start of payload */ payload_index = (hdrlen + 8); @@ -1173,11 +1173,11 @@ static signed int aes_decipher(u8 *key, uint hdrlen, /* Insert MIC into payload */ for (j = 0; j < 8; j++) - message[payload_index+j] = mic[j]; + message[payload_index + j] = mic[j]; payload_index = hdrlen + 8; for (i = 0; i < num_blocks; i++) { - construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector, i+1, + construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector, i + 1, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ aes128k128d(key, ctr_preload, aes_out); @@ -1190,13 +1190,13 @@ static signed int aes_decipher(u8 *key, uint hdrlen, /* If there is a short final block, then pad it,*/ /* encrypt it and copy the unpadded part back */ construct_ctr_preload(ctr_preload, a4_exists, qc_exists, message, pn_vector, - num_blocks+1, frtype); + num_blocks + 1, frtype); /* add for CONFIG_IEEE80211W, none 11w also can use */ for (j = 0; j < 16; j++) padded_buffer[j] = 0x00; for (j = 0; j < payload_remainder; j++) - padded_buffer[j] = message[payload_index+j]; + padded_buffer[j] = message[payload_index + j]; aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, padded_buffer, 16); @@ -1211,7 +1211,7 @@ static signed int aes_decipher(u8 *key, uint hdrlen, for (j = 0; j < 16; j++) padded_buffer[j] = 0x00; for (j = 0; j < 8; j++) - padded_buffer[j] = message[j+hdrlen+8+plen-8]; + padded_buffer[j] = message[j + hdrlen + 8 + plen - 8]; aes128k128d(key, ctr_preload, aes_out); crypto_xor_cpy(chain_buffer, aes_out, padded_buffer, 16); @@ -1298,7 +1298,7 @@ u32 rtw_aes_decrypt(struct adapter *padapter, u8 *precvframe) prwskey = &stainfo->dot118021x_UncstKey.skey[0]; } - length = ((union recv_frame *)precvframe)->u.hdr.len-prxattrib->hdrlen-prxattrib->iv_len; + length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len; res = aes_decipher(prwskey, prxattrib->hdrlen, pframe, length); @@ -1323,9 +1323,8 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe) __le16 le_tmp; __le64 le_tmp64 = 0; - ori_len = pattrib->pkt_len-WLAN_HDR_A3_LEN+BIP_AAD_SIZE; - BIP_AAD = rtw_zmalloc(ori_len); - + ori_len = pattrib->pkt_len - WLAN_HDR_A3_LEN + BIP_AAD_SIZE; + BIP_AAD = kzalloc(ori_len, GFP_KERNEL); if (!BIP_AAD) return _FAIL; @@ -1334,28 +1333,28 @@ u32 rtw_BIP_verify(struct adapter *padapter, u8 *precvframe) /* mapping to wlan header */ pwlanhdr = (struct ieee80211_hdr *)pframe; /* save the frame body + MME */ - memcpy(BIP_AAD+BIP_AAD_SIZE, pframe+WLAN_HDR_A3_LEN, pattrib->pkt_len-WLAN_HDR_A3_LEN); + memcpy(BIP_AAD + BIP_AAD_SIZE, pframe + WLAN_HDR_A3_LEN, pattrib->pkt_len - WLAN_HDR_A3_LEN); /* find MME IE pointer */ - p = rtw_get_ie(BIP_AAD+BIP_AAD_SIZE, WLAN_EID_MMIE, &len, pattrib->pkt_len-WLAN_HDR_A3_LEN); + p = rtw_get_ie(BIP_AAD + BIP_AAD_SIZE, WLAN_EID_MMIE, &len, pattrib->pkt_len - WLAN_HDR_A3_LEN); /* Baron */ if (p) { u16 keyid = 0; u64 temp_ipn = 0; /* save packet number */ - memcpy(&le_tmp64, p+4, 6); + memcpy(&le_tmp64, p + 4, 6); temp_ipn = le64_to_cpu(le_tmp64); /* BIP packet number should bigger than previous BIP packet */ if (temp_ipn <= pmlmeext->mgnt_80211w_IPN_rx) goto BIP_exit; /* copy key index */ - memcpy(&le_tmp, p+2, 2); + memcpy(&le_tmp, p + 2, 2); keyid = le16_to_cpu(le_tmp); if (keyid != padapter->securitypriv.dot11wBIPKeyid) goto BIP_exit; /* clear the MIC field of MME to zero */ - memset(p+2+len-8, 0, 8); + memset(p + 2 + len - 8, 0, 8); /* conscruct AAD, copy frame control field */ memcpy(BIP_AAD, &pwlanhdr->frame_control, 2); @@ -1515,7 +1514,7 @@ u8 rtw_handle_tkip_countermeasure(struct adapter *adapter, const char *caller) if (securitypriv->btkip_countermeasure) { unsigned long passing_ms = jiffies_to_msecs(jiffies - securitypriv->btkip_countermeasure_time); - if (passing_ms > 60*1000) { + if (passing_ms > 60 * 1000) { netdev_dbg(adapter->pnetdev, "%s(%s) countermeasure time:%lus > 60s\n", caller, ADPT_ARG(adapter), diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c index 656a1b49ad20c..ccfbef80c00ca 100644 --- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c +++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c @@ -341,7 +341,6 @@ void set_channel_bwmode(struct adapter *padapter, unsigned char channel, unsigne center_ch = rtw_get_center_ch(channel, bwmode, channel_offset); - /* set Channel */ if (mutex_lock_interruptible(&(adapter_to_dvobj(padapter)->setch_mutex))) return; @@ -692,6 +691,9 @@ int WMM_param_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE) return false; } + if (pIE->length != WLAN_WMM_LEN) + return false; + if (!memcmp(&(pmlmeinfo->WMM_param), (pIE->data + 6), sizeof(struct WMM_para_element))) return false; else @@ -1133,7 +1135,7 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len) if (memcmp(cur_network->network.mac_address, pbssid, 6)) return true; - bssid = rtw_zmalloc(sizeof(struct wlan_bssid_ex)); + bssid = kzalloc(sizeof(*bssid), GFP_KERNEL); if (!bssid) return true; diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c index 21690857fd62a..c9e0b91d75d3e 100644 --- a/drivers/staging/rtl8723bs/core/rtw_xmit.c +++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c @@ -78,7 +78,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) } pxmitpriv->pxmit_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_frame_buf), 4); - pxframe = (struct xmit_frame *) pxmitpriv->pxmit_frame_buf; + pxframe = (struct xmit_frame *)pxmitpriv->pxmit_frame_buf; for (i = 0; i < NR_XMITFRAME; i++) { INIT_LIST_HEAD(&pxframe->list); @@ -238,7 +238,9 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) pxmitbuf->padapter = padapter; pxmitbuf->buf_tag = XMITBUF_CMD; - res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, MAX_CMDBUF_SZ+XMITBUF_ALIGN_SZ, true); + res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, + MAX_CMDBUF_SZ + XMITBUF_ALIGN_SZ, + true); if (res == _FAIL) { res = _FAIL; goto exit; @@ -248,7 +250,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter) pxmitbuf->pend = pxmitbuf->pbuf + MAX_CMDBUF_SZ; pxmitbuf->len = 0; pxmitbuf->pdata = pxmitbuf->ptail = pxmitbuf->phead; - pxmitbuf->alloc_sz = MAX_CMDBUF_SZ+XMITBUF_ALIGN_SZ; + pxmitbuf->alloc_sz = MAX_CMDBUF_SZ + XMITBUF_ALIGN_SZ; } } @@ -274,7 +276,7 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv) { int i; struct adapter *padapter = pxmitpriv->adapter; - struct xmit_frame *pxmitframe = (struct xmit_frame *) pxmitpriv->pxmit_frame_buf; + struct xmit_frame *pxmitframe = (struct xmit_frame *)pxmitpriv->pxmit_frame_buf; struct xmit_buf *pxmitbuf = (struct xmit_buf *)pxmitpriv->pxmitbuf; rtw_hal_free_xmit_priv(padapter); @@ -321,7 +323,9 @@ void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv) for (i = 0; i < CMDBUF_MAX; i++) { pxmitbuf = &pxmitpriv->pcmd_xmitbuf[i]; if (pxmitbuf) - rtw_os_xmit_resource_free(padapter, pxmitbuf, MAX_CMDBUF_SZ+XMITBUF_ALIGN_SZ, true); + rtw_os_xmit_resource_free(padapter, pxmitbuf, + MAX_CMDBUF_SZ + XMITBUF_ALIGN_SZ, + true); } rtw_free_hwxmits(padapter); @@ -736,7 +740,7 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p pattrib->subtype = WIFI_DATA_TYPE; pattrib->priority = 0; - if (check_fwstate(pmlmepriv, WIFI_AP_STATE|WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE)) { + if (check_fwstate(pmlmepriv, WIFI_AP_STATE | WIFI_ADHOC_STATE | WIFI_ADHOC_MASTER_STATE)) { if (pattrib->qos_en) set_qos(&pktfile, pattrib); } else { @@ -787,15 +791,15 @@ static s32 xmitframe_addmic(struct adapter *padapter, struct xmit_frame *pxmitfr rtw_secmicsetkey(&micdata, &pattrib->dot11tkiptxmickey.skey[0]); } - if (pframe[1]&1) { /* ToDS == 1 */ + if (pframe[1] & 1) { /* ToDS == 1 */ rtw_secmicappend(&micdata, &pframe[16], 6); /* DA */ - if (pframe[1]&2) /* From Ds == 1 */ + if (pframe[1] & 2) /* From Ds == 1 */ rtw_secmicappend(&micdata, &pframe[24], 6); else rtw_secmicappend(&micdata, &pframe[10], 6); } else { /* ToDS == 0 */ rtw_secmicappend(&micdata, &pframe[4], 6); /* DA */ - if (pframe[1]&2) /* From Ds == 1 */ + if (pframe[1] & 2) /* From Ds == 1 */ rtw_secmicappend(&micdata, &pframe[16], 6); else rtw_secmicappend(&micdata, &pframe[10], 6); @@ -810,16 +814,20 @@ static s32 xmitframe_addmic(struct adapter *padapter, struct xmit_frame *pxmitfr for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) { payload = (u8 *)round_up((SIZE_PTR)(payload), 4); - payload = payload+pattrib->hdrlen+pattrib->iv_len; + payload = payload + pattrib->hdrlen + pattrib->iv_len; - if ((curfragnum+1) == pattrib->nr_frags) { - length = pattrib->last_txcmdsz-pattrib->hdrlen-pattrib->iv_len-((pattrib->bswenc) ? pattrib->icv_len : 0); + if ((curfragnum + 1) == pattrib->nr_frags) { + length = pattrib->last_txcmdsz - pattrib->hdrlen - + pattrib->iv_len - + ((pattrib->bswenc) ? pattrib->icv_len : 0); rtw_secmicappend(&micdata, payload, length); - payload = payload+length; + payload = payload + length; } else { - length = pxmitpriv->frag_len-pattrib->hdrlen-pattrib->iv_len-((pattrib->bswenc) ? pattrib->icv_len : 0); + length = pxmitpriv->frag_len - pattrib->hdrlen - + pattrib->iv_len - + ((pattrib->bswenc) ? pattrib->icv_len : 0); rtw_secmicappend(&micdata, payload, length); - payload = payload+length+pattrib->icv_len; + payload = payload + length + pattrib->icv_len; } } rtw_secgetmic(&micdata, &mic[0]); @@ -1108,8 +1116,10 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct if (bmcst || (rtw_endofpktfile(&pktfile) == true)) { pattrib->nr_frags = frg_inx; - pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len + ((pattrib->nr_frags == 1) ? llc_sz:0) + - ((pattrib->bswenc) ? pattrib->icv_len : 0) + mem_sz; + pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len + + ((pattrib->nr_frags == 1) ? llc_sz : 0) + + ((pattrib->bswenc) ? pattrib->icv_len : 0) + + mem_sz; ClearMFrag(mem_start); @@ -1158,8 +1168,8 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s mem_start = pframe = (u8 *)(pxmitframe->buf_addr) + TXDESC_OFFSET; pwlanhdr = (struct ieee80211_hdr *)pframe; - ori_len = BIP_AAD_SIZE+pattrib->pktlen; - tmp_buf = BIP_AAD = rtw_zmalloc(ori_len); + ori_len = BIP_AAD_SIZE + pattrib->pktlen; + tmp_buf = BIP_AAD = kzalloc(ori_len, GFP_ATOMIC); subtype = GetFrameSubType(pframe); /* bit(7)~bit(2) */ if (!BIP_AAD) @@ -1211,14 +1221,14 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s /* conscruct AAD, copy address 1 to address 3 */ memcpy(BIP_AAD + 2, &pwlanhdr->addrs, sizeof(pwlanhdr->addrs)); /* copy management fram body */ - memcpy(BIP_AAD+BIP_AAD_SIZE, MGMT_body, frame_body_len); + memcpy(BIP_AAD + BIP_AAD_SIZE, MGMT_body, frame_body_len); /* calculate mic */ if (omac1_aes_128(padapter->securitypriv.dot11wBIPKey[padapter->securitypriv.dot11wBIPKeyid].skey - , BIP_AAD, BIP_AAD_SIZE+frame_body_len, mic)) + , BIP_AAD, BIP_AAD_SIZE + frame_body_len, mic)) goto xmitframe_coalesce_fail; /* copy right BIP mic value, total is 128bits, we use the 0~63 bits */ - memcpy(pframe-8, mic, 8); + memcpy(pframe - 8, mic, 8); } else { /* unicast mgmt frame TX */ /* start to encrypt mgmt frame */ if (subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC || @@ -1267,9 +1277,10 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s memcpy(pframe, pattrib->iv, pattrib->iv_len); pframe += pattrib->iv_len; /* copy mgmt data portion after CCMP header */ - memcpy(pframe, tmp_buf+pattrib->hdrlen, pattrib->pktlen-pattrib->hdrlen); + memcpy(pframe, tmp_buf + pattrib->hdrlen, + pattrib->pktlen - pattrib->hdrlen); /* move pframe to end of mgmt pkt */ - pframe += pattrib->pktlen-pattrib->hdrlen; + pframe += pattrib->pktlen - pattrib->hdrlen; /* add 8 bytes CCMP IV header to length */ pattrib->pktlen += pattrib->iv_len; if ((pattrib->icv_len > 0) && (pattrib->bswenc)) { @@ -1375,7 +1386,7 @@ void rtw_count_tx_stats(struct adapter *padapter, struct xmit_frame *pxmitframe, struct mlme_priv *pmlmepriv = &padapter->mlmepriv; u8 pkt_num = 1; - if ((pxmitframe->frame_tag&0x0f) == DATA_FRAMETAG) { + if ((pxmitframe->frame_tag & 0x0f) == DATA_FRAMETAG) { pkt_num = pxmitframe->agg_num; pmlmepriv->LinkDetectInfo.NumTxOkInPeriod += pkt_num; @@ -1662,8 +1673,7 @@ struct xmit_frame *rtw_alloc_xmitframe_once(struct xmit_priv *pxmitpriv) struct xmit_frame *pxframe = NULL; u8 *alloc_addr; - alloc_addr = rtw_zmalloc(sizeof(struct xmit_frame) + 4); - + alloc_addr = kzalloc(sizeof(*pxframe) + 4, GFP_ATOMIC); if (!alloc_addr) goto exit; @@ -1836,8 +1846,7 @@ s32 rtw_alloc_hwxmits(struct adapter *padapter) pxmitpriv->hwxmits = NULL; - pxmitpriv->hwxmits = rtw_zmalloc(sizeof(struct hw_xmit) * pxmitpriv->hwxmit_entry); - + pxmitpriv->hwxmits = kcalloc(pxmitpriv->hwxmit_entry, sizeof(*hwxmits), GFP_ATOMIC); if (!pxmitpriv->hwxmits) return _FAIL; @@ -2070,7 +2079,7 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x spin_lock_bh(&psta->sleep_q.lock); - if (psta->state&WIFI_SLEEP_STATE) { + if (psta->state & WIFI_SLEEP_STATE) { u8 wmmps_ac = 0; if (pstapriv->sta_dz_bitmap & BIT(psta->aid)) { @@ -2083,20 +2092,20 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x switch (pattrib->priority) { case 1: case 2: - wmmps_ac = psta->uapsd_bk&BIT(0); + wmmps_ac = psta->uapsd_bk & BIT(0); break; case 4: case 5: - wmmps_ac = psta->uapsd_vi&BIT(0); + wmmps_ac = psta->uapsd_vi & BIT(0); break; case 6: case 7: - wmmps_ac = psta->uapsd_vo&BIT(0); + wmmps_ac = psta->uapsd_vo & BIT(0); break; case 0: case 3: default: - wmmps_ac = psta->uapsd_be&BIT(0); + wmmps_ac = psta->uapsd_be & BIT(0); break; } @@ -2214,20 +2223,20 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta) switch (pxmitframe->attrib.priority) { case 1: case 2: - wmmps_ac = psta->uapsd_bk&BIT(1); + wmmps_ac = psta->uapsd_bk & BIT(1); break; case 4: case 5: - wmmps_ac = psta->uapsd_vi&BIT(1); + wmmps_ac = psta->uapsd_vi & BIT(1); break; case 6: case 7: - wmmps_ac = psta->uapsd_vo&BIT(1); + wmmps_ac = psta->uapsd_vo & BIT(1); break; case 0: case 3: default: - wmmps_ac = psta->uapsd_be&BIT(1); + wmmps_ac = psta->uapsd_be & BIT(1); break; } @@ -2259,7 +2268,7 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta) pstapriv->tim_bitmap &= ~BIT(psta->aid); - if (psta->state&WIFI_SLEEP_STATE) + if (psta->state & WIFI_SLEEP_STATE) psta->state ^= WIFI_SLEEP_STATE; if (psta->state & WIFI_STA_ALIVE_CHK_STATE) { @@ -2274,7 +2283,7 @@ void wakeup_sta_to_xmit(struct adapter *padapter, struct sta_info *psta) if (!psta_bmc) goto _exit; - if ((pstapriv->sta_dz_bitmap&0xfffe) == 0x0) { /* no any sta in ps mode */ + if ((pstapriv->sta_dz_bitmap & 0xfffe) == 0x0) { /* no any sta in ps mode */ xmitframe_phead = get_list_head(&psta_bmc->sleep_q); list_for_each_safe(xmitframe_plist, tmp, xmitframe_phead) { pxmitframe = list_entry(xmitframe_plist, @@ -2327,20 +2336,20 @@ void xmit_delivery_enabled_frames(struct adapter *padapter, struct sta_info *pst switch (pxmitframe->attrib.priority) { case 1: case 2: - wmmps_ac = psta->uapsd_bk&BIT(1); + wmmps_ac = psta->uapsd_bk & BIT(1); break; case 4: case 5: - wmmps_ac = psta->uapsd_vi&BIT(1); + wmmps_ac = psta->uapsd_vi & BIT(1); break; case 6: case 7: - wmmps_ac = psta->uapsd_vo&BIT(1); + wmmps_ac = psta->uapsd_vo & BIT(1); break; case 0: case 3: default: - wmmps_ac = psta->uapsd_be&BIT(1); + wmmps_ac = psta->uapsd_be & BIT(1); break; } diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c index 399edfbf8ec6d..ca3fa3e399c94 100644 --- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c +++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c @@ -382,7 +382,7 @@ s32 rtl8723bs_init_recv_priv(struct adapter *padapter) spin_lock_init(&precvpriv->recv_buf_pending_queue.lock); n = NR_RECVBUFF * sizeof(struct recv_buf) + 4; - precvpriv->pallocated_recv_buf = rtw_zmalloc(n); + precvpriv->pallocated_recv_buf = kzalloc(n, GFP_KERNEL); if (!precvpriv->pallocated_recv_buf) { res = _FAIL; goto exit; diff --git a/drivers/staging/rtl8723bs/hal/sdio_ops.c b/drivers/staging/rtl8723bs/hal/sdio_ops.c index 8736c124f8574..54228c41579ee 100644 --- a/drivers/staging/rtl8723bs/hal/sdio_ops.c +++ b/drivers/staging/rtl8723bs/hal/sdio_ops.c @@ -895,7 +895,7 @@ void sd_int_dpc(struct adapter *adapter) if (hal->sdio_hisr & SDIO_HISR_C2HCMD) { struct c2h_evt_hdr_88xx *c2h_evt; - c2h_evt = rtw_zmalloc(16); + c2h_evt = kzalloc(16, GFP_ATOMIC); if (c2h_evt) { if (c2h_evt_read_88xx(adapter, (u8 *)c2h_evt) == _SUCCESS) { if (c2h_id_filter_ccx_8723b((u8 *)c2h_evt)) { diff --git a/drivers/staging/rtl8723bs/include/basic_types.h b/drivers/staging/rtl8723bs/include/basic_types.h index 1c2da18e62100..16b270fe0203d 100644 --- a/drivers/staging/rtl8723bs/include/basic_types.h +++ b/drivers/staging/rtl8723bs/include/basic_types.h @@ -12,6 +12,7 @@ #define FAIL (-1) #include +#include #define FIELD_OFFSET(s, field) ((__kernel_ssize_t)&((s *)(0))->field) diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c index 73fc1d4f92e6c..06bf5d176e7d6 100644 --- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c +++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c @@ -111,6 +111,7 @@ static struct ieee80211_supported_band *rtw_spt_band_alloc( { struct ieee80211_supported_band *spt_band = NULL; int n_channels, n_bitrates; + size_t alloc_sz; if (band == NL80211_BAND_2GHZ) { n_channels = RTW_2G_CHANNELS_NUM; @@ -119,9 +120,10 @@ static struct ieee80211_supported_band *rtw_spt_band_alloc( goto exit; } - spt_band = rtw_zmalloc(sizeof(struct ieee80211_supported_band) + - sizeof(struct ieee80211_channel) * n_channels + - sizeof(struct ieee80211_rate) * n_bitrates); + alloc_sz = sizeof(*spt_band); + alloc_sz = size_add(alloc_sz, array_size(n_channels, sizeof(struct ieee80211_channel))); + alloc_sz = size_add(alloc_sz, array_size(n_bitrates, sizeof(struct ieee80211_rate))); + spt_band = kzalloc(alloc_sz, GFP_KERNEL); if (!spt_band) goto exit; @@ -841,11 +843,9 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev, struct mlme_priv *pmlmepriv = &padapter->mlmepriv; param_len = sizeof(struct ieee_param) + params->key_len; - param = rtw_malloc(param_len); + param = kzalloc(param_len, GFP_KERNEL); if (!param) - return -1; - - memset(param, 0, param_len); + return -ENOMEM; param->cmd = IEEE_CMD_SET_ENCRYPTION; eth_broadcast_addr(param->sta_addr); @@ -1431,7 +1431,7 @@ static int rtw_cfg80211_set_wpa_ie(struct adapter *padapter, u8 *pie, size_t iel goto exit; } - buf = rtw_zmalloc(ielen); + buf = kzalloc(ielen, GFP_KERNEL); if (!buf) { ret = -ENOMEM; goto exit; @@ -1721,15 +1721,14 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev, if (wep_key_len > 0) { wep_key_len = wep_key_len <= 5 ? 5 : 13; - wep_total_len = wep_key_len + FIELD_OFFSET(struct ndis_802_11_wep, key_material); - pwep = rtw_malloc(wep_total_len); + wep_total_len = wep_key_len + + offsetof(struct ndis_802_11_wep, key_material); + pwep = kzalloc(wep_total_len, GFP_KERNEL); if (!pwep) { ret = -ENOMEM; goto exit; } - memset(pwep, 0, wep_total_len); - pwep->key_length = wep_key_len; pwep->length = wep_total_len; @@ -1958,7 +1957,7 @@ static u8 rtw_get_chan_type(struct adapter *adapter) else return NL80211_CHAN_NO_HT; case CHANNEL_WIDTH_40: - if (mlme_ext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_UPPER) + if (mlme_ext->cur_ch_offset == HAL_PRIME_CHNL_OFFSET_LOWER) return NL80211_CHAN_HT40PLUS; else return NL80211_CHAN_HT40MINUS; @@ -2034,6 +2033,8 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc /* Skip the ratio tap header */ skb_pull(skb, rtap_len); + if (skb->len < dot11_hdr_len) + goto fail; dot11_hdr = (struct ieee80211_hdr *)skb->data; frame_control = le16_to_cpu(dot11_hdr->frame_control); @@ -2046,6 +2047,8 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc qos_len = 2; if ((frame_control & 0x0300) == 0x0300) dot11_hdr_len += 6; + if (skb->len < dot11_hdr_len + qos_len + snap_len) + goto fail; memcpy(dst_mac_addr, dot11_hdr->addr1, sizeof(dst_mac_addr)); memcpy(src_mac_addr, dot11_hdr->addr2, sizeof(src_mac_addr)); @@ -2155,7 +2158,7 @@ static int rtw_cfg80211_add_monitor_if(struct adapter *padapter, char *name, str pnpi->sizeof_priv = sizeof(struct adapter); /* wdev */ - mon_wdev = rtw_zmalloc(sizeof(struct wireless_dev)); + mon_wdev = kzalloc(sizeof(*mon_wdev), GFP_KERNEL); if (!mon_wdev) { ret = -ENOMEM; goto out; @@ -2265,7 +2268,7 @@ static int rtw_add_beacon(struct adapter *adapter, const u8 *head, size_t head_l if (head_len < 24) return -EINVAL; - pbuf = rtw_zmalloc(head_len + tail_len); + pbuf = kzalloc(head_len + tail_len, GFP_KERNEL); if (!pbuf) return -ENOMEM; @@ -2736,7 +2739,7 @@ int rtw_wdev_alloc(struct adapter *padapter, struct device *dev) goto free_wiphy; /* wdev */ - wdev = rtw_zmalloc(sizeof(struct wireless_dev)); + wdev = kzalloc(sizeof(*wdev), GFP_KERNEL); if (!wdev) { ret = -ENOMEM; goto unregister_wiphy; diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c index 6ca6dc5488057..9f7f6b40093c8 100644 --- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c +++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c @@ -560,7 +560,7 @@ struct dvobj_priv *devobj_init(void) { struct dvobj_priv *pdvobj = NULL; - pdvobj = rtw_zmalloc(sizeof(*pdvobj)); + pdvobj = kzalloc(sizeof(*pdvobj), GFP_KERNEL); if (!pdvobj) return NULL; diff --git a/drivers/staging/rtl8723bs/os_dep/osdep_service.c b/drivers/staging/rtl8723bs/os_dep/osdep_service.c index a00f9f0c85c5b..a9502ed7e000b 100644 --- a/drivers/staging/rtl8723bs/os_dep/osdep_service.c +++ b/drivers/staging/rtl8723bs/os_dep/osdep_service.c @@ -223,12 +223,9 @@ struct rtw_cbuf *rtw_cbuf_alloc(u32 size) { struct rtw_cbuf *cbuf; - cbuf = rtw_malloc(struct_size(cbuf, bufs, size)); - - if (cbuf) { - cbuf->write = cbuf->read = 0; + cbuf = kzalloc(struct_size(cbuf, bufs, size), GFP_KERNEL); + if (cbuf) cbuf->size = size; - } return cbuf; } diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c index 139ace51486d2..ae379e99736ad 100644 --- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c +++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c @@ -307,8 +307,8 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct status = _SUCCESS; free_hal_data: - if (status != _SUCCESS && padapter->HalData) - kfree(padapter->HalData); + if (status != _SUCCESS) + rtw_hal_data_deinit(padapter); if (status != _SUCCESS) { rtw_wdev_unregister(padapter->rtw_wdev); diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c index 944b9c724b32e..20d2926a4480a 100644 --- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c +++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c @@ -46,7 +46,7 @@ signed int rtw_endofpktfile(struct pkt_file *pfile) int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag) { if (alloc_sz > 0) { - pxmitbuf->pallocated_buf = rtw_zmalloc(alloc_sz); + pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL); if (!pxmitbuf->pallocated_buf) return _FAIL; diff --git a/drivers/staging/sm750fb/Kconfig b/drivers/staging/sm750fb/Kconfig index 08bcccdd0f1c4..25fe422f55f2c 100644 --- a/drivers/staging/sm750fb/Kconfig +++ b/drivers/staging/sm750fb/Kconfig @@ -6,6 +6,7 @@ config FB_SM750 select FB_CFB_FILLRECT select FB_CFB_COPYAREA select FB_CFB_IMAGEBLIT + select FB_IOMEM_FOPS help Frame buffer driver for the Silicon Motion SM750 chip with 2D acceleration and dual head support. diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c index 24e927a1669d6..676ef44577248 100644 --- a/drivers/staging/sm750fb/sm750.c +++ b/drivers/staging/sm750fb/sm750.c @@ -273,7 +273,7 @@ static void lynxfb_ops_imageblit(struct fb_info *info, spin_lock(&sm750_dev->slock); sm750_dev->accel.de_imageblit(&sm750_dev->accel, - image->data, image->width >> 3, 0, + image->data, 0, base, pitch, bpp, image->dx, image->dy, image->width, image->height, @@ -603,7 +603,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par) crtc = &par->crtc; crtc->vidmem_size = sm750_dev->vidmem_size; - if (sm750_dev->fb_count > 1) + if (g_dualview) crtc->vidmem_size >>= 1; /* setup crtc and output member */ @@ -968,7 +968,7 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src) NO_PARAM: if (sm750_dev->revid != SM750LE_REVISION_ID) { - if (sm750_dev->fb_count > 1) { + if (g_dualview) { if (swap) sm750_dev->dataflow = sm750_dual_swap; else diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h index fcb7d586ebf02..2845b8ec48aca 100644 --- a/drivers/staging/sm750fb/sm750.h +++ b/drivers/staging/sm750fb/sm750.h @@ -73,7 +73,7 @@ struct lynx_accel { u32 rop2); int (*de_imageblit)(struct lynx_accel *accel, const char *p_srcbuf, - u32 src_delta, u32 start_bit, u32 d_base, u32 d_pitch, + u32 start_bit, u32 d_base, u32 d_pitch, u32 byte_per_pixel, u32 dx, u32 dy, u32 width, u32 height, u32 f_color, u32 b_color, u32 rop2); diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c index b07c1aa68621c..47230896a5b0b 100644 --- a/drivers/staging/sm750fb/sm750_accel.c +++ b/drivers/staging/sm750fb/sm750_accel.c @@ -300,8 +300,6 @@ static unsigned int deGetTransparency(struct lynx_accel *accel) * sm750_hw_imageblit * @accel: Acceleration device data * @pSrcbuf: pointer to start of source buffer in system memory - * @srcDelta: Pitch value (in bytes) of the source buffer, +ive means top down - * and -ive mean button up * @startBit: Mono data can start at any bit in a byte, this value should be * 0 to 7 * @dBase: Address of destination: offset in frame buffer @@ -316,7 +314,7 @@ static unsigned int deGetTransparency(struct lynx_accel *accel) * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *pSrcbuf, - u32 srcDelta, u32 startBit, u32 dBase, u32 dPitch, + u32 startBit, u32 dBase, u32 dPitch, u32 bytePerPixel, u32 dx, u32 dy, u32 width, u32 height, u32 fColor, u32 bColor, u32 rop2) { @@ -405,7 +403,7 @@ int sm750_hw_imageblit(struct lynx_accel *accel, const char *pSrcbuf, write_dpPort(accel, *(unsigned int *)ajRemain); } - pSrcbuf += srcDelta; + pSrcbuf += ulBytesPerScan; } return 0; diff --git a/drivers/staging/sm750fb/sm750_accel.h b/drivers/staging/sm750fb/sm750_accel.h index 2c79cb730a0a6..df48ed57a054e 100644 --- a/drivers/staging/sm750fb/sm750_accel.h +++ b/drivers/staging/sm750fb/sm750_accel.h @@ -220,8 +220,6 @@ int sm750_hw_copyarea(struct lynx_accel *accel, /** * sm750_hw_imageblit * @pSrcbuf: pointer to start of source buffer in system memory - * @srcDelta: Pitch value (in bytes) of the source buffer, +ive means top down - *>----- and -ive mean button up * @startBit: Mono data can start at any bit in a byte, this value should be *>----- 0 to 7 * @dBase: Address of destination: offset in frame buffer @@ -236,7 +234,7 @@ int sm750_hw_copyarea(struct lynx_accel *accel, * @rop2: ROP value */ int sm750_hw_imageblit(struct lynx_accel *accel, const char *pSrcbuf, - u32 srcDelta, u32 startBit, u32 dBase, u32 dPitch, + u32 startBit, u32 dBase, u32 dPitch, u32 bytePerPixel, u32 dx, u32 dy, u32 width, u32 height, u32 fColor, u32 bColor, u32 rop2); diff --git a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c index 1a0638b2dc4db..1182cd6557975 100644 --- a/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c +++ b/drivers/staging/vc04_services/interface/vchiq_arm/vchiq_core.c @@ -1590,7 +1590,7 @@ create_pagelist(struct vchiq_instance *instance, struct vchiq_bulk *bulk) for (actual_pages = 0; actual_pages < num_pages; actual_pages++) { struct page *pg = - vmalloc_to_page(((unsigned int *)bulk->offset + + vmalloc_to_page(((void *)bulk->offset + (actual_pages * PAGE_SIZE))); size_t bytes = PAGE_SIZE - off; diff --git a/drivers/staging/vc04_services/vc-sm-cma/vc_sm.c b/drivers/staging/vc04_services/vc-sm-cma/vc_sm.c index b0aff3974f24f..3d4e2232e353e 100644 --- a/drivers/staging/vc04_services/vc-sm-cma/vc_sm.c +++ b/drivers/staging/vc04_services/vc-sm-cma/vc_sm.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -1200,8 +1201,10 @@ static int vc_sm_cma_ioctl_alloc(struct vc_sm_privdata_t *private, buffer->alloc.sg_table = sgt; fd = dma_buf_fd(dmabuf, O_CLOEXEC); - if (fd < 0) + if (fd < 0) { + ret = fd; goto error; + } vc_sm_add_resource(private, buffer); @@ -1276,9 +1279,9 @@ static long vc_sm_cma_ioctl(struct file *file, unsigned int cmd, if (!ret && (copy_to_user((void *)arg, &ioparam, sizeof(ioparam)) != 0)) { - /* FIXME: Release allocation */ pr_err("[%s]: failed to copy-to-user for cmd %x\n", __func__, cmdnr); + close_fd(ioparam.handle); ret = -EFAULT; } break; @@ -1312,11 +1315,12 @@ static long vc_sm_cma_ioctl(struct file *file, unsigned int cmd, ioparam.vc_handle = buf->vc_handle; ioparam.dma_addr = buf->dma_addr; - if (ioparam.handle < 0 || - (copy_to_user((void *)arg, &ioparam, - sizeof(ioparam)) != 0)) { + if (ioparam.handle < 0) { dma_buf_put(new_dmabuf); - /* FIXME: Release allocation */ + ret = -EFAULT; + } else if (copy_to_user((void *)arg, &ioparam, + sizeof(ioparam)) != 0) { + close_fd(ioparam.handle); ret = -EFAULT; } } diff --git a/drivers/staging/vc04_services/vchiq-mmal/mmal-vchiq.c b/drivers/staging/vc04_services/vchiq-mmal/mmal-vchiq.c index ec42af0ed45a0..245cfec3ef370 100644 --- a/drivers/staging/vc04_services/vchiq-mmal/mmal-vchiq.c +++ b/drivers/staging/vc04_services/vchiq-mmal/mmal-vchiq.c @@ -1349,6 +1349,9 @@ static int port_parameter_set(struct vchiq_mmal_instance *instance, struct mmal_msg *rmsg; struct vchiq_header *rmsg_handle; + if (value_size > sizeof(m.u.port_parameter_set.value)) + return -EINVAL; + m.h.type = MMAL_MSG_TYPE_PORT_PARAMETER_SET; m.u.port_parameter_set.component_handle = port->component->handle; @@ -1390,6 +1393,9 @@ static int port_parameter_get(struct vchiq_mmal_instance *instance, struct mmal_msg *rmsg; struct vchiq_header *rmsg_handle; + if (*value_size > sizeof(rmsg->u.port_parameter_get_reply.value)) + return -EINVAL; + m.h.type = MMAL_MSG_TYPE_PORT_PARAMETER_GET; m.u.port_parameter_get.component_handle = port->component->handle; diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 9d05d014d9bf7..ce90f3d6194c2 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -1519,8 +1519,10 @@ __iscsit_check_dataout_hdr(struct iscsit_conn *conn, void *buf, */ if (se_cmd->transport_state & CMD_T_ABORTED) { if (hdr->flags & ISCSI_FLAG_CMD_FINAL && - --cmd->outstanding_r2ts < 1) + --cmd->outstanding_r2ts < 1) { iscsit_stop_dataout_timer(cmd); + target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED); + } return iscsit_dump_data_payload(conn, payload_length, 1); } diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index c2ac9a99ebbb2..6108a9f3bdd54 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -47,7 +47,7 @@ static struct iscsi_login *iscsi_login_init_conn(struct iscsit_conn *conn) login->conn = conn; login->first_request = 1; - login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); + login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS + 1, GFP_KERNEL); if (!login->req_buf) { pr_err("Unable to allocate memory for response buffer.\n"); goto out_login; diff --git a/drivers/target/target_core_fabric_configfs.c b/drivers/target/target_core_fabric_configfs.c index 7156a4dc1ca7d..15172e993be7c 100644 --- a/drivers/target/target_core_fabric_configfs.c +++ b/drivers/target/target_core_fabric_configfs.c @@ -690,6 +690,14 @@ static void target_fabric_port_unlink( } core_dev_del_lun(se_tpg, lun); + + if (tf->tf_ops->fabric_post_unlink) { + /* + * Allow fabrics to release state that must remain valid until + * core_dev_del_lun() has drained all active LUN references. + */ + tf->tf_ops->fabric_post_unlink(se_tpg, lun); + } } static void target_fabric_port_release(struct config_item *item) diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 66c292b7d74bc..4ec322f371bc2 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -902,7 +902,7 @@ static sense_reason_t iblock_execute_pr_out(struct se_cmd *cmd, u8 sa, u64 key, break; case PRO_PREEMPT: case PRO_PREEMPT_AND_ABORT: - if (!ops->pr_clear) { + if (!ops->pr_preempt) { pr_err("block_device does not support pr_preempt.\n"); return TCM_UNSUPPORTED_SCSI_OPCODE; } @@ -912,8 +912,8 @@ static sense_reason_t iblock_execute_pr_out(struct se_cmd *cmd, u8 sa, u64 key, sa == PRO_PREEMPT_AND_ABORT); break; case PRO_RELEASE: - if (!ops->pr_clear) { - pr_err("block_device does not support pr_pclear.\n"); + if (!ops->pr_release) { + pr_err("block_device does not support pr_release.\n"); return TCM_UNSUPPORTED_SCSI_OPCODE; } diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 88544c911949c..5ad2dcf730cd2 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1691,6 +1691,7 @@ int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, u32 data_length, int task_attr, int data_dir, int flags) { struct se_portal_group *se_tpg; + int ret; se_tpg = se_sess->se_tpg; BUG_ON(!se_tpg); @@ -1720,7 +1721,11 @@ int target_init_cmd(struct se_cmd *se_cmd, struct se_session *se_sess, * necessary for fabrics using TARGET_SCF_ACK_KREF that expect a second * kref_put() to happen during fabric packet acknowledgement. */ - return target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); + ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); + if (ret) + se_cmd->cmd_cnt = NULL; + + return ret; } EXPORT_SYMBOL_GPL(target_init_cmd); @@ -1996,8 +2001,10 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, * allocation failure. */ ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp); - if (ret < 0) + if (ret < 0) { + se_cmd->cmd_cnt = NULL; return -ENOMEM; + } if (tm_type == TMR_ABORT_TASK) se_cmd->se_tmr_req->ref_task_tag = tag; @@ -2005,6 +2012,7 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, /* See target_submit_cmd for commentary */ ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF); if (ret) { + se_cmd->cmd_cnt = NULL; core_tmr_release_req(se_cmd->se_tmr_req); return ret; } diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c index 16eb953e14bb6..e422e77b06fb8 100644 --- a/drivers/tee/optee/call.c +++ b/drivers/tee/optee/call.c @@ -602,7 +602,8 @@ static bool is_normal_memory(pgprot_t p) return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) || ((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK)); #elif defined(CONFIG_ARM64) - return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL); + return ((pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL)) || + ((pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED)); #else #error "Unsupported architecture" #endif diff --git a/drivers/thermal/airoha_thermal.c b/drivers/thermal/airoha_thermal.c index b9fd6bfc88e5e..829a7327fc403 100644 --- a/drivers/thermal/airoha_thermal.c +++ b/drivers/thermal/airoha_thermal.c @@ -273,7 +273,7 @@ static int airoha_thermal_set_trips(struct thermal_zone_device *tz, int low, if (low != -INT_MAX) { /* Validate low and clamp it to a supported value */ - low = clamp_t(int, high, RAW_TO_TEMP(priv, 0), + low = clamp_t(int, low, RAW_TO_TEMP(priv, 0), RAW_TO_TEMP(priv, FIELD_MAX(EN7581_DOUT_TADC_MASK))); /* We offset the low temp of 1°C to trigger correct event */ @@ -403,7 +403,7 @@ static void airoha_thermal_setup_monitor(struct airoha_thermal_priv *priv) * sen interval is 379 * 52.715us = 19.97ms */ writel(FIELD_PREP(EN7581_FILT_INTERVAL, 1) | - FIELD_PREP(EN7581_FILT_INTERVAL, 379), + FIELD_PREP(EN7581_SEN_INTERVAL, 379), priv->base + EN7581_TEMPMONCTL2); /* AHB poll is set to 146 * 68.64 = 10.02us */ diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c index 576f88b6a1b35..cf1706569e6d7 100644 --- a/drivers/thermal/db8500_thermal.c +++ b/drivers/thermal/db8500_thermal.c @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -82,7 +82,7 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th, unsigned long next_low, unsigned long next_high) { - prcmu_stop_temp_sense(); + db8500_prcmu_stop_temp_sense(); th->cur_index = idx; th->interpolated_temp = (next_low + next_high)/2; @@ -91,8 +91,8 @@ static void db8500_thermal_update_config(struct db8500_thermal_zone *th, * The PRCMU accept absolute temperatures in celsius so divide * down the millicelsius with 1000 */ - prcmu_config_hotmon((u8)(next_low/1000), (u8)(next_high/1000)); - prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME); + db8500_prcmu_config_hotmon((u8)(next_low / 1000), (u8)(next_high / 1000)); + db8500_prcmu_start_temp_sense(PRCMU_DEFAULT_MEASURE_TIME); } static irqreturn_t prcmu_low_irq_handler(int irq, void *irq_data) @@ -204,7 +204,7 @@ static int db8500_thermal_probe(struct platform_device *pdev) static int db8500_thermal_suspend(struct platform_device *pdev, pm_message_t state) { - prcmu_stop_temp_sense(); + db8500_prcmu_stop_temp_sense(); return 0; } diff --git a/drivers/thermal/imx_thermal.c b/drivers/thermal/imx_thermal.c index 38c993d1bcb32..9b1e2f24a4386 100644 --- a/drivers/thermal/imx_thermal.c +++ b/drivers/thermal/imx_thermal.c @@ -832,12 +832,12 @@ static int imx_thermal_runtime_resume(struct device *dev) ret = regmap_write(map, socdata->sensor_ctrl + REG_CLR, socdata->power_down_mask); if (ret) - return ret; + goto disable_clk; ret = regmap_write(map, socdata->sensor_ctrl + REG_SET, socdata->measure_temp_mask); if (ret) - return ret; + goto disable_clk; /* * According to the temp sensor designers, it may require up to ~17us @@ -846,6 +846,11 @@ static int imx_thermal_runtime_resume(struct device *dev) usleep_range(20, 50); return 0; + +disable_clk: + clk_disable_unprepare(data->thermal_clk); + + return ret; } static const struct dev_pm_ops imx_thermal_pm_ops = { diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index 908cc1bf57f19..2aa03ceb570e7 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -354,8 +354,10 @@ static void cleanup_odvp(struct int3400_thermal_priv *priv) kfree(priv->odvp_attrs[i].attr.attr.name); } kfree(priv->odvp_attrs); + priv->odvp_attrs = NULL; } kfree(priv->odvp); + priv->odvp = NULL; priv->odvp_count = 0; } @@ -635,7 +637,6 @@ static int int3400_thermal_probe(struct platform_device *pdev) acpi_remove_notify_handler(priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify); free_sysfs: - cleanup_odvp(priv); if (!ZERO_OR_NULL_PTR(priv->data_vault)) { device_remove_bin_file(&pdev->dev, &bin_attr_data_vault); kfree(priv->data_vault); @@ -649,6 +650,7 @@ static int int3400_thermal_probe(struct platform_device *pdev) acpi_thermal_rel_misc_device_remove(priv->adev->handle); thermal_zone_device_unregister(priv->thermal); free_art_trt: + cleanup_odvp(priv); kfree(priv->trts); kfree(priv->arts); free_priv: diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c index bd2fca7dc0176..8a2f441cd2ecf 100644 --- a/drivers/thermal/intel/intel_hfi.c +++ b/drivers/thermal/intel/intel_hfi.c @@ -592,7 +592,7 @@ static void hfi_disable_instance(void *ptr) hfi_disable(); } -static void hfi_syscore_resume(void) +static void hfi_syscore_resume(void *data) { /* This code runs only on the boot CPU. */ struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, 0); @@ -603,7 +603,7 @@ static void hfi_syscore_resume(void) hfi_enable_instance(hfi_instance); } -static int hfi_syscore_suspend(void) +static int hfi_syscore_suspend(void *data) { /* No locking needed. There is no concurrency with CPU offline. */ hfi_disable(); @@ -611,11 +611,15 @@ static int hfi_syscore_suspend(void) return 0; } -static struct syscore_ops hfi_pm_ops = { +static const struct syscore_ops hfi_pm_ops = { .resume = hfi_syscore_resume, .suspend = hfi_syscore_suspend, }; +static struct syscore hfi_pm = { + .ops = &hfi_pm_ops, +}; + static int hfi_thermal_notify(struct notifier_block *nb, unsigned long state, void *_notify) { @@ -710,7 +714,7 @@ void __init intel_hfi_init(void) if (thermal_genl_register_notifier(&hfi_thermal_nb)) goto err_nl_notif; - register_syscore_ops(&hfi_pm_ops); + register_syscore(&hfi_pm); return; diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c index d7f2e6ca92c2c..d1b086737bcd2 100644 --- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c +++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c @@ -369,9 +369,6 @@ static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp) if (ret < 0) return ret; - if (ret != IIO_VAL_INT) - return -EINVAL; - return 0; } diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index a2422ebee8169..40f27c1a1cd59 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -316,9 +316,66 @@ static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s) } /** - * tsens_hw_to_mC - Return sign-extended temperature in mCelsius. + * tsens_read_temp - Retrieve temperature readings from the hardware. * @s: Pointer to sensor struct * @field: Index into regmap_field array pointing to temperature data + * @temp: temperature in deciCelsius to be read from hardware + * + * This function handles temperature returned in ADC code or deciCelsius + * depending on IP version. + * + * Return: 0 on success, a negative errno will be returned in error cases + */ +static int tsens_read_temp(const struct tsens_sensor *s, int field, int *temp) +{ + struct tsens_priv *priv = s->priv; + int temp_val[MAX_READ_RETRY] = {0}; + u32 status; + int ret; + u32 last_temp_mask = GENMASK(priv->fields[LAST_TEMP_0].msb, + priv->fields[LAST_TEMP_0].lsb); + u32 valid_bit = priv->rf[VALID_0] ? BIT(priv->fields[VALID_0].lsb) : 0; + + for (int i = 0; i < MAX_READ_RETRY; i++) { + ret = regmap_read(priv->tm_map, priv->fields[field].reg, &status); + if (ret) + return ret; + + /* VER_0 doesn't have a VALID bit */ + if (!valid_bit) { + *temp = status & last_temp_mask; + return 0; + } + + temp_val[i] = status & last_temp_mask; + + if (status & valid_bit) { + *temp = temp_val[i]; + return 0; + } + } + + /* + * As per the HW guidelines, if none of the attempts observe a + * valid sample, a stable fallback value must be returned. If the + * first and second samples match, the second value is returned; + * otherwise, if the second and third samples match, the third + * value is returned. + */ + if (temp_val[0] == temp_val[1]) + *temp = temp_val[1]; + else if (temp_val[1] == temp_val[2]) + *temp = temp_val[2]; + else + return -EAGAIN; + + return 0; +} + +/** + * tsens_hw_to_mC - Return sign-extended temperature in mCelsius. + * @s: Pointer to sensor struct + * @temp: temperature in milliCelsius to be read from hardware * * This function handles temperature returned in ADC code or deciCelsius * depending on IP version. @@ -326,20 +383,14 @@ static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s) * Return: Temperature in milliCelsius on success, a negative errno will * be returned in error cases */ -static int tsens_hw_to_mC(const struct tsens_sensor *s, int field) +static int tsens_hw_to_mC(const struct tsens_sensor *s, int temp) { struct tsens_priv *priv = s->priv; u32 resolution; - u32 temp = 0; - int ret; resolution = priv->fields[LAST_TEMP_0].msb - priv->fields[LAST_TEMP_0].lsb; - ret = regmap_field_read(priv->rf[field], &temp); - if (ret) - return ret; - /* Convert temperature from ADC code to milliCelsius */ if (priv->feat->adc) return code_to_degc(temp, s) * 1000; @@ -514,8 +565,10 @@ static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, &d->crit_irq_mask); if (ret) return ret; - - d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id); + ret = regmap_field_read(priv->rf[CRIT_THRESH_0 + hw_id], &d->crit_thresh); + if (ret) + return ret; + d->crit_thresh = tsens_hw_to_mC(s, d->crit_thresh); } else { /* No mask register on older TSENS */ d->up_irq_mask = 0; @@ -525,8 +578,16 @@ static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, d->crit_thresh = 0; } - d->up_thresh = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id); - d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id); + ret = regmap_field_read(priv->rf[UP_THRESH_0 + hw_id], &d->up_thresh); + if (ret) + return ret; + + d->up_thresh = tsens_hw_to_mC(s, d->up_thresh); + ret = regmap_field_read(priv->rf[LOW_THRESH_0 + hw_id], &d->low_thresh); + if (ret) + return ret; + + d->low_thresh = tsens_hw_to_mC(s, d->low_thresh); dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n", hw_id, __func__, @@ -750,33 +811,15 @@ static void tsens_disable_irq(struct tsens_priv *priv) int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp) { - struct tsens_priv *priv = s->priv; int hw_id = s->hw_id; u32 temp_idx = LAST_TEMP_0 + hw_id; - u32 valid_idx = VALID_0 + hw_id; - u32 valid; int ret; - /* VER_0 doesn't have VALID bit */ - if (tsens_version(priv) == VER_0) - goto get_temp; - - /* Valid bit is 0 for 6 AHB clock cycles. - * At 19.2MHz, 1 AHB clock is ~60ns. - * We should enter this loop very, very rarely. - * Wait 1 us since it's the min of poll_timeout macro. - * Old value was 400 ns. - */ - ret = regmap_field_read_poll_timeout(priv->rf[valid_idx], valid, - valid, 1, 20 * USEC_PER_MSEC); - if (ret) - return ret; - -get_temp: - /* Valid bit is set, OK to read the temperature */ - *temp = tsens_hw_to_mC(s, temp_idx); + ret = tsens_read_temp(s, temp_idx, temp); + if (!ret) + *temp = tsens_hw_to_mC(s, *temp); - return 0; + return ret; } int get_temp_common(const struct tsens_sensor *s, int *temp) diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index 2a7afa4c899b9..ab57ad88c3f7c 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -21,6 +21,7 @@ #define THRESHOLD_MIN_ADC_CODE 0x0 #define MAX_SENSORS 16 +#define MAX_READ_RETRY 3 #include #include diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 01b58be0dcc64..4368c0a83b302 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -366,11 +366,20 @@ static int qoriq_tmu_resume(struct device *dev) if (data->ver > TMU_VER1) { ret = regmap_clear_bits(data->regmap, REGS_TMR, TMR_CMD); if (ret) - return ret; + goto disable_clk; } /* Enable monitoring */ - return regmap_update_bits(data->regmap, REGS_TMR, TMR_ME, TMR_ME); + ret = regmap_update_bits(data->regmap, REGS_TMR, TMR_ME, TMR_ME); + if (ret) + goto disable_clk; + + return 0; + +disable_clk: + clk_disable_unprepare(data->clk); + + return ret; } static DEFINE_SIMPLE_DEV_PM_OPS(qoriq_tmu_pm_ops, diff --git a/drivers/thermal/renesas/rcar_thermal.c b/drivers/thermal/renesas/rcar_thermal.c index fdd7afdc4ff69..27adc64757f55 100644 --- a/drivers/thermal/renesas/rcar_thermal.c +++ b/drivers/thermal/renesas/rcar_thermal.c @@ -492,12 +492,6 @@ static int rcar_thermal_probe(struct platform_device *pdev) "rcar_thermal", trips, ARRAY_SIZE(trips), priv, &rcar_thermal_zone_ops, NULL, 0, idle); - - ret = thermal_zone_device_enable(priv->zone); - if (ret) { - thermal_zone_device_unregister(priv->zone); - priv->zone = ERR_PTR(ret); - } } if (IS_ERR(priv->zone)) { dev_err(dev, "can't register thermal zone\n"); @@ -506,11 +500,12 @@ static int rcar_thermal_probe(struct platform_device *pdev) goto error_unregister; } - if (chip->use_of_thermal) { + if (chip->use_of_thermal) ret = thermal_add_hwmon_sysfs(priv->zone); - if (ret) - goto error_unregister; - } + else + ret = thermal_zone_device_enable(priv->zone); + if (ret) + goto error_unregister; rcar_thermal_irq_enable(priv); diff --git a/drivers/thermal/tegra/soctherm.c b/drivers/thermal/tegra/soctherm.c index 5d26b52beaba7..9d3eb3be2db02 100644 --- a/drivers/thermal/tegra/soctherm.c +++ b/drivers/thermal/tegra/soctherm.c @@ -1700,9 +1700,9 @@ static void soctherm_init_hw_throt_cdev(struct platform_device *pdev) stc->init = true; } else { - tcd = thermal_of_cooling_device_register(np_stcc, - (char *)name, ts, - &throt_cooling_ops); + tcd = devm_thermal_of_cooling_device_register(dev, np_stcc, + (char *)name, ts, + &throt_cooling_ops); if (IS_ERR_OR_NULL(tcd)) { dev_err(dev, "throttle-cfg: %s: failed to register cooling device\n", diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index c68b27b2fb027..342a5eb85093c 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -212,7 +212,8 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz) if (new_hwmon_device) hwmon_device_unregister(hwmon->device); free_mem: - kfree(hwmon); + if (new_hwmon_device) + kfree(hwmon); return result; } diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c index d80612506a334..e7fb5a5865db5 100644 --- a/drivers/thermal/thermal_sysfs.c +++ b/drivers/thermal/thermal_sysfs.c @@ -401,8 +401,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) struct thermal_trip_attrs *trip_attrs = &td->trip_attrs; /* create trip type attribute */ - snprintf(trip_attrs->type.name, THERMAL_NAME_LENGTH, - "trip_point_%d_type", i); + scnprintf(trip_attrs->type.name, sizeof(trip_attrs->type.name), + "trip_point_%d_type", i); sysfs_attr_init(&trip_attrs->type.attr.attr); trip_attrs->type.attr.attr.name = trip_attrs->type.name; @@ -411,8 +411,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) attrs[i] = &trip_attrs->type.attr.attr; /* create trip temp attribute */ - snprintf(trip_attrs->temp.name, THERMAL_NAME_LENGTH, - "trip_point_%d_temp", i); + scnprintf(trip_attrs->temp.name, sizeof(trip_attrs->temp.name), + "trip_point_%d_temp", i); sysfs_attr_init(&trip_attrs->temp.attr.attr); trip_attrs->temp.attr.attr.name = trip_attrs->temp.name; @@ -424,8 +424,8 @@ static int create_trip_attrs(struct thermal_zone_device *tz) } attrs[i + tz->num_trips] = &trip_attrs->temp.attr.attr; - snprintf(trip_attrs->hyst.name, THERMAL_NAME_LENGTH, - "trip_point_%d_hyst", i); + scnprintf(trip_attrs->hyst.name, sizeof(trip_attrs->hyst.name), + "trip_point_%d_hyst", i); sysfs_attr_init(&trip_attrs->hyst.attr.attr); trip_attrs->hyst.attr.attr.name = trip_attrs->hyst.name; diff --git a/drivers/thunderbolt/eeprom.c b/drivers/thunderbolt/eeprom.c index 1af65fece495f..62228f1ae753f 100644 --- a/drivers/thunderbolt/eeprom.c +++ b/drivers/thunderbolt/eeprom.c @@ -394,9 +394,16 @@ static int tb_drom_parse_entry_port(struct tb_switch *sw, return -EIO; } port->link_nr = entry->link_nr; - if (entry->has_dual_link_port) + if (entry->has_dual_link_port) { + if (entry->dual_link_port_nr > sw->config.max_port_number) { + tb_sw_warn(sw, + "port entry has invalid dual link port number %u\n", + entry->dual_link_port_nr); + return -EIO; + } port->dual_link_port = &port->sw->ports[entry->dual_link_port_nr]; + } } return 0; } diff --git a/drivers/thunderbolt/icm.c b/drivers/thunderbolt/icm.c index f213d9174dc57..cafd91a443bc8 100644 --- a/drivers/thunderbolt/icm.c +++ b/drivers/thunderbolt/icm.c @@ -587,6 +587,11 @@ static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, struct icm_fr_pkg_approve_xdomain request; int ret; + if (atomic_read(&xd->ntunnels) >= 1) { + tb_warn(tb, "only one tunnel is supported by the firmware\n"); + return -EOPNOTSUPP; + } + memset(&request, 0, sizeof(request)); request.hdr.code = ICM_APPROVE_XDOMAIN; request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link; @@ -1157,6 +1162,11 @@ static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd, struct icm_tr_pkg_approve_xdomain request; int ret; + if (atomic_read(&xd->ntunnels) >= 1) { + tb_warn(tb, "only one tunnel is supported by the firmware\n"); + return -EOPNOTSUPP; + } + memset(&request, 0, sizeof(request)); request.hdr.code = ICM_APPROVE_XDOMAIN; request.route_hi = upper_32_bits(xd->route); @@ -2325,7 +2335,7 @@ static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata, if (tx_data_len) { request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID; if (tx_data_len < ARRAY_SIZE(request.data)) - request.data_len_valid = + request.data_len_valid |= tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK; memcpy(request.data, tx_data, tx_data_len * sizeof(u32)); } diff --git a/drivers/thunderbolt/path.c b/drivers/thunderbolt/path.c index f9b11dadfbdd5..d8e547286127a 100644 --- a/drivers/thunderbolt/path.c +++ b/drivers/thunderbolt/path.c @@ -426,7 +426,8 @@ static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index, * in the USB4 spec so we clear them * only for pre-USB4 adapters. */ - if (!tb_switch_is_usb4(port->sw)) { + if (tb_port_is_null(port) || + !tb_switch_is_usb4(port->sw)) { hop.ingress_fc = 0; hop.ingress_shared_buffer = 0; } @@ -546,15 +547,18 @@ int tb_path_activate(struct tb_path *path) __tb_path_deactivate_hop(path->hops[i].in_port, path->hops[i].in_hop_index, path->clear_fc); - /* dword 0 */ + /* Needed for USB4 routers, read path config space before write */ + res = tb_port_read(path->hops[i].in_port, &hop, TB_CFG_HOPS, + 2 * path->hops[i].in_hop_index, 2); + if (res) + goto err; + hop.next_hop = path->hops[i].next_hop_index; hop.out_port = path->hops[i].out_port->port; - hop.initial_credits = path->hops[i].initial_credits; hop.pmps = path->hops[i].pm_support; hop.unknown1 = 0; hop.enable = 1; - /* dword 1 */ out_mask = (i == path->path_length - 1) ? TB_PATH_DESTINATION : TB_PATH_INTERNAL; in_mask = (i == 0) ? TB_PATH_SOURCE : TB_PATH_INTERNAL; @@ -564,12 +568,21 @@ int tb_path_activate(struct tb_path *path) hop.drop_packages = path->drop_packages; hop.counter = path->hops[i].in_counter_index; hop.counter_enable = path->hops[i].in_counter_index != -1; - hop.ingress_fc = path->ingress_fc_enable & in_mask; hop.egress_fc = path->egress_fc_enable & out_mask; - hop.ingress_shared_buffer = path->ingress_shared_buffer - & in_mask; - hop.egress_shared_buffer = path->egress_shared_buffer - & out_mask; + hop.egress_shared_buffer = path->egress_shared_buffer & out_mask; + /* + * Protocol adapters IFC and ISE bits, and Path Credits + * Allocated are vendor defined in the USB4 spec so we + * program them only for pre-USB4 and lane adapters. + */ + if (tb_port_is_null(path->hops[i].in_port) || + !tb_switch_is_usb4(path->hops[i].in_port->sw)) { + hop.initial_credits = path->hops[i].initial_credits; + hop.ingress_fc = path->ingress_fc_enable & in_mask; + hop.ingress_shared_buffer = + path->ingress_shared_buffer & in_mask; + } + hop.unknown3 = 0; tb_port_dbg(path->hops[i].in_port, "Writing hop %d\n", i); diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c index 0e07904aa73b4..a99d2db220802 100644 --- a/drivers/thunderbolt/switch.c +++ b/drivers/thunderbolt/switch.c @@ -1601,6 +1601,12 @@ static int tb_switch_reset_host(struct tb_switch *sw) ret = tb_port_reset(port); if (ret) return ret; + /* + * USB4 Lane 1 adapters do not have accessible + * path config space. + */ + if (tb_switch_is_usb4(sw) && !port->usb4) + continue; } else if (tb_port_is_usb3_down(port) || tb_port_is_usb3_up(port)) { tb_usb3_port_enable(port, false); diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c index 4a94cb406bdfb..205985eb197ec 100644 --- a/drivers/thunderbolt/tb.c +++ b/drivers/thunderbolt/tb.c @@ -98,7 +98,7 @@ static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug) if (!ev) return; - ev->tb = tb; + ev->tb = tb_domain_get(tb); ev->route = route; ev->port = port; ev->unplug = unplug; @@ -609,7 +609,7 @@ static int tb_consumed_dp_bandwidth(struct tb *tb, int *consumed_up, int *consumed_down) { - int group_reserved[MAX_GROUPS] = {}; + int group_reserved[MAX_GROUPS + 1] = {}; struct tb_cm *tcm = tb_priv(tb); struct tb_tunnel *tunnel; bool downstream; @@ -2527,6 +2527,9 @@ static void tb_handle_hotplug(struct work_struct *work) pm_runtime_mark_last_busy(&tb->dev); pm_runtime_put_autosuspend(&tb->dev); + /* Undo the refcount increased in tb_queue_hotplug() */ + tb_domain_put(tb); + kfree(ev); } @@ -2844,6 +2847,9 @@ static void tb_handle_dp_bandwidth_request(struct work_struct *work) /* Update other clients about the allocation change */ tb_recalc_estimated_bandwidth(tb); + + tb_dbg(tb, "checking if more DP tunnels can be established now\n"); + tb_tunnel_dp(tb); } put_sw: @@ -2949,6 +2955,7 @@ static void tb_stop(struct tb *tb) tb_tunnel_put(tunnel); } tb_switch_remove(tb->root_switch); + tb->root_switch = NULL; tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */ } diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h index 8e2762ff8d517..775ed48cc299a 100644 --- a/drivers/thunderbolt/tb.h +++ b/drivers/thunderbolt/tb.h @@ -1478,6 +1478,7 @@ int usb4_dp_port_allocate_bandwidth(struct tb_port *port, int bw); int usb4_dp_port_requested_bandwidth(struct tb_port *port); int usb4_pci_port_set_ext_encapsulation(struct tb_port *port, bool enable); +int usb4_pci_port_ltssm_state(struct tb_port *port); static inline bool tb_is_usb4_port_device(const struct device *dev) { diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h index 4e43b47f9f119..7f2dcb309e928 100644 --- a/drivers/thunderbolt/tb_regs.h +++ b/drivers/thunderbolt/tb_regs.h @@ -216,6 +216,7 @@ struct tb_regs_switch_header { #define ROUTER_CS_6_WOPS BIT(2) #define ROUTER_CS_6_WOUS BIT(3) #define ROUTER_CS_6_HCI BIT(18) +#define ROUTER_CS_6_RR BIT(24) #define ROUTER_CS_6_CR BIT(25) #define ROUTER_CS_7 0x07 #define ROUTER_CS_9 0x09 @@ -473,10 +474,25 @@ struct tb_regs_port_header { /* PCIe adapter registers */ #define ADP_PCIE_CS_0 0x00 +#define ADP_PCIE_CS_0_LTSSM_MASK GENMASK(28, 25) #define ADP_PCIE_CS_0_PE BIT(31) #define ADP_PCIE_CS_1 0x01 #define ADP_PCIE_CS_1_EE BIT(0) +enum tb_pcie_ltssm_state { + USB4_PCIE_LTSSM_DETECT, + USB4_PCIE_LTSSM_POLLING, + USB4_PCIE_LTSSM_CONFIG, + USB4_PCIE_LTSSM_CONFIG_IDLE, + USB4_PCIE_LTSSM_RECOVERY, + USB4_PCIE_LTSSM_RECOVERY_IDLE, + USB4_PCIE_LTSSM_L0, + USB4_PCIE_LTSSM_L1, + USB4_PCIE_LTSSM_L2, + USB4_PCIE_LTSSM_DISABLED, + USB4_PCIE_LTSSM_HOT_RESET, +}; + /* USB adapter registers */ #define ADP_USB3_CS_0 0x00 #define ADP_USB3_CS_0_V BIT(30) diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c index bfa0607b55744..6066355388b7f 100644 --- a/drivers/thunderbolt/tunnel.c +++ b/drivers/thunderbolt/tunnel.c @@ -296,6 +296,40 @@ static inline void tb_tunnel_changed(struct tb_tunnel *tunnel) tunnel->src_port, tunnel->dst_port); } +static int tb_pci_port_ltssm_state_detect(struct tb_port *port) +{ + ktime_t timeout = ktime_add_ms(ktime_get(), 500); + + do { + int ret; + + ret = usb4_pci_port_ltssm_state(port); + if (ret < 0) + return ret; + if (ret == USB4_PCIE_LTSSM_DETECT) + return 0; + + fsleep(50); + } while (ktime_before(ktime_get(), timeout)); + + return -ETIMEDOUT; +} + +static int tb_pci_pre_activate(struct tb_tunnel *tunnel) +{ + struct tb_port *down = tunnel->src_port; + struct tb_port *up = tunnel->dst_port; + int ret; + + ret = tb_switch_is_usb4(down->sw) ? + tb_pci_port_ltssm_state_detect(down) : 0; + if (ret) + return ret; + + return tb_switch_is_usb4(up->sw) ? + tb_pci_port_ltssm_state_detect(up) : 0; +} + static int tb_pci_set_ext_encapsulation(struct tb_tunnel *tunnel, bool enable) { struct tb_port *port = tb_upstream_port(tunnel->dst_port->sw); @@ -511,6 +545,7 @@ struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up, if (!tunnel) return NULL; + tunnel->pre_activate = tb_pci_pre_activate; tunnel->activate = tb_pci_activate; tunnel->src_port = down; tunnel->dst_port = up; diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c index 76f01713a8754..6383fd43203d7 100644 --- a/drivers/thunderbolt/usb4.c +++ b/drivers/thunderbolt/usb4.c @@ -294,7 +294,12 @@ int usb4_switch_setup(struct tb_switch *sw) /* TBT3 supported by the CM */ val &= ~ROUTER_CS_5_CNS; - return tb_sw_write(sw, &val, TB_CFG_SWITCH, ROUTER_CS_5, 1); + ret = tb_sw_write(sw, &val, TB_CFG_SWITCH, ROUTER_CS_5, 1); + if (ret) + return ret; + + return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_RR, + ROUTER_CS_6_RR, 500); } /** @@ -304,7 +309,7 @@ int usb4_switch_setup(struct tb_switch *sw) * Sets configuration valid bit for the router. Must be called before * any tunnels can be set through the router and after * usb4_switch_setup() has been called. Can be called to host and device - * routers (does nothing for the latter). + * routers (does nothing for the former). * * Return: %0 on success, negative errno otherwise. */ @@ -327,7 +332,7 @@ int usb4_switch_configuration_valid(struct tb_switch *sw) return ret; return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_CR, - ROUTER_CS_6_CR, 50); + ROUTER_CS_6_CR, 500); } /** @@ -3145,3 +3150,27 @@ int usb4_pci_port_set_ext_encapsulation(struct tb_port *port, bool enable) return tb_port_write(port, &val, TB_CFG_PORT, port->cap_adap + ADP_PCIE_CS_1, 1); } + +/** + * usb4_pci_port_ltssm_state() - Read PCIe adapter LTSSM state + * @port: PCIe adapter + * + * Return: + * * LTSSM state of @port. + * * Negative errno - On failure. + */ +int usb4_pci_port_ltssm_state(struct tb_port *port) +{ + u32 val; + int ret; + + if (!tb_port_is_pcie_down(port) && !tb_port_is_pcie_up(port)) + return -EINVAL; + + ret = tb_port_read(port, &val, TB_CFG_PORT, + port->cap_adap + ADP_PCIE_CS_0, 1); + if (ret) + return ret; + + return FIELD_GET(ADP_PCIE_CS_0_LTSSM_MASK, val); +} diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c index 458476907eab0..e2c46366c8160 100644 --- a/drivers/thunderbolt/xdomain.c +++ b/drivers/thunderbolt/xdomain.c @@ -139,6 +139,7 @@ static int __tb_xdomain_response(struct tb_ctl *ctl, const void *response, size_t size, enum tb_cfg_pkg_type type) { struct tb_cfg_request *req; + int ret; req = tb_cfg_request_alloc(); if (!req) @@ -150,7 +151,11 @@ static int __tb_xdomain_response(struct tb_ctl *ctl, const void *response, req->request_size = size; req->request_type = type; - return tb_cfg_request(ctl, req, response_ready, req); + ret = tb_cfg_request(ctl, req, response_ready, req); + if (ret) + tb_cfg_request_put(req); + + return ret; } /** @@ -751,7 +756,7 @@ static void tb_xdp_handle_request(struct work_struct *work) mutex_lock(&tb->lock); if (tb->root_switch) - uuid = tb->root_switch->uuid; + uuid = kmemdup(tb->root_switch->uuid, sizeof(*uuid), GFP_KERNEL); else uuid = NULL; mutex_unlock(&tb->lock); @@ -785,9 +790,13 @@ static void tb_xdp_handle_request(struct work_struct *work) * the xdomain related to this connection as well in * case there is a change in services it offers. */ - if (xd && device_is_registered(&xd->dev)) - queue_delayed_work(tb->wq, &xd->state_work, - msecs_to_jiffies(XDOMAIN_SHORT_TIMEOUT)); + if (xd) { + mutex_lock(&xd->lock); + if (!xd->removing && device_is_registered(&xd->dev)) + queue_delayed_work(tb->wq, &xd->state_work, + msecs_to_jiffies(XDOMAIN_SHORT_TIMEOUT)); + mutex_unlock(&xd->lock); + } break; case UUID_REQUEST_OLD: @@ -800,8 +809,12 @@ static void tb_xdp_handle_request(struct work_struct *work) * received UUID request from the remote host. */ if (!ret && xd && xd->state == XDOMAIN_STATE_ERROR) { - dev_dbg(&xd->dev, "restarting handshake\n"); - start_handshake(xd); + mutex_lock(&xd->lock); + if (!xd->removing) { + dev_dbg(&xd->dev, "restarting handshake\n"); + start_handshake(xd); + } + mutex_unlock(&xd->lock); } break; @@ -829,9 +842,13 @@ static void tb_xdp_handle_request(struct work_struct *work) ret = tb_xdp_link_state_change_response(ctl, route, sequence, 0); - xd->target_link_width = lsc->tlw; - queue_delayed_work(tb->wq, &xd->state_work, - msecs_to_jiffies(XDOMAIN_SHORT_TIMEOUT)); + mutex_lock(&xd->lock); + if (!xd->removing) { + xd->target_link_width = lsc->tlw; + queue_delayed_work(tb->wq, &xd->state_work, + msecs_to_jiffies(XDOMAIN_SHORT_TIMEOUT)); + } + mutex_unlock(&xd->lock); } else { tb_xdp_error_response(ctl, route, sequence, ERROR_NOT_READY); @@ -853,6 +870,7 @@ static void tb_xdp_handle_request(struct work_struct *work) } out: + kfree(uuid); kfree(xw->pkg); kfree(xw); @@ -1012,6 +1030,7 @@ static void tb_service_release(struct device *dev) ida_free(&xd->service_ids, svc->id); kfree(svc->key); kfree(svc); + tb_xdomain_put(xd); } const struct device_type tb_service_type = { @@ -1120,7 +1139,7 @@ static void enumerate_services(struct tb_xdomain *xd) svc->id = id; svc->dev.bus = &tb_bus_type; svc->dev.type = &tb_service_type; - svc->dev.parent = &xd->dev; + svc->dev.parent = get_device(&xd->dev); dev_set_name(&svc->dev, "%s.%d", dev_name(&xd->dev), svc->id); tb_service_debugfs_init(svc); @@ -1939,7 +1958,13 @@ static void tb_xdomain_link_exit(struct tb_xdomain *xd) if (tb_port_get_link_generation(down) >= 4) { down->bonded = false; down->dual_link_port->bonded = false; - } else if (xd->link_width > TB_LINK_WIDTH_SINGLE) { + return; + } + + if (!xd->bonding_possible) + return; + + if (xd->link_width > TB_LINK_WIDTH_SINGLE) { /* * Just return port structures back to way they were and * update credits. No need to update userspace because @@ -1996,6 +2021,7 @@ struct tb_xdomain *tb_xdomain_alloc(struct tb *tb, struct device *parent, INIT_DELAYED_WORK(&xd->state_work, tb_xdomain_state_work); INIT_DELAYED_WORK(&xd->properties_changed_work, tb_xdomain_properties_changed); + atomic_set(&xd->ntunnels, 0); xd->local_uuid = kmemdup(local_uuid, sizeof(uuid_t), GFP_KERNEL); if (!xd->local_uuid) @@ -2074,6 +2100,10 @@ void tb_xdomain_remove(struct tb_xdomain *xd) { tb_xdomain_debugfs_remove(xd); + mutex_lock(&xd->lock); + xd->removing = true; + mutex_unlock(&xd->lock); + stop_handshake(xd); device_for_each_child_reverse(&xd->dev, xd, unregister_service); @@ -2273,9 +2303,15 @@ int tb_xdomain_enable_paths(struct tb_xdomain *xd, int transmit_path, int transmit_ring, int receive_path, int receive_ring) { - return tb_domain_approve_xdomain_paths(xd->tb, xd, transmit_path, - transmit_ring, receive_path, - receive_ring); + int ret; + + ret = tb_domain_approve_xdomain_paths(xd->tb, xd, transmit_path, + transmit_ring, receive_path, + receive_ring); + if (ret) + return ret; + atomic_inc(&xd->ntunnels); + return 0; } EXPORT_SYMBOL_GPL(tb_xdomain_enable_paths); @@ -2298,9 +2334,15 @@ int tb_xdomain_disable_paths(struct tb_xdomain *xd, int transmit_path, int transmit_ring, int receive_path, int receive_ring) { - return tb_domain_disconnect_xdomain_paths(xd->tb, xd, transmit_path, - transmit_ring, receive_path, - receive_ring); + int ret; + + ret = tb_domain_disconnect_xdomain_paths(xd->tb, xd, transmit_path, + transmit_ring, receive_path, + receive_ring); + if (ret) + return ret; + atomic_dec(&xd->ntunnels); + return 0; } EXPORT_SYMBOL_GPL(tb_xdomain_disable_paths); @@ -2316,6 +2358,9 @@ static struct tb_xdomain *switch_find_xdomain(struct tb_switch *sw, { struct tb_port *port; + if (!sw) + return NULL; + tb_switch_for_each_port(sw, port) { struct tb_xdomain *xd; @@ -2484,8 +2529,12 @@ static int update_xdomain(struct device *dev, void *data) xd = tb_to_xdomain(dev); if (xd) { - queue_delayed_work(xd->tb->wq, &xd->properties_changed_work, - msecs_to_jiffies(50)); + mutex_lock(&xd->lock); + if (!xd->removing) + queue_delayed_work(xd->tb->wq, + &xd->properties_changed_work, + msecs_to_jiffies(50)); + mutex_unlock(&xd->lock); } return 0; diff --git a/drivers/tty/hvc/Kconfig b/drivers/tty/hvc/Kconfig index c2a4e88b328f3..5866195de26a6 100644 --- a/drivers/tty/hvc/Kconfig +++ b/drivers/tty/hvc/Kconfig @@ -79,7 +79,7 @@ config HVC_UDBG config HVC_DCC bool "ARM JTAG DCC console" - depends on ARM || ARM64 + depends on (ARM && (CPU_V6 || CPU_V6K || CPU_V7)) || ARM64 select HVC_DRIVER select SERIAL_CORE_CONSOLE help diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index fa0720a6bd111..c83f3ba6cd6e1 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -704,6 +704,7 @@ static void serial_8250_overrun_backoff_work(struct work_struct *work) int serial8250_register_8250_port(const struct uart_8250_port *up) { struct uart_8250_port *uart; + bool cons_flow; int ret; if (up->port.uartclk == 0) @@ -727,6 +728,9 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) if (uart->port.type == PORT_8250_CIR) return -ENODEV; + /* Preserve specified console flow control. */ + cons_flow = uart_cons_flow_enabled(&uart->port); + if (uart->port.dev) uart_remove_one_port(&serial8250_reg, &uart->port); @@ -757,6 +761,8 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) uart->lsr_save_mask = up->lsr_save_mask; uart->dma = up->dma; + uart_set_cons_flow_enabled(&uart->port, uart_cons_flow_enabled(&up->port) | cons_flow); + /* Take tx_loadsz from fifosize if it wasn't set separately */ if (uart->port.fifosize && !uart->tx_loadsz) uart->tx_loadsz = uart->port.fifosize; @@ -775,7 +781,7 @@ int serial8250_register_8250_port(const struct uart_8250_port *up) * Only call mctrl_gpio_init(), if the device has no ACPI * companion device */ - if (!has_acpi_companion(uart->port.dev)) { + if (uart->port.dev && !has_acpi_companion(uart->port.dev)) { struct mctrl_gpios *gpios = mctrl_gpio_init(&uart->port, 0); if (IS_ERR(gpios)) { ret = PTR_ERR(gpios); diff --git a/drivers/tty/serial/8250/8250_dma.c b/drivers/tty/serial/8250/8250_dma.c index 3b6452e759d5b..5a83e5269b415 100644 --- a/drivers/tty/serial/8250/8250_dma.c +++ b/drivers/tty/serial/8250/8250_dma.c @@ -211,11 +211,12 @@ void serial8250_rx_dma_flush(struct uart_8250_port *p) { struct uart_8250_dma *dma = p->dma; - if (dma->rx_running) { - dmaengine_pause(dma->rxchan); - __dma_rx_complete(p); - dmaengine_terminate_async(dma->rxchan); - } + if (!dma || !dma->rxchan || !dma->rx_running) + return; + + dmaengine_pause(dma->rxchan); + __dma_rx_complete(p); + dmaengine_terminate_async(dma->rxchan); } EXPORT_SYMBOL_GPL(serial8250_rx_dma_flush); @@ -324,6 +325,7 @@ void serial8250_release_dma(struct uart_8250_port *p) /* Release RX resources */ dmaengine_terminate_sync(dma->rxchan); + dma->rx_running = 0; dma_free_coherent(dma->rxchan->device->dev, dma->rx_size, dma->rx_buf, dma->rx_addr); dma_release_channel(dma->rxchan); diff --git a/drivers/tty/serial/8250/8250_mid.c b/drivers/tty/serial/8250/8250_mid.c index f88809ff370b7..82656645b8a64 100644 --- a/drivers/tty/serial/8250/8250_mid.c +++ b/drivers/tty/serial/8250/8250_mid.c @@ -318,9 +318,11 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!uart.port.membase) return -ENOMEM; - ret = mid->board->setup(mid, &uart.port); - if (ret) - return ret; + if (mid->board->setup) { + ret = mid->board->setup(mid, &uart.port); + if (ret) + return ret; + } ret = mid8250_dma_setup(mid, &uart); if (ret) @@ -336,7 +338,8 @@ static int mid8250_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err: - mid->board->exit(mid); + if (mid->board->exit) + mid->board->exit(mid); return ret; } @@ -346,7 +349,8 @@ static void mid8250_remove(struct pci_dev *pdev) serial8250_unregister_port(mid->line); - mid->board->exit(mid); + if (mid->board->exit) + mid->board->exit(mid); } static const struct mid8250_board pnw_board = { diff --git a/drivers/tty/serial/8250/8250_of.c b/drivers/tty/serial/8250/8250_of.c index d178b6c54ea18..58e52c0e066d2 100644 --- a/drivers/tty/serial/8250/8250_of.c +++ b/drivers/tty/serial/8250/8250_of.c @@ -81,6 +81,40 @@ static int of_platform_serial_clk_notifier_cb(struct notifier_block *nb, unsigne return NOTIFY_DONE; } +static int lpc32xx_handle_irq(struct uart_port *port) +{ + struct uart_8250_port *up = up_to_u8250p(port); + unsigned int iir; + u16 status; + + guard(serial8250_rpm)(up); + + iir = serial_port_in(port, UART_IIR); + if (iir & UART_IIR_NO_INT) + return 0; + + guard(uart_port_lock_check_sysrq_irqsave)(port); + + /* + * The LPC32xx UART can assert an RX character-timeout interrupt while + * the RX FIFO is empty: IIR reports UART_IIR_RX_TIMEOUT but LSR.DR is + * clear. The timeout is only cleared by reading RHR, but the core RX + * path skips that read when the FIFO is empty, so the level-triggered + * IRQ re-fires forever and livelocks this single-core SoC. Do one + * throwaway RHR read to clear it; a healthy UART never reports a + * timeout with DR/BI clear, so no received data is ever discarded. + */ + if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) { + status = serial_lsr_in(up); + if (!(status & (UART_LSR_DR | UART_LSR_BI))) + serial_port_in(port, UART_RX); + } + + serial8250_handle_irq_locked(port, iir); + + return 1; +} + /* * Fill a struct uart_port for a given device node */ @@ -173,6 +207,9 @@ static int of_platform_serial_setup(struct platform_device *ofdev, case PORT_NPCM: ret = npcm_setup(port); break; + case PORT_LPC3220: + port->handle_irq = lpc32xx_handle_irq; + break; default: /* Nothing to do */ ret = 0; @@ -369,6 +406,7 @@ static struct platform_driver of_platform_serial_driver = { module_platform_driver(of_platform_serial_driver); +MODULE_IMPORT_NS("SERIAL_8250"); MODULE_AUTHOR("Arnd Bergmann "); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Serial Port driver for Open Firmware platform devices"); diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c index f28a0ffa30057..3239fdcd35b0f 100644 --- a/drivers/tty/serial/8250/8250_port.c +++ b/drivers/tty/serial/8250/8250_port.c @@ -458,6 +458,8 @@ static void set_io_from_upio(struct uart_port *p) p->serial_out = io_serial_out; break; #endif + case UPIO_AU: + break; default: WARN(p->iotype != UPIO_PORT || p->iobase, "Unsupported UART type %x\n", p->iotype); @@ -1810,6 +1812,13 @@ void serial8250_handle_irq_locked(struct uart_port *port, unsigned int iir) status = serial_lsr_in(up); + /* + * Recover from no-data-ready and FIFO error condition to avoid getting + * stuck in the ISR. + */ + if (!(status & UART_LSR_DR) && (status & UART_LSR_FIFOE)) + serial8250_clear_and_reinit_fifos(up); + /* * If port is stopped and there are no error conditions in the * FIFO, then don't drain the FIFO, as this may lead to TTY buffer @@ -2000,8 +2009,14 @@ static void wait_for_xmitr(struct uart_8250_port *up, int bits) wait_for_lsr(up, bits); - /* Wait up to 1s for flow control if necessary */ - if (up->port.flags & UPF_CONS_FLOW) { + /* + * Wait up to 1s for flow control if necessary. + * When 'no_console_suspend' is active (in the window between + * suspend() and resume()), flow control is temporarily ignored + * because the canary workaround is not reliable in all situations, + * leading to flow control timeouts for every character. + */ + if (uart_console_hwflow_active(&up->port) && !up->canary) { for (tmout = 1000000; tmout; tmout--) { unsigned int msr = serial_in(up, UART_MSR); up->msr_saved_flags |= msr & MSR_SAVE_FLAGS; @@ -2795,6 +2810,12 @@ serial8250_do_set_termios(struct uart_port *port, struct ktermios *termios, serial8250_set_efr(port, termios); serial8250_set_divisor(port, baud, quot, frac); serial8250_set_fcr(port, termios); + /* Consoles manually poll CTS for hardware flow control. */ + if (uart_console(port) && + !(port->rs485.flags & SER_RS485_ENABLED) + && termios->c_cflag & CRTSCTS) { + port->mctrl |= TIOCM_RTS; + } serial8250_set_mctrl(port, port->mctrl); } @@ -3364,7 +3385,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s, * it regardless of the CTS state. Therefore, only use fifo * if we don't use control flow. */ - !(up->port.flags & UPF_CONS_FLOW); + !uart_console_hwflow_active(&up->port); if (likely(use_fifo)) serial8250_console_fifo_write(up, s, count); @@ -3434,6 +3455,9 @@ int serial8250_console_setup(struct uart_port *port, char *options, bool probe) if (ret) return ret; + /* Track user-specified console flow control. */ + uart_set_cons_flow_enabled(port, flow == 'r'); + if (port->dev) pm_runtime_get_sync(port->dev); diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 7068245dcbfd9..167ae0c0056ea 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -267,6 +267,7 @@ enum pl011_rs485_tx_state { WAIT_AFTER_RTS, SEND, WAIT_AFTER_SEND, + WAIT_AFTER_SEND_DELAY, }; /* @@ -1202,7 +1203,7 @@ static void pl011_dma_shutdown(struct uart_amba_port *uap) if (uap->using_tx_dma) { /* In theory, this should already be done by pl011_dma_flush_buffer */ - dmaengine_terminate_all(uap->dmatx.chan); + dmaengine_terminate_sync(uap->dmatx.chan); if (uap->dmatx.queued) { dma_unmap_single(uap->dmatx.chan->device->dev, uap->dmatx.dma, uap->dmatx.len, @@ -1215,12 +1216,12 @@ static void pl011_dma_shutdown(struct uart_amba_port *uap) } if (uap->using_rx_dma) { - dmaengine_terminate_all(uap->dmarx.chan); + if (uap->dmarx.poll_rate) + timer_delete_sync(&uap->dmarx.timer); + dmaengine_terminate_sync(uap->dmarx.chan); /* Clean up the RX DMA */ pl011_dmabuf_free(uap->dmarx.chan, &uap->dmarx.dbuf_a, DMA_FROM_DEVICE); pl011_dmabuf_free(uap->dmarx.chan, &uap->dmarx.dbuf_b, DMA_FROM_DEVICE); - if (uap->dmarx.poll_rate) - timer_delete_sync(&uap->dmarx.timer); uap->using_rx_dma = false; } } @@ -1289,11 +1290,30 @@ static inline bool pl011_dma_rx_running(struct uart_amba_port *uap) #define pl011_dma_flush_buffer NULL #endif -static void pl011_rs485_tx_stop(struct uart_amba_port *uap) +static void pl011_rs485_tx_stop_now(struct uart_amba_port *uap) { struct uart_port *port = &uap->port; u32 cr; + cr = pl011_read(uap, REG_CR); + + if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND) + cr &= ~UART011_CR_RTS; + else + cr |= UART011_CR_RTS; + + /* Disable the transmitter and reenable the transceiver */ + cr &= ~UART011_CR_TXE; + cr |= UART011_CR_RXE; + pl011_write(cr, uap, REG_CR); + + uap->rs485_tx_state = OFF; +} + +static void pl011_rs485_tx_stop(struct uart_amba_port *uap) +{ + struct uart_port *port = &uap->port; + if (uap->rs485_tx_state == SEND) uap->rs485_tx_state = WAIT_AFTER_SEND; @@ -1306,6 +1326,7 @@ static void pl011_rs485_tx_stop(struct uart_amba_port *uap) return; } if (port->rs485.delay_rts_after_send > 0) { + uap->rs485_tx_state = WAIT_AFTER_SEND_DELAY; hrtimer_start(&uap->trigger_stop_tx, ms_to_ktime(port->rs485.delay_rts_after_send), HRTIMER_MODE_REL); @@ -1316,19 +1337,7 @@ static void pl011_rs485_tx_stop(struct uart_amba_port *uap) hrtimer_try_to_cancel(&uap->trigger_start_tx); } - cr = pl011_read(uap, REG_CR); - - if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND) - cr &= ~UART011_CR_RTS; - else - cr |= UART011_CR_RTS; - - /* Disable the transmitter and reenable the transceiver */ - cr &= ~UART011_CR_TXE; - cr |= UART011_CR_RXE; - pl011_write(cr, uap, REG_CR); - - uap->rs485_tx_state = OFF; + pl011_rs485_tx_stop_now(uap); } static void pl011_stop_tx(struct uart_port *port) @@ -1371,7 +1380,8 @@ static void pl011_rs485_tx_start(struct uart_amba_port *uap) uap->rs485_tx_state = SEND; return; } - if (uap->rs485_tx_state == WAIT_AFTER_SEND) { + if (uap->rs485_tx_state == WAIT_AFTER_SEND || + uap->rs485_tx_state == WAIT_AFTER_SEND_DELAY) { hrtimer_try_to_cancel(&uap->trigger_stop_tx); uap->rs485_tx_state = SEND; return; @@ -1438,7 +1448,8 @@ static enum hrtimer_restart pl011_trigger_stop_tx(struct hrtimer *t) unsigned long flags; uart_port_lock_irqsave(&uap->port, &flags); - if (uap->rs485_tx_state == WAIT_AFTER_SEND) + if (uap->rs485_tx_state == WAIT_AFTER_SEND || + uap->rs485_tx_state == WAIT_AFTER_SEND_DELAY) pl011_rs485_tx_stop(uap); uart_port_unlock_irqrestore(&uap->port, flags); @@ -2041,11 +2052,20 @@ static void pl011_shutdown(struct uart_port *port) pl011_dma_shutdown(uap); - if ((port->rs485.flags & SER_RS485_ENABLED && uap->rs485_tx_state != OFF)) - pl011_rs485_tx_stop(uap); - free_irq(uap->port.irq, uap); + /* + * free_irq() drains the UART interrupt handler, which can arm either + * timer. Cancel the timers afterwards to drain their callbacks too. + */ + hrtimer_cancel(&uap->trigger_start_tx); + hrtimer_cancel(&uap->trigger_stop_tx); + + uart_port_lock_irq(port); + if (uap->rs485_tx_state != OFF) + pl011_rs485_tx_stop_now(uap); + uart_port_unlock_irq(port); + pl011_disable_uart(uap); /* @@ -2452,7 +2472,7 @@ static int pl011_console_setup(struct console *co, char *options) /* Allow pins to be muxed in and configured */ pinctrl_pm_select_default_state(uap->port.dev); - ret = clk_prepare(uap->clk); + ret = clk_prepare_enable(uap->clk); if (ret) return ret; @@ -2481,6 +2501,15 @@ static int pl011_console_setup(struct console *co, char *options) return uart_set_options(&uap->port, co, baud, parity, bits, flow); } +static int pl011_console_exit(struct console *co) +{ + struct uart_amba_port *uap = amba_ports[co->index]; + + clk_disable_unprepare(uap->clk); + + return 0; +} + /** * pl011_console_match - non-standard console matching * @co: registering console @@ -2550,8 +2579,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2568,8 +2595,6 @@ pl011_console_write_atomic(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2582,8 +2607,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!nbcon_enter_unsafe(wctxt)) return; - clk_enable(uap->clk); - if (!uap->vendor->always_enabled) { old_cr = pl011_read(uap, REG_CR); pl011_write((old_cr & ~UART011_CR_CTSEN) | (UART01x_CR_UARTEN | UART011_CR_TXE), @@ -2612,8 +2635,6 @@ pl011_console_write_thread(struct console *co, struct nbcon_write_context *wctxt if (!uap->vendor->always_enabled) pl011_write(old_cr, uap, REG_CR); - clk_disable(uap->clk); - nbcon_exit_unsafe(wctxt); } @@ -2634,6 +2655,7 @@ static struct console amba_console = { .name = "ttyAMA", .device = uart_console_device, .setup = pl011_console_setup, + .exit = pl011_console_exit, .match = pl011_console_match, .write_atomic = pl011_console_write_atomic, .write_thread = pl011_console_write_thread, @@ -2968,6 +2990,8 @@ static void pl011_remove(struct amba_device *dev) struct uart_amba_port *uap = amba_get_drvdata(dev); uart_remove_one_port(&amba_reg, &uap->port); + hrtimer_cancel(&uap->trigger_start_tx); + hrtimer_cancel(&uap->trigger_stop_tx); pl011_unregister_port(uap); } @@ -2975,21 +2999,45 @@ static void pl011_remove(struct amba_device *dev) static int pl011_suspend(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + int ret; if (!uap) return -EINVAL; - return uart_suspend_port(&amba_reg, &uap->port); + ret = uart_suspend_port(&amba_reg, &uap->port); + if (ret) + return ret; + + if (console_suspend_enabled && uap->port.suspended && + uart_console_registered(&uap->port)) + clk_disable_unprepare(uap->clk); + + return 0; } static int pl011_resume(struct device *dev) { struct uart_amba_port *uap = dev_get_drvdata(dev); + bool resume_console; + int ret; if (!uap) return -EINVAL; - return uart_resume_port(&amba_reg, &uap->port); + resume_console = console_suspend_enabled && + uap->port.suspended && + uart_console_registered(&uap->port); + if (resume_console) { + ret = clk_prepare_enable(uap->clk); + if (ret) + return ret; + } + + ret = uart_resume_port(&amba_reg, &uap->port); + if (ret && resume_console) + clk_disable_unprepare(uap->clk); + + return ret; } #endif diff --git a/drivers/tty/serial/bcm63xx_uart.c b/drivers/tty/serial/bcm63xx_uart.c index 51df9d2d8bfc5..544695cb184c3 100644 --- a/drivers/tty/serial/bcm63xx_uart.c +++ b/drivers/tty/serial/bcm63xx_uart.c @@ -675,7 +675,7 @@ static void wait_for_xmitr(struct uart_port *port) } /* Wait up to 1s for flow control if necessary */ - if (port->flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(port)) { tmout = 1000000; while (--tmout) { unsigned int val; diff --git a/drivers/tty/serial/imx.c b/drivers/tty/serial/imx.c index 90e2ea1e8afe5..1f2b4c12eead8 100644 --- a/drivers/tty/serial/imx.c +++ b/drivers/tty/serial/imx.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -2079,6 +2080,9 @@ static const struct uart_ops imx_uart_pops = { static struct imx_port *imx_uart_ports[UART_NR]; +/* Held across uart_add/remove_one_port(); console callbacks must not take it. */ +static DEFINE_MUTEX(imx_uart_ports_lock); + #if IS_ENABLED(CONFIG_SERIAL_IMX_CONSOLE) static void imx_uart_console_putchar(struct uart_port *port, unsigned char ch) { @@ -2631,11 +2635,19 @@ static int imx_uart_probe(struct platform_device *pdev) } } - imx_uart_ports[sport->port.line] = sport; - platform_set_drvdata(pdev, sport); - ret = uart_add_one_port(&imx_uart_uart_driver, &sport->port); + scoped_guard(mutex, &imx_uart_ports_lock) { + if (imx_uart_ports[sport->port.line]) { + ret = -EBUSY; + } else { + imx_uart_ports[sport->port.line] = sport; + ret = uart_add_one_port(&imx_uart_uart_driver, + &sport->port); + if (ret) + imx_uart_ports[sport->port.line] = NULL; + } + } err_clk: clk_disable_unprepare(sport->clk_ipg); @@ -2647,7 +2659,9 @@ static void imx_uart_remove(struct platform_device *pdev) { struct imx_port *sport = platform_get_drvdata(pdev); + guard(mutex)(&imx_uart_ports_lock); uart_remove_one_port(&imx_uart_uart_driver, &sport->port); + imx_uart_ports[sport->port.line] = NULL; } static void imx_uart_restore_context(struct imx_port *sport) diff --git a/drivers/tty/serial/ma35d1_serial.c b/drivers/tty/serial/ma35d1_serial.c index 285b0fe41a86a..920fe7ff5083b 100644 --- a/drivers/tty/serial/ma35d1_serial.c +++ b/drivers/tty/serial/ma35d1_serial.c @@ -608,8 +608,14 @@ static int __init ma35d1serial_console_setup(struct console *co, char *options) if (!np || !p) return -ENODEV; - if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) + if (of_property_read_u32_array(np, "reg", val32, ARRAY_SIZE(val32)) != 0) { + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; return -EINVAL; + } + + of_node_put(np); + ma35d1serial_uart_nodes[co->index] = NULL; p->port.iobase = val32[1]; p->port.membase = ioremap(p->port.iobase, MA35_UART_REG_SIZE); @@ -648,8 +654,10 @@ static void ma35d1serial_console_init_port(void) of_node_get(np); ma35d1serial_uart_nodes[i] = np; i++; - if (i == MA35_UART_NR) + if (i == MA35_UART_NR) { + of_node_put(np); break; + } } } } diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c index 0b85f47ff19e0..a689d190940cf 100644 --- a/drivers/tty/serial/omap-serial.c +++ b/drivers/tty/serial/omap-serial.c @@ -1092,7 +1092,7 @@ static void __maybe_unused wait_for_xmitr(struct uart_omap_port *up) } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ - if (up->port.flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(&up->port)) { for (tmout = 1000000; tmout; tmout--) { unsigned int msr = serial_in(up, UART_MSR); diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c index 9992fa231e4e8..c3f32e0590b7f 100644 --- a/drivers/tty/serial/pch_uart.c +++ b/drivers/tty/serial/pch_uart.c @@ -1451,7 +1451,7 @@ static void wait_for_xmitr(struct eg20t_port *up, int bits) } /* Wait up to 1s for flow control if necessary */ - if (up->port.flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(&up->port)) { unsigned int tmout; for (tmout = 1000000; tmout; tmout--) { unsigned int msr = ioread8(up->membase + UART_MSR); diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c index e395ff29c1a2c..027d936b50239 100644 --- a/drivers/tty/serial/pxa.c +++ b/drivers/tty/serial/pxa.c @@ -573,7 +573,7 @@ static void wait_for_xmitr(struct uart_pxa_port *up) } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ - if (up->port.flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(&up->port)) { tmout = 1000000; while (--tmout && ((serial_in(up, UART_MSR) & UART_MSR_CTS) == 0)) diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c index a34817a4428c9..953b737357103 100644 --- a/drivers/tty/serial/qcom_geni_serial.c +++ b/drivers/tty/serial/qcom_geni_serial.c @@ -136,6 +136,7 @@ struct qcom_geni_serial_port { unsigned int tx_remaining; unsigned int tx_queued; + bool tx_dma_stale; int wakeup_irq; bool rx_tx_swap; bool cts_rts_swap; @@ -149,6 +150,7 @@ static const struct uart_ops qcom_geni_uart_pops; static struct uart_driver qcom_geni_console_driver; static struct uart_driver qcom_geni_uart_driver; +static void qcom_geni_serial_stop_tx_dma(struct uart_port *uport); static void __qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport); static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport); static int qcom_geni_serial_port_setup(struct uart_port *uport); @@ -627,35 +629,34 @@ static unsigned int qcom_geni_serial_tx_empty(struct uart_port *uport) return !readl(uport->membase + SE_GENI_TX_FIFO_STATUS); } +static void qcom_geni_serial_flush_buffer_dma(struct uart_port *uport) +{ + struct qcom_geni_serial_port *port = to_dev_port(uport); + + qcom_geni_serial_stop_tx_dma(uport); + port->tx_remaining = 0; + port->tx_queued = 0; +} + static void qcom_geni_serial_stop_tx_dma(struct uart_port *uport) { struct qcom_geni_serial_port *port = to_dev_port(uport); - bool done; - if (!qcom_geni_serial_main_active(uport)) - return; + if (qcom_geni_serial_main_active(uport)) + __qcom_geni_serial_cancel_tx_cmd(uport); if (port->tx_dma_addr) { + writel(1, uport->membase + SE_DMA_TX_FSM_RST); + if (!qcom_geni_serial_poll_bit(uport, SE_DMA_TX_IRQ_STAT, + TX_RESET_DONE, true)) + dev_err_ratelimited(uport->dev, "TX DMA reset failed"); + writel(TX_RESET_DONE | TX_DMA_DONE, + uport->membase + SE_DMA_TX_IRQ_CLR); + geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining); port->tx_dma_addr = 0; - port->tx_remaining = 0; } - - geni_se_cancel_m_cmd(&port->se); - - done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS, - M_CMD_CANCEL_EN, true); - if (!done) { - geni_se_abort_m_cmd(&port->se); - done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS, - M_CMD_ABORT_EN, true); - if (!done) - dev_err_ratelimited(uport->dev, "M_CMD_ABORT_EN not set"); - writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR); - } - - writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR); } static void qcom_geni_serial_start_tx_dma(struct uart_port *uport) @@ -688,6 +689,7 @@ static void qcom_geni_serial_start_tx_dma(struct uart_port *uport) } port->tx_remaining = xmit_size; + port->tx_dma_stale = false; } static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport) @@ -1020,6 +1022,7 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) struct qcom_geni_serial_port *port = to_dev_port(uport); struct tty_port *tport = &uport->state->port; unsigned int fifo_len = kfifo_len(&tport->xmit_fifo); + bool tx_dma_stale = port->tx_dma_stale; /* * Only advance the kfifo if it still contains the bytes that were @@ -1030,12 +1033,13 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport) * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining * and triggering a spurious large DMA transfer of stale data. */ - if (fifo_len >= port->tx_remaining) + if (!tx_dma_stale && fifo_len >= port->tx_remaining) uart_xmit_advance(uport, port->tx_remaining); geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining); port->tx_dma_addr = 0; port->tx_remaining = 0; + port->tx_dma_stale = false; if (!kfifo_is_empty(&tport->xmit_fifo)) qcom_geni_serial_start_tx_dma(uport); @@ -1171,8 +1175,12 @@ static void qcom_geni_serial_shutdown(struct uart_port *uport) uart_port_unlock_irq(uport); } -static void qcom_geni_serial_flush_buffer(struct uart_port *uport) +static void qcom_geni_serial_flush_buffer_fifo(struct uart_port *uport) { + struct qcom_geni_serial_port *port = to_dev_port(uport); + + if (port->tx_dma_addr) + port->tx_dma_stale = true; qcom_geni_serial_cancel_tx_cmd(uport); } @@ -1695,7 +1703,7 @@ static const struct uart_ops qcom_geni_console_pops = { .request_port = qcom_geni_serial_request_port, .config_port = qcom_geni_serial_config_port, .shutdown = qcom_geni_serial_shutdown, - .flush_buffer = qcom_geni_serial_flush_buffer, + .flush_buffer = qcom_geni_serial_flush_buffer_fifo, .type = qcom_geni_serial_get_type, .set_mctrl = qcom_geni_serial_set_mctrl, .get_mctrl = qcom_geni_serial_get_mctrl, @@ -1718,6 +1726,7 @@ static const struct uart_ops qcom_geni_uart_pops = { .request_port = qcom_geni_serial_request_port, .config_port = qcom_geni_serial_config_port, .shutdown = qcom_geni_serial_shutdown, + .flush_buffer = qcom_geni_serial_flush_buffer_dma, .type = qcom_geni_serial_get_type, .set_mctrl = qcom_geni_serial_set_mctrl, .get_mctrl = qcom_geni_serial_get_mctrl, diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c index 7a0b89d856c2d..59d58fb743a31 100644 --- a/drivers/tty/serial/samsung_tty.c +++ b/drivers/tty/serial/samsung_tty.c @@ -311,7 +311,7 @@ static void s3c24xx_serial_stop_tx(struct uart_port *port) ourport->tx_enabled = 0; ourport->tx_in_progress = 0; - if (port->flags & UPF_CONS_FLOW) + if (uart_cons_flow_enabled(port)) s3c24xx_serial_rx_enable(port); ourport->tx_mode = 0; @@ -485,7 +485,7 @@ static void s3c24xx_serial_start_tx(struct uart_port *port) struct tty_port *tport = &port->state->port; if (!ourport->tx_enabled) { - if (port->flags & UPF_CONS_FLOW) + if (uart_cons_flow_enabled(port)) s3c24xx_serial_rx_disable(port); ourport->tx_enabled = 1; @@ -773,7 +773,7 @@ static void s3c24xx_serial_rx_drain_fifo(struct s3c24xx_uart_port *ourport) uerstat = rd_regl(port, S3C2410_UERSTAT); ch = rd_reg(port, S3C2410_URXH); - if (port->flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(port)) { bool txe = s3c24xx_serial_txempty_nofifo(port); if (ourport->rx_enabled) { @@ -1828,7 +1828,7 @@ static int s3c24xx_serial_init_port(struct s3c24xx_uart_port *ourport, if (cfg->uart_flags & UPF_CONS_FLOW) { dev_dbg(port->dev, "enabling flow control\n"); - port->flags |= UPF_CONS_FLOW; + uart_set_cons_flow_enabled(port, true); } /* sort our the physical and virtual addresses for each UART */ diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c index f6ccb1dd216ba..3bc6fa4f69c13 100644 --- a/drivers/tty/serial/sc16is7xx.c +++ b/drivers/tty/serial/sc16is7xx.c @@ -11,6 +11,7 @@ #define DEFAULT_SYMBOL_NAMESPACE "SERIAL_NXP_SC16IS7XX" #include +#include #include #include #include @@ -328,7 +329,7 @@ struct sc16is7xx_one_config { struct sc16is7xx_one { struct uart_port port; struct regmap *regmap; - struct mutex efr_lock; /* EFR registers access */ + struct mutex lock; /* For registers sharing same address space. */ struct kthread_work tx_work; struct kthread_work reg_work; struct kthread_delayed_work ms_work; @@ -436,7 +437,7 @@ static void sc16is7xx_efr_lock(struct uart_port *port) { struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); - mutex_lock(&one->efr_lock); + mutex_lock(&one->lock); /* Backup content of LCR. */ one->old_lcr = sc16is7xx_port_read(port, SC16IS7XX_LCR_REG); @@ -458,7 +459,7 @@ static void sc16is7xx_efr_unlock(struct uart_port *port) /* Restore original content of LCR */ sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, one->old_lcr); - mutex_unlock(&one->efr_lock); + mutex_unlock(&one->lock); } static void sc16is7xx_ier_clear(struct uart_port *port, u8 bit) @@ -593,7 +594,7 @@ static int sc16is7xx_set_baud(struct uart_port *port, int baud) SC16IS7XX_MCR_CLKSEL_BIT, prescaler == 1 ? 0 : SC16IS7XX_MCR_CLKSEL_BIT); - mutex_lock(&one->efr_lock); + mutex_lock(&one->lock); /* Backup LCR and access special register set (DLL/DLH) */ lcr = sc16is7xx_port_read(port, SC16IS7XX_LCR_REG); @@ -609,7 +610,7 @@ static int sc16is7xx_set_baud(struct uart_port *port, int baud) /* Restore LCR and access to general register set */ sc16is7xx_port_write(port, SC16IS7XX_LCR_REG, lcr); - mutex_unlock(&one->efr_lock); + mutex_unlock(&one->lock); return DIV_ROUND_CLOSEST((clk / prescaler) / 16, div); } @@ -756,7 +757,7 @@ static void sc16is7xx_update_mlines(struct sc16is7xx_one *one) unsigned long flags; unsigned int status, changed; - lockdep_assert_held_once(&one->efr_lock); + lockdep_assert_held_once(&one->lock); status = sc16is7xx_get_hwmctrl(port); changed = status ^ one->old_mctrl; @@ -782,18 +783,15 @@ static void sc16is7xx_update_mlines(struct sc16is7xx_one *one) static bool sc16is7xx_port_irq(struct sc16is7xx_port *s, int portno) { - bool rc = true; unsigned int iir, rxlen; struct uart_port *port = &s->p[portno].port; struct sc16is7xx_one *one = to_sc16is7xx_one(port, port); - mutex_lock(&one->efr_lock); + guard(mutex)(&one->lock); iir = sc16is7xx_port_read(port, SC16IS7XX_IIR_REG); - if (iir & SC16IS7XX_IIR_NO_INT_BIT) { - rc = false; - goto out_port_irq; - } + if (iir & SC16IS7XX_IIR_NO_INT_BIT) + return false; iir &= SC16IS7XX_IIR_ID_MASK; @@ -818,7 +816,7 @@ static bool sc16is7xx_port_irq(struct sc16is7xx_port *s, int portno) if (rxlen) sc16is7xx_handle_rx(port, rxlen, iir); else - rc = false; + return false; break; /* CTSRTS interrupt comes only when CTS goes inactive */ case SC16IS7XX_IIR_CTSRTS_SRC: @@ -835,10 +833,7 @@ static bool sc16is7xx_port_irq(struct sc16is7xx_port *s, int portno) break; } -out_port_irq: - mutex_unlock(&one->efr_lock); - - return rc; + return true; } static irqreturn_t sc16is7xx_irq(int irq, void *dev_id) @@ -880,9 +875,11 @@ static void sc16is7xx_tx_proc(struct kthread_work *ws) (port->rs485.delay_rts_before_send > 0)) msleep(port->rs485.delay_rts_before_send); - mutex_lock(&one->efr_lock); + guard(mutex)(&one->lock); + sc16is7xx_port_update(port, SC16IS7XX_IER_REG, + SC16IS7XX_IER_THRI_BIT, + SC16IS7XX_IER_THRI_BIT); sc16is7xx_handle_tx(port); - mutex_unlock(&one->efr_lock); } static void sc16is7xx_reconf_rs485(struct uart_port *port) @@ -949,9 +946,8 @@ static void sc16is7xx_ms_proc(struct kthread_work *ws) struct sc16is7xx_port *s = dev_get_drvdata(one->port.dev); if (one->port.state) { - mutex_lock(&one->efr_lock); - sc16is7xx_update_mlines(one); - mutex_unlock(&one->efr_lock); + scoped_guard(mutex, &one->lock) + sc16is7xx_update_mlines(one); kthread_queue_delayed_work(&s->kworker, &one->ms_work, HZ); } @@ -1333,6 +1329,17 @@ static int sc16is7xx_gpio_set(struct gpio_chip *chip, unsigned int offset, return 0; } +static int sc16is7xx_gpio_get_direction(struct gpio_chip *chip, unsigned int offset) +{ + struct sc16is7xx_port *s = gpiochip_get_data(chip); + struct uart_port *port = &s->p[0].port; + unsigned int val; + + val = sc16is7xx_port_read(port, SC16IS7XX_IODIR_REG); + + return val & BIT(offset) ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN; +} + static int sc16is7xx_gpio_direction_input(struct gpio_chip *chip, unsigned offset) { @@ -1410,6 +1417,7 @@ static int sc16is7xx_setup_gpio_chip(struct sc16is7xx_port *s) s->gpio.parent = dev; s->gpio.label = dev_name(dev); s->gpio.init_valid_mask = sc16is7xx_gpio_init_valid_mask; + s->gpio.get_direction = sc16is7xx_gpio_get_direction; s->gpio.direction_input = sc16is7xx_gpio_direction_input; s->gpio.get = sc16is7xx_gpio_get; s->gpio.direction_output = sc16is7xx_gpio_direction_output; @@ -1628,7 +1636,7 @@ int sc16is7xx_probe(struct device *dev, const struct sc16is7xx_devtype *devtype, s->p[i].old_mctrl = 0; s->p[i].regmap = regmaps[i]; - mutex_init(&s->p[i].efr_lock); + mutex_init(&s->p[i].lock); ret = uart_get_rs485_mode(&s->p[i].port); if (ret) diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c index ca72454b3eb09..96599dee967bf 100644 --- a/drivers/tty/serial/serial_core.c +++ b/drivers/tty/serial/serial_core.c @@ -2263,6 +2263,18 @@ uart_set_options(struct uart_port *port, struct console *co, port->mctrl |= TIOCM_DTR; port->ops->set_termios(port, &termios, &dummy); + + /* + * If console hardware flow control was specified and is supported, + * the related policy UPSTAT_CTS_ENABLE must be set to allow console + * drivers to identify if CTS should be used for polling. + */ + if (flow == 'r' && (termios.c_cflag & CRTSCTS)) { + /* Synchronize @status RMW update against the console. */ + guard(uart_port_lock_irqsave)(port); + port->status |= UPSTAT_CTS_ENABLE; + } + /* * Allow the setting of the UART parameters with a NULL console * too: @@ -2569,7 +2581,14 @@ uart_configure_port(struct uart_driver *drv, struct uart_state *state, * We probably don't need a spinlock around this, but */ scoped_guard(uart_port_lock_irqsave, port) { - port->mctrl &= TIOCM_DTR; + unsigned int mask = TIOCM_DTR; + + /* Console hardware flow control polls CTS. */ + if (uart_console_hwflow_active(port)) + mask |= TIOCM_RTS; + + port->mctrl &= mask; + if (!(port->rs485.flags & SER_RS485_ENABLED)) port->ops->set_mctrl(port, port->mctrl); } diff --git a/drivers/tty/serial/serial_txx9.c b/drivers/tty/serial/serial_txx9.c index 436a559234dfe..4ae9a45c8e3a3 100644 --- a/drivers/tty/serial/serial_txx9.c +++ b/drivers/tty/serial/serial_txx9.c @@ -422,7 +422,7 @@ static void wait_for_xmitr(struct uart_port *up) udelay(1); /* Wait up to 1s for flow control if necessary */ - if (up->flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(up)) { tmout = 1000000; while (--tmout && (sio_in(up, TXX9_SICISR) & TXX9_SICISR_CTSS)) @@ -857,7 +857,7 @@ serial_txx9_console_write(struct console *co, const char *s, unsigned int count) * Disable flow-control if enabled (and unnecessary) */ flcr = sio_in(up, TXX9_SIFLCR); - if (!(up->flags & UPF_CONS_FLOW) && (flcr & TXX9_SIFLCR_TES)) + if (!uart_cons_flow_enabled(up) && (flcr & TXX9_SIFLCR_TES)) sio_out(up, TXX9_SIFLCR, flcr & ~TXX9_SIFLCR_TES); uart_console_write(up, s, count, serial_txx9_console_putchar); diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c index 383141fe7ba0d..f71937fbe8aa9 100644 --- a/drivers/tty/serial/sunsu.c +++ b/drivers/tty/serial/sunsu.c @@ -1245,7 +1245,7 @@ static void wait_for_xmitr(struct uart_sunsu_port *up) } while (!uart_lsr_tx_empty(status)); /* Wait up to 1s for flow control if necessary */ - if (up->port.flags & UPF_CONS_FLOW) { + if (uart_cons_flow_enabled(&up->port)) { tmout = 1000000; while (--tmout && ((serial_in(up, UART_MSR) & UART_MSR_CTS) == 0)) diff --git a/drivers/tty/tty_io.c b/drivers/tty/tty_io.c index e2d92cf70eb78..f779f8fb138e8 100644 --- a/drivers/tty/tty_io.c +++ b/drivers/tty/tty_io.c @@ -3167,8 +3167,10 @@ static int tty_cdev_add(struct tty_driver *driver, dev_t dev, driver->cdevs[index]->ops = &tty_fops; driver->cdevs[index]->owner = driver->owner; err = cdev_add(driver->cdevs[index], dev, count); - if (err) + if (err) { kobject_put(&driver->cdevs[index]->kobj); + driver->cdevs[index] = NULL; + } return err; } diff --git a/drivers/tty/vt/keyboard.c b/drivers/tty/vt/keyboard.c index ee1d9c448c7eb..d9d8e91c09c90 100644 --- a/drivers/tty/vt/keyboard.c +++ b/drivers/tty/vt/keyboard.c @@ -1396,7 +1396,7 @@ static void kbd_keycode(unsigned int keycode, int down, bool hw_raw) struct keyboard_notifier_param param = { .vc = vc, .value = keycode, .down = down }; int rc; - tty = vc->port.tty; + tty = tty_port_tty_get(&vc->port); if (tty && (!tty->driver_data)) { /* No driver data? Strange. Okay we fix it then. */ @@ -1456,9 +1456,12 @@ static void kbd_keycode(unsigned int keycode, int down, bool hw_raw) * characters get aren't echoed locally. This makes key repeat * usable with slow applications and under heavy loads. */ + tty_kref_put(tty); return; } + tty_kref_put(tty); + param.shift = shift_final = (shift_state | kbd->slockstate) ^ kbd->lockstate; param.ledstate = kbd->ledflagstate; key_map = key_maps[shift_final]; diff --git a/drivers/tty/vt/vt_ioctl.c b/drivers/tty/vt/vt_ioctl.c index 28993a3d0acb6..deb3b3f934616 100644 --- a/drivers/tty/vt/vt_ioctl.c +++ b/drivers/tty/vt/vt_ioctl.c @@ -406,6 +406,8 @@ static int vt_k_ioctl(struct tty_struct *tty, unsigned int cmd, /* this could be folded into KDSKBMODE, but for compatibility reasons it is not so easy to fold KDGKBMETA into KDGKBMODE */ case KDSKBMETA: + if (!perm) + return -EPERM; return vt_do_kdskbmeta(console, arg); case KDGKBMETA: diff --git a/drivers/ufs/core/ufs-debugfs.c b/drivers/ufs/core/ufs-debugfs.c index e3baed6c70bd9..6dece24baad15 100644 --- a/drivers/ufs/core/ufs-debugfs.c +++ b/drivers/ufs/core/ufs-debugfs.c @@ -165,7 +165,7 @@ static ssize_t ufs_saved_err_write(struct file *file, const char __user *buf, char val_str[16] = { }; int val, ret; - if (count > sizeof(val_str)) + if (count >= sizeof(val_str)) return -EINVAL; if (copy_from_user(val_str, buf, count)) return -EFAULT; diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 8de3d0886a121..87578e8824d2d 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -1338,6 +1338,7 @@ static int ufshcd_wait_for_pending_cmds(struct ufs_hba *hba, break; } + __set_current_state(TASK_UNINTERRUPTIBLE); io_schedule_timeout(msecs_to_jiffies(20)); if (ktime_to_us(ktime_sub(ktime_get(), start)) > wait_timeout_us) { @@ -7128,7 +7129,7 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) } /** - * ufshcd_threaded_intr - Threaded interrupt service routine + * ufshcd_intr - Main interrupt service routine * @irq: irq number * @__hba: pointer to adapter instance * @@ -7136,7 +7137,7 @@ static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status) * IRQ_HANDLED - If interrupt is valid * IRQ_NONE - If invalid interrupt */ -static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) +static irqreturn_t ufshcd_intr(int irq, void *__hba) { u32 last_intr_status, intr_status, enabled_intr_status = 0; irqreturn_t retval = IRQ_NONE; @@ -7175,34 +7176,6 @@ static irqreturn_t ufshcd_threaded_intr(int irq, void *__hba) return retval; } -/** - * ufshcd_intr - Main interrupt service routine - * @irq: irq number - * @__hba: pointer to adapter instance - * - * Return: - * IRQ_HANDLED - If interrupt is valid - * IRQ_WAKE_THREAD - If handling is moved to threaded handled - * IRQ_NONE - If invalid interrupt - */ -static irqreturn_t ufshcd_intr(int irq, void *__hba) -{ - struct ufs_hba *hba = __hba; - u32 intr_status, enabled_intr_status; - - /* Move interrupt handling to thread when MCQ & ESI are not enabled */ - if (!hba->mcq_enabled || !hba->mcq_esi_enabled) - return IRQ_WAKE_THREAD; - - intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); - enabled_intr_status = intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE); - - ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); - - /* Directly handle interrupts since MCQ ESI handlers does the hard job */ - return ufshcd_sl_intr(hba, enabled_intr_status); -} - static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag) { int err = 0; @@ -9873,6 +9846,7 @@ static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op) req_link_state == UIC_LINK_ACTIVE_STATE) { ufshcd_disable_auto_bkops(hba); flush_work(&hba->eeh_work); + cancel_delayed_work_sync(&hba->ufs_rtc_update_work); goto vops_suspend; } @@ -10082,10 +10056,11 @@ static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) if (ret) goto set_old_link_state; ufshcd_set_timestamp_attr(hba); - schedule_delayed_work(&hba->ufs_rtc_update_work, - msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); } + schedule_delayed_work(&hba->ufs_rtc_update_work, + msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS)); + if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) ufshcd_enable_auto_bkops(hba); else @@ -10829,8 +10804,7 @@ int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq) ufshcd_readl(hba, REG_INTERRUPT_ENABLE); /* IRQ registration */ - err = devm_request_threaded_irq(dev, irq, ufshcd_intr, ufshcd_threaded_intr, - IRQF_ONESHOT | IRQF_SHARED, UFSHCD, hba); + err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba); if (err) { dev_err(hba->dev, "request irq failed\n"); goto out_disable; diff --git a/drivers/uio/uio.c b/drivers/uio/uio.c index d93ed4e86a174..4029585bf22cd 100644 --- a/drivers/uio/uio.c +++ b/drivers/uio/uio.c @@ -1051,6 +1051,11 @@ int __uio_register_device(struct module *owner, err_request_irq: uio_dev_del_attributes(idev); err_uio_dev_add_attributes: + mutex_lock(&idev->info_lock); + idev->info = NULL; + mutex_unlock(&idev->info_lock); + wake_up_interruptible(&idev->wait); + kill_fasync(&idev->async_queue, SIGIO, POLL_HUP); device_del(&idev->dev); err_device_create: uio_free_minor(idev->minor); diff --git a/drivers/uio/uio_hv_generic.c b/drivers/uio/uio_hv_generic.c index 3f8e2e27697fb..c47d46334a7c0 100644 --- a/drivers/uio/uio_hv_generic.c +++ b/drivers/uio/uio_hv_generic.c @@ -395,9 +395,15 @@ hv_uio_remove(struct hv_device *dev) vmbus_free_ring(dev->channel); } +static const struct hv_vmbus_device_id hv_uio_id_table[] = { + { HV_FCOPY_GUID }, + {} +}; +MODULE_DEVICE_TABLE(vmbus, hv_uio_id_table); + static struct hv_driver hv_uio_drv = { .name = "uio_hv_generic", - .id_table = NULL, /* only dynamic id's */ + .id_table = hv_uio_id_table, .probe = hv_uio_probe, .remove = hv_uio_remove, }; diff --git a/drivers/usb/atm/cxacru.c b/drivers/usb/atm/cxacru.c index 68a8e9de8b4fe..746021c31cf3a 100644 --- a/drivers/usb/atm/cxacru.c +++ b/drivers/usb/atm/cxacru.c @@ -700,6 +700,8 @@ static int cxacru_cm(struct cxacru_data *instance, enum cxacru_cm_request cm, ret = offd; usb_dbg(instance->usbatm, "cm %#x\n", cm); fail: + if (ret < 0) + usb_kill_urb(instance->rcv_urb); mutex_unlock(&instance->cm_serialize); err: return ret; diff --git a/drivers/usb/atm/ueagle-atm.c b/drivers/usb/atm/ueagle-atm.c index a943a5257a59f..f1f1fc16a4711 100644 --- a/drivers/usb/atm/ueagle-atm.c +++ b/drivers/usb/atm/ueagle-atm.c @@ -2550,6 +2550,7 @@ static struct usbatm_driver uea_usbatm_driver = { static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id) { struct usb_device *usb = interface_to_usbdev(intf); + bool single_iface = usb->config->desc.bNumInterfaces == 1; int ret; uea_dbg(usb, "ADSL device found with vid (%#X) pid (%#X) Rev (%#X): %s\n", @@ -2558,6 +2559,22 @@ static int uea_probe(struct usb_interface *intf, const struct usb_device_id *id) le16_to_cpu(usb->descriptor.bcdDevice), chip_name[UEA_CHIP_VERSION(id)]); + /* + * uea_probe() decides between the pre-firmware and post-firmware case + * from the USB id and stores a different object as interface data in + * each case: a struct completion for a pre-firmware device, a struct + * usbatm_data for a post-firmware one. uea_disconnect() instead tells + * the two apart by the number of interfaces (a pre-firmware device + * exposes a single interface, ADI930 has 2 and eagle has 3). A crafted + * device advertising a pre-firmware id together with a multi-interface + * descriptor (or the other way around) makes the two disagree, so that + * usbatm_usb_disconnect() treats the small completion object as a + * struct usbatm_data and reads out of bounds. Reject such inconsistent + * descriptors so both paths make the same decision. + */ + if (UEA_IS_PREFIRM(id) != single_iface) + return -ENODEV; + usb_reset_device(usb); if (UEA_IS_PREFIRM(id)) { diff --git a/drivers/usb/atm/usbatm.c b/drivers/usb/atm/usbatm.c index 5f3ad9a99d9e7..780be13af3648 100644 --- a/drivers/usb/atm/usbatm.c +++ b/drivers/usb/atm/usbatm.c @@ -917,8 +917,8 @@ static int usbatm_atm_init(struct usbatm_data *instance) instance->atm_dev = atm_dev; - atm_dev->ci_range.vpi_bits = ATM_CI_MAX; - atm_dev->ci_range.vci_bits = ATM_CI_MAX; + atm_dev->ci_range.vpi_bits = 8; + atm_dev->ci_range.vci_bits = 16; atm_dev->signal = ATM_PHY_SIG_UNKNOWN; /* temp init ATM device, set to 128kbit */ diff --git a/drivers/usb/atm/xusbatm.c b/drivers/usb/atm/xusbatm.c index 0befbf63d1cc8..5c1e1f5215555 100644 --- a/drivers/usb/atm/xusbatm.c +++ b/drivers/usb/atm/xusbatm.c @@ -79,7 +79,7 @@ static int xusbatm_bind(struct usbatm_data *usbatm, struct usb_interface *intf, const struct usb_device_id *id) { struct usb_device *usb_dev = interface_to_usbdev(intf); - int drv_ix = id - xusbatm_usb_ids; + int drv_ix = id->driver_info; int rx_alt = rx_altsetting[drv_ix]; int tx_alt = tx_altsetting[drv_ix]; struct usb_interface *rx_intf = xusbatm_find_intf(usb_dev, rx_alt, rx_endpoint[drv_ix]); @@ -168,7 +168,8 @@ static struct usb_driver xusbatm_usb_driver = { .name = xusbatm_driver_name, .probe = xusbatm_usb_probe, .disconnect = usbatm_usb_disconnect, - .id_table = xusbatm_usb_ids + .id_table = xusbatm_usb_ids, + .no_dynamic_id = 1, }; static int __init xusbatm_init(void) @@ -190,6 +191,7 @@ static int __init xusbatm_init(void) xusbatm_usb_ids[i].match_flags = USB_DEVICE_ID_MATCH_DEVICE; xusbatm_usb_ids[i].idVendor = vendor[i]; xusbatm_usb_ids[i].idProduct = product[i]; + xusbatm_usb_ids[i].driver_info = i; xusbatm_drivers[i].driver_name = xusbatm_driver_name; xusbatm_drivers[i].bind = xusbatm_bind; diff --git a/drivers/usb/c67x00/c67x00-sched.c b/drivers/usb/c67x00/c67x00-sched.c index a09fa68a6ce74..346cdac1c1e23 100644 --- a/drivers/usb/c67x00/c67x00-sched.c +++ b/drivers/usb/c67x00/c67x00-sched.c @@ -761,13 +761,13 @@ static int c67x00_add_iso_urb(struct c67x00_hcd *c67x00, struct urb *urb) ret); urb->iso_frame_desc[urbp->cnt].actual_length = 0; urb->iso_frame_desc[urbp->cnt].status = ret; - if (urbp->cnt + 1 == urb->number_of_packets) - c67x00_giveback_urb(c67x00, urb, 0); } urbp->ep_data->next_frame = frame_add(urbp->ep_data->next_frame, urb->interval); urbp->cnt++; + if (ret && urbp->cnt == urb->number_of_packets) + c67x00_giveback_urb(c67x00, urb, 0); } return 0; } diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c index 0252560cbc80b..1ffd3bb867073 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.c +++ b/drivers/usb/cdns3/cdnsp-gadget.c @@ -155,9 +155,9 @@ static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev) offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP); reg = base + offset + REG_CHICKEN_BITS_3_OFFSET; - val = le32_to_cpu(readl(reg)); + val = readl(reg); val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout); - writel(cpu_to_le32(val), reg); + writel(val, reg); } static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit) @@ -1820,6 +1820,82 @@ static void cdnsp_get_rev_cap(struct cdnsp_device *pdev) readl(&pdev->rev_cap->tx_buff_size)); } +static void cdnsp_set_event_deq(struct cdnsp_device *pdev) +{ + dma_addr_t deq; + u64 temp; + + deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, + pdev->event_ring->dequeue); + + /* Update controller event ring dequeue pointer */ + temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue); + temp &= ERST_PTR_MASK; + + /* + * Don't clear the EHB bit (which is RW1C) because + * there might be more events to service. + */ + temp &= ~ERST_EHB; + + cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp, + &pdev->ir_set->erst_dequeue); +} + +static void cdnsp_add_interrupter(struct cdnsp_device *pdev) +{ + u64 erst_base; + u32 erst_size; + + /* Set ERST count with the number of entries in the segment table. */ + erst_size = readl(&pdev->ir_set->erst_size); + erst_size &= ERST_SIZE_MASK; + erst_size |= ERST_NUM_SEGS; + writel(erst_size, &pdev->ir_set->erst_size); + + /* Set the segment table base address. */ + erst_base = cdnsp_read_64(&pdev->ir_set->erst_base); + erst_base &= ERST_PTR_MASK; + erst_base |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK); + cdnsp_write_64(erst_base, &pdev->ir_set->erst_base); + + /* Set the event ring dequeue address. */ + cdnsp_set_event_deq(pdev); +} + +/* Set up basic CDNSP registers */ +static void cdnsp_init(struct cdnsp_device *pdev) +{ + unsigned int val; + u64 val_64; + + val = readl(&pdev->op_regs->config_reg); + val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E; + writel(val, &pdev->op_regs->config_reg); + + /* Initialize the Command ring */ + cdnsp_ring_init(pdev, pdev->cmd_ring); + + /* Set the address in the Command Ring Control register */ + val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring); + val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) | + (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) | + pdev->cmd_ring->cycle_state; + cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring); + + /* Set Device Context Base Address Array pointer */ + cdnsp_write_64(pdev->dcbaa->dma, &pdev->op_regs->dcbaa_ptr); + + /* Set Doorbell array pointer */ + val = readl(&pdev->cap_regs->db_off); + val &= DBOFF_MASK; + pdev->dba = (void __iomem *)pdev->cap_regs + val; + + /* Initialize the Primary interrupter */ + cdnsp_ring_init(pdev, pdev->event_ring); + cdnsp_add_interrupter(pdev); +} + static int cdnsp_gen_setup(struct cdnsp_device *pdev) { int ret; @@ -1885,6 +1961,8 @@ static int cdnsp_gen_setup(struct cdnsp_device *pdev) if (ret) return ret; + cdnsp_init(pdev); + /* * Software workaround for U1: after transition * to U1 the controller starts gating clock, and in some cases, @@ -2015,9 +2093,6 @@ static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup) struct cdnsp_device *pdev = cdns->gadget_dev; unsigned long flags; - if (pdev->link_state == XDEV_U3) - return 0; - spin_lock_irqsave(&pdev->lock, flags); cdnsp_disconnect_gadget(pdev); cdnsp_stop(pdev); @@ -2031,12 +2106,38 @@ static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power) struct cdnsp_device *pdev = cdns->gadget_dev; enum usb_device_speed max_speed; unsigned long flags; + bool context_lost; + u32 val; int ret; if (!pdev->gadget_driver) return 0; spin_lock_irqsave(&pdev->lock, flags); + val = readl(&pdev->port3x_regs->mode_2); + context_lost = !!(val & CFG_3XPORT_U1_PIPE_CLK_GATE_EN) || lost_power; + + if (context_lost) { + cdnsp_halt(pdev); + cdnsp_set_apb_timeout_value(pdev); + + /* Reset the internal controller memory state and registers. */ + ret = cdnsp_reset(pdev); + if (ret) + goto unlock; + + val = readl(&pdev->port3x_regs->mode_2); + val &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN; + writel(val, &pdev->port3x_regs->mode_2); + + cdnsp_clear_cmd_ring(pdev); + + memset(pdev->event_ring->first_seg->trbs, 0, + sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT)); + + cdnsp_init(pdev); + } + max_speed = pdev->gadget_driver->max_speed; /* Limit speed if necessary. */ @@ -2044,9 +2145,10 @@ static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power) ret = cdnsp_run(pdev, max_speed); - if (pdev->link_state == XDEV_U3) + if (!context_lost && pdev->link_state == XDEV_U3) __cdnsp_gadget_wakeup(pdev); +unlock: spin_unlock_irqrestore(&pdev->lock, flags); return ret; diff --git a/drivers/usb/cdns3/cdnsp-gadget.h b/drivers/usb/cdns3/cdnsp-gadget.h index a91cca509db08..c157023c93761 100644 --- a/drivers/usb/cdns3/cdnsp-gadget.h +++ b/drivers/usb/cdns3/cdnsp-gadget.h @@ -1509,6 +1509,7 @@ int cdnsp_endpoint_init(struct cdnsp_device *pdev, int cdnsp_ring_expansion(struct cdnsp_device *pdev, struct cdnsp_ring *ring, unsigned int num_trbs, gfp_t flags); +void cdnsp_ring_init(struct cdnsp_device *pdev, struct cdnsp_ring *ring); struct cdnsp_ring *cdnsp_dma_to_transfer_ring(struct cdnsp_ep *ep, u64 address); int cdnsp_alloc_stream_info(struct cdnsp_device *pdev, struct cdnsp_ep *pep, diff --git a/drivers/usb/cdns3/cdnsp-mem.c b/drivers/usb/cdns3/cdnsp-mem.c index 3f35c8af271e0..4b66307baaeb5 100644 --- a/drivers/usb/cdns3/cdnsp-mem.c +++ b/drivers/usb/cdns3/cdnsp-mem.c @@ -394,13 +394,6 @@ static struct cdnsp_ring *cdnsp_ring_alloc(struct cdnsp_device *pdev, if (ret) goto fail; - /* Only event ring does not use link TRB. */ - if (type != TYPE_EVENT) - ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= - cpu_to_le32(LINK_TOGGLE); - - cdnsp_initialize_ring_info(ring); - trace_cdnsp_ring_alloc(ring); return ring; fail: kfree(ring); @@ -604,6 +597,7 @@ int cdnsp_alloc_stream_info(struct cdnsp_device *pdev, if (!cur_ring) goto cleanup_rings; + cdnsp_ring_init(pdev, cur_ring); cur_ring->stream_id = cur_stream; cur_ring->trb_address_map = &stream_info->trb_address_map; @@ -699,6 +693,8 @@ static int cdnsp_alloc_priv_device(struct cdnsp_device *pdev) if (!pdev->eps[0].ring) goto fail; + cdnsp_ring_init(pdev, pdev->eps[0].ring); + /* Point to output device context in dcbaa. */ pdev->dcbaa->dev_context_ptrs[1] = cpu_to_le64(pdev->out_ctx.dma); pdev->cmd.in_ctx = &pdev->in_ctx; @@ -992,6 +988,8 @@ int cdnsp_endpoint_init(struct cdnsp_device *pdev, if (!pep->ring) return -ENOMEM; + cdnsp_ring_init(pdev, pep->ring); + pep->skip = false; /* Fill the endpoint context */ @@ -1099,28 +1097,6 @@ void cdnsp_mem_cleanup(struct cdnsp_device *pdev) pdev->active_port = NULL; } -static void cdnsp_set_event_deq(struct cdnsp_device *pdev) -{ - dma_addr_t deq; - u64 temp; - - deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, - pdev->event_ring->dequeue); - - /* Update controller event ring dequeue pointer */ - temp = cdnsp_read_64(&pdev->ir_set->erst_dequeue); - temp &= ERST_PTR_MASK; - - /* - * Don't clear the EHB bit (which is RW1C) because - * there might be more events to service. - */ - temp &= ~ERST_EHB; - - cdnsp_write_64(((u64)deq & (u64)~ERST_PTR_MASK) | temp, - &pdev->ir_set->erst_dequeue); -} - static void cdnsp_add_in_port(struct cdnsp_device *pdev, struct cdnsp_port *port, __le32 __iomem *addr) @@ -1200,6 +1176,36 @@ static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev) return 0; } +static void cdnsp_initialize_ring_segments(struct cdnsp_device *pdev, struct cdnsp_ring *ring) +{ + struct cdnsp_segment *seg; + + /* Only event ring does not use link TRB. */ + if (ring->type == TYPE_EVENT) + return; + + seg = ring->first_seg; + + while (seg) { + struct cdnsp_segment *next = seg->next; + + cdnsp_link_segments(pdev, seg, next, ring->type); + if (next == ring->first_seg) + break; + + seg = next; + } + + ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |= cpu_to_le32(LINK_TOGGLE); +} + +void cdnsp_ring_init(struct cdnsp_device *pdev, struct cdnsp_ring *ring) +{ + cdnsp_initialize_ring_segments(pdev, ring); + cdnsp_initialize_ring_info(ring); + trace_cdnsp_ring_alloc(ring); +} + /* * Initialize memory for CDNSP (one-time init). * @@ -1211,10 +1217,8 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) { struct device *dev = pdev->dev; int ret = -ENOMEM; - unsigned int val; dma_addr_t dma; u32 page_size; - u64 val_64; /* * Use 4K pages, since that's common and the minimum the @@ -1222,10 +1226,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) */ page_size = 1 << 12; - val = readl(&pdev->op_regs->config_reg); - val |= ((val & ~MAX_DEVS) | CDNSP_DEV_MAX_SLOTS) | CONFIG_U3E; - writel(val, &pdev->op_regs->config_reg); - /* * Doorbell array must be physically contiguous * and 64-byte (cache line) aligned. @@ -1237,8 +1237,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) pdev->dcbaa->dma = dma; - cdnsp_write_64(dma, &pdev->op_regs->dcbaa_ptr); - /* * Initialize the ring segment pool. The ring must be a contiguous * structure comprised of TRBs. The TRBs must be 16 byte aligned, @@ -1264,17 +1262,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) if (!pdev->cmd_ring) goto destroy_device_pool; - /* Set the address in the Command Ring Control register */ - val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring); - val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) | - (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) | - pdev->cmd_ring->cycle_state; - cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring); - - val = readl(&pdev->cap_regs->db_off); - val &= DBOFF_MASK; - pdev->dba = (void __iomem *)pdev->cap_regs + val; - /* Set ir_set to interrupt register set 0 */ pdev->ir_set = &pdev->run_regs->ir_set[0]; @@ -1291,21 +1278,6 @@ int cdnsp_mem_init(struct cdnsp_device *pdev) if (ret) goto free_event_ring; - /* Set ERST count with the number of entries in the segment table. */ - val = readl(&pdev->ir_set->erst_size); - val &= ERST_SIZE_MASK; - val |= ERST_NUM_SEGS; - writel(val, &pdev->ir_set->erst_size); - - /* Set the segment table base address. */ - val_64 = cdnsp_read_64(&pdev->ir_set->erst_base); - val_64 &= ERST_PTR_MASK; - val_64 |= (pdev->erst.erst_dma_addr & (u64)~ERST_PTR_MASK); - cdnsp_write_64(val_64, &pdev->ir_set->erst_base); - - /* Set the event ring dequeue address. */ - cdnsp_set_event_deq(pdev); - ret = cdnsp_setup_port_arrays(pdev); if (ret) goto free_erst; diff --git a/drivers/usb/chipidea/core.c b/drivers/usb/chipidea/core.c index 4d3a89b9d3860..628b83099764a 100644 --- a/drivers/usb/chipidea/core.c +++ b/drivers/usb/chipidea/core.c @@ -1265,6 +1265,7 @@ static void ci_hdrc_remove(struct platform_device *pdev) usb_role_switch_unregister(ci->role_switch); if (ci->supports_runtime_pm) { + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_get_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); pm_runtime_put_noidle(&pdev->dev); diff --git a/drivers/usb/class/cdc-acm.c b/drivers/usb/class/cdc-acm.c index 12f9fe62199ba..d6a2f1591460a 100644 --- a/drivers/usb/class/cdc-acm.c +++ b/drivers/usb/class/cdc-acm.c @@ -797,6 +797,9 @@ static void acm_port_shutdown(struct tty_port *port) "ctrl polling restart failed after port close\n"); /* port_shutdown() cleared DTR/RTS; restore them */ acm_set_control(acm, USB_CDC_CTRL_DTR | USB_CDC_CTRL_RTS); + if (acm_submit_read_urbs(acm, GFP_KERNEL)) + dev_dbg(&acm->control->dev, + "read urb restart failed after port close\n"); } } @@ -1564,6 +1567,9 @@ static int acm_probe(struct usb_interface *intf, if (usb_submit_urb(acm->ctrlurb, GFP_KERNEL)) dev_warn(&intf->dev, "failed to start persistent ctrl polling\n"); + if (acm_submit_read_urbs(acm, GFP_KERNEL)) + dev_warn(&intf->dev, + "failed to start persistent bulk read polling\n"); } return 0; diff --git a/drivers/usb/class/usbtmc.c b/drivers/usb/class/usbtmc.c index fe88d555142ad..5bfbf801f9c4f 100644 --- a/drivers/usb/class/usbtmc.c +++ b/drivers/usb/class/usbtmc.c @@ -71,7 +71,6 @@ struct usbtmc_dev_capabilities { * allocated for each USBTMC device in the driver's probe function. */ struct usbtmc_device_data { - const struct usb_device_id *id; struct usb_device *usb_dev; struct usb_interface *intf; struct list_head file_list; @@ -2398,7 +2397,6 @@ static int usbtmc_probe(struct usb_interface *intf, return -ENOMEM; data->intf = intf; - data->id = id; data->usb_dev = usb_get_dev(interface_to_usbdev(intf)); usb_set_intfdata(intf, data); kref_init(&data->kref); diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c index 1c0f36c85bcfb..a09633f1dd940 100644 --- a/drivers/usb/core/config.c +++ b/drivers/usb/core/config.c @@ -927,6 +927,18 @@ int usb_get_configuration(struct usb_device *dev) unsigned char *bigbuffer; struct usb_config_descriptor *desc; int result; + size_t usb_config_req_size; + + /* + * We usually start by grabbing the first 9-bytes descriptor so we know + * how long the whole configuration is. Some devices with quirky + * firmware will fail enumeration, so if the quirk is set, use 255 instead, + * mirroring the behavior of Windows. + */ + if (dev->quirks & USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE) + usb_config_req_size = 255; + else + usb_config_req_size = USB_DT_CONFIG_SIZE; if (ncfg > USB_MAXCONFIG) { dev_notice(ddev, "too many configurations: %d, " @@ -953,15 +965,13 @@ int usb_get_configuration(struct usb_device *dev) if (!dev->rawdescriptors) return -ENOMEM; - desc = kmalloc(USB_DT_CONFIG_SIZE, GFP_KERNEL); + desc = kmalloc(usb_config_req_size, GFP_KERNEL); if (!desc) return -ENOMEM; for (cfgno = 0; cfgno < ncfg; cfgno++) { - /* We grab just the first descriptor so we know how long - * the whole configuration is */ result = usb_get_descriptor(dev, USB_DT_CONFIG, cfgno, - desc, USB_DT_CONFIG_SIZE); + desc, usb_config_req_size); if (result < 0) { dev_err(ddev, "unable to read config index %d " "descriptor/%s: %d\n", cfgno, "start", result); @@ -971,16 +981,14 @@ int usb_get_configuration(struct usb_device *dev) dev->descriptor.bNumConfigurations = cfgno; break; } else if (result < 4) { - dev_err(ddev, "config index %d descriptor too short " - "(expected %i, got %i)\n", cfgno, - USB_DT_CONFIG_SIZE, result); + dev_err(ddev, "config index %d descriptor too short (asked for %zu, got %i)\n", + cfgno, usb_config_req_size, result); result = -EINVAL; goto err; } length = max_t(int, le16_to_cpu(desc->wTotalLength), USB_DT_CONFIG_SIZE); - /* Now that we know the length, get the whole thing */ bigbuffer = kmalloc(length, GFP_KERNEL); if (!bigbuffer) { result = -ENOMEM; @@ -990,6 +998,13 @@ int usb_get_configuration(struct usb_device *dev) if (dev->quirks & USB_QUIRK_DELAY_INIT) msleep(200); + /* Skip the second read if we already got everything */ + if (result >= length) { + memcpy(bigbuffer, desc, length); + goto store_and_parse; + } + + /* Get the whole thing */ result = usb_get_descriptor(dev, USB_DT_CONFIG, cfgno, bigbuffer, length); if (result < 0) { @@ -1004,6 +1019,7 @@ int usb_get_configuration(struct usb_device *dev) length = result; } +store_and_parse: dev->rawdescriptors[cfgno] = bigbuffer; result = usb_parse_configuration(dev, cfgno, diff --git a/drivers/usb/core/devio.c b/drivers/usb/core/devio.c index f6ce6e26e0d45..53dcd0019e111 100644 --- a/drivers/usb/core/devio.c +++ b/drivers/usb/core/devio.c @@ -1113,7 +1113,6 @@ static int usbdev_release(struct inode *inode, struct file *file) if (!ps->suspend_allowed) usb_autosuspend_device(dev); usb_unlock_device(dev); - usb_put_dev(dev); put_pid(ps->disc_pid); put_cred(ps->cred); @@ -1122,6 +1121,7 @@ static int usbdev_release(struct inode *inode, struct file *file) free_async(as); as = async_getcompleted(ps); } + usb_put_dev(dev); kfree(ps); return 0; diff --git a/drivers/usb/core/driver.c b/drivers/usb/core/driver.c index 74b8bdc27dbf5..acd05f87b8e14 100644 --- a/drivers/usb/core/driver.c +++ b/drivers/usb/core/driver.c @@ -228,14 +228,16 @@ static void usb_free_dynids(struct usb_driver *usb_drv) } static const struct usb_device_id *usb_match_dynamic_id(struct usb_interface *intf, - const struct usb_driver *drv) + const struct usb_driver *drv, + struct usb_device_id *id_copy) { struct usb_dynid *dynid; guard(mutex)(&usb_dynids_lock); list_for_each_entry(dynid, &drv->dynids.list, node) { if (usb_match_one_id(intf, &dynid->id)) { - return &dynid->id; + *id_copy = dynid->id; + return id_copy; } } return NULL; @@ -321,6 +323,7 @@ static int usb_probe_interface(struct device *dev) struct usb_interface *intf = to_usb_interface(dev); struct usb_device *udev = interface_to_usbdev(intf); const struct usb_device_id *id; + struct usb_device_id id_copy; int error = -ENODEV; int lpm_disable_error = -ENODEV; @@ -340,7 +343,7 @@ static int usb_probe_interface(struct device *dev) return error; } - id = usb_match_dynamic_id(intf, driver); + id = usb_match_dynamic_id(intf, driver, &id_copy); if (!id) id = usb_match_id(intf, driver->id_table); if (!id) @@ -892,6 +895,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) struct usb_interface *intf; const struct usb_driver *usb_drv; const struct usb_device_id *id; + struct usb_device_id id_copy; /* device drivers never match interfaces */ if (is_usb_device_driver(drv)) @@ -904,7 +908,7 @@ static int usb_device_match(struct device *dev, const struct device_driver *drv) if (id) return 1; - id = usb_match_dynamic_id(intf, usb_drv); + id = usb_match_dynamic_id(intf, usb_drv, &id_copy); if (id) return 1; } diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c index dc217ad06ac53..19abce5345c88 100644 --- a/drivers/usb/core/hcd.c +++ b/drivers/usb/core/hcd.c @@ -450,7 +450,8 @@ rh_string(int id, struct usb_hcd const *hcd, u8 *data, unsigned len) /* Root hub control transfers execute synchronously */ -static int rh_call_control (struct usb_hcd *hcd, struct urb *urb) +static int rh_call_control(struct usb_hcd *hcd, + struct urb *urb, gfp_t mem_flags) { struct usb_ctrlrequest *cmd; u16 typeReq, wValue, wIndex, wLength; @@ -485,8 +486,8 @@ static int rh_call_control (struct usb_hcd *hcd, struct urb *urb) * tbuf should be at least as big as the * USB hub descriptor. */ - tbuf_size = max_t(u16, sizeof(struct usb_hub_descriptor), wLength); - tbuf = kzalloc(tbuf_size, GFP_KERNEL); + tbuf_size = max_t(u16, sizeof(struct usb_hub_descriptor), wLength); + tbuf = kzalloc(tbuf_size, mem_flags); if (!tbuf) { status = -ENOMEM; goto err_alloc; @@ -811,12 +812,13 @@ static int rh_queue_status (struct usb_hcd *hcd, struct urb *urb) return retval; } -static int rh_urb_enqueue (struct usb_hcd *hcd, struct urb *urb) +static int rh_urb_enqueue(struct usb_hcd *hcd, + struct urb *urb, gfp_t mem_flags) { if (usb_endpoint_xfer_int(&urb->ep->desc)) return rh_queue_status (hcd, urb); if (usb_endpoint_xfer_control(&urb->ep->desc)) - return rh_call_control (hcd, urb); + return rh_call_control(hcd, urb, mem_flags); return -EINVAL; } @@ -1537,7 +1539,7 @@ int usb_hcd_submit_urb (struct urb *urb, gfp_t mem_flags) */ if (is_root_hub(urb->dev)) { - status = rh_urb_enqueue(hcd, urb); + status = rh_urb_enqueue(hcd, urb, mem_flags); } else { status = map_urb_for_dma(hcd, urb, mem_flags); if (likely(status == 0)) { diff --git a/drivers/usb/core/hub.c b/drivers/usb/core/hub.c index 49107595caa8c..de8bec972a42a 100644 --- a/drivers/usb/core/hub.c +++ b/drivers/usb/core/hub.c @@ -757,10 +757,12 @@ void usb_wakeup_notification(struct usb_device *hdev, { struct usb_hub *hub; struct usb_port *port_dev; + unsigned long flags; if (!hdev) return; + spin_lock_irqsave(&device_state_lock, flags); hub = usb_hub_to_struct_hub(hdev); if (hub) { port_dev = hub->ports[portnum - 1]; @@ -770,6 +772,7 @@ void usb_wakeup_notification(struct usb_device *hdev, set_bit(portnum, hub->wakeup_bits); kick_hub_wq(hub); } + spin_unlock_irqrestore(&device_state_lock, flags); } EXPORT_SYMBOL_GPL(usb_wakeup_notification); @@ -995,10 +998,12 @@ static int hub_hub_status(struct usb_hub *hub, mutex_lock(&hub->status_mutex); ret = get_hub_status(hub->hdev, &hub->status->hub); - if (ret < 0) { + if (ret < (int)sizeof(hub->status->hub)) { if (ret != -ENODEV) dev_err(hub->intfdev, "%s failed (err = %d)\n", __func__, ret); + if (ret >= 0) + ret = -EIO; } else { *status = le16_to_cpu(hub->status->hub.wHubStatus); *change = le16_to_cpu(hub->status->hub.wHubChange); diff --git a/drivers/usb/core/port.c b/drivers/usb/core/port.c index a47df5d32f7c4..7cae194843556 100644 --- a/drivers/usb/core/port.c +++ b/drivers/usb/core/port.c @@ -726,6 +726,8 @@ static void connector_unbind(struct device *dev, struct device *connector, void sysfs_remove_link(&connector->kobj, dev_name(dev)); sysfs_remove_link(&dev->kobj, "connector"); + if (port_dev->child) + typec_deattach(port_dev->connector, &port_dev->child->dev); port_dev->connector = NULL; } diff --git a/drivers/usb/core/quirks.c b/drivers/usb/core/quirks.c index c921732836f7a..0f804a52ab763 100644 --- a/drivers/usb/core/quirks.c +++ b/drivers/usb/core/quirks.c @@ -143,6 +143,10 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) break; case 'q': flags |= USB_QUIRK_FORCE_ONE_CONFIG; + break; + case 'r': + flags |= USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE; + break; /* Ignore unrecognized flag characters */ } } @@ -590,6 +594,9 @@ static const struct usb_device_id usb_quirk_list[] = { { USB_DEVICE(0x2386, 0x350e), .driver_info = USB_QUIRK_NO_LPM }, + /* ShanWan Wireless Gamepad */ + { USB_DEVICE(0x2563, 0x0575), .driver_info = USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE }, + /* UGREEN 35871 - BOS descriptor fetch hangs at SuperSpeed Plus */ { USB_DEVICE(0x2b89, 0x5871), .driver_info = USB_QUIRK_NO_BOS }, diff --git a/drivers/usb/core/sysfs.c b/drivers/usb/core/sysfs.c index a07866f1060cf..d22dc78457d79 100644 --- a/drivers/usb/core/sysfs.c +++ b/drivers/usb/core/sysfs.c @@ -899,10 +899,15 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj, { struct device *dev = kobj_to_dev(kobj); struct usb_device *udev = to_usb_device(dev); - struct usb_host_bos *bos = udev->bos; + struct usb_host_bos *bos; struct usb_bos_descriptor *desc; size_t desclen, n = 0; + int rc; + rc = usb_lock_device_interruptible(udev); + if (rc < 0) + return -EINTR; + bos = udev->bos; if (bos) { desc = bos->desc; desclen = le16_to_cpu(desc->wTotalLength); @@ -911,6 +916,7 @@ bos_descriptors_read(struct file *filp, struct kobject *kobj, memcpy(buf, (void *) desc + off, n); } } + usb_unlock_device(udev); return n; } static const BIN_ATTR_RO(bos_descriptors, 65535); /* max-size BOS */ diff --git a/drivers/usb/dwc2/gadget.c b/drivers/usb/dwc2/gadget.c index 162dc25c5207c..94e73c82f30f4 100644 --- a/drivers/usb/dwc2/gadget.c +++ b/drivers/usb/dwc2/gadget.c @@ -4680,6 +4680,7 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) { struct dwc2_hsotg *hsotg = to_hsotg(gadget); unsigned long flags; + int ret = 0; dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on, hsotg->op_state); @@ -4691,6 +4692,13 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) } spin_lock_irqsave(&hsotg->lock, flags); + if (hsotg->in_ppd) { + ret = dwc2_exit_partial_power_down(hsotg, 0, true); + if (ret) { + dev_err(hsotg->dev, "exit partial_power_down failed\n"); + goto exit; + } + } if (is_on) { hsotg->enabled = 1; dwc2_hsotg_core_init_disconnected(hsotg, false); @@ -4704,9 +4712,10 @@ static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) } hsotg->gadget.speed = USB_SPEED_UNKNOWN; +exit: spin_unlock_irqrestore(&hsotg->lock, flags); - return 0; + return ret; } static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active) diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index a8ff8db610d34..fe479bba10c3a 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -304,7 +304,7 @@ void dwc3_ep0_out_start(struct dwc3 *dwc) dwc3_ep->flags &= ~DWC3_EP_DELAY_STOP; if (dwc->connected) - dwc3_stop_active_transfer(dwc3_ep, true, true); + dwc3_stop_active_transfer(dwc3_ep, false, true); else dwc3_remove_requests(dwc, dwc3_ep, -ESHUTDOWN); } diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index ea8487d4bbb3c..050ea2b463bf9 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -1004,7 +1004,7 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) * controller to generate an ERDY to initiate the * stream. */ - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); /* * All stream eps will reinitiate stream on NoStream @@ -1032,7 +1032,7 @@ void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) { struct dwc3_request *req; - dwc3_stop_active_transfer(dep, true, false); + dwc3_stop_active_transfer(dep, false, false); /* If endxfer is delayed, avoid unmapping requests */ if (dep->flags & DWC3_EP_DELAY_STOP) @@ -1720,7 +1720,7 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) if (ret == -EAGAIN) return ret; - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); list_for_each_entry_safe(req, tmp, &dep->started_list, list) dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); @@ -1757,6 +1757,11 @@ static int __dwc3_gadget_get_frame(struct dwc3 *dwc) * the controller won't update the TRB progress on command * completion. It also won't clear the HWO bit in the TRB. * The command will also not complete immediately in that case. + * + * Older programming guide revisions recommended setting ForceRM to 1 + * when ending a transfer. Newer programming guide revisions now + * recommend keeping ForceRM cleared, and TRBs are properly updated + * on command completion. */ static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) { @@ -1882,7 +1887,7 @@ static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) * to wait for the next XferNotReady to test the command again */ if (cmd_status == 0) { - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); return 0; } } @@ -2165,7 +2170,7 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, struct dwc3_request *t; /* wait until it is processed */ - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); /* * Remove any started request if the transfer is @@ -2242,7 +2247,7 @@ int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) return 0; } - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); list_for_each_entry_safe(req, tmp, &dep->started_list, list) dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); @@ -3357,7 +3362,7 @@ static void dwc3_nostream_work(struct work_struct *work) dwc3_send_gadget_generic_command(dwc, cmd, dep->number); } else { dep->flags |= DWC3_EP_DELAY_START; - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); spin_unlock_irqrestore(&dwc->lock, flags); return; } @@ -3497,6 +3502,7 @@ static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) } dwc3_debugfs_remove_endpoint_dir(dep); + cancel_delayed_work_sync(&dep->nostream_work); kfree(dep); } } @@ -3714,7 +3720,7 @@ static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && list_empty(&dep->started_list) && (list_empty(&dep->pending_list) || status == -EXDEV)) - dwc3_stop_active_transfer(dep, true, true); + dwc3_stop_active_transfer(dep, false, true); else if (dwc3_gadget_ep_should_continue(dep)) if (__dwc3_gadget_kick_transfer(dep) == 0) no_started_trb = false; diff --git a/drivers/usb/gadget/configfs.c b/drivers/usb/gadget/configfs.c index 6bcac85c55501..aa0a25efb6aa0 100644 --- a/drivers/usb/gadget/configfs.c +++ b/drivers/usb/gadget/configfs.c @@ -1177,7 +1177,7 @@ static ssize_t os_desc_qw_sign_show(struct config_item *item, char *page) struct gadget_info *gi = os_desc_item_to_gadget_info(item); int res; - res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN, + res = utf16s_to_utf8s((wchar_t *) gi->qw_sign, OS_STRING_QW_SIGN_LEN / 2, UTF16_LITTLE_ENDIAN, page, PAGE_SIZE - 1); page[res++] = '\n'; @@ -1199,7 +1199,7 @@ static ssize_t os_desc_qw_sign_store(struct config_item *item, const char *page, mutex_lock(&gi->lock); res = utf8s_to_utf16s(page, l, UTF16_LITTLE_ENDIAN, (wchar_t *) gi->qw_sign, - OS_STRING_QW_SIGN_LEN); + OS_STRING_QW_SIGN_LEN / 2); if (res > 0) res = len; mutex_unlock(&gi->lock); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 3116060977185..d73b082d5d835 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -549,6 +549,7 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf, if (ffs_setup_state_clear_cancelled(ffs) == FFS_SETUP_CANCELLED) return -EIDRM; +retry: /* Acquire mutex */ ret = ffs_mutex_lock(&ffs->mutex, file->f_flags & O_NONBLOCK); if (ret < 0) @@ -583,10 +584,15 @@ static ssize_t ffs_ep0_read(struct file *file, char __user *buf, break; } - if (wait_event_interruptible_exclusive_locked_irq(ffs->ev.waitq, - ffs->ev.count)) { - ret = -EINTR; - break; + if (!ffs->ev.count) { + spin_unlock_irq(&ffs->ev.waitq.lock); + mutex_unlock(&ffs->mutex); + + if (wait_event_interruptible_exclusive(ffs->ev.waitq, + ffs->ev.count)) + return -EINTR; + + goto retry; } /* unlocks spinlock */ @@ -857,9 +863,15 @@ static void ffs_user_copy_worker(struct work_struct *work) bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD; if (io_data->read && ret > 0) { - kthread_use_mm(io_data->mm); - ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data); - kthread_unuse_mm(io_data->mm); + if (mmget_not_zero(io_data->mm)) { + kthread_use_mm(io_data->mm); + ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data); + kthread_unuse_mm(io_data->mm); + mmput(io_data->mm); + } else { + ret = -EFAULT; + } + mmdrop(io_data->mm); } io_data->kiocb->ki_complete(io_data->kiocb, ret); @@ -1240,16 +1252,22 @@ static ssize_t ffs_epfile_write_iter(struct kiocb *kiocb, struct iov_iter *from) kiocb->private = p; - if (p->aio) + if (p->aio) { + mmgrab(p->mm); kiocb_set_cancel_fn(kiocb, ffs_aio_cancel); + } res = ffs_epfile_io(kiocb->ki_filp, p); if (res == -EIOCBQUEUED) return res; - if (p->aio) + if (p->aio) { + kiocb->ki_complete(kiocb, res); + mmdrop(p->mm); kfree(p); - else + return -EIOCBQUEUED; + } else { *from = p->data; + } return res; } @@ -1284,16 +1302,21 @@ static ssize_t ffs_epfile_read_iter(struct kiocb *kiocb, struct iov_iter *to) kiocb->private = p; - if (p->aio) + if (p->aio) { + mmgrab(p->mm); kiocb_set_cancel_fn(kiocb, ffs_aio_cancel); + } res = ffs_epfile_io(kiocb->ki_filp, p); if (res == -EIOCBQUEUED) return res; if (p->aio) { + kiocb->ki_complete(kiocb, res); + mmdrop(p->mm); kfree(p->to_free); kfree(p); + return -EIOCBQUEUED; } else { *to = p->data; } @@ -1660,13 +1683,13 @@ static int ffs_dmabuf_transfer(struct file *file, /* In the meantime, endpoint got disabled or changed. */ if (epfile->ep != ep) { ret = -ESHUTDOWN; - goto err_fence_put; + goto err_fence_free; } usb_req = usb_ep_alloc_request(ep->ep, GFP_ATOMIC); if (!usb_req) { ret = -ENOMEM; - goto err_fence_put; + goto err_fence_free; } /* @@ -1715,9 +1738,9 @@ static int ffs_dmabuf_transfer(struct file *file, return ret; -err_fence_put: +err_fence_free: spin_unlock_irq(&epfile->ffs->eps_lock); - dma_fence_put(&fence->base); + kfree(fence); err_resv_unlock: dma_resv_unlock(dmabuf->resv); err_attachment_put: diff --git a/drivers/usb/gadget/function/f_mass_storage.c b/drivers/usb/gadget/function/f_mass_storage.c index 6f275c3d11ac5..6970d0fa806b0 100644 --- a/drivers/usb/gadget/function/f_mass_storage.c +++ b/drivers/usb/gadget/function/f_mass_storage.c @@ -2748,6 +2748,9 @@ int fsg_common_set_num_buffers(struct fsg_common *common, unsigned int n) struct fsg_buffhd *bh, *buffhds; int i; + if (n < 2) + return -EINVAL; + buffhds = kcalloc(n, sizeof(*buffhds), GFP_KERNEL); if (!buffhds) return -ENOMEM; @@ -2961,7 +2964,7 @@ EXPORT_SYMBOL_GPL(fsg_common_create_lun); int fsg_common_create_luns(struct fsg_common *common, struct fsg_config *cfg) { - char buf[8]; /* enough for 100000000 different numbers, decimal */ + char buf[14]; int i, rc; fsg_common_remove_luns(common); diff --git a/drivers/usb/gadget/function/f_midi.c b/drivers/usb/gadget/function/f_midi.c index da82598fcef8a..e1d1d2b16eafb 100644 --- a/drivers/usb/gadget/function/f_midi.c +++ b/drivers/usb/gadget/function/f_midi.c @@ -878,7 +878,6 @@ static int f_midi_bind(struct usb_configuration *c, struct usb_function *f) int status, n, jack = 1, i = 0, endpoint_descriptor_index = 0; midi->gadget = cdev->gadget; - INIT_WORK(&midi->work, f_midi_in_work); status = f_midi_register_card(midi); if (status < 0) goto fail_register; @@ -1302,6 +1301,7 @@ static void f_midi_free(struct usb_function *f) opts = container_of(f->fi, struct f_midi_opts, func_inst); mutex_lock(&opts->lock); if (!--midi->free_ref) { + cancel_work_sync(&midi->work); kfree(midi->id); kfifo_free(&midi->in_req_fifo); kfree(midi); @@ -1370,6 +1370,7 @@ static struct usb_function *f_midi_alloc(struct usb_function_instance *fi) status = -ENOMEM; goto midi_free; } + INIT_WORK(&midi->work, f_midi_in_work); midi->out_ports = opts->out_ports; midi->index = opts->index; midi->buflen = opts->buflen; diff --git a/drivers/usb/gadget/function/f_midi2.c b/drivers/usb/gadget/function/f_midi2.c index de16b02d857e0..076d0aca647f3 100644 --- a/drivers/usb/gadget/function/f_midi2.c +++ b/drivers/usb/gadget/function/f_midi2.c @@ -1145,7 +1145,7 @@ static int f_midi2_alloc_ep_reqs(struct f_midi2_usb_ep *usb_ep) if (!usb_ep->reqs) return -EINVAL; - for (i = 0; i < midi2->info.num_reqs; i++) { + for (i = 0; i < usb_ep->num_reqs; i++) { if (usb_ep->reqs[i].req) continue; usb_ep->reqs[i].req = alloc_ep_req(usb_ep->usb_ep, @@ -1160,10 +1160,9 @@ static int f_midi2_alloc_ep_reqs(struct f_midi2_usb_ep *usb_ep) /* Free allocated requests */ static void f_midi2_free_ep_reqs(struct f_midi2_usb_ep *usb_ep) { - struct f_midi2 *midi2 = usb_ep->card; int i; - for (i = 0; i < midi2->info.num_reqs; i++) { + for (i = 0; i < usb_ep->num_reqs; i++) { if (!usb_ep->reqs[i].req) continue; free_ep_req(usb_ep->usb_ep, usb_ep->reqs[i].req); @@ -2179,13 +2178,13 @@ static ssize_t f_midi2_opts_bool_store(struct f_midi2_opts *opts, /* generic show/store for string */ static ssize_t f_midi2_opts_str_show(struct f_midi2_opts *opts, - const char *str, char *page) + const char **strp, char *page) { int result = 0; mutex_lock(&opts->lock); - if (str) - result = scnprintf(page, PAGE_SIZE, "%s\n", str); + if (*strp) + result = scnprintf(page, PAGE_SIZE, "%s\n", *strp); mutex_unlock(&opts->lock); return result; } @@ -2279,7 +2278,7 @@ static ssize_t f_midi2_block_opts_name_show(struct config_item *item, { struct f_midi2_block_opts *opts = to_f_midi2_block_opts(item); - return f_midi2_opts_str_show(opts->ep->opts, opts->info.name, page); + return f_midi2_opts_str_show(opts->ep->opts, &opts->info.name, page); } static ssize_t f_midi2_block_opts_name_store(struct config_item *item, @@ -2436,7 +2435,7 @@ static ssize_t f_midi2_ep_opts_##name##_show(struct config_item *item, \ char *page) \ { \ struct f_midi2_ep_opts *opts = to_f_midi2_ep_opts(item); \ - return f_midi2_opts_str_show(opts->opts, opts->info.name, page);\ + return f_midi2_opts_str_show(opts->opts, &opts->info.name, page);\ } \ \ static ssize_t f_midi2_ep_opts_##name##_store(struct config_item *item, \ @@ -2474,6 +2473,7 @@ static void f_midi2_ep_opts_release(struct config_item *item) { struct f_midi2_ep_opts *opts = to_f_midi2_ep_opts(item); + configfs_remove_default_groups(&opts->group); kfree(opts->info.ep_name); kfree(opts->info.product_id); kfree(opts); @@ -2590,7 +2590,7 @@ static ssize_t f_midi2_opts_iface_name_show(struct config_item *item, { struct f_midi2_opts *opts = to_f_midi2_opts(item); - return f_midi2_opts_str_show(opts, opts->info.iface_name, page); + return f_midi2_opts_str_show(opts, &opts->info.iface_name, page); } static ssize_t f_midi2_opts_iface_name_store(struct config_item *item, @@ -2640,6 +2640,7 @@ static void f_midi2_free_inst(struct usb_function_instance *f) opts = container_of(f, struct f_midi2_opts, func_inst); + configfs_remove_default_groups(&opts->func_inst.group); kfree(opts->info.iface_name); kfree(opts); } diff --git a/drivers/usb/gadget/function/f_ncm.c b/drivers/usb/gadget/function/f_ncm.c index 04be7047aa3d2..cc435680db688 100644 --- a/drivers/usb/gadget/function/f_ncm.c +++ b/drivers/usb/gadget/function/f_ncm.c @@ -1171,7 +1171,7 @@ static int ncm_unwrap_ntb(struct gether *port, unsigned char *ntb_ptr = skb->data; __le16 *tmp; unsigned index, index2; - int ndp_index; + unsigned int ndp_index; unsigned dg_len, dg_len2; unsigned ndp_len; unsigned block_len; @@ -1189,6 +1189,10 @@ static int ncm_unwrap_ntb(struct gether *port, frame_max = ncm_opts->max_segment_size; parse_ntb: + if (to_process < (int)opts->nth_size) { + INFO(port->func.config->cdev, "Packet too small for headers\n"); + goto err; + } tmp = (__le16 *)ntb_ptr; /* dwSignature */ @@ -1209,8 +1213,12 @@ static int ncm_unwrap_ntb(struct gether *port, tmp++; /* skip wSequence */ block_len = get_ncm(&tmp, opts->block_length); + if (block_len == 0) + block_len = to_process; + /* (d)wBlockLength */ - if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max)) { + if ((block_len < opts->nth_size + opts->ndp_size) || (block_len > ntb_max) || + (block_len > to_process)) { INFO(port->func.config->cdev, "Bad block length: %#X\n", block_len); goto err; } @@ -1273,7 +1281,7 @@ static int ncm_unwrap_ntb(struct gether *port, index = index2; /* wDatagramIndex[0] */ if ((index < opts->nth_size) || - (index > block_len - opts->dpe_size)) { + (index > block_len)) { INFO(port->func.config->cdev, "Bad index: %#X\n", index); goto err; @@ -1285,7 +1293,8 @@ static int ncm_unwrap_ntb(struct gether *port, * ethernet hdr + crc or larger than max frame size */ if ((dg_len < 14 + crc_len) || - (dg_len > frame_max)) { + (dg_len > frame_max) || + (dg_len > block_len - index)) { INFO(port->func.config->cdev, "Bad dgram length: %#X\n", dg_len); goto err; @@ -1310,7 +1319,7 @@ static int ncm_unwrap_ntb(struct gether *port, dg_len2 = get_ncm(&tmp, opts->dgram_item_len); /* wDatagramIndex[1] */ - if (index2 > block_len - opts->dpe_size) { + if (index2 > block_len) { INFO(port->func.config->cdev, "Bad index: %#X\n", index2); goto err; diff --git a/drivers/usb/gadget/function/f_printer.c b/drivers/usb/gadget/function/f_printer.c index a3884afa42b4b..3db07d0fba89c 100644 --- a/drivers/usb/gadget/function/f_printer.c +++ b/drivers/usb/gadget/function/f_printer.c @@ -431,7 +431,7 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr) { struct printer_dev *dev = fd->private_data; unsigned long flags; - size_t size; + size_t size, not_copied, copied; size_t bytes_copied; struct usb_request *req; /* This is a pointer to the current USB rx request. */ @@ -524,10 +524,12 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr) else size = len; - size -= copy_to_user(buf, current_rx_buf, size); - bytes_copied += size; - len -= size; - buf += size; + not_copied = copy_to_user(buf, current_rx_buf, size); + copied = size - not_copied; + + bytes_copied += copied; + len -= copied; + buf += copied; spin_lock_irqsave(&dev->lock, flags); @@ -542,6 +544,17 @@ printer_read(struct file *fd, char __user *buf, size_t len, loff_t *ptr) if (dev->interface < 0) goto out_disabled; + if (!copied) { + dev->current_rx_req = current_rx_req; + dev->current_rx_bytes = current_rx_bytes; + dev->current_rx_buf = current_rx_buf; + spin_unlock_irqrestore(&dev->lock, flags); + mutex_unlock(&dev->lock_printer_io); + return bytes_copied ? bytes_copied : -EFAULT; + } + + size = copied; + /* If we not returning all the data left in this RX request * buffer then adjust the amount of data left in the buffer. * Othewise if we are done with this RX request buffer then diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 7b27f8082ace6..02c5d9cfda45a 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1674,19 +1674,25 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, opts = container_of(tpg_instances[i].func_inst, struct f_tcm_opts, func_inst); - mutex_lock(&opts->dep_lock); - if (!opts->ready) - goto unlock_dep; + if (!READ_ONCE(opts->ready)) + goto unlock_inst; if (opts->has_dep) { if (!try_module_get(opts->dependent)) - goto unlock_dep; + goto unlock_inst; } else { + /* + * configfs_depend_item_unlocked() may acquire the configfs + * root inode lock when the target belongs to a different + * subsystem. Calling it under dep_lock would create a + * circular dependency: + * dep_lock -> configfs inode lock -> su_mutex -> dep_lock + */ ret = configfs_depend_item_unlocked( wwn->wwn_group.cg_subsys, &opts->func_inst.group.cg_item); if (ret) - goto unlock_dep; + goto unlock_inst; } tpg = kzalloc(sizeof(struct usbg_tpg), GFP_KERNEL); @@ -1713,7 +1719,6 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, tpg_instances[i].tpg = tpg; tpg->fi = tpg_instances[i].func_inst; - mutex_unlock(&opts->dep_lock); mutex_unlock(&tpg_instances_lock); return &tpg->se_tpg; @@ -1726,8 +1731,6 @@ static struct se_portal_group *usbg_make_tpg(struct se_wwn *wwn, module_put(opts->dependent); else configfs_undepend_item_unlocked(&opts->func_inst.group.cg_item); -unlock_dep: - mutex_unlock(&opts->dep_lock); unlock_inst: mutex_unlock(&tpg_instances_lock); @@ -2023,7 +2026,7 @@ static const struct target_core_fabric_ops usbg_ops = { .fabric_enable_tpg = usbg_enable_tpg, .fabric_drop_tpg = usbg_drop_tpg, .fabric_post_link = usbg_port_link, - .fabric_pre_unlink = usbg_port_unlink, + .fabric_post_unlink = usbg_port_unlink, .fabric_init_nodeacl = usbg_init_nodeacl, .tfc_wwn_attrs = usbg_wwn_attrs, @@ -2361,31 +2364,158 @@ static int tcm_bind(struct usb_configuration *c, struct usb_function *f) return -ENOTSUPP; } -struct guas_setup_wq { - struct work_struct work; - struct f_uas *fu; - unsigned int alt; -}; +static void tcm_cleanup_old_alt(struct f_uas *fu) +{ + if (fu->flags & USBG_IS_UAS) + uasp_cleanup_old_alt(fu); + else if (fu->flags & USBG_IS_BOT) + bot_cleanup_old_alt(fu); + fu->flags = 0; +} + +static void tcm_delayed_set_alt_done(struct f_uas *fu) +{ + unsigned long flags; + + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE; + fu->delayed_set_alt_cancel = false; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); +} + +static bool tcm_delayed_set_alt_cancelled(struct f_uas *fu) +{ + bool cancelled; + unsigned long flags; + + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + cancelled = fu->delayed_set_alt_cancel; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + + return cancelled; +} + +static bool tcm_complete_delayed_status(struct f_uas *fu) +{ + struct usb_composite_dev *cdev = fu->function.config->cdev; + struct usb_request *req = cdev->req; + unsigned long cdev_flags; + bool cancelled; + int ret; + + spin_lock_irqsave(&cdev->lock, cdev_flags); + spin_lock(&fu->delayed_set_alt_lock); + cancelled = fu->delayed_set_alt_cancel; + if (!cancelled) { + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE; + fu->delayed_set_alt_cancel = false; + } + spin_unlock(&fu->delayed_set_alt_lock); + + if (cancelled) { + spin_unlock_irqrestore(&cdev->lock, cdev_flags); + return false; + } + + if (cdev->delayed_status == 0) { + WARN(cdev, "%s: Unexpected call\n", __func__); + } else if (--cdev->delayed_status == 0) { + req->length = 0; + req->context = cdev; + ret = usb_ep_queue(cdev->gadget->ep0, req, GFP_ATOMIC); + if (ret == 0) { + cdev->setup_pending = true; + } else { + req->status = 0; + req->complete(cdev->gadget->ep0, req); + } + } + + spin_unlock_irqrestore(&cdev->lock, cdev_flags); + + return true; +} + +static bool tcm_cancel_delayed_set_alt(struct f_uas *fu) +{ + bool cleanup = false; + bool cancel = false; + unsigned long flags; + + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + switch (fu->delayed_set_alt_state) { + case USBG_DELAYED_SET_ALT_IDLE: + cleanup = true; + break; + case USBG_DELAYED_SET_ALT_QUEUED: + case USBG_DELAYED_SET_ALT_RUNNING: + fu->delayed_set_alt_cancel = true; + cancel = true; + break; + } + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + + if (cancel && cancel_work(&fu->delayed_set_alt)) { + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + if (fu->delayed_set_alt_state == USBG_DELAYED_SET_ALT_QUEUED) { + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE; + fu->delayed_set_alt_cancel = false; + cleanup = true; + } + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + } + + return cleanup; +} + +static void tcm_cancel_delayed_set_alt_sync(struct f_uas *fu) +{ + unsigned long flags; + + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_IDLE) + fu->delayed_set_alt_cancel = true; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + + cancel_work_sync(&fu->delayed_set_alt); + + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_IDLE; + fu->delayed_set_alt_cancel = false; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); +} static void tcm_delayed_set_alt(struct work_struct *wq) { - struct guas_setup_wq *work = container_of(wq, struct guas_setup_wq, - work); - struct f_uas *fu = work->fu; - int alt = work->alt; + struct f_uas *fu = container_of(wq, struct f_uas, delayed_set_alt); + unsigned long flags; + unsigned int alt; - kfree(work); + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_QUEUED) { + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + return; + } + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_RUNNING; + alt = fu->delayed_alt; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); - if (fu->flags & USBG_IS_BOT) - bot_cleanup_old_alt(fu); - if (fu->flags & USBG_IS_UAS) - uasp_cleanup_old_alt(fu); + tcm_cleanup_old_alt(fu); + + if (tcm_delayed_set_alt_cancelled(fu)) + goto out_done; if (alt == USB_G_ALT_INT_BBB) bot_set_alt(fu); else if (alt == USB_G_ALT_INT_UAS) uasp_set_alt(fu); - usb_composite_setup_continue(fu->function.config->cdev); + + if (tcm_complete_delayed_status(fu)) + return; + + tcm_cleanup_old_alt(fu); +out_done: + tcm_delayed_set_alt_done(fu); } static int tcm_get_alt(struct usb_function *f, unsigned intf) @@ -2411,15 +2541,20 @@ static int tcm_set_alt(struct usb_function *f, unsigned intf, unsigned alt) return -EOPNOTSUPP; if ((alt == USB_G_ALT_INT_BBB) || (alt == USB_G_ALT_INT_UAS)) { - struct guas_setup_wq *work; + unsigned long flags; - work = kmalloc(sizeof(*work), GFP_ATOMIC); - if (!work) - return -ENOMEM; - INIT_WORK(&work->work, tcm_delayed_set_alt); - work->fu = fu; - work->alt = alt; - schedule_work(&work->work); + spin_lock_irqsave(&fu->delayed_set_alt_lock, flags); + if (fu->delayed_set_alt_state != USBG_DELAYED_SET_ALT_IDLE) { + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, + flags); + return -EBUSY; + } + fu->delayed_alt = alt; + fu->delayed_set_alt_cancel = false; + fu->delayed_set_alt_state = USBG_DELAYED_SET_ALT_QUEUED; + spin_unlock_irqrestore(&fu->delayed_set_alt_lock, flags); + + schedule_work(&fu->delayed_set_alt); return USB_GADGET_DELAYED_STATUS; } return -EOPNOTSUPP; @@ -2429,11 +2564,8 @@ static void tcm_disable(struct usb_function *f) { struct f_uas *fu = to_f_uas(f); - if (fu->flags & USBG_IS_UAS) - uasp_cleanup_old_alt(fu); - else if (fu->flags & USBG_IS_BOT) - bot_cleanup_old_alt(fu); - fu->flags = 0; + if (tcm_cancel_delayed_set_alt(fu)) + tcm_cleanup_old_alt(fu); } static int tcm_setup(struct usb_function *f, @@ -2535,9 +2667,7 @@ static int tcm_set_name(struct usb_function_instance *f, const char *name) pr_debug("tcm: Activating %s\n", name); - mutex_lock(&opts->dep_lock); - opts->ready = true; - mutex_unlock(&opts->dep_lock); + WRITE_ONCE(opts->ready, true); return 0; } @@ -2581,11 +2711,16 @@ static void tcm_free(struct usb_function *f) { struct f_uas *tcm = to_f_uas(f); + tcm_cancel_delayed_set_alt_sync(tcm); kfree(tcm); } static void tcm_unbind(struct usb_configuration *c, struct usb_function *f) { + struct f_uas *fu = to_f_uas(f); + + tcm_cancel_delayed_set_alt_sync(fu); + tcm_cleanup_old_alt(fu); usb_free_all_descriptors(f); } @@ -2618,6 +2753,8 @@ static struct usb_function *tcm_alloc(struct usb_function_instance *fi) fu->function.disable = tcm_disable; fu->function.free_func = tcm_free; fu->tpg = tpg_instances[i].tpg; + INIT_WORK(&fu->delayed_set_alt, tcm_delayed_set_alt); + spin_lock_init(&fu->delayed_set_alt_lock); hash_init(fu->stream_hash); mutex_unlock(&tpg_instances_lock); diff --git a/drivers/usb/gadget/function/f_uac1_legacy.c b/drivers/usb/gadget/function/f_uac1_legacy.c index 4981af8337ab8..c234f9fea27cf 100644 --- a/drivers/usb/gadget/function/f_uac1_legacy.c +++ b/drivers/usb/gadget/function/f_uac1_legacy.c @@ -888,60 +888,10 @@ UAC1_INT_ATTRIBUTE(req_buf_size); UAC1_INT_ATTRIBUTE(req_count); UAC1_INT_ATTRIBUTE(audio_buf_size); -#define UAC1_STR_ATTRIBUTE(name) \ -static ssize_t f_uac1_opts_##name##_show(struct config_item *item, \ - char *page) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int result; \ - \ - mutex_lock(&opts->lock); \ - result = sprintf(page, "%s\n", opts->name); \ - mutex_unlock(&opts->lock); \ - \ - return result; \ -} \ - \ -static ssize_t f_uac1_opts_##name##_store(struct config_item *item, \ - const char *page, size_t len) \ -{ \ - struct f_uac1_legacy_opts *opts = to_f_uac1_opts(item); \ - int ret = -EBUSY; \ - char *tmp; \ - \ - mutex_lock(&opts->lock); \ - if (opts->refcnt) \ - goto end; \ - \ - tmp = kstrndup(page, len, GFP_KERNEL); \ - if (tmp) { \ - ret = -ENOMEM; \ - goto end; \ - } \ - if (opts->name##_alloc) \ - kfree(opts->name); \ - opts->name##_alloc = true; \ - opts->name = tmp; \ - ret = len; \ - \ -end: \ - mutex_unlock(&opts->lock); \ - return ret; \ -} \ - \ -CONFIGFS_ATTR(f_uac1_opts_, name) - -UAC1_STR_ATTRIBUTE(fn_play); -UAC1_STR_ATTRIBUTE(fn_cap); -UAC1_STR_ATTRIBUTE(fn_cntl); - static struct configfs_attribute *f_uac1_attrs[] = { &f_uac1_opts_attr_req_buf_size, &f_uac1_opts_attr_req_count, &f_uac1_opts_attr_audio_buf_size, - &f_uac1_opts_attr_fn_play, - &f_uac1_opts_attr_fn_cap, - &f_uac1_opts_attr_fn_cntl, NULL, }; @@ -956,12 +906,6 @@ static void f_audio_free_inst(struct usb_function_instance *f) struct f_uac1_legacy_opts *opts; opts = container_of(f, struct f_uac1_legacy_opts, func_inst); - if (opts->fn_play_alloc) - kfree(opts->fn_play); - if (opts->fn_cap_alloc) - kfree(opts->fn_cap); - if (opts->fn_cntl_alloc) - kfree(opts->fn_cntl); kfree(opts); } diff --git a/drivers/usb/gadget/function/f_uvc.c b/drivers/usb/gadget/function/f_uvc.c index 72601e15563c3..ab77e6bf001a1 100644 --- a/drivers/usb/gadget/function/f_uvc.c +++ b/drivers/usb/gadget/function/f_uvc.c @@ -889,9 +889,12 @@ uvc_function_bind(struct usb_configuration *c, struct usb_function *f) v4l2_error: v4l2_device_unregister(&uvc->v4l2_dev); error: - if (uvc->control_req) + if (uvc->control_req) { usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); + uvc->control_req = NULL; + } kfree(uvc->control_buf); + uvc->control_buf = NULL; usb_free_all_descriptors(f); return ret; @@ -1075,7 +1078,9 @@ static void uvc_function_unbind(struct usb_configuration *c, uvc->vdev_release_done = NULL; usb_ep_free_request(cdev->gadget->ep0, uvc->control_req); + uvc->control_req = NULL; kfree(uvc->control_buf); + uvc->control_buf = NULL; usb_free_all_descriptors(f); } diff --git a/drivers/usb/gadget/function/tcm.h b/drivers/usb/gadget/function/tcm.h index 009974d81d66b..e1d5a93916127 100644 --- a/drivers/usb/gadget/function/tcm.h +++ b/drivers/usb/gadget/function/tcm.h @@ -3,6 +3,7 @@ #define __TARGET_USB_GADGET_H__ #include +#include /* #include */ #include #include @@ -29,6 +30,12 @@ enum { #define USB_G_DEFAULT_SESSION_TAGS USBG_NUM_CMDS +enum { + USBG_DELAYED_SET_ALT_IDLE = 0, + USBG_DELAYED_SET_ALT_QUEUED, + USBG_DELAYED_SET_ALT_RUNNING, +}; + struct tcm_usbg_nexus { struct se_session *tvn_se_sess; }; @@ -132,6 +139,12 @@ struct f_uas { #define USBG_BOT_CMD_PEND (1 << 4) #define USBG_BOT_WEDGED (1 << 5) + struct work_struct delayed_set_alt; + spinlock_t delayed_set_alt_lock; /* protects delayed_set_alt_* */ + unsigned int delayed_alt; + unsigned int delayed_set_alt_state; + bool delayed_set_alt_cancel; + struct usbg_cdb cmd[USBG_NUM_CMDS]; struct usb_ep *ep_in; struct usb_ep *ep_out; diff --git a/drivers/usb/gadget/function/u_audio.c b/drivers/usb/gadget/function/u_audio.c index ca8dbec65f736..24c673dcbbd77 100644 --- a/drivers/usb/gadget/function/u_audio.c +++ b/drivers/usb/gadget/function/u_audio.c @@ -1177,6 +1177,20 @@ static struct snd_kcontrol_new u_audio_controls[] = { }, }; +static void u_audio_card_free(struct snd_card *card) +{ + struct snd_uac_chip *uac = card->private_data; + + if (!uac) + return; + + kfree(uac->p_prm.reqs); + kfree(uac->c_prm.reqs); + kfree(uac->p_prm.rbuf); + kfree(uac->c_prm.rbuf); + kfree(uac); +} + int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, const char *card_name) { @@ -1258,6 +1272,8 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, goto fail; uac->card = card; + card->private_data = uac; + card->private_free = u_audio_card_free; /* * Create first PCM device @@ -1426,6 +1442,8 @@ int g_audio_setup(struct g_audio *g_audio, const char *pcm_name, snd_fail: snd_card_free(card); + return err; + fail: kfree(uac->p_prm.reqs); kfree(uac->c_prm.reqs); @@ -1451,12 +1469,6 @@ void g_audio_cleanup(struct g_audio *g_audio) card = uac->card; if (card) snd_card_free_when_closed(card); - - kfree(uac->p_prm.reqs); - kfree(uac->c_prm.reqs); - kfree(uac->p_prm.rbuf); - kfree(uac->c_prm.rbuf); - kfree(uac); } EXPORT_SYMBOL_GPL(g_audio_cleanup); diff --git a/drivers/usb/gadget/function/u_uac1_legacy.h b/drivers/usb/gadget/function/u_uac1_legacy.h index b5df9bcbbeba7..b9ddae550ff3c 100644 --- a/drivers/usb/gadget/function/u_uac1_legacy.h +++ b/drivers/usb/gadget/function/u_uac1_legacy.h @@ -62,9 +62,6 @@ struct f_uac1_legacy_opts { char *fn_cap; char *fn_cntl; unsigned bound:1; - unsigned fn_play_alloc:1; - unsigned fn_cap_alloc:1; - unsigned fn_cntl_alloc:1; struct mutex lock; int refcnt; }; diff --git a/drivers/usb/gadget/function/uvc_v4l2.c b/drivers/usb/gadget/function/uvc_v4l2.c index 23bafb07133f2..b5384c88b84c9 100644 --- a/drivers/usb/gadget/function/uvc_v4l2.c +++ b/drivers/usb/gadget/function/uvc_v4l2.c @@ -200,6 +200,8 @@ uvc_send_response(struct uvc_device *uvc, struct uvc_request_data *data) return usb_ep_set_halt(cdev->gadget->ep0); req->length = min_t(unsigned int, uvc->event_length, data->length); + if (req->length > sizeof(data->data)) + req->length = sizeof(data->data); req->zero = data->length < uvc->event_length; memcpy(req->buf, data->data, req->length); diff --git a/drivers/usb/gadget/function/uvc_video.c b/drivers/usb/gadget/function/uvc_video.c index f24746e728774..dd9105a8ef781 100644 --- a/drivers/usb/gadget/function/uvc_video.c +++ b/drivers/usb/gadget/function/uvc_video.c @@ -821,7 +821,7 @@ int uvcg_video_init(struct uvc_video *video, struct uvc_device *uvc) /* Allocate a kthread for asynchronous hw submit handler. */ video->kworker = kthread_run_worker(0, "UVCG"); if (IS_ERR(video->kworker)) { - uvcg_err(&video->uvc->func, "failed to create UVCG kworker\n"); + uvcg_err(&uvc->func, "failed to create UVCG kworker\n"); return PTR_ERR(video->kworker); } diff --git a/drivers/usb/gadget/functions.c b/drivers/usb/gadget/functions.c index 203361a64212d..70e31c40e2673 100644 --- a/drivers/usb/gadget/functions.c +++ b/drivers/usb/gadget/functions.c @@ -70,7 +70,7 @@ void usb_put_function_instance(struct usb_function_instance *fi) { struct module *mod; - if (!fi) + if (!fi || !fi->fd) return; mod = fi->fd->mod; diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index 13c3da49348c5..7c72267c0c53b 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -1253,10 +1253,11 @@ ep0_poll (struct file *fd, poll_table *wait) static long gadget_dev_ioctl (struct file *fd, unsigned code, unsigned long value) { struct dev_data *dev = fd->private_data; - struct usb_gadget *gadget = dev->gadget; + struct usb_gadget *gadget; long ret = -ENOTTY; spin_lock_irq(&dev->lock); + gadget = dev->gadget; if (dev->state == STATE_DEV_OPENED || dev->state == STATE_DEV_UNBOUND) { /* Not bound to a UDC */ diff --git a/drivers/usb/gadget/udc/aspeed_udc.c b/drivers/usb/gadget/udc/aspeed_udc.c index 353bfb1ff0a12..94f5184412378 100644 --- a/drivers/usb/gadget/udc/aspeed_udc.c +++ b/drivers/usb/gadget/udc/aspeed_udc.c @@ -694,7 +694,7 @@ static int ast_udc_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req) struct ast_udc_dev *udc = ep->udc; struct ast_udc_request *req; unsigned long flags; - int rc = 0; + int rc = -EINVAL; spin_lock_irqsave(&udc->lock, flags); @@ -704,14 +704,11 @@ static int ast_udc_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req) list_del_init(&req->queue); ast_udc_done(ep, req, -ESHUTDOWN); _req->status = -ECONNRESET; + rc = 0; break; } } - /* dequeue request not found */ - if (&req->req != _req) - rc = -EINVAL; - spin_unlock_irqrestore(&udc->lock, flags); return rc; @@ -1434,25 +1431,12 @@ static void ast_udc_init_hw(struct ast_udc_dev *udc) ast_udc_write(udc, 0, AST_UDC_EP0_CTRL); } -static void ast_udc_remove(struct platform_device *pdev) +static void ast_udc_cleanup(struct platform_device *pdev) { struct ast_udc_dev *udc = platform_get_drvdata(pdev); unsigned long flags; u32 ctrl; - usb_del_gadget_udc(&udc->gadget); - if (udc->driver) { - /* - * This is broken as only some cleanup is skipped, *udev is - * freed and the register mapping goes away. Any further usage - * probably crashes. Also the device is unbound, so the skipped - * cleanup is never catched up later. - */ - dev_alert(&pdev->dev, - "Driver is busy and still going away. Fasten your seat belts!\n"); - return; - } - spin_lock_irqsave(&udc->lock, flags); /* Disable upstream port connection */ @@ -1472,6 +1456,26 @@ static void ast_udc_remove(struct platform_device *pdev) udc->ep0_buf = NULL; } +static void ast_udc_remove(struct platform_device *pdev) +{ + struct ast_udc_dev *udc = platform_get_drvdata(pdev); + + usb_del_gadget_udc(&udc->gadget); + if (udc->driver) { + /* + * This is broken as only some cleanup is skipped, *udev is + * freed and the register mapping goes away. Any further usage + * probably crashes. Also the device is unbound, so the skipped + * cleanup is never catched up later. + */ + dev_alert(&pdev->dev, + "Driver is busy and still going away. Fasten your seat belts!\n"); + return; + } + + ast_udc_cleanup(pdev); +} + static int ast_udc_probe(struct platform_device *pdev) { enum usb_device_speed max_speed; @@ -1524,6 +1528,12 @@ static int ast_udc_probe(struct platform_device *pdev) AST_UDC_NUM_ENDPOINTS, &udc->ep0_buf_dma, GFP_KERNEL); + if (!udc->ep0_buf) { + clk_disable_unprepare(udc->clk); + rc = -ENOMEM; + goto err; + } + udc->gadget.speed = USB_SPEED_UNKNOWN; udc->gadget.max_speed = USB_SPEED_HIGH; udc->creq = udc->reg + AST_UDC_SETUP0; @@ -1553,20 +1563,20 @@ static int ast_udc_probe(struct platform_device *pdev) udc->irq = platform_get_irq(pdev, 0); if (udc->irq < 0) { rc = udc->irq; - goto err; + goto err_cleanup; } rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0, KBUILD_MODNAME, udc); if (rc) { dev_err(&pdev->dev, "Failed to request interrupt\n"); - goto err; + goto err_cleanup; } rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget); if (rc) { dev_err(&pdev->dev, "Failed to add gadget udc\n"); - goto err; + goto err_cleanup; } dev_info(&pdev->dev, "Initialized udc in USB%s mode\n", @@ -1574,9 +1584,10 @@ static int ast_udc_probe(struct platform_device *pdev) return 0; +err_cleanup: + ast_udc_cleanup(pdev); err: dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc); - ast_udc_remove(pdev); return rc; } diff --git a/drivers/usb/gadget/udc/at91_udc.c b/drivers/usb/gadget/udc/at91_udc.c index 42b94d858e373..005152798000f 100644 --- a/drivers/usb/gadget/udc/at91_udc.c +++ b/drivers/usb/gadget/udc/at91_udc.c @@ -1794,6 +1794,19 @@ static void at91udc_of_init(struct at91_udc *udc, struct device_node *np) udc->caps = match->data; } +/* + * The work handler re-arms this timer, so shut the timer down before + * draining the work; otherwise it restarts the polling cycle. + */ +static void at91_udc_shutdown_vbus_timer(struct at91_udc *udc) +{ + if (!(udc->board.vbus_pin && udc->board.vbus_polled)) + return; + + timer_shutdown_sync(&udc->vbus_timer); + cancel_work_sync(&udc->vbus_timer_work); +} + static int at91udc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1907,7 +1920,7 @@ static int at91udc_probe(struct platform_device *pdev) } retval = usb_add_gadget_udc(dev, &udc->gadget); if (retval) - goto err_unprepare_iclk; + goto err_shutdown_vbus; dev_set_drvdata(dev, udc); device_init_wakeup(dev, 1); create_debug_file(udc); @@ -1915,6 +1928,8 @@ static int at91udc_probe(struct platform_device *pdev) INFO("%s version %s\n", driver_name, DRIVER_VERSION); return 0; +err_shutdown_vbus: + at91_udc_shutdown_vbus_timer(udc); err_unprepare_iclk: clk_unprepare(udc->iclk); err_unprepare_fclk: @@ -1933,6 +1948,9 @@ static void at91udc_remove(struct platform_device *pdev) DBG("remove\n"); usb_del_gadget_udc(&udc->gadget); + + at91_udc_shutdown_vbus_timer(udc); + if (udc->driver) { dev_err(&pdev->dev, "Driver still in use but removing anyhow\n"); diff --git a/drivers/usb/gadget/udc/bdc/bdc_core.c b/drivers/usb/gadget/udc/bdc/bdc_core.c index f47aac078f6be..de44cda273fb2 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_core.c +++ b/drivers/usb/gadget/udc/bdc/bdc_core.c @@ -586,9 +586,29 @@ static int bdc_probe(struct platform_device *pdev) static void bdc_remove(struct platform_device *pdev) { struct bdc *bdc; + unsigned long flags; + u32 temp; bdc = platform_get_drvdata(pdev); dev_dbg(bdc->dev, "%s ()\n", __func__); + /* + * Disable the device interrupt source before freeing the IRQ: + * clear BDC_GIE so the controller stops asserting interrupts, + * then free_irq drains any in-flight handler. + */ + spin_lock_irqsave(&bdc->lock, flags); + temp = bdc_readl(bdc->regs, BDC_BDCSC); + temp &= ~BDC_GIE; + bdc_writel(bdc->regs, BDC_BDCSC, temp); + spin_unlock_irqrestore(&bdc->lock, flags); + free_irq(bdc->irq, bdc); + /* + * Drain func_wake_notify after free_irq: the IRQ handler arms this + * delayed_work via bdc_sr_uspc -> handle_link_state_change -> + * schedule_delayed_work (self-rearmed in bdc_func_wake_timer), so + * the IRQ must be released first to prevent re-arm after cancel. + */ + cancel_delayed_work_sync(&bdc->func_wake_notify); bdc_udc_exit(bdc); bdc_hw_exit(bdc); bdc_phy_exit(bdc); diff --git a/drivers/usb/gadget/udc/bdc/bdc_udc.c b/drivers/usb/gadget/udc/bdc/bdc_udc.c index 23826fd7a8e69..7a12219edac6f 100644 --- a/drivers/usb/gadget/udc/bdc/bdc_udc.c +++ b/drivers/usb/gadget/udc/bdc/bdc_udc.c @@ -530,8 +530,8 @@ int bdc_udc_init(struct bdc *bdc) bdc->gadget.name = BRCM_BDC_NAME; - ret = devm_request_irq(bdc->dev, bdc->irq, bdc_udc_interrupt, - IRQF_SHARED, BRCM_BDC_NAME, bdc); + ret = request_irq(bdc->irq, bdc_udc_interrupt, IRQF_SHARED, + BRCM_BDC_NAME, bdc); if (ret) { dev_err(bdc->dev, "failed to request irq #%d %d\n", @@ -542,7 +542,7 @@ int bdc_udc_init(struct bdc *bdc) ret = bdc_init_ep(bdc); if (ret) { dev_err(bdc->dev, "bdc init ep fail: %d\n", ret); - return ret; + goto err0; } ret = usb_add_gadget_udc(bdc->dev, &bdc->gadget); @@ -571,6 +571,7 @@ int bdc_udc_init(struct bdc *bdc) err1: usb_del_gadget_udc(&bdc->gadget); err0: + free_irq(bdc->irq, bdc); bdc_free_ep(bdc); return ret; diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index ab1bcc5512b2c..fa49746d54036 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -714,6 +714,9 @@ static int usb_gadget_connect_locked(struct usb_gadget *gadget) goto out; } + if (gadget->connected) + goto out; + if (gadget->deactivated || !gadget->udc->allow_connect || !gadget->udc->started) { /* * If the gadget isn't usable (because it is deactivated, @@ -887,8 +890,10 @@ int usb_gadget_activate(struct usb_gadget *gadget) * If gadget has been connected before deactivation, or became connected * while it was being deactivated, we call usb_gadget_connect(). */ - if (gadget->connected) + if (gadget->connected) { + gadget->connected = false; ret = usb_gadget_connect_locked(gadget); + } unlock: mutex_unlock(&gadget->udc->connect_lock); diff --git a/drivers/usb/gadget/udc/dummy_hcd.c b/drivers/usb/gadget/udc/dummy_hcd.c index 4a78536644ba1..5e3cbe0338bff 100644 --- a/drivers/usb/gadget/udc/dummy_hcd.c +++ b/drivers/usb/gadget/udc/dummy_hcd.c @@ -278,6 +278,7 @@ struct dummy { unsigned ints_enabled:1; unsigned udc_suspended:1; unsigned pullup:1; + unsigned fifo_req_busy:1; /* * HOST side support @@ -329,6 +330,26 @@ static inline struct dummy *gadget_dev_to_dummy(struct device *dev) /* DEVICE/GADGET SIDE UTILITY ROUTINES */ +/* + * Give back a gadget request with dum->lock dropped around the callback. + * If @req is the shared fifo_req, clear fifo_req_busy afterward: the flag + * was set in dummy_queue() when the shared request was taken and must stay + * set until its completion callback has returned; list_del_init() alone + * makes the request look idle while the callback is still running. + * Caller holds dum->lock and has already done list_del_init() + status. + */ +static void dummy_giveback(struct dummy *dum, struct usb_ep *_ep, + struct dummy_request *req) +{ + bool fifo = req == &dum->fifo_req; + + spin_unlock(&dum->lock); + usb_gadget_giveback_request(_ep, &req->req); + spin_lock(&dum->lock); + if (fifo) + dum->fifo_req_busy = 0; +} + /* called with spinlock held */ static void nuke(struct dummy *dum, struct dummy_ep *ep) { @@ -339,9 +360,7 @@ static void nuke(struct dummy *dum, struct dummy_ep *ep) list_del_init(&req->queue); req->req.status = -ESHUTDOWN; - spin_unlock(&dum->lock); - usb_gadget_giveback_request(&ep->ep, &req->req); - spin_lock(&dum->lock); + dummy_giveback(dum, &ep->ep, req); } } @@ -728,10 +747,11 @@ static int dummy_queue(struct usb_ep *_ep, struct usb_request *_req, /* implement an emulated single-request FIFO */ if (ep->desc && (ep->desc->bEndpointAddress & USB_DIR_IN) && - list_empty(&dum->fifo_req.queue) && + !dum->fifo_req_busy && list_empty(&ep->queue) && _req->length <= FIFO_SIZE) { req = &dum->fifo_req; + dum->fifo_req_busy = 1; req->req = *_req; req->req.buf = dum->fifo_buf; memcpy(dum->fifo_buf, _req->buf, _req->length); @@ -785,9 +805,7 @@ static int dummy_dequeue(struct usb_ep *_ep, struct usb_request *_req) dev_dbg(udc_dev(dum), "dequeued req %p from %s, len %d buf %p\n", req, _ep->name, _req->length, _req->buf); - spin_unlock(&dum->lock); - usb_gadget_giveback_request(_ep, _req); - spin_lock(&dum->lock); + dummy_giveback(dum, _ep, req); } spin_unlock_irqrestore(&dum->lock, flags); return retval; @@ -1523,9 +1541,7 @@ static int transfer(struct dummy_hcd *dum_hcd, struct urb *urb, if (req->req.status != -EINPROGRESS) { list_del_init(&req->queue); - spin_unlock(&dum->lock); - usb_gadget_giveback_request(&ep->ep, &req->req); - spin_lock(&dum->lock); + dummy_giveback(dum, &ep->ep, req); /* requests might have been unlinked... */ rescan = 1; @@ -1910,9 +1926,7 @@ static enum hrtimer_restart dummy_timer(struct hrtimer *t) dev_dbg(udc_dev(dum), "stale req = %p\n", req); - spin_unlock(&dum->lock); - usb_gadget_giveback_request(&ep->ep, &req->req); - spin_lock(&dum->lock); + dummy_giveback(dum, &ep->ep, req); ep->already_seen = 0; goto restart; } diff --git a/drivers/usb/gadget/udc/fsl_udc_core.c b/drivers/usb/gadget/udc/fsl_udc_core.c index 4dea8bc30cf6d..ed311f0d74b5e 100644 --- a/drivers/usb/gadget/udc/fsl_udc_core.c +++ b/drivers/usb/gadget/udc/fsl_udc_core.c @@ -184,7 +184,7 @@ __acquires(ep->udc->lock) usb_gadget_unmap_request(&ep->udc->gadget, &req->req, ep_is_in(ep)); if (status && (status != -ESHUTDOWN)) - dev_vdbg(&udc->gadget.dev, "complete %s req %p stat %d len %u/%u\n", + dev_vdbg(udc->dev, "complete %s req %p stat %d len %u/%u\n", ep->ep.name, &req->req, status, req->req.actual, req->req.length); @@ -286,7 +286,7 @@ static int dr_controller_setup(struct fsl_udc *udc) timeout = jiffies + FSL_UDC_RESET_TIMEOUT; while (fsl_readl(&dr_regs->usbcmd) & USB_CMD_CTRL_RESET) { if (time_after(jiffies, timeout)) { - dev_err(&udc->gadget.dev, "udc reset timeout!\n"); + dev_err(udc->dev, "udc reset timeout!\n"); return -ETIMEDOUT; } cpu_relax(); @@ -309,7 +309,7 @@ static int dr_controller_setup(struct fsl_udc *udc) tmp &= USB_EP_LIST_ADDRESS_MASK; fsl_writel(tmp, &dr_regs->endpointlistaddr); - dev_vdbg(&udc->gadget.dev, + dev_vdbg(udc->dev, "vir[qh_base] is %p phy[qh_base] is 0x%8x reg is 0x%8x\n", udc->ep_qh, (int)tmp, fsl_readl(&dr_regs->endpointlistaddr)); @@ -500,7 +500,7 @@ static void struct_ep_qh_setup(struct fsl_udc *udc, unsigned char ep_num, tmp = max_pkt_len << EP_QUEUE_HEAD_MAX_PKT_LEN_POS; break; default: - dev_vdbg(&udc->gadget.dev, "error ep type is %d\n", ep_type); + dev_vdbg(udc->dev, "error ep type is %d\n", ep_type); return; } if (zlt) @@ -613,7 +613,7 @@ static int fsl_ep_enable(struct usb_ep *_ep, spin_unlock_irqrestore(&udc->lock, flags); retval = 0; - dev_vdbg(&udc->gadget.dev, "enabled %s (ep%d%s) maxpacket %d\n", + dev_vdbg(udc->dev, "enabled %s (ep%d%s) maxpacket %d\n", ep->ep.name, ep->ep.desc->bEndpointAddress & 0x0f, (desc->bEndpointAddress & USB_DIR_IN) ? "in" : "out", max); @@ -634,13 +634,8 @@ static int fsl_ep_disable(struct usb_ep *_ep) int ep_num; ep = container_of(_ep, struct fsl_ep, ep); - if (!_ep || !ep->ep.desc) { - /* - * dev_vdbg(&udc->gadget.dev, "%s not enabled\n", - * _ep ? ep->ep.name : NULL); - */ + if (!_ep || !ep->ep.desc) return -EINVAL; - } /* disable ep on controller */ ep_num = ep_index(ep); @@ -664,7 +659,7 @@ static int fsl_ep_disable(struct usb_ep *_ep) ep->stopped = 1; spin_unlock_irqrestore(&udc->lock, flags); - dev_vdbg(&udc->gadget.dev, "disabled %s OK\n", _ep->name); + dev_vdbg(udc->dev, "disabled %s OK\n", _ep->name); return 0; } @@ -724,9 +719,6 @@ static void fsl_queue_td(struct fsl_ep *ep, struct fsl_req *req) { u32 temp, bitmask, tmp_stat; - /* dev_vdbg(&udc->gadget.dev, "QH addr Register 0x%8x\n", dr_regs->endpointlistaddr); - dev_vdbg(&udc->gadget.dev, "ep_qh[%d] addr is 0x%8x\n", i, (u32)&(ep->udc->ep_qh[i])); */ - bitmask = ep_is_in(ep) ? (1 << (ep_index(ep) + 16)) : (1 << (ep_index(ep))); @@ -813,7 +805,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length, *is_last = 0; if ((*is_last) == 0) - dev_vdbg(&udc_controller->gadget.dev, "multi-dtd request!\n"); + dev_vdbg(udc_controller->dev, "multi-dtd request!\n"); /* Fill in the transfer size; set active bit */ swap_temp = ((*length << DTD_LENGTH_BIT_POS) | DTD_STATUS_ACTIVE); @@ -825,7 +817,7 @@ static struct ep_td_struct *fsl_build_dtd(struct fsl_req *req, unsigned *length, mb(); - dev_vdbg(&udc_controller->gadget.dev, "length = %d address= 0x%x\n", *length, (int)*dma); + dev_vdbg(udc_controller->dev, "length = %d address= 0x%x\n", *length, (int)*dma); return dtd; } @@ -876,11 +868,11 @@ fsl_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags) /* catch various bogus parameters */ if (!_req || !req->req.complete || !req->req.buf || !list_empty(&req->queue)) { - dev_vdbg(&udc->gadget.dev, "%s, bad params\n", __func__); + dev_vdbg(udc->dev, "%s, bad params\n", __func__); return -EINVAL; } if (unlikely(!ep->ep.desc)) { - dev_vdbg(&udc->gadget.dev, "%s, bad ep\n", __func__); + dev_vdbg(udc->dev, "%s, bad ep\n", __func__); return -EINVAL; } if (usb_endpoint_xfer_isoc(ep->ep.desc)) { @@ -1040,7 +1032,7 @@ static int fsl_ep_set_halt(struct usb_ep *_ep, int value) udc->ep0_dir = 0; } out: - dev_vdbg(&udc->gadget.dev, "%s %s halt stat %d\n", ep->ep.name, + dev_vdbg(udc->dev, "%s %s halt stat %d\n", ep->ep.name, value ? "set" : "clear", status); return status; @@ -1109,7 +1101,7 @@ static void fsl_ep_fifo_flush(struct usb_ep *_ep) /* Wait until flush complete */ while (fsl_readl(&dr_regs->endptflush)) { if (time_after(jiffies, timeout)) { - dev_err(&udc_controller->gadget.dev, + dev_err(udc_controller->dev, "ep flush timeout\n"); return; } @@ -1182,7 +1174,7 @@ static int fsl_vbus_session(struct usb_gadget *gadget, int is_active) udc = container_of(gadget, struct fsl_udc, gadget); spin_lock_irqsave(&udc->lock, flags); - dev_vdbg(&gadget->dev, "VBUS %s\n", str_on_off(is_active)); + dev_vdbg(udc->dev, "VBUS %s\n", str_on_off(is_active)); udc->vbus_active = (is_active != 0); if (can_pullup(udc)) fsl_writel((fsl_readl(&dr_regs->usbcmd) | USB_CMD_RUN_STOP), @@ -1548,7 +1540,7 @@ static void ep0_req_complete(struct fsl_udc *udc, struct fsl_ep *ep0, udc->ep0_state = WAIT_FOR_SETUP; break; case WAIT_FOR_SETUP: - dev_err(&udc->gadget.dev, "Unexpected ep0 packets\n"); + dev_err(udc->dev, "Unexpected ep0 packets\n"); break; default: ep0stall(udc); @@ -1617,7 +1609,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe, errors = hc32_to_cpu(curr_td->size_ioc_sts); if (errors & DTD_ERROR_MASK) { if (errors & DTD_STATUS_HALTED) { - dev_err(&udc->gadget.dev, "dTD error %08x QH=%d\n", errors, pipe); + dev_err(udc->dev, "dTD error %08x QH=%d\n", errors, pipe); /* Clear the errors and Halt condition */ tmp = hc32_to_cpu(curr_qh->size_ioc_int_sts); tmp &= ~errors; @@ -1628,26 +1620,26 @@ static int process_ep_req(struct fsl_udc *udc, int pipe, break; } if (errors & DTD_STATUS_DATA_BUFF_ERR) { - dev_vdbg(&udc->gadget.dev, "Transfer overflow\n"); + dev_vdbg(udc->dev, "Transfer overflow\n"); status = -EPROTO; break; } else if (errors & DTD_STATUS_TRANSACTION_ERR) { - dev_vdbg(&udc->gadget.dev, "ISO error\n"); + dev_vdbg(udc->dev, "ISO error\n"); status = -EILSEQ; break; } else - dev_err(&udc->gadget.dev, + dev_err(udc->dev, "Unknown error has occurred (0x%x)!\n", errors); } else if (hc32_to_cpu(curr_td->size_ioc_sts) & DTD_STATUS_ACTIVE) { - dev_vdbg(&udc->gadget.dev, "Request not complete\n"); + dev_vdbg(udc->dev, "Request not complete\n"); status = REQ_UNCOMPLETE; return status; } else if (remaining_length) { if (direction) { - dev_vdbg(&udc->gadget.dev, + dev_vdbg(udc->dev, "Transmit dTD remaining length not zero\n"); status = -EPROTO; break; @@ -1655,8 +1647,7 @@ static int process_ep_req(struct fsl_udc *udc, int pipe, break; } } else { - dev_vdbg(&udc->gadget.dev, - "dTD transmitted successful\n"); + dev_vdbg(udc->dev, "dTD transmitted successful\n"); } if (j != curr_req->dtd_count - 1) @@ -1699,7 +1690,7 @@ static void dtd_complete_irq(struct fsl_udc *udc) /* If the ep is configured */ if (!curr_ep->ep.name) { - dev_warn(&udc->gadget.dev, "Invalid EP?\n"); + dev_warn(udc->dev, "Invalid EP?\n"); continue; } @@ -1708,7 +1699,7 @@ static void dtd_complete_irq(struct fsl_udc *udc) queue) { status = process_ep_req(udc, i, curr_req); - dev_vdbg(&udc->gadget.dev, + dev_vdbg(udc->dev, "status of process_ep_req= %d, ep = %d\n", status, ep_num); if (status == REQ_UNCOMPLETE) @@ -1829,7 +1820,7 @@ static void reset_irq(struct fsl_udc *udc) while (fsl_readl(&dr_regs->endpointprime)) { /* Wait until all endptprime bits cleared */ if (time_after(jiffies, timeout)) { - dev_err(&udc->gadget.dev, "Timeout for reset\n"); + dev_err(udc->dev, "Timeout for reset\n"); break; } cpu_relax(); @@ -1839,7 +1830,7 @@ static void reset_irq(struct fsl_udc *udc) fsl_writel(0xffffffff, &dr_regs->endptflush); if (fsl_readl(&dr_regs->portsc1) & PORTSCX_PORT_RESET) { - dev_vdbg(&udc->gadget.dev, "Bus reset\n"); + dev_vdbg(udc->dev, "Bus reset\n"); /* Bus is reseting */ udc->bus_reset = 1; /* Reset all the queues, include XD, dTD, EP queue @@ -1847,7 +1838,7 @@ static void reset_irq(struct fsl_udc *udc) reset_queues(udc, true); udc->usb_state = USB_STATE_DEFAULT; } else { - dev_vdbg(&udc->gadget.dev, "Controller reset\n"); + dev_vdbg(udc->dev, "Controller reset\n"); /* initialize usb hw reg except for regs for EP, not * touch usbintr reg */ dr_controller_setup(udc); @@ -1881,7 +1872,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc) /* Clear notification bits */ fsl_writel(irq_src, &dr_regs->usbsts); - /* dev_vdbg(&udc->gadget.dev, "irq_src [0x%8x]", irq_src); */ + /* dev_vdbg(udc->dev, "irq_src [0x%8x]", irq_src); */ /* Need to resume? */ if (udc->usb_state == USB_STATE_SUSPENDED) @@ -1890,7 +1881,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc) /* USB Interrupt */ if (irq_src & USB_STS_INT) { - dev_vdbg(&udc->gadget.dev, "Packet int\n"); + dev_vdbg(udc->dev, "Packet int\n"); /* Setup package, we only support ep0 as control ep */ if (fsl_readl(&dr_regs->endptsetupstat) & EP_SETUP_STATUS_EP0) { tripwire_handler(udc, 0, @@ -1919,7 +1910,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc) /* Reset Received */ if (irq_src & USB_STS_RESET) { - dev_vdbg(&udc->gadget.dev, "reset int\n"); + dev_vdbg(udc->dev, "reset int\n"); reset_irq(udc); status = IRQ_HANDLED; } @@ -1931,7 +1922,7 @@ static irqreturn_t fsl_udc_irq(int irq, void *_udc) } if (irq_src & (USB_STS_ERR | USB_STS_SYS_ERR)) { - dev_vdbg(&udc->gadget.dev, "Error IRQ %x\n", irq_src); + dev_vdbg(udc->dev, "Error IRQ %x\n", irq_src); } spin_unlock_irqrestore(&udc->lock, flags); @@ -1967,7 +1958,7 @@ static int fsl_udc_start(struct usb_gadget *g, udc_controller->transceiver->otg, &udc_controller->gadget); if (retval < 0) { - dev_err(&udc_controller->gadget.dev, "can't bind to transceiver\n"); + dev_err(udc_controller->dev, "can't bind to transceiver\n"); udc_controller->driver = NULL; return retval; } @@ -2252,7 +2243,7 @@ static int struct_udc_setup(struct fsl_udc *udc, udc->eps = kcalloc(udc->max_ep, sizeof(struct fsl_ep), GFP_KERNEL); if (!udc->eps) { - dev_err(&udc->gadget.dev, "kmalloc udc endpoint status failed\n"); + dev_err(udc->dev, "kmalloc udc endpoint status failed\n"); goto eps_alloc_failed; } @@ -2267,7 +2258,7 @@ static int struct_udc_setup(struct fsl_udc *udc, udc->ep_qh = dma_alloc_coherent(&pdev->dev, size, &udc->ep_qh_dma, GFP_KERNEL); if (!udc->ep_qh) { - dev_err(&udc->gadget.dev, "malloc QHs for udc failed\n"); + dev_err(udc->dev, "malloc QHs for udc failed\n"); goto ep_queue_alloc_failed; } @@ -2278,14 +2269,14 @@ static int struct_udc_setup(struct fsl_udc *udc, udc->status_req = container_of(fsl_alloc_request(NULL, GFP_KERNEL), struct fsl_req, req); if (!udc->status_req) { - dev_err(&udc->gadget.dev, "kzalloc for udc status request failed\n"); + dev_err(udc->dev, "kzalloc for udc status request failed\n"); goto udc_status_alloc_failed; } /* allocate a small amount of memory to get valid address */ udc->status_req->req.buf = kmalloc(8, GFP_KERNEL); if (!udc->status_req->req.buf) { - dev_err(&udc->gadget.dev, "kzalloc for udc request buffer failed\n"); + dev_err(udc->dev, "kzalloc for udc request buffer failed\n"); goto udc_req_buf_alloc_failed; } @@ -2373,6 +2364,7 @@ static int fsl_udc_probe(struct platform_device *pdev) if (udc_controller == NULL) return -ENOMEM; + udc_controller->dev = &pdev->dev; pdata = dev_get_platdata(&pdev->dev); udc_controller->pdata = pdata; spin_lock_init(&udc_controller->lock); @@ -2382,7 +2374,7 @@ static int fsl_udc_probe(struct platform_device *pdev) if (pdata->operating_mode == FSL_USB2_DR_OTG) { udc_controller->transceiver = usb_get_phy(USB_PHY_TYPE_USB2); if (IS_ERR_OR_NULL(udc_controller->transceiver)) { - dev_err(&udc_controller->gadget.dev, "Can't find OTG driver!\n"); + dev_err(&pdev->dev, "Can't find OTG driver!\n"); ret = -ENODEV; goto err_kfree; } @@ -2398,7 +2390,7 @@ static int fsl_udc_probe(struct platform_device *pdev) if (pdata->operating_mode == FSL_USB2_DR_DEVICE) { if (!request_mem_region(res->start, resource_size(res), driver_name)) { - dev_err(&udc_controller->gadget.dev, "request mem region for %s failed\n", pdev->name); + dev_err(&pdev->dev, "failed to request mem region\n"); ret = -EBUSY; goto err_kfree; } @@ -2429,7 +2421,7 @@ static int fsl_udc_probe(struct platform_device *pdev) /* Read Device Controller Capability Parameters register */ dccparams = fsl_readl(&dr_regs->dccparams); if (!(dccparams & DCCPARAMS_DC)) { - dev_err(&udc_controller->gadget.dev, "This SOC doesn't support device role\n"); + dev_err(&pdev->dev, "This SOC doesn't support device role\n"); ret = -ENODEV; goto err_exit; } @@ -2447,14 +2439,14 @@ static int fsl_udc_probe(struct platform_device *pdev) ret = request_irq(udc_controller->irq, fsl_udc_irq, IRQF_SHARED, driver_name, udc_controller); if (ret != 0) { - dev_err(&udc_controller->gadget.dev, "cannot request irq %d err %d\n", + dev_err(&pdev->dev, "cannot request irq %d err %d\n", udc_controller->irq, ret); goto err_exit; } /* Initialize the udc structure including QH member and other member */ if (struct_udc_setup(udc_controller, pdev)) { - dev_err(&udc_controller->gadget.dev, "Can't initialize udc data structure\n"); + dev_err(&pdev->dev, "Can't initialize udc data structure\n"); ret = -ENOMEM; goto err_free_irq; } @@ -2474,7 +2466,6 @@ static int fsl_udc_probe(struct platform_device *pdev) udc_controller->gadget.name = driver_name; /* Setup gadget.dev and register with kernel */ - dev_set_name(&udc_controller->gadget.dev, "gadget"); udc_controller->gadget.dev.of_node = pdev->dev.of_node; if (!IS_ERR_OR_NULL(udc_controller->transceiver)) diff --git a/drivers/usb/gadget/udc/fsl_usb2_udc.h b/drivers/usb/gadget/udc/fsl_usb2_udc.h index cc1756f3e89d1..53922bc58ca01 100644 --- a/drivers/usb/gadget/udc/fsl_usb2_udc.h +++ b/drivers/usb/gadget/udc/fsl_usb2_udc.h @@ -470,6 +470,7 @@ struct fsl_ep { #define EP_DIR_OUT 0 struct fsl_udc { + struct device *dev; struct usb_gadget gadget; struct usb_gadget_driver *driver; struct fsl_usb2_platform_data *pdata; diff --git a/drivers/usb/gadget/udc/goku_udc.c b/drivers/usb/gadget/udc/goku_udc.c index b860c2e764494..c262749a29cc7 100644 --- a/drivers/usb/gadget/udc/goku_udc.c +++ b/drivers/usb/gadget/udc/goku_udc.c @@ -1616,7 +1616,8 @@ static irqreturn_t goku_irq(int irq, void *_dev) if (stat & INT_USBRESET) { /* hub reset done */ ACK(INT_USBRESET); INFO(dev, "USB reset done, gadget %s\n", - dev->driver->driver.name); + dev->driver ? dev->driver->driver.name : + ""); } // and INT_ERR on some endpoint's crc/bitstuff/... problem } diff --git a/drivers/usb/gadget/udc/r8a66597-udc.c b/drivers/usb/gadget/udc/r8a66597-udc.c index e5c2630e37114..29ae81c95d0a6 100644 --- a/drivers/usb/gadget/udc/r8a66597-udc.c +++ b/drivers/usb/gadget/udc/r8a66597-udc.c @@ -1951,7 +1951,6 @@ static int r8a66597_probe(struct platform_device *pdev) return 0; err_add_udc: - r8a66597_free_request(&r8a66597->ep[0].ep, r8a66597->ep0_req); clean_up2: if (r8a66597->pdata->on_chip) clk_disable_unprepare(r8a66597->clk); diff --git a/drivers/usb/gadget/udc/snps_udc_core.c b/drivers/usb/gadget/udc/snps_udc_core.c index 373942ceb0769..2f790cc65257b 100644 --- a/drivers/usb/gadget/udc/snps_udc_core.c +++ b/drivers/usb/gadget/udc/snps_udc_core.c @@ -3133,7 +3133,6 @@ int udc_probe(struct udc *dev) /* device struct setup */ dev->gadget.ops = &udc_ops; - dev_set_name(&dev->gadget.dev, "gadget"); dev->gadget.name = name; dev->gadget.max_speed = USB_SPEED_HIGH; diff --git a/drivers/usb/gadget/udc/snps_udc_plat.c b/drivers/usb/gadget/udc/snps_udc_plat.c index db842a6de643d..0ee3ed185d9c3 100644 --- a/drivers/usb/gadget/udc/snps_udc_plat.c +++ b/drivers/usb/gadget/udc/snps_udc_plat.c @@ -159,10 +159,9 @@ static int udc_plat_probe(struct platform_device *pdev) if (of_property_present(dev->of_node, "extcon")) { udc->edev = extcon_get_edev_by_phandle(dev, 0); if (IS_ERR(udc->edev)) { - if (PTR_ERR(udc->edev) == -EPROBE_DEFER) - return -EPROBE_DEFER; - dev_err(dev, "Invalid or missing extcon\n"); ret = PTR_ERR(udc->edev); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Invalid or missing extcon\n"); goto exit_phy; } diff --git a/drivers/usb/host/Kconfig b/drivers/usb/host/Kconfig index 08e80deee437b..4d878ee19a6fc 100644 --- a/drivers/usb/host/Kconfig +++ b/drivers/usb/host/Kconfig @@ -71,7 +71,7 @@ config USB_XHCI_HISTB config USB_XHCI_MTK tristate "xHCI support for MediaTek SoCs" select MFD_SYSCON - depends on (MIPS && SOC_MT7621) || ARCH_MEDIATEK || COMPILE_TEST + depends on (MIPS && SOC_MT7621) || ARCH_MEDIATEK || ARCH_AIROHA || COMPILE_TEST help Say 'Y' to enable the support for the xHCI host controller found in MediaTek SoCs. diff --git a/drivers/usb/host/xhci-dbgcap.c b/drivers/usb/host/xhci-dbgcap.c index ecda964e018ac..e2e0694d0c22a 100644 --- a/drivers/usb/host/xhci-dbgcap.c +++ b/drivers/usb/host/xhci-dbgcap.c @@ -260,7 +260,6 @@ xhci_dbc_queue_trb(struct xhci_ring *ring, u32 field1, trace_xhci_dbc_gadget_ep_queue(ring, &trb->generic, xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue)); - ring->num_trbs_free--; next = ++(ring->enqueue); if (TRB_TYPE_LINK_LE32(next->link.control)) { next->link.control ^= cpu_to_le32(TRB_CYCLE); @@ -281,7 +280,7 @@ static int xhci_dbc_queue_bulk_tx(struct dbc_ep *dep, num_trbs = count_trbs(req->dma, req->length); WARN_ON(num_trbs != 1); - if (ring->num_trbs_free < num_trbs) + if (xhci_num_trbs_free(ring) <= num_trbs) return -EBUSY; addr = req->dma; @@ -782,7 +781,6 @@ static void dbc_handle_xfer_event(struct xhci_dbc *dbc, union xhci_trb *event) } if (r->status == -COMP_STALL_ERROR) { dev_warn(dbc->dev, "Give back stale stalled req\n"); - ring->num_trbs_free++; xhci_dbc_giveback(r, 0); } } @@ -847,7 +845,6 @@ static void dbc_handle_xfer_event(struct xhci_dbc *dbc, union xhci_trb *event) break; } - ring->num_trbs_free++; req->actual = req->length - remain_length; xhci_dbc_giveback(req, status); } diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c index 90282e51e23ec..fda5ccaba27a9 100644 --- a/drivers/usb/host/xhci-dbgtty.c +++ b/drivers/usb/host/xhci-dbgtty.c @@ -628,8 +628,8 @@ int dbc_tty_init(void) dbc_tty_driver = tty_alloc_driver(64, TTY_DRIVER_REAL_RAW | TTY_DRIVER_DYNAMIC_DEV); if (IS_ERR(dbc_tty_driver)) { - idr_destroy(&dbc_tty_minors); - return PTR_ERR(dbc_tty_driver); + ret = PTR_ERR(dbc_tty_driver); + goto fail; } dbc_tty_driver->driver_name = "dbc_serial"; @@ -649,10 +649,17 @@ int dbc_tty_init(void) ret = tty_register_driver(dbc_tty_driver); if (ret) { pr_err("Can't register dbc tty driver\n"); - tty_driver_kref_put(dbc_tty_driver); - idr_destroy(&dbc_tty_minors); + goto fail_put; } + return ret; + +fail_put: + tty_driver_kref_put(dbc_tty_driver); +fail: + idr_destroy(&dbc_tty_minors); + dbc_tty_driver = NULL; + return ret; } diff --git a/drivers/usb/host/xhci-debugfs.c b/drivers/usb/host/xhci-debugfs.c index c6d44977193f0..ae50b667a5487 100644 --- a/drivers/usb/host/xhci-debugfs.c +++ b/drivers/usb/host/xhci-debugfs.c @@ -329,7 +329,7 @@ static int xhci_portsc_show(struct seq_file *s, void *unused) u32 portsc; char str[XHCI_MSG_MAX]; - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); seq_printf(s, "%s\n", xhci_decode_portsc(str, portsc)); return 0; @@ -359,7 +359,7 @@ static ssize_t xhci_port_write(struct file *file, const char __user *ubuf, return count; spin_lock_irqsave(&xhci->lock, flags); /* compliance mode can only be enabled on ports in RxDetect */ - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); if ((portsc & PORT_PLS_MASK) != XDEV_RXDETECT) { spin_unlock_irqrestore(&xhci->lock, flags); return -EPERM; @@ -367,7 +367,7 @@ static ssize_t xhci_port_write(struct file *file, const char __user *ubuf, portsc = xhci_port_state_to_neutral(portsc); portsc &= ~PORT_PLS_MASK; portsc |= PORT_LINK_STROBE | XDEV_COMP_MODE; - writel(portsc, port->addr); + xhci_portsc_writel(port, portsc); spin_unlock_irqrestore(&xhci->lock, flags); } else { return -EINVAL; @@ -613,20 +613,16 @@ void xhci_debugfs_remove_slot(struct xhci_hcd *xhci, int slot_id) static void xhci_debugfs_create_ports(struct xhci_hcd *xhci, struct dentry *parent) { - unsigned int num_ports; char port_name[8]; struct xhci_port *port; struct dentry *dir; - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - parent = debugfs_create_dir("ports", parent); - while (num_ports--) { - scnprintf(port_name, sizeof(port_name), "port%02d", - num_ports + 1); + for (int i = 0; i < xhci->max_ports; i++) { + scnprintf(port_name, sizeof(port_name), "port%02d", i + 1); dir = debugfs_create_dir(port_name, parent); - port = &xhci->hw_ports[num_ports]; + port = &xhci->hw_ports[i]; debugfs_create_file("portsc", 0644, dir, port, &port_fops); } } @@ -634,7 +630,6 @@ static void xhci_debugfs_create_ports(struct xhci_hcd *xhci, static int xhci_port_bw_show(struct xhci_hcd *xhci, u8 dev_speed, struct seq_file *s) { - unsigned int num_ports; unsigned int i; int ret; struct xhci_container_ctx *ctx; @@ -645,8 +640,6 @@ static int xhci_port_bw_show(struct xhci_hcd *xhci, u8 dev_speed, if (ret < 0) return ret; - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - ctx = xhci_alloc_port_bw_ctx(xhci, 0); if (!ctx) { pm_runtime_put_sync(dev); @@ -661,7 +654,7 @@ static int xhci_port_bw_show(struct xhci_hcd *xhci, u8 dev_speed, /* print all roothub ports available bandwidth * refer to xhci rev1_2 protocol 6.2.6 , byte 0 is reserved */ - for (i = 1; i < num_ports+1; i++) + for (i = 1; i <= xhci->max_ports; i++) seq_printf(s, "port[%d] available bw: %d%%.\n", i, ctx->bytes[i]); err_out: diff --git a/drivers/usb/host/xhci-hub.c b/drivers/usb/host/xhci-hub.c index 5e1442e91743d..cf358e5c66421 100644 --- a/drivers/usb/host/xhci-hub.c +++ b/drivers/usb/host/xhci-hub.c @@ -299,7 +299,7 @@ static void xhci_usb2_hub_descriptor(struct usb_hcd *hcd, struct xhci_hcd *xhci, */ memset(port_removable, 0, sizeof(port_removable)); for (i = 0; i < ports; i++) { - portsc = readl(rhub->ports[i]->addr); + portsc = xhci_portsc_readl(rhub->ports[i]); /* If a device is removable, PORTSC reports a 0, same as in the * hub descriptor DeviceRemovable bits. */ @@ -356,7 +356,7 @@ static void xhci_usb3_hub_descriptor(struct usb_hcd *hcd, struct xhci_hcd *xhci, port_removable = 0; /* bit 0 is reserved, bit 1 is for port 1, etc. */ for (i = 0; i < ports; i++) { - portsc = readl(rhub->ports[i]->addr); + portsc = xhci_portsc_readl(rhub->ports[i]); if (portsc & PORT_DEV_REMOVE) port_removable |= 1 << (i + 1); } @@ -566,19 +566,19 @@ static void xhci_disable_port(struct xhci_hcd *xhci, struct xhci_port *port) return; } - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); portsc = xhci_port_state_to_neutral(portsc); /* Write 1 to disable the port */ - writel(portsc | PORT_PE, port->addr); + xhci_portsc_writel(port, portsc | PORT_PE); - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); xhci_dbg(xhci, "disable port %d-%d, portsc: 0x%x\n", hcd->self.busnum, port->hcd_portnum + 1, portsc); } static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue, - u16 wIndex, __le32 __iomem *addr, u32 port_status) + u16 wIndex, struct xhci_port *port, u32 port_status) { char *port_change_bit; u32 status; @@ -621,8 +621,8 @@ static void xhci_clear_port_change_bit(struct xhci_hcd *xhci, u16 wValue, return; } /* Change bits are all write 1 to clear */ - writel(port_status | status, addr); - port_status = readl(addr); + xhci_portsc_writel(port, port_status | status); + port_status = xhci_portsc_readl(port); xhci_dbg(xhci, "clear port%d %s change, portsc: 0x%x\n", wIndex + 1, port_change_bit, port_status); @@ -650,7 +650,7 @@ static void xhci_set_port_power(struct xhci_hcd *xhci, struct xhci_port *port, u32 temp; hcd = port->rhub->hcd; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); xhci_dbg(xhci, "set port power %d-%d %s, portsc: 0x%x\n", hcd->self.busnum, port->hcd_portnum + 1, on ? "ON" : "OFF", temp); @@ -659,11 +659,11 @@ static void xhci_set_port_power(struct xhci_hcd *xhci, struct xhci_port *port, if (on) { /* Power on */ - writel(temp | PORT_POWER, port->addr); - readl(port->addr); + xhci_portsc_writel(port, temp | PORT_POWER); + xhci_portsc_readl(port); } else { /* Power off */ - writel(temp & ~PORT_POWER, port->addr); + xhci_portsc_writel(port, temp & ~PORT_POWER); } spin_unlock_irqrestore(&xhci->lock, *flags); @@ -683,9 +683,9 @@ static void xhci_port_set_test_mode(struct xhci_hcd *xhci, /* xhci only supports test mode for usb2 ports */ port = xhci->usb2_rhub.ports[wIndex]; - temp = readl(port->addr + PORTPMSC); + temp = readl(&port->port_reg->portpmsc); temp |= test_mode << PORT_TEST_MODE_SHIFT; - writel(temp, port->addr + PORTPMSC); + writel(temp, &port->port_reg->portpmsc); xhci->test_mode = test_mode; if (test_mode == USB_TEST_FORCE_ENABLE) xhci_start(xhci); @@ -700,7 +700,7 @@ static int xhci_enter_test_mode(struct xhci_hcd *xhci, /* Disable all Device Slots */ xhci_dbg(xhci, "Disable all slots\n"); spin_unlock_irqrestore(&xhci->lock, *flags); - for (i = 1; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { + for (i = 1; i <= xhci->max_slots; i++) { if (!xhci->devs[i]) continue; @@ -801,11 +801,11 @@ void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port, u32 temp; u32 portsc; - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); temp = xhci_port_state_to_neutral(portsc); temp &= ~PORT_PLS_MASK; temp |= PORT_LINK_STROBE | link_state; - writel(temp, port->addr); + xhci_portsc_writel(port, temp); xhci_dbg(xhci, "Set port %d-%d link state, portsc: 0x%x, write 0x%x", port->rhub->hcd->self.busnum, port->hcd_portnum + 1, @@ -817,7 +817,7 @@ static void xhci_set_remote_wake_mask(struct xhci_hcd *xhci, { u32 temp; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); temp = xhci_port_state_to_neutral(temp); if (wake_mask & USB_PORT_FEAT_REMOTE_WAKE_CONNECT) @@ -835,7 +835,7 @@ static void xhci_set_remote_wake_mask(struct xhci_hcd *xhci, else temp &= ~PORT_WKOC_E; - writel(temp, port->addr); + xhci_portsc_writel(port, temp); } /* Test and clear port RWC bit */ @@ -844,11 +844,11 @@ void xhci_test_and_clear_bit(struct xhci_hcd *xhci, struct xhci_port *port, { u32 temp; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if (temp & port_bit) { temp = xhci_port_state_to_neutral(temp); temp |= port_bit; - writel(temp, port->addr); + xhci_portsc_writel(port, temp); } } @@ -1002,7 +1002,7 @@ static int xhci_handle_usb2_port_link_resume(struct xhci_port *port, } xhci_ring_device(xhci, port->slot_id); } else { - int port_status = readl(port->addr); + int port_status = xhci_portsc_readl(port); xhci_warn(xhci, "Port resume timed out, port %d-%d: 0x%x\n", hcd->self.busnum, wIndex + 1, port_status); @@ -1263,7 +1263,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, wIndex--; port = ports[portnum1 - 1]; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if (temp == ~(u32)0) { xhci_hc_died(xhci); retval = -ENODEV; @@ -1288,7 +1288,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, retval = -EINVAL; break; } - port_li = readl(port->addr + PORTLI); + port_li = readl(&port->port_reg->portli); status = xhci_get_ext_port_status(temp, port_li); put_unaligned_le32(status, &buf[4]); } @@ -1309,7 +1309,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, port = ports[portnum1 - 1]; wIndex--; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if (temp == ~(u32)0) { xhci_hc_died(xhci); retval = -ENODEV; @@ -1319,7 +1319,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, /* FIXME: What new port features do we need to support? */ switch (wValue) { case USB_PORT_FEAT_SUSPEND: - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if ((temp & PORT_PLS_MASK) != XDEV_U0) { /* Resume the port to U0 first */ xhci_set_link_state(xhci, port, XDEV_U0); @@ -1331,7 +1331,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, * a port unless the port reports that it is in the * enabled (PED = ‘1’,PLS < ‘3’) state. */ - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if ((temp & PORT_PE) == 0 || (temp & PORT_RESET) || (temp & PORT_PLS_MASK) >= XDEV_U3) { xhci_warn(xhci, "USB core suspending port %d-%d not in U0/U1/U2\n", @@ -1354,11 +1354,11 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, msleep(10); /* wait device to enter */ spin_lock_irqsave(&xhci->lock, flags); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); bus_state->suspended_ports |= 1 << wIndex; break; case USB_PORT_FEAT_LINK_STATE: - temp = readl(port->addr); + temp = xhci_portsc_readl(port); /* Disable port */ if (link_state == USB_SS_PORT_LS_SS_DISABLED) { xhci_dbg(xhci, "Disable port %d-%d\n", @@ -1371,8 +1371,8 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, temp |= PORT_CSC | PORT_PEC | PORT_WRC | PORT_OCC | PORT_RC | PORT_PLC | PORT_CEC; - writel(temp | PORT_PE, port->addr); - temp = readl(port->addr); + xhci_portsc_writel(port, temp | PORT_PE); + temp = xhci_portsc_readl(port); break; } @@ -1381,7 +1381,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, xhci_dbg(xhci, "Enable port %d-%d\n", hcd->self.busnum, portnum1); xhci_set_link_state(xhci, port, link_state); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); break; } @@ -1414,7 +1414,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, hcd->self.busnum, portnum1); xhci_set_link_state(xhci, port, link_state); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); break; } /* Port must be enabled */ @@ -1462,7 +1462,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, xhci_dbg(xhci, "missing U0 port change event for port %d-%d\n", hcd->self.busnum, portnum1); spin_lock_irqsave(&xhci->lock, flags); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); break; } @@ -1480,12 +1480,12 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, spin_unlock_irqrestore(&xhci->lock, flags); while (retries--) { usleep_range(4000, 8000); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if ((temp & PORT_PLS_MASK) == XDEV_U3) break; } spin_lock_irqsave(&xhci->lock, flags); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); bus_state->suspended_ports |= 1 << wIndex; } break; @@ -1500,38 +1500,38 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, break; case USB_PORT_FEAT_RESET: temp = (temp | PORT_RESET); - writel(temp, port->addr); + xhci_portsc_writel(port, temp); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); xhci_dbg(xhci, "set port reset, actual port %d-%d status = 0x%x\n", hcd->self.busnum, portnum1, temp); break; case USB_PORT_FEAT_REMOTE_WAKE_MASK: xhci_set_remote_wake_mask(xhci, port, wake_mask); - temp = readl(port->addr); + temp = xhci_portsc_readl(port); xhci_dbg(xhci, "set port remote wake mask, actual port %d-%d status = 0x%x\n", hcd->self.busnum, portnum1, temp); break; case USB_PORT_FEAT_BH_PORT_RESET: temp |= PORT_WR; - writel(temp, port->addr); - temp = readl(port->addr); + xhci_portsc_writel(port, temp); + temp = xhci_portsc_readl(port); break; case USB_PORT_FEAT_U1_TIMEOUT: if (hcd->speed < HCD_USB3) goto error; - temp = readl(port->addr + PORTPMSC); + temp = readl(&port->port_reg->portpmsc); temp &= ~PORT_U1_TIMEOUT_MASK; temp |= PORT_U1_TIMEOUT(timeout); - writel(temp, port->addr + PORTPMSC); + writel(temp, &port->port_reg->portpmsc); break; case USB_PORT_FEAT_U2_TIMEOUT: if (hcd->speed < HCD_USB3) goto error; - temp = readl(port->addr + PORTPMSC); + temp = readl(&port->port_reg->portpmsc); temp &= ~PORT_U2_TIMEOUT_MASK; temp |= PORT_U2_TIMEOUT(timeout); - writel(temp, port->addr + PORTPMSC); + writel(temp, &port->port_reg->portpmsc); break; case USB_PORT_FEAT_TEST: /* 4.19.6 Port Test Modes (USB2 Test Mode) */ @@ -1547,7 +1547,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, goto error; } /* unblock any posted writes */ - temp = readl(port->addr); + temp = xhci_portsc_readl(port); break; case ClearPortFeature: if (!portnum1 || portnum1 > max_ports) @@ -1556,7 +1556,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, port = ports[portnum1 - 1]; wIndex--; - temp = readl(port->addr); + temp = xhci_portsc_readl(port); if (temp == ~(u32)0) { xhci_hc_died(xhci); retval = -ENODEV; @@ -1566,7 +1566,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, temp = xhci_port_state_to_neutral(temp); switch (wValue) { case USB_PORT_FEAT_SUSPEND: - temp = readl(port->addr); + temp = xhci_portsc_readl(port); xhci_dbg(xhci, "clear USB_PORT_FEAT_SUSPEND\n"); xhci_dbg(xhci, "PORTSC %04x\n", temp); if (temp & PORT_RESET) @@ -1603,8 +1603,7 @@ int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, case USB_PORT_FEAT_C_ENABLE: case USB_PORT_FEAT_C_PORT_LINK_STATE: case USB_PORT_FEAT_C_PORT_CONFIG_ERROR: - xhci_clear_port_change_bit(xhci, wValue, wIndex, - port->addr, temp); + xhci_clear_port_change_bit(xhci, wValue, wIndex, port, temp); break; case USB_PORT_FEAT_ENABLE: xhci_disable_port(xhci, port); @@ -1682,7 +1681,7 @@ int xhci_hub_status_data(struct usb_hcd *hcd, char *buf) /* For each port, did anything change? If so, set that bit in buf. */ for (i = 0; i < max_ports; i++) { - temp = readl(ports[i]->addr); + temp = xhci_portsc_readl(ports[i]); if (temp == ~(u32)0) { xhci_hc_died(xhci); retval = -ENODEV; @@ -1751,7 +1750,7 @@ int xhci_bus_suspend(struct usb_hcd *hcd) u32 t1, t2; int retries = 10; retry: - t1 = readl(ports[port_index]->addr); + t1 = xhci_portsc_readl(ports[port_index]); t2 = xhci_port_state_to_neutral(t1); portsc_buf[port_index] = 0; @@ -1829,7 +1828,7 @@ int xhci_bus_suspend(struct usb_hcd *hcd) spin_lock_irqsave(&xhci->lock, flags); } } - writel(portsc_buf[port_index], ports[port_index]->addr); + xhci_portsc_writel(ports[port_index], portsc_buf[port_index]); } hcd->state = HC_STATE_SUSPENDED; bus_state->next_statechange = jiffies + msecs_to_jiffies(10); @@ -1850,7 +1849,7 @@ static bool xhci_port_missing_cas_quirk(struct xhci_port *port) { u32 portsc; - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); /* if any of these are set we are not stuck */ if (portsc & (PORT_CONNECT | PORT_CAS)) @@ -1863,9 +1862,9 @@ static bool xhci_port_missing_cas_quirk(struct xhci_port *port) /* clear wakeup/change bits, and do a warm port reset */ portsc &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS); portsc |= PORT_WR; - writel(portsc, port->addr); + xhci_portsc_writel(port, portsc); /* flush write */ - readl(port->addr); + xhci_portsc_readl(port); return true; } @@ -1912,7 +1911,7 @@ int xhci_bus_resume(struct usb_hcd *hcd) } port_index = max_ports; while (port_index--) { - portsc = readl(ports[port_index]->addr); + portsc = xhci_portsc_readl(ports[port_index]); /* warm reset CAS limited ports stuck in polling/compliance */ if ((xhci->quirks & XHCI_MISSING_CAS) && @@ -1942,7 +1941,7 @@ int xhci_bus_resume(struct usb_hcd *hcd) } /* disable wake for all ports, write new link state if needed */ portsc &= ~(PORT_RWC_BITS | PORT_CEC | PORT_WAKE_BITS); - writel(portsc, ports[port_index]->addr); + xhci_portsc_writel(ports[port_index], portsc); } /* USB2 specific resume signaling delay and U0 link state transition */ @@ -1963,7 +1962,7 @@ int xhci_bus_resume(struct usb_hcd *hcd) /* poll for U0 link state complete, both USB2 and USB3 */ for_each_set_bit(port_index, &bus_state->bus_suspended, BITS_PER_LONG) { - sret = xhci_handshake(ports[port_index]->addr, PORT_PLC, + sret = xhci_handshake(&ports[port_index]->port_reg->portsc, PORT_PLC, PORT_PLC, 10 * 1000); if (sret) { xhci_warn(xhci, "port %d-%d resume PLC timeout\n", diff --git a/drivers/usb/host/xhci-mem.c b/drivers/usb/host/xhci-mem.c index c7c6e631cfc1b..8a8636279e844 100644 --- a/drivers/usb/host/xhci-mem.c +++ b/drivers/usb/host/xhci-mem.c @@ -317,12 +317,6 @@ void xhci_initialize_ring_info(struct xhci_ring *ring) * handling ring expansion, set the cycle state equal to the old ring. */ ring->cycle_state = 1; - - /* - * Each segment has a link TRB, and leave an extra TRB for SW - * accounting purpose - */ - ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1; } EXPORT_SYMBOL_GPL(xhci_initialize_ring_info); @@ -959,7 +953,7 @@ static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_i /* is this a hub device that added a tt_info to the tts list */ if (tt_info->slot_id == slot_id) { /* are any devices using this tt_info? */ - for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { + for (i = 1; i < xhci->max_slots; i++) { vdev = xhci->devs[i]; if (vdev && (vdev->tt_info == tt_info)) xhci_free_virt_devices_depth_first( @@ -1936,7 +1930,7 @@ EXPORT_SYMBOL_GPL(xhci_remove_secondary_interrupter); void xhci_mem_cleanup(struct xhci_hcd *xhci) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - int i, j, num_ports; + int i, j; cancel_delayed_work_sync(&xhci->cmd_timer); @@ -1955,8 +1949,7 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); xhci_cleanup_command_queue(xhci); - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - for (i = 0; i < num_ports && xhci->rh_bw; i++) { + for (i = 0; i < xhci->max_ports && xhci->rh_bw; i++) { struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; for (j = 0; j < XHCI_MAX_INTERVAL; j++) { struct list_head *ep = &bwt->interval_bw[j].endpoints; @@ -1965,7 +1958,7 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) } } - for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) + for (i = xhci->max_slots; i > 0; i--) xhci_free_virt_devices_depth_first(xhci, i); dma_pool_destroy(xhci->segment_pool); @@ -2001,7 +1994,7 @@ void xhci_mem_cleanup(struct xhci_hcd *xhci) if (!xhci->rh_bw) goto no_bw; - for (i = 0; i < num_ports; i++) { + for (i = 0; i < xhci->max_ports; i++) { struct xhci_tt_bw_info *tt, *n; list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { list_del(&tt->tt_list); @@ -2202,7 +2195,7 @@ static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, if (!rhub->ports) return; - for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { + for (i = 0; i < xhci->max_ports; i++) { if (xhci->hw_ports[i].rhub != rhub || xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) continue; @@ -2225,32 +2218,28 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) { void __iomem *base; u32 offset; - unsigned int num_ports; int i, j; int cap_count = 0; u32 cap_start; struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - num_ports = HCS_MAX_PORTS(xhci->hcs_params1); - xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports), - flags, dev_to_node(dev)); + xhci->hw_ports = kcalloc_node(xhci->max_ports, sizeof(*xhci->hw_ports), + flags, dev_to_node(dev)); if (!xhci->hw_ports) return -ENOMEM; - for (i = 0; i < num_ports; i++) { - xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base + - NUM_PORT_REGS * i; + for (i = 0; i < xhci->max_ports; i++) { + xhci->hw_ports[i].port_reg = &xhci->op_regs->port_regs[i]; xhci->hw_ports[i].hw_portnum = i; init_completion(&xhci->hw_ports[i].rexit_done); init_completion(&xhci->hw_ports[i].u3exit_done); } - xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags, - dev_to_node(dev)); + xhci->rh_bw = kcalloc_node(xhci->max_ports, sizeof(*xhci->rh_bw), flags, dev_to_node(dev)); if (!xhci->rh_bw) return -ENOMEM; - for (i = 0; i < num_ports; i++) { + for (i = 0; i < xhci->max_ports; i++) { struct xhci_interval_bw_table *bw_table; INIT_LIST_HEAD(&xhci->rh_bw[i].tts); @@ -2282,9 +2271,8 @@ static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) offset = cap_start; while (offset) { - xhci_add_in_port(xhci, num_ports, base + offset, cap_count); - if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == - num_ports) + xhci_add_in_port(xhci, xhci->max_ports, base + offset, cap_count); + if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == xhci->max_ports) break; offset = xhci_find_next_ext_cap(base, offset, XHCI_EXT_CAPS_PROTOCOL); diff --git a/drivers/usb/host/xhci-pci.c b/drivers/usb/host/xhci-pci.c index 86272de4bfe1e..0ad9e5f26d7fa 100644 --- a/drivers/usb/host/xhci-pci.c +++ b/drivers/usb/host/xhci-pci.c @@ -460,6 +460,7 @@ static void xhci_pci_quirks(struct device *dev, struct xhci_hcd *xhci) if (pdev->vendor == PCI_VENDOR_ID_VIA && pdev->device == PCI_DEVICE_ID_VIA_VL805) { xhci->quirks |= XHCI_LPM_SUPPORT; xhci->quirks |= XHCI_TRB_OVERFETCH; + xhci->dma_mask_bits = 40; xhci->quirks |= XHCI_EP_CTX_BROKEN_DCS; xhci->quirks |= XHCI_AVOID_DQ_ON_LINK; xhci->quirks |= XHCI_VLI_SS_BULK_OUT_BUG; @@ -913,9 +914,9 @@ static int xhci_pci_poweroff_late(struct usb_hcd *hcd, bool do_wakeup) if (!(xhci->quirks & XHCI_RESET_TO_DEFAULT)) return 0; - for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { + for (i = 0; i < xhci->max_ports; i++) { port = &xhci->hw_ports[i]; - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); if ((portsc & PORT_PLS_MASK) != XDEV_U3) continue; @@ -936,7 +937,7 @@ static int xhci_pci_poweroff_late(struct usb_hcd *hcd, bool do_wakeup) xhci_dbg(xhci, "port %d-%d in U3 without wakeup, disable it\n", port->rhub->hcd->self.busnum, port->hcd_portnum + 1); portsc = xhci_port_state_to_neutral(portsc); - writel(portsc | PORT_PE, port->addr); + xhci_portsc_writel(port, portsc | PORT_PE); } return 0; diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index d9e0a43381ebe..6f2832a0a3fff 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -339,7 +339,7 @@ static struct xhci_segment *trb_in_td(struct xhci_td *td, dma_addr_t suspect_dma * Only for transfer and command rings where driver is the producer, not for * event rings. */ -static unsigned int xhci_num_trbs_free(struct xhci_ring *ring) +unsigned int xhci_num_trbs_free(struct xhci_ring *ring) { struct xhci_segment *enq_seg = ring->enq_seg; union xhci_trb *enq = ring->enqueue; @@ -885,21 +885,18 @@ static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, usb_hcd_giveback_urb(hcd, urb, status); } -static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, - struct xhci_ring *ring, struct xhci_td *td) +static void xhci_unmap_one_bounce_buffer(struct xhci_hcd *xhci, + struct xhci_ring *ring, struct xhci_td *td, + struct xhci_segment *seg) { struct device *dev = xhci_to_hcd(xhci)->self.sysdev; - struct xhci_segment *seg = td->bounce_seg; struct urb *urb = td->urb; size_t len; - if (!ring || !seg || !urb) - return; - if (usb_urb_dir_out(urb)) { dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, DMA_TO_DEVICE); - return; + goto done; } dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, @@ -915,10 +912,29 @@ static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, seg->bounce_len); } +done: seg->bounce_len = 0; seg->bounce_offs = 0; } +static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, + struct xhci_ring *ring, struct xhci_td *td) +{ + struct xhci_segment *seg; + int i = 0; + + if (!td->bounce_seg || !ring || !td->urb) + return; + + /* td->bounce_seg is the last one bounced, unmap them all */ + for (seg = td->start_seg; i++ < ring->num_segs; seg = seg->next) { + if (seg->bounce_len) + xhci_unmap_one_bounce_buffer(xhci, ring, td, seg); + if (seg == td->bounce_seg) + break; + } +} + static void xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_ring *ep_ring, int status) { @@ -1437,7 +1453,7 @@ void xhci_hc_died(struct xhci_hcd *xhci) xhci_cleanup_command_queue(xhci); /* return any pending urbs, remove may be waiting for them */ - for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { + for (i = 0; i <= xhci->max_slots; i++) { if (!xhci->devs[i]) continue; for (j = 0; j < 31; j++) @@ -2038,7 +2054,6 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) struct usb_hcd *hcd; u32 port_id; u32 portsc, cmd_reg; - int max_ports; unsigned int hcd_portnum; struct xhci_bus_state *bus_state; bool bogus_port_status = false; @@ -2050,9 +2065,8 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) "WARN: xHC returned failed port status event\n"); port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); - max_ports = HCS_MAX_PORTS(xhci->hcs_params1); - if ((port_id <= 0) || (port_id > max_ports)) { + if ((port_id <= 0) || (port_id > xhci->max_ports)) { xhci_warn(xhci, "Port change event with invalid port ID %d\n", port_id); return; @@ -2070,7 +2084,7 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) vdev = xhci->devs[port->slot_id]; /* We might get interrupts after shared_hcd is removed */ - if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { + if (port->rhub == &xhci->usb3_rhub && xhci_get_usb3_hcd(xhci) == NULL) { xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); bogus_port_status = true; goto cleanup; @@ -2079,7 +2093,7 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) hcd = port->rhub->hcd; bus_state = &port->rhub->bus_state; hcd_portnum = port->hcd_portnum; - portsc = readl(port->addr); + portsc = xhci_portsc_readl(port); xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", hcd->self.busnum, hcd_portnum + 1, port_id, portsc); @@ -2614,6 +2628,7 @@ static void process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, td->status = 0; break; case COMP_SHORT_PACKET: + ep->err_count = 0; td->status = 0; break; case COMP_STOPPED_SHORT_PACKET: @@ -2697,6 +2712,17 @@ static bool xhci_spurious_success_tx_event(struct xhci_hcd *xhci, } } +static struct xhci_td *find_td_by_dma(struct xhci_ring *ep_ring, dma_addr_t dma) +{ + struct xhci_td *td; + + if (dma) + list_for_each_entry(td, &ep_ring->td_list, td_list) + if (trb_in_td(td, dma)) + return td; + return NULL; +} + /* * If this function returns an error condition, it means it got a Transfer * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address. @@ -2889,8 +2915,11 @@ static int handle_tx_event(struct xhci_hcd *xhci, xhci_dequeue_td(xhci, td, ep_ring, td->status); } - /* If the TRB pointer is NULL, missed TDs will be skipped on the next event */ - if (trb_comp_code == COMP_MISSED_SERVICE_ERROR && !ep_trb_dma) + /* + * We don't know how many TDs were missed when ep_trb_dma is zero (as permitted by + * xHCI 1.0) or bogus. Bail out leaving ep->skip set, next event will sort it out. + */ + if (trb_comp_code == COMP_MISSED_SERVICE_ERROR && !find_td_by_dma(ep_ring, ep_trb_dma)) return 0; if (list_empty(&ep_ring->td_list)) { @@ -3811,7 +3840,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, &trb_buff_len, ring->enq_seg)) { send_addr = ring->enq_seg->bounce_dma; - /* assuming TD won't span 2 segs */ + /* TD bounced at least, and last on this seg */ td->bounce_seg = ring->enq_seg; } } @@ -4475,6 +4504,7 @@ static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, u32 field3, u32 field4, bool command_must_succeed) { int reserved_trbs = xhci->cmd_ring_reserved_trbs; + struct usb_hcd *hcd = xhci_to_hcd(xhci); int ret; if ((xhci->xhc_state & XHCI_STATE_DYING) || @@ -4484,6 +4514,11 @@ static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, return -ESHUTDOWN; } + if (!HCD_HW_ACCESSIBLE(hcd)) { + xhci_warn(xhci, "Can't queue command, xHC not accessible\n"); + return -ESHUTDOWN; + } + if (!command_must_succeed) reserved_trbs++; diff --git a/drivers/usb/host/xhci-tegra.c b/drivers/usb/host/xhci-tegra.c index b0dcdede1fc8e..4620c2e23b887 100644 --- a/drivers/usb/host/xhci-tegra.c +++ b/drivers/usb/host/xhci-tegra.c @@ -2025,7 +2025,7 @@ static bool xhci_hub_ports_suspended(struct xhci_hub *hub) u32 value; for (i = 0; i < hub->num_ports; i++) { - value = readl(hub->ports[i]->addr); + value = xhci_portsc_readl(hub->ports[i]); if ((value & PORT_PE) == 0) continue; @@ -2151,7 +2151,7 @@ static void tegra_xhci_enable_phy_sleepwalk_wake(struct tegra_xusb *tegra) if (!is_host_mode_phy(tegra, i, j)) continue; - portsc = readl(rhub->ports[index]->addr); + portsc = xhci_portsc_readl(rhub->ports[index]); speed = tegra_xhci_portsc_to_speed(tegra, portsc); tegra_xusb_padctl_enable_phy_sleepwalk(padctl, phy, speed); tegra_xusb_padctl_enable_phy_wake(padctl, phy); @@ -2246,7 +2246,7 @@ static int tegra_xusb_enter_elpg(struct tegra_xusb *tegra, bool is_auto_resume) for (i = 0; i < xhci->usb2_rhub.num_ports; i++) { if (!xhci->usb2_rhub.ports[i]) continue; - portsc = readl(xhci->usb2_rhub.ports[i]->addr); + portsc = xhci_portsc_readl(xhci->usb2_rhub.ports[i]); tegra->lp0_utmi_pad_mask &= ~BIT(i); if (((portsc & PORT_PLS_MASK) == XDEV_U3) || ((portsc & DEV_SPEED_MASK) == XDEV_FS)) tegra->lp0_utmi_pad_mask |= BIT(i); @@ -2784,7 +2784,7 @@ static int tegra_xhci_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, while (i--) { if (!test_bit(i, &bus_state->resuming_ports)) continue; - portsc = readl(ports[i]->addr); + portsc = xhci_portsc_readl(ports[i]); if ((portsc & PORT_PLS_MASK) == XDEV_RESUME) tegra_phy_xusb_utmi_pad_power_on( tegra_xusb_get_phy(tegra, "usb2", (int) i)); @@ -2802,7 +2802,7 @@ static int tegra_xhci_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, if (!index || index > rhub->num_ports) return -EPIPE; ports = rhub->ports; - portsc = readl(ports[port]->addr); + portsc = xhci_portsc_readl(ports[port]); if (portsc & PORT_CONNECT) tegra_phy_xusb_utmi_pad_power_on(phy); } @@ -2821,7 +2821,7 @@ static int tegra_xhci_hub_control(struct usb_hcd *hcd, u16 type_req, u16 value, if ((type_req == ClearPortFeature) && (value == USB_PORT_FEAT_C_CONNECTION)) { ports = rhub->ports; - portsc = readl(ports[port]->addr); + portsc = xhci_portsc_readl(ports[port]); if (!(portsc & PORT_CONNECT)) { /* We don't suspend the PAD while HNP role swap happens on the OTG * port diff --git a/drivers/usb/host/xhci-trace.h b/drivers/usb/host/xhci-trace.h index 9abc904f17495..481becbcbf813 100644 --- a/drivers/usb/host/xhci-trace.h +++ b/drivers/usb/host/xhci-trace.h @@ -575,6 +575,11 @@ DEFINE_EVENT(xhci_log_portsc, xhci_hub_status_data, TP_ARGS(port, portsc) ); +DEFINE_EVENT(xhci_log_portsc, xhci_portsc_writel, + TP_PROTO(struct xhci_port *port, u32 portsc), + TP_ARGS(port, portsc) +); + DECLARE_EVENT_CLASS(xhci_log_doorbell, TP_PROTO(u32 slot, u32 doorbell), TP_ARGS(slot, doorbell), diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index ac5978215dcd6..6bf660cb313db 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -45,6 +45,19 @@ static int sandbag_lpm = 1; module_param(sandbag_lpm, int, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(sandbag_lpm, "Use relaxed U1/U2 port LPM timeouts"); +void xhci_portsc_writel(struct xhci_port *port, u32 val) +{ + trace_xhci_portsc_writel(port, val); + writel(val, &port->port_reg->portsc); +} +EXPORT_SYMBOL_GPL(xhci_portsc_writel); + +u32 xhci_portsc_readl(struct xhci_port *port) +{ + return readl(&port->port_reg->portsc); +} +EXPORT_SYMBOL_GPL(xhci_portsc_readl); + static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring) { struct xhci_segment *seg; @@ -282,8 +295,7 @@ static void xhci_zero_64b_regs(struct xhci_hcd *xhci) if (upper_32_bits(val)) xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring); - intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1), - ARRAY_SIZE(xhci->run_regs->ir_set)); + intrs = min_t(u32, xhci->max_interrupters, ARRAY_SIZE(xhci->run_regs->ir_set)); for (i = 0; i < intrs; i++) { struct xhci_intr_reg __iomem *ir; @@ -377,7 +389,7 @@ static void compliance_mode_recovery(struct timer_list *t) return; for (i = 0; i < rhub->num_ports; i++) { - temp = readl(rhub->ports[i]->addr); + temp = xhci_portsc_readl(rhub->ports[i]); if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) { /* * Compliance Mode Detected. Letting USB Core @@ -475,15 +487,13 @@ static void xhci_hcd_page_size(struct xhci_hcd *xhci) static void xhci_enable_max_dev_slots(struct xhci_hcd *xhci) { u32 config_reg; - u32 max_slots; - max_slots = HCS_MAX_SLOTS(xhci->hcs_params1); xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xHC can handle at most %d device slots", - max_slots); + xhci->max_slots); config_reg = readl(&xhci->op_regs->config_reg); config_reg &= ~HCS_SLOTS_MASK; - config_reg |= max_slots; + config_reg |= xhci->max_slots; xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Setting Max device slots reg = 0x%x", config_reg); @@ -900,7 +910,7 @@ static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci, spin_lock_irqsave(&xhci->lock, flags); for (i = 0; i < rhub->num_ports; i++) { - portsc = readl(rhub->ports[i]->addr); + portsc = xhci_portsc_readl(rhub->ports[i]); t1 = xhci_port_state_to_neutral(portsc); t2 = t1; @@ -913,7 +923,7 @@ static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci, t2 |= PORT_CSC; if (t1 != t2) { - writel(t2, rhub->ports[i]->addr); + xhci_portsc_writel(rhub->ports[i], t2); xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n", rhub->hcd->self.busnum, i + 1, portsc, t2); } @@ -940,7 +950,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) port_index = xhci->usb2_rhub.num_ports; ports = xhci->usb2_rhub.ports; while (port_index--) { - portsc = readl(ports[port_index]->addr); + portsc = xhci_portsc_readl(ports[port_index]); if (portsc & PORT_CHANGE_MASK || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; @@ -948,7 +958,7 @@ static bool xhci_pending_portevent(struct xhci_hcd *xhci) port_index = xhci->usb3_rhub.num_ports; ports = xhci->usb3_rhub.ports; while (port_index--) { - portsc = readl(ports[port_index]->addr); + portsc = xhci_portsc_readl(ports[port_index]); if (portsc & (PORT_CHANGE_MASK | PORT_CAS) || (portsc & PORT_PLS_MASK) == XDEV_RESUME) return true; @@ -1006,6 +1016,10 @@ int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup) /* step 1: stop endpoint */ /* skipped assuming that port suspend has done */ + /* Check if command ring is empty */ + if (!list_empty(&xhci->cmd_list)) + xhci_warn(xhci, "Suspending and stopping xHC with pending command!\n"); + /* step 2: clear Run/Stop bit */ command = readl(&xhci->op_regs->command); command &= ~CMD_RUN; @@ -4352,8 +4366,7 @@ int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev) xhci_err(xhci, "Error while assigning device slot ID: %s\n", xhci_trb_comp_code_string(command->status)); xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n", - HCS_MAX_SLOTS( - readl(&xhci->cap_regs->hcs_params1))); + xhci->max_slots); xhci_free_command(xhci, command); return 0; } @@ -4766,7 +4779,7 @@ static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd, { struct xhci_hcd *xhci = hcd_to_xhci(hcd); struct xhci_port **ports; - __le32 __iomem *pm_addr, *hlpm_addr; + struct xhci_port_regs __iomem *port_reg; u32 pm_val, hlpm_val, field; unsigned int port_num; unsigned long flags; @@ -4791,9 +4804,8 @@ static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd, ports = xhci->usb2_rhub.ports; port_num = udev->portnum - 1; - pm_addr = ports[port_num]->addr + PORTPMSC; - pm_val = readl(pm_addr); - hlpm_addr = ports[port_num]->addr + PORTHLPMC; + port_reg = ports[port_num]->port_reg; + pm_val = readl(&port_reg->portpmsc); xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n", str_enable_disable(enable), port_num + 1); @@ -4822,30 +4834,30 @@ static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd, spin_lock_irqsave(&xhci->lock, flags); hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev); - writel(hlpm_val, hlpm_addr); + writel(hlpm_val, &port_reg->porthlmpc); /* flush write */ - readl(hlpm_addr); + readl(&port_reg->porthlmpc); } else { hird = xhci_calculate_hird_besl(xhci, udev); } pm_val &= ~PORT_HIRD_MASK; pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id); - writel(pm_val, pm_addr); - pm_val = readl(pm_addr); + writel(pm_val, &port_reg->portpmsc); + pm_val = readl(&port_reg->portpmsc); pm_val |= PORT_HLE; - writel(pm_val, pm_addr); + writel(pm_val, &port_reg->portpmsc); /* flush write */ - readl(pm_addr); + readl(&port_reg->portpmsc); } else { pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK); - writel(pm_val, pm_addr); + writel(pm_val, &port_reg->portpmsc); /* flush write */ - readl(pm_addr); + readl(&port_reg->portpmsc); if (udev->usb2_hw_lpm_besl_capable) { spin_unlock_irqrestore(&xhci->lock, flags); xhci_change_max_exit_latency(xhci, udev, 0); - readl_poll_timeout(ports[port_num]->addr, pm_val, + readl_poll_timeout(&ports[port_num]->port_reg->portsc, pm_val, (pm_val & PORT_PLS_MASK) == XDEV_U0, 100, 10000); return 0; @@ -5538,6 +5550,8 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) */ struct device *dev = hcd->self.sysdev; int retval; + u32 hcs_params1; + u32 hc_capbase; /* Accept arbitrarily long scatter-gather lists */ hcd->self.sg_tablesize = ~0; @@ -5558,23 +5572,29 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) mutex_init(&xhci->mutex); xhci->main_hcd = hcd; xhci->cap_regs = hcd->regs; - xhci->op_regs = hcd->regs + - HC_LENGTH(readl(&xhci->cap_regs->hc_capbase)); + hc_capbase = readl(&xhci->cap_regs->hc_capbase); + if (hc_capbase == U32_MAX) { + xhci_warn(xhci, "Host controller not accessible, removed?\n"); + return -ENODEV; + } + xhci->op_regs = hcd->regs + HC_LENGTH(hc_capbase); xhci->run_regs = hcd->regs + (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK); /* Cache read-only capability registers */ - xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1); + hcs_params1 = readl(&xhci->cap_regs->hcs_params1); xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2); xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3); - xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase)); + xhci->hci_version = HC_VERSION(hc_capbase); xhci->hcc_params = readl(&xhci->cap_regs->hcc_params); if (xhci->hci_version > 0x100) xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2); + xhci->dma_mask_bits = 64; + xhci->max_slots = HCS_MAX_SLOTS(hcs_params1); + xhci->max_ports = HCS_MAX_PORTS(hcs_params1); /* xhci-plat or xhci-pci might have set max_interrupters already */ - if ((!xhci->max_interrupters) || - xhci->max_interrupters > HCS_MAX_INTRS(xhci->hcs_params1)) - xhci->max_interrupters = HCS_MAX_INTRS(xhci->hcs_params1); + if ((!xhci->max_interrupters) || xhci->max_interrupters > HCS_MAX_INTRS(hcs_params1)) + xhci->max_interrupters = HCS_MAX_INTRS(hcs_params1); xhci->quirks |= quirks; @@ -5617,12 +5637,16 @@ int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks) if (xhci->quirks & XHCI_NO_64BIT_SUPPORT) xhci->hcc_params &= ~BIT(0); - /* Set dma_mask and coherent_dma_mask to 64-bits, - * if xHC supports 64-bit addressing */ + /* + * Set dma_mask and coherent_dma_mask to 64-bits if xHC supports + * 64-bit addressing, unless a controller-specific quirk callback + * limits the usable address width. + */ if (HCC_64BIT_ADDR(xhci->hcc_params) && - !dma_set_mask(dev, DMA_BIT_MASK(64))) { - xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n"); - dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); + !dma_set_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits))) { + xhci_dbg(xhci, "Enabling %u-bit DMA addresses.\n", + xhci->dma_mask_bits); + dma_set_coherent_mask(dev, DMA_BIT_MASK(xhci->dma_mask_bits)); } else { /* * This is to avoid error in cases where a 32-bit USB diff --git a/drivers/usb/host/xhci.h b/drivers/usb/host/xhci.h index c170c6aa7f0b0..6d1bcbc92a302 100644 --- a/drivers/usb/host/xhci.h +++ b/drivers/usb/host/xhci.h @@ -66,13 +66,19 @@ struct xhci_cap_regs { /* Reserved up to (CAPLENGTH - 0x1C) */ }; -/* Number of registers per port */ -#define NUM_PORT_REGS 4 - -#define PORTSC 0 -#define PORTPMSC 1 -#define PORTLI 2 -#define PORTHLPMC 3 +/* + * struct xhci_port_regs - Host Controller USB Port Register Set. xHCI spec 5.4.8 + * @portsc: Port Status and Control + * @portpmsc: Port Power Management Status and Control + * @portli: Port Link Info + * @porthlmpc: Port Hardware LPM Control + */ +struct xhci_port_regs { + __le32 portsc; + __le32 portpmsc; + __le32 portli; + __le32 porthlmpc; +}; /** * struct xhci_op_regs - xHCI Host Controller Operational Registers. @@ -85,16 +91,7 @@ struct xhci_cap_regs { * @cmd_ring: CRP - 64-bit Command Ring Pointer * @dcbaa_ptr: DCBAAP - 64-bit Device Context Base Address Array Pointer * @config_reg: CONFIG - Configure Register - * @port_status_base: PORTSCn - base address for Port Status and Control - * Each port has a Port Status and Control register, - * followed by a Port Power Management Status and Control - * register, a Port Link Info register, and a reserved - * register. - * @port_power_base: PORTPMSCn - base address for - * Port Power Management Status and Control - * @port_link_base: PORTLIn - base address for Port Link Info (current - * Link PM state and control) for USB 2.1 and USB 3.0 - * devices. + * @port_regs: Port Register Sets, from 1 to MaxPorts (defined by HCSPARAMS1). */ struct xhci_op_regs { __le32 command; @@ -110,13 +107,7 @@ struct xhci_op_regs { __le32 config_reg; /* rsvd: offset 0x3C-3FF */ __le32 reserved4[241]; - /* port 1 registers, which serve as a base address for other ports */ - __le32 port_status_base; - __le32 port_power_base; - __le32 port_link_base; - __le32 reserved5; - /* registers for ports 2-255 */ - __le32 reserved6[NUM_PORT_REGS*254]; + struct xhci_port_regs port_regs[]; }; /* USBCMD - USB command - command bitmasks */ @@ -1379,7 +1370,6 @@ struct xhci_ring { u32 cycle_state; unsigned int stream_id; unsigned int num_segs; - unsigned int num_trbs_free; /* used only by xhci DbC */ unsigned int bounce_buf_len; enum xhci_ring_type type; u32 old_trb_comp_code; @@ -1476,7 +1466,7 @@ struct xhci_port_cap { }; struct xhci_port { - __le32 __iomem *addr; + struct xhci_port_regs __iomem *port_reg; int hw_portnum; int hcd_portnum; struct xhci_hub *rhub; @@ -1512,7 +1502,6 @@ struct xhci_hcd { struct xhci_doorbell_array __iomem *dba; /* Cached register copies of read-only HC data */ - __u32 hcs_params1; __u32 hcs_params2; __u32 hcs_params3; __u32 hcc_params; @@ -1523,9 +1512,12 @@ struct xhci_hcd { /* packed release number */ u16 hci_version; u16 max_interrupters; + u8 max_slots; + u8 max_ports; /* imod_interval in ns (I * 250ns) */ u32 imod_interval; u32 page_size; + unsigned int dma_mask_bits; /* MSI-X/MSI vectors */ int nvecs; /* optional clocks */ @@ -1959,6 +1951,7 @@ void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, void xhci_cleanup_command_queue(struct xhci_hcd *xhci); void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring); unsigned int count_trbs(u64 addr, u64 len); +unsigned int xhci_num_trbs_free(struct xhci_ring *ring); int xhci_stop_endpoint_sync(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, int suspend, gfp_t gfp_flags); void xhci_process_cancelled_tds(struct xhci_virt_ep *ep); @@ -1968,6 +1961,8 @@ void xhci_update_erst_dequeue(struct xhci_hcd *xhci, void xhci_add_interrupter(struct xhci_hcd *xhci, unsigned int intr_num); int xhci_usb_endpoint_maxp(struct usb_device *udev, struct usb_host_endpoint *host_ep); +void xhci_portsc_writel(struct xhci_port *port, u32 val); +u32 xhci_portsc_readl(struct xhci_port *port); /* xHCI roothub code */ void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port, diff --git a/drivers/usb/image/mdc800.c b/drivers/usb/image/mdc800.c index 10d72562e4d29..afc8da9cddf10 100644 --- a/drivers/usb/image/mdc800.c +++ b/drivers/usb/image/mdc800.c @@ -1000,13 +1000,13 @@ static int __init usb_mdc800_init (void) mdc800->downloaded = 0; mdc800->written = 0; - mdc800->irq_urb_buffer=kmalloc (8, GFP_KERNEL); + mdc800->irq_urb_buffer=kzalloc (8, GFP_KERNEL); if (!mdc800->irq_urb_buffer) goto cleanup_on_fail; mdc800->write_urb_buffer=kmalloc (8, GFP_KERNEL); if (!mdc800->write_urb_buffer) goto cleanup_on_fail; - mdc800->download_urb_buffer=kmalloc (64, GFP_KERNEL); + mdc800->download_urb_buffer=kzalloc (64, GFP_KERNEL); if (!mdc800->download_urb_buffer) goto cleanup_on_fail; diff --git a/drivers/usb/misc/usb-ljca.c b/drivers/usb/misc/usb-ljca.c index c562630d862c7..b19cc99edfeb6 100644 --- a/drivers/usb/misc/usb-ljca.c +++ b/drivers/usb/misc/usb-ljca.c @@ -585,6 +585,9 @@ static int ljca_enumerate_gpio(struct ljca_adapter *adap) if (gpio_num > LJCA_MAX_GPIO_NUM) return -EINVAL; + if (desc->bank_num > ARRAY_SIZE(valid_pin)) + return -EINVAL; + /* construct platform data */ gpio_info = kzalloc(sizeof *gpio_info, GFP_KERNEL); if (!gpio_info) diff --git a/drivers/usb/misc/usbio.c b/drivers/usb/misc/usbio.c index 7315cd2469234..7feae96b6f8ab 100644 --- a/drivers/usb/misc/usbio.c +++ b/drivers/usb/misc/usbio.c @@ -265,7 +265,7 @@ int usbio_bulk_msg(struct auxiliary_device *adev, u8 type, u8 cmd, bool last, lockdep_assert_held(&usbio->bulk_mutex); if ((obuf_len > (usbio->txbuf_len - sizeof(*bpkt))) || - (ibuf_len > (usbio->txbuf_len - sizeof(*bpkt)))) + (ibuf_len > (usbio->rxbuf_len - sizeof(*bpkt)))) return -EMSGSIZE; if (ibuf_len) diff --git a/drivers/usb/misc/usbtest.c b/drivers/usb/misc/usbtest.c index 5c92c8d8e2833..a015fbe4fc359 100644 --- a/drivers/usb/misc/usbtest.c +++ b/drivers/usb/misc/usbtest.c @@ -3054,6 +3054,7 @@ static struct usb_driver usbtest_driver = { .disconnect = usbtest_disconnect, .suspend = usbtest_suspend, .resume = usbtest_resume, + .no_dynamic_id = 1, }; /*-------------------------------------------------------------------------*/ diff --git a/drivers/usb/mtu3/mtu3_core.c b/drivers/usb/mtu3/mtu3_core.c index a3a6282893d09..ca8cb6c3a01bf 100644 --- a/drivers/usb/mtu3/mtu3_core.c +++ b/drivers/usb/mtu3/mtu3_core.c @@ -1037,9 +1037,14 @@ int ssusb_gadget_suspend(struct ssusb_mtk *ssusb, pm_message_t msg) if (!mtu->gadget_driver) return 0; - if (mtu->connected) + /* Prevent runtime suspend when active connection exists */ + if (mtu->connected && PMSG_IS_AUTO(msg)) return -EBUSY; + /* Perform soft disconnect for system suspend */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 0); + mtu3_dev_suspend(mtu); synchronize_irq(mtu->irq); @@ -1055,5 +1060,9 @@ int ssusb_gadget_resume(struct ssusb_mtk *ssusb, pm_message_t msg) mtu3_dev_resume(mtu); + /* Restore soft connect for system resume */ + if (mtu->softconnect && !PMSG_IS_AUTO(msg)) + mtu3_dev_on_off(mtu, 1); + return 0; } diff --git a/drivers/usb/musb/omap2430.c b/drivers/usb/musb/omap2430.c index aadabc415145b..77133ee77063c 100644 --- a/drivers/usb/musb/omap2430.c +++ b/drivers/usb/musb/omap2430.c @@ -455,7 +455,6 @@ static int omap2430_probe(struct platform_device *pdev) dev_err(&pdev->dev, "failed to register musb device\n"); goto err_disable_rpm; } - of_node_put(np); return 0; @@ -465,7 +464,6 @@ static int omap2430_probe(struct platform_device *pdev) if (!IS_ERR(glue->control_otghs)) put_device(glue->control_otghs); err_put_musb: - of_node_put(np); platform_device_put(musb); return ret; diff --git a/drivers/usb/phy/phy-fsl-usb.c b/drivers/usb/phy/phy-fsl-usb.c index e266a47c4d483..1c405e5dc32c3 100644 --- a/drivers/usb/phy/phy-fsl-usb.c +++ b/drivers/usb/phy/phy-fsl-usb.c @@ -46,7 +46,7 @@ static const char driver_name[] = "fsl-usb2-otg"; -const pm_message_t otg_suspend_state = { +static const pm_message_t otg_suspend_state = { .event = 1, }; @@ -57,11 +57,11 @@ static struct fsl_otg *fsl_otg_dev; static int srp_wait_done; /* FSM timers */ -struct fsl_otg_timer *a_wait_vrise_tmr, *a_wait_bcon_tmr, *a_aidl_bdis_tmr, +static struct fsl_otg_timer *a_wait_vrise_tmr, *a_wait_bcon_tmr, *a_aidl_bdis_tmr, *b_ase0_brst_tmr, *b_se0_srp_tmr; /* Driver specific timers */ -struct fsl_otg_timer *b_data_pulse_tmr, *b_vbus_pulse_tmr, *b_srp_fail_tmr, +static struct fsl_otg_timer *b_data_pulse_tmr, *b_vbus_pulse_tmr, *b_srp_fail_tmr, *b_srp_wait_tmr, *a_wait_enum_tmr; static struct list_head active_timers; @@ -102,7 +102,7 @@ static void (*_fsl_writel)(u32 v, unsigned __iomem *p); #define fsl_writel(val, addr) writel(val, addr) #endif /* CONFIG_PPC32 */ -int write_ulpi(u8 addr, u8 data) +static int write_ulpi(u8 addr, u8 data) { u32 temp; @@ -115,7 +115,7 @@ int write_ulpi(u8 addr, u8 data) /* Operations that will be called from OTG Finite State Machine */ /* Charge vbus for vbus pulsing in SRP */ -void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) +static void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) { u32 tmp; @@ -133,7 +133,7 @@ void fsl_otg_chrg_vbus(struct otg_fsm *fsm, int on) } /* Discharge vbus through a resistor to ground */ -void fsl_otg_dischrg_vbus(int on) +static void fsl_otg_dischrg_vbus(int on) { u32 tmp; @@ -151,7 +151,7 @@ void fsl_otg_dischrg_vbus(int on) } /* A-device driver vbus, controlled through PP bit in PORTSC */ -void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) +static void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) { u32 tmp; @@ -169,7 +169,7 @@ void fsl_otg_drv_vbus(struct otg_fsm *fsm, int on) * Pull-up D+, signalling connect by periperal. Also used in * data-line pulsing in SRP */ -void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) +static void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) { u32 tmp; @@ -188,7 +188,7 @@ void fsl_otg_loc_conn(struct otg_fsm *fsm, int on) * port. In host mode, controller will automatically send SOF. * Suspend will block the data on the port. */ -void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) +static void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) { u32 tmp; @@ -203,7 +203,7 @@ void fsl_otg_loc_sof(struct otg_fsm *fsm, int on) } /* Start SRP pulsing by data-line pulsing, followed with v-bus pulsing. */ -void fsl_otg_start_pulse(struct otg_fsm *fsm) +static void fsl_otg_start_pulse(struct otg_fsm *fsm) { u32 tmp; @@ -219,7 +219,7 @@ void fsl_otg_start_pulse(struct otg_fsm *fsm) fsl_otg_add_timer(fsm, b_data_pulse_tmr); } -void b_data_pulse_end(unsigned long foo) +static void b_data_pulse_end(unsigned long foo) { #ifdef HA_DATA_PULSE #else @@ -230,7 +230,7 @@ void b_data_pulse_end(unsigned long foo) fsl_otg_pulse_vbus(); } -void fsl_otg_pulse_vbus(void) +static void fsl_otg_pulse_vbus(void) { srp_wait_done = 0; fsl_otg_chrg_vbus(&fsl_otg_dev->fsm, 1); @@ -238,7 +238,7 @@ void fsl_otg_pulse_vbus(void) fsl_otg_add_timer(&fsl_otg_dev->fsm, b_vbus_pulse_tmr); } -void b_vbus_pulse_end(unsigned long foo) +static void b_vbus_pulse_end(unsigned long foo) { fsl_otg_chrg_vbus(&fsl_otg_dev->fsm, 0); @@ -251,7 +251,7 @@ void b_vbus_pulse_end(unsigned long foo) fsl_otg_add_timer(&fsl_otg_dev->fsm, b_srp_wait_tmr); } -void b_srp_end(unsigned long foo) +static void b_srp_end(unsigned long foo) { fsl_otg_dischrg_vbus(0); srp_wait_done = 1; @@ -266,7 +266,7 @@ void b_srp_end(unsigned long foo) * a_host will start by SRP. It needs to set b_hnp_enable before * actually suspending to start HNP */ -void a_wait_enum(unsigned long foo) +static void a_wait_enum(unsigned long foo) { VDBG("a_wait_enum timeout\n"); if (!fsl_otg_dev->phy.otg->host->b_hnp_enable) @@ -276,13 +276,13 @@ void a_wait_enum(unsigned long foo) } /* The timeout callback function to set time out bit */ -void set_tmout(unsigned long indicator) +static void set_tmout(unsigned long indicator) { *(int *)indicator = 1; } /* Initialize timers */ -int fsl_otg_init_timers(struct otg_fsm *fsm) +static int fsl_otg_init_timers(struct otg_fsm *fsm) { /* FSM used timers */ a_wait_vrise_tmr = otg_timer_initializer(&set_tmout, TA_WAIT_VRISE, @@ -339,7 +339,7 @@ int fsl_otg_init_timers(struct otg_fsm *fsm) } /* Uninitialize timers */ -void fsl_otg_uninit_timers(void) +static void fsl_otg_uninit_timers(void) { /* FSM used timers */ kfree(a_wait_vrise_tmr); @@ -391,7 +391,7 @@ static struct fsl_otg_timer *fsl_otg_get_timer(enum otg_fsm_timer t) } /* Add timer to timer list */ -void fsl_otg_add_timer(struct otg_fsm *fsm, void *gtimer) +static void fsl_otg_add_timer(struct otg_fsm *fsm, void *gtimer) { struct fsl_otg_timer *timer = gtimer; struct fsl_otg_timer *tmp_timer; @@ -421,7 +421,7 @@ static void fsl_otg_fsm_add_timer(struct otg_fsm *fsm, enum otg_fsm_timer t) } /* Remove timer from the timer list; clear timeout status */ -void fsl_otg_del_timer(struct otg_fsm *fsm, void *gtimer) +static void fsl_otg_del_timer(struct otg_fsm *fsm, void *gtimer) { struct fsl_otg_timer *timer = gtimer; struct fsl_otg_timer *tmp_timer, *del_tmp; @@ -443,7 +443,7 @@ static void fsl_otg_fsm_del_timer(struct otg_fsm *fsm, enum otg_fsm_timer t) } /* Reset controller, not reset the bus */ -void otg_reset_controller(void) +static void otg_reset_controller(void) { u32 command; @@ -455,7 +455,7 @@ void otg_reset_controller(void) } /* Call suspend/resume routines in host driver */ -int fsl_otg_start_host(struct otg_fsm *fsm, int on) +static int fsl_otg_start_host(struct otg_fsm *fsm, int on) { struct usb_otg *otg = fsm->otg; struct device *dev; @@ -522,7 +522,7 @@ int fsl_otg_start_host(struct otg_fsm *fsm, int on) * Call suspend and resume function in udc driver * to stop and start udc driver. */ -int fsl_otg_start_gadget(struct otg_fsm *fsm, int on) +static int fsl_otg_start_gadget(struct otg_fsm *fsm, int on) { struct usb_otg *otg = fsm->otg; struct device *dev; @@ -704,7 +704,7 @@ static int fsl_otg_start_hnp(struct usb_otg *otg) * intact. It needs to have knowledge of some USB interrupts * such as port change. */ -irqreturn_t fsl_otg_isr(int irq, void *dev_id) +static irqreturn_t fsl_otg_isr(int irq, void *dev_id) { struct otg_fsm *fsm = &((struct fsl_otg *)dev_id)->fsm; struct usb_otg *otg = ((struct fsl_otg *)dev_id)->phy.otg; @@ -830,7 +830,7 @@ static int fsl_otg_conf(struct platform_device *pdev) } /* OTG Initialization */ -int usb_otg_start(struct platform_device *pdev) +static int usb_otg_start(struct platform_device *pdev) { struct fsl_otg *p_otg; struct usb_phy *otg_trans = usb_get_phy(USB_PHY_TYPE_USB2); @@ -1002,7 +1002,7 @@ static void fsl_otg_remove(struct platform_device *pdev) pdata->exit(pdev); } -struct platform_driver fsl_otg_driver = { +static struct platform_driver fsl_otg_driver = { .probe = fsl_otg_probe, .remove = fsl_otg_remove, .driver = { diff --git a/drivers/usb/phy/phy-fsl-usb.h b/drivers/usb/phy/phy-fsl-usb.h index d70341ae5a926..cc841c4e47627 100644 --- a/drivers/usb/phy/phy-fsl-usb.h +++ b/drivers/usb/phy/phy-fsl-usb.h @@ -373,6 +373,6 @@ struct fsl_otg_config { #define FSL_OTG_NAME "fsl-usb2-otg" -void fsl_otg_add_timer(struct otg_fsm *fsm, void *timer); -void fsl_otg_del_timer(struct otg_fsm *fsm, void *timer); -void fsl_otg_pulse_vbus(void); +static void fsl_otg_add_timer(struct otg_fsm *fsm, void *timer); +static void fsl_otg_del_timer(struct otg_fsm *fsm, void *timer); +static void fsl_otg_pulse_vbus(void); diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index 8a79548e1569b..fddb57b8d8472 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -813,9 +813,6 @@ static void usbhs_remove(struct platform_device *pdev) flush_delayed_work(&priv->notify_hotplug_work); - usbhs_platform_call(priv, hardware_exit, pdev); - reset_control_assert(priv->rsts); - /* * Explicitly free the IRQ to ensure the interrupt handler is * disabled and synchronized before freeing resources. @@ -832,6 +829,9 @@ static void usbhs_remove(struct platform_device *pdev) if (!usbhs_get_dparam(priv, runtime_pwctrl)) usbhsc_power_ctrl(priv, 0); + usbhs_platform_call(priv, hardware_exit, pdev); + reset_control_assert(priv->rsts); + usbhsc_clk_put(priv); pm_runtime_disable(&pdev->dev); } diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index ceddfb1852e16..8eb50640e8b73 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -1075,6 +1075,8 @@ static const struct usb_device_id id_table_combined[] = { { USB_DEVICE_INTERFACE_NUMBER(ALTERA_VID, ALTERA_UB3_602E_PID, 3) }, /* Abacus Electrics */ { USB_DEVICE(FTDI_VID, ABACUS_OPTICAL_PROBE_PID) }, + /* Endress+Hauser AG devices */ + { USB_DEVICE(FTDI_VID, FTDI_EH_FXA291_PID) }, { } /* Terminating entry */ }; diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index 6c76cfebfd0e4..9c83c17853c87 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -313,6 +313,11 @@ #define FTDI_ELV_UDF77_PID 0xFB5E /* USB DCF Funkuhr (UDF 77) */ #define FTDI_ELV_UIO88_PID 0xFB5F /* USB-I/O Interface (UIO 88) */ +/* + * Endress+Hauser AG product ids (FTDI_VID) + */ +#define FTDI_EH_FXA291_PID 0xE510 + /* * EVER Eco Pro UPS (http://www.ever.com.pl/) */ diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c index ad73040b30c8d..9b8fb8467b537 100644 --- a/drivers/usb/serial/io_edgeport.c +++ b/drivers/usb/serial/io_edgeport.c @@ -646,7 +646,8 @@ static void edge_interrupt_callback(struct urb *urb) if (edge_port && edge_port->open) { spin_lock_irqsave(&edge_port->ep_lock, flags); - edge_port->txCredits += txCredits; + edge_port->txCredits = min(edge_port->txCredits + txCredits, + edge_port->maxTxCredits); spin_unlock_irqrestore(&edge_port->ep_lock, flags); dev_dbg(dev, "%s - txcredits for port%d = %d\n", diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c index f800bf5ac1de9..e500ffd150854 100644 --- a/drivers/usb/serial/keyspan_pda.c +++ b/drivers/usb/serial/keyspan_pda.c @@ -35,6 +35,8 @@ struct keyspan_pda_private { struct work_struct unthrottle_work; struct usb_serial *serial; struct usb_serial_port *port; + bool throttled; + bool throttle_req; }; static int keyspan_pda_write_start(struct usb_serial_port *port); @@ -150,6 +152,7 @@ static void keyspan_pda_rx_interrupt(struct urb *urb) int retval; int status = urb->status; struct keyspan_pda_private *priv; + bool throttled = false; unsigned long flags; priv = usb_get_serial_port_data(port); @@ -211,16 +214,24 @@ static void keyspan_pda_rx_interrupt(struct urb *urb) } exit: - retval = usb_submit_urb(urb, GFP_ATOMIC); - if (retval) - dev_err(&port->dev, - "%s - usb_submit_urb failed with result %d\n", - __func__, retval); + spin_lock_irqsave(&port->lock, flags); + if (priv->throttle_req) { + priv->throttled = true; + throttled = true; + } + spin_unlock_irqrestore(&port->lock, flags); + + if (!throttled) { + retval = usb_submit_urb(urb, GFP_ATOMIC); + if (retval) + dev_err(&port->dev, "failed to resubmit in urb: %d\n", retval); + } } static void keyspan_pda_rx_throttle(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; + struct keyspan_pda_private *priv = usb_get_serial_port_data(port); /* * Stop receiving characters. We just turn off the URB request, and @@ -230,16 +241,29 @@ static void keyspan_pda_rx_throttle(struct tty_struct *tty) * send an XOFF, although it might make sense to foist that off upon * the device too. */ - usb_kill_urb(port->interrupt_in_urb); + spin_lock_irq(&port->lock); + priv->throttle_req = true; + spin_unlock_irq(&port->lock); } static void keyspan_pda_rx_unthrottle(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; + struct keyspan_pda_private *priv = usb_get_serial_port_data(port); + bool throttled; + int ret; - /* just restart the receive interrupt URB */ - if (usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL)) - dev_dbg(&port->dev, "usb_submit_urb(read urb) failed\n"); + spin_lock_irq(&port->lock); + throttled = priv->throttled; + priv->throttled = false; + priv->throttle_req = false; + spin_unlock_irq(&port->lock); + + if (throttled) { + ret = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL); + if (ret) + dev_err(&port->dev, "failed to submit in urb: %d\n", ret); + } } static speed_t keyspan_pda_setbaud(struct usb_serial *serial, speed_t baud) @@ -579,6 +603,8 @@ static int keyspan_pda_open(struct tty_struct *tty, spin_lock_irq(&port->lock); priv->tx_room = rc; + priv->throttled = false; + priv->throttle_req = false; spin_unlock_irq(&port->lock); rc = usb_submit_urb(port->interrupt_in_urb, GFP_KERNEL); diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index b4b58e48e2253..5652155fc680b 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -2497,6 +2497,7 @@ static const struct usb_device_id option_ids[] = { .driver_info = RSVD(5) }, { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff), /* Rolling RW135R-GL (laptop MBIM) */ .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x3466, 0x3301, 0xff) }, /* TDTECH MT5710-CN */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */ { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) }, { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) }, @@ -2689,12 +2690,26 @@ static void option_instat_callback(struct urb *urb) dev_dbg(dev, "%s: NULL req_pkt\n", __func__); return; } + + if (urb->actual_length < sizeof(*req_pkt)) { + dev_err(dev, "%s: short packet: %u bytes\n", __func__, + urb->actual_length); + return; + } + if ((req_pkt->bRequestType == 0xA1) && (req_pkt->bRequest == 0x20)) { + unsigned char signals; int old_dcd_state; - unsigned char signals = *((unsigned char *) - urb->transfer_buffer + - sizeof(struct usb_ctrlrequest)); + + if (urb->actual_length < sizeof(*req_pkt) + 1) { + dev_err(dev, "%s: short interrupt transfer: %u bytes\n", + __func__, urb->actual_length); + return; + } + + signals = *((unsigned char *)urb->transfer_buffer + + sizeof(*req_pkt)); dev_dbg(dev, "%s: signal x%x\n", __func__, signals); diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c index 11077beb7232a..8418d812eb3cd 100644 --- a/drivers/usb/serial/spcp8x5.c +++ b/drivers/usb/serial/spcp8x5.c @@ -135,14 +135,14 @@ struct spcp8x5_private { static int spcp8x5_probe(struct usb_serial *serial, const struct usb_device_id *id) { - usb_set_serial_data(serial, (void *)id); + usb_set_serial_data(serial, (void *)id->driver_info); return 0; } static int spcp8x5_port_probe(struct usb_serial_port *port) { - const struct usb_device_id *id = usb_get_serial_data(port->serial); + unsigned int quirks = (unsigned int)(unsigned long)usb_get_serial_data(port->serial); struct spcp8x5_private *priv; priv = kzalloc(sizeof(*priv), GFP_KERNEL); @@ -150,7 +150,7 @@ static int spcp8x5_port_probe(struct usb_serial_port *port) return -ENOMEM; spin_lock_init(&priv->lock); - priv->quirks = id->driver_info; + priv->quirks = quirks; usb_set_serial_port_data(port, priv); @@ -237,18 +237,6 @@ static void spcp8x5_set_work_mode(struct usb_serial_port *port, u16 value, dev_err(&port->dev, "failed to set work mode: %d\n", ret); } -static int spcp8x5_carrier_raised(struct usb_serial_port *port) -{ - u8 msr; - int ret; - - ret = spcp8x5_get_msr(port, &msr); - if (ret || msr & MSR_STATUS_LINE_DCD) - return 1; - - return 0; -} - static void spcp8x5_dtr_rts(struct usb_serial_port *port, int on) { struct spcp8x5_private *priv = usb_get_serial_port_data(port); @@ -460,7 +448,6 @@ static struct usb_serial_driver spcp8x5_device = { .num_bulk_out = 1, .open = spcp8x5_open, .dtr_rts = spcp8x5_dtr_rts, - .carrier_raised = spcp8x5_carrier_raised, .set_termios = spcp8x5_set_termios, .init_termios = spcp8x5_init_termios, .tiocmget = spcp8x5_tiocmget, diff --git a/drivers/usb/storage/ene_ub6250.c b/drivers/usb/storage/ene_ub6250.c index 5b4cbd15c3c8a..31727753ef2a8 100644 --- a/drivers/usb/storage/ene_ub6250.c +++ b/drivers/usb/storage/ene_ub6250.c @@ -2358,7 +2358,9 @@ static int ene_ub6250_probe(struct usb_interface *intf, return result; /* probe card type */ + mutex_lock(&us->dev_mutex); result = ene_get_card_type(us, REG_CARD_STATUS, info->bbuf); + mutex_unlock(&us->dev_mutex); if (result != USB_STOR_XFER_GOOD) { usb_stor_disconnect(intf); return USB_STOR_TRANSPORT_ERROR; diff --git a/drivers/usb/storage/realtek_cr.c b/drivers/usb/storage/realtek_cr.c index 3cc243956fd4a..3601278fc5424 100644 --- a/drivers/usb/storage/realtek_cr.c +++ b/drivers/usb/storage/realtek_cr.c @@ -916,7 +916,6 @@ static int realtek_cr_autosuspend_setup(struct us_data *us) us->proto_handler = rts51x_invoke_transport; chip->timer_expires = 0; - timer_setup(&chip->rts51x_suspend_timer, rts51x_suspend_timer_fn, 0); fw5895_init(us); /* enable autosuspend function of the usb device */ @@ -934,10 +933,7 @@ static void realtek_cr_destructor(void *extra) return; #ifdef CONFIG_REALTEK_AUTOPM - if (ss_en) { - timer_delete(&chip->rts51x_suspend_timer); - chip->timer_expires = 0; - } + timer_shutdown_sync(&chip->rts51x_suspend_timer); #endif kfree(chip->status); } @@ -982,6 +978,9 @@ static int init_realtek_cr(struct us_data *us) us->extra = chip; us->extra_destructor = realtek_cr_destructor; +#ifdef CONFIG_REALTEK_AUTOPM + timer_setup(&chip->rts51x_suspend_timer, rts51x_suspend_timer_fn, 0); +#endif us->max_lun = chip->max_lun = rts51x_get_max_lun(us); chip->us = us; diff --git a/drivers/usb/storage/scsiglue.c b/drivers/usb/storage/scsiglue.c index d2f476e48d0c7..f81c997656f59 100644 --- a/drivers/usb/storage/scsiglue.c +++ b/drivers/usb/storage/scsiglue.c @@ -111,7 +111,8 @@ static int sdev_configure(struct scsi_device *sdev, struct queue_limits *lim) * let the queue segment size sort out the real limit. */ lim->max_hw_sectors = 0x7FFFFF; - } else if (us->pusb_dev->speed >= USB_SPEED_SUPER) { + } else if (us->pusb_dev->speed >= USB_SPEED_SUPER || + (us->pusb_dev->bos && us->pusb_dev->bos->ss_cap)) { /* * USB3 devices will be limited to 2048 sectors. This gives us * better throughput on most devices. @@ -119,6 +120,15 @@ static int sdev_configure(struct scsi_device *sdev, struct queue_limits *lim) lim->max_hw_sectors = 2048; } + /* + * The default of 240 sectors on 16K page platforms causes decomposition + * of transfers into lengths of 224 sectors aligned to 32 sectors. + * Middle-of-the-range devices show a modest sequential speed increase + * with 192 sectors (64s alignment). + */ + if (lim->max_hw_sectors == 240 && PAGE_SIZE == SZ_16K) + lim->max_hw_sectors = 192; + /* * The max_hw_sectors should be up to maximum size of a mapping for * the device. Otherwise, a DMA API might fail on swiotlb environment. diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index 255968f9ca42a..ac22fa3187343 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -395,6 +395,13 @@ UNUSUAL_DEV( 0x04b3, 0x4001, 0x0110, 0x0110, USB_SC_DEVICE, USB_PR_CB, NULL, US_FL_MAX_SECTORS_MIN), +/* Reported by Ai Chao */ +UNUSUAL_DEV( 0x04b4, 0xb708, 0x0000, 0xffff, + "Longmai Technologies", + "USB Key", + USB_SC_SCSI, USB_PR_BULK, NULL, + US_FL_NO_ATA_1X), + /* * Reported by Simon Levitt * This entry needs Sub and Proto fields diff --git a/drivers/usb/typec/altmodes/thunderbolt.c b/drivers/usb/typec/altmodes/thunderbolt.c index d09dd09cf1c31..1f4dab8d62646 100644 --- a/drivers/usb/typec/altmodes/thunderbolt.c +++ b/drivers/usb/typec/altmodes/thunderbolt.c @@ -303,6 +303,8 @@ static void tbt_altmode_remove(struct typec_altmode *alt) { struct tbt_altmode *tbt = typec_altmode_get_drvdata(alt); + disable_work_sync(&tbt->work); + for (int i = TYPEC_PLUG_SOP_PP; i >= 0; --i) { if (tbt->plug[i]) typec_altmode_put_plug(tbt->plug[i]); diff --git a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c index c8b1463e6e8b0..abb9d03210324 100644 --- a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c +++ b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_pdphy.c @@ -544,6 +544,8 @@ static void qcom_pmic_typec_pdphy_stop(struct pmic_typec *tcpm) for (i = 0; i < pmic_typec_pdphy->nr_irqs; i++) disable_irq(pmic_typec_pdphy->irq_data[i].irq); + cancel_work_sync(&pmic_typec_pdphy->reset_work); + qcom_pmic_typec_pdphy_reset_on(pmic_typec_pdphy); regulator_disable(pmic_typec_pdphy->vdd_pdphy); diff --git a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c index 8051eaa469913..0b3d125c50cb6 100644 --- a/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c +++ b/drivers/usb/typec/tcpm/qcom/qcom_pmic_typec_port.c @@ -683,6 +683,9 @@ static int qcom_pmic_typec_port_start(struct pmic_typec *tcpm, enable_irq(pmic_typec_port->irq_data[i].irq); done: + if (ret) + disable_delayed_work_sync(&pmic_typec_port->cc_debounce_dwork); + return ret; } @@ -693,6 +696,8 @@ static void qcom_pmic_typec_port_stop(struct pmic_typec *tcpm) for (i = 0; i < pmic_typec_port->nr_irqs; i++) disable_irq(pmic_typec_port->irq_data[i].irq); + + disable_delayed_work_sync(&pmic_typec_port->cc_debounce_dwork); } int qcom_pmic_typec_port_probe(struct platform_device *pdev, diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c index 2a951c585e92b..3be5d3f58c50d 100644 --- a/drivers/usb/typec/tcpm/tcpci.c +++ b/drivers/usb/typec/tcpm/tcpci.c @@ -37,6 +37,7 @@ struct tcpci { struct regmap *regmap; unsigned int alert_mask; + unsigned int rx_type_mask; bool controls_vbus; @@ -485,6 +486,8 @@ static int tcpci_set_pd_rx(struct tcpc_dev *tcpc, bool enable) if (tcpci->data->cable_comm_capable) reg |= TCPC_RX_DETECT_SOP1; } + + tcpci->rx_type_mask = reg; ret = regmap_write(tcpci->regmap, TCPC_RX_DETECT, reg); if (ret < 0) return ret; @@ -746,6 +749,7 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) if (status & TCPC_ALERT_RX_STATUS) { struct pd_message msg; unsigned int cnt, payload_cnt; + enum tcpm_transmit_type rx_type; u16 header; regmap_read(tcpci->regmap, TCPC_RX_BYTE_CNT, &cnt); @@ -770,10 +774,16 @@ irqreturn_t tcpci_irq(struct tcpci *tcpci) regmap_raw_read(tcpci->regmap, TCPC_RX_DATA, &msg.payload, payload_cnt); + ret = regmap_read(tcpci->regmap, TCPC_RX_BUF_FRAME_TYPE, &rx_type); + if (ret) + return ret; + /* Read complete, clear RX status alert bit */ tcpci_write16(tcpci, TCPC_ALERT, TCPC_ALERT_RX_STATUS); - tcpm_pd_receive(tcpci->port, &msg, TCPC_TX_SOP); + rx_type &= TCPC_RX_BUF_FRAME_TYPE_MASK; + if (tcpci->rx_type_mask & BIT(rx_type)) + tcpm_pd_receive(tcpci->port, &msg, rx_type); } if (tcpci->data->vbus_vsafe0v && (status & TCPC_ALERT_EXTENDED_STATUS)) { diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c index 2209fab8f4431..30cea27ff9320 100644 --- a/drivers/usb/typec/tcpm/tcpm.c +++ b/drivers/usb/typec/tcpm/tcpm.c @@ -6509,16 +6509,32 @@ static void tcpm_pd_event_handler(struct kthread_work *work) } } if (events & TCPM_SOURCING_VBUS) { - tcpm_log(port, "sourcing vbus"); /* * In fast role swap case TCPC autonomously sources vbus. Set vbus_source - * true as TCPM wouldn't have called tcpm_set_vbus. + * true conditionally as TCPM wouldn't have called tcpm_set_vbus. + * If TCPM calls tcpm_set_vbus to source vbus, vbus_source would already + * be true. * - * When vbus is sourced on the command on TCPM i.e. TCPM called - * tcpm_set_vbus to source vbus, vbus_source would already be true. + * When TCPM_FRS_EVENT and TCPM_SOURCING_VBUS arrive simultaneously, + * handling TCPM_FRS_EVENT above transitions the state to AMS_START + * with upcoming_state FR_SWAP_SEND. */ - port->vbus_source = true; - _tcpm_pd_vbus_on(port); + + if (tcpm_port_is_source(port) || + tcpm_port_is_debug_source(port) || + (port->state == AMS_START && port->upcoming_state == FR_SWAP_SEND) || + port->state == FR_SWAP_SEND || + port->state == FR_SWAP_SEND_TIMEOUT || + port->state == FR_SWAP_SNK_SRC_TRANSITION_TO_OFF || + port->state == FR_SWAP_SNK_SRC_NEW_SINK_READY || + port->state == FR_SWAP_SNK_SRC_SOURCE_VBUS_APPLIED) { + tcpm_log(port, "sourcing vbus"); + port->vbus_source = true; + _tcpm_pd_vbus_on(port); + } else { + tcpm_log(port, "Discarding sourcing vbus! Invalid state %s", + tcpm_states[port->state]); + } } if (events & TCPM_PORT_CLEAN) { tcpm_log(port, "port clean"); diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c index 0c711dbe0c973..d6fbe515e644e 100644 --- a/drivers/usb/typec/tipd/core.c +++ b/drivers/usb/typec/tipd/core.c @@ -114,7 +114,6 @@ struct tps6598x_intel_vid_status_reg { __le32 attention_vdo; __le16 enter_vdo; __le16 device_mode; - __le16 cable_mode; } __packed; /* Standard Task return codes */ @@ -694,9 +693,19 @@ static void cd321x_typec_update_mode(struct tps6598x *tps, struct cd321x_status cd321x->state.mode == TYPEC_TBT_MODE) return; - tbt_data.cable_mode = le16_to_cpu(st->intel_vid_status.cable_mode); - tbt_data.device_mode = le16_to_cpu(st->intel_vid_status.device_mode); - tbt_data.enter_vdo = le16_to_cpu(st->intel_vid_status.enter_vdo); + tbt_data.cable_mode = TBT_MODE | + TBT_SET_CABLE_SPEED(TPS_DATA_STATUS_TBT_CABLE_SPEED(st->data_status)) | + TBT_SET_CABLE_ROUNDED(TPS_DATA_STATUS_TBT_CABLE_GEN(st->data_status)); + if (st->data_status & TPS_DATA_STATUS_OPTICAL_CABLE) + tbt_data.cable_mode |= TBT_CABLE_OPTICAL; + if (st->data_status & TPS_DATA_STATUS_ACTIVE_LINK_TRAIN) + tbt_data.cable_mode |= TBT_CABLE_LINK_TRAINING; + if (st->data_status & TPS_DATA_STATUS_ACTIVE_CABLE) + tbt_data.cable_mode |= TBT_CABLE_ACTIVE_PASSIVE; + tbt_data.device_mode = TBT_MODE | + (u32)le16_to_cpu(st->intel_vid_status.device_mode) << 16; + tbt_data.enter_vdo = + (u32)le16_to_cpu(st->intel_vid_status.enter_vdo) << 16; cd321x->state.alt = cd321x->port_altmode_tbt; cd321x->state.mode = TYPEC_TBT_MODE; cd321x->state.data = &tbt_data; diff --git a/drivers/usb/typec/tipd/tps6598x.h b/drivers/usb/typec/tipd/tps6598x.h index 03edbb77bbd6d..d172c84ada74e 100644 --- a/drivers/usb/typec/tipd/tps6598x.h +++ b/drivers/usb/typec/tipd/tps6598x.h @@ -206,10 +206,10 @@ #define TPS_DATA_STATUS_DP_PIN_ASSIGNMENT(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_DP_PIN_ASSIGNMENT_MASK, (x)) #define TPS_DATA_STATUS_TBT_CABLE_SPEED_MASK GENMASK(27, 25) -#define TPS_DATA_STATUS_TBT_CABLE_SPEED \ +#define TPS_DATA_STATUS_TBT_CABLE_SPEED(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_TBT_CABLE_SPEED_MASK, (x)) #define TPS_DATA_STATUS_TBT_CABLE_GEN_MASK GENMASK(29, 28) -#define TPS_DATA_STATUS_TBT_CABLE_GEN \ +#define TPS_DATA_STATUS_TBT_CABLE_GEN(x) \ TPS_FIELD_GET(TPS_DATA_STATUS_TBT_CABLE_GEN_MASK, (x)) /* Map data status to DP spec assignments */ diff --git a/drivers/usb/typec/ucsi/debugfs.c b/drivers/usb/typec/ucsi/debugfs.c index f73f2b54554e2..414ffe1d23780 100644 --- a/drivers/usb/typec/ucsi/debugfs.c +++ b/drivers/usb/typec/ucsi/debugfs.c @@ -129,6 +129,7 @@ void ucsi_debugfs_unregister(struct ucsi *ucsi) debugfs_remove_recursive(ucsi->debugfs->dentry); kfree(ucsi->debugfs); + ucsi->debugfs = NULL; } void ucsi_debugfs_init(void) diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c index 7067f2561b846..8d2032d0762ca 100644 --- a/drivers/usb/typec/ucsi/displayport.c +++ b/drivers/usb/typec/ucsi/displayport.c @@ -74,7 +74,7 @@ static int ucsi_displayport_enter(struct typec_altmode *alt, u32 *vdo) cur = 0xff; } - if (cur != 0xff) { + if (cur < UCSI_MAX_ALTMODES) { ret = dp->con->port_altmode[cur] == alt ? 0 : -EBUSY; goto err_unlock; } diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c index b4b0c46d2fa44..2acd48a0a6a2b 100644 --- a/drivers/usb/typec/ucsi/ucsi.c +++ b/drivers/usb/typec/ucsi/ucsi.c @@ -78,7 +78,8 @@ int ucsi_sync_control_common(struct ucsi *ucsi, u64 command, u32 *cci, if (ret) goto out_clear_bit; - if (!wait_for_completion_timeout(&ucsi->complete, 5 * HZ)) + if (!wait_for_completion_timeout(&ucsi->complete, + msecs_to_jiffies(UCSI_TIMEOUT_MS))) ret = -ETIMEDOUT; out_clear_bit: @@ -487,6 +488,129 @@ static int ucsi_register_altmode(struct ucsi_connector *con, return ret; } +static void ucsi_dump_duplicate_altmode(struct ucsi_connector *con, + u8 recipient, u16 svid, + u32 existing_vdo, u32 new_vdo, + int offset) +{ + static const char * const recipient_names[] = { + [UCSI_RECIPIENT_CON] = "port", + [UCSI_RECIPIENT_SOP] = "partner", + [UCSI_RECIPIENT_SOP_P] = "plug", + [UCSI_RECIPIENT_SOP_PP] = "cable plug prime", + }; + + dev_warn(con->ucsi->dev, + "con%d: Firmware bug: duplicate %s altmode SVID 0x%04x at offset %d, ignoring but please contact the BIOS vendor to fix this issue.\n", + con->num, recipient_names[recipient], svid, offset); + + if (existing_vdo != new_vdo) + dev_warn(con->ucsi->dev, + "con%d: VDO mismatch: 0x%08x vs 0x%08x\n", + con->num, existing_vdo, new_vdo); +} + +/* Count altmodes in @altmodes that advertise @svid. */ +static int ucsi_altmode_count_svid(struct typec_altmode **altmodes, u16 svid) +{ + int count = 0; + int k; + + for (k = 0; k < UCSI_MAX_ALTMODES; k++) { + if (!altmodes[k]) + break; + if (altmodes[k]->svid == svid) + count++; + } + + return count; +} + +/* + * Check if an altmode is a duplicate. Some firmware implementations + * incorrectly return the same altmode multiple times, causing sysfs errors. + * Returns true if the altmode should be skipped. + * + * The matching rules differ by recipient: + * + * - UCSI_RECIPIENT_CON (port) and UCSI_RECIPIENT_SOP_P (plug): + * Two altmodes with identical SVID and VDO are byte-for-byte duplicates + * and the second has no observable function. Drop them. + * + * - UCSI_RECIPIENT_SOP (partner): + * The typec class binds each partner altmode to a port altmode of the + * same SVID via altmode_match()/device_find_child(), which returns the + * first port altmode with a matching SVID. If the partner advertises + * more altmodes for SVID X than the port advertises, the surplus + * partner altmode(s) collapse onto an already-paired port altmode and + * trigger a "duplicate filename .../partner" sysfs error during + * typec_altmode_create_links(). Use the port-side altmode count for + * SVID X as the authoritative cap and reject any partner altmode that + * would exceed it. This preserves legitimate multi-Mode partner + * altmodes (e.g. vendor SVIDs that the port really does advertise + * twice) while filtering the firmware-generated duplicates that have + * no port counterpart. + */ +static bool ucsi_altmode_is_duplicate(struct ucsi_connector *con, u8 recipient, + const struct ucsi_altmode *alt_batch, int batch_idx, + u16 svid, u32 vdo, int offset) +{ + struct typec_altmode **altmodes; + int port_count, partner_count; + int k; + + /* Check for duplicates within the current batch first */ + for (k = 0; k < batch_idx; k++) { + if (alt_batch[k].svid == svid && alt_batch[k].mid == vdo) { + ucsi_dump_duplicate_altmode(con, recipient, svid, + vdo, vdo, offset); + return true; + } + } + + switch (recipient) { + case UCSI_RECIPIENT_SOP: + /* + * Cap partner altmodes per SVID by the port-side count: + * any further partner altmode for that SVID would alias an + * already-paired port altmode and break typec sysfs. + */ + port_count = ucsi_altmode_count_svid(con->port_altmode, svid); + partner_count = ucsi_altmode_count_svid(con->partner_altmode, + svid); + if (port_count && partner_count >= port_count) { + ucsi_dump_duplicate_altmode(con, recipient, svid, + con->partner_altmode[partner_count - 1]->vdo, + vdo, offset); + return true; + } + return false; + case UCSI_RECIPIENT_CON: + altmodes = con->port_altmode; + break; + case UCSI_RECIPIENT_SOP_P: + altmodes = con->plug_altmode; + break; + default: + return false; + } + + /* CON and SOP_P: drop only exact SVID+VDO duplicates. */ + for (k = 0; k < UCSI_MAX_ALTMODES; k++) { + if (!altmodes[k]) + break; + + if (altmodes[k]->svid != svid || altmodes[k]->vdo != vdo) + continue; + + ucsi_dump_duplicate_altmode(con, recipient, svid, + altmodes[k]->vdo, vdo, offset); + return true; + } + + return false; +} + static int ucsi_register_altmodes_nvidia(struct ucsi_connector *con, u8 recipient) { @@ -541,19 +665,25 @@ ucsi_register_altmodes_nvidia(struct ucsi_connector *con, u8 recipient) /* now register altmodes */ for (i = 0; i < max_altmodes; i++) { - memset(&desc, 0, sizeof(desc)); - if (multi_dp) { - desc.svid = updated[i].svid; - desc.vdo = updated[i].mid; - } else { - desc.svid = orig[i].svid; - desc.vdo = orig[i].mid; - } - desc.roles = TYPEC_PORT_DRD; + struct ucsi_altmode *altmode_array = multi_dp ? updated : orig; - if (!desc.svid) + if (!altmode_array[i].svid) return 0; + /* + * Check for duplicates in current array and already + * registered altmodes. Skip if duplicate found. + */ + if (ucsi_altmode_is_duplicate(con, recipient, altmode_array, i, + altmode_array[i].svid, + altmode_array[i].mid, i)) + continue; + + memset(&desc, 0, sizeof(desc)); + desc.svid = altmode_array[i].svid; + desc.vdo = altmode_array[i].mid; + desc.roles = TYPEC_PORT_DRD; + ret = ucsi_register_altmode(con, &desc, recipient); if (ret) return ret; @@ -611,6 +741,15 @@ static int ucsi_register_altmodes(struct ucsi_connector *con, u8 recipient) if (!alt[j].svid) return 0; + /* + * Check for duplicates in current batch and already + * registered altmodes. Skip if duplicate found. + */ + if (ucsi_altmode_is_duplicate(con, recipient, alt, j, + alt[j].svid, alt[j].mid, + i - num + j)) + continue; + memset(&desc, 0, sizeof(desc)); desc.vdo = alt[j].mid; desc.svid = alt[j].svid; @@ -1603,6 +1742,7 @@ static int ucsi_register_port(struct ucsi *ucsi, struct ucsi_connector *con) INIT_WORK(&con->work, ucsi_handle_connector_change); init_completion(&con->complete); mutex_init(&con->lock); + lockdep_set_class(&con->lock, &con->lock_key); INIT_LIST_HEAD(&con->partner_tasks); con->ucsi = ucsi; @@ -1752,6 +1892,42 @@ static int ucsi_register_port(struct ucsi *ucsi, struct ucsi_connector *con) return ret; } +static void ucsi_unregister_port(struct ucsi_connector *con) +{ + struct ucsi_work *uwork; + + if (con->wq) { + mutex_lock(&con->lock); + ucsi_unregister_partner(con); + /* + * queue delayed items immediately so they can execute + * and free themselves before the wq is destroyed + */ + list_for_each_entry(uwork, &con->partner_tasks, node) { + if (cancel_delayed_work(&uwork->work)) + queue_delayed_work(con->wq, &uwork->work, 0); + } + mutex_unlock(&con->lock); + + destroy_workqueue(con->wq); + con->wq = NULL; + } else { + ucsi_unregister_partner(con); + } + + ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON); + ucsi_unregister_port_psy(con); + + usb_power_delivery_unregister_capabilities(con->port_sink_caps); + con->port_sink_caps = NULL; + usb_power_delivery_unregister_capabilities(con->port_source_caps); + con->port_source_caps = NULL; + usb_power_delivery_unregister(con->pd); + con->pd = NULL; + typec_unregister_port(con->port); + con->port = NULL; +} + static u64 ucsi_get_supported_notifications(struct ucsi *ucsi) { u16 features = ucsi->cap.features; @@ -1843,6 +2019,9 @@ static int ucsi_init(struct ucsi *ucsi) goto err_reset; } + for (i = 0; i < ucsi->cap.num_connectors; i++) + lockdep_register_key(&connector[i].lock_key); + /* Register all connectors */ for (i = 0; i < ucsi->cap.num_connectors; i++) { connector[i].num = i + 1; @@ -1872,22 +2051,11 @@ static int ucsi_init(struct ucsi *ucsi) return 0; err_unregister: - for (con = connector; con->port; con++) { - if (con->wq) - destroy_workqueue(con->wq); - ucsi_unregister_partner(con); - ucsi_unregister_altmodes(con, UCSI_RECIPIENT_CON); - ucsi_unregister_port_psy(con); - - usb_power_delivery_unregister_capabilities(con->port_sink_caps); - con->port_sink_caps = NULL; - usb_power_delivery_unregister_capabilities(con->port_source_caps); - con->port_source_caps = NULL; - usb_power_delivery_unregister(con->pd); - con->pd = NULL; - typec_unregister_port(con->port); - con->port = NULL; - } + for (con = connector; con->port; con++) + ucsi_unregister_port(con); + for (i = 0; i < ucsi->cap.num_connectors; i++) + lockdep_unregister_key(&connector[i].lock_key); + kfree(connector); err_reset: memset(&ucsi->cap, 0, sizeof(ucsi->cap)); @@ -2107,6 +2275,8 @@ void ucsi_unregister(struct ucsi *ucsi) cancel_delayed_work_sync(&ucsi->work); cancel_work_sync(&ucsi->resume_work); + ucsi_debugfs_unregister(ucsi); + /* Disable notifications */ ucsi->ops->async_control(ucsi, cmd); @@ -2115,33 +2285,8 @@ void ucsi_unregister(struct ucsi *ucsi) for (i = 0; i < ucsi->cap.num_connectors; i++) { cancel_work_sync(&ucsi->connector[i].work); - - if (ucsi->connector[i].wq) { - struct ucsi_work *uwork; - - mutex_lock(&ucsi->connector[i].lock); - /* - * queue delayed items immediately so they can execute - * and free themselves before the wq is destroyed - */ - list_for_each_entry(uwork, &ucsi->connector[i].partner_tasks, node) - mod_delayed_work(ucsi->connector[i].wq, &uwork->work, 0); - mutex_unlock(&ucsi->connector[i].lock); - destroy_workqueue(ucsi->connector[i].wq); - } - - ucsi_unregister_partner(&ucsi->connector[i]); - ucsi_unregister_altmodes(&ucsi->connector[i], - UCSI_RECIPIENT_CON); - ucsi_unregister_port_psy(&ucsi->connector[i]); - - usb_power_delivery_unregister_capabilities(ucsi->connector[i].port_sink_caps); - ucsi->connector[i].port_sink_caps = NULL; - usb_power_delivery_unregister_capabilities(ucsi->connector[i].port_source_caps); - ucsi->connector[i].port_source_caps = NULL; - usb_power_delivery_unregister(ucsi->connector[i].pd); - ucsi->connector[i].pd = NULL; - typec_unregister_port(ucsi->connector[i].port); + ucsi_unregister_port(&ucsi->connector[i]); + lockdep_unregister_key(&ucsi->connector[i].lock_key); } kfree(ucsi->connector); diff --git a/drivers/usb/typec/ucsi/ucsi.h b/drivers/usb/typec/ucsi/ucsi.h index a9144c2997391..520a1a4f91143 100644 --- a/drivers/usb/typec/ucsi/ucsi.h +++ b/drivers/usb/typec/ucsi/ucsi.h @@ -502,6 +502,7 @@ struct ucsi_connector { struct ucsi *ucsi; struct mutex lock; /* port lock */ + struct lock_class_key lock_key; struct work_struct work; struct completion complete; struct workqueue_struct *wq; diff --git a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c index c5965656babad..4d53f37ae1193 100644 --- a/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c +++ b/drivers/usb/typec/ucsi/ucsi_huawei_gaokun.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "ucsi.h" @@ -82,6 +83,10 @@ struct gaokun_ucsi_port { struct gaokun_ucsi *ucsi; struct auxiliary_device *bridge; + struct typec_mux *typec_mux; + struct typec_mux_state state; + struct typec_altmode dp_alt; + int idx; enum gaokun_ucsi_ccx ccx; enum gaokun_ucsi_mux mux; @@ -100,6 +105,7 @@ struct gaokun_ucsi { struct notifier_block nb; u16 version; u8 num_ports; + bool registered; }; /* -------------------------------------------------------------------------- */ @@ -226,19 +232,18 @@ static void gaokun_ucsi_port_update(struct gaokun_ucsi_port *port, port->hpd_state = FIELD_GET(GAOKUN_HPD_STATE_MASK, ddi); port->hpd_irq = FIELD_GET(GAOKUN_HPD_IRQ_MASK, ddi); - /* Mode and SVID are unused; keeping them to make things clearer */ switch (port->mode) { case USBC_DPAM_PAN_C: case USBC_DPAM_PAN_C_REVERSE: - port->mode = DP_PIN_ASSIGN_C; /* correct it for usb later */ + port->mode = TYPEC_DP_STATE_C; /* correct it for usb later */ break; case USBC_DPAM_PAN_D: case USBC_DPAM_PAN_D_REVERSE: - port->mode = DP_PIN_ASSIGN_D; + port->mode = TYPEC_DP_STATE_D; break; case USBC_DPAM_PAN_E: case USBC_DPAM_PAN_E_REVERSE: - port->mode = DP_PIN_ASSIGN_E; + port->mode = TYPEC_DP_STATE_E; break; case USBC_DPAM_PAN_NONE: port->mode = TYPEC_STATE_SAFE; @@ -287,18 +292,30 @@ static int gaokun_ucsi_refresh(struct gaokun_ucsi *uec) return idx; } -static void gaokun_ucsi_handle_altmode(struct gaokun_ucsi_port *port) +static void gaokun_ucsi_handle_usb_mode(struct gaokun_ucsi_port *port) { struct gaokun_ucsi *uec = port->ucsi; - int idx = port->idx; + int idx = port->idx, ret; + + /* + * For every typec port on this platform, the only mode-switch is + * controlled by its qmp combo phy which consumes svid and mode only. + */ + port->dp_alt.svid = port->svid; + port->state.mode = port->mode; + port->state.alt = &port->dp_alt; - if (idx >= uec->ucsi->cap.num_connectors) { + if (idx >= uec->num_ports) { dev_warn(uec->dev, "altmode port out of range: %d\n", idx); return; } + ret = typec_mux_set(port->typec_mux, &port->state); + if (ret) + dev_err(uec->dev, "failed to set mux %d\n", ret); + /* UCSI callback .connector_status() have set orientation */ - if (port->bridge) + if (port->bridge && port->svid == USB_TYPEC_DP_SID) drm_aux_hpd_bridge_notify(&port->bridge->dev, port->hpd_state ? connector_status_connected : @@ -307,7 +324,7 @@ static void gaokun_ucsi_handle_altmode(struct gaokun_ucsi_port *port) gaokun_ec_ucsi_pan_ack(uec->ec, port->idx); } -static void gaokun_ucsi_altmode_notify_ind(struct gaokun_ucsi *uec) +static void gaokun_ucsi_usb_notify_ind(struct gaokun_ucsi *uec) { int idx; @@ -320,7 +337,7 @@ static void gaokun_ucsi_altmode_notify_ind(struct gaokun_ucsi *uec) if (idx == GAOKUN_UCSI_NO_PORT_UPDATE) gaokun_ec_ucsi_pan_ack(uec->ec, idx); /* ack directly if no update */ else - gaokun_ucsi_handle_altmode(&uec->ports[idx]); + gaokun_ucsi_handle_usb_mode(&uec->ports[idx]); } /* @@ -352,7 +369,7 @@ static void gaokun_ucsi_handle_no_usb_event(struct gaokun_ucsi *uec, int idx) port = &uec->ports[idx]; if (!wait_for_completion_timeout(&port->usb_ack, 2 * HZ)) { dev_warn(uec->dev, "No USB EVENT, triggered by UCSI EVENT"); - gaokun_ucsi_altmode_notify_ind(uec); + gaokun_ucsi_usb_notify_ind(uec); } } @@ -366,7 +383,7 @@ static int gaokun_ucsi_notify(struct notifier_block *nb, switch (action) { case EC_EVENT_USB: gaokun_ucsi_complete_usb_ack(uec); - gaokun_ucsi_altmode_notify_ind(uec); + gaokun_ucsi_usb_notify_ind(uec); return NOTIFY_OK; case EC_EVENT_UCSI: @@ -429,8 +446,15 @@ static int gaokun_ucsi_ports_init(struct gaokun_ucsi *uec) fwnode_handle_put(fwnode); return PTR_ERR(ucsi_port->bridge); } - } + ucsi_port->typec_mux = fwnode_typec_mux_get(fwnode); + if (IS_ERR(ucsi_port->typec_mux)) { + fwnode_handle_put(fwnode); + return dev_err_probe(dev, PTR_ERR(ucsi_port->typec_mux), + "failed to acquire mode-switch for port: %d\n", + port); + } + } for (i = 0; i < num_ports; i++) { if (!uec->ports[i].bridge) continue; @@ -459,8 +483,13 @@ static void gaokun_ucsi_register_worker(struct work_struct *work) } ret = ucsi_register(ucsi); - if (ret) + if (ret) { dev_err_probe(ucsi->dev, ret, "ucsi register failed\n"); + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + return; + } + + uec->registered = true; } static int gaokun_ucsi_probe(struct auxiliary_device *adev, @@ -502,10 +531,17 @@ static int gaokun_ucsi_probe(struct auxiliary_device *adev, static void gaokun_ucsi_remove(struct auxiliary_device *adev) { struct gaokun_ucsi *uec = auxiliary_get_drvdata(adev); + int i; disable_delayed_work_sync(&uec->work); - gaokun_ec_unregister_notify(uec->ec, &uec->nb); - ucsi_unregister(uec->ucsi); + if (uec->registered) { + gaokun_ec_unregister_notify(uec->ec, &uec->nb); + ucsi_unregister(uec->ucsi); + } + + for (i = 0; i < uec->num_ports; ++i) + typec_mux_put(uec->ports[i].typec_mux); + ucsi_destroy(uec->ucsi); } diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c index d5865460e82d5..336fb4d92c6f5 100644 --- a/drivers/usb/usbip/vhci_sysfs.c +++ b/drivers/usb/usbip/vhci_sysfs.c @@ -59,6 +59,29 @@ static void port_show_vhci(char **out, int hub, int port, struct vhci_device *vd *out += sprintf(*out, "\n"); } +static ssize_t status_show_not_ready(int pdev_nr, char *out) +{ + char *s = out; + int i = 0; + + for (i = 0; i < VHCI_HC_PORTS; i++) { + out += sprintf(out, "hs %04u %03u ", + (pdev_nr * VHCI_PORTS) + i, + VDEV_ST_NOTASSIGNED); + out += sprintf(out, "000 00000000 0000000000000000 0-0"); + out += sprintf(out, "\n"); + } + + for (i = 0; i < VHCI_HC_PORTS; i++) { + out += sprintf(out, "ss %04u %03u ", + (pdev_nr * VHCI_PORTS) + VHCI_HC_PORTS + i, + VDEV_ST_NOTASSIGNED); + out += sprintf(out, "000 00000000 0000000000000000 0-0"); + out += sprintf(out, "\n"); + } + return out - s; +} + /* Sysfs entry to show port status */ static ssize_t status_show_vhci(int pdev_nr, char *out) { @@ -76,6 +99,12 @@ static ssize_t status_show_vhci(int pdev_nr, char *out) } hcd = platform_get_drvdata(pdev); + + if (!hcd) { + usbip_dbg_vhci_sysfs("show status error (hcd is NULL)\n"); + return status_show_not_ready(pdev_nr, out); + } + vhci_hcd = hcd_to_vhci_hcd(hcd); vhci = vhci_hcd->vhci; @@ -104,29 +133,6 @@ static ssize_t status_show_vhci(int pdev_nr, char *out) return out - s; } -static ssize_t status_show_not_ready(int pdev_nr, char *out) -{ - char *s = out; - int i = 0; - - for (i = 0; i < VHCI_HC_PORTS; i++) { - out += sprintf(out, "hs %04u %03u ", - (pdev_nr * VHCI_PORTS) + i, - VDEV_ST_NOTASSIGNED); - out += sprintf(out, "000 00000000 0000000000000000 0-0"); - out += sprintf(out, "\n"); - } - - for (i = 0; i < VHCI_HC_PORTS; i++) { - out += sprintf(out, "ss %04u %03u ", - (pdev_nr * VHCI_PORTS) + VHCI_HC_PORTS + i, - VDEV_ST_NOTASSIGNED); - out += sprintf(out, "000 00000000 0000000000000000 0-0"); - out += sprintf(out, "\n"); - } - return out - s; -} - static int status_name_to_id(const char *name) { char *c; diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 6658dc74d9150..0f54b520f14d2 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -724,7 +724,8 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, if (config->device_features & ~device_features) { IFCVF_ERR(pdev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n", config->device_features, device_features); - return -EINVAL; + ret = -EINVAL; + goto err; } device_features &= config->device_features; } @@ -734,15 +735,22 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, ret = dev_set_name(&vdpa_dev->dev, "%s", name); else ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index); + if (ret) { + IFCVF_ERR(pdev, "Failed to set device name"); + goto err; + } ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring); if (ret) { - put_device(&adapter->vdpa.dev); IFCVF_ERR(pdev, "Failed to register to vDPA bus"); - return ret; + goto err; } return 0; + +err: + put_device(&adapter->vdpa.dev); + return ret; } static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c index 8870a7169267e..5ec01d6724815 100644 --- a/drivers/vdpa/mlx5/core/mr.c +++ b/drivers/vdpa/mlx5/core/mr.c @@ -234,7 +234,8 @@ static int create_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr * cmds[i].out = cmd_mem->out; cmds[i].outlen = sizeof(cmd_mem->out); cmds[i].in = cmd_mem->in; - cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount); + cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount) - + offsetof(struct mlx5_create_mkey_mem, in); fill_create_direct_mr(mvdev, dmr, cmd_mem); @@ -481,7 +482,7 @@ static int add_direct_chain(struct mlx5_vdpa_dev *mvdev, return 0; err_alloc: - list_for_each_entry_safe(dmr, n, &mr->head, list) { + list_for_each_entry_safe(dmr, n, &tmp, list) { list_del_init(&dmr->list); unmap_direct_mr(mvdev, dmr); kfree(dmr); diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c index df8af6c1454cc..9aac6dbdaf3aa 100644 --- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c +++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c @@ -582,10 +582,10 @@ static const struct vdpa_mgmtdev_ops octep_vdpa_mgmt_dev_ops = { static bool get_device_ready_status(u8 __iomem *addr) { - u64 signature = readq(addr + OCTEP_VF_MBOX_DATA(0)); + u32 signature = readl(addr + OCTEP_VF_MBOX_DATA(0)); if (signature == OCTEP_DEV_READY_SIGNATURE) { - writeq(0, addr + OCTEP_VF_MBOX_DATA(0)); + writel(0, addr + OCTEP_VF_MBOX_DATA(0)); return true; } @@ -757,7 +757,7 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) rpvf = FIELD_GET(GENMASK_ULL(35, 32), val); if (done) { for (i = 0; i < pf->enabled_vfs; i++) - writeq(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf)); + writel(OCTEP_DEV_READY_SIGNATURE, addr + OCTEP_PF_MBOX_DATA(i * rpvf)); } return num_vfs; diff --git a/drivers/vdpa/solidrun/snet_main.c b/drivers/vdpa/solidrun/snet_main.c index 4588211d57ebc..db3c8e95de490 100644 --- a/drivers/vdpa/solidrun/snet_main.c +++ b/drivers/vdpa/solidrun/snet_main.c @@ -418,11 +418,15 @@ static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet) snet->vqs[i]->irq_name, snet->vqs[i]); if (ret) { SNET_ERR(pdev, "Failed to request IRQ\n"); - return ret; + goto err_free_irqs; } snet->vqs[i]->irq = irq; } return 0; + +err_free_irqs: + snet_free_irqs(snet); + return ret; } static void snet_set_status(struct vdpa_device *vdev, u8 status) diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index df9c7ddc5d782..445c45080d453 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -231,8 +231,11 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr, kthread_init_work(&vdpasim->work, vdpasim_work_fn); vdpasim->worker = kthread_run_worker(0, "vDPA sim worker: %s", dev_attr->name); - if (IS_ERR(vdpasim->worker)) + if (IS_ERR(vdpasim->worker)) { + ret = PTR_ERR(vdpasim->worker); + vdpasim->worker = NULL; goto err_iommu; + } mutex_init(&vdpasim->mutex); spin_lock_init(&vdpasim->iommu_lock); @@ -744,18 +747,24 @@ static void vdpasim_free(struct vdpa_device *vdpa) struct vdpasim *vdpasim = vdpa_to_sim(vdpa); int i; - kthread_cancel_work_sync(&vdpasim->work); - kthread_destroy_worker(vdpasim->worker); + if (vdpasim->worker) { + kthread_cancel_work_sync(&vdpasim->work); + kthread_destroy_worker(vdpasim->worker); + } - for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { - vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); - vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + if (vdpasim->vqs) { + for (i = 0; i < vdpasim->dev_attr.nvqs; i++) { + vringh_kiov_cleanup(&vdpasim->vqs[i].out_iov); + vringh_kiov_cleanup(&vdpasim->vqs[i].in_iov); + } } vdpasim->dev_attr.free(vdpasim); - for (i = 0; i < vdpasim->dev_attr.nas; i++) - vhost_iotlb_reset(&vdpasim->iommu[i]); + if (vdpasim->iommu) { + for (i = 0; i < vdpasim->dev_attr.nas; i++) + vhost_iotlb_reset(&vdpasim->iommu[i]); + } kfree(vdpasim->iommu); kfree(vdpasim->iommu_pt); kfree(vdpasim->vqs); diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index b137f36793439..770a40a20be9d 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -79,10 +79,11 @@ static void vdpasim_blk_buffer_unlock(struct vdpasim_blk *blk) static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, u64 num_sectors, u64 max_sectors) { - if (start_sector > VDPASIM_BLK_CAPACITY) { + if (start_sector >= VDPASIM_BLK_CAPACITY) { dev_dbg(&vdpasim->vdpa.dev, "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n", start_sector, VDPASIM_BLK_CAPACITY); + return false; } if (num_sectors > max_sectors) { diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c index 6caf09a1907b5..b596d4a532d01 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c @@ -225,10 +225,15 @@ static void vdpasim_net_work(struct vdpasim *vdpasim) break; } - ++tx_pkts; read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov, net->buffer, PAGE_SIZE); + if (read <= 0) { + ++tx_errors; + vdpasim_net_complete(txq, 0); + continue; + } + ++tx_pkts; tx_bytes += read; if (!receive_filter(vdpasim, read)) { diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 4352b5cf74f07..ec743bed361c1 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -124,7 +124,7 @@ static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain, if (!map->bounce_page) { head_map = &domain->bounce_maps[(iova & PAGE_MASK) >> BOUNCE_MAP_SHIFT]; if (!head_map->bounce_page) { - tmp_page = alloc_page(GFP_ATOMIC); + tmp_page = alloc_page(GFP_ATOMIC | __GFP_ZERO); if (!tmp_page) return -ENOMEM; if (cmpxchg(&head_map->bounce_page, NULL, tmp_page)) @@ -493,17 +493,15 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain, vduse_domain_free_iova(iovad, dma_addr, size); } -void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, - size_t size, dma_addr_t *dma_addr, - gfp_t flag) +dma_addr_t vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, + size_t size, void *orig) { struct iova_domain *iovad = &domain->consistent_iovad; unsigned long limit = domain->iova_limit; dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); - void *orig = alloc_pages_exact(size, flag); - if (!iova || !orig) - goto err; + if (!iova) + return DMA_MAPPING_ERROR; spin_lock(&domain->iotlb_lock); if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1, @@ -514,27 +512,20 @@ void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, } spin_unlock(&domain->iotlb_lock); - *dma_addr = iova; + return iova; - return orig; err: - *dma_addr = DMA_MAPPING_ERROR; - if (orig) - free_pages_exact(orig, size); - if (iova) - vduse_domain_free_iova(iovad, iova, size); + vduse_domain_free_iova(iovad, iova, size); - return NULL; + return DMA_MAPPING_ERROR; } void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, - void *vaddr, dma_addr_t dma_addr, - unsigned long attrs) + dma_addr_t dma_addr, unsigned long attrs) { struct iova_domain *iovad = &domain->consistent_iovad; struct vhost_iotlb_map *map; struct vdpa_map_file *map_file; - phys_addr_t pa; spin_lock(&domain->iotlb_lock); map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr, @@ -546,12 +537,10 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, map_file = (struct vdpa_map_file *)map->opaque; fput(map_file->file); kfree(map_file); - pa = map->addr; vhost_iotlb_map_free(domain->iotlb, map); spin_unlock(&domain->iotlb_lock); vduse_domain_free_iova(iovad, dma_addr, size); - free_pages_exact(phys_to_virt(pa), size); } static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf) diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h index 775cad5238f3a..ead5ab477079b 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.h +++ b/drivers/vdpa/vdpa_user/iova_domain.h @@ -67,13 +67,11 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs); -void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, - size_t size, dma_addr_t *dma_addr, - gfp_t flag); +dma_addr_t vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, + size_t size, void *orig); void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, - void *vaddr, dma_addr_t dma_addr, - unsigned long attrs); + dma_addr_t dma_addr, unsigned long attrs); void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain); diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index db7af4b3fd4ad..1cc826742ac9c 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,10 @@ struct vduse_umem { struct mm_struct *mm; }; +struct vduse_vq_group { + struct vduse_dev *dev; +}; + struct vduse_dev { struct vduse_vdpa *vdev; struct device *dev; @@ -115,6 +120,7 @@ struct vduse_dev { u32 vq_num; u32 vq_align; struct vduse_umem *umem; + struct vduse_vq_group *groups; struct mutex mem_lock; unsigned int bounce_size; struct mutex domain_lock; @@ -615,6 +621,16 @@ static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, return 0; } +static union virtio_map vduse_get_vq_map(struct vdpa_device *vdpa, u16 idx) +{ + struct vduse_dev *dev = vdpa_to_vduse(vdpa); + union virtio_map ret = { + .group = &dev->groups[0], + }; + + return ret; +} + static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, struct vdpa_vq_state *state) { @@ -834,6 +850,7 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = { .get_vq_affinity = vduse_vdpa_get_vq_affinity, .reset = vduse_vdpa_reset, .set_map = vduse_vdpa_set_map, + .get_vq_map = vduse_get_vq_map, .free = vduse_vdpa_free, }; @@ -841,7 +858,14 @@ static void vduse_dev_sync_single_for_device(union virtio_map token, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return; + + vdev = token.group->dev; + domain = vdev->domain; vduse_domain_sync_single_for_device(domain, dma_addr, size, dir); } @@ -850,7 +874,14 @@ static void vduse_dev_sync_single_for_cpu(union virtio_map token, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return; + + vdev = token.group->dev; + domain = vdev->domain; vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir); } @@ -860,7 +891,14 @@ static dma_addr_t vduse_dev_map_page(union virtio_map token, struct page *page, enum dma_data_direction dir, unsigned long attrs) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return DMA_MAPPING_ERROR; + + vdev = token.group->dev; + domain = vdev->domain; return vduse_domain_map_page(domain, page, offset, size, dir, attrs); } @@ -869,7 +907,14 @@ static void vduse_dev_unmap_page(union virtio_map token, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return; + + vdev = token.group->dev; + domain = vdev->domain; return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); } @@ -877,33 +922,58 @@ static void vduse_dev_unmap_page(union virtio_map token, dma_addr_t dma_addr, static void *vduse_dev_alloc_coherent(union virtio_map token, size_t size, dma_addr_t *dma_addr, gfp_t flag) { - struct vduse_iova_domain *domain = token.iova_domain; - unsigned long iova; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; void *addr; *dma_addr = DMA_MAPPING_ERROR; - addr = vduse_domain_alloc_coherent(domain, size, - (dma_addr_t *)&iova, flag); + if (!token.group) + return NULL; + + addr = alloc_pages_exact(size, flag | __GFP_ZERO); if (!addr) return NULL; - *dma_addr = (dma_addr_t)iova; + vdev = token.group->dev; + domain = vdev->domain; + *dma_addr = vduse_domain_alloc_coherent(domain, size, addr); + if (*dma_addr == DMA_MAPPING_ERROR) + goto err; return addr; + +err: + free_pages_exact(addr, size); + return NULL; } static void vduse_dev_free_coherent(union virtio_map token, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return; + + vdev = token.group->dev; + domain = vdev->domain; - vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); + vduse_domain_free_coherent(domain, size, dma_addr, attrs); + free_pages_exact(vaddr, size); } static bool vduse_dev_need_sync(union virtio_map token, dma_addr_t dma_addr) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return false; + + vdev = token.group->dev; + domain = vdev->domain; return dma_addr < domain->bounce_size; } @@ -917,7 +987,14 @@ static int vduse_dev_mapping_error(union virtio_map token, dma_addr_t dma_addr) static size_t vduse_dev_max_mapping_size(union virtio_map token) { - struct vduse_iova_domain *domain = token.iova_domain; + struct vduse_dev *vdev; + struct vduse_iova_domain *domain; + + if (!token.group) + return 0; + + vdev = token.group->dev; + domain = vdev->domain; return domain->bounce_size; } @@ -1716,6 +1793,7 @@ static int vduse_destroy_dev(char *name) if (dev->domain) vduse_domain_destroy(dev->domain); kfree(dev->name); + kfree(dev->groups); vduse_dev_destroy(dev); module_put(THIS_MODULE); @@ -1874,6 +1952,11 @@ static int vduse_create_dev(struct vduse_dev_config *config, dev->device_features = config->features; dev->device_id = config->device_id; dev->vendor_id = config->vendor_id; + dev->groups = kcalloc(1, sizeof(dev->groups[0]), GFP_KERNEL); + if (!dev->groups) + goto err_vq_groups; + dev->groups[0].dev = dev; + dev->name = kstrdup(config->name, GFP_KERNEL); if (!dev->name) goto err_str; @@ -1910,6 +1993,8 @@ static int vduse_create_dev(struct vduse_dev_config *config, err_idr: kfree(dev->name); err_str: + kfree(dev->groups); +err_vq_groups: vduse_dev_destroy(dev); err: return ret; @@ -2071,7 +2156,6 @@ static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, return -ENOMEM; } - dev->vdev->vdpa.vmap.iova_domain = dev->domain; ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); if (ret) { put_device(&dev->vdev->vdpa.dev); diff --git a/drivers/vfio/pci/vfio_pci_config.c b/drivers/vfio/pci/vfio_pci_config.c index 8f02f236b5b4b..1050bd7fd7ce6 100644 --- a/drivers/vfio/pci/vfio_pci_config.c +++ b/drivers/vfio/pci/vfio_pci_config.c @@ -1257,6 +1257,7 @@ static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos) ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags); if (ret) { kfree(vdev->msi_perm); + vdev->msi_perm = NULL; return ret; } diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index b2b90732b1d59..72bd44f2f1428 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -724,10 +724,12 @@ static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq, goto err; } - gso = buf + pad - sock_hlen; - - if (!sock_hlen) + if (!sock_hlen) { memset(buf, 0, pad); + gso = buf; + } else { + gso = buf + pad - sock_hlen; + } if ((gso->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && vhost16_to_cpu(vq, gso->csum_start) + @@ -1776,7 +1778,8 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return -EFAULT; /* Zero the trailing space provided by user-space, if any */ - if (clear_user(argp, size_mul(count - copied, sizeof(u64)))) + if (clear_user(argp + size_mul(copied, sizeof(u64)), + size_mul(count - copied, sizeof(u64)))) return -EFAULT; return 0; case VHOST_SET_FEATURES_ARRAY: diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 98e4f68f4e3cb..deb4c8bfd3d57 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -971,6 +971,9 @@ vhost_scsi_mapal(struct vhost_scsi *vs, struct vhost_scsi_cmd *cmd, if (prot_bytes) { sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes, VHOST_SCSI_PREALLOC_PROT_SGLS); + if (sgl_count < 0) + return sgl_count; + cmd->prot_table.sgl = cmd->prot_sgl; ret = sg_alloc_table_chained(&cmd->prot_table, sgl_count, cmd->prot_table.sgl, @@ -1416,6 +1419,11 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) * actual data payload length. */ if (prot_bytes) { + if (prot_bytes >= exp_data_len) { + vq_err(vq, "Protection data exceeds payload length\n"); + goto err; + } + exp_data_len -= prot_bytes; prot_iter = data_iter; iov_iter_truncate(&prot_iter, prot_bytes); @@ -2223,6 +2231,7 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) { struct vhost_virtqueue *vq; bool is_log, was_log; + u64 old_features; int i; if (features & ~VHOST_SCSI_FEATURES) @@ -2238,6 +2247,14 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) if (!vs->dev.nvqs) goto out; + old_features = vs->vqs[0].vq.acked_features; + if (vs->vs_tpg && + ((features ^ old_features) & + ~(1ULL << VHOST_F_LOG_ALL))) { + mutex_unlock(&vs->dev.mutex); + return -EBUSY; + } + is_log = features & (1 << VHOST_F_LOG_ALL); /* * All VQs should have same feature. @@ -2431,9 +2448,10 @@ vhost_scsi_ioctl(struct file *f, default: mutex_lock(&vs->dev.mutex); r = vhost_dev_ioctl(&vs->dev, ioctl, argp); - /* TODO: flush backend after dev ioctl. */ if (r == -ENOIOCTLCMD) r = vhost_vring_ioctl(&vs->dev, ioctl, argp); + else + vhost_scsi_flush(vs); mutex_unlock(&vs->dev.mutex); return r; } diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index a6c6c28bf2d9d..2dd2a5ccd09ee 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -53,9 +53,12 @@ struct vhost_vdpa { struct cdev cdev; atomic_t opened; u32 nvqs; + u16 vq_num_max; int virtio_id; int minor; struct eventfd_ctx *config_ctx; + /* Serialises vhost_vdpa_config_cb() against config_ctx being replaced. */ + spinlock_t config_lock; int in_batch; struct vdpa_iova_range range; u32 batch_asid; @@ -187,10 +190,12 @@ static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) static irqreturn_t vhost_vdpa_config_cb(void *private) { struct vhost_vdpa *v = private; - struct eventfd_ctx *config_ctx = v->config_ctx; + unsigned long flags; - if (config_ctx) - eventfd_signal(config_ctx); + spin_lock_irqsave(&v->config_lock, flags); + if (v->config_ctx) + eventfd_signal(v->config_ctx); + spin_unlock_irqrestore(&v->config_lock, flags); return IRQ_HANDLED; } @@ -229,7 +234,9 @@ static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid) static int _compat_vdpa_reset(struct vhost_vdpa *v) { struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; u32 flags = 0; + int ret; v->suspended = false; @@ -239,7 +246,14 @@ static int _compat_vdpa_reset(struct vhost_vdpa *v) VDPA_RESET_F_CLEAN_MAP : 0; } - return vdpa_reset(vdpa, flags); + v->vq_num_max = 0; + ret = vdpa_reset(vdpa, flags); + if (!ret) { + /* Some backends derive the max from mutable queue state. */ + v->vq_num_max = ops->get_vq_num_max(vdpa); + } + + return ret; } static int vhost_vdpa_reset(struct vhost_vdpa *v) @@ -511,15 +525,22 @@ static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) static void vhost_vdpa_config_put(struct vhost_vdpa *v) { - if (v->config_ctx) { - eventfd_ctx_put(v->config_ctx); - v->config_ctx = NULL; - } + struct eventfd_ctx *ctx; + unsigned long flags; + + spin_lock_irqsave(&v->config_lock, flags); + ctx = v->config_ctx; + v->config_ctx = NULL; + spin_unlock_irqrestore(&v->config_lock, flags); + + if (ctx) + eventfd_ctx_put(ctx); } static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) { struct vdpa_callback cb; + unsigned long flags; int fd; struct eventfd_ctx *ctx; @@ -529,18 +550,20 @@ static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp) return -EFAULT; ctx = fd == VHOST_FILE_UNBIND ? NULL : eventfd_ctx_fdget(fd); + if (IS_ERR(ctx)) + return PTR_ERR(ctx); + + spin_lock_irqsave(&v->config_lock, flags); swap(ctx, v->config_ctx); + spin_unlock_irqrestore(&v->config_lock, flags); - if (!IS_ERR_OR_NULL(ctx)) + /* + * The callback can no longer reach the old context, so this is the + * last reference to it. + */ + if (ctx) eventfd_ctx_put(ctx); - if (IS_ERR(v->config_ctx)) { - long ret = PTR_ERR(v->config_ctx); - - v->config_ctx = NULL; - return ret; - } - v->vdpa->config->set_config_cb(v->vdpa, &cb); return 0; @@ -641,9 +664,15 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, u32 idx; long r; - r = get_user(idx, (u32 __user *)argp); - if (r < 0) - return r; + if (cmd == VHOST_SET_VRING_NUM) { + if (copy_from_user(&s, argp, sizeof(s))) + return -EFAULT; + idx = s.index; + } else { + r = get_user(idx, (u32 __user *)argp); + if (r < 0) + return r; + } if (idx >= v->nvqs) return -ENOBUFS; @@ -652,6 +681,23 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, vq = &v->vqs[idx]; switch (cmd) { + case VHOST_SET_VRING_NUM: + mutex_lock(&vq->mutex); + if (vq->private_data) { + r = -EBUSY; + } else if (!s.num || s.num > 0xffff || + s.num > v->vq_num_max || + (s.num & (s.num - 1))) { + r = -EINVAL; + } else { + vq->num = s.num; + r = 0; + } + mutex_unlock(&vq->mutex); + if (r) + return r; + ops->set_vq_num(vdpa, idx, s.num); + return 0; case VHOST_VDPA_SET_VRING_ENABLE: if (copy_from_user(&s, argp, sizeof(s))) return -EFAULT; @@ -763,9 +809,6 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, ops->set_vq_cb(vdpa, idx, &cb); break; - case VHOST_SET_VRING_NUM: - ops->set_vq_num(vdpa, idx, vq->num); - break; } return r; @@ -1100,6 +1143,7 @@ static int vhost_vdpa_pa_map(struct vhost_vdpa *v, unsigned int gup_flags = FOLL_LONGTERM; unsigned long npages, cur_base, map_pfn, last_pfn = 0; unsigned long lock_limit, sz2pin, nchunks, i; + unsigned long page_offset; u64 start = iova; long pinned; int ret = 0; @@ -1112,7 +1156,13 @@ static int vhost_vdpa_pa_map(struct vhost_vdpa *v, if (perm & VHOST_ACCESS_WO) gup_flags |= FOLL_WRITE; - npages = PFN_UP(size + (iova & ~PAGE_MASK)); + page_offset = iova & ~PAGE_MASK; + if (size > ULONG_MAX - page_offset) { + ret = -EINVAL; + goto free; + } + + npages = PFN_UP(size + page_offset); if (!npages) { ret = -EINVAL; goto free; @@ -1597,6 +1647,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa) } atomic_set(&v->opened, 0); + spin_lock_init(&v->config_lock); v->minor = minor; v->vdpa = vdpa; v->nvqs = vdpa->nvqs; diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 6a7b22650aa72..583fa2ec9bc30 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -2131,6 +2131,14 @@ static long vhost_vring_set_num_addr(struct vhost_dev *d, BUG(); } + /* + * The metadata cache holds the IOTLB mapping that backed the previous + * desc/avail/used addresses and vring size, both of which are being + * replaced here. iotlb_access_ok() takes a cache hit as proof that the + * region was validated, so the stale entries have to go. + */ + __vhost_vq_meta_reset(vq); + mutex_unlock(&vq->mutex); return r; diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index d51777df12d16..0adb1e2fa5338 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -37,6 +37,7 @@ source "drivers/char/agp/Kconfig" source "drivers/gpu/vga/Kconfig" +source "drivers/gpu/Kconfig" source "drivers/gpu/host1x/Kconfig" source "drivers/gpu/ipu-v3/Kconfig" source "drivers/gpu/nova-core/Kconfig" diff --git a/drivers/video/fbdev/clps711x-fb.c b/drivers/video/fbdev/clps711x-fb.c index 5e61a349a4ab6..c350340fb7b95 100644 --- a/drivers/video/fbdev/clps711x-fb.c +++ b/drivers/video/fbdev/clps711x-fb.c @@ -332,8 +332,6 @@ static int clps711x_fb_probe(struct platform_device *pdev) return 0; - unregister_framebuffer(info); - out_fb_dealloc_cmap: regmap_update_bits(cfb->syscon, SYSCON_OFFSET, SYSCON1_LCDEN, 0); fb_dealloc_cmap(&info->cmap); diff --git a/drivers/video/fbdev/core/bitblit.c b/drivers/video/fbdev/core/bitblit.c index bed8ba18222b9..8f6a2d1736d39 100644 --- a/drivers/video/fbdev/core/bitblit.c +++ b/drivers/video/fbdev/core/bitblit.c @@ -274,9 +274,14 @@ static void bit_cursor(struct vc_data *vc, struct fb_info *info, bool enable, if (!vc->vc_font.data) return; - c = scr_readw((u16 *) vc->vc_pos); + c = scr_readw((u16 *) vc->vc_pos); attribute = get_attribute(info, c); - src = vc->vc_font.data + ((c & charmask) * (w * vc->vc_font.height)); + c &= charmask; + + /* Clamp to font size, same as bit_putcs_aligned() */ + if (c >= vc->vc_font.charcount) + c = 0; + src = vc->vc_font.data + (c * (w * vc->vc_font.height)); if (par->cursor_state.image.data != src || par->cursor_reset) { diff --git a/drivers/video/fbdev/core/fb_io_fops.c b/drivers/video/fbdev/core/fb_io_fops.c index 6ab60fcd00501..0798e88799ebb 100644 --- a/drivers/video/fbdev/core/fb_io_fops.c +++ b/drivers/video/fbdev/core/fb_io_fops.c @@ -61,6 +61,14 @@ ssize_t fb_io_read(struct fb_info *info, char __user *buf, size_t count, loff_t buf += c; cnt += c; count -= c; + + /* + * If there was a partial copy, the user buffer is faulty. + * Break out to avoid over-advancing the src pointer and + * reading out of bounds in the next iteration. + */ + if (trailing) + break; } kfree(buffer); diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 89e6f99f027b6..4548cf98223c0 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -2669,8 +2669,9 @@ void fbcon_suspended(struct fb_info *info) return; vc = vc_cons[par->currcon].d; - /* Clear cursor, restore saved data */ - fbcon_cursor(vc, false); + /* Clear cursor, restore saved data when in text mode */ + if ((vc->vc_mode == KD_TEXT) && con_is_visible(vc)) + fbcon_cursor(vc, false); } void fbcon_resumed(struct fb_info *info) @@ -2682,7 +2683,9 @@ void fbcon_resumed(struct fb_info *info) return; vc = vc_cons[par->currcon].d; - update_screen(vc); + /* Update screen when in text mode only */ + if ((vc->vc_mode == KD_TEXT) && con_is_visible(vc)) + update_screen(vc); } static void fbcon_modechanged(struct fb_info *info) diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c index c8b1dfa456a33..c44d6809859e5 100644 --- a/drivers/video/fbdev/kyro/fbdev.c +++ b/drivers/video/fbdev/kyro/fbdev.c @@ -369,6 +369,9 @@ static int kyro_dev_overlay_create(u32 ulWidth, static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight) { + u32 right; + u32 bottom; + if (deviceInfo.ulOverlayOffset == 0) /* probably haven't called CreateOverlay yet */ return -EINVAL; @@ -378,11 +381,30 @@ static int kyro_dev_overlay_viewport_set(u32 x, u32 y, u32 ulWidth, u32 ulHeight (x < 2 && ulWidth + 2 == 0)) return -EINVAL; + /* + * SetOverlayViewPort() adjusts X coordinates by +2 (left) and +1 + * (right) before packing them into 16-bit register fields. + */ + if (x > U16_MAX - 2 || y > U16_MAX) + return -EINVAL; + + right = x + ulWidth; + bottom = y + ulHeight; + + if (right < x || bottom < y) + return -EINVAL; + + right--; + bottom--; + + if (right > U16_MAX - 1 || bottom > U16_MAX) + return -EINVAL; + /* Stop Ramdac Output */ DisableRamdacOutput(deviceInfo.pSTGReg); SetOverlayViewPort(deviceInfo.pSTGReg, - x, y, x + ulWidth - 1, y + ulHeight - 1); + x, y, right, bottom); EnableOverlayPlane(deviceInfo.pSTGReg); /* Start Ramdac Output */ diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c index 1d75f27c6b80f..ce1d4d5f4034d 100644 --- a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c +++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c @@ -1151,14 +1151,14 @@ static int dsicm_probe(struct platform_device *pdev) dssdev->caps = OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE | OMAP_DSS_DISPLAY_CAP_TEAR_ELIM; + mutex_init(&ddata->lock); + r = omapdss_register_display(dssdev); if (r) { dev_err(dev, "Failed to register panel\n"); goto err_reg; } - mutex_init(&ddata->lock); - atomic_set(&ddata->do_update, 0); ddata->reset_gpio = devm_gpiod_get(&pdev->dev, "reset", GPIOD_OUT_LOW); diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c index f34429829b7d7..82408503c2812 100644 --- a/drivers/video/fbdev/pm2fb.c +++ b/drivers/video/fbdev/pm2fb.c @@ -1711,6 +1711,7 @@ static int pm2fb_probe(struct pci_dev *pdev, const struct pci_device_id *id) err_exit_both: kfree(info->pixmap.addr); err_exit_pixmap: + arch_phys_wc_del(default_par->wc_cookie); iounmap(info->screen_base); release_mem_region(pm2fb_fix.smem_start, pm2fb_fix.smem_len); err_exit_mmio: diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c index 0b8d23c12b773..d3459a56beefa 100644 --- a/drivers/video/fbdev/pvr2fb.c +++ b/drivers/video/fbdev/pvr2fb.c @@ -639,7 +639,7 @@ static irqreturn_t __maybe_unused pvr2fb_interrupt(int irq, void *dev_id) } #ifdef CONFIG_PVR2_DMA -static ssize_t pvr2fb_write(struct fb_info *info, const char *buf, +static ssize_t pvr2fb_write(struct fb_info *info, const char __user *buf, size_t count, loff_t *ppos) { unsigned long dst, start, end, len; @@ -1078,7 +1078,7 @@ static struct pvr2_board { #ifdef CONFIG_PCI { pvr2fb_pci_init, pvr2fb_pci_exit, "PCI PVR2" }, #endif - { 0, }, + { }, }; static int __init pvr2fb_init(void) diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index 83dd31fa1fab5..0df3ac694f4e0 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -72,6 +73,13 @@ struct ssd1307fb_par { struct i2c_client *client; u32 height; struct fb_info *info; + /* Pending damage, with exclusive x2/y2, protected by damage_lock. */ + spinlock_t damage_lock; + bool damage_pending; + u32 damage_x1; + u32 damage_x2; + u32 damage_y1; + u32 damage_y2; u8 lookup_table[4]; u32 page_offset; u32 col_offset; @@ -302,19 +310,49 @@ static int ssd1307fb_blank(int blank_mode, struct fb_info *info) return ssd1307fb_write_cmd(par->client, SSD1307FB_DISPLAY_ON); } +static void ssd1307fb_schedule_damage(struct fb_info *info, u32 x, u32 y, + u32 width, u32 height) +{ + struct ssd1307fb_par *par = info->par; + unsigned long flags; + u32 x2, y2; + + if (!width || !height || x >= par->width || y >= par->height) + return; + + x2 = x + min(width, par->width - x); + y2 = y + min(height, par->height - y); + + spin_lock_irqsave(&par->damage_lock, flags); + if (par->damage_pending) { + par->damage_x1 = min(par->damage_x1, x); + par->damage_y1 = min(par->damage_y1, y); + par->damage_x2 = max(par->damage_x2, x2); + par->damage_y2 = max(par->damage_y2, y2); + } else { + par->damage_x1 = x; + par->damage_y1 = y; + par->damage_x2 = x2; + par->damage_y2 = y2; + par->damage_pending = true; + } + spin_unlock_irqrestore(&par->damage_lock, flags); + + /* Advance an already-pending mmap update as well. */ + mod_delayed_work(system_wq, &info->deferred_work, 0); +} + static void ssd1307fb_defio_damage_range(struct fb_info *info, off_t off, size_t len) { struct ssd1307fb_par *par = info->par; - ssd1307fb_update_display(par); + ssd1307fb_schedule_damage(info, 0, 0, par->width, par->height); } static void ssd1307fb_defio_damage_area(struct fb_info *info, u32 x, u32 y, u32 width, u32 height) { - struct ssd1307fb_par *par = info->par; - - ssd1307fb_update_rect(par, x, y, width, height); + ssd1307fb_schedule_damage(info, x, y, width, height); } FB_GEN_DEFAULT_DEFERRED_SYSMEM_OPS(ssd1307fb, @@ -329,7 +367,30 @@ static const struct fb_ops ssd1307fb_ops = { static void ssd1307fb_deferred_io(struct fb_info *info, struct list_head *pagereflist) { - ssd1307fb_update_display(info->par); + struct ssd1307fb_par *par = info->par; + unsigned long flags; + u32 x, y, width, height; + + spin_lock_irqsave(&par->damage_lock, flags); + if (!list_empty(pagereflist)) { + x = 0; + y = 0; + width = par->width; + height = par->height; + par->damage_pending = false; + } else if (par->damage_pending) { + x = par->damage_x1; + y = par->damage_y1; + width = par->damage_x2 - par->damage_x1; + height = par->damage_y2 - par->damage_y1; + par->damage_pending = false; + } else { + spin_unlock_irqrestore(&par->damage_lock, flags); + return; + } + spin_unlock_irqrestore(&par->damage_lock, flags); + + ssd1307fb_update_rect(par, x, y, width, height); } static int ssd1307fb_init(struct ssd1307fb_par *par) @@ -601,6 +662,7 @@ static int ssd1307fb_probe(struct i2c_client *client) par = info->par; par->info = info; par->client = client; + spin_lock_init(&par->damage_lock); par->device_info = device_get_match_data(dev); diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c index c5095358209e1..eb8f33ae94e77 100644 --- a/drivers/video/fbdev/tdfxfb.c +++ b/drivers/video/fbdev/tdfxfb.c @@ -1383,7 +1383,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) return err; - err = pci_enable_device(pdev); + err = pcim_enable_device(pdev); if (err) { printk(KERN_ERR "tdfxfb: Can't enable pdev: %d\n", err); return err; diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c index b806da0538bc6..8330d80fb394b 100644 --- a/drivers/video/fbdev/uvesafb.c +++ b/drivers/video/fbdev/uvesafb.c @@ -1910,6 +1910,8 @@ static int uvesafb_init(void) err = 0; } } + if (err) + cn_del_callback(&uvesafb_cn_id); return err; } diff --git a/drivers/video/fbdev/vfb.c b/drivers/video/fbdev/vfb.c index 5b7965f36c5ed..de137b2bdaedc 100644 --- a/drivers/video/fbdev/vfb.c +++ b/drivers/video/fbdev/vfb.c @@ -78,6 +78,13 @@ static int vfb_pan_display(struct fb_var_screeninfo *var, static int vfb_mmap(struct fb_info *info, struct vm_area_struct *vma); +static void vfb_destroy(struct fb_info *info) +{ + vfree(info->screen_buffer); + fb_dealloc_cmap(&info->cmap); + framebuffer_release(info); +} + static const struct fb_ops vfb_ops = { .owner = THIS_MODULE, __FB_DEFAULT_SYSMEM_OPS_RDWR, @@ -87,6 +94,7 @@ static const struct fb_ops vfb_ops = { .fb_pan_display = vfb_pan_display, __FB_DEFAULT_SYSMEM_OPS_DRAW, .fb_mmap = vfb_mmap, + .fb_destroy = vfb_destroy, }; /* @@ -485,9 +493,6 @@ static void vfb_remove(struct platform_device *dev) if (info) { unregister_framebuffer(info); - vfree(videomemory); - fb_dealloc_cmap(&info->cmap); - framebuffer_release(info); } } diff --git a/drivers/video/sticore.c b/drivers/video/sticore.c index 88a1758616e02..91382628515bb 100644 --- a/drivers/video/sticore.c +++ b/drivers/video/sticore.c @@ -325,7 +325,7 @@ static void sti_rom_copy(unsigned long base, unsigned long count, void *dest) -static char default_sti_path[21] __read_mostly; +static char default_sti_path[32] __read_mostly; #ifndef MODULE static int __init sti_setup(char *str) @@ -1148,6 +1148,26 @@ static void sti_init_roms(void) pr_info("STI GSC/PCI core graphics driver " STI_DRIVERVERSION "\n"); + /* + * Find default console by hardware path which is either stored in + * console entry in stable storage or alternatively from console path + * in PAGE0 used by BCH and PDC. + */ + if (!default_sti_path[0]) { + struct pdc_module_path conspath; + struct device *dev = NULL; + + if (pdc_stable_read(0x60, &conspath, sizeof(conspath)) == PDC_OK) + dev = hwpath_to_device(&conspath.path); + if (!dev) + dev = hwpath_to_device(&PAGE0->mem_cons.dp.path); + if (dev && dev_is_pci(dev)) + print_pci_hwpath(to_pci_dev(dev), default_sti_path); + else if (dev && !dev_is_pci(dev)) + print_pa_hwpath(to_parisc_device(dev), default_sti_path); + pr_debug("default graphic card: %s\n", default_sti_path); + } + /* Register drivers for native & PCI cards */ register_parisc_driver(&pa_sti_driver); WARN_ON(pci_register_driver(&pci_sti_driver)); diff --git a/drivers/virt/acrn/irqfd.c b/drivers/virt/acrn/irqfd.c index 64d32c8fbf79b..b18d1a9021fb4 100644 --- a/drivers/virt/acrn/irqfd.c +++ b/drivers/virt/acrn/irqfd.c @@ -16,6 +16,9 @@ #include "acrn_drv.h" +/* Cleanup work has been queued; set via test_and_set_bit(). */ +#define HSM_IRQFD_FLAG_SHUTDOWN 0 + /** * struct hsm_irqfd - Properties of HSM irqfd * @vm: Associated VM pointer @@ -25,6 +28,7 @@ * @list: Entry within &acrn_vm.irqfds of irqfds of a VM * @pt: Structure for select/poll on the associated eventfd * @msi: MSI data + * @flags: Internal lifecycle flags (HSM_IRQFD_FLAG_*) */ struct hsm_irqfd { struct acrn_vm *vm; @@ -34,6 +38,7 @@ struct hsm_irqfd { struct list_head list; poll_table pt; struct acrn_msi_entry msi; + unsigned long flags; }; static void acrn_irqfd_inject(struct hsm_irqfd *irqfd) @@ -44,30 +49,29 @@ static void acrn_irqfd_inject(struct hsm_irqfd *irqfd) irqfd->msi.msi_data); } -static void hsm_irqfd_shutdown(struct hsm_irqfd *irqfd) +/* Queue the cleanup work at most once. Safe from atomic context. */ +static void hsm_irqfd_queue_shutdown(struct hsm_irqfd *irqfd) { - u64 cnt; - - lockdep_assert_held(&irqfd->vm->irqfds_lock); - - /* remove from wait queue */ - list_del_init(&irqfd->list); - eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); - eventfd_ctx_put(irqfd->eventfd); - kfree(irqfd); + if (!test_and_set_bit(HSM_IRQFD_FLAG_SHUTDOWN, &irqfd->flags)) + queue_work(irqfd->vm->irqfd_wq, &irqfd->shutdown); } +/* Sole owner of @irqfd: unhook waitqueue, drop eventfd ref, free. */ static void hsm_irqfd_shutdown_work(struct work_struct *work) { - struct hsm_irqfd *irqfd; - struct acrn_vm *vm; + struct hsm_irqfd *irqfd = container_of(work, struct hsm_irqfd, + shutdown); + struct acrn_vm *vm = irqfd->vm; + u64 cnt; - irqfd = container_of(work, struct hsm_irqfd, shutdown); - vm = irqfd->vm; mutex_lock(&vm->irqfds_lock); if (!list_empty(&irqfd->list)) - hsm_irqfd_shutdown(irqfd); + list_del_init(&irqfd->list); mutex_unlock(&vm->irqfds_lock); + + eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); + eventfd_ctx_put(irqfd->eventfd); + kfree(irqfd); } /* Called with wqh->lock held and interrupts disabled */ @@ -76,17 +80,16 @@ static int hsm_irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, { unsigned long poll_bits = (unsigned long)key; struct hsm_irqfd *irqfd; - struct acrn_vm *vm; irqfd = container_of(wait, struct hsm_irqfd, wait); - vm = irqfd->vm; + if (poll_bits & POLLIN) /* An event has been signaled, inject an interrupt */ acrn_irqfd_inject(irqfd); if (poll_bits & POLLHUP) - /* Do shutdown work in thread to hold wqh->lock */ - queue_work(vm->irqfd_wq, &irqfd->shutdown); + /* Defer teardown to the cleanup work; can't sleep here. */ + hsm_irqfd_queue_shutdown(irqfd); return 0; } @@ -142,6 +145,12 @@ static int acrn_irqfd_assign(struct acrn_vm *vm, struct acrn_irqfd *args) init_waitqueue_func_entry(&irqfd->wait, hsm_irqfd_wakeup); init_poll_funcptr(&irqfd->pt, hsm_irqfd_poll_func); + /* + * Hold irqfds_lock across waitqueue install and list_add so the + * irqfd is not visible to deassign/deinit before its waitqueue + * entry is in place, and any racing EPOLLHUP cleanup work blocks + * on irqfds_lock until publication completes. + */ mutex_lock(&vm->irqfds_lock); list_for_each_entry(tmp, &vm->irqfds, list) { if (irqfd->eventfd != tmp->eventfd) @@ -150,14 +159,12 @@ static int acrn_irqfd_assign(struct acrn_vm *vm, struct acrn_irqfd *args) mutex_unlock(&vm->irqfds_lock); goto fail; } - list_add_tail(&irqfd->list, &vm->irqfds); - mutex_unlock(&vm->irqfds_lock); - /* Check the pending event in this stage */ events = vfs_poll(fd_file(f), &irqfd->pt); - + list_add_tail(&irqfd->list, &vm->irqfds); if (events & EPOLLIN) acrn_irqfd_inject(irqfd); + mutex_unlock(&vm->irqfds_lock); return 0; fail: @@ -180,13 +187,17 @@ static int acrn_irqfd_deassign(struct acrn_vm *vm, mutex_lock(&vm->irqfds_lock); list_for_each_entry_safe(irqfd, tmp, &vm->irqfds, list) { if (irqfd->eventfd == eventfd) { - hsm_irqfd_shutdown(irqfd); + list_del_init(&irqfd->list); + hsm_irqfd_queue_shutdown(irqfd); break; } } mutex_unlock(&vm->irqfds_lock); eventfd_ctx_put(eventfd); + /* Wait for cleanup work to finish so the eventfd is fully detached. */ + flush_workqueue(vm->irqfd_wq); + return 0; } @@ -219,9 +230,15 @@ void acrn_irqfd_deinit(struct acrn_vm *vm) struct hsm_irqfd *irqfd, *next; dev_dbg(acrn_dev.this_device, "VM %u irqfd deinit.\n", vm->vmid); - destroy_workqueue(vm->irqfd_wq); + mutex_lock(&vm->irqfds_lock); - list_for_each_entry_safe(irqfd, next, &vm->irqfds, list) - hsm_irqfd_shutdown(irqfd); + list_for_each_entry_safe(irqfd, next, &vm->irqfds, list) { + list_del_init(&irqfd->list); + hsm_irqfd_queue_shutdown(irqfd); + } mutex_unlock(&vm->irqfds_lock); + + /* Drain all cleanup work before tearing the workqueue down. */ + flush_workqueue(vm->irqfd_wq); + destroy_workqueue(vm->irqfd_wq); } diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 66d00b6ceb789..a38df08da6fa0 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -16,54 +16,38 @@ /** * struct arm_cca_token_info - a descriptor for the token buffer. - * @challenge: Pointer to the challenge data - * @challenge_size: Size of the challenge data * @granule: PA of the granule to which the token will be written * @offset: Offset within granule to start of buffer in bytes - * @result: result of rsi_attestation_token_continue operation */ struct arm_cca_token_info { - void *challenge; - unsigned long challenge_size; phys_addr_t granule; unsigned long offset; - unsigned long result; }; -static void arm_cca_attestation_init(void *param) -{ - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; - - info->result = rsi_attestation_token_init(info->challenge, - info->challenge_size); -} - /** * arm_cca_attestation_continue - Retrieve the attestation token data. * - * @param: pointer to the arm_cca_token_info + * @info: pointer to the arm_cca_token_info * * Attestation token generation is a long running operation and therefore * the token data may not be retrieved in a single call. Moreover, the * token retrieval operation must be requested on the same CPU on which the * attestation token generation was initialised. - * This helper function is therefore scheduled on the same CPU multiple + * This helper function must therefore be executed on the same CPU multiple * times until the entire token data is retrieved. */ -static void arm_cca_attestation_continue(void *param) +static unsigned long +arm_cca_attestation_continue(struct arm_cca_token_info *info) { + unsigned long ret; unsigned long len; unsigned long size; - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; size = RSI_GRANULE_SIZE - info->offset; - info->result = rsi_attestation_token_continue(info->granule, - info->offset, size, &len); + ret = rsi_attestation_token_continue(info->granule, info->offset, size, + &len); info->offset += len; + return ret; } /** @@ -74,8 +58,8 @@ static void arm_cca_attestation_continue(void *param) * * Initialise the attestation token generation using the challenge data * passed in the TSM descriptor. Allocate memory for the attestation token - * and schedule calls to retrieve the attestation token on the same CPU - * on which the attestation token generation was initialised. + * and retrieve the attestation token on the same CPU on which the + * attestation token generation was initialised. * * The challenge data must be at least 32 bytes and no more than 64 bytes. If * less than 64 bytes are provided it will be zero padded to 64 bytes. @@ -85,12 +69,11 @@ static void arm_cca_attestation_continue(void *param) * * %-EINVAL - A parameter was not valid. * * %-ENOMEM - Out of memory. * * %-EFAULT - Failed to get IPA for memory page(s). - * * A negative status code as returned by smp_call_function_single(). */ static int arm_cca_report_new(struct tsm_report *report, void *data) { - int ret; - int cpu; + int ret = 0; + unsigned long rsi_result; long max_size; unsigned long token_size = 0; struct arm_cca_token_info info; @@ -103,37 +86,33 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) /* * The attestation token 'init' and 'continue' calls must be - * performed on the same CPU. smp_call_function_single() is used - * instead of simply calling get_cpu() because of the need to - * allocate outblob based on the returned value from the 'init' - * call and that cannot be done in an atomic context. + * performed on the same CPU, so disable CPU migration around + * those operations. */ - cpu = smp_processor_id(); + migrate_disable(); - info.challenge = desc->inblob; - info.challenge_size = desc->inblob_len; - - ret = smp_call_function_single(cpu, arm_cca_attestation_init, - &info, true); - if (ret) - return ret; - max_size = info.result; - - if (max_size <= 0) - return -EINVAL; + max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len); + if (max_size <= 0) { + ret = -EINVAL; + goto exit_migrate_enable; + } /* Allocate outblob */ token = kvzalloc(max_size, GFP_KERNEL); - if (!token) - return -ENOMEM; + if (!token) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* * Since the outblob may not be physically contiguous, use a page * to bounce the buffer from RMM. */ buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; + if (!buf) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* Get the PA of the memory page(s) that were allocated */ info.granule = (unsigned long)virt_to_phys(buf); @@ -144,21 +123,15 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) info.offset = 0; do { /* - * Schedule a call to retrieve a sub-granule chunk - * of data per loop iteration. + * Retrieve a sub-granule chunk of data per loop + * iteration. */ - ret = smp_call_function_single(cpu, - arm_cca_attestation_continue, - (void *)&info, true); - if (ret != 0) { - token_size = 0; - goto exit_free_granule_page; - } - } while (info.result == RSI_INCOMPLETE && + rsi_result = arm_cca_attestation_continue(&info); + } while (rsi_result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); /* Break out in case of failure */ - if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { + if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; @@ -173,12 +146,14 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) break; memcpy(&token[token_size], buf, info.offset); token_size += info.offset; - } while (info.result == RSI_INCOMPLETE); + } while (rsi_result == RSI_INCOMPLETE); report->outblob = no_free_ptr(token); exit_free_granule_page: report->outblob_len = token_size; free_pages_exact(buf, RSI_GRANULE_SIZE); +exit_migrate_enable: + migrate_enable(); return ret; } diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c index 4230b817a80bd..d66291def0f40 100644 --- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c +++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c @@ -82,8 +82,8 @@ static int mmio_guard_ioremap_hook(phys_addr_t phys, size_t size, if (protval != PROT_DEVICE_nGnRE && protval != PROT_DEVICE_nGnRnE) return 0; + end = PAGE_ALIGN(phys + size); phys = PAGE_ALIGN_DOWN(phys); - end = phys + PAGE_ALIGN(size); while (phys < end) { const int func_id = ARM_SMCCC_VENDOR_HYP_KVM_MMIO_GUARD_FUNC_ID; diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 5bdc6b82b30b4..4ced08886d0b8 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -401,6 +401,32 @@ static const struct cpumask *virtio_irq_get_affinity(struct device *_d, return dev->config->get_vq_affinity(dev, irq_vec); } +/** + * virtio_device_shutdown - break and reset a device on shutdown + * @dev: the device + * + * Drivers with their own .shutdown method should quiesce their activity and + * then call this to stop the device the way the generic shutdown path does. + */ +void virtio_device_shutdown(struct virtio_device *dev) +{ + /* + * Some devices get wedged if you kick them after they are + * reset. Mark all vqs as broken to make sure we don't. + */ + virtio_break_device(dev); + /* + * Guarantee that any callback will see vq->broken as true. + */ + virtio_synchronize_cbs(dev); + /* + * As IOMMUs are reset on shutdown, this will block device access to memory. + * Some devices get wedged if this happens, so reset to make sure it does not. + */ + dev->config->reset(dev); +} +EXPORT_SYMBOL_GPL(virtio_device_shutdown); + static void virtio_dev_shutdown(struct device *_d) { struct virtio_device *dev = dev_to_virtio(_d); @@ -419,20 +445,7 @@ static void virtio_dev_shutdown(struct device *_d) return; } - /* - * Some devices get wedged if you kick them after they are - * reset. Mark all vqs as broken to make sure we don't. - */ - virtio_break_device(dev); - /* - * Guarantee that any callback will see vq->broken as true. - */ - virtio_synchronize_cbs(dev); - /* - * As IOMMUs are reset on shutdown, this will block device access to memory. - * Some devices get wedged if this happens, so reset to make sure it does not. - */ - dev->config->reset(dev); + virtio_device_shutdown(dev); } static const struct bus_type virtio_bus = { @@ -582,8 +595,8 @@ void unregister_virtio_device(struct virtio_device *dev) { int index = dev->index; /* save for after device release */ - device_unregister(&dev->dev); virtio_debug_device_exit(dev); + device_unregister(&dev->dev); ida_free(&virtio_index_ida, index); } EXPORT_SYMBOL_GPL(unregister_virtio_device); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 1b93d8c643612..a3995e77a652c 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -1116,31 +1117,50 @@ static void remove_common(struct virtio_balloon *vb) vb->vdev->config->del_vqs(vb->vdev); } -static void virtballoon_remove(struct virtio_device *vdev) +/* + * Stop all asynchronous balloon work. The device must still be alive so that + * in-flight requests can drain via the host before it is reset or freed. + */ +static void virtballoon_quiesce(struct virtio_balloon *vb) { - struct virtio_balloon *vb = vdev->priv; + struct virtio_device *vdev = vb->vdev; - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_REPORTING)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_REPORTING)) page_reporting_unregister(&vb->pr_dev_info); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_DEFLATE_ON_OOM)) unregister_oom_notifier(&vb->oom_nb); - if (virtio_has_feature(vb->vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) virtio_balloon_unregister_shrinker(vb); + spin_lock_irq(&vb->stop_update_lock); vb->stop_update = true; spin_unlock_irq(&vb->stop_update_lock); cancel_work_sync(&vb->update_balloon_size_work); cancel_work_sync(&vb->update_balloon_stats_work); - if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) { + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) cancel_work_sync(&vb->report_free_page_work); +} + +static void virtballoon_remove(struct virtio_device *vdev) +{ + struct virtio_balloon *vb = vdev->priv; + + virtballoon_quiesce(vb); + + if (virtio_has_feature(vdev, VIRTIO_BALLOON_F_FREE_PAGE_HINT)) destroy_workqueue(vb->balloon_wq); - } remove_common(vb); kfree(vb); } +static void virtballoon_shutdown(struct virtio_device *vdev) +{ + virtballoon_quiesce(vdev->priv); + virtio_device_shutdown(vdev); +} + #ifdef CONFIG_PM_SLEEP static int virtballoon_freeze(struct virtio_device *vdev) { @@ -1185,6 +1205,11 @@ static int virtballoon_validate(struct virtio_device *vdev) else if (!virtio_has_feature(vdev, VIRTIO_BALLOON_F_PAGE_POISON)) __virtio_clear_bit(vdev, VIRTIO_BALLOON_F_REPORTING); + /* + * Disable indirect descriptors to avoid memory allocation in + * virtqueue_add during page reporting. + */ + __virtio_clear_bit(vdev, VIRTIO_RING_F_INDIRECT_DESC); __virtio_clear_bit(vdev, VIRTIO_F_ACCESS_PLATFORM); return 0; } @@ -1206,6 +1231,7 @@ static struct virtio_driver virtio_balloon_driver = { .validate = virtballoon_validate, .probe = virtballoon_probe, .remove = virtballoon_remove, + .shutdown = virtballoon_shutdown, .config_changed = virtballoon_changed, #ifdef CONFIG_PM_SLEEP .freeze = virtballoon_freeze, diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index d0728285b6ce9..698cbaef9eec1 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -46,9 +46,12 @@ static void virtinput_recv_events(struct virtqueue *vq) le16_to_cpu(event->code), le32_to_cpu(event->value)); spin_lock_irqsave(&vi->lock, flags); + if (!vi->ready) + continue; virtinput_queue_evtbuf(vi, event); } - virtqueue_kick(vq); + if (vi->ready) + virtqueue_kick(vq); } spin_unlock_irqrestore(&vi->lock, flags); } @@ -328,6 +331,7 @@ static int virtinput_probe(struct virtio_device *vdev) spin_lock_irqsave(&vi->lock, flags); vi->ready = false; spin_unlock_irqrestore(&vi->lock, flags); + virtio_reset_device(vdev); err_mt_init_slots: input_free_device(vi->idev); err_input_alloc: @@ -347,8 +351,9 @@ static void virtinput_remove(struct virtio_device *vdev) vi->ready = false; spin_unlock_irqrestore(&vi->lock, flags); - input_unregister_device(vi->idev); + /* Callbacks use vi->idev. */ virtio_reset_device(vdev); + input_unregister_device(vi->idev); while ((buf = virtqueue_detach_unused_buf(vi->sts)) != NULL) kfree(buf); vdev->config->del_vqs(vdev); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 4832c9835dff9..607e27b1feb29 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -87,6 +87,9 @@ struct virtio_mmio_device { void __iomem *base; unsigned long version; + + /* True if enable_irq_wake() succeeded for the shared IRQ. */ + bool wake_irq_enabled; }; /* Configuration interface */ @@ -329,11 +332,17 @@ static void vm_del_vqs(struct virtio_device *vdev) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtqueue *vq, *n; + int irq = platform_get_irq(vm_dev->pdev, 0); list_for_each_entry_safe(vq, n, &vdev->vqs, list) vm_del_vq(vq); - free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev); + if (vm_dev->wake_irq_enabled) { + disable_irq_wake(irq); + vm_dev->wake_irq_enabled = false; + } + + free_irq(irq, vm_dev); } static void vm_synchronize_cbs(struct virtio_device *vdev) @@ -460,8 +469,9 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, if (err) return err; - if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source")) - enable_irq_wake(irq); + if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source") && + !enable_irq_wake(irq)) + vm_dev->wake_irq_enabled = true; for (i = 0; i < nvqs; ++i) { struct virtqueue_info *vqi = &vqs_info[i]; diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 30c6627a5ce4d..64806d042dbb2 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -120,7 +120,9 @@ static irqreturn_t vp_interrupt(int irq, void *opaque) if (isr & VIRTIO_PCI_ISR_CONFIG) vp_config_changed(irq, opaque); - return vp_vring_interrupt(irq, opaque); + vp_vring_interrupt(irq, opaque); + + return IRQ_HANDLED; } static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, @@ -502,7 +504,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, if (!avq_num) return 0; sprintf(avq->name, "avq.%u", avq->vq_index); - vq = vp_setup_vq(vdev, queue_idx++, vp_modern_avq_done, avq->name, + vq = vp_setup_vq(vdev, avq->vq_index, vp_modern_avq_done, avq->name, false, VIRTIO_MSI_NO_VECTOR, &vp_dev->admin_vq.info); if (IS_ERR(vq)) { diff --git a/drivers/virtio/virtio_rtc_driver.c b/drivers/virtio/virtio_rtc_driver.c index 4419735b0f0dc..74616ba5be119 100644 --- a/drivers/virtio/virtio_rtc_driver.c +++ b/drivers/virtio/virtio_rtc_driver.c @@ -574,8 +574,8 @@ static int viortc_msg_xfer(struct viortc_vq *vq, struct viortc_msg *msg, * read requests */ -/** timeout for clock readings, where timeouts are considered non-fatal */ -#define VIORTC_MSG_READ_TIMEOUT secs_to_jiffies(60) +/** timeout for runtime requests, where timeouts are considered non-fatal */ +#define VIORTC_MSG_TIMEOUT secs_to_jiffies(60) /** * viortc_read() - VIRTIO_RTC_REQ_READ wrapper @@ -600,7 +600,7 @@ int viortc_read(struct viortc_dev *viortc, u16 vio_clk_id, u64 *reading) VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -642,7 +642,7 @@ int viortc_read_cross(struct viortc_dev *viortc, u16 vio_clk_id, u8 hw_counter, VIORTC_MSG_WRITE(hdl, hw_counter, &hw_counter); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - VIORTC_MSG_READ_TIMEOUT); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -809,7 +809,7 @@ int viortc_read_alarm(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, clock_id, &vio_clk_id); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -858,7 +858,7 @@ int viortc_set_alarm(struct viortc_dev *viortc, u16 vio_clk_id, u64 alarm_time, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); @@ -900,7 +900,7 @@ int viortc_set_alarm_enabled(struct viortc_dev *viortc, u16 vio_clk_id, VIORTC_MSG_WRITE(hdl, flags, &flags); ret = viortc_msg_xfer(&viortc->vqs[VIORTC_REQUESTQ], VIORTC_MSG(hdl), - 0); + VIORTC_MSG_TIMEOUT); if (ret) { dev_dbg(&viortc->vdev->dev, "%s: xfer returned %d\n", __func__, ret); diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c index e2a568c9a43aa..0c93bf338a6ef 100644 --- a/drivers/w1/masters/ds2482.c +++ b/drivers/w1/masters/ds2482.c @@ -310,6 +310,10 @@ static u8 ds2482_w1_triplet(void *data, u8 dbit) mutex_unlock(&pdev->access_lock); + /* On bus error, decode to 3 (no device responded) to abort the search */ + if (status < 0) + status = 3 << 5; + /* Decode the status */ return (status >> 5); } diff --git a/drivers/w1/slaves/w1_ds28e17.c b/drivers/w1/slaves/w1_ds28e17.c index 5738cbce1a37f..d98a1719d6b36 100644 --- a/drivers/w1/slaves/w1_ds28e17.c +++ b/drivers/w1/slaves/w1_ds28e17.c @@ -389,6 +389,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter, * another simple read in that case. */ if (msgs[i+1].flags & I2C_M_RECV_LEN) { + if (msgs[i+1].buf[0] > I2C_SMBUS_BLOCK_MAX) { + i = -EPROTO; + goto error; + } result = w1_f19_i2c_read(sl, msgs[i+1].addr, &(msgs[i+1].buf[1]), msgs[i+1].buf[0]); if (result < 0) { @@ -415,6 +419,10 @@ static int w1_f19_i2c_master_transfer(struct i2c_adapter *adapter, * another simple read in that case. */ if (msgs[i].flags & I2C_M_RECV_LEN) { + if (msgs[i].buf[0] > I2C_SMBUS_BLOCK_MAX) { + i = -EPROTO; + goto error; + } result = w1_f19_i2c_read(sl, msgs[i].addr, &(msgs[i].buf[1]), diff --git a/drivers/watchdog/airoha_wdt.c b/drivers/watchdog/airoha_wdt.c index dc8ca11c14d81..4bd333189b87e 100644 --- a/drivers/watchdog/airoha_wdt.c +++ b/drivers/watchdog/airoha_wdt.c @@ -147,6 +147,9 @@ static int airoha_wdt_probe(struct platform_device *pdev) /* Watchdog ticks at half the bus rate */ airoha_wdt->wdt_freq = clk_get_rate(bus_clk) / 2; + if (!airoha_wdt->wdt_freq) + return dev_err_probe(dev, -EINVAL, + "invalid clock frequency\n"); /* Initialize struct watchdog device */ wdog_dev = &airoha_wdt->wdog_dev; diff --git a/drivers/watchdog/at91sam9_wdt.c b/drivers/watchdog/at91sam9_wdt.c index aba66b8e9d033..80ba04df54adf 100644 --- a/drivers/watchdog/at91sam9_wdt.c +++ b/drivers/watchdog/at91sam9_wdt.c @@ -242,7 +242,7 @@ static int at91_wdt_init(struct platform_device *pdev, struct at91wdt *wdt) return 0; out_stop_timer: - timer_delete(&wdt->timer); + timer_shutdown_sync(&wdt->timer); return err; } @@ -378,7 +378,7 @@ static void at91wdt_remove(struct platform_device *pdev) watchdog_unregister_device(&wdt->wdd); pr_warn("I quit now, hardware will probably reboot!\n"); - timer_delete(&wdt->timer); + timer_shutdown_sync(&wdt->timer); } #if defined(CONFIG_OF) diff --git a/drivers/watchdog/bd96801_wdt.c b/drivers/watchdog/bd96801_wdt.c index 12b74fd2bc052..a25b7cf1488b2 100644 --- a/drivers/watchdog/bd96801_wdt.c +++ b/drivers/watchdog/bd96801_wdt.c @@ -169,7 +169,6 @@ static int bd96801_set_wdt_mode(struct wdtbd96801 *w, unsigned int hw_margin, int fastng, slowng, type, ret, reg, mask; struct device *dev = w->dev; - if (hw_margin_min * 1000 > FASTNG_MAX_US) { dev_err(dev, "Unsupported fast timeout %u uS [max %u]\n", hw_margin_min * 1000, FASTNG_MAX_US); @@ -258,10 +257,10 @@ static int bd96801_set_heartbeat_from_hw(struct wdtbd96801 *w, fast = FASTNG_MIN << sel; sel = (val & BD96801_WD_RATIO_MASK) + 1; - w->wdt.max_hw_heartbeat_ms = (fast << sel) / USEC_PER_MSEC; + w->wdt.max_hw_heartbeat_ms = (fast << sel) / 10; if ((conf_reg & BD96801_WD_TYPE_MASK) == BD96801_WD_TYPE_WIN) - w->wdt.min_hw_heartbeat_ms = fast / USEC_PER_MSEC; + w->wdt.min_hw_heartbeat_ms = fast / 10; return 0; } diff --git a/drivers/watchdog/db8500_wdt.c b/drivers/watchdog/db8500_wdt.c index 97148ac0aa54a..70ccea13288d8 100644 --- a/drivers/watchdog/db8500_wdt.c +++ b/drivers/watchdog/db8500_wdt.c @@ -16,7 +16,7 @@ #include #include -#include +#include #define WATCHDOG_TIMEOUT 600 /* 10 minutes */ @@ -37,24 +37,24 @@ MODULE_PARM_DESC(nowayout, static int db8500_wdt_start(struct watchdog_device *wdd) { - return prcmu_enable_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_enable_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_stop(struct watchdog_device *wdd) { - return prcmu_disable_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_disable_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_keepalive(struct watchdog_device *wdd) { - return prcmu_kick_a9wdog(PRCMU_WDOG_ALL); + return db8500_prcmu_kick_a9wdog(PRCMU_WDOG_ALL); } static int db8500_wdt_set_timeout(struct watchdog_device *wdd, unsigned int timeout) { db8500_wdt_stop(wdd); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(wdd); return 0; @@ -91,10 +91,10 @@ static int db8500_wdt_probe(struct platform_device *pdev) watchdog_set_nowayout(&db8500_wdt, nowayout); /* disable auto off on sleep */ - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); /* set HW initial value */ - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); ret = devm_watchdog_register_device(dev, &db8500_wdt); if (ret) @@ -110,9 +110,9 @@ static int db8500_wdt_suspend(struct platform_device *pdev, { if (watchdog_active(&db8500_wdt)) { db8500_wdt_stop(&db8500_wdt); - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, true); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(&db8500_wdt); } return 0; @@ -122,9 +122,9 @@ static int db8500_wdt_resume(struct platform_device *pdev) { if (watchdog_active(&db8500_wdt)) { db8500_wdt_stop(&db8500_wdt); - prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); + db8500_prcmu_config_a9wdog(PRCMU_WDOG_CPU1, false); - prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); + db8500_prcmu_load_a9wdog(PRCMU_WDOG_ALL, timeout * 1000); db8500_wdt_start(&db8500_wdt); } return 0; diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c index 03479110453ce..855dc9d5083a5 100644 --- a/drivers/watchdog/imx7ulp_wdt.c +++ b/drivers/watchdog/imx7ulp_wdt.c @@ -56,6 +56,7 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" struct imx_wdt_hw_feature { bool prescaler_enable; bool post_rcs_wait; + bool cpu_lpm_auto_cg; u32 wdog_clock_rate; }; @@ -360,7 +361,7 @@ static int __maybe_unused imx7ulp_wdt_suspend_noirq(struct device *dev) { struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev); - if (watchdog_active(&imx7ulp_wdt->wdd)) + if (watchdog_active(&imx7ulp_wdt->wdd) && !imx7ulp_wdt->hw->cpu_lpm_auto_cg) imx7ulp_wdt_stop(&imx7ulp_wdt->wdd); clk_disable_unprepare(imx7ulp_wdt->clk); @@ -408,10 +409,17 @@ static const struct imx_wdt_hw_feature imx93_wdt_hw = { .wdog_clock_rate = 125, }; +static const struct imx_wdt_hw_feature imx94_wdt_hw = { + .prescaler_enable = true, + .wdog_clock_rate = 125, + .cpu_lpm_auto_cg = true, +}; + static const struct of_device_id imx7ulp_wdt_dt_ids[] = { { .compatible = "fsl,imx7ulp-wdt", .data = &imx7ulp_wdt_hw, }, { .compatible = "fsl,imx8ulp-wdt", .data = &imx8ulp_wdt_hw, }, { .compatible = "fsl,imx93-wdt", .data = &imx93_wdt_hw, }, + { .compatible = "fsl,imx94-wdt", .data = &imx94_wdt_hw, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, imx7ulp_wdt_dt_ids); diff --git a/drivers/watchdog/lenovo_se10_wdt.c b/drivers/watchdog/lenovo_se10_wdt.c index cd0500e5080b4..503e220263f95 100644 --- a/drivers/watchdog/lenovo_se10_wdt.c +++ b/drivers/watchdog/lenovo_se10_wdt.c @@ -178,7 +178,7 @@ static int se10_wdt_probe(struct platform_device *pdev) return -EBUSY; chip_id = get_chipID(); - if (chip_id != 0x5632) { + if (chip_id != 0x5632 && chip_id != 0x5652) { release_region(CFG_PORT, CFG_SIZE); return -ENODEV; } @@ -224,7 +224,7 @@ static struct platform_driver se10_wdt_driver = { .probe = se10_wdt_probe, }; -static int se10_create_platform_device(const struct dmi_system_id *id) +static int se10_create_platform_device(void) { int err; @@ -233,9 +233,10 @@ static int se10_create_platform_device(const struct dmi_system_id *id) return -ENOMEM; err = platform_device_add(se10_pdev); - if (err) + if (err) { platform_device_put(se10_pdev); - + se10_pdev = NULL; + } return err; } @@ -246,7 +247,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), DMI_MATCH(DMI_PRODUCT_NAME, "12NH"), }, - .callback = se10_create_platform_device, }, { .ident = "LENOVO-SE10", @@ -254,7 +254,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), DMI_MATCH(DMI_PRODUCT_NAME, "12NJ"), }, - .callback = se10_create_platform_device, }, { .ident = "LENOVO-SE10", @@ -262,7 +261,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), DMI_MATCH(DMI_PRODUCT_NAME, "12NK"), }, - .callback = se10_create_platform_device, }, { .ident = "LENOVO-SE10", @@ -270,7 +268,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), DMI_MATCH(DMI_PRODUCT_NAME, "12NL"), }, - .callback = se10_create_platform_device, }, { .ident = "LENOVO-SE10", @@ -278,7 +275,62 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = { DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), DMI_MATCH(DMI_PRODUCT_NAME, "12NM"), }, - .callback = se10_create_platform_device, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13LJ"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13LK"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S1"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S2"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S3"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S4"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S5"), + }, + }, + { + .ident = "LENOVO-SE10-G2", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"), + DMI_MATCH(DMI_PRODUCT_NAME, "13S6"), + }, }, {} }; @@ -286,10 +338,20 @@ MODULE_DEVICE_TABLE(dmi, se10_dmi_table); static int __init se10_wdt_init(void) { + int err; + if (!dmi_check_system(se10_dmi_table)) return -ENODEV; - return platform_driver_register(&se10_wdt_driver); + err = platform_driver_register(&se10_wdt_driver); + if (err) + return err; + + err = se10_create_platform_device(); + if (err) + platform_driver_unregister(&se10_wdt_driver); + + return err; } static void __exit se10_wdt_exit(void) diff --git a/drivers/watchdog/msc313e_wdt.c b/drivers/watchdog/msc313e_wdt.c index 90171431fc594..eea26c6c95d84 100644 --- a/drivers/watchdog/msc313e_wdt.c +++ b/drivers/watchdog/msc313e_wdt.c @@ -32,20 +32,36 @@ struct msc313e_wdt_priv { struct clk *clk; }; +static u32 msc313e_wdt_get_hw_timeout(struct msc313e_wdt_priv *priv) +{ + u16 low, high; + + low = readw(priv->base + REG_WDT_MAX_PRD_L); + high = readw(priv->base + REG_WDT_MAX_PRD_H); + + return ((u32)high << 16) | low; +} + +static void msc313e_wdt_set_hw_timeout(struct msc313e_wdt_priv *priv, + unsigned int timeout) +{ + u32 t = timeout * clk_get_rate(priv->clk); + + writew(t & 0xffff, priv->base + REG_WDT_MAX_PRD_L); + writew((t >> 16) & 0xffff, priv->base + REG_WDT_MAX_PRD_H); + writew(1, priv->base + REG_WDT_CLR); +} + static int msc313e_wdt_start(struct watchdog_device *wdev) { struct msc313e_wdt_priv *priv = watchdog_get_drvdata(wdev); - u32 timeout; int err; err = clk_prepare_enable(priv->clk); if (err) return err; - timeout = wdev->timeout * clk_get_rate(priv->clk); - writew(timeout & 0xffff, priv->base + REG_WDT_MAX_PRD_L); - writew((timeout >> 16) & 0xffff, priv->base + REG_WDT_MAX_PRD_H); - writew(1, priv->base + REG_WDT_CLR); + msc313e_wdt_set_hw_timeout(priv, wdev->timeout); return 0; } @@ -70,9 +86,13 @@ static int msc313e_wdt_stop(struct watchdog_device *wdev) static int msc313e_wdt_settimeout(struct watchdog_device *wdev, unsigned int new_time) { + struct msc313e_wdt_priv *priv = watchdog_get_drvdata(wdev); + wdev->timeout = new_time; - return msc313e_wdt_start(wdev); + if (watchdog_hw_running(wdev) || watchdog_active(wdev)) + msc313e_wdt_set_hw_timeout(priv, wdev->timeout); + return 0; } static const struct watchdog_info msc313e_wdt_ident = { @@ -98,6 +118,8 @@ static int msc313e_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct msc313e_wdt_priv *priv; + unsigned long rate; + int ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -117,27 +139,51 @@ static int msc313e_wdt_probe(struct platform_device *pdev) priv->wdev.ops = &msc313e_wdt_ops, priv->wdev.parent = dev; priv->wdev.min_timeout = MSC313E_WDT_MIN_TIMEOUT; - priv->wdev.max_timeout = U32_MAX / clk_get_rate(priv->clk); + rate = clk_get_rate(priv->clk); + if (!rate) + return -EINVAL; + priv->wdev.max_timeout = U32_MAX / rate; priv->wdev.timeout = MSC313E_WDT_DEFAULT_TIMEOUT; - /* If the period is non-zero the WDT is running */ - if (readw(priv->base + REG_WDT_MAX_PRD_L) | (readw(priv->base + REG_WDT_MAX_PRD_H) << 16)) - set_bit(WDOG_HW_RUNNING, &priv->wdev.status); - watchdog_set_drvdata(&priv->wdev, priv); + platform_set_drvdata(pdev, priv); watchdog_init_timeout(&priv->wdev, timeout, dev); watchdog_stop_on_reboot(&priv->wdev); watchdog_stop_on_unregister(&priv->wdev); + watchdog_stop_ping_on_suspend(&priv->wdev); + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + /* If the period is non-zero the WDT is running */ + if (msc313e_wdt_get_hw_timeout(priv)) { + msc313e_wdt_set_hw_timeout(priv, priv->wdev.timeout); + set_bit(WDOG_HW_RUNNING, &priv->wdev.status); + /* + * Keep the clock enabled. The watchdog core will skip the next + * start() and a future stop() will balance the CCF reference + * count. + */ + } else { + clk_disable_unprepare(priv->clk); + } + + ret = devm_watchdog_register_device(dev, &priv->wdev); + + /* If the WDT is running and anything goes wrong, disable the clock. */ + if (ret && test_bit(WDOG_HW_RUNNING, &priv->wdev.status)) + clk_disable_unprepare(priv->clk); - return devm_watchdog_register_device(dev, &priv->wdev); + return ret; } static int __maybe_unused msc313e_wdt_suspend(struct device *dev) { struct msc313e_wdt_priv *priv = dev_get_drvdata(dev); - if (watchdog_active(&priv->wdev)) + if (watchdog_active(&priv->wdev) || watchdog_hw_running(&priv->wdev)) msc313e_wdt_stop(&priv->wdev); return 0; @@ -147,7 +193,7 @@ static int __maybe_unused msc313e_wdt_resume(struct device *dev) { struct msc313e_wdt_priv *priv = dev_get_drvdata(dev); - if (watchdog_active(&priv->wdev)) + if (watchdog_active(&priv->wdev) || watchdog_hw_running(&priv->wdev)) msc313e_wdt_start(&priv->wdev); return 0; diff --git a/drivers/watchdog/pseries-wdt.c b/drivers/watchdog/pseries-wdt.c index 7f53b52934090..bad4c63b83101 100644 --- a/drivers/watchdog/pseries-wdt.c +++ b/drivers/watchdog/pseries-wdt.c @@ -13,61 +13,10 @@ #include #include #include +#include #define DRV_NAME "pseries-wdt" -/* - * H_WATCHDOG Input - * - * R4: "flags": - * - * Bits 48-55: "operation" - */ -#define PSERIES_WDTF_OP_START 0x100UL /* start timer */ -#define PSERIES_WDTF_OP_STOP 0x200UL /* stop timer */ -#define PSERIES_WDTF_OP_QUERY 0x300UL /* query timer capabilities */ - -/* - * Bits 56-63: "timeoutAction" (for "Start Watchdog" only) - */ -#define PSERIES_WDTF_ACTION_HARD_POWEROFF 0x1UL /* poweroff */ -#define PSERIES_WDTF_ACTION_HARD_RESTART 0x2UL /* restart */ -#define PSERIES_WDTF_ACTION_DUMP_RESTART 0x3UL /* dump + restart */ - -/* - * H_WATCHDOG Output - * - * R3: Return code - * - * H_SUCCESS The operation completed. - * - * H_BUSY The hypervisor is too busy; retry the operation. - * - * H_PARAMETER The given "flags" are somehow invalid. Either the - * "operation" or "timeoutAction" is invalid, or a - * reserved bit is set. - * - * H_P2 The given "watchdogNumber" is zero or exceeds the - * supported maximum value. - * - * H_P3 The given "timeoutInMs" is below the supported - * minimum value. - * - * H_NOOP The given "watchdogNumber" is already stopped. - * - * H_HARDWARE The operation failed for ineffable reasons. - * - * H_FUNCTION The H_WATCHDOG hypercall is not supported by this - * hypervisor. - * - * R4: - * - * - For the "Query Watchdog Capabilities" operation, a 64-bit - * structure: - */ -#define PSERIES_WDTQ_MIN_TIMEOUT(cap) (((cap) >> 48) & 0xffff) -#define PSERIES_WDTQ_MAX_NUMBER(cap) (((cap) >> 32) & 0xffff) - static const unsigned long pseries_wdt_action[] = { [0] = PSERIES_WDTF_ACTION_HARD_POWEROFF, [1] = PSERIES_WDTF_ACTION_HARD_RESTART, diff --git a/drivers/watchdog/s32g_wdt.c b/drivers/watchdog/s32g_wdt.c index ad55063060af6..6422a694fc65f 100644 --- a/drivers/watchdog/s32g_wdt.c +++ b/drivers/watchdog/s32g_wdt.c @@ -56,8 +56,7 @@ MODULE_PARM_DESC(early_enable, static const struct watchdog_info s32g_wdt_info = { .identity = "s32g watchdog", - .options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | - WDIOC_GETTIMEOUT | WDIOC_GETTIMELEFT, + .options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE, }; static struct s32g_wdt_device *wdd_to_s32g_wdt(struct watchdog_device *wdd) diff --git a/drivers/watchdog/sunxi_wdt.c b/drivers/watchdog/sunxi_wdt.c index b6c761acc3de6..3db34524ed13b 100644 --- a/drivers/watchdog/sunxi_wdt.c +++ b/drivers/watchdog/sunxi_wdt.c @@ -128,6 +128,38 @@ static int sunxi_wdt_ping(struct watchdog_device *wdt_dev) return 0; } +static bool sunxi_wdt_is_running(struct watchdog_device *wdt_dev) +{ + struct sunxi_wdt_dev *sunxi_wdt = watchdog_get_drvdata(wdt_dev); + const struct sunxi_wdt_reg *regs = sunxi_wdt->wdt_regs; + + return readl(sunxi_wdt->wdt_base + regs->wdt_mode) & WDT_MODE_EN; +} + +static unsigned int sunxi_wdt_get_timeout(struct watchdog_device *wdt_dev) +{ + struct sunxi_wdt_dev *sunxi_wdt = watchdog_get_drvdata(wdt_dev); + const struct sunxi_wdt_reg *regs = sunxi_wdt->wdt_regs; + unsigned int timeout; + u32 interval; + + interval = readl(sunxi_wdt->wdt_base + regs->wdt_mode); + interval >>= regs->wdt_timeout_shift; + interval &= WDT_TIMEOUT_MASK; + /* Round the 0.5-second interval up to the minimum representable timeout. */ + if (!interval) + return WDT_MIN_TIMEOUT; + + for (timeout = WDT_MIN_TIMEOUT; + timeout < ARRAY_SIZE(wdt_timeout_map); timeout++) { + if (wdt_timeout_map[timeout] == interval) + return timeout; + } + + /* Reserved interval encoding. */ + return 0; +} + static int sunxi_wdt_set_timeout(struct watchdog_device *wdt_dev, unsigned int timeout) { @@ -259,6 +291,7 @@ static int sunxi_wdt_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct sunxi_wdt_dev *sunxi_wdt; + unsigned int running_timeout; int err; sunxi_wdt = devm_kzalloc(dev, sizeof(*sunxi_wdt), GFP_KERNEL); @@ -286,7 +319,17 @@ static int sunxi_wdt_probe(struct platform_device *pdev) watchdog_set_drvdata(&sunxi_wdt->wdt_dev, sunxi_wdt); - sunxi_wdt_stop(&sunxi_wdt->wdt_dev); + if (sunxi_wdt_is_running(&sunxi_wdt->wdt_dev)) { + running_timeout = sunxi_wdt_get_timeout(&sunxi_wdt->wdt_dev); + if (running_timeout) + sunxi_wdt->wdt_dev.timeout = running_timeout; + + err = sunxi_wdt_start(&sunxi_wdt->wdt_dev); + if (err) + return err; + + set_bit(WDOG_HW_RUNNING, &sunxi_wdt->wdt_dev.status); + } watchdog_stop_on_reboot(&sunxi_wdt->wdt_dev); err = devm_watchdog_register_device(dev, &sunxi_wdt->wdt_dev); diff --git a/drivers/watchdog/watchdog_hrtimer_pretimeout.c b/drivers/watchdog/watchdog_hrtimer_pretimeout.c index fbc7eecd8b203..49a05ea60c979 100644 --- a/drivers/watchdog/watchdog_hrtimer_pretimeout.c +++ b/drivers/watchdog/watchdog_hrtimer_pretimeout.c @@ -30,6 +30,7 @@ void watchdog_hrtimer_pretimeout_init(struct watchdog_device *wdd) void watchdog_hrtimer_pretimeout_start(struct watchdog_device *wdd) { if (!(wdd->info->options & WDIOF_PRETIMEOUT) && + wdd->pretimeout && !watchdog_pretimeout_invalid(wdd, wdd->pretimeout)) hrtimer_start(&wdd->wd_data->pretimeout_timer, ktime_set(wdd->timeout - wdd->pretimeout, 0), diff --git a/drivers/watchdog/watchdog_pretimeout.c b/drivers/watchdog/watchdog_pretimeout.c index 2526436dc74dd..ea05f84f07e18 100644 --- a/drivers/watchdog/watchdog_pretimeout.c +++ b/drivers/watchdog/watchdog_pretimeout.c @@ -167,6 +167,8 @@ void watchdog_unregister_governor(struct watchdog_governor *gov) } spin_lock_irq(&pretimeout_lock); + if (default_gov == gov) + default_gov = NULL; list_for_each_entry(p, &pretimeout_list, entry) if (p->wdd->gov == gov) p->wdd->gov = default_gov; diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 8c44a25a7d2b9..d4c99165b3a9c 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -679,7 +679,7 @@ void xen_free_ballooned_pages(unsigned int nr_pages, struct page **pages) } EXPORT_SYMBOL(xen_free_ballooned_pages); -static int __init balloon_add_regions(void) +static int __init balloon_add_regions(bool append) { unsigned long start_pfn, pages; unsigned long pfn, extra_pfn_end; @@ -703,19 +703,26 @@ static int __init balloon_add_regions(void) balloon_append(pfn_to_page(pfn)); /* - * Extra regions are accounted for in the physmap, but need - * decreasing from current_pages and target_pages to balloon - * down the initial allocation, because they are already - * accounted for in total_pages. + * There are two different use-cases depending on how the + * initial memory target is fetched. For PVH dom0 and PV the + * target is usually set to reflect the domain assigned memory, + * and hence extra regions need adding. + * + * OTOH for HVM and PVH domU the target is set to the amount of + * RAM reported in the memory map, and hence extra regions need + * subtracting to reflect the real memory usage. */ pages = extra_pfn_end - start_pfn; - if (pages >= balloon_stats.current_pages || - pages >= balloon_stats.target_pages) { + if (append) { + balloon_stats.total_pages += pages; + } else if (pages >= balloon_stats.current_pages || + pages >= balloon_stats.target_pages) { WARN(1, "Extra pages underflow current target"); return -ERANGE; + } else { + balloon_stats.current_pages -= pages; + balloon_stats.target_pages -= pages; } - balloon_stats.current_pages -= pages; - balloon_stats.target_pages -= pages; } return 0; @@ -724,7 +731,9 @@ static int __init balloon_add_regions(void) static int __init balloon_init(void) { struct task_struct *task; - unsigned long current_pages; + long current_pages = 0; + domid_t domid = DOMID_SELF; + bool append = true; int rc; if (!xen_domain()) @@ -732,15 +741,22 @@ static int __init balloon_init(void) pr_info("Initialising balloon driver\n"); - if (xen_pv_domain()) { - if (xen_released_pages >= xen_start_info->nr_pages) - goto underflow; - current_pages = min(xen_start_info->nr_pages - - xen_released_pages, max_pfn); - } else { - if (xen_unpopulated_pages >= get_num_physpages()) - goto underflow; - current_pages = get_num_physpages() - xen_unpopulated_pages; + if (xen_initial_domain()) + current_pages = HYPERVISOR_memory_op(XENMEM_current_reservation, + &domid); + if (current_pages <= 0) { + if (xen_pv_domain()) { + if (xen_released_pages >= xen_start_info->nr_pages) + goto underflow; + current_pages = min(xen_start_info->nr_pages - + xen_released_pages, max_pfn); + } else { + if (xen_unpopulated_pages >= get_num_physpages()) + goto underflow; + append = false; + current_pages = get_num_physpages() - + xen_unpopulated_pages; + } } balloon_stats.current_pages = current_pages; @@ -760,7 +776,7 @@ static int __init balloon_init(void) register_sysctl_init("xen/balloon", balloon_table); #endif - rc = balloon_add_regions(); + rc = balloon_add_regions(append); if (rc) return rc; diff --git a/drivers/xen/gntalloc.c b/drivers/xen/gntalloc.c index f93f73ecefeee..462f67dc32751 100644 --- a/drivers/xen/gntalloc.c +++ b/drivers/xen/gntalloc.c @@ -272,6 +272,7 @@ static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, int rc = 0; struct ioctl_gntalloc_alloc_gref op; uint32_t *gref_ids; + unsigned int limit_snapshot; pr_debug("%s: priv %p\n", __func__, priv); @@ -280,6 +281,12 @@ static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, goto out; } + limit_snapshot = READ_ONCE(limit); + if (op.count > limit_snapshot) { + rc = -ENOSPC; + goto out; + } + gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_KERNEL); if (!gref_ids) { rc = -ENOMEM; @@ -292,14 +299,16 @@ static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, * are about to enforce, removing them here is a good idea. */ do_cleanup(); - if (gref_size + op.count > limit) { + limit_snapshot = READ_ONCE(limit); + if (gref_size > limit_snapshot || + op.count > limit_snapshot - gref_size) { mutex_unlock(&gref_mutex); rc = -ENOSPC; goto out_free; } gref_size += op.count; op.index = priv->index; - priv->index += op.count * PAGE_SIZE; + priv->index += (uint64_t)op.count * PAGE_SIZE; mutex_unlock(&gref_mutex); rc = add_grefs(&op, gref_ids, priv); diff --git a/drivers/xen/xen-acpi-processor.c b/drivers/xen/xen-acpi-processor.c index 520756159d3d3..8d1860bd5d578 100644 --- a/drivers/xen/xen-acpi-processor.c +++ b/drivers/xen/xen-acpi-processor.c @@ -492,7 +492,7 @@ static void xen_acpi_processor_resume_worker(struct work_struct *dummy) pr_info("ACPI data upload failed, error = %d\n", rc); } -static void xen_acpi_processor_resume(void) +static void xen_acpi_processor_resume(void *data) { static DECLARE_WORK(wq, xen_acpi_processor_resume_worker); @@ -506,10 +506,14 @@ static void xen_acpi_processor_resume(void) schedule_work(&wq); } -static struct syscore_ops xap_syscore_ops = { +static const struct syscore_ops xap_syscore_ops = { .resume = xen_acpi_processor_resume, }; +static struct syscore xap_syscore = { + .ops = &xap_syscore_ops, +}; + static int __init xen_acpi_processor_init(void) { int i; @@ -560,7 +564,7 @@ static int __init xen_acpi_processor_init(void) if (rc) goto err_unregister; - register_syscore_ops(&xap_syscore_ops); + register_syscore(&xap_syscore); return 0; err_unregister: @@ -577,7 +581,7 @@ static void __exit xen_acpi_processor_exit(void) { int i; - unregister_syscore_ops(&xap_syscore_ops); + unregister_syscore(&xap_syscore); bitmap_free(acpi_ids_done); bitmap_free(acpi_id_present); bitmap_free(acpi_id_cst_present); diff --git a/drivers/xen/xen-front-pgdir-shbuf.c b/drivers/xen/xen-front-pgdir-shbuf.c index 223870a0111b2..e7fb06f607711 100644 --- a/drivers/xen/xen-front-pgdir-shbuf.c +++ b/drivers/xen/xen-front-pgdir-shbuf.c @@ -447,8 +447,10 @@ static int grant_references(struct xen_front_pgdir_shbuf *buf) unsigned long frame; cur_ref = gnttab_claim_grant_reference(&priv_gref_head); - if (cur_ref < 0) - return cur_ref; + if (cur_ref < 0) { + ret = cur_ref; + goto out_free_refs; + } frame = xen_page_to_gfn(virt_to_page(buf->directory + PAGE_SIZE * i)); @@ -459,11 +461,13 @@ static int grant_references(struct xen_front_pgdir_shbuf *buf) if (buf->ops->grant_refs_for_buffer) { ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j); if (ret) - return ret; + goto out_free_refs; } + ret = 0; +out_free_refs: gnttab_free_grant_references(priv_gref_head); - return 0; + return ret; } /* diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index 528682bf0c7f2..f312b9a57d971 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -912,19 +912,27 @@ int xs_init(void) int err; struct task_struct *task; - register_reboot_notifier(&xs_reboot_nb); + err = register_reboot_notifier(&xs_reboot_nb); + if (err) + return err; /* Initialize the shared memory rings to talk to xenstored */ err = xb_init_comms(); if (err) - return err; + goto err_unregister_reboot_notifier; task = kthread_run(xenwatch_thread, NULL, "xenwatch"); - if (IS_ERR(task)) - return PTR_ERR(task); + if (IS_ERR(task)) { + err = PTR_ERR(task); + goto err_unregister_reboot_notifier; + } /* shutdown watches for kexec boot */ xs_reset_watches(); return 0; + +err_unregister_reboot_notifier: + unregister_reboot_notifier(&xs_reboot_nb); + return err; } diff --git a/fs/9p/vfs_dir.c b/fs/9p/vfs_dir.c index e0d34e4e9076e..323f85352f6a4 100644 --- a/fs/9p/vfs_dir.c +++ b/fs/9p/vfs_dir.c @@ -27,6 +27,7 @@ * struct p9_rdir - readdir accounting * @head: start offset of current dirread buffer * @tail: end offset of current dirread buffer + * @offset: file position the data at @head corresponds to * @buf: dirread buffer * * private structure for keeping track of readdir @@ -36,6 +37,7 @@ struct p9_rdir { int head; int tail; + loff_t offset; uint8_t buf[]; }; @@ -70,7 +72,7 @@ static struct p9_rdir *v9fs_alloc_rdir_buf(struct file *filp, int buflen) struct p9_fid *fid = filp->private_data; if (!fid->rdir) - fid->rdir = kzalloc(sizeof(struct p9_rdir) + buflen, GFP_KERNEL); + fid->rdir = kvzalloc(sizeof(struct p9_rdir) + buflen, GFP_KERNEL); return fid->rdir; } @@ -102,6 +104,9 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) kvec.iov_base = rdir->buf; kvec.iov_len = buflen; + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + while (1) { if (rdir->tail == rdir->head) { struct iov_iter to; @@ -117,6 +122,7 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) rdir->head = 0; rdir->tail = n; + rdir->offset = ctx->pos; } while (rdir->head < rdir->tail) { err = p9stat_read(fid->clnt, rdir->buf + rdir->head, @@ -134,6 +140,7 @@ static int v9fs_dir_readdir(struct file *file, struct dir_context *ctx) rdir->head += err; ctx->pos += err; + rdir->offset = ctx->pos; } } } @@ -161,6 +168,9 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) if (!rdir) return -ENOMEM; + if (rdir->head < rdir->tail && rdir->offset != ctx->pos) + rdir->head = rdir->tail = 0; + while (1) { if (rdir->tail == rdir->head) { err = p9_client_readdir(fid, rdir->buf, buflen, @@ -170,6 +180,7 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) rdir->head = 0; rdir->tail = err; + rdir->offset = ctx->pos; } while (rdir->head < rdir->tail) { @@ -190,6 +201,7 @@ static int v9fs_dir_readdir_dotl(struct file *file, struct dir_context *ctx) ctx->pos = curdirent.d_off; rdir->head += err; + rdir->offset = ctx->pos; } } } diff --git a/fs/affs/affs.h b/fs/affs/affs.h index ac4e9a02910b7..99e54e3453ac1 100644 --- a/fs/affs/affs.h +++ b/fs/affs/affs.h @@ -225,11 +225,6 @@ static inline bool affs_validblock(struct super_block *sb, int block) block < AFFS_SB(sb)->s_partition_size); } -static inline void -affs_set_blocksize(struct super_block *sb, int size) -{ - sb_set_blocksize(sb, size); -} static inline struct buffer_head * affs_bread(struct super_block *sb, int block) { diff --git a/fs/affs/super.c b/fs/affs/super.c index 44f8aa883100e..994f1199beaf1 100644 --- a/fs/affs/super.c +++ b/fs/affs/super.c @@ -357,7 +357,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc) size = bdev_nr_sectors(sb->s_bdev); pr_debug("initial blocksize=%d, #blocks=%d\n", 512, size); - affs_set_blocksize(sb, PAGE_SIZE); + if (!sb_set_blocksize(sb, PAGE_SIZE)) + return -EINVAL; /* Try to find root block. Its location depends on the block size. */ i = bdev_logical_block_size(sb->s_bdev); @@ -373,7 +374,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc) if (ctx->root_block < 0) sbi->s_root_block = (ctx->reserved + size - 1) / 2; pr_debug("setting blocksize to %d\n", blocksize); - affs_set_blocksize(sb, blocksize); + if (!sb_set_blocksize(sb, blocksize)) + return -EINVAL; sbi->s_partition_size = size; /* The root block location that was calculated above is not diff --git a/fs/afs/addr_list.c b/fs/afs/addr_list.c index b1704de3d95f5..48d68f5aa7b43 100644 --- a/fs/afs/addr_list.c +++ b/fs/afs/addr_list.c @@ -394,8 +394,11 @@ void afs_set_peer_appdata(struct afs_server *server, struct rxrpc_peer *pn = new_alist->addrs[n].peer; struct rxrpc_peer *po = old_alist->addrs[o].peer; - if (pn == po) + if (pn == po) { + n++; + o++; continue; + } if (pn < po) { rxrpc_kernel_set_peer_data(pn, data); n++; diff --git a/fs/afs/callback.c b/fs/afs/callback.c index 0a09a9ca3c6b3..652c4301d56ea 100644 --- a/fs/afs/callback.c +++ b/fs/afs/callback.c @@ -138,6 +138,7 @@ static struct afs_volume *afs_lookup_volume_rcu(struct afs_cell *cell, if (volume && afs_try_get_volume(volume, afs_volume_trace_get_callback)) break; + volume = NULL; if (!need_seqretry(&cell->volume_lock, seq)) break; } diff --git a/fs/afs/dir_edit.c b/fs/afs/dir_edit.c index fd3aa9f97ce64..c31303059444a 100644 --- a/fs/afs/dir_edit.c +++ b/fs/afs/dir_edit.c @@ -415,7 +415,7 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, if (!afs_dir_init_iter(&iter, name)) return; - meta = afs_dir_find_block(&iter, 0); + meta = afs_dir_get_block(&iter, 0); if (!meta) return; @@ -442,7 +442,7 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, /* Check and clear the entry. */ de = &block->dirents[slot]; if (de->u.valid != 1) - goto error_unmap; + goto error; trace_afs_edit_dir(vnode, why, afs_edit_dir_delete, b, slot, ntohl(de->u.vnode), ntohl(de->u.unique), @@ -458,7 +458,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, /* Clear the constituent entries. */ next = de->u.hash_next; memset(de, 0, sizeof(*de) * iter.nr_slots); - kunmap_local(block); /* Adjust the hash chain: if iter->prev_entry is 0, the hashtable head * index is previous; otherwise it's slot number of the previous entry. @@ -485,7 +484,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, pde = &pblock->dirents[ps]; prev_next = pde->u.hash_next; if (prev_next != htons(entry)) { - kunmap_local(pblock); pr_warn("%llx:%llx:%x: not prev in chain b=%x p=%x,%x e=%x %*s", vnode->fid.vid, vnode->fid.vnode, vnode->fid.unique, iter.bucket, iter.prev_entry, prev_next, entry, @@ -493,7 +491,6 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, goto error; } pde->u.hash_next = next; - kunmap_local(pblock); } netfs_single_mark_inode_dirty(&vnode->netfs.inode); @@ -503,18 +500,16 @@ void afs_edit_dir_remove(struct afs_vnode *vnode, _debug("Remove %s from %u[%u]", name->name, b, slot); out_unmap: + afs_dir_end_iter(&iter); kunmap_local(meta); _leave(""); return; already_invalidated: - kunmap_local(block); trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_inval, 0, 0, 0, 0, name->name); goto out_unmap; -error_unmap: - kunmap_local(block); error: trace_afs_edit_dir(vnode, why, afs_edit_dir_delete_error, 0, 0, 0, 0, name->name); diff --git a/fs/afs/dir_search.c b/fs/afs/dir_search.c index d2516e55b5edb..770ef8002a658 100644 --- a/fs/afs/dir_search.c +++ b/fs/afs/dir_search.c @@ -75,10 +75,7 @@ union afs_xdr_dir_block *afs_dir_find_block(struct afs_dir_iter *iter, size_t bl _enter("%zx,%d", block, slot); - if (iter->block) { - kunmap_local(iter->block); - iter->block = NULL; - } + afs_dir_end_iter(iter); if (dvnode->directory_size < blend) goto fail; @@ -173,12 +170,8 @@ int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, ret = -ENOENT; found: - if (iter->block) { - kunmap_local(iter->block); - iter->block = NULL; - } - bad: + afs_dir_end_iter(iter); if (ret == -ESTALE) afs_invalidate_dir(iter->dvnode, afs_dir_invalid_iter_stale); _leave(" = %d", ret); diff --git a/fs/afs/fs_probe.c b/fs/afs/fs_probe.c index e0030ac74ea0b..a4c770d399b2d 100644 --- a/fs/afs/fs_probe.c +++ b/fs/afs/fs_probe.c @@ -258,6 +258,7 @@ int afs_fs_probe_fileserver(struct afs_net *net, struct afs_server *server, lockdep_is_held(&server->fs_lock)); if (old) { estate->responsive_set = old->responsive_set; + old_alist = old->addresses; if (!new_alist) new_alist = old->addresses; } diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c index bc9556991d7cd..2dfcd4a20d107 100644 --- a/fs/afs/fsclient.c +++ b/fs/afs/fsclient.c @@ -477,6 +477,9 @@ void afs_fs_fetch_data(struct afs_operation *op) if (!call) return afs_op_nomem(op); + if (op->flags & AFS_OPERATION_ASYNC) + call->async = true; + /* marshall the parameters */ bp = call->request; bp[0] = htonl(FSFETCHDATA); @@ -484,7 +487,7 @@ void afs_fs_fetch_data(struct afs_operation *op) bp[2] = htonl(vp->fid.vnode); bp[3] = htonl(vp->fid.unique); bp[4] = htonl(lower_32_bits(subreq->start + subreq->transferred)); - bp[5] = htonl(lower_32_bits(subreq->len + subreq->transferred)); + bp[5] = htonl(lower_32_bits(subreq->len - subreq->transferred)); call->fid = vp->fid; trace_afs_make_fs_call(call, &vp->fid); diff --git a/fs/afs/internal.h b/fs/afs/internal.h index bea8aae5364ec..25b5fb3eda95f 100644 --- a/fs/afs/internal.h +++ b/fs/afs/internal.h @@ -1129,6 +1129,14 @@ int afs_dir_search_bucket(struct afs_dir_iter *iter, const struct qstr *name, int afs_dir_search(struct afs_vnode *dvnode, const struct qstr *name, struct afs_fid *_fid, afs_dataversion_t *_dir_version); +static inline void afs_dir_end_iter(struct afs_dir_iter *iter) +{ + if (iter->block) { + kunmap_local(iter->block); + iter->block = NULL; + } +} + /* * dir_silly.c */ @@ -1420,7 +1428,7 @@ static inline void afs_make_op_call(struct afs_operation *op, struct afs_call *c { struct afs_addr_list *alist = op->estate->addresses; - op->call = call; + op->call = afs_get_call(call, afs_call_trace_get); op->type = call->type; call->op = op; call->key = op->key; @@ -1428,6 +1436,7 @@ static inline void afs_make_op_call(struct afs_operation *op, struct afs_call *c call->peer = rxrpc_kernel_get_peer(alist->addrs[op->addr_index].peer); call->service_id = op->server->service_id; afs_make_call(call, gfp); + afs_put_call(call); } static inline void afs_extract_begin(struct afs_call *call, void *buf, size_t size) diff --git a/fs/afs/server.c b/fs/afs/server.c index c4428ebddb1da..43520c2d9a571 100644 --- a/fs/afs/server.c +++ b/fs/afs/server.c @@ -242,7 +242,6 @@ struct afs_server *afs_lookup_server(struct afs_cell *cell, struct key *key, out: afs_put_addrlist(alist, afs_alist_trace_put_server_create); if (candidate) { - kfree(rcu_access_pointer(server->endpoint_state)); kfree(candidate); afs_dec_servers_outstanding(cell->net); } diff --git a/fs/autofs/inode.c b/fs/autofs/inode.c index f5c16ffba0134..9593d407cbc6f 100644 --- a/fs/autofs/inode.c +++ b/fs/autofs/inode.c @@ -322,8 +322,10 @@ static int autofs_fill_super(struct super_block *s, struct fs_context *fc) return -ENOMEM; root_inode = autofs_get_inode(s, S_IFDIR | 0755); - if (!root_inode) + if (!root_inode) { + autofs_free_ino(ino); return -ENOMEM; + } root_inode->i_uid = ctx->uid; root_inode->i_gid = ctx->gid; diff --git a/fs/backing-file.c b/fs/backing-file.c index e049a627d78fb..6f16e23f70461 100644 --- a/fs/backing-file.c +++ b/fs/backing-file.c @@ -34,7 +34,7 @@ struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, const struct cred *cred) { - const struct path *user_path = &user_file->f_path; + const struct path *user_path = file_user_path(user_file); struct file *f; int error; diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c index 8f430ff8e4458..cd9921b977a72 100644 --- a/fs/befs/linuxvfs.c +++ b/fs/befs/linuxvfs.c @@ -858,7 +858,8 @@ befs_fill_super(struct super_block *sb, struct fs_context *fc) */ sb->s_magic = BEFS_SUPER_MAGIC; /* Set real blocksize of fs */ - sb_set_blocksize(sb, (ulong) befs_sb->block_size); + if (!sb_set_blocksize(sb, (ulong) befs_sb->block_size)) + goto unacquire_priv_sbp; sb->s_op = &befs_sops; sb->s_export_op = &befs_export_operations; sb->s_time_min = 0; diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c index 984b365df0460..35b86dce3a788 100644 --- a/fs/bfs/inode.c +++ b/fs/bfs/inode.c @@ -342,7 +342,8 @@ static int bfs_fill_super(struct super_block *s, struct fs_context *fc) s->s_time_min = 0; s->s_time_max = U32_MAX; - sb_set_blocksize(s, BFS_BSIZE); + if (!sb_set_blocksize(s, BFS_BSIZE)) + goto out; sbh = sb_bread(s, 0); if (!sbh) diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c index a3d4e6973b299..7834ca77877db 100644 --- a/fs/binfmt_elf_fdpic.c +++ b/fs/binfmt_elf_fdpic.c @@ -231,6 +231,10 @@ static int load_elf_fdpic_binary(struct linux_binprm *bprm) for (i = 0; i < exec_params.hdr.e_phnum; i++, phdr++) { switch (phdr->p_type) { case PT_INTERP: + /* elf ABI allows only one interpreter */ + if (interpreter_name) + continue; + retval = -ENOMEM; if (phdr->p_filesz > PATH_MAX) goto error; diff --git a/fs/binfmt_misc.c b/fs/binfmt_misc.c index a8b1d79e4af07..9dadd2c14ab5e 100644 --- a/fs/binfmt_misc.c +++ b/fs/binfmt_misc.c @@ -161,8 +161,10 @@ static Node *get_binfmt_handler(struct binfmt_misc *misc, static void put_binfmt_handler(Node *e) { if (refcount_dec_and_test(&e->users)) { - if (e->flags & MISC_FMT_OPEN_FILE) + if (e->flags & MISC_FMT_OPEN_FILE) { + exe_file_allow_write_access(e->interp_file); filp_close(e->interp_file, NULL); + } kfree(e); } } @@ -227,9 +229,6 @@ static int load_misc_binary(struct linux_binprm *bprm) goto ret; } - if (fmt->flags & MISC_FMT_OPEN_BINARY) - bprm->have_execfd = 1; - /* make argv[1] be the path to the binary */ retval = copy_string_kernel(bprm->interp, bprm); if (retval < 0) @@ -249,8 +248,14 @@ static int load_misc_binary(struct linux_binprm *bprm) if (fmt->flags & MISC_FMT_OPEN_FILE) { interp_file = file_clone_open(fmt->interp_file); - if (!IS_ERR(interp_file)) - deny_write_access(interp_file); + if (!IS_ERR(interp_file)) { + int err = exe_file_deny_write_access(interp_file); + + if (err) { + fput(interp_file); + interp_file = ERR_PTR(err); + } + } } else { interp_file = open_exec(fmt->interpreter); } @@ -259,6 +264,8 @@ static int load_misc_binary(struct linux_binprm *bprm) goto ret; bprm->interpreter = interp_file; + if (fmt->flags & MISC_FMT_OPEN_BINARY) + bprm->have_execfd = 1; if (fmt->flags & MISC_FMT_CREDENTIALS) bprm->execfd_creds = 1; @@ -376,6 +383,10 @@ static Node *create_entry(const char __user *buffer, size_t count) pr_debug("register: delim: %#x {%c}\n", del, del); + /* A flag-char delimiter runs the flag scan off the buffer. */ + if (del == 'P' || del == 'O' || del == 'C' || del == 'F') + goto einval; + /* Pad the buffer with the delim to simplify parsing below. */ memset(buf + count, del, 8); @@ -917,18 +928,9 @@ static const struct file_operations bm_status_operations = { /* Superblock handling */ -static void bm_put_super(struct super_block *sb) -{ - struct user_namespace *user_ns = sb->s_fs_info; - - sb->s_fs_info = NULL; - put_user_ns(user_ns); -} - static const struct super_operations s_ops = { .statfs = simple_statfs, .evict_inode = bm_evict_inode, - .put_super = bm_put_super, }; static int bm_fill_super(struct super_block *sb, struct fs_context *fc) @@ -942,9 +944,14 @@ static int bm_fill_super(struct super_block *sb, struct fs_context *fc) /* last one */ {""} }; - if (WARN_ON(user_ns != current_user_ns())) + /* The fscontext fd may have been passed to another user namespace. */ + if (user_ns != current_user_ns()) return -EINVAL; + /* Never exec off this instance and never let anything stack on it. */ + sb->s_iflags |= SB_I_NOEXEC | SB_I_NODEV; + sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH; + /* * Lazily allocate a new binfmt_misc instance for this namespace, i.e. * do it here during the first mount of binfmt_misc. We don't need to @@ -982,13 +989,12 @@ static int bm_fill_super(struct super_block *sb, struct fs_context *fc) /* * When the binfmt_misc superblock for this userns is shutdown * ->enabled might have been set to false and we don't reinitialize - * ->enabled again in put_super() as someone might already be mounting - * binfmt_misc again. It also would be pointless since by the time - * ->put_super() is called we know that the binary type list for this - * bintfmt_misc mount is empty making load_misc_binary() return - * -ENOEXEC independent of whether ->enabled is true. Instead, if - * someone mounts binfmt_misc for the first time or again we simply - * reset ->enabled to true. + * ->enabled again during shutdown as someone might already be mounting + * binfmt_misc again. It also would be pointless since by then we know + * that the binary type list for this binfmt_misc mount is empty making + * load_misc_binary() return -ENOEXEC independent of whether ->enabled + * is true. Instead, if someone mounts binfmt_misc for the first time or + * again we simply reset ->enabled to true. */ misc->enabled = true; @@ -1014,6 +1020,14 @@ static const struct fs_context_operations bm_context_ops = { .get_tree = bm_get_tree, }; +static void bm_kill_sb(struct super_block *sb) +{ + struct user_namespace *user_ns = sb->s_fs_info; + + kill_litter_super(sb); + put_user_ns(user_ns); +} + static int bm_init_fs_context(struct fs_context *fc) { fc->ops = &bm_context_ops; @@ -1030,7 +1044,7 @@ static struct file_system_type bm_fs_type = { .name = "binfmt_misc", .init_fs_context = bm_init_fs_context, .fs_flags = FS_USERNS_MOUNT, - .kill_sb = kill_litter_super, + .kill_sb = bm_kill_sb, }; MODULE_ALIAS_FS("binfmt_misc"); diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index e050d0938dc45..0c47b4570a7bf 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -1408,12 +1408,12 @@ static int find_parent_nodes(struct btrfs_backref_walk_ctx *ctx, if (!path) return -ENOMEM; if (!ctx->trans) { - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; } if (ctx->time_seq == BTRFS_SEQ_LAST) - path->skip_locking = 1; + path->skip_locking = true; again: head = NULL; @@ -1560,7 +1560,7 @@ static int find_parent_nodes(struct btrfs_backref_walk_ctx *ctx, btrfs_release_path(path); - ret = add_missing_keys(ctx->fs_info, &preftrees, path->skip_locking == 0); + ret = add_missing_keys(ctx->fs_info, &preftrees, !path->skip_locking); if (ret) goto out; @@ -2833,8 +2833,8 @@ struct btrfs_backref_iter *btrfs_backref_iter_alloc(struct btrfs_fs_info *fs_inf } /* Current backref iterator only supports iteration in commit root */ - ret->path->search_commit_root = 1; - ret->path->skip_locking = 1; + ret->path->search_commit_root = true; + ret->path->skip_locking = true; ret->fs_info = fs_info; return ret; @@ -3307,8 +3307,8 @@ static int handle_indirect_tree_backref(struct btrfs_trans_handle *trans, level = cur->level + 1; /* Search the tree to find parent blocks referring to the block */ - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; path->lowest_level = level; ret = btrfs_search_slot(NULL, root, tree_key, path, 0, 0); path->lowest_level = 0; diff --git a/fs/btrfs/bio.c b/fs/btrfs/bio.c index 7d8aaa019c8ce..afe022a3b8fc3 100644 --- a/fs/btrfs/bio.c +++ b/fs/btrfs/bio.c @@ -172,7 +172,21 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, struct btrfs_inode *inode = repair_bbio->inode; struct btrfs_fs_info *fs_info = inode->root->fs_info; struct bio_vec *bv = bio_first_bvec_all(&repair_bbio->bio); + /* + * We can not move forward the saved_iter, as it will be later + * utilized by repair_bbio again. + */ + struct bvec_iter saved_iter = repair_bbio->saved_iter; + const u32 step = min(fs_info->sectorsize, PAGE_SIZE); + const u64 logical = repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT; + const u32 nr_steps = repair_bbio->saved_iter.bi_size / step; int mirror = repair_bbio->mirror_num; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; + phys_addr_t paddr; + unsigned int slot = 0; + + /* Repair bbio should be eaxctly one block sized. */ + ASSERT(repair_bbio->saved_iter.bi_size == fs_info->sectorsize); if (repair_bbio->bio.bi_status || !btrfs_data_csum_ok(repair_bbio, dev, 0, bvec_phys(bv))) { @@ -190,12 +204,17 @@ static void btrfs_end_repair_bio(struct btrfs_bio *repair_bbio, return; } + btrfs_bio_for_each_block(paddr, &repair_bbio->bio, &saved_iter, step) { + ASSERT(slot < nr_steps); + paddrs[slot] = paddr; + slot++; + } + do { mirror = prev_repair_mirror(fbio, mirror); btrfs_repair_io_failure(fs_info, btrfs_ino(inode), repair_bbio->file_offset, fs_info->sectorsize, - repair_bbio->saved_iter.bi_sector << SECTOR_SHIFT, - bvec_phys(bv), mirror); + logical, paddrs, step, mirror); } while (mirror != fbio->bbio->mirror_num); done: @@ -865,18 +884,36 @@ void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num) * * The I/O is issued synchronously to block the repair read completion from * freeing the bio. + * + * @ino: Offending inode number + * @fileoff: File offset inside the inode + * @length: Length of the repair write + * @logical: Logical address of the range + * @paddrs: Physical address array of the content + * @step: Length of for each paddrs + * @mirror_num: Mirror number to write to. Must not be zero */ -int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, - u64 length, u64 logical, phys_addr_t paddr, int mirror_num) +int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, + u32 length, u64 logical, const phys_addr_t paddrs[], + unsigned int step, int mirror_num) { + const u32 nr_steps = DIV_ROUND_UP_POW2(length, step); struct btrfs_io_stripe smap = { 0 }; - struct bio_vec bvec; - struct bio bio; + struct bio *bio = NULL; int ret = 0; ASSERT(!(fs_info->sb->s_flags & SB_RDONLY)); BUG_ON(!mirror_num); + /* Basic alignment checks. */ + ASSERT(IS_ALIGNED(logical, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(length, fs_info->sectorsize)); + ASSERT(IS_ALIGNED(fileoff, fs_info->sectorsize)); + /* Either it's a single data or metadata block. */ + ASSERT(length <= BTRFS_MAX_BLOCKSIZE); + ASSERT(step <= length); + ASSERT(is_power_of_2(step)); + if (btrfs_repair_one_zone(fs_info, logical)) return 0; @@ -896,24 +933,27 @@ int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, goto out_counter_dec; } - bio_init(&bio, smap.dev->bdev, &bvec, 1, REQ_OP_WRITE | REQ_SYNC); - bio.bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; - __bio_add_page(&bio, phys_to_page(paddr), length, offset_in_page(paddr)); - ret = submit_bio_wait(&bio); + bio = bio_alloc(smap.dev->bdev, nr_steps, REQ_OP_WRITE | REQ_SYNC, GFP_NOFS); + bio->bi_iter.bi_sector = smap.physical >> SECTOR_SHIFT; + for (int i = 0; i < nr_steps; i++) { + ret = bio_add_page(bio, phys_to_page(paddrs[i]), step, offset_in_page(paddrs[i])); + /* We should have allocated enough slots to contain all the different pages. */ + ASSERT(ret == step); + } + ret = submit_bio_wait(bio); + bio_put(bio); if (ret) { /* try to remap that extent elsewhere? */ btrfs_dev_stat_inc_and_print(smap.dev, BTRFS_DEV_STAT_WRITE_ERRS); - goto out_bio_uninit; + goto out_counter_dec; } btrfs_info_rl(fs_info, "read error corrected: ino %llu off %llu (dev %s sector %llu)", - ino, start, btrfs_dev_name(smap.dev), + ino, fileoff, btrfs_dev_name(smap.dev), smap.physical >> SECTOR_SHIFT); ret = 0; -out_bio_uninit: - bio_uninit(&bio); out_counter_dec: btrfs_bio_counter_dec(fs_info); return ret; diff --git a/fs/btrfs/bio.h b/fs/btrfs/bio.h index 69fe54f564fcb..fd09b6501c962 100644 --- a/fs/btrfs/bio.h +++ b/fs/btrfs/bio.h @@ -127,7 +127,8 @@ void btrfs_bio_end_io(struct btrfs_bio *bbio, blk_status_t status); void btrfs_submit_bbio(struct btrfs_bio *bbio, int mirror_num); void btrfs_submit_repair_write(struct btrfs_bio *bbio, int mirror_num, bool dev_replace); -int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 start, - u64 length, u64 logical, phys_addr_t paddr, int mirror_num); +int btrfs_repair_io_failure(struct btrfs_fs_info *fs_info, u64 ino, u64 fileoff, + u32 length, u64 logical, const phys_addr_t paddrs[], + unsigned int step, int mirror_num); #endif diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 1e57f7d04c47e..e9ee10bee2ece 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -613,8 +613,8 @@ static int sample_block_group_extent_item(struct btrfs_caching_control *caching_ extent_root = btrfs_extent_root(fs_info, max_t(u64, block_group->start, BTRFS_SUPER_INFO_OFFSET)); - path->skip_locking = 1; - path->search_commit_root = 1; + path->skip_locking = true; + path->search_commit_root = true; path->reada = READA_FORWARD; search_offset = index * div_u64(block_group->length, max_index); @@ -744,8 +744,8 @@ static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl) * root to add free space. So we skip locking and search the commit * root, since its read-only */ - path->skip_locking = 1; - path->search_commit_root = 1; + path->skip_locking = true; + path->search_commit_root = true; path->reada = READA_FORWARD; key.objectid = last; @@ -1057,8 +1057,7 @@ static int remove_block_group_item(struct btrfs_trans_handle *trans, if (ret < 0) return ret; - ret = btrfs_del_item(trans, root, path); - return ret; + return btrfs_del_item(trans, root, path); } int btrfs_remove_block_group(struct btrfs_trans_handle *trans, @@ -3073,7 +3072,7 @@ int btrfs_inc_block_group_ro(struct btrfs_block_group *cache, * We have allocated a new chunk. We also need to activate that chunk to * grant metadata tickets for zoned filesystem. */ - ret = btrfs_zoned_activate_one_bg(fs_info, space_info, true); + ret = btrfs_zoned_activate_one_bg(space_info, true); if (ret < 0) goto out; @@ -4350,25 +4349,29 @@ static void reserve_chunk_space(struct btrfs_trans_handle *trans, if (IS_ERR(bg)) { ret = PTR_ERR(bg); } else { + int activate_ret; + /* * We have a new chunk. We also need to activate it for * zoned filesystem. */ - ret = btrfs_zoned_activate_one_bg(fs_info, info, true); - if (ret < 0) - return; - - /* - * If we fail to add the chunk item here, we end up - * trying again at phase 2 of chunk allocation, at - * btrfs_create_pending_block_groups(). So ignore - * any error here. An ENOSPC here could happen, due to - * the cases described at do_chunk_alloc() - the system - * block group we just created was just turned into RO - * mode by a scrub for example, or a running discard - * temporarily removed its free space entries, etc. - */ - btrfs_chunk_alloc_add_chunk_item(trans, bg); + activate_ret = btrfs_zoned_activate_one_bg(info, true); + if (activate_ret < 0) { + ret = activate_ret; + } else { + /* + * If we fail to add the chunk item here, we end + * up trying again at phase 2 of chunk allocation, + * at btrfs_create_pending_block_groups(). So + * ignore any error here. An ENOSPC here could + * happen, due to the cases described at + * do_chunk_alloc() - the system block group we + * just created was just turned into RO mode by a + * scrub for example, or a running discard + * temporarily removed its free space entries, etc. + */ + btrfs_chunk_alloc_add_chunk_item(trans, bg); + } } } diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index 27e2adc2ee717..c7c2e4405c476 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -1709,9 +1709,9 @@ static struct extent_buffer *btrfs_search_slot_get_root(struct btrfs_root *root, level = btrfs_header_level(b); /* * Ensure that all callers have set skip_locking when - * p->search_commit_root = 1. + * p->search_commit_root is true. */ - ASSERT(p->skip_locking == 1); + ASSERT(p->skip_locking); goto out; } @@ -3858,10 +3858,10 @@ static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, } btrfs_release_path(path); - path->keep_locks = 1; - path->search_for_split = 1; + path->keep_locks = true; + path->search_for_split = true; ret = btrfs_search_slot(trans, root, &key, path, 0, 1); - path->search_for_split = 0; + path->search_for_split = false; if (ret > 0) ret = -EAGAIN; if (ret < 0) @@ -3888,11 +3888,11 @@ static noinline int setup_leaf_for_split(struct btrfs_trans_handle *trans, if (ret) goto err; - path->keep_locks = 0; + path->keep_locks = false; btrfs_unlock_up_safe(path, 1); return 0; err: - path->keep_locks = 0; + path->keep_locks = false; return ret; } @@ -3990,8 +3990,7 @@ int btrfs_split_item(struct btrfs_trans_handle *trans, if (ret) return ret; - ret = split_item(trans, path, new_key, split_offset); - return ret; + return split_item(trans, path, new_key, split_offset); } /* @@ -4610,11 +4609,11 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, u32 nritems; int level; int ret = 1; - int keep_locks = path->keep_locks; + const bool keep_locks = path->keep_locks; ASSERT(!path->nowait); ASSERT(path->lowest_level == 0); - path->keep_locks = 1; + path->keep_locks = true; again: cur = btrfs_read_lock_root_node(root); level = btrfs_header_level(cur); @@ -4704,7 +4703,7 @@ int btrfs_search_forward(struct btrfs_root *root, struct btrfs_key *min_key, * 0 is returned if another key is found, < 0 if there are any errors * and 1 is returned if there are no higher keys in the tree * - * path->keep_locks should be set to 1 on the search made before + * path->keep_locks should be set to true on the search made before * calling this function. */ int btrfs_find_next_key(struct btrfs_root *root, struct btrfs_path *path, @@ -4803,13 +4802,13 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, next = NULL; btrfs_release_path(path); - path->keep_locks = 1; + path->keep_locks = true; if (time_seq) { ret = btrfs_search_old_slot(root, &key, path, time_seq); } else { if (path->need_commit_sem) { - path->need_commit_sem = 0; + path->need_commit_sem = false; need_commit_sem = true; if (path->nowait) { if (!down_read_trylock(&fs_info->commit_root_sem)) { @@ -4822,7 +4821,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, } ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); } - path->keep_locks = 0; + path->keep_locks = false; if (ret < 0) goto done; @@ -4961,7 +4960,7 @@ int btrfs_next_old_leaf(struct btrfs_root *root, struct btrfs_path *path, if (need_commit_sem) { int ret2; - path->need_commit_sem = 1; + path->need_commit_sem = true; ret2 = finish_need_commit_sem_search(path); up_read(&fs_info->commit_root_sem); if (ret2) diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 16dd11c485313..692370fc07b28 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -65,21 +65,21 @@ struct btrfs_path { * set by btrfs_split_item, tells search_slot to keep all locks * and to force calls to keep space in the nodes */ - unsigned int search_for_split:1; + bool search_for_split:1; /* Keep some upper locks as we walk down. */ - unsigned int keep_locks:1; - unsigned int skip_locking:1; - unsigned int search_commit_root:1; - unsigned int need_commit_sem:1; - unsigned int skip_release_on_error:1; + bool keep_locks:1; + bool skip_locking:1; + bool search_commit_root:1; + bool need_commit_sem:1; + bool skip_release_on_error:1; /* * Indicate that new item (btrfs_search_slot) is extending already * existing item and ins_len contains only the data size and not item * header (ie. sizeof(struct btrfs_item) is not included). */ - unsigned int search_for_extension:1; + bool search_for_extension:1; /* Stop search if any locks need to be taken (for read) */ - unsigned int nowait:1; + bool nowait:1; }; #define BTRFS_PATH_AUTO_FREE(path_name) \ diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c index a4cc1bc635622..c59548a0b9c53 100644 --- a/fs/btrfs/defrag.c +++ b/fs/btrfs/defrag.c @@ -472,7 +472,7 @@ static int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, memcpy(&key, &root->defrag_progress, sizeof(key)); } - path->keep_locks = 1; + path->keep_locks = true; ret = btrfs_search_forward(root, &key, path, BTRFS_OLDEST_GENERATION); if (ret < 0) @@ -515,7 +515,7 @@ static int btrfs_defrag_leaves(struct btrfs_trans_handle *trans, /* * Now that we reallocated the node we can find the next key. Note that * btrfs_find_next_key() can release our path and do another search - * without COWing, this is because even with path->keep_locks = 1, + * without COWing, this is because even with path->keep_locks == true, * btrfs_search_slot() / ctree.c:unlock_up() does not keeps a lock on a * node when path->slots[node_level - 1] does not point to the last * item or a slot beyond the last item (ctree.c:unlock_up()). Therefore @@ -1150,20 +1150,15 @@ static_assert(PAGE_ALIGNED(CLUSTER_SIZE)); * * - Extent bits are locked */ -static int defrag_one_locked_target(struct btrfs_inode *inode, - struct defrag_target_range *target, - struct folio **folios, int nr_pages, - struct extent_state **cached_state) +static void defrag_one_locked_target(struct btrfs_inode *inode, + struct defrag_target_range *target, + struct folio **folios, int nr_pages, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_changeset *data_reserved = NULL; const u64 start = target->start; const u64 len = target->len; - int ret = 0; - ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); - if (ret < 0) - return ret; btrfs_clear_extent_bit(&inode->io_tree, start, start + len - 1, EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, cached_state); @@ -1184,10 +1179,6 @@ static int defrag_one_locked_target(struct btrfs_inode *inode, btrfs_folio_clamp_clear_checked(fs_info, folio, start, len); btrfs_folio_clamp_set_dirty(fs_info, folio, start, len); } - btrfs_delalloc_release_extents(inode, len); - extent_changeset_free(data_reserved); - - return ret; } static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, @@ -1203,6 +1194,8 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, u64 cur = start; const unsigned int nr_pages = ((start + len - 1) >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1; + struct extent_changeset *data_reserved = NULL; + u64 last_defrag_end = start; int ret = 0; ASSERT(nr_pages <= CLUSTER_SIZE / PAGE_SIZE); @@ -1212,6 +1205,22 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, if (!folios) return -ENOMEM; + /* + * Reserve delalloc space before locking the range and before locking + * and dirtying any folios - otherwise we could deadlock, for example + * after defrag of one range we dirty folios and keep them locked when + * we move to the next range, so reserving delalloc space right before + * each range could trigger flushing of delalloc and deadlock on the + * extent lock or trigger a transaction commit with flushoncommit, which + * can either deadlock on the lock of a folio made dirty in the previous + * range or the extent lock. + */ + ret = btrfs_delalloc_reserve_space(inode, &data_reserved, start, len); + if (ret < 0) { + kfree(folios); + return ret; + } + /* Prepare all pages */ for (int i = 0; cur < start + len && i < nr_pages; i++) { folios[i] = defrag_prepare_one_folio(inode, cur >> PAGE_SHIFT); @@ -1246,10 +1255,11 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, goto unlock_extent; list_for_each_entry(entry, &target_list, list) { - ret = defrag_one_locked_target(inode, entry, folios, nr_pages, - &cached_state); - if (ret < 0) - break; + defrag_one_locked_target(inode, entry, folios, nr_pages, &cached_state); + if (entry->start > last_defrag_end) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + entry->start - last_defrag_end, true); + last_defrag_end = entry->start + entry->len; } list_for_each_entry_safe(entry, tmp, &target_list, list) { @@ -1266,6 +1276,12 @@ static int defrag_one_range(struct btrfs_inode *inode, u64 start, u32 len, folio_put(folios[i]); } kfree(folios); + btrfs_delalloc_release_extents(inode, len); + if (last_defrag_end < start + len) + btrfs_delalloc_release_space(inode, data_reserved, last_defrag_end, + start + len - last_defrag_end, true); + extent_changeset_free(data_reserved); + return ret; } diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index ea48706a3d810..e1d1f71153fcd 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -1140,8 +1140,8 @@ __btrfs_commit_inode_delayed_items(struct btrfs_trans_handle *trans, ret = btrfs_record_root_in_trans(trans, node->root); if (ret) return ret; - ret = btrfs_update_delayed_inode(trans, node->root, path, node); - return ret; + + return btrfs_update_delayed_inode(trans, node->root, path, node); } /* diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index a4eaef60549ee..b6c7da8e1bc8b 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -489,8 +489,8 @@ static int mark_block_group_to_copy(struct btrfs_fs_info *fs_info, } path->reada = READA_FORWARD; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; key.objectid = src_dev->devid; key.type = BTRFS_DEV_EXTENT_KEY; diff --git a/fs/btrfs/direct-io.c b/fs/btrfs/direct-io.c index 3091132190b6b..5f2a99e683353 100644 --- a/fs/btrfs/direct-io.c +++ b/fs/btrfs/direct-io.c @@ -281,17 +281,24 @@ static int btrfs_get_blocks_direct_write(struct extent_map **map, em2 = btrfs_create_dio_extent(BTRFS_I(inode), dio_data, start, &file_extent, type); btrfs_dec_nocow_writers(bg); - if (type == BTRFS_ORDERED_PREALLOC) { - btrfs_free_extent_map(em); - *map = em2; - em = em2; - } - if (IS_ERR(em2)) { ret = PTR_ERR(em2); + btrfs_free_extent_map(em); + *map = NULL; goto out; } + /* + * True NOCOW writes don't need to create a new extent map, + * while PREALLOC writes must replace the existing one. + */ + if (em2) { + ASSERT(type == BTRFS_ORDERED_PREALLOC); + btrfs_free_extent_map(em); + *map = em2; + em = em2; + } + dio_data->nocow_done = true; } else { /* Our caller expects us to free the input extent map. */ diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 655eed981078b..8863479a19d3e 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -183,27 +183,32 @@ static int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num) { struct btrfs_fs_info *fs_info = eb->fs_info; - int ret = 0; + const u32 step = min(fs_info->nodesize, PAGE_SIZE); + const u32 nr_steps = eb->len / step; + phys_addr_t paddrs[BTRFS_MAX_BLOCKSIZE / PAGE_SIZE]; if (sb_rdonly(fs_info->sb)) return -EROFS; - for (int i = 0; i < num_extent_folios(eb); i++) { + for (int i = 0; i < num_extent_pages(eb); i++) { struct folio *folio = eb->folios[i]; - u64 start = max_t(u64, eb->start, folio_pos(folio)); - u64 end = min_t(u64, eb->start + eb->len, - folio_pos(folio) + eb->folio_size); - u32 len = end - start; - phys_addr_t paddr = PFN_PHYS(folio_pfn(folio)) + - offset_in_folio(folio, start); - - ret = btrfs_repair_io_failure(fs_info, 0, start, len, start, - paddr, mirror_num); - if (ret) - break; + + /* No large folio support yet. */ + ASSERT(folio_order(folio) == 0); + ASSERT(i < nr_steps); + + /* + * For nodesize < page size, there is just one paddr, with some + * offset inside the page. + * + * For nodesize >= page size, it's one or more paddrs, and eb->start + * must be aligned to page boundary. + */ + paddrs[i] = page_to_phys(&folio->page) + offset_in_page(eb->start); } - return ret; + return btrfs_repair_io_failure(fs_info, 0, eb->start, eb->len, + eb->start, paddrs, step, mirror_num); } /* @@ -273,14 +278,15 @@ int btree_csum_one_bio(struct btrfs_bio *bbio) return -EIO; /* - * If an extent_buffer is marked as EXTENT_BUFFER_ZONED_ZEROOUT, don't - * checksum it but zero-out its content. This is done to preserve - * ordering of I/O without unnecessarily writing out data. + * An extent_buffer marked EXTENT_BUFFER_ZONED_ZEROOUT is written out as + * zeros to preserve ordering of I/O without persisting the now + * unnecessary block. The bio is fed from the shared zero page (see + * write_one_eb()), so there is nothing to checksum here. Crucially, the + * buffer's own content is left intact: it may still be referenced, e.g. + * btrfs_free_tree_block() reads its header to add a delayed reference. */ - if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { - memzero_extent_buffer(eb, 0, eb->len); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) return 0; - } if (WARN_ON_ONCE(found_start != eb->start)) return -EIO; @@ -2183,11 +2189,10 @@ static int load_global_roots(struct btrfs_root *tree_root) return ret; if (!btrfs_fs_compat_ro(tree_root->fs_info, FREE_SPACE_TREE)) return ret; - ret = load_global_roots_objectid(tree_root, path, - BTRFS_FREE_SPACE_TREE_OBJECTID, - "free space"); - return ret; + return load_global_roots_objectid(tree_root, path, + BTRFS_FREE_SPACE_TREE_OBJECTID, + "free space"); } static int btrfs_read_roots(struct btrfs_fs_info *fs_info) @@ -3425,7 +3430,16 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device /* Update the values for the current filesystem. */ sb->s_blocksize = sectorsize; sb->s_blocksize_bits = blksize_bits(sectorsize); - memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); + /* + * When temp_fsid is active, fs_devices->fsid is assigned a random UUID + * at mount. This inconsistent UUID causes issues for layered filesystems + * like OverlayFS. Since metadata_uuid may or may not be set, provide the + * on-disk UUID directly from the super_copy. + */ + if (fs_info->fs_devices->temp_fsid) + memcpy(&sb->s_uuid, fs_info->super_copy->fsid, BTRFS_FSID_SIZE); + else + memcpy(&sb->s_uuid, fs_info->fs_devices->fsid, BTRFS_FSID_SIZE); mutex_lock(&fs_info->chunk_mutex); ret = btrfs_read_sys_array(fs_info); diff --git a/fs/btrfs/extent-io-tree.c b/fs/btrfs/extent-io-tree.c index bb2ca1c9c7b02..2a2bce0f1f7c8 100644 --- a/fs/btrfs/extent-io-tree.c +++ b/fs/btrfs/extent-io-tree.c @@ -187,18 +187,18 @@ static int add_extent_changeset(struct extent_state *state, u32 bits, struct extent_changeset *changeset, int set) { - int ret; - if (!changeset) return 0; if (set && (state->state & bits) == bits) return 0; if (!set && (state->state & bits) == 0) return 0; + changeset->bytes_changed += state->end - state->start + 1; - ret = ulist_add(&changeset->range_changed, state->start, state->end, - GFP_ATOMIC); - return ret; + if (!extent_changeset_tracks_ranges(changeset)) + return 0; + + return ulist_add(&changeset->range_changed, state->start, state->end, GFP_ATOMIC); } static inline struct extent_state *next_state(struct extent_state *state) diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 6ef7cc116bfcc..7c199d59a5465 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -788,7 +788,7 @@ int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, want = extent_ref_type(parent, owner); if (insert) { extra_size = btrfs_extent_inline_ref_size(want); - path->search_for_extension = 1; + path->search_for_extension = true; } else extra_size = -1; @@ -954,7 +954,7 @@ int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, if (!path->keep_locks) { btrfs_release_path(path); - path->keep_locks = 1; + path->keep_locks = true; goto again; } @@ -975,11 +975,11 @@ int lookup_inline_extent_backref(struct btrfs_trans_handle *trans, *ref_ret = (struct btrfs_extent_inline_ref *)ptr; out: if (path->keep_locks) { - path->keep_locks = 0; + path->keep_locks = false; btrfs_unlock_up_safe(path, 1); } if (insert) - path->search_for_extension = 0; + path->search_for_extension = false; return ret; } @@ -2557,7 +2557,6 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data) { struct btrfs_fs_info *fs_info = root->fs_info; u64 flags; - u64 ret; if (data) flags = BTRFS_BLOCK_GROUP_DATA; @@ -2566,8 +2565,7 @@ static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data) else flags = BTRFS_BLOCK_GROUP_METADATA; - ret = btrfs_get_alloc_profile(fs_info, flags); - return ret; + return btrfs_get_alloc_profile(fs_info, flags); } static u64 first_logical_byte(struct btrfs_fs_info *fs_info) @@ -4192,10 +4190,8 @@ static int find_free_extent_update_loop(struct btrfs_fs_info *fs_info, else trans = btrfs_join_transaction(root); - if (IS_ERR(trans)) { - ret = PTR_ERR(trans); - return ret; - } + if (IS_ERR(trans)) + return PTR_ERR(trans); ret = btrfs_chunk_alloc(trans, space_info, ffe_ctl->flags, CHUNK_ALLOC_FORCE_FOR_EXTENT); diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index f955b4003d424..90a6b161d7c11 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -1949,7 +1949,7 @@ static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *e btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN); percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, - -eb->len, + -(s64)eb->len, fs_info->dirty_metadata_batch); ret = true; } else { @@ -2182,14 +2182,17 @@ static struct extent_buffer *find_extent_buffer_nolock( static void end_bbio_meta_write(struct btrfs_bio *bbio) { struct extent_buffer *eb = bbio->private; - struct folio_iter fi; if (bbio->bio.bi_status != BLK_STS_OK) set_btree_ioerr(eb); - bio_for_each_folio_all(fi, &bbio->bio) { - btrfs_meta_folio_clear_writeback(fi.folio, eb); - } + /* + * Clear writeback on the buffer's own folios. The bio may carry the + * shared zero page instead (EXTENT_BUFFER_ZONED_ZEROOUT), so iterate + * the extent buffer folios rather than the bio folios. + */ + for (int i = 0; i < num_extent_folios(eb); i++) + btrfs_meta_folio_clear_writeback(eb->folios[i], eb); buffer_tree_clear_mark(eb, PAGECACHE_TAG_WRITEBACK); clear_and_wake_up_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags); @@ -2230,7 +2233,8 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, struct btrfs_fs_info *fs_info = eb->fs_info; struct btrfs_bio *bbio; - prepare_eb_write(eb); + if (!test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) + prepare_eb_write(eb); bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES, REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc), @@ -2250,8 +2254,21 @@ static noinline_for_stack void write_one_eb(struct extent_buffer *eb, btrfs_meta_folio_set_writeback(folio, eb); if (!folio_test_dirty(folio)) wbc->nr_to_write -= folio_nr_pages(folio); - bio_add_folio_nofail(&bbio->bio, folio, range_len, - offset_in_folio(folio, range_start)); + if (test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags)) { + u32 off = 0; + + while (off < range_len) { + u32 add = min_t(u32, PAGE_SIZE, range_len - off); + + bio_add_folio_nofail(&bbio->bio, + page_folio(ZERO_PAGE(0)), + add, 0); + off += add; + } + } else { + bio_add_folio_nofail(&bbio->bio, folio, range_len, + offset_in_folio(folio, range_start)); + } wbc_account_cgroup_owner(wbc, folio, range_len); folio_unlock(folio); } @@ -3731,7 +3748,7 @@ void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans, return; buffer_tree_clear_mark(eb, PAGECACHE_TAG_DIRTY); - percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len, + percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -(s64)eb->len, fs_info->dirty_metadata_batch); for (int i = 0; i < num_extent_folios(eb); i++) { diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 73571d5d3d5ad..65d1ed3fcd5bf 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -196,6 +196,25 @@ static inline void extent_changeset_init(struct extent_changeset *changeset) ulist_init(&changeset->range_changed); } +/* + * Sentinel value for range_changed.prealloc indicating that the changeset + * only tracks bytes_changed and does not record individual ranges. This + * avoids GFP_ATOMIC allocations inside add_extent_changeset() when the + * caller doesn't need to iterate the changed ranges afterwards. + */ +#define EXTENT_CHANGESET_BYTES_ONLY ((struct ulist_node *)1) + +static inline void extent_changeset_init_bytes_only(struct extent_changeset *changeset) +{ + changeset->bytes_changed = 0; + changeset->range_changed.prealloc = EXTENT_CHANGESET_BYTES_ONLY; +} + +static inline bool extent_changeset_tracks_ranges(const struct extent_changeset *changeset) +{ + return changeset->range_changed.prealloc != EXTENT_CHANGESET_BYTES_ONLY; +} + static inline struct extent_changeset *extent_changeset_alloc(void) { struct extent_changeset *ret; @@ -210,6 +229,7 @@ static inline struct extent_changeset *extent_changeset_alloc(void) static inline void extent_changeset_prealloc(struct extent_changeset *changeset, gfp_t gfp_mask) { + ASSERT(extent_changeset_tracks_ranges(changeset)); ulist_prealloc(&changeset->range_changed, gfp_mask); } @@ -218,7 +238,8 @@ static inline void extent_changeset_release(struct extent_changeset *changeset) if (!changeset) return; changeset->bytes_changed = 0; - ulist_release(&changeset->range_changed); + if (extent_changeset_tracks_ranges(changeset)) + ulist_release(&changeset->range_changed); } static inline void extent_changeset_free(struct extent_changeset *changeset) diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c index 7e38c23a0c1cb..21eb104ee2aef 100644 --- a/fs/btrfs/extent_map.c +++ b/fs/btrfs/extent_map.c @@ -854,13 +854,13 @@ void btrfs_drop_extent_map_range(struct btrfs_inode *inode, u64 start, u64 end, goto next; } - flags = em->flags; /* * In case we split the extent map, we want to preserve the * EXTENT_FLAG_LOGGING flag on our extent map, but we don't want * it on the new extent maps. */ - em->flags &= ~(EXTENT_FLAG_PINNED | EXTENT_FLAG_LOGGING); + flags = em->flags & ~EXTENT_FLAG_LOGGING; + em->flags &= ~EXTENT_FLAG_PINNED; modified = !list_empty(&em->list); /* diff --git a/fs/btrfs/fiemap.c b/fs/btrfs/fiemap.c index f2eaaef8422bf..30a0f66c9c9ef 100644 --- a/fs/btrfs/fiemap.c +++ b/fs/btrfs/fiemap.c @@ -661,7 +661,7 @@ static int extent_fiemap(struct btrfs_inode *inode, range_end = round_up(start + len, sectorsize); prev_extent_end = range_start; - btrfs_lock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_lock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); ret = fiemap_find_last_extent_offset(inode, path, &last_extent_end); if (ret < 0) @@ -841,7 +841,7 @@ static int extent_fiemap(struct btrfs_inode *inode, } out_unlock: - btrfs_unlock_extent(&inode->io_tree, range_start, range_end, &cached_state); + btrfs_unlock_extent(&inode->io_tree, range_start, range_end - 1, &cached_state); if (ret == BTRFS_FIEMAP_FLUSH_CACHE) { btrfs_release_path(path); diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 4b7c40f05e8f9..9b19b33e85b68 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -350,6 +350,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) const unsigned int nblocks = orig_len >> fs_info->sectorsize_bits; int ret = 0; u32 bio_offset = 0; + bool using_commit_root = false; if ((inode->flags & BTRFS_INODE_NODATASUM) || test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) @@ -394,8 +395,8 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) * between reading the free space cache and updating the csum tree. */ if (btrfs_is_free_space_inode(inode)) { - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; } /* @@ -423,8 +424,9 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) * from across transactions. */ if (bbio->csum_search_commit_root) { - path->search_commit_root = 1; - path->skip_locking = 1; + using_commit_root = true; + path->search_commit_root = true; + path->skip_locking = true; down_read(&fs_info->commit_root_sem); } @@ -455,6 +457,28 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) * assume this is the case. */ if (count == 0) { + /* + * If an extent is relocated in the current transaction + * then relocation writes a new csum without updating + * the extent map generation. Until the next commit, we + * will see a hole in that case, so we need to fallback + * to searching the transaction csum root. + * + * Note that a commit root lookup of a referenced extent can + * only miss, not return a stale csum. A freed extent's csum + * is deleted in the same transaction and its bytenr is not + * reusable until that transaction has committed and the + * extent is unpinned. + */ + if (using_commit_root) { + up_read(&fs_info->commit_root_sem); + using_commit_root = false; + path->search_commit_root = false; + path->skip_locking = false; + btrfs_release_path(path); + continue; + } + memset(csum_dst, 0, csum_size); count = 1; @@ -473,7 +497,7 @@ int btrfs_lookup_bio_sums(struct btrfs_bio *bbio) bio_offset += count * sectorsize; } - if (bbio->csum_search_commit_root) + if (using_commit_root) up_read(&fs_info->commit_root_sem); return ret; } @@ -1168,10 +1192,10 @@ int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, } btrfs_release_path(path); - path->search_for_extension = 1; + path->search_for_extension = true; ret = btrfs_search_slot(trans, root, &file_key, path, csum_size, 1); - path->search_for_extension = 0; + path->search_for_extension = false; if (ret < 0) goto out; diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 66c76f425afb5..a15062f976ba3 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -881,62 +881,56 @@ static noinline int prepare_one_folio(struct inode *inode, struct folio **folio_ /* * Locks the extent and properly waits for data=ordered extents to finish - * before allowing the folios to be modified if need. + * before allowing the folios to be modified. * * Return: - * 1 - the extent is locked - * 0 - the extent is not locked, and everything is OK + * 0 - the extent is locked * -EAGAIN - need to prepare the folios again */ static noinline int -lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, - loff_t pos, size_t write_bytes, - u64 *lockstart, u64 *lockend, bool nowait, - struct extent_state **cached_state) +lock_and_cleanup_extent(struct btrfs_inode *inode, struct folio *folio, + loff_t pos, size_t write_bytes, + u64 *lockstart, u64 *lockend, bool nowait, + struct extent_state **cached_state) { struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_ordered_extent *ordered; u64 start_pos; u64 last_pos; - int ret = 0; start_pos = round_down(pos, fs_info->sectorsize); last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1; - if (start_pos < inode->vfs_inode.i_size) { - struct btrfs_ordered_extent *ordered; - - if (nowait) { - if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, - last_pos, cached_state)) { - folio_unlock(folio); - folio_put(folio); - return -EAGAIN; - } - } else { - btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); - } - - ordered = btrfs_lookup_ordered_range(inode, start_pos, - last_pos - start_pos + 1); - if (ordered && - ordered->file_offset + ordered->num_bytes > start_pos && - ordered->file_offset <= last_pos) { - btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, - cached_state); + if (nowait) { + if (!btrfs_try_lock_extent(&inode->io_tree, start_pos, + last_pos, cached_state)) { folio_unlock(folio); folio_put(folio); - btrfs_start_ordered_extent(ordered); - btrfs_put_ordered_extent(ordered); return -EAGAIN; } - if (ordered) - btrfs_put_ordered_extent(ordered); + } else { + btrfs_lock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + } - *lockstart = start_pos; - *lockend = last_pos; - ret = 1; + ordered = btrfs_lookup_ordered_range(inode, start_pos, + last_pos - start_pos + 1); + if (ordered && + ordered->file_offset + ordered->num_bytes > start_pos && + ordered->file_offset <= last_pos) { + btrfs_unlock_extent(&inode->io_tree, start_pos, last_pos, + cached_state); + folio_unlock(folio); + folio_put(folio); + btrfs_start_ordered_extent(ordered); + btrfs_put_ordered_extent(ordered); + return -EAGAIN; } + if (ordered) + btrfs_put_ordered_extent(ordered); + + *lockstart = start_pos; + *lockend = last_pos; /* * We should be called after prepare_one_folio() which should have locked @@ -944,7 +938,7 @@ lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct folio *folio, */ WARN_ON(!folio_test_locked(folio)); - return ret; + return 0; } /* @@ -1201,7 +1195,6 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, const u64 reserved_start = round_down(start, fs_info->sectorsize); u64 reserved_len; struct folio *folio = NULL; - int extents_locked; u64 lockstart; u64 lockend; bool only_release_metadata = false; @@ -1256,18 +1249,15 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, reserved_len = last_block - reserved_start; } - extents_locked = lock_and_cleanup_extent_if_need(inode, folio, start, - write_bytes, &lockstart, - &lockend, nowait, - &cached_state); - if (extents_locked < 0) { - if (!nowait && extents_locked == -EAGAIN) + ret = lock_and_cleanup_extent(inode, folio, start, write_bytes, + &lockstart, &lockend, nowait, &cached_state); + if (ret < 0) { + if (!nowait) goto again; btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); - ret = extents_locked; return ret; } @@ -1292,11 +1282,8 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, /* No copied bytes, unlock, release reserved space and exit. */ if (copied == 0) { - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, - &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, + &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); release_space(inode, *data_reserved, reserved_start, reserved_len, only_release_metadata); @@ -1315,17 +1302,7 @@ static int copy_one_range(struct btrfs_inode *inode, struct iov_iter *iter, ret = btrfs_dirty_folio(inode, folio, start, copied, &cached_state, only_release_metadata); - /* - * If we have not locked the extent range, because the range's start - * offset is >= i_size, we might still have a non-NULL cached extent - * state, acquired while marking the extent range as delalloc through - * btrfs_dirty_page(). Therefore free any possible cached extent state - * to avoid a memory leak. - */ - if (extents_locked) - btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); - else - btrfs_free_extent_state(cached_state); + btrfs_unlock_extent(&inode->io_tree, lockstart, lockend, &cached_state); btrfs_delalloc_release_extents(inode, reserved_len); if (ret) { diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index ab873bd671920..7ee3d8f05c1a2 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -554,6 +554,9 @@ static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) u32 crc = ~(u32)0; unsigned offset = 0; + if (index >= io_ctl->num_pages) + return -EIO; + if (index == 0) offset = sizeof(u32) * io_ctl->num_pages; @@ -968,8 +971,8 @@ int load_free_space_cache(struct btrfs_block_group *block_group) path = btrfs_alloc_path(); if (!path) return 0; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; /* * We must pass a path with search_commit_root set to btrfs_iget in diff --git a/fs/btrfs/free-space-tree.c b/fs/btrfs/free-space-tree.c index c3734892d6548..fa8fbb19ff650 100644 --- a/fs/btrfs/free-space-tree.c +++ b/fs/btrfs/free-space-tree.c @@ -1533,9 +1533,7 @@ int btrfs_remove_block_group_free_space(struct btrfs_trans_handle *trans, btrfs_release_path(path); } - ret = 0; - - return ret; + return 0; } static int load_free_space_bitmaps(struct btrfs_caching_control *caching_ctl, @@ -1699,8 +1697,8 @@ int btrfs_load_free_space_tree(struct btrfs_caching_control *caching_ctl) * Just like caching_thread() doesn't want to deadlock on the extent * tree, we don't want to deadlock on the free space tree. */ - path->skip_locking = 1; - path->search_commit_root = 1; + path->skip_locking = true; + path->search_commit_root = true; path->reada = READA_FORWARD; info = btrfs_search_free_space_info(NULL, block_group, path, 0); diff --git a/fs/btrfs/inode-item.c b/fs/btrfs/inode-item.c index 7e14e1bbcf389..a864f8c997296 100644 --- a/fs/btrfs/inode-item.c +++ b/fs/btrfs/inode-item.c @@ -312,7 +312,7 @@ int btrfs_insert_inode_ref(struct btrfs_trans_handle *trans, if (!path) return -ENOMEM; - path->skip_release_on_error = 1; + path->skip_release_on_error = true; ret = btrfs_insert_empty_item(trans, root, path, &key, ins_len); if (ret == -EEXIST) { @@ -371,14 +371,13 @@ int btrfs_insert_empty_inode(struct btrfs_trans_handle *trans, struct btrfs_path *path, u64 objectid) { struct btrfs_key key; - int ret; + key.objectid = objectid; key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; - ret = btrfs_insert_empty_item(trans, root, path, &key, - sizeof(struct btrfs_inode_item)); - return ret; + return btrfs_insert_empty_item(trans, root, path, &key, + sizeof(struct btrfs_inode_item)); } int btrfs_lookup_inode(struct btrfs_trans_handle *trans, struct btrfs_root diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 30a34cd905ba4..3023e1b504b96 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -2348,7 +2348,6 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol u64 start, u64 end, struct writeback_control *wbc) { const bool zoned = btrfs_is_zoned(inode->root->fs_info); - int ret; /* * The range must cover part of the @locked_folio, or a return of 1 @@ -2356,10 +2355,8 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol */ ASSERT(!(end <= folio_pos(locked_folio) || start >= folio_end(locked_folio))); - if (should_nocow(inode, start, end)) { - ret = run_delalloc_nocow(inode, locked_folio, start, end); - return ret; - } + if (should_nocow(inode, start, end)) + return run_delalloc_nocow(inode, locked_folio, start, end); if (btrfs_inode_can_compress(inode) && inode_need_compress(inode, start, end) && @@ -2367,11 +2364,9 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct folio *locked_fol return 1; if (zoned) - ret = run_delalloc_cow(inode, locked_folio, start, end, wbc, - true); + return run_delalloc_cow(inode, locked_folio, start, end, wbc, true); else - ret = cow_file_range(inode, locked_folio, start, end, NULL, 0); - return ret; + return cow_file_range(inode, locked_folio, start, end, NULL, 0); } void btrfs_split_delalloc_extent(struct btrfs_inode *inode, @@ -3385,6 +3380,9 @@ int btrfs_finish_one_ordered(struct btrfs_ordered_extent *ordered_extent) */ btrfs_remove_ordered_extent(inode, ordered_extent); + /* Cleanup any remaining biocs attached to the OE. */ + btrfs_cleanup_ordered_bioc_list(ordered_extent); + /* once for us */ btrfs_put_ordered_extent(ordered_extent); /* once for the tree */ @@ -7170,8 +7168,8 @@ struct extent_map *btrfs_get_extent(struct btrfs_inode *inode, * point the commit_root has everything we need. */ if (btrfs_is_free_space_inode(inode)) { - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; } ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0); @@ -8171,7 +8169,8 @@ static int btrfs_getattr(struct mnt_idmap *idmap, stat->result_mask |= STATX_SUBVOL; spin_lock(&BTRFS_I(inode)->lock); - delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes; + delalloc_bytes = S_ISREG(inode->i_mode) ? + BTRFS_I(inode)->new_delalloc_bytes : 0; inode_bytes = inode_get_bytes(inode); spin_unlock(&BTRFS_I(inode)->lock); stat->blocks = (ALIGN(inode_bytes, blocksize) + @@ -10037,6 +10036,7 @@ ssize_t btrfs_do_encoded_write(struct kiocb *iocb, struct iov_iter *from, } kvfree(folios); out: + extent_changeset_free(data_reserved); if (ret >= 0) iocb->ki_pos += encoded->len; return ret; diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 2f1c5f5e2e725..45dbfa8cce964 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -330,14 +330,21 @@ int btrfs_fileattr_set(struct mnt_idmap *idmap, inode_flags |= BTRFS_INODE_NODATACOW; } } else { - /* - * Revert back under same assumptions as above - */ - if (S_ISREG(inode->vfs_inode.i_mode)) { - if (inode->vfs_inode.i_size == 0) - inode_flags &= ~(BTRFS_INODE_NODATACOW | - BTRFS_INODE_NODATASUM); - } else { + /* We can only change NODATACOW for zero-sized regular file. */ + if (S_ISREG(inode->vfs_inode.i_mode) && (inode->vfs_inode.i_size == 0)) { + inode_flags &= ~BTRFS_INODE_NODATACOW; + /* + * There is currently no way to change NODATASUM flag + * through fileattr API. If we unconditionally keep the + * current NODATASUM flag, chattr +C then chattr -C will + * keep the NODATASUM flag, and no way to remove that + * flag. + * + * So respect the current mount option for NODATASUM flag. + */ + if (!btrfs_test_opt(fs_info, NODATASUM)) + inode_flags &= ~BTRFS_INODE_NODATASUM; + } else if (!S_ISREG(inode->vfs_inode.i_mode)) { inode_flags &= ~BTRFS_INODE_NODATACOW; } } @@ -2046,7 +2053,6 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) struct btrfs_root_ref *rref; struct extent_buffer *leaf; unsigned long item_off; - unsigned long item_len; int slot; int ret = 0; @@ -2121,17 +2127,17 @@ static int btrfs_ioctl_get_subvol_info(struct inode *inode, void __user *argp) btrfs_item_key_to_cpu(leaf, &key, slot); if (key.objectid == subvol_info->treeid && key.type == BTRFS_ROOT_BACKREF_KEY) { + u16 name_len; + subvol_info->parent_id = key.offset; rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); subvol_info->dirid = btrfs_root_ref_dirid(leaf, rref); - item_off = btrfs_item_ptr_offset(leaf, slot) - + sizeof(struct btrfs_root_ref); - item_len = btrfs_item_size(leaf, slot) - - sizeof(struct btrfs_root_ref); + item_off = btrfs_item_ptr_offset(leaf, slot) + sizeof(*rref); read_extent_buffer(leaf, subvol_info->name, - item_off, item_len); + item_off, name_len); } else { ret = -ENOENT; goto out; diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 261aa65019207..f654639334fcd 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -694,7 +694,6 @@ void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info) static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src, u64 dst) { - int ret; struct btrfs_root *quota_root = trans->fs_info->quota_root; BTRFS_PATH_AUTO_FREE(path); struct btrfs_key key; @@ -707,8 +706,7 @@ static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src, key.type = BTRFS_QGROUP_RELATION_KEY; key.offset = dst; - ret = btrfs_insert_empty_item(trans, quota_root, path, &key, 0); - return ret; + return btrfs_insert_empty_item(trans, quota_root, path, &key, 0); } static int del_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src, @@ -833,9 +831,7 @@ static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid) if (ret > 0) return -ENOENT; - ret = btrfs_del_item(trans, quota_root, path); - - return ret; + return btrfs_del_item(trans, quota_root, path); } static int update_qgroup_limit_item(struct btrfs_trans_handle *trans, @@ -2682,10 +2678,8 @@ int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans, return ret; } - if (root_level == 0) { - ret = btrfs_qgroup_trace_leaf_items(trans, root_eb); - return ret; - } + if (root_level == 0) + return btrfs_qgroup_trace_leaf_items(trans, root_eb); path = btrfs_alloc_path(); if (!path) @@ -3882,8 +3876,8 @@ static void btrfs_qgroup_rescan_worker(struct btrfs_work *work) * Rescan should only search for commit root, and any later difference * should be recorded by qgroup */ - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; while (!ret && !(stopped = rescan_should_stop(fs_info))) { trans = btrfs_start_transaction(fs_info->fs_root, 0); @@ -4350,12 +4344,13 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, struct ulist_node *unode; struct ulist_iterator uiter; struct extent_changeset changeset; + const u32 sectorsize = root->fs_info->sectorsize; + const u64 aligned_start = round_down(start, sectorsize); + const u64 aligned_len = round_up(start + len, sectorsize) - aligned_start; u64 freed = 0; int ret; - extent_changeset_init(&changeset); - len = round_up(start + len, root->fs_info->sectorsize); - start = round_down(start, root->fs_info->sectorsize); + extent_changeset_init_bytes_only(&changeset); ULIST_ITER_INIT(&uiter); while ((unode = ulist_next(&reserved->range_changed, &uiter))) { @@ -4367,12 +4362,15 @@ static int qgroup_free_reserved_data(struct btrfs_inode *inode, extent_changeset_release(&changeset); - /* Only free range in range [start, start + len) */ - if (range_start >= start + len || - range_start + range_len <= start) + /* + * Only free the range within + * [aligned_start, aligned_start + aligned_len). + */ + if (range_start >= aligned_start + aligned_len || + range_start + range_len <= aligned_start) continue; - free_start = max(range_start, start); - free_len = min(start + len, range_start + range_len) - + free_start = max(range_start, aligned_start); + free_len = min(aligned_start + aligned_len, range_start + range_len) - free_start; /* * TODO: To also modify reserved->ranges_reserved to reflect @@ -4418,7 +4416,7 @@ static int __btrfs_qgroup_release_data(struct btrfs_inode *inode, WARN_ON(!free && reserved); if (free && reserved) return qgroup_free_reserved_data(inode, reserved, start, len, released); - extent_changeset_init(&changeset); + extent_changeset_init_bytes_only(&changeset); ret = btrfs_clear_record_extent_bits(&inode->io_tree, start, start + len - 1, EXTENT_QGROUP_RESERVED, &changeset); if (ret < 0) @@ -4673,6 +4671,7 @@ void btrfs_qgroup_check_reserved_leak(struct btrfs_inode *inode) WARN_ON(ret < 0); if (WARN_ON(changeset.bytes_changed)) { + ASSERT(extent_changeset_tracks_ranges(&changeset)); ULIST_ITER_INIT(&iter); while ((unode = ulist_next(&changeset.range_changed, &iter))) { btrfs_warn(inode->root->fs_info, diff --git a/fs/btrfs/raid-stripe-tree.c b/fs/btrfs/raid-stripe-tree.c index cc6f6095cc9fd..d31b292600473 100644 --- a/fs/btrfs/raid-stripe-tree.c +++ b/fs/btrfs/raid-stripe-tree.c @@ -279,8 +279,10 @@ static int update_raid_extent_item(struct btrfs_trans_handle *trans, ret = btrfs_search_slot(trans, trans->fs_info->stripe_root, key, path, 0, 1); - if (ret) - return (ret == 1 ? ret : -EINVAL); + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + return ret; leaf = path->nodes[0]; slot = path->slots[0]; @@ -306,7 +308,6 @@ int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, stripe_extent = kzalloc(item_size, GFP_NOFS); if (!unlikely(stripe_extent)) { btrfs_abort_transaction(trans, -ENOMEM); - btrfs_end_transaction(trans); return -ENOMEM; } @@ -345,7 +346,7 @@ int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, struct btrfs_ordered_extent *ordered_extent) { struct btrfs_io_context *bioc; - int ret; + int ret = 0; if (!btrfs_fs_incompat(trans->fs_info, RAID_STRIPE_TREE)) return 0; @@ -353,17 +354,23 @@ int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, list_for_each_entry(bioc, &ordered_extent->bioc_list, rst_ordered_entry) { ret = btrfs_insert_one_raid_extent(trans, bioc); if (ret) - return ret; + break; } - while (!list_empty(&ordered_extent->bioc_list)) { - bioc = list_first_entry(&ordered_extent->bioc_list, + btrfs_cleanup_ordered_bioc_list(ordered_extent); + return ret; +} + +void btrfs_cleanup_ordered_bioc_list(struct btrfs_ordered_extent *ordered) +{ + while (!list_empty(&ordered->bioc_list)) { + struct btrfs_io_context *bioc; + + bioc = list_first_entry(&ordered->bioc_list, typeof(*bioc), rst_ordered_entry); list_del(&bioc->rst_ordered_entry); btrfs_put_bioc(bioc); } - - return 0; } int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, @@ -394,8 +401,8 @@ int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, return -ENOMEM; if (stripe->rst_search_commit_root) { - path->skip_locking = 1; - path->search_commit_root = 1; + path->skip_locking = true; + path->search_commit_root = true; } ret = btrfs_search_slot(NULL, stripe_root, &stripe_key, path, 0, 0); diff --git a/fs/btrfs/raid-stripe-tree.h b/fs/btrfs/raid-stripe-tree.h index 69942ad431408..eb02cf48511bc 100644 --- a/fs/btrfs/raid-stripe-tree.h +++ b/fs/btrfs/raid-stripe-tree.h @@ -28,6 +28,7 @@ int btrfs_get_raid_extent_offset(struct btrfs_fs_info *fs_info, u32 stripe_index, struct btrfs_io_stripe *stripe); int btrfs_insert_raid_extent(struct btrfs_trans_handle *trans, struct btrfs_ordered_extent *ordered_extent); +void btrfs_cleanup_ordered_bioc_list(struct btrfs_ordered_extent *ordered); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS int btrfs_insert_one_raid_extent(struct btrfs_trans_handle *trans, diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 0135dceb7baaa..4d4852f2ba7eb 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1583,8 +1583,10 @@ static void verify_bio_data_sectors(struct btrfs_raid_bio *rbio, int ret; /* No csum for this sector, skip to the next sector. */ - if (!test_bit(total_sector_nr, rbio->csum_bitmap)) + if (!test_bit(total_sector_nr, rbio->csum_bitmap)) { + total_sector_nr++; continue; + } ret = btrfs_check_block_csum(fs_info, paddr, csum_buf, expected_csum); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index fc76013b1a3e0..5f05eb88a4fa3 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -496,6 +496,7 @@ static int __add_reloc_root(struct btrfs_root *root) btrfs_err(fs_info, "Duplicate root found for start=%llu while inserting into relocation tree", node->bytenr); + kfree(node); return -EEXIST; } @@ -1456,6 +1457,17 @@ static int insert_dirty_subvol(struct btrfs_trans_handle *trans, return 0; } +static void clear_reloc_root(struct btrfs_root *root) +{ + root->reloc_root = NULL; + /* + * Need barrier to ensure clear_bit() only happens after + * root->reloc_root = NULL. Pairs with have_reloc_root(). + */ + smp_wmb(); + clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); +} + static int clean_dirty_subvols(struct reloc_control *rc) { struct btrfs_root *root; @@ -1470,13 +1482,7 @@ static int clean_dirty_subvols(struct reloc_control *rc) struct btrfs_root *reloc_root = root->reloc_root; list_del_init(&root->reloc_dirty_list); - root->reloc_root = NULL; - /* - * Need barrier to ensure clear_bit() only happens after - * root->reloc_root = NULL. Pairs with have_reloc_root. - */ - smp_wmb(); - clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, &root->state); + clear_reloc_root(root); if (reloc_root) { /* * btrfs_drop_snapshot drops our ref we hold for @@ -1857,24 +1863,43 @@ void merge_reloc_roots(struct reloc_control *rc) * corruption, e.g. bad reloc tree key offset. */ ret = -EINVAL; + btrfs_put_root(root); goto out; } ret = merge_reloc_root(rc, root); - btrfs_put_root(root); if (ret) { - if (list_empty(&reloc_root->root_list)) + /* + * Clear the reloc root since below we will call + * free_reloc_roots(), otherwise we leave + * root->reloc_root pointing to a freed reloc + * root and trigger a use-after-free during + * unmount or elsewhere. + */ + clear_reloc_root(root); + btrfs_put_root(root); + /* + * We are adding the reloc_root to the local + * reloc_roots list, so we add a ref for this + * list which will be dropped below by the call + * to free_reloc_roots(). + */ + if (list_empty(&reloc_root->root_list)) { list_add_tail(&reloc_root->root_list, &reloc_roots); + btrfs_grab_root(reloc_root); + } + /* Now drop the ref for root->reloc_root. */ + btrfs_put_root(reloc_root); goto out; } + btrfs_put_root(root); } else { if (!IS_ERR(root)) { if (root->reloc_root == reloc_root) { - root->reloc_root = NULL; + clear_reloc_root(root); + /* Drop the ref for root->reloc_root. */ btrfs_put_root(reloc_root); } - clear_bit(BTRFS_ROOT_DEAD_RELOC_TREE, - &root->state); btrfs_put_root(root); } @@ -3175,8 +3200,8 @@ static int __add_tree_block(struct reloc_control *rc, key.offset = blocksize; } - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, 0, 0); if (ret < 0) return ret; @@ -3264,7 +3289,6 @@ static int delete_v1_space_cache(struct extent_buffer *leaf, struct btrfs_key key; bool found = false; int i; - int ret; if (btrfs_header_owner(leaf) != BTRFS_ROOT_TREE_OBJECTID) return 0; @@ -3288,8 +3312,8 @@ static int delete_v1_space_cache(struct extent_buffer *leaf, } if (!found) return -ENOENT; - ret = delete_block_group_cache(block_group, NULL, space_cache_ino); - return ret; + + return delete_block_group_cache(block_group, NULL, space_cache_ino); } /* @@ -3368,8 +3392,8 @@ int find_next_extent(struct reloc_control *rc, struct btrfs_path *path, key.type = BTRFS_EXTENT_ITEM_KEY; key.offset = 0; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; ret = btrfs_search_slot(NULL, rc->extent_root, &key, path, 0, 0); if (ret < 0) @@ -4054,6 +4078,24 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) return ret; } +static void release_recovered_fs_roots(struct list_head *roots, bool drop_reloc_refs) +{ + struct btrfs_root *root; + struct btrfs_root *next; + + list_for_each_entry_safe(root, next, roots, reloc_dirty_list) { + list_del_init(&root->reloc_dirty_list); + if (drop_reloc_refs) { + struct btrfs_root *reloc_root = root->reloc_root; + + ASSERT(reloc_root); + root->reloc_root = NULL; + btrfs_put_root(reloc_root); + } + btrfs_put_root(root); + } +} + /* * recover relocation interrupted by system crash. * @@ -4063,6 +4105,7 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) { LIST_HEAD(reloc_roots); + LIST_HEAD(recovered_roots); struct btrfs_key key; struct btrfs_root *fs_root; struct btrfs_root *reloc_root; @@ -4174,7 +4217,7 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) ret = PTR_ERR(fs_root); list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } ret = __add_reloc_root(reloc_root); @@ -4183,15 +4226,17 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) list_add_tail(&reloc_root->root_list, &reloc_roots); btrfs_put_root(fs_root); btrfs_end_transaction(trans); - goto out_unset; + goto out_drop_reloc_refs; } + ASSERT(list_empty(&fs_root->reloc_dirty_list)); fs_root->reloc_root = btrfs_grab_root(reloc_root); - btrfs_put_root(fs_root); + list_add_tail(&fs_root->reloc_dirty_list, &recovered_roots); } ret = btrfs_commit_transaction(trans); if (ret) - goto out_unset; + goto out_drop_reloc_refs; + release_recovered_fs_roots(&recovered_roots, false); merge_reloc_roots(rc); @@ -4207,6 +4252,8 @@ int btrfs_recover_relocation(struct btrfs_fs_info *fs_info) ret2 = clean_dirty_subvols(rc); if (ret2 < 0 && !ret) ret = ret2; +out_drop_reloc_refs: + release_recovered_fs_roots(&recovered_roots, true); out_unset: unset_reloc_control(rc); reloc_chunk_end(fs_info); diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 16936d17166ee..f02188acd0480 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -463,10 +463,10 @@ static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( refcount_set(&sctx->refs, 1); sctx->is_dev_replace = is_dev_replace; sctx->fs_info = fs_info; - sctx->extent_path.search_commit_root = 1; - sctx->extent_path.skip_locking = 1; - sctx->csum_path.search_commit_root = 1; - sctx->csum_path.skip_locking = 1; + sctx->extent_path.search_commit_root = true; + sctx->extent_path.skip_locking = true; + sctx->csum_path.search_commit_root = true; + sctx->csum_path.skip_locking = true; for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) { int ret; @@ -1049,6 +1049,10 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, skip: for_each_set_bit(sector_nr, &extent_bitmap, stripe->nr_sectors) { + const u64 sector_logical = stripe->logical + + ((u64)sector_nr << fs_info->sectorsize_bits); + const u64 sector_physical = physical + + ((u64)sector_nr << fs_info->sectorsize_bits); bool repaired = false; if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { @@ -1077,12 +1081,12 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, if (dev) { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on dev %s physical %llu", - stripe->logical, btrfs_dev_name(dev), - physical); + sector_logical, btrfs_dev_name(dev), + sector_physical); } else { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on mirror %u", - stripe->logical, stripe->mirror_num); + sector_logical, stripe->mirror_num); } continue; } @@ -1091,30 +1095,30 @@ static void scrub_stripe_report_errors(struct scrub_ctx *sctx, if (dev) { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on dev %s physical %llu", - stripe->logical, btrfs_dev_name(dev), - physical); + sector_logical, btrfs_dev_name(dev), + sector_physical); } else { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on mirror %u", - stripe->logical, stripe->mirror_num); + sector_logical, stripe->mirror_num); } if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("i/o error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_csum_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("checksum error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_meta_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("header error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); if (scrub_bitmap_test_bit_meta_gen_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("generation error", dev, false, - stripe->logical, physical); + sector_logical, sector_physical); } /* Update the device stats. */ @@ -2103,10 +2107,10 @@ static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, * as the data stripe bytenr may be smaller than previous extent. Thus * we have to use our own extent/csum paths. */ - extent_path.search_commit_root = 1; - extent_path.skip_locking = 1; - csum_path.search_commit_root = 1; - csum_path.skip_locking = 1; + extent_path.search_commit_root = true; + extent_path.skip_locking = true; + csum_path.search_commit_root = true; + csum_path.skip_locking = true; for (int i = 0; i < data_stripes; i++) { int stripe_index; @@ -2630,8 +2634,8 @@ int scrub_enumerate_chunks(struct scrub_ctx *sctx, return -ENOMEM; path->reada = READA_FORWARD; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; key.objectid = scrub_dev->devid; key.type = BTRFS_DEV_EXTENT_KEY; diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index 04473387ee8bf..22190902be81c 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -633,9 +633,9 @@ static struct btrfs_path *alloc_path_for_send(void) path = btrfs_alloc_path(); if (!path) return NULL; - path->search_commit_root = 1; - path->skip_locking = 1; - path->need_commit_sem = 1; + path->search_commit_root = true; + path->skip_locking = true; + path->need_commit_sem = true; return path; } @@ -2074,7 +2074,7 @@ static int will_overwrite_ref(struct send_ctx *sctx, u64 dir, u64 dir_gen, ret = is_inode_existent(sctx, dir, dir_gen, NULL, &parent_root_dir_gen); if (ret <= 0) - return 0; + return ret; /* * If we have a parent root we need to verify that the parent dir was @@ -7643,10 +7643,10 @@ static int btrfs_compare_trees(struct btrfs_root *left_root, goto out; } - left_path->search_commit_root = 1; - left_path->skip_locking = 1; - right_path->search_commit_root = 1; - right_path->skip_locking = 1; + left_path->search_commit_root = true; + left_path->skip_locking = true; + right_path->search_commit_root = true; + right_path->skip_locking = true; /* * Strategy: Go to the first items of both trees. Then do diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 736575f925179..105f21759c99f 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -1518,12 +1518,14 @@ static int btrfs_reconfigure(struct fs_context *fc) sync_filesystem(sb); set_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state); - if (!btrfs_check_options(fs_info, &ctx->mount_opt, fc->sb_flags)) - return -EINVAL; + if (!btrfs_check_options(fs_info, &ctx->mount_opt, fc->sb_flags)) { + ret = -EINVAL; + goto restore; + } ret = btrfs_check_features(fs_info, !(fc->sb_flags & SB_RDONLY)); if (ret < 0) - return ret; + goto restore; btrfs_ctx_to_info(fs_info, ctx); btrfs_remount_begin(fs_info, old_ctx.mount_opt, fc->sb_flags); @@ -2408,7 +2410,7 @@ static int btrfs_show_devname(struct seq_file *m, struct dentry *root) static long btrfs_nr_cached_objects(struct super_block *sb, struct shrink_control *sc) { struct btrfs_fs_info *fs_info = btrfs_sb(sb); - const s64 nr = percpu_counter_sum_positive(&fs_info->evictable_extent_maps); + const s64 nr = percpu_counter_read_positive(&fs_info->evictable_extent_maps); trace_btrfs_extent_map_shrinker_count(fs_info, nr); diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index bd6cf479b3272..32835d0271be9 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -456,8 +456,19 @@ static int record_root_in_trans(struct btrfs_trans_handle *trans, * through btrfs_record_root_in_trans without having to take the * lock. smp_wmb() makes sure that all the writes above are * done before we pop in the zero below + * + * If @force is true, it means the call is from + * qgroup_account_snapshot(), which only requires radix tree + * tracking. + * We should not force reloc root creation here, as the root + * may have already been modified, and in that case + * root->commit_root has already been dropped. + * + * Using that commit root will cause the reloc root to refer + * to a deleted extent, causing extent tree corruption. */ - ret = btrfs_init_reloc_root(trans, root); + if (!force) + ret = btrfs_init_reloc_root(trans, root); smp_mb__before_atomic(); clear_bit(BTRFS_ROOT_IN_TRANS_SETUP, &root->state); } diff --git a/fs/btrfs/tree-checker.c b/fs/btrfs/tree-checker.c index db7402836340a..79f297d9ebc4a 100644 --- a/fs/btrfs/tree-checker.c +++ b/fs/btrfs/tree-checker.c @@ -1288,6 +1288,37 @@ static int check_root_item(struct extent_buffer *leaf, struct btrfs_key *key, return 0; } +static int check_root_ref(struct extent_buffer *leaf, struct btrfs_key *key, int slot) +{ + struct btrfs_root_ref *rref; + u32 item_size = btrfs_item_size(leaf, slot); + u32 name_len; + + if (unlikely(item_size <= sizeof(*rref))) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect > %zu", + key->type, item_size, sizeof(*rref)); + return -EUCLEAN; + } + + rref = btrfs_item_ptr(leaf, slot, struct btrfs_root_ref); + name_len = btrfs_root_ref_name_len(leaf, rref); + if (unlikely(name_len > BTRFS_NAME_LEN)) { + generic_err(leaf, slot, + "root ref name too long for key type %u, have %u max %u", + key->type, name_len, BTRFS_NAME_LEN); + return -EUCLEAN; + } + if (unlikely(item_size != sizeof(*rref) + name_len)) { + generic_err(leaf, slot, + "invalid root ref item size for key type %u, have %u expect %zu", + key->type, item_size, sizeof(*rref) + name_len); + return -EUCLEAN; + } + + return 0; +} + __printf(3,4) __cold static void extent_err(const struct extent_buffer *eb, int slot, @@ -1782,6 +1813,12 @@ static int check_inode_ref(struct extent_buffer *leaf, iref = (struct btrfs_inode_ref *)ptr; namelen = btrfs_inode_ref_name_len(leaf, iref); + if (unlikely(namelen == 0 || namelen > BTRFS_NAME_LEN)) { + inode_ref_err(leaf, slot, + "invalid inode ref name length, has %u expect [1, %u]", + namelen, BTRFS_NAME_LEN); + return -EUCLEAN; + } if (unlikely(ptr + sizeof(*iref) + namelen > end)) { inode_ref_err(leaf, slot, "inode ref overflow, ptr %lu end %lu namelen %u", @@ -1965,6 +2002,10 @@ static enum btrfs_tree_block_status check_leaf_item(struct extent_buffer *leaf, case BTRFS_ROOT_ITEM_KEY: ret = check_root_item(leaf, key, slot); break; + case BTRFS_ROOT_REF_KEY: + case BTRFS_ROOT_BACKREF_KEY: + ret = check_root_ref(leaf, key, slot); + break; case BTRFS_EXTENT_ITEM_KEY: case BTRFS_METADATA_ITEM_KEY: ret = check_extent_item(leaf, key, slot, prev_key); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index a0e12b4fb9561..f84065b69464c 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -602,9 +602,9 @@ static int overwrite_item(struct walk_control *wc) insert: btrfs_release_path(wc->subvol_path); /* try to insert the key into the destination tree */ - wc->subvol_path->skip_release_on_error = 1; + wc->subvol_path->skip_release_on_error = true; ret = btrfs_insert_empty_item(trans, root, wc->subvol_path, &wc->log_key, item_size); - wc->subvol_path->skip_release_on_error = 0; + wc->subvol_path->skip_release_on_error = false; dst_eb = wc->subvol_path->nodes[0]; dst_slot = wc->subvol_path->slots[0]; @@ -5734,8 +5734,8 @@ static int btrfs_check_ref_name_override(struct extent_buffer *eb, search_path = btrfs_alloc_path(); if (!search_path) return -ENOMEM; - search_path->search_commit_root = 1; - search_path->skip_locking = 1; + search_path->search_commit_root = true; + search_path->skip_locking = true; while (cur_offset < item_size) { u64 parent; @@ -6052,8 +6052,8 @@ static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); if (WARN_ON_ONCE(ret > 0)) { @@ -6073,8 +6073,8 @@ static int conflicting_inode_is_dir(struct btrfs_root *root, u64 ino, } btrfs_release_path(path); - path->search_commit_root = 0; - path->skip_locking = 0; + path->search_commit_root = false; + path->skip_locking = false; return ret; } @@ -7232,8 +7232,8 @@ static int btrfs_log_all_parents(struct btrfs_trans_handle *trans, path = btrfs_alloc_path(); if (!path) return -ENOMEM; - path->skip_locking = 1; - path->search_commit_root = 1; + path->skip_locking = true; + path->search_commit_root = true; key.objectid = ino; key.type = BTRFS_INODE_REF_KEY; diff --git a/fs/btrfs/verity.c b/fs/btrfs/verity.c index 46bd8ca586708..7fc30a58d8d9c 100644 --- a/fs/btrfs/verity.c +++ b/fs/btrfs/verity.c @@ -731,14 +731,18 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, goto out; folio_lock(folio); - /* If it's not uptodate after we have the lock, we got a read error. */ - if (!folio_test_uptodate(folio)) { + /* Folio was truncated from mapping. */ + if (!folio->mapping) { folio_unlock(folio); folio_put(folio); - return ERR_PTR(-EIO); + goto again; } - folio_unlock(folio); - goto out; + /* Another reader may have filled the folio while we waited. */ + if (folio_test_uptodate(folio)) { + folio_unlock(folio); + goto out; + } + goto read_folio; } folio = filemap_alloc_folio(mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS), @@ -755,6 +759,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, return ERR_PTR(ret); } +read_folio: /* * Merkle item keys are indexed from byte 0 in the merkle tree. * They have the form: @@ -764,6 +769,7 @@ static struct page *btrfs_read_merkle_tree_page(struct inode *inode, ret = read_key_bytes(BTRFS_I(inode), BTRFS_VERITY_MERKLE_ITEM_KEY, off, folio_address(folio), PAGE_SIZE, folio); if (ret < 0) { + folio_unlock(folio); folio_put(folio); return ERR_PTR(ret); } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 630fb5885692b..792818b66dc4b 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -1839,8 +1839,8 @@ static int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes, } path->reada = READA_FORWARD; - path->search_commit_root = 1; - path->skip_locking = 1; + path->search_commit_root = true; + path->skip_locking = true; key.objectid = device->devid; key.type = BTRFS_DEV_EXTENT_KEY; @@ -2456,9 +2456,7 @@ int btrfs_rm_device(struct btrfs_fs_info *fs_info, free_fs_devices(cur_devices); } - ret = btrfs_commit_transaction(trans); - - return ret; + return btrfs_commit_transaction(trans); error_undo: if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { @@ -3035,7 +3033,11 @@ int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path error_sysfs: btrfs_sysfs_remove_device(device); mutex_lock(&fs_info->fs_devices->device_list_mutex); + if (seeding_dev) + btrfs_assign_next_active_device(device, seed_devices->latest_dev); mutex_lock(&fs_info->chunk_mutex); + if (!list_empty(&device->post_commit_list)) + list_del_init(&device->post_commit_list); list_del_rcu(&device->dev_list); list_del(&device->dev_alloc_list); fs_info->fs_devices->num_devices--; @@ -3721,7 +3723,11 @@ static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info, u64 chunk_type; cache = btrfs_lookup_block_group(fs_info, chunk_offset); - ASSERT(cache); + if (unlikely(!cache)) { + btrfs_err(fs_info, "balance: chunk at bytenr %llu has no corresponding block group", + chunk_offset); + return -EUCLEAN; + } chunk_type = cache->flags; btrfs_put_block_group(cache); @@ -7347,7 +7353,6 @@ static int read_one_dev(struct extent_buffer *leaf, struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; struct btrfs_device *device; u64 devid; - int ret; u8 fs_uuid[BTRFS_FSID_SIZE]; u8 dev_uuid[BTRFS_UUID_SIZE]; @@ -7446,8 +7451,8 @@ static int read_one_dev(struct extent_buffer *leaf, atomic64_add(device->total_bytes - device->bytes_used, &fs_info->free_chunk_space); } - ret = 0; - return ret; + + return 0; } int btrfs_read_sys_array(struct btrfs_fs_info *fs_info) @@ -7627,7 +7632,7 @@ int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info) * chunk tree, to keep it simple, just skip locking on the chunk tree. */ ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags)); - path->skip_locking = 1; + path->skip_locking = true; /* * Read all device items, and then all the chunk items. All diff --git a/fs/btrfs/xattr.c b/fs/btrfs/xattr.c index b6f01d6c79e7f..b6e91e8fb7e38 100644 --- a/fs/btrfs/xattr.c +++ b/fs/btrfs/xattr.c @@ -97,7 +97,7 @@ int btrfs_setxattr(struct btrfs_trans_handle *trans, struct inode *inode, path = btrfs_alloc_path(); if (!path) return -ENOMEM; - path->skip_release_on_error = 1; + path->skip_release_on_error = true; if (!value) { di = btrfs_lookup_xattr(trans, root, path, diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index b8fc97e33009b..862475ae49770 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -130,8 +130,10 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - BTRFS_SUPER_INFO_SIZE; + filemap_invalidate_lock(mapping); page[i] = read_cache_page_gfp(mapping, bytenr >> PAGE_SHIFT, GFP_NOFS); + filemap_invalidate_unlock(mapping); if (IS_ERR(page[i])) { if (i == 1) btrfs_release_disk_super(super[0]); @@ -2167,7 +2169,11 @@ static bool check_bg_is_active(struct btrfs_eb_write_context *ctx, if (fs_info->treelog_bg == block_group->start) { if (!btrfs_zone_activate(block_group)) { - int ret_fin = btrfs_zone_finish_one_bg(fs_info); + int ret_fin; + + btrfs_zoned_meta_io_unlock(fs_info); + ret_fin = btrfs_zone_finish_one_bg(fs_info); + btrfs_zoned_meta_io_lock(fs_info); if (ret_fin != 1 || !btrfs_zone_activate(block_group)) return false; @@ -2586,16 +2592,13 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ down_read(&dev_replace->rwsem); map = block_group->physical_map; for (i = 0; i < map->num_stripes; i++) { - ret = call_zone_finish(block_group, &map->stripes[i]); - if (ret) { - up_read(&dev_replace->rwsem); - return ret; - } + if (ret) + break; } up_read(&dev_replace->rwsem); - if (!fully_written) + if (!ret && !fully_written) btrfs_dec_block_group_ro(block_group); spin_lock(&fs_info->zone_active_bgs_lock); @@ -2608,7 +2611,7 @@ static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_writ clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); - return 0; + return ret; } int btrfs_zone_finish(struct btrfs_block_group *block_group) @@ -2959,10 +2962,9 @@ int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) return ret < 0 ? ret : 1; } -int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, - bool do_finish) +int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish) { + struct btrfs_fs_info *fs_info = space_info->fs_info; struct btrfs_block_group *bg; int index; @@ -3167,6 +3169,17 @@ int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num reclaimed = bg->alloc_offset; bg->zone_unusable = bg->length - bg->zone_capacity; bg->alloc_offset = 0; + /* + * The zone was just reset to empty, so alloc_offset went back to + * the start of the zone. For metadata/system block groups the + * write pointer must follow it back to the start of the zone; + * otherwise it stays stale at the previous (finished) zone end, + * and metadata written into the reused zone would sit behind the + * write pointer, could never be written out in sequential order, + * and would be stranded (pinning its folio) until unmount. + */ + if (bg->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) + bg->meta_write_pointer = bg->start; /* * This holds because we currently reset fully used then freed * block group. diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index 2b807a02d1a8a..5cefdeb08b7b7 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -93,8 +93,7 @@ bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info); void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, u64 length); int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info); -int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, bool do_finish); +int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish); void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info); int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num_bytes); #else /* CONFIG_BLK_DEV_ZONED */ @@ -261,8 +260,7 @@ static inline int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) return 1; } -static inline int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info, +static inline int btrfs_zoned_activate_one_bg(struct btrfs_space_info *space_info, bool do_finish) { /* Consider all the block groups are active */ diff --git a/fs/btrfs/zstd.c b/fs/btrfs/zstd.c index c9cddcfa337b9..baaf3c1436323 100644 --- a/fs/btrfs/zstd.c +++ b/fs/btrfs/zstd.c @@ -307,8 +307,17 @@ struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level) DEFINE_WAIT(wait); prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE); - schedule(); + /* + * Re-check after being queued: zstd_put_workspace() only wakes + * a queue that already has a sleeper, so a workspace returned + * since the failed allocation woke nobody. + */ + ws = zstd_find_workspace(fs_info, level); + if (!ws) + schedule(); finish_wait(&zwsm->wait, &wait); + if (ws) + return ws; goto again; } diff --git a/fs/buffer.c b/fs/buffer.c index b6b477ff7b75d..261f521dbc583 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -2199,6 +2199,7 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) { size_t block_start, block_end; bool partial = false; + bool uptodate = folio_test_uptodate(folio); unsigned blocksize; struct buffer_head *bh, *head; @@ -2221,6 +2222,8 @@ void block_commit_write(struct folio *folio, size_t from, size_t to) clear_buffer_new(bh); block_start = block_end; + if (uptodate && block_start >= to) + break; bh = bh->b_this_page; } while (bh != head); diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index d1edb2ac38376..eae5045d50081 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -499,7 +499,6 @@ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) ret = -EINVAL; if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { - fput(file); pr_notice("Cache does not support read_iter and write_iter\n"); goto err_unuse; } diff --git a/fs/cachefiles/xattr.c b/fs/cachefiles/xattr.c index 52383b1d0ba63..6214331821583 100644 --- a/fs/cachefiles/xattr.c +++ b/fs/cachefiles/xattr.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "internal.h" #define CACHEFILES_COOKIE_TYPE_DATA 1 @@ -50,7 +51,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) _enter("%x,#%d", object->debug_id, len); - buf = kmalloc(sizeof(struct cachefiles_xattr) + len, GFP_KERNEL); + buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL); if (!buf) return -ENOMEM; @@ -60,6 +61,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) buf->content = object->content_info; if (test_bit(FSCACHE_COOKIE_LOCAL_WRITE, &object->cookie->flags)) buf->content = CACHEFILES_CONTENT_DIRTY; + put_unaligned_be64(0, (__be64 *)buf->data); if (len > 0) memcpy(buf->data, fscache_get_aux(object->cookie), len); @@ -77,8 +79,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) trace_cachefiles_vfs_error(object, file_inode(file), ret, cachefiles_trace_setxattr_error); trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, + buf->data, buf->content, cachefiles_coherency_set_fail); if (ret != -ENOMEM) cachefiles_io_error_obj( @@ -86,8 +87,7 @@ int cachefiles_set_object_xattr(struct cachefiles_object *object) "Failed to set xattr with error %d", ret); } else { trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, + buf->data, buf->content, cachefiles_coherency_set_ok); } @@ -110,9 +110,10 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file int ret = -ESTALE; tlen = sizeof(struct cachefiles_xattr) + len; - buf = kmalloc(tlen, GFP_KERNEL); + buf = kmalloc(sizeof(struct cachefiles_xattr) + max(len, sizeof(__be64)), GFP_KERNEL); if (!buf) return -ENOMEM; + put_unaligned_be64(0, (__be64 *)buf->data); xlen = cachefiles_inject_read_error(); if (xlen == 0) @@ -148,8 +149,7 @@ int cachefiles_check_auxdata(struct cachefiles_object *object, struct file *file out: trace_cachefiles_coherency(object, file_inode(file)->i_ino, - be64_to_cpup((__be64 *)buf->data), - buf->content, why); + buf->data, buf->content, why); kfree(buf); return ret; } diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c index 3af6795cb3c15..faa3de95caff6 100644 --- a/fs/ceph/addr.c +++ b/fs/ceph/addr.c @@ -785,6 +785,9 @@ static int write_folio_nounlock(struct folio *folio, ceph_wbc.truncate_size, true); if (IS_ERR(req)) { folio_redirty_for_writepage(wbc, folio); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; return PTR_ERR(req); } @@ -804,6 +807,9 @@ static int write_folio_nounlock(struct folio *folio, folio_redirty_for_writepage(wbc, folio); folio_end_writeback(folio); ceph_osdc_put_request(req); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; return PTR_ERR(bounce_page); } } @@ -838,6 +844,9 @@ static int write_folio_nounlock(struct folio *folio, ceph_vinop(inode), folio); folio_redirty_for_writepage(wbc, folio); folio_end_writeback(folio); + if (atomic_long_dec_return(&fsc->writeback_count) < + CONGESTION_OFF_THRESH(fsc->mount_options->congestion_kb)) + fsc->write_congested = false; return err; } if (err == -EBLOCKLISTED) @@ -1412,6 +1421,16 @@ void ceph_shift_unused_folios_left(struct folio_batch *fbatch) fbatch->nr = n; } +static void ceph_undo_wrbuffer_claim(struct inode *inode, struct folio *folio) +{ + struct ceph_snap_context *snapc = folio_detach_private(folio); + + if (!snapc) + return; + ceph_put_wrbuffer_cap_refs(ceph_inode(inode), 1, snapc); + ceph_put_snap_context(snapc); +} + static int ceph_submit_write(struct address_space *mapping, struct writeback_control *wbc, @@ -1475,6 +1494,7 @@ int ceph_submit_write(struct address_space *mapping, if (!page) continue; + ceph_undo_wrbuffer_claim(inode, page_folio(page)); redirty_page_for_writepage(wbc, page); unlock_page(page); } diff --git a/fs/ceph/caps.c b/fs/ceph/caps.c index b1a8ff612c41d..7f4bb62df3def 100644 --- a/fs/ceph/caps.c +++ b/fs/ceph/caps.c @@ -978,6 +978,27 @@ int __ceph_caps_revoking_other(struct ceph_inode_info *ci, return 0; } +/* + * Return true if any cap of this inode holds caps which the MDS has + * revoked, but which we have not released yet. + */ +static bool __ceph_is_any_revoking(const struct ceph_inode_info *ci) +{ + const struct rb_node *p; + + lockdep_assert_held(&ci->i_ceph_lock); + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + const struct ceph_cap *cap = + rb_entry(p, struct ceph_cap, ci_node); + + if (cap->implemented & ~cap->issued) + return true; + } + + return false; +} + int __ceph_caps_used(struct ceph_inode_info *ci) { int used = 0; @@ -1420,6 +1441,9 @@ static void __prep_cap(struct cap_msg_args *arg, struct ceph_cap *cap, cap->implemented &= cap->issued | used; cap->mds_wanted = want; + if ((ci->i_ceph_flags & CEPH_I_FLUSH_FORCE) != 0 && !__ceph_is_any_revoking(ci)) + ci->i_ceph_flags &= ~CEPH_I_FLUSH_FORCE; + arg->session = cap->session; arg->ino = ceph_vino(inode).ino; arg->cid = cap->cap_id; @@ -2034,6 +2058,14 @@ void ceph_check_caps(struct ceph_inode_info *ci, int flags) if (ci->i_ceph_flags & CEPH_I_FLUSH) flags |= CHECK_CAPS_FLUSH; + /* + * A revoke whose response was deferred (see handle_cap_grant()) must + * still be acknowledged. Replay the forced flush here so that even a + * check triggered by writeback/invalidation completion sends a cap + * message to the MDS. + */ + if (ci->i_ceph_flags & CEPH_I_FLUSH_FORCE) + flags |= CHECK_CAPS_FLUSH_FORCE; retry: /* Caps wanted by virtue of active open files. */ file_wanted = __ceph_caps_file_wanted(ci); @@ -2585,9 +2617,14 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } } - list_for_each_entry(cf, &ci->i_cap_flush_list, i_list) { - if (cf->tid < first_tid) + cf = list_first_entry(&ci->i_cap_flush_list, struct ceph_cap_flush, i_list); + while (&cf->i_list != &ci->i_cap_flush_list) { + struct ceph_cap_flush *next; + + if (cf->tid < first_tid) { + cf = list_next_entry(cf, i_list); continue; + } cap = ci->i_auth_cap; if (!(cap && cap->session == session)) { @@ -2597,6 +2634,7 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } first_tid = cf->tid + 1; + next = list_next_entry(cf, i_list); if (!cf->is_capsnap) { struct cap_msg_args arg; @@ -2637,6 +2675,7 @@ static void __kick_flushing_caps(struct ceph_mds_client *mdsc, } spin_lock(&ci->i_ceph_lock); + cf = next; } } @@ -3090,7 +3129,19 @@ int __ceph_get_caps(struct inode *inode, struct ceph_file_info *fi, int need, ret = -ERESTARTSYS; break; } - wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + + /* + * If a cap update is lost after + * mds_wanted was raised, waiting + * forever will never make progress. + * Retry the renew path periodically + * so we can resend synchronously. + */ + if (!wait_woken(&wait, TASK_INTERRUPTIBLE, + CEPH_GET_CAPS_WAIT_TIMEOUT)) { + ret = -EUCLEAN; + break; + } } remove_wait_queue(&ci->i_cap_wq, &wait); @@ -3124,7 +3175,8 @@ int __ceph_get_caps(struct inode *inode, struct ceph_file_info *fi, int need, continue; } if (ret == -EUCLEAN) { - /* session was killed, try renew caps */ + /* session was killed or a waited cap + * request needs a retry */ ret = ceph_renew_caps(inode, flags); if (ret == 0) continue; @@ -3740,13 +3792,30 @@ static void handle_cap_grant(struct inode *inode, BUG_ON(cap->issued & ~cap->implemented); /* don't let check_caps skip sending a response to MDS for revoke msgs */ - if (!revoke_wait && le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) { - cap->mds_wanted = 0; - flags |= CHECK_CAPS_FLUSH_FORCE; - if (cap == ci->i_auth_cap) - check_caps = 1; /* check auth cap only */ - else - check_caps = 2; /* check all caps */ + if (le32_to_cpu(grant->op) == CEPH_CAP_OP_REVOKE) { + if (revoke_wait) { + /* + * We can't ack the revoke yet: the response is deferred + * until the writeback or cache invalidation queued above + * completes. Set the CEPH_I_FLUSH_FORCE flag to remember + * that a forced cap message is owed so that deferred + * completion (ceph_put_wrbuffer_cap_refs() or the + * invalidate worker, both of which call ceph_check_caps()) + * actually sends one, even if by then the revoked caps look + * unused, the inode is retaining caps, or the MDS has + * re-granted them. Without this, the cap message is never + * sent and the MDS hangs ("isn't responding to + * mclientcaps(revoke)"). + */ + ci->i_ceph_flags |= CEPH_I_FLUSH_FORCE; + } else { + cap->mds_wanted = 0; + flags |= CHECK_CAPS_FLUSH_FORCE; + if (cap == ci->i_auth_cap) + check_caps = 1; /* check auth cap only */ + else + check_caps = 2; /* check all caps */ + } } if (extra_info->inline_version > 0 && @@ -4364,6 +4433,7 @@ void ceph_handle_caps(struct ceph_mds_session *session, snaptrace = h + 1; snaptrace_len = le32_to_cpu(h->snap_trace_len); + ceph_decode_need(&snaptrace, end, snaptrace_len, bad); p = snaptrace + snaptrace_len; if (msg_version >= 2) { diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c index de823a50af9c5..a236035f3a7a4 100644 --- a/fs/ceph/crypto.c +++ b/fs/ceph/crypto.c @@ -351,6 +351,11 @@ int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int elen) * Otherwise, base64 decode the string, and then ask fscrypt to format it * for userland presentation. * + * Though the fscrypt/crypto subsystems broadly expect all buffers to be in the + * linear-mapped region, this function slightly relaxes those requirements: + * fname->ctext, fname->name, and oname->name may be vmalloc(), but not + * tname->name. + * * Returns 0 on success or negative error code on error. */ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, @@ -358,11 +363,15 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, { struct inode *dir = fname->dir; struct fscrypt_str _tname = FSTR_INIT(NULL, 0); + struct fscrypt_str _oname; struct fscrypt_str iname; char *name = fname->name; int name_len = fname->name_len; int ret; + if (WARN_ON_ONCE(tname && is_vmalloc_addr(tname->name))) + return -EIO; + /* Sanity check that the resulting name will fit in the buffer */ if (fname->name_len > NAME_MAX || fname->ctext_len > NAME_MAX) return -EIO; @@ -403,16 +412,19 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, goto out_inode; } + if ((!tname || !tname->name) && + (fname->ctext_len == 0 || + unlikely(is_vmalloc_addr(fname->ctext)) || + unlikely(is_vmalloc_addr(oname->name)))) { + ret = fscrypt_fname_alloc_buffer(NAME_MAX, &_tname); + if (ret) + goto out_inode; + tname = &_tname; + } + if (fname->ctext_len == 0) { int declen; - if (!tname) { - ret = fscrypt_fname_alloc_buffer(NAME_MAX, &_tname); - if (ret) - goto out_inode; - tname = &_tname; - } - declen = ceph_base64_decode(name, name_len, tname->name); if (declen <= 0) { ret = -EIO; @@ -420,13 +432,29 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, } iname.name = tname->name; iname.len = declen; + } else if (unlikely(is_vmalloc_addr(fname->ctext))) { + memcpy(tname->name, fname->ctext, fname->ctext_len); + + iname.name = tname->name; + iname.len = fname->ctext_len; } else { iname.name = fname->ctext; iname.len = fname->ctext_len; } - ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, oname); - if (!ret && (dir != fname->dir)) { + _oname.name = unlikely(is_vmalloc_addr(oname->name)) ? + tname->name : oname->name; + _oname.len = oname->len; + + ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, &_oname); + if (ret) + goto out; + + if (unlikely(is_vmalloc_addr(oname->name))) + memcpy(oname->name, _oname.name, _oname.len); + oname->len = _oname.len; + + if (dir != fname->dir) { char tmp_buf[CEPH_BASE64_CHARS(NAME_MAX)]; name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%ld", diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index 66e592c47e57a..24abe4dbf774d 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -546,11 +546,16 @@ static int ceph_readdir(struct file *file, struct dir_context *ctx) pr_warn_client(cl, "%p %llx.%llx rde->offset 0x%llx ctx->pos 0x%llx\n", inode, ceph_vinop(inode), rde->offset, ctx->pos); + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; return -EIO; } - if (WARN_ON_ONCE(!rde->inode.in)) + if (WARN_ON_ONCE(!rde->inode.in)) { + ceph_mdsc_put_request(dfi->last_readdir); + dfi->last_readdir = NULL; return -EIO; + } ctx->pos = rde->offset; doutc(cl, "%p %llx.%llx (%d/%d) -> %llx '%.*s' %p\n", inode, @@ -1168,7 +1173,7 @@ static struct dentry *ceph_mkdir(struct mnt_idmap *idmap, struct inode *dir, !req->r_reply_info.head->is_target && !req->r_reply_info.head->is_dentry) err = ceph_handle_notrace_create(dir, dentry); - ret = ERR_PTR(err); + ret = err ? ERR_PTR(err) : NULL; out_req: if (!IS_ERR(ret) && req->r_dentry != dentry) /* Some other dentry was spliced in */ @@ -1758,11 +1763,11 @@ static int __dir_lease_check(const struct dentry *dentry, if (ret > 0) { if (time_before(jiffies, di->time + lwc->dir_lease_ttl)) return STOP; + if (!lwc->expire_dir_lease) + return KEEP; /* Move dentry to tail of dir lease list if we don't want * to delete it. So dentries in the list are checked in a * round robin manner */ - if (!lwc->expire_dir_lease) - return TOUCH; if (dentry->d_lockref.count > 0 || (di->flags & CEPH_DENTRY_REFERENCED)) return TOUCH; @@ -1789,7 +1794,7 @@ int ceph_trim_dentries(struct ceph_mds_client *mdsc) lwc.dir_lease = false; lwc.nr_to_scan = CEPH_CAPS_PER_RELEASE * 2; freed = __dentry_leases_walk(mdsc, &lwc); - if (!lwc.nr_to_scan) /* more invalid leases */ + if (freed > 0 && !lwc.nr_to_scan) /* more invalid leases */ return -EAGAIN; if (lwc.nr_to_scan < CEPH_CAPS_PER_RELEASE) @@ -1799,6 +1804,10 @@ int ceph_trim_dentries(struct ceph_mds_client *mdsc) lwc.expire_dir_lease = freed < count; lwc.dir_lease_ttl = mdsc->fsc->mount_options->caps_wanted_delay_max * HZ; freed +=__dentry_leases_walk(mdsc, &lwc); + if (freed == 0 && count == 0) + /* no progress possible currently, retry futile */ + return 0; + if (!lwc.nr_to_scan) /* more to check */ return -EAGAIN; diff --git a/fs/ceph/export.c b/fs/ceph/export.c index b2f2af1046791..debb9634b9e3d 100644 --- a/fs/ceph/export.c +++ b/fs/ceph/export.c @@ -442,6 +442,16 @@ static struct dentry *ceph_fh_to_parent(struct super_block *sb, return dentry; } +static int ceph_export_copy_name(char *name, const char *src, u32 len) +{ + if (len > NAME_MAX) + return -ENAMETOOLONG; + + memcpy(name, src, len); + name[len] = '\0'; + return 0; +} + static int __get_snap_name(struct dentry *parent, char *name, struct dentry *child) { @@ -513,9 +523,8 @@ static int __get_snap_name(struct dentry *parent, char *name, BUG_ON(!rde->inode.in); if (ceph_snap(inode) == le64_to_cpu(rde->inode.in->snapid)) { - memcpy(name, rde->name, rde->name_len); - name[rde->name_len] = '\0'; - err = 0; + err = ceph_export_copy_name(name, rde->name, + rde->name_len); goto out; } } @@ -580,8 +589,8 @@ static int ceph_get_name(struct dentry *parent, char *name, rinfo = &req->r_reply_info; if (!IS_ENCRYPTED(dir)) { - memcpy(name, rinfo->dname, rinfo->dname_len); - name[rinfo->dname_len] = 0; + err = ceph_export_copy_name(name, rinfo->dname, + rinfo->dname_len); } else { struct fscrypt_str oname = FSTR_INIT(NULL, 0); struct ceph_fname fname = { .dir = dir, @@ -595,10 +604,9 @@ static int ceph_get_name(struct dentry *parent, char *name, goto out; err = ceph_fname_to_usr(&fname, NULL, &oname, NULL); - if (!err) { - memcpy(name, oname.name, oname.len); - name[oname.len] = 0; - } + if (!err) + err = ceph_export_copy_name(name, oname.name, + oname.len); ceph_fname_free_buffer(dir, &oname); } out: diff --git a/fs/ceph/file.c b/fs/ceph/file.c index ceb5706fe3665..fa582c8537c5f 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -295,7 +295,7 @@ static int ceph_init_file(struct inode *inode, struct file *file, int fmode) } /* - * try renew caps after session gets killed. + * Retry cap acquisition after a stale session or a lost cap update. */ int ceph_renew_caps(struct inode *inode, int fmode) { @@ -303,14 +303,15 @@ int ceph_renew_caps(struct inode *inode, int fmode) struct ceph_client *cl = mdsc->fsc->client; struct ceph_inode_info *ci = ceph_inode(inode); struct ceph_mds_request *req; - int err, flags, wanted; + int err, flags, wanted, issued; spin_lock(&ci->i_ceph_lock); __ceph_touch_fmode(ci, mdsc, fmode); wanted = __ceph_caps_file_wanted(ci); + issued = __ceph_caps_issued(ci, NULL); if (__ceph_is_any_real_caps(ci) && - (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap)) { - int issued = __ceph_caps_issued(ci, NULL); + (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap) && + (issued & wanted) == wanted) { spin_unlock(&ci->i_ceph_lock); doutc(cl, "%p %llx.%llx want %s issued %s updating mds_wanted\n", inode, ceph_vinop(inode), ceph_cap_string(wanted), @@ -2411,6 +2412,54 @@ static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from) if (err < 0) goto out; + /* + * For O_APPEND writes we may have waited for Fwx exclusive caps + * while the previous Fwx holder (another client) extended the + * file. i_size has been updated via the cap grant message from + * the MDS, but ki_pos is still the old EOF. Re-read i_size here + * (no extra MDS round-trip needed) and adjust ki_pos to the true + * EOF. Since we hold Fwx, no other client can change the file. + */ + if (iocb->ki_flags & IOCB_APPEND) { + loff_t cur_eof = i_size_read(inode); + + if (cur_eof != pos) { + doutc(cl, + "%p %llx.%llx O_APPEND: pos adjusted %lld -> %lld\n", + inode, ceph_vinop(inode), pos, cur_eof); + iocb->ki_pos = cur_eof; + pos = cur_eof; + if (pos >= limit) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, limit - pos); + count = iov_iter_count(from); + + /* + * ceph_get_caps() validated the old endoff + * against i_max_size; adjusting ki_pos forward + * may have shifted the write range beyond the + * granted max_size. Re-check and truncate if + * necessary. + */ + spin_lock(&ci->i_ceph_lock); + if (pos + count > (loff_t)ci->i_max_size) { + loff_t max_size = ci->i_max_size; + + spin_unlock(&ci->i_ceph_lock); + if (pos >= max_size) { + err = -EFBIG; + goto out_caps; + } + iov_iter_truncate(from, max_size - pos); + count = iov_iter_count(from); + } else { + spin_unlock(&ci->i_ceph_lock); + } + } + } + err = file_update_time(file); if (err) goto out_caps; diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c index ba9f96efc8ee7..a2a63ea1f0ac2 100644 --- a/fs/ceph/mds_client.c +++ b/fs/ceph/mds_client.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -495,9 +496,13 @@ static int parse_reply_info_readdir(void **p, void *end, * to do the base64_decode in-place. It's * safe because the decoded string should * always be shorter, which is 3/4 of origin - * string. + * string. If this message was allocated with + * vmalloc() (happens, but rarely), leave it + * NULL and let ceph_fname_to_usr() allocate + * suitable temporary working space instead. */ - tname.name = _name; + if (likely(!is_vmalloc_addr(_name))) + tname.name = _name; /* * Set oname to _name too, and this will be @@ -569,10 +574,36 @@ static int parse_reply_info_filelock(void **p, void *end, #define DELEGATED_INO_AVAILABLE xa_mk_value(1) +static int ceph_insert_deleg_ino(struct ceph_mds_session *s, u64 ino) +{ + struct ceph_client *cl = s->s_mdsc->fsc->client; + int err; + + /* + * Cap how many delegated inodes a single session may hold. This is + * the only place that grows the count, so atomic_add_unless() bounds + * it at exactly CEPH_MAX_DELEG_INOS; s_num_deleg_inos can never exceed + * that. + */ + if (!atomic_add_unless(&s->s_num_deleg_inos, 1, CEPH_MAX_DELEG_INOS)) { + pr_warn_ratelimited_client(cl, + "MDS session already holds %d delegated inodes\n", + CEPH_MAX_DELEG_INOS); + return -EOVERFLOW; + } + + err = xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, + GFP_KERNEL); + if (err) + atomic_dec(&s->s_num_deleg_inos); + return err; +} + static int ceph_parse_deleg_inos(void **p, void *end, struct ceph_mds_session *s) { struct ceph_client *cl = s->s_mdsc->fsc->client; + u64 msg_deleg_inos = 0; u32 sets; ceph_decode_32_safe(p, end, sets, bad); @@ -590,16 +621,34 @@ static int ceph_parse_deleg_inos(void **p, void *end, start, len); continue; } + + /* + * Bound the number of inodes one reply may delegate. + * ceph_insert_deleg_ino() separately caps the per-session + * population, so this only has to stop one reply from spinning + * the insert loop under an attacker-controlled len. + */ + if (len > (u64)CEPH_MAX_DELEG_INOS || + msg_deleg_inos > (u64)CEPH_MAX_DELEG_INOS - len) { + pr_warn_ratelimited_client(cl, + "MDS reply delegates too many inodes (have %llu, +%llu, max %d)\n", + msg_deleg_inos, len, CEPH_MAX_DELEG_INOS); + return -EIO; + } + msg_deleg_inos += len; + while (len--) { - int err = xa_insert(&s->s_delegated_inos, start++, - DELEGATED_INO_AVAILABLE, - GFP_KERNEL); + int err = ceph_insert_deleg_ino(s, start++); + if (!err) { doutc(cl, "added delegated inode 0x%llx\n", start - 1); } else if (err == -EBUSY) { pr_warn_client(cl, "MDS delegated inode 0x%llx more than once.\n", start - 1); + } else if (err == -EOVERFLOW) { + /* ceph_insert_deleg_ino() already warned. */ + return -EIO; } else { return err; } @@ -617,16 +666,17 @@ u64 ceph_get_deleg_ino(struct ceph_mds_session *s) xa_for_each(&s->s_delegated_inos, ino, val) { val = xa_erase(&s->s_delegated_inos, ino); - if (val == DELEGATED_INO_AVAILABLE) + if (val == DELEGATED_INO_AVAILABLE) { + atomic_dec(&s->s_num_deleg_inos); return ino; + } } return 0; } int ceph_restore_deleg_ino(struct ceph_mds_session *s, u64 ino) { - return xa_insert(&s->s_delegated_inos, ino, DELEGATED_INO_AVAILABLE, - GFP_KERNEL); + return ceph_insert_deleg_ino(s, ino); } #else /* BITS_PER_LONG == 64 */ /* @@ -1011,6 +1061,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, INIT_LIST_HEAD(&s->s_waiting); INIT_LIST_HEAD(&s->s_unsafe); xa_init(&s->s_delegated_inos); + atomic_set(&s->s_num_deleg_inos, 0); INIT_LIST_HEAD(&s->s_cap_releases); INIT_WORK(&s->s_cap_release_work, ceph_cap_release_work); @@ -3843,6 +3894,7 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) struct ceph_mds_reply_head *head = msg->front.iov_base; struct ceph_mds_reply_info_parsed *rinfo; /* parsed reply info */ struct ceph_snap_realm *realm; + unsigned int nofs_flags; u64 tid; int err, result; int mds = session->s_mds; @@ -3985,6 +4037,14 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) /* insert trace into our cache */ mutex_lock(&req->r_fill_mutex); + + /* disable fs reclaim while we are using current->journal_info + * for our own purposes, or else shrinkers of other + * filesystems might dereference this pointer as a different + * type + */ + nofs_flags = memalloc_nofs_save(); + current->journal_info = req; err = ceph_fill_trace(mdsc->fsc->sb, req); if (err == 0) { @@ -3993,6 +4053,7 @@ static void handle_reply(struct ceph_mds_session *session, struct ceph_msg *msg) err = ceph_readdir_prepopulate(req, req->r_session); } current->journal_info = NULL; + memalloc_nofs_restore(nofs_flags); mutex_unlock(&req->r_fill_mutex); up_read(&mdsc->snap_rwsem); @@ -4258,7 +4319,9 @@ static void handle_session(struct ceph_mds_session *session, pr_err_client(cl, "No memory for path\n"); goto fail; } - ceph_decode_copy(&p, cap_auths[i].match.path, _len); + ceph_decode_copy_safe(&p, end, + cap_auths[i].match.path, + _len, bad); /* Remove the tailing '/' */ while (_len && cap_auths[i].match.path[_len - 1] == '/') { @@ -4275,7 +4338,9 @@ static void handle_session(struct ceph_mds_session *session, pr_err_client(cl, "No memory for fs_name\n"); goto fail; } - ceph_decode_copy(&p, cap_auths[i].match.fs_name, _len); + ceph_decode_copy_safe(&p, end, + cap_auths[i].match.fs_name, + _len, bad); } ceph_decode_8_safe(&p, end, cap_auths[i].match.root_squash, bad); @@ -4898,6 +4963,7 @@ static void send_mds_reconnect(struct ceph_mds_client *mdsc, goto fail_nomsg; xa_destroy(&session->s_delegated_inos); + atomic_set(&session->s_num_deleg_inos, 0); mutex_lock(&session->s_mutex); session->s_state = CEPH_MDS_SESSION_RECONNECTING; @@ -5783,11 +5849,13 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) doutc(cl, "tpath '%s', mask %d, caller_uid %d, caller_gid %d\n", tpath, mask, caller_uid, caller_gid); + mutex_lock(&mdsc->mutex); for (i = 0; i < mdsc->s_cap_auths_num; i++) { struct ceph_mds_cap_auth *s = &mdsc->s_cap_auths[i]; err = ceph_mds_auth_match(mdsc, s, cred, tpath); if (err < 0) { + mutex_unlock(&mdsc->mutex); put_cred(cred); return err; } else if (err > 0) { @@ -5809,6 +5877,7 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) doutc(cl, "root_squash_perms %d, rw_perms_s %p\n", root_squash_perms, rw_perms_s); if (root_squash_perms && rw_perms_s == NULL) { + mutex_unlock(&mdsc->mutex); doutc(cl, "access allowed\n"); return 0; } @@ -5823,6 +5892,7 @@ int ceph_mds_check_access(struct ceph_mds_client *mdsc, char *tpath, int mask) !!(mask & MAY_READ), !!(mask & MAY_WRITE)); } doutc(cl, "access denied\n"); + mutex_unlock(&mdsc->mutex); return -EACCES; } diff --git a/fs/ceph/mds_client.h b/fs/ceph/mds_client.h index 0428a5eaf28c6..cbd7f1590717a 100644 --- a/fs/ceph/mds_client.h +++ b/fs/ceph/mds_client.h @@ -74,6 +74,7 @@ struct ceph_fs_client; struct ceph_cap; #define MDS_AUTH_UID_ANY -1 +#define CEPH_GET_CAPS_WAIT_TIMEOUT (5 * HZ) struct ceph_mds_cap_match { s64 uid; /* default to MDS_AUTH_UID_ANY */ @@ -251,6 +252,7 @@ struct ceph_mds_session { struct list_head s_waiting; /* waiting requests */ struct list_head s_unsafe; /* unsafe requests */ struct xarray s_delegated_inos; + atomic_t s_num_deleg_inos; }; /* @@ -544,6 +546,7 @@ struct ceph_mds_client { struct rw_semaphore pool_perm_rwsem; struct rb_root pool_perm_tree; + /* protected by mutex */ u32 s_cap_auths_num; struct ceph_mds_cap_auth *s_cap_auths; diff --git a/fs/ceph/mdsmap.c b/fs/ceph/mdsmap.c index b228e5ecfb926..c91c0e2e118ba 100644 --- a/fs/ceph/mdsmap.c +++ b/fs/ceph/mdsmap.c @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -15,7 +16,7 @@ #include "super.h" #define CEPH_MDS_IS_READY(i, ignore_laggy) \ - (m->m_info[i].state > 0 && ignore_laggy ? true : !m->m_info[i].laggy) + (m->m_info[i].state > 0 && (ignore_laggy ? true : !m->m_info[i].laggy)) static int __mdsmap_get_random_mds(struct ceph_mdsmap *m, bool ignore_laggy) { @@ -126,6 +127,7 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, u8 mdsmap_v; u16 mdsmap_ev; u32 target; + size_t export_targets_len; m = kzalloc(sizeof(*m), GFP_NOFS); if (!m) @@ -224,8 +226,11 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, *p += namelen; if (info_v >= 2) { ceph_decode_32_safe(p, end, num_export_targets, bad); + export_targets_len = size_mul(num_export_targets, + sizeof(u32)); + ceph_decode_need(p, end, export_targets_len, bad); pexport_targets = *p; - *p += num_export_targets * sizeof(u32); + *p += export_targets_len; } else { num_export_targets = 0; } @@ -264,6 +269,10 @@ struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, goto nomem; for (j = 0; j < num_export_targets; j++) { target = ceph_decode_32(&pexport_targets); + if (target >= CEPH_MAX_MDS) { + err = -EIO; + goto corrupt; + } info->export_targets[j] = target; } } else { diff --git a/fs/ceph/super.h b/fs/ceph/super.h index 29a980e22dc26..71870f2f7d244 100644 --- a/fs/ceph/super.h +++ b/fs/ceph/super.h @@ -624,6 +624,15 @@ static inline int ceph_ino_compare(struct inode *inode, void *data) #define CEPH_MDS_INO_LOG_OFFSET (2 * CEPH_MAX_MDS) #define CEPH_INO_SYSTEM_BASE ((6*CEPH_MAX_MDS) + (CEPH_MAX_MDS * CEPH_NUM_STRAY)) +/* + * Upper bound on the number of delegated inodes a single MDS session may + * hold. The MDS normally hands out a small preallocation window (the + * userspace mds_client_prealloc_inos option defaults to 1000) and refills + * it as the client consumes entries. This leaves generous headroom while + * bounding the CPU and memory a malformed delegation interval can consume. + */ +#define CEPH_MAX_DELEG_INOS 8192 + static inline bool ceph_vino_is_reserved(const struct ceph_vino vino) { if (vino.ino >= CEPH_INO_SYSTEM_BASE || @@ -672,6 +681,10 @@ static inline struct inode *ceph_find_inode(struct super_block *sb, #define CEPH_I_SHUTDOWN (1 << 13) /* inode is no longer usable */ #define CEPH_I_ASYNC_CHECK_CAPS (1 << 14) /* check caps immediately after async creating finishes */ +#define CEPH_I_FLUSH_FORCE (1 << 15) /* a revoke's response was deferred; + * force a cap message to the MDS once + * the deferred work completes + */ /* * Masks of ceph inode work. diff --git a/fs/ceph/xattr.c b/fs/ceph/xattr.c index caf0fe4d2b1b7..cc73b0874ce40 100644 --- a/fs/ceph/xattr.c +++ b/fs/ceph/xattr.c @@ -850,6 +850,7 @@ static int __build_xattrs(struct inode *inode) name = p; p += len; ceph_decode_32_safe(&p, end, len, bad); + ceph_decode_need(&p, end, len, bad); val = p; p += len; diff --git a/fs/configfs/dir.c b/fs/configfs/dir.c index 68203952766e6..293a3fcc05352 100644 --- a/fs/configfs/dir.c +++ b/fs/configfs/dir.c @@ -401,8 +401,17 @@ static void remove_dir(struct dentry * d) configfs_remove_dirent(d); - if (d_really_is_positive(d)) - simple_rmdir(d_inode(parent),d); + if (d_really_is_positive(d)) { + if (!simple_rmdir(d_inode(parent), d)) + /* + * configfs_get_config_item() takes a hashed dentry as + * proof that ->s_element is still alive. Our caller + * is about to drop the last reference to the item and + * the VFS will not unhash until after we return, so + * unhash it here. + */ + d_drop(d); + } pr_debug(" o %pd removing done (%d)\n", d, d_count(d)); @@ -1088,15 +1097,12 @@ static int configfs_dump(struct configfs_dirent *sd, int level) * much on the stack, though, so folks that need this function - be careful * about your stack! Patches will be accepted to make it iterative. */ -static int configfs_depend_prep(struct dentry *origin, +static int configfs_depend_prep(struct configfs_dirent *sd, struct config_item *target) { - struct configfs_dirent *child_sd, *sd; + struct configfs_dirent *child_sd; int ret = 0; - BUG_ON(!origin || !origin->d_fsdata); - sd = origin->d_fsdata; - if (sd->s_element == target) /* Boo-yah */ goto out; @@ -1104,8 +1110,7 @@ static int configfs_depend_prep(struct dentry *origin, if ((child_sd->s_type & CONFIGFS_DIR) && !(child_sd->s_type & CONFIGFS_USET_DROPPING) && !(child_sd->s_type & CONFIGFS_USET_CREATING)) { - ret = configfs_depend_prep(child_sd->s_dentry, - target); + ret = configfs_depend_prep(child_sd, target); if (!ret) goto out; /* Child path boo-yah */ } @@ -1126,7 +1131,7 @@ static int configfs_do_depend_item(struct dentry *subsys_dentry, spin_lock(&configfs_dirent_lock); /* Scan the tree, return 0 if found */ - ret = configfs_depend_prep(subsys_dentry, target); + ret = configfs_depend_prep(subsys_dentry->d_fsdata, target); if (ret) goto out_unlock_dirent_lock; diff --git a/fs/configfs/symlink.c b/fs/configfs/symlink.c index f3f79c67add5d..b42f07e2ea7f5 100644 --- a/fs/configfs/symlink.c +++ b/fs/configfs/symlink.c @@ -76,9 +76,9 @@ static int configfs_get_target_path(struct config_item *item, static int create_link(struct config_item *parent_item, struct config_item *item, + struct configfs_dirent *target_sd, struct dentry *dentry) { - struct configfs_dirent *target_sd = item->ci_dentry->d_fsdata; char *body; int ret; @@ -115,6 +115,7 @@ static int create_link(struct config_item *parent_item, static int get_target(const char *symname, struct config_item **target, + struct configfs_dirent **target_sd, struct super_block *sb) { struct path path __free(path_put) = {}; @@ -125,7 +126,20 @@ static int get_target(const char *symname, struct config_item **target, return ret; if (path.dentry->d_sb != sb) return -EPERM; - *target = configfs_get_config_item(path.dentry); + /* + * A hashed dentry guarantees that neither the item nor the dirent + * have been released yet, as removals unhash before dropping. + * Grab both references here. An item reference alone would not keep + * ->ci_dentry alive. + */ + spin_lock(&path.dentry->d_lock); + if (!d_unhashed(path.dentry)) { + struct configfs_dirent *sd = path.dentry->d_fsdata; + + *target = config_item_get(sd->s_element); + *target_sd = configfs_get(sd); + } + spin_unlock(&path.dentry->d_lock); if (!*target) return -ENOENT; return 0; @@ -139,6 +153,7 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct configfs_dirent *sd; struct config_item *parent_item; struct config_item *target_item = NULL; + struct configfs_dirent *target_sd = NULL; const struct config_item_type *type; sd = dentry->d_parent->d_fsdata; @@ -182,7 +197,7 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, * AV, a thoroughly annoyed bastard. */ inode_unlock(dir); - ret = get_target(symname, &target_item, dentry->d_sb); + ret = get_target(symname, &target_item, &target_sd, dentry->d_sb); inode_lock(dir); if (ret) goto out_put; @@ -196,13 +211,14 @@ int configfs_symlink(struct mnt_idmap *idmap, struct inode *dir, ret = type->ct_item_ops->allow_link(parent_item, target_item); if (!ret) { mutex_lock(&configfs_symlink_mutex); - ret = create_link(parent_item, target_item, dentry); + ret = create_link(parent_item, target_item, target_sd, dentry); mutex_unlock(&configfs_symlink_mutex); if (ret && type->ct_item_ops->drop_link) type->ct_item_ops->drop_link(parent_item, target_item); } + configfs_put(target_sd); config_item_put(target_item); out_put: diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c index 645cc49360729..500397ca8a26f 100644 --- a/fs/crypto/inline_crypt.c +++ b/fs/crypto/inline_crypt.c @@ -22,22 +22,14 @@ #include "fscrypt_private.h" -static struct block_device **fscrypt_get_devices(struct super_block *sb, - unsigned int *num_devs) +static unsigned int +fscrypt_get_devices(struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]) { - struct block_device **devs; - - if (sb->s_cop->get_devices) { - devs = sb->s_cop->get_devices(sb, num_devs); - if (devs) - return devs; - } - devs = kmalloc(sizeof(*devs), GFP_KERNEL); - if (!devs) - return ERR_PTR(-ENOMEM); + if (sb->s_cop->get_devices) + return sb->s_cop->get_devices(sb, devs); devs[0] = sb->s_bdev; - *num_devs = 1; - return devs; + return 1; } static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_inode_info *ci) @@ -96,7 +88,7 @@ int fscrypt_select_encryption_impl(struct fscrypt_inode_info *ci, const struct inode *inode = ci->ci_inode; struct super_block *sb = inode->i_sb; struct blk_crypto_config crypto_cfg; - struct block_device **devs; + struct block_device *devs[FSCRYPT_MAX_DEVICES]; unsigned int num_devs; unsigned int i; @@ -135,20 +127,15 @@ int fscrypt_select_encryption_impl(struct fscrypt_inode_info *ci, crypto_cfg.key_type = is_hw_wrapped_key ? BLK_CRYPTO_KEY_TYPE_HW_WRAPPED : BLK_CRYPTO_KEY_TYPE_RAW; - devs = fscrypt_get_devices(sb, &num_devs); - if (IS_ERR(devs)) - return PTR_ERR(devs); - + num_devs = fscrypt_get_devices(sb, devs); for (i = 0; i < num_devs; i++) { if (!blk_crypto_config_supported(devs[i], &crypto_cfg)) - goto out_free_devs; + return 0; } fscrypt_log_blk_crypto_impl(ci->ci_mode, devs, num_devs, &crypto_cfg); ci->ci_inlinecrypt = true; -out_free_devs: - kfree(devs); return 0; } @@ -164,7 +151,7 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, enum blk_crypto_key_type key_type = is_hw_wrapped ? BLK_CRYPTO_KEY_TYPE_HW_WRAPPED : BLK_CRYPTO_KEY_TYPE_RAW; struct blk_crypto_key *blk_key; - struct block_device **devs; + struct block_device *devs[FSCRYPT_MAX_DEVICES]; unsigned int num_devs; unsigned int i; int err; @@ -182,17 +169,12 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key, } /* Start using blk-crypto on all the filesystem's block devices. */ - devs = fscrypt_get_devices(sb, &num_devs); - if (IS_ERR(devs)) { - err = PTR_ERR(devs); - goto fail; - } + num_devs = fscrypt_get_devices(sb, devs); for (i = 0; i < num_devs; i++) { err = blk_crypto_start_using_key(devs[i], blk_key); if (err) break; } - kfree(devs); if (err) { fscrypt_err(inode, "error %d starting to use blk-crypto", err); goto fail; @@ -210,20 +192,21 @@ void fscrypt_destroy_inline_crypt_key(struct super_block *sb, struct fscrypt_prepared_key *prep_key) { struct blk_crypto_key *blk_key = prep_key->blk_key; - struct block_device **devs; + struct block_device *devs[FSCRYPT_MAX_DEVICES]; unsigned int num_devs; unsigned int i; if (!blk_key) return; - /* Evict the key from all the filesystem's block devices. */ - devs = fscrypt_get_devices(sb, &num_devs); - if (!IS_ERR(devs)) { - for (i = 0; i < num_devs; i++) - blk_crypto_evict_key(devs[i], blk_key); - kfree(devs); - } + /* + * Evict the key from all the filesystem's block devices. + * This *must* be done before the key is freed. + */ + num_devs = fscrypt_get_devices(sb, devs); + for (i = 0; i < num_devs; i++) + blk_crypto_evict_key(devs[i], blk_key); + kfree_sensitive(blk_key); } diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c index c4d05168522b0..7b2ee3c14d31c 100644 --- a/fs/crypto/keysetup_v1.c +++ b/fs/crypto/keysetup_v1.c @@ -190,13 +190,19 @@ find_or_insert_direct_key(struct fscrypt_direct_key *to_insert, if (memcmp(ci->ci_policy.v1.master_key_descriptor, dk->dk_descriptor, FSCRYPT_KEY_DESCRIPTOR_SIZE) != 0) continue; + /* The sb is used at eviction time, so it must be the same. */ + if (ci->ci_inode->i_sb != dk->dk_sb) + continue; if (ci->ci_mode != dk->dk_mode) continue; if (!fscrypt_is_key_prepared(&dk->dk_key, ci)) continue; if (crypto_memneq(raw_key, dk->dk_raw, ci->ci_mode->keysize)) continue; - /* using existing tfm with same (descriptor, mode, raw_key) */ + /* + * Use an existing prepared key with the same (descriptor, sb, + * mode, inlinecrypt, raw_key) combination. + */ refcount_inc(&dk->dk_refcount); spin_unlock(&fscrypt_direct_keys_lock); free_direct_key(to_insert); diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c index bbb2f5ced9880..254a251f94bcd 100644 --- a/fs/crypto/policy.c +++ b/fs/crypto/policy.c @@ -534,7 +534,7 @@ int fscrypt_ioctl_set_policy(struct file *filp, const void __user *arg) return -EFAULT; policy.version = version; - if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + if (!inode_owner_or_capable(file_mnt_idmap(filp), inode)) return -EACCES; ret = mnt_want_write_file(filp); diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c index 5c35cc67aca4c..ee55994ce90d8 100644 --- a/fs/dlm/memory.c +++ b/fs/dlm/memory.c @@ -48,8 +48,10 @@ int __init dlm_memory_init(void) if (!rsb_cache) goto rsb; - cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback), + cb_cache = kmem_cache_create_usercopy("dlm_cb", sizeof(struct dlm_callback), __alignof__(struct dlm_callback), 0, + offsetof(struct dlm_callback, lvbptr), + sizeof_field(struct dlm_callback, lvbptr), NULL); if (!cb_cache) goto cb; diff --git a/fs/ecryptfs/crypto.c b/fs/ecryptfs/crypto.c index 69536cacdea8d..70c204c995a74 100644 --- a/fs/ecryptfs/crypto.c +++ b/fs/ecryptfs/crypto.c @@ -1285,7 +1285,7 @@ static int ecryptfs_read_headers_virt(char *page_virt, } else set_default_header_data(crypt_stat); rc = ecryptfs_parse_packet_set(crypt_stat, (page_virt + offset), - ecryptfs_dentry); + PAGE_SIZE - offset, ecryptfs_dentry); out: return rc; } diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h index 9e6ab0b413376..5be8c31653fa6 100644 --- a/fs/ecryptfs/ecryptfs_kernel.h +++ b/fs/ecryptfs/ecryptfs_kernel.h @@ -585,7 +585,8 @@ int ecryptfs_generate_key_packet_set(char *dest_base, size_t *len, size_t max); int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - unsigned char *src, struct dentry *ecryptfs_dentry); + unsigned char *src, size_t src_size, + struct dentry *ecryptfs_dentry); int ecryptfs_truncate(struct dentry *dentry, loff_t new_length); ssize_t ecryptfs_getxattr_lower(struct dentry *lower_dentry, struct inode *lower_inode, diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c index 7f9f68c00ef63..970e375103948 100644 --- a/fs/ecryptfs/keystore.c +++ b/fs/ecryptfs/keystore.c @@ -937,6 +937,12 @@ ecryptfs_parse_tag_70_packet(char **filename, size_t *filename_size, "rc = [%d]\n", __func__, rc); goto out; } + if (s->parsed_tag_70_packet_size < (ECRYPTFS_SIG_SIZE + 2)) { + ecryptfs_printk(KERN_WARNING, "Invalid packet size [%zd]\n", + s->parsed_tag_70_packet_size); + rc = -EINVAL; + goto out; + } s->block_aligned_filename_size = (s->parsed_tag_70_packet_size - ECRYPTFS_SIG_SIZE - 1); if ((1 + s->packet_size_len + s->parsed_tag_70_packet_size) @@ -1424,10 +1430,20 @@ parse_tag_3_packet(struct ecryptfs_crypt_stat *crypt_stat, } (*new_auth_tok)->session_key.encrypted_key_size = (body_size - (ECRYPTFS_SALT_SIZE + 5)); + /* + * Although encrypted_key_size is copied into the + * encrypted_key[ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES] buffer here, + * it later bounds operations on a smaller buffer: + * decrypt_passphrase_encrypted_session_key() sets decrypted_key_size = + * encrypted_key_size and decrypts into + * decrypted_key[ECRYPTFS_MAX_KEY_BYTES], then memcpy's into + * crypt_stat->key[ECRYPTFS_MAX_KEY_BYTES]. Limit to + * ECRYPTFS_MAX_KEY_BYTES to protect those smaller buffers. + */ if ((*new_auth_tok)->session_key.encrypted_key_size - > ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES) { + > ECRYPTFS_MAX_KEY_BYTES) { printk(KERN_WARNING "Tag 3 packet contains key larger " - "than ECRYPTFS_MAX_ENCRYPTED_KEY_BYTES\n"); + "than ECRYPTFS_MAX_KEY_BYTES\n"); rc = -EINVAL; goto out_free; } @@ -1576,7 +1592,7 @@ parse_tag_11_packet(unsigned char *data, unsigned char *contents, } (*packet_size) += length_size; (*tag_11_contents_size) = (body_size - 14); - if (unlikely((*packet_size) + body_size + 1 > max_packet_size)) { + if (unlikely((*packet_size) + body_size > max_packet_size)) { printk(KERN_ERR "Packet size exceeds max\n"); rc = -EINVAL; goto out; @@ -1743,6 +1759,7 @@ decrypt_passphrase_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, * ecryptfs_parse_packet_set * @crypt_stat: The cryptographic context * @src: Virtual address of region of memory containing the packets + * @src_size: Size of the packet set buffer * @ecryptfs_dentry: The eCryptfs dentry associated with the packet set * * Get crypt_stat to have the file's session key if the requisite key @@ -1753,7 +1770,7 @@ decrypt_passphrase_encrypted_session_key(struct ecryptfs_auth_tok *auth_tok, * conditions. */ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, - unsigned char *src, + unsigned char *src, size_t src_size, struct dentry *ecryptfs_dentry) { size_t i = 0; @@ -1777,7 +1794,11 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, * added the our &auth_tok_list */ next_packet_is_auth_tok_packet = 1; while (next_packet_is_auth_tok_packet) { - size_t max_packet_size = ((PAGE_SIZE - 8) - i); + size_t max_packet_size; + + if (i >= src_size) + break; + max_packet_size = src_size - i; switch (src[i]) { case ECRYPTFS_TAG_3_PACKET_TYPE: @@ -1792,12 +1813,16 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } rc = parse_tag_11_packet((unsigned char *)&src[i], sig_tmp_space, ECRYPTFS_SIG_SIZE, &tag_11_contents_size, &tag_11_packet_size, - max_packet_size); + src_size - i); if (rc) { ecryptfs_printk(KERN_ERR, "No valid " "(ecryptfs-specific) literal " @@ -1809,6 +1834,10 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += tag_11_packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } if (ECRYPTFS_SIG_SIZE != tag_11_contents_size) { ecryptfs_printk(KERN_ERR, "Expected " "signature of size [%d]; " @@ -1836,6 +1865,10 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat, goto out_wipe_list; } i += packet_size; + if (i > src_size) { + rc = -EIO; + goto out_wipe_list; + } crypt_stat->flags |= ECRYPTFS_ENCRYPTED; break; case ECRYPTFS_TAG_11_PACKET_TYPE: diff --git a/fs/ecryptfs/messaging.c b/fs/ecryptfs/messaging.c index 6318f3500e5c5..75c35b559b37c 100644 --- a/fs/ecryptfs/messaging.c +++ b/fs/ecryptfs/messaging.c @@ -165,6 +165,7 @@ int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon) mutex_unlock(&daemon->mux); goto out; } + mutex_lock(&ecryptfs_msg_ctx_lists_mux); list_for_each_entry_safe(msg_ctx, msg_ctx_tmp, &daemon->msg_ctx_out_queue, daemon_out_list) { list_del(&msg_ctx->daemon_out_list); @@ -173,6 +174,7 @@ int ecryptfs_exorcise_daemon(struct ecryptfs_daemon *daemon) "the out queue of a dying daemon\n", __func__); ecryptfs_msg_ctx_alloc_to_free(msg_ctx); } + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); hlist_del(&daemon->euid_chain); mutex_unlock(&daemon->mux); kfree_sensitive(daemon); @@ -283,9 +285,16 @@ ecryptfs_send_message_locked(char *data, int data_len, u8 msg_type, mutex_unlock(&ecryptfs_msg_ctx_lists_mux); rc = ecryptfs_send_miscdev(data, data_len, *msg_ctx, msg_type, 0, daemon); - if (rc) + if (rc) { printk(KERN_ERR "%s: Error attempting to send message to " "userspace daemon; rc = [%d]\n", __func__, rc); + mutex_lock(&ecryptfs_msg_ctx_lists_mux); + mutex_lock(&(*msg_ctx)->mux); + ecryptfs_msg_ctx_alloc_to_free(*msg_ctx); + mutex_unlock(&(*msg_ctx)->mux); + mutex_unlock(&ecryptfs_msg_ctx_lists_mux); + *msg_ctx = NULL; + } out: return rc; } diff --git a/fs/ecryptfs/miscdev.c b/fs/ecryptfs/miscdev.c index 4e62c3cef70fb..23baa55d390f1 100644 --- a/fs/ecryptfs/miscdev.c +++ b/fs/ecryptfs/miscdev.c @@ -357,7 +357,7 @@ ecryptfs_miscdev_write(struct file *file, const char __user *buf, u32 seq; size_t packet_size, packet_size_length; char *data; - unsigned char packet_size_peek[ECRYPTFS_MAX_PKT_LEN_SIZE]; + unsigned char packet_size_peek[ECRYPTFS_MAX_PKT_LEN_SIZE] = { }; ssize_t rc; if (count == 0) { @@ -373,7 +373,8 @@ ecryptfs_miscdev_write(struct file *file, const char __user *buf, } if (copy_from_user(packet_size_peek, &buf[PKT_LEN_OFFSET], - sizeof(packet_size_peek))) { + min_t(size_t, count - PKT_LEN_OFFSET, + sizeof(packet_size_peek)))) { printk(KERN_WARNING "%s: Error while inspecting packet size\n", __func__); return -EFAULT; diff --git a/fs/ecryptfs/super.c b/fs/ecryptfs/super.c index e7b7f426fecfb..0e2108219a6f5 100644 --- a/fs/ecryptfs/super.c +++ b/fs/ecryptfs/super.c @@ -153,6 +153,13 @@ static int ecryptfs_show_options(struct seq_file *m, struct dentry *root) if (mount_crypt_stat->global_default_cipher_key_size) seq_printf(m, ",ecryptfs_key_bytes=%zd", mount_crypt_stat->global_default_cipher_key_size); + if (mount_crypt_stat->flags & ECRYPTFS_GLOBAL_ENCRYPT_FILENAMES) { + seq_printf(m, ",ecryptfs_fn_cipher=%s", + mount_crypt_stat->global_default_fn_cipher_name); + if (mount_crypt_stat->global_default_fn_cipher_key_bytes) + seq_printf(m, ",ecryptfs_fn_key_bytes=%zd", + mount_crypt_stat->global_default_fn_cipher_key_bytes); + } if (mount_crypt_stat->flags & ECRYPTFS_PLAINTEXT_PASSTHROUGH_ENABLED) seq_printf(m, ",ecryptfs_passthrough"); if (mount_crypt_stat->flags & ECRYPTFS_XATTR_METADATA_ENABLED) diff --git a/fs/efivarfs/super.c b/fs/efivarfs/super.c index 6de97565d5f78..ca5004774c0b1 100644 --- a/fs/efivarfs/super.c +++ b/fs/efivarfs/super.c @@ -89,12 +89,30 @@ static int efivarfs_statfs(struct dentry *dentry, struct kstatfs *buf) /* Some UEFI firmware does not implement QueryVariableInfo() */ storage_space = remaining_space = 0; if (efi_rt_services_supported(EFI_RT_SUPPORTED_QUERY_VARIABLE_INFO)) { - status = efivar_query_variable_info(attr, &storage_space, - &remaining_space, - &max_variable_size); - if (status != EFI_SUCCESS && status != EFI_UNSUPPORTED) - pr_warn_ratelimited("query_variable_info() failed: 0x%lx\n", - status); + static DEFINE_RATELIMIT_STATE(_rs, 2 * HZ, 5); + static u64 storage, remaining; + static DEFINE_SPINLOCK(lock); + + if (!__ratelimit(&_rs)) { + ratelimit_set_flags(&_rs, RATELIMIT_MSG_ON_RELEASE); + + spin_lock(&lock); + storage_space = storage; + remaining_space = remaining; + spin_unlock(&lock); + } else { + status = efivar_query_variable_info(attr, &storage_space, + &remaining_space, + &max_variable_size); + if (status != EFI_SUCCESS && status != EFI_UNSUPPORTED) + pr_warn("query_variable_info() failed: 0x%lx\n", + status); + + spin_lock(&lock); + storage = storage_space; + remaining = remaining_space; + spin_unlock(&lock); + } } /* diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index d81f3318417df..473a22167c154 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -120,6 +120,23 @@ config EROFS_FS_ZIP_LZMA If unsure, say N. +config EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS + int "EROFS LZMA default maximum decompression streams" if SMP + depends on EROFS_FS_ZIP_LZMA + range 1 NR_CPUS if SMP + range 1 1 if !SMP + default 16 if SMP && NR_CPUS >= 16 + default NR_CPUS if SMP + default 1 + help + By default EROFS allocates one LZMA decompression stream per CPU. + Each stream can hold a dictionary of up to 8 MiB taken from the + mounted image, so on systems with many CPUs this can reserve a lot + of memory. This caps the default; the lzma_streams module parameter + still overrides it. + + If unsure, keep the suggested default (16 or lower). + config EROFS_FS_ZIP_DEFLATE bool "EROFS DEFLATE compressed data support" depends on EROFS_FS_ZIP diff --git a/fs/erofs/data.c b/fs/erofs/data.c index 8ca29962a3dde..3e6b512efe21c 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -46,7 +46,7 @@ void *erofs_bread(struct erofs_buf *buf, erofs_off_t offset, bool need_kmap) return NULL; if (!buf->base) buf->base = kmap_local_page(buf->page); - return buf->base + (offset & ~PAGE_MASK); + return buf->base + ((buf->off + offset) & ~PAGE_MASK); } int erofs_init_metabuf(struct erofs_buf *buf, struct super_block *sb, diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index e4e59a4e0d90d..48a0c84cec755 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -7,8 +7,6 @@ #include "compress.h" #include -#define LZ4_MAX_DISTANCE_PAGES (DIV_ROUND_UP(LZ4_DISTANCE_MAX, PAGE_SIZE) + 1) - static int z_erofs_load_lz4_config(struct super_block *sb, struct erofs_super_block *dsb, void *data, int size) { @@ -21,8 +19,6 @@ static int z_erofs_load_lz4_config(struct super_block *sb, erofs_err(sb, "invalid lz4 cfgs, size=%u", size); return -EINVAL; } - distance = le16_to_cpu(lz4->max_distance); - sbi->lz4.max_pclusterblks = le16_to_cpu(lz4->max_pclusterblks); if (!sbi->lz4.max_pclusterblks) { sbi->lz4.max_pclusterblks = 1; /* reserved case */ @@ -36,45 +32,25 @@ static int z_erofs_load_lz4_config(struct super_block *sb, distance = le16_to_cpu(dsb->u1.lz4_max_distance); sbi->lz4.max_pclusterblks = 1; } - - sbi->lz4.max_distance_pages = distance ? - DIV_ROUND_UP(distance, PAGE_SIZE) + 1 : - LZ4_MAX_DISTANCE_PAGES; return z_erofs_gbuf_growsize(sbi->lz4.max_pclusterblks); } /* - * Fill all gaps with bounce pages if it's a sparse page list. Also check if - * all physical pages are consecutive, which can be seen for moderate CR. + * Fill all gaps with bounce pages if it's a sparse page list (for example some + * folios are already uptodate and thus can be mapped into userspace). Also + * check if pages are physically consecutive, which can be seen for moderate CR. */ -static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq, - struct page **pagepool) +static int z_erofs_oneshot_prepare_dstpages(struct z_erofs_decompress_req *rq, + struct page **pagepool) { - struct page *availables[LZ4_MAX_DISTANCE_PAGES] = { NULL }; - unsigned long bounced[DIV_ROUND_UP(LZ4_MAX_DISTANCE_PAGES, - BITS_PER_LONG)] = { 0 }; - unsigned int lz4_max_distance_pages = - EROFS_SB(rq->sb)->lz4.max_distance_pages; void *kaddr = NULL; - unsigned int i, j, top; + unsigned int i; - top = 0; - for (i = j = 0; i < rq->outpages; ++i, ++j) { - struct page *const page = rq->out[i]; - struct page *victim; - - if (j >= lz4_max_distance_pages) - j = 0; - - /* 'valid' bounced can only be tested after a complete round */ - if (!rq->fillgaps && test_bit(j, bounced)) { - DBG_BUGON(i < lz4_max_distance_pages); - DBG_BUGON(top >= lz4_max_distance_pages); - availables[top++] = rq->out[i - lz4_max_distance_pages]; - } + for (i = 0; i < rq->outpages; ++i) { + struct page *page, *victim; + page = rq->out[i]; if (page) { - __clear_bit(j, bounced); if (!PageHighMem(page)) { if (!i) { kaddr = page_address(page); @@ -86,21 +62,14 @@ static int z_erofs_lz4_prepare_dstpages(struct z_erofs_decompress_req *rq, continue; } } - kaddr = NULL; - continue; - } - kaddr = NULL; - __set_bit(j, bounced); - - if (top) { - victim = availables[--top]; } else { victim = __erofs_allocpage(pagepool, rq->gfp, true); if (!victim) return -ENOMEM; set_page_private(victim, Z_EROFS_SHORTLIVED_PAGE); + rq->out[i] = victim; } - rq->out[i] = victim; + kaddr = NULL; } return kaddr ? 1 : 0; } @@ -142,7 +111,6 @@ static void *z_erofs_lz4_handle_overlap(const struct z_erofs_decompress_req *rq, oend = rq->pageofs_out + rq->outputsize; omargin = PAGE_ALIGN(oend) - oend; if (!rq->partial_decoding && may_inplace && - rq->outpages >= rq->inpages && omargin >= LZ4_DECOMPRESS_INPLACE_MARGIN(rq->inputsize)) { for (i = 0; i < rq->inpages; ++i) if (rq->out[rq->outpages - rq->inpages + i] != @@ -272,7 +240,7 @@ static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, dst_maptype = 0; } else { /* general decoding path which can be used for all cases */ - ret = z_erofs_lz4_prepare_dstpages(rq, pagepool); + ret = z_erofs_oneshot_prepare_dstpages(rq, pagepool); if (ret < 0) return ret; if (ret > 0) { @@ -305,6 +273,7 @@ static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq, return -EOPNOTSUPP; if (rq->alg == Z_EROFS_COMPRESSION_INTERLACED) { cur = bs - (rq->pageofs_out & (bs - 1)); + DBG_BUGON(rq->pageofs_in & (bs - 1)); pi = (rq->pageofs_in + rq->inputsize - cur) & ~PAGE_MASK; cur = min(cur, rq->outputsize); if (cur && rq->out[0]) { diff --git a/fs/erofs/decompressor_lzma.c b/fs/erofs/decompressor_lzma.c index 832cffb83a662..288d6a201da46 100644 --- a/fs/erofs/decompressor_lzma.c +++ b/fs/erofs/decompressor_lzma.c @@ -5,6 +5,7 @@ struct z_erofs_lzma { struct z_erofs_lzma *next; struct xz_dec_microlzma *state; + unsigned int dict_size; u8 bounce[PAGE_SIZE]; }; @@ -51,7 +52,8 @@ static int __init z_erofs_lzma_init(void) /* by default, use # of possible CPUs instead */ if (!z_erofs_lzma_nstrms) - z_erofs_lzma_nstrms = num_possible_cpus(); + z_erofs_lzma_nstrms = min_t(unsigned int, num_possible_cpus(), + CONFIG_EROFS_FS_ZIP_LZMA_DEFAULT_MAX_STREAMS); for (i = 0; i < z_erofs_lzma_nstrms; ++i) { struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL); @@ -127,11 +129,19 @@ static int z_erofs_load_lzma_config(struct super_block *sb, err = 0; /* 2. walk each isolated stream and grow max dict_size if needed */ for (strm = head; strm; strm = strm->next) { + struct xz_dec_microlzma *state; + + if (strm->dict_size >= dict_size) + continue; + state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); + if (!state) { + err = -ENOMEM; + break; + } if (strm->state) xz_dec_microlzma_end(strm->state); - strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size); - if (!strm->state) - err = -ENOMEM; + strm->state = state; + strm->dict_size = dict_size; } /* 3. push back all to the global list and update max dict_size */ @@ -141,7 +151,8 @@ static int z_erofs_load_lzma_config(struct super_block *sb, spin_unlock(&z_erofs_lzma_lock); wake_up_all(&z_erofs_lzma_wq); - z_erofs_lzma_max_dictsize = dict_size; + if (!err) + z_erofs_lzma_max_dictsize = dict_size; mutex_unlock(&lzma_resize_mutex); return err; } diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 87edbb4366d16..845c71afb58d5 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -70,12 +70,8 @@ struct erofs_dev_context { bool flatdev; }; -/* all filesystem-wide lz4 configurations */ struct erofs_sb_lz4_info { - /* # of pages needed for EROFS lz4 rolling decompression */ - u16 max_distance_pages; - /* maximum possible blocks for pclusters in the filesystem */ - u16 max_pclusterblks; + u16 max_pclusterblks; /* maximum physical blocks for LZ4 pclusters */ }; struct erofs_domain { @@ -286,7 +282,7 @@ struct erofs_inode { #ifdef CONFIG_EROFS_FS_ZIP struct { unsigned short z_advise; - unsigned char z_algorithmtype[2]; + unsigned char z_algofmt[2]; unsigned char z_lclusterbits; union { u64 z_tailextent_headlcn; @@ -338,20 +334,19 @@ static inline struct folio *erofs_grab_folio_nowait(struct address_space *as, readahead_gfp_mask(as) & ~__GFP_RECLAIM); } -/* Has a disk mapping */ -#define EROFS_MAP_MAPPED 0x0001 +/* Allocated on disk at @m_pa (e.g. NOT a fragment extent) */ +#define EROFS_MAP_MAPPED 0x0001 /* Located in metadata (could be copied from bd_inode) */ -#define EROFS_MAP_META 0x0002 -/* The extent is encoded */ -#define EROFS_MAP_ENCODED 0x0004 -/* The length of extent is full */ -#define EROFS_MAP_FULL_MAPPED 0x0008 +#define EROFS_MAP_META 0x0002 +/* @m_llen may be truncated by the runtime compared to the on-disk record */ +#define EROFS_MAP_PARTIAL_MAPPED 0x0004 +/* The on-disk @m_llen may cover only part of the encoded data */ +#define EROFS_MAP_PARTIAL_REF 0x0008 /* Located in the special packed inode */ -#define __EROFS_MAP_FRAGMENT 0x0010 -/* The extent refers to partial decompressed data */ -#define EROFS_MAP_PARTIAL_REF 0x0020 - -#define EROFS_MAP_FRAGMENT (EROFS_MAP_MAPPED | __EROFS_MAP_FRAGMENT) +#define EROFS_MAP_FRAGMENT 0x0010 +/* The encoded on-disk data will be fully handled (decompressed) */ +#define EROFS_MAP_FULL(f) (!((f) & (EROFS_MAP_PARTIAL_MAPPED | \ + EROFS_MAP_PARTIAL_REF))) struct erofs_map_blocks { struct erofs_buf buf; diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c index 86b22b9f0c191..60527159bfe76 100644 --- a/fs/erofs/sysfs.c +++ b/fs/erofs/sysfs.c @@ -86,7 +86,6 @@ static struct attribute *erofs_attrs[] = { ATTRIBUTE_GROUPS(erofs); /* Features this copy of erofs supports */ -EROFS_ATTR_FEATURE(zero_padding); EROFS_ATTR_FEATURE(compr_cfgs); EROFS_ATTR_FEATURE(big_pcluster); EROFS_ATTR_FEATURE(chunked_file); @@ -96,11 +95,11 @@ EROFS_ATTR_FEATURE(sb_chksum); EROFS_ATTR_FEATURE(ztailpacking); EROFS_ATTR_FEATURE(fragments); EROFS_ATTR_FEATURE(dedupe); +EROFS_ATTR_FEATURE(xattr_prefixes); EROFS_ATTR_FEATURE(48bit); EROFS_ATTR_FEATURE(metabox); static struct attribute *erofs_feat_attrs[] = { - ATTR_LIST(zero_padding), ATTR_LIST(compr_cfgs), ATTR_LIST(big_pcluster), ATTR_LIST(chunked_file), @@ -110,6 +109,7 @@ static struct attribute *erofs_feat_attrs[] = { ATTR_LIST(ztailpacking), ATTR_LIST(fragments), ATTR_LIST(dedupe), + ATTR_LIST(xattr_prefixes), ATTR_LIST(48bit), ATTR_LIST(metabox), NULL, diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index a02ce7c06f9e1..0af259e22026f 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -128,7 +128,17 @@ struct z_erofs_pcluster_slab { #define _PCLP(n) { .maxpages = n } static struct z_erofs_pcluster_slab pcluster_pool[] __read_mostly = { - _PCLP(1), _PCLP(4), _PCLP(16), _PCLP(64), _PCLP(128), + _PCLP(1), + _PCLP(4), +#if Z_EROFS_PCLUSTER_MAX_PAGES > 16 + _PCLP(16), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 64 + _PCLP(64), +#endif +#if Z_EROFS_PCLUSTER_MAX_PAGES > 128 + _PCLP(128), +#endif _PCLP(Z_EROFS_PCLUSTER_MAX_PAGES + 1) }; @@ -520,7 +530,7 @@ static bool z_erofs_should_alloc_cache(struct z_erofs_frontend *fe) if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED) return false; - if (!(fe->map.m_flags & EROFS_MAP_FULL_MAPPED)) + if (fe->map.m_flags & EROFS_MAP_PARTIAL_MAPPED) return true; if (cachestrategy >= EROFS_ZIP_CACHE_READAROUND && @@ -1034,10 +1044,7 @@ static int z_erofs_scan_folio(struct z_erofs_frontend *f, /* bump split parts first to avoid several separate cases */ ++split; - if (!(map->m_flags & EROFS_MAP_MAPPED)) { - folio_zero_segment(folio, cur, end); - tight = false; - } else if (map->m_flags & __EROFS_MAP_FRAGMENT) { + if (map->m_flags & EROFS_MAP_FRAGMENT) { erofs_off_t fpos = offset + cur - map->m_la; err = z_erofs_read_fragment(inode->i_sb, folio, cur, @@ -1046,6 +1053,9 @@ static int z_erofs_scan_folio(struct z_erofs_frontend *f, if (err) break; tight = false; + } else if (!(map->m_flags & EROFS_MAP_MAPPED)) { + folio_zero_segment(folio, cur, end); + tight = false; } else { if (!f->pcl) { err = z_erofs_pcluster_begin(f); @@ -1081,14 +1091,13 @@ static int z_erofs_scan_folio(struct z_erofs_frontend *f, f->pcl->length = offset + end - map->m_la; f->pcl->pageofs_out = map->m_la & ~PAGE_MASK; } - if ((map->m_flags & EROFS_MAP_FULL_MAPPED) && - !(map->m_flags & EROFS_MAP_PARTIAL_REF) && + if (EROFS_MAP_FULL(map->m_flags) && f->pcl->length == map->m_llen) f->pcl->partial = false; } /* shorten the remaining extent to update progress */ map->m_llen = offset + cur - map->m_la; - map->m_flags &= ~EROFS_MAP_FULL_MAPPED; + map->m_flags |= EROFS_MAP_PARTIAL_MAPPED; if (cur <= pgs) { split = cur < pgs; tight = (bs == PAGE_SIZE); @@ -1258,7 +1267,7 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) const struct z_erofs_decompressor *decomp = z_erofs_decomp[pcl->algorithmformat]; bool try_free = true; - int i, j, jtop, err2, err = eio ? -EIO : 0; + int i, err2, err = eio ? -EIO : 0; struct page *page; bool overlapped; @@ -1333,7 +1342,6 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) be->compressed_pages >= be->onstack_pages + Z_EROFS_ONSTACK_PAGES) kvfree(be->compressed_pages); - jtop = 0; z_erofs_fill_other_copies(be, err); for (i = 0; i < be->nr_pages; ++i) { page = be->decompressed_pages[i]; @@ -1341,22 +1349,11 @@ static int z_erofs_decompress_pcluster(struct z_erofs_backend *be, bool eio) continue; DBG_BUGON(z_erofs_page_is_invalidated(page)); - if (!z_erofs_is_shortlived_page(page)) { + if (!z_erofs_is_shortlived_page(page)) erofs_onlinefolio_end(page_folio(page), err, true); - continue; - } - if (pcl->algorithmformat != Z_EROFS_COMPRESSION_LZ4) { + else erofs_pagepool_add(be->pagepool, page); - continue; - } - for (j = 0; j < jtop && be->decompressed_pages[j] != page; ++j) - ; - if (j >= jtop) /* this bounce page is newly detected */ - be->decompressed_pages[jtop++] = page; } - while (jtop) - erofs_pagepool_add(be->pagepool, - be->decompressed_pages[--jtop]); if (be->decompressed_pages != be->onstack_pages) kvfree(be->decompressed_pages); @@ -1829,7 +1826,7 @@ static void z_erofs_pcluster_readmore(struct z_erofs_frontend *f, map->m_la = end; err = z_erofs_map_blocks_iter(inode, map, EROFS_GET_BLOCKS_READMORE); - if (err || !(map->m_flags & EROFS_MAP_ENCODED)) + if (err || !(map->m_flags & EROFS_MAP_MAPPED)) return; /* expand ra for the trailing edge if readahead */ @@ -1841,7 +1838,7 @@ static void z_erofs_pcluster_readmore(struct z_erofs_frontend *f, end = round_up(end, PAGE_SIZE); } else { end = round_up(map->m_la, PAGE_SIZE); - if (!(map->m_flags & EROFS_MAP_ENCODED) || !map->m_llen) + if (!(map->m_flags & EROFS_MAP_MAPPED) || !map->m_llen) return; } diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index abf7ddc64c63b..b5d888883213c 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -418,7 +418,7 @@ static int z_erofs_map_blocks_fo(struct inode *inode, if ((flags & EROFS_GET_BLOCKS_FINDTAIL) && ztailpacking) vi->z_fragmentoff = m.nextpackoff; - map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; end = (m.lcn + 1ULL) << lclusterbits; if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && endoff >= m.clusterofs) { @@ -434,7 +434,7 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } else { if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { end = (m.lcn << lclusterbits) | m.clusterofs; - map->m_flags |= EROFS_MAP_FULL_MAPPED; + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; m.delta[0] = 1; } /* get the corresponding first chunk */ @@ -472,19 +472,14 @@ static int z_erofs_map_blocks_fo(struct inode *inode, } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { - if (map->m_llen > map->m_plen) { - DBG_BUGON(1); - err = -EFSCORRUPTED; - goto unmap_out; - } - if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) + if ((vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) && + !(map->m_flags & EROFS_MAP_META)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; - } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { - map->m_algorithmformat = vi->z_algorithmtype[1]; } else { - map->m_algorithmformat = vi->z_algorithmtype[0]; + map->m_algorithmformat = + vi->z_algofmt[m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || @@ -495,7 +490,7 @@ static int z_erofs_map_blocks_fo(struct inode *inode, map->m_llen >= i_blocksize(inode))) { err = z_erofs_get_extent_decompressedlen(&m); if (!err) - map->m_flags |= EROFS_MAP_FULL_MAPPED; + map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; } unmap_out: @@ -593,15 +588,19 @@ static int z_erofs_map_blocks_ext(struct inode *inode, if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) vi->z_fragmentoff |= map->m_pa << 32; } else if (map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) { - map->m_flags |= EROFS_MAP_MAPPED | - EROFS_MAP_FULL_MAPPED | EROFS_MAP_ENCODED; + map->m_flags |= EROFS_MAP_MAPPED; fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; - if (fmt) - map->m_algorithmformat = fmt - 1; - else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) + if (fmt) { + map->m_algorithmformat = --fmt; + if (fmt >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", + fmt, map->m_la, vi->nid); + return -EOPNOTSUPP; + } + } else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else @@ -619,7 +618,7 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; - int err = 0; + int err = 0, nr; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* @@ -662,12 +661,19 @@ static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) goto done; } - vi->z_algorithmtype[0] = h->h_algorithmtype & 15; - vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); + for (nr = 0; nr < 2; ++nr) { + vi->z_algofmt[nr] = (h->h_algorithmtype >> (4 * nr)) & 15; + if (vi->z_algofmt[nr] >= Z_EROFS_COMPRESSION_MAX) { + erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel", + nr + 1, vi->z_algofmt[nr], vi->nid); + err = -EOPNOTSUPP; + goto out_unlock; + } + } if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | @@ -713,17 +719,25 @@ static int z_erofs_map_sanity_check(struct inode *inode, struct erofs_sb_info *sbi = EROFS_I_SB(inode); u64 pend; - if (!(map->m_flags & EROFS_MAP_ENCODED)) + if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; - if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { - erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", - map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); - return -EOPNOTSUPP; - } - if (unlikely(map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX && - !(sbi->available_compr_algs & (1 << map->m_algorithmformat)))) { - erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", - map->m_algorithmformat, EROFS_I(inode)->nid); + + DBG_BUGON(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX); + if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { + if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { + erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", + map->m_algorithmformat, EROFS_I(inode)->nid); + return -EFSCORRUPTED; + } + if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && + map->m_la + map->m_llen < inode->i_size) { + erofs_err(inode->i_sb, "too much compressed data @ la %llu of nid %llu", + map->m_la, EROFS_I(inode)->nid); + return -EFSCORRUPTED; + } + } else if (map->m_llen > map->m_plen) { + erofs_err(inode->i_sb, "not enough plain data on disk @ la %llu of nid %llu", + map->m_la, EROFS_I(inode)->nid); return -EFSCORRUPTED; } if (unlikely(map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || @@ -780,10 +794,12 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, iomap->bdev = inode->i_sb->s_bdev; iomap->offset = map.m_la; iomap->length = map.m_llen; - if (map.m_flags & EROFS_MAP_MAPPED) { + if (map.m_flags & EROFS_MAP_FRAGMENT) { + iomap->type = IOMAP_MAPPED; + iomap->addr = IOMAP_NULL_ADDR; + } else if (map.m_flags & EROFS_MAP_MAPPED) { iomap->type = IOMAP_MAPPED; - iomap->addr = map.m_flags & __EROFS_MAP_FRAGMENT ? - IOMAP_NULL_ADDR : map.m_pa; + iomap->addr = map.m_pa; } else { iomap->type = IOMAP_HOLE; iomap->addr = IOMAP_NULL_ADDR; diff --git a/fs/erofs/zutil.c b/fs/erofs/zutil.c index 55ff2ab5128e0..54ce492e897f3 100644 --- a/fs/erofs/zutil.c +++ b/fs/erofs/zutil.c @@ -79,6 +79,8 @@ int z_erofs_gbuf_growsize(unsigned int nrpages) for (i = 0; i < z_erofs_gbuf_count; ++i) { gbuf = &z_erofs_gbufpool[i]; + if (gbuf->nrpages >= nrpages) + continue; tmp_pages = kcalloc(nrpages, sizeof(*tmp_pages), GFP_KERNEL); if (!tmp_pages) goto out; diff --git a/fs/exec.c b/fs/exec.c index 4298e7e08d5d7..c0eab916a3e08 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -736,7 +737,7 @@ int transfer_args_to_stack(struct linux_binprm *bprm, stop = bprm->p >> PAGE_SHIFT; sp = *sp_location; - for (index = MAX_ARG_PAGES - 1; index >= stop; index--) { + for (index = MAX_ARG_PAGES; index-- > stop; ) { unsigned int offset = index == stop ? bprm->p & ~PAGE_MASK : 0; char *src = kmap_local_page(bprm->page[index]) + offset; sp -= PAGE_SIZE - offset; @@ -851,6 +852,7 @@ static int exec_mmap(struct mm_struct *mm) /* Notify parent that we're no longer interested in the old VM */ tsk = current; old_mm = current->mm; + /* Clean up futexes and release the mm */ exec_mm_release(tsk, old_mm); ret = down_write_killable(&tsk->signal->exec_update_lock); @@ -899,9 +901,11 @@ static int exec_mmap(struct mm_struct *mm) setmax_mm_hiwater_rss(&tsk->signal->maxrss, old_mm); mm_update_next_owner(old_mm); mmput(old_mm); - return 0; + } else { + mmdrop_lazy_tlb(active_mm); } - mmdrop_lazy_tlb(active_mm); + + futex_exec_done(tsk); return 0; } @@ -1135,6 +1139,20 @@ int begin_new_exec(struct linux_binprm * bprm) if (retval) goto out; + /* + * We have to apply CLOEXEC before we change whether the process is + * dumpable (in setup_new_exec) to avoid a race with a process in userspace + * trying to access the should-be-closed file descriptors of a process + * undergoing exec(2). + * + * This can block on filesystem ->flush() handlers, including waiting + * for FUSE daemons, so do it before exec_mmap takes the + * exec_update_lock. + * This must happen after the point of no return, and after unsharing + * the FD table. + */ + do_close_on_exec(me->files); + /* * Must be called _before_ exec_mmap() as bprm->mm is * not visible until then. Doing it here also ensures @@ -1185,14 +1203,6 @@ int begin_new_exec(struct linux_binprm * bprm) clear_syscall_work_syscall_user_dispatch(me); - /* - * We have to apply CLOEXEC before we change whether the process is - * dumpable (in setup_new_exec) to avoid a race with a process in userspace - * trying to access the should-be-closed file descriptors of a process - * undergoing exec(2). - */ - do_close_on_exec(me->files); - if (bprm->secureexec) { /* Make sure parent cannot signal privileged process. */ me->pdeath_signal = 0; diff --git a/fs/exfat/nls.c b/fs/exfat/nls.c index 57db08a5271cf..055447edcf9a6 100644 --- a/fs/exfat/nls.c +++ b/fs/exfat/nls.c @@ -769,13 +769,18 @@ int exfat_create_upcase_table(struct super_block *sb) tbl_clu = le32_to_cpu(ep->dentry.upcase.start_clu); tbl_size = le64_to_cpu(ep->dentry.upcase.size); - - sector = exfat_cluster_to_sector(sbi, tbl_clu); - num_sectors = ((tbl_size - 1) >> blksize_bits) + 1; - ret = exfat_load_upcase_table(sb, sector, num_sectors, - le32_to_cpu(ep->dentry.upcase.checksum)); - + if (tbl_size) { + sector = exfat_cluster_to_sector(sbi, tbl_clu); + num_sectors = ((tbl_size - 1) >> blksize_bits) + 1; + ret = exfat_load_upcase_table(sb, sector, num_sectors, + le32_to_cpu(ep->dentry.upcase.checksum)); + } else { + exfat_fs_error(sb, + "bad upcase table size (0 bytes). Please run fsck"); + ret = -EINVAL; + } brelse(bh); + if (ret && ret != -EIO) { /* free memory from exfat_load_upcase_table call */ exfat_free_upcase_table(sbi); @@ -790,6 +795,8 @@ int exfat_create_upcase_table(struct super_block *sb) return -EIO; } + exfat_fs_error(sb, "no upcase table entry. Please run fsck"); + load_default: /* load default upcase table */ return exfat_load_default_upcase_table(sb); diff --git a/fs/ext4/crypto.c b/fs/ext4/crypto.c index f41f320f4437b..3971986de0289 100644 --- a/fs/ext4/crypto.c +++ b/fs/ext4/crypto.c @@ -144,7 +144,13 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, if (inode->i_ino == EXT4_ROOT_INO) return -EPERM; - if (WARN_ON_ONCE(IS_DAX(inode) && i_size_read(inode))) + /* + * For new encrypted inodes, S_DAX is never set in the first place. + * + * For existing inodes, this is called only on empty directories. ext4 + * never sets S_DAX on directories. + */ + if (WARN_ON_ONCE(IS_DAX(inode))) return -EINVAL; if (ext4_test_inode_flag(inode, EXT4_INODE_DAX)) @@ -163,6 +169,14 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, */ if (handle) { + /* + * __ext4_new_inode() should have already set the encrypt flag + * on the inode and avoided enabling inline data. + */ + if (WARN_ON_ONCE(!IS_ENCRYPTED(inode))) + return -EINVAL; + if (WARN_ON_ONCE(ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))) + return -EINVAL; /* * Since the inode is new it is ok to pass the * XATTR_CREATE flag. This is necessary to match the @@ -170,21 +184,10 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, * function with the credits allocated for the new * inode. */ - res = ext4_xattr_set_handle(handle, inode, - EXT4_XATTR_INDEX_ENCRYPTION, - EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, - ctx, len, XATTR_CREATE); - if (!res) { - ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - ext4_clear_inode_state(inode, - EXT4_STATE_MAY_INLINE_DATA); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ - ext4_set_inode_flags(inode, false); - } - return res; + return ext4_xattr_set_handle(handle, inode, + EXT4_XATTR_INDEX_ENCRYPTION, + EXT4_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); } res = dquot_initialize(inode); @@ -205,10 +208,7 @@ static int ext4_set_context(struct inode *inode, const void *ctx, size_t len, ctx, len, 0); if (!res) { ext4_set_inode_flag(inode, EXT4_INODE_ENCRYPT); - /* - * Update inode->i_flags - S_ENCRYPTED will be enabled, - * S_DAX may be disabled - */ + /* Update inode->i_flags to set S_ENCRYPTED. */ ext4_set_inode_flags(inode, false); res = ext4_mark_inode_dirty(handle, inode); if (res) diff --git a/fs/ext4/dir.c b/fs/ext4/dir.c index d4164c507a907..290a9aeac15f1 100644 --- a/fs/ext4/dir.c +++ b/fs/ext4/dir.c @@ -137,6 +137,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) struct buffer_head *bh = NULL; struct fscrypt_str fstr = FSTR_INIT(NULL, 0); struct dir_private_info *info = file->private_data; + bool has_csum = ext4_has_feature_metadata_csum(sb); err = fscrypt_prepare_readdir(inode); if (err) @@ -148,7 +149,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) return err; /* Can we just clear INDEX flag to ignore htree information? */ - if (!ext4_has_feature_metadata_csum(sb)) { + if (!has_csum) { /* * We don't set the inode dirty flag since it's not * critical that it gets flushed back to the disk. @@ -234,7 +235,10 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) * dirent right now. Scan from the start of the block * to make sure. */ if (!inode_eq_iversion(inode, info->cookie)) { - for (i = 0; i < sb->s_blocksize && i < offset; ) { + for (i = 0; + i <= sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode) && + i < offset;) { de = (struct ext4_dir_entry_2 *) (bh->b_data + i); /* It's too expensive to do a full @@ -256,6 +260,17 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) info->cookie = inode_query_iversion(inode); } + if (unlikely(offset < sb->s_blocksize && + offset > sb->s_blocksize - + ext4_dir_rec_len(1, has_csum ? NULL : inode))) { + EXT4_ERROR_FILE(file, bh->b_blocknr, + "bad entry in directory: %s - offset=%u, size=%lu", + "directory entry too close to block end", + offset, sb->s_blocksize); + ctx->pos = round_up(ctx->pos, sb->s_blocksize); + goto next_block; + } + while (ctx->pos < inode->i_size && offset < sb->s_blocksize) { de = (struct ext4_dir_entry_2 *) (bh->b_data + offset); @@ -311,6 +326,7 @@ static int ext4_readdir(struct file *file, struct dir_context *ctx) ctx->pos += ext4_rec_len_from_disk(de->rec_len, sb->s_blocksize); } +next_block: if ((ctx->pos < inode->i_size) && !dir_relax_shared(inode)) goto done; brelse(bh); diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index 54be698b9a1c1..996ffdc2dd13d 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h @@ -3074,6 +3074,7 @@ int do_journal_get_write_access(handle_t *handle, struct inode *inode, void ext4_set_inode_mapping_order(struct inode *inode); #define FALL_BACK_TO_NONDELALLOC 1 #define CONVERT_INLINE_DATA 2 +#define EXT4_WRITE_DATA_INLINE 4 typedef enum { EXT4_IGET_NORMAL = 0, diff --git a/fs/ext4/fast_commit.c b/fs/ext4/fast_commit.c index 2c4e628acb8d1..6b87aecedb082 100644 --- a/fs/ext4/fast_commit.c +++ b/fs/ext4/fast_commit.c @@ -1797,8 +1797,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, if (ret == 0) { /* Range is not mapped */ path = ext4_find_extent(inode, cur, path, 0); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } memset(&newex, 0, sizeof(newex)); newex.ee_block = cpu_to_le32(cur); ext4_ext_store_pblock( @@ -1810,8 +1813,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, path = ext4_ext_insert_extent(NULL, inode, path, &newex, 0); up_write((&EXT4_I(inode)->i_data_sem)); - if (IS_ERR(path)) + if (IS_ERR(path)) { + ret = PTR_ERR(path); + path = NULL; goto out; + } goto next; } @@ -1858,10 +1864,11 @@ static int ext4_fc_replay_add_range(struct super_block *sb, } ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); + ret = 0; out: ext4_free_ext_path(path); iput(inode); - return 0; + return ret; } /* Replay DEL_RANGE tag */ @@ -1922,9 +1929,10 @@ ext4_fc_replay_del_range(struct super_block *sb, ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >> sb->s_blocksize_bits); ext4_mark_inode_dirty(NULL, inode); + ret = 0; out: iput(inode); - return 0; + return ret; } static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb) diff --git a/fs/ext4/file.c b/fs/ext4/file.c index 7a8b309321892..149a7fe5a0962 100644 --- a/fs/ext4/file.c +++ b/fs/ext4/file.c @@ -292,6 +292,13 @@ static ssize_t ext4_buffered_write_iter(struct kiocb *iocb, return -EOPNOTSUPP; inode_lock(inode); + + /* + * Prevent concurrent direct I/O and buffered I/O to the same file + * range. Wait for in-flight DIO to finish before dirtying pages. + */ + inode_dio_wait(inode); + ret = ext4_write_checks(iocb, from); if (ret <= 0) goto out; diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c index b1bc1950c9f03..d8620dfd44ff9 100644 --- a/fs/ext4/ialloc.c +++ b/fs/ext4/ialloc.c @@ -995,6 +995,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto out; + if (encrypt) + i_flags |= EXT4_ENCRYPT_FL; } err = dquot_initialize(inode); @@ -1304,6 +1306,8 @@ struct inode *__ext4_new_inode(struct mnt_idmap *idmap, ei->i_extra_isize = sbi->s_want_extra_isize; ei->i_inline_off = 0; if (ext4_has_feature_inline_data(sb) && + /* Encrypted inodes cannot have inline data */ + !(ei->i_flags & EXT4_ENCRYPT_FL) && (!(ei->i_flags & (EXT4_DAX_FL|EXT4_EA_INODE_FL)) || S_ISDIR(mode))) ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); ret = inode; diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c index 408677fa81967..672085e4ec24f 100644 --- a/fs/ext4/inline.c +++ b/fs/ext4/inline.c @@ -812,7 +812,19 @@ int ext4_write_inline_data_end(struct inode *inode, loff_t pos, unsigned len, goto out; } ext4_write_lock_xattr(inode, &no_expand); - BUG_ON(!ext4_has_inline_data(inode)); + /* + * We could have raced with ext4_page_mkwrite() converting + * the inode and clearing the inline data flag, so we just + * release resources and retry the whole write. + */ + if (unlikely(!ext4_has_inline_data(inode))) { + ext4_write_unlock_xattr(inode, &no_expand); + brelse(iloc.bh); + folio_unlock(folio); + folio_put(folio); + ext4_journal_stop(handle); + return 0; + } /* * ei->i_inline_off may have changed since @@ -1454,6 +1466,8 @@ int ext4_read_inline_dir(struct file *file, /* for other entry, the real offset in * the buf has to be tuned accordingly. */ + if (i + ext4_dir_rec_len(1, NULL) > extra_size) + break; de = (struct ext4_dir_entry_2 *) (dir_buf + i - extra_offset); /* It's too expensive to do a full @@ -1488,10 +1502,17 @@ int ext4_read_inline_dir(struct file *file, continue; } + /* + * de lives at dir_buf + ctx->pos - extra_offset, within the + * kmalloc(inline_size) buffer. Make sure its header fits before + * ext4_check_dir_entry() dereferences de->rec_len. + */ + if (ctx->pos + ext4_dir_rec_len(1, NULL) > extra_size) + goto out; de = (struct ext4_dir_entry_2 *) (dir_buf + ctx->pos - extra_offset); if (ext4_check_dir_entry(inode, file, de, iloc.bh, dir_buf, - extra_size, ctx->pos)) + inline_size, ctx->pos)) goto out; if (le32_to_cpu(de->inode)) { if (!dir_emit(ctx, de->name, de->name_len, diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index c38ca1441b25f..3b2891f185409 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c @@ -1179,6 +1179,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, int nr_wait = 0; int i; bool should_journal_data = ext4_should_journal_data(inode); + bool folio_uptodate = folio_test_uptodate(folio); BUG_ON(!folio_test_locked(folio)); BUG_ON(to > folio_size(folio)); @@ -1190,13 +1191,13 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, bbits = ilog2(blocksize); block = (sector_t)folio->index << (PAGE_SHIFT - bbits); - for (bh = head, block_start = 0; bh != head || !block_start; + for (bh = head, block_start = 0; + block_start < to || (!folio_uptodate && bh != head); block++, block_start = block_end, bh = bh->b_this_page) { block_end = block_start + blocksize; if (block_end <= from || block_start >= to) { - if (folio_test_uptodate(folio)) { + if (folio_uptodate) set_buffer_uptodate(bh); - } continue; } if (WARN_ON_ONCE(buffer_new(bh))) @@ -1217,7 +1218,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, if (should_journal_data) do_journal_get_write_access(handle, inode, bh); - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { /* * Unlike __block_write_begin() we leave * dirtying of new uptodate buffers to @@ -1234,7 +1235,7 @@ int ext4_block_write_begin(handle_t *handle, struct folio *folio, continue; } } - if (folio_test_uptodate(folio)) { + if (folio_uptodate) { set_buffer_uptodate(bh); continue; } @@ -1299,6 +1300,8 @@ static int ext4_write_begin(const struct kiocb *iocb, if (unlikely(ret)) return ret; + *fsdata = (void *)((unsigned long)*fsdata & ~EXT4_WRITE_DATA_INLINE); + trace_ext4_write_begin(inode, pos, len); /* * Reserve one block more for addition to orphan list in case @@ -1313,8 +1316,10 @@ static int ext4_write_begin(const struct kiocb *iocb, foliop); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } /* @@ -1450,8 +1455,7 @@ static int ext4_write_end(const struct kiocb *iocb, trace_ext4_write_end(inode, pos, len, copied); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -1561,8 +1565,7 @@ static int ext4_journalled_write_end(const struct kiocb *iocb, BUG_ON(!ext4_handle_valid(handle)); - if (ext4_has_inline_data(inode) && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) + if ((unsigned long)fsdata & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -2686,13 +2689,25 @@ static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd) * page is already under writeback and we are not doing * a data integrity writeback, skip the page */ - if (!folio_test_dirty(folio) || - (folio_test_writeback(folio) && - (mpd->wbc->sync_mode == WB_SYNC_NONE)) || + if ((folio_test_writeback(folio) && + mpd->wbc->sync_mode == WB_SYNC_NONE) || unlikely(folio->mapping != mapping)) { folio_unlock(folio); continue; } + /* + * If the folio is clean, skip writing it back. + * Cycle the folio through the writeback state + * though, to clear stale xarray tags. + */ + if (!folio_test_dirty(folio)) { + if (!folio_test_writeback(folio)) { + __folio_start_writeback(folio, false); + folio_end_writeback(folio); + } + folio_unlock(folio); + continue; + } folio_wait_writeback(folio); BUG_ON(folio_test_writeback(folio)); @@ -3148,8 +3163,10 @@ static int ext4_da_write_begin(const struct kiocb *iocb, foliop, fsdata, true); if (ret < 0) return ret; - if (ret == 1) + if (ret == 1) { + *fsdata = (void *)((unsigned long)*fsdata | EXT4_WRITE_DATA_INLINE); return 0; + } } retry: @@ -3282,17 +3299,15 @@ static int ext4_da_write_end(const struct kiocb *iocb, struct folio *folio, void *fsdata) { struct inode *inode = mapping->host; - int write_mode = (int)(unsigned long)fsdata; + unsigned long write_mode = (unsigned long)fsdata; - if (write_mode == FALL_BACK_TO_NONDELALLOC) + if (write_mode & FALL_BACK_TO_NONDELALLOC) return ext4_write_end(iocb, mapping, pos, len, copied, folio, fsdata); trace_ext4_da_write_end(inode, pos, len, copied); - if (write_mode != CONVERT_INLINE_DATA && - ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) && - ext4_has_inline_data(inode)) + if (write_mode & EXT4_WRITE_DATA_INLINE) return ext4_write_inline_data_end(inode, pos, len, copied, folio); @@ -6433,6 +6448,16 @@ static int ext4_try_to_expand_extra_isize(struct inode *inode, if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) return -EOVERFLOW; + /* + * Skip expansion during mount (!SB_ACTIVE). Expanding extra isize + * may move xattrs to external blocks and release ea_inodes via iput. + * When !SB_ACTIVE, iput triggers write_inode_now() which acquires + * s_writepages_rwsem, causing a deadlock with the caller's active + * jbd2 handle (lock order: s_writepages_rwsem -> jbd2_handle). + */ + if (unlikely(!(inode->i_sb->s_flags & SB_ACTIVE))) + return -EBUSY; + /* * In nojournal mode, we can immediately attempt to expand * the inode. When journaled, we first need to obtain extra diff --git a/fs/ext4/migrate.c b/fs/ext4/migrate.c index 1b0dfd963d3f0..84e3b703ab463 100644 --- a/fs/ext4/migrate.c +++ b/fs/ext4/migrate.c @@ -458,6 +458,7 @@ int ext4_ext_migrate(struct inode *inode) if (IS_ERR(tmp_inode)) { retval = PTR_ERR(tmp_inode); ext4_journal_stop(handle); + tmp_inode = NULL; goto out_unlock; } /* @@ -585,9 +586,9 @@ int ext4_ext_migrate(struct inode *inode) ext4_journal_stop(handle); out_tmp_inode: unlock_new_inode(tmp_inode); - iput(tmp_inode); out_unlock: ext4_writepages_up_write(inode->i_sb, alloc_ctx); + iput(tmp_inode); return retval; } diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c index 2e35453a56f28..2c661155158f8 100644 --- a/fs/ext4/namei.c +++ b/fs/ext4/namei.c @@ -1467,6 +1467,8 @@ int ext4_search_dir(struct buffer_head *bh, char *search_buf, int buf_size, /* this code is executed quadratically often */ /* do minimal checking `by hand' */ if (de->name + de->name_len <= dlimit && + (!ext4_hash_in_dirent(dir) || + (char *)de + ext4_dir_rec_len(de->name_len, dir) <= dlimit) && ext4_match(dir, fname, de)) { /* found a match - just to be sure, do * a full check */ diff --git a/fs/ext4/orphan.c b/fs/ext4/orphan.c index fb57bba0d19d1..47289d17152b8 100644 --- a/fs/ext4/orphan.c +++ b/fs/ext4/orphan.c @@ -388,7 +388,7 @@ void ext4_orphan_cleanup(struct super_block *sb, struct ext4_super_block *es) struct ext4_orphan_info *oi = &EXT4_SB(sb)->s_orphan_info; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); - if (!es->s_last_orphan && !oi->of_blocks) { + if (!es->s_last_orphan && ext4_orphan_file_empty(sb)) { ext4_debug("no orphan inodes to clean up\n"); return; } @@ -572,6 +572,7 @@ int ext4_init_orphan_info(struct super_block *sb) int i, j; int ret; int free; + int loaded = 0; __le32 *bdata; int inodes_per_ob = ext4_inodes_per_orphan_block(sb); struct ext4_orphan_block_tail *ot; @@ -615,6 +616,7 @@ int ext4_init_orphan_info(struct super_block *sb) ret = -EIO; goto out_free; } + loaded++; ot = ext4_orphan_block_tail(sb, oi->of_binfo[i].ob_bh); if (le32_to_cpu(ot->ob_magic) != EXT4_ORPHAN_BLOCK_MAGIC) { ext4_error(sb, "orphan file block %d: bad magic", i); @@ -637,8 +639,10 @@ int ext4_init_orphan_info(struct super_block *sb) iput(inode); return 0; out_free: - for (i--; i >= 0; i--) - brelse(oi->of_binfo[i].ob_bh); + while (loaded > 0) { + loaded--; + brelse(oi->of_binfo[loaded].ob_bh); + } kvfree(oi->of_binfo); out_put: iput(inode); diff --git a/fs/ext4/super.c b/fs/ext4/super.c index cf4285db7fb8b..9b45d27515f9f 100644 --- a/fs/ext4/super.c +++ b/fs/ext4/super.c @@ -5291,7 +5291,7 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb) ext4_set_def_opts(sb, es); sbi->s_resuid = make_kuid(&init_user_ns, ext4_get_resuid(es)); - sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resuid(es)); + sbi->s_resgid = make_kgid(&init_user_ns, ext4_get_resgid(es)); sbi->s_commit_interval = JBD2_DEFAULT_MAX_COMMIT_AGE * HZ; sbi->s_min_batch_time = EXT4_DEF_MIN_BATCH_TIME; sbi->s_max_batch_time = EXT4_DEF_MAX_BATCH_TIME; diff --git a/fs/ext4/xattr.c b/fs/ext4/xattr.c index f0522a4538d19..02095ca17be0f 100644 --- a/fs/ext4/xattr.c +++ b/fs/ext4/xattr.c @@ -2075,12 +2075,13 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, * stable so we can check the additional * reference fits. */ - ref = le32_to_cpu(BHDR(new_bh)->h_refcount) + 1; - if (ref > EXT4_XATTR_REFCOUNT_MAX) { + ref = le32_to_cpu(BHDR(new_bh)->h_refcount); + if (ref >= EXT4_XATTR_REFCOUNT_MAX) { /* * Undo everything and check mbcache * again. */ + clear_bit(MBE_REUSABLE_B, &ce->e_flags); unlock_buffer(new_bh); dquot_free_block(inode, EXT4_C2B(EXT4_SB(sb), @@ -2091,6 +2092,7 @@ ext4_xattr_block_set(handle_t *handle, struct inode *inode, new_bh = NULL; goto inserted; } + ref++; BHDR(new_bh)->h_refcount = cpu_to_le32(ref); if (ref == EXT4_XATTR_REFCOUNT_MAX) clear_bit(MBE_REUSABLE_B, &ce->e_flags); @@ -2839,6 +2841,7 @@ int ext4_expand_extra_isize_ea(struct inode *inode, int new_extra_isize, s_min_extra_isize) { tried_min_extra_isize++; new_extra_isize = s_min_extra_isize; + error = 0; goto retry; } goto cleanup; diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index 872169b91913c..05ab77446b716 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c @@ -1453,8 +1453,11 @@ static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) old_blkaddr = dn->data_blkaddr; err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, &sum, seg_type, NULL); - if (err) + if (err) { + if (old_blkaddr == NULL_ADDR) + dec_valid_block_count(sbi, dn->inode, count); return err; + } if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c index 43a83bbd3bc5a..0650af1305577 100644 --- a/fs/f2fs/debug.c +++ b/fs/f2fs/debug.c @@ -351,10 +351,6 @@ static void update_mem_info(struct f2fs_sb_info *sbi) get_cache: si->cache_mem = 0; - /* build gc */ - if (sbi->gc_thread) - si->cache_mem += sizeof(struct f2fs_gc_kthread); - /* build merge flush thread */ if (SM_I(sbi)->fcc_info) si->cache_mem += sizeof(struct flush_cmd_control); diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index 48f4f98afb013..86ddd5f02bb65 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c @@ -313,6 +313,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir, de = find_in_block(dir, dentry_folio, fname, &max_slots, use_hash); if (IS_ERR(de)) { + f2fs_folio_put(dentry_folio, false); *res_folio = ERR_CAST(de); de = NULL; break; @@ -453,7 +454,7 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, folio_mark_dirty(folio); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); f2fs_folio_put(folio, true); } @@ -608,7 +609,7 @@ void f2fs_update_parent_metadata(struct inode *dir, struct inode *inode, clear_inode_flag(inode, FI_NEW_INODE); } inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (F2FS_I(dir)->i_current_depth != current_depth) f2fs_i_depth_write(dir, current_depth); @@ -920,7 +921,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct folio *folio, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h index f1d7a22293b37..f14c364cacc55 100644 --- a/fs/f2fs/f2fs.h +++ b/fs/f2fs/f2fs.h @@ -1683,6 +1683,33 @@ struct decompress_io_ctx { #define MAX_COMPRESS_LOG_SIZE 8 #define MAX_COMPRESS_WINDOW_SIZE(log_size) ((PAGE_SIZE) << (log_size)) +struct f2fs_gc_kthread { + struct task_struct *f2fs_gc_task; + wait_queue_head_t gc_wait_queue_head; + + /* for gc sleep time */ + unsigned int urgent_sleep_time; + unsigned int min_sleep_time; + unsigned int max_sleep_time; + unsigned int no_gc_sleep_time; + + /* for changing gc mode */ + bool gc_wake; + + /* for GC_MERGE mount option */ + wait_queue_head_t fggc_wq; /* + * caller of f2fs_balance_fs() + * will wait on this wait queue. + */ + + /* for gc control for zoned devices */ + unsigned int no_zoned_gc_percent; + unsigned int boost_zoned_gc_percent; + unsigned int valid_thresh_ratio; + unsigned int boost_gc_multiple; + unsigned int boost_gc_greedy; +}; + struct f2fs_sb_info { struct super_block *sb; /* pointer to VFS super block */ struct proc_dir_entry *s_proc; /* proc entry */ @@ -1818,7 +1845,7 @@ struct f2fs_sb_info { * semaphore for GC, avoid * race between GC and GC or CP */ - struct f2fs_gc_kthread *gc_thread; /* GC thread */ + struct f2fs_gc_kthread gc_thread; /* GC thread */ struct atgc_management am; /* atgc management */ unsigned int cur_victim_sec; /* current victim section num */ unsigned int gc_mode; /* current GC state */ @@ -4019,10 +4046,26 @@ static inline struct inode *fio_inode(struct f2fs_io_info *fio) #define MIN_FRAGMENT_SIZE 1 #define MAX_FRAGMENT_SIZE 512 -static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi) +static inline bool f2fs_need_rand_blk(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; +} + +static inline bool f2fs_need_rand_seg(struct f2fs_sb_info *sbi, + enum log_type type) +{ + if (type == CURSEG_COLD_DATA_PINNED) + return false; + return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG; +} + +static inline bool f2fs_need_rand_seg_blk(struct f2fs_sb_info *sbi, + enum log_type type) { - return F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG || - F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK; + return f2fs_need_rand_blk(sbi, type) || f2fs_need_rand_seg(sbi, type); } /* diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c index 271221485d66a..222b2a8bbc941 100644 --- a/fs/f2fs/file.c +++ b/fs/f2fs/file.c @@ -35,23 +35,69 @@ #include #include -static void f2fs_zero_post_eof_page(struct inode *inode, - loff_t new_size, bool lock) +static int fill_zero(struct inode *inode, pgoff_t index, + loff_t start, loff_t len); + +static int do_zero_post_eof_page(struct inode *inode, loff_t new_size) +{ + loff_t old_size = i_size_read(inode); + unsigned int offset, len; + pgoff_t index; + int err; + + offset = old_size & (PAGE_SIZE - 1); + + if (!offset) + return 0; + + len = min_t(loff_t, PAGE_SIZE - offset, new_size - old_size); + index = old_size >> PAGE_SHIFT; + + if (f2fs_has_inline_data(inode)) { + /* data post eof should be always zero */ + if (new_size <= MAX_INLINE_DATA(inode)) + return 0; + err = f2fs_convert_inline_inode(inode); + if (err) + return err; + } + + err = fill_zero(inode, index, offset, len); + if (err) + return err; + return filemap_write_and_wait_range(inode->i_mapping, + old_size, old_size + len - 1); +} + +static int f2fs_zero_post_eof_page(struct inode *inode, + loff_t new_size, bool lock, bool writeback) { loff_t old_size = i_size_read(inode); + bool strict = + F2FS_OPTION(F2FS_I_SB(inode)).fsync_mode == FSYNC_MODE_STRICT; if (old_size >= new_size) - return; + return 0; - if (mapping_empty(inode->i_mapping)) - return; + if (!strict && mapping_empty(inode->i_mapping)) + return 0; if (lock) filemap_invalidate_lock(inode->i_mapping); /* zero or drop pages only in range of [old_size, new_size] */ - truncate_inode_pages_range(inode->i_mapping, old_size, new_size); + truncate_inode_pages_range(inode->i_mapping, old_size, new_size - 1); if (lock) filemap_invalidate_unlock(inode->i_mapping); + + if (!writeback || !strict) + return 0; + /* + * In fsync_mode=strict, when we expand an unaligned EOF size, we + * should zero post EOF data and writeback the data immediately, + * so that it can avoid exposing stale data after metadata flush + * and POR. + */ + return do_zero_post_eof_page(inode, new_size); } static vm_fault_t f2fs_filemap_fault(struct vm_fault *vmf) @@ -122,7 +168,10 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) f2fs_bug_on(sbi, f2fs_has_inline_data(inode)); - f2fs_zero_post_eof_page(inode, (folio->index + 1) << PAGE_SHIFT, true); + err = f2fs_zero_post_eof_page(inode, + (folio->index + 1) << PAGE_SHIFT, true, false); + if (err) + goto out_pagefault; file_update_time(vmf->vma->vm_file); filemap_invalidate_lock_shared(inode->i_mapping); @@ -179,7 +228,7 @@ static vm_fault_t f2fs_vm_page_mkwrite(struct vm_fault *vmf) out_sem: filemap_invalidate_unlock_shared(inode->i_mapping); - +out_pagefault: sb_end_pagefault(inode->i_sb); out: ret = vmf_fs_error(err); @@ -1087,6 +1136,8 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, return -EPERM; if ((attr->ia_valid & ATTR_SIZE)) { + if (mapping_large_folio_support(inode->i_mapping)) + return -EOPNOTSUPP; if (!f2fs_is_compress_backend_ready(inode) || IS_DEVICE_ALIASING(inode)) return -EOPNOTSUPP; @@ -1094,17 +1145,23 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, !IS_ALIGNED(attr->ia_size, F2FS_BLK_TO_BYTES(fi->i_cluster_size))) return -EINVAL; - /* - * To prevent scattered pin block generation, we don't allow - * smaller/equal size unaligned truncation for pinned file. - * We only support overwrite IO to pinned file, so don't - * care about larger size truncation. - */ - if (f2fs_is_pinned_file(inode) && - attr->ia_size <= i_size_read(inode) && - !IS_ALIGNED(attr->ia_size, - F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) - return -EINVAL; + + if (f2fs_is_pinned_file(inode)) { + /* + * It may break section-aligned fallocate recovery + * mechanism, so do not allow larger size truncation. + */ + if (attr->ia_size > i_size_read(inode)) + return -EINVAL; + /* + * To prevent scattered pin block generation, we don't + * allow smaller/equal size unaligned truncation for + * pinned file. + */ + else if (!IS_ALIGNED(attr->ia_size, + F2FS_BLK_TO_BYTES(CAP_BLKS_PER_SEC(sbi)))) + return -EINVAL; + } } if (is_quota_modification(idmap, inode, attr)) { @@ -1154,8 +1211,12 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, f2fs_down_write(&fi->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - if (attr->ia_size > old_size) - f2fs_zero_post_eof_page(inode, attr->ia_size, false); + if (attr->ia_size > old_size) { + err = f2fs_zero_post_eof_page(inode, + attr->ia_size, false, true); + if (err) + goto err_out; + } truncate_setsize(inode, attr->ia_size); if (attr->ia_size <= old_size) @@ -1164,6 +1225,7 @@ int f2fs_setattr(struct mnt_idmap *idmap, struct dentry *dentry, * do not trim all blocks after i_size if target size is * larger than i_size. */ +err_out: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&fi->i_gc_rwsem[WRITE]); if (err) @@ -1274,7 +1336,9 @@ static int f2fs_punch_hole(struct inode *inode, loff_t offset, loff_t len) if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1559,7 +1623,9 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(inode->i_mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; f2fs_lock_op(sbi); f2fs_drop_extent_tree(inode); @@ -1567,6 +1633,7 @@ static int f2fs_do_collapse(struct inode *inode, loff_t offset, loff_t len) ret = __exchange_data_block(inode, inode, end, start, nrpages - end, true); f2fs_unlock_op(sbi); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); return ret; @@ -1599,11 +1666,14 @@ static int f2fs_collapse_range(struct inode *inode, loff_t offset, loff_t len) /* write out all moved pages, if possible */ filemap_invalidate_lock(inode->i_mapping); - filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + ret = filemap_write_and_wait_range(inode->i_mapping, offset, LLONG_MAX); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); new_size = i_size_read(inode) - len; ret = f2fs_truncate_blocks(inode, new_size, true); +out_unlock: filemap_invalidate_unlock(inode->i_mapping); if (!ret) f2fs_i_size_write(inode, new_size); @@ -1685,7 +1755,9 @@ static int f2fs_zero_range(struct inode *inode, loff_t offset, loff_t len, if (ret) return ret; - f2fs_zero_post_eof_page(inode, offset + len, true); + ret = f2fs_zero_post_eof_page(inode, offset + len, true, false); + if (ret) + return ret; pg_start = ((unsigned long long) offset) >> PAGE_SHIFT; pg_end = ((unsigned long long) offset + len) >> PAGE_SHIFT; @@ -1819,7 +1891,9 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); filemap_invalidate_lock(mapping); - f2fs_zero_post_eof_page(inode, offset + len, false); + ret = f2fs_zero_post_eof_page(inode, offset + len, false, false); + if (ret) + goto out_unlock; truncate_pagecache(inode, offset); while (!ret && idx > pg_start) { @@ -1835,6 +1909,7 @@ static int f2fs_insert_range(struct inode *inode, loff_t offset, loff_t len) idx + delta, nr, false); f2fs_unlock_op(sbi); } +out_unlock: filemap_invalidate_unlock(mapping); f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); if (ret) @@ -1877,7 +1952,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, if (err) return err; - f2fs_zero_post_eof_page(inode, offset + len, true); + err = f2fs_zero_post_eof_page(inode, offset + len, true, true); + if (err) + return err; f2fs_balance_fs(sbi, true); @@ -1898,8 +1975,9 @@ static int f2fs_expand_inode_data(struct inode *inode, loff_t offset, block_t sec_len; if (map.m_lblk % sec_blks) { - map.m_lblk = rounddown(map.m_lblk, sec_blks); - map.m_len = pg_end - map.m_lblk; + pg_start = rounddown(map.m_lblk, sec_blks); + map.m_lblk = pg_start; + map.m_len = pg_end - pg_start; if (off_end) map.m_len++; } @@ -3059,6 +3137,9 @@ static int f2fs_ioc_defragment(struct file *filp, unsigned long arg) if (f2fs_readonly(sbi->sb)) return -EROFS; + if (IS_DEVICE_ALIASING(inode)) + return -EOPNOTSUPP; + if (copy_from_user(&range, (struct f2fs_defragment __user *)arg, sizeof(range))) return -EFAULT; @@ -3096,8 +3177,9 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, struct inode *src = file_inode(file_in); struct inode *dst = file_inode(file_out); struct f2fs_sb_info *sbi = F2FS_I_SB(src); - size_t olen = len, dst_max_i_size = 0; - size_t dst_osize; + size_t olen = len; + loff_t dst_max_i_size = 0; + loff_t dst_osize, dst_end; int ret; if (file_in->f_path.mnt != file_out->f_path.mnt || @@ -3110,7 +3192,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (!S_ISREG(src->i_mode) || !S_ISREG(dst->i_mode)) return -EINVAL; - if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst)) + if (IS_ENCRYPTED(src) || IS_ENCRYPTED(dst) || + IS_DEVICE_ALIASING(src) || IS_DEVICE_ALIASING(dst)) return -EOPNOTSUPP; if (pos_out < 0 || pos_in < 0) @@ -3119,8 +3202,6 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, if (src == dst) { if (pos_in == pos_out) return 0; - if (pos_out > pos_in && pos_out < pos_in + len) - return -EINVAL; } inode_lock(src); @@ -3146,6 +3227,8 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, goto out_unlock; if (len == 0) olen = len = src->i_size - pos_in; + if (src == dst && pos_out > pos_in && pos_out < pos_in + len) + goto out_unlock; if (pos_in + len == src->i_size) len = ALIGN(src->i_size, F2FS_BLKSIZE) - pos_in; if (len == 0) { @@ -3154,8 +3237,15 @@ static int f2fs_move_file_range(struct file *file_in, loff_t pos_in, } dst_osize = dst->i_size; - if (pos_out + olen > dst->i_size) - dst_max_i_size = pos_out + olen; + if (olen > LLONG_MAX - pos_out) + goto out_unlock; + dst_end = pos_out + olen; + if (dst_end > dst->i_size) { + ret = inode_newsize_ok(dst, dst_end); + if (ret) + goto out_unlock; + dst_max_i_size = dst_end; + } /* verify the end result is block aligned */ if (!IS_ALIGNED(pos_in, F2FS_BLKSIZE) || @@ -4441,7 +4531,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) page_idx = folio_next_index(folio); } while (page_len < len); - do { + while (redirty_idx < page_idx) { folio = filemap_lock_folio(mapping, redirty_idx); /* It will never fail, when folio has pinned above */ @@ -4454,7 +4544,7 @@ static int redirty_blocks(struct inode *inode, pgoff_t page_idx, int len) redirty_idx = folio_next_index(folio); folio_unlock(folio); folio_put_refs(folio, 2); - } while (redirty_idx < page_idx); + } return ret; } @@ -4933,8 +5023,10 @@ static ssize_t f2fs_write_checks(struct kiocb *iocb, struct iov_iter *from) if (err) return err; - f2fs_zero_post_eof_page(inode, - iocb->ki_pos + iov_iter_count(from), true); + err = f2fs_zero_post_eof_page(inode, + iocb->ki_pos + iov_iter_count(from), true, true); + if (err) + return err; return count; } diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c index 3d41c6348b3c1..ca1c87e147006 100644 --- a/fs/f2fs/gc.c +++ b/fs/f2fs/gc.c @@ -31,9 +31,9 @@ static unsigned int count_bits(const unsigned long *addr, static int gc_thread_func(void *data) { struct f2fs_sb_info *sbi = data; - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; - wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head; - wait_queue_head_t *fggc_wq = &sbi->gc_thread->fggc_wq; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; + wait_queue_head_t *wq = &sbi->gc_thread.gc_wait_queue_head; + wait_queue_head_t *fggc_wq = &sbi->gc_thread.fggc_wq; unsigned int wait_ms; struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -191,13 +191,9 @@ static int gc_thread_func(void *data) int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; dev_t dev = sbi->sb->s_bdev->bd_dev; - gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL); - if (!gc_th) - return -ENOMEM; - gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME; gc_th->valid_thresh_ratio = DEF_GC_THREAD_VALID_THRESH_RATIO; gc_th->boost_gc_multiple = BOOST_GC_MULTIPLE; @@ -219,16 +215,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) gc_th->gc_wake = false; - sbi->gc_thread = gc_th; - init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head); - init_waitqueue_head(&sbi->gc_thread->fggc_wq); - sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi, + init_waitqueue_head(&gc_th->gc_wait_queue_head); + init_waitqueue_head(&gc_th->fggc_wq); + gc_th->f2fs_gc_task = kthread_run(gc_thread_func, sbi, "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev)); if (IS_ERR(gc_th->f2fs_gc_task)) { int err = PTR_ERR(gc_th->f2fs_gc_task); - kfree(gc_th); - sbi->gc_thread = NULL; + gc_th->f2fs_gc_task = NULL; return err; } @@ -237,14 +231,14 @@ int f2fs_start_gc_thread(struct f2fs_sb_info *sbi) void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi) { - struct f2fs_gc_kthread *gc_th = sbi->gc_thread; + struct f2fs_gc_kthread *gc_th = &sbi->gc_thread; - if (!gc_th) + if (!gc_th->f2fs_gc_task) return; + kthread_stop(gc_th->f2fs_gc_task); + gc_th->f2fs_gc_task = NULL; wake_up_all(&gc_th->fggc_wq); - kfree(gc_th); - sbi->gc_thread = NULL; } static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type) @@ -312,10 +306,11 @@ static void select_policy(struct f2fs_sb_info *sbi, int gc_type, p->max_search = sbi->max_victim_search; /* let's select beginning hot/small space first. */ - if (f2fs_need_rand_seg(sbi)) + if (f2fs_need_rand_seg_blk(sbi, type)) { p->offset = get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); - else if (type == CURSEG_HOT_DATA || IS_NODESEG(type)) + SIT_I(sbi)->last_victim[p->gc_mode] = p->offset; + } else if (type == CURSEG_HOT_DATA || IS_NODESEG(type)) p->offset = 0; else p->offset = SIT_I(sbi)->last_victim[p->gc_mode]; @@ -791,7 +786,7 @@ int f2fs_get_victim(struct f2fs_sb_info *sbi, unsigned int *result, if (one_time) { p.one_time_gc = one_time; if (has_enough_free_secs(sbi, 0, NR_PERSISTENT_LOG)) - valid_thresh_ratio = sbi->gc_thread->valid_thresh_ratio; + valid_thresh_ratio = sbi->gc_thread.valid_thresh_ratio; } retry: @@ -1793,9 +1788,9 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi, if (f2fs_sb_has_blkzoned(sbi) && !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent)) + sbi->gc_thread.boost_zoned_gc_percent)) window_granularity *= - sbi->gc_thread->boost_gc_multiple; + sbi->gc_thread.boost_gc_multiple; end_segno = start_segno + window_granularity; } @@ -2170,8 +2165,9 @@ int f2fs_gc_range(struct f2fs_sb_info *sbi, static int free_segment_range(struct f2fs_sb_info *sbi, unsigned int secs, bool dry_run) { - unsigned int next_inuse, start, end; + unsigned int secno, next_inuse, start, end, end_secno; struct cp_control cpc = { CP_RESIZE, 0, 0, 0 }; + unsigned int freed_secs = 0; int gc_mode, gc_type; int err = 0; int type; @@ -2180,6 +2176,7 @@ static int free_segment_range(struct f2fs_sb_info *sbi, MAIN_SECS(sbi) -= secs; start = MAIN_SECS(sbi) * SEGS_PER_SEC(sbi); end = MAIN_SEGS(sbi) - 1; + end_secno = GET_SEC_FROM_SEG(sbi, end); mutex_lock(&DIRTY_I(sbi)->seglist_lock); for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++) @@ -2191,8 +2188,16 @@ static int free_segment_range(struct f2fs_sb_info *sbi, sbi->next_victim_seg[gc_type] = NULL_SEGNO; mutex_unlock(&DIRTY_I(sbi)->seglist_lock); + spin_lock(&FREE_I(sbi)->segmap_lock); + for (secno = MAIN_SECS(sbi); secno <= end_secno; secno++) { + if (!test_bit(secno, FREE_I(sbi)->free_secmap)) + freed_secs++; + } + FREE_I(sbi)->free_sections -= freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); + /* Move out cursegs from the target range */ - for (type = CURSEG_HOT_DATA; type < NR_CURSEG_PERSIST_TYPE; type++) { + for (type = CURSEG_HOT_DATA; type < NR_CURSEG_TYPE; type++) { err = f2fs_allocate_segment_for_resize(sbi, type, start, end); if (err) goto out; @@ -2215,6 +2220,9 @@ static int free_segment_range(struct f2fs_sb_info *sbi, f2fs_bug_on(sbi, 1); } out: + spin_lock(&FREE_I(sbi)->segmap_lock); + FREE_I(sbi)->free_sections += freed_secs; + spin_unlock(&FREE_I(sbi)->segmap_lock); MAIN_SECS(sbi) += secs; return err; } diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h index 24e8b1c27acc4..2568d8017bf01 100644 --- a/fs/f2fs/gc.h +++ b/fs/f2fs/gc.h @@ -45,32 +45,7 @@ #define NR_GC_CHECKPOINT_SECS (3) /* data/node/dentry sections */ -struct f2fs_gc_kthread { - struct task_struct *f2fs_gc_task; - wait_queue_head_t gc_wait_queue_head; - - /* for gc sleep time */ - unsigned int urgent_sleep_time; - unsigned int min_sleep_time; - unsigned int max_sleep_time; - unsigned int no_gc_sleep_time; - - /* for changing gc mode */ - bool gc_wake; - - /* for GC_MERGE mount option */ - wait_queue_head_t fggc_wq; /* - * caller of f2fs_balance_fs() - * will wait on this wait queue. - */ - /* for gc control for zoned devices */ - unsigned int no_zoned_gc_percent; - unsigned int boost_zoned_gc_percent; - unsigned int valid_thresh_ratio; - unsigned int boost_gc_multiple; - unsigned int boost_gc_greedy; -}; struct gc_inode_list { struct list_head ilist; @@ -197,6 +172,6 @@ static inline bool need_to_boost_gc(struct f2fs_sb_info *sbi) { if (f2fs_sb_has_blkzoned(sbi)) return !has_enough_free_blocks(sbi, - sbi->gc_thread->boost_zoned_gc_percent); + sbi->gc_thread.boost_zoned_gc_percent); return has_enough_invalid_blocks(sbi); } diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index ea5680406e23d..55dbbbfe738f5 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c @@ -506,6 +506,12 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) bit_pos++; continue; } + if (unlikely(le16_to_cpu(de->name_len) > F2FS_NAME_LEN || + bit_pos + GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)) > + d.max)) { + err = -EFSCORRUPTED; + goto punch_dentry_pages; + } /* * We only need the disk_name and hash to move the dentry. @@ -526,6 +532,7 @@ static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry) bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); } return 0; + punch_dentry_pages: truncate_inode_pages(&dir->i_data, 0); f2fs_truncate_blocks(dir, 0, false); @@ -720,7 +727,7 @@ void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, f2fs_folio_put(folio, true); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); - f2fs_mark_inode_dirty_sync(dir, false); + f2fs_mark_inode_dirty_sync(dir, true); if (inode) f2fs_drop_nlink(dir, inode); diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c index fb0b0b35ad26d..2e5085d199b2d 100644 --- a/fs/f2fs/namei.c +++ b/fs/f2fs/namei.c @@ -681,15 +681,16 @@ static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, * performance regression. */ if (!err) { - filemap_write_and_wait_range(inode->i_mapping, 0, - disk_link.len - 1); + err = filemap_write_and_wait_range(inode->i_mapping, 0, + disk_link.len - 1); - if (IS_DIRSYNC(dir)) + if (!err && IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); - } else { - f2fs_unlink(dir, dentry); } + if (err) + f2fs_unlink(dir, dentry); + f2fs_balance_fs(sbi, true); goto out_free_encrypted_link; @@ -1046,7 +1047,7 @@ static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, f2fs_up_write(&F2FS_I(old_inode)->i_sem); inode_set_ctime_current(old_inode); - f2fs_mark_inode_dirty_sync(old_inode, false); + f2fs_mark_inode_dirty_sync(old_inode, true); f2fs_delete_entry(old_entry, old_folio, old_dir, NULL); old_folio = NULL; @@ -1215,7 +1216,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(old_dir, old_nlink > 0); f2fs_up_write(&F2FS_I(old_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(old_dir, false); + f2fs_mark_inode_dirty_sync(old_dir, true); /* update directory entry info of new dir inode */ f2fs_set_link(new_dir, new_entry, new_folio, old_inode); @@ -1234,7 +1235,7 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, f2fs_i_links_write(new_dir, new_nlink > 0); f2fs_up_write(&F2FS_I(new_dir)->i_sem); } - f2fs_mark_inode_dirty_sync(new_dir, false); + f2fs_mark_inode_dirty_sync(new_dir, true); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { f2fs_add_ino_entry(sbi, old_dir->i_ino, TRANS_DIR_INO); diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 3918021553311..823b78c4aa5b1 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c @@ -1596,7 +1596,7 @@ static struct folio *__get_node_folio(struct f2fs_sb_info *sbi, pgoff_t nid, if (!err) return folio; out_err: - folio_clear_uptodate(folio); + clear_node_folio_dirty(folio); out_put_err: /* ENOENT comes from read_node_folio which is not an error. */ if (err != -ENOENT) @@ -1767,7 +1767,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync, /* get old block addr of this node page */ nid = nid_of_node(folio); - if (f2fs_sanity_check_node_footer(sbi, folio, nid, + if (f2fs_sanity_check_node_footer(sbi, folio, folio->index, NODE_TYPE_REGULAR, false)) { f2fs_handle_critical_error(sbi, STOP_CP_REASON_CORRUPTED_NID); goto redirty_out; @@ -1986,6 +1986,11 @@ int f2fs_fsync_node_pages(struct f2fs_sb_info *sbi, struct inode *inode, f2fs_debug(sbi, "Retry to write fsync mark: ino=%u, idx=%lx", ino, last_folio->index); folio_lock(last_folio); + if (unlikely(!is_node_folio(last_folio))) { + f2fs_folio_put(last_folio, true); + ret = -EAGAIN; + goto out; + } f2fs_folio_wait_writeback(last_folio, NODE, true, true); folio_mark_dirty(last_folio); folio_unlock(last_folio); diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c index 06674e694b271..e954e356c8089 100644 --- a/fs/f2fs/recovery.c +++ b/fs/f2fs/recovery.c @@ -169,7 +169,8 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, struct inode *dir, *einode; struct fsync_inode_entry *entry; int err = 0; - char *name; + const char *name; + int name_len; entry = get_fsync_inode(dir_list, pino); if (!entry) { @@ -228,12 +229,16 @@ static int recover_dentry(struct inode *inode, struct folio *ifolio, out_put: f2fs_folio_put(folio, false); out: - if (file_enc_name(inode)) + if (file_enc_name(inode)) { name = ""; - else + name_len = sizeof("") - 1; + } else { name = raw_inode->i_name; - f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %s, dir = %lx, err = %d", - __func__, ino_of_node(ifolio), name, + name_len = min_t(unsigned int, le32_to_cpu(raw_inode->i_namelen), + F2FS_NAME_LEN); + } + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, dir = %lx, err = %d", + __func__, ino_of_node(ifolio), name_len, name, IS_ERR(dir) ? 0 : dir->i_ino, err); return err; } @@ -281,7 +286,8 @@ static int recover_inode(struct inode *inode, struct folio *folio) { struct f2fs_inode *raw = F2FS_INODE(folio); struct f2fs_inode_info *fi = F2FS_I(inode); - char *name; + const char *name; + int name_len; int err; inode->i_mode = le16_to_cpu(raw->i_mode); @@ -330,13 +336,18 @@ static int recover_inode(struct inode *inode, struct folio *folio) f2fs_mark_inode_dirty_sync(inode, true); - if (file_enc_name(inode)) + if (file_enc_name(inode)) { name = ""; - else - name = F2FS_INODE(folio)->i_name; + name_len = sizeof("") - 1; + } else { + name = raw->i_name; + name_len = min_t(unsigned int, le32_to_cpu(raw->i_namelen), + F2FS_NAME_LEN); + } - f2fs_notice(F2FS_I_SB(inode), "recover_inode: ino = %x, name = %s, inline = %x", - ino_of_node(folio), name, raw->i_inline); + f2fs_notice(F2FS_I_SB(inode), "%s: ino = %x, name = %.*s, inline = %x", + __func__, ino_of_node(folio), name_len, name, + raw->i_inline); return 0; } diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c index 5e2789cd96b5e..2fa2a005c384f 100644 --- a/fs/f2fs/segment.c +++ b/fs/f2fs/segment.c @@ -450,15 +450,14 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need) f2fs_submit_merged_write(sbi, DATA); f2fs_submit_all_merged_ipu_writes(sbi); - if (test_opt(sbi, GC_MERGE) && sbi->gc_thread && - sbi->gc_thread->f2fs_gc_task) { + if (test_opt(sbi, GC_MERGE) && sbi->gc_thread.f2fs_gc_task) { DEFINE_WAIT(wait); - prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait, + prepare_to_wait(&sbi->gc_thread.fggc_wq, &wait, TASK_UNINTERRUPTIBLE); - wake_up(&sbi->gc_thread->gc_wait_queue_head); + wake_up(&sbi->gc_thread.gc_wait_queue_head); io_schedule(); - finish_wait(&sbi->gc_thread->fggc_wq, &wait); + finish_wait(&sbi->gc_thread.fggc_wq, &wait); } else { struct f2fs_gc_control gc_control = { .victim_segno = NULL_SEGNO, @@ -2459,9 +2458,6 @@ static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_ent unsigned int segno, block_t blkaddr, unsigned int offset, int del) { bool exist; -#ifdef CONFIG_F2FS_CHECK_FS - bool mir_exist; -#endif int i; int del_count = -del; @@ -2469,15 +2465,6 @@ static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_ent for (i = 0; i < del_count; i++) { exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map); -#ifdef CONFIG_F2FS_CHECK_FS - mir_exist = f2fs_test_and_clear_bit(offset + i, - se->cur_valid_map_mir); - if (unlikely(exist != mir_exist)) { - f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d", - blkaddr + i, exist); - f2fs_bug_on(sbi, 1); - } -#endif if (unlikely(!exist)) { f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i); f2fs_bug_on(sbi, 1); @@ -2518,20 +2505,8 @@ static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry unsigned int segno, block_t blkaddr, unsigned int offset, int del) { bool exist; -#ifdef CONFIG_F2FS_CHECK_FS - bool mir_exist; -#endif exist = f2fs_test_and_set_bit(offset, se->cur_valid_map); -#ifdef CONFIG_F2FS_CHECK_FS - mir_exist = f2fs_test_and_set_bit(offset, - se->cur_valid_map_mir); - if (unlikely(exist != mir_exist)) { - f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d", - blkaddr, exist); - f2fs_bug_on(sbi, 1); - } -#endif if (unlikely(exist)) { f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr); f2fs_bug_on(sbi, 1); @@ -2964,7 +2939,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) sanity_check_seg_type(sbi, seg_type); if (__is_large_section(sbi)) { - if (f2fs_need_rand_seg(sbi)) { + if (f2fs_need_rand_seg_blk(sbi, type)) { unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno); if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint) @@ -2973,7 +2948,7 @@ static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type) GET_SEG_FROM_SEC(sbi, hint + 1) - 1); } return curseg->segno; - } else if (f2fs_need_rand_seg(sbi)) { + } else if (f2fs_need_rand_seg_blk(sbi, type)) { return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi)); } @@ -3029,7 +3004,7 @@ static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec) curseg->next_segno = segno; reset_curseg(sbi, type, 1); curseg->alloc_type = LFS; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) curseg->fragment_remained_chunk = get_random_u32_inclusive(1, sbi->max_fragment_chunk); return 0; @@ -3388,10 +3363,13 @@ int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi) err = f2fs_gc_range(sbi, 0, sbi->first_seq_zone_segno - 1, true, ZONED_PIN_SEC_REQUIRED_COUNT); f2fs_up_write(&sbi->gc_lock); - - gc_required = false; - if (!err) + if (err) + return err; + err = f2fs_sync_fs(sbi->sb, 1); + if (!err) { + gc_required = false; goto retry; + } } return err; @@ -3832,7 +3810,7 @@ int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct folio *folio, curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg); } else { curseg->next_blkoff++; - if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + if (f2fs_need_rand_blk(sbi, type)) f2fs_randomize_chunk(sbi, curseg); } if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno)) @@ -4759,11 +4737,6 @@ void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc) int offset, sit_offset; se = get_seg_entry(sbi, segno); -#ifdef CONFIG_F2FS_CHECK_FS - if (memcmp(se->cur_valid_map, se->cur_valid_map_mir, - SIT_VBLOCK_MAP_SIZE)) - f2fs_bug_on(sbi, 1); -#endif /* add discard candidates */ if (!(cpc->reason & CP_DISCARD)) { @@ -4854,11 +4827,7 @@ static int build_sit_info(struct f2fs_sb_info *sbi) if (!sit_i->dirty_sentries_bitmap) return -ENOMEM; -#ifdef CONFIG_F2FS_CHECK_FS - bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map); -#else bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map); -#endif sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL); if (!sit_i->bitmap) return -ENOMEM; @@ -4872,11 +4841,6 @@ static int build_sit_info(struct f2fs_sb_info *sbi) sit_i->sentries[start].ckpt_valid_map = bitmap; bitmap += SIT_VBLOCK_MAP_SIZE; -#ifdef CONFIG_F2FS_CHECK_FS - sit_i->sentries[start].cur_valid_map_mir = bitmap; - bitmap += SIT_VBLOCK_MAP_SIZE; -#endif - if (discard_map) { sit_i->sentries[start].discard_map = bitmap; bitmap += SIT_VBLOCK_MAP_SIZE; diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h index d05d133c89aff..21ada383f19a4 100644 --- a/fs/f2fs/segment.h +++ b/fs/f2fs/segment.h @@ -172,9 +172,6 @@ struct seg_entry { unsigned int ckpt_valid_blocks:10; /* # of valid blocks last cp */ unsigned int padding:6; /* padding */ unsigned char *cur_valid_map; /* validity bitmap of blocks */ -#ifdef CONFIG_F2FS_CHECK_FS - unsigned char *cur_valid_map_mir; /* mirror of current valid bitmap */ -#endif /* * # of valid blocks and the validity bitmap stored in the last * checkpoint pack. This information is used by the SSR mode. @@ -403,9 +400,6 @@ static inline void seg_info_from_raw_sit(struct seg_entry *se, se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs); memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE); -#ifdef CONFIG_F2FS_CHECK_FS - memcpy(se->cur_valid_map_mir, rs->valid_map, SIT_VBLOCK_MAP_SIZE); -#endif se->type = GET_SIT_TYPE(rs); se->mtime = le64_to_cpu(rs->mtime); } diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c index f6b75ce11d1c1..74b12c1ef8211 100644 --- a/fs/f2fs/super.c +++ b/fs/f2fs/super.c @@ -2401,9 +2401,9 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root) seq_puts(seq, "adaptive"); else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_LFS) seq_puts(seq, "lfs"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_SEG) + else if (f2fs_need_rand_seg(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:segment"); - else if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) + else if (f2fs_need_rand_blk(sbi, NO_CHECK_TYPE)) seq_puts(seq, "fragment:block"); seq_printf(seq, ",active_logs=%u", F2FS_OPTION(sbi).active_logs); if (test_opt(sbi, RESERVE_ROOT) || test_opt(sbi, RESERVE_NODE)) @@ -2846,11 +2846,11 @@ static int __f2fs_remount(struct fs_context *fc, struct super_block *sb) if ((flags & SB_RDONLY) || (F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_OFF && !test_opt(sbi, GC_MERGE))) { - if (sbi->gc_thread) { + if (sbi->gc_thread.f2fs_gc_task) { f2fs_stop_gc_thread(sbi); need_restart_gc = true; } - } else if (!sbi->gc_thread) { + } else if (!sbi->gc_thread.f2fs_gc_task) { err = f2fs_start_gc_thread(sbi); if (err) goto restore_opts; @@ -3659,24 +3659,27 @@ static bool f2fs_has_stable_inodes(struct super_block *sb) return true; } -static struct block_device **f2fs_get_devices(struct super_block *sb, - unsigned int *num_devs) +static unsigned int +f2fs_get_devices(struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]) { struct f2fs_sb_info *sbi = F2FS_SB(sb); - struct block_device **devs; + int ndevs; int i; - if (!f2fs_is_multi_device(sbi)) - return NULL; + static_assert(MAX_DEVICES <= FSCRYPT_MAX_DEVICES); - devs = kmalloc_array(sbi->s_ndevs, sizeof(*devs), GFP_KERNEL); - if (!devs) - return ERR_PTR(-ENOMEM); + if (!f2fs_is_multi_device(sbi)) { + devs[0] = sb->s_bdev; + return 1; + } + ndevs = sbi->s_ndevs; + if (WARN_ON_ONCE(ndevs > FSCRYPT_MAX_DEVICES)) + ndevs = FSCRYPT_MAX_DEVICES; - for (i = 0; i < sbi->s_ndevs; i++) + for (i = 0; i < ndevs; i++) devs[i] = FDEV(i).bdev; - *num_devs = sbi->s_ndevs; - return devs; + return ndevs; } static const struct fscrypt_operations f2fs_cryptops = { diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c index 1f5982a0a6326..3f9643d34f568 100644 --- a/fs/f2fs/sysfs.c +++ b/fs/f2fs/sysfs.c @@ -74,7 +74,7 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a, static unsigned char *__struct_ptr(struct f2fs_sb_info *sbi, int struct_type) { if (struct_type == GC_THREAD) - return (unsigned char *)sbi->gc_thread; + return (unsigned char *)&sbi->gc_thread; else if (struct_type == SM_INFO) return (unsigned char *)SM_I(sbi); else if (struct_type == DCC_INFO) @@ -547,7 +547,7 @@ static ssize_t __sbi_store(struct f2fs_attr *a, return -EINVAL; cprc->ckpt_thread_ioprio = IOPRIO_PRIO_VALUE(class, level); - if (test_opt(sbi, MERGE_CHECKPOINT)) { + if (cprc->f2fs_issue_ckpt) { ret = set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio); if (ret) @@ -648,20 +648,20 @@ static ssize_t __sbi_store(struct f2fs_attr *a, sbi->gc_mode = GC_NORMAL; } else if (t == 1) { sbi->gc_mode = GC_URGENT_HIGH; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); wake_up_discard_thread(sbi, true); } } else if (t == 2) { sbi->gc_mode = GC_URGENT_LOW; } else if (t == 3) { sbi->gc_mode = GC_URGENT_MID; - if (sbi->gc_thread) { - sbi->gc_thread->gc_wake = true; + if (sbi->gc_thread.f2fs_gc_task) { + sbi->gc_thread.gc_wake = true; wake_up_interruptible_all( - &sbi->gc_thread->gc_wait_queue_head); + &sbi->gc_thread.gc_wait_queue_head); } } else { return -EINVAL; @@ -918,14 +918,14 @@ static ssize_t __sbi_store(struct f2fs_attr *a, if (!strcmp(a->attr.name, "gc_boost_gc_multiple")) { if (t < 1 || t > SEGS_PER_SEC(sbi)) return -EINVAL; - sbi->gc_thread->boost_gc_multiple = (unsigned int)t; + sbi->gc_thread.boost_gc_multiple = (unsigned int)t; return count; } if (!strcmp(a->attr.name, "gc_boost_gc_greedy")) { if (t > GC_GREEDY) return -EINVAL; - sbi->gc_thread->boost_gc_greedy = (unsigned int)t; + sbi->gc_thread.boost_gc_greedy = (unsigned int)t; return count; } @@ -962,13 +962,14 @@ static ssize_t f2fs_sbi_store(struct f2fs_attr *a, ssize_t ret; bool gc_entry = (!strcmp(a->attr.name, "gc_urgent") || a->struct_type == GC_THREAD); + bool thread_entry = !strcmp(a->attr.name, "ckpt_thread_ioprio"); - if (gc_entry) { + if (gc_entry || thread_entry) { if (!down_read_trylock(&sbi->sb->s_umount)) return -EAGAIN; } ret = __sbi_store(a, sbi, buf, count); - if (gc_entry) + if (gc_entry || thread_entry) up_read(&sbi->sb->s_umount); return ret; diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c index 8cb0de2a5bcf9..84687552a6810 100644 --- a/fs/f2fs/xattr.c +++ b/fs/f2fs/xattr.c @@ -118,7 +118,7 @@ static int f2fs_xattr_advise_set(const struct xattr_handler *handler, unsigned char old_advise = F2FS_I(inode)->i_advise; unsigned char new_advise; - if (!inode_owner_or_capable(&nop_mnt_idmap, inode)) + if (!inode_owner_or_capable(idmap, inode)) return -EPERM; if (value == NULL) return -EINVAL; diff --git a/fs/fat/misc.c b/fs/fat/misc.c index 950da09f09614..5c45603bf77ea 100644 --- a/fs/fat/misc.c +++ b/fs/fat/misc.c @@ -133,7 +133,11 @@ int fat_chain_add(struct inode *inode, int new_dclus, int nr_cluster) ret = fat_ent_read(inode, &fatent, last); if (ret >= 0) { int wait = inode_needs_sync(inode); + int old = ret; + ret = fat_ent_write(inode, &fatent, new_dclus, wait); + if (ret < 0) + fat_ent_write(inode, &fatent, old, wait); fatent_brelse(&fatent); } if (ret < 0) diff --git a/fs/fat/nfs.c b/fs/fat/nfs.c index 509eea96a457d..6e1b371711edd 100644 --- a/fs/fat/nfs.c +++ b/fs/fat/nfs.c @@ -250,8 +250,10 @@ struct inode *fat_rebuild_parent(struct super_block *sb, int parent_logstart) MSDOS_I(dummy_grand_parent)->i_pos = -1; } - if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) + if (!fat_scan_logstart(dummy_grand_parent, clus_to_match, &sinfo)) { parent = fat_build_inode(sb, sinfo.de, sinfo.i_pos); + brelse(sinfo.bh); + } brelse(parent_bh); iput(dummy_grand_parent); diff --git a/fs/freevxfs/vxfs_bmap.c b/fs/freevxfs/vxfs_bmap.c index 26d367e3668da..de4003597b555 100644 --- a/fs/freevxfs/vxfs_bmap.c +++ b/fs/freevxfs/vxfs_bmap.c @@ -227,7 +227,8 @@ vxfs_bmap_typed(struct inode *ip, long iblock) return 0; } default: - BUG(); + WARN_ON_ONCE(1); + return 0; } } diff --git a/fs/fuse/args.h b/fs/fuse/args.h new file mode 100644 index 0000000000000..ecfe51a192af0 --- /dev/null +++ b/fs/fuse/args.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _FS_FUSE_ARGS_H +#define _FS_FUSE_ARGS_H + +#include + +struct fuse_mount; + +/** One input argument of a request */ +struct fuse_in_arg { + unsigned size; + const void *value; +}; + +/** One output argument of a request */ +struct fuse_arg { + unsigned size; + void *value; +}; + +struct fuse_args { + u64 nodeid; + u32 opcode; + u32 uid; + u32 gid; + u32 pid; + u8 in_numargs; + u8 out_numargs; + u8 ext_idx; + bool force:1; + bool noreply:1; + bool nocreds:1; + bool in_pages:1; + bool out_pages:1; + bool user_pages:1; + bool out_argvar:1; + bool page_zeroing:1; + bool page_replace:1; + bool may_block:1; + bool is_ext:1; + bool is_pinned:1; + bool invalidate_vmap:1; + bool abort_on_kill:1; + struct fuse_in_arg in_args[4]; + struct fuse_arg out_args[2]; + void (*end)(struct fuse_args *args, int error); + /* Used for kvec iter backed by vmalloc address */ + void *vmap_base; +}; + +/** FUSE folio descriptor */ +struct fuse_folio_desc { + unsigned int length; + unsigned int offset; +}; + +struct fuse_args_pages { + struct fuse_args args; + struct folio **folios; + struct fuse_folio_desc *descs; + unsigned int num_folios; +}; + +#endif /* _FS_FUSE_ARGS_H */ diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c index 2bb98bf81b079..792620c2fa4b2 100644 --- a/fs/fuse/cuse.c +++ b/fs/fuse/cuse.c @@ -306,6 +306,7 @@ struct cuse_init_args { struct cuse_init_out out; struct folio *folio; struct fuse_folio_desc desc; + struct fuse_conn *fc; }; /** @@ -319,11 +320,10 @@ struct cuse_init_args { * required data structures for it. Please read the comment at the * top of this file for high level overview. */ -static void cuse_process_init_reply(struct fuse_mount *fm, - struct fuse_args *args, int error) +static void cuse_process_init_reply(struct fuse_args *args, int error) { - struct fuse_conn *fc = fm->fc; struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args); + struct fuse_conn *fc = ia->fc; struct fuse_args_pages *ap = &ia->ap; struct cuse_conn *cc = fc_to_cc(fc), *pos; struct cuse_init_out *arg = &ia->out; @@ -468,6 +468,7 @@ static int cuse_send_init(struct cuse_conn *cc) ap->descs = &ia->desc; ia->folio = folio; ia->desc.length = ap->args.out_args[1].size; + ia->fc = &cc->fc; ap->args.end = cuse_process_init_reply; rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL); @@ -649,6 +650,11 @@ static void __exit cuse_exit(void) { misc_deregister(&cuse_miscdev); class_destroy(cuse_class); + /* + * Wait for pending call_rcu() callbacks that call back into + * this module via fc->release (cuse_fc_release). + */ + rcu_barrier(); } module_init(cuse_init); diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c index 77c3982bca3d7..7c884532d9e73 100644 --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -172,7 +172,13 @@ void fuse_set_initialized(struct fuse_conn *fc) static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background) { - return !fc->initialized || (for_background && fc->blocked) || + if (!fc->initialized) + return true; + + /* Pairs with smp_wmb() in fuse_set_initialized() */ + smp_rmb(); + + return (for_background && fc->blocked) || (fc->io_uring && fc->connected && !fuse_uring_ready(fc)); } @@ -212,8 +218,6 @@ static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap, (TASK_KILLABLE | TASK_FREEZABLE))) goto out; } - /* Matches smp_wmb() in fuse_set_initialized() */ - smp_rmb(); err = -ENOTCONN; if (!fc->connected) @@ -353,7 +357,8 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq, void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req) { spin_lock(&fiq->lock); - if (list_empty(&req->intr_entry)) { + /* Repeat FR_SENT test after obtaining the lock to prevent race with fuse_resend() */ + if (list_empty(&req->intr_entry) && test_bit(FR_SENT, &req->flags)) { list_add_tail(&req->intr_entry, &fiq->interrupts); /* * Pairs with smp_mb() implied by test_and_set_bit() @@ -512,7 +517,7 @@ void fuse_request_end(struct fuse_req *req) } if (test_bit(FR_ASYNC, &req->flags)) - req->args->end(fm, req->args, req->out.h.error); + req->args->end(req->args, req->out.h.error); put_request: fuse_put_request(req); } @@ -580,7 +585,7 @@ static void request_wait_answer(struct fuse_req *req) if (req->args->abort_on_kill) { fuse_abort_conn(fc); - return; + goto wait_for_finish; } if (test_bit(FR_URING, &req->flags)) @@ -591,6 +596,7 @@ static void request_wait_answer(struct fuse_req *req) return; } +wait_for_finish: /* * Either request is already in userspace, or it was forced. * Wait it out. @@ -1859,8 +1865,7 @@ struct fuse_retrieve_args { struct fuse_notify_retrieve_in inarg; }; -static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void fuse_retrieve_end(struct fuse_args *args, int error) { struct fuse_retrieve_args *ra = container_of(args, typeof(*ra), ap.args); @@ -1955,7 +1960,7 @@ static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode, err = fuse_simple_notify_reply(fm, args, outarg->notify_unique); if (err) - fuse_retrieve_end(fm, args, err); + fuse_retrieve_end(args, err); return err; } @@ -2011,7 +2016,7 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size, static void fuse_resend(struct fuse_conn *fc) { struct fuse_dev *fud; - struct fuse_req *req, *next; + struct fuse_req *req; struct fuse_iqueue *fiq = &fc->iq; LIST_HEAD(to_queue); unsigned int i; @@ -2026,24 +2031,20 @@ static void fuse_resend(struct fuse_conn *fc) struct fuse_pqueue *fpq = &fud->pq; spin_lock(&fpq->lock); - for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) - list_splice_tail_init(&fpq->processing[i], &to_queue); + for (i = 0; i < FUSE_PQ_HASH_SIZE; i++) { + struct list_head *this_queue = &fpq->processing[i]; + + list_for_each_entry(req, this_queue, list) + clear_bit(FR_SENT, &req->flags); + list_splice_tail_init(this_queue, &to_queue); + } spin_unlock(&fpq->lock); } spin_unlock(&fc->lock); - list_for_each_entry_safe(req, next, &to_queue, list) { - set_bit(FR_PENDING, &req->flags); - clear_bit(FR_SENT, &req->flags); - /* mark the request as resend request */ - req->in.h.unique |= FUSE_UNIQUE_RESEND; - } - spin_lock(&fiq->lock); if (!fiq->connected) { spin_unlock(&fiq->lock); - list_for_each_entry(req, &to_queue, list) - clear_bit(FR_PENDING, &req->flags); fuse_dev_end_requests(&to_queue); return; } @@ -2052,6 +2053,10 @@ static void fuse_resend(struct fuse_conn *fc) * intr_entry on fiq->interrupts after the request is re-queued. */ list_for_each_entry(req, &to_queue, list) { + set_bit(FR_PENDING, &req->flags); + /* mark the request as resend request */ + req->in.h.unique |= FUSE_UNIQUE_RESEND; + if (test_bit(FR_INTERRUPTED, &req->flags)) list_del_init(&req->intr_entry); } diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c index 42c82b5476593..9b57544935b50 100644 --- a/fs/fuse/dev_uring.c +++ b/fs/fuse/dev_uring.c @@ -18,7 +18,8 @@ MODULE_PARM_DESC(enable_uring, "Enable userspace communication through io-uring"); #define FUSE_URING_IOV_SEGS 2 /* header and payload */ - +#define FUSE_URING_IOV_HEADERS 0 +#define FUSE_URING_IOV_PAYLOAD 1 bool fuse_uring_enabled(void) { @@ -31,6 +32,15 @@ struct fuse_uring_pdu { static const struct fuse_iqueue_ops fuse_io_uring_ops; +enum fuse_uring_header_type { + /* struct fuse_in_header / struct fuse_out_header */ + FUSE_URING_HEADER_IN_OUT, + /* per op code header */ + FUSE_URING_HEADER_OP, + /* struct fuse_uring_ent_in_out header */ + FUSE_URING_HEADER_RING_ENT, +}; + static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd, struct fuse_ring_ent *ring_ent) { @@ -194,7 +204,7 @@ void fuse_uring_destruct(struct fuse_conn *fc) return; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); struct fuse_ring_ent *ent, *next; if (!queue) @@ -213,7 +223,7 @@ void fuse_uring_destruct(struct fuse_conn *fc) kfree(queue->fpq.processing); kfree(queue); - ring->queues[qid] = NULL; + WRITE_ONCE(ring->queues[qid], NULL); } kfree(ring->queues); @@ -307,9 +317,11 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring, } /* - * write_once and lock as the caller mostly doesn't take the lock at all + * fc->lock serializes concurrent creators for this qid. + * smp_store_release() are for the lockless readers who must see a + * fully initialized queue after &ring->queues[qid] is set */ - WRITE_ONCE(ring->queues[qid], queue); + smp_store_release(&ring->queues[qid], queue); spin_unlock(&fc->lock); return queue; @@ -406,7 +418,7 @@ static void fuse_uring_log_ent_state(struct fuse_ring *ring) struct fuse_ring_ent *ent; for (qid = 0; qid < ring->nr_queues; qid++) { - struct fuse_ring_queue *queue = ring->queues[qid]; + struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]); if (!queue) continue; @@ -579,6 +591,82 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh, return err; } +static int ring_header_type_offset(enum fuse_uring_header_type type) +{ + switch (type) { + case FUSE_URING_HEADER_IN_OUT: + return 0; + case FUSE_URING_HEADER_OP: + return offsetof(struct fuse_uring_req_header, op_in); + case FUSE_URING_HEADER_RING_ENT: + return offsetof(struct fuse_uring_req_header, ring_ent_in_out); + default: + WARN_ONCE(1, "Invalid header type: %d\n", type); + return -EINVAL; + } +} + +static int copy_header_to_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, + const void *header, size_t header_size) +{ + int offset = ring_header_type_offset(type); + void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_to_user(ring, header, header_size)) { + pr_info_ratelimited("Copying header to ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int copy_header_from_ring(struct fuse_ring_ent *ent, + enum fuse_uring_header_type type, void *header, + size_t header_size) +{ + int offset = ring_header_type_offset(type); + const void __user *ring; + + if (offset < 0) + return offset; + + ring = (void __user *)ent->headers + offset; + + if (copy_from_user(header, ring, header_size)) { + pr_info_ratelimited("Copying header from ring failed.\n"); + return -EFAULT; + } + + return 0; +} + +static int setup_fuse_copy_state(struct fuse_copy_state *cs, + struct fuse_ring *ring, struct fuse_req *req, + struct fuse_ring_ent *ent, int dir, + struct iov_iter *iter) +{ + int err; + + err = import_ubuf(dir, ent->payload, ring->max_payload_sz, iter); + if (err) { + pr_info_ratelimited("fuse: Import of user buffer failed\n"); + return err; + } + + fuse_copy_init(cs, dir == ITER_DEST, iter); + + cs->is_uring = true; + cs->req = req; + + return 0; +} + static int fuse_uring_copy_from_ring(struct fuse_ring *ring, struct fuse_req *req, struct fuse_ring_ent *ent) @@ -589,20 +677,15 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring, int err; struct fuse_uring_ent_in_out ring_in_out; - err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out, - sizeof(ring_in_out)); + err = copy_header_from_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ring_in_out, sizeof(ring_in_out)); if (err) - return -EFAULT; + return err; - err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz, - &iter); + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_SOURCE, &iter); if (err) return err; - fuse_copy_init(&cs, false, &iter); - cs.is_uring = true; - cs.req = req; - err = fuse_copy_out_args(&cs, args, ring_in_out.payload_sz); fuse_copy_finish(&cs); return err; @@ -625,15 +708,9 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, .commit_id = req->in.h.unique, }; - err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter); - if (err) { - pr_info_ratelimited("fuse: Import of user buffer failed\n"); + err = setup_fuse_copy_state(&cs, ring, req, ent, ITER_DEST, &iter); + if (err) return err; - } - - fuse_copy_init(&cs, true, &iter); - cs.is_uring = true; - cs.req = req; if (num_args > 0) { /* @@ -641,13 +718,11 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, * Some op code have that as zero size. */ if (args->in_args[0].size > 0) { - err = copy_to_user(&ent->headers->op_in, in_args->value, - in_args->size); - if (err) { - pr_info_ratelimited( - "Copying the header failed.\n"); - return -EFAULT; - } + err = copy_header_to_ring(ent, FUSE_URING_HEADER_OP, + in_args->value, + in_args->size); + if (err) + return err; } in_args++; num_args--; @@ -663,9 +738,8 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req, } ent_in_out.payload_sz = cs.ring.copied_sz; - err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out, - sizeof(ent_in_out)); - return err ? -EFAULT : 0; + return copy_header_to_ring(ent, FUSE_URING_HEADER_RING_ENT, + &ent_in_out, sizeof(ent_in_out)); } static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, @@ -673,6 +747,7 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; + struct fuse_in_header in_header; int err; err = -EIO; @@ -694,14 +769,9 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent, } /* copy fuse_in_header */ - err = copy_to_user(&ent->headers->in_out, &req->in.h, - sizeof(req->in.h)); - if (err) { - err = -EFAULT; - return err; - } - - return 0; + in_header = req->in.h; + return copy_header_to_ring(ent, FUSE_URING_HEADER_IN_OUT, &in_header, + sizeof(in_header)); } static int fuse_uring_prepare_send(struct fuse_ring_ent *ent, @@ -835,11 +905,13 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req, { struct fuse_ring *ring = ent->queue->ring; struct fuse_conn *fc = ring->fc; + struct fuse_out_header out_header; ssize_t err = -EFAULT; - if (copy_from_user(&req->out.h, &ent->headers->in_out, - sizeof(req->out.h))) + if (copy_header_from_ring(ent, FUSE_URING_HEADER_IN_OUT, &out_header, + sizeof(out_header))) goto out; + req->out.h = out_header; err = fuse_uring_out_header_has_err(&req->out.h, req, fc); if (err) { @@ -911,7 +983,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags, if (qid >= ring->nr_queues) return -EINVAL; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) return err; fpq = &queue->fpq; @@ -980,7 +1052,7 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) if (current_qid == qid) continue; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { ready = false; break; @@ -998,15 +1070,26 @@ static bool is_ring_ready(struct fuse_ring *ring, int current_qid) /* * fuse_uring_req_fetch command handling */ -static void fuse_uring_do_register(struct fuse_ring_ent *ent, - struct io_uring_cmd *cmd, - unsigned int issue_flags) +static int fuse_uring_do_register(struct fuse_ring_ent *ent, + struct io_uring_cmd *cmd, + unsigned int issue_flags) { struct fuse_ring_queue *queue = ent->queue; struct fuse_ring *ring = queue->ring; struct fuse_conn *fc = ring->fc; struct fuse_iqueue *fiq = &fc->iq; + spin_lock(&fc->lock); + /* abort teardown path is running or has run */ + if (!fc->connected) { + spin_unlock(&fc->lock); + if (atomic_dec_and_test(&ring->queue_refs)) + wake_up_all(&ring->stop_waitq); + kfree(ent); + return -ECONNABORTED; + } + spin_unlock(&fc->lock); + fuse_uring_prepare_cancel(cmd, issue_flags, ent); spin_lock(&queue->lock); @@ -1023,11 +1106,12 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent, wake_up_all(&fc->blocked_waitq); } } + return 0; } /* - * sqe->addr is a ptr to an iovec array, iov[0] has the headers, iov[1] - * the payload + * sqe->addr is a ptr to an iovec array, iov[FUSE_URING_IOV_HEADERS] has the + * headers, iov[FUSE_URING_IOV_PAYLOAD] the payload */ static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe, struct iovec iov[FUSE_URING_IOV_SEGS]) @@ -1057,8 +1141,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, { struct fuse_ring *ring = queue->ring; struct fuse_ring_ent *ent; - size_t payload_size; struct iovec iov[FUSE_URING_IOV_SEGS]; + struct iovec *headers, *payload; int err; err = fuse_uring_get_iovec_from_sqe(cmd->sqe, iov); @@ -1069,15 +1153,16 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, } err = -EINVAL; - if (iov[0].iov_len < sizeof(struct fuse_uring_req_header)) { - pr_info_ratelimited("Invalid header len %zu\n", iov[0].iov_len); + headers = &iov[FUSE_URING_IOV_HEADERS]; + if (headers->iov_len < sizeof(struct fuse_uring_req_header)) { + pr_info_ratelimited("Invalid header len %zu\n", headers->iov_len); return ERR_PTR(err); } - payload_size = iov[1].iov_len; - if (payload_size < ring->max_payload_sz) { + payload = &iov[FUSE_URING_IOV_PAYLOAD]; + if (payload->iov_len < ring->max_payload_sz) { pr_info_ratelimited("Invalid req payload len %zu\n", - payload_size); + payload->iov_len); return ERR_PTR(err); } @@ -1089,8 +1174,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd, INIT_LIST_HEAD(&ent->list); ent->queue = queue; - ent->headers = iov[0].iov_base; - ent->payload = iov[1].iov_base; + ent->headers = headers->iov_base; + ent->payload = payload->iov_base; atomic_inc(&ring->queue_refs); return ent; @@ -1122,7 +1207,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, return -EINVAL; } - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); if (!queue) { queue = fuse_uring_create_queue(ring, qid); if (!queue) @@ -1138,9 +1223,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd, if (IS_ERR(ent)) return PTR_ERR(ent); - fuse_uring_do_register(ent, cmd, issue_flags); - - return 0; + return fuse_uring_do_register(ent, cmd, issue_flags); } /* @@ -1276,7 +1359,7 @@ static struct fuse_ring_queue *fuse_uring_task_to_queue(struct fuse_ring *ring) ring->nr_queues)) qid = 0; - queue = ring->queues[qid]; + queue = READ_ONCE(ring->queues[qid]); WARN_ONCE(!queue, "Missing queue for qid %d\n", qid); return queue; diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c index 1bc6982b5d6aa..a013fad843b2c 100644 --- a/fs/fuse/dir.c +++ b/fs/fuse/dir.c @@ -1976,10 +1976,8 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, filemap_invalidate_lock(mapping); fault_blocked = true; err = fuse_dax_break_layouts(inode, 0, -1); - if (err) { - filemap_invalidate_unlock(mapping); - return err; - } + if (err) + goto unlock; } if (attr->ia_valid & ATTR_OPEN) { @@ -2006,7 +2004,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, ATTR_TIMES_SET)) { err = write_inode_now(inode, true); if (err) - return err; + goto unlock; fuse_set_nowrite(inode); fuse_release_nowrite(inode); @@ -2114,6 +2112,7 @@ int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry, clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); +unlock: if (fault_blocked) filemap_invalidate_unlock(mapping); return err; diff --git a/fs/fuse/file.c b/fs/fuse/file.c index 79d1b502e7319..0812eee881519 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -91,8 +91,7 @@ static struct fuse_file *fuse_file_get(struct fuse_file *ff) return ff; } -static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void fuse_release_end(struct fuse_args *args, int error) { struct fuse_release_args *ra = container_of(args, typeof(*ra), args); @@ -112,15 +111,15 @@ static void fuse_file_put(struct fuse_file *ff, bool sync) if (!args) { /* Do nothing when server does not implement 'opendir' */ } else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) { - fuse_release_end(ff->fm, args, 0); + fuse_release_end(args, 0); } else if (sync) { fuse_simple_request(ff->fm, args); - fuse_release_end(ff->fm, args, 0); + fuse_release_end(args, 0); } else { args->end = fuse_release_end; if (fuse_simple_background(ff->fm, args, GFP_KERNEL | __GFP_NOFAIL)) - fuse_release_end(ff->fm, args, -ENOTCONN); + fuse_release_end(args, -ENOTCONN); } kfree(ff); } @@ -268,7 +267,7 @@ static int fuse_open(struct inode *inode, struct file *file) filemap_invalidate_lock(inode->i_mapping); err = fuse_dax_break_layouts(inode, 0, -1); if (err) - goto out_inode_unlock; + goto out_unlock; } if (is_wb_truncate || dax_truncate) @@ -292,9 +291,9 @@ static int fuse_open(struct inode *inode, struct file *file) else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) invalidate_inode_pages2(inode->i_mapping); } +out_unlock: if (dax_truncate) filemap_invalidate_unlock(inode->i_mapping); -out_inode_unlock: if (is_wb_truncate || dax_truncate) inode_unlock(inode); @@ -709,8 +708,7 @@ static void fuse_io_free(struct fuse_io_args *ia) kfree(ia); } -static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, - int err) +static void fuse_aio_complete_req(struct fuse_args *args, int err) { struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); struct fuse_io_priv *io = ia->io; @@ -758,7 +756,7 @@ static ssize_t fuse_async_req_send(struct fuse_mount *fm, ia->ap.args.may_block = io->should_dirty; err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); if (err) - fuse_aio_complete_req(fm, &ia->ap.args, err); + fuse_aio_complete_req(&ia->ap.args, err); return num_bytes; } @@ -881,8 +879,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter, return fuse_do_readfolio(file, folio, off, len); } -static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, - int err) +static void fuse_readpages_end(struct fuse_args *args, int err) { int i; struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); @@ -947,7 +944,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file, res = fuse_simple_request(fm, &ap->args); err = res < 0 ? res : 0; } - fuse_readpages_end(fm, &ap->args, err); + fuse_readpages_end(&ap->args, err); } static void fuse_readahead(struct readahead_control *rac) @@ -1950,8 +1947,7 @@ __acquires(fi->lock) } } -static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void fuse_writepage_end(struct fuse_args *args, int error) { struct fuse_writepage_args *wpa = container_of(args, typeof(*wpa), ia.ap.args); @@ -2140,7 +2136,10 @@ static bool fuse_writepage_need_send(struct fuse_conn *fc, loff_t pos, WARN_ON(!ap->num_folios); - /* Reached max pages */ + /* Reached max pages or max folio slots */ + if (ap->num_folios >= fc->max_pages) + return true; + if ((bytes + PAGE_SIZE - 1) >> PAGE_SHIFT > fc->max_pages) return true; @@ -2525,8 +2524,9 @@ static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) struct fuse_file *ff = file->private_data; /* emulate flock with POSIX locks */ - ff->flock = true; err = fuse_setlk(file, fl, 1); + if (!err) + ff->flock = true; } return err; diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h index c288f28f6c6ea..7d26159af4059 100644 --- a/fs/fuse/fuse_i.h +++ b/fs/fuse/fuse_i.h @@ -13,6 +13,7 @@ # define pr_fmt(fmt) "fuse: " fmt #endif +#include "args.h" #include #include #include @@ -296,58 +297,6 @@ struct fuse_file { bool flock:1; }; -/** One input argument of a request */ -struct fuse_in_arg { - unsigned size; - const void *value; -}; - -/** One output argument of a request */ -struct fuse_arg { - unsigned size; - void *value; -}; - -/** FUSE folio descriptor */ -struct fuse_folio_desc { - unsigned int length; - unsigned int offset; -}; - -struct fuse_args { - uint64_t nodeid; - uint32_t opcode; - uint8_t in_numargs; - uint8_t out_numargs; - uint8_t ext_idx; - bool force:1; - bool noreply:1; - bool nocreds:1; - bool in_pages:1; - bool out_pages:1; - bool user_pages:1; - bool out_argvar:1; - bool page_zeroing:1; - bool page_replace:1; - bool may_block:1; - bool is_ext:1; - bool is_pinned:1; - bool invalidate_vmap:1; - bool abort_on_kill:1; - struct fuse_in_arg in_args[4]; - struct fuse_arg out_args[2]; - void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error); - /* Used for kvec iter backed by vmalloc address */ - void *vmap_base; -}; - -struct fuse_args_pages { - struct fuse_args args; - struct folio **folios; - struct fuse_folio_desc *descs; - unsigned int num_folios; -}; - struct fuse_release_args { struct fuse_args args; struct fuse_release_in inarg; diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c index e9ed693fc7b37..a0a64dfffb759 100644 --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -1311,13 +1311,14 @@ struct fuse_init_args { struct fuse_args args; struct fuse_init_in in; struct fuse_init_out out; + struct fuse_mount *fm; }; -static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args, - int error) +static void process_init_reply(struct fuse_args *args, int error) { - struct fuse_conn *fc = fm->fc; struct fuse_init_args *ia = container_of(args, typeof(*ia), args); + struct fuse_mount *fm = ia->fm; + struct fuse_conn *fc = fm->fc; struct fuse_init_out *arg = &ia->out; bool ok = true; @@ -1486,6 +1487,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm) ia = kzalloc(sizeof(*ia), GFP_KERNEL | __GFP_NOFAIL); + ia->fm = fm; ia->in.major = FUSE_KERNEL_VERSION; ia->in.minor = FUSE_KERNEL_MINOR_VERSION; ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE; @@ -1559,7 +1561,7 @@ int fuse_send_init(struct fuse_mount *fm) if (!err) return 0; } - process_init_reply(fm, &ia->args, err); + process_init_reply(&ia->args, err); if (fm->fc->conn_error) return -ENOTCONN; return 0; @@ -1741,6 +1743,8 @@ static int fuse_fill_super_submount(struct super_block *sb, fuse_fill_attr_from_inode(&root_attr, parent_fi); root = fuse_iget(sb, parent_fi->nodeid, 0, &root_attr, 0, 0, fuse_get_evict_ctr(fm->fc)); + if (!root) + return -ENOMEM; /* * This inode is just a duplicate, so it is not looked up and * its nlookup should not be incremented. fuse_iget() does diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c index b2f6486fe1d56..2b3daa93e299a 100644 --- a/fs/fuse/virtio_fs.c +++ b/fs/fuse/virtio_fs.c @@ -988,7 +988,9 @@ static int virtio_fs_setup_vqs(struct virtio_device *vdev, kfree(vqs); if (ret) { kfree(fs->vqs); + fs->vqs = NULL; kfree(fs->mq_map); + fs->mq_map = NULL; } return ret; } diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c index fdcac8e3f2ba2..836e260099c39 100644 --- a/fs/gfs2/bmap.c +++ b/fs/gfs2/bmap.c @@ -1321,6 +1321,19 @@ static int gfs2_block_zero_range(struct inode *inode, loff_t from, loff_t length &gfs2_iomap_write_ops, NULL); } +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end) +{ + loff_t isize = i_size_read(inode); + unsigned int len = isize & ~PAGE_MASK; + + if (!len || isize >= end) + return 0; + len = PAGE_SIZE - len; + if (end - isize < len) + len = end - isize; + return gfs2_block_zero_range(inode, isize, len); +} + #define GFS2_JTRUNC_REVOKES 8192 /** @@ -2096,6 +2109,12 @@ static int do_grow(struct inode *inode, u64 size) unstuff = 1; } + if (!unstuff) { + error = gfs2_clear_beyond_eof(inode, size); + if (error) + goto do_grow_qunlock; + } + error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT + (unstuff && gfs2_is_jdata(ip) ? RES_JDATA : 0) + diff --git a/fs/gfs2/bmap.h b/fs/gfs2/bmap.h index 6cdc72dd55a3f..e3d6efdfd8903 100644 --- a/fs/gfs2/bmap.h +++ b/fs/gfs2/bmap.h @@ -58,6 +58,7 @@ int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock, unsigned int *extlen); int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock, unsigned *extlen, bool *new); +int gfs2_clear_beyond_eof(struct inode *inode, loff_t end); int gfs2_setattr_size(struct inode *inode, u64 size); int gfs2_truncatei_resume(struct gfs2_inode *ip); int gfs2_file_dealloc(struct gfs2_inode *ip); diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c index bc67fa058c845..ef57d6aa42ebb 100644 --- a/fs/gfs2/file.c +++ b/fs/gfs2/file.c @@ -1057,6 +1057,10 @@ static ssize_t gfs2_file_buffered_write(struct kiocb *iocb, goto out_unlock; } + ret = gfs2_clear_beyond_eof(inode, iocb->ki_pos); + if (ret) + goto out_unlock; + pagefault_disable(); ret = iomap_file_buffered_write(iocb, from, &gfs2_iomap_ops, &gfs2_iomap_write_ops, NULL); @@ -1266,6 +1270,12 @@ static long __gfs2_fallocate(struct file *file, int mode, loff_t offset, loff_t next = (next + 1) << sdp->sd_sb.sb_bsize_shift; + if (!(mode & FALLOC_FL_KEEP_SIZE)) { + error = gfs2_clear_beyond_eof(inode, offset + len); + if (error) + return error; + } + offset &= bsize_mask; len = next - offset; diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c index 898fc3937b449..4620f586411a6 100644 --- a/fs/gfs2/quota.c +++ b/fs/gfs2/quota.c @@ -254,9 +254,13 @@ static struct gfs2_quota_data *qd_alloc(unsigned hash, struct gfs2_sbd *sdp, str return NULL; } -static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash, - const struct gfs2_sbd *sdp, - struct kqid qid) +/* + * Lookup variant for callers which already hold qd_lock + bucket lock. + */ +static struct gfs2_quota_data * +gfs2_qd_search_bucket_noref(unsigned int hash, + const struct gfs2_sbd *sdp, + struct kqid qid) { struct gfs2_quota_data *qd; struct hlist_bl_node *h; @@ -264,12 +268,22 @@ static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash, hlist_bl_for_each_entry_rcu(qd, h, &qd_hash_table[hash], qd_hlist) { if (!qid_eq(qd->qd_id, qid)) continue; - if (qd->qd_sbd != sdp) - continue; - if (lockref_get_not_dead(&qd->qd_lockref)) { - list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + if (qd->qd_sbd == sdp) return qd; - } + } + + return NULL; +} + +static struct gfs2_quota_data * +gfs2_qd_search_bucket(unsigned int hash, const struct gfs2_sbd *sdp, struct kqid qid) +{ + struct gfs2_quota_data *qd; + + qd = gfs2_qd_search_bucket_noref(hash, sdp, qid); + if (qd && lockref_get_not_dead(&qd->qd_lockref)) { + list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru); + return qd; } return NULL; @@ -1433,7 +1447,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) qc = (struct gfs2_quota_change *)(bh->b_data + sizeof(struct gfs2_meta_header)); for (y = 0; y < sdp->sd_qc_per_block && slot < sdp->sd_quota_slots; - y++, slot++) { + y++, slot++, qc++) { struct gfs2_quota_data *old_qd, *qd; s64 qc_change = be64_to_cpu(qc->qc_change); u32 qc_flags = be32_to_cpu(qc->qc_flags); @@ -1441,7 +1455,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) USRQUOTA : GRPQUOTA; struct kqid qc_id = make_kqid(&init_user_ns, qtype, be32_to_cpu(qc->qc_id)); - qc++; if (!qc_change) continue; @@ -1458,7 +1471,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) spin_lock(&qd_lock); spin_lock_bucket(hash); - old_qd = gfs2_qd_search_bucket(hash, sdp, qc_id); + old_qd = gfs2_qd_search_bucket_noref(hash, sdp, qc_id); if (old_qd) { fs_err(sdp, "Corruption found in quota_change%u" "file: duplicate identifier in " @@ -1467,7 +1480,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp) spin_unlock_bucket(hash); spin_unlock(&qd_lock); - qd_put(old_qd); gfs2_glock_put(qd->qd_gl); kmem_cache_free(gfs2_quotad_cachep, qd); diff --git a/fs/hfs/catalog.c b/fs/hfs/catalog.c index b80ba40e38776..ccdbbffaaf7c1 100644 --- a/fs/hfs/catalog.c +++ b/fs/hfs/catalog.c @@ -340,7 +340,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) { struct super_block *sb; struct hfs_find_data fd; - struct hfs_readdir_data *rd; int res, type; hfs_dbg("name %s, cnid %u\n", str ? str->name : NULL, cnid); @@ -366,14 +365,6 @@ int hfs_cat_delete(u32 cnid, struct inode *dir, const struct qstr *str) } } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFS_I(dir)->open_dir_lock); - list_for_each_entry(rd, &HFS_I(dir)->open_dir_list, list) { - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFS_I(dir)->open_dir_lock); - res = hfs_brec_remove(&fd); if (res) goto out; diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c index 86a6b317b474a..130c2f3a417f0 100644 --- a/fs/hfs/dir.c +++ b/fs/hfs/dir.c @@ -97,7 +97,15 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfs_cat_key)); + err = hfs_brec_find(&fd); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; @@ -146,7 +154,6 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc(sizeof(struct hfs_readdir_data), GFP_KERNEL); if (!rd) { @@ -154,15 +161,8 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFS_I(inode)->open_dir_list); - spin_unlock(&HFS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfs_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, &fd.key->cat, sizeof(struct hfs_cat_key)); out: hfs_find_exit(&fd); @@ -171,13 +171,7 @@ static int hfs_readdir(struct file *file, struct dir_context *ctx) static int hfs_dir_release(struct inode *inode, struct file *file) { - struct hfs_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } diff --git a/fs/hfs/hfs.h b/fs/hfs/hfs.h index 6f194d0768b6f..f46d12ce04a3f 100644 --- a/fs/hfs/hfs.h +++ b/fs/hfs/hfs.h @@ -281,8 +281,7 @@ struct hfs_mdb { /*======== Data structures kept in memory ========*/ struct hfs_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfs_cat_key key; }; diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h index 38854df4c1b42..d9665226f5474 100644 --- a/fs/hfs/hfs_fs.h +++ b/fs/hfs/hfs_fs.h @@ -37,8 +37,6 @@ struct hfs_inode_info { struct hfs_cat_key cat_key; - struct list_head open_dir_list; - spinlock_t open_dir_lock; struct inode *rsrc_inode; struct mutex extents_lock; diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c index cd43eff72d13d..cd3dd5bd456e0 100644 --- a/fs/hfs/inode.c +++ b/fs/hfs/inode.c @@ -192,8 +192,6 @@ struct inode *hfs_new_inode(struct inode *dir, const struct qstr *name, umode_t return NULL; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); hfs_cat_build_key(sb, (btree_key *)&HFS_I(inode)->cat_key, dir->i_ino, name); next_id = atomic64_inc_return(&HFS_SB(sb)->next_id); BUG_ON(next_id > U32_MAX); @@ -334,8 +332,6 @@ static int hfs_read_inode(struct inode *inode, void *data) HFS_I(inode)->flags = 0; HFS_I(inode)->rsrc_inode = NULL; mutex_init(&HFS_I(inode)->extents_lock); - INIT_LIST_HEAD(&HFS_I(inode)->open_dir_list); - spin_lock_init(&HFS_I(inode)->open_dir_lock); /* Initialize the inode */ inode->i_uid = hsb->s_uid; diff --git a/fs/hfsplus/catalog.c b/fs/hfsplus/catalog.c index 6c8380f7208df..97333ca50ec8f 100644 --- a/fs/hfsplus/catalog.c +++ b/fs/hfsplus/catalog.c @@ -204,7 +204,7 @@ int hfsplus_find_cat(struct super_block *sb, u32 cnid, return err; type = be16_to_cpu(tmp.type); - if (type != HFSPLUS_FOLDER_THREAD && type != HFSPLUS_FILE_THREAD) { + if (!is_hfs_thread_record_type(type)) { pr_err("found bad thread record in catalog\n"); return -EIO; } @@ -332,7 +332,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) struct super_block *sb = dir->i_sb; struct hfs_find_data fd; struct hfsplus_fork_raw fork; - struct list_head *pos; int err, off; u16 type; @@ -350,23 +349,22 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) goto out; if (!str) { - int len; + hfsplus_cat_entry entry = {0}; hfsplus_cat_build_key_with_cnid(sb, fd.search_key, cnid); - err = hfs_brec_find(&fd, hfs_find_rec_by_key); + err = hfsplus_brec_read_cat(&fd, &entry); if (err) goto out; - off = fd.entryoffset + - offsetof(struct hfsplus_cat_thread, nodeName); - fd.search_key->cat.parent = cpu_to_be32(dir->i_ino); - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.length, off, 2); - len = be16_to_cpu(fd.search_key->cat.name.length) * 2; - hfs_bnode_read(fd.bnode, - &fd.search_key->cat.name.unicode, - off + 2, len); - fd.search_key->key_len = cpu_to_be16(6 + len); + type = be16_to_cpu(entry.type); + if (!is_hfs_thread_record_type(type)) { + pr_err("found bad thread record in catalog\n"); + err = -EIO; + goto out; + } + + hfsplus_cat_build_key_uni(fd.search_key, dir->i_ino, + &entry.thread.nodeName); } else { err = hfsplus_cat_build_key(sb, fd.search_key, dir->i_ino, str); if (unlikely(err)) @@ -391,16 +389,6 @@ int hfsplus_delete_cat(u32 cnid, struct inode *dir, const struct qstr *str) hfsplus_free_fork(sb, cnid, &fork, HFSPLUS_TYPE_RSRC); } - /* we only need to take spinlock for exclusion with ->release() */ - spin_lock(&HFSPLUS_I(dir)->open_dir_lock); - list_for_each(pos, &HFSPLUS_I(dir)->open_dir_list) { - struct hfsplus_readdir_data *rd = - list_entry(pos, struct hfsplus_readdir_data, list); - if (fd.tree->keycmp(fd.search_key, (void *)&rd->key) < 0) - rd->file->f_pos--; - } - spin_unlock(&HFSPLUS_I(dir)->open_dir_lock); - err = hfs_brec_remove(&fd); if (err) goto out; diff --git a/fs/hfsplus/dir.c b/fs/hfsplus/dir.c index 8aeb861969d37..8254d6c92eb94 100644 --- a/fs/hfsplus/dir.c +++ b/fs/hfsplus/dir.c @@ -185,7 +185,15 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) } if (ctx->pos >= inode->i_size) goto out; - err = hfs_brec_goto(&fd, ctx->pos - 1); + rd = file->private_data; + if (rd && rd->pos == ctx->pos) { + memcpy(fd.search_key, &rd->key, sizeof(struct hfsplus_cat_key)); + err = hfs_brec_find(&fd, hfs_find_rec_by_key); + if (err == -ENOENT) + err = hfs_brec_goto(&fd, 1); + } else { + err = hfs_brec_goto(&fd, ctx->pos - 1); + } if (err) goto out; for (;;) { @@ -261,7 +269,6 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) if (err) goto out; } - rd = file->private_data; if (!rd) { rd = kmalloc(sizeof(struct hfsplus_readdir_data), GFP_KERNEL); if (!rd) { @@ -269,15 +276,8 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) goto out; } file->private_data = rd; - rd->file = file; - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_add(&rd->list, &HFSPLUS_I(inode)->open_dir_list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); } - /* - * Can be done after the list insertion; exclusion with - * hfsplus_delete_cat() is provided by directory lock. - */ + rd->pos = ctx->pos; memcpy(&rd->key, fd.key, sizeof(struct hfsplus_cat_key)); out: kfree(strbuf); @@ -287,13 +287,7 @@ static int hfsplus_readdir(struct file *file, struct dir_context *ctx) static int hfsplus_dir_release(struct inode *inode, struct file *file) { - struct hfsplus_readdir_data *rd = file->private_data; - if (rd) { - spin_lock(&HFSPLUS_I(inode)->open_dir_lock); - list_del(&rd->list); - spin_unlock(&HFSPLUS_I(inode)->open_dir_lock); - kfree(rd); - } + kfree(file->private_data); return 0; } diff --git a/fs/hfsplus/hfsplus_fs.h b/fs/hfsplus/hfsplus_fs.h index 2da2bd52d200e..08ed2381c719f 100644 --- a/fs/hfsplus/hfsplus_fs.h +++ b/fs/hfsplus/hfsplus_fs.h @@ -215,8 +215,6 @@ struct hfsplus_inode_info { sector_t fs_blocks; u8 userflags; /* BSD user file flags */ u32 subfolders; /* Subfolder count (HFSX only) */ - struct list_head open_dir_list; - spinlock_t open_dir_lock; loff_t phys_size; struct inode vfs_inode; @@ -266,8 +264,7 @@ struct hfs_find_data { }; struct hfsplus_readdir_data { - struct list_head list; - struct file *file; + loff_t pos; struct hfsplus_cat_key key; }; @@ -514,6 +511,12 @@ static inline u32 hfsplus_cat_thread_size(const struct hfsplus_cat_thread *threa be16_to_cpu(thread->nodeName.length) * sizeof(hfsplus_unichr); } +static inline +bool is_hfs_thread_record_type(u16 type) +{ + return type == HFSPLUS_FOLDER_THREAD || type == HFSPLUS_FILE_THREAD; +} + int hfsplus_brec_read_cat(struct hfs_find_data *fd, hfsplus_cat_entry *entry); /* diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c index 6153e5cc6eb65..ab1c343955b10 100644 --- a/fs/hfsplus/inode.c +++ b/fs/hfsplus/inode.c @@ -125,9 +125,44 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) struct file *file = iocb->ki_filp; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; + loff_t isize; size_t count = iov_iter_count(iter); + loff_t end = iocb->ki_pos + count; ssize_t ret; + /* + * The hfsplus_get_block() only allows creating the next sequential block. + * For direct writes beyond EOF, expand the file first. + */ + if (iov_iter_rw(iter) == WRITE && iocb->ki_pos > i_size_read(inode)) { + loff_t start_off, end_off; + loff_t start_page, end_page; + + isize = i_size_read(inode); + + /* + * Wait for any in-flight DIO on this inode to finish before + * calling generic_cont_expand_simple(). + */ + inode_dio_wait(inode); + + ret = generic_cont_expand_simple(inode, iocb->ki_pos); + if (ret) + return ret; + + start_off = isize; + end_off = (end > 0) ? end - 1 : end; + + ret = filemap_write_and_wait_range(mapping, start_off, end_off); + if (ret) + return ret; + + start_page = start_off >> PAGE_SHIFT; + end_page = end_off >> PAGE_SHIFT; + + invalidate_inode_pages2_range(mapping, start_page, end_page); + } + ret = blockdev_direct_IO(iocb, inode, iter, hfsplus_get_block); /* @@ -135,8 +170,7 @@ static ssize_t hfsplus_direct_IO(struct kiocb *iocb, struct iov_iter *iter) * blocks outside i_size. Trim these off again. */ if (unlikely(iov_iter_rw(iter) == WRITE && ret < 0)) { - loff_t isize = i_size_read(inode); - loff_t end = iocb->ki_pos + count; + isize = i_size_read(inode); if (end > isize) hfsplus_write_failed(mapping, end); @@ -425,8 +459,6 @@ struct inode *hfsplus_new_inode(struct super_block *sb, struct inode *dir, simple_inode_init_ts(inode); hip = HFSPLUS_I(inode); - INIT_LIST_HEAD(&hip->open_dir_list); - spin_lock_init(&hip->open_dir_lock); mutex_init(&hip->extents_lock); atomic_set(&hip->opencnt, 0); hip->extent_state = 0; diff --git a/fs/hfsplus/super.c b/fs/hfsplus/super.c index e1e2833f528d1..d031c77fb4eb3 100644 --- a/fs/hfsplus/super.c +++ b/fs/hfsplus/super.c @@ -91,8 +91,6 @@ struct inode *hfsplus_iget(struct super_block *sb, unsigned long ino) HFSPLUS_I(inode)->fs_blocks = 0; HFSPLUS_I(inode)->userflags = 0; HFSPLUS_I(inode)->subfolders = 0; - INIT_LIST_HEAD(&HFSPLUS_I(inode)->open_dir_list); - spin_lock_init(&HFSPLUS_I(inode)->open_dir_lock); HFSPLUS_I(inode)->phys_size = 0; if (inode->i_ino >= HFSPLUS_FIRSTUSER_CNID || diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c index 8ab85e7ac91eb..553e6b6b4600b 100644 --- a/fs/hpfs/super.c +++ b/fs/hpfs/super.c @@ -522,7 +522,8 @@ static int hpfs_fill_super(struct super_block *s, struct fs_context *fc) hpfs_lock(s); /*sbi->sb_mounting = 1;*/ - sb_set_blocksize(s, 512); + if (!sb_set_blocksize(s, 512)) + goto bail0; sbi->sb_fs_size = -1; if (!(bootblock = hpfs_map_sector(s, 0, &bh0, 0))) goto bail1; if (!(superblock = hpfs_map_sector(s, 16, &bh1, 1))) goto bail2; diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index f42548ee9083c..34ed9966f2ea8 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -1433,7 +1433,8 @@ hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc) goto out_free; return 0; out_free: - kfree(sbinfo->spool); + if (sbinfo->spool) + hugepage_put_subpool(sbinfo->spool); kfree(sbinfo); return -ENOMEM; } diff --git a/fs/inode.c b/fs/inode.c index 2c55ec49b0239..a931126398ced 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -2703,8 +2703,8 @@ struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 t { trace_inode_set_ctime_to_ts(inode, &ts); set_normalized_timespec64(&ts, ts.tv_sec, ts.tv_nsec); - inode->i_ctime_sec = ts.tv_sec; - inode->i_ctime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_ctime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_ctime_nsec, ts.tv_nsec); return ts; } EXPORT_SYMBOL(inode_set_ctime_to_ts); @@ -2778,7 +2778,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) */ cns = smp_load_acquire(&inode->i_ctime_nsec); if (cns & I_CTIME_QUERIED) { - struct timespec64 ctime = { .tv_sec = inode->i_ctime_sec, + struct timespec64 ctime = { .tv_sec = inode_get_ctime_sec(inode), .tv_nsec = cns & ~I_CTIME_QUERIED }; if (timespec64_compare(&now, &ctime) <= 0) { @@ -2790,7 +2790,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) mgtime_counter_inc(mg_ctime_updates); /* No need to cmpxchg if it's exactly the same */ - if (cns == now.tv_nsec && inode->i_ctime_sec == now.tv_sec) { + if (cns == now.tv_nsec && inode_get_ctime_sec(inode) == now.tv_sec) { trace_ctime_xchg_skip(inode, &now); goto out; } @@ -2799,7 +2799,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) /* Try to swap the nsec value into place. */ if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { /* If swap occurred, then we're (mostly) done */ - inode->i_ctime_sec = now.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, now.tv_sec); trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); mgtime_counter_inc(mg_ctime_swaps); } else { @@ -2814,7 +2814,7 @@ struct timespec64 inode_set_ctime_current(struct inode *inode) goto retry; } /* Otherwise, keep the existing ctime */ - now.tv_sec = inode->i_ctime_sec; + now.tv_sec = inode_get_ctime_sec(inode); now.tv_nsec = cur & ~I_CTIME_QUERIED; } out: @@ -2847,7 +2847,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u /* pairs with try_cmpxchg below */ cur = smp_load_acquire(&inode->i_ctime_nsec); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); /* If the update is older than the existing value, skip it. */ if (timespec64_compare(&update, &cur_ts) <= 0) @@ -2873,7 +2873,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u retry: old = cur; if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { - inode->i_ctime_sec = update.tv_sec; + WRITE_ONCE(inode->i_ctime_sec, update.tv_sec); mgtime_counter_inc(mg_ctime_swaps); return update; } @@ -2889,7 +2889,7 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 u goto retry; /* Otherwise, it was a new timestamp. */ - cur_ts.tv_sec = inode->i_ctime_sec; + cur_ts.tv_sec = inode_get_ctime_sec(inode); cur_ts.tv_nsec = cur & ~I_CTIME_QUERIED; return cur_ts; } diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index c0fa6acf19375..c9ece378687b9 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -47,11 +47,13 @@ static bool ifs_set_range_uptodate(struct folio *folio, struct iomap_folio_state *ifs, size_t off, size_t len) { struct inode *inode = folio->mapping->host; - unsigned int first_blk = off >> inode->i_blkbits; - unsigned int last_blk = (off + len - 1) >> inode->i_blkbits; - unsigned int nr_blks = last_blk - first_blk + 1; + unsigned int first_blk, last_blk; - bitmap_set(ifs->state, first_blk, nr_blks); + if (len) { + first_blk = off >> inode->i_blkbits; + last_blk = (off + len - 1) >> inode->i_blkbits; + bitmap_set(ifs->state, first_blk, last_blk - first_blk + 1); + } return ifs_is_fully_uptodate(folio, ifs); } @@ -127,13 +129,17 @@ static void ifs_clear_range_dirty(struct folio *folio, { struct inode *inode = folio->mapping->host; unsigned int blks_per_folio = i_blocks_per_folio(inode, folio); - unsigned int first_blk = (off >> inode->i_blkbits); - unsigned int last_blk = (off + len - 1) >> inode->i_blkbits; - unsigned int nr_blks = last_blk - first_blk + 1; + unsigned int first_blk = round_up(off, i_blocksize(inode)) >> + inode->i_blkbits; + unsigned int last_blk = (off + len) >> inode->i_blkbits; unsigned long flags; + if (first_blk >= last_blk) + return; + spin_lock_irqsave(&ifs->state_lock, flags); - bitmap_clear(ifs->state, first_blk + blks_per_folio, nr_blks); + bitmap_clear(ifs->state, first_blk + blks_per_folio, + last_blk - first_blk); spin_unlock_irqrestore(&ifs->state_lock, flags); } @@ -150,13 +156,17 @@ static void ifs_set_range_dirty(struct folio *folio, { struct inode *inode = folio->mapping->host; unsigned int blks_per_folio = i_blocks_per_folio(inode, folio); - unsigned int first_blk = (off >> inode->i_blkbits); - unsigned int last_blk = (off + len - 1) >> inode->i_blkbits; - unsigned int nr_blks = last_blk - first_blk + 1; + unsigned int first_blk, last_blk; unsigned long flags; + if (!len) + return; + + first_blk = off >> inode->i_blkbits; + last_blk = (off + len - 1) >> inode->i_blkbits; spin_lock_irqsave(&ifs->state_lock, flags); - bitmap_set(ifs->state, first_blk + blks_per_folio, nr_blks); + bitmap_set(ifs->state, first_blk + blks_per_folio, + last_blk - first_blk + 1); spin_unlock_irqrestore(&ifs->state_lock, flags); } diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index e73c71f39bd45..92f32e02f77f4 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -74,7 +74,7 @@ static void iomap_dio_submit_bio(const struct iomap_iter *iter, /* Sync dio can't be polled reliably */ if ((iocb->ki_flags & IOCB_HIPRI) && !is_sync_kiocb(iocb)) { - bio_set_polled(bio, iocb); + bio->bi_opf |= REQ_POLLED; WRITE_ONCE(iocb->private, bio); } diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index 6c75fbf4d5af4..2a86b05c88efc 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -11,6 +11,7 @@ struct bio_set iomap_ioend_bioset; EXPORT_SYMBOL_GPL(iomap_ioend_bioset); +static struct bio_set iomap_ioend_split_bioset; struct iomap_ioend *iomap_init_ioend(struct inode *inode, struct bio *bio, loff_t file_offset, u16 ioend_flags) @@ -409,7 +410,8 @@ struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; - split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); + split = bio_split(bio, sector_offset, GFP_NOFS, + &iomap_ioend_split_bioset); if (IS_ERR(split)) return ERR_CAST(split); split->bi_private = bio->bi_private; @@ -432,8 +434,23 @@ EXPORT_SYMBOL_GPL(iomap_split_ioend); static int __init iomap_ioend_init(void) { - return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), + const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE); + int error; + + error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries, offsetof(struct iomap_ioend, io_bio), BIOSET_NEED_BVECS); + if (error) + return error; + error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries, + offsetof(struct iomap_ioend, io_bio), + BIOSET_NEED_BVECS); + if (error) + goto out_exit_ioend_bioset; + return 0; + +out_exit_ioend_bioset: + bioset_exit(&iomap_ioend_bioset); + return error; } fs_initcall(iomap_ioend_init); diff --git a/fs/isofs/compress.c b/fs/isofs/compress.c index 5f3b6da0e0225..4500b14f802a6 100644 --- a/fs/isofs/compress.c +++ b/fs/isofs/compress.c @@ -65,12 +65,14 @@ static loff_t zisofs_uncompress_block(struct inode *inode, loff_t block_start, /* Empty block? */ if (block_size == 0) { for ( i = 0 ; i < pcount ; i++ ) { + unsigned int off = i ? 0 : poffset; + if (!pages[i]) continue; - memzero_page(pages[i], 0, PAGE_SIZE); + memzero_page(pages[i], off, PAGE_SIZE - off); SetPageUptodate(pages[i]); } - return ((loff_t)pcount) << PAGE_SHIFT; + return (((loff_t)pcount) << PAGE_SHIFT) - poffset; } /* Because zlib is not thread-safe, do all the I/O at the top. */ @@ -291,6 +293,7 @@ static int zisofs_fill_pages(struct inode *inode, int full_page, int pcount, memzero_page(*pages, poffset, PAGE_SIZE - poffset); SetPageUptodate(*pages); } + brelse(bh); return 0; } diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c index ad3143d4066bf..3433fa14cd014 100644 --- a/fs/isofs/inode.c +++ b/fs/isofs/inode.c @@ -818,7 +818,8 @@ static int isofs_fill_super(struct super_block *s, struct fs_context *fc) * entries. By forcing the blocksize in this way, we ensure * that we will never be required to do this. */ - sb_set_blocksize(s, orig_zonesize); + if (!sb_set_blocksize(s, orig_zonesize)) + goto out_freesbi; sbi->s_nls_iocharset = NULL; diff --git a/fs/jbd2/checkpoint.c b/fs/jbd2/checkpoint.c index 6dbfb9cb07d7a..d0a89ca1a4e44 100644 --- a/fs/jbd2/checkpoint.c +++ b/fs/jbd2/checkpoint.c @@ -358,15 +358,16 @@ int jbd2_cleanup_journal_tail(journal_t *journal) /* * journal_shrink_one_cp_list * - * Find all the written-back checkpoint buffers in the given list - * and try to release them. If the whole transaction is released, set - * the 'released' parameter. Return the number of released checkpointed - * buffers. + * Find written-back checkpoint buffers in the given list and try to release + * them. If 'nr_to_scan' is set, scan at most that many buffers. If the whole + * transaction is released, set the 'released' parameter. Return the number of + * released checkpointed buffers. * * Called with j_list_lock held. */ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, enum jbd2_shrink_type type, + unsigned long *nr_to_scan, bool *released) { struct journal_head *last_jh; @@ -375,13 +376,15 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, int ret; *released = false; - if (!jh) + if (!jh || (nr_to_scan && !*nr_to_scan)) return 0; last_jh = jh->b_cpprev; do { jh = next_jh; next_jh = jh->b_cpnext; + if (nr_to_scan) + (*nr_to_scan)--; if (type == JBD2_SHRINK_DESTROY) { ret = __jbd2_journal_remove_checkpoint(jh); @@ -389,7 +392,7 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, ret = jbd2_journal_try_remove_checkpoint(jh); if (ret < 0) { if (type == JBD2_SHRINK_BUSY_SKIP) - continue; + goto next; break; } } @@ -400,9 +403,10 @@ static unsigned long journal_shrink_one_cp_list(struct journal_head *jh, break; } +next: if (need_resched()) break; - } while (jh != last_jh); + } while (jh != last_jh && (!nr_to_scan || *nr_to_scan)); return nr_freed; } @@ -424,7 +428,6 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, tid_t first_tid = 0, last_tid = 0, next_tid = 0; tid_t tid = 0; unsigned long nr_freed = 0; - unsigned long freed; bool first_set = false; again: @@ -457,10 +460,9 @@ unsigned long jbd2_journal_shrink_checkpoint_list(journal_t *journal, next_transaction = transaction->t_cpnext; tid = transaction->t_tid; - freed = journal_shrink_one_cp_list(transaction->t_checkpoint_list, - JBD2_SHRINK_BUSY_SKIP, &released); - nr_freed += freed; - (*nr_to_scan) -= min(*nr_to_scan, freed); + nr_freed += journal_shrink_one_cp_list(transaction->t_checkpoint_list, + JBD2_SHRINK_BUSY_SKIP, + nr_to_scan, &released); if (*nr_to_scan == 0) break; if (need_resched() || spin_needbreak(&journal->j_list_lock)) @@ -516,7 +518,7 @@ void __jbd2_journal_clean_checkpoint_list(journal_t *journal, transaction = next_transaction; next_transaction = transaction->t_cpnext; journal_shrink_one_cp_list(transaction->t_checkpoint_list, - type, &released); + type, NULL, &released); /* * This function only frees up some memory if possible so we * dont have an obligation to finish processing. Bail out if diff --git a/fs/jfs/super.c b/fs/jfs/super.c index 3cfb86c5a36e8..20b66fb022244 100644 --- a/fs/jfs/super.c +++ b/fs/jfs/super.c @@ -491,7 +491,8 @@ static int jfs_fill_super(struct super_block *sb, struct fs_context *fc) /* * Initialize blocksize to 4K. */ - sb_set_blocksize(sb, PSIZE); + if (!sb_set_blocksize(sb, PSIZE)) + goto out_unload; /* * Set method vectors. diff --git a/fs/lockd/clntproc.c b/fs/lockd/clntproc.c index cebcc283b7ce2..8728b53cf2e8a 100644 --- a/fs/lockd/clntproc.c +++ b/fs/lockd/clntproc.c @@ -487,9 +487,12 @@ static const struct file_lock_operations nlmclnt_lock_ops = { static void nlmclnt_locks_init_private(struct file_lock *fl, struct nlm_host *host) { fl->fl_u.nfs_fl.state = 0; + fl->fl_ops = NULL; fl->fl_u.nfs_fl.owner = nlmclnt_find_lockowner(host, fl->c.flc_owner); INIT_LIST_HEAD(&fl->fl_u.nfs_fl.list); + if (!fl->fl_u.nfs_fl.owner) + return; fl->fl_ops = &nlmclnt_lock_ops; } diff --git a/fs/lockd/svcsubs.c b/fs/lockd/svcsubs.c index d11beea76ddb5..4075d0eab263f 100644 --- a/fs/lockd/svcsubs.c +++ b/fs/lockd/svcsubs.c @@ -301,12 +301,10 @@ nlm_file_inuse(struct nlm_file *file) return 0; } -static void nlm_close_files(struct nlm_file *file) +static void nlm_file_release(struct nlm_file *file) { - if (file->f_file[O_RDONLY]) - nlmsvc_ops->fclose(file->f_file[O_RDONLY]); - if (file->f_file[O_WRONLY]) - nlmsvc_ops->fclose(file->f_file[O_WRONLY]); + if (!nlm_file_inuse(file)) + nlm_delete_file(file); } /* @@ -316,32 +314,41 @@ static int nlm_traverse_files(void *data, nlm_host_match_fn_t match, int (*is_failover_file)(void *data, struct nlm_file *file)) { - struct hlist_node *next; - struct nlm_file *file; + struct nlm_file *file, *next; int i, ret = 0; mutex_lock(&nlm_file_mutex); for (i = 0; i < FILE_NRHASH; i++) { - hlist_for_each_entry_safe(file, next, &nlm_files[i], f_list) { - if (is_failover_file && !is_failover_file(data, file)) - continue; + file = hlist_entry_safe(nlm_files[i].first, + struct nlm_file, f_list); + if (file) file->f_count++; - mutex_unlock(&nlm_file_mutex); - - /* Traverse locks, blocks and shares of this file - * and update file->f_locks count */ - if (nlm_inspect_file(data, file, match)) - ret = 1; + while (file) { + /* + * Pin the next neighbour before we drop the mutex + * for nlm_inspect_file(); a concurrent + * nlm_release_file() under the same mutex would + * otherwise be free to unlink and kfree it during + * the unlock window, leaving us to dereference a + * freed slab when we walked to next afterwards. + */ + next = hlist_entry_safe(file->f_list.next, + struct nlm_file, f_list); + if (next) + next->f_count++; + + if (!is_failover_file || is_failover_file(data, file)) { + mutex_unlock(&nlm_file_mutex); + + if (nlm_inspect_file(data, file, match)) + ret = 1; + + mutex_lock(&nlm_file_mutex); + } - mutex_lock(&nlm_file_mutex); file->f_count--; - /* No more references to this file. Let go of it. */ - if (list_empty(&file->f_blocks) && !file->f_locks - && !file->f_shares && !file->f_count) { - hlist_del(&file->f_list); - nlm_close_files(file); - kfree(file); - } + nlm_file_release(file); + file = next; } } mutex_unlock(&nlm_file_mutex); @@ -501,7 +508,7 @@ EXPORT_SYMBOL_GPL(nlmsvc_unlock_all_by_sb); static int nlmsvc_match_ip(void *datap, struct nlm_host *host) { - return rpc_cmp_addr(nlm_srcaddr(host), datap); + return rpc_cmp_addr(nlm_srcaddr(datap), (struct sockaddr *)host); } /** diff --git a/fs/minix/inode.c b/fs/minix/inode.c index 7bdd240ea1584..5bfe42599a585 100644 --- a/fs/minix/inode.c +++ b/fs/minix/inode.c @@ -272,7 +272,8 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc) sbi->s_namelen = 60; sbi->s_version = MINIX_V3; sbi->s_mount_state = MINIX_VALID_FS; - sb_set_blocksize(s, m3s->s_blocksize); + if (!sb_set_blocksize(s, m3s->s_blocksize)) + goto out; s->s_max_links = MINIX2_LINK_MAX; } else goto out_no_fs; diff --git a/fs/namespace.c b/fs/namespace.c index 5fd8b021785a4..d886ae255e40a 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -4352,6 +4352,11 @@ SYSCALL_DEFINE3(fsmount, int, fs_fd, unsigned int, flags, ret = PTR_ERR(newmount.mnt); goto err_unlock; } + if (newmount.mnt->mnt_sb->s_flags & SB_NOUSER) { + mntput(newmount.mnt); + ret = -EINVAL; + goto err_unlock; + } newmount.dentry = dget(fc->root); newmount.mnt->mnt_flags = mnt_flags; diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index fab3181c7f869..cccc860893f0a 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -102,8 +102,10 @@ static ssize_t netfs_prepare_read_iterator(struct netfs_io_subrequest *subreq, added = rolling_buffer_load_from_ra(&rreq->buffer, ractl, &put_batch); - if (added < 0) + if (added < 0) { + folio_batch_release(&put_batch); return added; + } rreq->submitted += added; } folio_batch_release(&put_batch); @@ -658,7 +660,7 @@ int netfs_write_begin(struct netfs_inode *ctx, * within the cache granule containing the EOF, in which case we need * to preload the granule. */ - if (!netfs_is_cache_enabled(ctx) && + if (!netfs_is_cache_maybe_enabled(ctx) && netfs_skip_folio_read(folio, pos, len, false)) { netfs_stat(&netfs_n_rh_write_zskip); goto have_folio_no_wait; diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c index dd0ce7b769ce0..0f61746bee2ac 100644 --- a/fs/netfs/buffered_write.c +++ b/fs/netfs/buffered_write.c @@ -278,7 +278,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter, * caching service temporarily because the backing store got * culled. */ - if (netfs_is_cache_enabled(ctx)) { + if (netfs_is_cache_maybe_enabled(ctx)) { if (finfo) { netfs_stat(&netfs_n_wh_wstream_conflict); goto flush_content; diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c index f9ab69de3e298..e05f69eb56a6c 100644 --- a/fs/netfs/direct_write.c +++ b/fs/netfs/direct_write.c @@ -21,7 +21,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) /* Okay, declare that all I/O is complete. */ trace_netfs_rreq(wreq, netfs_rreq_trace_write_done); - if (!wreq->error) + if (wreq->transferred) netfs_update_i_size(ictx, &ictx->inode, wreq->start, wreq->transferred); if (wreq->origin == NETFS_DIO_WRITE && @@ -51,7 +51,7 @@ static void netfs_unbuffered_write_done(struct netfs_io_request *wreq) wreq->iocb->ki_pos += written; if (wreq->iocb->ki_complete) { trace_netfs_rreq(wreq, netfs_rreq_trace_ki_complete); - wreq->iocb->ki_complete(wreq->iocb, wreq->error ?: written); + wreq->iocb->ki_complete(wreq->iocb, written ?: wreq->error); } wreq->iocb = VFS_PTR_POISON; } @@ -95,7 +95,7 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) { struct netfs_io_subrequest *subreq = NULL; struct netfs_io_stream *stream = &wreq->io_streams[0]; - int ret; + int ret = 0; _enter("%llx", wreq->len); @@ -110,6 +110,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) if (!subreq) { netfs_prepare_write(wreq, stream, wreq->start + wreq->transferred); subreq = stream->construct; + if (!subreq) { + wreq->error = -ENOMEM; + ret = -ENOMEM; + break; + } stream->construct = NULL; } @@ -121,8 +126,14 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) } iov_iter_truncate(&subreq->io_iter, wreq->len - wreq->transferred); - if (!iov_iter_count(&subreq->io_iter)) + if (!iov_iter_count(&subreq->io_iter)) { + pr_warn("netfs: Unexpected zero-length iterator R=%08x\n", + wreq->debug_id); + __set_bit(NETFS_SREQ_FAILED, &subreq->flags); + netfs_write_subrequest_terminated(subreq, -EIO); + wreq->error = -EIO; break; + } subreq->len = netfs_limit_iter(&subreq->io_iter, 0, stream->sreq_max_len, @@ -139,13 +150,11 @@ static int netfs_unbuffered_write(struct netfs_io_request *wreq) if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { retry = true; } else if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { - ret = subreq->error; - wreq->error = ret; + wreq->error = subreq->error; netfs_see_subrequest(subreq, netfs_sreq_trace_see_failed); subreq = NULL; break; } - ret = 0; if (!retry) { netfs_unbuffered_write_collect(wreq, stream, subreq); @@ -292,11 +301,11 @@ ssize_t netfs_unbuffered_write_iter_locked(struct kiocb *iocb, struct iov_iter * ret = -EIOCBQUEUED; } else { ret = netfs_unbuffered_write(wreq); - if (ret < 0) { - _debug("begin = %zd", ret); - } else { + if (wreq->transferred) { iocb->ki_pos += wreq->transferred; - ret = wreq->transferred ?: wreq->error; + ret = wreq->transferred; + } else if (wreq->error) { + ret = wreq->error; } netfs_put_request(wreq, netfs_rreq_trace_put_complete); diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 645996ecfc803..d889caa401dc2 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -239,6 +239,18 @@ static inline bool netfs_is_cache_enabled(struct netfs_inode *ctx) #endif } +static inline bool netfs_is_cache_maybe_enabled(struct netfs_inode *ctx) +{ +#if IS_ENABLED(CONFIG_FSCACHE) + struct fscache_cookie *cookie = ctx->cache; + + return fscache_cookie_valid(cookie) && + test_bit(FSCACHE_COOKIE_IS_CACHING, &cookie->flags); +#else + return false; +#endif +} + /* * Get a ref on a netfs group attached to a dirty page (e.g. a ceph snap). */ diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c index a1489aa29f782..7eacc58abadb7 100644 --- a/fs/netfs/read_pgpriv2.c +++ b/fs/netfs/read_pgpriv2.c @@ -54,6 +54,7 @@ static void netfs_pgpriv2_copy_folio(struct netfs_io_request *creq, struct folio /* Attach the folio to the rolling buffer. */ if (rolling_buffer_append(&creq->buffer, folio, 0) < 0) { + folio_end_private_2(folio); clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &creq->flags); return; } diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c index 999177426141a..2b42758e01ec9 100644 --- a/fs/netfs/read_retry.c +++ b/fs/netfs/read_retry.c @@ -98,7 +98,12 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) goto abandon; } - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || @@ -175,7 +180,9 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) list_for_each_entry_safe_from(subreq, tmp, &stream->subrequests, rreq_link) { trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); + spin_lock(&rreq->lock); list_del(&subreq->rreq_link); + spin_unlock(&rreq->lock); netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); if (subreq == to) break; @@ -203,8 +210,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) refcount_read(&subreq->ref), netfs_sreq_trace_new); + spin_lock(&rreq->lock); list_add(&subreq->rreq_link, &to->rreq_link); - to = list_next_entry(to, rreq_link); + spin_unlock(&rreq->lock); + to = subreq; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); stream->sreq_max_len = umin(len, rreq->rsize); diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index 76614339554ea..ceb1a684a758c 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -170,6 +170,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq, rolling_buffer_make_space(&wreq->buffer); subreq = netfs_alloc_subrequest(wreq); + if (!subreq) + return; subreq->source = stream->source; subreq->start = start; subreq->stream_nr = stream->stream_nr; @@ -730,6 +732,7 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq, size_t iter_off = 0; size_t fsize = folio_size(folio), flen; loff_t fpos = folio_pos(folio); + ssize_t ret; bool to_eof = false; bool no_debug = false; @@ -758,7 +761,11 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq, /* Attach the folio to the rolling buffer. */ folio_get(folio); - rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK); + ret = rolling_buffer_append(&wreq->buffer, folio, NETFS_ROLLBUF_PUT_MARK); + if (ret < 0) { + folio_put(folio); + return ret; + } /* Move the submission point forward to allow for write-streaming data * not starting at the front of the page. We don't do write-streaming diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c index 29489a23a2209..058bc7a166a59 100644 --- a/fs/netfs/write_retry.c +++ b/fs/netfs/write_retry.c @@ -72,7 +72,12 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) return; - list_for_each_continue(next, &stream->subrequests) { + for (;;) { + /* Read pointer to subreq before reading subreq state. */ + next = smp_load_acquire(&next->next); + if (next == &stream->subrequests) + break; + subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); if (subreq->start + subreq->transferred != start + len || test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || @@ -130,7 +135,9 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, list_for_each_entry_safe_from(subreq, tmp, &stream->subrequests, rreq_link) { trace_netfs_sreq(subreq, netfs_sreq_trace_discard); + spin_lock(&wreq->lock); list_del(&subreq->rreq_link); + spin_unlock(&wreq->lock); netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); if (subreq == to) break; @@ -153,8 +160,10 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq, netfs_sreq_trace_new); trace_netfs_sreq(subreq, netfs_sreq_trace_split); + spin_lock(&wreq->lock); list_add(&subreq->rreq_link, &to->rreq_link); - to = list_next_entry(to, rreq_link); + spin_unlock(&wreq->lock); + to = subreq; trace_netfs_sreq(subreq, netfs_sreq_trace_retry); stream->sreq_max_len = len; diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index 134d7f760a33a..4b0d1bb50dd6e 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -85,15 +85,17 @@ bl_free_device(struct pnfs_block_dev *dev) { bl_unregister_dev(dev); - if (dev->nr_children) { + if (dev->children) { int i; for (i = 0; i < dev->nr_children; i++) bl_free_device(&dev->children[i]); kfree(dev->children); - } else { - if (dev->bdev_file) - fput(dev->bdev_file); + dev->children = NULL; + dev->nr_children = 0; + } else if (dev->bdev_file) { + fput(dev->bdev_file); + dev->bdev_file = NULL; } } @@ -434,6 +436,7 @@ bl_parse_scsi(struct nfs_server *server, struct pnfs_block_dev *d, out_blkdev_put: fput(d->bdev_file); + d->bdev_file = NULL; return error; } @@ -469,8 +472,11 @@ bl_parse_concat(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->concat.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->concat.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; d->children[i].start += len; @@ -498,8 +504,11 @@ bl_parse_stripe(struct nfs_server *server, struct pnfs_block_dev *d, for (i = 0; i < v->stripe.volumes_count; i++) { ret = bl_parse_deviceid(server, &d->children[i], volumes, v->stripe.volumes[i], gfp_mask); - if (ret) + if (ret) { + bl_free_device(&d->children[i]); + bl_free_device(d); return ret; + } d->nr_children++; len += d->children[i].len; diff --git a/fs/nfs/client.c b/fs/nfs/client.c index 404e75b0444d7..e3cd92b5aaa39 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -215,9 +215,21 @@ static void nfs_cb_idr_remove_locked(struct nfs_client *clp) { struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); - if (clp->cl_cb_ident) + if (clp->cl_cb_ident) { idr_remove(&nn->cb_ident_idr, clp->cl_cb_ident); + clp->cl_cb_ident = 0; + } +} + +void nfs_cb_idr_remove(struct nfs_client *clp) +{ + struct nfs_net *nn = net_generic(clp->cl_net, nfs_net_id); + + spin_lock(&nn->nfs_client_lock); + nfs_cb_idr_remove_locked(clp); + spin_unlock(&nn->nfs_client_lock); } +EXPORT_SYMBOL_GPL(nfs_cb_idr_remove); static void pnfs_init_server(struct nfs_server *server) { diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index 31ca8bbfb9702..38b0834e3ebf1 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -2275,6 +2275,13 @@ int nfs_atomic_open_v23(struct inode *dir, struct dentry *dentry, if (open_flags & O_CREAT) { error = nfs_do_create(dir, dentry, mode, open_flags); if (!error) { + /* With UNCHECKED mode, a server may return NFS3_OK for + * a pre-existing non-regular file (e.g. a symlink). + * Let the VFS handle it; calling finish_open() would + * hit no_open() and return -ENXIO. + */ + if (!d_is_reg(dentry)) + return finish_no_open(file, NULL); file->f_mode |= FMODE_CREATED; return finish_open(file, dentry, NULL); } else if (error != -EEXIST || open_flags & O_EXCL) diff --git a/fs/nfs/filelayout/filelayoutdev.c b/fs/nfs/filelayout/filelayoutdev.c index df79aeb68db42..e13d880de581f 100644 --- a/fs/nfs/filelayout/filelayoutdev.c +++ b/fs/nfs/filelayout/filelayoutdev.c @@ -170,7 +170,8 @@ nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, goto out_err_free_deviceid; } - dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + dsaddr->ds_list[i] = nfs4_pnfs_ds_add(net, &dsaddrs, 4, + gfp_flags); if (!dsaddr->ds_list[i]) goto out_err_drain_dsaddrs; trace_fl_getdevinfo(server, &pdev->dev_id, dsaddr->ds_list[i]->ds_remotestr); diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 4604f10882722..8099bfc4fe2f8 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -1324,7 +1324,8 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task, struct pnfs_layout_hdr *lo = lseg->pls_layout; struct inode *inode = lo->plh_inode; struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, dss_id); - struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table; + struct nfs4_slot_table *tbl = nfs4_has_session(clp) ? + &clp->cl_session->fc_slot_table : clp->cl_slot_tbl; switch (op_status) { case NFS4_OK: diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index c2d8a13a9dbdd..cb88c51346d99 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -149,7 +149,8 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev, new_ds->ds_versions = ds_versions; new_ds->ds_versions_cnt = version_count; - new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, gfp_flags); + new_ds->ds = nfs4_pnfs_ds_add(net, &dsaddrs, ds_versions[0].version, + gfp_flags); if (!new_ds->ds) goto out_err_drain_dsaddrs; diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index 8c2bfcc323e02..970e67768d67e 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -1546,9 +1546,7 @@ int nfs_clear_invalid_mapping(struct address_space *mapping) ret = nfs_invalidate_mapping(inode, mapping); trace_nfs_invalidate_mapping_exit(inode, ret); - clear_bit_unlock(NFS_INO_INVALIDATING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_INVALIDATING); + clear_and_wake_up_bit(NFS_INO_INVALIDATING, bitlock); out: return ret; } diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index ea239a7e5ef68..8d0d238b7f881 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -225,6 +225,7 @@ void nfs_server_copy_userdata(struct nfs_server *, struct nfs_server *); extern void nfs_put_client(struct nfs_client *); extern void nfs_free_client(struct nfs_client *); +void nfs_cb_idr_remove(struct nfs_client *clp); extern struct nfs_client *nfs4_find_client_ident(struct net *, int); extern struct nfs_client * nfs4_find_client_sessionid(struct net *, const struct sockaddr *, diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c index c08520828708b..1c66e1bb19652 100644 --- a/fs/nfs/nfs42proc.c +++ b/fs/nfs/nfs42proc.c @@ -81,12 +81,17 @@ static int _nfs42_proc_fallocate(struct rpc_message *msg, struct file *filep, status = nfs4_call_sync(server->client, server, msg, &args.seq_args, &res.seq_res, 0); if (status == 0) { - if (nfs_should_remove_suid(inode)) { - spin_lock(&inode->i_lock); + loff_t newsize = offset + len; + + spin_lock(&inode->i_lock); + if (newsize > i_size_read(inode)) + i_size_write(inode, newsize); + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BLOCKS); + if (nfs_should_remove_suid(inode)) nfs_set_cache_invalid(inode, - NFS_INO_REVAL_FORCED | NFS_INO_INVALID_MODE); - spin_unlock(&inode->i_lock); - } + NFS_INO_REVAL_FORCED | + NFS_INO_INVALID_MODE); + spin_unlock(&inode->i_lock); status = nfs_post_op_update_inode_force_wcc(inode, res.falloc_fattr); } diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 495cace1e5646..c0880a966b334 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -272,6 +272,7 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) return clp; error: + nfs_cb_idr_remove(clp); nfs_free_client(clp); return ERR_PTR(err); } @@ -1095,20 +1096,22 @@ static int nfs4_server_common_setup(struct nfs_server *server, return error; /* data servers support only a subset of NFSv4.1 */ - if (is_ds_only_client(server->nfs_client)) - return -EPROTONOSUPPORT; + if (is_ds_only_client(server->nfs_client)) { + error = -EPROTONOSUPPORT; + goto out_free_delegation_hash; + } /* We must ensure the session is initialised first */ error = nfs4_init_session(server->nfs_client); if (error < 0) - return error; + goto out_free_delegation_hash; nfs_server_set_init_caps(server); /* Probe the root fh to retrieve its FSID and filehandle */ error = nfs4_get_rootfh(server, mntfh, auth_probe); if (error < 0) - return error; + goto out_free_delegation_hash; dprintk("Server FSID: %llx:%llx\n", (unsigned long long) server->fsid.major, @@ -1117,7 +1120,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, error = nfs_probe_server(server, mntfh); if (error < 0) - return error; + goto out_free_delegation_hash; nfs4_session_limit_rwsize(server); nfs4_session_limit_xasize(server); @@ -1129,6 +1132,11 @@ static int nfs4_server_common_setup(struct nfs_server *server, server->mount_time = jiffies; server->destroy = nfs4_destroy_server; return 0; + +out_free_delegation_hash: + kfree(server->delegation_hash_table); + server->delegation_hash_table = NULL; + return error; } /* diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 403f7f8dfea2b..84ba899bf0a74 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7862,6 +7862,7 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, { struct inode *inode = file_inode(file); fmode_t type = arg == F_RDLCK ? FMODE_READ : FMODE_WRITE; + fl_owner_t owner = (*lease)->c.flc_owner; int ret; /* No delegation, no lease */ @@ -7871,7 +7872,8 @@ static int nfs4_add_lease(struct file *file, int arg, struct file_lease **lease, if (ret || nfs4_have_delegation(inode, type, 0)) return ret; /* We raced with a delegation return */ - nfs4_delete_lease(file, priv); + dprintk("%s: raced with a delegation return\n", __func__); + nfs4_delete_lease(file, &owner); return -EAGAIN; } @@ -10634,6 +10636,7 @@ static void nfs41_free_stateid_release(void *calldata) struct nfs_free_stateid_data *data = calldata; struct nfs_client *clp = data->server->nfs_client; + nfs_sb_deactive(data->server->super); nfs_put_client(clp); kfree(calldata); } @@ -10675,6 +10678,10 @@ static int nfs41_free_stateid(struct nfs_server *server, if (!refcount_inc_not_zero(&clp->cl_count)) return -EIO; + if (!nfs_sb_active(server->super)) { + nfs_put_client(clp); + return -EIO; + } nfs4_state_protect(server->nfs_client, NFS_SP4_MACH_CRED_STATEID, &task_setup.rpc_client, &msg); diff --git a/fs/nfs/nfs4session.c b/fs/nfs/nfs4session.c index 5db460476bf2a..c59e710eb7ccc 100644 --- a/fs/nfs/nfs4session.c +++ b/fs/nfs/nfs4session.c @@ -634,16 +634,22 @@ int nfs4_init_ds_session(struct nfs_client *clp, unsigned long lease_time) int ret; spin_lock(&clp->cl_lock); - if (test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { - /* - * Do not set NFS_CS_CHECK_LEASE_TIME instead set the - * DS lease to be equal to the MDS lease. - */ + /* + * Do not set NFS_CS_CHECK_LEASE_TIME instead set the + * DS lease to be equal to the MDS lease. + * + * A v4.0 DS has no session, so seed the lease every time. + */ + if (!session || + test_and_clear_bit(NFS4_SESSION_INITING, &session->session_state)) { clp->cl_lease_time = lease_time; clp->cl_last_renewal = jiffies; } spin_unlock(&clp->cl_lock); + if (!session) + return 0; + ret = nfs41_check_session_ready(clp); if (ret) return ret; diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index f84086437f531..cf0dcf9e35d75 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -1389,7 +1389,7 @@ pnfs_layout_need_return(struct pnfs_layout_hdr *lo) return false; return pnfs_mark_layout_stateid_return(lo, &lo->plh_return_segs, lo->plh_return_iomode, - lo->plh_return_seq) != EBUSY; + lo->plh_return_seq) != -EBUSY; } static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo) @@ -2088,15 +2088,6 @@ static bool pnfs_is_first_layoutget(struct pnfs_layout_hdr *lo) return test_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); } -static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo) -{ - unsigned long *bitlock = &lo->plh_flags; - - clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET); -} - static void _add_to_server_list(struct pnfs_layout_hdr *lo, struct nfs_server *server) { @@ -2272,7 +2263,8 @@ pnfs_update_layout(struct inode *ino, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_INVALID_OPEN); nfs4_schedule_stateid_recovery(server, ctx->state); - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); pnfs_put_layout_hdr(lo); goto lookup_again; } @@ -2341,7 +2333,8 @@ pnfs_update_layout(struct inode *ino, if (!exception.retry) goto out_put_layout_hdr; if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY); pnfs_put_layout_hdr(lo); @@ -2353,7 +2346,7 @@ pnfs_update_layout(struct inode *ino, out_put_layout_hdr: if (first) - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg, PNFS_UPDATE_LAYOUT_EXIT); pnfs_put_layout_hdr(lo); @@ -2445,7 +2438,7 @@ static void _lgopen_prepare_attached(struct nfs4_opendata *data, lgp = pnfs_alloc_init_layoutget_args(ino, ctx, ¤t_stateid, &rng, nfs_io_gfp_mask()); if (!lgp) { - pnfs_clear_first_layoutget(lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, &lo->plh_flags); nfs_layoutget_end(lo); pnfs_put_layout_hdr(lo); return; @@ -2549,7 +2542,8 @@ void nfs4_lgopen_release(struct nfs4_layoutget *lgp) { if (lgp != NULL) { if (lgp->lo) { - pnfs_clear_first_layoutget(lgp->lo); + clear_and_wake_up_bit(NFS_LAYOUT_FIRST_LAYOUTGET, + &lgp->lo->plh_flags); nfs_layoutget_end(lgp->lo); } pnfs_layoutget_free(lgp); @@ -2631,6 +2625,7 @@ pnfs_layout_process(struct nfs4_layoutget *lgp) spin_unlock(&ino->i_lock); lseg->pls_layout = lo; NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg); + pnfs_free_lseg_list(&free_me); return ERR_PTR(-EAGAIN); } @@ -3261,15 +3256,6 @@ pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc) } EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages); -static void pnfs_clear_layoutcommitting(struct inode *inode) -{ - unsigned long *bitlock = &NFS_I(inode)->flags; - - clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock); - smp_mb__after_atomic(); - wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING); -} - /* * There can be multiple RW segments. */ @@ -3294,7 +3280,7 @@ static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *lis pnfs_put_lseg(lseg); } - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); } void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg) @@ -3434,7 +3420,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) spin_unlock(&inode->i_lock); kfree(data); clear_layoutcommitting: - pnfs_clear_layoutcommitting(inode); + clear_and_wake_up_bit(NFS_INO_LAYOUTCOMMITTING, &NFS_I(inode)->flags); goto out; } EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode); diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h index 3db8f13d8fe4e..7b3ba2e64e6f3 100644 --- a/fs/nfs/pnfs.h +++ b/fs/nfs/pnfs.h @@ -63,6 +63,7 @@ struct nfs4_pnfs_ds { const struct net *ds_net; struct nfs_client *ds_clp; refcount_t ds_count; + u32 ds_version; /* cache key, with ds_addrs */ unsigned long ds_state; #define NFS4DS_CONNECTING 0 /* ds is establishing connection */ }; @@ -417,7 +418,7 @@ void pnfs_generic_write_commit_done(struct rpc_task *task, void *data); void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds); struct nfs4_pnfs_ds *nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, - gfp_t gfp_flags); + u32 version, gfp_t gfp_flags); void nfs4_pnfs_v3_ds_connect_unload(void); int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, struct nfs4_deviceid_node *devid, unsigned int timeo, diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index 1e291fd23d46f..5e85ea0ef5846 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -603,15 +603,17 @@ _same_data_server_addrs_locked(const struct list_head *dsaddrs1, } /* - * Lookup DS by addresses. nfs4_ds_cache_lock is held + * Lookup DS by addresses and NFS version. nfs4_ds_cache_lock is held */ static struct nfs4_pnfs_ds * -_data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs) +_data_server_lookup_locked(const struct nfs_net *nn, + const struct list_head *dsaddrs, u32 version) { struct nfs4_pnfs_ds *ds; list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node) - if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) + if (ds->ds_version == version && + _same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs)) return ds; return NULL; } @@ -719,7 +721,8 @@ nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags) * uncached and return cached struct nfs4_pnfs_ds. */ struct nfs4_pnfs_ds * -nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags) +nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, u32 version, + gfp_t gfp_flags) { struct nfs_net *nn = net_generic(net, nfs_net_id); struct nfs4_pnfs_ds *tmp_ds, *ds = NULL; @@ -738,7 +741,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags); spin_lock(&nn->nfs4_data_server_lock); - tmp_ds = _data_server_lookup_locked(nn, dsaddrs); + tmp_ds = _data_server_lookup_locked(nn, dsaddrs, version); if (tmp_ds == NULL) { INIT_LIST_HEAD(&ds->ds_addrs); list_splice_init(dsaddrs, &ds->ds_addrs); @@ -747,6 +750,7 @@ nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_fla INIT_LIST_HEAD(&ds->ds_node); ds->ds_net = net; ds->ds_clp = NULL; + ds->ds_version = version; list_add(&ds->ds_node, &nn->nfs4_data_server_cache); dprintk("%s add new data server %s\n", __func__, ds->ds_remotestr); diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index dd715cdb6c043..85aa03a7b020c 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -292,8 +292,22 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, nfs_fh, pnf, fmode); if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) { - /* Delete the cached file when racing with nfs_uuid_put() */ + /* + * Delete the cached file when racing with nfs_uuid_put(). + * Since nfl->nfs_uuid was never published via + * rcu_assign_pointer(), nfs_close_local_fh() will early-return + * and cannot clean up after us. Drop the slot's file ref and + * its paired net ref, then drop the caller-owned nfsd_file ref + * (+1) and the entry-time nfsd_net ref carried via nf->nf_net, + * and return -ENXIO so the caller never dereferences the + * now-cleared localio. + */ + struct nfsd_file __rcu *tmp = + (struct nfsd_file __force __rcu *)localio; + nfs_to_nfsd_file_put_local(pnf); + nfs_to_nfsd_file_put_local(&tmp); + localio = ERR_PTR(-ENXIO); } nfs_to_nfsd_net_put(net); diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c index a238b6725008a..7c2d7fca4a9ef 100644 --- a/fs/nfsd/filecache.c +++ b/fs/nfsd/filecache.c @@ -55,6 +55,17 @@ /* We only care about NFSD_MAY_READ/WRITE for this cache */ #define NFSD_FILE_MAY_MASK (NFSD_MAY_READ|NFSD_MAY_WRITE|NFSD_MAY_LOCALIO) +/* If the shrinker runs between calls to list_lru_walk_node() in + * nfsd_file_gc(), the "remaining" count will be wrong. This could + * result in premature freeing of some files. This may not matter much + * but is easy to fix with this spinlock which temporarily disables + * the shrinker. + * + * It also serializes callers of nfsd_file_dispose_list_delayed() + * against per-net shutdown. + */ +static DEFINE_SPINLOCK(nfsd_gc_lock); + static DEFINE_PER_CPU(unsigned long, nfsd_file_cache_hits); static DEFINE_PER_CPU(unsigned long, nfsd_file_acquisitions); static DEFINE_PER_CPU(unsigned long, nfsd_file_allocations); @@ -62,11 +73,6 @@ static DEFINE_PER_CPU(unsigned long, nfsd_file_releases); static DEFINE_PER_CPU(unsigned long, nfsd_file_total_age); static DEFINE_PER_CPU(unsigned long, nfsd_file_evictions); -struct nfsd_fcache_disposal { - spinlock_t lock; - struct list_head freeme; -}; - static struct kmem_cache *nfsd_file_slab; static struct kmem_cache *nfsd_file_mark_slab; static struct list_lru nfsd_file_lru; @@ -327,8 +333,11 @@ static void nfsd_file_lru_add(struct nfsd_file *nf) refcount_inc(&nf->nf_ref); if (list_lru_add_obj(&nfsd_file_lru, &nf->nf_lru)) trace_nfsd_file_lru_add(nf); - else - WARN_ON(1); + else { + refcount_dec(&nf->nf_ref); + WARN_ON_ONCE(1); + return; + } nfsd_file_schedule_laundrette(); } @@ -419,25 +428,31 @@ nfsd_file_dispose_list(struct list_head *dispose) } /** - * nfsd_file_dispose_list_delayed - move list of dead files to net's freeme list + * nfsd_file_dispose_list_delayed - queue dead files for nfsd thread disposal * @dispose: list of nfsd_files to be disposed * - * Transfers each file to the "freeme" list for its nfsd_net, to eventually - * be disposed of by the per-net garbage collector. + * Transfers each file to the dispose list in its nfsd_net and wakes an nfsd + * thread to do the actual close. This keeps the cost of fput() in the nfsd + * threads rather than in the shrinker or GC worker. + * + * All callers must hold nfsd_gc_lock, so that nfsd_file_cache_shutdown_net() + * can synchronize against them before draining the per-net dispose list. + * This guarantees nf_net is still live when we call net_generic(). */ static void nfsd_file_dispose_list_delayed(struct list_head *dispose) { - while(!list_empty(dispose)) { + lockdep_assert_held(&nfsd_gc_lock); + + while (!list_empty(dispose)) { struct nfsd_file *nf = list_first_entry(dispose, struct nfsd_file, nf_gc); struct nfsd_net *nn = net_generic(nf->nf_net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; struct svc_serv *serv; - spin_lock(&l->lock); - list_move_tail(&nf->nf_gc, &l->freeme); - spin_unlock(&l->lock); + spin_lock(&nn->fcache_dispose_lock); + list_move_tail(&nf->nf_gc, &nn->fcache_dispose_list); + spin_unlock(&nn->fcache_dispose_lock); /* * The filecache laundrette is shut down after the @@ -461,21 +476,28 @@ nfsd_file_dispose_list_delayed(struct list_head *dispose) */ void nfsd_file_net_dispose(struct nfsd_net *nn) { - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - if (!list_empty(&l->freeme)) { + if (!list_empty(&nn->fcache_dispose_list)) { LIST_HEAD(dispose); int i; - spin_lock(&l->lock); - for (i = 0; i < 8 && !list_empty(&l->freeme); i++) - list_move(l->freeme.next, &dispose); - spin_unlock(&l->lock); - if (!list_empty(&l->freeme)) - /* Wake up another thread to share the work + spin_lock(&nn->fcache_dispose_lock); + for (i = 0; i < 8 && !list_empty(&nn->fcache_dispose_list); i++) + list_move(nn->fcache_dispose_list.next, &dispose); + spin_unlock(&nn->fcache_dispose_lock); + if (!list_empty(&nn->fcache_dispose_list)) { + /* + * Wake up another thread to share the work * *before* doing any actual disposing. + * + * The filecache laundrette is shut down after + * the nn->nfsd_serv pointer is cleared, but + * before the svc_serv is freed. */ - svc_wake_up(nn->nfsd_serv); + struct svc_serv *serv = nn->nfsd_serv; + + if (serv) + svc_wake_up(serv); + } nfsd_file_dispose_list(&dispose); } } @@ -552,13 +574,6 @@ nfsd_file_gc_cb(struct list_head *item, struct list_lru_one *lru, return nfsd_file_lru_cb(item, lru, arg); } -/* If the shrinker runs between calls to list_lru_walk_node() in - * nfsd_file_gc(), the "remaining" count will be wrong. This could - * result in premature freeing of some files. This may not matter much - * but is easy to fix with this spinlock which temporarily disables - * the shrinker. - */ -static DEFINE_SPINLOCK(nfsd_gc_lock); static void nfsd_file_gc(void) { @@ -581,9 +596,9 @@ nfsd_file_gc(void) remaining = 0; } } + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_gc_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); } static void @@ -611,9 +626,9 @@ nfsd_file_lru_scan(struct shrinker *s, struct shrink_control *sc) ret = list_lru_shrink_walk(&nfsd_file_lru, sc, nfsd_file_lru_cb, &dispose); + nfsd_file_dispose_list_delayed(&dispose); spin_unlock(&nfsd_gc_lock); trace_nfsd_file_shrinker_removed(ret, list_lru_count(&nfsd_file_lru)); - nfsd_file_dispose_list_delayed(&dispose); return ret; } @@ -686,11 +701,11 @@ nfsd_file_queue_for_close(struct inode *inode, struct list_head *dispose) } /** - * nfsd_file_close_inode - attempt a delayed close of a nfsd_file + * nfsd_file_close_inode - attempt a deferred close of a nfsd_file * @inode: inode of the file to attempt to remove * * Close out any open nfsd_files that can be reaped for @inode. The - * actual freeing is deferred to the dispose_list_delayed infrastructure. + * actual freeing is deferred to the nfsd service threads. * * This is used by the fsnotify callbacks and setlease notifier. */ @@ -699,8 +714,10 @@ nfsd_file_close_inode(struct inode *inode) { LIST_HEAD(dispose); + spin_lock(&nfsd_gc_lock); nfsd_file_queue_for_close(inode, &dispose); nfsd_file_dispose_list_delayed(&dispose); + spin_unlock(&nfsd_gc_lock); } /** @@ -894,42 +911,14 @@ __nfsd_file_cache_purge(struct net *net) nfsd_file_dispose_list(&dispose); } -static struct nfsd_fcache_disposal * -nfsd_alloc_fcache_disposal(void) -{ - struct nfsd_fcache_disposal *l; - - l = kmalloc(sizeof(*l), GFP_KERNEL); - if (!l) - return NULL; - spin_lock_init(&l->lock); - INIT_LIST_HEAD(&l->freeme); - return l; -} - -static void -nfsd_free_fcache_disposal(struct nfsd_fcache_disposal *l) -{ - nfsd_file_dispose_list(&l->freeme); - kfree(l); -} - -static void -nfsd_free_fcache_disposal_net(struct net *net) -{ - struct nfsd_net *nn = net_generic(net, nfsd_net_id); - struct nfsd_fcache_disposal *l = nn->fcache_disposal; - - nfsd_free_fcache_disposal(l); -} - int nfsd_file_cache_start_net(struct net *net) { struct nfsd_net *nn = net_generic(net, nfsd_net_id); - nn->fcache_disposal = nfsd_alloc_fcache_disposal(); - return nn->fcache_disposal ? 0 : -ENOMEM; + spin_lock_init(&nn->fcache_dispose_lock); + INIT_LIST_HEAD(&nn->fcache_dispose_list); + return 0; } /** @@ -948,8 +937,18 @@ nfsd_file_cache_purge(struct net *net) void nfsd_file_cache_shutdown_net(struct net *net) { + struct nfsd_net *nn = net_generic(net, nfsd_net_id); + nfsd_file_cache_purge(net); - nfsd_free_fcache_disposal_net(net); + /* + * Ensure any in-progress shrinker, GC, or fsnotify/lease callback + * (all of which hold nfsd_gc_lock while calling + * nfsd_file_dispose_list_delayed()) has fully completed before + * draining the per-net dispose list. + */ + spin_lock(&nfsd_gc_lock); + spin_unlock(&nfsd_gc_lock); + nfsd_file_dispose_list(&nn->fcache_dispose_list); } void diff --git a/fs/nfsd/flexfilelayoutxdr.c b/fs/nfsd/flexfilelayoutxdr.c index f9f7e38cba13f..374e52d3064a6 100644 --- a/fs/nfsd/flexfilelayoutxdr.c +++ b/fs/nfsd/flexfilelayoutxdr.c @@ -30,19 +30,24 @@ nfsd4_ff_encode_layoutget(struct xdr_stream *xdr, struct ff_idmap uid; struct ff_idmap gid; - fh_len = 4 + fl->fh.size; + fh_len = 4 + xdr_align_size(fl->fh.size); uid.len = sprintf(uid.buf, "%u", from_kuid(&init_user_ns, fl->uid)); gid.len = sprintf(gid.buf, "%u", from_kgid(&init_user_ns, fl->gid)); - /* 8 + len for recording the length, name, and padding */ - ds_len = 20 + sizeof(stateid_opaque_t) + 4 + fh_len + - 8 + uid.len + 8 + gid.len; + /* data server entry: deviceid + efficiency + stateid + fh list + + * user + group + flags + stats_collect_hint + */ + ds_len = 16 + 4 + 4 + sizeof(stateid_opaque_t) + 4 + fh_len + + 4 + xdr_align_size(uid.len) + + 4 + xdr_align_size(gid.len) + + 4 + 4; + /* mirror: ds_count + ds */ mirror_len = 4 + ds_len; - /* The layout segment */ - len = 20 + mirror_len; + /* stripe_unit + mirror_count + mirror */ + len = 12 + mirror_len; p = xdr_reserve_space(xdr, sizeof(__be32) + len); if (!p) @@ -94,7 +99,8 @@ nfsd4_ff_encode_getdeviceinfo(struct xdr_stream *xdr, } /* len + padding for two strings */ - addr_len = 16 + da->netaddr.netid_len + da->netaddr.addr_len; + addr_len = 8 + xdr_align_size(da->netaddr.netid_len) + + xdr_align_size(da->netaddr.addr_len); ver_len = 20; len = 4 + ver_len + 4 + addr_len; diff --git a/fs/nfsd/localio.c b/fs/nfsd/localio.c index be710d809a3ba..c3eb0557b3e1b 100644 --- a/fs/nfsd/localio.c +++ b/fs/nfsd/localio.c @@ -97,11 +97,15 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, } nfsd_file_get(localio); again: + rcu_read_lock(); new = unrcu_pointer(cmpxchg(pnf, NULL, RCU_INITIALIZER(localio))); if (new) { /* Some other thread installed an nfsd_file */ - if (nfsd_file_get(new) == NULL) + if (nfsd_file_get(new) == NULL) { + rcu_read_unlock(); goto again; + } + rcu_read_unlock(); /* * Drop the ref we were going to install (both file and * net) and the one we were going to return (only file). @@ -110,6 +114,8 @@ nfsd_open_local_fh(struct net *net, struct auth_domain *dom, nfsd_net_put(net); nfsd_file_put(localio); localio = new; + } else { + rcu_read_unlock(); } } else nfsd_net_put(net); diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index fe8338735e7cc..f3b440ae98fed 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -69,6 +69,7 @@ struct nfsd_net { bool grace_end_forced; bool client_tracking_active; time64_t boot_time; + time64_t boot_time_bt; /* same instant in CLOCK_BOOTTIME */ struct dentry *nfsd_client_dir; @@ -100,6 +101,10 @@ struct nfsd_net { */ struct list_head client_lru; struct list_head close_lru; + + /* protects del_recall_lru and delegation hash/unhash; + * nests outside client_lock */ + spinlock_t deleg_lock ____cacheline_aligned; struct list_head del_recall_lru; /* protected by blocked_locks_lock */ @@ -107,7 +112,8 @@ struct nfsd_net { struct delayed_work laundromat_work; - /* client_lock protects the client lru list and session hash table */ + /* client_lock protects the client lru list and session hash + * table; nests inside deleg_lock */ spinlock_t client_lock; /* protects blocked_locks_lru */ @@ -200,7 +206,8 @@ struct nfsd_net { /* utsname taken from the process that starts the server */ char nfsd_name[UNX_MAXNODENAME+1]; - struct nfsd_fcache_disposal *fcache_disposal; + spinlock_t fcache_dispose_lock; + struct list_head fcache_dispose_list; siphash_key_t siphash_key; diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c index 76305b86c1a95..2998640f259de 100644 --- a/fs/nfsd/nfs2acl.c +++ b/fs/nfsd/nfs2acl.c @@ -115,14 +115,19 @@ static __be32 nfsacld_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); - if (error) - goto out_drop_lock; + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + if (error) + goto out_drop_lock; + } inode_unlock(inode); @@ -248,22 +253,21 @@ nfsaclsvc_encode_getaclres(struct svc_rqst *rqstp, struct xdr_stream *xdr) if (!svcxdr_encode_stat(xdr, resp->status)) return false; - - if (dentry == NULL || d_really_is_negative(dentry)) - return true; - inode = d_inode(dentry); - - if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) - return false; - if (xdr_stream_encode_u32(xdr, resp->mask) < 0) - return false; - - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, - resp->mask & NFS_ACL, 0)) - return false; - if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, - resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) - return false; + switch (resp->status) { + case nfs_ok: + inode = d_inode(dentry); + if (!svcxdr_encode_fattr(rqstp, xdr, &resp->fh, &resp->stat)) + return false; + if (xdr_stream_encode_u32(xdr, resp->mask) < 0) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_access, + resp->mask & NFS_ACL, 0)) + return false; + if (!nfs_stream_encode_acl(xdr, inode, resp->acl_default, + resp->mask & NFS_DFACL, NFS_ACL_DEFAULT)) + return false; + break; + } return true; } diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c index e87731380be85..a87f9d7f32be2 100644 --- a/fs/nfsd/nfs3acl.c +++ b/fs/nfsd/nfs3acl.c @@ -105,12 +105,17 @@ static __be32 nfsd3_proc_setacl(struct svc_rqst *rqstp) inode_lock(inode); - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_ACCESS, - argp->acl_access); - if (error) - goto out_drop_lock; - error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, ACL_TYPE_DEFAULT, - argp->acl_default); + error = 0; + if (argp->mask & NFS_ACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_ACCESS, argp->acl_access); + if (error) + goto out_drop_lock; + } + if (argp->mask & NFS_DFACL) { + error = set_posix_acl(&nop_mnt_idmap, fh->fh_dentry, + ACL_TYPE_DEFAULT, argp->acl_default); + } out_drop_lock: inode_unlock(inode); diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c index b6d03e1ef5f7a..fbf573c317193 100644 --- a/fs/nfsd/nfs3proc.c +++ b/fs/nfsd/nfs3proc.c @@ -29,6 +29,25 @@ static int nfs3_ftypes[] = { S_IFIFO, /* NF3FIFO */ }; +/* + * Reject a client-supplied atime or mtime whose nanoseconds field is out + * of range. Such a value is well-formed on the wire but is not a valid + * timespec64, and storing it verbatim can corrupt on-disk timestamps. + * tv_nsec is a long, so it is cast to unsigned long (the same width) to + * catch both an over-large value and one that became negative when an + * out-of-range u32 wire nseconds was assigned to a 32-bit long. + */ +static bool nfsd3_time_in_range(const struct iattr *iap) +{ + if ((iap->ia_valid & ATTR_ATIME_SET) && + (unsigned long)iap->ia_atime.tv_nsec >= NSEC_PER_SEC) + return false; + if ((iap->ia_valid & ATTR_MTIME_SET) && + (unsigned long)iap->ia_mtime.tv_nsec >= NSEC_PER_SEC) + return false; + return true; +} + static __be32 nfsd3_map_status(__be32 status) { switch (status) { @@ -101,9 +120,14 @@ nfsd3_proc_setattr(struct svc_rqst *rqstp) SVCFH_fmt(&argp->fh)); fh_copy(&resp->fh, &argp->fh); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->check_guard) guardtime = &argp->guardtime; resp->status = nfsd_setattr(rqstp, &resp->fh, &attrs, guardtime); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -265,6 +289,8 @@ nfsd3_create_file(struct svc_rqst *rqstp, struct svc_fh *fhp, trace_nfsd_vfs_create(rqstp, fhp, S_IFREG, argp->name, argp->len); + if (!nfsd3_time_in_range(iap)) + return nfserr_inval; if (isdotent(argp->name, argp->len)) return nfserr_exist; if (!(iap->ia_valid & ATTR_MODE)) @@ -404,8 +430,13 @@ nfsd3_proc_mkdir(struct svc_rqst *rqstp) argp->attrs.ia_valid &= ~ATTR_SIZE; fh_copy(&resp->dirfh, &argp->fh); fh_init(&resp->fh, NFS3_FHSIZE); + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, S_IFDIR, 0, &resp->fh); +out: resp->status = nfsd3_map_status(resp->status); return rpc_success; } @@ -419,6 +450,10 @@ nfsd3_proc_symlink(struct svc_rqst *rqstp) .na_iattr = &argp->attrs, }; + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } if (argp->tlen == 0) { resp->status = nfserr_inval; goto out; @@ -475,6 +510,11 @@ nfsd3_proc_mknod(struct svc_rqst *rqstp) goto out; } + if (!nfsd3_time_in_range(&argp->attrs)) { + resp->status = nfserr_inval; + goto out; + } + type = nfs3_ftypes[argp->ftype]; resp->status = nfsd_create(rqstp, &resp->dirfh, argp->name, argp->len, &attrs, type, rdev, &resp->fh); diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index e00b2aea8da2b..441a55c251481 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -108,6 +108,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_access(xdr, &access)) return -EIO; + if (access.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_atime.tv_sec = access.seconds; fattr->ncf_cb_atime.tv_nsec = access.nseconds; @@ -117,6 +119,8 @@ static int decode_cb_fattr4(struct xdr_stream *xdr, uint32_t *bitmap, if (!xdrgen_decode_fattr4_time_deleg_modify(xdr, &modify)) return -EIO; + if (modify.nseconds >= NSEC_PER_SEC) + return -EIO; fattr->ncf_cb_mtime.tv_sec = modify.seconds; fattr->ncf_cb_mtime.tv_nsec = modify.nseconds; @@ -456,13 +460,20 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, const struct nfsd4_callback *cb, struct nfs4_cb_compound_hdr *hdr) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; struct nfsd4_referring_call_list *rcl; __be32 *p; if (hdr->minorversion == 0) return; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + return; + } + encode_nfs_cb_opnum4(xdr, OP_CB_SEQUENCE); encode_sessionid4(xdr, session); @@ -478,6 +489,7 @@ static void encode_cb_sequence4args(struct xdr_stream *xdr, encode_referring_call_list4(xdr, rcl); hdr->nops++; + rcu_read_unlock(); } static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) @@ -529,21 +541,32 @@ static void update_cb_slot_table(struct nfsd4_session *ses, u32 target) static int decode_cb_sequence4resok(struct xdr_stream *xdr, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; int status = -ESERVERFAULT; __be32 *p; u32 seqid, slotid, target; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + cb->cb_seq_status = -NFS4ERR_BADSESSION; + return -NFS4ERR_BADSESSION; + } + /* * If the server returns different values for sessionID, slotID or * sequence number, the server is looney tunes. */ p = xdr_inline_decode(xdr, NFS4_MAX_SESSIONID_LEN + 4 + 4 + 4 + 4); - if (unlikely(p == NULL)) + if (unlikely(p == NULL)) { + rcu_read_unlock(); goto out_overflow; + } if (memcmp(p, session->se_sessionid.data, NFS4_MAX_SESSIONID_LEN)) { dprintk("NFS: %s Invalid session id\n", __func__); + rcu_read_unlock(); goto out; } p += XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN); @@ -551,12 +574,14 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, seqid = be32_to_cpup(p++); if (seqid != session->se_cb_seq_nr[cb->cb_held_slot]) { dprintk("NFS: %s Invalid sequence number\n", __func__); + rcu_read_unlock(); goto out; } slotid = be32_to_cpup(p++); if (slotid != cb->cb_held_slot) { dprintk("NFS: %s Invalid slotid\n", __func__); + rcu_read_unlock(); goto out; } @@ -564,6 +589,7 @@ static int decode_cb_sequence4resok(struct xdr_stream *xdr, target = be32_to_cpup(p++); update_cb_slot_table(session, target); + rcu_read_unlock(); status = 0; out: cb->cb_seq_status = status; @@ -1174,9 +1200,8 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c } else { if (!conn->cb_xprt || !ses) return -EINVAL; - clp->cl_cb_session = ses; args.bc_xprt = conn->cb_xprt; - args.prognumber = clp->cl_cb_session->se_cb_prog; + args.prognumber = ses->se_cb_prog; args.protocol = conn->cb_xprt->xpt_class->xcl_ident | XPRT_TRANSPORT_BC; args.authflavor = ses->se_cb_sec.flavor; @@ -1194,8 +1219,10 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c return -ENOMEM; } - if (clp->cl_minorversion != 0) + if (clp->cl_minorversion != 0) { clp->cl_cb_conn.cb_xprt = conn->cb_xprt; + rcu_assign_pointer(clp->cl_cb_session, ses); + } clp->cl_cb_client = client; clp->cl_cb_cred = cred; rcu_read_lock(); @@ -1302,18 +1329,33 @@ static int grab_slot(struct nfsd4_session *ses) static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) return true; + + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (!ses) { + rcu_read_unlock(); + rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); + return false; + } cb->cb_held_slot = grab_slot(ses); if (cb->cb_held_slot < 0) { + rcu_read_unlock(); rpc_sleep_on(&clp->cl_cb_waitq, task, NULL); /* Race breaker */ - cb->cb_held_slot = grab_slot(ses); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) + cb->cb_held_slot = grab_slot(ses); + rcu_read_unlock(); if (cb->cb_held_slot < 0) return false; rpc_wake_up_queued_task(&clp->cl_cb_waitq, task); + } else { + rcu_read_unlock(); } return true; } @@ -1321,12 +1363,17 @@ static bool nfsd41_cb_get_slot(struct nfsd4_callback *cb, struct rpc_task *task) static void nfsd41_cb_release_slot(struct nfsd4_callback *cb) { struct nfs4_client *clp = cb->cb_clp; - struct nfsd4_session *ses = clp->cl_cb_session; + struct nfsd4_session *ses; if (cb->cb_held_slot >= 0) { - spin_lock(&ses->se_lock); - ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); - spin_unlock(&ses->se_lock); + rcu_read_lock(); + ses = rcu_dereference(clp->cl_cb_session); + if (ses) { + spin_lock(&ses->se_lock); + ses->se_cb_slot_avail |= BIT(cb->cb_held_slot); + spin_unlock(&ses->se_lock); + } + rcu_read_unlock(); cb->cb_held_slot = -1; rpc_wake_up_next(&clp->cl_cb_waitq); } @@ -1458,22 +1505,35 @@ static void nfsd4_cb_prepare(struct rpc_task *task, void *calldata) trace_nfsd_cb_rpc_prepare(clp); cb->cb_seq_status = 1; cb->cb_status = 0; - if (minorversion && !nfsd41_cb_get_slot(cb, task)) - return; + if (minorversion) { + if (!rcu_access_pointer(clp->cl_cb_session)) { + rpc_exit(task, -EIO); + return; + } + if (!nfsd41_cb_get_slot(cb, task)) + return; + } rpc_call_start(task); } /* Returns true if CB_COMPOUND processing should continue */ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback *cb) { - struct nfsd4_session *session = cb->cb_clp->cl_cb_session; + struct nfsd4_session *session; bool ret = false; if (cb->cb_held_slot < 0) goto requeue; + rcu_read_lock(); + session = rcu_dereference(cb->cb_clp->cl_cb_session); + if (!session) { + rcu_read_unlock(); + goto requeue; + } + /* This is the operation status code for CB_SEQUENCE */ - trace_nfsd_cb_seq_status(task, cb); + trace_nfsd_cb_seq_status(task, cb, session); switch (cb->cb_seq_status) { case 0: /* @@ -1505,12 +1565,16 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback fallthrough; case -NFS4ERR_BADSESSION: nfsd4_mark_cb_fault(cb->cb_clp); + rcu_read_unlock(); goto requeue; case -NFS4ERR_DELAY: cb->cb_seq_status = 1; - if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) + if (RPC_SIGNALLED(task) || !rpc_restart_call(task)) { + rcu_read_unlock(); goto requeue; + } rpc_delay(task, 2 * HZ); + rcu_read_unlock(); return false; case -NFS4ERR_SEQ_MISORDERED: case -NFS4ERR_BADSLOT: @@ -1522,11 +1586,13 @@ static bool nfsd4_cb_sequence_done(struct rpc_task *task, struct nfsd4_callback */ nfsd4_mark_cb_fault(cb->cb_clp); cb->cb_held_slot = -1; + rcu_read_unlock(); goto retry_nowait; default: nfsd4_mark_cb_fault(cb->cb_clp); } - trace_nfsd_cb_free_slot(task, cb); + trace_nfsd_cb_free_slot(task, cb, session); + rcu_read_unlock(); nfsd41_cb_release_slot(cb); return ret; retry_nowait: @@ -1648,7 +1714,15 @@ static struct nfsd4_conn * __nfsd4_find_backchannel(struct nfs4_client *clp) * Note there isn't a lot of locking in this code; instead we depend on * the fact that it is run from clp->cl_callback_wq, which won't run two * work items at once. So, for example, clp->cl_callback_wq handles all - * access of cl_cb_client and all calls to rpc_create or rpc_shutdown_client. + * access of cl_cb_client, and all calls to rpc_create or + * rpc_shutdown_client. + * + * cl_cb_session is written only from cl_callback_wq (via + * rcu_assign_pointer) and read from rpciod under rcu_read_lock (via + * rcu_dereference) by encode_cb_sequence4args(), decode_cb_sequence4resok(), + * nfsd4_cb_sequence_done(), and the cb-slot helpers. Sessions are freed + * with kfree_rcu() so that rpciod readers in an RCU read-side critical + * section never dereference a freed session. */ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) { @@ -1700,6 +1774,7 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) nfsd4_mark_cb_down(clp); if (c) svc_xprt_put(c->cn_xprt); + rcu_assign_pointer(clp->cl_cb_session, ses); return; } } diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c index 62762e43f810b..2121ef26e7a98 100644 --- a/fs/nfsd/nfs4layouts.c +++ b/fs/nfsd/nfs4layouts.c @@ -249,11 +249,17 @@ nfsd4_alloc_layout_stateid(struct nfsd4_compound_state *cstate, nfsd4_init_cb(&ls->ls_recall, clp, &nfsd4_cb_layout_ops, NFSPROC4_CLNT_CB_LAYOUT); - if (parent->sc_type == SC_TYPE_DELEG) + if (parent->sc_type == SC_TYPE_DELEG) { + spin_lock(&fp->fi_lock); ls->ls_file = nfsd_file_get(fp->fi_deleg_file); - else + spin_unlock(&fp->fi_lock); + } else { ls->ls_file = find_any_file(fp); - BUG_ON(!ls->ls_file); + } + if (!ls->ls_file) { + nfs4_put_stid(stp); + return NULL; + } if (nfsd4_layout_setlease(ls)) { nfs4_put_stid(stp); diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 98086d217aa22..bc14c3939a5c4 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -800,7 +800,23 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) return status; + /* Sanitize cr_type to avoid returning ATTRNOTSUPP. */ + switch (create->cr_type) { + case NF4LNK: + case NF4BLK: + case NF4CHR: + case NF4SOCK: + case NF4FIFO: + case NF4DIR: + break; + default: + status = nfserr_badtype; + goto out_aftermask; + } + status = nfsd4_acl_to_attr(create->cr_type, create->cr_acl, &attrs); + if (status != nfs_ok) + goto out_aftermask; current->fs->umask = create->cr_umask; switch (create->cr_type) { case NF4LNK: @@ -867,6 +883,7 @@ nfsd4_create(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, fh_put(&resfh); out_umask: current->fs->umask = 0; +out_aftermask: nfsd_attrs_free(&attrs); return status; } @@ -1196,7 +1213,7 @@ nfsd4_setattr(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (deleg_attrs) { status = nfserr_bad_stateid; - if (st->sc_type & SC_TYPE_DELEG) { + if (st && (st->sc_type & SC_TYPE_DELEG)) { struct nfs4_delegation *dp = delegstateid(st); /* Only for *_ATTRS_DELEG flavors */ @@ -1428,18 +1445,30 @@ static void nfs4_put_copy(struct nfsd4_copy *copy) { if (!refcount_dec_and_test(©->refcount)) return; + /* Drop the task_struct pinned in nfsd4_copy(); NULL on sync copies. */ + if (copy->copy_task) + put_task_struct(copy->copy_task); kfree(copy->cp_src); kfree(copy); } +static void release_copy_files(struct nfsd4_copy *copy); + static void nfsd4_stop_copy(struct nfsd4_copy *copy) { trace_nfsd_copy_async_cancel(copy); - if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags)) { - kthread_stop(copy->copy_task); - copy->nfserr = nfs_ok; - set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); - } + /* + * Join the kthread before releasing its resources. The task_struct is + * pinned in nfsd4_copy(), so kthread_stop() is safe even after the + * one-shot kthread has exited. The caller already unlinked the copy, + * so this runs once per copy. + */ + set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); + kthread_stop(copy->copy_task); + copy->nfserr = nfs_ok; + set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); + + release_copy_files(copy); nfs4_put_copy(copy); } @@ -1452,7 +1481,13 @@ static struct nfsd4_copy *nfsd4_unhash_copy(struct nfs4_client *clp) copy = list_first_entry(&clp->async_copies, struct nfsd4_copy, copies); refcount_inc(©->refcount); - copy->cp_clp = NULL; + /* + * Unlinking hides the copy from the reaper, so drop its + * s2s_cp_stateids entry here while cp_clp is still valid. + */ + nfs4_free_copy_state(copy); + /* Pairs with smp_load_acquire() in nfsd4_send_cb_offload(). */ + smp_store_release(©->cp_clp, NULL); if (!list_empty(©->copies)) list_del_init(©->copies); } @@ -1464,8 +1499,11 @@ void nfsd4_shutdown_copy(struct nfs4_client *clp) { struct nfsd4_copy *copy; - while ((copy = nfsd4_unhash_copy(clp)) != NULL) + while ((copy = nfsd4_unhash_copy(clp)) != NULL) { nfsd4_stop_copy(copy); + /* Reaper can't reach the unhashed copy; drop its membership ref. */ + nfs4_put_copy(copy); + } } #ifdef CONFIG_NFSD_V4_2_INTER_SSC @@ -1756,6 +1794,8 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb) container_of(cbo, struct nfsd4_copy, cp_cb_offload); set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); + /* Drop the copy reference taken in nfsd4_send_cb_offload(). */ + nfs4_put_copy(copy); } static int nfsd4_cb_offload_done(struct nfsd4_callback *cb, @@ -1805,6 +1845,7 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, /* See RFC 7862 p.67: */ if (bytes_total == 0) bytes_total = ULLONG_MAX; + since = READ_ONCE(dst->f_wb_err); do { /* Only async copies can be stopped here */ if (kthread_should_stop()) @@ -1820,13 +1861,14 @@ static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy, } while (bytes_total > 0 && nfsd4_copy_is_async(copy)); /* for a non-zero asynchronous copy do a commit of data */ if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) { - since = READ_ONCE(dst->f_wb_err); end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1; status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0); if (!status) status = filemap_check_wb_err(dst->f_mapping, since); if (!status) set_bit(NFSD4_COPY_F_COMMITTED, ©->cp_flags); + else if (status != -EAGAIN && status != -ESTALE) + nfsd_reset_write_verifier(copy->cp_nn); } return bytes_copied; } @@ -1881,34 +1923,52 @@ static void release_copy_files(struct nfsd4_copy *copy) nfsd_file_put(copy->nf_dst); } +/* + * Called from the reaper and from nfsd4_copy()'s error path; in both + * cases the copy is already unreachable from clp->async_copies. + */ static void cleanup_async_copy(struct nfsd4_copy *copy) { nfs4_free_copy_state(copy); release_copy_files(copy); - if (copy->cp_clp) { - spin_lock(©->cp_clp->async_lock); - if (!list_empty(©->copies)) - list_del_init(©->copies); - spin_unlock(©->cp_clp->async_lock); - } nfs4_put_copy(copy); } static void nfsd4_send_cb_offload(struct nfsd4_copy *copy) { struct nfsd4_cb_offload *cbo = ©->cp_cb_offload; + struct nfs4_client *clp; + + /* + * Pairs with smp_store_release(&cp_clp) in find_async_copy() and + * nfsd4_unhash_copy(); the set_bit/clear_bit writers are unordered. + * cp_clp is NULL once the copy was canceled; skip the callback, the + * canceling path owns the notification. + */ + clp = smp_load_acquire(©->cp_clp); + if (!clp) { + set_bit(NFSD4_COPY_F_OFFLOAD_DONE, ©->cp_flags); + return; + } memcpy(&cbo->co_res, ©->cp_res, sizeof(copy->cp_res)); memcpy(&cbo->co_fh, ©->fh, sizeof(copy->fh)); cbo->co_nfserr = copy->nfserr; cbo->co_retries = 5; - nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops, + /* + * Hold the copy across the in-flight callback; co_cb is embedded in + * the copy, so it must outlive the callback. The reference is dropped + * in nfsd4_cb_offload_release(). + */ + refcount_inc(©->refcount); + + nfsd4_init_cb(&cbo->co_cb, clp, &nfsd4_cb_offload_ops, NFSPROC4_CLNT_CB_OFFLOAD); nfsd41_cb_referring_call(&cbo->co_cb, &cbo->co_referring_sessionid, cbo->co_referring_slotid, cbo->co_referring_seqno); - trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid, + trace_nfsd_cb_offload(clp, &cbo->co_res.cb_stateid, &cbo->co_fh, copy->cp_count, copy->nfserr); nfsd4_try_run_cb(&cbo->co_cb); } @@ -1950,16 +2010,20 @@ static int nfsd4_do_async_copy(void *data) } do_callback: - /* The kthread exits forthwith. Ensure that a subsequent - * OFFLOAD_CANCEL won't try to kill it again. */ - set_bit(NFSD4_COPY_F_STOPPED, ©->cp_flags); - + /* + * Don't set NFSD4_COPY_F_STOPPED here: it tells a teardown caller it + * may skip kthread_stop(), which would then release nf_dst and the + * client while still in use. Only nfsd4_stop_copy() sets it, after + * joining. + */ set_bit(NFSD4_COPY_F_COMPLETED, ©->cp_flags); trace_nfsd_copy_async_done(copy); atomic_dec(©->cp_nn->pending_async_copies); if (copy->cp_res.wr_bytes_written > 0 && copy->attr_update) nfsd_update_cmtime_attr(copy->nf_dst->nf_file, 0); nfsd4_send_cb_offload(copy); + /* Drop the kthread's reference (taken in nfsd4_copy()); copy may be freed after this. */ + nfs4_put_copy(copy); return 0; } @@ -1985,21 +2049,21 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, } status = nfsd4_setup_inter_ssc(rqstp, cstate, copy); if (status) { - trace_nfsd_copy_done(copy, status); - return nfserr_offload_denied; + status = nfserr_offload_denied; + goto out; } } else { trace_nfsd_copy_intra(copy); status = nfsd4_setup_intra_ssc(rqstp, cstate, copy); - if (status) { - trace_nfsd_copy_done(copy, status); - return status; - } + if (status) + goto out; } memcpy(©->fh, &cstate->current_fh.fh_handle, sizeof(struct knfsd_fh)); if (nfsd4_copy_is_async(copy)) { + struct task_struct *task; + async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL); if (!async_copy) goto out_err; @@ -2014,11 +2078,12 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, async_copy->cp_src = kmalloc(sizeof(*async_copy->cp_src), GFP_KERNEL); if (!async_copy->cp_src) goto out_dec_async_copy_err; - if (!nfs4_init_copy_state(nn, copy)) + dup_copy_fields(copy, async_copy); + + if (!nfs4_init_copy_state(nn, async_copy)) goto out_dec_async_copy_err; - memcpy(&result->cb_stateid, ©->cp_stateid.cs_stid, + memcpy(&result->cb_stateid, &async_copy->cp_stateid.cs_stid, sizeof(result->cb_stateid)); - dup_copy_fields(copy, async_copy); if ((READ_ONCE(copy->nf_dst->nf_file->f_mode) & FMODE_NOCMTIME) != 0) async_copy->attr_update = true; @@ -2027,15 +2092,27 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, NFS4_MAX_SESSIONID_LEN); async_copy->cp_cb_offload.co_referring_slotid = cstate->slot->sl_index; async_copy->cp_cb_offload.co_referring_seqno = cstate->slot->sl_seqid; - async_copy->copy_task = kthread_create(nfsd4_do_async_copy, - async_copy, "%s", "copy thread"); - if (IS_ERR(async_copy->copy_task)) + task = kthread_create(nfsd4_do_async_copy, async_copy, + "%s", "copy thread"); + if (IS_ERR(task)) goto out_dec_async_copy_err; + /* + * Pin the task_struct so kthread_stop() is safe after this + * one-shot kthread exits. Released by nfs4_put_copy(). + */ + get_task_struct(task); + async_copy->copy_task = task; + /* + * Take the kthread's ref and wake it before publishing, so the + * publisher touches async_copy no further and teardown can + * drain it. + */ + refcount_inc(&async_copy->refcount); + wake_up_process(async_copy->copy_task); spin_lock(&async_copy->cp_clp->async_lock); list_add(&async_copy->copies, &async_copy->cp_clp->async_copies); spin_unlock(&async_copy->cp_clp->async_lock); - wake_up_process(async_copy->copy_task); status = nfs_ok; } else { status = nfsd4_do_copy(copy, copy->nf_src->nf_file, @@ -2089,8 +2166,18 @@ find_async_copy(struct nfs4_client *clp, stateid_t *stateid) spin_lock(&clp->async_lock); copy = find_async_copy_locked(clp, stateid); - if (copy) + if (copy) { refcount_inc(©->refcount); + nfs4_free_copy_state(copy); + /* + * Mirror nfsd4_unhash_copy(): unlink and clear cp_clp under + * async_lock so the reaper can't reach it. Caller drops the + * membership ref after nfsd4_stop_copy(). + */ + smp_store_release(©->cp_clp, NULL); + if (!list_empty(©->copies)) + list_del_init(©->copies); + } spin_unlock(&clp->async_lock); return copy; } @@ -2109,8 +2196,11 @@ nfsd4_offload_cancel(struct svc_rqst *rqstp, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); return manage_cpntf_state(nn, &os->stateid, clp, NULL); - } else + } else { nfsd4_stop_copy(copy); + /* find_async_copy() unlinked it from the reaper; drop the membership ref. */ + nfs4_put_copy(copy); + } return nfs_ok; } @@ -2124,7 +2214,6 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct nfs4_stid *stid = NULL; struct nfs4_cpntf_state *cps; - struct nfs4_client *clp = cstate->clp; status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh, &cn->cpn_src_stateid, RD_STATE, NULL, @@ -2138,12 +2227,14 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, cn->cpn_lease_time.tv_nsec = 0; status = nfserrno(-ENOMEM); + /* + * The returned cps is published and fully initialized, and carries an + * extra reference for us; drop it once we are done with it. + */ cps = nfs4_alloc_init_cpntf_state(nn, stid); if (!cps) goto out; memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t)); - memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t)); - memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t)); /* For now, only return one server address in cpn_src, the * address used by the client to connect to this server. @@ -2152,10 +2243,11 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfsd4_set_netaddr((struct sockaddr *)&rqstp->rq_daddr, &cn->cpn_src->u.nl4_addr); WARN_ON_ONCE(status); - if (status) { - nfs4_put_cpntf_state(nn, cps); - goto out; - } + /* + * Drop our extra reference. The membership reference keeps the entry + * alive for a later inter-server READ, or until the laundromat reaps it. + */ + nfs4_put_cpntf_state(nn, cps); out: nfs4_put_stid(stid); return status; @@ -2929,9 +3021,22 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) op->status = nfsd4_open_omfg(rqstp, cstate, op); goto encode_op; } - if (!current_fh->fh_dentry && - !HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { - if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + if (!current_fh->fh_dentry) { + if (HAS_FH_FLAG(current_fh, NFSD4_FH_FOREIGN)) { + /* + * FOREIGN fh from inter-SSC PUTFH: only + * SAVEFH may proceed with a NULL fh_dentry. + * Per RFC 7862 S15.2.3, validation of a + * foreign fh is deferred to the operation + * that consumes it, and NFS4ERR_STALE is + * returned at that point. + */ + if (op->opnum != OP_SAVEFH && + !(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { + op->status = nfserr_stale; + goto encode_op; + } + } else if (!(op->opdesc->op_flags & ALLOWED_WITHOUT_FH)) { op->status = nfserr_nofilehandle; goto encode_op; } @@ -3317,7 +3422,7 @@ static u32 nfsd4_get_dir_delegation_rsize(const struct svc_rqst *rqstp, op_encode_stateid_maxsz + 2 /* gddr_notification */ + 2 /* gddr_child_attributes */ + - 2 /* gddr_dir_attributes */); + 2 /* gddr_dir_attributes */) * sizeof(__be32); } #ifdef CONFIG_NFSD_PNFS diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index c9d9a7d46e7b2..fbaf2f1352748 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -91,13 +91,6 @@ static void deleg_reaper(struct nfsd_net *nn); /* Locking: */ -/* - * Currently used for the del_recall_lru and file hash table. In an - * effort to decrease the scope of the client_mutex, this spinlock may - * eventually cover more: - */ -static DEFINE_SPINLOCK(state_lock); - enum nfsd4_st_mutex_lock_subclass { OPEN_STATEID_MUTEX = 0, LOCK_STATEID_MUTEX = 1, @@ -207,18 +200,28 @@ renew_client_locked(struct nfs4_client *clp) clp->cl_state = NFSD4_ACTIVE; } +/* + * Finish a cl_rpc_users unpin with the client_lock held. A + * revocation walk clears @renew so the client whose state it is + * revoking is not revived; every other caller renews the lease of + * a still-active client. + */ +static void __put_client_locked(struct nfs4_client *clp, bool renew) +{ + if (is_client_expired(clp)) + wake_up_all(&expiry_wq); + else if (renew) + renew_client_locked(clp); +} + static void put_client_renew_locked(struct nfs4_client *clp) { struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); lockdep_assert_held(&nn->client_lock); - if (!atomic_dec_and_test(&clp->cl_rpc_users)) - return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, true); } static void put_client_renew(struct nfs4_client *clp) @@ -227,10 +230,27 @@ static void put_client_renew(struct nfs4_client *clp) if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) return; - if (!is_client_expired(clp)) - renew_client_locked(clp); - else - wake_up_all(&expiry_wq); + __put_client_locked(clp, true); + spin_unlock(&nn->client_lock); +} + +static void put_client_no_renew_locked(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + lockdep_assert_held(&nn->client_lock); + + if (atomic_dec_and_test(&clp->cl_rpc_users)) + __put_client_locked(clp, false); +} + +static void put_client_no_renew(struct nfs4_client *clp) +{ + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); + + if (!atomic_dec_and_lock(&clp->cl_rpc_users, &nn->client_lock)) + return; + __put_client_locked(clp, false); spin_unlock(&nn->client_lock); } @@ -331,6 +351,16 @@ free_blocked_lock(struct nfsd4_blocked_lock *nbl) kref_put(&nbl->nbl_kref, free_nbl); } +/* A blocked lock's flc_owner is its nfs4_lockowner. */ +static struct nfs4_client * +nbl_client(struct nfsd4_blocked_lock *nbl) +{ + struct nfs4_lockowner *lo; + + lo = (struct nfs4_lockowner *)nbl->nbl_lock.c.flc_owner; + return lo->lo_owner.so_client; +} + static void remove_blocked_locks(struct nfs4_lockowner *lo) { @@ -944,7 +974,7 @@ struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla * Create a unique stateid_t to represent each COPY. */ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, - unsigned char cs_type) + unsigned char cs_type, struct nfs4_stid *p_stid) { int new_id; @@ -954,19 +984,34 @@ static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid, idr_preload(GFP_KERNEL); spin_lock(&nn->s2s_cp_lock); new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT); - stid->cs_stid.si_opaque.so_id = new_id; - stid->cs_stid.si_generation = 1; + if (new_id >= 0) { + stid->cs_stid.si_opaque.so_id = new_id; + stid->cs_stid.si_generation = 1; + /* + * Set cs_type and link onto sc_cp_list under the same lock + * that installed the IDR entry, so a concurrent + * manage_cpntf_state() sees either no entry or a fully + * linked cp_list. + */ + stid->cs_type = cs_type; + if (p_stid) { + struct nfs4_cpntf_state *cps = + container_of(stid, struct nfs4_cpntf_state, + cp_stateid); + + list_add(&cps->cp_list, &p_stid->sc_cp_list); + } + } spin_unlock(&nn->s2s_cp_lock); idr_preload_end(); if (new_id < 0) return 0; - stid->cs_type = cs_type; return 1; } int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy) { - return nfs4_init_cp_state(nn, ©->cp_stateid, NFS4_COPY_STID); + return nfs4_init_cp_state(nn, ©->cp_stateid, NFS4_COPY_STID, NULL); } struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, @@ -977,13 +1022,21 @@ struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn, cps = kzalloc(sizeof(struct nfs4_cpntf_state), GFP_KERNEL); if (!cps) return NULL; + /* So a stale list_del_init() before linking is a no-op. */ + INIT_LIST_HEAD(&cps->cp_list); cps->cpntf_time = ktime_get_boottime_seconds(); - refcount_set(&cps->cp_stateid.cs_count, 1); - if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID)) + /* + * Fully initialize the entry before nfs4_init_cp_state() publishes it, + * since a concurrent OFFLOAD_CANCEL could then free it. Take an extra + * reference for the caller (dropped with nfs4_put_cpntf_state()). + */ + memcpy(&cps->cp_p_stateid, &p_stid->sc_stateid, sizeof(stateid_t)); + memcpy(&cps->cp_p_clid, &p_stid->sc_client->cl_clientid, + sizeof(clientid_t)); + refcount_set(&cps->cp_stateid.cs_count, 2); + if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID, + p_stid)) goto out_free; - spin_lock(&nn->s2s_cp_lock); - list_add(&cps->cp_list, &p_stid->sc_cp_list); - spin_unlock(&nn->s2s_cp_lock); return cps; out_free: kfree(cps); @@ -1003,18 +1056,66 @@ void nfs4_free_copy_state(struct nfsd4_copy *copy) spin_unlock(&nn->s2s_cp_lock); } +/* + * Drop the parent's reference on an already-unlinked cpntf entry. If a + * concurrent holder still owns a reference, its nfs4_put_cpntf_state() does + * the final free. + * + * nn->s2s_cp_lock must be held. + */ +static void put_cpntf_state_unlinked_locked(struct nfs4_cpntf_state *cps) +{ + WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); + WARN_ON_ONCE(!list_empty(&cps->cp_list)); + + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) + kfree(cps); +} + +/* + * Unhash from the IDR and sc_cp_list. Gated on list_empty() to avoid + * evicting a recycled so_id. + */ +static void nfsd4_unhash_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) +{ + lockdep_assert_held(&nn->s2s_cp_lock); + + if (!list_empty(&cps->cp_list)) { + list_del_init(&cps->cp_list); + idr_remove(&nn->s2s_cp_stateids, cps->cp_stateid.cs_stid.si_opaque.so_id); + } +} + +/* + * Revoke a copy-notify stateid: unlink it from the IDR and sc_cp_list first + * so no new finder can discover it, then drop the membership reference. Every + * revoke path (cancel, laundromat, drain) must use this rather than + * _free_cpntf_state_locked(), which unlinks only at refcount zero and so could + * let a second revoke free the entry under a concurrent reader. + * + * nn->s2s_cp_lock must be held. + */ +static void revoke_cpntf_state_locked(struct nfsd_net *nn, + struct nfs4_cpntf_state *cps) +{ + nfsd4_unhash_cpntf_state(nn, cps); + put_cpntf_state_unlinked_locked(cps); +} + static void nfs4_free_cpntf_statelist(struct net *net, struct nfs4_stid *stid) { - struct nfs4_cpntf_state *cps; + struct nfs4_cpntf_state *cps, *tmp; struct nfsd_net *nn; nn = net_generic(net, nfsd_net_id); spin_lock(&nn->s2s_cp_lock); - while (!list_empty(&stid->sc_cp_list)) { - cps = list_first_entry(&stid->sc_cp_list, - struct nfs4_cpntf_state, cp_list); - _free_cpntf_state_locked(nn, cps); - } + /* + * Revoke unlinks each entry before dropping the parent's reference, so + * the drain terminates in one pass per entry regardless of cs_count; a + * concurrent holder does the final kfree via nfs4_put_cpntf_state(). + */ + list_for_each_entry_safe(cps, tmp, &stid->sc_cp_list, cp_list) + revoke_cpntf_state_locked(nn, cps); spin_unlock(&nn->s2s_cp_lock); } @@ -1279,8 +1380,9 @@ nfs4_delegation_exists(struct nfs4_client *clp, struct nfs4_file *fp) { struct nfs4_delegation *searchdp = NULL; struct nfs4_client *searchclp = NULL; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); - lockdep_assert_held(&state_lock); + lockdep_assert_held(&nn->deleg_lock); lockdep_assert_held(&fp->fi_lock); list_for_each_entry(searchdp, &fp->fi_delegations, dl_perfile) { @@ -1309,8 +1411,9 @@ static int hash_delegation_locked(struct nfs4_delegation *dp, struct nfs4_file *fp) { struct nfs4_client *clp = dp->dl_stid.sc_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); - lockdep_assert_held(&state_lock); + lockdep_assert_held(&nn->deleg_lock); lockdep_assert_held(&fp->fi_lock); lockdep_assert_held(&clp->cl_lock); @@ -1332,8 +1435,10 @@ static bool unhash_delegation_locked(struct nfs4_delegation *dp, unsigned short statusmask) { struct nfs4_file *fp = dp->dl_stid.sc_file; + struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, + nfsd_net_id); - lockdep_assert_held(&state_lock); + lockdep_assert_held(&nn->deleg_lock); if (!delegation_hashed(dp)) return false; @@ -1358,10 +1463,12 @@ unhash_delegation_locked(struct nfs4_delegation *dp, unsigned short statusmask) static void destroy_delegation(struct nfs4_delegation *dp) { bool unhashed; + struct nfsd_net *nn = net_generic(dp->dl_stid.sc_client->net, + nfsd_net_id); - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); unhashed = unhash_delegation_locked(dp, SC_STATUS_CLOSED); - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); if (unhashed) destroy_unhashed_deleg(dp); } @@ -1771,13 +1878,23 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) struct nfs4_client *clp; retry: list_for_each_entry(clp, head, cl_idhash) { - struct nfs4_stid *stid = find_one_sb_stid(clp, sb, - sc_types); + struct nfs4_stid *stid; + + /* + * force_expire_client() ignores cl_rpc_users once + * its wait_event() has passed, so pinning cannot + * keep an already-expiring client alive; the + * expiry path revokes its states instead. + */ + if (is_client_expired(clp)) + continue; + stid = find_one_sb_stid(clp, sb, sc_types); if (stid) { struct nfs4_ol_stateid *stp; struct nfs4_delegation *dp; struct nfs4_layout_stateid *ls; + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); switch (stid->sc_type) { case SC_TYPE_OPEN: @@ -1829,11 +1946,11 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) */ refcount_inc(&stid->sc_count); dp = delegstateid(stid); - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); if (!unhash_delegation_locked( dp, SC_STATUS_ADMIN_REVOKED)) dp = NULL; - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); if (dp) revoke_delegation(dp); else @@ -1861,6 +1978,7 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb) */ nn->nfs40_last_revoke = ktime_get_boottime_seconds(); + put_client_no_renew_locked(clp); goto retry; } } @@ -2185,7 +2303,7 @@ static void __free_session(struct nfsd4_session *ses) { free_session_slots(ses, 0); xa_destroy(&ses->se_slots); - kfree(ses); + kfree_rcu(ses, rcu_head); } static void free_session(struct nfsd4_session *ses) @@ -2499,13 +2617,13 @@ __destroy_client(struct nfs4_client *clp) struct nfs4_delegation *dp; LIST_HEAD(reaplist); - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); while (!list_empty(&clp->cl_delegations)) { dp = list_entry(clp->cl_delegations.next, struct nfs4_delegation, dl_perclnt); unhash_delegation_locked(dp, SC_STATUS_CLOSED); list_add(&dp->dl_recall_lru, &reaplist); } - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); while (!list_empty(&reaplist)) { dp = list_entry(reaplist.next, struct nfs4_delegation, dl_recall_lru); list_del_init(&dp->dl_recall_lru); @@ -2522,14 +2640,24 @@ __destroy_client(struct nfs4_client *clp) release_openowner(oo); } for (i = 0; i < OWNER_HASH_SIZE; i++) { - struct nfs4_stateowner *so, *tmp; + struct nfs4_stateowner *so; - list_for_each_entry_safe(so, tmp, &clp->cl_ownerstr_hashtbl[i], - so_strhash) { + spin_lock(&clp->cl_lock); + while (!list_empty(&clp->cl_ownerstr_hashtbl[i])) { + so = list_first_entry(&clp->cl_ownerstr_hashtbl[i], + struct nfs4_stateowner, so_strhash); /* Should be no openowners at this point */ WARN_ON_ONCE(so->so_is_open_owner); + nfs4_get_stateowner(so); + unhash_lockowner_locked(lockowner(so)); + spin_unlock(&clp->cl_lock); + remove_blocked_locks(lockowner(so)); + nfs4_put_stateowner(so); + + spin_lock(&clp->cl_lock); } + spin_unlock(&clp->cl_lock); } nfsd4_return_all_client_layouts(clp); nfsd4_shutdown_copy(clp); @@ -3137,6 +3265,11 @@ static void force_expire_client(struct nfs4_client *clp) trace_nfsd_clid_admin_expired(&clp->cl_clientid); + /* + * cl_time is cleared under client_lock before the wait so a + * revocation walk pinning cl_rpc_users under it either skips + * this client or is seen by this wait_event(). + */ spin_lock(&nn->client_lock); clp->cl_time = 0; spin_unlock(&nn->client_lock); @@ -3292,7 +3425,7 @@ static struct nfs4_client *create_client(struct xdr_netobj name, clp->cl_time = ktime_get_boottime_seconds(); copy_verf(clp, verf); memcpy(&clp->cl_addr, sa, sizeof(struct sockaddr_storage)); - clp->cl_cb_session = NULL; + RCU_INIT_POINTER(clp->cl_cb_session, NULL); clp->net = net; clp->cl_nfsd_dentry = nfsd_client_mkdir( nn, &clp->cl_nfsdfs, @@ -4396,6 +4529,19 @@ static void nfsd4_construct_sequence_response(struct nfsd4_session *session, seq->status_flags |= SEQ4_STATUS_ADMIN_STATE_REVOKED; } +static bool nfsd4_slots_inuse(struct nfsd4_session *ses, int from) +{ + int i; + + for (i = from; i < ses->se_fchannel.maxreqs; i++) { + struct nfsd4_slot *slot = xa_load(&ses->se_slots, i); + + if (slot->sl_flags & NFSD4_SLOT_INUSE) + return true; + } + return false; +} + __be32 nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) @@ -4475,7 +4621,9 @@ nfsd4_sequence(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (session->se_target_maxslots < session->se_fchannel.maxreqs && slot->sl_generation == session->se_slot_gen && - seq->maxslots <= session->se_target_maxslots) + seq->maxslots <= session->se_target_maxslots && + seq->slotid < session->se_target_maxslots && + !nfsd4_slots_inuse(session, session->se_target_maxslots)) /* Client acknowledged our reduce maxreqs */ free_session_slots(session, session->se_target_maxslots); @@ -5072,6 +5220,7 @@ static void nfsd4_drop_revoked_stid(struct nfs4_stid *s) case SC_TYPE_DELEG: dp = delegstateid(s); list_del_init(&dp->dl_recall_lru); + s->sc_status |= SC_STATUS_FREED; spin_unlock(&cl->cl_lock); nfs4_put_stid(s); break; @@ -5443,12 +5592,12 @@ static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb) * If the dl_time != 0, then we know that it has already been * queued for a lease break. Don't queue it again. */ - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); if (delegation_hashed(dp) && dp->dl_time == 0) { dp->dl_time = ktime_get_boottime_seconds(); list_add_tail(&dp->dl_recall_lru, &nn->del_recall_lru); } - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); } static int nfsd4_cb_recall_done(struct nfsd4_callback *cb, @@ -5515,8 +5664,10 @@ static void nfsd_break_one_deleg(struct nfs4_delegation *dp) refcount_inc(&dp->dl_stid.sc_count); queued = nfsd4_run_cb(&dp->dl_recall); WARN_ON_ONCE(!queued); - if (!queued) + if (!queued) { refcount_dec(&dp->dl_stid.sc_count); + clear_bit(NFSD4_CALLBACK_RUNNING, &dp->dl_recall.cb_flags); + } } /* Called from break_lease() with flc_lock held. */ @@ -6060,6 +6211,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, { bool deleg_ts = nfsd4_want_deleg_timestamps(open); struct nfs4_client *clp = stp->st_stid.sc_client; + struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); struct nfs4_file *fp = stp->st_stid.sc_file; struct nfs4_clnt_odstate *odstate = stp->st_clnt_odstate; struct nfs4_delegation *dp; @@ -6119,7 +6271,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, return ERR_PTR(-EOPNOTSUPP); } - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); spin_lock(&fp->fi_lock); if (nfs4_delegation_exists(clp, fp)) status = -EAGAIN; @@ -6134,7 +6286,7 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, } else fp->fi_delegees++; spin_unlock(&fp->fi_lock); - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); if (nf) nfsd_file_put(nf); if (status) @@ -6178,13 +6330,13 @@ nfs4_set_delegation(struct nfsd4_open *open, struct nfs4_ol_stateid *stp, if (fp->fi_had_conflict) goto out_unlock; - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); spin_lock(&clp->cl_lock); spin_lock(&fp->fi_lock); status = hash_delegation_locked(dp, fp); spin_unlock(&fp->fi_lock); spin_unlock(&clp->cl_lock); - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); if (status) goto out_unlock; @@ -6670,7 +6822,7 @@ bool nfsd4_force_end_grace(struct nfsd_net *nn) */ static bool clients_still_reclaiming(struct nfsd_net *nn) { - time64_t double_grace_period_end = nn->boot_time + + time64_t double_grace_period_end = nn->boot_time_bt + 2 * nn->nfsd4_lease; if (READ_ONCE(nn->grace_end_forced)) @@ -6738,30 +6890,36 @@ static void nfsd4_ssc_shutdown_umount(struct nfsd_net *nn) static void nfsd4_ssc_expire_umount(struct nfsd_net *nn) { bool do_wakeup = false; - struct nfsd4_ssc_umount_item *ni = NULL; - struct nfsd4_ssc_umount_item *tmp; + struct nfsd4_ssc_umount_item *ni; +restart: spin_lock(&nn->nfsd_ssc_lock); - list_for_each_entry_safe(ni, tmp, &nn->nfsd_ssc_mount_list, nsui_list) { - if (time_after(jiffies, ni->nsui_expire)) { - if (refcount_read(&ni->nsui_refcnt) > 1) - continue; + list_for_each_entry(ni, &nn->nfsd_ssc_mount_list, nsui_list) { + if (!time_after(jiffies, ni->nsui_expire)) + break; + if (refcount_read(&ni->nsui_refcnt) > 1) + continue; - /* mark being unmount */ - ni->nsui_busy = true; - spin_unlock(&nn->nfsd_ssc_lock); - mntput(ni->nsui_vfsmount); - spin_lock(&nn->nfsd_ssc_lock); + /* Prevent concurrent setup during unmount */ + ni->nsui_busy = true; + spin_unlock(&nn->nfsd_ssc_lock); + mntput(ni->nsui_vfsmount); + spin_lock(&nn->nfsd_ssc_lock); - /* waiters need to start from begin of list */ - list_del(&ni->nsui_list); - kfree(ni); + /* Force concurrent scanners to restart */ + list_del(&ni->nsui_list); + kfree(ni); - /* wakeup ssc_connect waiters */ - do_wakeup = true; - continue; - } - break; + /* wakeup ssc_connect waiters */ + do_wakeup = true; + /* + * Concurrent nfsd4_ssc_cancel_dul() can free any item + * on the list under nfsd_ssc_lock while mntput() runs + * above. Restart from the head; the list is short and + * the expire worker is periodic, so this is cheap. + */ + spin_unlock(&nn->nfsd_ssc_lock); + goto restart; } if (do_wakeup) wake_up_all(&nn->nfsd_ssc_waitq); @@ -6911,16 +7069,20 @@ static void nfs40_clean_admin_revoked(struct nfsd_net *nn, if (atomic_read(&clp->cl_admin_revoked) == 0) continue; + if (is_client_expired(clp)) + continue; spin_lock(&clp->cl_lock); idr_for_each_entry_ul(&clp->cl_stateids, stid, tmp, id) if (stid->sc_status & SC_STATUS_ADMIN_REVOKED) { refcount_inc(&stid->sc_count); + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); /* this function drops ->cl_lock */ nfsd4_drop_revoked_stid(stid); nfs4_put_stid(stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); goto retry; } spin_unlock(&clp->cl_lock); @@ -6941,6 +7103,7 @@ nfs4_laundromat(struct nfsd_net *nn) .new_timeo = nn->nfsd4_lease }; struct nfs4_cpntf_state *cps; + struct nfs4_client *clp; copy_stateid_t *cps_t; int i; @@ -6955,7 +7118,7 @@ nfs4_laundromat(struct nfsd_net *nn) cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid); if (cps->cp_stateid.cs_type == NFS4_COPYNOTIFY_STID && state_expired(<, cps->cpntf_time)) - _free_cpntf_state_locked(nn, cps); + revoke_cpntf_state_locked(nn, cps); } spin_unlock(&nn->s2s_cp_lock); nfsd4_async_copy_reaper(nn); @@ -6964,21 +7127,35 @@ nfs4_laundromat(struct nfsd_net *nn) nfs40_clean_admin_revoked(nn, <); - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); list_for_each_safe(pos, next, &nn->del_recall_lru) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); if (!state_expired(<, dp->dl_time)) break; + clp = dp->dl_stid.sc_client; + spin_lock(&nn->client_lock); + if (is_client_expired(clp)) { + spin_unlock(&nn->client_lock); + continue; + } + /* + * Pin without reviving: get_client_locked() would + * flip a courtesy client back to NFSD4_ACTIVE. + */ + atomic_inc(&clp->cl_rpc_users); + spin_unlock(&nn->client_lock); refcount_inc(&dp->dl_stid.sc_count); unhash_delegation_locked(dp, SC_STATUS_REVOKED); list_add(&dp->dl_recall_lru, &reaplist); } - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); while (!list_empty(&reaplist)) { dp = list_first_entry(&reaplist, struct nfs4_delegation, dl_recall_lru); + clp = dp->dl_stid.sc_client; list_del_init(&dp->dl_recall_lru); revoke_delegation(dp); + put_client_no_renew(clp); } spin_lock(&nn->client_lock); @@ -6988,11 +7165,16 @@ nfs4_laundromat(struct nfsd_net *nn) if (!state_expired(<, oo->oo_time)) break; list_del_init(&oo->oo_close_lru); + clp = oo->oo_owner.so_client; + if (is_client_expired(clp)) + continue; stp = oo->oo_last_closed_stid; oo->oo_last_closed_stid = NULL; + atomic_inc(&clp->cl_rpc_users); spin_unlock(&nn->client_lock); nfs4_put_stid(&stp->st_stid); spin_lock(&nn->client_lock); + put_client_no_renew_locked(clp); } spin_unlock(&nn->client_lock); @@ -7008,22 +7190,29 @@ nfs4_laundromat(struct nfsd_net *nn) * indefinitely once the lock does become free. */ BUG_ON(!list_empty(&reaplist)); + spin_lock(&nn->client_lock); spin_lock(&nn->blocked_locks_lock); - while (!list_empty(&nn->blocked_locks_lru)) { - nbl = list_first_entry(&nn->blocked_locks_lru, - struct nfsd4_blocked_lock, nbl_lru); + list_for_each_safe(pos, next, &nn->blocked_locks_lru) { + nbl = list_entry(pos, struct nfsd4_blocked_lock, nbl_lru); if (!state_expired(<, nbl->nbl_time)) break; + clp = nbl_client(nbl); + if (is_client_expired(clp)) + continue; + atomic_inc(&clp->cl_rpc_users); list_move(&nbl->nbl_lru, &reaplist); list_del_init(&nbl->nbl_list); } spin_unlock(&nn->blocked_locks_lock); + spin_unlock(&nn->client_lock); while (!list_empty(&reaplist)) { nbl = list_first_entry(&reaplist, struct nfsd4_blocked_lock, nbl_lru); + clp = nbl_client(nbl); list_del_init(&nbl->nbl_lru); free_blocked_lock(nbl); + put_client_no_renew(clp); } #ifdef CONFIG_NFSD_V4_2_INTER_SSC /* service the server-to-server copy delayed unmount list */ @@ -7074,12 +7263,12 @@ deleg_reaper(struct nfsd_net *nn) continue; if (atomic_read(&clp->cl_delegs_in_recall)) continue; - if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) - continue; if (ktime_get_boottime_seconds() - clp->cl_ra_time < 5) continue; if (clp->cl_cb_state != NFSD4_CB_UP) continue; + if (test_and_set_bit(NFSD4_CALLBACK_RUNNING, &clp->cl_ra->ra_cb.cb_flags)) + continue; /* release in nfsd4_cb_recall_any_release */ kref_get(&clp->cl_nfsdfs.cl_ref); @@ -7342,16 +7531,14 @@ nfs4_check_file(struct svc_rqst *rqstp, struct svc_fh *fhp, struct nfs4_stid *s, out: return status; } -static void -_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) + +static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps) { WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID); - if (!refcount_dec_and_test(&cps->cp_stateid.cs_count)) - return; - list_del(&cps->cp_list); - idr_remove(&nn->s2s_cp_stateids, - cps->cp_stateid.cs_stid.si_opaque.so_id); - kfree(cps); + if (refcount_dec_and_test(&cps->cp_stateid.cs_count)) { + nfsd4_unhash_cpntf_state(nn, cps); + kfree(cps); + } } /* * A READ from an inter server to server COPY will have a @@ -7376,10 +7563,20 @@ __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st, state = NULL; goto unlock; } - if (!clp) + if (!clp) { refcount_inc(&state->cp_stateid.cs_count); - else - _free_cpntf_state_locked(nn, state); + } else if (memcmp(&clp->cl_clientid, &state->cp_p_clid, + sizeof(clientid_t))) { + /* + * OFFLOAD_CANCEL: only the creating client may cancel. + * so_id is guessable, so without this check any client + * could free another's cpntf state. + */ + state = NULL; + goto unlock; + } else { + revoke_cpntf_state_locked(nn, state); + } } unlock: spin_unlock(&nn->s2s_cp_lock); @@ -7666,7 +7863,7 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, return status; stp = openlockstateid(s); if (nfsd4_cstate_assign_replay(cstate, stp->st_stateowner) == -EAGAIN) { - nfs4_put_stateowner(stp->st_stateowner); + nfs4_put_stid(&stp->st_stid); goto retry; } @@ -7914,6 +8111,10 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (status) goto put_stateid; + status = nfs4_check_fh(&cstate->current_fh, &dp->dl_stid); + if (status) + goto put_stateid; + trace_nfsd_deleg_return(stateid); destroy_delegation(dp); smp_mb__after_atomic(); @@ -8381,6 +8582,9 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, status = nfserr_no_grace; if (!locks_in_grace(net) && lock->lk_reclaim) goto out; + if (lock->lk_reclaim && + test_bit(NFSD4_CLIENT_RECLAIM_COMPLETE, &cstate->clp->cl_flags)) + goto out; if (lock->lk_reclaim) flags |= FL_RECLAIM; @@ -8417,10 +8621,11 @@ nfsd4_lock(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, goto out; } - if (lock->lk_type & (NFS4_READW_LT | NFS4_WRITEW_LT) && - nfsd4_has_session(cstate) && - locks_can_async_lock(nf->nf_file->f_op)) - flags |= FL_SLEEP; + if ((lock->lk_type == NFS4_READW_LT || + lock->lk_type == NFS4_WRITEW_LT) && + nfsd4_has_session(cstate) && + locks_can_async_lock(nf->nf_file->f_op)) + flags |= FL_SLEEP; nbl = find_or_allocate_block(lock_sop, &fp->fi_fhandle, nn); if (!nbl) { @@ -8993,6 +9198,7 @@ static int nfs4_state_create_net(struct net *net) nn->conf_name_tree = RB_ROOT; nn->unconf_name_tree = RB_ROOT; nn->boot_time = ktime_get_real_seconds(); + nn->boot_time_bt = ktime_get_boottime_seconds(); nn->grace_ended = false; nn->grace_end_forced = false; nn->client_tracking_active = false; @@ -9001,6 +9207,7 @@ static int nfs4_state_create_net(struct net *net) INIT_LIST_HEAD(&nn->client_lru); INIT_LIST_HEAD(&nn->close_lru); INIT_LIST_HEAD(&nn->del_recall_lru); + spin_lock_init(&nn->deleg_lock); spin_lock_init(&nn->client_lock); spin_lock_init(&nn->s2s_cp_lock); idr_init(&nn->s2s_cp_stateids); @@ -9135,13 +9342,13 @@ nfs4_state_shutdown_net(struct net *net) locks_end_grace(&nn->nfsd4_manager); INIT_LIST_HEAD(&reaplist); - spin_lock(&state_lock); + spin_lock(&nn->deleg_lock); list_for_each_safe(pos, next, &nn->del_recall_lru) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); unhash_delegation_locked(dp, SC_STATUS_CLOSED); list_add(&dp->dl_recall_lru, &reaplist); } - spin_unlock(&state_lock); + spin_unlock(&nn->deleg_lock); list_for_each_safe(pos, next, &reaplist) { dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru); list_del_init(&dp->dl_recall_lru); diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index a96029c967e6b..9f49e3c9e7527 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -526,6 +526,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_access(argp->xdr, &access)) return nfserr_bad_xdr; + if (access.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_atime.tv_sec = access.seconds; iattr->ia_atime.tv_nsec = access.nseconds; iattr->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET | ATTR_DELEG; @@ -535,6 +537,8 @@ nfsd4_decode_fattr4(struct nfsd4_compoundargs *argp, u32 *bmval, u32 bmlen, if (!xdrgen_decode_fattr4_time_deleg_modify(argp->xdr, &modify)) return nfserr_bad_xdr; + if (modify.nseconds >= NSEC_PER_SEC) + return nfserr_inval; iattr->ia_mtime.tv_sec = modify.seconds; iattr->ia_mtime.tv_nsec = modify.nseconds; iattr->ia_ctime.tv_sec = modify.seconds; @@ -822,6 +826,10 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, union nfsd4_op_u *u) case NF4LNK: if (xdr_stream_decode_u32(argp->xdr, &create->cr_datalen) < 0) return nfserr_bad_xdr; + if (create->cr_datalen == 0) + return nfserr_inval; + if (create->cr_datalen > NFS4_MAXPATHLEN) + return nfserr_nametoolong; p = xdr_inline_decode(argp->xdr, create->cr_datalen); if (!p) return nfserr_bad_xdr; @@ -1969,6 +1977,7 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, { struct nfs42_netaddr *naddr; __be32 *p; + u32 str_len; if (xdr_stream_decode_u32(argp->xdr, &ns->nl4_type) < 0) return nfserr_bad_xdr; @@ -1998,6 +2007,18 @@ static __be32 nfsd4_decode_nl4_server(struct nfsd4_compoundargs *argp, return nfserr_bad_xdr; memcpy(naddr->addr, p, naddr->addr_len); break; + case NL4_NAME: + case NL4_URL: + /* + * Well-formed XDR, but only NL4_NETADDR is supported. Consume + * the utf8str_cis to keep the stream aligned, then return + * NFS4ERR_NOTSUPP rather than the misleading NFS4ERR_BADXDR. + */ + if (xdr_stream_decode_u32(argp->xdr, &str_len) < 0) + return nfserr_bad_xdr; + if (!xdr_inline_decode(argp->xdr, str_len)) + return nfserr_bad_xdr; + return nfserr_notsupp; default: return nfserr_bad_xdr; } @@ -4478,7 +4499,7 @@ static __be32 nfsd4_encode_splice_read( static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, struct nfsd4_read *read, - struct file *file, unsigned long maxcount) + unsigned long maxcount) { struct xdr_stream *xdr = resp->xdr; unsigned int base = xdr->buf->page_len & ~PAGE_MASK; @@ -4489,7 +4510,7 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, if (xdr_reserve_space_vec(xdr, maxcount) < 0) return nfserr_resource; - nfserr = nfsd_iter_read(resp->rqstp, read->rd_fhp, file, + nfserr = nfsd_iter_read(resp->rqstp, read->rd_fhp, read->rd_nf, read->rd_offset, &maxcount, base, &read->rd_eof); read->rd_length = maxcount; @@ -4536,7 +4557,7 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr, if (file->f_op->splice_read && splice_ok) nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); else - nfserr = nfsd4_encode_readv(resp, read, file, maxcount); + nfserr = nfsd4_encode_readv(resp, read, maxcount); if (nfserr) { xdr_truncate_encode(xdr, eof_offset); return nfserr; @@ -5432,7 +5453,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, if (file->f_op->splice_read && splice_ok) nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); else - nfserr = nfsd4_encode_readv(resp, read, file, maxcount); + nfserr = nfsd4_encode_readv(resp, read, maxcount); if (nfserr) return nfserr; @@ -5984,9 +6005,12 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp) { struct nfsd4_compoundargs *args = rqstp->rq_argp; + args->opcnt = 0; if (args->ops != args->iops) { - vfree(args->ops); + void *old_ops = args->ops; + args->ops = args->iops; + kvfree_rcu_mightsleep(old_ops); } while (args->to_free) { struct svcxdr_tmpbuf *tb = args->to_free; diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index ab13ee9c7fd84..ed1748a6d319f 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -200,14 +200,14 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) nn->nfsd_reply_cache_shrinker->seeks = 1; nn->nfsd_reply_cache_shrinker->private_data = nn; - shrinker_register(nn->nfsd_reply_cache_shrinker); - for (i = 0; i < hashsize; i++) { INIT_LIST_HEAD(&nn->drc_hashtbl[i].lru_head); spin_lock_init(&nn->drc_hashtbl[i].cache_lock); } nn->drc_hashsize = hashsize; + shrinker_register(nn->nfsd_reply_cache_shrinker); + return 0; out_shrinker: kvfree(nn->drc_hashtbl); @@ -275,7 +275,7 @@ nfsd_prune_bucket_locked(struct nfsd_net *nn, struct nfsd_drc_bucket *b, nfsd_cacherep_unlink_locked(nn, b, rp); list_add(&rp->c_lru, dispose); - if (max && ++freed > max) + if (max && ++freed >= max) break; } } diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 1da5c40d6722a..2fcf14d530b2e 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -298,7 +298,7 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb); mutex_lock(&nfsd_mutex); nn = net_generic(netns(file), nfsd_net_id); - if (nn->nfsd_serv) + if (nn->nfsd_net_up) nfsd4_revoke_states(nn, path.dentry->d_sb); else error = -EINVAL; @@ -1411,6 +1411,21 @@ static int create_proc_exports_entry(void) unsigned int nfsd_net_id; +struct nfsd_genl_rqstp { + struct sockaddr_storage rq_daddr; + struct sockaddr_storage rq_saddr; + unsigned long rq_flags; + ktime_t rq_stime; + __be32 rq_xid; + u32 rq_vers; + u32 rq_prog; + u32 rq_proc; + + /* NFSv4 compound */ + u32 rq_opcnt; + u32 rq_opnum[16]; +}; + static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, struct netlink_callback *cb, struct nfsd_genl_rqstp *genl_rqstp) @@ -1431,9 +1446,9 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME, ktime_to_us(genl_rqstp->rq_stime), NFSD_A_RPC_STATUS_PAD)) - return -ENOBUFS; + goto out_cancel; - switch (genl_rqstp->rq_saddr.sa_family) { + switch (genl_rqstp->rq_saddr.ss_family) { case AF_INET: { const struct sockaddr_in *s_in, *d_in; @@ -1447,7 +1462,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin_port)) - return -ENOBUFS; + goto out_cancel; break; } case AF_INET6: { @@ -1463,7 +1478,7 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, s_in->sin6_port) || nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT, d_in->sin6_port)) - return -ENOBUFS; + goto out_cancel; break; } } @@ -1471,10 +1486,14 @@ static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb, for (i = 0; i < genl_rqstp->rq_opcnt; i++) if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS, genl_rqstp->rq_opnum[i])) - return -ENOBUFS; + goto out_cancel; genlmsg_end(skb, hdr); return 0; + +out_cancel: + genlmsg_cancel(skb, hdr); + return -ENOBUFS; } /** @@ -1502,18 +1521,28 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, for (i = 0; i < nn->nfsd_serv->sv_nrpools; i++) { struct svc_rqst *rqstp; + long thread_skip = 0; if (i < cb->args[0]) /* already consumed */ continue; + /* + * The saved thread index only applies to the pool the dump + * was resumed in. Subsequent pools must start from thread 0, + * otherwise their first cb->args[1] threads are silently + * skipped. + */ + if (i == cb->args[0]) + thread_skip = cb->args[1]; + rqstp_index = 0; list_for_each_entry_rcu(rqstp, &nn->nfsd_serv->sv_pools[i].sp_all_threads, rq_all) { - struct nfsd_genl_rqstp genl_rqstp; + struct nfsd_genl_rqstp genl_rqstp = {}; unsigned int status_counter; - if (rqstp_index++ < cb->args[1]) /* already consumed */ + if (rqstp_index++ < thread_skip) /* already consumed */ continue; /* * Acquire rq_status_counter before parsing the rqst @@ -1534,9 +1563,9 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, genl_rqstp.rq_stime = rqstp->rq_stime; genl_rqstp.rq_opcnt = 0; memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp), - sizeof(struct sockaddr)); + sizeof(struct sockaddr_storage)); #ifdef CONFIG_NFSD_V4 if (rqstp->rq_vers == NFS4_VERSION && @@ -1555,17 +1584,26 @@ int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb, #endif /* CONFIG_NFSD_V4 */ /* - * Acquire rq_status_counter before reporting the rqst - * fields to the user. + * Read-side load-load fence: order the field reads + * above before the counter re-read below, mirroring + * the smp_rmb() in the standard seqcount retry. The + * begin-side smp_load_acquire() above pairs with the + * smp_store_release() in nfsd_dispatch(). */ - if (smp_load_acquire(&rqstp->rq_status_counter) != - status_counter) + smp_rmb(); + if (READ_ONCE(rqstp->rq_status_counter) != status_counter) continue; ret = nfsd_genl_rpc_status_compose_msg(skb, cb, &genl_rqstp); - if (ret) + if (ret) { + if (skb->len) { + cb->args[0] = i; + cb->args[1] = rqstp_index - 1; + ret = skb->len; + } goto out; + } } } @@ -1875,6 +1913,60 @@ int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info) return err; } +/** + * nfsd_nl_validate_listeners - sanity-check the listener list from userland + * @info: netlink metadata and command arguments + * + * Walk every NFSD_A_SERVER_SOCK_ADDR attribute and confirm that each entry + * is well-formed: it parses against the policy, carries both an address and + * a transport name, and the address is long enough for its family. Doing + * this up front lets the callers below assume every entry is valid and + * guarantees we make no changes when the request is malformed. + * + * Return: 0 if every entry is valid, or a negative errno otherwise. + */ +static int nfsd_nl_validate_listeners(struct genl_info *info) +{ + const struct nlattr *attr; + int rem; + + nlmsg_for_each_attr_type(attr, NFSD_A_SERVER_SOCK_ADDR, info->nlhdr, + GENL_HDRLEN, rem) { + struct nlattr *tb[NFSD_A_SOCK_MAX + 1]; + struct sockaddr *sa; + int err; + + err = nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, + nfsd_sock_nl_policy, info->extack); + if (err < 0) + return err; + + if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) + return -EINVAL; + + sa = nla_data(tb[NFSD_A_SOCK_ADDR]); + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(sa->sa_family)) + return -EINVAL; + + switch (sa->sa_family) { + case AF_INET: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in)) + return -EINVAL; + break; + case AF_INET6: + if (nla_len(tb[NFSD_A_SOCK_ADDR]) < + sizeof(struct sockaddr_in6)) + return -EINVAL; + break; + default: + return -EAFNOSUPPORT; + } + } + + return 0; +} + /** * nfsd_nl_listener_set_doit - set the nfs running sockets * @skb: reply buffer @@ -1893,6 +1985,15 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) bool delete = false; int err, rem; + /* + * Validate the entire listener list before making any changes, so a + * malformed request fails cleanly without creating a serv or touching + * the existing listeners. + */ + err = nfsd_nl_validate_listeners(info); + if (err) + return err; + mutex_lock(&nfsd_mutex); err = nfsd_create_serv(net); @@ -1919,16 +2020,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) const char *xcl_name; struct sockaddr *sa; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -1980,16 +2076,11 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info) struct sockaddr *sa; int ret; + /* validated up front in nfsd_nl_validate_listeners() */ if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr, nfsd_sock_nl_policy, info->extack) < 0) continue; - if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME]) - continue; - - if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa)) - continue; - xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]); sa = nla_data(tb[NFSD_A_SOCK_ADDR]); @@ -2250,11 +2341,12 @@ static int __init init_nfsd(void) { int retval; - nfsd_debugfs_init(); - retval = nfsd4_init_slabs(); if (retval) return retval; + + nfsd_debugfs_init(); + retval = nfsd4_init_pnfs(); if (retval) goto out_free_slabs; @@ -2299,8 +2391,8 @@ static int __init init_nfsd(void) out_free_pnfs: nfsd4_exit_pnfs(); out_free_slabs: - nfsd4_free_slabs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); return retval; } @@ -2315,9 +2407,9 @@ static void __exit exit_nfsd(void) unregister_pernet_subsys(&nfsd_net_ops); nfsd_drc_slab_free(); nfsd_lockd_shutdown(); - nfsd4_free_slabs(); nfsd4_exit_pnfs(); nfsd_debugfs_exit(); + nfsd4_free_slabs(); } MODULE_AUTHOR("Olaf Kirch "); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 8cecccdaf9ee5..02026ab030d28 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -60,21 +60,6 @@ struct readdir_cd { /* Maximum number of operations per session compound */ #define NFSD_MAX_OPS_PER_COMPOUND 200 -struct nfsd_genl_rqstp { - struct sockaddr rq_daddr; - struct sockaddr rq_saddr; - unsigned long rq_flags; - ktime_t rq_stime; - __be32 rq_xid; - u32 rq_vers; - u32 rq_prog; - u32 rq_proc; - - /* NFSv4 compound */ - u32 rq_opcnt; - u32 rq_opnum[16]; -}; - extern struct svc_program nfsd_programs[]; extern const struct svc_version nfsd_version2, nfsd_version3, nfsd_version4; extern struct mutex nfsd_mutex; diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c index ed85dd43da18e..3ab4b1b6e2358 100644 --- a/fs/nfsd/nfsfh.c +++ b/fs/nfsd/nfsfh.c @@ -69,10 +69,8 @@ nfsd_mode_check(struct dentry *dentry, umode_t requested) if (requested == 0) /* the caller doesn't care */ return nfs_ok; if (mode == requested) { - if (mode == S_IFDIR && !d_can_lookup(dentry)) { - WARN_ON_ONCE(1); + if (mode == S_IFDIR && !d_can_lookup(dentry)) return nfserr_notdir; - } return nfs_ok; } if (mode == S_IFLNK) { @@ -279,15 +277,19 @@ static __be32 nfsd_set_fh_dentry(struct svc_rqst *rqstp, struct net *net, if (dentry->d_sb->s_export_op->flags & EXPORT_OP_NOWCC) fhp->fh_no_wcc = true; fhp->fh_64bit_cookies = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } break; case NFS_FHSIZE: fhp->fh_no_wcc = true; if (EX_WGATHER(exp)) fhp->fh_use_wgather = true; - if (exp->ex_flags & NFSEXP_V4ROOT) + if (exp->ex_flags & NFSEXP_V4ROOT) { + dput(dentry); goto out; + } } fhp->fh_dentry = dentry; diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c index 906a672578900..35f409b655523 100644 --- a/fs/nfsd/nfsproc.c +++ b/fs/nfsd/nfsproc.c @@ -82,6 +82,7 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) .na_iattr = iap, }; struct svc_fh *fhp; + int hosterr; dprintk("nfsd: SETATTR %s, valid=%x, size=%ld\n", SVCFH_fmt(&argp->fh), @@ -117,6 +118,12 @@ nfsd_proc_setattr(struct svc_rqst *rqstp) if (resp->status != nfs_ok) goto out; + hosterr = fh_want_write(fhp); + if (hosterr) { + resp->status = nfserrno(hosterr); + goto out; + } + if (delta < 0) delta = -delta; if (delta < MAX_TOUCH_TIME_ERROR && diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index fcb47f344e32f..242b8c7b6e0f1 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -835,7 +835,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, ret->mismatch.lovers = NFSD_ACL_NRVERS; for (i = NFSD_ACL_MINVERS; i < NFSD_ACL_NRVERS; i++) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.lovers = i; break; @@ -845,7 +845,7 @@ nfsd_acl_init_request(struct svc_rqst *rqstp, return rpc_prog_unavail; ret->mismatch.hivers = NFSD_ACL_MINVERS; for (i = NFSD_ACL_NRVERS - 1; i >= NFSD_ACL_MINVERS; i--) { - if (nfsd_support_acl_version(rqstp->rq_vers) && + if (nfsd_support_acl_version(i) && nfsd_vers(nn, i, NFSD_TEST)) { ret->mismatch.hivers = i; break; diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c index fc262ceafca97..e7e309594f844 100644 --- a/fs/nfsd/nfsxdr.c +++ b/fs/nfsd/nfsxdr.c @@ -9,6 +9,16 @@ #include "xdr.h" #include "auth.h" +/* + * Sun convention: a sattr time-useconds field of one full second (an + * otherwise out-of-range value) means "set this time to the current + * server time." It's needed to make permissions checks for the "touch" + * program across NFSv2 mounts work correctly. See description of + * sattr in section 6.1 of "NFS Illustrated" by Brent Callaghan, + * Addison-Wesley, ISBN 0-201-32750-5 + */ +#define NFS2_SATTR_SET_TO_SERVER_TIME (1000000) + /* * Mapping of S_IF* types to NFS file types */ @@ -172,27 +182,29 @@ svcxdr_decode_sattr(struct svc_rqst *rqstp, struct xdr_stream *xdr, tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + /* + * Range test here to prevent the multiplication from + * wrapping to a valid (but incorrect) value on 32-bit + * platforms. + */ + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_ATIME | ATTR_ATIME_SET; iap->ia_atime.tv_sec = tmp1; iap->ia_atime.tv_nsec = tmp2 * NSEC_PER_USEC; + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) + iap->ia_valid &= ~ATTR_ATIME_SET; } tmp1 = be32_to_cpup(p++); tmp2 = be32_to_cpup(p++); if (tmp1 != (u32)-1 && tmp2 != (u32)-1) { + if (tmp2 > NFS2_SATTR_SET_TO_SERVER_TIME) + return false; iap->ia_valid |= ATTR_MTIME | ATTR_MTIME_SET; iap->ia_mtime.tv_sec = tmp1; iap->ia_mtime.tv_nsec = tmp2 * NSEC_PER_USEC; - /* - * Passing the invalid value useconds=1000000 for mtime - * is a Sun convention for "set both mtime and atime to - * current server time". It's needed to make permissions - * checks for the "touch" program across v2 mounts to - * Solaris and Irix boxes work correctly. See description of - * sattr in section 6.1 of "NFS Illustrated" by - * Brent Callaghan, Addison-Wesley, ISBN 0-201-32750-5 - */ - if (tmp2 == 1000000) + if (tmp2 == NFS2_SATTR_SET_TO_SERVER_TIME) iap->ia_valid &= ~(ATTR_ATIME_SET|ATTR_MTIME_SET); } diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 2eed2c5b3cd0c..926e4e8063b33 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -123,7 +123,7 @@ struct nfs4_stid { #define SC_TYPE_LAYOUT BIT(3) unsigned short sc_type; -/* state_lock protects sc_status for delegation stateids. +/* nn->deleg_lock protects sc_status for delegation stateids. * ->cl_lock protects sc_status for open and lock stateids. * ->st_mutex also protect sc_status for open stateids. * ->ls_lock protects sc_status for layout stateids. @@ -383,6 +383,7 @@ struct nfsd4_session { u16 se_slot_gen; bool se_dead; u32 se_target_maxslots; + struct rcu_head rcu_head; }; /* formatted contents of nfs4_sessionid */ @@ -494,7 +495,7 @@ struct nfs4_client { #define NFSD4_CB_FAULT 3 int cl_cb_state; struct nfsd4_callback cl_cb_null; - struct nfsd4_session *cl_cb_session; + struct nfsd4_session __rcu *cl_cb_session; /* for all client information that callback code might need: */ spinlock_t cl_lock; diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 6e2c8e2aab10a..ba48d3963e9ff 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -236,7 +236,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify, TP_CONDITION(rqstp != NULL), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -275,7 +275,7 @@ TRACE_EVENT_CONDITION(nfsd_fh_verify_err, TP_CONDITION(rqstp != NULL && error), TP_STRUCT__entry( __field(unsigned int, netns_ino) - __sockaddr(server, rqstp->rq_xprt->xpt_remotelen) + __sockaddr(server, rqstp->rq_xprt->xpt_locallen) __sockaddr(client, rqstp->rq_xprt->xpt_remotelen) __field(u32, xid) __field(u32, fh_hash) @@ -1688,9 +1688,10 @@ DEFINE_NFSD_CB_LIFETIME_EVENT(bc_shutdown); TRACE_EVENT(nfsd_cb_seq_status, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1702,8 +1703,6 @@ TRACE_EVENT(nfsd_cb_seq_status, __field(int, seq_status) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; @@ -1729,9 +1728,10 @@ TRACE_EVENT(nfsd_cb_seq_status, TRACE_EVENT(nfsd_cb_free_slot, TP_PROTO( const struct rpc_task *task, - const struct nfsd4_callback *cb + const struct nfsd4_callback *cb, + const struct nfsd4_session *session ), - TP_ARGS(task, cb), + TP_ARGS(task, cb, session), TP_STRUCT__entry( __field(unsigned int, task_id) __field(unsigned int, client_id) @@ -1742,8 +1742,6 @@ TRACE_EVENT(nfsd_cb_free_slot, __field(u32, slot_seqno) ), TP_fast_assign( - const struct nfs4_client *clp = cb->cb_clp; - const struct nfsd4_session *session = clp->cl_cb_session; const struct nfsd4_sessionid *sid = (struct nfsd4_sessionid *)&session->se_sessionid; diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 2033037238780..53b55ee4e00ee 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -136,8 +136,10 @@ nfsd_cross_mnt(struct svc_rqst *rqstp, struct dentry **dpp, follow_flags = LOOKUP_AUTOMOUNT; err = follow_down(&path, follow_flags); - if (err < 0) + if (err < 0) { + path_put(&path); goto out; + } if (path.mnt == exp->ex_path.mnt && path.dentry == dentry && nfsd_mountpoint(dentry, exp) == 2) { /* This is only a mountpoint in some other namespace */ @@ -418,21 +420,22 @@ nfsd_sanitize_attrs(struct inode *inode, struct iattr *iap) } static __be32 -nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct iattr *iap) +nfsd_may_truncate(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct iattr *iap) { struct inode *inode = d_inode(fhp->fh_dentry); - if (iap->ia_size < inode->i_size) { - __be32 err; + if (iap->ia_size >= i_size_read(inode)) + return nfs_ok; - err = nfsd_permission(&rqstp->rq_cred, - fhp->fh_export, fhp->fh_dentry, - NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); - if (err) - return err; - } - return nfserrno(get_write_access(inode)); + return nfsd_permission(&rqstp->rq_cred, fhp->fh_export, fhp->fh_dentry, + NFSD_MAY_TRUNC | NFSD_MAY_OWNER_OVERRIDE); +} + +static __be32 +nfsd_get_write_access(struct svc_fh *fhp) +{ + return nfserrno(get_write_access(d_inode(fhp->fh_dentry))); } static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap) @@ -559,12 +562,17 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp, * setattr call. */ if (size_change) { - err = nfsd_get_write_access(rqstp, fhp, iap); + err = nfsd_get_write_access(fhp); if (err) return err; } inode_lock(inode); + if (size_change) { + err = nfsd_may_truncate(rqstp, fhp, iap); + if (err) + goto out_unlock; + } err = fh_fill_pre_attrs(fhp); if (err) goto out_unlock; @@ -1078,7 +1086,7 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, * nfsd_iter_read - Perform a VFS read using an iterator * @rqstp: RPC transaction context * @fhp: file handle of file to be read - * @file: opened struct file of file to be read + * @nf: opened struct nfsd_file of file to be read * @offset: starting byte offset * @count: IN: requested number of bytes; OUT: number of bytes read * @base: offset in first page of read buffer @@ -1091,9 +1099,10 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, * returned. */ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct file *file, loff_t offset, unsigned long *count, + struct nfsd_file *nf, loff_t offset, unsigned long *count, unsigned int base, u32 *eof) { + struct file *file = nf->nf_file; unsigned long v, total; struct iov_iter iter; struct kiocb kiocb; @@ -1202,7 +1211,7 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long exp_op_flags = 0; unsigned int pflags = current->flags; bool restore_flags = false; - unsigned int nvecs; + int nvecs; trace_nfsd_write_opened(rqstp, fhp, offset, *cnt); @@ -1242,7 +1251,13 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, } nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload); + if (nvecs < 0) { + host_err = nvecs; + goto out_nfserr; + } + iov_iter_bvec(&iter, ITER_SOURCE, rqstp->rq_bvec, nvecs, *cnt); + since = READ_ONCE(file->f_wb_err); if (verf) nfsd_copy_write_verifier(verf, nn); @@ -1346,7 +1361,7 @@ __be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp, if (file->f_op->splice_read && nfsd_read_splice_ok(rqstp)) err = nfsd_splice_read(rqstp, fhp, file, offset, count, eof); else - err = nfsd_iter_read(rqstp, fhp, file, offset, count, 0, eof); + err = nfsd_iter_read(rqstp, fhp, nf, offset, count, 0, eof); nfsd_file_put(nf); trace_nfsd_read_done(rqstp, fhp, offset, *count); diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 9652c1de53ec1..1dd3ae3ceb3a8 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -122,7 +122,7 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long *count, u32 *eof); __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct file *file, loff_t offset, + struct nfsd_file *nf, loff_t offset, unsigned long *count, unsigned int base, u32 *eof); bool nfsd_read_splice_ok(struct svc_rqst *rqstp); diff --git a/fs/nilfs2/bmap.c b/fs/nilfs2/bmap.c index ccc1a7aa52d20..efa01f5b35df6 100644 --- a/fs/nilfs2/bmap.c +++ b/fs/nilfs2/bmap.c @@ -175,7 +175,7 @@ static int nilfs_bmap_do_delete(struct nilfs_bmap *bmap, __u64 key) return ret; } - return bmap->b_ops->bop_delete(bmap, key); + return bmap->b_ops->bop_delete(bmap, key, false); } /** diff --git a/fs/nilfs2/bmap.h b/fs/nilfs2/bmap.h index 4656df3927228..a72f3c308a5d2 100644 --- a/fs/nilfs2/bmap.h +++ b/fs/nilfs2/bmap.h @@ -63,7 +63,7 @@ struct nilfs_bmap_operations { int (*bop_lookup_contig)(const struct nilfs_bmap *, __u64, __u64 *, unsigned int); int (*bop_insert)(struct nilfs_bmap *, __u64, __u64); - int (*bop_delete)(struct nilfs_bmap *, __u64); + int (*bop_delete)(struct nilfs_bmap *bmap, __u64 key, bool deform); void (*bop_clear)(struct nilfs_bmap *); int (*bop_propagate)(struct nilfs_bmap *, struct buffer_head *); diff --git a/fs/nilfs2/btree.c b/fs/nilfs2/btree.c index dd0c8e560ef6a..a47ee0ad57f7f 100644 --- a/fs/nilfs2/btree.c +++ b/fs/nilfs2/btree.c @@ -1425,6 +1425,28 @@ static void nilfs_btree_shrink(struct nilfs_bmap *btree, path[level].bp_bh = NULL; } +/** + * nilfs_btree_discard - discard the last node for the mapping transformation + * @btree: bmap struct of btree + * @path: array of nilfs_btree_path struct + * @level: level of the B-tree node being operated on + * @keyp: argument for passing a key (unused) + * @ptrp: argument for passing a pointer (unused) + */ +static void nilfs_btree_discard(struct nilfs_bmap *btree, + struct nilfs_btree_path *path, int level, + __u64 *keyp, __u64 *ptrp) +{ + struct nilfs_btree_node *root = nilfs_btree_get_root(btree); + + nilfs_btree_node_delete(root, 0, NULL, NULL, + NILFS_BTREE_ROOT_NCHILDREN_MAX); + nilfs_btree_node_set_level(root, level); + + nilfs_btnode_delete(path[level].bp_bh); + path[level].bp_bh = NULL; +} + static void nilfs_btree_nop(struct nilfs_bmap *btree, struct nilfs_btree_path *path, int level, __u64 *keyp, __u64 *ptrp) @@ -1435,7 +1457,7 @@ static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree, struct nilfs_btree_path *path, int *levelp, struct nilfs_bmap_stats *stats, - struct inode *dat) + struct inode *dat, bool deform) { struct buffer_head *bh; struct nilfs_btree_node *node, *parent, *sib; @@ -1522,15 +1544,17 @@ static int nilfs_btree_prepare_delete(struct nilfs_bmap *btree, if (nilfs_btree_node_get_nchildren(node) - 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) { path[level].bp_op = nilfs_btree_shrink; - stats->bs_nblocks += 2; - level++; - path[level].bp_op = nilfs_btree_nop; - goto shrink_root_child; + } else if (deform) { + path[level].bp_op = nilfs_btree_discard; } else { path[level].bp_op = nilfs_btree_do_delete; stats->bs_nblocks++; goto out; } + stats->bs_nblocks += 2; + level++; + path[level].bp_op = nilfs_btree_nop; + goto shrink_root_child; } } @@ -1581,7 +1605,7 @@ static void nilfs_btree_commit_delete(struct nilfs_bmap *btree, nilfs_bmap_set_dirty(btree); } -static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key) +static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key, bool deform) { struct nilfs_btree_path *path; @@ -1601,7 +1625,8 @@ static int nilfs_btree_delete(struct nilfs_bmap *btree, __u64 key) dat = NILFS_BMAP_USE_VBN(btree) ? nilfs_bmap_get_dat(btree) : NULL; - ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat); + ret = nilfs_btree_prepare_delete(btree, path, &level, &stats, dat, + deform); if (ret < 0) goto out; nilfs_btree_commit_delete(btree, path, level, dat); diff --git a/fs/nilfs2/direct.c b/fs/nilfs2/direct.c index 2d8dc6b35b547..834ce02f2afb2 100644 --- a/fs/nilfs2/direct.c +++ b/fs/nilfs2/direct.c @@ -144,7 +144,7 @@ static int nilfs_direct_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr) return ret; } -static int nilfs_direct_delete(struct nilfs_bmap *bmap, __u64 key) +static int nilfs_direct_delete(struct nilfs_bmap *bmap, __u64 key, bool deform) { union nilfs_bmap_ptr_req req; struct inode *dat; @@ -234,7 +234,7 @@ int nilfs_direct_delete_and_convert(struct nilfs_bmap *bmap, /* no need to allocate any resource for conversion */ /* delete */ - ret = bmap->b_ops->bop_delete(bmap, key); + ret = bmap->b_ops->bop_delete(bmap, key, true); if (ret < 0) return ret; diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index 7fa02146f1e06..913e1a472c8fe 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -527,6 +527,7 @@ static int nilfs_ioctl_get_bdescs(struct inode *inode, struct file *filp, * Return: 0 on success, or one of the following negative error codes on * failure: * * %-EEXIST - Block conflict detected. + * * %-EINVAL - Invalid virtual block descriptor. * * %-EIO - I/O error. * * %-ENOENT - Requested block doesn't exist. * * %-ENOMEM - Insufficient memory available. @@ -536,15 +537,30 @@ static int nilfs_ioctl_move_inode_block(struct inode *inode, struct list_head *buffers) { struct buffer_head *bh; + __u64 limit_blkidx = (__u64)inode->i_sb->s_maxbytes >> inode->i_blkbits; int ret; - if (vdesc->vd_flags == 0) + /* + * vblocknr 0 is reserved as an invalid pointer. Also, limit_blkidx + * ensures that the page index converted from vd_vblocknr never + * overflows the page cache limit and respects the architecture's bmap + * key width. + */ + if (unlikely(vdesc->vd_vblocknr == 0 || + vdesc->vd_vblocknr >= limit_blkidx)) + return -EINVAL; + + if (vdesc->vd_flags == 0) { + if (unlikely(vdesc->vd_offset >= limit_blkidx)) + return -EINVAL; + ret = nilfs_gccache_submit_read_data( inode, vdesc->vd_offset, vdesc->vd_blocknr, vdesc->vd_vblocknr, &bh); - else + } else { ret = nilfs_gccache_submit_read_node( inode, vdesc->vd_blocknr, vdesc->vd_vblocknr, &bh); + } if (unlikely(ret < 0)) { if (ret == -ENOENT) diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 56c4da417b6a1..89180e2f59fef 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c @@ -243,6 +243,7 @@ static void nilfs_copy_folio(struct folio *dst, struct folio *src, int nilfs_copy_dirty_pages(struct address_space *dmap, struct address_space *smap) { + struct inode *smap_inode = smap->host; struct folio_batch fbatch; unsigned int i; pgoff_t index = 0; @@ -258,8 +259,19 @@ int nilfs_copy_dirty_pages(struct address_space *dmap, struct folio *folio = fbatch.folios[i], *dfolio; folio_lock(folio); - if (unlikely(!folio_test_dirty(folio))) - NILFS_FOLIO_BUG(folio, "inconsistent dirty state"); + if (unlikely(!folio_test_dirty(folio))) { + if (WARN_ONCE(!sb_rdonly(smap_inode->i_sb), + "inconsistent dirty state\n")) + goto unlock_folio; + + /* + * If the filesystem has been forced to read-only + * due to metadata corruption. + */ + folio_unlock(folio); + err = -EROFS; + break; + } dfolio = filemap_grab_folio(dmap, folio->index); if (IS_ERR(dfolio)) { @@ -277,6 +289,7 @@ int nilfs_copy_dirty_pages(struct address_space *dmap, folio_unlock(dfolio); folio_put(dfolio); +unlock_folio: folio_unlock(folio); } folio_batch_release(&fbatch); diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 0bc1f0f02f31d..c20340f4e0dc3 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2561,6 +2561,10 @@ int nilfs_clean_segments(struct super_block *sb, struct nilfs_argv *argv, break; nilfs_warn(sb, "error %d cleaning segments", err); + + if (unlikely(err == -EROFS)) + goto out_unlock; + set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(sci->sc_interval); } diff --git a/fs/nilfs2/the_nilfs.c b/fs/nilfs2/the_nilfs.c index d0bcf744c553a..9911a484398a0 100644 --- a/fs/nilfs2/the_nilfs.c +++ b/fs/nilfs2/the_nilfs.c @@ -461,6 +461,12 @@ static int nilfs_store_disk_layout(struct the_nilfs *nilfs, nilfs->ns_inode_size); return -EINVAL; } + if (NILFS_SR_BYTES(nilfs->ns_inode_size) > nilfs->ns_blocksize) { + nilfs_err(nilfs->ns_sb, + "too large inode size for super root: %d bytes", + nilfs->ns_inode_size); + return -EINVAL; + } nilfs->ns_first_ino = le32_to_cpu(sbp->s_first_ino); if (nilfs->ns_first_ino < NILFS_USER_INO) { diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c index bfe884d624e7b..cdb69c23e2e6c 100644 --- a/fs/notify/fanotify/fanotify.c +++ b/fs/notify/fanotify/fanotify.c @@ -601,9 +601,9 @@ static struct fanotify_event *fanotify_alloc_perm_event(const void *data, pevent->hdr.pad = 0; pevent->hdr.len = 0; pevent->state = FAN_EVENT_INIT; + pevent->watchdog_cnt = 0; pevent->path = *path; - /* NULL ppos means no range info */ - pevent->ppos = range ? &range->pos : NULL; + pevent->pos = range ? range->pos : FANOTIFY_NO_RANGE; pevent->count = range ? range->count : 0; path_get(path); diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h index 39e60218df7ce..eb2ba71d5c81a 100644 --- a/fs/notify/fanotify/fanotify.h +++ b/fs/notify/fanotify/fanotify.h @@ -427,6 +427,8 @@ FANOTIFY_ME(struct fanotify_event *event) return container_of(event, struct fanotify_mnt_event, fae); } +#define FANOTIFY_NO_RANGE ((loff_t)-1) + /* * Structure for permission fanotify events. It gets allocated and freed in * fanotify_handle_event() since we wait there for user response. When the @@ -437,7 +439,7 @@ FANOTIFY_ME(struct fanotify_event *event) struct fanotify_perm_event { struct fanotify_event fae; struct path path; - const loff_t *ppos; /* optional file range info */ + loff_t pos; /* FANOTIFY_NO_RANGE if unavailable */ size_t count; u32 response; /* userspace answer to the event */ unsigned short state; /* state of the event */ @@ -467,7 +469,7 @@ static inline bool fanotify_event_has_access_range(struct fanotify_event *event) if (!(event->mask & FANOTIFY_PRE_CONTENT_EVENTS)) return false; - return FANOTIFY_PERM(event)->ppos; + return FANOTIFY_PERM(event)->pos != FANOTIFY_NO_RANGE; } static inline struct fanotify_event *FANOTIFY_E(struct fsnotify_event *fse) diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index 5c30f4d8c7c16..629407c3e8737 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -111,7 +111,12 @@ static DECLARE_DELAYED_WORK(perm_group_work, perm_group_watchdog); static void perm_group_watchdog_schedule(void) { - schedule_delayed_work(&perm_group_work, secs_to_jiffies(perm_group_timeout)); + int timeout = READ_ONCE(perm_group_timeout); + + if (!timeout) + return; + + schedule_delayed_work(&perm_group_work, secs_to_jiffies(timeout)); } static void perm_group_watchdog(struct work_struct *work) @@ -674,12 +679,9 @@ static size_t copy_range_info_to_user(struct fanotify_event *event, if (WARN_ON_ONCE(info_len > count)) return -EFAULT; - if (WARN_ON_ONCE(!pevent->ppos)) - return -EINVAL; - info.hdr.info_type = FAN_EVENT_INFO_TYPE_RANGE; info.hdr.len = info_len; - info.offset = *(pevent->ppos); + info.offset = pevent->pos; info.count = pevent->count; if (copy_to_user(buf, &info, info_len)) @@ -1156,11 +1158,13 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar { struct fsnotify_group *group; struct fsnotify_event *fsn_event; + unsigned int info_mode; void __user *p; int ret = -ENOTTY; size_t send_len = 0; group = file->private_data; + info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES); p = (void __user *) arg; @@ -1168,7 +1172,8 @@ static long fanotify_ioctl(struct file *file, unsigned int cmd, unsigned long ar case FIONREAD: spin_lock(&group->notification_lock); list_for_each_entry(fsn_event, &group->notification_list, list) - send_len += FAN_EVENT_METADATA_LEN; + send_len += fanotify_event_len(info_mode, + FANOTIFY_E(fsn_event)); spin_unlock(&group->notification_lock); ret = put_user(send_len, (int __user *) p); break; @@ -1327,16 +1332,18 @@ static bool fanotify_mark_update_flags(struct fsnotify_mark *fsn_mark, static bool fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, __u32 mask, unsigned int fan_flags) { + __u32 old_mask; bool recalc; spin_lock(&fsn_mark->lock); - if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) + if (!(fan_flags & FANOTIFY_MARK_IGNORE_BITS)) { + old_mask = fsn_mark->mask; fsn_mark->mask |= mask; - else + recalc = old_mask != fsn_mark->mask; + } else { fsn_mark->ignore_mask |= mask; - - recalc = fsnotify_calc_mask(fsn_mark) & - ~fsnotify_conn_mask(fsn_mark->connector); + recalc = true; + } recalc |= fanotify_mark_update_flags(fsn_mark, fan_flags); spin_unlock(&fsn_mark->lock); diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 0d813c52ff9c3..6eaf0d382eec8 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -539,7 +539,6 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, { struct fsnotify_mark *fsn_mark; struct inotify_inode_mark *i_mark; - __u32 old_mask, new_mask; int replace = !(arg & IN_MASK_ADD); int create = (arg & IN_MASK_CREATE); int ret; @@ -555,27 +554,15 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); spin_lock(&fsn_mark->lock); - old_mask = fsn_mark->mask; if (replace) { fsn_mark->mask = 0; fsn_mark->flags &= ~INOTIFY_MARK_FLAGS; } fsn_mark->mask |= inotify_arg_to_mask(inode, arg); fsn_mark->flags |= inotify_arg_to_flags(arg); - new_mask = fsn_mark->mask; spin_unlock(&fsn_mark->lock); - if (old_mask != new_mask) { - /* more bits in old than in new? */ - int dropped = (old_mask & ~new_mask); - /* more bits in this fsn_mark than the inode's mask? */ - int do_inode = (new_mask & ~READ_ONCE(inode->i_fsnotify_mask)); - - /* update the inode with this new fsn_mark */ - if (dropped || do_inode) - fsnotify_recalc_mask(inode->i_fsnotify_marks); - - } + fsnotify_recalc_mask(fsn_mark->connector); /* return the wd */ ret = i_mark->wd; diff --git a/fs/nsfs.c b/fs/nsfs.c index e2f9a725883c7..4dee1e728569d 100644 --- a/fs/nsfs.c +++ b/fs/nsfs.c @@ -566,7 +566,7 @@ static struct dentry *nsfs_fh_to_dentry(struct super_block *sb, struct fid *fh, return ERR_PTR(-EOPNOTSUPP); } - if (owning_ns && !ns_capable(owning_ns, CAP_SYS_ADMIN)) { + if (owning_ns && !may_see_all_namespaces()) { ns->ops->put(ns); return ERR_PTR(-EPERM); } diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c index d41d02372c747..a68eff8cf1d66 100644 --- a/fs/ntfs3/dir.c +++ b/fs/ntfs3/dir.c @@ -304,7 +304,9 @@ static inline bool ntfs_dir_emit(struct ntfs_sb_info *sbi, if (sbi->options->nohidden && (fname->dup.fa & FILE_ATTRIBUTE_HIDDEN)) return true; - if (fname->name_len + sizeof(struct NTFS_DE) > le16_to_cpu(e->size)) + if (sizeof(struct NTFS_DE) + + offsetof(struct ATTR_FILE_NAME, name) + + fname->name_len * sizeof(short) > le16_to_cpu(e->size)) return true; name_len = ntfs_utf16_to_nls(sbi, fname->name, fname->name_len, name, diff --git a/fs/ntfs3/frecord.c b/fs/ntfs3/frecord.c index e904c5da91922..7aa37cf01df5a 100644 --- a/fs/ntfs3/frecord.c +++ b/fs/ntfs3/frecord.c @@ -766,10 +766,23 @@ int ni_create_attr_list(struct ntfs_inode *ni) rs = sbi->record_size; /* - * Skip estimating exact memory requirement. - * Looks like one record_size is always enough. + * Compute the exact size of the attribute list. Each attribute in the + * record yields one ATTR_LIST_ENTRY of le_size(name_len) bytes. The + * minimum on-disk attribute is SIZEOF_RESIDENT (0x18) bytes, but an + * unnamed one expands to le_size(0) (0x20) here, so a record crafted + * with many such attributes needs more than a single record_size; the + * previous fixed kzalloc(record_size) could therefore be overflowed by + * an attacker-controlled record. */ - le = kmalloc(al_aligned(rs), GFP_NOFS); + lsize = 0; + attr = NULL; + while ((attr = mi_enum_attr(ni, &ni->mi, attr))) + lsize += le_size(attr->name_len); + + if (!lsize) + return -EINVAL; + + le = kzalloc(al_aligned(lsize), GFP_NOFS); if (!le) return -ENOMEM; @@ -779,7 +792,6 @@ int ni_create_attr_list(struct ntfs_inode *ni) attr = NULL; nb = 0; free_b = 0; - attr = NULL; for (; (attr = mi_enum_attr(ni, &ni->mi, attr)); le = Add2Ptr(le, sz)) { sz = le_size(attr->name_len); @@ -2652,6 +2664,15 @@ int ni_read_frame(struct ntfs_inode *ni, u64 frame_vbo, struct page **pages, err = unc_size; else if (!unc_size || unc_size > frame_size) err = -EINVAL; + else if (unc_size < frame_size) { + /* + * Partial decompress: zero the [unc_size, frame_size) + * tail. decompress_lznt() leaves it untouched, so + * without this the freshly vmapped pages would expose + * uninitialized kernel memory to userspace. + */ + memset(frame_mem + unc_size, 0, frame_size - unc_size); + } } if (!err && valid_size < frame_vbo + frame_size) { size_t ok = valid_size - frame_vbo; diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c index a0503a3344237..ae15c2fc48646 100644 --- a/fs/ntfs3/fslog.c +++ b/fs/ntfs3/fslog.c @@ -647,6 +647,14 @@ static inline void *enum_rstbl(struct RESTART_TABLE *t, void *c) return NULL; } +/* + * dp_range_ok - true if [j, j + count) fits in a page_lcns[cap] array. + */ +static inline bool dp_range_ok(size_t j, u32 count, u32 cap) +{ + return j < cap && count <= cap - j; +} + /* * find_dp - Search for a @vcn in Dirty Page Table. */ @@ -789,6 +797,20 @@ static bool check_rstbl(const struct RESTART_TABLE *rt, size_t bytes) return true; } +static bool check_dp_table(const struct RESTART_TABLE *dptbl) +{ + u32 rsize = le16_to_cpu(dptbl->size); + struct DIR_PAGE_ENTRY *dp = NULL; + + while ((dp = enum_rstbl((struct RESTART_TABLE *)dptbl, dp))) { + if (struct_size(dp, page_lcns, le32_to_cpu(dp->lcns_follow)) > + rsize) + return false; + } + + return true; +} + /* * free_rsttbl_idx - Free a previously allocated index a Restart Table. */ @@ -853,6 +875,9 @@ static inline struct RESTART_TABLE *extend_rsttbl(struct RESTART_TABLE *tbl, u32 used = le16_to_cpu(tbl->used); struct RESTART_TABLE *rt; + if (used + add > U16_MAX) + return NULL; + rt = init_rsttbl(esize, used + add); if (!rt) return NULL; @@ -2274,7 +2299,15 @@ static int read_log_rec_buf(struct ntfs_log *log, */ for (;;) { bool usa_error; - u32 tail = log->page_size - off; + u32 tail; + + /* off comes from the on-disk restart area; bound it. */ + if (off > log->page_size) { + err = -EINVAL; + goto out; + } + + tail = log->page_size - off; if (tail >= data_len) tail = data_len; @@ -2608,11 +2641,12 @@ static int read_next_log_rec(struct ntfs_log *log, struct lcb *lcb, u64 *lsn) bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) { + const bool has_subnode = hdr_has_subnode(hdr); __le16 mask; u32 min_de, de_off, used, total; const struct NTFS_DE *e; - if (hdr_has_subnode(hdr)) { + if (has_subnode) { min_de = sizeof(struct NTFS_DE) + sizeof(u64); mask = NTFS_IE_HAS_SUBNODES; } else { @@ -2629,20 +2663,33 @@ bool check_index_header(const struct INDEX_HDR *hdr, size_t bytes) return false; } - e = Add2Ptr(hdr, de_off); + e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); for (;;) { u16 esize = le16_to_cpu(e->size); - struct NTFS_DE *next = Add2Ptr(e, esize); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; - if (esize < min_de || PtrOffset(hdr, next) > used || + if (!IS_ALIGNED(esize, 8) || esize < min_de || (e->flags & NTFS_IE_HAS_SUBNODES) != mask) { return false; } - if (de_is_last(e)) + if (size_add(de_off, esize) > used) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + break; + } + + data_size = esize - min_de; + if (key_size > data_size) + return false; - e = next; + de_off += esize; + e = (const struct NTFS_DE *)((const u8 *)hdr + de_off); } return true; @@ -4280,6 +4327,11 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) goto out; } + if (!check_dp_table(rt)) { + err = -EINVAL; + goto out; + } + dptbl = kmemdup(rt, t32, GFP_NOFS); if (!dptbl) { err = -ENOMEM; @@ -5072,6 +5124,13 @@ int log_replay(struct ntfs_inode *ni, bool *initialized) /* Shorten length by any Lcns which were deleted. */ saved_len = dlen; + if (!dp_range_ok(le64_to_cpu(lrh->target_vcn) - le64_to_cpu(dp->vcn), + le16_to_cpu(lrh->lcns_follow), + le32_to_cpu(dp->lcns_follow))) { + err = -EINVAL; + goto out; + } + for (i = le16_to_cpu(lrh->lcns_follow); i; i--) { size_t j; u32 alen, voff; diff --git a/fs/ntfs3/fsntfs.c b/fs/ntfs3/fsntfs.c index 83df92df1ee0c..5e3506815c279 100644 --- a/fs/ntfs3/fsntfs.c +++ b/fs/ntfs3/fsntfs.c @@ -2293,8 +2293,8 @@ int ntfs_reparse_init(struct ntfs_sb_info *sbi) goto out; } - root_r = resident_data(attr); - if (root_r->type != ATTR_ZERO || + root_r = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root_r || root_r->type != ATTR_ZERO || root_r->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; @@ -2331,8 +2331,8 @@ int ntfs_objid_init(struct ntfs_sb_info *sbi) goto out; } - root = resident_data(attr); - if (root->type != ATTR_ZERO || + root = resident_data_ex(attr, sizeof(struct INDEX_ROOT)); + if (!root || root->type != ATTR_ZERO || root->rule != NTFS_COLLATION_TYPE_UINTS) { err = -EINVAL; goto out; diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c index a9c5dcc23a5ce..edfcd28573ef9 100644 --- a/fs/ntfs3/index.c +++ b/fs/ntfs3/index.c @@ -613,16 +613,51 @@ static const struct NTFS_DE *hdr_insert_head(struct INDEX_HDR *hdr, */ static bool index_hdr_check(const struct INDEX_HDR *hdr, u32 bytes) { + const bool has_subnode = hdr_has_subnode(hdr); + const u16 min_size = sizeof(struct NTFS_DE) + + (has_subnode ? sizeof(u64) : 0); u32 end = le32_to_cpu(hdr->used); u32 tot = le32_to_cpu(hdr->total); u32 off = le32_to_cpu(hdr->de_off); + const struct NTFS_DE *e; if (!IS_ALIGNED(off, 8) || tot > bytes || end > tot || - size_add(off, sizeof(struct NTFS_DE)) > end) { + size_add(off, min_size) > end) { /* incorrect index buffer. */ return false; } + /* Ensure every key stays inside its entry before lookup walks it. */ + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + for (;;) { + u16 e_size = le16_to_cpu(e->size); + u16 key_size = le16_to_cpu(e->key_size); + u16 data_size; + + if (!IS_ALIGNED(e_size, 8) || e_size < min_size || + de_has_vcn(e) != has_subnode) { + /* incorrect index entry. */ + return false; + } + + if (size_add(off, e_size) > end) + return false; + + if (de_is_last(e)) { + if (key_size) + return false; + + break; + } + + data_size = e_size - min_size; + if (key_size > data_size) + return false; + + off += e_size; + e = (const struct NTFS_DE *)((const u8 *)hdr + off); + } + return true; } @@ -759,6 +794,10 @@ static struct NTFS_DE *hdr_find_e(const struct ntfs_index *indx, binary_search: e_key_len = le16_to_cpu(e->key_size); + /* Validate key_size fits within the entry data area. */ + if (e_key_len > le16_to_cpu(e->size) - sizeof(struct NTFS_DE)) + return NULL; + diff2 = (*cmp)(key, key_len, e + 1, e_key_len, ctx); if (diff2 > 0) { if (found) { diff --git a/fs/ntfs3/record.c b/fs/ntfs3/record.c index 714c7ecedca83..0f8ba76094fe1 100644 --- a/fs/ntfs3/record.c +++ b/fs/ntfs3/record.c @@ -202,7 +202,7 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, u32 used = le32_to_cpu(rec->used); u32 t32, off, asize, prev_type; u16 t16; - u64 data_size, alloc_size, tot_size; + u64 svcn, evcn, data_size, alloc_size, tot_size; if (!attr) { u32 total = le32_to_cpu(rec->total); @@ -310,10 +310,38 @@ struct ATTRIB *mi_enum_attr(struct ntfs_inode *ni, struct mft_inode *mi, if (t32 && le16_to_cpu(attr->name_off) + t32 > t16) goto out; - /* Check start/end vcn. */ - if (le64_to_cpu(attr->nres.svcn) > le64_to_cpu(attr->nres.evcn) + 1) + /* + * Check start/end vcn. svcn == 0 with evcn == -1 (U64_MAX) is the + * sentinel for an empty non-resident attribute (no allocated + * clusters) and must be accepted: "svcn > evcn + 1" tolerates it, + * since "(u64)-1 + 1" is 0 and "0 > 0" is false. + * + * For a non-empty attribute evcn is a cluster index and must lie + * within the volume (sbi->used.bitmap.nbits, set up in + * ntfs_init_from_boot() before any caller of mi_enum_attr() runs). + * Bounding evcn also prevents a malformed value close to U64_MAX + * from slipping through the near-wrap "evcn + 1" upper bound. + */ + svcn = le64_to_cpu(attr->nres.svcn); + evcn = le64_to_cpu(attr->nres.evcn); + if (svcn > evcn + 1) goto out; + if (is_attr_ext(attr)) { + /* sparsed/compressed attribute. */ +#ifdef CONFIG_NTFS3_64BIT_CLUSTER + /* No limits. */ +#else + /* Check evcn fits into 32 bits. */ + if (evcn != U64_MAX && evcn >= (1ull << 32)) + goto out; +#endif + } else { + /* Check out of volume for normal attribute. */ + if (evcn != U64_MAX && evcn >= mi->sbi->used.bitmap.nbits) + goto out; + } + data_size = le64_to_cpu(attr->nres.data_size); if (le64_to_cpu(attr->nres.valid_size) > data_size) goto out; diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c index 9a2e3d0efd998..66ad6d82bf5c5 100644 --- a/fs/ntfs3/super.c +++ b/fs/ntfs3/super.c @@ -57,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -918,7 +919,7 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, struct ntfs_sb_info *sbi = sb->s_fs_info; int err; u32 mb, gb, boot_sector_size, sct_per_clst, record_size; - u64 sectors, clusters, mlcn, mlcn2, dev_size0; + u64 sectors, clusters, mlcn, mlcn2, mft_pos, mft2_pos, dev_size0; struct NTFS_BOOT *boot; struct buffer_head *bh; struct MFT_REC *rec; @@ -987,7 +988,15 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, mlcn2 = le64_to_cpu(boot->mft2_clst); sectors = le64_to_cpu(boot->sectors_per_volume); - if (mlcn * sct_per_clst >= sectors || mlcn2 * sct_per_clst >= sectors) { + /* + * Convert mlcn/mlcn2 to sector positions before comparing with + * 'sectors'. All three are u64 values that come from the boot + * sector, so use check_mul_overflow() to keep a wraparound from + * silently bypassing the comparison. + */ + if (check_mul_overflow(mlcn, (u64)sct_per_clst, &mft_pos) || + check_mul_overflow(mlcn2, (u64)sct_per_clst, &mft2_pos) || + mft_pos >= sectors || mft2_pos >= sectors) { ntfs_err( sb, "%s: start of MFT 0x%llx (0x%llx) is out of volume 0x%llx.", @@ -1134,7 +1143,10 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, rec->total = cpu_to_le32(sbi->record_size); ((struct ATTRIB *)Add2Ptr(rec, ao))->type = ATTR_END; - sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE)); + if (!sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE))) { + err = -EINVAL; + goto out; + } sbi->block_mask = sb->s_blocksize - 1; sbi->blocks_per_cluster = sbi->cluster_size >> sb->s_blocksize_bits; @@ -1185,7 +1197,8 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size, /* * Try alternative boot (last sector) */ - sb_set_blocksize(sb, block_size); + if (!sb_set_blocksize(sb, block_size)) + return -EINVAL; hint = "Alternative boot"; dev_size = dev_size0; /* restore original size. */ goto read_boot; diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c index d719e1073dbb1..0dbb410d51dfe 100644 --- a/fs/ntfs3/xattr.c +++ b/fs/ntfs3/xattr.c @@ -146,26 +146,29 @@ static int ntfs_read_ea(struct ntfs_inode *ni, struct EA_FULL **ea, for (off = 0; off < size; off += ea_size) { const struct EA_FULL *ef = Add2Ptr(ea_p, off); u32 bytes = size - off; + size_t need; /* Check if we can use field ea->size. */ if (bytes < sizeof(ef->size)) goto out1; + /* Check if we can use fields ef->name_len and ef->elength. */ + if (bytes < offsetof(struct EA_FULL, name)) + goto out1; + + /* Size needed to hold this record's name and value. */ + need = struct_size(ef, name, + 1 + ef->name_len + le16_to_cpu(ef->elength)); + if (ef->size) { ea_size = le32_to_cpu(ef->size); - if (ea_size > bytes) + /* ef->size must fit the list and cover the record. */ + if (ea_size > bytes || ea_size < need) goto out1; continue; } - /* Check if we can use fields ef->name_len and ef->elength. */ - if (bytes < offsetof(struct EA_FULL, name)) - goto out1; - - ea_size = ALIGN(struct_size(ef, name, - 1 + ef->name_len + - le16_to_cpu(ef->elength)), - 4); + ea_size = ALIGN(need, 4); if (ea_size > bytes) goto out1; } @@ -867,7 +870,9 @@ static noinline int ntfs_setxattr(const struct xattr_handler *handler, if (!strcmp(name, SYSTEM_DOS_ATTRIB)) { if (sizeof(u8) != size) goto out; - new_fa = cpu_to_le32(*(u8 *)value); + /* system.dos_attrib only covers the low DOS attribute byte. */ + new_fa = (ni->std_fa & ~cpu_to_le32(0xff)) | + cpu_to_le32(*(u8 *)value); goto set_new_fa; } diff --git a/fs/ocfs2/acl.c b/fs/ocfs2/acl.c index 62464d194da3f..095ce96c2484a 100644 --- a/fs/ocfs2/acl.c +++ b/fs/ocfs2/acl.c @@ -109,8 +109,7 @@ static void *ocfs2_acl_to_xattr(const struct posix_acl *acl, size_t *size) return ocfs2_acl; } -static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, - int type, +static struct posix_acl *ocfs2_get_acl_nolock(struct inode *inode, int type, struct buffer_head *di_bh) { int name_index; @@ -348,63 +347,105 @@ int ocfs2_acl_chmod(struct inode *inode, struct buffer_head *bh) * Initialize the ACLs of a new inode. If parent directory has default ACL, * then clone to new inode. Called from ocfs2_mknod. */ -int ocfs2_init_acl(handle_t *handle, - struct inode *inode, - struct inode *dir, - struct buffer_head *di_bh, - struct buffer_head *dir_bh, - struct ocfs2_alloc_context *meta_ac, - struct ocfs2_alloc_context *data_ac) +void ocfs2_acl_init_release(struct ocfs2_acl_state *state) +{ + posix_acl_release(state->default_acl); + posix_acl_release(state->acl); + state->default_acl = NULL; + state->acl = NULL; +} + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state) { struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - struct posix_acl *acl = NULL; - int ret = 0, ret2; - umode_t mode; - - if (!S_ISLNK(inode->i_mode)) { - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl = ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, - dir_bh); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (IS_ERR(acl)) - return PTR_ERR(acl); + int ret = 0; + + state->default_acl = NULL; + state->acl = NULL; + state->mode = inode->i_mode; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + down_read(&OCFS2_I(dir)->ip_xattr_sem); + state->default_acl = + ocfs2_get_acl_nolock(dir, ACL_TYPE_DEFAULT, dir_bh); + up_read(&OCFS2_I(dir)->ip_xattr_sem); + if (IS_ERR(state->default_acl)) { + ret = PTR_ERR(state->default_acl); + state->default_acl = NULL; + return ret; } - if (!acl) { - mode = inode->i_mode & ~current_umask(); - ret = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret) { - mlog_errno(ret); + if (state->default_acl) { + state->acl = posix_acl_dup(state->default_acl); + if (!state->acl) { + ret = -ENOMEM; goto cleanup; } + ret = __posix_acl_create(&state->acl, GFP_NOFS, + &state->mode); + if (ret < 0) + goto cleanup; + if (ret == 0) { + posix_acl_release(state->acl); + state->acl = NULL; + } + if (!S_ISDIR(inode->i_mode)) { + posix_acl_release(state->default_acl); + state->default_acl = NULL; + } + } else { + state->mode &= ~current_umask(); } + } else { + state->mode &= ~current_umask(); } - if ((osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) && acl) { - if (S_ISDIR(inode->i_mode)) { + + return 0; +cleanup: + ocfs2_acl_init_release(state); + return ret; +} + +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state) +{ + struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); + int ret = 0; + + if (S_ISLNK(inode->i_mode)) + return 0; + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (S_ISDIR(inode->i_mode) && state->default_acl) { ret = ocfs2_set_acl(handle, inode, di_bh, - ACL_TYPE_DEFAULT, acl, - meta_ac, data_ac); + ACL_TYPE_DEFAULT, + state->default_acl, meta_ac, + data_ac); if (ret) - goto cleanup; + return ret; } - mode = inode->i_mode; - ret = __posix_acl_create(&acl, GFP_NOFS, &mode); - if (ret < 0) - return ret; + } - ret2 = ocfs2_acl_set_mode(inode, di_bh, handle, mode); - if (ret2) { - mlog_errno(ret2); - ret = ret2; - goto cleanup; - } - if (ret > 0) { - ret = ocfs2_set_acl(handle, inode, - di_bh, ACL_TYPE_ACCESS, - acl, meta_ac, data_ac); + ret = ocfs2_acl_set_mode(inode, di_bh, handle, state->mode); + if (ret) { + mlog_errno(ret); + return ret; + } + + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (state->acl) { + ret = ocfs2_set_acl(handle, inode, di_bh, + ACL_TYPE_ACCESS, state->acl, + meta_ac, data_ac); } } -cleanup: - posix_acl_release(acl); + return ret; } diff --git a/fs/ocfs2/acl.h b/fs/ocfs2/acl.h index 667c6f03fa606..a91f9ce278d6d 100644 --- a/fs/ocfs2/acl.h +++ b/fs/ocfs2/acl.h @@ -20,9 +20,20 @@ struct posix_acl *ocfs2_iop_get_acl(struct inode *inode, int type, bool rcu); int ocfs2_iop_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, struct posix_acl *acl, int type); extern int ocfs2_acl_chmod(struct inode *, struct buffer_head *); -extern int ocfs2_init_acl(handle_t *, struct inode *, struct inode *, - struct buffer_head *, struct buffer_head *, - struct ocfs2_alloc_context *, - struct ocfs2_alloc_context *); +struct ocfs2_acl_state { + struct posix_acl *default_acl; + struct posix_acl *acl; + umode_t mode; +}; + +int ocfs2_acl_init_prepare(struct inode *inode, struct inode *dir, + struct buffer_head *dir_bh, + struct ocfs2_acl_state *state); +void ocfs2_acl_init_release(struct ocfs2_acl_state *state); +int ocfs2_init_acl(handle_t *handle, struct inode *inode, + struct buffer_head *di_bh, + struct ocfs2_alloc_context *meta_ac, + struct ocfs2_alloc_context *data_ac, + struct ocfs2_acl_state *state); #endif /* OCFS2_ACL_H */ diff --git a/fs/ocfs2/cluster/heartbeat.c b/fs/ocfs2/cluster/heartbeat.c index 724350925aff3..38bf8fadb346e 100644 --- a/fs/ocfs2/cluster/heartbeat.c +++ b/fs/ocfs2/cluster/heartbeat.c @@ -43,6 +43,14 @@ static DECLARE_RWSEM(o2hb_callback_sem); * whenever any of the threads sees activity from the node in its region. */ static DEFINE_SPINLOCK(o2hb_live_lock); +/* + * Serializes region pin/unpin dependency management (o2hb_dependent_users + * and the o2nm_depend_item()/o2nm_undepend_item() calls). o2hb_region_pin() + * has to drop o2hb_live_lock across the sleeping o2nm_depend_item(), so the + * spinlock alone can no longer keep pin and unpin mutually exclusive; this + * mutex, taken outside o2hb_live_lock, does. + */ +static DEFINE_MUTEX(o2hb_dependency_mutex); static struct list_head o2hb_live_slots[O2NM_MAX_NODES]; static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; static LIST_HEAD(o2hb_node_events); @@ -138,7 +146,7 @@ static unsigned int o2hb_dependent_users; * In global heartbeat mode, we pin/unpin all o2hb regions. This solution * works for both file system and userdlm domains. */ -static int o2hb_region_pin(const char *region_uuid); +static int o2hb_region_pin(const char *region_uuid, bool from_callback); static void o2hb_region_unpin(const char *region_uuid); /* Only sets a new threshold if there are no active regions. @@ -203,6 +211,7 @@ struct o2hb_region { /* protected by the hr_callback_sem */ struct task_struct *hr_task; + u8 hr_node_num; unsigned int hr_blocks; unsigned long long hr_start_block; @@ -350,12 +359,12 @@ static void o2hb_disarm_timeout(struct o2hb_region *reg) cancel_delayed_work_sync(®->hr_nego_timeout_work); } -static int o2hb_send_nego_msg(int key, int type, u8 target) +static int o2hb_send_nego_msg(int key, int type, u8 target, u8 node_num) { struct o2hb_nego_msg msg; int status, ret; - msg.node_num = o2nm_this_node(); + msg.node_num = node_num; again: ret = o2net_send_message(type, key, &msg, sizeof(msg), target, &status); @@ -373,8 +382,10 @@ static void o2hb_nego_timeout(struct work_struct *work) unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; int master_node, i, ret; struct o2hb_region *reg; + u8 node_num; reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work); + node_num = reg->hr_node_num; /* don't negotiate timeout if last hb failed since it is very * possible io failed. Should let write timeout fence self. */ @@ -385,10 +396,10 @@ static void o2hb_nego_timeout(struct work_struct *work) /* lowest node as master node to make negotiate decision. */ master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES); - if (master_node == o2nm_this_node()) { + if (master_node == node_num) { if (!test_bit(master_node, reg->hr_nego_node_bitmap)) { printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg)); set_bit(master_node, reg->hr_nego_node_bitmap); } @@ -417,7 +428,7 @@ static void o2hb_nego_timeout(struct work_struct *work) mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i); ret = o2hb_send_nego_msg(reg->hr_key, - O2HB_NEGO_APPROVE_MSG, i); + O2HB_NEGO_APPROVE_MSG, i, node_num); if (ret) mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n", i, ret); @@ -425,10 +436,10 @@ static void o2hb_nego_timeout(struct work_struct *work) } else { /* negotiate timeout with master node. */ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n", - o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(®->hr_item), + node_num, O2HB_NEGO_TIMEOUT_MS / 1000, config_item_name(®->hr_item), reg_bdev(reg), master_node); ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG, - master_node); + master_node, node_num); if (ret) mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n", master_node, ret); @@ -601,7 +612,9 @@ static int o2hb_issue_node_write(struct o2hb_region *reg, o2hb_bio_wait_init(write_wc); - slot = o2nm_this_node(); + slot = reg->hr_node_num; + if (slot >= O2NM_MAX_NODES) + return -EINVAL; bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE | REQ_SYNC); @@ -670,8 +683,12 @@ static int o2hb_check_own_slot(struct o2hb_region *reg) struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; char *errstr; + u8 node_num = reg->hr_node_num; + + if (node_num >= O2NM_MAX_NODES) + return 0; - slot = ®->hr_slots[o2nm_this_node()]; + slot = ®->hr_slots[node_num]; /* Don't check on our 1st timestamp */ if (!slot->ds_last_time) return 0; @@ -712,7 +729,10 @@ static inline void o2hb_prepare_block(struct o2hb_region *reg, struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; - node_num = o2nm_this_node(); + node_num = reg->hr_node_num; + if (node_num >= O2NM_MAX_NODES) + return; + slot = ®->hr_slots[node_num]; hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block; @@ -1206,7 +1226,7 @@ static int o2hb_thread(void *data) set_user_nice(current, MIN_NICE); /* Pin node */ - ret = o2nm_depend_this_node(); + ret = o2nm_depend_node(reg->hr_node_num); if (ret) { mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret); reg->hr_node_deleted = 1; @@ -1215,7 +1235,8 @@ static int o2hb_thread(void *data) } while (!kthread_should_stop() && - !reg->hr_unclean_stop && !reg->hr_aborted_start) { + !reg->hr_unclean_stop && !reg->hr_aborted_start && + o2nm_this_node() == reg->hr_node_num) { /* We track the time spent inside * o2hb_do_disk_heartbeat so that we avoid more than * hr_timeout_ms between disk writes. On busy systems @@ -1264,7 +1285,7 @@ static int o2hb_thread(void *data) } /* Unpin node */ - o2nm_undepend_this_node(); + o2nm_undepend_node(reg->hr_node_num); mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n"); @@ -1456,13 +1477,38 @@ void o2hb_init(void) o2hb_debug_init(); } -/* if we're already in a callback then we're already serialized by the sem */ -static void o2hb_fill_node_map_from_callback(unsigned long *map, - unsigned int bits) +static void __o2hb_fill_node_map(unsigned long *map, unsigned int bits) { bitmap_copy(map, o2hb_live_node_bitmap, bits); } +void o2hb_callback_read_lock(void) +{ + down_read(&o2hb_callback_sem); +} + +void o2hb_callback_read_unlock(void) +{ + up_read(&o2hb_callback_sem); +} + +void o2hb_synchronize_callbacks(void) +{ + down_write(&o2hb_callback_sem); + up_write(&o2hb_callback_sem); +} + +/* + * Callers must already hold o2hb_callback_sem for read or write so the copy + * stays serialized with callback delivery. + */ +void o2hb_fill_node_map_locked(unsigned long *map, unsigned int bits) +{ + spin_lock(&o2hb_live_lock); + __o2hb_fill_node_map(map, bits); + spin_unlock(&o2hb_live_lock); +} + /* * get a map of all nodes that are heartbeating in any regions */ @@ -1470,11 +1516,9 @@ void o2hb_fill_node_map(unsigned long *map, unsigned int bits) { /* callers want to serialize this map and callbacks so that they * can trust that they don't miss nodes coming to the party */ - down_read(&o2hb_callback_sem); - spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(map, bits); - spin_unlock(&o2hb_live_lock); - up_read(&o2hb_callback_sem); + o2hb_callback_read_lock(); + o2hb_fill_node_map_locked(map, bits); + o2hb_callback_read_unlock(); } EXPORT_SYMBOL_GPL(o2hb_fill_node_map); @@ -1776,7 +1820,8 @@ static ssize_t o2hb_region_dev_store(struct config_item *item, /* We can't heartbeat without having had our node number * configured yet. */ - if (o2nm_this_node() == O2NM_MAX_NODES) + reg->hr_node_num = o2nm_this_node(); + if (reg->hr_node_num == O2NM_MAX_NODES) return -EINVAL; ret = kstrtol(p, 0, &fd); @@ -2009,6 +2054,7 @@ static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *g ret = -ENAMETOOLONG; goto free; } + reg->hr_node_num = O2NM_MAX_NODES; spin_lock(&o2hb_live_lock); reg->hr_region_num = 0; @@ -2115,6 +2161,7 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, * If global heartbeat active and there are dependent users, * pin all regions if quorum region count <= CUT_OFF */ + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); if (!o2hb_dependent_users) @@ -2122,10 +2169,11 @@ static void o2hb_heartbeat_group_drop_item(struct config_group *group, if (bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) - o2hb_region_pin(NULL); + o2hb_region_pin(NULL, true); unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); } static ssize_t o2hb_heartbeat_group_dead_threshold_show(struct config_item *item, @@ -2263,48 +2311,113 @@ EXPORT_SYMBOL_GPL(o2hb_setup_callback); * In local, we only pin the matching region. In global we pin all the active * regions. */ -static int o2hb_region_pin(const char *region_uuid) +static int o2hb_region_pin(const char *region_uuid, bool from_callback) { - int ret = 0, found = 0; - struct o2hb_region *reg; + int ret = 0, found; + struct o2hb_region *reg, *pinned; char *uuid; assert_spin_locked(&o2hb_live_lock); - list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { - if (reg->hr_item_dropped) - continue; + do { + found = 0; + pinned = NULL; - uuid = config_item_name(®->hr_item); + list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { + if (reg->hr_item_dropped) + continue; - /* local heartbeat */ - if (region_uuid) { - if (strcmp(region_uuid, uuid)) + uuid = config_item_name(®->hr_item); + + /* local heartbeat */ + if (region_uuid) { + if (strcmp(region_uuid, uuid)) + continue; + found = 1; + } + + if (reg->hr_item_pinned || reg->hr_item_dropped) { + if (found) + break; continue; - found = 1; + } + + /* + * Found a region that needs pinning. Take a reference + * so it stays alive while we drop the lock below. + */ + pinned = reg; + config_item_get(®->hr_item); + break; } - if (reg->hr_item_pinned || reg->hr_item_dropped) - goto skip_pin; + if (!pinned) + break; + + uuid = config_item_name(&pinned->hr_item); + + /* + * o2nm_depend_item() -> configfs_depend_item() can sleep (it + * takes the configfs root inode rwsem), so it must not run + * under o2hb_live_lock. Drop the lock across it; @pinned is + * kept alive by the reference taken above. The region list may + * change while unlocked, so we rescan from the top afterwards. + */ + spin_unlock(&o2hb_live_lock); /* Ignore ENOENT only for local hb (userdlm domain) */ - ret = o2nm_depend_item(®->hr_item); + if (from_callback) + ret = o2nm_depend_item_unlocked(&pinned->hr_item); + else + ret = o2nm_depend_item(&pinned->hr_item); + + spin_lock(&o2hb_live_lock); if (!ret) { - mlog(ML_CLUSTER, "Pin region %s\n", uuid); - reg->hr_item_pinned = 1; - } else { - if (ret == -ENOENT && found) - ret = 0; - else { - mlog(ML_ERROR, "Pin region %s fails with %d\n", - uuid, ret); + /* + * o2hb_live_lock was dropped across o2nm_depend_item(). + * o2hb_set_quorum_device() runs in the heartbeat thread + * without o2hb_dependency_mutex, so for global heartbeat + * it may have crossed O2HB_PIN_CUT_OFF and unpinned the + * regions while we slept. If that happened this pin is + * no longer wanted; undo it and stop rather than + * resurrecting it on the rescan below. + */ + if (!region_uuid && + bitmap_weight(o2hb_quorum_region_bitmap, + O2NM_MAX_REGIONS) > O2HB_PIN_CUT_OFF) { + o2nm_undepend_item(&pinned->hr_item); + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); break; } + mlog(ML_CLUSTER, "Pin region %s\n", uuid); + pinned->hr_item_pinned = 1; + } else if (ret == -ENOENT && (found || !region_uuid)) { + /* + * For local hb (found): ignore ENOENT from userdlm + * domains as before. For global hb (!region_uuid): + * the region may have been detached from configfs + * while the lock was dropped — skip it and continue + * pinning the remaining regions. + */ + ret = 0; + } else { + mlog(ML_ERROR, "Pin region %s fails with %d\n", + uuid, ret); } -skip_pin: - if (found) - break; - } + + /* + * config_item_put() may drop the last reference and run + * o2hb_region_release(), which also grabs o2hb_live_lock and + * can sleep, so it must happen with the lock released. + */ + spin_unlock(&o2hb_live_lock); + config_item_put(&pinned->hr_item); + spin_lock(&o2hb_live_lock); + + /* local hb pins a single matching region */ + } while (!ret && !region_uuid); return ret; } @@ -2349,12 +2462,13 @@ static int o2hb_region_inc_user(const char *region_uuid) { int ret = 0; + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); /* local heartbeat */ if (!o2hb_global_heartbeat_active()) { - ret = o2hb_region_pin(region_uuid); - goto unlock; + ret = o2hb_region_pin(region_uuid, false); + goto unlock; } /* @@ -2366,16 +2480,23 @@ static int o2hb_region_inc_user(const char *region_uuid) goto unlock; if (bitmap_weight(o2hb_quorum_region_bitmap, - O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) - ret = o2hb_region_pin(NULL); + O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) { + ret = o2hb_region_pin(NULL, false); + if (ret) { + o2hb_region_unpin(NULL); + o2hb_dependent_users--; + } + } unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); return ret; } static void o2hb_region_dec_user(const char *region_uuid) { + mutex_lock(&o2hb_dependency_mutex); spin_lock(&o2hb_live_lock); /* local heartbeat */ @@ -2394,6 +2515,7 @@ static void o2hb_region_dec_user(const char *region_uuid) unlock: spin_unlock(&o2hb_live_lock); + mutex_unlock(&o2hb_dependency_mutex); } int o2hb_register_callback(const char *region_uuid, @@ -2468,7 +2590,7 @@ int o2hb_check_node_heartbeating_no_sem(u8 node_num) unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; spin_lock(&o2hb_live_lock); - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + __o2hb_fill_node_map(testing_map, O2NM_MAX_NODES); spin_unlock(&o2hb_live_lock); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, @@ -2485,7 +2607,7 @@ int o2hb_check_node_heartbeating_from_callback(u8 node_num) { unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; - o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); + o2hb_fill_node_map_locked(testing_map, O2NM_MAX_NODES); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, "node (%u) does not have heartbeating enabled.\n", diff --git a/fs/ocfs2/cluster/heartbeat.h b/fs/ocfs2/cluster/heartbeat.h index 8ef8c1b9eeb76..2ca2b657583c0 100644 --- a/fs/ocfs2/cluster/heartbeat.h +++ b/fs/ocfs2/cluster/heartbeat.h @@ -58,6 +58,11 @@ int o2hb_register_callback(const char *region_uuid, struct o2hb_callback_func *hc); void o2hb_unregister_callback(const char *region_uuid, struct o2hb_callback_func *hc); +void o2hb_callback_read_lock(void); +void o2hb_callback_read_unlock(void); +void o2hb_synchronize_callbacks(void); +void o2hb_fill_node_map_locked(unsigned long *map, + unsigned int bits); void o2hb_fill_node_map(unsigned long *map, unsigned int bits); void o2hb_exit(void); diff --git a/fs/ocfs2/cluster/nodemanager.c b/fs/ocfs2/cluster/nodemanager.c index 2f61d39e4e509..b65fc6eeae6e0 100644 --- a/fs/ocfs2/cluster/nodemanager.c +++ b/fs/ocfs2/cluster/nodemanager.c @@ -325,6 +325,7 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, struct o2nm_node *node = to_o2nm_node(item); struct o2nm_cluster *cluster; unsigned long tmp; + bool starting = false; char *p = (char *)page; ssize_t ret; @@ -361,11 +362,13 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, ret = o2net_start_listening(node); if (ret) goto out; + starting = true; } if (!tmp && cluster->cl_has_local && cluster->cl_local_node == node->nd_num) { o2net_stop_listening(node); + cluster->cl_has_local = 0; cluster->cl_local_node = O2NM_INVALID_NODE_NUM; } @@ -373,6 +376,8 @@ static ssize_t o2nm_node_local_store(struct config_item *item, const char *page, if (node->nd_local) { cluster->cl_has_local = tmp; cluster->cl_local_node = node->nd_num; + if (starting) + o2net_complete_start_listening(node); } ret = count; @@ -776,17 +781,23 @@ int o2nm_depend_item(struct config_item *item) return configfs_depend_item(&o2nm_cluster_group.cs_subsys, item); } +int o2nm_depend_item_unlocked(struct config_item *item) +{ + return configfs_depend_item_unlocked(&o2nm_cluster_group.cs_subsys, + item); +} + void o2nm_undepend_item(struct config_item *item) { configfs_undepend_item(item); } -int o2nm_depend_this_node(void) +int o2nm_depend_node(u8 node_num) { int ret = 0; struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); if (!local_node) { ret = -EINVAL; goto out; @@ -799,17 +810,27 @@ int o2nm_depend_this_node(void) return ret; } -void o2nm_undepend_this_node(void) +void o2nm_undepend_node(u8 node_num) { struct o2nm_node *local_node; - local_node = o2nm_get_node_by_num(o2nm_this_node()); + local_node = o2nm_get_node_by_num(node_num); BUG_ON(!local_node); o2nm_undepend_item(&local_node->nd_item); o2nm_node_put(local_node); } +int o2nm_depend_this_node(void) +{ + return o2nm_depend_node(o2nm_this_node()); +} + +void o2nm_undepend_this_node(void) +{ + o2nm_undepend_node(o2nm_this_node()); +} + static void __exit exit_o2nm(void) { diff --git a/fs/ocfs2/cluster/nodemanager.h b/fs/ocfs2/cluster/nodemanager.h index 3490e77a952d5..ca3483fb54504 100644 --- a/fs/ocfs2/cluster/nodemanager.h +++ b/fs/ocfs2/cluster/nodemanager.h @@ -64,7 +64,10 @@ void o2nm_node_get(struct o2nm_node *node); void o2nm_node_put(struct o2nm_node *node); int o2nm_depend_item(struct config_item *item); +int o2nm_depend_item_unlocked(struct config_item *item); void o2nm_undepend_item(struct config_item *item); +int o2nm_depend_node(u8 node_num); +void o2nm_undepend_node(u8 node_num); int o2nm_depend_this_node(void); void o2nm_undepend_this_node(void); diff --git a/fs/ocfs2/cluster/tcp.c b/fs/ocfs2/cluster/tcp.c index b05d4e9d13b28..822553530e746 100644 --- a/fs/ocfs2/cluster/tcp.c +++ b/fs/ocfs2/cluster/tcp.c @@ -105,6 +105,8 @@ static struct socket *o2net_listen_sock; * destroying the work queue. */ static struct workqueue_struct *o2net_wq; +/* Heartbeat callbacks stay registered across local-node off/on. */ +static bool o2net_listening; static struct work_struct o2net_listen_work; static struct o2hb_callback_func o2net_hb_up, o2net_hb_down; @@ -1692,6 +1694,19 @@ static void o2net_still_up(struct work_struct *work) /* ------------------------------------------------------------ */ +static void o2net_hb_node_up(struct o2net_node *nn) +{ + /* ensure an immediate connect attempt */ + nn->nn_last_connect_attempt = jiffies - + (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + + spin_lock(&nn->nn_lock); + atomic_set(&nn->nn_timeout, 0); + if (nn->nn_persistent_error) + o2net_set_nn_state(nn, NULL, 0, 0); + spin_unlock(&nn->nn_lock); +} + void o2net_disconnect_node(struct o2nm_node *node) { struct o2net_node *nn = o2net_nn_from_num(node->nd_num); @@ -1713,41 +1728,39 @@ void o2net_disconnect_node(struct o2nm_node *node) static void o2net_hb_node_down_cb(struct o2nm_node *node, int node_num, void *data) { + u8 this_node; + o2quo_hb_down(node_num); if (!node) - return; + goto out; - if (node_num != o2nm_this_node()) + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + goto out; + + if (node_num != this_node) o2net_disconnect_node(node); +out: BUG_ON(atomic_read(&o2net_connected_peers) < 0); } static void o2net_hb_node_up_cb(struct o2nm_node *node, int node_num, void *data) { - struct o2net_node *nn = o2net_nn_from_num(node_num); + u8 this_node; o2quo_hb_up(node_num); BUG_ON(!node); - /* ensure an immediate connect attempt */ - nn->nn_last_connect_attempt = jiffies - - (msecs_to_jiffies(o2net_reconnect_delay()) + 1); + this_node = o2nm_this_node(); + if (!READ_ONCE(o2net_listening) || this_node == O2NM_MAX_NODES) + return; - if (node_num != o2nm_this_node()) { - /* believe it or not, accept and node heartbeating testing - * can succeed for this node before we got here.. so - * only use set_nn_state to clear the persistent error - * if that hasn't already happened */ - spin_lock(&nn->nn_lock); - atomic_set(&nn->nn_timeout, 0); - if (nn->nn_persistent_error) - o2net_set_nn_state(nn, NULL, 0, 0); - spin_unlock(&nn->nn_lock); - } + if (node_num != this_node) + o2net_hb_node_up(o2net_nn_from_num(node_num)); } void o2net_unregister_hb_callbacks(void) @@ -1756,6 +1769,37 @@ void o2net_unregister_hb_callbacks(void) o2hb_unregister_callback(NULL, &o2net_hb_down); } +/* + * Delay heartbeat-driven network work until the local node is fully published + * through o2nm_this_node(), then replay the nodes that are already live while + * callback delivery stays blocked. + */ +void o2net_complete_start_listening(struct o2nm_node *node) +{ + unsigned long live_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; + unsigned long node_num; + u8 local_node; + + local_node = o2nm_this_node(); + if (WARN_ON_ONCE(local_node == O2NM_MAX_NODES)) + return; + if (WARN_ON_ONCE(local_node != node->nd_num)) + return; + if (WARN_ON_ONCE(!o2net_wq)) + return; + + o2hb_callback_read_lock(); + WRITE_ONCE(o2net_listening, true); + o2hb_fill_node_map_locked(live_nodes, O2NM_MAX_NODES); + for_each_set_bit(node_num, live_nodes, O2NM_MAX_NODES) { + if (node_num == local_node) + continue; + + o2net_hb_node_up(o2net_nn_from_num(node_num)); + } + o2hb_callback_read_unlock(); +} + int o2net_register_hb_callbacks(void) { int ret; @@ -2034,6 +2078,8 @@ int o2net_start_listening(struct o2nm_node *node) { int ret = 0; + if (WARN_ON_ONCE(READ_ONCE(o2net_listening))) + return -EBUSY; BUG_ON(o2net_wq != NULL); BUG_ON(o2net_listen_sock != NULL); @@ -2065,6 +2111,9 @@ void o2net_stop_listening(struct o2nm_node *node) BUG_ON(o2net_wq == NULL); BUG_ON(o2net_listen_sock == NULL); + WRITE_ONCE(o2net_listening, false); + o2hb_synchronize_callbacks(); + /* stop the listening socket from generating work */ write_lock_bh(&sock->sk->sk_callback_lock); sock->sk->sk_data_ready = sock->sk->sk_user_data; diff --git a/fs/ocfs2/cluster/tcp.h b/fs/ocfs2/cluster/tcp.h index a75b551d31c7b..2e86d42b5faf9 100644 --- a/fs/ocfs2/cluster/tcp.h +++ b/fs/ocfs2/cluster/tcp.h @@ -96,6 +96,7 @@ struct o2nm_node; int o2net_register_hb_callbacks(void); void o2net_unregister_hb_callbacks(void); int o2net_start_listening(struct o2nm_node *node); +void o2net_complete_start_listening(struct o2nm_node *node); void o2net_stop_listening(struct o2nm_node *node); void o2net_disconnect_node(struct o2nm_node *node); int o2net_num_connected_peers(void); diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c index 8c9c4825f9841..fbfce8cd131aa 100644 --- a/fs/ocfs2/dir.c +++ b/fs/ocfs2/dir.c @@ -580,7 +580,7 @@ static int ocfs2_validate_dx_root(struct super_block *sb, mlog(ML_ERROR, "Checksum failed for dir index root block %llu\n", (unsigned long long)bh->b_blocknr); - return ret; + goto bail; } if (!OCFS2_IS_VALID_DX_ROOT(dx_root)) { @@ -588,8 +588,54 @@ static int ocfs2_validate_dx_root(struct super_block *sb, "Dir Index Root # %llu has bad signature %.*s\n", (unsigned long long)le64_to_cpu(dx_root->dr_blkno), 7, dx_root->dr_signature); + goto bail; } + if (!(dx_root->dr_flags & OCFS2_DX_FLAG_INLINE)) { + struct ocfs2_extent_list *el = &dx_root->dr_list; + + if (le16_to_cpu(el->l_count) != ocfs2_extent_recs_per_dx_root(sb)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid l_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(el->l_count), + ocfs2_extent_recs_per_dx_root(sb)); + goto bail; + } + + if (le16_to_cpu(el->l_next_free_rec) > le16_to_cpu(el->l_count)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid l_next_free_rec %u (l_count %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(el->l_next_free_rec), + le16_to_cpu(el->l_count)); + goto bail; + } + } else { + struct ocfs2_dx_entry_list *dl_list = &dx_root->dr_entries; + + if (le16_to_cpu(dl_list->de_count) != + ocfs2_dx_entries_per_root(sb)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_count), + ocfs2_dx_entries_per_root(sb)); + goto bail; + } + + if (le16_to_cpu(dl_list->de_num_used) > + le16_to_cpu(dl_list->de_count)) { + ret = ocfs2_error(sb, + "Dir Index Root # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_root->dr_blkno), + le16_to_cpu(dl_list->de_num_used), + le16_to_cpu(dl_list->de_count)); + goto bail; + } + } + +bail: return ret; } @@ -626,10 +672,25 @@ static int ocfs2_validate_dx_leaf(struct super_block *sb, return ret; } - if (!OCFS2_IS_VALID_DX_LEAF(dx_leaf)) { - ret = ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n", - 7, dx_leaf->dl_signature); - } + if (!OCFS2_IS_VALID_DX_LEAF(dx_leaf)) + return ocfs2_error(sb, "Dir Index Leaf has bad signature %.*s\n", + 7, dx_leaf->dl_signature); + + if (le16_to_cpu(dx_leaf->dl_list.de_count) != + ocfs2_dx_entries_per_leaf(sb)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_count %u (expected %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_count), + ocfs2_dx_entries_per_leaf(sb)); + + if (le16_to_cpu(dx_leaf->dl_list.de_num_used) > + le16_to_cpu(dx_leaf->dl_list.de_count)) + return ocfs2_error(sb, + "Dir Index Leaf # %llu has invalid de_num_used %u (de_count %u)\n", + (unsigned long long)le64_to_cpu(dx_leaf->dl_blkno), + le16_to_cpu(dx_leaf->dl_list.de_num_used), + le16_to_cpu(dx_leaf->dl_list.de_count)); return ret; } @@ -1888,7 +1949,7 @@ static int ocfs2_dir_foreach_blk_el(struct inode *inode, i += le16_to_cpu(de->rec_len); } offset = i; - ctx->pos = (ctx->pos & ~(sb->s_blocksize - 1)) + ctx->pos = (ctx->pos & ~((loff_t)sb->s_blocksize - 1)) | offset; *f_version = inode_query_iversion(inode); } diff --git a/fs/ocfs2/dlm/dlmmaster.c b/fs/ocfs2/dlm/dlmmaster.c index 4145e06d2c080..4ab8c48582b15 100644 --- a/fs/ocfs2/dlm/dlmmaster.c +++ b/fs/ocfs2/dlm/dlmmaster.c @@ -3101,6 +3101,12 @@ int dlm_migrate_request_handler(struct o2net_msg *msg, u32 len, void *data, name = migrate->name; namelen = migrate->namelen; + if (namelen > DLM_LOCKID_NAME_MAX) { + mlog(ML_ERROR, "%s: invalid name length %u in migrate request\n", + dlm->name, namelen); + ret = -EINVAL; + goto leave; + } hash = dlm_lockid_hash(name, namelen); /* preallocate.. if this fails, abort */ diff --git a/fs/ocfs2/dlm/dlmrecovery.c b/fs/ocfs2/dlm/dlmrecovery.c index 843ee02bd85fd..2379225f0ee69 100644 --- a/fs/ocfs2/dlm/dlmrecovery.c +++ b/fs/ocfs2/dlm/dlmrecovery.c @@ -1357,6 +1357,15 @@ int dlm_mig_lockres_handler(struct o2net_msg *msg, u32 len, void *data, if (!dlm_grab(dlm)) return -EINVAL; + if (mres->lockname_len > DLM_LOCKID_NAME_MAX || + mres->num_locks > DLM_MAX_MIGRATABLE_LOCKS || + be16_to_cpu(msg->data_len) < struct_size(mres, ml, mres->num_locks)) { + mlog(ML_ERROR, "%s: invalid lockres migration message from %u\n", + dlm->name, mres->master); + dlm_put(dlm); + return -EINVAL; + } + if (!dlm_joined(dlm)) { mlog(ML_ERROR, "Domain %s not joined! " "lockres %.*s, master %u\n", diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index d247d69349e4a..9ec3a7657b5b6 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -1538,6 +1538,22 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if ((le32_to_cpu(di->i_flags) & OCFS2_ORPHANED_FL) && + le16_to_cpu(di->i_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_orphaned_slot)); + goto bail; + } + + if ((le32_to_cpu(di->i_flags) & OCFS2_DIO_ORPHANED_FL) && + le16_to_cpu(di->i_dio_orphaned_slot) >= OCFS2_SB(sb)->max_slots) { + rc = ocfs2_error(sb, "Invalid dinode %llu: DIO orphaned slot %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_dio_orphaned_slot)); + goto bail; + } + /* * Reject dinodes whose i_mode does not name one of the seven * canonical POSIX file types. ocfs2_populate_inode() copies diff --git a/fs/ocfs2/namei.c b/fs/ocfs2/namei.c index c90b254da75eb..20276ad6e17e2 100644 --- a/fs/ocfs2/namei.c +++ b/fs/ocfs2/namei.c @@ -255,6 +255,7 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, sigset_t oldset; int did_block_signals = 0; struct ocfs2_dentry_lock *dl = NULL; + struct ocfs2_acl_state acl_state = { 0 }; trace_ocfs2_mknod(dir, dentry, dentry->d_name.len, dentry->d_name.name, (unsigned long long)OCFS2_I(dir)->ip_blkno, @@ -329,10 +330,14 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, } } + status = ocfs2_acl_init_prepare(inode, dir, parent_fe_bh, &acl_state); + if (status < 0) + goto leave; + /* calculate meta data/clusters for setting security and acl xattr */ - status = ocfs2_calc_xattr_init(dir, parent_fe_bh, mode, - &si, &want_clusters, - &xattr_credits, &want_meta); + status = ocfs2_calc_xattr_init(dir, mode, &si, &want_clusters, + &xattr_credits, &want_meta, + &acl_state); if (status < 0) { mlog_errno(status); goto leave; @@ -410,8 +415,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, inc_nlink(dir); } - status = ocfs2_init_acl(handle, inode, dir, new_fe_bh, parent_fe_bh, - meta_ac, data_ac); + status = ocfs2_init_acl(handle, inode, new_fe_bh, meta_ac, data_ac, + &acl_state); if (status < 0) { mlog_errno(status); @@ -476,6 +481,8 @@ static int ocfs2_mknod(struct mnt_idmap *idmap, brelse(parent_fe_bh); kfree(si.value); + ocfs2_acl_init_release(&acl_state); + ocfs2_free_dir_lookup_result(&lookup); if (inode_ac) diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 267b50e8e42e5..54a030e6779da 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -116,6 +116,33 @@ static int ocfs2_validate_refcount_block(struct super_block *sb, le32_to_cpu(rb->rf_fs_generation)); goto out; } + + /* + * rf_records (rl_count/rl_used/rl_recs[]) is only meaningful when + * this block is not an interior tree block (OCFS2_REFCOUNT_TREE_FL); + * in that case the same union bytes hold an extent list (rf_list) + * instead, which is validated by ocfs2_validate_extent_block(). + */ + if (!(le32_to_cpu(rb->rf_flags) & OCFS2_REFCOUNT_TREE_FL)) { + if (le16_to_cpu(rb->rf_records.rl_count) != + ocfs2_refcount_recs_per_rb(sb)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_count of %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + + if (le16_to_cpu(rb->rf_records.rl_used) > + le16_to_cpu(rb->rf_records.rl_count)) { + rc = ocfs2_error(sb, + "Refcount block #%llu has an invalid rl_used of %u (rl_count %u)\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(rb->rf_records.rl_used), + le16_to_cpu(rb->rf_records.rl_count)); + goto out; + } + } out: return rc; } @@ -3355,10 +3382,9 @@ static int ocfs2_replace_cow(struct ocfs2_cow_context *context) cow_start += num_clusters; } - if (ocfs2_dealloc_has_cluster(&context->dealloc)) { + if (ocfs2_dealloc_has_cluster(&context->dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &context->dealloc); - } + ocfs2_run_deallocs(osb, &context->dealloc); return ret; } @@ -3841,10 +3867,9 @@ static int ocfs2_attach_refcount_tree(struct inode *inode, ocfs2_unlock_refcount_tree(osb, ref_tree, 1); brelse(ref_root_bh); - if (!ret && ocfs2_dealloc_has_cluster(&dealloc)) { + if (!ret && ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); out: /* * Empty the extent map so that we may get the right extent @@ -4130,10 +4155,9 @@ static int ocfs2_create_reflink_node(struct inode *s_inode, ocfs2_unlock_refcount_tree(osb, ref_tree, 1); brelse(ref_root_bh); out: - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); return ret; } @@ -4686,10 +4710,9 @@ loff_t ocfs2_reflink_remap_blocks(struct inode *s_inode, } out: - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(osb, 1); - ocfs2_run_deallocs(osb, &dealloc); - } + ocfs2_run_deallocs(osb, &dealloc); return ret; } diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 8f7018bad283b..3a796d9159d8e 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -607,13 +607,10 @@ int ocfs2_calc_security_init(struct inode *dir, return ret; } -int ocfs2_calc_xattr_init(struct inode *dir, - struct buffer_head *dir_bh, - umode_t mode, +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, struct ocfs2_security_xattr_info *si, - int *want_clusters, - int *xattr_credits, - int *want_meta) + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state) { int ret = 0; struct ocfs2_super *osb = OCFS2_SB(dir->i_sb); @@ -624,19 +621,15 @@ int ocfs2_calc_xattr_init(struct inode *dir, si->value_len); if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { - down_read(&OCFS2_I(dir)->ip_xattr_sem); - acl_len = ocfs2_xattr_get_nolock(dir, dir_bh, - OCFS2_XATTR_INDEX_POSIX_ACL_DEFAULT, - "", NULL, 0); - up_read(&OCFS2_I(dir)->ip_xattr_sem); - if (acl_len > 0) { - a_size = ocfs2_xattr_entry_real_size(0, acl_len); - if (S_ISDIR(mode)) - a_size <<= 1; - } else if (acl_len != 0 && acl_len != -ENODATA) { - ret = acl_len; - mlog_errno(ret); - return ret; + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + a_size += ocfs2_xattr_entry_real_size(0, acl_len); } } @@ -679,14 +672,33 @@ int ocfs2_calc_xattr_init(struct inode *dir, new_clusters); *want_clusters += new_clusters; } - if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL && - acl_len > OCFS2_XATTR_INLINE_SIZE) { - /* for directory, it has DEFAULT and ACCESS two types of acls */ - new_clusters = (S_ISDIR(mode) ? 2 : 1) * - ocfs2_clusters_for_bytes(dir->i_sb, acl_len); - *xattr_credits += ocfs2_clusters_to_blocks(dir->i_sb, - new_clusters); - *want_clusters += new_clusters; + if (osb->s_mount_opt & OCFS2_MOUNT_POSIX_ACL) { + if (acl_state->default_acl && S_ISDIR(mode)) { + acl_len = acl_state->default_acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } + if (acl_state->acl) { + acl_len = acl_state->acl->a_count * + sizeof(struct ocfs2_acl_entry); + if (acl_len > OCFS2_XATTR_INLINE_SIZE) { + new_clusters = + ocfs2_clusters_for_bytes(dir->i_sb, + acl_len); + *xattr_credits += + ocfs2_clusters_to_blocks(dir->i_sb, + new_clusters); + *want_clusters += new_clusters; + } + } } return ret; @@ -736,12 +748,10 @@ static int ocfs2_xattr_extend_allocation(struct inode *inode, prev_clusters; if (why != RESTART_NONE && clusters_to_add) { - /* - * We can only fail in case the alloc file doesn't give - * up enough clusters. - */ - BUG_ON(why == RESTART_META); - + if (why == RESTART_META) { + status = -ENOSPC; + break; + } credits = ocfs2_calc_extend_credits(inode->i_sb, &vb->vb_xv->xr_list); status = ocfs2_extend_trans(handle, credits); @@ -3210,6 +3220,14 @@ static int ocfs2_calc_xattr_set_need(struct inode *inode, } else credits += OCFS2_SUBALLOC_ALLOC + 1; + /* + * Reserve metadata for the new xattr's value extent tree. + * The not_found path above adds credits for this tree but + * omits meta_add, leaving meta_ac NULL for large values. + */ + if (xi->xi_value_len > OCFS2_XATTR_INLINE_SIZE) + meta_add += ocfs2_extend_meta_needed(&def_xv.xv.xr_list); + /* * This cluster will be used either for new bucket or for * new xattr block. @@ -3439,9 +3457,10 @@ static int __ocfs2_xattr_set_handle(struct inode *inode, } /* - * This function only called duing creating inode - * for init security/acl xattrs of the new inode. - * All transanction credits have been reserved in mknod. + * This helper is only for setting initial ACL or security xattrs on an inode + * that is still unpublished, unhashed, and unattached to a dentry. + * Ordinary xattr updates must use ocfs2_xattr_set(). + * All transaction credits have been reserved in mknod or symlink callers. */ int ocfs2_xattr_set_handle(handle_t *handle, struct inode *inode, @@ -3498,8 +3517,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, xis.inode_bh = xbs.inode_bh = di_bh; di = (struct ocfs2_dinode *)di_bh->b_data; - down_write(&OCFS2_I(inode)->ip_xattr_sem); - ret = ocfs2_xattr_ibody_find(inode, name_index, name, &xis); if (ret) goto cleanup; @@ -3512,7 +3529,6 @@ int ocfs2_xattr_set_handle(handle_t *handle, ret = __ocfs2_xattr_set_handle(inode, di, &xi, &xis, &xbs, &ctxt); cleanup: - up_write(&OCFS2_I(inode)->ip_xattr_sem); brelse(xbs.xattr_bh); ocfs2_xattr_bucket_free(xbs.bucket); @@ -7181,10 +7197,9 @@ int ocfs2_reflink_xattrs(struct inode *old_inode, ref_tree, 1); brelse(ref_root_bh); - if (ocfs2_dealloc_has_cluster(&dealloc)) { + if (ocfs2_dealloc_has_cluster(&dealloc)) ocfs2_schedule_truncate_log_flush(OCFS2_SB(old_inode->i_sb), 1); - ocfs2_run_deallocs(OCFS2_SB(old_inode->i_sb), &dealloc); - } + ocfs2_run_deallocs(OCFS2_SB(old_inode->i_sb), &dealloc); out: return ret; @@ -7203,6 +7218,7 @@ int ocfs2_init_security_and_acl(struct inode *dir, { int ret = 0; struct buffer_head *dir_bh = NULL; + struct ocfs2_acl_state acl_state = { 0 }; ret = ocfs2_init_security_get(inode, dir, qstr, NULL); if (ret) { @@ -7215,10 +7231,17 @@ int ocfs2_init_security_and_acl(struct inode *dir, mlog_errno(ret); goto leave; } - ret = ocfs2_init_acl(NULL, inode, dir, NULL, dir_bh, NULL, NULL); + + ret = ocfs2_acl_init_prepare(inode, dir, dir_bh, &acl_state); + if (ret) + goto unlock; + + ret = ocfs2_init_acl(NULL, inode, NULL, NULL, NULL, &acl_state); if (ret) mlog_errno(ret); +unlock: + ocfs2_acl_init_release(&acl_state); ocfs2_inode_unlock(dir, 0); brelse(dir_bh); leave: diff --git a/fs/ocfs2/xattr.h b/fs/ocfs2/xattr.h index 65e9aa7439199..5cdd6c6b40064 100644 --- a/fs/ocfs2/xattr.h +++ b/fs/ocfs2/xattr.h @@ -55,9 +55,12 @@ int ocfs2_init_security_set(handle_t *, struct inode *, int ocfs2_calc_security_init(struct inode *, struct ocfs2_security_xattr_info *, int *, int *, struct ocfs2_alloc_context **); -int ocfs2_calc_xattr_init(struct inode *, struct buffer_head *, - umode_t, struct ocfs2_security_xattr_info *, - int *, int *, int *); + +struct ocfs2_acl_state; +int ocfs2_calc_xattr_init(struct inode *dir, umode_t mode, + struct ocfs2_security_xattr_info *si, + int *want_clusters, int *xattr_credits, + int *want_meta, struct ocfs2_acl_state *acl_state); /* * xattrs can live inside an inode, as part of an external xattr block, diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c index 31218d89b7580..14369efc70d81 100644 --- a/fs/omfs/inode.c +++ b/fs/omfs/inode.c @@ -477,7 +477,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_time_min = 0; sb->s_time_max = U64_MAX / MSEC_PER_SEC; - sb_set_blocksize(sb, 0x200); + if (!sb_set_blocksize(sb, 0x200)) + goto end; bh = sb_bread(sb, 0); if (!bh) @@ -529,7 +530,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc) * Use sys_blocksize as the fs block since it is smaller than a * page while the fs blocksize can be larger. */ - sb_set_blocksize(sb, sbi->s_sys_blocksize); + if (!sb_set_blocksize(sb, sbi->s_sys_blocksize)) + goto out_brelse_bh; /* * ...and the difference goes into a shift. sys_blocksize is always diff --git a/fs/orangefs/devorangefs-req.c b/fs/orangefs/devorangefs-req.c index 33ee8cb32f831..e5c60da7e6773 100644 --- a/fs/orangefs/devorangefs-req.c +++ b/fs/orangefs/devorangefs-req.c @@ -474,6 +474,7 @@ static ssize_t orangefs_devreq_write_iter(struct kiocb *iocb, op->downcall.trailer_size, iter)) { gossip_err("%s: failed to copy trailer.\n", __func__); vfree(op->downcall.trailer_buf); + op->downcall.trailer_buf = NULL; goto Efault; } diff --git a/fs/orangefs/orangefs-debugfs.c b/fs/orangefs/orangefs-debugfs.c index 79267b3419f2c..4b7e6c52d4a37 100644 --- a/fs/orangefs/orangefs-debugfs.c +++ b/fs/orangefs/orangefs-debugfs.c @@ -569,6 +569,7 @@ static int orangefs_prepare_cdm_array(char *debug_array_string) cds_delimiter = strchr(cds_head, '\n'); *cds_delimiter = '\0'; + cds_head = skip_spaces(cds_head); keyword_len = strcspn(cds_head, " "); cdm_array[i].keyword = kzalloc(keyword_len + 1, GFP_KERNEL); diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c index 97f373fd98797..89317e1e7361b 100644 --- a/fs/overlayfs/copy_up.c +++ b/fs/overlayfs/copy_up.c @@ -864,7 +864,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) { struct ovl_fs *ofs = OVL_FS(c->dentry->d_sb); struct inode *udir = d_inode(c->destdir); - struct dentry *temp, *upper; + struct dentry *temp, *upper, *newdentry = NULL; struct file *tmpfile; struct ovl_cu_creds cc; int err; @@ -901,6 +901,14 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) err = PTR_ERR(upper); if (!IS_ERR(upper)) { err = ovl_do_link(ofs, temp, udir, upper); + if (!err) { + /* + * Record the linked dentry -- not the disconnected + * O_TMPFILE dentry -- so that ->d_revalidate() on + * the upper fs sees the real parent/name. + */ + newdentry = dget(upper); + } dput(upper); } inode_unlock(udir); @@ -916,7 +924,7 @@ static int ovl_copy_up_tmpfile(struct ovl_copy_up_ctx *c) if (!c->metacopy) ovl_set_upperdata(d_inode(c->dentry)); - ovl_inode_update(d_inode(c->dentry), dget(temp)); + ovl_inode_update(d_inode(c->dentry), newdentry); out: ovl_end_write(c->dentry); diff --git a/fs/overlayfs/dir.c b/fs/overlayfs/dir.c index e924321b64025..1abe533304ba4 100644 --- a/fs/overlayfs/dir.c +++ b/fs/overlayfs/dir.c @@ -191,7 +191,6 @@ struct dentry *ovl_create_real(struct ovl_fs *ofs, struct dentry *parent, if (!err && ofs->casefold != ovl_dentry_casefolded(newdentry)) { pr_warn_ratelimited("wrong inherited casefold (%pd2)\n", newdentry); - dput(newdentry); err = -EINVAL; } break; diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index 3fedfdddfa758..a857ce377b0cf 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -541,9 +541,9 @@ static loff_t ovl_copyfile(struct file *file_in, loff_t pos_in, struct file *file_out, loff_t pos_out, loff_t len, unsigned int flags, enum ovl_copyop op) { + struct inode *inode_in = file_inode(file_in); struct inode *inode_out = file_inode(file_out); struct file *realfile_in, *realfile_out; - const struct cred *old_cred; loff_t ret; inode_lock(inode_out); @@ -565,25 +565,38 @@ static loff_t ovl_copyfile(struct file *file_in, loff_t pos_in, if (IS_ERR(realfile_in)) goto out_unlock; - old_cred = ovl_override_creds(file_inode(file_out)->i_sb); - switch (op) { - case OVL_COPY: - ret = vfs_copy_file_range(realfile_in, pos_in, - realfile_out, pos_out, len, flags); - break; - - case OVL_CLONE: - ret = vfs_clone_file_range(realfile_in, pos_in, - realfile_out, pos_out, len, flags); - break; - - case OVL_DEDUPE: - ret = vfs_dedupe_file_range_one(realfile_in, pos_in, - realfile_out, pos_out, len, - flags); - break; + /* + * For cross-sb copy, vfs_copy_file_range() will verify read access with + * the mounter creds of the dest fs mounter, so we need to explicitly + * verify read access with the source mounter creds. + */ + if (unlikely(inode_in->i_sb != inode_out->i_sb)) { + with_ovl_creds(inode_in->i_sb) { + ret = rw_verify_area(READ, realfile_in, &pos_in, len); + if (unlikely(ret)) + goto out_unlock; + } + } + + with_ovl_creds(inode_out->i_sb) { + switch (op) { + case OVL_COPY: + ret = vfs_copy_file_range(realfile_in, pos_in, + realfile_out, pos_out, len, flags); + break; + + case OVL_CLONE: + ret = vfs_clone_file_range(realfile_in, pos_in, + realfile_out, pos_out, len, flags); + break; + + case OVL_DEDUPE: + ret = vfs_dedupe_file_range_one(realfile_in, pos_in, + realfile_out, pos_out, len, + flags); + break; + } } - ovl_revert_creds(old_cred); /* Update size */ ovl_file_modified(file_out); diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h index d1eb1cbe7a456..696b13d9e1f10 100644 --- a/fs/overlayfs/overlayfs.h +++ b/fs/overlayfs/overlayfs.h @@ -445,6 +445,11 @@ struct dentry *ovl_workdir(struct dentry *dentry); const struct cred *ovl_override_creds(struct super_block *sb); void ovl_revert_creds(const struct cred *old_cred); +EXTEND_CLASS(override_creds, _ovl, ovl_override_creds(sb), struct super_block *sb) + +#define with_ovl_creds(sb) \ + scoped_class(override_creds_ovl, __UNIQUE_ID(label), sb) + static inline const struct cred *ovl_creds(struct super_block *sb) { return OVL_FS(sb)->creator_cred; diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c index 0e971ea97580f..d727e56ef4b1f 100644 --- a/fs/overlayfs/super.c +++ b/fs/overlayfs/super.c @@ -1381,7 +1381,8 @@ int ovl_fill_super(struct super_block *sb, struct fs_context *fc) int err; err = -EIO; - if (WARN_ON(fc->user_ns != current_user_ns())) + /* The fscontext fd may have been passed to another user namespace. */ + if (fc->user_ns != current_user_ns()) goto out_err; ovl_set_d_op(sb); diff --git a/fs/overlayfs/xattrs.c b/fs/overlayfs/xattrs.c index 88055deca9360..c86f1a6c8b937 100644 --- a/fs/overlayfs/xattrs.c +++ b/fs/overlayfs/xattrs.c @@ -13,7 +13,7 @@ static bool ovl_is_escaped_xattr(struct super_block *sb, const char *name) OVL_XATTR_ESCAPE_USER_PREFIX_LEN) == 0; else return strncmp(name, OVL_XATTR_ESCAPE_TRUSTED_PREFIX, - OVL_XATTR_ESCAPE_TRUSTED_PREFIX_LEN - 1) == 0; + OVL_XATTR_ESCAPE_TRUSTED_PREFIX_LEN) == 0; } static bool ovl_is_own_xattr(struct super_block *sb, const char *name) diff --git a/fs/pidfs.c b/fs/pidfs.c index 34987fcdd9a87..4341492b91e12 100644 --- a/fs/pidfs.c +++ b/fs/pidfs.c @@ -456,7 +456,7 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct task_struct *task __free(put_task) = NULL; struct nsproxy *nsp __free(put_nsproxy) = NULL; struct ns_common *ns_common = NULL; - struct pid_namespace *pid_ns; + int error; if (!pidfs_ioctl_valid(cmd)) return -ENOIOCTLCMD; @@ -480,92 +480,126 @@ static long pidfd_ioctl(struct file *file, unsigned int cmd, unsigned long arg) if (arg) return -EINVAL; + /* + * We're trying to open a file descriptor to the namespace so perform a + * filesystem cred ptrace check. Hold @task's exec_update_lock for the + * duration of the ptrace check and the namespace lookup so that the + * credentials used for the access decision match those of @task at the + * time its namespace is read, preventing a concurrent execve() from + * swapping the task's credentials in between the check and the use. We + * mirror nsfs behavior. + */ + error = down_read_killable(&task->signal->exec_update_lock); + if (error) + return error; + + if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { + error = -EACCES; + goto out_unlock; + } + scoped_guard(task_lock, task) { nsp = task->nsproxy; if (nsp) get_nsproxy(nsp); } - if (!nsp) - return -ESRCH; /* just pretend it didn't exist */ - - /* - * We're trying to open a file descriptor to the namespace so perform a - * filesystem cred ptrace check. Also, we mirror nsfs behavior. - */ - if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) - return -EACCES; + if (!nsp) { + error = -ESRCH; /* just pretend it didn't exist */ + goto out_unlock; + } switch (cmd) { /* Namespaces that hang of nsproxy. */ case PIDFD_GET_CGROUP_NAMESPACE: - if (IS_ENABLED(CONFIG_CGROUPS)) { - get_cgroup_ns(nsp->cgroup_ns); - ns_common = to_ns_common(nsp->cgroup_ns); - } +#ifdef CONFIG_CGROUPS + if (!ns_ref_get(nsp->cgroup_ns)) + break; + ns_common = to_ns_common(nsp->cgroup_ns); +#endif break; case PIDFD_GET_IPC_NAMESPACE: - if (IS_ENABLED(CONFIG_IPC_NS)) { - get_ipc_ns(nsp->ipc_ns); - ns_common = to_ns_common(nsp->ipc_ns); - } +#ifdef CONFIG_IPC_NS + if (!ns_ref_get(nsp->ipc_ns)) + break; + ns_common = to_ns_common(nsp->ipc_ns); +#endif break; case PIDFD_GET_MNT_NAMESPACE: - get_mnt_ns(nsp->mnt_ns); + if (!ns_ref_get(nsp->mnt_ns)) + break; ns_common = to_ns_common(nsp->mnt_ns); break; case PIDFD_GET_NET_NAMESPACE: - if (IS_ENABLED(CONFIG_NET_NS)) { - ns_common = to_ns_common(nsp->net_ns); - get_net_ns(ns_common); - } +#ifdef CONFIG_NET_NS + if (!ns_ref_get(nsp->net_ns)) + break; + ns_common = to_ns_common(nsp->net_ns); +#endif break; case PIDFD_GET_PID_FOR_CHILDREN_NAMESPACE: - if (IS_ENABLED(CONFIG_PID_NS)) { - get_pid_ns(nsp->pid_ns_for_children); - ns_common = to_ns_common(nsp->pid_ns_for_children); - } +#ifdef CONFIG_PID_NS + if (!ns_ref_get(nsp->pid_ns_for_children)) + break; + ns_common = to_ns_common(nsp->pid_ns_for_children); +#endif break; case PIDFD_GET_TIME_NAMESPACE: - if (IS_ENABLED(CONFIG_TIME_NS)) { - get_time_ns(nsp->time_ns); - ns_common = to_ns_common(nsp->time_ns); - } +#ifdef CONFIG_TIME_NS + if (!ns_ref_get(nsp->time_ns)) + break; + ns_common = to_ns_common(nsp->time_ns); +#endif break; case PIDFD_GET_TIME_FOR_CHILDREN_NAMESPACE: - if (IS_ENABLED(CONFIG_TIME_NS)) { - get_time_ns(nsp->time_ns_for_children); - ns_common = to_ns_common(nsp->time_ns_for_children); - } +#ifdef CONFIG_TIME_NS + if (!ns_ref_get(nsp->time_ns_for_children)) + break; + ns_common = to_ns_common(nsp->time_ns_for_children); +#endif break; case PIDFD_GET_UTS_NAMESPACE: - if (IS_ENABLED(CONFIG_UTS_NS)) { - get_uts_ns(nsp->uts_ns); - ns_common = to_ns_common(nsp->uts_ns); - } +#ifdef CONFIG_UTS_NS + if (!ns_ref_get(nsp->uts_ns)) + break; + ns_common = to_ns_common(nsp->uts_ns); +#endif break; /* Namespaces that don't hang of nsproxy. */ case PIDFD_GET_USER_NAMESPACE: - if (IS_ENABLED(CONFIG_USER_NS)) { - rcu_read_lock(); - ns_common = to_ns_common(get_user_ns(task_cred_xxx(task, user_ns))); - rcu_read_unlock(); +#ifdef CONFIG_USER_NS + scoped_guard(rcu) { + struct user_namespace *user_ns; + + user_ns = task_cred_xxx(task, user_ns); + if (!ns_ref_get(user_ns)) + break; + ns_common = to_ns_common(user_ns); } +#endif break; case PIDFD_GET_PID_NAMESPACE: - if (IS_ENABLED(CONFIG_PID_NS)) { - rcu_read_lock(); +#ifdef CONFIG_PID_NS + scoped_guard(rcu) { + struct pid_namespace *pid_ns; + pid_ns = task_active_pid_ns(task); - if (pid_ns) - ns_common = to_ns_common(get_pid_ns(pid_ns)); - rcu_read_unlock(); + if (!ns_ref_get(pid_ns)) + break; + ns_common = to_ns_common(pid_ns); } +#endif break; default: - return -ENOIOCTLCMD; + error = -ENOIOCTLCMD; } - if (!ns_common) - return -EOPNOTSUPP; + if (!error && !ns_common) + error = -EOPNOTSUPP; + +out_unlock: + up_read(&task->signal->exec_update_lock); + if (error) + return error; /* open_namespace() unconditionally consumes the reference */ return open_namespace(ns_common); diff --git a/fs/posix_acl.c b/fs/posix_acl.c index 4050942ab52f9..0d92d6a631e1d 100644 --- a/fs/posix_acl.c +++ b/fs/posix_acl.c @@ -93,6 +93,13 @@ static void __forget_cached_acl(struct posix_acl **p) { struct posix_acl *old; + /* + * ACL_DONT_CACHE is expected to be a "const" value and xchg it with + * ACL_NOT_CACHED would enable acl caching for the inode - + * clearly not what the caller has intended. + */ + if (READ_ONCE(*p) == ACL_DONT_CACHE) + return; old = xchg(p, ACL_NOT_CACHED); if (!is_uncached_acl(old)) posix_acl_release(old); diff --git a/fs/proc/namespaces.c b/fs/proc/namespaces.c index 2f46f13967445..ea6ec61a0430b 100644 --- a/fs/proc/namespaces.c +++ b/fs/proc/namespaces.c @@ -46,7 +46,7 @@ static const char *proc_ns_get_link(struct dentry *dentry, const struct proc_ns_operations *ns_ops = PROC_I(inode)->ns_ops; struct task_struct *task; struct path ns_path; - int error = -EACCES; + int error; if (!dentry) return ERR_PTR(-ECHILD); @@ -59,6 +59,7 @@ static const char *proc_ns_get_link(struct dentry *dentry, if (error) goto out_put_task; + error = -EACCES; if (!ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) goto out; @@ -90,6 +91,7 @@ static int proc_ns_readlink(struct dentry *dentry, char __user *buffer, int bufl if (res) goto out_put_task; + res = -EACCES; if (ptrace_may_access(task, PTRACE_MODE_READ_FSCREDS)) { res = ns_get_name(name, sizeof(name), task, ns_ops); if (res >= 0) diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 38b9d47426ca9..5c3d966b70f26 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1073,10 +1073,12 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, bool present = false; struct folio *folio; + if (pmd_none(*pmd)) + return; if (pmd_present(*pmd)) { page = vm_normal_page_pmd(vma, addr, *pmd); present = true; - } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) { + } else if (unlikely(thp_migration_supported())) { swp_entry_t entry = pmd_to_swp_entry(*pmd); if (is_pfn_swap_entry(entry)) @@ -1989,90 +1991,102 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, return make_pme(frame, flags); } -static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, - struct mm_walk *walk) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, + unsigned long end, struct vm_area_struct *vma, + struct pagemapread *pm) { - struct vm_area_struct *vma = walk->vma; - struct pagemapread *pm = walk->private; - spinlock_t *ptl; - pte_t *pte, *orig_pte; + unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; + u64 flags = 0, frame = 0; + pmd_t pmd = *pmdp; + struct page *page = NULL; + struct folio *folio = NULL; int err = 0; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - ptl = pmd_trans_huge_lock(pmdp, vma); - if (ptl) { - unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; - u64 flags = 0, frame = 0; - pmd_t pmd = *pmdp; - struct page *page = NULL; - struct folio *folio = NULL; + if (vma->vm_flags & VM_SOFTDIRTY) + flags |= PM_SOFT_DIRTY; - if (vma->vm_flags & VM_SOFTDIRTY) - flags |= PM_SOFT_DIRTY; + if (pmd_none(pmd)) + goto populate_pagemap; - if (pmd_present(pmd)) { - page = pmd_page(pmd); + if (pmd_present(pmd)) { + page = pmd_page(pmd); - flags |= PM_PRESENT; - if (pmd_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; - if (pmd_uffd_wp(pmd)) - flags |= PM_UFFD_WP; - if (pm->show_pfn) - frame = pmd_pfn(pmd) + idx; - } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION - else if (is_swap_pmd(pmd)) { - swp_entry_t entry = pmd_to_swp_entry(pmd); - unsigned long offset; - - if (pm->show_pfn) { - if (is_pfn_swap_entry(entry)) - offset = swp_offset_pfn(entry) + idx; - else - offset = swp_offset(entry) + idx; - frame = swp_type(entry) | - (offset << MAX_SWAPFILES_SHIFT); - } - flags |= PM_SWAP; - if (pmd_swp_soft_dirty(pmd)) - flags |= PM_SOFT_DIRTY; - if (pmd_swp_uffd_wp(pmd)) - flags |= PM_UFFD_WP; - VM_BUG_ON(!is_pmd_migration_entry(pmd)); - page = pfn_swap_entry_to_page(entry); - } -#endif + flags |= PM_PRESENT; + if (pmd_soft_dirty(pmd)) + flags |= PM_SOFT_DIRTY; + if (pmd_uffd_wp(pmd)) + flags |= PM_UFFD_WP; + if (pm->show_pfn) + frame = pmd_pfn(pmd) + idx; + } else if (thp_migration_supported()) { + swp_entry_t entry = pmd_to_swp_entry(pmd); + unsigned long offset; - if (page) { - folio = page_folio(page); - if (!folio_test_anon(folio)) - flags |= PM_FILE; + if (pm->show_pfn) { + if (is_pfn_swap_entry(entry)) + offset = swp_offset_pfn(entry) + idx; + else + offset = swp_offset(entry) + idx; + frame = swp_type(entry) | + (offset << MAX_SWAPFILES_SHIFT); } + flags |= PM_SWAP; + if (pmd_swp_soft_dirty(pmd)) + flags |= PM_SOFT_DIRTY; + if (pmd_swp_uffd_wp(pmd)) + flags |= PM_UFFD_WP; + VM_WARN_ON_ONCE(!pmd_is_migration_entry(pmd)); + page = pfn_swap_entry_to_page(entry); + } - for (; addr != end; addr += PAGE_SIZE, idx++) { - u64 cur_flags = flags; - pagemap_entry_t pme; + if (page) { + folio = page_folio(page); + if (!folio_test_anon(folio)) + flags |= PM_FILE; + } - if (folio && (flags & PM_PRESENT) && - __folio_page_mapped_exclusively(folio, page)) - cur_flags |= PM_MMAP_EXCLUSIVE; +populate_pagemap: + for (; addr != end; addr += PAGE_SIZE, idx++) { + u64 cur_flags = flags; + pagemap_entry_t pme; - pme = make_pme(frame, cur_flags); - err = add_to_pagemap(&pme, pm); - if (err) - break; - if (pm->show_pfn) { - if (flags & PM_PRESENT) - frame++; - else if (flags & PM_SWAP) - frame += (1 << MAX_SWAPFILES_SHIFT); - } + if (folio && (flags & PM_PRESENT) && + __folio_page_mapped_exclusively(folio, page)) + cur_flags |= PM_MMAP_EXCLUSIVE; + + pme = make_pme(frame, cur_flags); + err = add_to_pagemap(&pme, pm); + if (err) + break; + if (pm->show_pfn) { + if (flags & PM_PRESENT) + frame++; + else if (flags & PM_SWAP) + frame += (1 << MAX_SWAPFILES_SHIFT); } + } + return err; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, + struct mm_walk *walk) +{ + struct vm_area_struct *vma = walk->vma; + struct pagemapread *pm = walk->private; + spinlock_t *ptl; + pte_t *pte, *orig_pte; + int err = 0; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + ptl = pmd_trans_huge_lock(pmdp, vma); + if (ptl) { + err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); spin_unlock(ptl); return err; } -#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ +#endif /* * We can assume that @vma always points to a valid one and @end never @@ -2326,6 +2340,19 @@ static unsigned long pagemap_page_category(struct pagemap_scan_private *p, { unsigned long categories = 0; + if (pte_none(pte)) { + /* + * An unpopulated pte carries no uffd-wp marker, i.e. it is not + * write-protected, the same condition under which the present + * and swap cases below report PAGE_IS_WRITTEN. Report it here + * too so this generic path agrees with the PAGE_IS_WRITTEN fast + * path in pagemap_scan_pmd_entry(), which reports pte_none as + * written and, under PM_SCAN_WP_MATCHING, arms a marker. The + * fast path applies no VMA test, so neither does this. + */ + return PAGE_IS_WRITTEN; + } + if (pte_present(pte)) { struct page *page; @@ -2390,6 +2417,9 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, { unsigned long categories = PAGE_IS_HUGE; + if (pmd_none(pmd)) + return categories; + if (pmd_present(pmd)) { struct page *page; @@ -2407,7 +2437,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, categories |= PAGE_IS_PFNZERO; if (pmd_soft_dirty(pmd)) categories |= PAGE_IS_SOFT_DIRTY; - } else if (is_swap_pmd(pmd)) { + } else { swp_entry_t swp; categories |= PAGE_IS_SWAPPED; @@ -2485,15 +2515,19 @@ static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, psize = huge_page_size(hstate_vma(vma)); - if (is_hugetlb_entry_migration(ptent)) + if (is_hugetlb_entry_migration(ptent)) { set_huge_pte_at(vma->vm_mm, addr, ptep, pte_swp_mkuffd_wp(ptent), psize); - else if (!huge_pte_none(ptent)) - huge_ptep_modify_prot_commit(vma, addr, ptep, ptent, - huge_pte_mkuffd_wp(ptent)); - else + } else if (!huge_pte_none(ptent)) { + pte_t old_pte, new_pte; + + old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); + new_pte = huge_pte_mkuffd_wp(old_pte); + huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); + } else { set_huge_pte_at(vma->vm_mm, addr, ptep, make_pte_marker(PTE_MARKER_UFFD_WP), psize); + } } #endif /* CONFIG_HUGETLB_PAGE */ @@ -2914,12 +2948,28 @@ static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, { struct pagemap_scan_private *p = walk->private; struct vm_area_struct *vma = walk->vma; + unsigned long categories; int ret, err; - if (!vma || !pagemap_scan_is_interesting_page(p->cur_vma_category, p)) + if (!vma) + return 0; + + /* + * In a uffd-wp VMA an unpopulated range is treated as written: + * uffd-wp registration populates page tables and installs markers + * with WP_UNPOPULATED, so a missing marker means the range was + * zapped. See the pte_none() handling in pagemap_page_category(). + * + * hugetlb differs, see pagemap_hugetlb_category(). + */ + categories = p->cur_vma_category; + if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma)) + categories |= PAGE_IS_WRITTEN; + + if (!pagemap_scan_is_interesting_page(categories, p)) return 0; - ret = pagemap_scan_output(p->cur_vma_category, p, addr, &end); + ret = pagemap_scan_output(categories, p, addr, &end); if (addr == end) return ret; diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c index e399e2dd3a124..7a15ad20d98a2 100644 --- a/fs/qnx4/inode.c +++ b/fs/qnx4/inode.c @@ -202,7 +202,8 @@ static int qnx4_fill_super(struct super_block *s, struct fs_context *fc) return -ENOMEM; s->s_fs_info = qs; - sb_set_blocksize(s, QNX4_BLOCK_SIZE); + if (!sb_set_blocksize(s, QNX4_BLOCK_SIZE)) + return -EINVAL; s->s_op = &qnx4_sops; s->s_magic = QNX4_SUPER_MAGIC; diff --git a/fs/resctrl/ctrlmondata.c b/fs/resctrl/ctrlmondata.c index 0d0ef54fc4de1..eeb12ec594c7b 100644 --- a/fs/resctrl/ctrlmondata.c +++ b/fs/resctrl/ctrlmondata.c @@ -547,7 +547,7 @@ struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, } void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, - struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, + struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, cpumask_t *cpumask, int evtid, int first) { int cpu; @@ -596,9 +596,9 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) struct kernfs_open_file *of = m->private; enum resctrl_res_level resid; enum resctrl_event_id evtid; + struct rdt_l3_mon_domain *d; struct rdt_domain_hdr *hdr; struct rmid_read rr = {0}; - struct rdt_mon_domain *d; struct rdtgroup *rdtgrp; int domid, cpu, ret = 0; struct rdt_resource *r; @@ -653,7 +653,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg) ret = -ENOENT; goto out; } - d = container_of(hdr, struct rdt_mon_domain, hdr); + d = container_of(hdr, struct rdt_l3_mon_domain, hdr); mon_event_read(&rr, r, d, rdtgrp, &d->hdr.cpu_mask, evtid, false); } diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h index cf1fd82dc5a99..77f6e531e261b 100644 --- a/fs/resctrl/internal.h +++ b/fs/resctrl/internal.h @@ -123,7 +123,7 @@ struct mon_data { struct rmid_read { struct rdtgroup *rgrp; struct rdt_resource *r; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; enum resctrl_event_id evtid; bool first; struct cacheinfo *ci; @@ -351,23 +351,27 @@ int closids_supported(void); void closid_free(int closid); +int setup_rmid_lru_list(void); + +void free_rmid_lru_list(void); + int alloc_rmid(u32 closid); void free_rmid(u32 closid, u32 rmid); -void resctrl_mon_resource_exit(void); +int resctrl_l3_mon_resource_init(void); + +void resctrl_l3_mon_resource_exit(void); void mon_event_count(void *info); int rdtgroup_mondata_show(struct seq_file *m, void *arg); void mon_event_read(struct rmid_read *rr, struct rdt_resource *r, - struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, + struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, cpumask_t *cpumask, int evtid, int first); -int resctrl_mon_resource_init(void); - -void mbm_setup_overflow_handler(struct rdt_mon_domain *dom, +void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, int exclude_cpu); @@ -375,14 +379,14 @@ void mbm_handle_overflow(struct work_struct *work); bool is_mba_sc(struct rdt_resource *r); -void cqm_setup_limbo_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, +void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, int exclude_cpu); void cqm_handle_limbo(struct work_struct *work); -bool has_busy_rmid(struct rdt_mon_domain *d); +bool has_busy_rmid(struct rdt_l3_mon_domain *d); -void __check_limbo(struct rdt_mon_domain *d, bool force_free); +void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free); void resctrl_file_fflags_init(const char *config, unsigned long fflags); diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c index 7012a75814ac3..55db47e969ce6 100644 --- a/fs/resctrl/monitor.c +++ b/fs/resctrl/monitor.c @@ -130,7 +130,7 @@ static void limbo_release_entry(struct rmid_entry *entry) * decrement the count. If the busy count gets to zero on an RMID, we * free the RMID */ -void __check_limbo(struct rdt_mon_domain *d, bool force_free) +void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); u32 idx_limit = resctrl_arch_system_num_rmid_idx(); @@ -193,7 +193,7 @@ void __check_limbo(struct rdt_mon_domain *d, bool force_free) resctrl_arch_mon_ctx_free(r, QOS_L3_OCCUP_EVENT_ID, arch_mon_ctx); } -bool has_busy_rmid(struct rdt_mon_domain *d) +bool has_busy_rmid(struct rdt_l3_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); @@ -294,7 +294,7 @@ int alloc_rmid(u32 closid) static void add_rmid_to_limbo(struct rmid_entry *entry) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; u32 idx; lockdep_assert_held(&rdtgroup_mutex); @@ -347,7 +347,7 @@ void free_rmid(u32 closid, u32 rmid) list_add_tail(&entry->list, &rmid_free_lru); } -static struct mbm_state *get_mbm_state(struct rdt_mon_domain *d, u32 closid, +static struct mbm_state *get_mbm_state(struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id evtid) { u32 idx = resctrl_arch_rmid_idx_encode(closid, rmid); @@ -367,7 +367,7 @@ static struct mbm_state *get_mbm_state(struct rdt_mon_domain *d, u32 closid, * Return: * Valid counter ID on success, or -ENOENT on failure. */ -static int mbm_cntr_get(struct rdt_resource *r, struct rdt_mon_domain *d, +static int mbm_cntr_get(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { int cntr_id; @@ -394,7 +394,7 @@ static int mbm_cntr_get(struct rdt_resource *r, struct rdt_mon_domain *d, * Return: * Valid counter ID on success, or -ENOSPC on failure. */ -static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_mon_domain *d, +static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { int cntr_id; @@ -413,17 +413,17 @@ static int mbm_cntr_alloc(struct rdt_resource *r, struct rdt_mon_domain *d, /* * mbm_cntr_free() - Clear the counter ID configuration details in the domain @d. */ -static void mbm_cntr_free(struct rdt_mon_domain *d, int cntr_id) +static void mbm_cntr_free(struct rdt_l3_mon_domain *d, int cntr_id) { memset(&d->cntr_cfg[cntr_id], 0, sizeof(*d->cntr_cfg)); } -static int __mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) +static int __l3_mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) { int cpu = smp_processor_id(); u32 closid = rdtgrp->closid; u32 rmid = rdtgrp->mon.rmid; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; int cntr_id = -ENOENT; struct mbm_state *m; int err, ret; @@ -499,6 +499,17 @@ static int __mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) return ret; } +static int __mon_event_count(struct rdtgroup *rdtgrp, struct rmid_read *rr) +{ + switch (rr->r->rid) { + case RDT_RESOURCE_L3: + return __l3_mon_event_count(rdtgrp, rr); + default: + rr->err = -EINVAL; + return -EINVAL; + } +} + /* * mbm_bw_count() - Update bw count from values previously read by * __mon_event_count(). @@ -617,7 +628,7 @@ static struct rdt_ctrl_domain *get_ctrl_domain_from_cpu(int cpu, * throttle MSRs already have low percentage values. To avoid * unnecessarily restricting such rdtgroups, we also increase the bandwidth. */ -static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_mon_domain *dom_mbm) +static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_l3_mon_domain *dom_mbm) { u32 closid, rmid, cur_msr_val, new_msr_val; struct mbm_state *pmbm_data, *cmbm_data; @@ -685,7 +696,7 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_mon_domain *dom_mbm) resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val); } -static void mbm_update_one_event(struct rdt_resource *r, struct rdt_mon_domain *d, +static void mbm_update_one_event(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { struct rmid_read rr = {0}; @@ -717,7 +728,7 @@ static void mbm_update_one_event(struct rdt_resource *r, struct rdt_mon_domain * resctrl_arch_mon_ctx_free(rr.r, rr.evtid, rr.arch_mon_ctx); } -static void mbm_update(struct rdt_resource *r, struct rdt_mon_domain *d, +static void mbm_update(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp) { /* @@ -738,12 +749,12 @@ static void mbm_update(struct rdt_resource *r, struct rdt_mon_domain *d, void cqm_handle_limbo(struct work_struct *work) { unsigned long delay = msecs_to_jiffies(CQM_LIMBOCHECK_INTERVAL); - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; cpus_read_lock(); mutex_lock(&rdtgroup_mutex); - d = container_of(work, struct rdt_mon_domain, cqm_limbo.work); + d = container_of(work, struct rdt_l3_mon_domain, cqm_limbo.work); __check_limbo(d, false); @@ -766,7 +777,7 @@ void cqm_handle_limbo(struct work_struct *work) * @exclude_cpu: Which CPU the handler should not run on, * RESCTRL_PICK_ANY_CPU to pick any CPU. */ -void cqm_setup_limbo_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, +void cqm_setup_limbo_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, int exclude_cpu) { unsigned long delay = msecs_to_jiffies(delay_ms); @@ -783,7 +794,7 @@ void mbm_handle_overflow(struct work_struct *work) { unsigned long delay = msecs_to_jiffies(MBM_OVERFLOW_INTERVAL); struct rdtgroup *prgrp, *crgrp; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; struct list_head *head; struct rdt_resource *r; @@ -798,7 +809,7 @@ void mbm_handle_overflow(struct work_struct *work) goto out_unlock; r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - d = container_of(work, struct rdt_mon_domain, mbm_over.work); + d = container_of(work, struct rdt_l3_mon_domain, mbm_over.work); list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) { mbm_update(r, d, prgrp); @@ -832,7 +843,7 @@ void mbm_handle_overflow(struct work_struct *work) * @exclude_cpu: Which CPU the handler should not run on, * RESCTRL_PICK_ANY_CPU to pick any CPU. */ -void mbm_setup_overflow_handler(struct rdt_mon_domain *dom, unsigned long delay_ms, +void mbm_setup_overflow_handler(struct rdt_l3_mon_domain *dom, unsigned long delay_ms, int exclude_cpu) { unsigned long delay = msecs_to_jiffies(delay_ms); @@ -851,42 +862,29 @@ void mbm_setup_overflow_handler(struct rdt_mon_domain *dom, unsigned long delay_ schedule_delayed_work_on(cpu, &dom->mbm_over, delay); } -static int dom_data_init(struct rdt_resource *r) +int setup_rmid_lru_list(void) { - u32 idx_limit = resctrl_arch_system_num_rmid_idx(); - u32 num_closid = resctrl_arch_get_num_closid(r); struct rmid_entry *entry = NULL; - int err = 0, i; + u32 idx_limit; u32 idx; + int i; - mutex_lock(&rdtgroup_mutex); - if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { - u32 *tmp; - - /* - * If the architecture hasn't provided a sanitised value here, - * this may result in larger arrays than necessary. Resctrl will - * use a smaller system wide value based on the resources in - * use. - */ - tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL); - if (!tmp) { - err = -ENOMEM; - goto out_unlock; - } + if (!resctrl_arch_mon_capable()) + return 0; - closid_num_dirty_rmid = tmp; - } + /* + * Called on every mount, but the number of RMIDs cannot change + * after the first mount, so keep using the same set of rmid_ptrs[] + * until resctrl_exit(). Note that the limbo handler continues to + * access rmid_ptrs[] after resctrl is unmounted. + */ + if (rmid_ptrs) + return 0; + idx_limit = resctrl_arch_system_num_rmid_idx(); rmid_ptrs = kcalloc(idx_limit, sizeof(struct rmid_entry), GFP_KERNEL); - if (!rmid_ptrs) { - if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { - kfree(closid_num_dirty_rmid); - closid_num_dirty_rmid = NULL; - } - err = -ENOMEM; - goto out_unlock; - } + if (!rmid_ptrs) + return -ENOMEM; for (i = 0; i < idx_limit; i++) { entry = &rmid_ptrs[i]; @@ -899,35 +897,24 @@ static int dom_data_init(struct rdt_resource *r) /* * RESCTRL_RESERVED_CLOSID and RESCTRL_RESERVED_RMID are special and * are always allocated. These are used for the rdtgroup_default - * control group, which will be setup later in resctrl_init(). + * control group, which was setup earlier in rdtgroup_setup_default(). */ idx = resctrl_arch_rmid_idx_encode(RESCTRL_RESERVED_CLOSID, RESCTRL_RESERVED_RMID); entry = __rmid_entry(idx); list_del(&entry->list); -out_unlock: - mutex_unlock(&rdtgroup_mutex); - - return err; + return 0; } -static void dom_data_exit(struct rdt_resource *r) +void free_rmid_lru_list(void) { - mutex_lock(&rdtgroup_mutex); - - if (!r->mon_capable) - goto out_unlock; - - if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { - kfree(closid_num_dirty_rmid); - closid_num_dirty_rmid = NULL; - } + if (!resctrl_arch_mon_capable()) + return; + mutex_lock(&rdtgroup_mutex); kfree(rmid_ptrs); rmid_ptrs = NULL; - -out_unlock: mutex_unlock(&rdtgroup_mutex); } @@ -1087,7 +1074,7 @@ ssize_t resctrl_mbm_assign_on_mkdir_write(struct kernfs_open_file *of, char *buf * mbm_cntr_free_all() - Clear all the counter ID configuration details in the * domain @d. Called when mbm_assign_mode is changed. */ -static void mbm_cntr_free_all(struct rdt_resource *r, struct rdt_mon_domain *d) +static void mbm_cntr_free_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { memset(d->cntr_cfg, 0, sizeof(*d->cntr_cfg) * r->mon.num_mbm_cntrs); } @@ -1096,7 +1083,7 @@ static void mbm_cntr_free_all(struct rdt_resource *r, struct rdt_mon_domain *d) * resctrl_reset_rmid_all() - Reset all non-architecture states for all the * supported RMIDs. */ -static void resctrl_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d) +static void resctrl_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); enum resctrl_event_id evt; @@ -1117,7 +1104,7 @@ static void resctrl_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain * Assign the counter if @assign is true else unassign the counter. Reset the * associated non-architectural state. */ -static void rdtgroup_assign_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +static void rdtgroup_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, enum resctrl_event_id evtid, u32 rmid, u32 closid, u32 cntr_id, bool assign) { @@ -1137,7 +1124,7 @@ static void rdtgroup_assign_cntr(struct rdt_resource *r, struct rdt_mon_domain * * Return: * 0 on success, < 0 on failure. */ -static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, struct mon_evt *mevt) { int cntr_id; @@ -1172,7 +1159,7 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_mon_dom * Return: * 0 on success, < 0 on failure. */ -static int rdtgroup_assign_cntr_event(struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, +static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, struct mon_evt *mevt) { struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); @@ -1222,7 +1209,7 @@ void rdtgroup_assign_cntrs(struct rdtgroup *rdtgrp) * rdtgroup_free_unassign_cntr() - Unassign and reset the counter ID configuration * for the event pointed to by @mevt within the domain @d and resctrl group @rdtgrp. */ -static void rdtgroup_free_unassign_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +static void rdtgroup_free_unassign_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, struct mon_evt *mevt) { int cntr_id; @@ -1243,7 +1230,7 @@ static void rdtgroup_free_unassign_cntr(struct rdt_resource *r, struct rdt_mon_d * the event structure @mevt from the domain @d and the group @rdtgrp. Unassign * the counters from all the domains if @d is NULL else unassign from @d. */ -static void rdtgroup_unassign_cntr_event(struct rdt_mon_domain *d, struct rdtgroup *rdtgrp, +static void rdtgroup_unassign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, struct mon_evt *mevt) { struct rdt_resource *r = resctrl_arch_get_resource(mevt->rid); @@ -1318,7 +1305,7 @@ static int resctrl_parse_mem_transactions(char *tok, u32 *val) static void rdtgroup_update_cntr_event(struct rdt_resource *r, struct rdtgroup *rdtgrp, enum resctrl_event_id evtid) { - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; int cntr_id; list_for_each_entry(d, &r->mon_domains, hdr.list) { @@ -1424,7 +1411,7 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; int ret = 0; bool enable; @@ -1497,7 +1484,7 @@ int resctrl_num_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, void *v) { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); - struct rdt_mon_domain *dom; + struct rdt_l3_mon_domain *dom; bool sep = false; cpus_read_lock(); @@ -1521,7 +1508,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, struct seq_file *s, void *v) { struct rdt_resource *r = rdt_kn_parent_priv(of->kn); - struct rdt_mon_domain *dom; + struct rdt_l3_mon_domain *dom; bool sep = false; u32 cntrs, i; int ret = 0; @@ -1562,7 +1549,7 @@ int resctrl_available_mbm_cntrs_show(struct kernfs_open_file *of, int mbm_L3_assignments_show(struct kernfs_open_file *of, struct seq_file *s, void *v) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; struct rdtgroup *rdtgrp; struct mon_evt *mevt; int ret = 0; @@ -1625,7 +1612,7 @@ static struct mon_evt *mbm_get_mon_event_by_name(struct rdt_resource *r, char *n return NULL; } -static int rdtgroup_modify_assign_state(char *assign, struct rdt_mon_domain *d, +static int rdtgroup_modify_assign_state(char *assign, struct rdt_l3_mon_domain *d, struct rdtgroup *rdtgrp, struct mon_evt *mevt) { int ret = 0; @@ -1651,7 +1638,7 @@ static int rdtgroup_modify_assign_state(char *assign, struct rdt_mon_domain *d, static int resctrl_parse_mbm_assignment(struct rdt_resource *r, struct rdtgroup *rdtgrp, char *event, char *tok) { - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; unsigned long dom_id = 0; char *dom_str, *id_str; struct mon_evt *mevt; @@ -1746,19 +1733,59 @@ ssize_t mbm_L3_assignments_write(struct kernfs_open_file *of, char *buf, return ret ?: nbytes; } +static int closid_num_dirty_rmid_alloc(struct rdt_resource *r) +{ + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { + u32 num_closid = resctrl_arch_get_num_closid(r); + u32 *tmp; + + /* For ARM memory ordering access to closid_num_dirty_rmid */ + mutex_lock(&rdtgroup_mutex); + + /* + * If the architecture hasn't provided a sanitised value here, + * this may result in larger arrays than necessary. Resctrl will + * use a smaller system wide value based on the resources in + * use. + */ + tmp = kcalloc(num_closid, sizeof(*tmp), GFP_KERNEL); + if (!tmp) { + mutex_unlock(&rdtgroup_mutex); + return -ENOMEM; + } + + closid_num_dirty_rmid = tmp; + + mutex_unlock(&rdtgroup_mutex); + } + + return 0; +} + +static void closid_num_dirty_rmid_free(void) +{ + if (IS_ENABLED(CONFIG_RESCTRL_RMID_DEPENDS_ON_CLOSID)) { + mutex_lock(&rdtgroup_mutex); + kfree(closid_num_dirty_rmid); + closid_num_dirty_rmid = NULL; + mutex_unlock(&rdtgroup_mutex); + } +} + /** - * resctrl_mon_resource_init() - Initialise global monitoring structures. + * resctrl_l3_mon_resource_init() - Initialise global monitoring structures. * * Allocate and initialise global monitor resources that do not belong to a - * specific domain. i.e. the rmid_ptrs[] used for the limbo and free lists. + * specific domain. i.e. the closid_num_dirty_rmid[] used to find the CLOSID + * with the cleanest set of RMIDs. * Called once during boot after the struct rdt_resource's have been configured * but before the filesystem is mounted. * Resctrl's cpuhp callbacks may be called before this point to bring a domain * online. * - * Returns 0 for success, or -ENOMEM. + * Return: 0 for success, or -ENOMEM. */ -int resctrl_mon_resource_init(void) +int resctrl_l3_mon_resource_init(void) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); int ret; @@ -1766,7 +1793,7 @@ int resctrl_mon_resource_init(void) if (!r->mon_capable) return 0; - ret = dom_data_init(r); + ret = closid_num_dirty_rmid_alloc(r); if (ret) return ret; @@ -1808,9 +1835,12 @@ int resctrl_mon_resource_init(void) return 0; } -void resctrl_mon_resource_exit(void) +void resctrl_l3_mon_resource_exit(void) { struct rdt_resource *r = resctrl_arch_get_resource(RDT_RESOURCE_L3); - dom_data_exit(r); + if (!r->mon_capable) + return; + + closid_num_dirty_rmid_free(); } diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c index 7cfb0086451c5..1f81bd6711551 100644 --- a/fs/resctrl/rdtgroup.c +++ b/fs/resctrl/rdtgroup.c @@ -581,14 +581,20 @@ static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of, * * On resource group creation via a mkdir, an extra kernfs_node reference is * taken to ensure that the rdtgroup structure remains accessible for the - * rdtgroup_kn_unlock() calls where it is removed. + * rdtgroup_kn_unlock() calls where it is removed. The default group is + * statically allocated: it does not have an extra reference but will have + * RDT_DELETED set on unmount to support safe access to its associated files + * via rdtgroup_kn_lock_live/rdtgroup_kn_unlock(). * - * Drop the extra reference here, then free the rdtgroup structure. + * For all but the default group: drop the extra reference, then free the + * rdtgroup structure. * * Return: void */ static void rdtgroup_remove(struct rdtgroup *rdtgrp) { + if (rdtgrp == &rdtgroup_default) + return; kernfs_put(rdtgrp->kn); kfree(rdtgrp); } @@ -1620,7 +1626,7 @@ static void mondata_config_read(struct resctrl_mon_config_info *mon_info) static int mbm_config_show(struct seq_file *s, struct rdt_resource *r, u32 evtid) { struct resctrl_mon_config_info mon_info; - struct rdt_mon_domain *dom; + struct rdt_l3_mon_domain *dom; bool sep = false; cpus_read_lock(); @@ -1668,7 +1674,7 @@ static int mbm_local_bytes_config_show(struct kernfs_open_file *of, } static void mbm_config_write_domain(struct rdt_resource *r, - struct rdt_mon_domain *d, u32 evtid, u32 val) + struct rdt_l3_mon_domain *d, u32 evtid, u32 val) { struct resctrl_mon_config_info mon_info = {0}; @@ -1710,7 +1716,7 @@ static int mon_config_write(struct rdt_resource *r, char *tok, u32 evtid) { char *dom_str = NULL, *id_str; unsigned long dom_id, val; - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; /* Walking r->domains, ensure it can't race with cpuhp */ lockdep_assert_cpus_held(); @@ -2522,15 +2528,24 @@ static void rdtgroup_kn_get(struct rdtgroup *rdtgrp, struct kernfs_node *kn) static void rdtgroup_kn_put(struct rdtgroup *rdtgrp, struct kernfs_node *kn) { - if (atomic_dec_and_test(&rdtgrp->waitcount) && - (rdtgrp->flags & RDT_DELETED)) { + bool needs_free; + + if (!atomic_dec_and_mutex_lock(&rdtgrp->waitcount, &rdtgroup_mutex)) { + kernfs_unbreak_active_protection(kn); + return; + } + + needs_free = rdtgrp->flags & RDT_DELETED; + + mutex_unlock(&rdtgroup_mutex); + + kernfs_unbreak_active_protection(kn); + + if (needs_free) { if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP || rdtgrp->mode == RDT_MODE_PSEUDO_LOCKED) rdtgroup_pseudo_lock_remove(rdtgrp); - kernfs_unbreak_active_protection(kn); rdtgroup_remove(rdtgrp); - } else { - kernfs_unbreak_active_protection(kn); } } @@ -2718,7 +2733,7 @@ static int rdt_get_tree(struct fs_context *fc) { struct rdt_fs_context *ctx = rdt_fc2context(fc); unsigned long flags = RFTYPE_CTRL_BASE; - struct rdt_mon_domain *dom; + struct rdt_l3_mon_domain *dom; struct rdt_resource *r; int ret; @@ -2732,6 +2747,16 @@ static int rdt_get_tree(struct fs_context *fc) goto out; } + /* Avoid races from pending operations from a previous mount */ + if (atomic_read(&rdtgroup_default.waitcount) != 0) { + ret = -EBUSY; + goto out; + } + + ret = setup_rmid_lru_list(); + if (ret) + goto out; + ret = rdtgroup_setup_root(ctx); if (ret) goto out; @@ -3090,6 +3115,7 @@ static void resctrl_fs_teardown(void) mon_put_kn_priv(); rdt_pseudo_lock_release(); rdtgroup_default.mode = RDT_MODE_SHAREABLE; + rdtgroup_default.flags = RDT_DELETED; closid_exit(); schemata_list_destroy(); rdtgroup_destroy_root(); @@ -3169,7 +3195,7 @@ static void mon_rmdir_one_subdir(struct kernfs_node *pkn, char *name, char *subn * when last domain being summed is removed. */ static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - struct rdt_mon_domain *d) + struct rdt_l3_mon_domain *d) { struct rdtgroup *prgrp, *crgrp; char subname[32]; @@ -3189,7 +3215,7 @@ static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, } } -static int mon_add_all_files(struct kernfs_node *kn, struct rdt_mon_domain *d, +static int mon_add_all_files(struct kernfs_node *kn, struct rdt_l3_mon_domain *d, struct rdt_resource *r, struct rdtgroup *prgrp, bool do_sum) { @@ -3218,7 +3244,7 @@ static int mon_add_all_files(struct kernfs_node *kn, struct rdt_mon_domain *d, } static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, - struct rdt_mon_domain *d, + struct rdt_l3_mon_domain *d, struct rdt_resource *r, struct rdtgroup *prgrp) { struct kernfs_node *kn, *ckn; @@ -3280,7 +3306,7 @@ static int mkdir_mondata_subdir(struct kernfs_node *parent_kn, * and "monitor" groups with given domain id. */ static void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, - struct rdt_mon_domain *d) + struct rdt_l3_mon_domain *d) { struct kernfs_node *parent_kn; struct rdtgroup *prgrp, *crgrp; @@ -3302,7 +3328,7 @@ static int mkdir_mondata_subdir_alldom(struct kernfs_node *parent_kn, struct rdt_resource *r, struct rdtgroup *prgrp) { - struct rdt_mon_domain *dom; + struct rdt_l3_mon_domain *dom; int ret; /* Walking r->domains, ensure it can't race with cpuhp */ @@ -4144,6 +4170,7 @@ static int rdtgroup_setup_root(struct rdt_fs_context *ctx) ctx->kfc.root = rdt_root; rdtgroup_default.kn = kernfs_root_to_node(rdt_root); + rdtgroup_default.flags = 0; return 0; } @@ -4170,7 +4197,7 @@ static void rdtgroup_setup_default(void) mutex_unlock(&rdtgroup_mutex); } -static void domain_destroy_mon_state(struct rdt_mon_domain *d) +static void domain_destroy_l3_mon_state(struct rdt_l3_mon_domain *d) { int idx; @@ -4192,7 +4219,7 @@ void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain mutex_unlock(&rdtgroup_mutex); } -void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d) +void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { mutex_lock(&rdtgroup_mutex); @@ -4218,13 +4245,13 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d cancel_delayed_work(&d->cqm_limbo); } - domain_destroy_mon_state(d); + domain_destroy_l3_mon_state(d); mutex_unlock(&rdtgroup_mutex); } /** - * domain_setup_mon_state() - Initialise domain monitoring structures. + * domain_setup_l3_mon_state() - Initialise domain monitoring structures. * @r: The resource for the newly online domain. * @d: The newly online domain. * @@ -4232,11 +4259,11 @@ void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d * Called when the first CPU of a domain comes online, regardless of whether * the filesystem is mounted. * During boot this may be called before global allocations have been made by - * resctrl_mon_resource_init(). + * resctrl_l3_mon_resource_init(). * - * Returns 0 for success, or -ENOMEM. + * Return: 0 for success, or -ENOMEM. */ -static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_mon_domain *d) +static int domain_setup_l3_mon_state(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { u32 idx_limit = resctrl_arch_system_num_rmid_idx(); size_t tsize = sizeof(*d->mbm_states[0]); @@ -4292,13 +4319,13 @@ int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d return err; } -int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d) +int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d) { int err; mutex_lock(&rdtgroup_mutex); - err = domain_setup_mon_state(r, d); + err = domain_setup_l3_mon_state(r, d); if (err) goto out_unlock; @@ -4344,10 +4371,10 @@ static void clear_childcpus(struct rdtgroup *r, unsigned int cpu) } } -static struct rdt_mon_domain *get_mon_domain_from_cpu(int cpu, +static struct rdt_l3_mon_domain *get_mon_domain_from_cpu(int cpu, struct rdt_resource *r) { - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; lockdep_assert_cpus_held(); @@ -4363,7 +4390,7 @@ static struct rdt_mon_domain *get_mon_domain_from_cpu(int cpu, void resctrl_offline_cpu(unsigned int cpu) { struct rdt_resource *l3 = resctrl_arch_get_resource(RDT_RESOURCE_L3); - struct rdt_mon_domain *d; + struct rdt_l3_mon_domain *d; struct rdtgroup *rdtgrp; mutex_lock(&rdtgroup_mutex); @@ -4413,13 +4440,13 @@ int resctrl_init(void) thread_throttle_mode_init(); - ret = resctrl_mon_resource_init(); + ret = resctrl_l3_mon_resource_init(); if (ret) return ret; ret = sysfs_create_mount_point(fs_kobj, "resctrl"); if (ret) { - resctrl_mon_resource_exit(); + resctrl_l3_mon_resource_exit(); return ret; } @@ -4454,7 +4481,7 @@ int resctrl_init(void) cleanup_mountpoint: sysfs_remove_mount_point(fs_kobj, "resctrl"); - resctrl_mon_resource_exit(); + resctrl_l3_mon_resource_exit(); return ret; } @@ -4490,7 +4517,7 @@ static bool resctrl_online_domains_exist(void) * When called by the architecture code, all CPUs and resctrl domains must be * offline. This ensures the limbo and overflow handlers are not scheduled to * run, meaning the data structures they access can be freed by - * resctrl_mon_resource_exit(). + * resctrl_l3_mon_resource_exit(). * * After resctrl_exit() returns, the architecture code should return an * error from all resctrl_arch_ functions that can do this. @@ -4517,5 +4544,6 @@ void resctrl_exit(void) * it can be used to umount resctrl. */ - resctrl_mon_resource_exit(); + resctrl_l3_mon_resource_exit(); + free_rmid_lru_list(); } diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c index de687b491df46..cef855456063a 100644 --- a/fs/smb/client/cached_dir.c +++ b/fs/smb/client/cached_dir.c @@ -118,7 +118,7 @@ static const char *path_no_prefix(struct cifs_sb_info *cifs_sb, if (!*path) return path; - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) && + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) && cifs_sb->prepath) { len = strlen(cifs_sb->prepath) + 1; if (unlikely(len > strlen(path))) diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c index 1fb71d2d31b5d..3045db555c3cc 100644 --- a/fs/smb/client/cifs_debug.c +++ b/fs/smb/client/cifs_debug.c @@ -776,7 +776,7 @@ static ssize_t cifs_stats_proc_write(struct file *file, atomic_set(&server->smb2slowcmd[i], 0); server->time_per_cmd[i] = 0; server->slowest_cmd[i] = 0; - server->fastest_cmd[0] = 0; + server->fastest_cmd[i] = 0; } #endif /* CONFIG_CIFS_STATS2 */ list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) { @@ -1307,11 +1307,11 @@ static const struct proc_ops cifs_mount_params_proc_ops = { }; #else -inline void cifs_proc_init(void) +void cifs_proc_init(void) { } -inline void cifs_proc_clean(void) +void cifs_proc_clean(void) { } #endif /* PROC_FS */ diff --git a/fs/smb/client/cifs_fs_sb.h b/fs/smb/client/cifs_fs_sb.h index 5e8d163cb5f80..fc75999ffdc44 100644 --- a/fs/smb/client/cifs_fs_sb.h +++ b/fs/smb/client/cifs_fs_sb.h @@ -10,6 +10,7 @@ #ifndef _CIFS_FS_SB_H #define _CIFS_FS_SB_H +#include #include #define CIFS_MOUNT_NO_PERM 1 /* do not do client vfs_perm check */ @@ -55,7 +56,7 @@ struct cifs_sb_info { struct nls_table *local_nls; struct smb3_fs_context *ctx; atomic_t active; - unsigned int mnt_cifs_flags; + atomic_t mnt_cifs_flags; struct delayed_work prune_tlinks; struct rcu_head rcu; @@ -72,4 +73,9 @@ struct cifs_sb_info { */ struct dentry *root; }; + +/* Mount flags are accessed locklessly by mount/remount and reconnect paths. */ +#define cifs_sb_flags(cifs_sb) \ + ((unsigned int)atomic_read(&(cifs_sb)->mnt_cifs_flags)) + #endif /* _CIFS_FS_SB_H */ diff --git a/fs/smb/client/cifs_ioctl.h b/fs/smb/client/cifs_ioctl.h index b51ce64fcccf9..23d0931fb9cb7 100644 --- a/fs/smb/client/cifs_ioctl.h +++ b/fs/smb/client/cifs_ioctl.h @@ -125,7 +125,7 @@ struct smb3_notify_info { static inline bool cifs_forced_shutdown(struct cifs_sb_info *sbi) { - if (CIFS_MOUNT_SHUTDOWN & sbi->mnt_cifs_flags) + if (CIFS_MOUNT_SHUTDOWN & cifs_sb_flags(sbi)) return true; else return false; diff --git a/fs/smb/client/cifs_spnego.h b/fs/smb/client/cifs_spnego.h index e4d751b0c8127..e70929db3611f 100644 --- a/fs/smb/client/cifs_spnego.h +++ b/fs/smb/client/cifs_spnego.h @@ -27,10 +27,8 @@ struct cifs_spnego_msg { uint8_t data[]; }; -#ifdef __KERNEL__ extern struct key_type cifs_spnego_key_type; extern struct key *cifs_get_spnego_key(struct cifs_ses *sesInfo, struct TCP_Server_Info *server); -#endif /* KERNEL */ #endif /* _CIFS_SPNEGO_H */ diff --git a/fs/smb/client/cifs_unicode.c b/fs/smb/client/cifs_unicode.c index f8659d36793f1..b6100ae68e618 100644 --- a/fs/smb/client/cifs_unicode.c +++ b/fs/smb/client/cifs_unicode.c @@ -16,9 +16,9 @@ int cifs_remap(struct cifs_sb_info *cifs_sb) { int map_type; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SFM_CHR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SFM_CHR) map_type = SFM_MAP_UNI_RSVD; - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SPECIAL_CHR) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SPECIAL_CHR) map_type = SFU_MAP_UNI_RSVD; else map_type = NO_MAP_UNI_RSVD; diff --git a/fs/smb/client/cifs_unicode.h b/fs/smb/client/cifs_unicode.h index e137a0dfbbe9a..6e4b99786498d 100644 --- a/fs/smb/client/cifs_unicode.h +++ b/fs/smb/client/cifs_unicode.h @@ -54,7 +54,6 @@ #define SFM_MAP_UNI_RSVD 1 #define SFU_MAP_UNI_RSVD 2 -#ifdef __KERNEL__ int cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, const struct nls_table *cp, int map_type); int cifs_utf16_bytes(const __le16 *from, int maxbytes, @@ -69,8 +68,6 @@ extern int cifs_remap(struct cifs_sb_info *cifs_sb); extern __le16 *cifs_strndup_to_utf16(const char *src, const int maxlen, int *utf16_len, const struct nls_table *cp, int remap); -#endif - wchar_t cifs_toupper(wchar_t in); #endif /* _CIFS_UNICODE_H */ diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c index 92ee03cd01b39..ef72ec55c60f9 100644 --- a/fs/smb/client/cifsacl.c +++ b/fs/smb/client/cifsacl.c @@ -69,6 +69,9 @@ cifs_idmap_key_instantiate(struct key *key, struct key_preparsed_payload *prep) { char *payload; + if (prep->datalen > U16_MAX) + return -EINVAL; + /* * If the payload is less than or equal to the size of a pointer, then * an allocation here is wasteful. Just copy the data directly to the @@ -98,8 +101,23 @@ cifs_idmap_key_destroy(struct key *key) kfree(key->payload.data[0]); } +static int +cifs_idmap_key_vet_description(const char *description) +{ + /* + * cifs.idmap descriptions are authority-bearing inputs to the + * cifs.idmap upcall helper. Only allow the kernel to create this + * type of key using the private root_cred installed in + * init_cifs_idmap; reject userspace request_key(2)/add_key(2). + */ + if (current_cred() != root_cred) + return -EPERM; + return 0; +} + static struct key_type cifs_idmap_key_type = { .name = "cifs.idmap", + .vet_description = cifs_idmap_key_vet_description, .instantiate = cifs_idmap_key_instantiate, .destroy = cifs_idmap_key_destroy, .describe = user_describe, @@ -355,7 +373,7 @@ sid_to_id(struct cifs_sb_info *cifs_sb, struct smb_sid *psid, return -EIO; } - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) || + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UID_FROM_ACL) || (cifs_sb_master_tcon(cifs_sb)->posix_extensions)) { uint32_t unix_id; bool is_group; @@ -1005,13 +1023,13 @@ unsigned int setup_special_user_owner_ACE(struct smb_ace *pntace) static void populate_new_aces(char *nacl_base, struct smb_sid *pownersid, struct smb_sid *pgrpsid, - __u64 *pnmode, u16 *pnum_aces, u16 *pnsize, + __u64 *pnmode, u16 *pnum_aces, u32 *pnsize, bool modefromsid, bool posix) { __u64 nmode; u16 num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; __u64 user_mode; __u64 group_mode; __u64 other_mode; @@ -1110,16 +1128,17 @@ static void populate_new_aces(char *nacl_base, *pnsize = nsize; } -static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, - struct smb_sid *pownersid, struct smb_sid *pgrpsid, - struct smb_sid *pnownersid, struct smb_sid *pngrpsid) +static int replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *pndacl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_sid *pnownersid, struct smb_sid *pngrpsid, + int *aclflag, u16 *pnsize) { int i; u16 size = 0; struct smb_ace *pntace = NULL; char *acl_base = NULL; u16 src_num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; struct smb_ace *pnntace = NULL; char *nacl_base = NULL; u16 ace_size = 0; @@ -1136,18 +1155,24 @@ static __u16 replace_sids_and_copy_aces(struct smb_acl *pdacl, struct smb_acl *p pntace = (struct smb_ace *) (acl_base + size); pnntace = (struct smb_ace *) (nacl_base + nsize); - if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) + if (pnownersid && compare_sids(&pntace->sid, pownersid) == 0) { ace_size = cifs_copy_ace(pnntace, pntace, pnownersid); - else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) + *aclflag |= CIFS_ACL_DACL; + } else if (pngrpsid && compare_sids(&pntace->sid, pgrpsid) == 0) { ace_size = cifs_copy_ace(pnntace, pntace, pngrpsid); - else + *aclflag |= CIFS_ACL_DACL; + } else { ace_size = cifs_copy_ace(pnntace, pntace, NULL); + } size += le16_to_cpu(pntace->size); nsize += ace_size; + if (nsize > U16_MAX) + return -EOVERFLOW; } - return nsize; + *pnsize = nsize; + return 0; } static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, @@ -1159,7 +1184,7 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, struct smb_ace *pntace = NULL; char *acl_base = NULL; u16 src_num_aces = 0; - u16 nsize = 0; + u32 nsize = 0; struct smb_ace *pnntace = NULL; char *nacl_base = NULL; u16 num_aces = 0; @@ -1210,6 +1235,8 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, nsize += cifs_copy_ace(pnntace, pntace, NULL); num_aces++; + if (nsize > U16_MAX) + return -EOVERFLOW; next_ace: size += le16_to_cpu(pntace->size); @@ -1226,6 +1253,10 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl, } finalize_dacl: + /* The DACL size field is 16-bit on the wire, see MS-DTYP 2.4.5 */ + if (nsize > U16_MAX) + return -EOVERFLOW; + pndacl->num_aces = cpu_to_le16(num_aces); pndacl->size = cpu_to_le16(nsize); @@ -1386,6 +1417,8 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, rc = set_chmod_dacl(dacl_ptr, ndacl_ptr, owner_sid_ptr, group_sid_ptr, pnmode, mode_from_sid, posix); + if (rc) + return rc; sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size); /* copy the non-dacl portion of secdesc */ @@ -1461,9 +1494,12 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd, if (dacloffset) { /* Replace ACEs for old owner with new one */ - size = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, - owner_sid_ptr, group_sid_ptr, - nowner_sid_ptr, ngroup_sid_ptr); + rc = replace_sids_and_copy_aces(dacl_ptr, ndacl_ptr, + owner_sid_ptr, group_sid_ptr, + nowner_sid_ptr, ngroup_sid_ptr, + aclflag, &size); + if (rc) + goto chown_chgrp_exit; ndacl_ptr->size = cpu_to_le16(size); } @@ -1678,7 +1714,7 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, kuid_t uid, kgid_t gid) { int rc = 0; - int aclflag = CIFS_ACL_DACL; /* default flag to set */ + int aclflag = 0; __u32 secdesclen = 0; __u32 nsecdesclen = 0; __u32 dacloffset = 0; @@ -1716,12 +1752,12 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, return rc; } - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID) mode_from_sid = true; else mode_from_sid = false; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UID_FROM_ACL) id_from_sid = true; else id_from_sid = false; @@ -1752,11 +1788,13 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, cifs_put_tlink(tlink); return rc; } - if (mode_from_sid) - nsecdesclen += - le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); - else /* cifsacl */ - nsecdesclen += le16_to_cpu(dacl_ptr->size); + /* + * Worst case: every ACE is rewritten with a new SID of + * SID_MAX_SUB_AUTHORITIES sub-auths -> sizeof(smb_ace) each, + * plus the smb_acl header replace_sids_and_copy_aces() emits. + */ + nsecdesclen += sizeof(struct smb_acl) + + le16_to_cpu(dacl_ptr->num_aces) * sizeof(struct smb_ace); } } @@ -1779,14 +1817,23 @@ id_mode_to_cifs_acl(struct inode *inode, const char *path, __u64 *pnmode, cifs_dbg(NOISY, "build_sec_desc rc: %d\n", rc); - if (ops->set_acl == NULL) - rc = -EOPNOTSUPP; + if (rc != 0) + goto id_mode_to_cifs_acl_exit; - if (!rc) { - /* Set the security descriptor */ - rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); - cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); + if (aclflag == 0) { + cifs_dbg(FYI, "set_cifs_acl aclflag=0, no change mapped\n"); + goto id_mode_to_cifs_acl_exit; + } + + if (ops->set_acl == NULL) { + rc = -EOPNOTSUPP; + goto id_mode_to_cifs_acl_exit; } + + /* Set the security descriptor */ + rc = ops->set_acl(pnntsd, nsecdesclen, inode, path, aclflag); + cifs_dbg(NOISY, "set_cifs_acl rc: %d\n", rc); + id_mode_to_cifs_acl_exit: cifs_put_tlink(tlink); diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 9059c2efbcc05..7bd950b28deb5 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include #include #include "cifsfs.h" @@ -35,10 +37,9 @@ #define DECLARE_GLOBALS_HERE #include "cifsglob.h" #include "cifsproto.h" +#include "smb2proto.h" #include "cifs_debug.h" #include "cifs_fs_sb.h" -#include -#include #include "cifs_spnego.h" #include "fscache.h" #ifdef CONFIG_CIFS_DFS_UPCALL @@ -235,7 +236,7 @@ cifs_read_super(struct super_block *sb) cifs_sb = CIFS_SB(sb); tcon = cifs_sb_master_tcon(cifs_sb); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIXACL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIXACL) sb->s_flags |= SB_POSIXACL; if (tcon->snapshot_time) @@ -311,7 +312,7 @@ cifs_read_super(struct super_block *sb) } #ifdef CONFIG_CIFS_NFSD_EXPORT - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) { cifs_dbg(FYI, "export ops supported\n"); sb->s_export_op = &cifs_export_ops; } @@ -428,7 +429,7 @@ static int cifs_permission(struct mnt_idmap *idmap, cifs_sb = CIFS_SB(inode->i_sb); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_PERM) { if ((mask & MAY_EXEC) && !execute_ok(inode)) return -EACCES; else @@ -576,13 +577,13 @@ cifs_show_cache_flavor(struct seq_file *s, struct cifs_sb_info *cifs_sb) { seq_puts(s, ",cache="); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_STRICT_IO) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_STRICT_IO) seq_puts(s, "strict"); - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DIRECT_IO) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DIRECT_IO) seq_puts(s, "none"); - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RW_CACHE) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE) seq_puts(s, "singleclient"); /* assume only one client access */ - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RO_CACHE) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RO_CACHE) seq_puts(s, "ro"); /* read only caching assumed */ else seq_puts(s, "loose"); @@ -678,14 +679,14 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_printf(s, ",uid=%u", from_kuid_munged(&init_user_ns, cifs_sb->ctx->linux_uid)); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_UID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID) seq_puts(s, ",forceuid"); else seq_puts(s, ",noforceuid"); seq_printf(s, ",gid=%u", from_kgid_munged(&init_user_ns, cifs_sb->ctx->linux_gid)); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_GID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID) seq_puts(s, ",forcegid"); else seq_puts(s, ",noforcegid"); @@ -728,53 +729,53 @@ cifs_show_options(struct seq_file *s, struct dentry *root) seq_puts(s, ",unix"); else seq_puts(s, ",nounix"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_DFS) seq_puts(s, ",nodfs"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) seq_puts(s, ",posixpaths"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) seq_puts(s, ",setuids"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UID_FROM_ACL) seq_puts(s, ",idsfromsid"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) seq_puts(s, ",serverino"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RWPIDFORWARD) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RWPIDFORWARD) seq_puts(s, ",rwpidforward"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) seq_puts(s, ",forcemand"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_XATTR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) seq_puts(s, ",nouser_xattr"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SPECIAL_CHR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SPECIAL_CHR) seq_puts(s, ",mapchars"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SFM_CHR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SFM_CHR) seq_puts(s, ",mapposix"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL) seq_puts(s, ",sfu"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_BRL) seq_puts(s, ",nobrl"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_HANDLE_CACHE) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_HANDLE_CACHE) seq_puts(s, ",nohandlecache"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID) seq_puts(s, ",modefromsid"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) seq_puts(s, ",cifsacl"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM) seq_puts(s, ",dynperm"); if (root->d_sb->s_flags & SB_POSIXACL) seq_puts(s, ",acl"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) seq_puts(s, ",mfsymlinks"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_FSCACHE) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_FSCACHE) seq_puts(s, ",fsc"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOSSYNC) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOSSYNC) seq_puts(s, ",nostrictsync"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_PERM) seq_puts(s, ",noperm"); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_BACKUPUID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_BACKUPUID) seq_printf(s, ",backupuid=%u", from_kuid_munged(&init_user_ns, cifs_sb->ctx->backupuid)); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_BACKUPGID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_BACKUPGID) seq_printf(s, ",backupgid=%u", from_kgid_munged(&init_user_ns, cifs_sb->ctx->backupgid)); @@ -917,7 +918,7 @@ static int cifs_drop_inode(struct inode *inode) struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); /* no serverino => unconditional eviction */ - return !(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) || + return !(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) || inode_generic_drop(inode); } @@ -955,7 +956,7 @@ cifs_get_root(struct smb3_fs_context *ctx, struct super_block *sb) char *s, *p; char sep; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) return dget(sb->s_root); full_path = cifs_build_path_to_root(ctx, cifs_sb, @@ -1364,8 +1365,19 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off, */ lock_two_nondirectories(target_inode, src_inode); - if (len == 0) - len = src_inode->i_size - off; + if (len == 0) { + loff_t src_size = i_size_read(src_inode); + + if (off > src_size) { + rc = -EINVAL; + goto unlock; + } + len = src_size - off; + if (!len) { + rc = 0; + goto unlock; + } + } cifs_dbg(FYI, "clone range\n"); diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h index e9534258d1efd..3d632d22da6fc 100644 --- a/fs/smb/client/cifsfs.h +++ b/fs/smb/client/cifsfs.h @@ -133,9 +133,11 @@ extern ssize_t cifs_file_copychunk_range(unsigned int xid, struct file *dst_file, loff_t destoff, size_t len, unsigned int flags); -extern long cifs_ioctl(struct file *filep, unsigned int cmd, unsigned long arg); -extern void cifs_setsize(struct inode *inode, loff_t offset); +long cifs_ioctl(struct file *filep, unsigned int command, unsigned long arg); +void cifs_setsize(struct inode *inode, loff_t offset); +void cifs_resize_file_locked(struct inode *inode, loff_t offset); +struct fs_context; struct smb3_fs_context; extern struct dentry *cifs_smb3_do_mount(struct file_system_type *fs_type, int flags, struct smb3_fs_context *ctx); diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 69deb29502a8c..1837f4820bd67 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -671,12 +671,6 @@ struct cifs_mnt_data { int flags; }; -static inline unsigned int -get_rfc1002_length(void *buf) -{ - return be32_to_cpu(*((__be32 *)buf)) & 0xffffff; -} - struct TCP_Server_Info { struct list_head tcp_ses_list; struct list_head smb_ses_list; @@ -1546,9 +1540,13 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, #define CIFS_CACHE_RW_FLG (CIFS_CACHE_READ_FLG | CIFS_CACHE_WRITE_FLG) #define CIFS_CACHE_RHW_FLG (CIFS_CACHE_RW_FLG | CIFS_CACHE_HANDLE_FLG) -#define CIFS_CACHE_READ(cinode) ((cinode->oplock & CIFS_CACHE_READ_FLG) || (CIFS_SB(cinode->netfs.inode.i_sb)->mnt_cifs_flags & CIFS_MOUNT_RO_CACHE)) +#define CIFS_CACHE_READ(cinode) \ + (((cinode)->oplock & CIFS_CACHE_READ_FLG) || \ + (cifs_sb_flags(CIFS_SB((cinode)->netfs.inode.i_sb)) & CIFS_MOUNT_RO_CACHE)) #define CIFS_CACHE_HANDLE(cinode) (cinode->oplock & CIFS_CACHE_HANDLE_FLG) -#define CIFS_CACHE_WRITE(cinode) ((cinode->oplock & CIFS_CACHE_WRITE_FLG) || (CIFS_SB(cinode->netfs.inode.i_sb)->mnt_cifs_flags & CIFS_MOUNT_RW_CACHE)) +#define CIFS_CACHE_WRITE(cinode) \ + (((cinode)->oplock & CIFS_CACHE_WRITE_FLG) || \ + (cifs_sb_flags(CIFS_SB((cinode)->netfs.inode.i_sb)) & CIFS_MOUNT_RW_CACHE)) /* * One of these for each file inode @@ -1611,7 +1609,7 @@ CIFS_FILE_SB(struct file *file) static inline char CIFS_DIR_SEP(const struct cifs_sb_info *cifs_sb) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) return '/'; else return '\\'; @@ -2116,11 +2114,9 @@ extern mempool_t cifs_io_subrequest_pool; /* Operations for different SMB versions */ #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY -extern struct smb_version_operations smb1_operations; -extern struct smb_version_values smb1_values; extern struct smb_version_operations smb20_operations; extern struct smb_version_values smb20_values; -#endif /* CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ extern struct smb_version_operations smb21_operations; extern struct smb_version_values smb21_values; extern struct smb_version_values smbdefault_values; @@ -2391,4 +2387,10 @@ static inline int cifs_open_create_options(unsigned int oflags, int opts) */ #define CIFS_INO_BLOCKS(size) DIV_ROUND_UP_ULL((u64)(size), 512) +struct cifs_calc_sig_ctx { + struct md5_ctx *md5; + struct hmac_sha256_ctx *hmac; + struct shash_desc *shash; +}; + #endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifspdu.h b/fs/smb/client/cifspdu.h index d9cf7db0ac35e..c86a329e58221 100644 --- a/fs/smb/client/cifspdu.h +++ b/fs/smb/client/cifspdu.h @@ -1323,14 +1323,6 @@ typedef struct smb_com_ntransact_rsp { /* parms and data follow */ } __attribute__((packed)) NTRANSACT_RSP; -/* See MS-SMB 2.2.7.2.1.1 */ -struct srv_copychunk { - __le64 SourceOffset; - __le64 DestinationOffset; - __le32 CopyLength; - __u32 Reserved; -} __packed; - typedef struct smb_com_transaction_ioctl_req { struct smb_hdr hdr; /* wct = 23 */ __u8 MaxSetupCount; diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index 4be719c9fd520..325408073888e 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -14,6 +14,7 @@ #ifdef CONFIG_CIFS_DFS_UPCALL #include "dfs_cache.h" #endif +#include "smb1proto.h" struct statfs; struct smb_rqst; @@ -82,7 +83,6 @@ extern char *cifs_build_path_to_root(struct smb3_fs_context *ctx, struct cifs_sb_info *cifs_sb, struct cifs_tcon *tcon, int add_treename); -extern char *build_wildcard_path_from_dentry(struct dentry *direntry); char *cifs_build_devname(char *nodename, const char *prepath); extern void delete_mid(struct mid_q_entry *mid); void __release_mid(struct kref *refcount); @@ -166,8 +166,6 @@ extern bool is_valid_oplock_break(char *, struct TCP_Server_Info *); extern bool backup_cred(struct cifs_sb_info *); extern bool is_size_safe_to_change(struct cifsInodeInfo *cifsInode, __u64 eof, bool from_readdir); -extern void cifs_update_eof(struct cifsInodeInfo *cifsi, loff_t offset, - unsigned int bytes_written); void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, ssize_t result); extern struct cifsFileInfo *find_writable_file(struct cifsInodeInfo *, int); extern int cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, @@ -194,8 +192,6 @@ extern void header_assemble(struct smb_hdr *, char /* command */ , extern int small_smb_init_no_tc(const int smb_cmd, const int wct, struct cifs_ses *ses, void **request_buf); -extern enum securityEnum select_sectype(struct TCP_Server_Info *server, - enum securityEnum requested); extern int CIFS_SessSetup(const unsigned int xid, struct cifs_ses *ses, struct TCP_Server_Info *server, const struct nls_table *nls_cp); @@ -435,16 +431,6 @@ extern int CIFSSMBSetFileSize(const unsigned int xid, struct cifs_tcon *tcon, struct cifsFileInfo *cfile, __u64 size, bool set_allocation); -struct cifs_unix_set_info_args { - __u64 ctime; - __u64 atime; - __u64 mtime; - __u64 mode; - kuid_t uid; - kgid_t gid; - dev_t device; -}; - extern int CIFSSMBUnixSetFileInfo(const unsigned int xid, struct cifs_tcon *tcon, const struct cifs_unix_set_info_args *args, @@ -604,7 +590,7 @@ extern int cifs_do_set_acl(const unsigned int xid, struct cifs_tcon *tcon, const struct nls_table *nls_codepage, int remap); extern int CIFSGetExtAttr(const unsigned int xid, struct cifs_tcon *tcon, const int netfid, __u64 *pExtAttrBits, __u64 *pMask); -#endif /* CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ extern void cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb); extern bool couldbe_mf_symlink(const struct cifs_fattr *fattr); extern int check_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, @@ -636,11 +622,6 @@ int cifs_create_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_sb_info *cifs_sb, const unsigned char *path, char *pbuf, unsigned int *pbytes_written); -struct cifs_calc_sig_ctx { - struct md5_ctx *md5; - struct hmac_sha256_ctx *hmac; - struct shash_desc *shash; -}; int __cifs_calc_signature(struct smb_rqst *rqst, struct TCP_Server_Info *server, char *signature, struct cifs_calc_sig_ctx *ctx); enum securityEnum cifs_select_sectype(struct TCP_Server_Info *, @@ -651,7 +632,6 @@ void cifs_free_hash(struct shash_desc **sdesc); int cifs_try_adding_channels(struct cifs_ses *ses); bool is_ses_using_iface(struct cifs_ses *ses, struct cifs_server_iface *iface); -void cifs_ses_mark_for_reconnect(struct cifs_ses *ses); int cifs_ses_get_chan_index(struct cifs_ses *ses, @@ -790,4 +770,9 @@ static inline void cifs_free_open_info(struct cifs_open_info_data *data) memset(data, 0, sizeof(*data)); } +#define smb_EIO2(trace, info, info2) ({ \ + trace_smb3_eio(trace, info, info2); \ + -EIO; \ +}) + #endif /* _CIFSPROTO_H */ diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index 7694ca283fdc3..5956d6657383e 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -527,6 +527,152 @@ CIFSSMBNegotiate(const unsigned int xid, return rc; } +/* + * Issue a TREE_CONNECT request. + */ +int +CIFSTCon(const unsigned int xid, struct cifs_ses *ses, + const char *tree, struct cifs_tcon *tcon, + const struct nls_table *nls_codepage) +{ + struct smb_hdr *smb_buffer; + struct smb_hdr *smb_buffer_response; + TCONX_REQ *pSMB; + TCONX_RSP *pSMBr; + unsigned char *bcc_ptr; + int rc = 0; + int length; + __u16 bytes_left, count; + + if (ses == NULL) + return -EIO; + + smb_buffer = cifs_buf_get(); + if (smb_buffer == NULL) + return -ENOMEM; + + smb_buffer_response = smb_buffer; + + header_assemble(smb_buffer, SMB_COM_TREE_CONNECT_ANDX, + NULL /*no tid */, 4 /*wct */); + + smb_buffer->Mid = get_next_mid(ses->server); + smb_buffer->Uid = ses->Suid; + pSMB = (TCONX_REQ *) smb_buffer; + pSMBr = (TCONX_RSP *) smb_buffer_response; + + pSMB->AndXCommand = 0xFF; + pSMB->Flags = cpu_to_le16(TCON_EXTENDED_SECINFO); + bcc_ptr = &pSMB->Password[0]; + + pSMB->PasswordLength = cpu_to_le16(1); /* minimum */ + *bcc_ptr = 0; /* password is null byte */ + bcc_ptr++; /* skip password */ + /* already aligned so no need to do it below */ + + if (ses->server->sign) + smb_buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + if (ses->capabilities & CAP_STATUS32) + smb_buffer->Flags2 |= SMBFLG2_ERR_STATUS; + + if (ses->capabilities & CAP_DFS) + smb_buffer->Flags2 |= SMBFLG2_DFS; + + if (ses->capabilities & CAP_UNICODE) { + smb_buffer->Flags2 |= SMBFLG2_UNICODE; + length = + cifs_strtoUTF16((__le16 *) bcc_ptr, tree, + 6 /* max utf8 char length in bytes */ * + (/* server len*/ + 256 /* share len */), nls_codepage); + bcc_ptr += 2 * length; /* convert num 16 bit words to bytes */ + bcc_ptr += 2; /* skip trailing null */ + } else { /* ASCII */ + strcpy(bcc_ptr, tree); + bcc_ptr += strlen(tree) + 1; + } + strcpy(bcc_ptr, "?????"); + bcc_ptr += strlen("?????"); + bcc_ptr += 1; + count = bcc_ptr - &pSMB->Password[0]; + be32_add_cpu(&pSMB->hdr.smb_buf_length, count); + pSMB->ByteCount = cpu_to_le16(count); + + rc = SendReceive(xid, ses, smb_buffer, smb_buffer_response, &length, + 0); + + /* above now done in SendReceive */ + if (rc == 0) { + bool is_unicode; + + tcon->tid = smb_buffer_response->Tid; + bcc_ptr = pByteArea(smb_buffer_response); + bytes_left = get_bcc(smb_buffer_response); + if (bytes_left < 2) { + rc = smb_EIO2(smb_eio_trace_tcon_bcc_too_small, + bytes_left, 2); + goto out; + } + length = strnlen(bcc_ptr, bytes_left - 2); + if (smb_buffer->Flags2 & SMBFLG2_UNICODE) + is_unicode = true; + else + is_unicode = false; + + + /* skip service field (NB: this field is always ASCII) */ + if (length == 3) { + if ((bcc_ptr[0] == 'I') && (bcc_ptr[1] == 'P') && + (bcc_ptr[2] == 'C')) { + cifs_dbg(FYI, "IPC connection\n"); + tcon->ipc = true; + tcon->pipe = true; + } + } else if (length == 2) { + if ((bcc_ptr[0] == 'A') && (bcc_ptr[1] == ':')) { + /* the most common case */ + cifs_dbg(FYI, "disk share connection\n"); + } + } + bcc_ptr += length + 1; + bytes_left -= (length + 1); + strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); + + /* mostly informational -- no need to fail on error here */ + kfree(tcon->nativeFileSystem); + tcon->nativeFileSystem = cifs_strndup_from_utf16(bcc_ptr, + bytes_left, is_unicode, + nls_codepage); + + cifs_dbg(FYI, "nativeFileSystem=%s\n", tcon->nativeFileSystem); + + if ((smb_buffer_response->WordCount == 3) || + (smb_buffer_response->WordCount == 7)) + /* field is in same location */ + tcon->Flags = le16_to_cpu(pSMBr->OptionalSupport); + else + tcon->Flags = 0; + cifs_dbg(FYI, "Tcon flags: 0x%x\n", tcon->Flags); + + /* + * reset_cifs_unix_caps calls QFSInfo which requires + * need_reconnect to be false, but we would not need to call + * reset_caps if this were not a reconnect case so must check + * need_reconnect flag here. The caller will also clear + * need_reconnect when tcon was successful but needed to be + * cleared earlier in the case of unix extensions reconnect + */ + if (tcon->need_reconnect && tcon->unix_ext) { + cifs_dbg(FYI, "resetting caps for %s\n", tcon->tree_name); + tcon->need_reconnect = false; + reset_cifs_unix_caps(xid, tcon, NULL, NULL); + } + } +out: + cifs_buf_release(smb_buffer); + return rc; +} + int CIFSSMBTDis(const unsigned int xid, struct cifs_tcon *tcon) { @@ -615,7 +761,7 @@ CIFSSMBEcho(struct TCP_Server_Info *server) iov[0].iov_len = 4; iov[0].iov_base = smb; - iov[1].iov_len = get_rfc1002_length(smb); + iov[1].iov_len = get_rfc1002_len(smb); iov[1].iov_base = (char *)smb + 4; rc = cifs_call_async(server, &rqst, NULL, cifs_echo_callback, NULL, @@ -1459,7 +1605,7 @@ cifs_async_readv(struct cifs_io_subrequest *rdata) rdata->iov[0].iov_base = smb; rdata->iov[0].iov_len = 4; rdata->iov[1].iov_base = (char *)smb + 4; - rdata->iov[1].iov_len = get_rfc1002_length(smb); + rdata->iov[1].iov_len = get_rfc1002_len(smb); trace_smb3_read_enter(rdata->rreq->debug_id, rdata->subreq.debug_index, @@ -1515,8 +1661,10 @@ CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); /* tcon and ses pointer are checked in smb_init */ - if (tcon->ses->server == NULL) + if (!tcon->ses->server) { + cifs_small_buf_release(pSMB); return -ECONNABORTED; + } pSMB->AndXCommand = 0xFF; /* none */ pSMB->Fid = netfid; @@ -1545,6 +1693,14 @@ CIFSSMBRead(const unsigned int xid, struct cifs_io_parms *io_parms, pSMBr = (READ_RSP *)rsp_iov.iov_base; if (rc) { cifs_dbg(VFS, "Send error in read = %d\n", rc); + } else if (rsp_iov.iov_len < tcon->ses->server->vals->read_rsp_size) { + /* check that the received response can hold a whole READ_RSP */ + cifs_dbg(FYI, "%s: server returned short header. got=%zu expected=%zu\n", + __func__, rsp_iov.iov_len, + tcon->ses->server->vals->read_rsp_size); + rc = smb_EIO2(smb_eio_trace_read_rsp_short, + rsp_iov.iov_len, tcon->ses->server->vals->read_rsp_size); + *nbytes = 0; } else { int data_length = le16_to_cpu(pSMBr->DataLengthHigh); data_length = data_length << 16; @@ -1628,8 +1784,10 @@ CIFSSMBWrite(const unsigned int xid, struct cifs_io_parms *io_parms, pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); /* tcon and ses pointer are checked in smb_init */ - if (tcon->ses->server == NULL) + if (!tcon->ses->server) { + cifs_buf_release(pSMB); return -ECONNABORTED; + } pSMB->AndXCommand = 0xFF; /* none */ pSMB->Fid = netfid; @@ -1831,7 +1989,7 @@ cifs_async_writev(struct cifs_io_subrequest *wdata) /* 4 for RFC1001 length + 1 for BCC */ iov[0].iov_len = 4; iov[0].iov_base = smb; - iov[1].iov_len = get_rfc1002_length(smb) + 1; + iov[1].iov_len = get_rfc1002_len(smb) + 1; iov[1].iov_base = (char *)smb + 4; rqst.rq_iov = iov; @@ -1908,8 +2066,10 @@ CIFSSMBWrite2(const unsigned int xid, struct cifs_io_parms *io_parms, pSMB->hdr.PidHigh = cpu_to_le16((__u16)(pid >> 16)); /* tcon and ses pointer are checked in smb_init */ - if (tcon->ses->server == NULL) + if (!tcon->ses->server) { + cifs_small_buf_release(pSMB); return -ECONNABORTED; + } pSMB->AndXCommand = 0xFF; /* none */ pSMB->Fid = netfid; diff --git a/fs/smb/client/compress.c b/fs/smb/client/compress.c index db709f5cd2e1f..fb2a48f1d2ad0 100644 --- a/fs/smb/client/compress.c +++ b/fs/smb/client/compress.c @@ -44,7 +44,7 @@ struct bucket { unsigned int count; }; -/** +/* * has_low_entropy() - Compute Shannon entropy of the sampled data. * @bkt: Bytes counts of the sample. * @slen: Size of the sample. @@ -82,7 +82,7 @@ static bool has_low_entropy(struct bucket *bkt, size_t slen) #define BYTE_DIST_BAD 0 #define BYTE_DIST_GOOD 1 #define BYTE_DIST_MAYBE 2 -/** +/* * calc_byte_distribution() - Compute byte distribution on the sampled data. * @bkt: Byte counts of the sample. * @slen: Size of the sample. @@ -182,7 +182,7 @@ static int collect_sample(const struct iov_iter *source, ssize_t max, u8 *sample return s; } -/** +/* * is_compressible() - Determines if a chunk of data is compressible. * @data: Iterator containing uncompressed data. * @@ -261,6 +261,21 @@ static bool is_compressible(const struct iov_iter *data) return ret; } +/* + * should_compress() - Determines if a request (write) or the response to a + * request (read) should be compressed. + * @tcon: tcon of the request is being sent to + * @rqst: request to evaluate + * + * Return: true iff: + * - compression was successfully negotiated with server + * - server has enabled compression for the share + * - it's a read or write request + * - (write only) request length is >= SMB_COMPRESS_MIN_LEN + * - (write only) is_compressible() returns 1 + * + * Return false otherwise. + */ bool should_compress(const struct cifs_tcon *tcon, const struct smb_rqst *rq) { const struct smb2_hdr *shdr = rq->rq_iov->iov_base; diff --git a/fs/smb/client/compress.h b/fs/smb/client/compress.h index f3ed1d3e52fbf..63aea32fbe921 100644 --- a/fs/smb/client/compress.h +++ b/fs/smb/client/compress.h @@ -29,26 +29,11 @@ #ifdef CONFIG_CIFS_COMPRESSION typedef int (*compress_send_fn)(struct TCP_Server_Info *, int, struct smb_rqst *); -int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, compress_send_fn send_fn); -/** - * should_compress() - Determines if a request (write) or the response to a - * request (read) should be compressed. - * @tcon: tcon of the request is being sent to - * @rqst: request to evaluate - * - * Return: true iff: - * - compression was successfully negotiated with server - * - server has enabled compression for the share - * - it's a read or write request - * - (write only) request length is >= SMB_COMPRESS_MIN_LEN - * - (write only) is_compressible() returns 1 - * - * Return false otherwise. - */ +int smb_compress(struct TCP_Server_Info *server, struct smb_rqst *rq, compress_send_fn send_fn); bool should_compress(const struct cifs_tcon *tcon, const struct smb_rqst *rq); -/** +/* * smb_compress_alg_valid() - Validate a compression algorithm. * @alg: Compression algorithm to check. * @valid_none: Conditional check whether NONE algorithm should be diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 2ee2199d2a6a2..ae4d3f7400653 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -1295,7 +1295,7 @@ cifs_demultiplex_thread(void *p) * The right amount was read from socket - 4 bytes, * so we can now interpret the length field. */ - pdu_length = get_rfc1002_length(buf); + pdu_length = get_rfc1002_len(buf); cifs_dbg(FYI, "RFC1002 header 0x%x\n", pdu_length); if (!is_smb_response(server, buf[0])) @@ -1871,14 +1871,6 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, * this will succeed. No need for try_module_get(). */ __module_get(THIS_MODULE); - tcp_ses->tsk = kthread_run(cifs_demultiplex_thread, - tcp_ses, "cifsd"); - if (IS_ERR(tcp_ses->tsk)) { - rc = PTR_ERR(tcp_ses->tsk); - cifs_dbg(VFS, "error %d create cifsd thread\n", rc); - module_put(THIS_MODULE); - goto out_err_crypto_release; - } tcp_ses->min_offload = ctx->min_offload; tcp_ses->retrans = ctx->retrans; /* @@ -1886,9 +1878,7 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, * to the struct since the kernel thread not created yet * no need to spinlock this update of tcpStatus */ - spin_lock(&tcp_ses->srv_lock); tcp_ses->tcpStatus = CifsNeedNegotiate; - spin_unlock(&tcp_ses->srv_lock); if ((ctx->max_credits < 20) || (ctx->max_credits > 60000)) tcp_ses->max_credits = SMB2_MAX_CREDITS_AVAILABLE; @@ -1897,7 +1887,16 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, tcp_ses->nr_targets = 1; tcp_ses->ignore_signature = ctx->ignore_signature; - /* thread spawned, put it on the list */ + + tcp_ses->tsk = kthread_create(cifs_demultiplex_thread, + tcp_ses, "cifsd"); + if (IS_ERR(tcp_ses->tsk)) { + rc = PTR_ERR(tcp_ses->tsk); + cifs_dbg(VFS, "error %d create cifsd thread\n", rc); + module_put(THIS_MODULE); + goto out_err_crypto_release; + } + /* thread created, put it on the list */ spin_lock(&cifs_tcp_ses_lock); list_add(&tcp_ses->tcp_ses_list, &cifs_tcp_ses_list); spin_unlock(&cifs_tcp_ses_lock); @@ -1905,6 +1904,12 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, /* queue echo request delayed work */ queue_delayed_work(cifsiod_wq, &tcp_ses->echo, tcp_ses->echo_interval); + /* + * Use split create/wake logic to ensure that tcp_ses is fully populated + * and tcp_ses->tsk is valid + */ + wake_up_process(tcp_ses->tsk); + return tcp_ses; out_err_crypto_release: @@ -2917,8 +2922,8 @@ compare_mount_options(struct super_block *sb, struct cifs_mnt_data *mnt_data) { struct cifs_sb_info *old = CIFS_SB(sb); struct cifs_sb_info *new = mnt_data->cifs_sb; - unsigned int oldflags = old->mnt_cifs_flags & CIFS_MOUNT_MASK; - unsigned int newflags = new->mnt_cifs_flags & CIFS_MOUNT_MASK; + unsigned int oldflags = cifs_sb_flags(old) & CIFS_MOUNT_MASK; + unsigned int newflags = cifs_sb_flags(new) & CIFS_MOUNT_MASK; if ((sb->s_flags & CIFS_MS_MASK) != (mnt_data->flags & CIFS_MS_MASK)) return 0; @@ -2973,9 +2978,9 @@ static int match_prepath(struct super_block *sb, struct smb3_fs_context *ctx = mnt_data->ctx; struct cifs_sb_info *old = CIFS_SB(sb); struct cifs_sb_info *new = mnt_data->cifs_sb; - bool old_set = (old->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) && + bool old_set = (cifs_sb_flags(old) & CIFS_MOUNT_USE_PREFIX_PATH) && old->prepath; - bool new_set = (new->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) && + bool new_set = (cifs_sb_flags(new) & CIFS_MOUNT_USE_PREFIX_PATH) && new->prepath; if (tcon->origin_fullpath && @@ -3006,7 +3011,7 @@ cifs_match_super(struct super_block *sb, void *data) cifs_sb = CIFS_SB(sb); /* We do not want to use a superblock that has been shutdown */ - if (CIFS_MOUNT_SHUTDOWN & cifs_sb->mnt_cifs_flags) { + if (CIFS_MOUNT_SHUTDOWN & cifs_sb_flags(cifs_sb)) { spin_unlock(&cifs_tcp_ses_lock); return 0; } @@ -3531,8 +3536,7 @@ void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, else if (CIFS_UNIX_POSIX_ACL_CAP & cap) { cifs_dbg(FYI, "negotiated posix acl support\n"); if (cifs_sb) - cifs_sb->mnt_cifs_flags |= - CIFS_MOUNT_POSIXACL; + atomic_or(CIFS_MOUNT_POSIXACL, &cifs_sb->mnt_cifs_flags); } if (ctx && ctx->posix_paths == 0) @@ -3540,8 +3544,7 @@ void reset_cifs_unix_caps(unsigned int xid, struct cifs_tcon *tcon, else if (cap & CIFS_UNIX_POSIX_PATHNAMES_CAP) { cifs_dbg(FYI, "negotiate posix pathnames\n"); if (cifs_sb) - cifs_sb->mnt_cifs_flags |= - CIFS_MOUNT_POSIX_PATHS; + atomic_or(CIFS_MOUNT_POSIX_PATHS, &cifs_sb->mnt_cifs_flags); } cifs_dbg(FYI, "Negotiate caps 0x%x\n", (int)cap); @@ -3609,11 +3612,10 @@ int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) cifs_dbg(FYI, "mounting share using direct i/o\n"); if (ctx->cache_ro) { cifs_dbg(VFS, "mounting share with read only caching. Ensure that the share will not be modified while in use.\n"); - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_RO_CACHE; + atomic_or(CIFS_MOUNT_RO_CACHE, &cifs_sb->mnt_cifs_flags); } else if (ctx->cache_rw) { cifs_dbg(VFS, "mounting share in single client RW caching mode. Ensure that no other systems will be accessing the share.\n"); - cifs_sb->mnt_cifs_flags |= (CIFS_MOUNT_RO_CACHE | - CIFS_MOUNT_RW_CACHE); + atomic_or(CIFS_MOUNT_RO_CACHE | CIFS_MOUNT_RW_CACHE, &cifs_sb->mnt_cifs_flags); } if ((ctx->cifs_acl) && (ctx->dynperm)) @@ -3623,7 +3625,7 @@ int cifs_setup_cifs_sb(struct cifs_sb_info *cifs_sb) cifs_sb->prepath = kstrdup(ctx->prepath, GFP_KERNEL); if (cifs_sb->prepath == NULL) return -ENOMEM; - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); } return 0; @@ -3643,7 +3645,7 @@ void cifs_mount_put_conns(struct cifs_mount_ctx *mnt_ctx) mnt_ctx->ses = NULL; mnt_ctx->tcon = NULL; mnt_ctx->server = NULL; - mnt_ctx->cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_POSIX_PATHS; + atomic_andnot(CIFS_MOUNT_POSIX_PATHS, &mnt_ctx->cifs_sb->mnt_cifs_flags); free_xid(mnt_ctx->xid); } @@ -3724,9 +3726,9 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) * path (i.e., do not remap / and \ and do not map any special characters) */ if (tcon->posix_extensions) { - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_POSIX_PATHS; - cifs_sb->mnt_cifs_flags &= ~(CIFS_MOUNT_MAP_SFM_CHR | - CIFS_MOUNT_MAP_SPECIAL_CHR); + atomic_or(CIFS_MOUNT_POSIX_PATHS, &cifs_sb->mnt_cifs_flags); + atomic_andnot(CIFS_MOUNT_MAP_SFM_CHR | CIFS_MOUNT_MAP_SPECIAL_CHR, + &cifs_sb->mnt_cifs_flags); } #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY @@ -3753,11 +3755,11 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) /* do not care if a following call succeed - informational */ if (!tcon->pipe && server->ops->qfs_tcon) { server->ops->qfs_tcon(mnt_ctx->xid, tcon, cifs_sb); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RO_CACHE) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RO_CACHE) { if (tcon->fsDevInfo.DeviceCharacteristics & cpu_to_le32(FILE_READ_ONLY_DEVICE)) cifs_dbg(VFS, "mounted to read only share\n"); - else if ((cifs_sb->mnt_cifs_flags & + else if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE) == 0) cifs_dbg(VFS, "read only mount of RW share\n"); /* no need to log a RW mount of a typical RW share */ @@ -3770,7 +3772,7 @@ int cifs_mount_get_tcon(struct cifs_mount_ctx *mnt_ctx) * Inside cifs_fscache_get_super_cookie it checks * that we do not get super cookie twice. */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_FSCACHE) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_FSCACHE) cifs_fscache_get_super_cookie(tcon); out: @@ -3893,7 +3895,7 @@ int cifs_is_path_remote(struct cifs_mount_ctx *mnt_ctx) cifs_sb, full_path, tcon->Flags & SMB_SHARE_IS_IN_DFS); if (rc != 0) { cifs_server_dbg(VFS, "cannot query dirs between root and final path, enabling CIFS_MOUNT_USE_PREFIX_PATH\n"); - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); rc = 0; } } @@ -3924,7 +3926,7 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) * Force the use of prefix path to support failover on DFS paths that resolve to targets * that have different prefix paths. */ - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); kfree(cifs_sb->prepath); cifs_sb->prepath = ctx->prepath; ctx->prepath = NULL; @@ -3987,148 +3989,6 @@ int cifs_mount(struct cifs_sb_info *cifs_sb, struct smb3_fs_context *ctx) } #endif -#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY -/* - * Issue a TREE_CONNECT request. - */ -int -CIFSTCon(const unsigned int xid, struct cifs_ses *ses, - const char *tree, struct cifs_tcon *tcon, - const struct nls_table *nls_codepage) -{ - struct smb_hdr *smb_buffer; - struct smb_hdr *smb_buffer_response; - TCONX_REQ *pSMB; - TCONX_RSP *pSMBr; - unsigned char *bcc_ptr; - int rc = 0; - int length; - __u16 bytes_left, count; - - if (ses == NULL) - return -EIO; - - smb_buffer = cifs_buf_get(); - if (smb_buffer == NULL) - return -ENOMEM; - - smb_buffer_response = smb_buffer; - - header_assemble(smb_buffer, SMB_COM_TREE_CONNECT_ANDX, - NULL /*no tid */, 4 /*wct */); - - smb_buffer->Mid = get_next_mid(ses->server); - smb_buffer->Uid = ses->Suid; - pSMB = (TCONX_REQ *) smb_buffer; - pSMBr = (TCONX_RSP *) smb_buffer_response; - - pSMB->AndXCommand = 0xFF; - pSMB->Flags = cpu_to_le16(TCON_EXTENDED_SECINFO); - bcc_ptr = &pSMB->Password[0]; - - pSMB->PasswordLength = cpu_to_le16(1); /* minimum */ - *bcc_ptr = 0; /* password is null byte */ - bcc_ptr++; /* skip password */ - /* already aligned so no need to do it below */ - - if (ses->server->sign) - smb_buffer->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; - - if (ses->capabilities & CAP_STATUS32) - smb_buffer->Flags2 |= SMBFLG2_ERR_STATUS; - - if (ses->capabilities & CAP_DFS) - smb_buffer->Flags2 |= SMBFLG2_DFS; - - if (ses->capabilities & CAP_UNICODE) { - smb_buffer->Flags2 |= SMBFLG2_UNICODE; - length = - cifs_strtoUTF16((__le16 *) bcc_ptr, tree, - 6 /* max utf8 char length in bytes */ * - (/* server len*/ + 256 /* share len */), nls_codepage); - bcc_ptr += 2 * length; /* convert num 16 bit words to bytes */ - bcc_ptr += 2; /* skip trailing null */ - } else { /* ASCII */ - strcpy(bcc_ptr, tree); - bcc_ptr += strlen(tree) + 1; - } - strcpy(bcc_ptr, "?????"); - bcc_ptr += strlen("?????"); - bcc_ptr += 1; - count = bcc_ptr - &pSMB->Password[0]; - be32_add_cpu(&pSMB->hdr.smb_buf_length, count); - pSMB->ByteCount = cpu_to_le16(count); - - rc = SendReceive(xid, ses, smb_buffer, smb_buffer_response, &length, - 0); - - /* above now done in SendReceive */ - if (rc == 0) { - bool is_unicode; - - tcon->tid = smb_buffer_response->Tid; - bcc_ptr = pByteArea(smb_buffer_response); - bytes_left = get_bcc(smb_buffer_response); - length = strnlen(bcc_ptr, bytes_left - 2); - if (smb_buffer->Flags2 & SMBFLG2_UNICODE) - is_unicode = true; - else - is_unicode = false; - - - /* skip service field (NB: this field is always ASCII) */ - if (length == 3) { - if ((bcc_ptr[0] == 'I') && (bcc_ptr[1] == 'P') && - (bcc_ptr[2] == 'C')) { - cifs_dbg(FYI, "IPC connection\n"); - tcon->ipc = true; - tcon->pipe = true; - } - } else if (length == 2) { - if ((bcc_ptr[0] == 'A') && (bcc_ptr[1] == ':')) { - /* the most common case */ - cifs_dbg(FYI, "disk share connection\n"); - } - } - bcc_ptr += length + 1; - bytes_left -= (length + 1); - strscpy(tcon->tree_name, tree, sizeof(tcon->tree_name)); - - /* mostly informational -- no need to fail on error here */ - kfree(tcon->nativeFileSystem); - tcon->nativeFileSystem = cifs_strndup_from_utf16(bcc_ptr, - bytes_left, is_unicode, - nls_codepage); - - cifs_dbg(FYI, "nativeFileSystem=%s\n", tcon->nativeFileSystem); - - if ((smb_buffer_response->WordCount == 3) || - (smb_buffer_response->WordCount == 7)) - /* field is in same location */ - tcon->Flags = le16_to_cpu(pSMBr->OptionalSupport); - else - tcon->Flags = 0; - cifs_dbg(FYI, "Tcon flags: 0x%x\n", tcon->Flags); - - /* - * reset_cifs_unix_caps calls QFSInfo which requires - * need_reconnect to be false, but we would not need to call - * reset_caps if this were not a reconnect case so must check - * need_reconnect flag here. The caller will also clear - * need_reconnect when tcon was successful but needed to be - * cleared earlier in the case of unix extensions reconnect - */ - if (tcon->need_reconnect && tcon->unix_ext) { - cifs_dbg(FYI, "resetting caps for %s\n", tcon->tree_name); - tcon->need_reconnect = false; - reset_cifs_unix_caps(xid, tcon, NULL, NULL); - } - } - cifs_buf_release(smb_buffer); - return rc; -} -#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ - static void delayed_free(struct rcu_head *p) { struct cifs_sb_info *cifs_sb = container_of(p, struct cifs_sb_info, rcu); @@ -4540,7 +4400,7 @@ cifs_sb_tlink(struct cifs_sb_info *cifs_sb) kuid_t fsuid = current_fsuid(); int err; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MULTIUSER)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) return cifs_get_tlink(cifs_sb_master_tlink(cifs_sb)); spin_lock(&cifs_sb->tlink_tree_lock); diff --git a/fs/smb/client/dfs_cache.c b/fs/smb/client/dfs_cache.c index f2ad0ccd08a77..ce9c0dc27b944 100644 --- a/fs/smb/client/dfs_cache.c +++ b/fs/smb/client/dfs_cache.c @@ -122,6 +122,8 @@ static inline void free_tgts(struct cache_entry *ce) kfree(t->name); kfree(t); } + + WRITE_ONCE(ce->tgthint, NULL); } static inline void flush_cache_ent(struct cache_entry *ce) @@ -869,13 +871,22 @@ int dfs_cache_find(const unsigned int xid, struct cifs_ses *ses, const struct nl goto out_free_path; } - if (ref) - rc = setup_referral(path, ce, ref, get_tgt_name(ce)); - else + if (ref) { + char *target = get_tgt_name(ce); + + if (IS_ERR(target)) { + rc = PTR_ERR(target); + goto out_unlock; + } + rc = setup_referral(path, ce, ref, target); + } else { rc = 0; + } + if (!rc && tgt_list) rc = get_targets(ce, tgt_list); +out_unlock: up_read(&htable_rw_lock); out_free_path: @@ -915,10 +926,17 @@ int dfs_cache_noreq_find(const char *path, struct dfs_info3_param *ref, goto out_unlock; } - if (ref) - rc = setup_referral(path, ce, ref, get_tgt_name(ce)); - else + if (ref) { + char *target = get_tgt_name(ce); + + if (IS_ERR(target)) { + rc = PTR_ERR(target); + goto out_unlock; + } + rc = setup_referral(path, ce, ref, target); + } else { rc = 0; + } if (!rc && tgt_list) rc = get_targets(ce, tgt_list); @@ -959,7 +977,8 @@ void dfs_cache_noreq_update_tgthint(const char *path, const struct dfs_cache_tgt t = READ_ONCE(ce->tgthint); - if (unlikely(!strcasecmp(it->it_name, t->name))) + /* Check 't' in case ce->tgthint was cleared by free_tgts() */ + if (t && unlikely(!strcasecmp(it->it_name, t->name))) goto out_unlock; list_for_each_entry(t, &ce->tlist, list) { @@ -1333,7 +1352,7 @@ int dfs_cache_remount_fs(struct cifs_sb_info *cifs_sb) * Force the use of prefix path to support failover on DFS paths that resolve to targets * that have different prefix paths. */ - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); refresh_tcon_referral(tcon, true); return 0; diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c index 50c7c6ec068f9..30368ddd0a1e6 100644 --- a/fs/smb/client/dir.c +++ b/fs/smb/client/dir.c @@ -97,7 +97,7 @@ char *__build_path_from_dentry_optional_prefix(struct dentry *direntry, void *pa else dfsplen = 0; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) pplen = cifs_sb->prepath ? strlen(cifs_sb->prepath) + 1 : 0; s = dentry_path_raw(direntry, page, PATH_MAX); @@ -124,7 +124,7 @@ char *__build_path_from_dentry_optional_prefix(struct dentry *direntry, void *pa if (dfsplen) { s -= dfsplen; memcpy(s, tree, dfsplen); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) { int i; for (i = 0; i < dfsplen; i++) { if (s[i] == '\\') @@ -161,7 +161,7 @@ check_name(struct dentry *direntry, struct cifs_tcon *tcon) le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength))) return -ENAMETOOLONG; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS)) { for (i = 0; i < direntry->d_name.len; i++) { if (direntry->d_name.name[i] == '\\') { cifs_dbg(FYI, "Invalid file name\n"); @@ -376,7 +376,7 @@ static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned .device = 0, }; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) { args.uid = current_fsuid(); if (inode->i_mode & S_ISGID) args.gid = inode->i_gid; @@ -413,9 +413,9 @@ static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned if (server->ops->set_lease_key) server->ops->set_lease_key(newinode, fid); if ((*oplock & CIFS_CREATE_ACTION) && S_ISREG(newinode->i_mode)) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM) newinode->i_mode = mode; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) { newinode->i_uid = current_fsuid(); if (inode->i_mode & S_ISGID) newinode->i_gid = inode->i_gid; @@ -458,7 +458,7 @@ static int cifs_do_create(struct inode *inode, struct dentry *direntry, unsigned int cifs_atomic_open(struct inode *inode, struct dentry *direntry, - struct file *file, unsigned oflags, umode_t mode) + struct file *file, unsigned int oflags, umode_t mode) { int rc; unsigned int xid; @@ -539,8 +539,8 @@ cifs_atomic_open(struct inode *inode, struct dentry *direntry, } if (file->f_flags & O_DIRECT && - CIFS_SB(inode->i_sb)->mnt_cifs_flags & CIFS_MOUNT_STRICT_IO) { - if (CIFS_SB(inode->i_sb)->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + cifs_sb_flags(CIFS_SB(inode->i_sb)) & CIFS_MOUNT_STRICT_IO) { + if (cifs_sb_flags(CIFS_SB(inode->i_sb)) & CIFS_MOUNT_NO_BRL) file->f_op = &cifs_file_direct_nobrl_ops; else file->f_op = &cifs_file_direct_ops; diff --git a/fs/smb/client/dns_resolve.h b/fs/smb/client/dns_resolve.h index 0dc706f2c4225..36bc4a6a55bfb 100644 --- a/fs/smb/client/dns_resolve.h +++ b/fs/smb/client/dns_resolve.h @@ -15,8 +15,6 @@ #include "cifsglob.h" #include "cifsproto.h" -#ifdef __KERNEL__ - int dns_resolve_name(const char *dom, const char *name, size_t namelen, struct sockaddr *ip_addr); @@ -36,6 +34,4 @@ static inline int dns_resolve_unc(const char *dom, const char *unc, return dns_resolve_name(dom, name, namelen, ip_addr); } -#endif /* KERNEL */ - #endif /* _DNS_RESOLVE_H */ diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c index a34457d5143e1..235e365f60934 100644 --- a/fs/smb/client/file.c +++ b/fs/smb/client/file.c @@ -282,7 +282,7 @@ static int cifs_init_request(struct netfs_io_request *rreq, struct file *file) open_file = file->private_data; rreq->netfs_priv = file->private_data; req->cfile = cifsFileInfo_get(open_file); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RWPIDFORWARD) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RWPIDFORWARD) req->pid = req->cfile->pid; } else if (rreq->origin != NETFS_WRITEBACK) { WARN_ON_ONCE(1); @@ -896,7 +896,7 @@ void _cifsFileInfo_put(struct cifsFileInfo *cifs_file, * close because it may cause a error when we open this file * again and get at least level II oplock. */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_STRICT_IO) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_STRICT_IO) set_bit(CIFS_INO_INVALID_MAPPING, &cifsi->flags); cifs_set_oplock_level(cifsi, 0); } @@ -949,7 +949,7 @@ int cifs_file_flush(const unsigned int xid, struct inode *inode, struct cifs_tcon *tcon; int rc; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOSSYNC) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOSSYNC) return 0; if (cfile && (OPEN_FMODE(cfile->f_flags) & FMODE_WRITE)) { @@ -994,6 +994,7 @@ static int cifs_do_truncate(const unsigned int xid, struct dentry *dentry) if (!rc) { netfs_resize_file(&cinode->netfs, 0, true); cifs_setsize(inode, 0); + cifs_invalidate_cache(inode, 0); } } if (cfile) @@ -1046,8 +1047,8 @@ int cifs_open(struct inode *inode, struct file *file) inode, file->f_flags, full_path); if (file->f_flags & O_DIRECT && - cifs_sb->mnt_cifs_flags & CIFS_MOUNT_STRICT_IO) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + cifs_sb_flags(cifs_sb) & CIFS_MOUNT_STRICT_IO) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_BRL) file->f_op = &cifs_file_direct_nobrl_ops; else file->f_op = &cifs_file_direct_ops; @@ -1211,7 +1212,7 @@ cifs_relock_file(struct cifsFileInfo *cfile) #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY if (cap_unix(tcon->ses) && (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && - ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0)) + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) rc = cifs_push_posix_locks(cfile); else #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ @@ -2007,7 +2008,7 @@ cifs_push_locks(struct cifsFileInfo *cfile) #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY if (cap_unix(tcon->ses) && (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && - ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0)) + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) rc = cifs_push_posix_locks(cfile); else #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ @@ -2415,7 +2416,7 @@ int cifs_flock(struct file *file, int cmd, struct file_lock *fl) if (cap_unix(tcon->ses) && (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && - ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0)) + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) posix_lck = true; if (!lock && !unlock) { @@ -2465,7 +2466,7 @@ int cifs_lock(struct file *file, int cmd, struct file_lock *flock) if (cap_unix(tcon->ses) && (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && - ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0)) + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) posix_lck = true; /* * BB add code here to normalize offset and length to account for @@ -2519,7 +2520,7 @@ struct cifsFileInfo *find_readable_file(struct cifsInodeInfo *cifs_inode, struct cifs_sb_info *cifs_sb = CIFS_SB(cifs_inode->netfs.inode.i_sb); /* only filter by fsuid on multiuser mounts */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MULTIUSER)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) fsuid_only = false; spin_lock(&cifs_inode->open_file_lock); @@ -2575,7 +2576,7 @@ cifs_get_writable_file(struct cifsInodeInfo *cifs_inode, int flags, cifs_sb = CIFS_SB(cifs_inode->netfs.inode.i_sb); /* only filter by fsuid on multiuser mounts */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MULTIUSER)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER)) fsuid_only = false; spin_lock(&cifs_inode->open_file_lock); @@ -2784,7 +2785,7 @@ int cifs_fsync(struct file *file, loff_t start, loff_t end, int datasync) file, datasync); tcon = tlink_tcon(smbfile->tlink); - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOSSYNC)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOSSYNC)) { server = tcon->ses->server; if (server->ops->flush == NULL) { rc = -ENOSYS; @@ -2853,7 +2854,7 @@ cifs_writev(struct kiocb *iocb, struct iov_iter *from) if (rc <= 0) goto out; - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) && + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) && (cifs_find_lock_conflict(cfile, iocb->ki_pos, iov_iter_count(from), server->vals->exclusive_lock_type, 0, NULL, CIFS_WRITE_OP))) { @@ -2889,7 +2890,7 @@ cifs_strict_writev(struct kiocb *iocb, struct iov_iter *from) if (CIFS_CACHE_WRITE(cinode)) { if (cap_unix(tcon->ses) && (CIFS_UNIX_FCNTL_CAP & le64_to_cpu(tcon->fsUnixInfo.Capability)) && - ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0)) { + ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0)) { written = netfs_file_write_iter(iocb, from); goto out; } @@ -2994,7 +2995,7 @@ cifs_strict_readv(struct kiocb *iocb, struct iov_iter *to) if (!CIFS_CACHE_READ(cinode)) return netfs_unbuffered_read_iter(iocb, to); - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NOPOSIXBRL) == 0) { + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NOPOSIXBRL) == 0) { if (iocb->ki_flags & IOCB_DIRECT) return netfs_unbuffered_read_iter(iocb, to); return netfs_buffered_read_iter(iocb, to); @@ -3116,7 +3117,7 @@ bool is_size_safe_to_change(struct cifsInodeInfo *cifsInode, __u64 end_of_file, struct cifs_sb_info *cifs_sb; cifs_sb = CIFS_SB(cifsInode->netfs.inode.i_sb); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DIRECT_IO) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DIRECT_IO) { /* since no page cache to corrupt on directio we can change size safely */ return true; @@ -3150,8 +3151,11 @@ void cifs_oplock_break(struct work_struct *work) TASK_UNINTERRUPTIBLE); tlink = cifs_sb_tlink(cifs_sb); - if (IS_ERR(tlink)) + if (IS_ERR(tlink)) { + /* drop the reference taken when the break was queued */ + _cifsFileInfo_put(cfile, false /* do not wait for ourself */, false); goto out; + } tcon = tlink_tcon(tlink); server = tcon->ses->server; diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index f207c7cef0467..6e0e3deedd5a4 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -505,7 +505,7 @@ cifs_parse_smb_version(struct fs_context *fc, char *value, struct smb3_fs_contex case Smb_20: cifs_errorf(fc, "vers=2.0 mount not permitted when legacy dialects disabled\n"); return 1; -#endif /* CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ case Smb_21: ctx->ops = &smb21_operations; ctx->vals = &smb21_values; @@ -1992,138 +1992,138 @@ smb3_cleanup_fs_context(struct smb3_fs_context *ctx) void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb) { struct smb3_fs_context *ctx = cifs_sb->ctx; + unsigned int sbflags = cifs_sb_flags(cifs_sb); if (ctx->nodfs) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NO_DFS; + sbflags |= CIFS_MOUNT_NO_DFS; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NO_DFS; + sbflags &= ~CIFS_MOUNT_NO_DFS; if (ctx->noperm) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NO_PERM; + sbflags |= CIFS_MOUNT_NO_PERM; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NO_PERM; + sbflags &= ~CIFS_MOUNT_NO_PERM; if (ctx->setuids) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_SET_UID; + sbflags |= CIFS_MOUNT_SET_UID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SET_UID; + sbflags &= ~CIFS_MOUNT_SET_UID; if (ctx->setuidfromacl) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_UID_FROM_ACL; + sbflags |= CIFS_MOUNT_UID_FROM_ACL; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_UID_FROM_ACL; + sbflags &= ~CIFS_MOUNT_UID_FROM_ACL; if (ctx->server_ino) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_SERVER_INUM; + sbflags |= CIFS_MOUNT_SERVER_INUM; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SERVER_INUM; + sbflags &= ~CIFS_MOUNT_SERVER_INUM; if (ctx->remap) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_MAP_SFM_CHR; + sbflags |= CIFS_MOUNT_MAP_SFM_CHR; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MAP_SFM_CHR; + sbflags &= ~CIFS_MOUNT_MAP_SFM_CHR; if (ctx->sfu_remap) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_MAP_SPECIAL_CHR; + sbflags |= CIFS_MOUNT_MAP_SPECIAL_CHR; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MAP_SPECIAL_CHR; + sbflags &= ~CIFS_MOUNT_MAP_SPECIAL_CHR; if (ctx->no_xattr) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NO_XATTR; + sbflags |= CIFS_MOUNT_NO_XATTR; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NO_XATTR; + sbflags &= ~CIFS_MOUNT_NO_XATTR; if (ctx->sfu_emul) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_UNX_EMUL; + sbflags |= CIFS_MOUNT_UNX_EMUL; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_UNX_EMUL; + sbflags &= ~CIFS_MOUNT_UNX_EMUL; if (ctx->nobrl) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NO_BRL; + sbflags |= CIFS_MOUNT_NO_BRL; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NO_BRL; + sbflags &= ~CIFS_MOUNT_NO_BRL; if (ctx->nohandlecache) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NO_HANDLE_CACHE; + sbflags |= CIFS_MOUNT_NO_HANDLE_CACHE; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NO_HANDLE_CACHE; + sbflags &= ~CIFS_MOUNT_NO_HANDLE_CACHE; if (ctx->nostrictsync) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NOSSYNC; + sbflags |= CIFS_MOUNT_NOSSYNC; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NOSSYNC; + sbflags &= ~CIFS_MOUNT_NOSSYNC; if (ctx->mand_lock) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_NOPOSIXBRL; + sbflags |= CIFS_MOUNT_NOPOSIXBRL; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_NOPOSIXBRL; + sbflags &= ~CIFS_MOUNT_NOPOSIXBRL; if (ctx->rwpidforward) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_RWPIDFORWARD; + sbflags |= CIFS_MOUNT_RWPIDFORWARD; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_RWPIDFORWARD; + sbflags &= ~CIFS_MOUNT_RWPIDFORWARD; if (ctx->mode_ace) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_MODE_FROM_SID; + sbflags |= CIFS_MOUNT_MODE_FROM_SID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MODE_FROM_SID; + sbflags &= ~CIFS_MOUNT_MODE_FROM_SID; if (ctx->cifs_acl) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_CIFS_ACL; + sbflags |= CIFS_MOUNT_CIFS_ACL; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_CIFS_ACL; + sbflags &= ~CIFS_MOUNT_CIFS_ACL; if (ctx->backupuid_specified) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_CIFS_BACKUPUID; + sbflags |= CIFS_MOUNT_CIFS_BACKUPUID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_CIFS_BACKUPUID; + sbflags &= ~CIFS_MOUNT_CIFS_BACKUPUID; if (ctx->backupgid_specified) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_CIFS_BACKUPGID; + sbflags |= CIFS_MOUNT_CIFS_BACKUPGID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_CIFS_BACKUPGID; + sbflags &= ~CIFS_MOUNT_CIFS_BACKUPGID; if (ctx->override_uid) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_OVERR_UID; + sbflags |= CIFS_MOUNT_OVERR_UID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_OVERR_UID; + sbflags &= ~CIFS_MOUNT_OVERR_UID; if (ctx->override_gid) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_OVERR_GID; + sbflags |= CIFS_MOUNT_OVERR_GID; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_OVERR_GID; + sbflags &= ~CIFS_MOUNT_OVERR_GID; if (ctx->dynperm) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_DYNPERM; + sbflags |= CIFS_MOUNT_DYNPERM; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_DYNPERM; + sbflags &= ~CIFS_MOUNT_DYNPERM; if (ctx->fsc) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_FSCACHE; + sbflags |= CIFS_MOUNT_FSCACHE; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_FSCACHE; + sbflags &= ~CIFS_MOUNT_FSCACHE; if (ctx->multiuser) - cifs_sb->mnt_cifs_flags |= (CIFS_MOUNT_MULTIUSER | - CIFS_MOUNT_NO_PERM); + sbflags |= CIFS_MOUNT_MULTIUSER | CIFS_MOUNT_NO_PERM; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MULTIUSER; + sbflags &= ~CIFS_MOUNT_MULTIUSER; if (ctx->strict_io) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_STRICT_IO; + sbflags |= CIFS_MOUNT_STRICT_IO; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_STRICT_IO; + sbflags &= ~CIFS_MOUNT_STRICT_IO; if (ctx->direct_io) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_DIRECT_IO; + sbflags |= CIFS_MOUNT_DIRECT_IO; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_DIRECT_IO; + sbflags &= ~CIFS_MOUNT_DIRECT_IO; if (ctx->mfsymlinks) - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_MF_SYMLINKS; + sbflags |= CIFS_MOUNT_MF_SYMLINKS; else - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_MF_SYMLINKS; + sbflags &= ~CIFS_MOUNT_MF_SYMLINKS; if (ctx->mfsymlinks) { if (ctx->sfu_emul) { /* @@ -2143,7 +2143,7 @@ void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb) cifs_dbg(VFS, "mount options mfsymlinks and sfu both enabled\n"); } } - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SHUTDOWN; + sbflags &= ~CIFS_MOUNT_SHUTDOWN; - return; + atomic_set(&cifs_sb->mnt_cifs_flags, sbflags); } diff --git a/fs/smb/client/fs_context.h b/fs/smb/client/fs_context.h index b0fec6b9a23b4..f88a3d795799b 100644 --- a/fs/smb/client/fs_context.h +++ b/fs/smb/client/fs_context.h @@ -359,18 +359,20 @@ static inline enum cifs_symlink_type cifs_symlink_type(struct cifs_sb_info *cifs return CIFS_SYMLINK_TYPE_NONE; } -extern int smb3_init_fs_context(struct fs_context *fc); -extern void smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx); -extern void smb3_cleanup_fs_context(struct smb3_fs_context *ctx); +int smb3_init_fs_context(struct fs_context *fc); +void smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx); +void smb3_cleanup_fs_context(struct smb3_fs_context *ctx); static inline struct smb3_fs_context *smb3_fc2context(const struct fs_context *fc) { return fc->fs_private; } -extern int smb3_fs_context_dup(struct smb3_fs_context *new_ctx, struct smb3_fs_context *ctx); -extern int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, struct cifs_ses *ses); -extern void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb); +int smb3_fs_context_dup(struct smb3_fs_context *new_ctx, + struct smb3_fs_context *ctx); +int smb3_sync_session_ctx_passwords(struct cifs_sb_info *cifs_sb, + struct cifs_ses *ses); +void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb); /* * max deferred close timeout (jiffies) - 2^30 @@ -378,7 +380,7 @@ extern void smb3_update_mnt_flags(struct cifs_sb_info *cifs_sb); #define SMB3_MAX_DCLOSETIMEO (1 << 30) #define SMB3_DEF_DCLOSETIMEO (1 * HZ) /* even 1 sec enough to help eg open/write/close/open/read */ #define MAX_CACHED_FIDS 16 -extern char *cifs_sanitize_prepath(char *prepath, gfp_t gfp); +char *cifs_sanitize_prepath(char *prepath, gfp_t gfp); extern struct mutex cifs_mount_mutex; diff --git a/fs/smb/client/fscache.h b/fs/smb/client/fscache.h index f06cb24f5f3cf..b6c94db5edb94 100644 --- a/fs/smb/client/fscache.h +++ b/fs/smb/client/fscache.h @@ -38,12 +38,12 @@ struct cifs_fscache_inode_coherency_data { /* * fscache.c */ -extern int cifs_fscache_get_super_cookie(struct cifs_tcon *); -extern void cifs_fscache_release_super_cookie(struct cifs_tcon *); +int cifs_fscache_get_super_cookie(struct cifs_tcon *tcon); +void cifs_fscache_release_super_cookie(struct cifs_tcon *tcon); -extern void cifs_fscache_get_inode_cookie(struct inode *inode); -extern void cifs_fscache_release_inode_cookie(struct inode *); -extern void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update); +void cifs_fscache_get_inode_cookie(struct inode *inode); +void cifs_fscache_release_inode_cookie(struct inode *inode); +void cifs_fscache_unuse_inode_cookie(struct inode *inode, bool update); static inline void cifs_fscache_fill_coherency(struct inode *inode, diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c index 594798a6b0fb4..7f8d2edb828ee 100644 --- a/fs/smb/client/inode.c +++ b/fs/smb/client/inode.c @@ -63,18 +63,18 @@ static void cifs_set_ops(struct inode *inode) switch (inode->i_mode & S_IFMT) { case S_IFREG: inode->i_op = &cifs_file_inode_ops; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DIRECT_IO) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DIRECT_IO) { set_bit(NETFS_ICTX_UNBUFFERED, &ictx->flags); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_BRL) inode->i_fop = &cifs_file_direct_nobrl_ops; else inode->i_fop = &cifs_file_direct_ops; - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_STRICT_IO) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_STRICT_IO) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_BRL) inode->i_fop = &cifs_file_strict_nobrl_ops; else inode->i_fop = &cifs_file_strict_ops; - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_BRL) + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_BRL) inode->i_fop = &cifs_file_nobrl_ops; else { /* not direct, send byte range locks */ inode->i_fop = &cifs_file_ops; @@ -212,7 +212,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr, /* if dynperm is set, don't clobber existing mode */ if (inode->i_state & I_NEW || - !(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM)) + !(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM)) inode->i_mode = fattr->cf_mode; cifs_i->cifsAttrs = fattr->cf_cifsattrs; @@ -259,7 +259,7 @@ cifs_fill_uniqueid(struct super_block *sb, struct cifs_fattr *fattr) { struct cifs_sb_info *cifs_sb = CIFS_SB(sb); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) return; fattr->cf_uniqueid = iunique(sb, ROOT_I); @@ -329,7 +329,7 @@ cifs_unix_basic_to_fattr(struct cifs_fattr *fattr, FILE_UNIX_BASIC_INFO *info, } fattr->cf_uid = cifs_sb->ctx->linux_uid; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_UID)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID)) { u64 id = le64_to_cpu(info->Uid); if (id < ((uid_t)-1)) { kuid_t uid = make_kuid(&init_user_ns, id); @@ -339,7 +339,7 @@ cifs_unix_basic_to_fattr(struct cifs_fattr *fattr, FILE_UNIX_BASIC_INFO *info, } fattr->cf_gid = cifs_sb->ctx->linux_gid; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_GID)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID)) { u64 id = le64_to_cpu(info->Gid); if (id < ((gid_t)-1)) { kgid_t gid = make_kgid(&init_user_ns, id); @@ -393,7 +393,7 @@ static int update_inode_info(struct super_block *sb, * * If file type or uniqueid is different, return error. */ - if (unlikely((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) && + if (unlikely((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) && CIFS_I(*inode)->uniqueid != fattr->cf_uniqueid)) { CIFS_I(*inode)->time = 0; /* force reval */ return -ESTALE; @@ -479,7 +479,7 @@ static int cifs_get_unix_fattr(const unsigned char *full_path, cifs_fill_uniqueid(sb, fattr); /* check for Minshall+French symlinks */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); } @@ -1092,7 +1092,7 @@ cifs_backup_query_path_info(int xid, else if ((tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find) == 0) info.info_level = SMB_FIND_FILE_INFO_STANDARD; - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) info.info_level = SMB_FIND_FILE_ID_FULL_DIR_INFO; else /* no srvino useful for fallback to some netapp */ info.info_level = SMB_FIND_FILE_DIRECTORY_INFO; @@ -1120,7 +1120,7 @@ static void cifs_set_fattr_ino(int xid, struct cifs_tcon *tcon, struct super_blo struct TCP_Server_Info *server = tcon->ses->server; int rc; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) { if (*inode) fattr->cf_uniqueid = CIFS_I(*inode)->uniqueid; else @@ -1383,14 +1383,14 @@ static int cifs_get_fattr(struct cifs_open_info_data *data, #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ /* query for SFU type info if supported and needed */ if ((fattr->cf_cifsattrs & ATTR_SYSTEM) && - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL)) { + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL)) { tmprc = cifs_sfu_type(fattr, full_path, cifs_sb, xid); if (tmprc) cifs_dbg(FYI, "cifs_sfu_type failed: %d\n", tmprc); } /* fill in 0777 bits from ACL */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID) { rc = cifs_acl_to_fattr(cifs_sb, fattr, *inode, true, full_path, fid); if (rc == -EREMOTE) @@ -1400,7 +1400,7 @@ static int cifs_get_fattr(struct cifs_open_info_data *data, __func__, rc); goto out; } - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) { + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) { rc = cifs_acl_to_fattr(cifs_sb, fattr, *inode, false, full_path, fid); if (rc == -EREMOTE) @@ -1410,7 +1410,7 @@ static int cifs_get_fattr(struct cifs_open_info_data *data, __func__, rc); goto out; } - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL) /* fill in remaining high mode bits e.g. SUID, VTX */ cifs_sfu_mode(fattr, full_path, cifs_sb, xid); else if (!(tcon->posix_extensions)) @@ -1420,7 +1420,7 @@ static int cifs_get_fattr(struct cifs_open_info_data *data, /* check for Minshall+French symlinks */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); } @@ -1520,7 +1520,7 @@ static int smb311_posix_get_fattr(struct cifs_open_info_data *data, * 3. Tweak fattr based on mount options */ /* check for Minshall+French symlinks */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { tmprc = check_mf_symlink(xid, tcon, cifs_sb, fattr, full_path); cifs_dbg(FYI, "check_mf_symlink: %d\n", tmprc); } @@ -1671,7 +1671,7 @@ struct inode *cifs_root_iget(struct super_block *sb) int len; int rc; - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_USE_PREFIX_PATH) + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_USE_PREFIX_PATH) && cifs_sb->prepath) { len = strlen(cifs_sb->prepath); path = kzalloc(len + 2 /* leading sep + null */, GFP_KERNEL); @@ -2162,7 +2162,7 @@ cifs_mkdir_qinfo(struct inode *parent, struct dentry *dentry, umode_t mode, .mtime = NO_CHANGE_64, .device = 0, }; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) { args.uid = current_fsuid(); if (parent->i_mode & S_ISGID) args.gid = parent->i_gid; @@ -2180,14 +2180,14 @@ cifs_mkdir_qinfo(struct inode *parent, struct dentry *dentry, umode_t mode, { #endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ struct TCP_Server_Info *server = tcon->ses->server; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) && + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) && (mode & S_IWUGO) == 0 && server->ops->mkdir_setinfo) server->ops->mkdir_setinfo(inode, full_path, cifs_sb, tcon, xid); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM) inode->i_mode = (mode | S_IFDIR); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) { inode->i_uid = current_fsuid(); if (inode->i_mode & S_ISGID) inode->i_gid = parent->i_gid; @@ -2233,7 +2233,7 @@ cifs_posix_mkdir(struct inode *inode, struct dentry *dentry, umode_t mode, /* no return info, go query for it */ goto posix_mkdir_get_info; /* - * BB check (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID ) to see if + * BB check (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID ) to see if * need to set uid/gid. */ @@ -2750,7 +2750,7 @@ cifs_dentry_needs_reval(struct dentry *dentry) } /* hardlinked files w/ noserverino get "special" treatment */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) && + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) && S_ISREG(inode->i_mode) && inode->i_nlink != 1) return true; @@ -2791,7 +2791,7 @@ cifs_revalidate_mapping(struct inode *inode) if (test_and_clear_bit(CIFS_INO_INVALID_MAPPING, flags)) { /* for cache=singleclient, do not invalidate */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_RW_CACHE) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE) goto skip_invalidate; cifs_inode->netfs.zero_point = cifs_inode->netfs.remote_i_size; @@ -2975,12 +2975,12 @@ int cifs_getattr(struct mnt_idmap *idmap, const struct path *path, * enabled, and the admin hasn't overridden them, set the ownership * to the fsuid/fsgid of the current process. */ - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MULTIUSER) && - !(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) && + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MULTIUSER) && + !(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) && !tcon->unix_ext) { - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_UID)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID)) stat->uid = current_fsuid(); - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_GID)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID)) stat->gid = current_fsgid(); } return 0; @@ -3039,6 +3039,24 @@ void cifs_setsize(struct inode *inode, loff_t offset) inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); truncate_pagecache(inode, offset); netfs_wait_for_outstanding_io(inode); + fscache_resize_cookie(cifs_inode_cookie(inode), offset); +} + +void cifs_resize_file_locked(struct inode *inode, loff_t offset) +{ + struct fscache_cookie *cookie = cifs_inode_cookie(inode); + + lockdep_assert_held_write(&inode->i_rwsem); + + netfs_resize_file(netfs_inode(inode), offset, true); + cifs_setsize(inode, offset); + + if (!cookie) + return; + + fscache_use_cookie(cookie, true); + fscache_resize_cookie(cookie, offset); + cifs_fscache_unuse_inode_cookie(inode, true); } int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, @@ -3079,6 +3097,7 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, size, false); cifs_dbg(FYI, "%s: set_file_size: rc = %d\n", __func__, rc); cifsFileInfo_put(open_file); + tcon = NULL; } } @@ -3104,10 +3123,8 @@ int cifs_file_set_size(const unsigned int xid, struct dentry *dentry, cifs_put_tlink(tlink); set_size_out: - if (rc == 0) { - netfs_resize_file(&cifsInode->netfs, size, true); - cifs_setsize(inode, size); - } + if (rc == 0) + cifs_resize_file_locked(inode, size); return rc; } @@ -3133,7 +3150,7 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) xid = get_xid(); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_PERM) attrs->ia_valid |= ATTR_FORCE; rc = setattr_prepare(&nop_mnt_idmap, direntry, attrs); @@ -3182,11 +3199,15 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) attrs->ia_valid &= ~(ATTR_CTIME | ATTR_MTIME); } - /* skip mode change if it's just for clearing setuid/setgid */ - if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) - attrs->ia_valid &= ~ATTR_MODE; + /* + * This function is only called when Unix extensions are in effect, + * so the mode is always sent to and stored on the server. Do not + * skip the mode change when clearing setuid/setgid bits: dropping + * ATTR_MODE here would leave those bits set on the server after a + * write, which is a security issue. + */ - args = kmalloc(sizeof(*args), GFP_KERNEL); + args = kmalloc_obj(*args); if (args == NULL) { rc = -ENOMEM; goto out; @@ -3257,13 +3278,6 @@ cifs_setattr_unix(struct dentry *direntry, struct iattr *attrs) if (rc) goto out; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); @@ -3291,6 +3305,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) kgid_t gid = INVALID_GID; struct inode *inode = d_inode(direntry); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); + unsigned int sbflags = cifs_sb_flags(cifs_sb); struct cifsInodeInfo *cifsInode = CIFS_I(inode); struct cifsFileInfo *cfile = NULL; const char *full_path; @@ -3305,7 +3320,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) cifs_dbg(FYI, "setattr on file %pd attrs->ia_valid 0x%x\n", direntry, attrs->ia_valid); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_PERM) + if (sbflags & CIFS_MOUNT_NO_PERM) attrs->ia_valid |= ATTR_FORCE; rc = setattr_prepare(&nop_mnt_idmap, direntry, attrs); @@ -3366,8 +3381,8 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (attrs->ia_valid & ATTR_GID) gid = attrs->ia_gid; - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) || - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID)) { + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) || + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID)) { if (uid_valid(uid) || gid_valid(gid)) { mode = NO_CHANGE_64; rc = id_mode_to_cifs_acl(inode, full_path, &mode, @@ -3378,19 +3393,34 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) goto cifs_setattr_exit; } } - } else - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID)) + } else if (!(sbflags & CIFS_MOUNT_SET_UID)) { attrs->ia_valid &= ~(ATTR_UID | ATTR_GID); + } - /* skip mode change if it's just for clearing setuid/setgid */ - if (attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) + /* + * Skip the mode change if it is only being done to clear the + * setuid/setgid bits *and* the mode is emulated via the DOS + * read-only attribute (the default, non-ACL case), which cannot + * represent the setuid/setgid bits anyway. + * + * When the mode is instead stored on the server - i.e. with the + * cifsacl or modefromsid mount options (via an ACL) or with the + * SMB3.1.1 POSIX extensions - the cleared mode must be pushed to + * the server. Dropping ATTR_MODE here would leave the setuid/ + * setgid bit set on the server after a write, which is a security + * issue (the bits are not stripped as they are on local + * filesystems). + */ + if ((attrs->ia_valid & (ATTR_KILL_SUID|ATTR_KILL_SGID)) && + !((sbflags & (CIFS_MOUNT_CIFS_ACL | CIFS_MOUNT_MODE_FROM_SID)) || + cifs_sb_master_tcon(cifs_sb)->posix_extensions)) attrs->ia_valid &= ~ATTR_MODE; if (attrs->ia_valid & ATTR_MODE) { mode = attrs->ia_mode; rc = 0; - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) || - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) || + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) || + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID) || posix) { rc = id_mode_to_cifs_acl(inode, full_path, &mode, INVALID_UID, INVALID_GID); @@ -3413,7 +3443,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) dosattr = cifsInode->cifsAttrs | ATTR_READONLY; /* fix up mode if we're not using dynperm */ - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM) == 0) + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM) == 0) attrs->ia_mode = inode->i_mode & ~S_IWUGO; } else if ((mode & S_IWUGO) && (cifsInode->cifsAttrs & ATTR_READONLY)) { @@ -3424,7 +3454,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) dosattr |= ATTR_NORMAL; /* reset local inode permissions to normal */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM)) { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM)) { attrs->ia_mode &= ~(S_IALLUGO); if (S_ISDIR(inode->i_mode)) attrs->ia_mode |= @@ -3433,7 +3463,7 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) attrs->ia_mode |= cifs_sb->ctx->file_mode; } - } else if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_DYNPERM)) { + } else if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_DYNPERM)) { /* ignore mode change - ATTR_READONLY hasn't changed */ attrs->ia_valid &= ~ATTR_MODE; } @@ -3459,13 +3489,6 @@ cifs_setattr_nounix(struct dentry *direntry, struct iattr *attrs) if (rc) goto cifs_setattr_exit; - if ((attrs->ia_valid & ATTR_SIZE) && - attrs->ia_size != i_size_read(inode)) { - truncate_setsize(inode, attrs->ia_size); - netfs_resize_file(&cifsInode->netfs, attrs->ia_size, true); - fscache_resize_cookie(cifs_inode_cookie(inode), attrs->ia_size); - } - setattr_copy(&nop_mnt_idmap, inode, attrs); mark_inode_dirty(inode); diff --git a/fs/smb/client/ioctl.c b/fs/smb/client/ioctl.c index 4607d007d9504..2731491876628 100644 --- a/fs/smb/client/ioctl.c +++ b/fs/smb/client/ioctl.c @@ -217,7 +217,7 @@ static int cifs_shutdown(struct super_block *sb, unsigned long arg) */ case CIFS_GOING_FLAGS_LOGFLUSH: case CIFS_GOING_FLAGS_NOLOGFLUSH: - sbi->mnt_cifs_flags |= CIFS_MOUNT_SHUTDOWN; + atomic_or(CIFS_MOUNT_SHUTDOWN, &sbi->mnt_cifs_flags); goto shutdown_good; default: rc = -EINVAL; diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c index 70f3c0c67eebd..ac31564ce9265 100644 --- a/fs/smb/client/link.c +++ b/fs/smb/client/link.c @@ -234,7 +234,7 @@ cifs_query_mf_symlink(unsigned int xid, struct cifs_tcon *tcon, struct cifs_open_parms oparms; struct cifs_io_parms io_parms = {0}; int buf_type = CIFS_NO_BUFFER; - struct cifs_open_info_data query_data; + struct cifs_open_info_data query_data = {}; oparms = (struct cifs_open_parms) { .tcon = tcon, @@ -593,14 +593,14 @@ cifs_symlink(struct mnt_idmap *idmap, struct inode *inode, break; case CIFS_SYMLINK_TYPE_MFSYMLINKS: - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) { rc = create_mf_symlink(xid, pTcon, cifs_sb, full_path, symname); } break; case CIFS_SYMLINK_TYPE_SFU: - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL) { rc = __cifs_sfu_make_node(xid, inode, direntry, pTcon, full_path, S_IFLNK, 0, symname); diff --git a/fs/smb/client/misc.c b/fs/smb/client/misc.c index f378437113129..25e29b1e554ef 100644 --- a/fs/smb/client/misc.c +++ b/fs/smb/client/misc.c @@ -18,6 +18,7 @@ #include "nterr.h" #include "cifs_unicode.h" #include "smb2pdu.h" +#include "smb2proto.h" #include "cifsfs.h" #ifdef CONFIG_CIFS_DFS_UPCALL #include "dns_resolve.h" @@ -565,13 +566,13 @@ dump_smb(void *buf, int smb_buf_length) void cifs_autodisable_serverino(struct cifs_sb_info *cifs_sb) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) { struct cifs_tcon *tcon = NULL; if (cifs_sb->master_tlink) tcon = cifs_sb_master_tcon(cifs_sb); - cifs_sb->mnt_cifs_flags &= ~CIFS_MOUNT_SERVER_INUM; + atomic_andnot(CIFS_MOUNT_SERVER_INUM, &cifs_sb->mnt_cifs_flags); cifs_sb->mnt_cifs_serverino_autodisabled = true; cifs_dbg(VFS, "Autodisabling the use of server inode numbers on %s\n", tcon ? tcon->tree_name : "new server"); @@ -657,10 +658,11 @@ void cifs_queue_oplock_break(struct cifsFileInfo *cfile) * open_file_lock to enforce the validity of it for the oplock * break handler. The matching put is done at the end of the * handler. + * + * Only take a reference if the work is actually queued. */ - cifsFileInfo_get(cfile); - - queue_work(cifsoplockd_wq, &cfile->oplock_break); + if (queue_work(cifsoplockd_wq, &cfile->oplock_break)) + cifsFileInfo_get(cfile); } void cifs_done_oplock_break(struct cifsInodeInfo *cinode) @@ -672,11 +674,11 @@ void cifs_done_oplock_break(struct cifsInodeInfo *cinode) bool backup_cred(struct cifs_sb_info *cifs_sb) { - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_BACKUPUID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_BACKUPUID) { if (uid_eq(cifs_sb->ctx->backupuid, current_fsuid())) return true; } - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_BACKUPGID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_BACKUPGID) { if (in_group_p(cifs_sb->ctx->backupgid)) return true; } @@ -957,6 +959,8 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, int i, rc = 0; char *data_end; struct dfs_referral_level_3 *ref; + unsigned int path_consumed; + size_t search_name_len; if (rsp_size < sizeof(*rsp)) { cifs_dbg(VFS | ONCE, @@ -1004,6 +1008,7 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, rc = -ENOMEM; goto parse_DFS_referrals_exit; } + search_name_len = strlen(searchName); /* collect necessary data from referrals */ for (i = 0; i < *num_of_nodes; i++) { @@ -1012,21 +1017,34 @@ parse_dfs_referrals(struct get_dfs_referral_rsp *rsp, u32 rsp_size, struct dfs_info3_param *node = (*target_nodes)+i; node->flags = le32_to_cpu(rsp->DFSFlags); + path_consumed = le16_to_cpu(rsp->PathConsumed); if (is_unicode) { - __le16 *tmp = kmalloc(strlen(searchName)*2 + 2, - GFP_KERNEL); - if (tmp == NULL) { + size_t search_name_utf16_len = search_name_len * 2 + 2; + __le16 *tmp; + + if (path_consumed > search_name_utf16_len) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + tmp = kmalloc(search_name_utf16_len, GFP_KERNEL); + if (!tmp) { rc = -ENOMEM; goto parse_DFS_referrals_exit; } - cifsConvertToUTF16((__le16 *) tmp, searchName, + cifsConvertToUTF16((__le16 *)tmp, searchName, PATH_MAX, nls_codepage, remap); - node->path_consumed = cifs_utf16_bytes(tmp, - le16_to_cpu(rsp->PathConsumed), - nls_codepage); + node->path_consumed = cifs_utf16_bytes(tmp, path_consumed, + nls_codepage); kfree(tmp); - } else - node->path_consumed = le16_to_cpu(rsp->PathConsumed); + } else { + if (path_consumed > search_name_len) { + rc = -EINVAL; + goto parse_DFS_referrals_exit; + } + + node->path_consumed = path_consumed; + } node->server_type = le16_to_cpu(ref->ServerType); node->ref_flag = le16_to_cpu(ref->ReferralEntryFlags); @@ -1193,8 +1211,14 @@ static void tcon_super_cb(struct super_block *sb, void *arg) t1->ses->dfs_root_ses == t2->ses->dfs_root_ses) && t1->ses->server == t2->ses->server && t2->origin_fullpath && - dfs_src_pathname_equal(t2->origin_fullpath, t1->origin_fullpath)) + dfs_src_pathname_equal(t2->origin_fullpath, t1->origin_fullpath)) { + /* + * Take the active reference while iterate_supers_type() still + * holds s_umount shared. + */ + cifs_sb_active(sb); sd->sb = sb; + } spin_unlock(&t2->tc_lock); } @@ -1211,15 +1235,8 @@ static struct super_block *__cifs_get_super(void (*f)(struct super_block *, void for (; *fs_type; fs_type++) { iterate_supers_type(*fs_type, f, &sd); - if (sd.sb) { - /* - * Grab an active reference in order to prevent automounts (DFS links) - * of expiring and then freeing up our cifs superblock pointer while - * we're doing failover. - */ - cifs_sb_active(sd.sb); + if (sd.sb) return sd.sb; - } } pr_warn_once("%s: could not find dfs superblock\n", __func__); return ERR_PTR(-EINVAL); @@ -1289,7 +1306,7 @@ int cifs_update_super_prepath(struct cifs_sb_info *cifs_sb, char *prefix) convert_delimiter(cifs_sb->prepath, CIFS_DIR_SEP(cifs_sb)); } - cifs_sb->mnt_cifs_flags |= CIFS_MOUNT_USE_PREFIX_PATH; + atomic_or(CIFS_MOUNT_USE_PREFIX_PATH, &cifs_sb->mnt_cifs_flags); return 0; } @@ -1318,7 +1335,7 @@ int cifs_inval_name_dfs_link_error(const unsigned int xid, * look up or tcon is not DFS. */ if (strlen(full_path) < 2 || !cifs_sb || - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS) || + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_DFS) || !is_tcon_dfs(tcon)) return 0; diff --git a/fs/smb/client/netmisc.c b/fs/smb/client/netmisc.c index 9ec20601cee2e..4e0bb1920eae0 100644 --- a/fs/smb/client/netmisc.c +++ b/fs/smb/client/netmisc.c @@ -200,7 +200,7 @@ cifs_set_port(struct sockaddr *addr, const unsigned short int port) } /***************************************************************************** -convert a NT status code to a dos class/code + *convert a NT status code to a dos class/code *****************************************************************************/ /* NT status -> dos error map */ static const struct { diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c index f0ce26622a140..9ca859a2451ea 100644 --- a/fs/smb/client/readdir.c +++ b/fs/smb/client/readdir.c @@ -122,7 +122,7 @@ cifs_prime_dcache(struct dentry *parent, struct qstr *name, * want to clobber the existing one with the one that * the readdir code created. */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM)) + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) fattr->cf_uniqueid = CIFS_I(inode)->uniqueid; /* @@ -216,11 +216,11 @@ cifs_fill_common_info(struct cifs_fattr *fattr, struct cifs_sb_info *cifs_sb) * may look wrong since the inodes may not have timed out by the time * "ls" does a stat() call on them. */ - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_CIFS_ACL) || - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID)) + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_CIFS_ACL) || + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MODE_FROM_SID)) fattr->cf_flags |= CIFS_FATTR_NEED_REVAL; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL && + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL && fattr->cf_cifsattrs & ATTR_SYSTEM) { if (fattr->cf_eof == 0) { fattr->cf_mode &= ~S_IFMT; @@ -396,7 +396,7 @@ _initiate_cifs_search(const unsigned int xid, struct file *file, else if ((tcon->ses->capabilities & tcon->ses->server->vals->cap_nt_find) == 0) { cifsFile->srch_inf.info_level = SMB_FIND_FILE_INFO_STANDARD; - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM) { + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM) { cifsFile->srch_inf.info_level = SMB_FIND_FILE_ID_FULL_DIR_INFO; } else /* not srvinos - BB fixme add check for backlevel? */ { cifsFile->srch_inf.info_level = SMB_FIND_FILE_FULL_DIRECTORY_INFO; @@ -413,7 +413,7 @@ _initiate_cifs_search(const unsigned int xid, struct file *file, if (rc == 0) { cifsFile->invalidHandle = false; } else if ((rc == -EOPNOTSUPP) && - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM)) { + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) { cifs_autodisable_serverino(cifs_sb); goto ffirst_retry; } @@ -949,7 +949,7 @@ static bool cifs_dir_emit(struct dir_context *ctx, static int cifs_filldir(char *find_entry, struct file *file, struct dir_context *ctx, char *scratch_buf, unsigned int max_len, - struct cached_fid *cfid) + char *end_of_smb, struct cached_fid *cfid) { struct cifsFileInfo *file_info = file->private_data; struct super_block *sb = file_inode(file)->i_sb; @@ -970,6 +970,11 @@ static int cifs_filldir(char *find_entry, struct file *file, return -EINVAL; } + if (de.name + de.namelen > end_of_smb) { + cifs_dbg(VFS, "search entry name extends past end of SMB\n"); + return -EINVAL; + } + /* skip . and .. since we added them first */ if (cifs_entry_is_dot(&de, file_info->srch_inf.unicode)) return 0; @@ -1020,14 +1025,14 @@ static int cifs_filldir(char *find_entry, struct file *file, break; } - if (de.ino && (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SERVER_INUM)) { + if (de.ino && (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SERVER_INUM)) { fattr.cf_uniqueid = de.ino; } else { fattr.cf_uniqueid = iunique(sb, ROOT_I); cifs_autodisable_serverino(cifs_sb); } - if ((cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MF_SYMLINKS) && + if ((cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MF_SYMLINKS) && couldbe_mf_symlink(&fattr)) /* * trying to get the type and mode can be slow, @@ -1190,7 +1195,7 @@ int cifs_readdir(struct file *file, struct dir_context *ctx) */ *tmp_buf = 0; rc = cifs_filldir(current_entry, file, ctx, - tmp_buf, max_len, cfid); + tmp_buf, max_len, end_of_smb, cfid); if (rc) { if (rc > 0) rc = 0; diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index 10c84c095fe78..6ec6b8c46ba22 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -83,7 +83,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, .symlink_target = symlink_target, }; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) && + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && symroot && symname[0] == '/') { /* * This is a request to create an absolute symlink on the server @@ -164,7 +164,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, * mask these characters in NT object prefix by '_' and then change * them back. */ - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') sym[0] = sym[1] = sym[2] = sym[5] = '_'; path = cifs_convert_path_to_utf16(sym, cifs_sb); @@ -173,7 +173,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, goto out; } - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { sym[0] = '\\'; sym[1] = sym[2] = '?'; sym[5] = ':'; @@ -197,7 +197,7 @@ static int create_native_symlink(const unsigned int xid, struct inode *inode, slen = 2 * UniStrnlen((wchar_t *)path, REPARSE_SYM_PATH_MAX); poff = 0; plen = slen; - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && symname[0] == '/') { /* * For absolute NT symlinks skip leading "\\??\\" in PrintName as * PrintName is user visible location in DOS/Win32 format (not in NT format). @@ -822,7 +822,7 @@ int smb2_parse_native_symlink(char **target, const char *buf, unsigned int len, goto out; } - if (!(cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) && + if (!(cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) && symroot && !relative) { /* * This is an absolute symlink from the server which does not @@ -953,7 +953,8 @@ int smb2_parse_native_symlink(char **target, const char *buf, unsigned int len, linux_target[i*3 + 1] = '.'; linux_target[i*3 + 2] = sep; } - memcpy(linux_target + levels*3, smb_target+1, smb_target_len); /* +1 to skip leading sep */ + /* +1 to skip leading sep */ + memcpy(linux_target + levels*3, smb_target+1, smb_target_len-1); } else { /* * This is either an absolute symlink in POSIX-style format @@ -1112,6 +1113,7 @@ static bool wsl_to_fattr(struct cifs_open_info_data *data, bool have_xattr_dev = false; u32 next = 0; + fattr->cf_mode &= ~S_IFMT; switch (tag) { case IO_REPARSE_TAG_LX_SYMLINK: fattr->cf_mode |= S_IFLNK; @@ -1176,6 +1178,7 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, struct cifs_open_info_data *data) { struct reparse_nfs_data_buffer *buf = (struct reparse_nfs_data_buffer *)data->reparse.buf; + umode_t ftype; if (buf == NULL) return true; @@ -1191,7 +1194,7 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, WARN_ON_ONCE(1); return false; } - fattr->cf_mode |= S_IFCHR; + ftype = S_IFCHR; fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); break; case NFS_SPECFILE_BLK: @@ -1199,22 +1202,23 @@ static bool posix_reparse_to_fattr(struct cifs_sb_info *cifs_sb, WARN_ON_ONCE(1); return false; } - fattr->cf_mode |= S_IFBLK; + ftype = S_IFBLK; fattr->cf_rdev = reparse_mkdev(buf->DataBuffer); break; case NFS_SPECFILE_FIFO: - fattr->cf_mode |= S_IFIFO; + ftype = S_IFIFO; break; case NFS_SPECFILE_SOCK: - fattr->cf_mode |= S_IFSOCK; + ftype = S_IFSOCK; break; case NFS_SPECFILE_LNK: - fattr->cf_mode |= S_IFLNK; + ftype = S_IFLNK; break; default: WARN_ON_ONCE(1); return false; } + fattr->cf_mode = (fattr->cf_mode & ~S_IFMT) | ftype; return true; } @@ -1242,6 +1246,7 @@ bool cifs_reparse_point_to_fattr(struct cifs_sb_info *cifs_sb, break; case 0: /* SMB1 symlink */ case IO_REPARSE_TAG_SYMLINK: + fattr->cf_mode &= ~S_IFMT; fattr->cf_mode |= S_IFLNK; break; default: diff --git a/fs/smb/client/reparse.h b/fs/smb/client/reparse.h index 66269c10beba9..d7d205cde8689 100644 --- a/fs/smb/client/reparse.h +++ b/fs/smb/client/reparse.h @@ -32,7 +32,7 @@ static inline kuid_t wsl_make_kuid(struct cifs_sb_info *cifs_sb, { u32 uid = le32_to_cpu(*(__le32 *)ptr); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_UID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_UID) return cifs_sb->ctx->linux_uid; return make_kuid(current_user_ns(), uid); } @@ -42,7 +42,7 @@ static inline kgid_t wsl_make_kgid(struct cifs_sb_info *cifs_sb, { u32 gid = le32_to_cpu(*(__le32 *)ptr); - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_OVERR_GID) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_OVERR_GID) return cifs_sb->ctx->linux_gid; return make_kgid(current_user_ns(), gid); } diff --git a/fs/smb/client/sess.c b/fs/smb/client/sess.c index ef3b498b0a02a..ca0ac52522e51 100644 --- a/fs/smb/client/sess.c +++ b/fs/smb/client/sess.c @@ -234,9 +234,9 @@ int cifs_try_adding_channels(struct cifs_ses *ses) cifs_dbg(VFS, "failed to open extra channel on iface:%pIS rc=%d\n", &iface->sockaddr, rc); - kref_put(&iface->refcount, release_iface); /* failure to add chan should increase weight */ iface->weight_fulfilled++; + kref_put(&iface->refcount, release_iface); continue; } diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index ca8f3dd7ff63b..925bc882f3a72 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -238,146 +238,6 @@ cifs_get_next_mid(struct TCP_Server_Info *server) return mid; } -/* - return codes: - 0 not a transact2, or all data present - >0 transact2 with that much data missing - -EINVAL invalid transact2 - */ -static int -check2ndT2(char *buf) -{ - struct smb_hdr *pSMB = (struct smb_hdr *)buf; - struct smb_t2_rsp *pSMBt; - int remaining; - __u16 total_data_size, data_in_this_rsp; - - if (pSMB->Command != SMB_COM_TRANSACTION2) - return 0; - - /* check for plausible wct, bcc and t2 data and parm sizes */ - /* check for parm and data offset going beyond end of smb */ - if (pSMB->WordCount != 10) { /* coalesce_t2 depends on this */ - cifs_dbg(FYI, "Invalid transact2 word count\n"); - return -EINVAL; - } - - pSMBt = (struct smb_t2_rsp *)pSMB; - - total_data_size = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); - data_in_this_rsp = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); - - if (total_data_size == data_in_this_rsp) - return 0; - else if (total_data_size < data_in_this_rsp) { - cifs_dbg(FYI, "total data %d smaller than data in frame %d\n", - total_data_size, data_in_this_rsp); - return -EINVAL; - } - - remaining = total_data_size - data_in_this_rsp; - - cifs_dbg(FYI, "missing %d bytes from transact2, check next response\n", - remaining); - if (total_data_size > CIFSMaxBufSize) { - cifs_dbg(VFS, "TotalDataSize %d is over maximum buffer %d\n", - total_data_size, CIFSMaxBufSize); - return -EINVAL; - } - return remaining; -} - -static int -coalesce_t2(char *second_buf, struct smb_hdr *target_hdr) -{ - struct smb_t2_rsp *pSMBs = (struct smb_t2_rsp *)second_buf; - struct smb_t2_rsp *pSMBt = (struct smb_t2_rsp *)target_hdr; - char *data_area_of_tgt; - char *data_area_of_src; - int remaining; - unsigned int byte_count, total_in_tgt; - __u16 tgt_total_cnt, src_total_cnt, total_in_src; - - src_total_cnt = get_unaligned_le16(&pSMBs->t2_rsp.TotalDataCount); - tgt_total_cnt = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); - - if (tgt_total_cnt != src_total_cnt) - cifs_dbg(FYI, "total data count of primary and secondary t2 differ source=%hu target=%hu\n", - src_total_cnt, tgt_total_cnt); - - total_in_tgt = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); - - remaining = tgt_total_cnt - total_in_tgt; - - if (remaining < 0) { - cifs_dbg(FYI, "Server sent too much data. tgt_total_cnt=%hu total_in_tgt=%u\n", - tgt_total_cnt, total_in_tgt); - return -EPROTO; - } - - if (remaining == 0) { - /* nothing to do, ignore */ - cifs_dbg(FYI, "no more data remains\n"); - return 0; - } - - total_in_src = get_unaligned_le16(&pSMBs->t2_rsp.DataCount); - if (remaining < total_in_src) - cifs_dbg(FYI, "transact2 2nd response contains too much data\n"); - - /* find end of first SMB data area */ - data_area_of_tgt = (char *)&pSMBt->hdr.Protocol + - get_unaligned_le16(&pSMBt->t2_rsp.DataOffset); - - /* validate target area */ - data_area_of_src = (char *)&pSMBs->hdr.Protocol + - get_unaligned_le16(&pSMBs->t2_rsp.DataOffset); - - data_area_of_tgt += total_in_tgt; - - total_in_tgt += total_in_src; - /* is the result too big for the field? */ - if (total_in_tgt > USHRT_MAX) { - cifs_dbg(FYI, "coalesced DataCount too large (%u)\n", - total_in_tgt); - return -EPROTO; - } - put_unaligned_le16(total_in_tgt, &pSMBt->t2_rsp.DataCount); - - /* fix up the BCC */ - byte_count = get_bcc(target_hdr); - byte_count += total_in_src; - /* is the result too big for the field? */ - if (byte_count > USHRT_MAX) { - cifs_dbg(FYI, "coalesced BCC too large (%u)\n", byte_count); - return -EPROTO; - } - put_bcc(byte_count, target_hdr); - - byte_count = be32_to_cpu(target_hdr->smb_buf_length); - byte_count += total_in_src; - /* don't allow buffer to overflow */ - if (byte_count > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { - cifs_dbg(FYI, "coalesced BCC exceeds buffer size (%u)\n", - byte_count); - return -ENOBUFS; - } - target_hdr->smb_buf_length = cpu_to_be32(byte_count); - - /* copy second buffer into end of first buffer */ - memcpy(data_area_of_tgt, data_area_of_src, total_in_src); - - if (remaining != total_in_src) { - /* more responses to go */ - cifs_dbg(FYI, "waiting for more secondary responses\n"); - return 1; - } - - /* we are done */ - cifs_dbg(FYI, "found the last secondary response\n"); - return 0; -} - static void cifs_downgrade_oplock(struct TCP_Server_Info *server, struct cifsInodeInfo *cinode, __u32 oplock, @@ -386,37 +246,6 @@ cifs_downgrade_oplock(struct TCP_Server_Info *server, cifs_set_oplock_level(cinode, oplock); } -static bool -cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, - char *buf, int malformed) -{ - if (malformed) - return false; - if (check2ndT2(buf) <= 0) - return false; - mid->multiRsp = true; - if (mid->resp_buf) { - /* merge response - fix up 1st*/ - malformed = coalesce_t2(buf, mid->resp_buf); - if (malformed > 0) - return true; - /* All parts received or packet is malformed. */ - mid->multiEnd = true; - dequeue_mid(mid, malformed); - return true; - } - if (!server->large_buf) { - /*FIXME: switch to already allocated largebuf?*/ - cifs_dbg(VFS, "1st trans2 resp needs bigbuf\n"); - } else { - /* Have first buffer */ - mid->resp_buf = buf; - mid->large_buf = true; - server->bigbuf = NULL; - } - return true; -} - static bool cifs_need_neg(struct TCP_Server_Info *server) { @@ -727,7 +556,7 @@ static int cifs_query_path_info(const unsigned int xid, ea->ea_value_length = cpu_to_le16(SMB2_WSL_XATTR_DEV_SIZE); memcpy(&ea->ea_data[0], SMB2_WSL_XATTR_DEV, SMB2_WSL_XATTR_NAME_LEN + 1); data->wsl.eas_len += ALIGN(sizeof(*ea) + SMB2_WSL_XATTR_NAME_LEN + 1 + - SMB2_WSL_XATTR_MODE_SIZE, 4); + SMB2_WSL_XATTR_DEV_SIZE, 4); rc = 0; } else if (rc >= 0) { /* It is an error if EA $LXDEV has wrong size. */ @@ -962,7 +791,7 @@ smb_set_file_info(struct inode *inode, const char *full_path, struct cifs_open_parms oparms; struct cifsFileInfo *open_file; FILE_BASIC_INFO new_buf; - struct cifs_open_info_data query_data; + struct cifs_open_info_data query_data = {}; __le64 write_time = buf->LastWriteTime; struct cifsInodeInfo *cinode = CIFS_I(inode); struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); @@ -1309,7 +1138,7 @@ cifs_make_node(unsigned int xid, struct inode *inode, .mtime = NO_CHANGE_64, .device = dev, }; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_SET_UID) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_SET_UID) { args.uid = current_fsuid(); args.gid = current_fsgid(); } else { @@ -1328,7 +1157,7 @@ cifs_make_node(unsigned int xid, struct inode *inode, if (rc == 0) d_instantiate(dentry, newinode); return rc; - } else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) { + } else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL) { /* * Check if mounted with mount parm 'sfu' mount parm. * SFU emulation should work with all servers diff --git a/fs/smb/client/smb1proto.h b/fs/smb/client/smb1proto.h new file mode 100644 index 0000000000000..8dfc6976ceba1 --- /dev/null +++ b/fs/smb/client/smb1proto.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1 */ +/* + * + * Copyright (c) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * + */ +#ifndef _SMB1PROTO_H +#define _SMB1PROTO_H + +struct cifs_unix_set_info_args { + __u64 ctime; + __u64 atime; + __u64 mtime; + __u64 mode; + kuid_t uid; + kgid_t gid; + dev_t device; +}; + +#ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY + +/* + * cifssmb.c + */ + +/* + * smb1ops.c + */ +extern struct smb_version_operations smb1_operations; +extern struct smb_version_values smb1_values; + +/* + * smb1transport.c + */ +bool cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, + char *buf, int malformed); + +#endif /* CONFIG_CIFS_ALLOW_INSECURE_LEGACY */ +#endif /* _SMB1PROTO_H */ diff --git a/fs/smb/client/smb1transport.c b/fs/smb/client/smb1transport.c index 518f764dedcce..419d8e706eb5b 100644 --- a/fs/smb/client/smb1transport.c +++ b/fs/smb/client/smb1transport.c @@ -165,7 +165,7 @@ SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, int resp_buf_type; iov[0].iov_base = in_buf; - iov[0].iov_len = get_rfc1002_length(in_buf) + 4; + iov[0].iov_len = get_rfc1002_len(in_buf) + 4; flags |= CIFS_NO_RSP_BUF; rc = SendReceive2(xid, ses, iov, 1, &resp_buf_type, flags, &rsp_iov); cifs_dbg(NOISY, "SendRcvNoRsp flags %d rc %d\n", flags, rc); @@ -177,7 +177,7 @@ int cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, bool log_error) { - unsigned int len = get_rfc1002_length(mid->resp_buf) + 4; + unsigned int len = get_rfc1002_len(mid->resp_buf) + 4; dump_smb(mid->resp_buf, min_t(u32, 92, len)); @@ -370,7 +370,7 @@ SendReceive(const unsigned int xid, struct cifs_ses *ses, goto out; } - *pbytes_returned = get_rfc1002_length(midQ->resp_buf); + *pbytes_returned = get_rfc1002_len(midQ->resp_buf); memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); rc = cifs_check_receive(midQ, server, 0); out: @@ -554,7 +554,7 @@ SendReceiveBlockingLock(const unsigned int xid, struct cifs_tcon *tcon, goto out; } - *pbytes_returned = get_rfc1002_length(midQ->resp_buf); + *pbytes_returned = get_rfc1002_len(midQ->resp_buf); memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); rc = cifs_check_receive(midQ, server, 0); out: @@ -563,3 +563,201 @@ SendReceiveBlockingLock(const unsigned int xid, struct cifs_tcon *tcon, return -ERESTARTSYS; return rc; } + +/* + return codes: + 0 not a transact2, or all data present + >0 transact2 with that much data missing + -EINVAL invalid transact2 + */ +static int +check2ndT2(char *buf) +{ + struct smb_hdr *pSMB = (struct smb_hdr *)buf; + struct smb_t2_rsp *pSMBt; + int remaining; + __u16 total_data_size, data_in_this_rsp; + + if (pSMB->Command != SMB_COM_TRANSACTION2) + return 0; + + /* check for plausible wct, bcc and t2 data and parm sizes */ + /* check for parm and data offset going beyond end of smb */ + if (pSMB->WordCount != 10) { /* coalesce_t2 depends on this */ + cifs_dbg(FYI, "Invalid transact2 word count\n"); + return -EINVAL; + } + + pSMBt = (struct smb_t2_rsp *)pSMB; + + total_data_size = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); + data_in_this_rsp = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); + + if (total_data_size == data_in_this_rsp) + return 0; + else if (total_data_size < data_in_this_rsp) { + cifs_dbg(FYI, "total data %d smaller than data in frame %d\n", + total_data_size, data_in_this_rsp); + return -EINVAL; + } + + remaining = total_data_size - data_in_this_rsp; + + cifs_dbg(FYI, "missing %d bytes from transact2, check next response\n", + remaining); + if (total_data_size > CIFSMaxBufSize) { + cifs_dbg(VFS, "TotalDataSize %d is over maximum buffer %d\n", + total_data_size, CIFSMaxBufSize); + return -EINVAL; + } + return remaining; +} + +static int +coalesce_t2(char *second_buf, struct smb_hdr *target_hdr) +{ + struct smb_t2_rsp *pSMBs = (struct smb_t2_rsp *)second_buf; + struct smb_t2_rsp *pSMBt = (struct smb_t2_rsp *)target_hdr; + char *data_area_of_tgt; + char *data_area_of_src; + int remaining; + unsigned int byte_count, total_in_tgt; + __u16 tgt_total_cnt, src_total_cnt, total_in_src; + + src_total_cnt = get_unaligned_le16(&pSMBs->t2_rsp.TotalDataCount); + tgt_total_cnt = get_unaligned_le16(&pSMBt->t2_rsp.TotalDataCount); + + if (tgt_total_cnt != src_total_cnt) + cifs_dbg(FYI, "total data count of primary and secondary t2 differ source=%hu target=%hu\n", + src_total_cnt, tgt_total_cnt); + + total_in_tgt = get_unaligned_le16(&pSMBt->t2_rsp.DataCount); + + remaining = tgt_total_cnt - total_in_tgt; + + if (remaining < 0) { + cifs_dbg(FYI, "Server sent too much data. tgt_total_cnt=%hu total_in_tgt=%u\n", + tgt_total_cnt, total_in_tgt); + return -EPROTO; + } + + if (remaining == 0) { + /* nothing to do, ignore */ + cifs_dbg(FYI, "no more data remains\n"); + return 0; + } + + total_in_src = get_unaligned_le16(&pSMBs->t2_rsp.DataCount); + if (remaining < total_in_src) + cifs_dbg(FYI, "transact2 2nd response contains too much data\n"); + + /* find end of first SMB data area */ + data_area_of_tgt = (char *)&pSMBt->hdr.Protocol + + get_unaligned_le16(&pSMBt->t2_rsp.DataOffset); + + data_area_of_src = (char *)&pSMBs->hdr.Protocol + + get_unaligned_le16(&pSMBs->t2_rsp.DataOffset); + + data_area_of_tgt += total_in_tgt; + + /* + * DataOffset fields are server-supplied and not validated against + * buffer bounds; check both data pointers before mutating the + * target header. + */ + if (data_area_of_tgt < (char *)target_hdr + + sizeof(struct smb_t2_rsp) + sizeof(__le16) || + data_area_of_tgt + total_in_src > + (char *)target_hdr + CIFSMaxBufSize + MAX_CIFS_HDR_SIZE) { + cifs_dbg(VFS, "%s: target data area out of bounds\n", __func__); + return -EPROTO; + } + if (data_area_of_src < second_buf + + sizeof(struct smb_t2_rsp) + sizeof(__le16) || + data_area_of_src + total_in_src > + second_buf + smbCalcSize((struct smb_hdr *)second_buf)) { + cifs_dbg(VFS, "%s: secondary data area out of bounds\n", __func__); + return -EPROTO; + } + + total_in_tgt += total_in_src; + /* is the result too big for the field? */ + if (total_in_tgt > USHRT_MAX) { + cifs_dbg(FYI, "coalesced DataCount too large (%u)\n", + total_in_tgt); + return -EPROTO; + } + put_unaligned_le16(total_in_tgt, &pSMBt->t2_rsp.DataCount); + + /* fix up the BCC */ + byte_count = get_bcc(target_hdr); + byte_count += total_in_src; + /* is the result too big for the field? */ + if (byte_count > USHRT_MAX) { + cifs_dbg(FYI, "coalesced BCC too large (%u)\n", byte_count); + return -EPROTO; + } + put_bcc(byte_count, target_hdr); + + byte_count = be32_to_cpu(target_hdr->smb_buf_length); + byte_count += total_in_src; + /* don't allow buffer to overflow */ + if (byte_count > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { + cifs_dbg(FYI, "coalesced BCC exceeds buffer size (%u)\n", + byte_count); + return -ENOBUFS; + } + target_hdr->smb_buf_length = cpu_to_be32(byte_count); + + /* copy second buffer into end of first buffer */ + memcpy(data_area_of_tgt, data_area_of_src, total_in_src); + + if (remaining != total_in_src) { + /* more responses to go */ + cifs_dbg(FYI, "waiting for more secondary responses\n"); + return 1; + } + + /* we are done */ + cifs_dbg(FYI, "found the last secondary response\n"); + return 0; +} + +bool +cifs_check_trans2(struct mid_q_entry *mid, struct TCP_Server_Info *server, + char *buf, int malformed) +{ + if (malformed || check2ndT2(buf) <= 0) { + /* mid->multiRsp blocks the server buf detach in handle_mid(); + * returning false here would leak resp_buf and leave a dangling + * server->smallbuf/bigbuf after the user thread frees resp_buf. + */ + if (mid->multiRsp) { + mid->multiEnd = true; + dequeue_mid(mid, true); + return true; + } + return false; + } + mid->multiRsp = true; + if (mid->resp_buf) { + /* merge response - fix up 1st*/ + malformed = coalesce_t2(buf, mid->resp_buf); + if (malformed > 0) + return true; + /* All parts received or packet is malformed. */ + mid->multiEnd = true; + dequeue_mid(mid, malformed); + return true; + } + if (!server->large_buf) { + /*FIXME: switch to already allocated largebuf?*/ + cifs_dbg(VFS, "1st trans2 resp needs bigbuf\n"); + } else { + /* Have first buffer */ + mid->resp_buf = buf; + mid->large_buf = true; + server->bigbuf = NULL; + } + return true; +} diff --git a/fs/smb/client/smb2file.c b/fs/smb/client/smb2file.c index cc1ff2461b003..36efebef82e41 100644 --- a/fs/smb/client/smb2file.c +++ b/fs/smb/client/smb2file.c @@ -30,6 +30,19 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) u8 *end = (u8 *)err + iov->iov_len; u32 len; + /* + * Per [MS-SMB2] section 2.2.2, a STATUS_STOPPED_ON_SYMLINK response has to + * carry a Symbolic Link Error Response, so ByteCount cannot be zero. Some + * servers (e.g. the macOS built-in SMB server) violate this and return an + * empty error response, with both ErrorContextCount and ByteCount set to + * zero, i.e. without the symlink target. Detect this and return -ENODATA + * so that callers can tell "server did not send the target" apart from a + * malformed response, and retrieve the target with FSCTL_GET_REPARSE_POINT + * instead. + */ + if (!err->ErrorContextCount && !le32_to_cpu(err->ByteCount)) + return ERR_PTR(-ENODATA); + if (err->ErrorContextCount) { struct smb2_error_context_rsp *p; @@ -48,7 +61,10 @@ static struct smb2_symlink_err_rsp *symlink_data(const struct kvec *iov) cifs_dbg(FYI, "%s: skipping unhandled error context: 0x%x\n", __func__, le32_to_cpu(p->ErrorId)); - len = ALIGN(le32_to_cpu(p->ErrorDataLength), 8); + len = le32_to_cpu(p->ErrorDataLength); + if (len > end - ((u8 *)p + sizeof(*p))) + return ERR_PTR(-EINVAL); + len = ALIGN(len, 8); if (len > end - ((u8 *)p + sizeof(*p))) return ERR_PTR(-EINVAL); @@ -77,7 +93,7 @@ int smb2_fix_symlink_target_type(char **target, bool directory, struct cifs_sb_i * POSIX server does not distinguish between symlinks to file and * symlink directory. So nothing is needed to fix on the client side. */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_POSIX_PATHS) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_POSIX_PATHS) return 0; if (!*target) @@ -147,7 +163,8 @@ int smb2_parse_symlink_response(struct cifs_sb_info *cifs_sb, const struct kvec cifs_sb); } -int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 *oplock, void *buf) +int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, + __u32 *oplock, void *buf) { int rc; __le16 *smb2_path; @@ -200,6 +217,14 @@ int smb2_open_file(const unsigned int xid, struct cifs_open_parms *oparms, __u32 rc = smb2_parse_symlink_response(oparms->cifs_sb, &err_iov, oparms->path, &data->symlink_target); + /* + * If smb2_parse_symlink_response returned -ENODATA then the + * symlink_target was not sent. Treat this as if the SMB2_open() + * failed with STATUS_IO_REPARSE_TAG_NOT_HANDLED status, which is + * indicated by the -EIO errno. + */ + if (rc == -ENODATA) + rc = -EIO; if (!rc) { memset(smb2_data, 0, sizeof(*smb2_data)); oparms->create_options |= OPEN_REPARSE_POINT; diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index e13bc8a97c449..b5351b620a570 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -21,7 +21,6 @@ #include "cifs_unicode.h" #include "fscache.h" #include "smb2glob.h" -#include "smb2pdu.h" #include "smb2proto.h" #include "cached_dir.h" #include "../common/smb2status.h" @@ -896,9 +895,19 @@ static int parse_create_response(struct cifs_open_info_data *data, rc = smb2_parse_symlink_response(cifs_sb, iov, full_path, &data->symlink_target); - if (rc) + if (rc != 0 && rc != -ENODATA) return rc; - tag = IO_REPARSE_TAG_SYMLINK; + /* + * -ENODATA means that the response was parsed but did not contain + * the symlink target at all (see symlink_data()). Treat it like + * STATUS_IO_REPARSE_TAG_NOT_HANDLED, which does not contain it + * either: leave the tag unset and clear rc, so that the caller + * retrieves the target with SMB2_OP_GET_REPARSE. + */ + if (rc == -ENODATA) + rc = 0; + else + tag = IO_REPARSE_TAG_SYMLINK; reparse_point = true; break; case STATUS_SUCCESS: @@ -1091,7 +1100,14 @@ int smb2_query_path_info(const unsigned int xid, rc = -EOPNOTSUPP; } - if (data->reparse.tag == IO_REPARSE_TAG_SYMLINK && !rc) { + /* + * If the symlink was already parsed in create response then it is needed to fix + * its type now (after the second call with OPEN_REPARSE_POINT which filled the + * data->fi.Attributes). If the symlink was not parsed in create response then + * the data->symlink_target was not filled yet and then the type will be fixed + * later after data->symlink_target is filled. + */ + if (data->reparse.tag == IO_REPARSE_TAG_SYMLINK && !rc && data->symlink_target) { bool directory = le32_to_cpu(data->fi.Attributes) & ATTR_DIRECTORY; rc = smb2_fix_symlink_target_type(&data->symlink_target, directory, cifs_sb); } diff --git a/fs/smb/client/smb2maperror.c b/fs/smb/client/smb2maperror.c index 12c2b868789fd..29cf456dade9e 100644 --- a/fs/smb/client/smb2maperror.c +++ b/fs/smb/client/smb2maperror.c @@ -10,10 +10,9 @@ #include #include "cifsglob.h" #include "cifs_debug.h" -#include "smb2pdu.h" #include "smb2proto.h" -#include "../common/smb2status.h" #include "smb2glob.h" +#include "../common/smb2status.h" #include "trace.h" struct status_to_posix_error { diff --git a/fs/smb/client/smb2misc.c b/fs/smb/client/smb2misc.c index c0c0ac3342570..d701bdfa585f0 100644 --- a/fs/smb/client/smb2misc.c +++ b/fs/smb/client/smb2misc.c @@ -496,9 +496,9 @@ cifs_convert_path_to_utf16(const char *from, struct cifs_sb_info *cifs_sb) __le16 *to; int map_type; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SFM_CHR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SFM_CHR) map_type = SFM_MAP_UNI_RSVD; - else if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MAP_SPECIAL_CHR) + else if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_MAP_SPECIAL_CHR) map_type = SFU_MAP_UNI_RSVD; else map_type = NO_MAP_UNI_RSVD; diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 786a7e963f1ff..67a7f0c4beac6 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -17,9 +17,9 @@ #include #include "cifsfs.h" #include "cifsglob.h" -#include "smb2pdu.h" -#include "smb2proto.h" #include "cifsproto.h" +#include "smb2proto.h" +#include "smb2pdu.h" #include "cifs_debug.h" #include "cifs_unicode.h" #include "../common/smb2status.h" @@ -1008,7 +1008,7 @@ smb2_is_path_accessible(const unsigned int xid, struct cifs_tcon *tcon, rc = -EREMOTE; } if (rc == -EREMOTE && IS_ENABLED(CONFIG_CIFS_DFS_UPCALL) && - (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_DFS)) + (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_DFS)) rc = -EOPNOTSUPP; goto out; } @@ -1539,7 +1539,7 @@ smb2_close_getattr(const unsigned int xid, struct cifs_tcon *tcon, static int SMB2_request_res_key(const unsigned int xid, struct cifs_tcon *tcon, u64 persistent_fid, u64 volatile_fid, - struct copychunk_ioctl *pcchunk) + struct copychunk_ioctl_req *pcchunk) { int rc; unsigned int ret_data_len; @@ -1831,31 +1831,31 @@ smb2_ioctl_query_info(const unsigned int xid, * * @tcon: destination file tcon * @bytes_left: how many bytes are left to copy + * @chunk_size: maximum size of a single chunk * * Return: maximum number of chunks with which Chunks[] can be filled. */ static inline u32 -calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left) +calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left, u32 chunk_size) { u32 max_chunks = READ_ONCE(tcon->max_chunks); u32 max_bytes_copy = READ_ONCE(tcon->max_bytes_copy); - u32 max_bytes_chunk = READ_ONCE(tcon->max_bytes_chunk); u64 need; u32 allowed; - if (!max_bytes_chunk || !max_bytes_copy || !max_chunks) + if (!chunk_size || !max_bytes_copy || !max_chunks) return 0; /* chunks needed for the remaining bytes */ - need = DIV_ROUND_UP_ULL(bytes_left, max_bytes_chunk); + need = DIV_ROUND_UP_ULL(bytes_left, chunk_size); /* chunks allowed per cc request */ - allowed = DIV_ROUND_UP(max_bytes_copy, max_bytes_chunk); + allowed = DIV_ROUND_UP(max_bytes_copy, chunk_size); return (u32)umin(need, umin(max_chunks, allowed)); } /** - * smb2_copychunk_range - server-side copy of data range + * __smb2_copychunk_range - server-side copy of data range * * @xid: transaction id * @src_file: source file @@ -1867,28 +1867,30 @@ calc_chunk_count(struct cifs_tcon *tcon, u64 bytes_left) * Obtains a resume key for @src_file and issues FSCTL_SRV_COPYCHUNK_WRITE * IOCTLs, splitting the request into chunks limited by tcon->max_*. * - * Return: @len on success; negative errno on failure. + * Return: 0 on success; negative errno on failure. */ -static ssize_t -smb2_copychunk_range(const unsigned int xid, - struct cifsFileInfo *src_file, - struct cifsFileInfo *dst_file, - u64 src_off, - u64 len, - u64 dst_off) +static int +__smb2_copychunk_range(const unsigned int xid, + struct cifsFileInfo *src_file, + struct cifsFileInfo *dst_file, + u64 src_off, + u64 len, + u64 dst_off) { int rc = 0; unsigned int ret_data_len = 0; - struct copychunk_ioctl *cc_req = NULL; + struct copychunk_ioctl_req *cc_req = NULL; struct copychunk_ioctl_rsp *cc_rsp = NULL; struct cifs_tcon *tcon; - struct copychunk *chunk; - u32 chunks, chunk_count, chunk_bytes; + struct srv_copychunk *chunk; + u32 chunks, chunk_count, chunk_bytes, chunk_size; u32 copy_bytes, copy_bytes_left; u32 chunks_written, bytes_written; u64 total_bytes_left = len; u64 src_off_prev, dst_off_prev; + u64 max_chunk = 0; u32 retries = 0; + bool reverse = false; tcon = tlink_tcon(dst_file->tlink); @@ -1896,8 +1898,50 @@ smb2_copychunk_range(const unsigned int xid, dst_file->fid.volatile_fid, tcon->tid, tcon->ses->Suid, src_off, dst_off, len); + /* + * Same-file left shifts are safe in forward order. For a right shift, + * let L be the copy length, delta the distance between the source and + * destination, and C the normal chunk size: + * + * delta >= L: copy forwards using C + * delta < L: + * delta >= C: copy backwards using C + * delta < C: copy backwards with chunks limited to delta + * + * Copying backwards prevents one chunk from overwriting data needed by + * a later chunk. Limiting the chunk size to delta prevents an individual + * chunk from overlapping itself. + * This limit can be removed once all supported servers handle overlapping + * descriptors safely. + * + * A small right shift over a large range may therefore require many + * chunks. + */ + if (src_file == dst_file && dst_off > src_off) { + u64 delta = dst_off - src_off; + + if (delta < len) { + reverse = true; + max_chunk = delta; + } + } + + /* + * A backward copy walks the offsets down from the end of the range. + * Do this once, outside the retry loop, so a retry does not move the + * offsets again. + */ + if (reverse) { + src_off += len; + dst_off += len; + } + retry: - chunk_count = calc_chunk_count(tcon, total_bytes_left); + chunk_size = READ_ONCE(tcon->max_bytes_chunk); + if (max_chunk && max_chunk < chunk_size) + chunk_size = (u32)max_chunk; + + chunk_count = calc_chunk_count(tcon, total_bytes_left, chunk_size); if (!chunk_count) { rc = -EOPNOTSUPP; goto out; @@ -1926,22 +1970,33 @@ smb2_copychunk_range(const unsigned int xid, src_off_prev = src_off; dst_off_prev = dst_off; + /* + * __counted_by_le(ChunkCount): set to allocated chunks before + * populating Chunks[] + */ + cc_req->ChunkCount = cpu_to_le32(chunk_count); + chunks = 0; copy_bytes = 0; copy_bytes_left = umin(total_bytes_left, tcon->max_bytes_copy); while (copy_bytes_left > 0 && chunks < chunk_count) { chunk = &cc_req->Chunks[chunks++]; + chunk_bytes = umin(copy_bytes_left, chunk_size); + if (reverse) { + src_off -= chunk_bytes; + dst_off -= chunk_bytes; + } + chunk->SourceOffset = cpu_to_le64(src_off); chunk->TargetOffset = cpu_to_le64(dst_off); - - chunk_bytes = umin(copy_bytes_left, tcon->max_bytes_chunk); - chunk->Length = cpu_to_le32(chunk_bytes); /* Buffer is zeroed, no need to set chunk->Reserved = 0 */ - src_off += chunk_bytes; - dst_off += chunk_bytes; + if (!reverse) { + src_off += chunk_bytes; + dst_off += chunk_bytes; + } copy_bytes_left -= chunk_bytes; copy_bytes += chunk_bytes; @@ -1982,6 +2037,18 @@ smb2_copychunk_range(const unsigned int xid, goto out; } + /* + * A successful COPYCHUNK should copy every descriptor (MS-SMB2 + * 3.3.5.15.6). Reject a short backward copy because the rewind + * below only supports forward copying. + */ + if (unlikely(reverse && bytes_written < copy_bytes)) { + cifs_tcon_dbg(VFS, "Copychunk short write %u/%u (reverse)\n", + bytes_written, copy_bytes); + rc = -EIO; + goto out; + } + /* Partial write: rewind */ if (bytes_written < copy_bytes) { u32 delta = copy_bytes - bytes_written; @@ -2043,10 +2110,27 @@ smb2_copychunk_range(const unsigned int xid, trace_smb3_copychunk_done(xid, src_file->fid.volatile_fid, dst_file->fid.volatile_fid, tcon->tid, tcon->ses->Suid, src_off, dst_off, len); - return len; + return 0; } } +static ssize_t +smb2_copychunk_range(const unsigned int xid, + struct cifsFileInfo *src_file, + struct cifsFileInfo *dst_file, + u64 src_off, + u64 len, + u64 dst_off) +{ + int rc; + + rc = __smb2_copychunk_range(xid, src_file, dst_file, src_off, len, + dst_off); + if (rc) + return rc; + return len; +} + static int smb2_flush_file(const unsigned int xid, struct cifs_tcon *tcon, struct cifs_fid *fid) @@ -2197,8 +2281,7 @@ smb2_duplicate_extents(const unsigned int xid, rc = smb2_set_file_size(xid, tcon, trgtfile, dest_off + len, false); if (rc) goto duplicate_extents_out; - netfs_resize_file(netfs_inode(inode), dest_off + len, true); - cifs_setsize(inode, dest_off + len); + cifs_resize_file_locked(inode, dest_off + len); } rc = SMB2_ioctl(xid, tcon, trgtfile->fid.persistent_fid, trgtfile->fid.volatile_fid, @@ -3381,6 +3464,13 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, trace_smb3_zero_enter(xid, cfile->fid.persistent_fid, tcon->tid, ses->Suid, offset, len); + new_size = offset + len; + if (!keep_size && i_size_read(inode) < new_size) { + rc = inode_newsize_ok(inode, new_size); + if (rc) + goto out; + } + filemap_invalidate_lock(inode->i_mapping); i_size = i_size_read(inode); @@ -3412,7 +3502,6 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, /* * do we also need to change the size of the file? */ - new_size = offset + len; if (keep_size == false && (unsigned long long)i_size_read(inode) < new_size) { rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_size); @@ -3427,6 +3516,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon, zero_range_exit: filemap_invalidate_unlock(inode->i_mapping); + out: free_xid(xid); if (rc) trace_smb3_zero_err(xid, cfile->fid.persistent_fid, tcon->tid, @@ -3457,6 +3547,15 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon, goto out; filemap_invalidate_lock(inode->i_mapping); + /* + * Flush dirty data first, otherwise a dirty folio spanning the punched + * range may be written back after the ioctl and refill the hole. + */ + rc = filemap_write_and_wait_range(inode->i_mapping, offset, + offset + len - 1); + if (rc < 0) + goto unlock; + /* * We implement the punch hole through ioctl, so we need remove the page * caches first, otherwise the data may be inconsistent with the server. @@ -3508,7 +3607,7 @@ static int smb3_simple_fallocate_write_range(unsigned int xid, char *buf) { struct cifs_io_parms io_parms = {0}; - int nbytes; + unsigned int nbytes; int rc = 0; struct kvec iov[2]; @@ -3529,9 +3628,10 @@ static int smb3_simple_fallocate_write_range(unsigned int xid, rc = SMB2_write(xid, &io_parms, &nbytes, iov, 1); if (rc) break; + if (!nbytes) + return -EIO; if (nbytes > len) return -EINVAL; - buf += nbytes; off += nbytes; len -= nbytes; } @@ -3544,11 +3644,25 @@ static int smb3_simple_fallocate_range(unsigned int xid, loff_t off, loff_t len) { struct file_allocated_range_buffer in_data, *out_data = NULL, *tmp_data; + struct inode *inode = d_inode(cfile->dentry); u32 out_data_len; char *buf = NULL; + u64 range_start, range_len, range_end; loff_t l; int rc; + buf = kvzalloc(min_t(loff_t, len, SMB2_MAX_BUFFER_SIZE), GFP_KERNEL); + if (!buf) { + rc = -ENOMEM; + goto out; + } + + if (off >= i_size_read(inode)) { + rc = smb3_simple_fallocate_write_range(xid, tcon, cfile, + off, len, buf); + goto out; + } + in_data.file_offset = cpu_to_le64(off); in_data.length = cpu_to_le64(len); rc = SMB2_ioctl(xid, tcon, cfile->fid.persistent_fid, @@ -3560,12 +3674,6 @@ static int smb3_simple_fallocate_range(unsigned int xid, if (rc) goto out; - buf = kvzalloc(1024 * 1024, GFP_KERNEL); - if (buf == NULL) { - rc = -ENOMEM; - goto out; - } - tmp_data = out_data; while (len) { /* @@ -3582,13 +3690,21 @@ static int smb3_simple_fallocate_range(unsigned int xid, goto out; } - if (off < le64_to_cpu(tmp_data->file_offset)) { + range_start = le64_to_cpu(tmp_data->file_offset); + range_len = le64_to_cpu(tmp_data->length); + if (check_add_overflow(range_start, range_len, &range_end) || + range_end > S64_MAX) { + rc = -EINVAL; + goto out; + } + + if (off < range_start) { /* * We are at a hole. Write until the end of the region * or until the next allocated data, * whichever comes next. */ - l = le64_to_cpu(tmp_data->file_offset) - off; + l = range_start - off; if (len < l) l = len; rc = smb3_simple_fallocate_write_range(xid, tcon, @@ -3605,11 +3721,13 @@ static int smb3_simple_fallocate_range(unsigned int xid, * until the end of the data or the end of the region * we are supposed to fallocate, whichever comes first. */ - l = le64_to_cpu(tmp_data->length); - if (len < l) - l = len; - off += l; - len -= l; + if (off < range_end) { + l = range_end - off; + if (len < l) + l = len; + off += l; + len -= l; + } tmp_data = &tmp_data[1]; out_data_len -= sizeof(struct file_allocated_range_buffer); @@ -3630,18 +3748,22 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; long rc = -EOPNOTSUPP; unsigned int xid; - loff_t new_eof; + loff_t old_eof, new_eof; + struct smb2_file_all_info file_inf; + u64 asize; + int qrc; xid = get_xid(); inode = d_inode(cfile->dentry); cifsi = CIFS_I(inode); + old_eof = i_size_read(inode); trace_smb3_falloc_enter(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len); /* if file not oplocked can't be sure whether asking to extend size */ if (!CIFS_CACHE_READ(cifsi)) - if (keep_size == false) { + if (!keep_size) { trace_smb3_falloc_err(xid, cfile->fid.persistent_fid, tcon->tid, tcon->ses->Suid, off, len, rc); free_xid(xid); @@ -3651,21 +3773,73 @@ static long smb3_simple_falloc(struct file *file, struct cifs_tcon *tcon, /* * Extending the file */ - if ((keep_size == false) && i_size_read(inode) < off + len) { + if (!keep_size && old_eof < off + len) { rc = inode_newsize_ok(inode, off + len); if (rc) goto out; + /* + * A small range at or beyond EOF can be allocated by writing + * zeroes. For off > old_eof, this preserves the intervening + * hole instead of allocating from offset 0. + */ + if (off > old_eof || + (off == old_eof && old_eof != 0 && + (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE))) { + if (len > 1024 * 1024) { + rc = -EOPNOTSUPP; + goto out; + } + + rc = smb3_simple_fallocate_range(xid, tcon, cfile, + off, len); + if (rc) { + spin_lock(&inode->i_lock); + cifsi->time = 0; + spin_unlock(&inode->i_lock); + goto out; + } + + new_eof = off + len; + cifs_resize_file_locked(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; + } + spin_unlock(&inode->i_lock); + goto out; + } + if (cifsi->cifsAttrs & FILE_ATTRIBUTE_SPARSE_FILE) smb2_set_sparse(xid, tcon, cfile, inode, false); new_eof = off + len; rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid, cfile->fid.volatile_fid, cfile->pid, new_eof); - if (rc == 0) { - netfs_resize_file(&cifsi->netfs, new_eof, true); - cifs_setsize(inode, new_eof); + if (rc) + goto out; + + cifs_resize_file_locked(inode, new_eof); + + qrc = SMB2_query_info(xid, tcon, + cfile->fid.persistent_fid, + cfile->fid.volatile_fid, &file_inf); + spin_lock(&inode->i_lock); + if (qrc == 0) { + asize = le64_to_cpu(file_inf.AllocationSize); + if (asize >= new_eof) + inode->i_blocks = CIFS_INO_BLOCKS(asize); + } else { + cifsi->time = 0; } + spin_unlock(&inode->i_lock); goto out; } @@ -3776,8 +3950,8 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon, ictx->zero_point = old_eof; netfs_wait_for_outstanding_io(inode); - rc = smb2_copychunk_range(xid, cfile, cfile, off + len, - old_eof - off - len, off); + rc = __smb2_copychunk_range(xid, cfile, cfile, off + len, + old_eof - off - len, off); if (rc < 0) goto out_2; @@ -3808,7 +3982,7 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, struct cifsFileInfo *cfile = file->private_data; struct inode *inode = file_inode(file); struct cifsInodeInfo *cifsi = CIFS_I(inode); - __u64 count, old_eof, new_eof; + loff_t old_eof, new_eof; xid = get_xid(); @@ -3818,8 +3992,18 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, goto out; } - count = old_eof - off; - new_eof = old_eof + len; + if (check_add_overflow(old_eof, len, &new_eof)) { + rc = -EFBIG; + goto out; + } + rc = inode_newsize_ok(inode, new_eof); + if (rc) + goto out; + + /* SET_ZERO_DATA creates a hole only in a sparse file. */ + rc = smb2_set_sparse(xid, tcon, cfile, inode, true); + if (rc) + goto out; filemap_invalidate_lock(inode->i_mapping); rc = filemap_write_and_wait_range(inode->i_mapping, off, new_eof - 1); @@ -3837,7 +4021,12 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon, netfs_resize_file(&cifsi->netfs, i_size_read(inode), true); fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode)); - rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len); + /* + * Move [off, old_eof) right by len. The helper copies backwards if the + * source and destination ranges overlap. + */ + rc = __smb2_copychunk_range(xid, cfile, cfile, off, old_eof - off, + off + len); if (rc < 0) goto out_2; cifsi->netfs.zero_point = new_eof; @@ -5222,7 +5411,7 @@ int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, { struct TCP_Server_Info *server = tcon->ses->server; struct cifs_open_parms oparms; - struct cifs_open_info_data idata; + struct cifs_open_info_data idata = {}; struct cifs_io_parms io_parms = {}; struct cifs_sb_info *cifs_sb = CIFS_SB(inode->i_sb); struct cifs_fid fid; @@ -5271,16 +5460,13 @@ int __cifs_sfu_make_node(unsigned int xid, struct inode *inode, data = (u8 *)symname_utf16; break; case S_IFSOCK: - type_len = 8; - strscpy(type, "LnxSOCK"); - data = (u8 *)&pdev; - data_len = sizeof(pdev); + /* SFU socket is system file with one zero byte */ + type_len = 1; + type[0] = '\0'; break; case S_IFIFO: - type_len = 8; - strscpy(type, "LnxFIFO"); - data = (u8 *)&pdev; - data_len = sizeof(pdev); + /* SFU fifo is system file which is empty */ + type_len = 0; break; default: rc = -EPERM; @@ -5377,7 +5563,7 @@ static int smb2_make_node(unsigned int xid, struct inode *inode, * supports block and char device, socket & fifo, * and was used by default in earlier versions of Windows */ - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) { + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_UNX_EMUL) { rc = cifs_sfu_make_node(xid, inode, dentry, tcon, full_path, mode, dev); } else if (CIFS_REPARSE_SUPPORT(tcon)) { diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c index 4828cff55fb7f..e4f18766f0a1d 100644 --- a/fs/smb/client/smb2pdu.c +++ b/fs/smb/client/smb2pdu.c @@ -26,8 +26,8 @@ #include #include #include "cifsglob.h" -#include "cifsacl.h" #include "cifsproto.h" +#include "cifsacl.h" #include "smb2proto.h" #include "cifs_unicode.h" #include "cifs_debug.h" @@ -3118,7 +3118,7 @@ SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, bool set_mode; bool set_owner; - if ((oparms->cifs_sb->mnt_cifs_flags & CIFS_MOUNT_MODE_FROM_SID) && + if ((cifs_sb_flags(oparms->cifs_sb) & CIFS_MOUNT_MODE_FROM_SID) && (oparms->mode != ACL_NO_MODE)) set_mode = true; else { @@ -3126,7 +3126,7 @@ SMB2_open_init(struct cifs_tcon *tcon, struct TCP_Server_Info *server, oparms->mode = ACL_NO_MODE; } - if (oparms->cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UID_FROM_ACL) + if (cifs_sb_flags(oparms->cifs_sb) & CIFS_MOUNT_UID_FROM_ACL) set_owner = true; else set_owner = false; @@ -3921,7 +3921,8 @@ int SMB2_query_info(const unsigned int xid, struct cifs_tcon *tcon, /* currently unused, as now we are doing compounding instead (see smb311_posix_query_path_info) */ int SMB311_posix_query_info(const unsigned int xid, struct cifs_tcon *tcon, - u64 persistent_fid, u64 volatile_fid, struct smb311_posix_qinfo *data, u32 *plen) + u64 persistent_fid, u64 volatile_fid, + struct smb311_posix_qinfo *data, u32 *plen) { size_t output_len = sizeof(struct smb311_posix_qinfo) + (sizeof(struct smb_sid) * 2) + (PATH_MAX * 2); @@ -4453,8 +4454,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, if (rc) return rc; - if (server == NULL) - return -ECONNABORTED; + if (!server) { + rc = -ECONNABORTED; + goto free_req; + } shdr = &req->hdr; shdr->Id.SyncId.ProcessId = cpu_to_le32(io_parms->pid); @@ -4485,8 +4488,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, rdata->mr = smbd_register_mr(server->smbd_conn, &rdata->subreq.io_iter, true, need_invalidate); - if (!rdata->mr) - return -EAGAIN; + if (!rdata->mr) { + rc = -EAGAIN; + goto free_req; + } req->Channel = SMB2_CHANNEL_RDMA_V1_INVALIDATE; if (need_invalidate) @@ -4529,6 +4534,10 @@ smb2_new_read_req(void **buf, unsigned int *total_len, *buf = req; return rc; + +free_req: + cifs_small_buf_release(req); + return rc; } static void diff --git a/fs/smb/client/smb2pdu.h b/fs/smb/client/smb2pdu.h index 101024f8f7256..cdf7c588cc00d 100644 --- a/fs/smb/client/smb2pdu.h +++ b/fs/smb/client/smb2pdu.h @@ -193,36 +193,12 @@ struct crt_sd_ctxt { struct smb3_sd sd; } __packed; - -#define COPY_CHUNK_RES_KEY_SIZE 24 struct resume_key_req { char ResumeKey[COPY_CHUNK_RES_KEY_SIZE]; __le32 ContextLength; /* MBZ */ char Context[]; /* ignored, Windows sets to 4 bytes of zero */ } __packed; - -struct copychunk { - __le64 SourceOffset; - __le64 TargetOffset; - __le32 Length; - __le32 Reserved; -} __packed; - -/* this goes in the ioctl buffer when doing a copychunk request */ -struct copychunk_ioctl { - char SourceKey[COPY_CHUNK_RES_KEY_SIZE]; - __le32 ChunkCount; - __le32 Reserved; - struct copychunk Chunks[]; -} __packed; - -struct copychunk_ioctl_rsp { - __le32 ChunksWritten; - __le32 ChunkBytesWritten; - __le32 TotalBytesWritten; -} __packed; - /* See MS-FSCC 2.3.29 and 2.3.30 */ struct get_retrieval_pointer_count_req { __le64 StartingVcn; /* virtual cluster number (signed) */ diff --git a/fs/smb/client/smb2proto.h b/fs/smb/client/smb2proto.h index 5241daaae5433..92e6b93d72c49 100644 --- a/fs/smb/client/smb2proto.h +++ b/fs/smb/client/smb2proto.h @@ -9,8 +9,10 @@ */ #ifndef _SMB2PROTO_H #define _SMB2PROTO_H + #include #include +#include "cached_dir.h" struct statfs; struct smb_rqst; @@ -39,15 +41,11 @@ extern struct mid_q_entry *smb2_setup_async_request( struct TCP_Server_Info *server, struct smb_rqst *rqst); extern struct cifs_tcon *smb2_find_smb_tcon(struct TCP_Server_Info *server, __u64 ses_id, __u32 tid); -extern void smb2_echo_request(struct work_struct *work); extern __le32 smb2_get_lease_state(struct cifsInodeInfo *cinode); extern bool smb2_is_valid_oplock_break(char *buffer, struct TCP_Server_Info *srv); extern int smb3_handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid); -extern int smb2_query_reparse_tag(const unsigned int xid, struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, const char *path, - __u32 *reparse_tag); struct inode *smb2_create_reparse_inode(struct cifs_open_info_data *data, struct super_block *sb, const unsigned int xid, @@ -300,17 +298,9 @@ extern int smb2_query_info_compound(const unsigned int xid, struct kvec *rsp, int *buftype, struct cifs_sb_info *cifs_sb); /* query path info from the server using SMB311 POSIX extensions*/ -int smb311_posix_query_path_info(const unsigned int xid, - struct cifs_tcon *tcon, - struct cifs_sb_info *cifs_sb, - const char *full_path, - struct cifs_open_info_data *data); int posix_info_parse(const void *beg, const void *end, struct smb2_posix_info_parsed *out); int posix_info_sid_size(const void *beg, const void *end); -int smb2_make_nfs_node(unsigned int xid, struct inode *inode, - struct dentry *dentry, struct cifs_tcon *tcon, - const char *full_path, umode_t mode, dev_t dev); int smb2_rename_pending_delete(const char *full_path, struct dentry *dentry, const unsigned int xid); diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h index dbed7c082af32..90fd665ff8dd0 100644 --- a/fs/smb/client/trace.h +++ b/fs/smb/client/trace.h @@ -20,6 +20,137 @@ /* * Specify enums for tracing information. */ +#define smb_eio_traces \ + EM(smb_eio_trace_compress_copy, "compress_copy") \ + EM(smb_eio_trace_copychunk_inv_rsp, "copychunk_inv_rsp") \ + EM(smb_eio_trace_copychunk_overcopy_b, "copychunk_overcopy_b") \ + EM(smb_eio_trace_copychunk_overcopy_c, "copychunk_overcopy_c") \ + EM(smb_eio_trace_create_rsp_too_small, "create_rsp_too_small") \ + EM(smb_eio_trace_dfsref_no_rsp, "dfsref_no_rsp") \ + EM(smb_eio_trace_ea_overrun, "ea_overrun") \ + EM(smb_eio_trace_extract_will_pin, "extract_will_pin") \ + EM(smb_eio_trace_forced_shutdown, "forced_shutdown") \ + EM(smb_eio_trace_getacl_bcc_too_small, "getacl_bcc_too_small") \ + EM(smb_eio_trace_getcifsacl_param_count, "getcifsacl_param_count") \ + EM(smb_eio_trace_getdfsrefer_bcc_too_small, "getdfsrefer_bcc_too_small") \ + EM(smb_eio_trace_getextattr_bcc_too_small, "getextattr_bcc_too_small") \ + EM(smb_eio_trace_getextattr_inv_size, "getextattr_inv_size") \ + EM(smb_eio_trace_getsrvinonum_bcc_too_small, "getsrvinonum_bcc_too_small") \ + EM(smb_eio_trace_getsrvinonum_size, "getsrvinonum_size") \ + EM(smb_eio_trace_ioctl_data_len, "ioctl_data_len") \ + EM(smb_eio_trace_ioctl_no_rsp, "ioctl_no_rsp") \ + EM(smb_eio_trace_ioctl_out_off, "ioctl_out_off") \ + EM(smb_eio_trace_lock_bcc_too_small, "lock_bcc_too_small") \ + EM(smb_eio_trace_lock_data_too_small, "lock_data_too_small") \ + EM(smb_eio_trace_malformed_ksid_key, "malformed_ksid_key") \ + EM(smb_eio_trace_malformed_sid_key, "malformed_sid_key") \ + EM(smb_eio_trace_mkdir_no_rsp, "mkdir_no_rsp") \ + EM(smb_eio_trace_neg_bad_rsplen, "neg_bad_rsplen") \ + EM(smb_eio_trace_neg_decode_token, "neg_decode_token") \ + EM(smb_eio_trace_neg_info_caps, "neg_info_caps") \ + EM(smb_eio_trace_neg_info_dialect, "neg_info_dialect") \ + EM(smb_eio_trace_neg_info_fail, "neg_info_fail") \ + EM(smb_eio_trace_neg_info_sec_mode, "neg_info_sec_mode") \ + EM(smb_eio_trace_neg_inval_dialect, "neg_inval_dialect") \ + EM(smb_eio_trace_neg_no_crypt_key, "neg_no_crypt_key") \ + EM(smb_eio_trace_neg_sec_blob_too_small, "neg_sec_blob_too_small") \ + EM(smb_eio_trace_neg_unreq_dialect, "neg_unreq_dialect") \ + EM(smb_eio_trace_no_auth_key, "no_auth_key") \ + EM(smb_eio_trace_no_lease_key, "no_lease_key") \ + EM(smb_eio_trace_not_netfs_writeback, "not_netfs_writeback") \ + EM(smb_eio_trace_null_pointers, "null_pointers") \ + EM(smb_eio_trace_oldqfsinfo_bcc_too_small, "oldqfsinfo_bcc_too_small") \ + EM(smb_eio_trace_pend_del_fail, "pend_del_fail") \ + EM(smb_eio_trace_qalleas_bcc_too_small, "qalleas_bcc_too_small") \ + EM(smb_eio_trace_qalleas_ea_overlong, "qalleas_ea_overlong") \ + EM(smb_eio_trace_qalleas_overlong, "qalleas_overlong") \ + EM(smb_eio_trace_qfileinfo_bcc_too_small, "qfileinfo_bcc_too_small") \ + EM(smb_eio_trace_qfileinfo_invalid, "qfileinfo_invalid") \ + EM(smb_eio_trace_qfsattrinfo_bcc_too_small, "qfsattrinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsdevinfo_bcc_too_small, "qfsdevinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsinfo_bcc_too_small, "qfsinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsposixinfo_bcc_too_small, "qfsposixinfo_bcc_too_small") \ + EM(smb_eio_trace_qfsunixinfo_bcc_too_small, "qfsunixinfo_bcc_too_small") \ + EM(smb_eio_trace_qpathinfo_bcc_too_small, "qpathinfo_bcc_too_small") \ + EM(smb_eio_trace_qpathinfo_invalid, "qpathinfo_invalid") \ + EM(smb_eio_trace_qreparse_data_area, "qreparse_data_area") \ + EM(smb_eio_trace_qreparse_rep_datalen, "qreparse_rep_datalen") \ + EM(smb_eio_trace_qreparse_ret_datalen, "qreparse_ret_datalen") \ + EM(smb_eio_trace_qreparse_setup_count, "qreparse_setup_count") \ + EM(smb_eio_trace_qreparse_sizes_wrong, "qreparse_sizes_wrong") \ + EM(smb_eio_trace_qsym_bcc_too_small, "qsym_bcc_too_small") \ + EM(smb_eio_trace_read_mid_state_unknown, "read_mid_state_unknown") \ + EM(smb_eio_trace_read_overlarge, "read_overlarge") \ + EM(smb_eio_trace_read_rsp_malformed, "read_rsp_malformed") \ + EM(smb_eio_trace_read_rsp_short, "read_rsp_short") \ + EM(smb_eio_trace_read_too_far, "read_too_far") \ + EM(smb_eio_trace_reparse_data_len, "reparse_data_len") \ + EM(smb_eio_trace_reparse_native_len, "reparse_native_len") \ + EM(smb_eio_trace_reparse_native_nul, "reparse_native_nul") \ + EM(smb_eio_trace_reparse_native_sym_len, "reparse_native_sym_len") \ + EM(smb_eio_trace_reparse_nfs_dev, "reparse_nfs_dev") \ + EM(smb_eio_trace_reparse_nfs_nul, "reparse_nfs_nul") \ + EM(smb_eio_trace_reparse_nfs_sockfifo, "reparse_nfs_sockfifo") \ + EM(smb_eio_trace_reparse_nfs_symbuf, "reparse_nfs_symbuf") \ + EM(smb_eio_trace_reparse_nfs_too_short, "reparse_nfs_too_short") \ + EM(smb_eio_trace_reparse_overlong, "reparse_overlong") \ + EM(smb_eio_trace_reparse_rdlen, "reparse_rdlen") \ + EM(smb_eio_trace_reparse_wsl_nul, "reparse_wsl_nul") \ + EM(smb_eio_trace_reparse_wsl_symbuf, "reparse_wsl_symbuf") \ + EM(smb_eio_trace_reparse_wsl_ver, "reparse_wsl_ver") \ + EM(smb_eio_trace_rx_b_read_short, "rx_b_read_short") \ + EM(smb_eio_trace_rx_bad_datalen, "rx_bad_datalen") \ + EM(smb_eio_trace_rx_both_buf, "rx_both_buf") \ + EM(smb_eio_trace_rx_calc_len_too_big, "rx_calc_len_too_big") \ + EM(smb_eio_trace_rx_check_rsp, "rx_check_rsp") \ + EM(smb_eio_trace_rx_copy_to_iter, "rx_copy_to_iter") \ + EM(smb_eio_trace_rx_insuff_res, "rx_insuff_res") \ + EM(smb_eio_trace_rx_inv_bcc, "rx_inv_bcc") \ + EM(smb_eio_trace_rx_mid_unready, "rx_mid_unready") \ + EM(smb_eio_trace_rx_neg_sess_resp, "rx_neg_sess_resp") \ + EM(smb_eio_trace_rx_overlong, "rx_overlong") \ + EM(smb_eio_trace_rx_overpage, "rx_overpage") \ + EM(smb_eio_trace_rx_pos_sess_resp, "rx_pos_sess_resp") \ + EM(smb_eio_trace_rx_rfc1002_magic, "rx_rfc1002_magic") \ + EM(smb_eio_trace_rx_sync_mid_invalid, "rx_sync_mid_invalid") \ + EM(smb_eio_trace_rx_sync_mid_malformed, "rx_sync_mid_malformed") \ + EM(smb_eio_trace_rx_too_short, "rx_too_short") \ + EM(smb_eio_trace_rx_trans2_extract, "rx_trans2_extract") \ + EM(smb_eio_trace_rx_unknown_resp, "rx_unknown_resp") \ + EM(smb_eio_trace_rx_unspec_error, "rx_unspec_error") \ + EM(smb_eio_trace_sess_buf_off, "sess_buf_off") \ + EM(smb_eio_trace_sess_exiting, "sess_exiting") \ + EM(smb_eio_trace_sess_krb_wcc, "sess_krb_wcc") \ + EM(smb_eio_trace_sess_nl2_wcc, "sess_nl2_wcc") \ + EM(smb_eio_trace_sess_rawnl_auth_wcc, "sess_rawnl_auth_wcc") \ + EM(smb_eio_trace_sess_rawnl_neg_wcc, "sess_rawnl_neg_wcc") \ + EM(smb_eio_trace_short_symlink_write, "short_symlink_write") \ + EM(smb_eio_trace_sid_too_many_auth, "sid_too_many_auth") \ + EM(smb_eio_trace_sig_data_too_small, "sig_data_too_small") \ + EM(smb_eio_trace_sig_iter, "sig_iter") \ + EM(smb_eio_trace_smb1_received_error, "smb1_received_error") \ + EM(smb_eio_trace_smb2_received_error, "smb2_received_error") \ + EM(smb_eio_trace_sym_slash, "sym_slash") \ + EM(smb_eio_trace_sym_target_len, "sym_target_len") \ + EM(smb_eio_trace_symlink_file_size, "symlink_file_size") \ + EM(smb_eio_trace_tcon_bcc_too_small, "tcon_bcc_too_small") \ + EM(smb_eio_trace_tdis_in_reconnect, "tdis_in_reconnect") \ + EM(smb_eio_trace_tx_chained_async, "tx_chained_async") \ + EM(smb_eio_trace_tx_compress_failed, "tx_compress_failed") \ + EM(smb_eio_trace_tx_copy_iter_to_buf, "tx_copy_iter_to_buf") \ + EM(smb_eio_trace_tx_copy_to_buf, "tx_copy_to_buf") \ + EM(smb_eio_trace_tx_max_compound, "tx_max_compound") \ + EM(smb_eio_trace_tx_miscopy_to_buf, "tx_miscopy_to_buf") \ + EM(smb_eio_trace_tx_need_transform, "tx_need_transform") \ + EM(smb_eio_trace_tx_too_long, "sr_too_long") \ + EM(smb_eio_trace_unixqfileinfo_bcc_too_small, "unixqfileinfo_bcc_too_small") \ + EM(smb_eio_trace_unixqpathinfo_bcc_too_small, "unixqpathinfo_bcc_too_small") \ + EM(smb_eio_trace_user_iter, "user_iter") \ + EM(smb_eio_trace_write_bad_buf_type, "write_bad_buf_type") \ + EM(smb_eio_trace_write_mid_state_unknown, "write_mid_state_unknown") \ + EM(smb_eio_trace_write_rsp_malformed, "write_rsp_malformed") \ + E_(smb_eio_trace_write_too_far, "write_too_far") + #define smb3_rw_credits_traces \ EM(cifs_trace_rw_credits_call_readv_adjust, "rd-call-adj") \ EM(cifs_trace_rw_credits_call_writev_adjust, "wr-call-adj") \ @@ -84,6 +215,7 @@ #define EM(a, b) a, #define E_(a, b) a +enum smb_eio_trace { smb_eio_traces }; enum smb3_rw_credits_trace { smb3_rw_credits_traces } __mode(byte); enum smb3_tcon_ref_trace { smb3_tcon_ref_traces } __mode(byte); @@ -97,6 +229,7 @@ enum smb3_tcon_ref_trace { smb3_tcon_ref_traces } __mode(byte); #define EM(a, b) TRACE_DEFINE_ENUM(a); #define E_(a, b) TRACE_DEFINE_ENUM(a); +smb_eio_traces; smb3_rw_credits_traces; smb3_tcon_ref_traces; @@ -1621,6 +1754,24 @@ TRACE_EVENT(smb3_rw_credits, __entry->server_credits, __entry->in_flight) ); +TRACE_EVENT(smb3_eio, + TP_PROTO(enum smb_eio_trace trace, unsigned long info, unsigned long info2), + TP_ARGS(trace, info, info2), + TP_STRUCT__entry( + __field(enum smb_eio_trace, trace) + __field(unsigned long, info) + __field(unsigned long, info2) + ), + TP_fast_assign( + __entry->trace = trace; + __entry->info = info; + __entry->info2 = info2; + ), + TP_printk("%s info=%lx,%lx", + __print_symbolic(__entry->trace, smb_eio_traces), + __entry->info, __entry->info2) + ); + #undef EM #undef E_ diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 2d7140767dc43..4da3d62b6481e 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -105,12 +105,11 @@ void __release_mid(struct kref *refcount) trace_smb3_slow_rsp(smb_cmd, midEntry->mid, midEntry->pid, midEntry->when_sent, midEntry->when_received); if (cifsFYI & CIFS_TIMER) { - pr_debug("slow rsp: cmd %d mid %llu", - midEntry->command, midEntry->mid); - cifs_info("A: 0x%lx S: 0x%lx R: 0x%lx\n", - now - midEntry->when_alloc, - now - midEntry->when_sent, - now - midEntry->when_received); + pr_debug("slow rsp: cmd %d mid %llu A: 0x%lx S: 0x%lx R: 0x%lx\n", + midEntry->command, midEntry->mid, + now - midEntry->when_alloc, + now - midEntry->when_sent, + now - midEntry->when_received); } } #endif diff --git a/fs/smb/client/xattr.c b/fs/smb/client/xattr.c index 029910d56c22e..3c8b6d6e26c61 100644 --- a/fs/smb/client/xattr.c +++ b/fs/smb/client/xattr.c @@ -150,7 +150,7 @@ static int cifs_xattr_set(const struct xattr_handler *handler, break; } - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_XATTR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) goto out; if (pTcon->ses->server->ops->set_EA) { @@ -310,7 +310,7 @@ static int cifs_xattr_get(const struct xattr_handler *handler, break; } - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_XATTR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) goto out; if (pTcon->ses->server->ops->query_all_EAs) @@ -399,7 +399,7 @@ ssize_t cifs_listxattr(struct dentry *direntry, char *data, size_t buf_size) if (unlikely(cifs_forced_shutdown(cifs_sb))) return -EIO; - if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_NO_XATTR) + if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_NO_XATTR) return -EOPNOTSUPP; tlink = cifs_sb_tlink(cifs_sb); diff --git a/fs/smb/common/cifsglob.h b/fs/smb/common/cifsglob.h index eda5e666a7617..371160fec1cdf 100644 --- a/fs/smb/common/cifsglob.h +++ b/fs/smb/common/cifsglob.h @@ -40,6 +40,11 @@ struct smb_version_values { size_t create_posix_size; }; +static inline unsigned int get_rfc1002_len(void *buf) +{ + return be32_to_cpu(*((__be32 *)buf)) & 0xffffff; +} + static inline void inc_rfc1001_len(void *buf, int count) { be32_add_cpu((__be32 *)buf, count); diff --git a/fs/smb/common/smb2pdu.h b/fs/smb/common/smb2pdu.h index f79a5165a7cc6..528ba77247e80 100644 --- a/fs/smb/common/smb2pdu.h +++ b/fs/smb/common/smb2pdu.h @@ -1388,6 +1388,35 @@ struct smb2_ioctl_req { __u8 Buffer[]; } __packed; +/* See MS-SMB2 2.2.31.1.1 */ +struct srv_copychunk { + __le64 SourceOffset; + __le64 TargetOffset; + __le32 Length; + __le32 Reserved; +} __packed; + +#define COPY_CHUNK_RES_KEY_SIZE 24 + +/* See MS-SMB2 2.2.31.1 */ +/* this goes in the ioctl buffer when doing a copychunk request */ +struct copychunk_ioctl_req { + union { + char SourceKey[COPY_CHUNK_RES_KEY_SIZE]; + __le64 SourceKeyU64[3]; + }; + __le32 ChunkCount; + __le32 Reserved; + struct srv_copychunk Chunks[] __counted_by_le(ChunkCount); +} __packed; + +/* See MS-SMB2 2.2.32.1 */ +struct copychunk_ioctl_rsp { + __le32 ChunksWritten; + __le32 ChunkBytesWritten; + __le32 TotalBytesWritten; +} __packed; + struct smb2_ioctl_rsp { struct smb2_hdr hdr; __le16 StructureSize; /* Must be 49 */ diff --git a/fs/smb/server/Kconfig b/fs/smb/server/Kconfig index 12594879cb640..8b91a6323d936 100644 --- a/fs/smb/server/Kconfig +++ b/fs/smb/server/Kconfig @@ -9,7 +9,6 @@ config SMB_SERVER select CRYPTO select CRYPTO_ECB select CRYPTO_LIB_ARC4 - select CRYPTO_LIB_DES select CRYPTO_LIB_MD5 select CRYPTO_LIB_SHA256 select CRYPTO_LIB_SHA512 diff --git a/fs/smb/server/auth.c b/fs/smb/server/auth.c index a778275207fe7..9844aaad7ab3d 100644 --- a/fs/smb/server/auth.c +++ b/fs/smb/server/auth.c @@ -24,7 +24,6 @@ #include #include -#include #include "server.h" #include "smb_common.h" @@ -436,6 +435,7 @@ int ksmbd_krb5_authenticate(struct ksmbd_session *sess, char *in_blob, resp_ext = ksmbd_ipc_login_request_ext(resp->login_response.account); user = ksmbd_alloc_user(&resp->login_response, resp_ext); + kvfree(resp_ext); if (!user) { ksmbd_debug(AUTH, "login failure\n"); retval = -ENOMEM; @@ -710,7 +710,7 @@ void ksmbd_gen_smb311_encryptionkey(struct ksmbd_conn *conn, int ksmbd_gen_preauth_integrity_hash(struct ksmbd_conn *conn, char *buf, __u8 *pi_hash) { - struct smb2_hdr *rcv_hdr = smb2_get_msg(buf); + struct smb2_hdr *rcv_hdr = smb_get_msg(buf); char *all_bytes_msg = (char *)&rcv_hdr->ProtocolId; int msg_size = get_rfc1002_len(buf); struct sha512_ctx sha_ctx; @@ -837,7 +837,7 @@ int ksmbd_crypt_message(struct ksmbd_work *work, struct kvec *iov, unsigned int nvec, int enc) { struct ksmbd_conn *conn = work->conn; - struct smb2_transform_hdr *tr_hdr = smb2_get_msg(iov[0].iov_base); + struct smb2_transform_hdr *tr_hdr = smb_get_msg(iov[0].iov_base); unsigned int assoc_data_len = sizeof(struct smb2_transform_hdr) - 20; int rc; DECLARE_CRYPTO_WAIT(wait); diff --git a/fs/smb/server/connection.c b/fs/smb/server/connection.c index 9274953eb064c..ff6cdd720dd43 100644 --- a/fs/smb/server/connection.c +++ b/fs/smb/server/connection.c @@ -11,6 +11,7 @@ #include "server.h" #include "smb_common.h" #include "mgmt/ksmbd_ida.h" +#include "mgmt/user_session.h" #include "connection.h" #include "transport_tcp.h" #include "transport_rdma.h" @@ -123,6 +124,7 @@ void ksmbd_conn_free(struct ksmbd_conn *conn) kvfree(conn->request_buf); kfree(conn->preauth_info); kfree(conn->mechToken); + ksmbd_preauth_session_destroy(conn); ksmbd_conn_put(conn); } @@ -161,6 +163,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void) init_waitqueue_head(&conn->r_count_q); INIT_LIST_HEAD(&conn->requests); INIT_LIST_HEAD(&conn->async_requests); + INIT_LIST_HEAD(&conn->preauth_sess_table); spin_lock_init(&conn->request_lock); spin_lock_init(&conn->credits_lock); ida_init(&conn->async_ida); @@ -252,6 +255,22 @@ void ksmbd_all_conn_set_status(u64 sess_id, u32 status) up_read(&conn_list_lock); } +void ksmbd_conn_abort(struct ksmbd_conn *conn) +{ + bool shutdown = false; + + spin_lock(&conn->request_lock); + if (!ksmbd_conn_exiting(conn) && !ksmbd_conn_releasing(conn)) { + ksmbd_conn_set_exiting(conn); + shutdown = true; + } + spin_unlock(&conn->request_lock); + wake_up_all(&conn->req_running_q); + + if (shutdown && conn->transport->ops->shutdown) + conn->transport->ops->shutdown(conn->transport); +} + void ksmbd_conn_wait_idle(struct ksmbd_conn *conn) { wait_event(conn->req_running_q, atomic_read(&conn->req_running) < 2); @@ -376,8 +395,11 @@ bool ksmbd_conn_alive(struct ksmbd_conn *conn) return true; } -#define SMB1_MIN_SUPPORTED_HEADER_SIZE (sizeof(struct smb_hdr)) -#define SMB2_MIN_SUPPORTED_HEADER_SIZE (sizeof(struct smb2_hdr) + 4) +/* "+2" for BCC field (ByteCount, 2 bytes) */ +#define SMB1_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb_hdr) + 2) +#define SMB2_MIN_SUPPORTED_PDU_SIZE (sizeof(struct smb2_pdu)) +#define SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE \ + (sizeof(struct smb2_transform_hdr) + sizeof(struct smb2_hdr)) /** * ksmbd_conn_handler_loop() - session thread to listen on new smb requests @@ -392,6 +414,7 @@ int ksmbd_conn_handler_loop(void *p) struct ksmbd_conn *conn = (struct ksmbd_conn *)p; struct ksmbd_transport *t = conn->transport; unsigned int pdu_size, max_allowed_pdu_size, max_req; + __le32 proto; char hdr_buf[4] = {0,}; int size; @@ -444,7 +467,7 @@ int ksmbd_conn_handler_loop(void *p) if (pdu_size > MAX_STREAM_PROT_LEN) break; - if (pdu_size < SMB1_MIN_SUPPORTED_HEADER_SIZE) + if (pdu_size < SMB1_MIN_SUPPORTED_PDU_SIZE) break; /* 4 for rfc1002 length field */ @@ -475,11 +498,14 @@ int ksmbd_conn_handler_loop(void *p) if (!ksmbd_smb_request(conn)) break; - if (((struct smb2_hdr *)smb2_get_msg(conn->request_buf))->ProtocolId == - SMB2_PROTO_NUMBER) { - if (pdu_size < SMB2_MIN_SUPPORTED_HEADER_SIZE) - break; - } + proto = *(__le32 *)smb_get_msg(conn->request_buf); + if (proto == SMB2_PROTO_NUMBER && + pdu_size < SMB2_MIN_SUPPORTED_PDU_SIZE) + break; + + if (proto == SMB2_TRANSFORM_PROTO_NUM && + pdu_size < SMB2_TRANSFORM_MIN_SUPPORTED_PDU_SIZE) + break; if (!default_conn_ops.process_fn) { pr_err("No connection request callback\n"); diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index 8d985568a82fc..ceb1c1aefbf64 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -168,6 +168,7 @@ struct ksmbd_conn *ksmbd_conn_alloc(void); void ksmbd_conn_free(struct ksmbd_conn *conn); struct ksmbd_conn *ksmbd_conn_get(struct ksmbd_conn *conn); void ksmbd_conn_put(struct ksmbd_conn *conn); +void ksmbd_conn_abort(struct ksmbd_conn *conn); int ksmbd_conn_wq_init(void); void ksmbd_conn_wq_destroy(void); bool ksmbd_conn_lookup_dialect(struct ksmbd_conn *c); @@ -197,6 +198,11 @@ void ksmbd_conn_r_count_dec(struct ksmbd_conn *conn); * This is a hack. We will move status to a proper place once we land * a multi-sessions support. */ +static inline bool ksmbd_conn_new(struct ksmbd_conn *conn) +{ + return READ_ONCE(conn->status) == KSMBD_SESS_NEW; +} + static inline bool ksmbd_conn_good(struct ksmbd_conn *conn) { return READ_ONCE(conn->status) == KSMBD_SESS_GOOD; diff --git a/fs/smb/server/mgmt/share_config.c b/fs/smb/server/mgmt/share_config.c index c9b1108d6e96d..8813a81ca6629 100644 --- a/fs/smb/server/mgmt/share_config.c +++ b/fs/smb/server/mgmt/share_config.c @@ -88,9 +88,9 @@ static struct ksmbd_share_config *__share_lookup(const char *name) static int parse_veto_list(struct ksmbd_share_config *share, char *veto_list, - int veto_list_sz) + size_t veto_list_sz) { - int sz = 0; + size_t sz; if (!veto_list_sz) return 0; @@ -98,7 +98,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, while (veto_list_sz > 0) { struct ksmbd_veto_pattern *p; - sz = strlen(veto_list); + sz = strnlen(veto_list, veto_list_sz); if (!sz) break; @@ -106,7 +106,7 @@ static int parse_veto_list(struct ksmbd_share_config *share, if (!p) return -ENOMEM; - p->pattern = kstrdup(veto_list, KSMBD_DEFAULT_GFP); + p->pattern = kstrndup(veto_list, sz, KSMBD_DEFAULT_GFP); if (!p->pattern) { kfree(p); return -ENOMEM; @@ -114,6 +114,9 @@ static int parse_veto_list(struct ksmbd_share_config *share, list_add(&p->list, &share->veto_list); + if (sz == veto_list_sz) + break; + veto_list += sz + 1; veto_list_sz -= (sz + 1); } @@ -158,15 +161,34 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, atomic_set(&share->refcount, 1); INIT_LIST_HEAD(&share->veto_list); share->name = kstrdup(name, KSMBD_DEFAULT_GFP); + if (!share->name) { + kill_share(share); + share = NULL; + goto out; + } if (!test_share_config_flag(share, KSMBD_SHARE_FLAG_PIPE)) { - int path_len = PATH_MAX; + size_t path_len; - if (resp->payload_sz) + if (resp->payload_sz <= resp->veto_list_sz) { + ret = -EINVAL; + } else { path_len = resp->payload_sz - resp->veto_list_sz; - - share->path = kstrndup(ksmbd_share_config_path(resp), path_len, - KSMBD_DEFAULT_GFP); + if (resp->veto_list_sz) + path_len--; + + if (!path_len) { + ret = -EINVAL; + } else { + share->path = kstrndup( + ksmbd_share_config_path(resp), + path_len, KSMBD_DEFAULT_GFP); + if (!share->path) + ret = -ENOMEM; + else + ret = 0; + } + } if (share->path) { share->path_sz = strlen(share->path); while (share->path_sz > 1 && @@ -179,7 +201,8 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->force_directory_mode = resp->force_directory_mode; share->force_uid = resp->force_uid; share->force_gid = resp->force_gid; - ret = parse_veto_list(share, + if (!ret) + ret = parse_veto_list(share, KSMBD_SHARE_CONFIG_VETO_LIST(resp), resp->veto_list_sz); if (!ret && share->path) { @@ -199,7 +222,7 @@ static struct ksmbd_share_config *share_config_request(struct ksmbd_work *work, share->path = NULL; } } - if (ret || !share->name) { + if (ret) { kill_share(share); share = NULL; goto out; diff --git a/fs/smb/server/mgmt/user_config.c b/fs/smb/server/mgmt/user_config.c index d051ac3bc831c..53db14e91bfd8 100644 --- a/fs/smb/server/mgmt/user_config.c +++ b/fs/smb/server/mgmt/user_config.c @@ -26,6 +26,7 @@ struct ksmbd_user *ksmbd_login_user(const char *account) resp_ext = ksmbd_ipc_login_request_ext(account); user = ksmbd_alloc_user(resp, resp_ext); + kvfree(resp_ext); out: kvfree(resp); return user; diff --git a/fs/smb/server/mgmt/user_session.c b/fs/smb/server/mgmt/user_session.c index 5bc2f18d68bbc..920eb3963ae9f 100644 --- a/fs/smb/server/mgmt/user_session.c +++ b/fs/smb/server/mgmt/user_session.c @@ -212,10 +212,21 @@ static void ksmbd_expire_session(struct ksmbd_conn *conn) int ksmbd_session_register(struct ksmbd_conn *conn, struct ksmbd_session *sess) { + int ret; + sess->dialect = conn->dialect; memcpy(sess->ClientGUID, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE); ksmbd_expire_session(conn); - return xa_err(xa_store(&conn->sessions, sess->id, sess, KSMBD_DEFAULT_GFP)); + ret = xa_err(xa_store(&conn->sessions, sess->id, sess, + KSMBD_DEFAULT_GFP)); + if (ret) { + down_write(&sessions_table_lock); + hash_del(&sess->hlist); + up_write(&sessions_table_lock); + ksmbd_user_session_put(sess); + } + + return ret; } static int ksmbd_chann_del(struct ksmbd_conn *conn, struct ksmbd_session *sess) @@ -236,22 +247,19 @@ void ksmbd_sessions_deregister(struct ksmbd_conn *conn) { struct ksmbd_session *sess; unsigned long id; + struct hlist_node *tmp; + int bkt; down_write(&sessions_table_lock); - if (conn->binding) { - int bkt; - struct hlist_node *tmp; - - hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { - if (!ksmbd_chann_del(conn, sess) && - xa_empty(&sess->ksmbd_chann_list)) { - hash_del(&sess->hlist); - down_write(&conn->session_lock); - xa_erase(&conn->sessions, sess->id); - up_write(&conn->session_lock); - if (atomic_dec_and_test(&sess->refcnt)) - ksmbd_session_destroy(sess); - } + hash_for_each_safe(sessions_table, bkt, tmp, sess, hlist) { + if (!ksmbd_chann_del(conn, sess) && + xa_empty(&sess->ksmbd_chann_list)) { + hash_del(&sess->hlist); + down_write(&conn->session_lock); + xa_erase(&conn->sessions, sess->id); + up_write(&conn->session_lock); + if (atomic_dec_and_test(&sess->refcnt)) + ksmbd_session_destroy(sess); } } @@ -374,6 +382,17 @@ struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, return sess; } +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn) +{ + struct preauth_session *sess, *tmp; + + list_for_each_entry_safe(sess, tmp, &conn->preauth_sess_table, + preauth_entry) { + list_del(&sess->preauth_entry); + kfree(sess); + } +} + void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id) { diff --git a/fs/smb/server/mgmt/user_session.h b/fs/smb/server/mgmt/user_session.h index cba7f688f6b57..525650d6876c3 100644 --- a/fs/smb/server/mgmt/user_session.h +++ b/fs/smb/server/mgmt/user_session.h @@ -101,6 +101,7 @@ void destroy_previous_session(struct ksmbd_conn *conn, struct ksmbd_user *user, u64 id); struct preauth_session *ksmbd_preauth_session_alloc(struct ksmbd_conn *conn, u64 sess_id); +void ksmbd_preauth_session_destroy(struct ksmbd_conn *conn); struct preauth_session *ksmbd_preauth_session_lookup(struct ksmbd_conn *conn, unsigned long long id); diff --git a/fs/smb/server/misc.c b/fs/smb/server/misc.c index 53ddcae31f719..d6abe574fc89b 100644 --- a/fs/smb/server/misc.c +++ b/fs/smb/server/misc.c @@ -121,7 +121,9 @@ int parse_stream_name(char *filename, char **stream_name, int *s_type) char *stream_type; char *s_name; int rc = 0; + bool has_stream_type = false; + *stream_name = NULL; s_name = filename; filename = strsep(&s_name, ":"); ksmbd_debug(SMB, "filename : %s, streams : %s\n", filename, s_name); @@ -137,14 +139,20 @@ int parse_stream_name(char *filename, char **stream_name, int *s_type) ksmbd_debug(SMB, "stream name : %s, stream type : %s\n", s_name, stream_type); - if (!strncasecmp("$data", stream_type, 5)) + if (!strncasecmp("$data", stream_type, 5)) { *s_type = DATA_STREAM; - else if (!strncasecmp("$index_allocation", stream_type, 17)) + has_stream_type = true; + } else if (!strncasecmp("$index_allocation", stream_type, 17)) { *s_type = DIR_STREAM; - else + has_stream_type = true; + } else { rc = -ENOENT; + } } + if (has_stream_type && !s_name[0] && *s_type == DATA_STREAM) + goto out; + *stream_name = s_name; out: return rc; diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c index a019c4496ae43..6183bd52ca9d1 100644 --- a/fs/smb/server/oplock.c +++ b/fs/smb/server/oplock.c @@ -5,6 +5,7 @@ */ #include +#include #include "glob.h" #include "oplock.h" @@ -19,6 +20,20 @@ static LIST_HEAD(lease_table_list); static DEFINE_RWLOCK(lease_list_lock); +#define SMB2_LEASE_STATE_MASK_LE (SMB2_LEASE_READ_CACHING_LE | \ + SMB2_LEASE_HANDLE_CACHING_LE | \ + SMB2_LEASE_WRITE_CACHING_LE) + +static bool lease_state_valid(__le32 state) +{ + return !(state & ~SMB2_LEASE_STATE_MASK_LE); +} + +static bool lease_v2_flags_valid(__le32 flags) +{ + return !(flags & ~SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE); +} + /** * alloc_opinfo() - allocate a new opinfo object for oplock info * @work: smb work @@ -193,6 +208,18 @@ void opinfo_put(struct oplock_info *opinfo) free_opinfo(opinfo); } +static bool ksmbd_inode_has_lease(struct ksmbd_inode *ci) +{ + struct oplock_info *opinfo = opinfo_get_list(ci); + bool is_lease; + + if (!opinfo) + return false; + is_lease = opinfo->is_lease; + opinfo_put(opinfo); + return is_lease; +} + static void opinfo_add(struct oplock_info *opinfo, struct ksmbd_file *fp) { struct ksmbd_inode *ci = fp->f_ci; @@ -505,7 +532,7 @@ static inline int compare_guid_key(struct oplock_info *opinfo, * Return: oplock(lease) object on success, otherwise NULL */ static struct oplock_info *same_client_has_lease(struct ksmbd_inode *ci, - char *client_guid, + const char *client_guid, struct lease_ctx_info *lctx) { int ret; @@ -663,7 +690,7 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) goto out; } - rsp_hdr = smb2_get_msg(work->response_buf); + rsp_hdr = smb_get_msg(work->response_buf); memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; @@ -677,7 +704,7 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) rsp_hdr->SessionId = 0; memset(rsp_hdr->Signature, 0, 16); - rsp = smb2_get_msg(work->response_buf); + rsp = smb_get_msg(work->response_buf); rsp->StructureSize = cpu_to_le16(24); if (!br_info->open_trunc && @@ -705,33 +732,70 @@ static void __smb2_oplock_break_noti(struct work_struct *wk) out: ksmbd_free_work_struct(work); ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); +} + +/* + * Select and pin the connection used for an oplock break before doing any + * allocations which may sleep. The caller of oplock_break() holds a live + * reference on ci (a file being opened, a file being operated on, or an + * explicit ksmbd_inode_lookup_lock() reference in the parent lease break + * paths), so the inode cannot be freed during the call. + * + * opinfo->conn is cleared under ci->m_lock by session_fd_check() when the + * durable handle owning the oplock is disconnected, reassigned by + * ksmbd_reopen_durable_fd() under the same lock, and the last + * ksmbd_conn_put() of the old connection frees it. Holding the read lock + * excludes both writers, so the connection cannot be freed while it is + * selected. + */ +static struct ksmbd_conn *smb2_oplock_break_conn_get(struct oplock_info *opinfo, + struct ksmbd_inode *ci) +{ + struct ksmbd_conn *conn; + + down_read(&ci->m_lock); + conn = READ_ONCE(opinfo->conn); + if (conn && !ksmbd_conn_releasing(conn)) + conn = ksmbd_conn_get(conn); + else + conn = NULL; + up_read(&ci->m_lock); + + return conn; } /** * smb2_oplock_break_noti() - send smb2 exclusive/batch to level2 oplock * break command from server to client * @opinfo: oplock info object + * @ci: inode owning the break target's oplock list, pinned by + * the caller * * Return: 0 on success, otherwise error */ -static int smb2_oplock_break_noti(struct oplock_info *opinfo) +static int smb2_oplock_break_noti(struct oplock_info *opinfo, + struct ksmbd_inode *ci) { struct ksmbd_conn *conn; struct oplock_break_info *br_info; int ret = 0; struct ksmbd_work *work; - conn = READ_ONCE(opinfo->conn); + conn = smb2_oplock_break_conn_get(opinfo, ci); if (!conn) return 0; work = ksmbd_alloc_work_struct(); - if (!work) + if (!work) { + ksmbd_conn_put(conn); return -ENOMEM; + } br_info = kmalloc(sizeof(struct oplock_break_info), KSMBD_DEFAULT_GFP); if (!br_info) { ksmbd_free_work_struct(work); + ksmbd_conn_put(conn); return -ENOMEM; } @@ -740,6 +804,7 @@ static int smb2_oplock_break_noti(struct oplock_info *opinfo) br_info->open_trunc = opinfo->open_trunc; work->request_buf = (char *)br_info; + /* Transfer the reference acquired by smb2_oplock_break_conn_get(). */ work->conn = conn; work->sess = opinfo->sess; @@ -775,7 +840,7 @@ static void __smb2_lease_break_noti(struct work_struct *wk) goto out; } - rsp_hdr = smb2_get_msg(work->response_buf); + rsp_hdr = smb_get_msg(work->response_buf); memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; @@ -789,7 +854,7 @@ static void __smb2_lease_break_noti(struct work_struct *wk) rsp_hdr->SessionId = 0; memset(rsp_hdr->Signature, 0, 16); - rsp = smb2_get_msg(work->response_buf); + rsp = smb_get_msg(work->response_buf); rsp->StructureSize = cpu_to_le16(44); rsp->Epoch = br_info->epoch; rsp->Flags = 0; @@ -814,6 +879,7 @@ static void __smb2_lease_break_noti(struct work_struct *wk) out: ksmbd_free_work_struct(work); ksmbd_conn_r_count_dec(conn); + ksmbd_conn_put(conn); } /** @@ -853,7 +919,7 @@ static int smb2_lease_break_noti(struct oplock_info *opinfo) memcpy(br_info->lease_key, lease->lease_key, SMB2_LEASE_KEY_SIZE); work->request_buf = (char *)br_info; - work->conn = conn; + work->conn = ksmbd_conn_get(conn); work->sess = opinfo->sess; ksmbd_conn_r_count_inc(conn); @@ -888,8 +954,8 @@ static void wait_lease_breaking(struct oplock_info *opinfo) } } -static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, - struct ksmbd_work *in_work) +static int oplock_break(struct oplock_info *brk_opinfo, struct ksmbd_inode *ci, + int req_op_level, struct ksmbd_work *in_work) { int err = 0; @@ -955,7 +1021,7 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, if (brk_opinfo->is_lease) err = smb2_lease_break_noti(brk_opinfo); else - err = smb2_oplock_break_noti(brk_opinfo); + err = smb2_oplock_break_noti(brk_opinfo, ci); ksmbd_debug(OPLOCK, "oplock granted = %d\n", brk_opinfo->level); if (brk_opinfo->op_state == OPLOCK_CLOSING) @@ -967,6 +1033,43 @@ static int oplock_break(struct oplock_info *brk_opinfo, int req_op_level, return err; } +struct oplock_break_entry { + struct list_head list; + struct oplock_info *opinfo; +}; + +/* + * Collect an oplock for a deferred break. oplock_break() may block for + * the client's break acknowledgment, and the close that wakes that wait + * needs ci->m_lock for write, so the walks that hold ci->m_lock defer the + * break until the lock is released. + */ +static int oplock_break_add(struct list_head *head, struct oplock_info *opinfo) +{ + struct oplock_break_entry *ent; + + ent = kmalloc_obj(struct oplock_break_entry, KSMBD_DEFAULT_GFP); + if (!ent) + return -ENOMEM; + + ent->opinfo = opinfo; + list_add_tail(&ent->list, head); + return 0; +} + +static void oplock_break_drain_none(struct list_head *head, + struct ksmbd_inode *ci) +{ + struct oplock_break_entry *ent, *tmp; + + list_for_each_entry_safe(ent, tmp, head, list) { + oplock_break(ent->opinfo, ci, SMB2_OPLOCK_LEVEL_NONE, NULL); + list_del(&ent->list); + opinfo_put(ent->opinfo); + kfree(ent); + } +} + void destroy_lease_table(struct ksmbd_conn *conn) { struct lease_table *lb, *lbtmp; @@ -997,7 +1100,7 @@ void destroy_lease_table(struct ksmbd_conn *conn) write_unlock(&lease_list_lock); } -int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, struct lease_ctx_info *lctx) { struct oplock_info *opinfo; @@ -1014,7 +1117,7 @@ int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, } list_for_each_entry(lb, &lease_table_list, l_entry) { - if (!memcmp(lb->client_guid, sess->ClientGUID, + if (!memcmp(lb->client_guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) goto found; } @@ -1030,7 +1133,7 @@ int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, rcu_read_unlock(); if (opinfo->o_fp->f_ci == ci) goto op_next; - err = compare_guid_key(opinfo, sess->ClientGUID, + err = compare_guid_key(opinfo, conn->ClientGUID, lctx->lease_key); if (err) { err = -EINVAL; @@ -1063,6 +1166,9 @@ static void copy_lease(struct oplock_info *op1, struct oplock_info *op2) lease2->flags = lease1->flags; lease2->epoch = lease1->epoch; lease2->version = lease1->version; + lease2->is_dir = lease1->is_dir; + memcpy(lease2->parent_lease_key, lease1->parent_lease_key, + SMB2_LEASE_KEY_SIZE); } static void add_lease_global_list(struct oplock_info *opinfo, @@ -1110,6 +1216,7 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, { struct oplock_info *opinfo; struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); if (lctx->version != 2) return; @@ -1135,12 +1242,14 @@ void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, continue; } - oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); - opinfo_put(opinfo); + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); } } up_read(&p_ci->m_lock); + oplock_break_drain_none(&brk_list, p_ci); + ksmbd_inode_put(p_ci); } @@ -1148,6 +1257,7 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) { struct oplock_info *opinfo; struct ksmbd_inode *p_ci = NULL; + LIST_HEAD(brk_list); rcu_read_lock(); opinfo = rcu_dereference(fp->f_opinfo); @@ -1176,12 +1286,14 @@ void smb_lazy_parent_lease_break_close(struct ksmbd_file *fp) continue; } - oplock_break(opinfo, SMB2_OPLOCK_LEVEL_NONE, NULL); - opinfo_put(opinfo); + if (oplock_break_add(&brk_list, opinfo)) + opinfo_put(opinfo); } } up_read(&p_ci->m_lock); + oplock_break_drain_none(&brk_list, p_ci); + ksmbd_inode_put(p_ci); } @@ -1201,7 +1313,6 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct ksmbd_file *fp, __u16 tid, struct lease_ctx_info *lctx, int share_ret) { - struct ksmbd_session *sess = work->sess; int err = 0; struct oplock_info *opinfo = NULL, *prev_opinfo = NULL; struct ksmbd_inode *ci = fp->f_ci; @@ -1232,10 +1343,22 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, if (!opinfo_count(fp)) goto set_lev; - /* grant none-oplock if second open is trunc */ - if (fp->attrib_only && fp->cdoption != FILE_OVERWRITE_IF_LE && + /* + * A stat open that only requests metadata access must not break the + * existing caching state. READ_CONTROL (reading the security + * descriptor) does not conflict with a lease, but it does conflict + * with an oplock, so only treat a read-control-only open as a stat + * open when the existing holder is a lease. + */ + if (fp->cdoption != FILE_OVERWRITE_IF_LE && fp->cdoption != FILE_OVERWRITE_LE && - fp->cdoption != FILE_SUPERSEDE_LE) { + fp->cdoption != FILE_SUPERSEDE_LE && + (fp->attrib_only || + (!(fp->daccess & ~(FILE_READ_ATTRIBUTES_LE | + FILE_WRITE_ATTRIBUTES_LE | + FILE_SYNCHRONIZE_LE | + FILE_READ_CONTROL_LE)) && + ksmbd_inode_has_lease(ci)))) { req_op_level = SMB2_OPLOCK_LEVEL_NONE; goto set_lev; } @@ -1244,12 +1367,12 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, struct oplock_info *m_opinfo; /* is lease already granted ? */ - m_opinfo = same_client_has_lease(ci, sess->ClientGUID, + m_opinfo = same_client_has_lease(ci, work->conn->ClientGUID, lctx); if (m_opinfo) { copy_lease(m_opinfo, opinfo); if (atomic_read(&m_opinfo->breaking_cnt)) - opinfo->o_lease->flags = + opinfo->o_lease->flags |= SMB2_LEASE_FLAG_BREAK_IN_PROGRESS_LE; opinfo_put(m_opinfo); goto out; @@ -1278,7 +1401,7 @@ int smb_grant_oplock(struct ksmbd_work *work, int req_op_level, u64 pid, goto op_break_not_needed; } - err = oplock_break(prev_opinfo, SMB2_OPLOCK_LEVEL_II, work); + err = oplock_break(prev_opinfo, fp->f_ci, SMB2_OPLOCK_LEVEL_II, work); opinfo_put(prev_opinfo); if (err == -ENOENT) goto set_lev; @@ -1364,7 +1487,7 @@ static void smb_break_all_write_oplock(struct ksmbd_work *work, } brk_opinfo->open_trunc = is_trunc; - oplock_break(brk_opinfo, SMB2_OPLOCK_LEVEL_II, work); + oplock_break(brk_opinfo, fp->f_ci, SMB2_OPLOCK_LEVEL_II, work); opinfo_put(brk_opinfo); } @@ -1381,6 +1504,7 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, struct oplock_info *op, *brk_op; struct ksmbd_inode *ci; struct ksmbd_conn *conn = work->conn; + LIST_HEAD(brk_list); if (!test_share_config_flag(work->tcon->share_conf, KSMBD_SHARE_FLAG_OPLOCKS)) @@ -1402,14 +1526,8 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, continue; } - if (brk_op->is_lease && (brk_op->o_lease->state & - (~(SMB2_LEASE_READ_CACHING_LE | - SMB2_LEASE_HANDLE_CACHING_LE)))) { - ksmbd_debug(OPLOCK, "unexpected lease state(0x%x)\n", - brk_op->o_lease->state); - goto next; - } else if (brk_op->level != - SMB2_OPLOCK_LEVEL_II) { + if (!brk_op->is_lease && + brk_op->level != SMB2_OPLOCK_LEVEL_II) { ksmbd_debug(OPLOCK, "unexpected oplock(0x%x)\n", brk_op->level); goto next; @@ -1428,12 +1546,21 @@ void smb_break_all_levII_oplock(struct ksmbd_work *work, struct ksmbd_file *fp, SMB2_LEASE_KEY_SIZE)) goto next; brk_op->open_trunc = is_trunc; - oplock_break(brk_op, SMB2_OPLOCK_LEVEL_NONE, NULL); + + /* + * Defer the break until ci->m_lock is released: oplock_break() + * may block waiting for the break acknowledgment, and the + * close that wakes that wait needs ci->m_lock for write. + */ + if (!oplock_break_add(&brk_list, brk_op)) + continue; next: opinfo_put(brk_op); } up_read(&ci->m_lock); + oplock_break_drain_none(&brk_list, ci); + if (op) opinfo_put(op); } @@ -1461,15 +1588,13 @@ void smb_break_all_oplock(struct ksmbd_work *work, struct ksmbd_file *fp) */ __u8 smb2_map_lease_to_oplock(__le32 lease_state) { - if (lease_state == (SMB2_LEASE_HANDLE_CACHING_LE | - SMB2_LEASE_READ_CACHING_LE | - SMB2_LEASE_WRITE_CACHING_LE)) { + if ((lease_state & SMB2_LEASE_WRITE_CACHING_LE) && + (lease_state & SMB2_LEASE_HANDLE_CACHING_LE)) { return SMB2_OPLOCK_LEVEL_BATCH; - } else if (lease_state != SMB2_LEASE_WRITE_CACHING_LE && - lease_state & SMB2_LEASE_WRITE_CACHING_LE) { - if (!(lease_state & SMB2_LEASE_HANDLE_CACHING_LE)) - return SMB2_OPLOCK_LEVEL_EXCLUSIVE; - } else if (lease_state & SMB2_LEASE_READ_CACHING_LE) { + } else if (lease_state & SMB2_LEASE_WRITE_CACHING_LE) { + return SMB2_OPLOCK_LEVEL_EXCLUSIVE; + } else if (lease_state & (SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE)) { return SMB2_OPLOCK_LEVEL_II; } return 0; @@ -1537,12 +1662,14 @@ struct lease_ctx_info *parse_lease_state(void *open_req) struct lease_ctx_info *lreq; cc = smb2_find_context_vals(req, SMB2_CREATE_REQUEST_LEASE, 4); - if (IS_ERR_OR_NULL(cc)) + if (IS_ERR(cc)) + return ERR_CAST(cc); + if (!cc) return NULL; lreq = kzalloc(sizeof(struct lease_ctx_info), KSMBD_DEFAULT_GFP); if (!lreq) - return NULL; + return ERR_PTR(-ENOMEM); if (sizeof(struct lease_context_v2) == le32_to_cpu(cc->DataLength)) { struct create_lease_v2 *lc = (struct create_lease_v2 *)cc; @@ -1556,11 +1683,14 @@ struct lease_ctx_info *parse_lease_state(void *open_req) lreq->flags = lc->lcontext.LeaseFlags; lreq->epoch = lc->lcontext.Epoch; lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state) || + !lease_v2_flags_valid(lreq->flags)) + goto err_out; if (lreq->flags == SMB2_LEASE_FLAG_PARENT_LEASE_KEY_SET_LE) memcpy(lreq->parent_lease_key, lc->lcontext.ParentLeaseKey, SMB2_LEASE_KEY_SIZE); lreq->version = 2; - } else { + } else if (sizeof(struct lease_context) == le32_to_cpu(cc->DataLength)) { struct create_lease *lc = (struct create_lease *)cc; if (le16_to_cpu(cc->DataOffset) + le32_to_cpu(cc->DataLength) < @@ -1571,12 +1701,15 @@ struct lease_ctx_info *parse_lease_state(void *open_req) lreq->req_state = lc->lcontext.LeaseState; lreq->flags = lc->lcontext.LeaseFlags; lreq->duration = lc->lcontext.LeaseDuration; + if (!lease_state_valid(lreq->req_state)) + goto err_out; lreq->version = 1; - } + } else + goto err_out; return lreq; err_out: kfree(lreq); - return NULL; + return ERR_PTR(-EINVAL); } /** diff --git a/fs/smb/server/oplock.h b/fs/smb/server/oplock.h index d91a8266e065e..795a9119dad9a 100644 --- a/fs/smb/server/oplock.h +++ b/fs/smb/server/oplock.h @@ -116,7 +116,7 @@ void create_posix_rsp_buf(char *cc, struct ksmbd_file *fp); struct create_context *smb2_find_context_vals(void *open_req, const char *tag, int tag_len); struct oplock_info *lookup_lease_in_table(struct ksmbd_conn *conn, char *lease_key); -int find_same_lease_key(struct ksmbd_session *sess, struct ksmbd_inode *ci, +int find_same_lease_key(struct ksmbd_conn *conn, struct ksmbd_inode *ci, struct lease_ctx_info *lctx); void destroy_lease_table(struct ksmbd_conn *conn); void smb_send_parent_lease_break_noti(struct ksmbd_file *fp, diff --git a/fs/smb/server/server.c b/fs/smb/server/server.c index b78126bb23711..f2bc73a7ee821 100644 --- a/fs/smb/server/server.c +++ b/fs/smb/server/server.c @@ -95,7 +95,7 @@ static inline int check_conn_state(struct ksmbd_work *work) if (ksmbd_conn_exiting(work->conn) || ksmbd_conn_need_reconnect(work->conn)) { - rsp_hdr = work->response_buf; + rsp_hdr = smb_get_msg(work->response_buf); rsp_hdr->Status.CifsError = STATUS_CONNECTION_DISCONNECTED; return 1; } @@ -170,8 +170,10 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, if (conn->ops->is_transform_hdr && conn->ops->is_transform_hdr(work->request_buf)) { rc = conn->ops->decrypt_req(work); - if (rc < 0) + if (rc < 0) { + ksmbd_conn_abort(conn); return; + } work->encrypted = true; } @@ -195,6 +197,12 @@ static void __handle_ksmbd_work(struct ksmbd_work *work, else conn->ops->set_rsp_status(work, STATUS_USER_SESSION_DELETED); + if (conn->ops->is_sign_req(work, conn->ops->get_cmd_val(work))) { + struct smb2_hdr *rsp_hdr; + + rsp_hdr = ksmbd_resp_buf_curr(work); + rsp_hdr->Flags |= SMB2_FLAGS_SIGNED; + } goto send; } else if (rc > 0) { rc = conn->ops->get_ksmbd_tcon(work); diff --git a/fs/smb/server/smb2misc.c b/fs/smb/server/smb2misc.c index 67a2d7a793f6e..05da29c72c4f0 100644 --- a/fs/smb/server/smb2misc.c +++ b/fs/smb/server/smb2misc.c @@ -262,8 +262,12 @@ static int smb2_calc_size(void *buf, unsigned int *len) static inline int smb2_query_info_req_len(struct smb2_query_info_req *h) { - return le32_to_cpu(h->InputBufferLength) + - le32_to_cpu(h->OutputBufferLength); + return le32_to_cpu(h->InputBufferLength); +} + +static inline int smb2_query_info_resp_len(struct smb2_query_info_req *h) +{ + return le32_to_cpu(h->OutputBufferLength); } static inline int smb2_set_info_req_len(struct smb2_set_info_req *h) @@ -309,6 +313,7 @@ static int smb2_validate_credit_charge(struct ksmbd_conn *conn, switch (hdr->Command) { case SMB2_QUERY_INFO: req_len = smb2_query_info_req_len(__hdr); + expect_resp_len = smb2_query_info_resp_len(__hdr); break; case SMB2_SET_INFO: req_len = smb2_set_info_req_len(__hdr); @@ -400,6 +405,11 @@ int ksmbd_smb2_check_message(struct ksmbd_work *work) return 1; } + if (len < __SMB2_HEADER_STRUCTURE_SIZE + sizeof(__le16)) { + ksmbd_debug(SMB, "Message is too small for StructureSize2\n"); + return 1; + } + if (smb2_req_struct_sizes[command] != pdu->StructureSize2) { if (!(command == SMB2_OPLOCK_BREAK_HE && (le16_to_cpu(pdu->StructureSize2) == OP_BREAK_STRUCT_SIZE_20 || diff --git a/fs/smb/server/smb2ops.c b/fs/smb/server/smb2ops.c index bcf05caa2304d..4c50b7f57f763 100644 --- a/fs/smb/server/smb2ops.c +++ b/fs/smb/server/smb2ops.c @@ -262,8 +262,10 @@ void init_smb3_02_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; + /* + * Durable handles are in-memory only. Do not advertise persistent + * handles until CA recovery and fencing are implemented. + */ } /** @@ -286,10 +288,7 @@ int init_smb3_11_server(struct ksmbd_conn *conn) if (server_conf.flags & KSMBD_GLOBAL_FLAG_SMB3_MULTICHANNEL) conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_MULTI_CHANNEL; - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE) - conn->vals->req_capabilities |= SMB2_GLOBAL_CAP_PERSISTENT_HANDLES; - - INIT_LIST_HEAD(&conn->preauth_sess_table); + /* See init_smb3_02_server(): persistent handles require CA recovery. */ return 0; } diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c index a69bc75c85ad0..cd7bfc73cf087 100644 --- a/fs/smb/server/smb2pdu.c +++ b/fs/smb/server/smb2pdu.c @@ -48,8 +48,8 @@ static void __wbuf(struct ksmbd_work *work, void **req, void **rsp) *req = ksmbd_req_buf_next(work); *rsp = ksmbd_resp_buf_next(work); } else { - *req = smb2_get_msg(work->request_buf); - *rsp = smb2_get_msg(work->response_buf); + *req = smb_get_msg(work->request_buf); + *rsp = smb_get_msg(work->response_buf); } } @@ -155,7 +155,7 @@ void smb2_set_err_rsp(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) err_rsp = ksmbd_resp_buf_next(work); else - err_rsp = smb2_get_msg(work->response_buf); + err_rsp = smb_get_msg(work->response_buf); if (err_rsp->hdr.Status != STATUS_STOPPED_ON_SYMLINK) { int err; @@ -181,7 +181,7 @@ void smb2_set_err_rsp(struct ksmbd_work *work) */ bool is_smb2_neg_cmd(struct ksmbd_work *work) { - struct smb2_hdr *hdr = smb2_get_msg(work->request_buf); + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); /* is it SMB2 header ? */ if (hdr->ProtocolId != SMB2_PROTO_NUMBER) @@ -205,7 +205,7 @@ bool is_smb2_neg_cmd(struct ksmbd_work *work) */ bool is_smb2_rsp(struct ksmbd_work *work) { - struct smb2_hdr *hdr = smb2_get_msg(work->response_buf); + struct smb2_hdr *hdr = smb_get_msg(work->response_buf); /* is it SMB2 header ? */ if (hdr->ProtocolId != SMB2_PROTO_NUMBER) @@ -231,7 +231,7 @@ u16 get_smb2_cmd_val(struct ksmbd_work *work) if (work->next_smb2_rcv_hdr_off) rcv_hdr = ksmbd_req_buf_next(work); else - rcv_hdr = smb2_get_msg(work->request_buf); + rcv_hdr = smb_get_msg(work->request_buf); return le16_to_cpu(rcv_hdr->Command); } @@ -244,7 +244,7 @@ void set_smb2_rsp_status(struct ksmbd_work *work, __le32 err) { struct smb2_hdr *rsp_hdr; - rsp_hdr = smb2_get_msg(work->response_buf); + rsp_hdr = smb_get_msg(work->response_buf); rsp_hdr->Status = err; work->iov_idx = 0; @@ -267,7 +267,7 @@ int init_smb2_neg_rsp(struct ksmbd_work *work) struct ksmbd_conn *conn = work->conn; int err; - rsp_hdr = smb2_get_msg(work->response_buf); + rsp_hdr = smb_get_msg(work->response_buf); memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); rsp_hdr->ProtocolId = SMB2_PROTO_NUMBER; rsp_hdr->StructureSize = SMB2_HEADER_STRUCTURE_SIZE; @@ -281,7 +281,7 @@ int init_smb2_neg_rsp(struct ksmbd_work *work) rsp_hdr->SessionId = 0; memset(rsp_hdr->Signature, 0, 16); - rsp = smb2_get_msg(work->response_buf); + rsp = smb_get_msg(work->response_buf); WARN_ON(ksmbd_conn_good(conn)); @@ -455,7 +455,7 @@ static void init_chained_smb2_rsp(struct ksmbd_work *work) */ bool is_chained_smb2_message(struct ksmbd_work *work) { - struct smb2_hdr *hdr = smb2_get_msg(work->request_buf); + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); unsigned int len, next_cmd; if (hdr->ProtocolId != SMB2_PROTO_NUMBER) @@ -506,8 +506,8 @@ bool is_chained_smb2_message(struct ksmbd_work *work) */ int init_smb2_rsp_hdr(struct ksmbd_work *work) { - struct smb2_hdr *rsp_hdr = smb2_get_msg(work->response_buf); - struct smb2_hdr *rcv_hdr = smb2_get_msg(work->request_buf); + struct smb2_hdr *rsp_hdr = smb_get_msg(work->response_buf); + struct smb2_hdr *rcv_hdr = smb_get_msg(work->request_buf); memset(rsp_hdr, 0, sizeof(struct smb2_hdr) + 2); rsp_hdr->ProtocolId = rcv_hdr->ProtocolId; @@ -536,7 +536,7 @@ int init_smb2_rsp_hdr(struct ksmbd_work *work) */ int smb2_allocate_rsp_buf(struct ksmbd_work *work) { - struct smb2_hdr *hdr = smb2_get_msg(work->request_buf); + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); size_t small_sz = MAX_CIFS_SMALL_BUFFER_SIZE; size_t large_sz = small_sz + work->conn->vals->max_trans_size; size_t sz = small_sz; @@ -552,7 +552,7 @@ int smb2_allocate_rsp_buf(struct ksmbd_work *work) offsetof(struct smb2_query_info_req, OutputBufferLength)) return -EINVAL; - req = smb2_get_msg(work->request_buf); + req = smb_get_msg(work->request_buf); if ((req->InfoType == SMB2_O_INFO_FILE && (req->FileInfoClass == FILE_FULL_EA_INFORMATION || req->FileInfoClass == FILE_ALL_INFORMATION)) || @@ -726,10 +726,10 @@ void smb2_send_interim_resp(struct ksmbd_work *work, __le32 status) } in_work->conn = work->conn; - memcpy(smb2_get_msg(in_work->response_buf), ksmbd_resp_buf_next(work), + memcpy(smb_get_msg(in_work->response_buf), ksmbd_resp_buf_next(work), __SMB2_HEADER_STRUCTURE_SIZE); - rsp_hdr = smb2_get_msg(in_work->response_buf); + rsp_hdr = smb_get_msg(in_work->response_buf); rsp_hdr->Flags |= SMB2_FLAGS_ASYNC_COMMAND; rsp_hdr->Id.AsyncId = cpu_to_le64(work->async_id); smb2_set_err_rsp(in_work); @@ -1102,26 +1102,21 @@ static __le32 deassemble_neg_contexts(struct ksmbd_conn *conn, * smb2_handle_negotiate() - handler for smb2 negotiate command * @work: smb work containing smb request buffer * + * The caller holds conn->srv_mutex. + * * Return: 0 */ int smb2_handle_negotiate(struct ksmbd_work *work) { struct ksmbd_conn *conn = work->conn; - struct smb2_negotiate_req *req = smb2_get_msg(work->request_buf); - struct smb2_negotiate_rsp *rsp = smb2_get_msg(work->response_buf); + struct smb2_negotiate_req *req = smb_get_msg(work->request_buf); + struct smb2_negotiate_rsp *rsp = smb_get_msg(work->response_buf); int rc = 0; unsigned int smb2_buf_len, smb2_neg_size, neg_ctxt_len = 0; __le32 status; ksmbd_debug(SMB, "Received negotiate request\n"); conn->need_neg = false; - if (ksmbd_conn_good(conn)) { - pr_err("conn->tcp_status is already in CifsGood State\n"); - work->send_no_response = 1; - return rc; - } - - ksmbd_conn_lock(conn); smb2_buf_len = get_rfc1002_len(work->request_buf); smb2_neg_size = offsetof(struct smb2_negotiate_req, Dialects); if (smb2_neg_size > smb2_buf_len) { @@ -1175,7 +1170,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) KSMBD_DEFAULT_GFP); if (!conn->preauth_info) { rc = -ENOMEM; - rsp->hdr.Status = STATUS_INVALID_PARAMETER; + rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; goto err_out; } @@ -1272,8 +1267,7 @@ int smb2_handle_negotiate(struct ksmbd_work *work) ksmbd_conn_set_need_setup(conn); err_out: - ksmbd_conn_unlock(conn); - if (rc) + if (rc && rsp->hdr.Status == STATUS_SUCCESS) rsp->hdr.Status = STATUS_INSUFFICIENT_RESOURCES; if (!rc) @@ -1492,11 +1486,6 @@ static int ntlm_authenticate(struct ksmbd_work *work, return -EPERM; } - /* Check for previous session */ - prev_id = le64_to_cpu(req->PreviousSessionId); - if (prev_id && prev_id != sess->id) - destroy_previous_session(conn, user, prev_id); - if (sess->state == SMB2_SESSION_VALID) { /* * Reuse session if anonymous try to connect @@ -1534,6 +1523,10 @@ static int ntlm_authenticate(struct ksmbd_work *work, } } + prev_id = le64_to_cpu(req->PreviousSessionId); + if (prev_id && prev_id != sess->id) + destroy_previous_session(conn, sess->user, prev_id); + /* * If session state is SMB2_SESSION_VALID, We can assume * that it is reauthentication. And the user/password @@ -1759,7 +1752,7 @@ int smb2_sess_setup(struct ksmbd_work *work) goto out_err; } - if (strncmp(conn->ClientGUID, sess->ClientGUID, + if (memcmp(conn->ClientGUID, sess->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { rc = -ENOENT; goto out_err; @@ -1802,6 +1795,13 @@ int smb2_sess_setup(struct ksmbd_work *work) } else { sess = ksmbd_session_lookup(conn, le64_to_cpu(req->hdr.SessionId)); + if (!sess) { + sess = ksmbd_session_lookup_slowpath(le64_to_cpu(req->hdr.SessionId)); + if (sess && !lookup_chann_list(sess, conn)) { + ksmbd_user_session_put(sess); + sess = NULL; + } + } if (!sess) { rc = -ENOENT; goto out_err; @@ -2056,12 +2056,11 @@ int smb2_tree_connect(struct ksmbd_work *work) write_unlock(&sess->tree_conns_lock); rsp->StructureSize = cpu_to_le16(16); out_err1: - if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && share && - test_share_config_flag(share, - KSMBD_SHARE_FLAG_CONTINUOUS_AVAILABILITY)) - rsp->Capabilities = SMB2_SHARE_CAP_CONTINUOUS_AVAILABILITY; - else - rsp->Capabilities = 0; + /* + * A configured CA share is not continuously available until persistent + * open recovery, ownership fencing, and failover are implemented. + */ + rsp->Capabilities = 0; rsp->Reserved = 0; /* default manual caching */ rsp->ShareFlags = SMB2_SHAREFLAG_MANUAL_CACHING; @@ -3012,6 +3011,17 @@ int smb2_open(struct ksmbd_work *work) if (server_conf.flags & KSMBD_GLOBAL_FLAG_DURABLE_HANDLE && req->CreateContextsOffset) { lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } rc = parse_durable_handle_context(work, req, lc, &dh_info); if (rc) { ksmbd_debug(SMB, "error parsing durable handle context\n"); @@ -3043,8 +3053,19 @@ int smb2_open(struct ksmbd_work *work) goto reconnected_fp; } - } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) + } else if (req_op_level == SMB2_OPLOCK_LEVEL_LEASE) { lc = parse_lease_state(req); + if (IS_ERR(lc)) { + rc = PTR_ERR(lc); + lc = NULL; + goto err_out2; + } + if (lc && lc->version == 2 && conn->dialect < SMB30_PROT_ID) { + kfree(lc); + lc = NULL; + req_op_level = SMB2_OPLOCK_LEVEL_NONE; + } + } if (le32_to_cpu(req->ImpersonationLevel) > le32_to_cpu(IL_DELEGATE)) { pr_err("Invalid impersonationlevel : 0x%x\n", @@ -3386,14 +3407,16 @@ int smb2_open(struct ksmbd_work *work) if (posix_acl_rc) ksmbd_debug(SMB, "inherit posix acl failed : %d\n", posix_acl_rc); - if (test_share_config_flag(work->tcon->share_conf, - KSMBD_SHARE_FLAG_ACL_XATTR)) { - rc = smb_inherit_dacl(conn, &path, sess->user->uid, - sess->user->gid); - } + rc = smb2_create_sd_buffer(work, req, &path); + if (rc && rc != -ENOENT) + goto err_out; - if (rc) { - rc = smb2_create_sd_buffer(work, req, &path); + if (rc == -ENOENT) { + if (test_share_config_flag(work->tcon->share_conf, + KSMBD_SHARE_FLAG_ACL_XATTR)) { + rc = smb_inherit_dacl(conn, &path, sess->user->uid, + sess->user->gid); + } if (rc) { if (posix_acl_rc) ksmbd_vfs_set_init_posix_acl(idmap, @@ -3519,7 +3542,7 @@ int smb2_open(struct ksmbd_work *work) ksmbd_debug(SMB, "lease req for(%s) req oplock state 0x%x, lease state 0x%x\n", name, req_op_level, lc->req_state); - rc = find_same_lease_key(sess, fp->f_ci, lc); + rc = find_same_lease_key(conn, fp->f_ci, lc); if (rc) goto err_out1; } else if (open_flags == O_RDONLY && @@ -3535,8 +3558,10 @@ int smb2_open(struct ksmbd_work *work) goto err_out1; } - if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) + if (req->CreateOptions & FILE_DELETE_ON_CLOSE_LE) { + smb_break_all_levII_oplock(work, fp, 0); ksmbd_fd_set_delete_on_close(fp, file_info); + } if (need_truncate) { rc = smb2_create_truncate(&fp->filp->f_path); @@ -3772,8 +3797,10 @@ int smb2_open(struct ksmbd_work *work) err_out2: if (!rc) { - ksmbd_update_fstate(&work->sess->file_table, fp, FP_INITED); - rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); + rc = ksmbd_update_fstate(&work->sess->file_table, fp, + FP_INITED); + if (!rc) + rc = ksmbd_iov_pin_rsp(work, (void *)rsp, iov_len); } if (rc) { if (rc == -EINVAL) @@ -6095,7 +6122,7 @@ int smb2_close(struct ksmbd_work *work) */ int smb2_echo(struct ksmbd_work *work) { - struct smb2_echo_rsp *rsp = smb2_get_msg(work->response_buf); + struct smb2_echo_rsp *rsp = smb_get_msg(work->response_buf); ksmbd_debug(SMB, "Received smb2 echo request\n"); @@ -6451,7 +6478,8 @@ static int set_rename_info(struct ksmbd_work *work, struct ksmbd_file *fp, return smb2_rename(work, fp, rename_info, work->conn->local_nls); } -static int set_file_disposition_info(struct ksmbd_file *fp, +static int set_file_disposition_info(struct ksmbd_work *work, + struct ksmbd_file *fp, struct smb2_file_disposition_info *file_info) { struct inode *inode; @@ -6466,6 +6494,7 @@ static int set_file_disposition_info(struct ksmbd_file *fp, if (S_ISDIR(inode->i_mode) && ksmbd_vfs_empty_dir(fp) == -ENOTEMPTY) return -EBUSY; + smb_break_all_levII_oplock(work, fp, 0); ksmbd_set_inode_pending_delete(fp); } else { ksmbd_clear_inode_pending_delete(fp); @@ -6595,7 +6624,7 @@ static int smb2_set_info_file(struct ksmbd_work *work, struct ksmbd_file *fp, if (buf_len < sizeof(struct smb2_file_disposition_info)) return -EINVAL; - return set_file_disposition_info(fp, + return set_file_disposition_info(work, fp, (struct smb2_file_disposition_info *)buffer); } case FILE_FULL_EA_INFORMATION: @@ -6673,8 +6702,8 @@ int smb2_set_info(struct ksmbd_work *work) pid = work->compound_pfid; } } else { - req = smb2_get_msg(work->request_buf); - rsp = smb2_get_msg(work->response_buf); + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); } if (!test_tree_conn_flag(work->tcon, KSMBD_TREE_CONN_FLAG_WRITABLE)) { @@ -6778,13 +6807,18 @@ static noinline int smb2_read_pipe(struct ksmbd_work *work) } aux_payload_buf = - kvmalloc(rpc_resp->payload_sz, KSMBD_DEFAULT_GFP); + kvmalloc(ALIGN(rpc_resp->payload_sz, 8), + KSMBD_DEFAULT_GFP); if (!aux_payload_buf) { err = -ENOMEM; goto out; } memcpy(aux_payload_buf, rpc_resp->payload, rpc_resp->payload_sz); + if (rpc_resp->payload_sz & 7) + memset(aux_payload_buf + rpc_resp->payload_sz, 0, + ALIGN(rpc_resp->payload_sz, 8) - + rpc_resp->payload_sz); nbytes = rpc_resp->payload_sz; err = ksmbd_iov_pin_rsp_read(work, (void *)rsp, @@ -6902,8 +6936,8 @@ int smb2_read(struct ksmbd_work *work) pid = work->compound_pfid; } } else { - req = smb2_get_msg(work->request_buf); - rsp = smb2_get_msg(work->response_buf); + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); } if (!has_file_id(id)) { @@ -7331,7 +7365,7 @@ int smb2_flush(struct ksmbd_work *work) int smb2_cancel(struct ksmbd_work *work) { struct ksmbd_conn *conn = work->conn; - struct smb2_hdr *hdr = smb2_get_msg(work->request_buf); + struct smb2_hdr *hdr = smb_get_msg(work->request_buf); struct smb2_hdr *chdr; struct ksmbd_work *iter; struct list_head *command_list; @@ -7348,7 +7382,7 @@ int smb2_cancel(struct ksmbd_work *work) spin_lock(&conn->request_lock); list_for_each_entry(iter, command_list, async_request_entry) { - chdr = smb2_get_msg(iter->request_buf); + chdr = smb_get_msg(iter->request_buf); if (iter->async_id != le64_to_cpu(hdr->Id.AsyncId)) @@ -7380,7 +7414,7 @@ int smb2_cancel(struct ksmbd_work *work) spin_lock(&conn->request_lock); list_for_each_entry(iter, command_list, request_entry) { - chdr = smb2_get_msg(iter->request_buf); + chdr = smb_get_msg(iter->request_buf); if (chdr->MessageId != hdr->MessageId || iter == work) @@ -7931,11 +7965,11 @@ static int fsctl_copychunk(struct ksmbd_work *work, } src_fp = ksmbd_lookup_foreign_fd(work, - le64_to_cpu(ci_req->ResumeKey[0])); + le64_to_cpu(ci_req->SourceKeyU64[0])); dst_fp = ksmbd_lookup_fd_slow(work, volatile_id, persistent_id); ret = -EINVAL; if (!src_fp || - src_fp->persistent_id != le64_to_cpu(ci_req->ResumeKey[1])) { + src_fp->persistent_id != le64_to_cpu(ci_req->SourceKeyU64[1])) { rsp->hdr.Status = STATUS_OBJECT_NAME_NOT_FOUND; goto out; } @@ -8136,7 +8170,7 @@ static int fsctl_validate_negotiate_info(struct ksmbd_conn *conn, goto err_out; } - if (strncmp(neg_req->Guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { + if (memcmp(neg_req->Guid, conn->ClientGUID, SMB2_CLIENT_GUID_SIZE)) { ret = -EINVAL; goto err_out; } @@ -8337,8 +8371,8 @@ int smb2_ioctl(struct ksmbd_work *work) id = work->compound_fid; } } else { - req = smb2_get_msg(work->request_buf); - rsp = smb2_get_msg(work->response_buf); + req = smb_get_msg(work->request_buf); + rsp = smb_get_msg(work->response_buf); } if (!has_file_id(id)) @@ -8697,11 +8731,10 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) struct smb2_oplock_break *rsp; struct ksmbd_file *fp; struct oplock_info *opinfo = NULL; - __le32 err = 0; - int ret = 0; + __le32 status = STATUS_SUCCESS; + int ret; u64 volatile_id, persistent_id; char req_oplevel = 0, rsp_oplevel = 0; - unsigned int oplock_change_type; WORK_BUFFERS(work, req, rsp); @@ -8727,71 +8760,55 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) return; } - if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { - rsp->hdr.Status = STATUS_INVALID_OPLOCK_PROTOCOL; + if (opinfo->op_state != OPLOCK_ACK_WAIT) { + ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", + opinfo->op_state); + status = STATUS_INVALID_DEVICE_STATE; goto err_out; } - if (opinfo->op_state == OPLOCK_STATE_NONE) { - ksmbd_debug(SMB, "unexpected oplock state 0x%x\n", opinfo->op_state); - rsp->hdr.Status = STATUS_UNSUCCESSFUL; + if (req_oplevel == SMB2_OPLOCK_LEVEL_LEASE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_PARAMETER; goto err_out; } - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - (req_oplevel != SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE)) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - oplock_change_type = OPLOCK_READ_TO_NONE; - } else if (req_oplevel == SMB2_OPLOCK_LEVEL_II || - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - err = STATUS_INVALID_DEVICE_STATE; - if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_II) { - oplock_change_type = OPLOCK_WRITE_TO_READ; - } else if ((opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE || - opinfo->level == SMB2_OPLOCK_LEVEL_BATCH) && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_WRITE_TO_NONE; - } else if (opinfo->level == SMB2_OPLOCK_LEVEL_II && - req_oplevel == SMB2_OPLOCK_LEVEL_NONE) { - oplock_change_type = OPLOCK_READ_TO_NONE; - } else { - oplock_change_type = 0; - } - } else { - oplock_change_type = 0; + if (opinfo->level == SMB2_OPLOCK_LEVEL_NONE) { + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - switch (oplock_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_II; - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; - break; - default: - pr_err("unknown oplock change 0x%x -> 0x%x\n", - opinfo->level, rsp_oplevel); + if (opinfo->level == SMB2_OPLOCK_LEVEL_EXCLUSIVE && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; } - if (ret < 0) { - rsp->hdr.Status = err; + if (opinfo->level == SMB2_OPLOCK_LEVEL_BATCH && + req_oplevel != SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE && + req_oplevel != SMB2_OPLOCK_LEVEL_EXCLUSIVE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; goto err_out; } + if (opinfo->level == SMB2_OPLOCK_LEVEL_II && + req_oplevel != SMB2_OPLOCK_LEVEL_NONE) { + opinfo->level = SMB2_OPLOCK_LEVEL_NONE; + status = STATUS_INVALID_OPLOCK_PROTOCOL; + goto err_out; + } + + if (req_oplevel == SMB2_OPLOCK_LEVEL_EXCLUSIVE) + rsp_oplevel = SMB2_OPLOCK_LEVEL_NONE; + else + rsp_oplevel = req_oplevel; + + opinfo->level = rsp_oplevel; + rsp->StructureSize = cpu_to_le16(24); rsp->OplockLevel = rsp_oplevel; rsp->Reserved = 0; @@ -8799,27 +8816,33 @@ static void smb20_oplock_break_ack(struct ksmbd_work *work) rsp->VolatileFid = volatile_id; rsp->PersistentFid = persistent_id; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_oplock_break)); - if (ret) { + if (ret) + ksmbd_debug(SMB, "failed to pin oplock break response: %d\n", + ret); + goto out; + err_out: - smb2_set_err_rsp(work); - } + rsp->hdr.Status = status; + smb2_set_err_rsp(work); +out: opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); opinfo_put(opinfo); ksmbd_fd_put(work, fp); } -static int check_lease_state(struct lease *lease, __le32 req_state) +static bool smb2_lease_state_valid(__le32 state) { - if ((lease->new_state == - (SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE)) && - !(req_state & SMB2_LEASE_WRITE_CACHING_LE)) { - lease->new_state = req_state; - return 0; - } + return !(state & ~(SMB2_LEASE_READ_CACHING_LE | + SMB2_LEASE_HANDLE_CACHING_LE | + SMB2_LEASE_WRITE_CACHING_LE)); +} - if (lease->new_state == req_state) +static int check_lease_state(struct lease *lease, __le32 req_state) +{ + if (smb2_lease_state_valid(req_state) && + !(req_state & ~lease->new_state)) return 0; return 1; @@ -8837,9 +8860,7 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) struct smb2_lease_ack *req; struct smb2_lease_ack *rsp; struct oplock_info *opinfo; - __le32 err = 0; int ret = 0; - unsigned int lease_change_type; __le32 lease_state; struct lease *lease; @@ -8863,80 +8884,23 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) goto err_out; } - if (check_lease_state(lease, req->LeaseState)) { - rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; - ksmbd_debug(OPLOCK, - "req lease state: 0x%x, expected state: 0x%x\n", - req->LeaseState, lease->new_state); - goto err_out; - } - if (!atomic_read(&opinfo->breaking_cnt)) { rsp->hdr.Status = STATUS_UNSUCCESSFUL; goto err_out; } - /* check for bad lease state */ - if (req->LeaseState & - (~(SMB2_LEASE_READ_CACHING_LE | SMB2_LEASE_HANDLE_CACHING_LE))) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else if (lease->state == SMB2_LEASE_READ_CACHING_LE && - req->LeaseState != SMB2_LEASE_NONE_LE) { - err = STATUS_INVALID_OPLOCK_PROTOCOL; - lease_change_type = OPLOCK_READ_TO_NONE; - ksmbd_debug(OPLOCK, "handle bad lease state 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } else { - /* valid lease state changes */ - err = STATUS_INVALID_DEVICE_STATE; - if (req->LeaseState == SMB2_LEASE_NONE_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_NONE; - else - lease_change_type = OPLOCK_READ_TO_NONE; - } else if (req->LeaseState & SMB2_LEASE_READ_CACHING_LE) { - if (lease->state & SMB2_LEASE_WRITE_CACHING_LE) - lease_change_type = OPLOCK_WRITE_TO_READ; - else - lease_change_type = OPLOCK_READ_HANDLE_TO_READ; - } else { - lease_change_type = 0; - } - } - - switch (lease_change_type) { - case OPLOCK_WRITE_TO_READ: - ret = opinfo_write_to_read(opinfo); - break; - case OPLOCK_READ_HANDLE_TO_READ: - ret = opinfo_read_handle_to_read(opinfo); - break; - case OPLOCK_WRITE_TO_NONE: - ret = opinfo_write_to_none(opinfo); - break; - case OPLOCK_READ_TO_NONE: - ret = opinfo_read_to_none(opinfo); - break; - default: - ksmbd_debug(OPLOCK, "unknown lease change 0x%x -> 0x%x\n", - le32_to_cpu(lease->state), - le32_to_cpu(req->LeaseState)); - } - - if (ret < 0) { - rsp->hdr.Status = err; + if (check_lease_state(lease, req->LeaseState)) { + rsp->hdr.Status = STATUS_REQUEST_NOT_ACCEPTED; + ksmbd_debug(OPLOCK, + "req lease state: 0x%x, expected state: 0x%x\n", + req->LeaseState, lease->new_state); goto err_out; } - lease_state = lease->state; + lease_state = req->LeaseState; + lease->state = lease_state; + lease->new_state = SMB2_LEASE_NONE_LE; + opinfo->level = smb2_map_lease_to_oplock(lease_state); rsp->StructureSize = cpu_to_le16(36); rsp->Reserved = 0; @@ -8945,16 +8909,20 @@ static void smb21_lease_break_ack(struct ksmbd_work *work) rsp->LeaseState = lease_state; rsp->LeaseDuration = 0; ret = ksmbd_iov_pin_rsp(work, rsp, sizeof(struct smb2_lease_ack)); - if (ret) { -err_out: - smb2_set_err_rsp(work); - } + if (ret) + goto err_out; opinfo->op_state = OPLOCK_STATE_NONE; wake_up_interruptible_all(&opinfo->oplock_q); atomic_dec(&opinfo->breaking_cnt); wake_up_interruptible_all(&opinfo->oplock_brk); opinfo_put(opinfo); + return; + +err_out: + smb2_set_err_rsp(work); + opinfo_put(opinfo); + return; } /** @@ -9024,7 +8992,7 @@ int smb2_notify(struct ksmbd_work *work) */ bool smb2_is_sign_req(struct ksmbd_work *work, unsigned int command) { - struct smb2_hdr *rcv_hdr2 = smb2_get_msg(work->request_buf); + struct smb2_hdr *rcv_hdr2 = smb_get_msg(work->request_buf); if ((rcv_hdr2->Flags & SMB2_FLAGS_SIGNED) && command != SMB2_NEGOTIATE_HE && @@ -9049,7 +9017,7 @@ int smb2_check_sign_req(struct ksmbd_work *work) struct kvec iov[1]; size_t len; - hdr = smb2_get_msg(work->request_buf); + hdr = smb_get_msg(work->request_buf); if (work->next_smb2_rcv_hdr_off) hdr = ksmbd_req_buf_next(work); @@ -9123,7 +9091,7 @@ int smb3_check_sign_req(struct ksmbd_work *work) struct kvec iov[1]; size_t len; - hdr = smb2_get_msg(work->request_buf); + hdr = smb_get_msg(work->request_buf); if (work->next_smb2_rcv_hdr_off) hdr = ksmbd_req_buf_next(work); @@ -9259,7 +9227,7 @@ void smb3_preauth_hash_rsp(struct ksmbd_work *work) static void fill_transform_hdr(void *tr_buf, char *old_buf, __le16 cipher_type) { struct smb2_transform_hdr *tr_hdr = tr_buf + 4; - struct smb2_hdr *hdr = smb2_get_msg(old_buf); + struct smb2_hdr *hdr = smb_get_msg(old_buf); unsigned int orig_len = get_rfc1002_len(old_buf); /* tr_buf must be cleared by the caller */ @@ -9298,7 +9266,7 @@ int smb3_encrypt_resp(struct ksmbd_work *work) bool smb3_is_transform_hdr(void *buf) { - struct smb2_transform_hdr *trhdr = smb2_get_msg(buf); + struct smb2_transform_hdr *trhdr = smb_get_msg(buf); return trhdr->ProtocolId == SMB2_TRANSFORM_PROTO_NUM; } @@ -9310,7 +9278,7 @@ int smb3_decrypt_req(struct ksmbd_work *work) unsigned int pdu_length = get_rfc1002_len(buf); struct kvec iov[2]; int buf_data_size = pdu_length - sizeof(struct smb2_transform_hdr); - struct smb2_transform_hdr *tr_hdr = smb2_get_msg(buf); + struct smb2_transform_hdr *tr_hdr = smb_get_msg(buf); int rc = 0; if (pdu_length < sizeof(struct smb2_transform_hdr) || @@ -9351,7 +9319,7 @@ bool smb3_11_final_sess_setup_resp(struct ksmbd_work *work) { struct ksmbd_conn *conn = work->conn; struct ksmbd_session *sess = work->sess; - struct smb2_hdr *rsp = smb2_get_msg(work->response_buf); + struct smb2_hdr *rsp = smb_get_msg(work->response_buf); if (conn->dialect < SMB30_PROT_ID) return false; diff --git a/fs/smb/server/smb2pdu.h b/fs/smb/server/smb2pdu.h index 5163d5241b90d..10591d757f2c1 100644 --- a/fs/smb/server/smb2pdu.h +++ b/fs/smb/server/smb2pdu.h @@ -187,26 +187,6 @@ struct resume_key_ioctl_rsp { __u8 Context[4]; /* ignored, Windows sets to 4 bytes of zero */ } __packed; -struct srv_copychunk { - __le64 SourceOffset; - __le64 TargetOffset; - __le32 Length; - __le32 Reserved; -} __packed; - -struct copychunk_ioctl_req { - __le64 ResumeKey[3]; - __le32 ChunkCount; - __le32 Reserved; - struct srv_copychunk Chunks[] __counted_by_le(ChunkCount); -} __packed; - -struct copychunk_ioctl_rsp { - __le32 ChunksWritten; - __le32 ChunkBytesWritten; - __le32 TotalBytesWritten; -} __packed; - struct file_sparse { __u8 SetSparse; } __packed; @@ -490,15 +470,6 @@ int smb2_ioctl(struct ksmbd_work *work); int smb2_oplock_break(struct ksmbd_work *work); int smb2_notify(struct ksmbd_work *ksmbd_work); -/* - * Get the body of the smb2 message excluding the 4 byte rfc1002 headers - * from request/response buffer. - */ -static inline void *smb2_get_msg(void *buf) -{ - return buf + 4; -} - #define POSIX_TYPE_FILE 0 #define POSIX_TYPE_DIR 1 #define POSIX_TYPE_SYMLINK 2 diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c index b23203a1c2865..3e3758f5ed42a 100644 --- a/fs/smb/server/smb_common.c +++ b/fs/smb/server/smb_common.c @@ -140,7 +140,7 @@ int ksmbd_verify_smb_message(struct ksmbd_work *work) if (smb2_hdr->ProtocolId == SMB2_PROTO_NUMBER) return ksmbd_smb2_check_message(work); - hdr = work->request_buf; + hdr = smb_get_msg(work->request_buf); if (*(__le32 *)hdr->Protocol == SMB1_PROTO_NUMBER && hdr->Command == SMB_COM_NEGOTIATE) { work->conn->outstanding_credits++; @@ -163,7 +163,7 @@ bool ksmbd_smb_request(struct ksmbd_conn *conn) if (conn->request_buf[0] != 0) return false; - proto = (__le32 *)smb2_get_msg(conn->request_buf); + proto = (__le32 *)smb_get_msg(conn->request_buf); if (*proto == SMB2_COMPRESSION_TRANSFORM_ID) { pr_err_ratelimited("smb2 compression not support yet"); return false; @@ -259,14 +259,14 @@ int ksmbd_lookup_dialect_by_id(__le16 *cli_dialects, __le16 dialects_count) static int ksmbd_negotiate_smb_dialect(void *buf) { int smb_buf_length = get_rfc1002_len(buf); - __le32 proto = ((struct smb2_hdr *)smb2_get_msg(buf))->ProtocolId; + __le32 proto = ((struct smb2_hdr *)smb_get_msg(buf))->ProtocolId; if (proto == SMB2_PROTO_NUMBER) { struct smb2_negotiate_req *req; int smb2_neg_size = offsetof(struct smb2_negotiate_req, Dialects); - req = (struct smb2_negotiate_req *)smb2_get_msg(buf); + req = (struct smb2_negotiate_req *)smb_get_msg(buf); if (smb2_neg_size > smb_buf_length) goto err_out; @@ -278,15 +278,14 @@ static int ksmbd_negotiate_smb_dialect(void *buf) req->DialectCount); } - proto = *(__le32 *)((struct smb_hdr *)buf)->Protocol; if (proto == SMB1_PROTO_NUMBER) { struct smb_negotiate_req *req; - req = (struct smb_negotiate_req *)buf; + req = (struct smb_negotiate_req *)smb_get_msg(buf); if (le16_to_cpu(req->ByteCount) < 2) goto err_out; - if (offsetof(struct smb_negotiate_req, DialectsArray) - 4 + + if (offsetof(struct smb_negotiate_req, DialectsArray) + le16_to_cpu(req->ByteCount) > smb_buf_length) { goto err_out; } @@ -320,8 +319,8 @@ static u16 get_smb1_cmd_val(struct ksmbd_work *work) */ static int init_smb1_rsp_hdr(struct ksmbd_work *work) { - struct smb_hdr *rsp_hdr = (struct smb_hdr *)work->response_buf; - struct smb_hdr *rcv_hdr = (struct smb_hdr *)work->request_buf; + struct smb_hdr *rsp_hdr = (struct smb_hdr *)smb_get_msg(work->response_buf); + struct smb_hdr *rcv_hdr = (struct smb_hdr *)smb_get_msg(work->request_buf); rsp_hdr->Command = SMB_COM_NEGOTIATE; *(__le32 *)rsp_hdr->Protocol = SMB1_PROTO_NUMBER; @@ -412,9 +411,10 @@ static int init_smb1_server(struct ksmbd_conn *conn) int ksmbd_init_smb_server(struct ksmbd_conn *conn) { + struct smb_hdr *rcv_hdr = (struct smb_hdr *)smb_get_msg(conn->request_buf); __le32 proto; - proto = *(__le32 *)((struct smb_hdr *)conn->request_buf)->Protocol; + proto = *(__le32 *)rcv_hdr->Protocol; if (conn->need_neg == false) { if (proto == SMB1_PROTO_NUMBER) return -EINVAL; @@ -572,12 +572,12 @@ static int __smb2_negotiate(struct ksmbd_conn *conn) static int smb_handle_negotiate(struct ksmbd_work *work) { - struct smb_negotiate_rsp *neg_rsp = work->response_buf; + struct smb_negotiate_rsp *neg_rsp = smb_get_msg(work->response_buf); ksmbd_debug(SMB, "Unsupported SMB1 protocol\n"); - if (ksmbd_iov_pin_rsp(work, (void *)neg_rsp + 4, - sizeof(struct smb_negotiate_rsp) - 4)) + if (ksmbd_iov_pin_rsp(work, (void *)neg_rsp, + sizeof(struct smb_negotiate_rsp))) return -ENOMEM; neg_rsp->hdr.Status.CifsError = STATUS_SUCCESS; @@ -592,23 +592,46 @@ int ksmbd_smb_negotiate_common(struct ksmbd_work *work, unsigned int command) struct ksmbd_conn *conn = work->conn; int ret; - conn->dialect = - ksmbd_negotiate_smb_dialect(work->request_buf); - ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); - if (command == SMB2_NEGOTIATE_HE) { + /* + * An SMB2 NEGOTIATE is valid for a new connection, or after an + * SMB1 multi-protocol negotiate has selected SMB2. Do not allow + * a second SMB2 NEGOTIATE to replace connection-wide state + * while a session setup is pending. KSMBD_SESS_NEED_RECONNECT + * is a transient session state and does not restart transport + * negotiation. + */ + ksmbd_conn_lock(conn); + if (!ksmbd_conn_new(conn) && + !ksmbd_conn_need_negotiate(conn)) { + work->send_no_response = 1; + ksmbd_conn_set_exiting(conn); + ksmbd_conn_unlock(conn); + return 0; + } + + conn->dialect = + ksmbd_negotiate_smb_dialect(work->request_buf); + ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); ret = smb2_handle_negotiate(work); + ksmbd_conn_unlock(conn); return ret; } if (command == SMB_COM_NEGOTIATE) { + ksmbd_conn_lock(conn); + conn->dialect = + ksmbd_negotiate_smb_dialect(work->request_buf); + ksmbd_debug(SMB, "conn->dialect 0x%x\n", conn->dialect); if (__smb2_negotiate(conn)) { init_smb3_11_server(conn); - init_smb2_neg_rsp(work); + ret = init_smb2_neg_rsp(work); ksmbd_debug(SMB, "Upgrade to SMB2 negotiation\n"); - return 0; + } else { + ret = smb_handle_negotiate(work); } - return smb_handle_negotiate(work); + ksmbd_conn_unlock(conn); + return ret; } pr_err("Unknown SMB negotiation command: %u\n", command); diff --git a/fs/smb/server/smb_common.h b/fs/smb/server/smb_common.h index e3be583e803da..95062e922de93 100644 --- a/fs/smb/server/smb_common.h +++ b/fs/smb/server/smb_common.h @@ -3,8 +3,8 @@ * Copyright (C) 2018 Samsung Electronics Co., Ltd. */ -#ifndef __SMB_COMMON_H__ -#define __SMB_COMMON_H__ +#ifndef __SMB_SERVER_COMMON_H__ +#define __SMB_SERVER_COMMON_H__ #include @@ -424,8 +424,12 @@ unsigned int ksmbd_server_side_copy_max_total_size(void); bool is_asterisk(char *p); __le32 smb_map_generic_desired_access(__le32 daccess); -static inline unsigned int get_rfc1002_len(void *buf) +/* + * Get the body of the smb message excluding the 4 byte rfc1002 headers + * from request/response buffer. + */ +static inline void *smb_get_msg(void *buf) { - return be32_to_cpu(*((__be32 *)buf)) & 0xffffff; + return buf + 4; } -#endif /* __SMB_COMMON_H__ */ +#endif /* __SMB_SERVER_COMMON_H__ */ diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c index b09bc8d9389a2..2a46646b3a0e4 100644 --- a/fs/smb/server/smbacl.c +++ b/fs/smb/server/smbacl.c @@ -258,6 +258,7 @@ static int sid_to_id(struct mnt_idmap *idmap, struct smb_sid *psid, uint sidtype, struct smb_fattr *fattr) { + const struct smb_sid *sid_prefix; int rc = -EINVAL; /* @@ -279,6 +280,12 @@ static int sid_to_id(struct mnt_idmap *idmap, kuid_t uid; uid_t id; + /* Only the server domain RID has a local uid representation. */ + sid_prefix = &server_conf.domain_sid; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); uid = KUIDT_INIT(id); uid = from_vfsuid(idmap, &init_user_ns, VFSUIDT_INIT(uid)); @@ -290,6 +297,12 @@ static int sid_to_id(struct mnt_idmap *idmap, kgid_t gid; gid_t id; + /* Local gids are represented by S-1-22-2-. */ + sid_prefix = &sid_unix_groups; + if (psid->num_subauth != sid_prefix->num_subauth + 1 || + compare_sids(psid, sid_prefix)) + return -EINVAL; + id = le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]); gid = KGIDT_INIT(id); gid = from_vfsgid(idmap, &init_user_ns, VFSGIDT_INIT(gid)); @@ -367,10 +380,10 @@ void free_acl_state(struct posix_acl_state *state) kfree(state->groups); } -static void parse_dacl(struct mnt_idmap *idmap, - struct smb_acl *pdacl, char *end_of_acl, - struct smb_sid *pownersid, struct smb_sid *pgrpsid, - struct smb_fattr *fattr) +static int parse_dacl(struct mnt_idmap *idmap, + struct smb_acl *pdacl, char *end_of_acl, + struct smb_sid *pownersid, struct smb_sid *pgrpsid, + struct smb_fattr *fattr) { int i, ret; u16 num_aces = 0; @@ -384,13 +397,13 @@ static void parse_dacl(struct mnt_idmap *idmap, bool owner_found = false, group_found = false, others_found = false; if (!pdacl) - return; + return 0; /* validate that we do not go past end of acl */ if (end_of_acl < (char *)pdacl + sizeof(struct smb_acl) || end_of_acl < (char *)pdacl + le16_to_cpu(pdacl->size)) { pr_err("ACL too small to parse DACL\n"); - return; + return -EINVAL; } ksmbd_debug(SMB, "DACL revision %d size %d num aces %d\n", @@ -402,31 +415,31 @@ static void parse_dacl(struct mnt_idmap *idmap, num_aces = le16_to_cpu(pdacl->num_aces); if (num_aces <= 0) - return; + return 0; dacl_size = le16_to_cpu(pdacl->size); if (dacl_size < sizeof(struct smb_acl)) - return; + return -EINVAL; if (num_aces > (dacl_size - sizeof(struct smb_acl)) / (offsetof(struct smb_ace, sid) + offsetof(struct smb_sid, sub_auth) + sizeof(__le16))) - return; + return -EINVAL; ret = init_acl_state(&acl_state, num_aces); if (ret) - return; + return ret; ret = init_acl_state(&default_acl_state, num_aces); if (ret) { free_acl_state(&acl_state); - return; + return ret; } ppace = kmalloc_array(num_aces, sizeof(struct smb_ace *), KSMBD_DEFAULT_GFP); if (!ppace) { free_acl_state(&default_acl_state); free_acl_state(&acl_state); - return; + return -ENOMEM; } /* @@ -435,8 +448,10 @@ static void parse_dacl(struct mnt_idmap *idmap, * user/group/other have no permissions */ for (i = 0; i < num_aces; ++i) { - if (end_of_acl - acl_base < acl_size) - break; + if (end_of_acl - acl_base < acl_size) { + ret = -EINVAL; + goto out; + } ppace[i] = (struct smb_ace *)(acl_base + acl_size); acl_base = (char *)ppace[i]; @@ -449,8 +464,10 @@ static void parse_dacl(struct mnt_idmap *idmap, (end_of_acl - acl_base < acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth) || (le16_to_cpu(ppace[i]->size) < - acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) - break; + acl_size + sizeof(__le32) * ppace[i]->sid.num_subauth)) { + ret = -EINVAL; + goto out; + } acl_size = le16_to_cpu(ppace[i]->size); ppace[i]->access_req = @@ -508,8 +525,8 @@ static void parse_dacl(struct mnt_idmap *idmap, temp_fattr.cf_uid = INVALID_UID; ret = sid_to_id(idmap, &ppace[i]->sid, SIDOWNER, &temp_fattr); if (ret || uid_eq(temp_fattr.cf_uid, INVALID_UID)) { - pr_err("%s: Error %d mapping Owner SID to uid\n", - __func__, ret); + pr_err_ratelimited("%s: Error %d mapping Owner SID to uid\n", + __func__, ret); continue; } @@ -525,7 +542,6 @@ static void parse_dacl(struct mnt_idmap *idmap, ((acl_mode & 0700) >> 6) | 0004; } } - kfree(ppace); if (owner_found) { /* The owner must be set to at least read-only. */ @@ -568,10 +584,12 @@ static void parse_dacl(struct mnt_idmap *idmap, fattr->cf_acls = posix_acl_alloc(acl_state.users->n + acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); - if (fattr->cf_acls) { - cf_pace = fattr->cf_acls->a_entries; - posix_state_to_acl(&acl_state, cf_pace); + if (!fattr->cf_acls) { + ret = -ENOMEM; + goto out; } + cf_pace = fattr->cf_acls->a_entries; + posix_state_to_acl(&acl_state, cf_pace); } } @@ -582,20 +600,27 @@ static void parse_dacl(struct mnt_idmap *idmap, fattr->cf_dacls = posix_acl_alloc(default_acl_state.users->n + default_acl_state.groups->n + 4, KSMBD_DEFAULT_GFP); - if (fattr->cf_dacls) { - cf_pdace = fattr->cf_dacls->a_entries; - posix_state_to_acl(&default_acl_state, cf_pdace); + if (!fattr->cf_dacls) { + ret = -ENOMEM; + goto out; } + cf_pdace = fattr->cf_dacls->a_entries; + posix_state_to_acl(&default_acl_state, cf_pdace); } } + ret = 0; +out: + kfree(ppace); free_acl_state(&acl_state); free_acl_state(&default_acl_state); + return ret; } static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, struct smb_ace *pndace, struct smb_fattr *fattr, u16 *num_aces, - u16 *size, u32 nt_aces_num) + u16 *size, u16 existing_nt_aces, + bool had_nt_aces) { struct posix_acl_entry *pace; struct smb_sid *sid; @@ -627,14 +652,14 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, gid = posix_acl_gid_translate(idmap, pace); id_to_sid(gid, SIDUNIX_GROUP, sid); - } else if (pace->e_tag == ACL_OTHER && !nt_aces_num) { + } else if (pace->e_tag == ACL_OTHER && !had_nt_aces) { smb_copy_sid(sid, &sid_everyone); } else { kfree(sid); continue; } ntace = pndace; - for (j = 0; j < nt_aces_num; j++) { + for (j = 0; j < existing_nt_aces; j++) { if (ntace->sid.sub_auth[ntace->sid.num_subauth - 1] == sid->sub_auth[sid->num_subauth - 1]) goto pass_same_sid; @@ -649,6 +674,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags, pace->e_perm, 0777); if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; kfree(sid); break; } @@ -663,6 +689,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x03, pace->e_perm, 0777); if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; kfree(sid); break; } @@ -676,7 +703,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, kfree(sid); } - if (nt_aces_num) + if (had_nt_aces) return; posix_default_acl: @@ -708,6 +735,7 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap, ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b, pace->e_perm, 0777); if (check_add_overflow(*size, ace_sz, size)) { + *size -= ace_sz; kfree(sid); break; } @@ -729,6 +757,7 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, { struct smb_ace *ntace, *pndace; u16 nt_num_aces = le16_to_cpu(nt_dacl->num_aces), num_aces = 0; + u16 copied_nt_aces; unsigned short size = 0; int i; @@ -738,24 +767,41 @@ static void set_ntacl_dacl(struct mnt_idmap *idmap, for (i = 0; i < nt_num_aces; i++) { unsigned short nt_ace_size; - if (offsetof(struct smb_ace, access_req) > aces_size) + if (aces_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; nt_ace_size = le16_to_cpu(ntace->size); - if (nt_ace_size > aces_size) + if (nt_ace_size > aces_size || + nt_ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE) break; + if (ntace->sid.num_subauth == 0 || + ntace->sid.num_subauth > SID_MAX_SUB_AUTHORITIES || + nt_ace_size < offsetof(struct smb_ace, sid) + + CIFS_SID_BASE_SIZE + + sizeof(__le32) * + ntace->sid.num_subauth) + goto next_ace; + memcpy((char *)pndace + size, ntace, nt_ace_size); - if (check_add_overflow(size, nt_ace_size, &size)) + if (check_add_overflow(size, nt_ace_size, &size)) { + size -= nt_ace_size; break; + } + num_aces++; + +next_ace: aces_size -= nt_ace_size; ntace = (struct smb_ace *)((char *)ntace + nt_ace_size); - num_aces++; } } + copied_nt_aces = num_aces; set_posix_acl_entries_dacl(idmap, pndace, fattr, - &num_aces, &size, nt_num_aces); + &num_aces, &size, copied_nt_aces, + nt_num_aces != 0); pndacl->num_aces = cpu_to_le16(num_aces); pndacl->size = cpu_to_le16(le16_to_cpu(pndacl->size) + size); } @@ -773,7 +819,7 @@ static void set_mode_dacl(struct mnt_idmap *idmap, if (fattr->cf_acls) { set_posix_acl_entries_dacl(idmap, pndace, fattr, - &num_aces, &size, num_aces); + &num_aces, &size, num_aces, false); goto out; } @@ -894,9 +940,9 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, rc = sid_to_id(idmap, owner_sid_ptr, SIDOWNER, fattr); if (rc) { - pr_err("%s: Error %d mapping Owner SID to uid\n", - __func__, rc); + ksmbd_debug(SMB, "Owner SID has no Unix uid mapping\n"); owner_sid_ptr = NULL; + rc = 0; } } @@ -912,9 +958,9 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, } rc = sid_to_id(idmap, group_sid_ptr, SIDUNIX_GROUP, fattr); if (rc) { - pr_err("%s: Error %d mapping Group SID to gid\n", - __func__, rc); + ksmbd_debug(SMB, "Group SID has no Unix gid mapping\n"); group_sid_ptr = NULL; + rc = 0; } } @@ -928,8 +974,10 @@ int parse_sec_desc(struct mnt_idmap *idmap, struct smb_ntsd *pntsd, if (dacloffset < sizeof(struct smb_ntsd)) return -EINVAL; - parse_dacl(idmap, dacl_ptr, end_of_acl, - owner_sid_ptr, group_sid_ptr, fattr); + rc = parse_dacl(idmap, dacl_ptr, end_of_acl, + owner_sid_ptr, group_sid_ptr, fattr); + if (rc) + return rc; } return 0; @@ -1449,7 +1497,7 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, DELETE; ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) @@ -1473,7 +1521,7 @@ int smb_check_perm_dacl(struct ksmbd_conn *conn, const struct path *path, id_to_sid(uid, sid_type, &sid); ace = (struct smb_ace *)((char *)pdacl + sizeof(struct smb_acl)); - aces_size = acl_size - sizeof(struct smb_acl); + aces_size = pdacl_size - sizeof(struct smb_acl); for (i = 0; i < le16_to_cpu(pdacl->num_aces); i++) { if (aces_size < offsetof(struct smb_ace, sid) + CIFS_SID_BASE_SIZE) diff --git a/fs/smb/server/transport_ipc.c b/fs/smb/server/transport_ipc.c index 1c5645238bd3f..7c30b50f23f04 100644 --- a/fs/smb/server/transport_ipc.c +++ b/fs/smb/server/transport_ipc.c @@ -498,6 +498,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_rpc_command *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_rpc_command)) + return -EINVAL; + if (check_add_overflow(sizeof(struct ksmbd_rpc_command), resp->payload_sz, &msg_sz)) return -EINVAL; @@ -507,6 +510,9 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_spnego_authen_response *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_spnego_authen_response)) + return -EINVAL; + msg_sz = sizeof(struct ksmbd_spnego_authen_response) + resp->session_key_len + resp->spnego_blob_len; break; @@ -515,20 +521,33 @@ static int ipc_validate_msg(struct ipc_msg_table_entry *entry) { struct ksmbd_share_config_response *resp = entry->response; - if (resp->payload_sz) { - if (resp->payload_sz < resp->veto_list_sz) - return -EINVAL; + if (entry->msg_sz < sizeof(struct ksmbd_share_config_response)) + return -EINVAL; - if (check_add_overflow(sizeof(struct ksmbd_share_config_response), - resp->payload_sz, &msg_sz)) - return -EINVAL; - } + if (strnlen(resp->share_name, sizeof(resp->share_name)) == + sizeof(resp->share_name)) + return -EINVAL; + + if (resp->veto_list_sz > resp->payload_sz) + return -EINVAL; + + if (resp->flags != KSMBD_SHARE_FLAG_INVALID && + !(resp->flags & KSMBD_SHARE_FLAG_PIPE) && + resp->payload_sz <= resp->veto_list_sz) + return -EINVAL; + + if (check_add_overflow(sizeof(struct ksmbd_share_config_response), + resp->payload_sz, &msg_sz)) + return -EINVAL; break; } case KSMBD_EVENT_LOGIN_REQUEST_EXT: { struct ksmbd_login_response_ext *resp = entry->response; + if (entry->msg_sz < sizeof(struct ksmbd_login_response_ext)) + return -EINVAL; + if (resp->ngroups) { if (resp->ngroups < 0 || resp->ngroups > NGROUPS_MAX) { diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index a09b07b3b108d..78c909d66814f 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -454,6 +454,11 @@ static void ksmbd_tcp_disconnect(struct ksmbd_transport *t) atomic_dec(&active_num_conn); } +static void ksmbd_tcp_shutdown(struct ksmbd_transport *t) +{ + kernel_sock_shutdown(TCP_TRANS(t)->sock, SHUT_RDWR); +} + static void tcp_destroy_socket(struct socket *ksmbd_socket) { int ret; @@ -709,5 +714,6 @@ static const struct ksmbd_transport_ops ksmbd_tcp_transport_ops = { .read = ksmbd_tcp_read, .writev = ksmbd_tcp_writev, .disconnect = ksmbd_tcp_disconnect, + .shutdown = ksmbd_tcp_shutdown, .free_transport = ksmbd_tcp_free_transport, }; diff --git a/fs/smb/server/vfs.c b/fs/smb/server/vfs.c index 764eca75291ce..ec9d04e4de416 100644 --- a/fs/smb/server/vfs.c +++ b/fs/smb/server/vfs.c @@ -327,9 +327,6 @@ static int check_lock_range(struct file *filp, loff_t start, loff_t end, struct file_lock_context *ctx = locks_inode_context(file_inode(filp)); int error = 0; - if (start == end) - return 0; - if (!ctx || list_empty_careful(&ctx->flc_posix)) return 0; @@ -748,6 +745,12 @@ int ksmbd_vfs_rename(struct ksmbd_work *work, const struct path *old_path, goto out3; } + if (!work->tcon->posix_extensions && d_is_dir(old_child) && + ksmbd_has_open_files(old_child)) { + err = -EACCES; + goto out3; + } + parent_fp = ksmbd_lookup_fd_inode(old_child->d_parent); if (parent_fp) { if (parent_fp->daccess & FILE_DELETE_LE) { @@ -1569,8 +1572,8 @@ int ksmbd_vfs_set_sd_xattr(struct ksmbd_conn *conn, if (rc < 0) pr_err("Failed to store XATTR ntacl :%d\n", rc); - kfree(sd_ndr.data); out: + kfree(sd_ndr.data); kfree(acl_ndr.data); kfree(smb_acl); kfree(def_smb_acl); @@ -1586,7 +1589,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, struct ndr n; struct inode *inode = d_inode(dentry); struct ndr acl_ndr = {0}; - struct xattr_ntacl acl; + struct xattr_ntacl acl = {0}; struct xattr_smb_acl *smb_acl = NULL, *def_smb_acl = NULL; __u8 cmp_hash[XATTR_SD_HASH_SIZE] = {0}; @@ -1597,7 +1600,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, n.length = rc; rc = ndr_decode_v4_ntacl(&n, &acl); if (rc) - goto free_n_data; + goto out_free; smb_acl = ksmbd_vfs_make_xattr_posix_acl(idmap, inode, ACL_TYPE_ACCESS); @@ -1623,6 +1626,7 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, *pntsd = acl.sd_buf; if (acl.sd_size < sizeof(struct smb_ntsd)) { pr_err("sd size is invalid\n"); + rc = -EINVAL; goto out_free; } @@ -1642,8 +1646,6 @@ int ksmbd_vfs_get_sd_xattr(struct ksmbd_conn *conn, kfree(acl.sd_buf); *pntsd = NULL; } - -free_n_data: kfree(n.data); return rc; } @@ -1658,14 +1660,15 @@ int ksmbd_vfs_set_dos_attrib_xattr(struct mnt_idmap *idmap, err = ndr_encode_dos_attr(&n, da); if (err) - return err; + goto out; err = ksmbd_vfs_setxattr(idmap, path, XATTR_NAME_DOS_ATTRIBUTE, (void *)n.data, n.offset, 0, get_write); if (err) ksmbd_debug(SMB, "failed to store dos attribute in xattr\n"); - kfree(n.data); +out: + kfree(n.data); return err; } @@ -1968,10 +1971,6 @@ int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, const struct path *path, struct inode *parent_inode) { struct posix_acl *acls; - struct posix_acl_entry *pace; - struct dentry *dentry = path->dentry; - struct inode *inode = d_inode(dentry); - int rc, i; if (!IS_ENABLED(CONFIG_FS_POSIX_ACL)) return -EOPNOTSUPP; @@ -1979,27 +1978,7 @@ int ksmbd_vfs_inherit_posix_acl(struct mnt_idmap *idmap, acls = get_inode_acl(parent_inode, ACL_TYPE_DEFAULT); if (IS_ERR_OR_NULL(acls)) return -ENOENT; - pace = acls->a_entries; - - for (i = 0; i < acls->a_count; i++, pace++) { - if (pace->e_tag == ACL_MASK) { - pace->e_perm = 0x07; - break; - } - } - - rc = set_posix_acl(idmap, dentry, ACL_TYPE_ACCESS, acls); - if (rc < 0) - ksmbd_debug(SMB, "Set posix acl(ACL_TYPE_ACCESS) failed, rc : %d\n", - rc); - if (S_ISDIR(inode->i_mode)) { - rc = set_posix_acl(idmap, dentry, ACL_TYPE_DEFAULT, - acls); - if (rc < 0) - ksmbd_debug(SMB, "Set posix acl(ACL_TYPE_DEFAULT) failed, rc : %d\n", - rc); - } posix_acl_release(acls); - return rc; + return 0; } diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c index 5b6d8bb8edb27..ce6e9886df8be 100644 --- a/fs/smb/server/vfs_cache.c +++ b/fs/smb/server/vfs_cache.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "glob.h" #include "vfs_cache.h" @@ -358,13 +359,13 @@ static void ksmbd_remove_durable_fd(struct ksmbd_file *fp) static void __ksmbd_remove_fd(struct ksmbd_file_table *ft, struct ksmbd_file *fp) { - if (!has_file_id(fp->volatile_id)) - return; - down_write(&fp->f_ci->m_lock); list_del_init(&fp->node); up_write(&fp->f_ci->m_lock); + if (!has_file_id(fp->volatile_id)) + return; + write_lock(&ft->lock); idr_remove(ft->idr, fp->volatile_id); write_unlock(&ft->lock); @@ -487,6 +488,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) { struct ksmbd_file *fp; struct ksmbd_file_table *ft; + bool closed = false; if (!has_file_id(id)) return 0; @@ -501,6 +503,9 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) fp = NULL; else { fp->f_state = FP_CLOSED; + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + closed = true; if (!atomic_dec_and_test(&fp->refcount)) fp = NULL; } @@ -508,7 +513,7 @@ int ksmbd_close_fd(struct ksmbd_work *work, u64 id) write_unlock(&ft->lock); if (!fp) - return -EINVAL; + return closed ? 0 : -EINVAL; __put_fd_final(work, fp); return 0; @@ -645,6 +650,30 @@ struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry) return NULL; } +bool ksmbd_has_open_files(struct dentry *dentry) +{ + struct ksmbd_file *fp; + unsigned int id; + bool ret = false; + + read_lock(&global_ft.lock); + idr_for_each_entry(global_ft.idr, fp, id) { + struct dentry *fp_dentry = fp->filp->f_path.dentry; + + if (fp->f_state != FP_INITED) + continue; + if (fp_dentry == dentry) + continue; + if (is_subdir(fp_dentry, dentry)) { + ret = true; + break; + } + } + read_unlock(&global_ft.lock); + + return ret; +} + #define OPEN_ID_TYPE_VOLATILE_ID (0) #define OPEN_ID_TYPE_PERSISTENT_ID (1) @@ -669,7 +698,8 @@ static int __open_id(struct ksmbd_file_table *ft, struct ksmbd_file *fp, idr_preload(KSMBD_DEFAULT_GFP); write_lock(&ft->lock); - ret = idr_alloc_cyclic(ft->idr, fp, 0, INT_MAX - 1, GFP_NOWAIT); + ret = idr_alloc_cyclic(ft->idr, fp, KSMBD_START_FID, INT_MAX - 1, + GFP_NOWAIT); if (ret >= 0) { id = ret; ret = 0; @@ -744,15 +774,58 @@ struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp) return ERR_PTR(ret); } -void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, - unsigned int state) +/** + * ksmbd_update_fstate() - update an fp state under the file-table lock + * @ft: file table that publishes @fp's volatile id + * @fp: file pointer to update + * @state: new state + * + * Return: 0 on success. The FP_NEW -> FP_INITED transition is special: + * -ENOENT if teardown already unpublished @fp by advancing the state or + * clearing the volatile id. Other state updates preserve the historical + * fire-and-forget behavior. + */ +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state) { + int ret; + if (!fp) - return; + return -ENOENT; write_lock(&ft->lock); - fp->f_state = state; + if (state == FP_INITED && + (fp->f_state != FP_NEW || !has_file_id(fp->volatile_id))) { + ret = -ENOENT; + } else { + fp->f_state = state; + ret = 0; + } write_unlock(&ft->lock); + + return ret; +} + +/* + * ksmbd_mark_fp_closed() - mark fp closed under ft->lock and return how many + * refs the teardown path owns. + * + * FP_INITED has a normal idr-owned reference, so teardown owns both that + * reference and the transient lookup reference. FP_NEW is still owned by the + * in-flight opener/reopener, which will drop the original reference after + * ksmbd_update_fstate(..., FP_INITED) observes the cleared volatile id. + * FP_CLOSED on entry means an earlier ksmbd_close_fd() already consumed the + * idr-owned ref. + */ +static int ksmbd_mark_fp_closed(struct ksmbd_file *fp) +{ + if (fp->f_state == FP_INITED) { + set_close_state_blocked_works(fp); + fp->f_state = FP_CLOSED; + return 2; + } + + return 1; } static int @@ -760,7 +833,8 @@ __close_file_table_ids(struct ksmbd_session *sess, struct ksmbd_tree_connect *tcon, bool (*skip)(struct ksmbd_tree_connect *tcon, struct ksmbd_file *fp, - struct ksmbd_user *user)) + struct ksmbd_user *user), + bool skip_preserves_fp) { struct ksmbd_file_table *ft = &sess->file_table; struct ksmbd_file *fp; @@ -768,32 +842,120 @@ __close_file_table_ids(struct ksmbd_session *sess, int num = 0; while (1) { + int n_to_drop; + write_lock(&ft->lock); fp = idr_get_next(ft->idr, &id); if (!fp) { write_unlock(&ft->lock); break; } - - if (skip(tcon, fp, sess->user) || - !atomic_dec_and_test(&fp->refcount)) { + if (!atomic_inc_not_zero(&fp->refcount)) { id++; write_unlock(&ft->lock); continue; } - set_close_state_blocked_works(fp); - idr_remove(ft->idr, fp->volatile_id); - fp->volatile_id = KSMBD_NO_FID; - write_unlock(&ft->lock); + if (skip_preserves_fp) { + /* + * Session teardown: skip() is session_fd_check(), + * which may sleep and mutates fp->conn / fp->tcon / + * fp->volatile_id when it chooses to preserve fp + * for durable reconnect. Unpublish fp from the + * session idr here, under ft->lock, so that + * __ksmbd_lookup_fd() through this session cannot + * grant a new ksmbd_fp_get() reference to an fp + * whose fields are about to be rewritten outside + * the lock. Durable reconnect still reaches fp via + * global_ft. + */ + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + write_unlock(&ft->lock); + + if (skip(tcon, fp, sess->user)) { + /* + * session_fd_check() has converted fp to + * durable-preserve state and cleared its + * per-conn fields. fp is already unpublished + * above; the original idr-owned ref keeps it + * alive for the durable scavenger. Drop only + * the transient ref. atomic_dec() is safe -- + * atomic_inc_not_zero() succeeded on a + * positive value and we added one more, so + * refcount cannot be zero here. + */ + atomic_dec(&fp->refcount); + id++; + continue; + } + /* + * Keep the close-state decision under the same lock + * observed by ksmbd_update_fstate(), which is how an + * in-flight FP_NEW opener learns that teardown has + * cleared its volatile id. + */ + write_lock(&ft->lock); + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } else { + /* + * Tree teardown: skip() is tree_conn_fd_check(), a + * cheap pointer compare that doesn't sleep and has + * no side effects, so keep the skip decision plus + * the unpublish-and-mark-closed sequence atomic + * under ft->lock. fps belonging to other tree + * connects (skip() == true) stay fully published in + * the session idr with no lock window. + */ + if (skip(tcon, fp, sess->user)) { + atomic_dec(&fp->refcount); + write_unlock(&ft->lock); + id++; + continue; + } + idr_remove(ft->idr, id); + fp->volatile_id = KSMBD_NO_FID; + n_to_drop = ksmbd_mark_fp_closed(fp); + write_unlock(&ft->lock); + } + + /* + * fp->volatile_id is already cleared to prevent stale idr + * removal from a deferred final close. Remove fp from + * m_fp_list here because __ksmbd_remove_fd() will skip the + * list unlink when volatile_id is KSMBD_NO_FID. + */ down_write(&fp->f_ci->m_lock); list_del_init(&fp->node); up_write(&fp->f_ci->m_lock); - __ksmbd_close_fd(ft, fp); - - num++; + /* + * Drop the references this iteration owns: + * + * n_to_drop == 2: we observed FP_INITED and committed + * the FP_CLOSED transition ourselves, so we own the + * transient (+1) and the still-intact idr-owned ref. + * + * n_to_drop == 1: either a prior ksmbd_close_fd() + * already consumed the idr-owned ref, or fp was still + * FP_NEW and the in-flight opener/reopener must keep + * the original reference until ksmbd_update_fstate() + * observes the cleared volatile id. + * + * If we end up as the final putter, finalize fp and + * account the open_files_count decrement via the caller's + * atomic_sub(num, ...). Otherwise the remaining user's + * ksmbd_fd_put() reaches __put_fd_final(), which does its + * own atomic_dec(&open_files_count), so we must not count + * this fp here -- doing so would double-decrement the + * connection-wide counter. + */ + if (atomic_sub_and_test(n_to_drop, &fp->refcount)) { + __ksmbd_close_fd(NULL, fp); + num++; + } id++; } @@ -967,9 +1129,12 @@ void ksmbd_launch_ksmbd_durable_scavenger(void) server_conf.dh_task = kthread_run(ksmbd_durable_scavenger, (void *)NULL, "ksmbd-durable-scavenger"); - if (IS_ERR(server_conf.dh_task)) + if (IS_ERR(server_conf.dh_task)) { pr_err("cannot start conn thread, err : %ld\n", PTR_ERR(server_conf.dh_task)); + server_conf.dh_task = NULL; + durable_scavenger_running = false; + } mutex_unlock(&durable_scavenger_lock); } @@ -1067,6 +1232,9 @@ static bool session_fd_check(struct ksmbd_tree_connect *tcon, if (!is_reconnectable(fp)) return false; + if (fp->f_state != FP_INITED) + return false; + if (WARN_ON_ONCE(!fp->conn)) return false; @@ -1118,7 +1286,8 @@ void ksmbd_close_tree_conn_fds(struct ksmbd_work *work) { int num = __close_file_table_ids(work->sess, work->tcon, - tree_conn_fd_check); + tree_conn_fd_check, + false); atomic_sub(num, &work->conn->stats.open_files_count); } @@ -1127,7 +1296,8 @@ void ksmbd_close_session_fds(struct ksmbd_work *work) { int num = __close_file_table_ids(work->sess, work->tcon, - session_fd_check); + session_fd_check, + true); atomic_sub(num, &work->conn->stats.open_files_count); } @@ -1267,7 +1437,7 @@ void ksmbd_destroy_file_table(struct ksmbd_session *sess) if (!ft->idr) return; - __close_file_table_ids(sess, NULL, session_fd_check); + __close_file_table_ids(sess, NULL, session_fd_check, true); idr_destroy(ft->idr); kfree(ft->idr); ft->idr = NULL; diff --git a/fs/smb/server/vfs_cache.h b/fs/smb/server/vfs_cache.h index 10f4172ec8017..38ccf2957c174 100644 --- a/fs/smb/server/vfs_cache.h +++ b/fs/smb/server/vfs_cache.h @@ -23,7 +23,12 @@ #define FILE_GENERIC_WRITE 0x120116 #define FILE_GENERIC_EXECUTE 0X1200a0 -#define KSMBD_START_FID 0 +/* + * Start volatile/persistent file id allocation at 1. A file id of 0 yields an + * SMB2 FileId of {0, 0}, which clients (e.g. Windows, Samba) treat as a null + * handle and never close, leaking the open on the server. + */ +#define KSMBD_START_FID 1 #define KSMBD_NO_FID (INT_MAX) #define SMB2_NO_FID (0xFFFFFFFFFFFFFFFFULL) @@ -165,6 +170,7 @@ struct ksmbd_file *ksmbd_lookup_durable_fd(unsigned long long id); void ksmbd_put_durable_fd(struct ksmbd_file *fp); struct ksmbd_file *ksmbd_lookup_fd_cguid(char *cguid); struct ksmbd_file *ksmbd_lookup_fd_inode(struct dentry *dentry); +bool ksmbd_has_open_files(struct dentry *dentry); unsigned int ksmbd_open_durable_fd(struct ksmbd_file *fp); struct ksmbd_file *ksmbd_open_fd(struct ksmbd_work *work, struct file *filp); void ksmbd_launch_ksmbd_durable_scavenger(void); @@ -175,8 +181,8 @@ int ksmbd_close_inode_fds(struct ksmbd_work *work, struct inode *inode); int ksmbd_init_global_file_table(void); void ksmbd_free_global_file_table(void); void ksmbd_set_fd_limit(unsigned long limit); -void ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, - unsigned int state); +int ksmbd_update_fstate(struct ksmbd_file_table *ft, struct ksmbd_file *fp, + unsigned int state); bool ksmbd_vfs_compare_durable_owner(struct ksmbd_file *fp, struct ksmbd_user *user); diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c index 92fb857d2c761..c8756f65bc548 100644 --- a/fs/squashfs/cache.c +++ b/fs/squashfs/cache.c @@ -299,7 +299,7 @@ int squashfs_copy_data(void *buffer, struct squashfs_cache_entry *entry, { int remaining = length; - if (length == 0) + if (length == 0 || offset < 0) return 0; else if (buffer == NULL) return min(length, entry->length - offset); diff --git a/fs/stat.c b/fs/stat.c index 6c79661e1b961..8c110555b3d7b 100644 --- a/fs/stat.c +++ b/fs/stat.c @@ -53,7 +53,7 @@ void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode) } stat->mtime = inode_get_mtime(inode); - stat->ctime.tv_sec = inode->i_ctime_sec; + stat->ctime.tv_sec = inode_get_ctime_sec(inode); stat->ctime.tv_nsec = (u32)atomic_read(pcn); if (!(stat->ctime.tv_nsec & I_CTIME_QUERIED)) stat->ctime.tv_nsec = ((u32)atomic_fetch_or(I_CTIME_QUERIED, pcn)); diff --git a/fs/super.c b/fs/super.c index ce6e5b4ff3c5f..c293b2efebe0a 100644 --- a/fs/super.c +++ b/fs/super.c @@ -1139,18 +1139,35 @@ void emergency_remount(void) } } +static inline bool get_active_super(struct super_block *sb) +{ + bool active = false; + + if (super_lock_excl(sb)) { + active = atomic_inc_not_zero(&sb->s_active); + super_unlock_excl(sb); + } + return active; +} + static void do_thaw_all_callback(struct super_block *sb, void *unused) { + if (!get_active_super(sb)) + return; + + /* fs_bdev_thaw() acquires s_umount so it must not be held here */ if (IS_ENABLED(CONFIG_BLOCK)) while (sb->s_bdev && !bdev_thaw(sb->s_bdev)) pr_warn("Emergency Thaw on %pg\n", sb->s_bdev); - thaw_super_locked(sb, FREEZE_HOLDER_USERSPACE, NULL); - return; + + if (super_lock_excl(sb)) + thaw_super_locked(sb, FREEZE_HOLDER_USERSPACE, NULL); + deactivate_super(sb); } static void do_thaw_all(struct work_struct *work) { - __iterate_supers(do_thaw_all_callback, NULL, SUPER_ITER_EXCL); + __iterate_supers(do_thaw_all_callback, NULL, SUPER_ITER_UNLOCKED); kfree(work); printk(KERN_WARNING "Emergency Thaw complete\n"); } @@ -1171,17 +1188,6 @@ void emergency_thaw_all(void) } } -static inline bool get_active_super(struct super_block *sb) -{ - bool active = false; - - if (super_lock_excl(sb)) { - active = atomic_inc_not_zero(&sb->s_active); - super_unlock_excl(sb); - } - return active; -} - static const char *filesystems_freeze_ptr = "filesystems_freeze"; static void filesystems_freeze_callback(struct super_block *sb, void *freeze_all_ptr) @@ -2077,7 +2083,7 @@ int freeze_super(struct super_block *sb, enum freeze_holder who, const void *fre int ret; if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while freezing!"); + WARN_ONCE(1, "Dying superblock while freezing!"); return -EINVAL; } atomic_inc(&sb->s_active); @@ -2181,11 +2187,14 @@ static int thaw_super_locked(struct super_block *sb, enum freeze_holder who, goto out_unlock; /* - * All freezers share a single active reference. - * So just unlock in case there are any left. + * All freezers share a single active reference. If other freezers + * remain, drop our hold and report success; the superblock stays + * frozen until the last holder thaws it. */ - if (freeze_dec(sb, who)) + if (freeze_dec(sb, who)) { + error = 0; goto out_unlock; + } if (sb_rdonly(sb)) { sb->s_writers.frozen = SB_UNFROZEN; @@ -2241,7 +2250,7 @@ int thaw_super(struct super_block *sb, enum freeze_holder who, const void *freeze_owner) { if (!super_lock_excl(sb)) { - WARN_ON_ONCE("Dying superblock while thawing!"); + WARN_ONCE(1, "Dying superblock while thawing!"); return -EINVAL; } return thaw_super_locked(sb, who, freeze_owner); diff --git a/fs/tracefs/event_inode.c b/fs/tracefs/event_inode.c index 02d56ed6ad20e..a7b082a665bd1 100644 --- a/fs/tracefs/event_inode.c +++ b/fs/tracefs/event_inode.c @@ -436,6 +436,8 @@ static inline struct eventfs_inode *init_ei(struct eventfs_inode *ei, const char if (!ei->name) return NULL; kref_init(&ei->kref); + INIT_LIST_HEAD(&ei->children); + INIT_LIST_HEAD(&ei->list); return ei; } @@ -727,8 +729,6 @@ struct eventfs_inode *eventfs_create_dir(const char *name, struct eventfs_inode ei->entries = entries; ei->nr_entries = size; ei->data = data; - INIT_LIST_HEAD(&ei->children); - INIT_LIST_HEAD(&ei->list); mutex_lock(&eventfs_mutex); if (!parent->is_freed) @@ -801,9 +801,6 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry ei->attr.uid = uid; ei->attr.gid = gid; - INIT_LIST_HEAD(&ei->children); - INIT_LIST_HEAD(&ei->list); - ti = get_tracefs(inode); ti->flags |= TRACEFS_EVENT_INODE; ti->private = ei; @@ -849,7 +846,7 @@ struct eventfs_inode *eventfs_create_events_dir(const char *name, struct dentry */ static void eventfs_remove_rec(struct eventfs_inode *ei, int level) { - struct eventfs_inode *ei_child; + struct eventfs_inode *ei_child, *tmp; /* * Check recursion depth. It should never be greater than 3: @@ -862,7 +859,7 @@ static void eventfs_remove_rec(struct eventfs_inode *ei, int level) return; /* search for nested folders or files */ - list_for_each_entry(ei_child, &ei->children, list) + list_for_each_entry_safe(ei_child, tmp, &ei->children, list) eventfs_remove_rec(ei_child, level + 1); list_del_rcu(&ei->list); diff --git a/fs/ubifs/auth.c b/fs/ubifs/auth.c index a4a0158f712d3..1f770795ee70f 100644 --- a/fs/ubifs/auth.c +++ b/fs/ubifs/auth.c @@ -217,7 +217,7 @@ int ubifs_sb_verify_signature(struct ubifs_info *c, signode = snod->node; - if (le32_to_cpu(signode->len) > snod->len + sizeof(struct ubifs_sig_node)) { + if (le32_to_cpu(signode->len) > snod->len - sizeof(struct ubifs_sig_node)) { ubifs_err(c, "invalid signature len %d", le32_to_cpu(signode->len)); err = -EINVAL; goto out_destroy; diff --git a/fs/udf/inode.c b/fs/udf/inode.c index a79d73f28aa78..271e3f0d1fb10 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -337,65 +337,6 @@ const struct address_space_operations udf_aops = { .migrate_folio = buffer_migrate_folio, }; -/* - * Expand file stored in ICB to a normal one-block-file - * - * This function requires i_mutex held - */ -int udf_expand_file_adinicb(struct inode *inode) -{ - struct folio *folio; - struct udf_inode_info *iinfo = UDF_I(inode); - int err; - - WARN_ON_ONCE(!inode_is_locked(inode)); - if (!iinfo->i_lenAlloc) { - down_write(&iinfo->i_data_sem); - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - up_write(&iinfo->i_data_sem); - mark_inode_dirty(inode); - return 0; - } - - folio = __filemap_get_folio(inode->i_mapping, 0, - FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); - if (IS_ERR(folio)) - return PTR_ERR(folio); - - if (!folio_test_uptodate(folio)) - udf_adinicb_read_folio(folio); - down_write(&iinfo->i_data_sem); - memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, - iinfo->i_lenAlloc); - iinfo->i_lenAlloc = 0; - if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) - iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; - else - iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; - folio_mark_dirty(folio); - folio_unlock(folio); - up_write(&iinfo->i_data_sem); - err = filemap_fdatawrite(inode->i_mapping); - if (err) { - /* Restore everything back so that we don't lose data... */ - folio_lock(folio); - down_write(&iinfo->i_data_sem); - memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, - folio, 0, inode->i_size); - folio_unlock(folio); - iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; - iinfo->i_lenAlloc = inode->i_size; - up_write(&iinfo->i_data_sem); - } - folio_put(folio); - mark_inode_dirty(inode); - - return err; -} - #define UDF_MAP_CREATE 0x01 /* Mapping can allocate new blocks */ #define UDF_MAP_NOPREALLOC 0x02 /* Do not preallocate blocks */ @@ -456,6 +397,76 @@ static int udf_map_block(struct inode *inode, struct udf_map_rq *map) return ret; } +/* + * Expand file stored in ICB to a normal one-block-file + * + * This function requires i_mutex held + */ +int udf_expand_file_adinicb(struct inode *inode) +{ + struct folio *folio; + struct udf_inode_info *iinfo = UDF_I(inode); + struct udf_map_rq map = { + .lblk = 0, + .iflags = UDF_MAP_CREATE, + }; + int err; + + WARN_ON_ONCE(!inode_is_locked(inode)); + if (!iinfo->i_lenAlloc) { + down_write(&iinfo->i_data_sem); + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + mark_inode_dirty(inode); + return 0; + } + + folio = __filemap_get_folio(inode->i_mapping, 0, + FGP_LOCK | FGP_ACCESSED | FGP_CREAT, GFP_KERNEL); + if (IS_ERR(folio)) + return PTR_ERR(folio); + + if (!folio_test_uptodate(folio)) + udf_adinicb_read_folio(folio); + down_write(&iinfo->i_data_sem); + memset(iinfo->i_data + iinfo->i_lenEAttr, 0x00, + iinfo->i_lenAlloc); + iinfo->i_lenAlloc = 0; + if (UDF_QUERY_FLAG(inode->i_sb, UDF_FLAG_USE_SHORT_AD)) + iinfo->i_alloc_type = ICBTAG_FLAG_AD_SHORT; + else + iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; + up_write(&iinfo->i_data_sem); + + /* Allocate the block underlying the data */ + err = udf_map_block(inode, &map); + if (err < 0) + goto restore; + + folio_mark_dirty(folio); + folio_unlock(folio); + err = filemap_fdatawrite(inode->i_mapping); + if (err) { + /* Restore everything back so that we don't lose data... */ + folio_lock(folio); +restore: + down_write(&iinfo->i_data_sem); + memcpy_from_folio(iinfo->i_data + iinfo->i_lenEAttr, + folio, 0, inode->i_size); + iinfo->i_alloc_type = ICBTAG_FLAG_AD_IN_ICB; + iinfo->i_lenAlloc = inode->i_size; + up_write(&iinfo->i_data_sem); + folio_unlock(folio); + } + folio_put(folio); + mark_inode_dirty(inode); + + return err; +} + static int __udf_get_block(struct inode *inode, sector_t block, struct buffer_head *bh_result, int flags) { @@ -532,7 +543,7 @@ static int udf_do_extend_file(struct inode *inode, sb->s_blocksize - 1) & ~(sb->s_blocksize - 1)); iinfo->i_lenExtents = (iinfo->i_lenExtents + sb->s_blocksize - 1) & - ~(sb->s_blocksize - 1); + ~((u64)sb->s_blocksize - 1); } add = 0; diff --git a/fs/udf/partition.c b/fs/udf/partition.c index 2b85c9501bed8..ad8dcedca263d 100644 --- a/fs/udf/partition.c +++ b/fs/udf/partition.c @@ -55,7 +55,7 @@ uint32_t udf_get_pblock_virt15(struct super_block *sb, uint32_t block, map = &sbi->s_partmaps[partition]; vdata = &map->s_type_specific.s_virtual; - if (block > vdata->s_num_entries) { + if (block >= vdata->s_num_entries) { udf_debug("Trying to access block beyond end of VAT (%u max %u)\n", block, vdata->s_num_entries); return 0xFFFFFFFF; diff --git a/fs/udf/super.c b/fs/udf/super.c index 5a82ae2af93ec..ff170bc0962b5 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -2048,6 +2048,17 @@ static int udf_load_vrs(struct super_block *sb, struct udf_options *uopt, return 0; } +static void udf_mark_buffer_dirty(struct buffer_head *bh) +{ + /* + * We set buffer uptodate unconditionally here to avoid spurious + * warnings from mark_buffer_dirty() when previous EIO has marked + * the buffer as !uptodate + */ + set_buffer_uptodate(bh); + mark_buffer_dirty(bh); +} + static void udf_finalize_lvid(struct logicalVolIntegrityDesc *lvid) { struct timespec64 ts; @@ -2083,7 +2094,7 @@ static void udf_open_lvid(struct super_block *sb) UDF_SET_FLAG(sb, UDF_FLAG_INCONSISTENT); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make opening of filesystem visible on the media immediately */ @@ -2116,14 +2127,8 @@ static void udf_close_lvid(struct super_block *sb) if (!UDF_QUERY_FLAG(sb, UDF_FLAG_INCONSISTENT)) lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_CLOSE); - /* - * We set buffer uptodate unconditionally here to avoid spurious - * warnings from mark_buffer_dirty() when previous EIO has marked - * the buffer as !uptodate - */ - set_buffer_uptodate(bh); udf_finalize_lvid(lvid); - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; mutex_unlock(&sbi->s_alloc_mutex); /* Make closing of filesystem visible on the media immediately */ @@ -2405,7 +2410,7 @@ static int udf_sync_fs(struct super_block *sb, int wait) * Blockdevice will be synced later so we don't have to submit * the buffer for IO */ - mark_buffer_dirty(bh); + udf_mark_buffer_dirty(bh); sbi->s_lvid_dirty = 0; } mutex_unlock(&sbi->s_alloc_mutex); diff --git a/fs/udf/symlink.c b/fs/udf/symlink.c index fe03745d09b18..a05d1888a2bab 100644 --- a/fs/udf/symlink.c +++ b/fs/udf/symlink.c @@ -36,6 +36,8 @@ static int udf_pc_to_char(struct super_block *sb, unsigned char *from, /* Reserve one byte for terminating \0 */ tolen--; while (elen < fromlen) { + if (fromlen - elen < sizeof(struct pathComponent)) + return -EIO; pc = (struct pathComponent *)(from + elen); elen += sizeof(struct pathComponent); switch (pc->componentType) { diff --git a/fs/ufs/cylinder.c b/fs/ufs/cylinder.c index a2813270c303e..b930ee1cf8530 100644 --- a/fs/ufs/cylinder.c +++ b/fs/ufs/cylinder.c @@ -68,6 +68,16 @@ static bool ufs_read_cylinder(struct super_block *sb, ucpi->c_clustersumoff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clustersumoff); ucpi->c_clusteroff = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_clusteroff); ucpi->c_nclusterblks = fs32_to_cpu(sb, ucg->cg_u.cg_44.cg_nclusterblks); + + /* these on-disk values become array and bitmap indices */ + if (ucpi->c_cgx != cgno || + ucpi->c_rotor >= uspi->s_fpg || + ucpi->c_frotor >= uspi->s_fpg || + ucpi->c_irotor >= uspi->s_ipg) { + ufs_error(sb, __func__, + "inconsistent metadata in cylinder group %u\n", cgno); + goto failed; + } UFSD("EXIT\n"); return true; diff --git a/fs/ufs/dir.c b/fs/ufs/dir.c index 0388a1bae326b..ec58edaf57c9e 100644 --- a/fs/ufs/dir.c +++ b/fs/ufs/dir.c @@ -589,7 +589,7 @@ int ufs_empty_dir(struct inode * inode) kaddr = ufs_get_folio(inode, i, &folio); if (IS_ERR(kaddr)) - continue; + return 0; de = (struct ufs_dir_entry *)kaddr; kaddr += ufs_last_byte(inode, i) - UFS_DIR_REC_LEN(1); diff --git a/fs/ufs/super.c b/fs/ufs/super.c index 6e4585169f94b..4fc7418567867 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -1200,6 +1200,15 @@ static int ufs_fill_super(struct super_block *sb, struct fs_context *fc) sb->s_maxbytes = ufs_max_bytes(sb); sb->s_max_links = UFS_LINK_MAX; + ufs_setup_cstotal(sb); + /* + * Read cylinder group structures + */ + if (!sb_rdonly(sb)) + if (!ufs_read_cylinder_structures(sb)) + goto failed; + + /* create the root dentry last, once UFS_SB(sb) is fully set up */ inode = ufs_iget(sb, UFS_ROOTINO); if (IS_ERR(inode)) { ret = PTR_ERR(inode); @@ -1211,14 +1220,6 @@ static int ufs_fill_super(struct super_block *sb, struct fs_context *fc) goto failed; } - ufs_setup_cstotal(sb); - /* - * Read cylinder group structures - */ - if (!sb_rdonly(sb)) - if (!ufs_read_cylinder_structures(sb)) - goto failed; - UFSD("EXIT\n"); return 0; diff --git a/fs/verity/measure.c b/fs/verity/measure.c index 388734132f018..433d963edf3bb 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -122,11 +122,11 @@ __bpf_kfunc int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *di { struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; const struct inode *inode = file_inode(file); - u32 dynptr_sz = __bpf_dynptr_size(digest_ptr); + u64 dynptr_sz = __bpf_dynptr_size(digest_ptr); struct fsverity_digest *arg; const struct fsverity_info *vi; const struct fsverity_hash_alg *hash_alg; - int out_digest_sz; + u64 out_digest_sz; if (dynptr_sz < sizeof(struct fsverity_digest)) return -EINVAL; @@ -144,17 +144,20 @@ __bpf_kfunc int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *di hash_alg = vi->tree_params.hash_alg; + out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); + if (out_digest_sz < hash_alg->digest_size) + return -EOVERFLOW; + arg->digest_algorithm = hash_alg - fsverity_hash_algs; arg->digest_size = hash_alg->digest_size; - out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); - /* copy digest */ - memcpy(arg->digest, vi->file_digest, min_t(int, hash_alg->digest_size, out_digest_sz)); + memcpy(arg->digest, vi->file_digest, hash_alg->digest_size); /* fill the extra buffer with zeros */ if (out_digest_sz > hash_alg->digest_size) - memset(arg->digest + arg->digest_size, 0, out_digest_sz - hash_alg->digest_size); + memset(arg->digest + hash_alg->digest_size, 0, + out_digest_sz - hash_alg->digest_size); return 0; } diff --git a/fs/xfs/libxfs/xfs_attr_leaf.c b/fs/xfs/libxfs/xfs_attr_leaf.c index b858e3c2ad50a..14dddee4f8fa6 100644 --- a/fs/xfs/libxfs/xfs_attr_leaf.c +++ b/fs/xfs/libxfs/xfs_attr_leaf.c @@ -268,6 +268,13 @@ xfs_attr3_leaf_verify_entry( */ if (ent->flags & XFS_ATTR_LOCAL) { lentry = xfs_attr3_leaf_name_local(leaf, idx); + + /* Validate lentry pointer is within bounds before field access */ + if ((char *)lentry >= buf_end) + return __this_address; + if ((char *)lentry + offsetof(struct xfs_attr_leaf_name_local, nameval) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_local(lentry->namelen, be16_to_cpu(lentry->valuelen)); name_end = (char *)lentry + namesize; @@ -275,6 +282,13 @@ xfs_attr3_leaf_verify_entry( return __this_address; } else { rentry = xfs_attr3_leaf_name_remote(leaf, idx); + + /* Validate rentry pointer is within bounds before field access */ + if ((char *)rentry >= buf_end) + return __this_address; + if ((char *)rentry + offsetof(struct xfs_attr_leaf_name_remote, name) > buf_end) + return __this_address; + namesize = xfs_attr_leaf_entsize_remote(rentry->namelen); name_end = (char *)rentry + namesize; if (rentry->namelen == 0) diff --git a/fs/xfs/libxfs/xfs_btree_mem.c b/fs/xfs/libxfs/xfs_btree_mem.c index f2f7b4305413e..242c09b827fc4 100644 --- a/fs/xfs/libxfs/xfs_btree_mem.c +++ b/fs/xfs/libxfs/xfs_btree_mem.c @@ -117,6 +117,7 @@ xfbtree_init( struct xfs_buftarg *btp, const struct xfs_btree_ops *ops) { + unsigned long long owner = xfbt->owner; unsigned int blocklen = xfbtree_rec_bytes(mp, ops); unsigned int keyptr_len; int error; @@ -133,6 +134,7 @@ xfbtree_init( memset(xfbt, 0, sizeof(*xfbt)); xfbt->target = btp; + xfbt->owner = owner; /* Set up min/maxrecs for this btree. */ keyptr_len = ops->key_len + sizeof(__be64); diff --git a/fs/xfs/libxfs/xfs_btree_staging.c b/fs/xfs/libxfs/xfs_btree_staging.c index 5ed84f9cc877e..a8e90cb2cdc21 100644 --- a/fs/xfs/libxfs/xfs_btree_staging.c +++ b/fs/xfs/libxfs/xfs_btree_staging.c @@ -337,8 +337,10 @@ xfs_btree_bload_prep_block( xfs_btree_set_sibling(cur, *blockp, &new_ptr, XFS_BB_RIGHTSIB); ret = xfs_btree_bload_drop_buf(bbl, buffers_list, bpp); - if (ret) + if (ret) { + xfs_buf_relse(new_bp); return ret; + } /* Initialize the new btree block. */ xfs_btree_init_block_cur(cur, new_bp, level, nr_this_block); diff --git a/fs/xfs/libxfs/xfs_defer.c b/fs/xfs/libxfs/xfs_defer.c index 43f352d89172f..ed2443abb47d6 100644 --- a/fs/xfs/libxfs/xfs_defer.c +++ b/fs/xfs/libxfs/xfs_defer.c @@ -583,7 +583,7 @@ xfs_defer_finish_one( const struct xfs_defer_op_type *ops = dfp->dfp_ops; struct xfs_btree_cur *state = NULL; struct list_head *li, *n; - int error; + int error = 0; trace_xfs_defer_pending_finish(tp->t_mountp, dfp); diff --git a/fs/xfs/libxfs/xfs_exchmaps.c b/fs/xfs/libxfs/xfs_exchmaps.c index f021fe3c9cb64..0cabbef65547e 100644 --- a/fs/xfs/libxfs/xfs_exchmaps.c +++ b/fs/xfs/libxfs/xfs_exchmaps.c @@ -959,6 +959,26 @@ xmi_can_exchange_reflink_flags( { struct xfs_mount *mp = req->ip1->i_mount; + /* + * The INO1_WRITTEN optimization can skip exchanging hole and + * unwritten mappings, which means we cannot guarantee that all + * shared extents actually moved to the other file. Clearing the + * reflink flag of an inode that still holds shared extents breaks + * the CoW write path, so refuse to exchange the flags in that case. + */ + if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) + return false; + + /* + * The INO1_WRITTEN optimization can skip exchanging hole and + * unwritten mappings, which means we cannot guarantee that all + * shared extents actually moved to the other file. Clearing the + * reflink flag of an inode that still holds shared extents breaks + * the CoW write path, so refuse to exchange the flags in that case. + */ + if (req->flags & XFS_EXCHMAPS_INO1_WRITTEN) + return false; + if (hweight32(reflink_state) != 1) return false; if (req->startoff1 != 0 || req->startoff2 != 0) diff --git a/fs/xfs/libxfs/xfs_parent.c b/fs/xfs/libxfs/xfs_parent.c index 69366c44a7015..f47dda640c0ef 100644 --- a/fs/xfs/libxfs/xfs_parent.c +++ b/fs/xfs/libxfs/xfs_parent.c @@ -193,7 +193,7 @@ xfs_parent_addname( const struct xfs_name *parent_name, struct xfs_inode *child) { - int error; + int error, local; error = xfs_parent_iread_extents(tp, child); if (error) @@ -202,6 +202,11 @@ xfs_parent_addname( xfs_inode_to_parent_rec(&ppargs->rec, dp); xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, child->i_ino, parent_name); + + /* Growing the attr fork needs a real reservation in args->total. */ + ppargs->args.total = xfs_attr_calc_size(&ppargs->args, &local); + ASSERT(local); + xfs_attr_defer_add(&ppargs->args, XFS_ATTR_DEFER_SET); return 0; } @@ -239,7 +244,7 @@ xfs_parent_replacename( const struct xfs_name *new_name, struct xfs_inode *child) { - int error; + int error, local; error = xfs_parent_iread_extents(tp, child); if (error) @@ -249,6 +254,10 @@ xfs_parent_replacename( xfs_parent_da_args_init(&ppargs->args, tp, &ppargs->rec, child, child->i_ino, old_name); + /* Growing the attr fork needs a real reservation in args->total. */ + ppargs->args.total = xfs_attr_calc_size(&ppargs->args, &local); + ASSERT(local); + xfs_inode_to_parent_rec(&ppargs->new_rec, new_dp); ppargs->args.new_name = new_name->name; ppargs->args.new_namelen = new_name->len; diff --git a/fs/xfs/libxfs/xfs_rtrefcount_btree.c b/fs/xfs/libxfs/xfs_rtrefcount_btree.c index ac11e94b42ae3..771ababe7edf0 100644 --- a/fs/xfs/libxfs/xfs_rtrefcount_btree.c +++ b/fs/xfs/libxfs/xfs_rtrefcount_btree.c @@ -201,7 +201,7 @@ xfs_rtrefcountbt_verify( if (fa) return fa; level = be16_to_cpu(block->bb_level); - if (level > mp->m_rtrefc_maxlevels) + if (level >= mp->m_rtrefc_maxlevels) return __this_address; return xfs_btree_fsblock_verify(bp, mp->m_rtrefc_mxr[level != 0]); @@ -489,8 +489,11 @@ xfs_rtrefcountbt_maxlevels_ondisk(void) minrecs[0] = xfs_rtrefcountbt_block_maxrecs(blocklen, true) / 2; minrecs[1] = xfs_rtrefcountbt_block_maxrecs(blocklen, false) / 2; - /* We need at most one record for every block in an rt group. */ - return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_RGBLOCKS); + /* + * We need at most one record for every block in an rt group, and + * one extra level for the inode root. + */ + return xfs_btree_compute_maxlevels(minrecs, XFS_MAX_RGBLOCKS) + 1; } int __init @@ -651,7 +654,7 @@ xfs_iformat_rtrefcount( numrecs = be16_to_cpu(dfp->bb_numrecs); level = be16_to_cpu(dfp->bb_level); - if (level > mp->m_rtrefc_maxlevels || + if (level >= mp->m_rtrefc_maxlevels || xfs_rtrefcount_droot_space_calc(level, numrecs) > dsize) { xfs_inode_mark_sick(ip, XFS_SICK_INO_CORE); return -EFSCORRUPTED; diff --git a/fs/xfs/libxfs/xfs_rtrmap_btree.c b/fs/xfs/libxfs/xfs_rtrmap_btree.c index 55f9031657695..d017bf22f48f7 100644 --- a/fs/xfs/libxfs/xfs_rtrmap_btree.c +++ b/fs/xfs/libxfs/xfs_rtrmap_btree.c @@ -618,7 +618,7 @@ xfs_rtrmapbt_mem_cursor( struct xfs_btree_cur *cur; cur = xfs_btree_alloc_cursor(mp, tp, &xfs_rtrmapbt_mem_ops, - mp->m_rtrmap_maxlevels, xfs_rtrmapbt_cur_cache); + xfs_rtrmapbt_maxlevels_ondisk(), xfs_rtrmapbt_cur_cache); cur->bc_mem.xfbtree = xfbt; cur->bc_nlevels = xfbt->nlevels; cur->bc_group = xfs_group_hold(rtg_group(rtg)); @@ -716,10 +716,12 @@ xfs_rtrmapbt_maxlevels_ondisk(void) * happens, which means that we must compute the max height based on * what the btree will look like if it consumes almost all the blocks * in the data device due to maximal sharing factor. + * + * Add one extra level for the inode root. */ max_dblocks = -1U; /* max ag count */ max_dblocks *= XFS_MAX_CRC_AG_BLOCKS; - return xfs_btree_space_to_height(minrecs, max_dblocks); + return xfs_btree_space_to_height(minrecs, max_dblocks) + 1; } int __init diff --git a/fs/xfs/libxfs/xfs_sb.c b/fs/xfs/libxfs/xfs_sb.c index 94c272a2ae262..823ff59ba7118 100644 --- a/fs/xfs/libxfs/xfs_sb.c +++ b/fs/xfs/libxfs/xfs_sb.c @@ -1118,10 +1118,10 @@ xfs_sb_read_verify( * because _verify_common checks the on-disk values. */ __xfs_sb_from_disk(&sb, dsb, false); - error = xfs_validate_sb_common(mp, bp, &sb); + error = xfs_validate_sb_read(mp, &sb); if (error) goto out_error; - error = xfs_validate_sb_read(mp, &sb); + error = xfs_validate_sb_common(mp, bp, &sb); out_error: if (error == -EFSCORRUPTED || error == -EFSBADCRC) diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index 303374df44bd2..5b8d66b20159e 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -18,6 +18,8 @@ #include "xfs_inode.h" #include "scrub/scrub.h" #include "scrub/common.h" +#include "scrub/bitmap.h" +#include "scrub/agino_bitmap.h" int xchk_setup_agheader( @@ -266,7 +268,7 @@ xchk_superblock( xchk_block_set_corrupt(sc, bp); if (sb->sb_gquotino != cpu_to_be64(0)) - xchk_block_set_preen(sc, bp); + xchk_block_set_corrupt(sc, bp); } else { if (sb->sb_uquotino != cpu_to_be64(mp->m_sb.sb_uquotino)) xchk_block_set_preen(sc, bp); @@ -416,6 +418,13 @@ xchk_superblock( xchk_block_set_corrupt(sc, bp); } + if (xfs_has_zoned(mp)) { + if (sb->sb_rtstart != cpu_to_be64(mp->m_sb.sb_rtstart)) + xchk_block_set_corrupt(sc, bp); + if (sb->sb_rtreserved != cpu_to_be64(mp->m_sb.sb_rtreserved)) + xchk_block_set_corrupt(sc, bp); + } + /* Everything else must be zero. */ sblen = xchk_superblock_ondisk_size(mp); if (memchr_inv((char *)sb + sblen, 0, BBTOB(bp->b_length) - sblen)) @@ -932,41 +941,85 @@ xchk_agi_xref( /* scrub teardown will take care of sc->sa for us */ } +/* + * Walk the incore unlinked list for a particular AGI bucket to construct + * the unlinked inode bitmap for later reconstruction of the unlinked list. + * Returns 1 if we should keep checking, 0 to stop checking, or a negative + * errno. + */ +static int +xchk_iunlink_bucket( + struct xfs_scrub *sc, + unsigned int bucket, + xfs_agino_t agino) +{ + struct xagino_bitmap seen; + int ret; + + xagino_bitmap_init(&seen); + + while (agino != NULLAGINO) { + struct xfs_inode *ip; + unsigned int len = 1; + + if (agino % XFS_AGI_UNLINKED_BUCKETS != bucket) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + if (xagino_bitmap_test(&seen, agino, &len)) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + ip = xfs_iunlink_lookup(sc->sa.pag, agino); + if (!ip) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + if (!xfs_inode_on_unlinked_list(ip)) { + xchk_block_set_corrupt(sc, sc->sa.agi_bp); + goto bad; + } + + ret = xagino_bitmap_set(&seen, agino, 1); + if (ret) + goto out_bitmap; + + agino = ip->i_next_unlinked; + } + ret = 1; + +out_bitmap: + xagino_bitmap_destroy(&seen); + return ret; +bad: + ret = 0; + goto out_bitmap; +} + /* * Check the unlinked buckets for links to bad inodes. We hold the AGI, so * there cannot be any threads updating unlinked list pointers in this AG. */ -STATIC void +STATIC int xchk_iunlink( struct xfs_scrub *sc, struct xfs_agi *agi) { unsigned int i; - struct xfs_inode *ip; for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { - xfs_agino_t agino = be32_to_cpu(agi->agi_unlinked[i]); - - while (agino != NULLAGINO) { - if (agino % XFS_AGI_UNLINKED_BUCKETS != i) { - xchk_block_set_corrupt(sc, sc->sa.agi_bp); - return; - } - - ip = xfs_iunlink_lookup(sc->sa.pag, agino); - if (!ip) { - xchk_block_set_corrupt(sc, sc->sa.agi_bp); - return; - } - - if (!xfs_inode_on_unlinked_list(ip)) { - xchk_block_set_corrupt(sc, sc->sa.agi_bp); - return; - } - - agino = ip->i_next_unlinked; - } + int ret; + + ret = xchk_iunlink_bucket(sc, i, + be32_to_cpu(agi->agi_unlinked[i])); + if (ret < 1) + return ret; } + + return 0; } /* Scrub the AGI. */ @@ -1053,7 +1106,9 @@ xchk_agi( if (pag->pagi_freecount != be32_to_cpu(agi->agi_freecount)) xchk_block_set_corrupt(sc, sc->sa.agi_bp); - xchk_iunlink(sc, agi); + error = xchk_iunlink(sc, agi); + if (error) + goto out; xchk_agi_xref(sc); out: diff --git a/fs/xfs/scrub/agheader_repair.c b/fs/xfs/scrub/agheader_repair.c index 6e3fef36d6614..925b761e56fbb 100644 --- a/fs/xfs/scrub/agheader_repair.c +++ b/fs/xfs/scrub/agheader_repair.c @@ -668,14 +668,16 @@ xrep_agfl_init_header( struct xfs_scrub *sc, struct xfs_buf *agfl_bp, struct xagb_bitmap *agfl_extents, - xfs_agblock_t flcount) + xfs_agblock_t flcount, + struct xfs_agfl *old_agfl) { struct xrep_agfl_fill af = { .sc = sc, .flcount = flcount, }; struct xfs_mount *mp = sc->mp; - struct xfs_agfl *agfl; + struct xfs_agfl *agfl = XFS_BUF_TO_AGFL(agfl_bp); + const size_t agfl_sz = BBTOB(agfl_bp->b_length); int error; ASSERT(flcount <= xfs_agfl_size(mp)); @@ -684,8 +686,8 @@ xrep_agfl_init_header( * Start rewriting the header by setting the bno[] array to * NULLAGBLOCK, then setting AGFL header fields. */ - agfl = XFS_BUF_TO_AGFL(agfl_bp); - memset(agfl, 0xFF, BBTOB(agfl_bp->b_length)); + memcpy(old_agfl, agfl, agfl_sz); + memset(agfl, 0xFF, agfl_sz); agfl->agfl_magicnum = cpu_to_be32(XFS_AGFL_MAGIC); agfl->agfl_seqno = cpu_to_be32(pag_agno(sc->sa.pag)); uuid_copy(&agfl->agfl_uuid, &mp->m_sb.sb_meta_uuid); @@ -697,16 +699,23 @@ xrep_agfl_init_header( */ xagb_bitmap_init(&af.used_extents); af.agfl_bno = xfs_buf_to_agfl_bno(agfl_bp); - xagb_bitmap_walk(agfl_extents, xrep_agfl_fill, &af); + error = xagb_bitmap_walk(agfl_extents, xrep_agfl_fill, &af); + if (error && error != -ECANCELED) + goto err_undo; error = xagb_bitmap_disunion(agfl_extents, &af.used_extents); if (error) - return error; + goto err_undo; /* Write new AGFL to disk. */ xfs_trans_buf_set_type(sc->tp, agfl_bp, XFS_BLFT_AGFL_BUF); - xfs_trans_log_buf(sc->tp, agfl_bp, 0, BBTOB(agfl_bp->b_length) - 1); + xfs_trans_log_buf(sc->tp, agfl_bp, 0, agfl_sz - 1); xagb_bitmap_destroy(&af.used_extents); return 0; + +err_undo: + xagb_bitmap_destroy(&af.used_extents); + memcpy(agfl, old_agfl, agfl_sz); + return error; } /* Repair the AGFL. */ @@ -718,6 +727,7 @@ xrep_agfl( struct xfs_mount *mp = sc->mp; struct xfs_buf *agf_bp; struct xfs_buf *agfl_bp; + struct xfs_agfl *old_agfl; xfs_agblock_t flcount; int error; @@ -725,6 +735,10 @@ xrep_agfl( if (!xfs_has_rmapbt(mp)) return -EOPNOTSUPP; + old_agfl = kzalloc(BBTOB(XFS_FSS_TO_BB(mp, 1)), XCHK_GFP_FLAGS); + if (!old_agfl) + return -ENOMEM; + xagb_bitmap_init(&agfl_extents); /* @@ -734,7 +748,7 @@ xrep_agfl( */ error = xfs_alloc_read_agf(sc->sa.pag, sc->tp, 0, &agf_bp); if (error) - return error; + goto err_old_agfl; /* * Make sure we have the AGFL buffer, as scrub might have decided it @@ -745,7 +759,7 @@ xrep_agfl( XFS_AGFL_DADDR(mp)), XFS_FSS_TO_BB(mp, 1), 0, &agfl_bp, NULL); if (error) - return error; + goto err_old_agfl; agfl_bp->b_ops = &xfs_agfl_buf_ops; /* Gather all the extents we're going to put on the new AGFL. */ @@ -762,10 +776,11 @@ xrep_agfl( * we adjust the AGF flcount (which can fail) so avoid updating any * buffers until we know that part works. */ - xrep_agfl_update_agf(sc, agf_bp, flcount); - error = xrep_agfl_init_header(sc, agfl_bp, &agfl_extents, flcount); + error = xrep_agfl_init_header(sc, agfl_bp, &agfl_extents, flcount, + old_agfl); if (error) goto err; + xrep_agfl_update_agf(sc, agf_bp, flcount); /* * Ok, the AGFL should be ready to go now. Roll the transaction to @@ -785,6 +800,8 @@ xrep_agfl( err: xagb_bitmap_destroy(&agfl_extents); +err_old_agfl: + kfree(old_agfl); return error; } @@ -979,6 +996,13 @@ xrep_agi_calc_from_btrees( return error; } +/* + * Magic value that means "not unlinked" because xfarrays don't support storing + * totally zeroed elements. There can't be a cluster that starts in daddr 0 so + * there can't be an inode #1 either. + */ +#define LINKED_AGINO (0x1) + /* * Record a forwards unlinked chain pointer from agino -> next_agino in our * staging information. @@ -1034,31 +1058,40 @@ xrep_iunlink_next( * the chain or if we should stop walking the chain due to corruption; or a * per-AG inode number. */ -STATIC xfs_agino_t +STATIC int xrep_iunlink_reload_next( struct xrep_agi *ragi, xfs_agino_t prev_agino, - xfs_agino_t agino) + xfs_agino_t agino, + xfs_agino_t *next_agino) { struct xfs_scrub *sc = ragi->sc; struct xfs_inode *ip; - xfs_agino_t ret = NULLAGINO; int error; + *next_agino = NULLAGINO; + error = xchk_iget(ragi->sc, xfs_agino_to_ino(sc->sa.pag, agino), &ip); if (error) - return ret; + return 0; trace_xrep_iunlink_reload_next(ip, prev_agino); /* If this is a linked inode, stop processing the chain. */ if (VFS_I(ip)->i_nlink != 0) { - xrep_iunlink_store_next(ragi, agino, NULLAGINO); + error = xrep_iunlink_store_next(ragi, agino, NULLAGINO); + if (error) + return error; + + error = xrep_iunlink_store_prev(ragi, agino, LINKED_AGINO); + if (error) + return error; + goto rele; } ip->i_prev_unlinked = prev_agino; - ret = ip->i_next_unlinked; + *next_agino = ip->i_next_unlinked; /* * Drop the inode reference that we just took. We hold the AGI, so @@ -1067,7 +1100,7 @@ xrep_iunlink_reload_next( */ rele: xchk_irele(sc, ip); - return ret; + return 0; } /* @@ -1080,18 +1113,22 @@ xrep_iunlink_walk_ondisk_bucket( struct xrep_agi *ragi, unsigned int bucket) { + struct xagino_bitmap seen; struct xfs_scrub *sc = ragi->sc; - struct xfs_agi *agi = sc->sa.agi_bp->b_addr; + struct xfs_agi *agi = ragi->agi_bp->b_addr; xfs_agino_t prev_agino = NULLAGINO; xfs_agino_t next_agino; int error = 0; + xagino_bitmap_init(&seen); + next_agino = be32_to_cpu(agi->agi_unlinked[bucket]); while (next_agino != NULLAGINO) { xfs_agino_t agino = next_agino; + unsigned int len = 1; if (xchk_should_terminate(ragi->sc, &error)) - return error; + goto out_bitmap; trace_xrep_iunlink_walk_ondisk_bucket(sc->sa.pag, bucket, prev_agino, agino); @@ -1099,15 +1136,27 @@ xrep_iunlink_walk_ondisk_bucket( if (bucket != agino % XFS_AGI_UNLINKED_BUCKETS) break; + if (xagino_bitmap_test(&seen, agino, &len)) + break; + next_agino = xrep_iunlink_next(sc, agino); - if (!next_agino) - next_agino = xrep_iunlink_reload_next(ragi, prev_agino, - agino); + if (!next_agino) { + error = xrep_iunlink_reload_next(ragi, prev_agino, + agino, &next_agino); + if (error) + break; + } + + error = xagino_bitmap_set(&seen, agino, 1); + if (error) + goto out_bitmap; prev_agino = agino; } - return 0; +out_bitmap: + xagino_bitmap_destroy(&seen); + return error; } /* Decide if this is an unlinked inode in this AG. */ @@ -1302,7 +1351,7 @@ xrep_iunlink_mark_ondisk_rec( * iunlink_bmp. We haven't checked the inobt yet, so we don't error out if * the btree is corrupt. */ -STATIC void +STATIC int xrep_iunlink_mark_ondisk( struct xrep_agi *ragi) { @@ -1314,6 +1363,14 @@ xrep_iunlink_mark_ondisk( cur = xfs_inobt_init_cursor(sc->sa.pag, sc->tp, agi_bp); error = xfs_btree_query_all(cur, xrep_iunlink_mark_ondisk_rec, ragi); xfs_btree_del_cursor(cur, error); + + /* + * Don't proceed if we couldn't set a bit in the bitmap. All other + * errors we ignore because we haven't actually checked the inobt yet. + */ + if (error == -ENOMEM) + return -ENOMEM; + return 0; } /* @@ -1326,15 +1383,32 @@ xrep_iunlink_resolve_bucket( struct xrep_agi *ragi, unsigned int bucket) { + struct xagino_bitmap seen; struct xfs_scrub *sc = ragi->sc; struct xfs_inode *ip; xfs_agino_t prev_agino = NULLAGINO; xfs_agino_t next_agino = ragi->iunlink_heads[bucket]; int error = 0; + xagino_bitmap_init(&seen); + while (next_agino != NULLAGINO) { + unsigned int len = 1; + if (xchk_should_terminate(ragi->sc, &error)) - return error; + goto out_bitmap; + + /* Inode already seen? We're stuck in a loop */ + if (xagino_bitmap_test(&seen, next_agino, &len)) { + trace_xrep_iunlink_resolve_infinite_loop(sc->sa.pag, + bucket, prev_agino, next_agino); + next_agino = NULLAGINO; + break; + } + + error = xagino_bitmap_set(&seen, next_agino, 1); + if (error) + goto out_bitmap; /* Find the next inode in the chain. */ ip = xfs_iunlink_lookup(sc->sa.pag, next_agino); @@ -1347,6 +1421,35 @@ xrep_iunlink_resolve_bucket( break; } + if (VFS_I(ip)->i_nlink != 0) { + /* + * Inode is linked somewhere! Blow out both unlinked + * list pointers, advance the list, and pretend we + * didn't see this inode. Clear it from iunlink_bmp + * because it's linked. + */ + trace_xrep_iunlink_resolve_allocated(sc->sa.pag, + bucket, prev_agino, next_agino); + + error = xrep_iunlink_store_next(ragi, next_agino, + NULLAGINO); + if (error) + goto out_bitmap; + + error = xrep_iunlink_store_prev(ragi, next_agino, + LINKED_AGINO); + if (error) + goto out_bitmap; + + error = xagino_bitmap_clear(&ragi->iunlink_bmp, + next_agino, 1); + if (error) + goto out_bitmap; + + next_agino = ip->i_next_unlinked; + continue; + } + if (next_agino % XFS_AGI_UNLINKED_BUCKETS != bucket) { /* * Inode is in the wrong bucket. Advance the list, @@ -1382,20 +1485,20 @@ xrep_iunlink_resolve_bucket( */ error = xagino_bitmap_clear(&ragi->iunlink_bmp, next_agino, 1); if (error) - return error; + goto out_bitmap; /* Remember the previous inode's next pointer. */ if (prev_agino != NULLAGINO) { error = xrep_iunlink_store_next(ragi, prev_agino, next_agino); if (error) - return error; + goto out_bitmap; } /* Remember this inode's previous pointer. */ error = xrep_iunlink_store_prev(ragi, next_agino, prev_agino); if (error) - return error; + goto out_bitmap; /* Advance the list and remember this inode. */ prev_agino = next_agino; @@ -1406,10 +1509,12 @@ xrep_iunlink_resolve_bucket( if (prev_agino != NULLAGINO) { error = xrep_iunlink_store_next(ragi, prev_agino, next_agino); if (error) - return error; + goto out_bitmap; } - return 0; +out_bitmap: + xagino_bitmap_destroy(&seen); + return error; } /* Reinsert this unlinked inode into the head of the staged bucket list. */ @@ -1434,6 +1539,10 @@ xrep_iunlink_add_to_bucket( if (error) return error; + error = xrep_iunlink_store_prev(ragi, agino, NULLAGINO); + if (error) + return error; + /* Remember the head inode's previous pointer. */ if (current_head != NULLAGINO) { error = xrep_iunlink_store_prev(ragi, current_head, agino); @@ -1501,7 +1610,9 @@ xrep_iunlink_rebuild_buckets( * If there are ondisk inodes that are unlinked and are not been loaded * into cache, record them in iunlink_bmp. */ - xrep_iunlink_mark_ondisk(ragi); + error = xrep_iunlink_mark_ondisk(ragi); + if (error) + return error; /* * Walk each iunlink bucket to (re)construct as much of the incore list @@ -1523,6 +1634,24 @@ xrep_iunlink_rebuild_buckets( xrep_iunlink_add_lost_inodes, ragi); } +static inline void +set_inode_prev_unlinked( + struct xfs_inode *ip, + xfs_agino_t prev_agino) +{ + /* + * Magic value that means "not unlinked" because xfarrays don't support + * storing totally zeroed elements. + */ + if (prev_agino == LINKED_AGINO) + prev_agino = 0; + + if (ip->i_prev_unlinked != prev_agino) { + trace_xrep_iunlink_relink_prev(ip, prev_agino); + ip->i_prev_unlinked = prev_agino; + } +} + /* Update i_next_iunlinked for the inode @agino. */ STATIC int xrep_iunlink_relink_next( @@ -1556,8 +1685,7 @@ xrep_iunlink_relink_next( if (error) goto out_rele; - trace_xrep_iunlink_relink_prev(ip, prev_agino); - ip->i_prev_unlinked = prev_agino; + set_inode_prev_unlinked(ip, prev_agino); } /* Update the forward pointer. */ @@ -1612,7 +1740,7 @@ xrep_iunlink_relink_prev( want_rele = true; /* Set the forward pointer since this just came off disk. */ - error = xfarray_load(ragi->iunlink_prev, agino, &next_agino); + error = xfarray_load(ragi->iunlink_next, agino, &next_agino); if (error) goto out_rele; @@ -1624,11 +1752,7 @@ xrep_iunlink_relink_prev( ip->i_next_unlinked = next_agino; } - /* Update the backward pointer. */ - if (ip->i_prev_unlinked != prev_agino) { - trace_xrep_iunlink_relink_prev(ip, prev_agino); - ip->i_prev_unlinked = prev_agino; - } + set_inode_prev_unlinked(ip, prev_agino); out_rele: /* @@ -1658,6 +1782,8 @@ xrep_iunlink_commit( if (error) return error; } + if (error < 0) + return error; /* Fix all the back links */ idx = XFARRAY_CURSOR_INIT; @@ -1666,6 +1792,8 @@ xrep_iunlink_commit( if (error) return error; } + if (error < 0) + return error; /* Copy the staged iunlink buckets to the new AGI. */ for (i = 0; i < XFS_AGI_UNLINKED_BUCKETS; i++) { diff --git a/fs/xfs/scrub/alloc.c b/fs/xfs/scrub/alloc.c index 8b282138097fb..c77934f5b7240 100644 --- a/fs/xfs/scrub/alloc.c +++ b/fs/xfs/scrub/alloc.c @@ -136,7 +136,7 @@ xchk_allocbt_rec( const union xfs_btree_rec *rec) { struct xfs_alloc_rec_incore irec; - struct xchk_alloc *ca = bs->private; + struct xchk_alloc *ca = bs->private; xfs_alloc_btrec_to_irec(rec, &irec); if (xfs_alloc_check_irec(to_perag(bs->cur->bc_group), &irec) != NULL) { @@ -144,7 +144,8 @@ xchk_allocbt_rec( return 0; } - xchk_allocbt_mergeable(bs, ca, &irec); + if (bs->sc->sm->sm_type == XFS_SCRUB_TYPE_BNOBT) + xchk_allocbt_mergeable(bs, ca, &irec); xchk_allocbt_xref(bs->sc, &irec); return 0; diff --git a/fs/xfs/scrub/alloc_repair.c b/fs/xfs/scrub/alloc_repair.c index 35035d02a2316..3f4e164bec5a6 100644 --- a/fs/xfs/scrub/alloc_repair.c +++ b/fs/xfs/scrub/alloc_repair.c @@ -571,7 +571,7 @@ xrep_abt_dispose_one( * allocation, and blocks that didn't get used can be freed via the usual * (deferred) means. */ -STATIC void +STATIC int xrep_abt_dispose_reservations( struct xrep_abt *ra, int error) @@ -582,9 +582,13 @@ xrep_abt_dispose_reservations( goto junkit; list_for_each_entry_safe(resv, n, &ra->new_bnobt.resv_list, list) { - error = xrep_abt_dispose_one(ra, resv); - if (error) + int error2 = xrep_abt_dispose_one(ra, resv); + + if (error2) { + if (!error) + error = error2; goto junkit; + } } junkit: @@ -596,6 +600,7 @@ xrep_abt_dispose_reservations( xrep_newbt_cancel(&ra->new_bnobt); xrep_newbt_cancel(&ra->new_cntbt); + return error; } /* Retrieve free space data for bulk load. */ @@ -801,7 +806,9 @@ xrep_abt_build_new_trees( goto err_newbt; /* Dispose of any unused blocks and the accounting information. */ - xrep_abt_dispose_reservations(ra, error); + error = xrep_abt_dispose_reservations(ra, error); + if (error) + return error; return xrep_roll_ag_trans(sc); @@ -812,8 +819,7 @@ xrep_abt_build_new_trees( xfs_btree_del_cursor(cnt_cur, error); xfs_btree_del_cursor(bno_cur, error); err_newbt: - xrep_abt_dispose_reservations(ra, error); - return error; + return xrep_abt_dispose_reservations(ra, error); } /* diff --git a/fs/xfs/scrub/attr_repair.c b/fs/xfs/scrub/attr_repair.c index dd24044c44efd..fa68ecabd983f 100644 --- a/fs/xfs/scrub/attr_repair.c +++ b/fs/xfs/scrub/attr_repair.c @@ -1427,7 +1427,8 @@ xrep_xattr_rebuild_tree( * If we didn't find any attributes to salvage, repair the file by * zapping its attr fork. */ - if (rx->attrs_found == 0) { + if (rx->attrs_found == 0 && + (!xfs_has_parent(sc->mp) || xfarray_length(rx->pptr_recs) == 0)) { xfs_trans_ijoin(sc->tp, sc->ip, 0); error = xrep_xattr_reset_fork(sc); if (error) diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index 4f1e2574660d9..47b5902431d0a 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -274,7 +274,7 @@ xchk_bmap_xref_rmap_cow( unsigned long long rmap_end; uint64_t owner = XFS_RMAP_OWN_COW; - if (!info->sc->sa.rmap_cur || xchk_skip_xref(info->sc->sm)) + if (xchk_skip_xref(info->sc->sm)) return; /* Find the rmap record for this irec. */ @@ -1040,7 +1040,7 @@ xchk_bmap( case XFS_COW_FORK: /* No CoW forks filesystem doesn't support out of place writes */ if (!xfs_has_reflink(mp) && !xfs_has_zoned(mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } break; @@ -1052,7 +1052,7 @@ xchk_bmap( * attr here. */ if (!xfs_has_attr(mp)) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); break; default: ASSERT(whichfork == XFS_DATA_FORK); @@ -1137,7 +1137,7 @@ xchk_bmap_data( int error; if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTD_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1165,11 +1165,16 @@ xchk_bmap_attr( * returning immediately. */ if (xchk_file_looks_zapped(sc, XFS_SICK_INO_BMBTA_ZAPPED)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } error = xchk_bmap(sc, XFS_ATTR_FORK); + /* A repaired, empty attr fork no longer has mappings to check. */ + if (error == -ENOENT && (sc->flags & XREP_ALREADY_FIXED)) { + xchk_mark_healthy_if_clean(sc, XFS_SICK_INO_BMBTA_ZAPPED); + return 0; + } if (error) return error; diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index 975c879c8d7f5..58f2ed19ef5ac 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -1099,7 +1099,7 @@ xchk_install_live_inode( struct xfs_inode *ip) { if (!igrab(VFS_I(ip))) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); return -EFSCORRUPTED; } @@ -1428,13 +1428,13 @@ xchk_metadata_inode_forks( /* Metadata inodes don't live on the rt device. */ if (sc->ip->i_diflags & XFS_DIFLAG_REALTIME) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } /* They should never participate in reflink. */ if (xfs_is_reflink_inode(sc->ip)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -1451,7 +1451,7 @@ xchk_metadata_inode_forks( &error)) return error; if (shared) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } /* @@ -1460,7 +1460,7 @@ xchk_metadata_inode_forks( */ if (xfs_inode_hasattr(sc->ip)) { if (!xfs_has_metadir(sc->mp)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/common.h b/fs/xfs/scrub/common.h index f2ecc68538f0c..ae998501f8aa9 100644 --- a/fs/xfs/scrub/common.h +++ b/fs/xfs/scrub/common.h @@ -30,6 +30,8 @@ void xchk_set_corrupt(struct xfs_scrub *sc); void xchk_block_set_corrupt(struct xfs_scrub *sc, struct xfs_buf *bp); void xchk_ino_set_corrupt(struct xfs_scrub *sc, xfs_ino_t ino); +#define xchk_ip_set_corrupt(_sc, _ip) \ + xchk_ino_set_corrupt((_sc), (_ip)->i_ino) void xchk_fblock_set_corrupt(struct xfs_scrub *sc, int whichfork, xfs_fileoff_t offset); #ifdef CONFIG_XFS_QUOTA diff --git a/fs/xfs/scrub/cow_repair.c b/fs/xfs/scrub/cow_repair.c index fc70fd5b975a3..87627ab8943c1 100644 --- a/fs/xfs/scrub/cow_repair.c +++ b/fs/xfs/scrub/cow_repair.c @@ -80,12 +80,6 @@ struct xrep_cow { unsigned int next_bno; }; -/* CoW staging extent. */ -struct xrep_cow_extent { - xfs_fsblock_t fsbno; - xfs_extlen_t len; -}; - /* * Mark the part of the file range that corresponds to the given physical * space. Caller must ensure that the physical range is within xc->irec. @@ -401,22 +395,21 @@ xrep_cow_find_bad_rt( STATIC int xrep_cow_alloc( struct xfs_scrub *sc, - xfs_extlen_t maxlen, - struct xrep_cow_extent *repl) + struct xfs_bmbt_irec *del) { struct xfs_alloc_arg args = { .tp = sc->tp, .mp = sc->mp, .oinfo = XFS_RMAP_OINFO_SKIP_UPDATE, .minlen = 1, - .maxlen = maxlen, + .maxlen = del->br_blockcount, .prod = 1, .resv = XFS_AG_RESV_NONE, .datatype = XFS_ALLOC_USERDATA, }; int error; - error = xfs_trans_reserve_more(sc->tp, maxlen, 0); + error = xfs_trans_reserve_more(sc->tp, del->br_blockcount, 0); if (error) return error; @@ -429,8 +422,8 @@ xrep_cow_alloc( xfs_refcount_alloc_cow_extent(sc->tp, false, args.fsbno, args.len); - repl->fsbno = args.fsbno; - repl->len = args.len; + del->br_startblock = args.fsbno; + del->br_blockcount = args.len; return 0; } @@ -441,10 +434,12 @@ xrep_cow_alloc( STATIC int xrep_cow_alloc_rt( struct xfs_scrub *sc, - xfs_extlen_t maxlen, - struct xrep_cow_extent *repl) + struct xfs_bmbt_irec *del) { - xfs_rtxlen_t maxrtx = xfs_rtb_to_rtx(sc->mp, maxlen); + xfs_fsblock_t fsbno; + xfs_rtxlen_t maxrtx = + min(U32_MAX, xfs_blen_to_rtbxlen(sc->mp, del->br_blockcount)); + xfs_extlen_t len; int error; error = xfs_trans_reserve_more(sc->tp, 0, maxrtx); @@ -452,11 +447,14 @@ xrep_cow_alloc_rt( return error; error = xfs_rtallocate_rtgs(sc->tp, NULLRTBLOCK, 1, maxrtx, 1, false, - false, &repl->fsbno, &repl->len); + false, &fsbno, &len); if (error) return error; - xfs_refcount_alloc_cow_extent(sc->tp, true, repl->fsbno, repl->len); + xfs_refcount_alloc_cow_extent(sc->tp, true, fsbno, len); + + del->br_startblock = fsbno; + del->br_blockcount = len; return 0; } @@ -470,19 +468,19 @@ static inline int xrep_cow_find_mapping( struct xrep_cow *xc, struct xfs_iext_cursor *icur, - xfs_fileoff_t startoff, - struct xfs_bmbt_irec *got) + xfs_fileoff_t badoff, + xfs_extlen_t badlen, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) { struct xfs_inode *ip = xc->sc->ip; struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); - if (!xfs_iext_lookup_extent(ip, ifp, startoff, icur, got)) + if (!xfs_iext_lookup_extent(ip, ifp, badoff, icur, got)) goto bad; + memcpy(rep, got, sizeof(*rep)); - if (got->br_startoff > startoff) - goto bad; - - if (got->br_blockcount == 0) + if (got->br_startoff > badoff) goto bad; if (isnullstartblock(got->br_startblock)) @@ -491,6 +489,24 @@ xrep_cow_find_mapping( if (xfs_bmap_is_written_extent(got)) goto bad; + if (got->br_startoff < badoff) { + const int64_t delta = badoff - got->br_startoff; + + rep->br_blockcount -= delta; + rep->br_startoff += delta; + rep->br_startblock += delta; + } + + if (got->br_startoff + got->br_blockcount > badoff + badlen) { + const int64_t delta = (got->br_startoff + got->br_blockcount) - + (badoff + badlen); + + rep->br_blockcount -= delta; + } + + if (got->br_blockcount == 0) + goto bad; + return 0; bad: ASSERT(0); @@ -501,46 +517,92 @@ xrep_cow_find_mapping( #define REPLACE_RIGHT_SIDE (1U << 1) /* - * Given a CoW fork mapping @got and a replacement mapping @repl, remap the - * beginning of @got with the space described by @rep. + * Given a CoW fork mapping @got and a replacement mapping @rep, map the space + * described by @rep into the cow fork, pushing aside @got as necessary. @icur + * must point to iext tree leaf containing @got. */ static inline void xrep_cow_replace_mapping( - struct xfs_inode *ip, - struct xfs_iext_cursor *icur, - const struct xfs_bmbt_irec *got, - const struct xrep_cow_extent *repl) + struct xfs_inode *ip, + struct xfs_iext_cursor *icur, + struct xfs_bmbt_irec *got, + struct xfs_bmbt_irec *rep) { - struct xfs_bmbt_irec new = *got; /* struct copy */ + struct xfs_ifork *ifp = xfs_ifork_ptr(ip, XFS_COW_FORK); + xfs_fileoff_t rep_endoff = + rep->br_startoff + rep->br_blockcount; + xfs_fileoff_t got_endoff = + got->br_startoff + got->br_blockcount; + uint32_t state = BMAP_COWFORK; - ASSERT(repl->len > 0); + ASSERT(rep->br_blockcount > 0); ASSERT(!isnullstartblock(got->br_startblock)); + ASSERT(got->br_startoff <= rep->br_startoff); + ASSERT(got_endoff >= rep_endoff); + + trace_xrep_cow_replace_mapping(ip, got, rep); - trace_xrep_cow_replace_mapping(ip, got, repl->fsbno, repl->len); + if (got->br_startoff == rep->br_startoff) + state |= BMAP_LEFT_FILLING; + if (got_endoff == rep_endoff) + state |= BMAP_RIGHT_FILLING; - if (got->br_blockcount == repl->len) { + switch (state & (BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING)) { + case BMAP_LEFT_FILLING | BMAP_RIGHT_FILLING: /* - * The new extent is a complete replacement for the existing - * extent. Update the COW fork record. + * Replacement matches the whole mapping, update the record. */ - new.br_startblock = repl->fsbno; - xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new); - return; - } + xfs_iext_update_extent(ip, state, icur, rep); + break; + case BMAP_LEFT_FILLING: + /* + * Replace the first part of the mapping: Update the cursor + * position with the new mapping, then add a record with the + * tail of the old mapping. + */ + got->br_startoff = rep_endoff; + got->br_blockcount -= rep->br_blockcount; + got->br_startblock += rep->br_blockcount; + + xfs_iext_update_extent(ip, state, icur, rep); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, got, state); + break; + case BMAP_RIGHT_FILLING: + /* + * Replacing the last part of the mapping. Shorten the current + * mapping then add a record with the new mapping. + */ + got->br_blockcount -= rep->br_blockcount; - /* - * The new extent can replace the beginning of the COW fork record. - * Move the left side of @got upwards, then insert the new record. - */ - new.br_startoff += repl->len; - new.br_startblock += repl->len; - new.br_blockcount -= repl->len; - xfs_iext_update_extent(ip, BMAP_COWFORK, icur, &new); - - new.br_startoff = got->br_startoff; - new.br_startblock = repl->fsbno; - new.br_blockcount = repl->len; - xfs_iext_insert(ip, icur, &new, BMAP_COWFORK); + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + break; + case 0: + /* + * Replacing the middle of the extent. Shorten the current + * mapping, add a new record with the new mapping, and add a + * second new record with the tail of the old mapping. + */ + got->br_blockcount = rep->br_startoff - got->br_startoff; + + struct xfs_bmbt_irec new = { + .br_startoff = rep_endoff, + .br_blockcount = got_endoff - rep_endoff, + .br_state = got->br_state, + .br_startblock = got->br_startblock + + rep->br_blockcount + + got->br_blockcount, + }; + + xfs_iext_update_extent(ip, state, icur, got); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, rep, state); + xfs_iext_next(ifp, icur); + xfs_iext_insert(ip, icur, &new, state); + break; + } } /* @@ -554,33 +616,30 @@ xrep_cow_replace_range( xfs_extlen_t *blockcount) { struct xfs_iext_cursor icur; - struct xrep_cow_extent repl; - struct xfs_bmbt_irec got; + struct xfs_bmbt_irec got, rep; struct xfs_scrub *sc = xc->sc; - xfs_fileoff_t nextoff; - xfs_extlen_t alloc_len; + xfs_fsblock_t old_fsbno; int error; /* - * Put the existing CoW fork mapping in @got. If @got ends before - * @rep, truncate @rep so we only replace one extent mapping at a time. + * Put the existing CoW fork mapping in @got, and put in @rep the + * contents of @got trimmed to @startoff/@blockcount. We only want + * to replace the bad region, and only one mapping at a time. */ - error = xrep_cow_find_mapping(xc, &icur, startoff, &got); + error = xrep_cow_find_mapping(xc, &icur, startoff, *blockcount, &got, + &rep); if (error) return error; - nextoff = min(startoff + *blockcount, - got.br_startoff + got.br_blockcount); + old_fsbno = rep.br_startblock; /* * Allocate a replacement extent. If we don't fill all the blocks, * shorten the quantity that will be deleted in this step. */ - alloc_len = min_t(xfs_fileoff_t, XFS_MAX_BMBT_EXTLEN, - nextoff - startoff); if (XFS_IS_REALTIME_INODE(sc->ip)) - error = xrep_cow_alloc_rt(sc, alloc_len, &repl); + error = xrep_cow_alloc_rt(sc, &rep); else - error = xrep_cow_alloc(sc, alloc_len, &repl); + error = xrep_cow_alloc(sc, &rep); if (error) return error; @@ -588,7 +647,7 @@ xrep_cow_replace_range( * Replace the old mapping with the new one, and commit the metadata * changes made so far. */ - xrep_cow_replace_mapping(sc->ip, &icur, &got, &repl); + xrep_cow_replace_mapping(sc->ip, &icur, &got, &rep); xfs_inode_set_cowblocks_tag(sc->ip); error = xfs_defer_finish(&sc->tp); @@ -597,15 +656,15 @@ xrep_cow_replace_range( /* Note the old CoW staging extents; we'll reap them all later. */ if (XFS_IS_REALTIME_INODE(sc->ip)) - error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, - got.br_startblock, repl.len); + error = xrtb_bitmap_set(&xc->old_cowfork_rtblocks, old_fsbno, + rep.br_blockcount); else - error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, - got.br_startblock, repl.len); + error = xfsb_bitmap_set(&xc->old_cowfork_fsblocks, old_fsbno, + rep.br_blockcount); if (error) return error; - *blockcount = repl.len; + *blockcount = rep.br_blockcount; return 0; } diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c index 4f849d98cbdd2..95ddb991e4787 100644 --- a/fs/xfs/scrub/dir.c +++ b/fs/xfs/scrub/dir.c @@ -1075,7 +1075,7 @@ xchk_directory( /* Plausible size? */ if (sc->ip->i_disk_size < xfs_dir2_sf_hdr_size(0)) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } diff --git a/fs/xfs/scrub/dir_repair.c b/fs/xfs/scrub/dir_repair.c index e0fa5e6ca1fe8..b43ea86db074f 100644 --- a/fs/xfs/scrub/dir_repair.c +++ b/fs/xfs/scrub/dir_repair.c @@ -484,18 +484,24 @@ xrep_dir_recover_data( while (offset < end) { struct xfs_dir2_data_unused *dup = bp->b_addr + offset; struct xfs_dir2_data_entry *dep = bp->b_addr + offset; + unsigned int advance; if (xchk_should_terminate(rd->sc, &error)) return error; /* Skip unused entries. */ if (be16_to_cpu(dup->freetag) == XFS_DIR2_DATA_FREE_TAG) { + if (!dup->length) + break; offset += be16_to_cpu(dup->length); continue; } /* Don't walk off the end of the block. */ - offset += xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + advance = xfs_dir2_data_entsize(rd->sc->mp, dep->namelen); + if (!advance) + break; + offset += advance; if (offset > end) break; @@ -721,7 +727,7 @@ xrep_dir_replay_removename( const struct xfs_name *name, xfs_extlen_t total) { - struct xfs_inode *dp = rd->args.dp; + struct xfs_inode *dp = rd->sc->tempip; ASSERT(S_ISDIR(VFS_I(dp)->i_mode)); diff --git a/fs/xfs/scrub/dirtree.c b/fs/xfs/scrub/dirtree.c index 7301e9cbdab91..2b5c5fa72339a 100644 --- a/fs/xfs/scrub/dirtree.c +++ b/fs/xfs/scrub/dirtree.c @@ -259,6 +259,7 @@ xchk_dirtree_create_path( dl->nr_paths++; return 0; out_path: + xino_bitmap_destroy(&path->seen_inodes); kfree(path); return error; } @@ -368,12 +369,38 @@ xchk_dirpath_step_up( struct xfs_inode *dp; xfs_ino_t parent_ino = be64_to_cpu(dl->pptr_rec.p_ino); unsigned int lock_mode; - int error; + int error = 0; + + if (xchk_should_terminate(sc, &error)) + return error; /* Grab and lock the parent directory. */ error = xchk_iget(sc, parent_ino, &dp); - if (error) + switch (error) { + case -EINVAL: + case -ENOENT: + mutex_lock(&dl->lock); + + if (dl->stale) { + /* live update detected a change in this path */ + error = -ESTALE; + } else { + /* inode doesn't exist, path invalid */ + error = -EFSCORRUPTED; + + trace_xchk_dirpath_badino(dl->sc, path->path_nr, + path->nr_steps, &dl->xname, + &dl->pptr_rec); + } + + mutex_unlock(&dl->lock); + return error; + case 0: + /* keep going */ + break; + default: return error; + } lock_mode = xfs_ilock_attr_map_shared(dp); mutex_lock(&dl->lock); @@ -979,10 +1006,10 @@ xchk_dirtree( xchk_dirtree_evaluate(dl, &oc); if (xchk_dirtree_parentless(dl)) { if (oc.good || oc.bad || oc.suspect) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { if (oc.bad || oc.good + oc.suspect != 1) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); if (oc.suspect) xchk_ino_xref_set_corrupt(sc, sc->ip->i_ino); } diff --git a/fs/xfs/scrub/dirtree_repair.c b/fs/xfs/scrub/dirtree_repair.c index 5c04e70ba9518..219c6b4c6b7a1 100644 --- a/fs/xfs/scrub/dirtree_repair.c +++ b/fs/xfs/scrub/dirtree_repair.c @@ -349,6 +349,8 @@ xrep_dirtree_unlink_iolock( ASSERT(sc->ilock_flags & XFS_IOLOCK_EXCL); + if (sc->ip == dp) + return 0; if (xfs_ilock_nowait(dp, XFS_IOLOCK_EXCL)) return 0; @@ -400,8 +402,18 @@ xrep_dirtree_unlink( * directory code can handle a reservationless update. */ resblks = xfs_remove_space_res(mp, step->name_len); - error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, sc->ip, - &resblks, &sc->tp, &dontcare); + if (sc->ip == dp) { +again: + error = xfs_trans_alloc_inode(dp, &M_RES(mp)->tr_remove, + resblks, 0, false, &sc->tp); + if ((error == -ENOSPC || error == -EDQUOT) && resblks > 0) { + resblks = 0; + goto again; + } + } else { + error = xfs_trans_alloc_dir(dp, &M_RES(mp)->tr_remove, sc->ip, + &resblks, &sc->tp, &dontcare); + } if (error) goto out_iolock; @@ -467,6 +479,7 @@ xrep_dirtree_unlink( } if (xfs_has_parent(sc->mp)) { + memset(&dl->ppargs, 0, sizeof(dl->ppargs)); error = xfs_parent_removename(sc->tp, &dl->ppargs, dp, &dl->xname, sc->ip); if (error) @@ -489,9 +502,11 @@ xrep_dirtree_unlink( xchk_trans_cancel(sc); out_ilock: xfs_iunlock(sc->ip, XFS_ILOCK_EXCL); - xfs_iunlock(dp, XFS_ILOCK_EXCL); + if (dp != sc->ip) + xfs_iunlock(dp, XFS_ILOCK_EXCL); out_iolock: - xfs_iunlock(dp, XFS_IOLOCK_EXCL); + if (dp != sc->ip) + xfs_iunlock(dp, XFS_IOLOCK_EXCL); return error; } @@ -604,6 +619,7 @@ xrep_dirtree_create_adoption_path( return 0; out_path: + xino_bitmap_destroy(&path->seen_inodes); kfree(path); return error; } diff --git a/fs/xfs/scrub/findparent.c b/fs/xfs/scrub/findparent.c index 84487072b6dd6..ccf1d8d70270c 100644 --- a/fs/xfs/scrub/findparent.c +++ b/fs/xfs/scrub/findparent.c @@ -139,32 +139,52 @@ xrep_findparent_dirent( return 0; } -/* - * If this is a directory, walk the dirents looking for any that point to the - * scrub target inode. - */ -STATIC int -xrep_findparent_walk_directory( - struct xrep_findparent_info *fpi) +static inline bool +xrep_findparent_want_scan_file( + const struct xrep_findparent_info *fpi) { - struct xfs_scrub *sc = fpi->sc; - struct xfs_inode *dp = fpi->dp; - unsigned int lock_mode; - int error = 0; + const struct xfs_scrub *sc = fpi->sc; + const struct xfs_inode *dp = fpi->dp; + + /* Only directories can be parents */ + if (!S_ISDIR(VFS_IC(dp)->i_mode)) + return false; /* * The inode being scanned cannot be its own parent, nor can any * temporary directory we created to stage this repair. */ if (dp == sc->ip || dp == sc->tempip) - return 0; + return false; /* * Similarly, temporary files created to stage a repair cannot be the * parent of this inode. */ if (xrep_is_tempfile(dp)) + return false; + + return true; +} + +/* + * If this is a directory, walk the dirents looking for any that point to the + * scrub target inode. + */ +STATIC int +xrep_findparent_walk_file( + struct xrep_findparent_info *fpi) +{ + struct xfs_scrub *sc = fpi->sc; + struct xfs_inode *dp = fpi->dp; + unsigned int lock_mode; + int error = 0; + + if (!xrep_findparent_want_scan_file(fpi)) { + if (fpi->parent_scan) + xchk_iscan_mark_visited(&fpi->parent_scan->iscan, dp); return 0; + } /* * Scan the directory to see if there it contains an entry pointing to @@ -201,6 +221,8 @@ xrep_findparent_walk_directory( goto out_unlock; out_unlock: + if (fpi->parent_scan) + xchk_iscan_mark_visited(&fpi->parent_scan->iscan, dp); xfs_iunlock(dp, lock_mode); return error; } @@ -309,11 +331,7 @@ xrep_findparent_scan( ASSERT(S_ISDIR(VFS_IC(sc->ip)->i_mode)); while ((ret = xchk_iscan_iter(&pscan->iscan, &fpi.dp)) == 1) { - if (S_ISDIR(VFS_I(fpi.dp)->i_mode)) - ret = xrep_findparent_walk_directory(&fpi); - else - ret = 0; - xchk_iscan_mark_visited(&pscan->iscan, fpi.dp); + ret = xrep_findparent_walk_file(&fpi); xchk_irele(sc, fpi.dp); if (ret) break; @@ -402,7 +420,7 @@ xrep_findparent_confirm( goto out_rele; } - error = xrep_findparent_walk_directory(&fpi); + error = xrep_findparent_walk_file(&fpi); if (error) goto out_rele; diff --git a/fs/xfs/scrub/ialloc.c b/fs/xfs/scrub/ialloc.c index 4dc7c83dc08a4..389451cd8756b 100644 --- a/fs/xfs/scrub/ialloc.c +++ b/fs/xfs/scrub/ialloc.c @@ -85,6 +85,8 @@ xchk_inobt_xref_finobt( goto no_record; error = xfs_inobt_get_rec(cur, &frec, &has_record); + if (error) + return error; if (!has_record) return -EFSCORRUPTED; @@ -188,6 +190,8 @@ xchk_finobt_xref_inobt( goto no_record; error = xfs_inobt_get_rec(cur, &irec, &has_record); + if (error) + return error; if (!has_record) return -EFSCORRUPTED; diff --git a/fs/xfs/scrub/inode_repair.c b/fs/xfs/scrub/inode_repair.c index 5c45a68793d03..fdb227dcfd21d 100644 --- a/fs/xfs/scrub/inode_repair.c +++ b/fs/xfs/scrub/inode_repair.c @@ -1956,7 +1956,7 @@ xrep_inode_cowextsize( /* Fix misaligned CoW extent size hints on a directory. */ if ((sc->ip->i_diflags & XFS_DIFLAG_RTINHERIT) && (sc->ip->i_diflags2 & XFS_DIFLAG2_COWEXTSIZE) && - sc->ip->i_extsize % sc->mp->m_sb.sb_rextsize > 0) { + xfs_extlen_to_rtxmod(sc->mp, sc->ip->i_cowextsize) > 0) { sc->ip->i_cowextsize = 0; sc->ip->i_diflags2 &= ~XFS_DIFLAG2_COWEXTSIZE; } diff --git a/fs/xfs/scrub/metapath.c b/fs/xfs/scrub/metapath.c index 378ec7c8d38ee..3bae758156278 100644 --- a/fs/xfs/scrub/metapath.c +++ b/fs/xfs/scrub/metapath.c @@ -314,7 +314,7 @@ xchk_metapath( /* Parent required to do anything else. */ if (mpath->dp == NULL) { - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); return 0; } @@ -329,7 +329,7 @@ xchk_metapath( trace_xchk_metapath_lookup(sc, mpath->path, mpath->dp, ino); if (error == -ENOENT) { /* No directory entry at all */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); error = 0; goto out_ilock; } @@ -337,7 +337,7 @@ xchk_metapath( goto out_ilock; if (ino != sc->ip->i_ino) { /* Pointing to wrong inode */ - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } out_ilock: @@ -397,7 +397,7 @@ xrep_metapath_unlink( /* Figure out if we're removing a parent pointer too. */ if (xfs_has_parent(mp)) { - xfs_inode_to_parent_rec(&rec, ip); + xfs_inode_to_parent_rec(&rec, mpath->dp); error = xfs_parent_lookup(sc->tp, ip, &mpath->xname, &rec, &mpath->pptr_args); switch (error) { @@ -556,6 +556,8 @@ xrep_metapath_try_unlink( error = xchk_metapath_ilock_parent_and_child(mpath, ip); if (error) { xchk_trans_cancel(sc); + if (ip) + xchk_irele(sc, ip); return error; } xfs_trans_ijoin(sc->tp, mpath->dp, 0); diff --git a/fs/xfs/scrub/nlinks.c b/fs/xfs/scrub/nlinks.c index dec3b9b47453e..988262fc99533 100644 --- a/fs/xfs/scrub/nlinks.c +++ b/fs/xfs/scrub/nlinks.c @@ -382,6 +382,12 @@ xchk_nlinks_ilock_dir( { uint lock_mode = XFS_ILOCK_SHARED; + /* + * Take the IOLOCK so that other threads cannot start a directory + * update while we're scanning. + */ + xfs_ilock(ip, XFS_IOLOCK_SHARED); + /* * We're going to scan the directory entries, so we must be ready to * pull the data fork mappings into memory if they aren't already. @@ -397,13 +403,8 @@ xchk_nlinks_ilock_dir( xfs_need_iread_extents(&ip->i_af)) lock_mode = XFS_ILOCK_EXCL; - /* - * Take the IOLOCK so that other threads cannot start a directory - * update while we're scanning. - */ - lock_mode |= XFS_IOLOCK_SHARED; xfs_ilock(ip, lock_mode); - return lock_mode; + return lock_mode | XFS_IOLOCK_SHARED; } /* Walk a directory to bump the observed link counts of the children. */ @@ -718,7 +719,7 @@ xchk_nlinks_compare_inode( * count, but it will let them decrease it. */ if (total_links > XFS_NLINK_PINNED) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } else if (total_links > XFS_MAXLINK) { xchk_ino_set_warning(sc, ip->i_ino); @@ -726,7 +727,7 @@ xchk_nlinks_compare_inode( /* Link counts should match. */ if (total_links != actual_nlink) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -747,7 +748,7 @@ xchk_nlinks_compare_inode( * back references. */ if (obs.backrefs != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } @@ -756,7 +757,7 @@ xchk_nlinks_compare_inode( * children. */ if (obs.children != 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } @@ -769,7 +770,7 @@ xchk_nlinks_compare_inode( * the root directory. */ if (obs.parents != 1) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } else if (actual_nlink > 0) { @@ -778,7 +779,7 @@ xchk_nlinks_compare_inode( * least one parent. */ if (obs.parents == 0) { - xchk_ino_set_corrupt(sc, ip->i_ino); + xchk_ip_set_corrupt(sc, ip); goto out_corrupt; } } diff --git a/fs/xfs/scrub/nlinks_repair.c b/fs/xfs/scrub/nlinks_repair.c index 6ef2ee9c3814a..c3540f3dd2c77 100644 --- a/fs/xfs/scrub/nlinks_repair.c +++ b/fs/xfs/scrub/nlinks_repair.c @@ -232,9 +232,14 @@ xrep_nlinks_repair_inode( * unlinked list, put it on the unlinked list. */ if (total_links == 0 && !xfs_inode_on_unlinked_list(ip)) { + if (actual_nlink) + clear_nlink(VFS_I(ip)); error = xfs_iunlink(sc->tp, ip); - if (error) + if (error) { + if (actual_nlink) + set_nlink(VFS_I(ip), actual_nlink); goto out_trans; + } dirty = true; } diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c index f2ee520cc9429..76836fd472835 100644 --- a/fs/xfs/scrub/parent.c +++ b/fs/xfs/scrub/parent.c @@ -486,7 +486,7 @@ xchk_parent_scan_attr( valuelen, &parent_ino, NULL); if (error) { xchk_fblock_set_corrupt(sc, XFS_ATTR_FORK, 0); - return error; + return -ECANCELED; } /* No self-referential parent pointers. */ @@ -729,10 +729,10 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink == 0 && pp->pptrs_found > 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); else if (VFS_I(sc->ip)->i_nlink > 0 && pp->pptrs_found == 0) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } else { /* * Starting with metadir, we allow checking of parent pointers @@ -743,7 +743,7 @@ xchk_parent_count_pptrs( pp->pptrs_found++; if (VFS_I(sc->ip)->i_nlink != pp->pptrs_found) - xchk_ino_set_corrupt(sc, sc->ip->i_ino); + xchk_ip_set_corrupt(sc, sc->ip); } return 0; diff --git a/fs/xfs/scrub/quota_repair.c b/fs/xfs/scrub/quota_repair.c index 8f4c8d41f3083..811eb6093bb6a 100644 --- a/fs/xfs/scrub/quota_repair.c +++ b/fs/xfs/scrub/quota_repair.c @@ -327,7 +327,7 @@ xrep_quota_block( * If there's nothing that would impede a dqiterate, we're * done. */ - if ((ddq->d_type & XFS_DQTYPE_REC_MASK) != dqtype || + if ((ddq->d_type & XFS_DQTYPE_REC_MASK) == dqtype && id == be32_to_cpu(ddq->d_id)) { xfs_trans_brelse(sc->tp, bp); return 0; @@ -365,11 +365,18 @@ xrep_quota_block( ddq->d_rtbcount, &ddq->d_rtbtimer, defq->rtb.time); + /* + * This transaction operates on raw disk buffers, so we don't + * have a dquot log item to assign the LSN for us. Instead, + * set it to zero so that log recovery will always replay any + * logged dquot item atop this buffer. + */ + dqblk->dd_lsn = 0; + /* We only support v5 filesystems so always set these. */ uuid_copy(&dqblk->dd_uuid, &sc->mp->m_sb.sb_meta_uuid); xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk), XFS_DQUOT_CRC_OFF); - dqblk->dd_lsn = 0; } switch (dqtype) { case XFS_DQTYPE_USER: @@ -457,8 +464,7 @@ xrep_quota_data_fork( if (truncate) { /* Erase everything after the block containing the max dquot */ - error = xfs_bunmapi_range(&sc->tp, sc->ip, 0, - max_dqid_off * sc->mp->m_sb.sb_blocksize, + error = xfs_bunmapi_range(&sc->tp, sc->ip, 0, max_dqid_off + 1, XFS_MAX_FILEOFF); if (error) goto out; diff --git a/fs/xfs/scrub/quotacheck.c b/fs/xfs/scrub/quotacheck.c index 38e855fe0feeb..b85a0df017c23 100644 --- a/fs/xfs/scrub/quotacheck.c +++ b/fs/xfs/scrub/quotacheck.c @@ -263,8 +263,10 @@ xqcheck_mod_live_ino_dqtrx( dqa->tx_id = p->tx_id; error = rhashtable_insert_fast(&xqc->shadow_dquot_acct, &dqa->hash, xqcheck_dqacct_hash_params); - if (error) + if (error) { + kfree(dqa); goto out_abort; + } } /* Find the shadow dqtrx (or an empty slot) here. */ diff --git a/fs/xfs/scrub/reap.c b/fs/xfs/scrub/reap.c index 07f5bb8a64212..db32356e517da 100644 --- a/fs/xfs/scrub/reap.c +++ b/fs/xfs/scrub/reap.c @@ -601,7 +601,7 @@ xreap_configure_agextent_limits( /* Maximum overhead of invalidating one buffer. */ const unsigned int per_binval = - xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + xfs_buf_inval_log_space(1, XFS_FSB_TO_B(mp, max_binval)); /* * For each transaction in a reap chain, we can delete some number of @@ -680,7 +680,7 @@ xreap_configure_agcow_limits( /* Overhead of invalidating one buffer */ const unsigned int per_binval = - xfs_buf_inval_log_space(1, XFS_B_TO_FSBT(mp, max_binval)); + xfs_buf_inval_log_space(1, XFS_FSB_TO_B(mp, max_binval)); /* * For each transaction in a reap chain, we can delete some number of @@ -1400,7 +1400,7 @@ xreap_bmapi_binval( * far we've gotten. */ if (!xreap_inc_binval(rs)) { - imap->br_blockcount = agbno_next - bno; + imap->br_blockcount = bno - agbno; goto out; } } diff --git a/fs/xfs/scrub/refcount.c b/fs/xfs/scrub/refcount.c index d465280230154..0951b3def0be1 100644 --- a/fs/xfs/scrub/refcount.c +++ b/fs/xfs/scrub/refcount.c @@ -411,7 +411,7 @@ xchk_refcount_mergeable( const struct xfs_refcount_irec *r1 = &rrc->prev_rec; /* Ignore if prev_rec is not yet initialized. */ - if (r1->rc_blockcount > 0) + if (r1->rc_blockcount == 0) return false; if (r1->rc_domain != r2->rc_domain) @@ -582,8 +582,12 @@ xchk_xref_is_cow_staging( if (rc.rc_domain != XFS_REFC_DOMAIN_COW) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + /* Can't start after bno */ + if (rc.rc_startblock > agbno) + xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); + /* Must be at least as long as what was passed in */ - if (rc.rc_blockcount < len) + if (rc.rc_startblock + rc.rc_blockcount < agbno + len) xchk_btree_xref_set_corrupt(sc, sc->sa.refc_cur, 0); } diff --git a/fs/xfs/scrub/rgsuper.c b/fs/xfs/scrub/rgsuper.c index 7b962890a8941..c44666cedc447 100644 --- a/fs/xfs/scrub/rgsuper.c +++ b/fs/xfs/scrub/rgsuper.c @@ -36,8 +36,10 @@ xchk_rgsuperblock_xref( if (sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) return; - xchk_xref_is_used_rt_space(sc, xfs_rgbno_to_rtb(sc->sr.rtg, 0), 1); - xchk_xref_is_only_rt_owned_by(sc, 0, 1, &XFS_RMAP_OINFO_FS); + xchk_xref_is_used_rt_space(sc, xfs_rgbno_to_rtb(sc->sr.rtg, 0), + sc->mp->m_sb.sb_rextsize); + xchk_xref_is_only_rt_owned_by(sc, 0, sc->mp->m_sb.sb_rextsize, + &XFS_RMAP_OINFO_FS); } int diff --git a/fs/xfs/scrub/rtbitmap.c b/fs/xfs/scrub/rtbitmap.c index 8efde8c26e619..68b8338707608 100644 --- a/fs/xfs/scrub/rtbitmap.c +++ b/fs/xfs/scrub/rtbitmap.c @@ -200,13 +200,13 @@ xchk_rtbitmap( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rtb->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is sb_rextslog correct? */ if (mp->m_sb.sb_rextslog != rtb->rextslog) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -215,17 +215,17 @@ xchk_rtbitmap( * case can we exceed 4bn bitmap blocks since the super field is a u32. */ if (rtb->rbmblocks > U32_MAX) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } if (mp->m_sb.sb_rbmblocks != rtb->rbmblocks) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* The bitmap file length must be aligned to an fsblock. */ if (rbmip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } @@ -235,7 +235,7 @@ xchk_rtbitmap( * file can be larger than sb_rbmblocks. */ if (rbmip->i_disk_size < XFS_FSB_TO_B(mp, rtb->rbmblocks)) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } diff --git a/fs/xfs/scrub/rtbitmap_repair.c b/fs/xfs/scrub/rtbitmap_repair.c index 41d6736a529d0..d2669082d0934 100644 --- a/fs/xfs/scrub/rtbitmap_repair.c +++ b/fs/xfs/scrub/rtbitmap_repair.c @@ -36,6 +36,24 @@ /* rt bitmap content repairs */ +/* + * Reserve enough blocks to write out a completely new bitmap file, plus twice + * as many blocks as we would need if we can only allocate one block per data + * fork mapping. This should cover the preallocation of the temporary file and + * exchanging the extent mappings. + * + * We cannot use xfs_exchmaps_estimate because we have not yet constructed the + * replacement bitmap and therefore do not know how many extents it will use. + * By the time we do, we will have a dirty transaction (which we cannot drop + * because we cannot drop the rtbitmap ILOCK) and cannot ask for more + * reservation. + */ +static inline unsigned long long +xrep_rtbitmap_calc_blocks(struct xfs_mount *mp, unsigned long long blocks) +{ + return blocks + (xfs_bmbt_calc_size(mp, blocks) * 2); +} + /* Set up to repair the realtime bitmap for this group. */ int xrep_setup_rtbitmap( @@ -56,20 +74,7 @@ xrep_setup_rtbitmap( if (error) return error; - /* - * Reserve enough blocks to write out a completely new bitmap file, - * plus twice as many blocks as we would need if we can only allocate - * one block per data fork mapping. This should cover the - * preallocation of the temporary file and exchanging the extent - * mappings. - * - * We cannot use xfs_exchmaps_estimate because we have not yet - * constructed the replacement bitmap and therefore do not know how - * many extents it will use. By the time we do, we will have a dirty - * transaction (which we cannot drop because we cannot drop the - * rtbitmap ILOCK) and cannot ask for more reservation. - */ - blocks += xfs_bmbt_calc_size(mp, blocks) * 2; + blocks = xrep_rtbitmap_calc_blocks(mp, mp->m_sb.sb_rbmblocks); if (blocks > UINT_MAX) return -EOPNOTSUPP; @@ -512,7 +517,7 @@ xrep_rtbitmap( struct xchk_rtbitmap *rtb = sc->buf; struct xfs_mount *mp = sc->mp; struct xfs_group *xg = rtg_group(sc->sr.rtg); - unsigned long long blocks = 0; + unsigned long long blocks; unsigned int busy_gen; int error; @@ -532,15 +537,20 @@ xrep_rtbitmap( * figure out if we need to adjust the block reservation in the * transaction. */ - blocks = xfs_bmbt_calc_size(mp, rtb->rbmblocks); + blocks = xrep_rtbitmap_calc_blocks(mp, rtb->rbmblocks); if (blocks > UINT_MAX) return -EOPNOTSUPP; if (blocks > rtb->resblks) { - error = xfs_trans_reserve_more(sc->tp, blocks, 0); + uint64_t delta = blocks - rtb->resblks; + + if (delta > UINT_MAX) + return -EOPNOTSUPP; + + error = xfs_trans_reserve_more(sc->tp, delta, 0); if (error) return error; - rtb->resblks += blocks; + rtb->resblks += delta; } /* Fix inode core and forks. */ diff --git a/fs/xfs/scrub/rtrefcount.c b/fs/xfs/scrub/rtrefcount.c index 24eb2a664aab8..a6f2d66de3a8b 100644 --- a/fs/xfs/scrub/rtrefcount.c +++ b/fs/xfs/scrub/rtrefcount.c @@ -20,6 +20,7 @@ #include "xfs_metafile.h" #include "xfs_rtrefcount_btree.h" #include "xfs_rtalloc.h" +#include "xfs_ag.h" #include "scrub/scrub.h" #include "scrub/common.h" #include "scrub/btree.h" @@ -376,7 +377,7 @@ xchk_rtrefcount_mergeable( const struct xfs_refcount_irec *r1 = &rrc->prev_rec; /* Ignore if prev_rec is not yet initialized. */ - if (r1->rc_blockcount > 0) + if (r1->rc_blockcount == 0) return false; if (r1->rc_startblock + r1->rc_blockcount != r2->rc_startblock) @@ -429,7 +430,7 @@ static inline void xchk_rtrefcountbt_xref_gaps( struct xfs_scrub *sc, struct xchk_rtrefcbt_records *rrc, - xfs_rtblock_t bno) + xfs_rgblock_t bno) { struct xfs_rmap_irec low; struct xfs_rmap_irec high; @@ -505,30 +506,75 @@ xchk_rtrefcountbt_rec( return 0; } +/* Count the number of blocks used by the rtrefcount btree file in this AG. */ +static int +xchk_rtrefcount_count_agblocks( + struct xfs_scrub *sc, + xfs_agnumber_t agno, + const struct xfs_owner_info *btree_oinfo, + xfs_filblks_t *blocks) +{ + xfs_filblks_t agblocks = 0; + int error; + + error = xchk_ag_init_existing(sc, agno, &sc->sa); + if (error) + goto out_free; + + /* + * If we don't have an rmap cursor, we can't complete the cross + * referencing, so return EFSCORRUPTED to end the loop and trigger the + * XFAIL flag. + */ + if (!sc->sa.rmap_cur) { + error = -EFSCORRUPTED; + goto out_free; + } + + error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, btree_oinfo, + &agblocks); + if (error) + goto out_free; + + *blocks += agblocks; +out_free: + xchk_ag_free(sc, &sc->sa); + return error; +} + /* Make sure we have as many refc blocks as the rmap says. */ STATIC void -xchk_refcount_xref_rmap( +xchk_rtrefcount_xref_rmap( struct xfs_scrub *sc, const struct xfs_owner_info *btree_oinfo, xfs_extlen_t cow_blocks) { xfs_filblks_t refcbt_blocks = 0; - xfs_filblks_t blocks; - int error; + xfs_filblks_t blocks = 1; /* one for the iroot */ + xfs_agnumber_t agno; + int error = 0; - if (!sc->sr.rmap_cur || !sc->sa.rmap_cur || xchk_skip_xref(sc->sm)) + if (!xfs_has_rmapbt(sc->mp) || xchk_skip_xref(sc->sm)) return; /* Check that we saw as many refcbt blocks as the rmap knows about. */ error = xfs_btree_count_blocks(sc->sr.refc_cur, &refcbt_blocks); if (!xchk_btree_process_error(sc, sc->sr.refc_cur, 0, &error)) return; - error = xchk_count_rmap_ownedby_ag(sc, sc->sa.rmap_cur, btree_oinfo, - &blocks); - if (!xchk_should_check_xref(sc, &error, &sc->sa.rmap_cur)) + + for (agno = 0; agno < sc->mp->m_sb.sb_agcount; agno++) { + error = xchk_rtrefcount_count_agblocks(sc, agno, btree_oinfo, + &blocks); + if (error) + break; + } + if (!xchk_fblock_xref_process_error(sc, XFS_DATA_FORK, 0, &error)) return; if (blocks != refcbt_blocks) - xchk_btree_xref_set_corrupt(sc, sc->sa.rmap_cur, 0); + xchk_fblock_xref_set_corrupt(sc, XFS_DATA_FORK, 0); + + if (!sc->sr.rmap_cur || xchk_skip_xref(sc->sm)) + return; /* Check that we saw as many cow blocks as the rmap knows about. */ error = xchk_count_rmap_ownedby_ag(sc, sc->sr.rmap_cur, @@ -539,7 +585,7 @@ xchk_refcount_xref_rmap( xchk_btree_xref_set_corrupt(sc, sc->sr.rmap_cur, 0); } -/* Scrub the refcount btree for some AG. */ +/* Scrub the refcount btree for some rtgroup. */ int xchk_rtrefcountbt( struct xfs_scrub *sc) @@ -565,11 +611,11 @@ xchk_rtrefcountbt( /* * Check that all blocks between the last refcount > 1 record and the - * end of the rt volume have at most one reverse mapping. + * end of the rtgroup have at most one reverse mapping. */ - xchk_rtrefcountbt_xref_gaps(sc, &rrc, sc->mp->m_sb.sb_rblocks); - - xchk_refcount_xref_rmap(sc, &btree_oinfo, rrc.cow_blocks); + xchk_rtrefcountbt_xref_gaps(sc, &rrc, + xfs_rtx_to_rgbno(sc->sr.rtg, sc->mp->m_sb.sb_rgextents)); + xchk_rtrefcount_xref_rmap(sc, &btree_oinfo, rrc.cow_blocks); return 0; } @@ -610,8 +656,12 @@ xchk_xref_is_rt_cow_staging( if (rc.rc_domain != XFS_REFC_DOMAIN_COW) xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + /* Can't start after bno */ + if (rc.rc_startblock > bno) + xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); + /* Must be at least as long as what was passed in */ - if (rc.rc_blockcount < len) + if (rc.rc_startblock + rc.rc_blockcount < bno + len) xchk_btree_xref_set_corrupt(sc, sc->sr.refc_cur, 0); } diff --git a/fs/xfs/scrub/rtsummary.c b/fs/xfs/scrub/rtsummary.c index fb78cff2ac3a1..dbbbafe8b92d6 100644 --- a/fs/xfs/scrub/rtsummary.c +++ b/fs/xfs/scrub/rtsummary.c @@ -315,25 +315,25 @@ xchk_rtsummary( /* Is sb_rextents correct? */ if (mp->m_sb.sb_rextents != rts->rextents) { - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ip_set_corrupt(sc, rbmip); return 0; } /* Is m_rsumlevels correct? */ if (mp->m_rsumlevels != rts->rsumlevels) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* Is m_rsumsize correct? */ if (mp->m_rsumblocks != rts->rsumblocks) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } /* The summary file length must be aligned to an fsblock. */ if (rsumip->i_disk_size & mp->m_blockmask) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -343,7 +343,7 @@ xchk_rtsummary( * the file can be larger than rsumsize. */ if (rsumip->i_disk_size < XFS_FSB_TO_B(mp, rts->rsumblocks)) { - xchk_ino_set_corrupt(sc, rsumip->i_ino); + xchk_ip_set_corrupt(sc, rsumip); return 0; } @@ -359,7 +359,7 @@ xchk_rtsummary( * EFSCORRUPTED means the rtbitmap is corrupt, which is an xref * error since we're checking the summary file. */ - xchk_ino_set_corrupt(sc, rbmip->i_ino); + xchk_ino_xref_set_corrupt(sc, rbmip->i_ino); return 0; } if (error) diff --git a/fs/xfs/scrub/rtsummary_repair.c b/fs/xfs/scrub/rtsummary_repair.c index d593977d70df2..c94e485a8837d 100644 --- a/fs/xfs/scrub/rtsummary_repair.c +++ b/fs/xfs/scrub/rtsummary_repair.c @@ -164,9 +164,10 @@ xrep_rtsummary( /* * Now exchange the contents. Nothing in repair uses the temporary * buffer, so we can reuse it for the tempfile exchrange information. + * Use XFS_MAX_FILEOFF here so that we correct the rtsummary file size. */ error = xrep_tempexch_trans_reserve(sc, XFS_DATA_FORK, 0, - rts->rsumblocks, &rts->tempexch); + XFS_MAX_FILEOFF, &rts->tempexch); if (error) return error; diff --git a/fs/xfs/scrub/scrub.c b/fs/xfs/scrub/scrub.c index c312f0a672e65..3f6f7647e3a3a 100644 --- a/fs/xfs/scrub/scrub.c +++ b/fs/xfs/scrub/scrub.c @@ -765,8 +765,7 @@ xfs_scrub_metadata( out_teardown: error = xchk_teardown(sc, error); out_sc: - if (error != -ENOENT) - xchk_stats_merge(mp, sm, &run); + xchk_stats_merge(mp, sm, error, &run); kfree(sc); out: trace_xchk_done(XFS_I(file_inode(file)), sm, error); diff --git a/fs/xfs/scrub/stats.c b/fs/xfs/scrub/stats.c index f8a37ea977919..425a90b38e250 100644 --- a/fs/xfs/scrub/stats.c +++ b/fs/xfs/scrub/stats.c @@ -84,6 +84,7 @@ static const char *name_map[XFS_SCRUB_TYPE_NR] = { [XFS_SCRUB_TYPE_RGSUPER] = "rgsuper", [XFS_SCRUB_TYPE_RTRMAPBT] = "rtrmapbt", [XFS_SCRUB_TYPE_RTREFCBT] = "rtrefcountbt", + [XFS_SCRUB_TYPE_HEALTHY] = "healthy", }; /* Format the scrub stats into a text buffer, similar to pcp style. */ @@ -99,25 +100,31 @@ xchk_stats_format( int ret = 0; for (i = 0; i < XFS_SCRUB_TYPE_NR; i++, css++) { + struct xchk_scrub_stats fss; + if (!name_map[i]) continue; + spin_lock(&css->css_lock); + memcpy(&fss, css, offsetof(struct xchk_scrub_stats, css_lock)); + spin_unlock(&css->css_lock); + ret = scnprintf(buf, remaining, "%s %u %u %u %u %u %u %u %u %u %llu %u %u %llu\n", name_map[i], - (unsigned int)css->invocations, - (unsigned int)css->clean, - (unsigned int)css->corrupt, - (unsigned int)css->preen, - (unsigned int)css->xfail, - (unsigned int)css->xcorrupt, - (unsigned int)css->incomplete, - (unsigned int)css->warning, - (unsigned int)css->retries, - (unsigned long long)css->checktime_us, - (unsigned int)css->repair_invocations, - (unsigned int)css->repair_success, - (unsigned long long)css->repairtime_us); + (unsigned int)fss.invocations, + (unsigned int)fss.clean, + (unsigned int)fss.corrupt, + (unsigned int)fss.preen, + (unsigned int)fss.xfail, + (unsigned int)fss.xcorrupt, + (unsigned int)fss.incomplete, + (unsigned int)fss.warning, + (unsigned int)fss.retries, + (unsigned long long)fss.checktime_us, + (unsigned int)fss.repair_invocations, + (unsigned int)fss.repair_success, + (unsigned long long)fss.repairtime_us); if (ret <= 0) break; @@ -188,31 +195,37 @@ STATIC void xchk_stats_merge_one( struct xchk_stats *cs, const struct xfs_scrub_metadata *sm, + int error, const struct xchk_stats_run *run) { struct xchk_scrub_stats *css; + unsigned int sm_flags = sm->sm_flags; if (sm->sm_type >= XFS_SCRUB_TYPE_NR) { ASSERT(sm->sm_type < XFS_SCRUB_TYPE_NR); return; } + /* caller applies this same transformation after we return */ + if (error == -EFSCORRUPTED || error == -EFSBADCRC) + sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; + css = &cs->cs_stats[sm->sm_type]; spin_lock(&css->css_lock); css->invocations++; - if (!(sm->sm_flags & XFS_SCRUB_OFLAG_UNCLEAN)) + if (!(sm_flags & XFS_SCRUB_OFLAG_UNCLEAN)) css->clean++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT) + if (sm_flags & XFS_SCRUB_OFLAG_CORRUPT) css->corrupt++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_PREEN) + if (sm_flags & XFS_SCRUB_OFLAG_PREEN) css->preen++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_XFAIL) + if (sm_flags & XFS_SCRUB_OFLAG_XFAIL) css->xfail++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_XCORRUPT) + if (sm_flags & XFS_SCRUB_OFLAG_XCORRUPT) css->xcorrupt++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) + if (sm_flags & XFS_SCRUB_OFLAG_INCOMPLETE) css->incomplete++; - if (sm->sm_flags & XFS_SCRUB_OFLAG_WARNING) + if (sm_flags & XFS_SCRUB_OFLAG_WARNING) css->warning++; css->retries += run->retries; css->checktime_us += howmany_64(run->scrub_ns, NSEC_PER_USEC); @@ -230,10 +243,14 @@ void xchk_stats_merge( struct xfs_mount *mp, const struct xfs_scrub_metadata *sm, + int error, const struct xchk_stats_run *run) { - xchk_stats_merge_one(&global_stats, sm, run); - xchk_stats_merge_one(mp->m_scrub_stats, sm, run); + if (error == -ENOENT) + return; + + xchk_stats_merge_one(&global_stats, sm, error, run); + xchk_stats_merge_one(mp->m_scrub_stats, sm, error, run); } /* debugfs boilerplate */ diff --git a/fs/xfs/scrub/stats.h b/fs/xfs/scrub/stats.h index b358ad8d8b90a..221052b95dd07 100644 --- a/fs/xfs/scrub/stats.h +++ b/fs/xfs/scrub/stats.h @@ -27,7 +27,7 @@ void xchk_stats_register(struct xchk_stats *cs, struct dentry *parent); void xchk_stats_unregister(struct xchk_stats *cs); void xchk_stats_merge(struct xfs_mount *mp, const struct xfs_scrub_metadata *sm, - const struct xchk_stats_run *run); + int error, const struct xchk_stats_run *run); static inline u64 xchk_stats_now(void) { return ktime_get_ns(); } static inline u64 xchk_stats_elapsed_ns(u64 since) @@ -53,7 +53,7 @@ static inline u64 xchk_stats_elapsed_ns(u64 since) # define xchk_stats_unregister(cs) ((void)0) # define xchk_stats_now() (0) # define xchk_stats_elapsed_ns(x) (0 * (x)) -# define xchk_stats_merge(mp, sm, run) ((void)0) +# define xchk_stats_merge(mp, sm, error, run) ((void)0) #endif /* CONFIG_XFS_ONLINE_SCRUB_STATS */ #endif /* __XFS_SCRUB_STATS_H__ */ diff --git a/fs/xfs/scrub/symlink_repair.c b/fs/xfs/scrub/symlink_repair.c index df629892462fe..6f8bc5c1fd689 100644 --- a/fs/xfs/scrub/symlink_repair.c +++ b/fs/xfs/scrub/symlink_repair.c @@ -291,7 +291,7 @@ xrep_symlink_swap_prep( if (error) return error; - xfs_trans_log_inode(sc->tp, sc->ip, 0); + xfs_trans_log_inode(sc->tp, sc->tempip, logflags); error = xfs_defer_finish(&sc->tp); if (error) diff --git a/fs/xfs/scrub/tempfile.c b/fs/xfs/scrub/tempfile.c index cf99e0ca51b00..fabf6ae2e5120 100644 --- a/fs/xfs/scrub/tempfile.c +++ b/fs/xfs/scrub/tempfile.c @@ -174,6 +174,7 @@ xrep_tempfile_create( xfs_iunlock(sc->tempip, XFS_ILOCK_EXCL); xfs_finish_inode_setup(sc->tempip); xchk_irele(sc, sc->tempip); + sc->tempip = NULL; } out_release_dquots: xfs_qm_dqrele(udqp); diff --git a/fs/xfs/scrub/trace.h b/fs/xfs/scrub/trace.h index 286c5f5e05444..ed88557140c72 100644 --- a/fs/xfs/scrub/trace.h +++ b/fs/xfs/scrub/trace.h @@ -1640,7 +1640,7 @@ DECLARE_EVENT_CLASS(xchk_pptr_class, __entry->dev = ip->i_mount->m_super->s_dev; __entry->ino = ip->i_ino; __entry->namelen = name->len; - memcpy(__get_str(name), name, name->len); + memcpy(__get_str(name), name->name, name->len); __entry->far_ino = far_ino; ), TP_printk("dev %d:%d ino 0x%llx name '%.*s' far_ino 0x%llx", @@ -1706,6 +1706,39 @@ DEFINE_EVENT(xchk_dirtree_class, name, \ DEFINE_XCHK_DIRTREE_EVENT(xchk_dirtree_create_path); DEFINE_XCHK_DIRTREE_EVENT(xchk_dirpath_walk_upwards); +TRACE_EVENT(xchk_dirpath_badino, + TP_PROTO(struct xfs_scrub *sc, unsigned int path_nr, + unsigned int step_nr, const struct xfs_name *name, + const struct xfs_parent_rec *pptr), + TP_ARGS(sc, path_nr, step_nr, name, pptr), + TP_STRUCT__entry( + __field(dev_t, dev) + __field(unsigned int, path_nr) + __field(unsigned int, step_nr) + __field(xfs_ino_t, parent_ino) + __field(unsigned int, parent_gen) + __field(unsigned int, namelen) + __dynamic_array(char, name, name->len) + ), + TP_fast_assign( + __entry->dev = sc->mp->m_super->s_dev; + __entry->path_nr = path_nr; + __entry->step_nr = step_nr; + __entry->parent_ino = be64_to_cpu(pptr->p_ino); + __entry->parent_gen = be32_to_cpu(pptr->p_gen); + __entry->namelen = name->len; + memcpy(__get_str(name), name->name, name->len); + ), + TP_printk("dev %d:%d path %u step %u parent_ino 0x%llx parent_gen 0x%x name '%.*s'", + MAJOR(__entry->dev), MINOR(__entry->dev), + __entry->path_nr, + __entry->step_nr, + __entry->parent_ino, + __entry->parent_gen, + __entry->namelen, + __get_str(name)) +); + DECLARE_EVENT_CLASS(xchk_dirpath_class, TP_PROTO(struct xfs_scrub *sc, struct xfs_inode *ip, unsigned int path_nr, unsigned int step_nr, @@ -2672,9 +2705,9 @@ TRACE_EVENT(xrep_cow_mark_file_range, ); TRACE_EVENT(xrep_cow_replace_mapping, - TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *irec, - xfs_fsblock_t new_startblock, xfs_extlen_t new_blockcount), - TP_ARGS(ip, irec, new_startblock, new_blockcount), + TP_PROTO(struct xfs_inode *ip, const struct xfs_bmbt_irec *got, + const struct xfs_bmbt_irec *rep), + TP_ARGS(ip, got, rep), TP_STRUCT__entry( __field(dev_t, dev) __field(xfs_ino_t, ino) @@ -2682,28 +2715,34 @@ TRACE_EVENT(xrep_cow_replace_mapping, __field(xfs_fileoff_t, startoff) __field(xfs_filblks_t, blockcount) __field(xfs_exntst_t, state) + __field(xfs_fileoff_t, new_startoff) __field(xfs_fsblock_t, new_startblock) __field(xfs_extlen_t, new_blockcount) + __field(xfs_exntst_t, new_state) ), TP_fast_assign( __entry->dev = ip->i_mount->m_super->s_dev; __entry->ino = ip->i_ino; - __entry->startoff = irec->br_startoff; - __entry->startblock = irec->br_startblock; - __entry->blockcount = irec->br_blockcount; - __entry->state = irec->br_state; - __entry->new_startblock = new_startblock; - __entry->new_blockcount = new_blockcount; + __entry->startoff = got->br_startoff; + __entry->startblock = got->br_startblock; + __entry->blockcount = got->br_blockcount; + __entry->state = got->br_state; + __entry->new_startoff = rep->br_startoff; + __entry->new_startblock = rep->br_startblock; + __entry->new_blockcount = rep->br_blockcount; + __entry->new_state = rep->br_state; ), - TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startblock 0x%llx new_fsbcount 0x%x", + TP_printk("dev %d:%d ino 0x%llx startoff 0x%llx startblock 0x%llx fsbcount 0x%llx state 0x%x new_startoff 0x%llx new_startblock 0x%llx new_fsbcount 0x%x new_state 0x%x", MAJOR(__entry->dev), MINOR(__entry->dev), __entry->ino, __entry->startoff, __entry->startblock, __entry->blockcount, __entry->state, + __entry->new_startoff, __entry->new_startblock, - __entry->new_blockcount) + __entry->new_blockcount, + __entry->new_state) ); TRACE_EVENT(xrep_cow_free_staging, @@ -3573,10 +3612,12 @@ DEFINE_EVENT(xrep_iunlink_resolve_class, name, \ TP_PROTO(const struct xfs_perag *pag, unsigned int bucket, \ xfs_agino_t prev_agino, xfs_agino_t next_agino), \ TP_ARGS(pag, bucket, prev_agino, next_agino)) +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_infinite_loop); DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_uncached); DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_wronglist); DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_nolist); DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_ok); +DEFINE_REPAIR_IUNLINK_RESOLVE_EVENT(xrep_iunlink_resolve_allocated); TRACE_EVENT(xrep_iunlink_relink_next, TP_PROTO(struct xfs_inode *ip, xfs_agino_t next_agino), diff --git a/fs/xfs/xfs_buf.c b/fs/xfs/xfs_buf.c index 1ca95ef46a73d..7b3ab1e3ebbd0 100644 --- a/fs/xfs/xfs_buf.c +++ b/fs/xfs/xfs_buf.c @@ -111,13 +111,13 @@ xfs_buf_free( ASSERT(list_empty(&bp->b_lru)); if (!xfs_buftarg_is_mem(bp->b_target) && size >= PAGE_SIZE) - mm_account_reclaimed_pages(howmany(size, PAGE_SHIFT)); + mm_account_reclaimed_pages(howmany(size, PAGE_SIZE)); if (is_vmalloc_addr(bp->b_addr)) vfree(bp->b_addr); else if (bp->b_flags & _XBF_KMEM) kfree(bp->b_addr); - else + else if (bp->b_addr) folio_put(virt_to_folio(bp->b_addr)); call_rcu(&bp->b_rcu, xfs_buf_free_callback); diff --git a/fs/xfs/xfs_buf_item_recover.c b/fs/xfs/xfs_buf_item_recover.c index e4c8af8736324..54cf93b2a196a 100644 --- a/fs/xfs/xfs_buf_item_recover.c +++ b/fs/xfs/xfs_buf_item_recover.c @@ -461,7 +461,7 @@ xlog_recover_validate_buf_type( * given buffer. The bitmap in the buf log format structure indicates * where to place the logged data. */ -STATIC void +STATIC int xlog_recover_do_reg_buffer( struct xfs_mount *mp, struct xlog_recover_item *item, @@ -489,8 +489,24 @@ xlog_recover_do_reg_buffer( ASSERT(nbits > 0); ASSERT(item->ri_buf[i].iov_base != NULL); ASSERT(item->ri_buf[i].iov_len % XFS_BLF_CHUNK == 0); - ASSERT(BBTOB(bp->b_length) >= - ((uint)bit << XFS_BLF_SHIFT) + (nbits << XFS_BLF_SHIFT)); + /* + * The bitmap is only trustworthy to the extent that it + * describes a region that actually fits inside the buffer we + * read in based on the (attacker-controlled) blf_len. Do not + * rely on an ASSERT() for this -- it compiles away entirely on + * non-DEBUG kernels, which is exactly where this matters, so + * validate it for real and abort recovery of this buffer rather + * than copying past the end of it. + */ + if (XFS_IS_CORRUPT(mp, BBTOB(bp->b_length) < + ((uint)bit << XFS_BLF_SHIFT) + + (nbits << XFS_BLF_SHIFT))) { + xfs_alert(mp, + "Bad buffer log item dirty bitmap (bit %d, nbits %d) for %d-byte buffer at daddr 0x%llx.", + bit, nbits, BBTOB(bp->b_length), + xfs_buf_daddr(bp)); + return -EFSCORRUPTED; + } /* * The dirty regions logged in the buffer, even though @@ -544,6 +560,7 @@ xlog_recover_do_reg_buffer( ASSERT(i == item->ri_total); xlog_recover_validate_buf_type(mp, bp, buf_f, current_lsn); + return 0; } /* @@ -552,10 +569,10 @@ xlog_recover_do_reg_buffer( * (ie. USR or GRP), then just toss this buffer away; don't recover it. * Else, treat it as a regular buffer and do recovery. * - * Return false if the buffer was tossed and true if we recovered the buffer to - * indicate to the caller if the buffer needs writing. + * Return 0 if the buffer was not recovered (tossed), 1 if it was recovered and + * needs writing, or a negative errno if recovery of the buffer failed. */ -STATIC bool +STATIC int xlog_recover_do_dquot_buffer( struct xfs_mount *mp, struct xlog *log, @@ -564,6 +581,7 @@ xlog_recover_do_dquot_buffer( struct xfs_buf_log_format *buf_f) { uint type; + int error; trace_xfs_log_recover_buf_dquot_buf(log, buf_f); @@ -571,7 +589,7 @@ xlog_recover_do_dquot_buffer( * Filesystems are required to send in quota flags at mount time. */ if (!mp->m_qflags) - return false; + return 0; type = 0; if (buf_f->blf_flags & XFS_BLF_UDQUOT_BUF) @@ -584,10 +602,12 @@ xlog_recover_do_dquot_buffer( * This type of quotas was turned off, so ignore this buffer */ if (log->l_quotaoffs_flag & type) - return false; + return 0; - xlog_recover_do_reg_buffer(mp, item, bp, buf_f, NULLCOMMITLSN); - return true; + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, NULLCOMMITLSN); + if (error) + return error; + return 1; } /* @@ -724,7 +744,9 @@ xlog_recover_do_primary_sb_buffer( xfs_rgnumber_t orig_rgcount = mp->m_sb.sb_rgcount; int error; - xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); + if (error) + return error; if (orig_agcount == 0) { xfs_alert(mp, "Trying to grow file system without AGs"); @@ -1081,11 +1103,11 @@ xlog_recover_buf_commit_pass2( goto out_release; } else if (buf_f->blf_flags & (XFS_BLF_UDQUOT_BUF|XFS_BLF_PDQUOT_BUF|XFS_BLF_GDQUOT_BUF)) { - bool dirty; - - dirty = xlog_recover_do_dquot_buffer(mp, log, item, bp, buf_f); - if (!dirty) + error = xlog_recover_do_dquot_buffer(mp, log, item, bp, buf_f); + if (error <= 0) goto out_release; + /* write dirty buffer */ + error = 0; } else if ((xfs_blft_from_flags(buf_f) & XFS_BLFT_SB_BUF) && xfs_buf_daddr(bp) == 0) { error = xlog_recover_do_primary_sb_buffer(mp, item, bp, buf_f, @@ -1105,7 +1127,10 @@ xlog_recover_buf_commit_pass2( xfs_buf_relse(rtsb_bp); } } else { - xlog_recover_do_reg_buffer(mp, item, bp, buf_f, current_lsn); + error = xlog_recover_do_reg_buffer(mp, item, bp, buf_f, + current_lsn); + if (error) + goto out_release; } /* diff --git a/fs/xfs/xfs_dquot.c b/fs/xfs/xfs_dquot.c index 896ff8080b75b..8c89f701537f8 100644 --- a/fs/xfs/xfs_dquot.c +++ b/fs/xfs/xfs_dquot.c @@ -778,7 +778,7 @@ xfs_dq_get_next_id( lock_flags = xfs_ilock_data_map_shared(quotip); error = xfs_iread_extents(NULL, quotip, XFS_DATA_FORK); if (error) - return error; + goto out_unlock; if (xfs_iext_lookup_extent(quotip, "ip->i_df, start, &cur, &got)) { /* contiguous chunk, bump startoff for the id calculation */ @@ -789,6 +789,7 @@ xfs_dq_get_next_id( error = -ENOENT; } +out_unlock: xfs_iunlock(quotip, lock_flags); return error; diff --git a/fs/xfs/xfs_dquot_item_recover.c b/fs/xfs/xfs_dquot_item_recover.c index 89bc9bcaf51e5..590412c8e6150 100644 --- a/fs/xfs/xfs_dquot_item_recover.c +++ b/fs/xfs/xfs_dquot_item_recover.c @@ -173,7 +173,7 @@ xlog_recover_dquot_commit_pass2( out_release: xfs_buf_relse(bp); - return 0; + return error; } const struct xlog_recover_item_ops xlog_dquot_item_ops = { diff --git a/fs/xfs/xfs_exchmaps_item.c b/fs/xfs/xfs_exchmaps_item.c index 229cbe0adf17f..debc102dab289 100644 --- a/fs/xfs/xfs_exchmaps_item.c +++ b/fs/xfs/xfs_exchmaps_item.c @@ -347,7 +347,17 @@ xfs_xmi_validate( if (!xfs_verify_fileext(mp, xlf->xmi_startoff1, xlf->xmi_blockcount)) return false; - return xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount); + if (!xfs_verify_fileext(mp, xlf->xmi_startoff2, xlf->xmi_blockcount)) + return false; + + if (xlf->xmi_flags & XFS_EXCHMAPS_SET_SIZES) { + if ((int64_t)xlf->xmi_isize1 < 0) + return false; + if ((int64_t)xlf->xmi_isize2 < 0) + return false; + } + + return true; } /* @@ -406,6 +416,13 @@ xfs_xmi_item_recover_intent( *ipp1 = ip1; *ipp2 = ip2; xmi = xfs_exchmaps_init_intent(req); + + /* Restore intended file sizes from recovered logged item */ + if (req->flags & XFS_EXCHMAPS_SET_SIZES) { + xmi->xmi_isize1 = xlf->xmi_isize1; + xmi->xmi_isize2 = xlf->xmi_isize2; + } + xfs_defer_add_item(dfp, &xmi->xmi_list); return xmi; diff --git a/fs/xfs/xfs_exchrange.c b/fs/xfs/xfs_exchrange.c index 0b41bdfecdfbc..e0f092ff77e9b 100644 --- a/fs/xfs/xfs_exchrange.c +++ b/fs/xfs/xfs_exchrange.c @@ -504,6 +504,9 @@ xfs_exchange_range_finish( { int error; + if (fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN) + return 0; + error = file_remove_privs(fxr->file1); if (error) return error; @@ -783,9 +786,12 @@ xfs_exchange_range( if (ret) return ret; - fsnotify_modify(fxr->file1); - if (fxr->file2 != fxr->file1) - fsnotify_modify(fxr->file2); + if (!(fxr->flags & XFS_EXCHANGE_RANGE_DRY_RUN)) { + fsnotify_modify(fxr->file1); + if (fxr->file2 != fxr->file1) + fsnotify_modify(fxr->file2); + } + return 0; } diff --git a/fs/xfs/xfs_rtalloc.c b/fs/xfs/xfs_rtalloc.c index 907c88034c8dc..365cfe22c0eba 100644 --- a/fs/xfs/xfs_rtalloc.c +++ b/fs/xfs/xfs_rtalloc.c @@ -734,7 +734,7 @@ xfs_rtginode_ensure( xfs_trans_cancel(tp); if (error != -ENOENT) - return 0; + return error; return xfs_rtginode_create(rtg, type, true); } diff --git a/include/acpi/processor.h b/include/acpi/processor.h index d0eccbd920e5c..ff864c1cee3a4 100644 --- a/include/acpi/processor.h +++ b/include/acpi/processor.h @@ -423,6 +423,8 @@ int acpi_processor_power_init(struct acpi_processor *pr); int acpi_processor_power_exit(struct acpi_processor *pr); int acpi_processor_power_state_has_changed(struct acpi_processor *pr); int acpi_processor_hotplug(struct acpi_processor *pr); +void acpi_processor_register_idle_driver(void); +void acpi_processor_unregister_idle_driver(void); #else static inline int acpi_processor_power_init(struct acpi_processor *pr) { diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index b89c7e3a20474..8862f77241e6c 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -62,6 +62,8 @@ struct ms_hyperv_info { }; }; u64 shared_gpa_boundary; + bool msi_ext_dest_id; + bool confidential_vmbus_available; }; extern struct ms_hyperv_info ms_hyperv; extern bool hv_nested; diff --git a/include/drm/bridge/dw_dp.h b/include/drm/bridge/dw_dp.h index d05df49fd8846..22105c3e8e4d6 100644 --- a/include/drm/bridge/dw_dp.h +++ b/include/drm/bridge/dw_dp.h @@ -11,10 +11,18 @@ struct drm_encoder; struct dw_dp; +enum { + DW_DP_MP_SINGLE_PIXEL, + DW_DP_MP_DUAL_PIXEL, + DW_DP_MP_QUAD_PIXEL, +}; + struct dw_dp_plat_data { u32 max_link_rate; + u8 pixel_mode; }; struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder, const struct dw_dp_plat_data *plat_data); +void dw_dp_unbind(struct dw_dp *dp); #endif /* __DW_DP__ */ diff --git a/include/drm/drm_buddy.h b/include/drm/drm_buddy.h index c2e05a2812522..3054369bebffb 100644 --- a/include/drm/drm_buddy.h +++ b/include/drm/drm_buddy.h @@ -6,166 +6,13 @@ #ifndef __DRM_BUDDY_H__ #define __DRM_BUDDY_H__ -#include -#include -#include -#include -#include +#include -#include +struct drm_printer; -#define DRM_BUDDY_RANGE_ALLOCATION BIT(0) -#define DRM_BUDDY_TOPDOWN_ALLOCATION BIT(1) -#define DRM_BUDDY_CONTIGUOUS_ALLOCATION BIT(2) -#define DRM_BUDDY_CLEAR_ALLOCATION BIT(3) -#define DRM_BUDDY_CLEARED BIT(4) -#define DRM_BUDDY_TRIM_DISABLE BIT(5) - -struct drm_buddy_block { -#define DRM_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) -#define DRM_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) -#define DRM_BUDDY_ALLOCATED (1 << 10) -#define DRM_BUDDY_FREE (2 << 10) -#define DRM_BUDDY_SPLIT (3 << 10) -#define DRM_BUDDY_HEADER_CLEAR GENMASK_ULL(9, 9) -/* Free to be used, if needed in the future */ -#define DRM_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6) -#define DRM_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0) - u64 header; - - struct drm_buddy_block *left; - struct drm_buddy_block *right; - struct drm_buddy_block *parent; - - void *private; /* owned by creator */ - - /* - * While the block is allocated by the user through drm_buddy_alloc*, - * the user has ownership of the link, for example to maintain within - * a list, if so desired. As soon as the block is freed with - * drm_buddy_free* ownership is given back to the mm. - */ - union { - struct rb_node rb; - struct list_head link; - }; - - struct list_head tmp_link; -}; - -/* Order-zero must be at least SZ_4K */ -#define DRM_BUDDY_MAX_ORDER (63 - 12) - -/* - * Binary Buddy System. - * - * Locking should be handled by the user, a simple mutex around - * drm_buddy_alloc* and drm_buddy_free* should suffice. - */ -struct drm_buddy { - /* Maintain a free list for each order. */ - struct rb_root **free_trees; - - /* - * Maintain explicit binary tree(s) to track the allocation of the - * address space. This gives us a simple way of finding a buddy block - * and performing the potentially recursive merge step when freeing a - * block. Nodes are either allocated or free, in which case they will - * also exist on the respective free list. - */ - struct drm_buddy_block **roots; - - /* - * Anything from here is public, and remains static for the lifetime of - * the mm. Everything above is considered do-not-touch. - */ - unsigned int n_roots; - unsigned int max_order; - - /* Must be at least SZ_4K */ - u64 chunk_size; - u64 size; - u64 avail; - u64 clear_avail; -}; - -static inline u64 -drm_buddy_block_offset(const struct drm_buddy_block *block) -{ - return block->header & DRM_BUDDY_HEADER_OFFSET; -} - -static inline unsigned int -drm_buddy_block_order(struct drm_buddy_block *block) -{ - return block->header & DRM_BUDDY_HEADER_ORDER; -} - -static inline unsigned int -drm_buddy_block_state(struct drm_buddy_block *block) -{ - return block->header & DRM_BUDDY_HEADER_STATE; -} - -static inline bool -drm_buddy_block_is_allocated(struct drm_buddy_block *block) -{ - return drm_buddy_block_state(block) == DRM_BUDDY_ALLOCATED; -} - -static inline bool -drm_buddy_block_is_clear(struct drm_buddy_block *block) -{ - return block->header & DRM_BUDDY_HEADER_CLEAR; -} - -static inline bool -drm_buddy_block_is_free(struct drm_buddy_block *block) -{ - return drm_buddy_block_state(block) == DRM_BUDDY_FREE; -} - -static inline bool -drm_buddy_block_is_split(struct drm_buddy_block *block) -{ - return drm_buddy_block_state(block) == DRM_BUDDY_SPLIT; -} - -static inline u64 -drm_buddy_block_size(struct drm_buddy *mm, - struct drm_buddy_block *block) -{ - return mm->chunk_size << drm_buddy_block_order(block); -} - -int drm_buddy_init(struct drm_buddy *mm, u64 size, u64 chunk_size); - -void drm_buddy_fini(struct drm_buddy *mm); - -struct drm_buddy_block * -drm_get_buddy(struct drm_buddy_block *block); - -int drm_buddy_alloc_blocks(struct drm_buddy *mm, - u64 start, u64 end, u64 size, - u64 min_page_size, - struct list_head *blocks, - unsigned long flags); - -int drm_buddy_block_trim(struct drm_buddy *mm, - u64 *start, - u64 new_size, - struct list_head *blocks); - -void drm_buddy_reset_clear(struct drm_buddy *mm, bool is_clear); - -void drm_buddy_free_block(struct drm_buddy *mm, struct drm_buddy_block *block); - -void drm_buddy_free_list(struct drm_buddy *mm, - struct list_head *objects, - unsigned int flags); - -void drm_buddy_print(struct drm_buddy *mm, struct drm_printer *p); -void drm_buddy_block_print(struct drm_buddy *mm, - struct drm_buddy_block *block, +/* DRM-specific GPU Buddy Allocator print helpers */ +void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p); +void drm_buddy_block_print(struct gpu_buddy *mm, + struct gpu_buddy_block *block, struct drm_printer *p); #endif diff --git a/include/drm/drm_client.h b/include/drm/drm_client.h index 146ca80e35db6..8e2e51f984306 100644 --- a/include/drm/drm_client.h +++ b/include/drm/drm_client.h @@ -173,20 +173,12 @@ struct drm_client_buffer { */ struct drm_client_dev *client; - /** - * @pitch: Buffer pitch - */ - u32 pitch; - /** * @gem: GEM object backing this buffer * - * FIXME: The dependency on GEM here isn't required, we could - * convert the driver handle to a dma-buf instead and use the - * backend-agnostic dma-buf vmap support instead. This would - * require that the handle2fd prime ioctl is reworked to pull the - * fd_install step out of the driver backend hooks, to make that - * final step optional for internal users. + * FIXME: The DRM framebuffer holds a reference on its GEM + * buffer objects. Do not use this field in new code and + * update existing users. */ struct drm_gem_object *gem; @@ -203,7 +195,7 @@ struct drm_client_buffer { struct drm_client_buffer * drm_client_framebuffer_create(struct drm_client_dev *client, u32 width, u32 height, u32 format); -void drm_client_framebuffer_delete(struct drm_client_buffer *buffer); +void drm_client_buffer_delete(struct drm_client_buffer *buffer); int drm_client_framebuffer_flush(struct drm_client_buffer *buffer, struct drm_rect *rect); int drm_client_buffer_vmap_local(struct drm_client_buffer *buffer, struct iosys_map *map_copy); diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index bd0fe8a8b9ef5..1ea4cf8b61e29 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -2513,13 +2513,13 @@ struct drm_tile_group { struct kref refcount; struct drm_device *dev; int id; - u8 group_data[8]; + u8 group_data[9]; }; struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev, - const char topology[8]); + const char topology_id[9]); struct drm_tile_group *drm_mode_get_tile_group(struct drm_device *dev, - const char topology[8]); + const char topology_id[9]); void drm_mode_put_tile_group(struct drm_device *dev, struct drm_tile_group *tg); diff --git a/include/drm/drm_exec.h b/include/drm/drm_exec.h index aa786b828a0a4..dee6ebdbe416f 100644 --- a/include/drm/drm_exec.h +++ b/include/drm/drm_exec.h @@ -65,31 +65,35 @@ drm_exec_obj(struct drm_exec *exec, unsigned long index) return index < exec->num_objects ? exec->objects[index] : NULL; } +/* Helper for drm_exec_for_each_locked_object(). Internal use only. */ +#define __drm_exec_for_each_locked_object(exec, obj, __index) \ + for (unsigned long __index = 0; ((obj) = drm_exec_obj(exec, __index)); ++__index) /** * drm_exec_for_each_locked_object - iterate over all the locked objects * @exec: drm_exec object - * @index: unsigned long index for the iteration * @obj: the current GEM object * * Iterate over all the locked GEM objects inside the drm_exec object. */ -#define drm_exec_for_each_locked_object(exec, index, obj) \ - for ((index) = 0; ((obj) = drm_exec_obj(exec, index)); ++(index)) +#define drm_exec_for_each_locked_object(exec, obj) \ + __drm_exec_for_each_locked_object(exec, obj, __UNIQUE_ID(drm_exec)) +/* Helper for drm_exec_for_each_locked_object_reverse(). Internal use only. */ +#define __drm_exec_for_each_locked_object_reverse(exec, obj, __index) \ + for (unsigned long __index = (exec)->num_objects - 1; \ + ((obj) = drm_exec_obj(exec, __index)); --__index) /** * drm_exec_for_each_locked_object_reverse - iterate over all the locked * objects in reverse locking order * @exec: drm_exec object - * @index: unsigned long index for the iteration * @obj: the current GEM object * * Iterate over all the locked GEM objects inside the drm_exec object in - * reverse locking order. Note that @index may go below zero and wrap, + * reverse locking order. Note that the internal index may wrap around, * but that will be caught by drm_exec_obj(), returning a NULL object. */ -#define drm_exec_for_each_locked_object_reverse(exec, index, obj) \ - for ((index) = (exec)->num_objects - 1; \ - ((obj) = drm_exec_obj(exec, index)); --(index)) +#define drm_exec_for_each_locked_object_reverse(exec, obj) \ + __drm_exec_for_each_locked_object_reverse(exec, obj, __UNIQUE_ID(drm_exec)) /** * drm_exec_until_all_locked - loop until all GEM objects are locked diff --git a/include/drm/drm_fb_helper.h b/include/drm/drm_fb_helper.h index c1d38d54a1120..2faf8aaf98dc7 100644 --- a/include/drm/drm_fb_helper.h +++ b/include/drm/drm_fb_helper.h @@ -256,8 +256,6 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var, int drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper); -struct fb_info *drm_fb_helper_alloc_info(struct drm_fb_helper *fb_helper); -void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper); void drm_fb_helper_unregister_info(struct drm_fb_helper *fb_helper); void drm_fb_helper_fill_info(struct fb_info *info, struct drm_fb_helper *fb_helper, @@ -340,16 +338,6 @@ drm_fb_helper_restore_fbdev_mode_unlocked(struct drm_fb_helper *fb_helper) return 0; } -static inline struct fb_info * -drm_fb_helper_alloc_info(struct drm_fb_helper *fb_helper) -{ - return NULL; -} - -static inline void drm_fb_helper_release_info(struct drm_fb_helper *fb_helper) -{ -} - static inline void drm_fb_helper_unregister_info(struct drm_fb_helper *fb_helper) { } diff --git a/include/drm/drm_utils.h b/include/drm/drm_utils.h index 6a46f755daba0..7e077484c5bbf 100644 --- a/include/drm/drm_utils.h +++ b/include/drm/drm_utils.h @@ -19,6 +19,7 @@ int drm_get_panel_orientation_quirk(int width, int height); struct drm_panel_backlight_quirk { u16 min_brightness; u32 brightness_mask; + bool force_pwm; }; const struct drm_panel_backlight_quirk * diff --git a/include/hyperv/hvhdk.h b/include/hyperv/hvhdk.h index 1057455b84f28..cdc74199ba3fb 100644 --- a/include/hyperv/hvhdk.h +++ b/include/hyperv/hvhdk.h @@ -540,9 +540,15 @@ union hv_interrupt_control { u64 as_uint64; struct { u32 interrupt_type; /* enum hv_interrupt_type */ +#if IS_ENABLED(CONFIG_X86) u32 level_triggered : 1; u32 logical_dest_mode : 1; u32 rsvd : 30; +#elif IS_ENABLED(CONFIG_ARM64) + u32 rsvd1 : 2; + u32 asserted : 1; + u32 rsvd2 : 29; +#endif } __packed; }; diff --git a/include/hyperv/hvhdk_mini.h b/include/hyperv/hvhdk_mini.h index 858f6a3925b30..45827d49c9a50 100644 --- a/include/hyperv/hvhdk_mini.h +++ b/include/hyperv/hvhdk_mini.h @@ -126,8 +126,9 @@ enum hv_dynamic_processor_feature_property { struct hv_input_get_system_property { u32 property_id; /* enum hv_system_property */ + u32 reserved; union { - u32 as_uint32; + u64 as_uint64; #if IS_ENABLED(CONFIG_X86) /* enum hv_dynamic_processor_feature_property */ u32 hv_processor_feature; diff --git a/include/kunit/run-in-irq-context.h b/include/kunit/run-in-irq-context.h index bfe60d6cf28d8..3802b6fb218ed 100644 --- a/include/kunit/run-in-irq-context.h +++ b/include/kunit/run-in-irq-context.h @@ -38,11 +38,13 @@ static enum hrtimer_restart kunit_irq_test_timer_func(struct hrtimer *timer) softirq_calls = atomic_read(&state->softirq_func_calls); /* - * If the timer is firing too often for the softirq or task to ever have - * a chance to run, increase the timer interval. This is needed on very - * slow systems. + * If the hrtimer is running much faster than the bh_work or the task, + * then it is firing too fast and might be starving those contexts as + * well as the actual system timer tick. Increase the interval. */ - if (hardirq_calls >= 20 && (softirq_calls == 0 || task_calls == 0)) + if (hardirq_calls >= 20 && + (hardirq_calls / 2 > softirq_calls || + hardirq_calls / 2 > task_calls)) state->interval = ktime_add_ns(state->interval, 250); if (!state->func(state->test_specific_state)) diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 5ff5d99f6ead5..8871f2edca712 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -779,6 +779,18 @@ const char *acpi_get_subsystem_id(acpi_handle handle); int acpi_mrrm_max_mem_region(void); #endif +#define ACPI_CMOS_RTC_IDS \ + { "PNP0B00", }, \ + { "PNP0B01", }, \ + { "PNP0B02", }, \ + { "", } + +#ifdef CONFIG_X86 +extern bool cmos_rtc_platform_device_present; +#else +#define cmos_rtc_platform_device_present false +#endif + #else /* !CONFIG_ACPI */ #define acpi_disabled 1 @@ -1104,6 +1116,8 @@ static inline int acpi_mrrm_max_mem_region(void) return 1; } +#define cmos_rtc_platform_device_present false + #endif /* !CONFIG_ACPI */ #ifdef CONFIG_ACPI_HMAT diff --git a/include/linux/adi-axi-common.h b/include/linux/adi-axi-common.h index 37962ba530dfc..e7ba393061ee6 100644 --- a/include/linux/adi-axi-common.h +++ b/include/linux/adi-axi-common.h @@ -51,6 +51,7 @@ enum adi_axi_fpga_technology { ADI_AXI_FPGA_TECH_SERIES7, ADI_AXI_FPGA_TECH_ULTRASCALE, ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS, + ADI_AXI_FPGA_TECH_VERSAL, }; enum adi_axi_fpga_family { @@ -71,6 +72,7 @@ enum adi_axi_fpga_speed_grade { ADI_AXI_FPGA_SPEED_2 = 20, ADI_AXI_FPGA_SPEED_2L = 21, ADI_AXI_FPGA_SPEED_2LV = 22, + ADI_AXI_FPGA_SPEED_2MP = 23, ADI_AXI_FPGA_SPEED_3 = 30, }; diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index 02de2ede560f3..7e7cdc7612bed 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -134,6 +134,8 @@ static inline bool mem_alloc_profiling_enabled(void) &mem_alloc_profiling_key); } +bool mem_alloc_profiling_permanently_disabled(void); + static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) { struct alloc_tag_counters v = { 0, 0 }; @@ -239,6 +241,7 @@ static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) #define DEFINE_ALLOC_TAG(_alloc_tag) static inline bool mem_alloc_profiling_enabled(void) { return false; } +static inline bool mem_alloc_profiling_permanently_disabled(void) { return true; } static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) {} static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index 81e603839c4a5..62d67dae8b703 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -445,7 +445,7 @@ ffa_mem_desc_offset(struct ffa_mem_region *buf, int count, u32 ffa_version) if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(ffa_version)) offset += offsetof(struct ffa_mem_region, ep_mem_offset); else - offset += sizeof(struct ffa_mem_region); + offset += buf->ep_mem_offset; return offset; } diff --git a/include/linux/audit.h b/include/linux/audit.h index b8d8029c6c480..09d06684174ba 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -16,7 +16,7 @@ #include #include -#define AUDIT_INO_UNSET ((unsigned long)-1) +#define AUDIT_INO_UNSET ((u64)-1) #define AUDIT_DEV_UNSET ((dev_t)-1) struct audit_sig_info { @@ -125,8 +125,8 @@ enum audit_nfcfgop { AUDIT_NFT_OP_INVALID, }; -extern int __init audit_register_class(int class, unsigned *list); -extern int audit_classify_syscall(int abi, unsigned syscall); +extern int __init audit_register_class(int class, unsigned int *list); +extern int audit_classify_syscall(int abi, unsigned int syscall); extern int audit_classify_arch(int arch); /* audit_names->type values */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 2b8153791e6a5..a35069a6c15de 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -21,13 +21,13 @@ enum auditsc_class_t { AUDITSC_NVALS /* count */ }; -extern int audit_classify_compat_syscall(int abi, unsigned syscall); +extern int audit_classify_compat_syscall(int abi, unsigned int syscall); /* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; +extern unsigned int compat_write_class[]; +extern unsigned int compat_read_class[]; +extern unsigned int compat_dir_class[]; +extern unsigned int compat_chattr_class[]; +extern unsigned int compat_signal_class[]; #endif diff --git a/include/linux/bio.h b/include/linux/bio.h index 16c1c85613b76..fb747b053b98c 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -324,6 +324,8 @@ extern struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs); int bio_split_io_at(struct bio *bio, const struct queue_limits *lim, unsigned *segs, unsigned max_bytes, unsigned len_align); +u8 bio_seg_gap(struct request_queue *q, struct bio *prev, struct bio *next, + u8 gaps_bit); /** * bio_next_split - get next @sectors from a bio, splitting if necessary @@ -376,17 +378,27 @@ void submit_bio(struct bio *bio); extern void bio_endio(struct bio *); -static inline void bio_io_error(struct bio *bio) +/** + * bio_endio_status - end I/O on a bio with a specific status + * @bio: bio + * @status: status to set + * + * Set @bio->bi_status to @status and call bio_endio(). + **/ +static inline void bio_endio_status(struct bio *bio, blk_status_t status) { - bio->bi_status = BLK_STS_IOERR; + bio->bi_status = status; bio_endio(bio); } +static inline void bio_io_error(struct bio *bio) +{ + bio_endio_status(bio, BLK_STS_IOERR); +} + static inline void bio_wouldblock_error(struct bio *bio) { - bio_set_flag(bio, BIO_QUIET); - bio->bi_status = BLK_STS_AGAIN; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_AGAIN); } /* @@ -678,20 +690,6 @@ static inline bool bioset_initialized(struct bio_set *bs) return bs->bio_slab != NULL; } -/* - * Mark a bio as polled. Note that for async polled IO, the caller must - * expect -EWOULDBLOCK if we cannot allocate a request (or other resources). - * We cannot block waiting for requests on polled IO, as those completions - * must be found by the caller. This is different than IRQ driven IO, where - * it's safe to wait for IO to complete. - */ -static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) -{ - bio->bi_opf |= REQ_POLLED; - if (kiocb->ki_flags & IOCB_NOWAIT) - bio->bi_opf |= REQ_NOWAIT; -} - static inline void bio_clear_polled(struct bio *bio) { bio->bi_opf &= ~REQ_POLLED; diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 5355f8f806a97..2b3127e712b93 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -42,7 +42,7 @@ * FIELD_MODIFY(REG_FIELD_C, ®, c); */ -#define __bf_shf(x) (__builtin_ffsll(x) - 1) +#define __bf_shf __builtin_ctzll #define __scalar_type_to_unsigned_cases(type) \ unsigned type: (unsigned type)0, \ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index b25d12545f46d..b54506b3b76d9 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -152,6 +152,14 @@ struct request { unsigned short nr_phys_segments; unsigned short nr_integrity_segments; + /* + * The lowest set bit for address gaps between physical segments. This + * provides information necessary for dma optimization opprotunities, + * like for testing if the segments can be coalesced against the + * device's iommu granule. + */ + unsigned char phys_gap_bit; + #ifdef CONFIG_BLK_INLINE_ENCRYPTION struct bio_crypt_ctx *crypt_ctx; struct blk_crypto_keyslot *crypt_keyslot; @@ -208,6 +216,14 @@ struct request { void *end_io_data; }; +/* + * Returns a mask with all bits starting at req->phys_gap_bit set to 1. + */ +static inline unsigned long req_phys_gap_mask(const struct request *req) +{ + return ~(((1 << req->phys_gap_bit) >> 1) - 1); +} + static inline enum req_op req_op(const struct request *req) { return req->cmd_flags & REQ_OP_MASK; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 4e2e3aed32f5f..47fe5631d9e2a 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -218,6 +218,18 @@ struct bio { enum rw_hint bi_write_hint; u8 bi_write_stream; blk_status_t bi_status; + + /* + * The bvec gap bit indicates the lowest set bit in any address offset + * between all bi_io_vecs. This field is initialized only after the bio + * is split to the hardware limits (see bio_split_io_at()). The value + * may be used to consider DMA optimization when performing that + * mapping. The value is compared to a power of two mask where the + * result depends on any bit set within the mask, so saving the lowest + * bit is sufficient to know if any segment gap collides with the mask. + */ + u8 bi_bvec_gap_bit; + atomic_t __bi_remaining; struct bvec_iter bi_iter; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h b/include/linux/bnxt/ulp.h similarity index 100% rename from drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h rename to include/linux/bnxt/ulp.h diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h index 25df9260d206d..1f6e9c54a2a5e 100644 --- a/include/linux/bootconfig.h +++ b/include/linux/bootconfig.h @@ -265,6 +265,9 @@ static inline struct xbc_node * __init xbc_node_get_subkey(struct xbc_node *node int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node *node, char *buf, size_t size); +/* Render key/value pairs under @root as a flat cmdline string */ +int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root); + /** * xbc_node_compose_key() - Compose full key string of the XBC node * @node: An XBC node. diff --git a/include/linux/bpf.h b/include/linux/bpf.h index e264c695f31ce..862d6e0d5a287 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -536,7 +536,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -559,7 +559,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) @@ -821,6 +821,7 @@ enum bpf_arg_type { ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ + ARG_SCALAR, /* scalar argument */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ @@ -1044,21 +1045,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 4867b0e8b5d9d..d305d30ebe95c 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -305,6 +305,8 @@ struct bpf_func_state { * | number of simulations is tracked in frame N */ u32 callback_depth; + /* Instructions processed in this frame and callees on the current path. */ + u32 insns_subtotal; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -648,6 +650,8 @@ struct bpf_subprog_info { u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; + u32 insns_total; + u32 insns_self; /* offsets in range [stack_depth .. fastcall_stack_off) * are used for bpf_fastcall spills and fills. */ @@ -794,6 +798,8 @@ struct bpf_verifier_env { u32 prev_insn_processed, insn_processed; /* number of jmps, calls, exits analyzed so far */ u32 prev_jmps_processed, jmps_processed; + /* maximum combined stack depth */ + u32 max_stack_depth; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -967,7 +973,9 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type) static inline bool type_is_ptr_alloc_obj(u32 type) { - return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; + return base_type(type) == PTR_TO_BTF_ID && + type_flag(type) & MEM_ALLOC && + !(type_flag(type) & PTR_UNTRUSTED); } static inline bool type_is_non_owning_ref(u32 type) @@ -1051,12 +1059,14 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie u32 frameno, bool print_all); void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); +u32 bpf_vlog_alignment(u32 pos); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); int bpf_jmp_offset(struct bpf_insn *insn); int bpf_insn_successors(struct bpf_prog *prog, u32 idx, u32 succ[2]); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx); +bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); diff --git a/include/linux/can/core.h b/include/linux/can/core.h index 5fb8d0e3f9c1d..3287232e3cad9 100644 --- a/include/linux/can/core.h +++ b/include/linux/can/core.h @@ -58,6 +58,7 @@ extern void can_rx_unregister(struct net *net, struct net_device *dev, void *data); extern int can_send(struct sk_buff *skb, int loop); +void can_set_skb_uid(struct sk_buff *skb); void can_sock_destruct(struct sock *sk); #endif /* !_CAN_CORE_H */ diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h index 1abc25a8d144e..869ea574a40a1 100644 --- a/include/linux/can/skb.h +++ b/include/linux/can/skb.h @@ -49,13 +49,11 @@ bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb); /** * struct can_skb_priv - private additional data inside CAN sk_buffs * @ifindex: ifindex of the first interface the CAN frame appeared on - * @skbcnt: atomic counter to have an unique id together with skb pointer * @frame_len: length of CAN frame in data link layer * @cf: align to the following CAN frame at skb->data */ struct can_skb_priv { int ifindex; - int skbcnt; unsigned int frame_len; struct can_frame cf[]; }; diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index a1194e44b5276..bacefa0f1512f 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -261,6 +261,10 @@ const volatile void * __must_check_fn(const volatile void *val) * CLASS(name, var)(args...): * declare the variable @var as an instance of the named class * + * CLASS_INIT(name, var, init_expr): + * declare the variable @var as an instance of the named class with + * custom initialization expression. + * * Ex. * * DEFINE_CLASS(fdget, struct fd, fdput(_T), fdget(fd), int fd) @@ -290,6 +294,9 @@ static inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \ class_##_name##_t var __cleanup(class_##_name##_destructor) = \ class_##_name##_constructor +#define CLASS_INIT(_name, _var, _init_expr) \ + class_##_name##_t _var __cleanup(class_##_name##_destructor) = (_init_expr) + #define __scoped_class(_name, var, _label, args...) \ for (CLASS(_name, var)(args); ; ({ goto _label; })) \ if (0) { \ diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 173d9c07a8952..ff2d5fb29161e 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -56,6 +56,7 @@ enum compact_result { }; struct alloc_context; /* in mm/internal.h */ +struct capture_control; /* in mm/internal.h */ /* * Number of free order-0 pages that should be available above given watermark @@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, enum compact_priority prio, - struct page **page); + struct capture_control *capc); extern void reset_isolation_suitable(pg_data_t *pgdat); extern bool compaction_suitable(struct zone *zone, int order, unsigned long watermark, int highest_zoneidx); diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h new file mode 100644 index 0000000000000..f8af63045281d --- /dev/null +++ b/include/linux/compiler-context-analysis.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Macros and attributes for compiler-based static context analysis. + */ + +#ifndef _LINUX_COMPILER_CONTEXT_ANALYSIS_H +#define _LINUX_COMPILER_CONTEXT_ANALYSIS_H + +#ifdef __CHECKER__ + +/* Sparse context/lock checking support. */ +# define __must_hold(x) __attribute__((context(x,1,1))) +# define __acquires(x) __attribute__((context(x,0,1))) +# define __cond_acquires(x) __attribute__((context(x,0,-1))) +# define __releases(x) __attribute__((context(x,1,0))) +# define __acquire(x) __context__(x,1) +# define __release(x) __context__(x,-1) +# define __cond_lock(x, c) ((c) ? ({ __acquire(x); 1; }) : 0) + +#else /* !__CHECKER__ */ + +# define __must_hold(x) +# define __acquires(x) +# define __cond_acquires(x) +# define __releases(x) +# define __acquire(x) (void)0 +# define __release(x) (void)0 +# define __cond_lock(x, c) (c) + +#endif /* __CHECKER__ */ + +#endif /* _LINUX_COMPILER_CONTEXT_ANALYSIS_H */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 2f18b8a01afe6..ba49413f704be 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -37,6 +37,8 @@ # define BTF_TYPE_TAG(value) /* nothing */ #endif +#include + /* sparse defines __CHECKER__; see Documentation/dev-tools/sparse.rst */ #ifdef __CHECKER__ /* address spaces */ @@ -47,14 +49,6 @@ # define __rcu __attribute__((noderef, address_space(__rcu))) static inline void __chk_user_ptr(const volatile void __user *ptr) { } static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } -/* context/locking */ -# define __must_hold(x) __attribute__((context(x,1,1))) -# define __acquires(x) __attribute__((context(x,0,1))) -# define __cond_acquires(x) __attribute__((context(x,0,-1))) -# define __releases(x) __attribute__((context(x,1,0))) -# define __acquire(x) __context__(x,1) -# define __release(x) __context__(x,-1) -# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) /* other */ # define __force __attribute__((force)) # define __nocast __attribute__((nocast)) @@ -75,14 +69,6 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } # define __chk_user_ptr(x) (void)0 # define __chk_io_ptr(x) (void)0 -/* context/locking */ -# define __must_hold(x) -# define __acquires(x) -# define __cond_acquires(x) -# define __releases(x) -# define __acquire(x) (void)0 -# define __release(x) (void)0 -# define __cond_lock(x,c) (c) /* other */ # define __force # define __nocast diff --git a/include/linux/cred.h b/include/linux/cred.h index 89ae50ad2acea..be2cd07b174c5 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h @@ -20,6 +20,8 @@ struct cred; struct inode; +extern struct task_struct init_task; + /* * COW Supplementary groups list */ @@ -156,6 +158,11 @@ extern struct cred *prepare_exec_creds(void); extern int commit_creds(struct cred *); extern void abort_creds(struct cred *); extern struct cred *prepare_kernel_cred(struct task_struct *); +static inline const struct cred *kernel_cred(void) +{ + /* shut up sparse */ + return rcu_dereference_raw(init_task.cred); +} extern int set_security_override(struct cred *, u32); extern int set_security_override_from_ctx(struct cred *, const char *); extern int set_create_files_as(struct cred *, struct inode *); @@ -180,6 +187,14 @@ static inline const struct cred *revert_creds(const struct cred *revert_cred) return rcu_replace_pointer(current->cred, revert_cred, 1); } +DEFINE_CLASS(override_creds, + const struct cred *, + revert_creds(_T), + override_creds(override_cred), const struct cred *override_cred) + +#define scoped_with_kernel_creds() \ + scoped_class(override_creds, __UNIQUE_ID(cred), kernel_cred()) + /** * get_cred_many - Get references on a set of credentials * @cred: The credentials to reference diff --git a/include/linux/designware_i2c.h b/include/linux/designware_i2c.h new file mode 100644 index 0000000000000..95eac4430427c --- /dev/null +++ b/include/linux/designware_i2c.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Synopsys DesignWare I2C register definitions + * + * Copyright (C) 2026, Intel Corporation + */ + +#ifndef __LINUX_DESIGNWARE_I2C_H +#define __LINUX_DESIGNWARE_I2C_H + +#include + +/* + * Registers offset + */ +#define DW_IC_CON 0x00 +#define DW_IC_TAR 0x04 +#define DW_IC_SAR 0x08 +#define DW_IC_DATA_CMD 0x10 +#define DW_IC_SS_SCL_HCNT 0x14 +#define DW_IC_SS_SCL_LCNT 0x18 +#define DW_IC_FS_SCL_HCNT 0x1c +#define DW_IC_FS_SCL_LCNT 0x20 +#define DW_IC_HS_SCL_HCNT 0x24 +#define DW_IC_HS_SCL_LCNT 0x28 +#define DW_IC_INTR_STAT 0x2c +#define DW_IC_INTR_MASK 0x30 +#define DW_IC_RAW_INTR_STAT 0x34 +#define DW_IC_RX_TL 0x38 +#define DW_IC_TX_TL 0x3c +#define DW_IC_CLR_INTR 0x40 +#define DW_IC_CLR_RX_UNDER 0x44 +#define DW_IC_CLR_RX_OVER 0x48 +#define DW_IC_CLR_TX_OVER 0x4c +#define DW_IC_CLR_RD_REQ 0x50 +#define DW_IC_CLR_TX_ABRT 0x54 +#define DW_IC_CLR_RX_DONE 0x58 +#define DW_IC_CLR_ACTIVITY 0x5c +#define DW_IC_CLR_STOP_DET 0x60 +#define DW_IC_CLR_START_DET 0x64 +#define DW_IC_CLR_GEN_CALL 0x68 +#define DW_IC_ENABLE 0x6c +#define DW_IC_STATUS 0x70 +#define DW_IC_TXFLR 0x74 +#define DW_IC_RXFLR 0x78 +#define DW_IC_SDA_HOLD 0x7c +#define DW_IC_TX_ABRT_SOURCE 0x80 +#define DW_IC_ENABLE_STATUS 0x9c +#define DW_IC_CLR_RESTART_DET 0xa8 +#define DW_IC_SCL_STUCK_AT_LOW_TIMEOUT 0xac +#define DW_IC_SDA_STUCK_AT_LOW_TIMEOUT 0xb0 +#define DW_IC_SMBUS_INTR_STAT 0xc8 +#define DW_IC_SMBUS_INTR_MASK 0xcc +#define DW_IC_CLR_SMBUS_INTR 0xd4 +#define DW_IC_COMP_PARAM_1 0xf4 +#define DW_IC_COMP_VERSION 0xf8 +#define DW_IC_COMP_TYPE 0xfc + +/* DW_IC_CON bits */ +#define DW_IC_CON_MASTER BIT(0) +#define DW_IC_CON_SPEED_STD (1 << 1) +#define DW_IC_CON_SPEED_FAST (2 << 1) +#define DW_IC_CON_SPEED_HIGH (3 << 1) +#define DW_IC_CON_SPEED_MASK GENMASK(2, 1) +#define DW_IC_CON_10BITADDR_SLAVE BIT(3) +#define DW_IC_CON_10BITADDR_MASTER BIT(4) +#define DW_IC_CON_RESTART_EN BIT(5) +#define DW_IC_CON_SLAVE_DISABLE BIT(6) +#define DW_IC_CON_STOP_DET_IFADDRESSED BIT(7) +#define DW_IC_CON_TX_EMPTY_CTRL BIT(8) +#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) +#define DW_IC_CON_BUS_CLEAR_CTRL BIT(11) + +/* DW_IC_DATA_CMD bits */ +#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) +#define DW_IC_DATA_CMD_FIRST_DATA_BYTE BIT(11) + +/* DW_IC_INTR_* bits */ +#define DW_IC_INTR_RX_UNDER BIT(0) +#define DW_IC_INTR_RX_OVER BIT(1) +#define DW_IC_INTR_RX_FULL BIT(2) +#define DW_IC_INTR_TX_OVER BIT(3) +#define DW_IC_INTR_TX_EMPTY BIT(4) +#define DW_IC_INTR_RD_REQ BIT(5) +#define DW_IC_INTR_TX_ABRT BIT(6) +#define DW_IC_INTR_RX_DONE BIT(7) +#define DW_IC_INTR_ACTIVITY BIT(8) +#define DW_IC_INTR_STOP_DET BIT(9) +#define DW_IC_INTR_START_DET BIT(10) +#define DW_IC_INTR_GEN_CALL BIT(11) +#define DW_IC_INTR_RESTART_DET BIT(12) +#define DW_IC_INTR_MST_ON_HOLD BIT(13) + +/* DW_IC_ENABLE bits */ +#define DW_IC_ENABLE_ENABLE BIT(0) +#define DW_IC_ENABLE_ABORT BIT(1) +#define DW_IC_ENABLE_BUS_RECOVERY BIT(3) + +/* DW_IC_STATUS bits */ +#define DW_IC_STATUS_ACTIVITY BIT(0) +#define DW_IC_STATUS_TFE BIT(2) +#define DW_IC_STATUS_RFNE BIT(3) +#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) +#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) +#define DW_IC_STATUS_MASTER_HOLD_TX_FIFO_EMPTY BIT(7) +#define DW_IC_STATUS_SDA_STUCK_NOT_RECOVERED BIT(11) + +/* DW_IC_SMBUS_INTR_* bits */ +#define DW_IC_SMBUS_INTR_ALERT BIT(10) + +#endif /* __LINUX_DESIGNWARE_I2C_H */ diff --git a/include/linux/device.h b/include/linux/device.h index dc1252a06480c..56e5a9314367d 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -485,10 +485,14 @@ struct device_physical_location { * * @DEV_FLAG_READY_TO_PROBE: If set then device_add() has finished enough * initialization that probe could be called. + * @DEV_FLAG_CAN_MATCH: The device has matched with a driver at least once or it + * is in a bus (like AMBA) which can't check for matching drivers + * until other devices probe successfully. * @DEV_FLAG_COUNT: Number of defined struct_device_flags. */ enum struct_device_flags { DEV_FLAG_READY_TO_PROBE = 0, + DEV_FLAG_CAN_MATCH = 1, DEV_FLAG_COUNT }; @@ -575,9 +579,6 @@ enum struct_device_flags { * @state_synced: The hardware state of this device has been synced to match * the software state of this device by calling the driver/bus * sync_state() callback. - * @can_match: The device has matched with a driver at least once or it is in - * a bus (like AMBA) which can't check for matching drivers until - * other devices probe successfully. * @dma_coherent: this particular device is dma coherent, even if the * architecture supports non-coherent devices. * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the @@ -696,7 +697,6 @@ struct device { bool offline:1; bool of_node_reused:1; bool state_synced:1; - bool can_match:1; #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) @@ -738,6 +738,7 @@ static inline bool dev_test_and_set_##accessor_name(struct device *dev) \ } __create_dev_flag_accessors(ready_to_probe, DEV_FLAG_READY_TO_PROBE); +__create_dev_flag_accessors(can_match, DEV_FLAG_CAN_MATCH); #undef __create_dev_flag_accessors diff --git a/include/linux/dibs.h b/include/linux/dibs.h index c75607f8a5cf9..d3e0777f25ae0 100644 --- a/include/linux/dibs.h +++ b/include/linux/dibs.h @@ -439,7 +439,7 @@ static inline void *dibs_get_priv(struct dibs_dev *dev, /** * dibs_dev_alloc() - allocate and reference device structure * - * The following fields will be valid upon successful return: dev + * The following fields will be valid upon successful return: dev, lock * NOTE: Use put_device(dibs_get_dev(@dibs)) to give up your reference instead * of freeing @dibs @dev directly once you have successfully called this * function. diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7b2968612b7e6..6746838940649 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -27,7 +27,7 @@ struct page_list { struct page *page; }; -typedef void (*io_notify_fn)(unsigned int long error, void *context); +typedef void (*io_notify_fn)(unsigned long int error, unsigned long int unsup, void *context); enum dm_io_mem_type { DM_IO_PAGE_LIST,/* Page list */ @@ -80,8 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits, - unsigned short ioprio); + struct dm_io_region *region, unsigned long int *sync_error_bits, + unsigned long int *sync_unsup_bits, unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index d58e329ac0e71..0b0b5fbb6cc1f 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -582,6 +582,11 @@ void dma_buf_unpin(struct dma_buf_attachment *attach); struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info); int dma_buf_fd(struct dma_buf *dmabuf, int flags); +/* + * This tree has no DMA-BUF fd tracepoint, so publishing a reserved fd only + * requires fd_install(). Call this after all fallible work has succeeded. + */ +#define dma_buf_fd_install(dmabuf, fd) fd_install((fd), (dmabuf)->file) struct dma_buf *dma_buf_get(int fd); void dma_buf_put(struct dma_buf *dmabuf); diff --git a/include/linux/dma/qcom_bam_dma.h b/include/linux/dma/qcom_bam_dma.h index 68fc0e643b1b9..d9d07a9ab3132 100644 --- a/include/linux/dma/qcom_bam_dma.h +++ b/include/linux/dma/qcom_bam_dma.h @@ -13,9 +13,12 @@ * supported by BAM DMA Engine. * * @cmd_and_addr - upper 8 bits command and lower 24 bits register address. - * @data - for write command: content to be written into peripheral register. - * for read command: dest addr to write peripheral register value. - * @mask - register mask. + * @data - For write command: content to be written into peripheral register. + * For read command: lower 32 bits of destination address. + * @mask - For write command: register write mask. + * For read command on BAM v1.6.0+: upper 4 bits of destination address. + * For read command on BAM < v1.6.0: ignored by hardware. + * Setting to 0 ensures 32-bit addressing compatibility. * @reserved - for future usage. * */ @@ -42,6 +45,10 @@ enum bam_command_type { * @addr: target address * @cmd: BAM command * @data: actual data for write and dest addr for read in le32 + * + * For BAM v1.6.0+, the mask field behavior depends on command type: + * - Write commands: mask = write mask (typically 0xffffffff) + * - Read commands: mask = upper 4 bits of destination address (0 for 32-bit) */ static inline void bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, @@ -50,7 +57,11 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, bam_ce->cmd_and_addr = cpu_to_le32((addr & 0xffffff) | ((cmd & 0xff) << 24)); bam_ce->data = data; - bam_ce->mask = cpu_to_le32(0xffffffff); + if (cmd == BAM_READ_COMMAND) + bam_ce->mask = cpu_to_le32(0x0); /* 32-bit addressing */ + else + bam_ce->mask = cpu_to_le32(0xffffffff); /* Write mask */ + bam_ce->reserved = 0; } /* @@ -60,7 +71,7 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, * @bam_ce: BAM command element * @addr: target address * @cmd: BAM command - * @data: actual data for write and dest addr for read + * @data: actual data for write and destination address for read */ static inline void bam_prep_ce(struct bam_cmd_element *bam_ce, u32 addr, diff --git a/include/linux/efi.h b/include/linux/efi.h index a98cc39e7aaa8..0ec9f3dc596d4 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -1228,8 +1228,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *), /* * efi_runtime_service() function identifiers. - * "NONE" is used by efi_recover_from_page_fault() to check if the page - * fault happened while executing an efi runtime service. + * "NONE" is used by efi_crash_gracefully_on_page_fault() to check if the + * page fault happened while executing an efi runtime service. */ enum efi_rts_ids { EFI_NONE, diff --git a/include/linux/file.h b/include/linux/file.h index af1768d934a01..27484b444d315 100644 --- a/include/linux/file.h +++ b/include/linux/file.h @@ -127,4 +127,129 @@ extern void __fput_sync(struct file *); extern unsigned int sysctl_nr_open_min, sysctl_nr_open_max; +/* + * fd_prepare: Combined fd + file allocation cleanup class. + * @err: Error code to indicate if allocation succeeded. + * @__fd: Allocated fd (may not be accessed directly) + * @__file: Allocated struct file pointer (may not be accessed directly) + * + * Allocates an fd and a file together. On error paths, automatically cleans + * up whichever resource was successfully allocated. Allows flexible file + * allocation with different functions per usage. + * + * Do not use directly. + */ +struct fd_prepare { + s32 err; + s32 __fd; /* do not access directly */ + struct file *__file; /* do not access directly */ +}; + +/* Typedef for fd_prepare cleanup guards. */ +typedef struct fd_prepare class_fd_prepare_t; + +/* + * Accessors for fd_prepare class members. + * _Generic() is used for zero-cost type safety. + */ +#define fd_prepare_fd(_fdf) \ + (_Generic((_fdf), struct fd_prepare: (_fdf).__fd)) + +#define fd_prepare_file(_fdf) \ + (_Generic((_fdf), struct fd_prepare: (_fdf).__file)) + +/* Do not use directly. */ +static inline void class_fd_prepare_destructor(const struct fd_prepare *fdf) +{ + if (unlikely(fdf->__fd >= 0)) + put_unused_fd(fdf->__fd); + if (unlikely(!IS_ERR_OR_NULL(fdf->__file))) + fput(fdf->__file); +} + +/* Do not use directly. */ +static inline int class_fd_prepare_lock_err(const struct fd_prepare *fdf) +{ + if (unlikely(fdf->err)) + return fdf->err; + if (unlikely(fdf->__fd < 0)) + return fdf->__fd; + if (unlikely(IS_ERR(fdf->__file))) + return PTR_ERR(fdf->__file); + if (unlikely(!fdf->__file)) + return -ENOMEM; + return 0; +} + +/* + * __FD_PREPARE_INIT - Helper to initialize fd_prepare class. + * @_fd_flags: flags for get_unused_fd_flags() + * @_file_owned: expression that returns struct file * + * + * Returns a struct fd_prepare with fd, file, and err set. + * If fd allocation fails, fd will be negative and err will be set. If + * fd succeeds but file_init_expr fails, file will be ERR_PTR and err + * will be set. The err field is the single source of truth for error + * checking. + */ +#define __FD_PREPARE_INIT(_fd_flags, _file_owned) \ + ({ \ + struct fd_prepare fdf = { \ + .__fd = get_unused_fd_flags((_fd_flags)), \ + }; \ + if (likely(fdf.__fd >= 0)) \ + fdf.__file = (_file_owned); \ + fdf.err = ACQUIRE_ERR(fd_prepare, &fdf); \ + fdf; \ + }) + +/* + * FD_PREPARE - Macro to declare and initialize an fd_prepare variable. + * + * Declares and initializes an fd_prepare variable with automatic + * cleanup. No separate scope required - cleanup happens when variable + * goes out of scope. + * + * @_fdf: name of struct fd_prepare variable to define + * @_fd_flags: flags for get_unused_fd_flags() + * @_file_owned: struct file to take ownership of (can be expression) + */ +#define FD_PREPARE(_fdf, _fd_flags, _file_owned) \ + CLASS_INIT(fd_prepare, _fdf, __FD_PREPARE_INIT(_fd_flags, _file_owned)) + +/* + * fd_publish - Publish prepared fd and file to the fd table. + * @_fdf: struct fd_prepare variable + */ +#define fd_publish(_fdf) \ + ({ \ + struct fd_prepare *fdp = &(_fdf); \ + VFS_WARN_ON_ONCE(fdp->err); \ + VFS_WARN_ON_ONCE(fdp->__fd < 0); \ + VFS_WARN_ON_ONCE(IS_ERR_OR_NULL(fdp->__file)); \ + fd_install(fdp->__fd, fdp->__file); \ + retain_and_null_ptr(fdp->__file); \ + take_fd(fdp->__fd); \ + }) + +/* Do not use directly. */ +#define __FD_ADD(_fdf, _fd_flags, _file_owned) \ + ({ \ + FD_PREPARE(_fdf, _fd_flags, _file_owned); \ + s32 ret = _fdf.err; \ + if (likely(!ret)) \ + ret = fd_publish(_fdf); \ + ret; \ + }) + +/* + * FD_ADD - Allocate and install an fd and file in one step. + * @_fd_flags: flags for get_unused_fd_flags() + * @_file_owned: struct file to take ownership of + * + * Returns the allocated fd number, or negative error code on failure. + */ +#define FD_ADD(_fd_flags, _file_owned) \ + __FD_ADD(__UNIQUE_ID(fd_prepare), _fd_flags, _file_owned) + #endif /* __LINUX_FILE_H */ diff --git a/include/linux/filter.h b/include/linux/filter.h index 2469fd2e40157..260a190634055 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -383,6 +383,37 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index ee80ca4bb0d0c..f0d0f980b0798 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -582,7 +582,7 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) /** * Request INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE - * Sync call to send a signed certificate + * Async call to send a signed certificate * * Call register usage: * a0 INTEL_SIP_SMC_FCS_SEND_CERTIFICATE @@ -591,7 +591,7 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) * a3-a7 not used * * Return status: - * a0 INTEL_SIP_SMC_STATUS_OK or INTEL_SIP_SMC_FCS_REJECTED + * a0 INTEL_SIP_SMC_STATUS_OK or INTEL_SIP_SMC_REJECTED * a1-a3 not used */ #define INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE 93 @@ -600,24 +600,22 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) /** * Request INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * Sync call to dump all the fuses and key hashes + * Async call to dump all the fuses and key hashes * * Call register usage: * a0 INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * a1 the physical address for firmware to write structure of fuse and - * key hashes - * a2-a7 not used + * a1-a7 not used * * Return status: - * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_FCS_ERROR or - * INTEL_SIP_SMC_FCS_REJECTED - * a1 mailbox error + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_ERROR or + * INTEL_SIP_SMC_STATUS_REJECTED + * a1 mailbox error if a0 is INTEL_SIP_SMC_STATUS_ERROR * a2 physical address for the structure of fuse and key hashes * a3 the size of structure * */ #define INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA 94 #define INTEL_SIP_SMC_FCS_GET_PROVISION_DATA \ - INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) #endif diff --git a/include/linux/firmware/qcom/qcom_scm.h b/include/linux/firmware/qcom/qcom_scm.h index a55ca771286bf..5045f8fe876da 100644 --- a/include/linux/firmware/qcom/qcom_scm.h +++ b/include/linux/firmware/qcom/qcom_scm.h @@ -72,13 +72,27 @@ struct qcom_scm_pas_metadata { ssize_t size; }; -int qcom_scm_pas_init_image(u32 peripheral, const void *metadata, size_t size, +struct qcom_scm_pas_context { + struct device *dev; + u32 pas_id; + phys_addr_t mem_phys; + size_t mem_size; + void *ptr; + dma_addr_t phys; + ssize_t size; +}; + +struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev, + u32 pas_id, + phys_addr_t mem_phys, + size_t mem_size); +int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size, struct qcom_scm_pas_metadata *ctx); void qcom_scm_pas_metadata_release(struct qcom_scm_pas_metadata *ctx); -int qcom_scm_pas_mem_setup(u32 peripheral, phys_addr_t addr, phys_addr_t size); -int qcom_scm_pas_auth_and_reset(u32 peripheral); -int qcom_scm_pas_shutdown(u32 peripheral); -bool qcom_scm_pas_supported(u32 peripheral); +int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size); +int qcom_scm_pas_auth_and_reset(u32 pas_id); +int qcom_scm_pas_shutdown(u32 pas_id); +bool qcom_scm_pas_supported(u32 pas_id); int qcom_scm_io_readl(phys_addr_t addr, unsigned int *val); int qcom_scm_io_writel(phys_addr_t addr, unsigned int val); diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index ae48d619c4e0a..2f1c71f0cea76 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -164,6 +164,7 @@ enum pm_api_cb_id { enum pm_api_id { PM_API_FEATURES = 0, PM_GET_API_VERSION = 1, + PM_GET_NODE_STATUS = 3, PM_REGISTER_NOTIFIER = 5, PM_FORCE_POWERDOWN = 8, PM_REQUEST_WAKEUP = 10, @@ -544,6 +545,18 @@ enum pm_gem_config_type { GEM_CONFIG_FIXED = 2, }; +/** + * enum pm_node_status - Device node status provided by xilpm fw + * @PM_NODE_UNUSED: Device is not used + * @PM_NODE_RUNNING: Device is power-on and out of reset + * @PM_NODE_HALT: Device is power-on but in the reset state + */ +enum pm_node_status { + PM_NODE_UNUSED = 0, + PM_NODE_RUNNING = 1, + PM_NODE_HALT = 12, +}; + /** * struct zynqmp_pm_query_data - PM query data * @qid: query ID diff --git a/include/linux/fs.h b/include/linux/fs.h index 14f5accc97b74..55ecd9d26b4fb 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1783,12 +1783,12 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, static inline time64_t inode_get_atime_sec(const struct inode *inode) { - return inode->i_atime_sec; + return READ_ONCE(inode->i_atime_sec); } static inline long inode_get_atime_nsec(const struct inode *inode) { - return inode->i_atime_nsec; + return READ_ONCE(inode->i_atime_nsec); } static inline struct timespec64 inode_get_atime(const struct inode *inode) @@ -1802,8 +1802,8 @@ static inline struct timespec64 inode_get_atime(const struct inode *inode) static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_atime_sec = ts.tv_sec; - inode->i_atime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_atime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_atime_nsec, ts.tv_nsec); return ts; } @@ -1818,12 +1818,12 @@ static inline struct timespec64 inode_set_atime(struct inode *inode, static inline time64_t inode_get_mtime_sec(const struct inode *inode) { - return inode->i_mtime_sec; + return READ_ONCE(inode->i_mtime_sec); } static inline long inode_get_mtime_nsec(const struct inode *inode) { - return inode->i_mtime_nsec; + return READ_ONCE(inode->i_mtime_nsec); } static inline struct timespec64 inode_get_mtime(const struct inode *inode) @@ -1836,8 +1836,8 @@ static inline struct timespec64 inode_get_mtime(const struct inode *inode) static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_mtime_sec = ts.tv_sec; - inode->i_mtime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_mtime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_mtime_nsec, ts.tv_nsec); return ts; } @@ -1862,12 +1862,12 @@ static inline struct timespec64 inode_set_mtime(struct inode *inode, static inline time64_t inode_get_ctime_sec(const struct inode *inode) { - return inode->i_ctime_sec; + return READ_ONCE(inode->i_ctime_sec); } static inline long inode_get_ctime_nsec(const struct inode *inode) { - return inode->i_ctime_nsec & ~I_CTIME_QUERIED; + return READ_ONCE(inode->i_ctime_nsec) & ~I_CTIME_QUERIED; } static inline struct timespec64 inode_get_ctime(const struct inode *inode) diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 516aba5b858b5..c009d4afe9185 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -57,6 +57,9 @@ struct fscrypt_name { /* Maximum value for the third parameter of fscrypt_operations.set_context(). */ #define FSCRYPT_SET_CONTEXT_MAX_SIZE 40 +/* Maximum supported number of block devices per filesystem */ +#define FSCRYPT_MAX_DEVICES 8 + #ifdef CONFIG_FS_ENCRYPTION /* Crypto operations for filesystems */ @@ -181,21 +184,20 @@ struct fscrypt_operations { bool (*has_stable_inodes)(struct super_block *sb); /* - * Return an array of pointers to the block devices to which the - * filesystem may write encrypted file contents, NULL if the filesystem - * only has a single such block device, or an ERR_PTR() on error. + * Retrieve the list of block devices to which the filesystem may write + * encrypted file contents. * - * On successful non-NULL return, *num_devs is set to the number of - * devices in the returned array. The caller must free the returned - * array using kfree(). + * This writes the block_device pointers to @devs and returns the count + * (between 1 and FSCRYPT_MAX_DEVICES inclusively). * * If the filesystem can use multiple block devices (other than block * devices that aren't used for encrypted file contents, such as * external journal devices), and wants to support inline encryption, * then it must implement this function. Otherwise it's not needed. */ - struct block_device **(*get_devices)(struct super_block *sb, - unsigned int *num_devs); + unsigned int (*get_devices)( + struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]); }; int fscrypt_d_revalidate(struct inode *dir, const struct qstr *name, diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index c3b1c74bdd7a4..62e85b2c65168 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -797,8 +797,9 @@ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec); unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec); extern ftrace_func_t ftrace_trace_function; +struct trace_array; -int ftrace_regex_open(struct ftrace_ops *ops, int flag, +int ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file); ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos); @@ -1008,7 +1009,7 @@ static inline unsigned long ftrace_location(unsigned long ip) * have them defined when ftrace is not enabled, but these * functions may still be called. Use a macro instead of inline. */ -#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; }) +#define ftrace_regex_open(tr, ops, flag, inode, file) ({ -ENODEV; }) #define ftrace_set_early_filter(ops, buf, enable) do { } while (0) #define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; }) #define ftrace_set_filter_ips(ops, ips, cnt, remove, reset) ({ -ENODEV; }) diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e9750f049805..e93f2be22779f 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -77,6 +77,7 @@ static inline void futex_init_task(struct task_struct *tsk) void futex_exit_recursive(struct task_struct *tsk); void futex_exit_release(struct task_struct *tsk); void futex_exec_release(struct task_struct *tsk); +void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3); @@ -98,6 +99,7 @@ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } static inline void futex_exit_release(struct task_struct *tsk) { } static inline void futex_exec_release(struct task_struct *tsk) { } +static inline void futex_exec_done(struct task_struct *tsk) { } static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3) diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h new file mode 100644 index 0000000000000..07ac65db6d2e8 --- /dev/null +++ b/include/linux/gpu_buddy.h @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __GPU_BUDDY_H__ +#define __GPU_BUDDY_H__ + +#include +#include +#include +#include +#include + +#define GPU_BUDDY_RANGE_ALLOCATION BIT(0) +#define GPU_BUDDY_TOPDOWN_ALLOCATION BIT(1) +#define GPU_BUDDY_CONTIGUOUS_ALLOCATION BIT(2) +#define GPU_BUDDY_CLEAR_ALLOCATION BIT(3) +#define GPU_BUDDY_CLEARED BIT(4) +#define GPU_BUDDY_TRIM_DISABLE BIT(5) + +enum gpu_buddy_free_tree { + GPU_BUDDY_CLEAR_TREE = 0, + GPU_BUDDY_DIRTY_TREE, + GPU_BUDDY_MAX_FREE_TREES, +}; + +#define for_each_free_tree(tree) \ + for ((tree) = 0; (tree) < GPU_BUDDY_MAX_FREE_TREES; (tree)++) + +struct gpu_buddy_block { +#define GPU_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) +#define GPU_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) +#define GPU_BUDDY_ALLOCATED (1 << 10) +#define GPU_BUDDY_FREE (2 << 10) +#define GPU_BUDDY_SPLIT (3 << 10) +#define GPU_BUDDY_HEADER_CLEAR GENMASK_ULL(9, 9) +/* Free to be used, if needed in the future */ +#define GPU_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6) +#define GPU_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0) + u64 header; + + struct gpu_buddy_block *left; + struct gpu_buddy_block *right; + struct gpu_buddy_block *parent; + + void *private; /* owned by creator */ + + /* + * While the block is allocated by the user through gpu_buddy_alloc*, + * the user has ownership of the link, for example to maintain within + * a list, if so desired. As soon as the block is freed with + * gpu_buddy_free* ownership is given back to the mm. + */ + union { + struct rb_node rb; + struct list_head link; + }; + + struct list_head tmp_link; +}; + +/* Order-zero must be at least SZ_4K */ +#define GPU_BUDDY_MAX_ORDER (63 - 12) + +/* + * Binary Buddy System. + * + * Locking should be handled by the user, a simple mutex around + * gpu_buddy_alloc* and gpu_buddy_free* should suffice. + */ +struct gpu_buddy { + /* Maintain a free list for each order. */ + struct rb_root **free_trees; + + /* + * Maintain explicit binary tree(s) to track the allocation of the + * address space. This gives us a simple way of finding a buddy block + * and performing the potentially recursive merge step when freeing a + * block. Nodes are either allocated or free, in which case they will + * also exist on the respective free list. + */ + struct gpu_buddy_block **roots; + + /* + * Anything from here is public, and remains static for the lifetime of + * the mm. Everything above is considered do-not-touch. + */ + unsigned int n_roots; + unsigned int max_order; + + /* Must be at least SZ_4K */ + u64 chunk_size; + u64 size; + u64 avail; + u64 clear_avail; +}; + +static inline u64 +gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_OFFSET; +} + +static inline unsigned int +gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_ORDER; +} + +static inline unsigned int +gpu_buddy_block_state(struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_STATE; +} + +static inline bool +gpu_buddy_block_is_allocated(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_state(block) == GPU_BUDDY_ALLOCATED; +} + +static inline bool +gpu_buddy_block_is_clear(struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_CLEAR; +} + +static inline bool +gpu_buddy_block_is_free(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_state(block) == GPU_BUDDY_FREE; +} + +static inline bool +gpu_buddy_block_is_split(struct gpu_buddy_block *block) +{ + return gpu_buddy_block_state(block) == GPU_BUDDY_SPLIT; +} + +static inline u64 +gpu_buddy_block_size(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + return mm->chunk_size << gpu_buddy_block_order(block); +} + +int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size); + +void gpu_buddy_fini(struct gpu_buddy *mm); + +struct gpu_buddy_block * +gpu_get_buddy(struct gpu_buddy_block *block); + +int gpu_buddy_alloc_blocks(struct gpu_buddy *mm, + u64 start, u64 end, u64 size, + u64 min_page_size, + struct list_head *blocks, + unsigned long flags); + +int gpu_buddy_block_trim(struct gpu_buddy *mm, + u64 *start, + u64 new_size, + struct list_head *blocks); + +void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear); + +void gpu_buddy_free_block(struct gpu_buddy *mm, struct gpu_buddy_block *block); + +void gpu_buddy_free_list(struct gpu_buddy *mm, + struct list_head *objects, + unsigned int flags); + +void gpu_buddy_print(struct gpu_buddy *mm); +void gpu_buddy_block_print(struct gpu_buddy *mm, + struct gpu_buddy_block *block); +#endif diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 240cbc6764809..b55a4a342f578 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -364,13 +364,12 @@ unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long add unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); -bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins); int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order); int min_order_for_split(struct folio *folio); int split_folio_to_list(struct folio *folio, struct list_head *list); -bool folio_split_supported(struct folio *folio, unsigned int new_order, - bool uniform_split, bool warns); +int folio_check_splittable(struct folio *folio, unsigned int new_order, + bool uniform_split); int folio_split(struct folio *folio, unsigned int new_order, struct page *page, struct list_head *list); /* @@ -390,7 +389,7 @@ int folio_split(struct folio *folio, unsigned int new_order, struct page *page, static inline int try_folio_split_to_order(struct folio *folio, struct page *page, unsigned int new_order) { - if (!folio_split_supported(folio, new_order, false, /* warns= */ false)) + if (folio_check_splittable(folio, new_order, false)) return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); return folio_split(folio, new_order, page, NULL); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index a3f6cdf8a2b6d..ca6d6eb817118 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -173,6 +173,7 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); extern int sysctl_hugetlb_shm_group; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); bool hugetlb_bootmem_allocated(void); extern nodemask_t hugetlb_bootmem_nodes; @@ -1277,6 +1278,8 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -1314,6 +1317,10 @@ static inline bool hugetlb_bootmem_allocated(void) { return false; } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 301a83afbd663..f5330e7ae247c 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -13,6 +13,7 @@ #define _HWMON_H_ #include +#include struct device; struct attribute_group; @@ -495,6 +496,8 @@ char *devm_hwmon_sanitize_name(struct device *dev, const char *name); void hwmon_lock(struct device *dev); void hwmon_unlock(struct device *dev); +DEFINE_GUARD(hwmon_lock, struct device *, hwmon_lock(_T), hwmon_unlock(_T)) + /** * hwmon_is_bad_char - Is the char invalid in a hwmon name * @ch: the char to be considered diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 1aa1efd835b86..df0132048a102 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -228,6 +228,8 @@ struct i3c_dev_desc { * every time the I3C device is rediscovered with a different dynamic * address assigned * @bus: I3C bus this device is attached to + * @node: unregistered device list node, only for use by + * i3c_master_register_new_i3c_devs(), it is not protected by a lock * * I3C device object exposed to I3C device drivers. The takes care of linking * this object to the relevant &struct_i3c_dev_desc one. @@ -238,6 +240,7 @@ struct i3c_device { struct device dev; struct i3c_dev_desc *desc; struct i3c_bus *bus; + struct list_head node; }; /* diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h new file mode 100644 index 0000000000000..132acced7d798 --- /dev/null +++ b/include/linux/ieee80211-uhr.h @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * IEEE 802.11 UHR definitions + * + * Copyright (c) 2025-2026 Intel Corporation + */ +#ifndef LINUX_IEEE80211_UHR_H +#define LINUX_IEEE80211_UHR_H + +#include +#include + +#define IEEE80211_UHR_OPER_PARAMS_DPS_ENA 0x0001 +#define IEEE80211_UHR_OPER_PARAMS_NPCA_ENA 0x0002 +#define IEEE80211_UHR_OPER_PARAMS_DBE_ENA 0x0004 +#define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA 0x0008 + +struct ieee80211_uhr_operation { + __le16 params; + u8 basic_mcs_nss_set[4]; + u8 variable[]; +} __packed; + +#define IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS 0x0000000F +#define IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH 0x000000F0 +#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY 0x00003F00 +#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY 0x000FC000 +#define IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC 0x00300000 +#define IEEE80211_UHR_NPCA_PARAMS_MOPLEN 0x00400000 +#define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES 0x00800000 + +struct ieee80211_uhr_npca_info { + __le32 params; + __le16 dis_subch_bmap[]; +} __packed; + +static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len, + bool beacon) +{ + const struct ieee80211_uhr_operation *oper = (const void *)data; + u8 needed = sizeof(*oper); + + if (len < needed) + return false; + + /* nothing else present in beacons */ + if (beacon) + return true; + + /* FIXME: DPS, DBE, P-EDCA (consider order, also relative to NPCA) */ + + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA)) { + const struct ieee80211_uhr_npca_info *npca = + (const void *)oper->variable; + + needed += sizeof(*npca); + + if (len < needed) + return false; + + if (npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES)) + needed += sizeof(npca->dis_subch_bmap[0]); + } + + return len >= needed; +} + +/* + * Note: cannot call this on the element coming from a beacon, + * must ensure ieee80211_uhr_oper_size_ok(..., false) first + */ +static inline const struct ieee80211_uhr_npca_info * +ieee80211_uhr_npca_info(const struct ieee80211_uhr_operation *oper) +{ + if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA))) + return NULL; + + /* FIXME: DPS */ + + return (const void *)oper->variable; +} + +static inline const __le16 * +ieee80211_uhr_npca_dis_subch_bitmap(const struct ieee80211_uhr_operation *oper) +{ + const struct ieee80211_uhr_npca_info *npca; + + npca = ieee80211_uhr_npca_info(oper); + if (!npca) + return NULL; + if (!(npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES))) + return NULL; + return npca->dis_subch_bmap; +} + +#define IEEE80211_UHR_MAC_CAP0_DPS_SUPP 0x01 +#define IEEE80211_UHR_MAC_CAP0_DPS_ASSIST_SUPP 0x02 +#define IEEE80211_UHR_MAC_CAP0_DPS_AP_STATIC_HCM_SUPP 0x04 +#define IEEE80211_UHR_MAC_CAP0_NPCA_SUPP 0x10 +#define IEEE80211_UHR_MAC_CAP0_ENH_BSR_SUPP 0x20 +#define IEEE80211_UHR_MAC_CAP0_ADD_MAP_TID_SUPP 0x40 +#define IEEE80211_UHR_MAC_CAP0_EOTSP_SUPP 0x80 + +#define IEEE80211_UHR_MAC_CAP1_DSO_SUPP 0x01 +#define IEEE80211_UHR_MAC_CAP1_PEDCA_SUPP 0x02 +#define IEEE80211_UHR_MAC_CAP1_DBE_SUPP 0x04 +#define IEEE80211_UHR_MAC_CAP1_UL_LLI_SUPP 0x08 +#define IEEE80211_UHR_MAC_CAP1_P2P_LLI_SUPP 0x10 +#define IEEE80211_UHR_MAC_CAP1_PUO_SUPP 0x20 +#define IEEE80211_UHR_MAC_CAP1_AP_PUO_SUPP 0x40 +#define IEEE80211_UHR_MAC_CAP1_DUO_SUPP 0x80 + +#define IEEE80211_UHR_MAC_CAP2_OMC_UL_MU_DIS_RX_SUPP 0x01 +#define IEEE80211_UHR_MAC_CAP2_AOM_SUPP 0x02 +#define IEEE80211_UHR_MAC_CAP2_IFCS_LOC_SUPP 0x04 +#define IEEE80211_UHR_MAC_CAP2_UHR_TRS_SUPP 0x08 +#define IEEE80211_UHR_MAC_CAP2_TXSPG_SUPP 0x10 +#define IEEE80211_UHR_MAC_CAP2_TXOP_RET_IN_TXSPG 0x20 +#define IEEE80211_UHR_MAC_CAP2_UHR_OM_PU_TO_LOW 0xC0 + +#define IEEE80211_UHR_MAC_CAP3_UHR_OM_PU_TO_HIGH 0x03 +#define IEEE80211_UHR_MAC_CAP3_PARAM_UPD_ADV_NOTIF_INTV 0x1C +#define IEEE80211_UHR_MAC_CAP3_UPD_IND_TIM_INTV_LOW 0xE0 + +#define IEEE80211_UHR_MAC_CAP4_UPD_IND_TIM_INTV_HIGH 0x03 +#define IEEE80211_UHR_MAC_CAP4_BOUNDED_ESS 0x04 +#define IEEE80211_UHR_MAC_CAP4_BTM_ASSURANCE 0x08 +#define IEEE80211_UHR_MAC_CAP4_CO_BF_SUPP 0x10 + +#define IEEE80211_UHR_MAC_CAP_DBE_MAX_BW 0x07 +#define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES 0x08 +#define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES 0x10 + +struct ieee80211_uhr_cap_mac { + u8 mac_cap[5]; +} __packed; + +struct ieee80211_uhr_cap { + struct ieee80211_uhr_cap_mac mac; + /* DBE, PHY capabilities */ + u8 variable[]; +} __packed; + +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_LE80 0x01 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_LE80 0x02 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_160 0x04 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_160 0x08 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_320 0x10 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_320 0x20 +#define IEEE80211_UHR_PHY_CAP_ELR_RX 0x40 +#define IEEE80211_UHR_PHY_CAP_ELR_TX 0x80 + +struct ieee80211_uhr_cap_phy { + u8 cap; +} __packed; + +static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len, + bool from_ap) +{ + const struct ieee80211_uhr_cap *cap = (const void *)data; + size_t needed = sizeof(*cap) + sizeof(struct ieee80211_uhr_cap_phy); + + if (len < needed) + return false; + + /* + * A non-AP STA does not include the DBE Capability Parameters field + * in the UHR MAC Capabilities Information field. + */ + if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) { + u8 dbe; + + needed += 1; + if (len < needed) + return false; + + dbe = cap->variable[0]; + + if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) + needed += 3; + + if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) + needed += 3; + } + + return len >= needed; +} + +static inline const struct ieee80211_uhr_cap_phy * +ieee80211_uhr_phy_cap(const struct ieee80211_uhr_cap *cap, bool from_ap) +{ + u8 offs = 0; + + if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) { + u8 dbe = cap->variable[0]; + + offs += 1; + + if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) + offs += 3; + + if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) + offs += 3; + } + + return (const void *)&cap->variable[offs]; +} + +#define IEEE80211_SMD_INFO_CAPA_DL_DATA_FWD 0x01 +#define IEEE80211_SMD_INFO_CAPA_MAX_NUM_PREP 0x0E +#define IEEE80211_SMD_INFO_CAPA_TYPE 0x10 +#define IEEE80211_SMD_INFO_CAPA_PTK_PER_AP_MLD 0x20 + +struct ieee80211_smd_info { + u8 id[ETH_ALEN]; + u8 capa; + __le16 timeout; +} __packed; + +#endif /* LINUX_IEEE80211_UHR_H */ diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 992a0c09c1e53..2526ccd8a4a06 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -9,7 +9,7 @@ * Copyright (c) 2006, Michael Wu * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright (c) 2016 - 2017 Intel Deutschland GmbH - * Copyright (c) 2018 - 2025 Intel Corporation + * Copyright (c) 2018 - 2026 Intel Corporation */ #ifndef LINUX_IEEE80211_H @@ -1381,8 +1381,9 @@ struct ieee80211_mgmt { #define BSS_MEMBERSHIP_SELECTOR_SAE_H2E 123 #define BSS_MEMBERSHIP_SELECTOR_HE_PHY 122 #define BSS_MEMBERSHIP_SELECTOR_EHT_PHY 121 +#define BSS_MEMBERSHIP_SELECTOR_UHR_PHY 120 -#define BSS_MEMBERSHIP_SELECTOR_MIN BSS_MEMBERSHIP_SELECTOR_EHT_PHY +#define BSS_MEMBERSHIP_SELECTOR_MIN BSS_MEMBERSHIP_SELECTOR_UHR_PHY /* mgmt header + 1 byte category code */ #define IEEE80211_MIN_ACTION_SIZE offsetof(struct ieee80211_mgmt, u.action.u) @@ -2117,6 +2118,15 @@ enum ieee80211_eid_ext { WLAN_EID_EXT_BANDWIDTH_INDICATION = 135, WLAN_EID_EXT_KNOWN_STA_IDENTIFCATION = 136, WLAN_EID_EXT_NON_AP_STA_REG_CON = 137, + WLAN_EID_EXT_UHR_OPER = 151, + WLAN_EID_EXT_UHR_CAPA = 152, + WLAN_EID_EXT_MACP = 153, + WLAN_EID_EXT_SMD = 154, + WLAN_EID_EXT_BSS_SMD_TRANS_PARAMS = 155, + WLAN_EID_EXT_CHAN_USAGE = 156, + WLAN_EID_EXT_UHR_MODE_CHG = 157, + WLAN_EID_EXT_UHR_PARAM_UPD = 158, + WLAN_EID_EXT_TXPI = 159, }; /* Action category code */ @@ -3352,6 +3362,22 @@ static inline bool for_each_element_completed(const struct element *element, #define WLAN_RSNX_CAPA_PROTECTED_TWT BIT(4) #define WLAN_RSNX_CAPA_SAE_H2E BIT(5) +/* EBPCC = Enhanced BSS Parameter Change Count */ +#define IEEE80211_ENH_CRIT_UPD_EBPCC 0x0F +#define IEEE80211_ENH_CRIT_UPD_TYPE 0x70 +#define IEEE80211_ENH_CRIT_UPD_TYPE_NO_UHR 0 +#define IEEE80211_ENH_CRIT_UPD_TYPE_UHR 1 +#define IEEE80211_ENH_CRIT_UPD_ALL 0x80 + +/** + * struct ieee80211_enh_crit_upd - enhanced critical update (UHR) + * @v: value of the enhanced critical update data, + * see %IEEE80211_ENH_CRIT_UPD_* to parse the bits + */ +struct ieee80211_enh_crit_upd { + u8 v; +} __packed; + /* * reduced neighbor report, based on Draft P802.11ax_D6.1, * section 9.4.2.170 and accepted contributions. @@ -3370,6 +3396,7 @@ static inline bool for_each_element_completed(const struct element *element, #define IEEE80211_RNR_TBTT_PARAMS_COLOC_ESS 0x10 #define IEEE80211_RNR_TBTT_PARAMS_PROBE_ACTIVE 0x20 #define IEEE80211_RNR_TBTT_PARAMS_COLOC_AP 0x40 +#define IEEE80211_RNR_TBTT_PARAMS_SAME_SMD 0x80 #define IEEE80211_RNR_TBTT_PARAMS_PSD_NO_LIMIT 127 #define IEEE80211_RNR_TBTT_PARAMS_PSD_RESERVED -128 @@ -3422,6 +3449,7 @@ struct ieee80211_tbtt_info_ge_11 { u8 bss_params; s8 psd_20; struct ieee80211_rnr_mld_params mld_params; + struct ieee80211_enh_crit_upd enh_crit_upd; } __packed; /* NAN operation mode, as defined in Wi-Fi Aware (TM) specification Table 81 */ @@ -3445,6 +3473,7 @@ struct ieee80211_tbtt_info_ge_11 { #include "ieee80211-vht.h" #include "ieee80211-he.h" #include "ieee80211-eht.h" +#include "ieee80211-uhr.h" #include "ieee80211-mesh.h" #endif /* LINUX_IEEE80211_H */ diff --git a/include/linux/igmp.h b/include/linux/igmp.h index 073b30a9b8508..401e67056fbfc 100644 --- a/include/linux/igmp.h +++ b/include/linux/igmp.h @@ -57,20 +57,21 @@ struct ip_mc_socklist { }; struct ip_sf_list { - struct ip_sf_list *sf_next; + struct ip_sf_list __rcu *sf_next; unsigned long sf_count[2]; /* include/exclude counts */ __be32 sf_inaddr; unsigned char sf_gsresp; /* include in g & s response? */ unsigned char sf_oldin; /* change state */ unsigned char sf_crcount; /* retrans. left to send */ + struct rcu_head rcu; }; struct ip_mc_list { struct in_device *interface; __be32 multiaddr; unsigned int sfmode; - struct ip_sf_list *sources; - struct ip_sf_list *tomb; + struct ip_sf_list __rcu *sources; + struct ip_sf_list __rcu *tomb; unsigned long sfcount[2]; union { struct ip_mc_list *next; diff --git a/include/linux/ima.h b/include/linux/ima.h index abf8923f8fc51..8e08baf16c2fd 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -11,6 +11,7 @@ #include #include #include +#include #include struct linux_binprm; @@ -73,14 +74,8 @@ int ima_validate_range(phys_addr_t phys, size_t size); #endif #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT -extern bool arch_ima_get_secureboot(void); extern const char * const *arch_get_ima_policy(void); #else -static inline bool arch_ima_get_secureboot(void) -{ - return false; -} - static inline const char * const *arch_get_ima_policy(void) { return NULL; diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 2ee408d076b3f..4f1607e00359b 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -17,6 +17,14 @@ enum { * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. */ IOU_F_TWQ_LAZY_WAKE = 1, + + /* + * Set when task_work is queued from a waitqueue wakeup handler, where + * an arbitrary provider waitqueue lock is held. Signaling the CQ ring + * eventfd inline from there can recurse back into that lock through + * epoll, so the eventfd signal must be deferred. + */ + IOU_F_TWQ_IN_WAKE = 2, }; enum io_uring_cmd_flags { @@ -474,6 +482,7 @@ struct io_ring_ctx { * ONLY core io_uring.c should instantiate this struct. */ struct io_tw_state { + bool cancel; }; /* Alias to use in code that doesn't instantiate struct io_tw_state */ typedef struct io_tw_state io_tw_token_t; @@ -617,7 +626,11 @@ enum { REQ_F_SQE_COPIED = IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT), }; -typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw); +struct io_tw_req { + struct io_kiocb *req; +}; + +typedef void (*io_req_tw_func_t)(struct io_tw_req tw_req, io_tw_token_t tw); struct io_task_work { struct llist_node node; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 84cf41f51f05d..6e51f5a48c74e 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -1657,6 +1657,7 @@ void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); +void iopf_group_dequeue(struct iopf_group *group); #else static inline int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) @@ -1702,5 +1703,9 @@ static inline void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { } + +static inline void iopf_group_dequeue(struct iopf_group *group) +{ +} #endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h index 92f3843d9ebb8..e135dacaa90f3 100644 --- a/include/linux/kcsan-checks.h +++ b/include/linux/kcsan-checks.h @@ -282,7 +282,7 @@ static inline void __kcsan_disable_current(void) { } * @size: size of access */ #define __kcsan_check_write(ptr, size) \ - __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + __kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE) /** * __kcsan_check_read_write - check regular read-write access for races @@ -308,7 +308,7 @@ static inline void __kcsan_disable_current(void) { } * @size: size of access */ #define kcsan_check_write(ptr, size) \ - kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE) /** * kcsan_check_read_write - check regular read-write access for races @@ -331,7 +331,7 @@ static inline void __kcsan_disable_current(void) { } #define kcsan_check_atomic_read(ptr, size) \ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC) #define kcsan_check_atomic_write(ptr, size) \ - kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) + kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) #define kcsan_check_atomic_read_write(ptr, size) \ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND) #endif diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 8c4f3bb244295..e6de7ae55bda0 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -181,6 +181,7 @@ struct kprobe_blacklist_entry { struct list_head list; unsigned long start_addr; unsigned long end_addr; + struct rcu_head rcu; }; #ifdef CONFIG_KPROBES diff --git a/include/linux/libata.h b/include/linux/libata.h index 5c3f98a418c38..868af0a08f527 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -389,7 +389,7 @@ enum { /* This should match the actual table size of * ata_eh_cmd_timeout_table in libata-eh.c. */ - ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 8, + ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 9, /* * Quirk flags: may be set by libata or controller drivers on drives. @@ -1151,6 +1151,7 @@ extern int ata_scsi_ioctl(struct scsi_device *dev, unsigned int cmd, #define ATA_SCSI_COMPAT_IOCTL /* empty */ #endif extern int ata_scsi_queuecmd(struct Scsi_Host *h, struct scsi_cmnd *cmd); +enum scsi_timeout_action ata_scsi_eh_timed_out(struct scsi_cmnd *cmd); #if IS_REACHABLE(CONFIG_ATA) bool ata_scsi_dma_need_drain(struct request *rq); #else @@ -1460,6 +1461,7 @@ extern const struct attribute_group *ata_common_sdev_groups[]; .ioctl = ata_scsi_ioctl, \ ATA_SCSI_COMPAT_IOCTL \ .queuecommand = ata_scsi_queuecmd, \ + .eh_timed_out = ata_scsi_eh_timed_out, \ .dma_need_drain = ata_scsi_dma_need_drain, \ .this_id = ATA_SHT_THIS_ID, \ .emulated = ATA_SHT_EMULATED, \ diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 28f086c4a1873..d929d83abf3be 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -126,6 +126,15 @@ struct nd_mapping_desc { struct bio; struct resource; struct nd_region; + +/* + * Provider flush callback return values: + * 0: flush completed synchronously + * <0: flush failed + * >0: flush completion was queued and @bio will be completed later + */ +#define NVDIMM_FLUSH_ASYNC 1 + struct nd_region_desc { struct resource *res; struct nd_mapping_desc *mapping; diff --git a/include/linux/list.h b/include/linux/list.h index 00ea8e5fb88b0..0121e2f88f890 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -150,10 +150,13 @@ static inline bool __list_del_entry_valid(struct list_head *entry) * * This is only for internal list manipulation where we know * the prev/next entries already! + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) +static __always_inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) { if (!__list_add_valid(new, prev, next)) return; @@ -171,8 +174,12 @@ static inline void __list_add(struct list_head *new, * * Insert a new entry after the specified head. * This is good for implementing stacks. + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void list_add(struct list_head *new, struct list_head *head) +static __always_inline void list_add(struct list_head *new, + struct list_head *head) { __list_add(new, head, head->next); } diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index d9930fc43ca54..ba838f529056c 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -268,7 +268,7 @@ void nsm_release(struct nsm_handle *nsm); * This is used in garbage collection and resource reclaim * A return value != 0 means destroy the lock/block/share */ -typedef int (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref); +typedef int (*nlm_host_match_fn_t)(void *owner, struct nlm_host *ref); /* * Server-side lock handling diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h index 1db0069c27c52..e6af0721198b4 100644 --- a/include/linux/mailbox_controller.h +++ b/include/linux/mailbox_controller.h @@ -114,6 +114,7 @@ struct mbox_controller { * @txdone_method: Way to detect TXDone chosen by the API * @cl: Pointer to the current owner of this channel * @tx_complete: Transmission completion + * @tx_status: Transmission status * @active_req: Currently active request hook * @msg_count: No. of mssg currently queued * @msg_free: Index of next available mssg slot @@ -126,6 +127,7 @@ struct mbox_chan { unsigned txdone_method; struct mbox_client *cl; struct completion tx_complete; + int tx_status; void *active_req; unsigned msg_count, msg_free; void *msg_data[MBOX_TX_QUEUE_LEN]; diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 66f98a3da8d8b..2ba53c0654678 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -539,7 +539,7 @@ void mas_destroy(struct ma_state *mas); int mas_expected_entries(struct ma_state *mas, unsigned long nr_entries); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 133591199933a..706ddda4a60b1 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1674,6 +1674,11 @@ int alloc_shrinker_info(struct mem_cgroup *memcg); void free_shrinker_info(struct mem_cgroup *memcg); void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id); void reparent_shrinker_deferred(struct mem_cgroup *memcg); + +static inline int shrinker_id(struct shrinker *shrinker) +{ + return shrinker->id; +} #else #define mem_cgroup_sockets_enabled 0 @@ -1705,6 +1710,11 @@ static inline void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id) { } + +static inline int shrinker_id(struct shrinker *shrinker) +{ + return -1; +} #endif #ifdef CONFIG_MEMCG diff --git a/include/linux/memory.h b/include/linux/memory.h index 0c214256216f6..1c0c8f3c35fbb 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -19,6 +19,7 @@ #include #include #include +#include #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) @@ -64,10 +65,22 @@ struct memory_group { }; }; +enum memory_block_state { + /* These states are exposed to userspace as text strings in sysfs */ + MEM_ONLINE, /* exposed to userspace */ + MEM_GOING_OFFLINE, /* exposed to userspace */ + MEM_OFFLINE, /* exposed to userspace */ + MEM_GOING_ONLINE, + MEM_CANCEL_ONLINE, + MEM_CANCEL_OFFLINE, + MEM_PREPARE_ONLINE, + MEM_FINISH_OFFLINE, +}; + struct memory_block { unsigned long start_section_nr; - unsigned long state; /* serialized by the dev->lock */ - int online_type; /* for passing data to online routine */ + enum memory_block_state state; /* serialized by the dev->lock */ + enum mmop online_type; /* for passing data to online routine */ int nid; /* NID for this memory block */ /* * The single zone of this memory block if all PFNs of this memory block @@ -89,16 +102,6 @@ int arch_get_memory_phys_device(unsigned long start_pfn); unsigned long memory_block_size_bytes(void); int set_memory_block_size_order(unsigned int order); -/* These states are exposed to userspace as text strings in sysfs */ -#define MEM_ONLINE (1<<0) /* exposed to userspace */ -#define MEM_GOING_OFFLINE (1<<1) /* exposed to userspace */ -#define MEM_OFFLINE (1<<2) /* exposed to userspace */ -#define MEM_GOING_ONLINE (1<<3) -#define MEM_CANCEL_ONLINE (1<<4) -#define MEM_CANCEL_OFFLINE (1<<5) -#define MEM_PREPARE_ONLINE (1<<6) -#define MEM_FINISH_OFFLINE (1<<7) - struct memory_notify { /* * The altmap_start_pfn and altmap_nr_pages fields are designated for diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 23f038a162319..bb01cd2671312 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -16,11 +16,8 @@ struct resource; struct vmem_altmap; struct dev_pagemap; -#ifdef CONFIG_MEMORY_HOTPLUG -struct page *pfn_to_online_page(unsigned long pfn); - /* Types for control the zone type of onlined and offlined memory */ -enum { +enum mmop { /* Offline the memory. */ MMOP_OFFLINE = 0, /* Online the memory. Zone depends, see default_zone_for_pfn(). */ @@ -31,6 +28,9 @@ enum { MMOP_ONLINE_MOVABLE, }; +#ifdef CONFIG_MEMORY_HOTPLUG +struct page *pfn_to_online_page(unsigned long pfn); + /* Flags for add_memory() and friends to specify memory hotplug details. */ typedef int __bitwise mhp_t; @@ -302,9 +302,9 @@ static inline void __remove_memory(u64 start, u64 size) {} #ifdef CONFIG_MEMORY_HOTPLUG /* Default online_type (MMOP_*) when new memory blocks are added. */ -extern int mhp_get_default_online_type(void); -extern void mhp_set_default_online_type(int online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +extern enum mmop mhp_get_default_online_type(void); +extern void mhp_set_default_online_type(enum mmop online_type); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, @@ -326,8 +326,8 @@ extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap); extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum); -extern struct zone *zone_for_pfn_range(int online_type, int nid, - struct memory_group *group, unsigned long start_pfn, +extern struct zone *zone_for_pfn_range(enum mmop online_type, + int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages); extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); diff --git a/include/linux/mfd/db8500-prcmu.h b/include/linux/mfd/db8500-prcmu.h index a62de3d155edc..c939c9a1170a0 100644 --- a/include/linux/mfd/db8500-prcmu.h +++ b/include/linux/mfd/db8500-prcmu.h @@ -12,6 +12,9 @@ #include #include +#include + +#include /* For clock identifiers */ /* * Registers @@ -24,6 +27,38 @@ #define DB8500_PRCM_DSI_SW_RESET_DSI1_SW_RESETN BIT(1) #define DB8500_PRCM_DSI_SW_RESET_DSI2_SW_RESETN BIT(2) +/* Offset for the firmware version within the TCPM */ +#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 + +#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 + +/* + * CLKOUT sources + */ +#define PRCMU_CLKSRC_CLK38M 0x00 +#define PRCMU_CLKSRC_ACLK 0x01 +#define PRCMU_CLKSRC_SYSCLK 0x02 +#define PRCMU_CLKSRC_LCDCLK 0x03 +#define PRCMU_CLKSRC_SDMMCCLK 0x04 +#define PRCMU_CLKSRC_TVCLK 0x05 +#define PRCMU_CLKSRC_TIMCLK 0x06 +#define PRCMU_CLKSRC_CLK009 0x07 +/* These are only valid for CLKOUT1: */ +#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 +#define PRCMU_CLKSRC_I2CCLK 0x41 +#define PRCMU_CLKSRC_MSP02CLK 0x42 +#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 +#define PRCMU_CLKSRC_HSIRXCLK 0x44 +#define PRCMU_CLKSRC_HSITXCLK 0x45 +#define PRCMU_CLKSRC_ARMCLKFIX 0x46 +#define PRCMU_CLKSRC_HDMICLK 0x47 + +/* + * Definitions for controlling ESRAM0 in deep sleep. + */ +#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 +#define ESRAM0_DEEP_SLEEP_STATE_RET 2 + /* This portion previously known as */ /** @@ -451,10 +486,173 @@ enum prcmu_power_status { PRCMU_ARMPENDINGIT_ER = 0x93, }; +/* PRCMU Wakeup defines */ +enum prcmu_wakeup_index { + PRCMU_WAKEUP_INDEX_RTC, + PRCMU_WAKEUP_INDEX_RTT0, + PRCMU_WAKEUP_INDEX_RTT1, + PRCMU_WAKEUP_INDEX_HSI0, + PRCMU_WAKEUP_INDEX_HSI1, + PRCMU_WAKEUP_INDEX_USB, + PRCMU_WAKEUP_INDEX_ABB, + PRCMU_WAKEUP_INDEX_ABB_FIFO, + PRCMU_WAKEUP_INDEX_ARM, + PRCMU_WAKEUP_INDEX_CD_IRQ, + NUM_PRCMU_WAKEUP_INDICES +}; + +#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) + +/** + * enum prcmu_wdog_id - PRCMU watchdog IDs + * @PRCMU_WDOG_ALL: use all timers + * @PRCMU_WDOG_CPU1: use first CPU timer only + * @PRCMU_WDOG_CPU2: use second CPU timer conly + */ +enum prcmu_wdog_id { + PRCMU_WDOG_ALL = 0x00, + PRCMU_WDOG_CPU1 = 0x01, + PRCMU_WDOG_CPU2 = 0x02, +}; + +/** + * enum ape_opp - APE OPP states definition + * @APE_OPP_INIT: + * @APE_NO_CHANGE: The APE operating point is unchanged + * @APE_100_OPP: The new APE operating point is ape100opp + * @APE_50_OPP: 50% + * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. + */ +enum ape_opp { + APE_OPP_INIT = 0x00, + APE_NO_CHANGE = 0x01, + APE_100_OPP = 0x02, + APE_50_OPP = 0x03, + APE_50_PARTLY_25_OPP = 0xFF, +}; + +/** + * enum arm_opp - ARM OPP states definition + * @ARM_OPP_INIT: + * @ARM_NO_CHANGE: The ARM operating point is unchanged + * @ARM_100_OPP: The new ARM operating point is arm100opp + * @ARM_50_OPP: The new ARM operating point is arm50opp + * @ARM_MAX_OPP: Operating point is "max" (more than 100) + * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 + * @ARM_EXTCLK: The new ARM operating point is armExtClk + */ +enum arm_opp { + ARM_OPP_INIT = 0x00, + ARM_NO_CHANGE = 0x01, + ARM_100_OPP = 0x02, + ARM_50_OPP = 0x03, + ARM_MAX_OPP = 0x04, + ARM_MAX_FREQ100OPP = 0x05, + ARM_EXTCLK = 0x07 +}; + +/** + * enum ddr_opp - DDR OPP states definition + * @DDR_100_OPP: The new DDR operating point is ddr100opp + * @DDR_50_OPP: The new DDR operating point is ddr50opp + * @DDR_25_OPP: The new DDR operating point is ddr25opp + */ +enum ddr_opp { + DDR_100_OPP = 0x00, + DDR_50_OPP = 0x01, + DDR_25_OPP = 0x02, +}; + +/** + * enum ddr_pwrst - DDR power states definition + * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged + * @DDR_PWR_STATE_ON: + * @DDR_PWR_STATE_OFFLOWLAT: + * @DDR_PWR_STATE_OFFHIGHLAT: + */ +enum ddr_pwrst { + DDR_PWR_STATE_UNCHANGED = 0x00, + DDR_PWR_STATE_ON = 0x01, + DDR_PWR_STATE_OFFLOWLAT = 0x02, + DDR_PWR_STATE_OFFHIGHLAT = 0x03 +}; + /* * Definitions for autonomous power management configuration. */ +/* EPOD (power domain) IDs */ + +/* + * DB8500 EPODs + * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP + * - EPOD_ID_SVAPIPE: power domain for SVA pipe + * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP + * - EPOD_ID_SIAPIPE: power domain for SIA pipe + * - EPOD_ID_SGA: power domain for SGA + * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE + * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 + * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 + * - NUM_EPOD_ID: number of power domains + * + * TODO: These should be prefixed. + */ +#define EPOD_ID_SVAMMDSP 0 +#define EPOD_ID_SVAPIPE 1 +#define EPOD_ID_SIAMMDSP 2 +#define EPOD_ID_SIAPIPE 3 +#define EPOD_ID_SGA 4 +#define EPOD_ID_B2R2_MCDE 5 +#define EPOD_ID_ESRAM12 6 +#define EPOD_ID_ESRAM34 7 +#define NUM_EPOD_ID 8 + +/* + * state definition for EPOD (power domain) + * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged + * - EPOD_STATE_OFF: The EPOD is switched off + * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in + * retention + * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off + * - EPOD_STATE_ON: Same as above, but with clock enabled + */ +#define EPOD_STATE_NO_CHANGE 0x00 +#define EPOD_STATE_OFF 0x01 +#define EPOD_STATE_RAMRET 0x02 +#define EPOD_STATE_ON_CLK_OFF 0x03 +#define EPOD_STATE_ON 0x04 + +#define PRCMU_FW_PROJECT_U8500 2 +#define PRCMU_FW_PROJECT_U8400 3 +#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8500_MBB 5 +#define PRCMU_FW_PROJECT_U8500_C1 6 +#define PRCMU_FW_PROJECT_U8500_C2 7 +#define PRCMU_FW_PROJECT_U8500_C3 8 +#define PRCMU_FW_PROJECT_U8500_C4 9 +#define PRCMU_FW_PROJECT_U9500_MBL 10 +#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8520 13 +#define PRCMU_FW_PROJECT_U8420 14 +#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 +#define PRCMU_FW_PROJECT_A9420 20 +/* [32..63] 9540 and derivatives */ +#define PRCMU_FW_PROJECT_U9540 32 +/* [64..95] 8540 and derivatives */ +#define PRCMU_FW_PROJECT_L8540 64 +/* [96..126] 8580 and derivatives */ +#define PRCMU_FW_PROJECT_L8580 96 + +#define PRCMU_FW_PROJECT_NAME_LEN 20 + +/* PRCMU QoS APE OPP class */ +#define PRCMU_QOS_APE_OPP 1 +#define PRCMU_QOS_DDR_OPP 2 +#define PRCMU_QOS_ARM_OPP 3 +#define PRCMU_QOS_DEFAULT_VALUE -1 + #define PRCMU_AUTO_PM_OFF 0 #define PRCMU_AUTO_PM_ON 1 @@ -469,6 +667,14 @@ enum prcmu_auto_pm_policy { PRCMU_AUTO_PM_POLICY_DSP_CLK_OFF_HWP_CLK_OFF, }; +struct prcmu_fw_version { + u32 project; /* Notice, project shifted with 8 on ux540 */ + u8 api_version; + u8 func_version; + u8 errata; + char project_name[PRCMU_FW_PROJECT_NAME_LEN]; +}; + /** * struct prcmu_auto_pm_config - Autonomous power management configuration. * @sia_auto_pm_enable: SIA autonomous pm enable. (PRCMU_AUTO_PM_{OFF,ON}) @@ -501,6 +707,9 @@ void prcmu_configure_auto_pm(struct prcmu_auto_pm_config *sleep, bool prcmu_is_auto_pm_enabled(void); int prcmu_config_clkout(u8 clkout, u8 source, u8 div); +unsigned long prcmu_clock_rate(u8 clock); +long prcmu_round_clock_rate(u8 clock, unsigned long rate); +int prcmu_set_clock_rate(u8 clock, unsigned long rate); int prcmu_set_clock_divider(u8 clock, u8 divider); int db8500_prcmu_config_hotdog(u8 threshold); int db8500_prcmu_config_hotmon(u8 low, u8 high); @@ -508,6 +717,8 @@ int db8500_prcmu_start_temp_sense(u16 cycles32k); int db8500_prcmu_stop_temp_sense(void); int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); +int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, + u8 *mask, u8 size); int prcmu_ac_wake_req(void); void prcmu_ac_sleep_req(void); @@ -610,6 +821,21 @@ static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) return 0; } +static inline unsigned long prcmu_clock_rate(u8 clock) +{ + return 0; +} + +static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + +static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + static inline int prcmu_set_clock_divider(u8 clock, u8 divider) { return 0; @@ -637,12 +863,18 @@ static inline int db8500_prcmu_stop_temp_sense(void) static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; } static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; +} + +static inline int prcmu_abb_write_masked(u8 slave, u8 reg, + u8 *value, u8 *mask, u8 size) +{ + return -EINVAL; } static inline int prcmu_ac_wake_req(void) @@ -745,4 +977,20 @@ static inline void db8500_prcmu_write_masked(unsigned int reg, u32 mask, #endif /* !CONFIG_MFD_DB8500_PRCMU */ +static inline int prcmu_qos_add_requirement(int prcmu_qos_class, + char *name, s32 value) +{ + return 0; +} + +static inline int prcmu_qos_update_requirement(int prcmu_qos_class, + char *name, s32 new_value) +{ + return 0; +} + +static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) +{ +} + #endif /* __MFD_DB8500_PRCMU_H */ diff --git a/include/linux/mfd/dbx500-prcmu.h b/include/linux/mfd/dbx500-prcmu.h deleted file mode 100644 index 828362b7860c6..0000000000000 --- a/include/linux/mfd/dbx500-prcmu.h +++ /dev/null @@ -1,575 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST Ericsson SA 2011 - * - * STE Ux500 PRCMU API - */ -#ifndef __MACH_PRCMU_H -#define __MACH_PRCMU_H - -#include -#include -#include - -#include /* For clock identifiers */ - -/* Offset for the firmware version within the TCPM */ -#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 -#define DBX540_PRCMU_FW_VERSION_OFFSET 0xA8 - -/* PRCMU Wakeup defines */ -enum prcmu_wakeup_index { - PRCMU_WAKEUP_INDEX_RTC, - PRCMU_WAKEUP_INDEX_RTT0, - PRCMU_WAKEUP_INDEX_RTT1, - PRCMU_WAKEUP_INDEX_HSI0, - PRCMU_WAKEUP_INDEX_HSI1, - PRCMU_WAKEUP_INDEX_USB, - PRCMU_WAKEUP_INDEX_ABB, - PRCMU_WAKEUP_INDEX_ABB_FIFO, - PRCMU_WAKEUP_INDEX_ARM, - PRCMU_WAKEUP_INDEX_CD_IRQ, - NUM_PRCMU_WAKEUP_INDICES -}; -#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) - -/* EPOD (power domain) IDs */ - -/* - * DB8500 EPODs - * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP - * - EPOD_ID_SVAPIPE: power domain for SVA pipe - * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP - * - EPOD_ID_SIAPIPE: power domain for SIA pipe - * - EPOD_ID_SGA: power domain for SGA - * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE - * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 - * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 - * - NUM_EPOD_ID: number of power domains - * - * TODO: These should be prefixed. - */ -#define EPOD_ID_SVAMMDSP 0 -#define EPOD_ID_SVAPIPE 1 -#define EPOD_ID_SIAMMDSP 2 -#define EPOD_ID_SIAPIPE 3 -#define EPOD_ID_SGA 4 -#define EPOD_ID_B2R2_MCDE 5 -#define EPOD_ID_ESRAM12 6 -#define EPOD_ID_ESRAM34 7 -#define NUM_EPOD_ID 8 - -/* - * state definition for EPOD (power domain) - * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged - * - EPOD_STATE_OFF: The EPOD is switched off - * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in - * retention - * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off - * - EPOD_STATE_ON: Same as above, but with clock enabled - */ -#define EPOD_STATE_NO_CHANGE 0x00 -#define EPOD_STATE_OFF 0x01 -#define EPOD_STATE_RAMRET 0x02 -#define EPOD_STATE_ON_CLK_OFF 0x03 -#define EPOD_STATE_ON 0x04 - -/* - * CLKOUT sources - */ -#define PRCMU_CLKSRC_CLK38M 0x00 -#define PRCMU_CLKSRC_ACLK 0x01 -#define PRCMU_CLKSRC_SYSCLK 0x02 -#define PRCMU_CLKSRC_LCDCLK 0x03 -#define PRCMU_CLKSRC_SDMMCCLK 0x04 -#define PRCMU_CLKSRC_TVCLK 0x05 -#define PRCMU_CLKSRC_TIMCLK 0x06 -#define PRCMU_CLKSRC_CLK009 0x07 -/* These are only valid for CLKOUT1: */ -#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 -#define PRCMU_CLKSRC_I2CCLK 0x41 -#define PRCMU_CLKSRC_MSP02CLK 0x42 -#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 -#define PRCMU_CLKSRC_HSIRXCLK 0x44 -#define PRCMU_CLKSRC_HSITXCLK 0x45 -#define PRCMU_CLKSRC_ARMCLKFIX 0x46 -#define PRCMU_CLKSRC_HDMICLK 0x47 - -/** - * enum prcmu_wdog_id - PRCMU watchdog IDs - * @PRCMU_WDOG_ALL: use all timers - * @PRCMU_WDOG_CPU1: use first CPU timer only - * @PRCMU_WDOG_CPU2: use second CPU timer conly - */ -enum prcmu_wdog_id { - PRCMU_WDOG_ALL = 0x00, - PRCMU_WDOG_CPU1 = 0x01, - PRCMU_WDOG_CPU2 = 0x02, -}; - -/** - * enum ape_opp - APE OPP states definition - * @APE_OPP_INIT: - * @APE_NO_CHANGE: The APE operating point is unchanged - * @APE_100_OPP: The new APE operating point is ape100opp - * @APE_50_OPP: 50% - * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. - */ -enum ape_opp { - APE_OPP_INIT = 0x00, - APE_NO_CHANGE = 0x01, - APE_100_OPP = 0x02, - APE_50_OPP = 0x03, - APE_50_PARTLY_25_OPP = 0xFF, -}; - -/** - * enum arm_opp - ARM OPP states definition - * @ARM_OPP_INIT: - * @ARM_NO_CHANGE: The ARM operating point is unchanged - * @ARM_100_OPP: The new ARM operating point is arm100opp - * @ARM_50_OPP: The new ARM operating point is arm50opp - * @ARM_MAX_OPP: Operating point is "max" (more than 100) - * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 - * @ARM_EXTCLK: The new ARM operating point is armExtClk - */ -enum arm_opp { - ARM_OPP_INIT = 0x00, - ARM_NO_CHANGE = 0x01, - ARM_100_OPP = 0x02, - ARM_50_OPP = 0x03, - ARM_MAX_OPP = 0x04, - ARM_MAX_FREQ100OPP = 0x05, - ARM_EXTCLK = 0x07 -}; - -/** - * enum ddr_opp - DDR OPP states definition - * @DDR_100_OPP: The new DDR operating point is ddr100opp - * @DDR_50_OPP: The new DDR operating point is ddr50opp - * @DDR_25_OPP: The new DDR operating point is ddr25opp - */ -enum ddr_opp { - DDR_100_OPP = 0x00, - DDR_50_OPP = 0x01, - DDR_25_OPP = 0x02, -}; - -/* - * Definitions for controlling ESRAM0 in deep sleep. - */ -#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 -#define ESRAM0_DEEP_SLEEP_STATE_RET 2 - -/** - * enum ddr_pwrst - DDR power states definition - * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged - * @DDR_PWR_STATE_ON: - * @DDR_PWR_STATE_OFFLOWLAT: - * @DDR_PWR_STATE_OFFHIGHLAT: - */ -enum ddr_pwrst { - DDR_PWR_STATE_UNCHANGED = 0x00, - DDR_PWR_STATE_ON = 0x01, - DDR_PWR_STATE_OFFLOWLAT = 0x02, - DDR_PWR_STATE_OFFHIGHLAT = 0x03 -}; - -#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 - -#define PRCMU_FW_PROJECT_U8500 2 -#define PRCMU_FW_PROJECT_U8400 3 -#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8500_MBB 5 -#define PRCMU_FW_PROJECT_U8500_C1 6 -#define PRCMU_FW_PROJECT_U8500_C2 7 -#define PRCMU_FW_PROJECT_U8500_C3 8 -#define PRCMU_FW_PROJECT_U8500_C4 9 -#define PRCMU_FW_PROJECT_U9500_MBL 10 -#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8520 13 -#define PRCMU_FW_PROJECT_U8420 14 -#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 -#define PRCMU_FW_PROJECT_A9420 20 -/* [32..63] 9540 and derivatives */ -#define PRCMU_FW_PROJECT_U9540 32 -/* [64..95] 8540 and derivatives */ -#define PRCMU_FW_PROJECT_L8540 64 -/* [96..126] 8580 and derivatives */ -#define PRCMU_FW_PROJECT_L8580 96 - -#define PRCMU_FW_PROJECT_NAME_LEN 20 -struct prcmu_fw_version { - u32 project; /* Notice, project shifted with 8 on ux540 */ - u8 api_version; - u8 func_version; - u8 errata; - char project_name[PRCMU_FW_PROJECT_NAME_LEN]; -}; - -#include - -#if defined(CONFIG_UX500_SOC_DB8500) - -static inline void __init prcmu_early_init(void) -{ - db8500_prcmu_early_init(); -} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return db8500_prcmu_set_power_state(state, keep_ulp_clk, - keep_ap_pll); -} - -static inline u8 prcmu_get_power_state_result(void) -{ - return db8500_prcmu_get_power_state_result(); -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return db8500_prcmu_set_epod(epod_id, epod_state); -} - -static inline void prcmu_enable_wakeups(u32 wakeups) -{ - db8500_prcmu_enable_wakeups(wakeups); -} - -static inline void prcmu_disable_wakeups(void) -{ - prcmu_enable_wakeups(0); -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) -{ - db8500_prcmu_config_abb_event_readout(abb_events); -} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - db8500_prcmu_get_abb_event_buffer(buf); -} - -int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, u8 size); - -int prcmu_config_clkout(u8 clkout, u8 source, u8 div); - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return db8500_prcmu_request_clock(clock, enable); -} - -unsigned long prcmu_clock_rate(u8 clock); -long prcmu_round_clock_rate(u8 clock, unsigned long rate); -int prcmu_set_clock_rate(u8 clock, unsigned long rate); - -static inline int prcmu_get_ddr_opp(void) -{ - return db8500_prcmu_get_ddr_opp(); -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return db8500_prcmu_set_arm_opp(opp); -} - -static inline int prcmu_get_arm_opp(void) -{ - return db8500_prcmu_get_arm_opp(); -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return db8500_prcmu_set_ape_opp(opp); -} - -static inline int prcmu_get_ape_opp(void) -{ - return db8500_prcmu_get_ape_opp(); -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return db8500_prcmu_request_ape_opp_100_voltage(enable); -} - -static inline void prcmu_system_reset(u16 reset_code) -{ - db8500_prcmu_system_reset(reset_code); -} - -static inline u16 prcmu_get_reset_code(void) -{ - return db8500_prcmu_get_reset_code(); -} - -int prcmu_ac_wake_req(void); -void prcmu_ac_sleep_req(void); -static inline void prcmu_modem_reset(void) -{ - db8500_prcmu_modem_reset(); -} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return db8500_prcmu_is_ac_wake_requested(); -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return db8500_prcmu_config_esram0_deep_sleep(state); -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return db8500_prcmu_config_hotdog(threshold); -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return db8500_prcmu_config_hotmon(low, high); -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return db8500_prcmu_start_temp_sense(cycles32k); -} - -static inline int prcmu_stop_temp_sense(void) -{ - return db8500_prcmu_stop_temp_sense(); -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return db8500_prcmu_read(reg); -} - -static inline void prcmu_write(unsigned int reg, u32 value) -{ - db8500_prcmu_write(reg, value); -} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) -{ - db8500_prcmu_write_masked(reg, mask, value); -} - -static inline int prcmu_enable_a9wdog(u8 id) -{ - return db8500_prcmu_enable_a9wdog(id); -} - -static inline int prcmu_disable_a9wdog(u8 id) -{ - return db8500_prcmu_disable_a9wdog(id); -} - -static inline int prcmu_kick_a9wdog(u8 id) -{ - return db8500_prcmu_kick_a9wdog(id); -} - -static inline int prcmu_load_a9wdog(u8 id, u32 timeout) -{ - return db8500_prcmu_load_a9wdog(id, timeout); -} - -static inline int prcmu_config_a9wdog(u8 num, bool sleep_auto_off) -{ - return db8500_prcmu_config_a9wdog(num, sleep_auto_off); -} -#else - -static inline void prcmu_early_init(void) {} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return 0; -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return 0; -} - -static inline void prcmu_enable_wakeups(u32 wakeups) {} - -static inline void prcmu_disable_wakeups(void) {} - -static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, - u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) -{ - return 0; -} - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return 0; -} - -static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline unsigned long prcmu_clock_rate(u8 clock) -{ - return 0; -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_ape_opp(void) -{ - return APE_100_OPP; -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return 0; -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_arm_opp(void) -{ - return ARM_100_OPP; -} - -static inline int prcmu_get_ddr_opp(void) -{ - return DDR_100_OPP; -} - -static inline void prcmu_system_reset(u16 reset_code) {} - -static inline u16 prcmu_get_reset_code(void) -{ - return 0; -} - -static inline int prcmu_ac_wake_req(void) -{ - return 0; -} - -static inline void prcmu_ac_sleep_req(void) {} - -static inline void prcmu_modem_reset(void) {} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return false; -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return 0; -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) {} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - *buf = NULL; -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return 0; -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return 0; -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return 0; -} - -static inline int prcmu_stop_temp_sense(void) -{ - return 0; -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return 0; -} - -static inline void prcmu_write(unsigned int reg, u32 value) {} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) {} - -#endif - -static inline void prcmu_set(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, bits); -} - -static inline void prcmu_clear(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, 0); -} - -/* PRCMU QoS APE OPP class */ -#define PRCMU_QOS_APE_OPP 1 -#define PRCMU_QOS_DDR_OPP 2 -#define PRCMU_QOS_ARM_OPP 3 -#define PRCMU_QOS_DEFAULT_VALUE -1 - -static inline int prcmu_qos_add_requirement(int prcmu_qos_class, - char *name, s32 value) -{ - return 0; -} - -static inline int prcmu_qos_update_requirement(int prcmu_qos_class, - char *name, s32 new_value) -{ - return 0; -} - -static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) -{ -} - -#endif /* __MACH_PRCMU_H */ diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 85c2b3d358ec1..647dcf144ae90 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -1142,6 +1142,7 @@ int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev); bool mlx5_lag_is_roce(struct mlx5_core_dev *dev); bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev); bool mlx5_lag_is_active(struct mlx5_core_dev *dev); +int mlx5_lag_query_bond_speed(struct mlx5_core_dev *dev, u32 *speed); bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev); bool mlx5_lag_is_master(struct mlx5_core_dev *dev); bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev); diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 6ac76a0c38277..7bf2449c53b20 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -116,6 +116,7 @@ enum mlx5_flow_namespace_type { }; enum { + FDB_DROP_ROOT, FDB_BYPASS_PATH, FDB_CRYPTO_INGRESS, FDB_TC_OFFLOAD, diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 1b0b36aa2a767..0250fdbe082c0 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -1071,7 +1071,9 @@ struct mlx5_ifc_e_switch_cap_bits { u8 esw_shared_ingress_acl[0x1]; u8 esw_uplink_ingress_acl[0x1]; u8 root_ft_on_other_esw[0x1]; - u8 reserved_at_a[0xf]; + u8 reserved_at_a[0x1]; + u8 esw_vport_state_max_tx_speed[0x1]; + u8 reserved_at_c[0xd]; u8 esw_functions_changed[0x1]; u8 reserved_at_1a[0x1]; u8 ecpf_vport_exists[0x1]; @@ -5443,7 +5445,8 @@ struct mlx5_ifc_query_vport_state_out_bits { u8 reserved_at_40[0x20]; - u8 reserved_at_60[0x18]; + u8 max_tx_speed[0x10]; + u8 reserved_at_70[0x8]; u8 admin_state[0x4]; u8 state[0x4]; }; @@ -7776,7 +7779,7 @@ struct mlx5_ifc_modify_vport_state_in_bits { u8 reserved_at_41[0xf]; u8 vport_number[0x10]; - u8 reserved_at_60[0x10]; + u8 max_tx_speed[0x10]; u8 ingress_connect[0x1]; u8 egress_connect[0x1]; u8 ingress_connect_valid[0x1]; @@ -12141,18 +12144,7 @@ struct mlx5_ifc_mcia_reg_bits { u8 reserved_at_60[0x20]; - u8 dword_0[0x20]; - u8 dword_1[0x20]; - u8 dword_2[0x20]; - u8 dword_3[0x20]; - u8 dword_4[0x20]; - u8 dword_5[0x20]; - u8 dword_6[0x20]; - u8 dword_7[0x20]; - u8 dword_8[0x20]; - u8 dword_9[0x20]; - u8 dword_10[0x20]; - u8 dword_11[0x20]; + u8 dwords[0x400]; }; struct mlx5_ifc_dcbx_param_bits { diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index b98aaa471ac22..e53a5d033d72d 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -41,6 +41,8 @@ (MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) && \ mlx5_core_is_pf(mdev)) +#define MLX5_MAX_TX_SPEED_UNIT 100 + enum { MLX5_CAP_INLINE_MODE_L2, MLX5_CAP_INLINE_MODE_VPORT_CONTEXT, @@ -58,6 +60,11 @@ enum { u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport); int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u8 state); +int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state); +int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, + u16 vport, u8 other_vport, u16 max_tx_speed); int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, u16 vport, bool other, u8 *addr); int mlx5_query_mac_address(struct mlx5_core_dev *mdev, u8 *addr); diff --git a/include/linux/mm.h b/include/linux/mm.h index 92ba2489f2374..50710a5d4a05f 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -4101,8 +4101,6 @@ int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); -int vmemmap_undo_hvo(unsigned long start, unsigned long end, int node, - unsigned long headsize); void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); void vmemmap_populate_print_last(void); diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 8b1045c51e0a6..f9c552744b71a 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -285,6 +285,8 @@ typedef struct { unsigned long val; } swp_entry_t; +typedef swp_entry_t softleaf_t; + #if defined(CONFIG_MEMCG) || defined(CONFIG_SLAB_OBJ_EXT) /* We have some extra room after the refcount in tail pages. */ #define NR_PAGES_IN_LARGE_FOLIO diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 95ee1f224c492..81c4d552bfab6 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -397,6 +397,9 @@ static inline void mmap_read_unlock(struct mm_struct *mm) DEFINE_GUARD(mmap_read_lock, struct mm_struct *, mmap_read_lock(_T), mmap_read_unlock(_T)) +DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T)) +DEFINE_GUARD(mmap_write_lock, struct mm_struct *, + mmap_write_lock(_T), mmap_write_unlock(_T)) static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) { diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 673cbdf434533..dce89c1106910 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -111,6 +111,7 @@ #define SDIO_VENDOR_ID_MEDIATEK 0x037a #define SDIO_DEVICE_ID_MEDIATEK_MT7663 0x7663 #define SDIO_DEVICE_ID_MEDIATEK_MT7668 0x7668 +#define SDIO_DEVICE_ID_MEDIATEK_MT7902 0x790a #define SDIO_DEVICE_ID_MEDIATEK_MT7961 0x7961 #define SDIO_VENDOR_ID_MICROCHIP_WILC 0x0296 diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index 0075f6e5c3da9..f887ffba95ca2 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -227,6 +227,7 @@ struct mr_table_ops { /** * struct mr_table - a multicast routing table + * @work: used for table destruction * @list: entry within a list of multicast routing tables * @net: net where this table belongs * @ops: protocol specific operations @@ -244,6 +245,7 @@ struct mr_table_ops { * @mroute_reg_vif_num: PIM-device vif index */ struct mr_table { + struct rcu_work work; struct list_head list; possible_net_t net; struct mr_table_ops ops; @@ -275,6 +277,7 @@ void vif_device_init(struct vif_device *v, unsigned short flags, unsigned short get_iflink_mask); +void mr_table_free(struct mr_table *mrt); struct mr_table * mr_table_alloc(struct net *net, u32 id, struct mr_table_ops *ops, diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 43ea96ecedc3a..7be5b78b2bff7 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -300,9 +300,11 @@ struct hh_cache { * We could use other alignment values, but we must maintain the * relationship HH alignment <= LL alignment. */ -#define LL_RESERVED_SPACE(dev) \ - ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \ +#define LL_RESERVED_SPACE_EX(dev, hlen) \ + ((((hlen) + READ_ONCE((dev)->needed_headroom)) \ & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD) +#define LL_RESERVED_SPACE(dev) \ + LL_RESERVED_SPACE_EX(dev, (dev)->hard_header_len) #define LL_RESERVED_SPACE_EXTRA(dev,extra) \ ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \ & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD) @@ -2143,6 +2145,7 @@ struct net_device { unsigned long state; unsigned int flags; unsigned short hard_header_len; + enum netdev_stat_type pcpu_stat_type:8; netdev_features_t features; struct inet6_dev __rcu *ip6_ptr; __cacheline_group_end(net_device_read_txrx); @@ -2394,8 +2397,6 @@ struct net_device { void *ml_priv; enum netdev_ml_priv_type ml_priv_type; - enum netdev_stat_type pcpu_stat_type:8; - #if IS_ENABLED(CONFIG_GARP) struct garp_port __rcu *garp_port; #endif @@ -3457,11 +3458,6 @@ static inline bool dev_validate_header(const struct net_device *dev, if (len < dev->min_header_len) return false; - if (capable(CAP_SYS_RAWIO)) { - memset(ll_header + len, 0, dev->hard_header_len - len); - return true; - } - if (dev->header_ops && dev->header_ops->validate) return dev->header_ops->validate(ll_header, len); diff --git a/include/linux/netfilter/ipset/ip_set.h b/include/linux/netfilter/ipset/ip_set.h index b98331572ad29..cadae9b2578f1 100644 --- a/include/linux/netfilter/ipset/ip_set.h +++ b/include/linux/netfilter/ipset/ip_set.h @@ -273,7 +273,7 @@ struct ip_set { /* Number of elements (vs timeout) */ u32 elements; /* Size of the dynamic extensions (vs timeout) */ - size_t ext_size; + atomic64_t ext_size; /* Element data size */ size_t dsize; /* Offsets to extensions in elements */ diff --git a/include/linux/netfilter/nf_conntrack_sip.h b/include/linux/netfilter/nf_conntrack_sip.h index dbc614dfe0d56..aafa0c04f917e 100644 --- a/include/linux/netfilter/nf_conntrack_sip.h +++ b/include/linux/netfilter/nf_conntrack_sip.h @@ -115,7 +115,7 @@ struct nf_nat_sip_hooks { unsigned int *datalen); void (*seq_adjust)(struct sk_buff *skb, - unsigned int protoff, s16 off); + unsigned int protoff, s32 off); unsigned int (*expect)(struct sk_buff *skb, unsigned int protoff, diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 972059a016c9a..e8a3ae2cebb6e 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -351,7 +351,7 @@ struct nfs4_layoutreturn { struct nfs4_xdr_opaque_data ld_private; }; -#define PNFS_LAYOUTSTATS_MAXSIZE 256 +#define PNFS_LAYOUTSTATS_MAXSIZE 384 struct nfs42_layoutstat_args; struct nfs42_layoutstat_devinfo; diff --git a/include/linux/ns_common.h b/include/linux/ns_common.h index f3c52904343e7..08ff44e631456 100644 --- a/include/linux/ns_common.h +++ b/include/linux/ns_common.h @@ -156,7 +156,12 @@ static __always_inline __must_check bool __ns_ref_get(struct ns_common *ns) return refcount_inc_not_zero(&ns->__ns_ref); } -#define ns_ref_read(__ns) refcount_read(&to_ns_common((__ns))->__ns_ref) +static __always_inline __must_check int __ns_ref_read(const struct ns_common *ns) +{ + return refcount_read(&ns->__ns_ref); +} + +#define ns_ref_read(__ns) __ns_ref_read(to_ns_common((__ns))) #define ns_ref_inc(__ns) refcount_inc(&to_ns_common((__ns))->__ns_ref) #define ns_ref_get(__ns) __ns_ref_get(to_ns_common((__ns))) #define ns_ref_put(__ns) __ns_ref_put(to_ns_common((__ns))) diff --git a/include/linux/nsproxy.h b/include/linux/nsproxy.h index bd118a187dec1..538ba8dba1840 100644 --- a/include/linux/nsproxy.h +++ b/include/linux/nsproxy.h @@ -93,7 +93,7 @@ static inline struct cred *nsset_cred(struct nsset *set) */ int copy_namespaces(u64 flags, struct task_struct *tsk); -void exit_task_namespaces(struct task_struct *tsk); +void exit_nsproxy_namespaces(struct task_struct *tsk); void switch_task_namespaces(struct task_struct *tsk, struct nsproxy *new); int exec_task_namespaces(void); void free_nsproxy(struct nsproxy *ns); diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 655d194f8e722..b9d3dbe2564d7 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -2332,4 +2332,8 @@ enum nvme_pr_change_ptpl { #define NVME_PR_IGNORE_KEY (1 << 3) +/* Section 8.3.4.5.2 of the NVMe 2.1 */ +#define NVME_AUTH_DHCHAP_MAX_HASH_IDS 30 +#define NVME_AUTH_DHCHAP_MAX_DH_IDS 30 + #endif /* _LINUX_NVME_H */ diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index f3b13da78aac9..6063fe5b7784a 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -214,6 +214,12 @@ static inline int nvmem_layout_register(struct nvmem_layout *layout) static inline void nvmem_layout_unregister(struct nvmem_layout *layout) {} +static inline int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, + struct device_node *np) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_NVMEM */ #if IS_ENABLED(CONFIG_NVMEM) && IS_ENABLED(CONFIG_OF) diff --git a/include/linux/oa_tc6.h b/include/linux/oa_tc6.h index 15f58e3c56c74..71cffcb041b1c 100644 --- a/include/linux/oa_tc6.h +++ b/include/linux/oa_tc6.h @@ -10,6 +10,56 @@ #include #include +/* OPEN Alliance TC6 registers */ +/* Standard Capabilities Register */ +#define OA_TC6_REG_STDCAP 0x0002 +#define OA_TC6_STDCAP_DIRECT_PHY_REG_ACCESS BIT(8) + +/* Reset Control and Status Register */ +#define OA_TC6_REG_RESET 0x0003 +#define OA_TC6_RESET_SWRESET BIT(0) /* Software Reset */ + +/* Configuration Register #0 */ +#define OA_TC6_REG_CONFIG0 0x0004 +#define OA_TC6_CONFIG0_SYNC BIT(15) +#define OA_TC6_CONFIG0_ZARFE_ENABLE BIT(12) +#define OA_TC6_CONFIG0_PROTE BIT(5) + +/* Status Register #0 */ +#define OA_TC6_REG_STATUS0 0x0008 +#define OA_TC6_STATUS0_RESETC BIT(6) /* Reset Complete */ +#define OA_TC6_STATUS0_HEADER_ERROR BIT(5) +#define OA_TC6_STATUS0_LOSS_OF_FRAME_ERROR BIT(4) +#define OA_TC6_STATUS0_RX_BUFFER_OVERFLOW_ERROR BIT(3) +#define OA_TC6_STATUS0_TX_PROTOCOL_ERROR BIT(0) + +/* Buffer Status Register */ +#define OA_TC6_REG_BUFFER_STATUS 0x000B +#define OA_TC6_BUFFER_STATUS_TX_CREDITS_AVAILABLE GENMASK(15, 8) +#define OA_TC6_BUFFER_STATUS_RX_CHUNKS_AVAILABLE GENMASK(7, 0) + +/* Interrupt Mask Register #0 */ +#define OA_TC6_REG_INT_MASK0 0x000C +#define OA_TC6_INT_MASK0_HEADER_ERR_MASK BIT(5) +#define OA_TC6_INT_MASK0_LOSS_OF_FRAME_ERR_MASK BIT(4) +#define OA_TC6_INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK BIT(3) +#define OA_TC6_INT_MASK0_TX_PROTOCOL_ERR_MASK BIT(0) +#define OA_TC6_INT_MASK0_ALL_INTERRUPTS (GENMASK(5, 0) | \ + GENMASK(12, 7)) + +/* PHY Clause 22 registers base address and mask */ +#define OA_TC6_PHY_STD_REG_ADDR_BASE 0xFF00 +#define OA_TC6_PHY_STD_REG_ADDR_MASK 0x1F + +/* PHY – Clause 45 registers memory map selector (MMS) as per table 6 in the + * OPEN Alliance specification. + */ +#define OA_TC6_PHY_C45_PCS_MMS2 2 /* MMD 3 */ +#define OA_TC6_PHY_C45_PMA_PMD_MMS3 3 /* MMD 1 */ +#define OA_TC6_PHY_C45_VS_PLCA_MMS4 4 /* MMD 31 */ +#define OA_TC6_PHY_C45_AUTO_NEG_MMS5 5 /* MMD 7 */ +#define OA_TC6_PHY_C45_POWER_UNIT_MMS6 6 /* MMD 13 */ + struct oa_tc6; struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev); diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index d930651473b4d..044f67ced6ff8 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -81,6 +81,9 @@ void __iomem *pci_ecam_map_bus(struct pci_bus *bus, unsigned int devfn, /* default ECAM ops */ extern const struct pci_ecam_ops pci_generic_ecam_ops; +/* default CAM ops */ +extern const struct pci_ecam_ops pci_generic_cam_ops; + #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) extern const struct pci_ecam_ops pci_32b_ops; /* 32-bit accesses only */ extern const struct pci_ecam_ops pci_32b_read_ops; /* 32-bit read only */ diff --git a/include/linux/pci.h b/include/linux/pci.h index 505bf6e009b18..148b71af80b32 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -72,12 +72,24 @@ /* return bus from PCI devid = ((u16)bus_number) << 8) | devfn */ #define PCI_BUS_NUM(x) (((x) >> 8) & 0xff) +/* + * PCI_SLOT_ALL_DEVICES indicates a slot that covers all devices on the bus. + * Used for PCIe hotplug where the physical slot is the entire secondary bus, + * and, if ARI Forwarding is enabled, functions may appear to be on multiple + * devices. + */ +#define PCI_SLOT_ALL_DEVICES 0xfeff + +/* Used to identify a slot as a placeholder */ +#define PCI_SLOT_PLACEHOLDER 0xffff + /* pci_slot represents a physical slot */ struct pci_slot { struct pci_bus *bus; /* Bus this slot is on */ struct list_head list; /* Node in list of slots */ struct hotplug_slot *hotplug; /* Hotplug info (move here) */ - unsigned char number; /* PCI_SLOT(pci_dev->devfn) */ + u16 number; /* Device nr, or PCI_SLOT_ALL_DEVICES */ + unsigned int per_func_slot:1; /* Allow per function slot */ struct kobject kobj; }; diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index f82a28040594b..ecaa40370830a 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -55,7 +55,7 @@ struct riscv_pmu { irqreturn_t (*handle_irq)(int irq_num, void *dev); - unsigned long cmask; + DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); u64 (*ctr_read)(struct perf_event *event); int (*ctr_get_idx)(struct perf_event *event); int (*ctr_get_width)(int idx); diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index de14923720a53..6ed1c4c5ce2ef 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -228,6 +228,7 @@ struct cros_ec_platform { /** * struct cros_ec_dev - ChromeOS EC device entry point. * @class_dev: Device structure used in sysfs. + * @group: sysfs attributes groups for this EC. * @ec_dev: cros_ec_device structure to talk to the physical device. * @dev: Pointer to the platform device. * @debug_info: cros_ec_debugfs structure for debugging information. @@ -237,6 +238,7 @@ struct cros_ec_platform { */ struct cros_ec_dev { struct device class_dev; + const struct attribute_group *group; struct cros_ec_device *ec_dev; struct device *dev; struct cros_ec_debugfs *debug_info; diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 0b436e15f4cd6..911d7a4d32c15 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -637,6 +637,30 @@ DEFINE_GUARD_COND(pm_runtime_active_auto, _try, DEFINE_GUARD_COND(pm_runtime_active_auto, _try_enabled, pm_runtime_resume_and_get(_T), _RET == 0) +/* ACQUIRE() wrapper macros for the guards defined above. */ + +#define PM_RUNTIME_ACQUIRE(_dev, _var) \ + ACQUIRE(pm_runtime_active_try, _var)(_dev) + +#define PM_RUNTIME_ACQUIRE_AUTOSUSPEND(_dev, _var) \ + ACQUIRE(pm_runtime_active_auto_try, _var)(_dev) + +#define PM_RUNTIME_ACQUIRE_IF_ENABLED(_dev, _var) \ + ACQUIRE(pm_runtime_active_try_enabled, _var)(_dev) + +#define PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(_dev, _var) \ + ACQUIRE(pm_runtime_active_auto_try_enabled, _var)(_dev) + +/* + * ACQUIRE_ERR() wrapper macro for guard pm_runtime_active. + * + * Always check PM_RUNTIME_ACQUIRE_ERR() after using one of the + * PM_RUNTIME_ACQUIRE*() macros defined above (yes, it can be used with + * any of them) and if it is nonzero, avoid accessing the given device. + */ +#define PM_RUNTIME_ACQUIRE_ERR(_var_ptr) \ + ACQUIRE_ERR(pm_runtime_active, _var_ptr) + /** * pm_runtime_put_sync - Drop device usage counter and run "idle check" if 0. * @dev: Target device. diff --git a/include/linux/psi.h b/include/linux/psi.h index e0745873e3f26..7966e3ac03b99 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of); + struct kernfs_open_file *of, + bool *need_rtpoll_worker); +int psi_trigger_create_rtpoll_worker(struct psi_group *group); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 6dbc5a1bf6a8c..d2fa1c41bfd2b 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -87,18 +87,18 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } /* - * Template for declaring augmented rbtree callbacks (generic case) + * Template for declaring augmented rbtree callbacks (generic multi fields) * * RBSTATIC: 'static' or empty * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + * RBCOPY: name of function that copies the RBAUGMENTED datas + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas */ -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ @@ -114,14 +114,14 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ RBCOMPUTE(old, false); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ @@ -130,6 +130,27 @@ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .rotate = RBNAME ## _rotate \ }; +/* + * Template for declaring augmented rbtree callbacks (generic single field) + * + * RBSTATIC: 'static' or empty + * RBNAME: name of the rb_augment_callbacks structure + * RBSTRUCT: struct type of the tree nodes + * RBFIELD: name of struct rb_node field within RBSTRUCT + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + */ + +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +static inline void \ +RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ +{ \ + new->RBAUGMENTED = old->RBAUGMENTED; \ +} \ +RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) + /* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index b4795698d8c2a..de98462d58889 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -37,279 +37,13 @@ #include #include +#include #include #include #include #include #include - -/** - * struct resource_table - firmware resource table header - * @ver: version number - * @num: number of resource entries - * @reserved: reserved (must be zero) - * @offset: array of offsets pointing at the various resource entries - * - * A resource table is essentially a list of system resources required - * by the remote processor. It may also include configuration entries. - * If needed, the remote processor firmware should contain this table - * as a dedicated ".resource_table" ELF section. - * - * Some resources entries are mere announcements, where the host is informed - * of specific remoteproc configuration. Other entries require the host to - * do something (e.g. allocate a system resource). Sometimes a negotiation - * is expected, where the firmware requests a resource, and once allocated, - * the host should provide back its details (e.g. address of an allocated - * memory region). - * - * The header of the resource table, as expressed by this structure, - * contains a version number (should we need to change this format in the - * future), the number of available resource entries, and their offsets - * in the table. - * - * Immediately following this header are the resource entries themselves, - * each of which begins with a resource entry header (as described below). - */ -struct resource_table { - u32 ver; - u32 num; - u32 reserved[2]; - u32 offset[]; -} __packed; - -/** - * struct fw_rsc_hdr - firmware resource entry header - * @type: resource type - * @data: resource data - * - * Every resource entry begins with a 'struct fw_rsc_hdr' header providing - * its @type. The content of the entry itself will immediately follow - * this header, and it should be parsed according to the resource type. - */ -struct fw_rsc_hdr { - u32 type; - u8 data[]; -} __packed; - -/** - * enum fw_resource_type - types of resource entries - * - * @RSC_CARVEOUT: request for allocation of a physically contiguous - * memory region. - * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. - * @RSC_TRACE: announces the availability of a trace buffer into which - * the remote processor will be writing logs. - * @RSC_VDEV: declare support for a virtio device, and serve as its - * virtio header. - * @RSC_LAST: just keep this one at the end of standard resources - * @RSC_VENDOR_START: start of the vendor specific resource types range - * @RSC_VENDOR_END: end of the vendor specific resource types range - * - * For more details regarding a specific resource type, please see its - * dedicated structure below. - * - * Please note that these values are used as indices to the rproc_handle_rsc - * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to - * check the validity of an index before the lookup table is accessed, so - * please update it as needed. - */ -enum fw_resource_type { - RSC_CARVEOUT = 0, - RSC_DEVMEM = 1, - RSC_TRACE = 2, - RSC_VDEV = 3, - RSC_LAST = 4, - RSC_VENDOR_START = 128, - RSC_VENDOR_END = 512, -}; - -#define FW_RSC_ADDR_ANY (-1) - -/** - * struct fw_rsc_carveout - physically contiguous memory request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested memory region - * - * This resource entry requests the host to allocate a physically contiguous - * memory region. - * - * These request entries should precede other firmware resource entries, - * as other entries might request placing other data objects inside - * these memory regions (e.g. data/code segments, trace resource entries, ...). - * - * Allocating memory this way helps utilizing the reserved physical memory - * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries - * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB - * pressure is important; it may have a substantial impact on performance. - * - * If the firmware is compiled with static addresses, then @da should specify - * the expected device address of this memory region. If @da is set to - * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then - * overwrite @da with the dynamically allocated address. - * - * We will always use @da to negotiate the device addresses, even if it - * isn't using an iommu. In that case, though, it will obviously contain - * physical addresses. - * - * Some remote processors needs to know the allocated physical address - * even if they do use an iommu. This is needed, e.g., if they control - * hardware accelerators which access the physical memory directly (this - * is the case with OMAP4 for instance). In that case, the host will - * overwrite @pa with the dynamically allocated physical address. - * Generally we don't want to expose physical addresses if we don't have to - * (remote processors are generally _not_ trusted), so we might want to - * change this to happen _only_ when explicitly required by the hardware. - * - * @flags is used to provide IOMMU protection flags, and @name should - * (optionally) contain a human readable name of this carveout region - * (mainly for debugging purposes). - */ -struct fw_rsc_carveout { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_devmem - iommu mapping request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested region to be mapped - * - * This resource entry requests the host to iommu map a physically contiguous - * memory region. This is needed in case the remote processor requires - * access to certain memory-based peripherals; _never_ use it to access - * regular memory. - * - * This is obviously only needed if the remote processor is accessing memory - * via an iommu. - * - * @da should specify the required device address, @pa should specify - * the physical address we want to map, @len should specify the size of - * the mapping and @flags is the IOMMU protection flags. As always, @name may - * (optionally) contain a human readable name of this mapping (mainly for - * debugging purposes). - * - * Note: at this point we just "trust" those devmem entries to contain valid - * physical addresses, but this isn't safe and will be changed: eventually we - * want remoteproc implementations to provide us ranges of physical addresses - * the firmware is allowed to request, and not allow firmwares to request - * access to physical addresses that are outside those ranges. - */ -struct fw_rsc_devmem { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_trace - trace buffer declaration - * @da: device address - * @len: length (in bytes) - * @reserved: reserved (must be zero) - * @name: human-readable name of the trace buffer - * - * This resource entry provides the host information about a trace buffer - * into which the remote processor will write log messages. - * - * @da specifies the device address of the buffer, @len specifies - * its size, and @name may contain a human readable name of the trace buffer. - * - * After booting the remote processor, the trace buffers are exposed to the - * user via debugfs entries (called trace0, trace1, etc..). - */ -struct fw_rsc_trace { - u32 da; - u32 len; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_vdev_vring - vring descriptor entry - * @da: device address - * @align: the alignment between the consumer and producer parts of the vring - * @num: num of buffers supported by this vring (must be power of two) - * @notifyid: a unique rproc-wide notify index for this vring. This notify - * index is used when kicking a remote processor, to let it know that this - * vring is triggered. - * @pa: physical address - * - * This descriptor is not a resource entry by itself; it is part of the - * vdev resource type (see below). - * - * Note that @da should either contain the device address where - * the remote processor is expecting the vring, or indicate that - * dynamically allocation of the vring's device address is supported. - */ -struct fw_rsc_vdev_vring { - u32 da; - u32 align; - u32 num; - u32 notifyid; - u32 pa; -} __packed; - -/** - * struct fw_rsc_vdev - virtio device header - * @id: virtio device id (as in virtio_ids.h) - * @notifyid: a unique rproc-wide notify index for this vdev. This notify - * index is used when kicking a remote processor, to let it know that the - * status/features of this vdev have changes. - * @dfeatures: specifies the virtio device features supported by the firmware - * @gfeatures: a place holder used by the host to write back the - * negotiated features that are supported by both sides. - * @config_len: the size of the virtio config space of this vdev. The config - * space lies in the resource table immediate after this vdev header. - * @status: a place holder where the host will indicate its virtio progress. - * @num_of_vrings: indicates how many vrings are described in this vdev header - * @reserved: reserved (must be zero) - * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. - * - * This resource is a virtio device header: it provides information about - * the vdev, and is then used by the host and its peer remote processors - * to negotiate and share certain virtio properties. - * - * By providing this resource entry, the firmware essentially asks remoteproc - * to statically allocate a vdev upon registration of the rproc (dynamic vdev - * allocation is not yet supported). - * - * Note: - * 1. unlike virtualization systems, the term 'host' here means - * the Linux side which is running remoteproc to control the remote - * processors. We use the name 'gfeatures' to comply with virtio's terms, - * though there isn't really any virtualized guest OS here: it's the host - * which is responsible for negotiating the final features. - * Yeah, it's a bit confusing. - * - * 2. immediately following this structure is the virtio config space for - * this vdev (which is specific to the vdev; for more info, read the virtio - * spec). The size of the config space is specified by @config_len. - */ -struct fw_rsc_vdev { - u32 id; - u32 notifyid; - u32 dfeatures; - u32 gfeatures; - u32 config_len; - u8 status; - u8 num_of_vrings; - u8 reserved[2]; - struct fw_rsc_vdev_vring vring[]; -} __packed; +#include struct rproc; @@ -412,7 +146,6 @@ struct rproc_ops { * a message. * @RPROC_RUNNING: device is up and running * @RPROC_CRASHED: device has crashed; need to start recovery - * @RPROC_DELETED: device is deleted * @RPROC_ATTACHED: device has been booted by another entity and the core * has attached to it * @RPROC_DETACHED: device has been booted by another entity and waiting @@ -430,10 +163,9 @@ enum rproc_state { RPROC_SUSPENDED = 1, RPROC_RUNNING = 2, RPROC_CRASHED = 3, - RPROC_DELETED = 4, - RPROC_ATTACHED = 5, - RPROC_DETACHED = 6, - RPROC_LAST = 7, + RPROC_ATTACHED = 4, + RPROC_DETACHED = 5, + RPROC_LAST = 6, }; /** @@ -526,7 +258,10 @@ enum rproc_features { * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device + * @attach_work: workqueue for attaching rproc * @crash_handler: workqueue for handling a crash + * @crash_handler_lock: serializes crash handler queueing and deletion + * @deleting: remoteproc deletion has begun * @crash_cnt: crash counter * @recovery_disabled: flag that state if recovery was disabled * @max_notifyid: largest allocated notify id. @@ -568,7 +303,10 @@ struct rproc { struct list_head subdevs; struct idr notifyids; int index; + struct work_struct attach_work; struct work_struct crash_handler; + spinlock_t crash_handler_lock; + bool deleting; unsigned int crash_cnt; bool recovery_disabled; int max_notifyid; diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index a7d92718b653f..f9bed457fc5f1 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -169,7 +169,7 @@ struct mbm_cntr_cfg { }; /** - * struct rdt_mon_domain - group of CPUs sharing a resctrl monitor resource + * struct rdt_l3_mon_domain - group of CPUs sharing RDT_RESOURCE_L3 monitoring * @hdr: common header for different domain types * @ci_id: cache info id for this domain * @rmid_busy_llc: bitmap of which limbo RMIDs are above threshold @@ -183,7 +183,7 @@ struct mbm_cntr_cfg { * @cntr_cfg: array of assignable counters' configuration (indexed * by counter ID) */ -struct rdt_mon_domain { +struct rdt_l3_mon_domain { struct rdt_domain_hdr hdr; unsigned int ci_id; unsigned long *rmid_busy_llc; @@ -355,10 +355,10 @@ struct resctrl_cpu_defaults { }; struct resctrl_mon_config_info { - struct rdt_resource *r; - struct rdt_mon_domain *d; - u32 evtid; - u32 mon_config; + struct rdt_resource *r; + struct rdt_l3_mon_domain *d; + u32 evtid; + u32 mon_config; }; /** @@ -495,9 +495,9 @@ int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d, u32 closid, enum resctrl_conf_type type); int resctrl_online_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d); -int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d); +int resctrl_online_mon_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d); void resctrl_offline_ctrl_domain(struct rdt_resource *r, struct rdt_ctrl_domain *d); -void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_mon_domain *d); +void resctrl_offline_mon_domain(struct rdt_resource *r, struct rdt_l3_mon_domain *d); void resctrl_online_cpu(unsigned int cpu); void resctrl_offline_cpu(unsigned int cpu); @@ -526,7 +526,7 @@ void resctrl_offline_cpu(unsigned int cpu); * Return: * 0 on success, or -EIO, -EINVAL etc on error. */ -int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_mon_domain *d, +int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid, u64 *val, void *arch_mon_ctx); @@ -573,7 +573,7 @@ struct rdt_domain_hdr *resctrl_find_domain(struct list_head *h, int id, * * This can be called from any CPU. */ -void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, enum resctrl_event_id eventid); @@ -586,7 +586,7 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_mon_domain *d, * * This can be called from any CPU. */ -void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_mon_domain *d); +void resctrl_arch_reset_rmid_all(struct rdt_resource *r, struct rdt_l3_mon_domain *d); /** * resctrl_arch_reset_all_ctrls() - Reset the control for each CLOSID to its @@ -612,7 +612,7 @@ void resctrl_arch_reset_all_ctrls(struct rdt_resource *r); * * This can be called from any CPU. */ -void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, enum resctrl_event_id evtid, u32 rmid, u32 closid, u32 cntr_id, bool assign); @@ -635,7 +635,7 @@ void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, * Return: * 0 on success, or -EIO, -EINVAL etc on error. */ -int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_mon_domain *d, +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val); @@ -650,7 +650,7 @@ int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_mon_domain *d, * * This can be called from any CPU. */ -void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_mon_domain *d, +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 closid, u32 rmid, int cntr_id, enum resctrl_event_id eventid); diff --git a/include/linux/rmap.h b/include/linux/rmap.h index daa92a58585d9..7afc6abe1c235 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -92,6 +92,7 @@ struct anon_vma_chain { }; enum ttu_flags { + TTU_USE_SHARED_ZEROPAGE = 0x2, /* for unused pages of large folios */ TTU_SPLIT_HUGE_PMD = 0x4, /* split huge PMD if any */ TTU_IGNORE_MLOCK = 0x8, /* ignore mlock */ TTU_SYNC = 0x10, /* avoid racy checks with PVMW_SYNC */ @@ -1000,12 +1001,8 @@ int mapping_wrprotect_range(struct address_space *mapping, pgoff_t pgoff, int pfn_mkclean_range(unsigned long pfn, unsigned long nr_pages, pgoff_t pgoff, struct vm_area_struct *vma); -enum rmp_flags { - RMP_LOCKED = 1 << 0, - RMP_USE_SHARED_ZEROPAGE = 1 << 1, -}; - -void remove_migration_ptes(struct folio *src, struct folio *dst, int flags); +void remove_migration_ptes(struct folio *src, struct folio *dst, + enum ttu_flags flags); /* * rmap_walk_control: To control rmap traversing for specific needs diff --git a/include/linux/rsc_table.h b/include/linux/rsc_table.h new file mode 100644 index 0000000000000..4cef11a2e3a2c --- /dev/null +++ b/include/linux/rsc_table.h @@ -0,0 +1,364 @@ +/* + * Resource table and its types data structure + * + * Copyright(c) 2011 Texas Instruments, Inc. + * Copyright(c) 2011 Google, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name Texas Instruments nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RSC_TABLE_H +#define RSC_TABLE_H + +/** + * struct resource_table - firmware resource table header + * @ver: version number + * @num: number of resource entries + * @reserved: reserved (must be zero) + * @offset: array of offsets pointing at the various resource entries + * + * A resource table is essentially a list of system resources required + * by the remote processor. It may also include configuration entries. + * If needed, the remote processor firmware should contain this table + * as a dedicated ".resource_table" ELF section. + * + * Some resources entries are mere announcements, where the host is informed + * of specific remoteproc configuration. Other entries require the host to + * do something (e.g. allocate a system resource). Sometimes a negotiation + * is expected, where the firmware requests a resource, and once allocated, + * the host should provide back its details (e.g. address of an allocated + * memory region). + * + * The header of the resource table, as expressed by this structure, + * contains a version number (should we need to change this format in the + * future), the number of available resource entries, and their offsets + * in the table. + * + * Immediately following this header are the resource entries themselves, + * each of which begins with a resource entry header (as described below). + */ +struct resource_table { + u32 ver; + u32 num; + u32 reserved[2]; + u32 offset[]; +} __packed; + +/** + * struct fw_rsc_hdr - firmware resource entry header + * @type: resource type + * @data: resource data + * + * Every resource entry begins with a 'struct fw_rsc_hdr' header providing + * its @type. The content of the entry itself will immediately follow + * this header, and it should be parsed according to the resource type. + */ +struct fw_rsc_hdr { + u32 type; + u8 data[]; +} __packed; + +/** + * enum fw_resource_type - types of resource entries + * + * @RSC_CARVEOUT: request for allocation of a physically contiguous + * memory region. + * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. + * @RSC_TRACE: announces the availability of a trace buffer into which + * the remote processor will be writing logs. + * @RSC_VDEV: declare support for a virtio device, and serve as its + * virtio header. + * @RSC_LAST: just keep this one at the end of standard resources + * @RSC_VENDOR_START: start of the vendor specific resource types range + * @RSC_VENDOR_END: end of the vendor specific resource types range + * + * For more details regarding a specific resource type, please see its + * dedicated structure below. + * + * Please note that these values are used as indices to the rproc_handle_rsc + * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to + * check the validity of an index before the lookup table is accessed, so + * please update it as needed. + */ +enum fw_resource_type { + RSC_CARVEOUT = 0, + RSC_DEVMEM = 1, + RSC_TRACE = 2, + RSC_VDEV = 3, + RSC_LAST = 4, + RSC_VENDOR_START = 128, + RSC_VENDOR_END = 512, +}; + +#define FW_RSC_ADDR_ANY (-1) + +/** + * struct fw_rsc_carveout - physically contiguous memory request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested memory region + * + * This resource entry requests the host to allocate a physically contiguous + * memory region. + * + * These request entries should precede other firmware resource entries, + * as other entries might request placing other data objects inside + * these memory regions (e.g. data/code segments, trace resource entries, ...). + * + * Allocating memory this way helps utilizing the reserved physical memory + * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries + * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB + * pressure is important; it may have a substantial impact on performance. + * + * If the firmware is compiled with static addresses, then @da should specify + * the expected device address of this memory region. If @da is set to + * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then + * overwrite @da with the dynamically allocated address. + * + * We will always use @da to negotiate the device addresses, even if it + * isn't using an iommu. In that case, though, it will obviously contain + * physical addresses. + * + * Some remote processors needs to know the allocated physical address + * even if they do use an iommu. This is needed, e.g., if they control + * hardware accelerators which access the physical memory directly (this + * is the case with OMAP4 for instance). In that case, the host will + * overwrite @pa with the dynamically allocated physical address. + * Generally we don't want to expose physical addresses if we don't have to + * (remote processors are generally _not_ trusted), so we might want to + * change this to happen _only_ when explicitly required by the hardware. + * + * @flags is used to provide IOMMU protection flags, and @name should + * (optionally) contain a human readable name of this carveout region + * (mainly for debugging purposes). + */ +struct fw_rsc_carveout { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_devmem - iommu mapping request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested region to be mapped + * + * This resource entry requests the host to iommu map a physically contiguous + * memory region. This is needed in case the remote processor requires + * access to certain memory-based peripherals; _never_ use it to access + * regular memory. + * + * This is obviously only needed if the remote processor is accessing memory + * via an iommu. + * + * @da should specify the required device address, @pa should specify + * the physical address we want to map, @len should specify the size of + * the mapping and @flags is the IOMMU protection flags. As always, @name may + * (optionally) contain a human readable name of this mapping (mainly for + * debugging purposes). + * + * Note: at this point we just "trust" those devmem entries to contain valid + * physical addresses, but this isn't safe and will be changed: eventually we + * want remoteproc implementations to provide us ranges of physical addresses + * the firmware is allowed to request, and not allow firmwares to request + * access to physical addresses that are outside those ranges. + */ +struct fw_rsc_devmem { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_trace - trace buffer declaration + * @da: device address + * @len: length (in bytes) + * @reserved: reserved (must be zero) + * @name: human-readable name of the trace buffer + * + * This resource entry provides the host information about a trace buffer + * into which the remote processor will write log messages. + * + * @da specifies the device address of the buffer, @len specifies + * its size, and @name may contain a human readable name of the trace buffer. + * + * After booting the remote processor, the trace buffers are exposed to the + * user via debugfs entries (called trace0, trace1, etc..). + */ +struct fw_rsc_trace { + u32 da; + u32 len; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_vdev_vring - vring descriptor entry + * @da: device address + * @align: the alignment between the consumer and producer parts of the vring + * @num: num of buffers supported by this vring (must be power of two) + * @notifyid: a unique rproc-wide notify index for this vring. This notify + * index is used when kicking a remote processor, to let it know that this + * vring is triggered. + * @pa: physical address + * + * This descriptor is not a resource entry by itself; it is part of the + * vdev resource type (see below). + * + * Note that @da should either contain the device address where + * the remote processor is expecting the vring, or indicate that + * dynamically allocation of the vring's device address is supported. + */ +struct fw_rsc_vdev_vring { + u32 da; + u32 align; + u32 num; + u32 notifyid; + u32 pa; +} __packed; + +/** + * struct fw_rsc_vdev - virtio device header + * @id: virtio device id (as in virtio_ids.h) + * @notifyid: a unique rproc-wide notify index for this vdev. This notify + * index is used when kicking a remote processor, to let it know that the + * status/features of this vdev have changes. + * @dfeatures: specifies the virtio device features supported by the firmware + * @gfeatures: a place holder used by the host to write back the + * negotiated features that are supported by both sides. + * @config_len: the size of the virtio config space of this vdev. The config + * space lies in the resource table immediate after this vdev header. + * @status: a place holder where the host will indicate its virtio progress. + * @num_of_vrings: indicates how many vrings are described in this vdev header + * @reserved: reserved (must be zero) + * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. + * + * This resource is a virtio device header: it provides information about + * the vdev, and is then used by the host and its peer remote processors + * to negotiate and share certain virtio properties. + * + * By providing this resource entry, the firmware essentially asks remoteproc + * to statically allocate a vdev upon registration of the rproc (dynamic vdev + * allocation is not yet supported). + * + * Note: + * 1. unlike virtualization systems, the term 'host' here means + * the Linux side which is running remoteproc to control the remote + * processors. We use the name 'gfeatures' to comply with virtio's terms, + * though there isn't really any virtualized guest OS here: it's the host + * which is responsible for negotiating the final features. + * Yeah, it's a bit confusing. + * + * 2. immediately following this structure is the virtio config space for + * this vdev (which is specific to the vdev; for more info, read the virtio + * spec). The size of the config space is specified by @config_len. + */ +struct fw_rsc_vdev { + u32 id; + u32 notifyid; + u32 dfeatures; + u32 gfeatures; + u32 config_len; + u8 status; + u8 num_of_vrings; + u8 reserved[2]; + struct fw_rsc_vdev_vring vring[]; +} __packed; + +/** + * rsc_table_for_each_entry() - iterate over all entries in a resource table + * @table: pointer to the resource table + * @table_sz: total size of the table buffer in bytes + * @dev: device used for error logging + * @cb: callback invoked for each entry: + * @type - value from enum fw_resource_type + * @rsc - pointer to the entry payload (past struct fw_rsc_hdr) + * @offset - byte offset of the payload within the table; callers + * that write back into the table (e.g. to record a + * dynamically allocated address) use this to locate the + * entry for later update + * @avail - bytes available in the payload + * @data - caller-supplied private pointer + * Return 0 to continue iteration, non-zero to stop. + * @data: private pointer forwarded to @cb on every call + * + * Iterates over every resource entry in @table, performing the standard + * truncation check, and invokes @cb for each one. Iteration stops on the + * first non-zero return from @cb or on a malformed table. + * + * Returns 0 after a complete iteration, -EINVAL if the table is truncated, + * or the first non-zero value returned by @cb. + */ +static inline int rsc_table_for_each_entry(struct resource_table *table, + size_t table_sz, + struct device *dev, + int (*cb)(u32 type, void *rsc, + int offset, int avail, + void *data), + void *data) { + int i, ret; + + for (i = 0; i < table->num; i++) { + u32 offset = table->offset[i]; + struct fw_rsc_hdr *hdr; + int avail, rsc_offset; + void *rsc; + + if (offset < sizeof(*table) || offset >= table_sz || + table_sz - offset < sizeof(*hdr)) { + dev_err(dev, "rsc table is truncated\n"); + return -EINVAL; + } + + hdr = (void *)table + offset; + avail = table_sz - offset - sizeof(*hdr); + rsc_offset = offset + sizeof(*hdr); + rsc = (void *)hdr + sizeof(*hdr); + + ret = cb(hdr->type, rsc, rsc_offset, avail, data); + if (ret) + return ret; + } + + return 0; +} + +#endif /* RSC_TABLE_H */ diff --git a/include/linux/rtsx_usb.h b/include/linux/rtsx_usb.h index 276b509c03e36..0fc5a74700a8b 100644 --- a/include/linux/rtsx_usb.h +++ b/include/linux/rtsx_usb.h @@ -61,6 +61,9 @@ struct rtsx_ucr { struct timer_list sg_timer; struct mutex dev_mutex; + + u16 card_status_cache; + bool card_status_valid; }; /* buffer size */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 5dea369fcfc96..e20e0bd7e96b3 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -587,15 +587,10 @@ struct sched_entity { u64 sum_exec_runtime; u64 prev_sum_exec_runtime; u64 vruntime; - union { - /* - * When !@on_rq this field is vlag. - * When cfs_rq->curr == se (which implies @on_rq) - * this field is vprot. See protect_slice(). - */ - s64 vlag; - u64 vprot; - }; + /* Approximated virtual lag: */ + s64 vlag; + /* 'Protected' deadline, to give out minimum quantums: */ + u64 vprot; u64 slice; u64 nr_migrations; @@ -1545,6 +1540,14 @@ struct task_struct { /* Collect coverage from softirq context: */ unsigned int kcov_softirq; + + /* Temporary storage for preempting remote coverage collection: */ + unsigned int kcov_saved_mode; + unsigned int kcov_saved_size; + void *kcov_saved_area; + struct kcov *kcov_saved_kcov; + int kcov_saved_sequence; + #endif #ifdef CONFIG_MEMCG_V1 @@ -1732,7 +1735,7 @@ static inline char task_state_to_char(struct task_struct *tsk) return task_index_to_char(task_state_index(tsk)); } -extern struct pid *cad_pid; +extern struct pid __rcu *cad_pid; /* * Per process flags diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 4e3338103654c..922935cc33833 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -52,8 +52,10 @@ static inline bool rt_or_dl_task_policy(struct task_struct *tsk) #ifdef CONFIG_RT_MUTEXES extern void rt_mutex_pre_schedule(void); +extern void rt_mutex_futex_pre_schedule(void); extern void rt_mutex_schedule(void); extern void rt_mutex_post_schedule(void); +extern void rt_mutex_futex_post_schedule(void); /* * Must hold either p->pi_lock or task_rq(p)->lock. diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 7d6449982822e..6e6e72b7b3623 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -561,10 +561,7 @@ static inline sigset_t *sigmask_to_save(void) return res; } -static inline int kill_cad_pid(int sig, int priv) -{ - return kill_pid(cad_pid, sig, priv); -} +int kill_cad_pid(int sig, int priv); /* These can be the second arg to send_sig_info/send_group_sig_info. */ #define SEND_SIG_NOINFO ((struct kernel_siginfo *) 0) diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 4cc52698e214e..8d7e5521f7cdd 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -25,7 +25,8 @@ struct user_struct { #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \ defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ - defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) + defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) || \ + defined(CONFIG_SECRETMEM) atomic_long_t locked_vm; #endif #ifdef CONFIG_WATCH_QUEUE diff --git a/include/linux/secure_boot.h b/include/linux/secure_boot.h new file mode 100644 index 0000000000000..d17e923515672 --- /dev/null +++ b/include/linux/secure_boot.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved. + * + * Author: Coiby Xu + */ + +#ifndef _LINUX_SECURE_BOOT_H +#define _LINUX_SECURE_BOOT_H + +#include + +#ifdef CONFIG_HAVE_ARCH_GET_SECUREBOOT +/* + * Returns true if the platform secure boot is enabled. + * Returns false if disabled or not supported. + */ +bool arch_get_secureboot(void); +#else +static inline bool arch_get_secureboot(void) { return false; } +#endif + +#endif /* _LINUX_SECURE_BOOT_H */ diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 6ab7924f33f59..6729245aa1543 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1224,7 +1224,7 @@ struct ss_tmp { spinlock_t *lock_irqsave; }; -static inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) +static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) { if (sst->lock) spin_unlock(sst->lock); @@ -1234,11 +1234,15 @@ static inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) extern void __scoped_seqlock_invalid_target(void); -#if defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000 +#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \ + defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT) /* * For some reason some GCC-8 architectures (nios2, alpha) have trouble * determining that the ss_done state is impossible in __scoped_seqlock_next() * below. + * + * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough + * to break this. But we don't care about code quality for such builds anyway. */ static inline void __scoped_seqlock_bug(void) { } #else @@ -1249,7 +1253,7 @@ static inline void __scoped_seqlock_bug(void) { } extern void __scoped_seqlock_bug(void); #endif -static inline void +static __always_inline void __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) { switch (sst->state) { diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 110ad4e2aef99..bdc214386e4a5 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -533,6 +533,7 @@ struct uart_port { #define UPF_HARD_FLOW ((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS)) /* Port has hardware-assisted s/w flow control */ #define UPF_SOFT_FLOW ((__force upf_t) BIT_ULL(22)) +/* Deprecated: use uart_set_cons_flow_enabled()/uart_cons_flow_enabled() instead. */ #define UPF_CONS_FLOW ((__force upf_t) BIT_ULL(23)) #define UPF_SHARE_IRQ ((__force upf_t) BIT_ULL(24)) #define UPF_EXAR_EFR ((__force upf_t) BIT_ULL(25)) @@ -567,6 +568,7 @@ struct uart_port { #define UPSTAT_SYNC_FIFO ((__force upstat_t) (1 << 5)) bool hw_stopped; /* sw-assisted CTS flow state */ + bool cons_flow; /* user specified console flow control */ unsigned int mctrl; /* current modem ctrl settings */ unsigned int frame_time; /* frame timing in ns */ unsigned int type; /* port type */ @@ -1163,6 +1165,24 @@ static inline bool uart_softcts_mode(struct uart_port *uport) return ((uport->status & mask) == UPSTAT_CTS_ENABLE); } +static inline void uart_set_cons_flow_enabled(struct uart_port *uport, bool enabled) +{ + uport->cons_flow = enabled; +} + +static inline bool uart_cons_flow_enabled(const struct uart_port *uport) +{ + return uport->cons_flow; +} + +static inline bool uart_console_hwflow_active(struct uart_port *uport) +{ + return uart_console(uport) && + !(uport->rs485.flags & SER_RS485_ENABLED) && + uart_cons_flow_enabled(uport) && + uart_cts_enabled(uport); +} + /* * The following are helper functions for the low level drivers. */ diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index def9cca948178..c9abf7e3accf4 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3066,6 +3066,11 @@ static inline bool skb_transport_header_was_set(const struct sk_buff *skb) return skb->transport_header != (typeof(skb->transport_header))~0U; } +static inline void skb_unset_transport_header(struct sk_buff *skb) +{ + skb->transport_header = (typeof(skb->transport_header))~0U; +} + static inline unsigned char *skb_transport_header(const struct sk_buff *skb) { DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); @@ -3110,6 +3115,30 @@ static inline void skb_set_transport_header(struct sk_buff *skb, skb->transport_header += offset; } +/** + * skb_set_transport_header_careful - conditionally set transport header + * @skb: buffer to alter + * @offset: offset to add to skb->data + * + * Hardened version of skb_set_transport_header(). + * + * Returns: true if the operation was a success. + */ +static inline bool __must_check +skb_set_transport_header_careful(struct sk_buff *skb, const int offset) +{ + long thoff = skb->data - skb->head + offset; + + if (unlikely(thoff != (typeof(skb->transport_header))thoff)) + return false; + + if (unlikely(thoff == (typeof(skb->transport_header))~0U)) + return false; + + skb->transport_header = thoff; + return true; +} + static inline unsigned char *skb_network_header(const struct sk_buff *skb) { return skb->head + skb->network_header; diff --git a/include/linux/slab.h b/include/linux/slab.h index 22daf3f34a76f..a12601cc93a1b 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -238,7 +238,7 @@ enum _slab_flag_bits { #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ -/* Slab created using create_boot_cache */ +/* Slab caches without obj_exts array */ #ifdef CONFIG_SLAB_OBJ_EXT #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) #else @@ -641,6 +641,9 @@ enum kmalloc_cache_type { #endif #ifndef CONFIG_MEMCG KMALLOC_CGROUP = KMALLOC_NORMAL, +#endif +#ifndef CONFIG_SLAB_OBJ_EXT + KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL, #endif KMALLOC_RANDOM_START = KMALLOC_NORMAL, KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR, @@ -654,6 +657,9 @@ enum kmalloc_cache_type { #endif #ifdef CONFIG_MEMCG KMALLOC_CGROUP, +#endif +#ifdef CONFIG_SLAB_OBJ_EXT + KMALLOC_NO_OBJ_EXT, #endif NR_KMALLOC_TYPES }; diff --git a/include/linux/soc/airoha/airoha_offload.h b/include/linux/soc/airoha/airoha_offload.h index d4f6e8124a493..7589fccfeef6d 100644 --- a/include/linux/soc/airoha/airoha_offload.h +++ b/include/linux/soc/airoha/airoha_offload.h @@ -6,6 +6,7 @@ #ifndef AIROHA_OFFLOAD_H #define AIROHA_OFFLOAD_H +#include #include #include diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 0a984e2579fe2..16d68622954ab 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -118,6 +118,7 @@ struct geni_se { #define SE_DMA_RX_FSM_RST 0xd58 #define SE_HW_PARAM_0 0xe24 #define SE_HW_PARAM_1 0xe28 +#define SE_HW_PARAM_2 0xe2c /* GENI_FORCE_DEFAULT_REG fields */ #define FORCE_DEFAULT BIT(0) @@ -285,6 +286,9 @@ struct geni_se { #define RX_FIFO_DEPTH_MSK GENMASK(21, 16) #define RX_FIFO_DEPTH_SHFT 16 +/* SE_HW_PARAM_2 fields */ +#define PROG_RAM_DEPTH_MSK GENMASK(10, 0) + #define HW_VER_MAJOR_MASK GENMASK(31, 28) #define HW_VER_MAJOR_SHFT 28 #define HW_VER_MINOR_MASK GENMASK(27, 16) diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index d9dfc9edc1b2f..d3bb18990f84d 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,6 +8,7 @@ #define __QCOM_UBWC_H__ #include +#include #include #include diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index e6a3476bcef1a..bf2ddd429620f 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -1093,6 +1093,8 @@ int sdw_slave_get_current_bank(struct sdw_slave *sdev); int sdw_slave_get_scale_index(struct sdw_slave *slave, u8 *base); +int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms); + /* messaging and data APIs */ int sdw_read(struct sdw_slave *slave, u32 addr); int sdw_write(struct sdw_slave *slave, u32 addr, u8 value); @@ -1136,6 +1138,12 @@ static inline int sdw_slave_get_current_bank(struct sdw_slave *sdev) return -EINVAL; } +static inline int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms) +{ + WARN_ONCE(1, "SoundWire API is disabled"); + return -EINVAL; +} + /* messaging and data APIs */ static inline int sdw_read(struct sdw_slave *slave, u32 addr) { diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h index 7516ad0fae808..655681cf8fedc 100644 --- a/include/linux/sunrpc/svc_rdma_pcl.h +++ b/include/linux/sunrpc/svc_rdma_pcl.h @@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk) */ #define pcl_for_each_segment(pos, chunk) \ for (pos = &(chunk)->ch_segments[0]; \ - pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \ + pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \ pos++) /** diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index 963bbe251e521..de37069aba908 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -26,6 +26,9 @@ struct svc_sock { void (*sk_odata)(struct sock *); void (*sk_owspace)(struct sock *); + /* For sends (protected by xpt_mutex) */ + struct bio_vec *sk_bvec; + /* private TCP part */ /* On-the-wire fragment header: */ __be32 sk_marker; diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 152597750f55b..656e0ad323af1 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -138,8 +138,23 @@ void xdr_terminate_string(const struct xdr_buf *, const u32); size_t xdr_buf_pagecount(const struct xdr_buf *buf); int xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp); void xdr_free_bvec(struct xdr_buf *buf); -unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, - const struct xdr_buf *xdr); +int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, + const struct xdr_buf *xdr); +int xdr_buf_to_sg(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg, unsigned int nsg); +int xdr_buf_to_sg_alloc(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg_head, + unsigned int sg_head_nents, + struct scatterlist **sg_overflow, gfp_t gfp); + +/* + * Inline scatterlist entries for xdr_buf_to_sg_alloc(). Sized to cover the + * head kvec, tail kvec, and a few page fragments without any heap allocation. + */ +enum { + XDR_BUF_TO_SG_NENTS = 8, +}; + static inline __be32 *xdr_encode_array(__be32 *p, const void *s, unsigned int len) { diff --git a/include/linux/swapops.h b/include/linux/swapops.h index a73c5f14b5912..1966b4890d469 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -120,6 +120,8 @@ static inline unsigned long swp_offset_pfn(swp_entry_t entry) return swp_offset(entry) & SWP_PFN_MASK; } +#define softleaf_to_pfn swp_offset_pfn + /* check whether a pte points to a swap entry */ static inline int is_swap_pte(pte_t pte) { @@ -139,6 +141,8 @@ static inline swp_entry_t pte_to_swp_entry(pte_t pte) return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +#define softleaf_from_pte pte_to_swp_entry + /* * Convert the arch-independent representation of a swp_entry_t into the * arch-dependent pte representation. @@ -570,9 +574,15 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -static inline int is_pmd_migration_entry(pmd_t pmd) +static inline int pmd_is_migration_entry(pmd_t pmd) { - return is_swap_pmd(pmd) && is_migration_entry(pmd_to_swp_entry(pmd)); + swp_entry_t entry; + + if (pmd_present(pmd)) + return 0; + + entry = pmd_to_swp_entry(pmd); + return is_migration_entry(entry); } #else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, @@ -599,7 +609,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -static inline int is_pmd_migration_entry(pmd_t pmd) +static inline int pmd_is_migration_entry(pmd_t pmd) { return 0; } diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index cdb58d61c1526..724da6c08bf7d 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -42,6 +42,7 @@ enum switchtec_gen { SWITCHTEC_GEN3, SWITCHTEC_GEN4, SWITCHTEC_GEN5, + SWITCHTEC_GEN6, }; struct mrpc_regs { diff --git a/include/linux/syscore_ops.h b/include/linux/syscore_ops.h index ae4d48e4c970a..ac6d71be5c382 100644 --- a/include/linux/syscore_ops.h +++ b/include/linux/syscore_ops.h @@ -11,14 +11,19 @@ #include struct syscore_ops { + int (*suspend)(void *data); + void (*resume)(void *data); + void (*shutdown)(void *data); +}; + +struct syscore { struct list_head node; - int (*suspend)(void); - void (*resume)(void); - void (*shutdown)(void); + const struct syscore_ops *ops; + void *data; }; -extern void register_syscore_ops(struct syscore_ops *ops); -extern void unregister_syscore_ops(struct syscore_ops *ops); +extern void register_syscore(struct syscore *syscore); +extern void unregister_syscore(struct syscore *syscore); #ifdef CONFIG_PM_SLEEP extern int syscore_suspend(void); extern void syscore_resume(void); diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 0ba112175bb39..466f315f0be7b 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -209,6 +209,8 @@ enum tb_link_width { * @link_width: Width of the downstream facing link * @link_usb4: Downstream link is USB4 * @is_unplugged: The XDomain is unplugged + * @removing: Set by tb_xdomain_remove() under @lock to prevent + * concurrent delayed work queueing * @needs_uuid: If the XDomain does not have @remote_uuid it will be * queried first * @service_ids: Used to generate IDs for the services @@ -228,6 +230,7 @@ enum tb_link_width { * changed notification * @bonding_possible: True if lane bonding is possible on local side * @target_link_width: Target link width from the remote host + * @ntunnels: Keeps track of how many tunnels go through this XDomain * @link: Root switch link the remote domain is connected (ICM only) * @depth: Depth in the chain the remote domain is connected (ICM only) * @@ -257,6 +260,7 @@ struct tb_xdomain { enum tb_link_width link_width; bool link_usb4; bool is_unplugged; + bool removing; bool needs_uuid; struct ida service_ids; struct ida in_hopids; @@ -273,6 +277,7 @@ struct tb_xdomain { int properties_changed_retries; bool bonding_possible; u8 target_link_width; + atomic_t ntunnels; u8 link; u8 depth; }; diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index c514d0e5a45cb..06a83e198dedc 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -8,6 +8,7 @@ #include #include #include +#include struct user_namespace; extern struct user_namespace init_user_ns; @@ -38,8 +39,6 @@ static inline struct time_namespace *to_time_ns(struct ns_common *ns) return container_of(ns, struct time_namespace, ns); } void __init time_ns_init(void); -extern int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns); extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns); static inline struct time_namespace *get_time_ns(struct time_namespace *ns) @@ -117,12 +116,6 @@ static inline void __init time_ns_init(void) { } -static inline int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns) -{ - return 0; -} - static inline void timens_commit(struct task_struct *tsk, struct time_namespace *ns) { @@ -175,4 +168,6 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif +DEFINE_FREE(time_ns, struct time_namespace *, if (_T) put_time_ns(_T)) + #endif /* _LINUX_TIMENS_H */ diff --git a/include/linux/usb.h b/include/linux/usb.h index 75be53a39b8aa..2f5e72804ec1d 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -1185,7 +1185,8 @@ extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf); * interface. It may also use usb_set_interface() to specify the * appropriate altsetting. If unwilling to manage the interface, * return -ENODEV, if genuine IO errors occurred, an appropriate - * negative errno value. + * negative errno value. The usb_device_id parameter is only valid during + * probe. * @disconnect: Called when the interface is no longer accessible, usually * because its device has been (or is being) disconnected or the * driver module is being unloaded. diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index b3cc7beab4a3c..a4043b33c2c2a 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -81,4 +81,7 @@ /* Device claims zero configurations, forcing to 1 */ #define USB_QUIRK_FORCE_ONE_CONFIG BIT(18) +/* Use a 255 bytes config descriptor request mirroring windows behavior */ +#define USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE BIT(19) + #endif /* __LINUX_USB_QUIRKS_H */ diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index f7f5cfbdef120..9b46a6bc762cc 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -144,6 +144,7 @@ #define TCPC_RX_BUF_FRAME_TYPE 0x31 #define TCPC_RX_BUF_FRAME_TYPE_SOP 0 #define TCPC_RX_BUF_FRAME_TYPE_SOP1 1 +#define TCPC_RX_BUF_FRAME_TYPE_MASK GENMASK(2, 0) #define TCPC_RX_HDR 0x32 #define TCPC_RX_DATA 0x34 /* through 0x4f */ diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index c0e716aec26aa..796a380664bd8 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -214,7 +214,10 @@ static inline bool vma_can_userfault(struct vm_area_struct *vma, { vm_flags &= __VM_UFFD_FLAGS; - if (vma->vm_flags & VM_DROPPABLE) + if (vma->vm_flags & (VM_DROPPABLE | VM_SHADOW_STACK)) + return false; + + if (!is_vm_hugetlb_page(vma) && (vma->vm_flags & VM_SPECIAL)) return false; if ((vm_flags & VM_UFFD_MINOR) && diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 132a474e59140..9918538f59ca8 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -41,13 +41,13 @@ struct virtqueue { void *priv; }; -struct vduse_iova_domain; +struct vduse_vq_group; union virtio_map { /* Device that performs DMA */ struct device *dma_dev; - /* VDUSE specific mapping data */ - struct vduse_iova_domain *iova_domain; + /* VDUSE specific virtqueue group for doing map */ + struct vduse_vq_group *group; }; int virtqueue_add_outbuf(struct virtqueue *vq, @@ -204,6 +204,7 @@ int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); #endif void virtio_reset_device(struct virtio_device *dev); +void virtio_device_shutdown(struct virtio_device *dev); int virtio_device_reset_prepare(struct virtio_device *dev); int virtio_device_reset_done(struct virtio_device *dev); diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index f36d21b5bc19e..c381b916c1b54 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -179,6 +180,9 @@ static inline int __virtio_net_hdr_to_skb(struct sk_buff *skb, if (skb->ip_summed == CHECKSUM_PARTIAL && skb->csum_offset != offsetof(struct tcphdr, check)) return -EINVAL; + + BUILD_BUG_ON(TCP_MIN_GSO_SIZE * GSO_MAX_SEGS < GSO_MAX_SIZE); + gso_size = max(gso_size, TCP_MIN_GSO_SIZE); break; } diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 29dd5b91dd7d6..3fc04b849e4e3 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -32,11 +32,17 @@ extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset); extern void vtime_account_softirq(struct task_struct *tsk); extern void vtime_account_hardirq(struct task_struct *tsk); extern void vtime_flush(struct task_struct *tsk); +extern void vtime_reset(void); +extern void vtime_dyntick_start(void); +extern void vtime_dyntick_stop(void); #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static inline void vtime_account_irq(struct task_struct *tsk, unsigned int offset) { } static inline void vtime_account_softirq(struct task_struct *tsk) { } static inline void vtime_account_hardirq(struct task_struct *tsk) { } static inline void vtime_flush(struct task_struct *tsk) { } +static inline void vtime_reset(void) { } +static inline void vtime_dyntick_start(void) { } +static inline void vtime_dyntick_stop(void) { } #endif /* diff --git a/include/linux/wait.h b/include/linux/wait.h index f648044466d5f..412ccf76abc04 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -1213,6 +1213,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); #define DEFINE_WAIT_FUNC(name, function) \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 9e29d79fc790a..88f8afd6b6e1b 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode); struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit); extern void __init wait_bit_init(void); +extern struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg); int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); diff --git a/include/media/rc-map.h b/include/media/rc-map.h index d90e4611b0664..d95ed3e96de28 100644 --- a/include/media/rc-map.h +++ b/include/media/rc-map.h @@ -262,7 +262,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_GENIUS_TVGO_A11MCE "rc-genius-tvgo-a11mce" #define RC_MAP_GOTVIEW7135 "rc-gotview7135" #define RC_MAP_HAUPPAUGE "rc-hauppauge" -#define RC_MAP_HAUPPAUGE_NEW "rc-hauppauge" #define RC_MAP_HISI_POPLAR "rc-hisi-poplar" #define RC_MAP_HISI_TV_DEMO "rc-hisi-tv-demo" #define RC_MAP_IMON_MCE "rc-imon-mce" @@ -309,7 +308,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_PROTEUS_2309 "rc-proteus-2309" #define RC_MAP_PURPLETV "rc-purpletv" #define RC_MAP_PV951 "rc-pv951" -#define RC_MAP_RC5_TV "rc-rc5-tv" #define RC_MAP_RC6_MCE "rc-rc6-mce" #define RC_MAP_REAL_AUDIO_220_32_KEYS "rc-real-audio-220-32-keys" #define RC_MAP_REDDO "rc-reddo" diff --git a/include/media/v4l2-async.h b/include/media/v4l2-async.h index f26c323e9c963..54a2d9620ed5b 100644 --- a/include/media/v4l2-async.h +++ b/include/media/v4l2-async.h @@ -333,8 +333,10 @@ int __v4l2_async_register_subdev(struct v4l2_subdev *sd, struct module *module); * An error is returned if the module is no longer loaded on any attempts * to register it. */ +#define v4l2_async_register_subdev_sensor(sd) \ + __v4l2_async_register_subdev_sensor(sd, THIS_MODULE) int __must_check -v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd); +__v4l2_async_register_subdev_sensor(struct v4l2_subdev *sd, struct module *module); /** * v4l2_async_unregister_subdev - unregisters a sub-device to the asynchronous diff --git a/include/media/v4l2-hevc.h b/include/media/v4l2-hevc.h new file mode 100644 index 0000000000000..973c96be16be4 --- /dev/null +++ b/include/media/v4l2-hevc.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Helper functions for HEVC stateless codecs. + */ + +#ifndef _MEDIA_V4L2_HEVC_H +#define _MEDIA_V4L2_HEVC_H + +#include +#include + +/** + * v4l2_hevc_pps_num_tile_columns - number of HEVC tile columns, bounded + * @pps: the V4L2 HEVC PPS control + * + * Return the number of tile columns (num_tile_columns_minus1 + 1) clamped to + * the capacity of column_width_minus1[]. The control validation already + * rejects out-of-range counts; this keeps the consuming drivers bounded too. + */ +static inline unsigned int +v4l2_hevc_pps_num_tile_columns(const struct v4l2_ctrl_hevc_pps *pps) +{ + return min_t(unsigned int, pps->num_tile_columns_minus1 + 1, + ARRAY_SIZE(pps->column_width_minus1)); +} + +/** + * v4l2_hevc_pps_num_tile_rows - number of HEVC tile rows, bounded + * @pps: the V4L2 HEVC PPS control + * + * Return the number of tile rows (num_tile_rows_minus1 + 1) clamped to the + * capacity of row_height_minus1[]. + */ +static inline unsigned int +v4l2_hevc_pps_num_tile_rows(const struct v4l2_ctrl_hevc_pps *pps) +{ + return min_t(unsigned int, pps->num_tile_rows_minus1 + 1, + ARRAY_SIZE(pps->row_height_minus1)); +} + +#endif /* _MEDIA_V4L2_HEVC_H */ diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h index c75b4f14581a3..d7bf9feadf0bb 100644 --- a/include/media/videobuf2-core.h +++ b/include/media/videobuf2-core.h @@ -1121,8 +1121,8 @@ __poll_t vb2_core_poll(struct vb2_queue *q, struct file *file, * @ppos: file handle position tracking pointer * @nonblock: mode selector (1 means blocking calls, 0 means nonblocking) */ -size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count, - loff_t *ppos, int nonblock); +ssize_t vb2_read(struct vb2_queue *q, char __user *data, size_t count, + loff_t *ppos, int nonblock); /** * vb2_write() - implements write() syscall logic. * @q: pointer to &struct vb2_queue with videobuf2 queue. @@ -1131,8 +1131,8 @@ size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count, * @ppos: file handle position tracking pointer * @nonblock: mode selector (1 means blocking calls, 0 means nonblocking) */ -size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count, - loff_t *ppos, int nonblock); +ssize_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count, + loff_t *ppos, int nonblock); /** * typedef vb2_thread_fnc - callback function for use with vb2_thread. diff --git a/include/net/act_api.h b/include/net/act_api.h index a6d6f09dd0cd6..bac933abcd4cc 100644 --- a/include/net/act_api.h +++ b/include/net/act_api.h @@ -242,7 +242,7 @@ static inline void tcf_action_update_bstats(struct tc_action *a, static inline void tcf_action_inc_drop_qstats(struct tc_action *a) { if (likely(a->cpu_qstats)) { - qstats_drop_inc(this_cpu_ptr(a->cpu_qstats)); + qstats_cpu_drop_inc(a->cpu_qstats); return; } atomic_inc(&a->tcfa_drops); @@ -270,6 +270,25 @@ int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, struct tcf_chain *newchain); +/* Range check for a control action supplied by user space. + * + * This is the same test tcf_action_check_ctrlact() applies to the primary + * control action, factored out for the *fallback* control actions + * (act_gact's TCA_GACT_PROB.paction and act_police's TCA_POLICE_RESULT), + * which must not reach tcf_action_check_ctrlact() because they have no + * goto_chain to allocate. Without it, user space can store kernel-internal + * verdicts such as TC_ACT_CONSUMED, which is TC_ACT_VALUE_MAX + 1 and is + * deliberately not part of the UAPI value range. + */ +static inline bool tcf_action_valid(int action) +{ + int opcode = TC_ACT_EXT_OPCODE(action); + + if (!opcode) + return action <= TC_ACT_VALUE_MAX; + return opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC; +} + #ifdef CONFIG_INET DECLARE_STATIC_KEY_FALSE(tcf_frag_xmit_count); #endif diff --git a/include/net/addrconf.h b/include/net/addrconf.h index a106e8d6bd6ef..b353dea9aedc8 100644 --- a/include/net/addrconf.h +++ b/include/net/addrconf.h @@ -399,8 +399,8 @@ static inline struct inet6_dev *in6_dev_get(const struct net_device *dev) rcu_read_lock(); idev = rcu_dereference(dev->ip6_ptr); - if (idev) - refcount_inc(&idev->refcnt); + if (idev && !refcount_inc_not_zero(&idev->refcnt)) + idev = NULL; rcu_read_unlock(); return idev; } diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 2613cfe5b8a39..87b3f99e6ebf1 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -647,6 +647,8 @@ enum { #define HCI_LE_EXT_ADV 0x10 #define HCI_LE_PERIODIC_ADV 0x20 #define HCI_LE_CHAN_SEL_ALG2 0x40 +#define HCI_LE_PAST_SENDER 0x01 +#define HCI_LE_PAST_RECEIVER 0x02 #define HCI_LE_CIS_CENTRAL 0x10 #define HCI_LE_CIS_PERIPHERAL 0x20 #define HCI_LE_ISO_BROADCASTER 0x40 @@ -2068,6 +2070,44 @@ struct hci_cp_le_set_privacy_mode { __u8 mode; } __packed; +#define HCI_OP_LE_PAST 0x205a +struct hci_cp_le_past { + __le16 handle; + __le16 service_data; + __le16 sync_handle; +} __packed; + +struct hci_rp_le_past { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_PAST_SET_INFO 0x205b +struct hci_cp_le_past_set_info { + __le16 handle; + __le16 service_data; + __u8 adv_handle; +} __packed; + +struct hci_rp_le_past_set_info { + __u8 status; + __le16 handle; +} __packed; + +#define HCI_OP_LE_PAST_PARAMS 0x205c +struct hci_cp_le_past_params { + __le16 handle; + __u8 mode; + __le16 skip; + __le16 sync_timeout; + __u8 cte_type; +} __packed; + +struct hci_rp_le_past_params { + __u8 status; + __le16 handle; +} __packed; + #define HCI_OP_LE_READ_BUFFER_SIZE_V2 0x2060 struct hci_rp_le_read_buffer_size_v2 { __u8 status; @@ -2800,6 +2840,20 @@ struct hci_evt_le_ext_adv_set_term { __u8 num_evts; } __packed; +#define HCI_EV_LE_PAST_RECEIVED 0x18 +struct hci_ev_le_past_received { + __u8 status; + __le16 handle; + __le16 service_data; + __le16 sync_handle; + __u8 sid; + __u8 bdaddr_type; + bdaddr_t bdaddr; + __u8 phy; + __le16 interval; + __u8 clock_accuracy; +} __packed; + #define HCI_EVT_LE_CIS_ESTABLISHED 0x19 struct hci_evt_le_cis_established { __u8 status; diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index b5640980fbf4e..9dea729fd2325 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -916,9 +916,9 @@ static inline void hci_discovery_filter_clear(struct hci_dev *hdev) hdev->discovery.result_filtering = false; hdev->discovery.report_invalid_rssi = true; hdev->discovery.rssi = HCI_RSSI_INVALID; - hdev->discovery.uuid_count = 0; spin_lock(&hdev->discovery.lock); + hdev->discovery.uuid_count = 0; kfree(hdev->discovery.uuids); hdev->discovery.uuids = NULL; spin_unlock(&hdev->discovery.lock); @@ -2054,6 +2054,18 @@ void hci_conn_del_sysfs(struct hci_conn *conn); #define sync_recv_capable(dev) \ ((dev)->le_features[3] & HCI_LE_ISO_SYNC_RECEIVER) #define sync_recv_enabled(dev) (le_enabled(dev) && sync_recv_capable(dev)) +#define past_sender_capable(dev) \ + ((dev)->le_features[3] & HCI_LE_PAST_SENDER) +#define past_receiver_capable(dev) \ + ((dev)->le_features[3] & HCI_LE_PAST_RECEIVER) +#define past_capable(dev) \ + (past_sender_capable(dev) || past_receiver_capable(dev)) +#define past_sender_enabled(dev) \ + (le_enabled(dev) && past_sender_capable(dev)) +#define past_receiver_enabled(dev) \ + (le_enabled(dev) && past_receiver_capable(dev)) +#define past_enabled(dev) \ + (past_sender_enabled(dev) || past_receiver_enabled(dev)) #define mws_transport_config_capable(dev) (((dev)->commands[30] & 0x08) && \ (!hci_test_quirk((dev), HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG))) diff --git a/include/net/bluetooth/l2cap.h b/include/net/bluetooth/l2cap.h index 265afda2e13bc..2f7c367d68ce4 100644 --- a/include/net/bluetooth/l2cap.h +++ b/include/net/bluetooth/l2cap.h @@ -702,7 +702,12 @@ struct l2cap_rx_busy { struct l2cap_pinfo { struct bt_sock bt; + + /* With owning sk_socket chan may be read without lock, other access + * should hold lock_sock. + */ struct l2cap_chan *chan; + struct list_head rx_busy; }; diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h index f5be96f08b9d9..8234915854b68 100644 --- a/include/net/bluetooth/mgmt.h +++ b/include/net/bluetooth/mgmt.h @@ -119,6 +119,8 @@ struct mgmt_rp_read_index_list { #define MGMT_SETTING_ISO_BROADCASTER BIT(20) #define MGMT_SETTING_ISO_SYNC_RECEIVER BIT(21) #define MGMT_SETTING_LL_PRIVACY BIT(22) +#define MGMT_SETTING_PAST_SENDER BIT(23) +#define MGMT_SETTING_PAST_RECEIVER BIT(24) #define MGMT_OP_READ_INFO 0x0004 #define MGMT_READ_INFO_SIZE 0 diff --git a/include/net/bluetooth/rfcomm.h b/include/net/bluetooth/rfcomm.h index c058824769002..7e40d3c5f5a97 100644 --- a/include/net/bluetooth/rfcomm.h +++ b/include/net/bluetooth/rfcomm.h @@ -229,6 +229,9 @@ int rfcomm_send_rpn(struct rfcomm_session *s, int cr, u8 dlci, u8 bit_rate, u8 data_bits, u8 stop_bits, u8 parity, u8 flow_ctrl_settings, u8 xon_char, u8 xoff_char, u16 param_mask); +int rfcomm_dlc_send_rpn(struct rfcomm_dlc *d, u8 bit_rate, u8 data_bits, + u8 stop_bits, u8 parity, u8 flow_ctrl_settings, + u8 xon_char, u8 xoff_char, u16 param_mask); /* ---- RFCOMM DLCs (channels) ---- */ struct rfcomm_dlc *rfcomm_dlc_alloc(gfp_t prio); diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h index 820e299f06b5e..0b9bc2247d323 100644 --- a/include/net/cfg80211.h +++ b/include/net/cfg80211.h @@ -7,7 +7,7 @@ * Copyright 2006-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include @@ -126,6 +126,7 @@ struct wiphy; * @IEEE80211_CHAN_NO_4MHZ: 4 MHz bandwidth is not permitted on this channel. * @IEEE80211_CHAN_NO_8MHZ: 8 MHz bandwidth is not permitted on this channel. * @IEEE80211_CHAN_NO_16MHZ: 16 MHz bandwidth is not permitted on this channel. + * @IEEE80211_CHAN_NO_UHR: UHR operation is not permitted on this channel. */ enum ieee80211_channel_flags { IEEE80211_CHAN_DISABLED = BIT(0), @@ -143,6 +144,7 @@ enum ieee80211_channel_flags { IEEE80211_CHAN_NO_10MHZ = BIT(12), IEEE80211_CHAN_NO_HE = BIT(13), /* can use free bits here */ + IEEE80211_CHAN_NO_UHR = BIT(18), IEEE80211_CHAN_NO_320MHZ = BIT(19), IEEE80211_CHAN_NO_EHT = BIT(20), IEEE80211_CHAN_DFS_CONCURRENT = BIT(21), @@ -429,6 +431,18 @@ struct ieee80211_sta_eht_cap { u8 eht_ppe_thres[IEEE80211_EHT_PPE_THRES_MAX_LEN]; }; +/** + * struct ieee80211_sta_uhr_cap - STA's UHR capabilities + * @has_uhr: true iff UHR is supported and data is valid + * @mac: fixed MAC capabilities + * @phy: fixed PHY capabilities + */ +struct ieee80211_sta_uhr_cap { + bool has_uhr; + struct ieee80211_uhr_cap_mac mac; + struct ieee80211_uhr_cap_phy phy; +}; + /* sparse defines __CHECKER__; see Documentation/dev-tools/sparse.rst */ #ifdef __CHECKER__ /* @@ -454,6 +468,7 @@ struct ieee80211_sta_eht_cap { * @he_6ghz_capa: HE 6 GHz capabilities, must be filled in for a * 6 GHz band channel (and 0 may be valid value). * @eht_cap: STA's EHT capabilities + * @uhr_cap: STA's UHR capabilities * @vendor_elems: vendor element(s) to advertise * @vendor_elems.data: vendor element(s) data * @vendor_elems.len: vendor element(s) length @@ -463,6 +478,7 @@ struct ieee80211_sband_iftype_data { struct ieee80211_sta_he_cap he_cap; struct ieee80211_he_6ghz_capa he_6ghz_capa; struct ieee80211_sta_eht_cap eht_cap; + struct ieee80211_sta_uhr_cap uhr_cap; struct { const u8 *data; unsigned int len; @@ -704,6 +720,26 @@ ieee80211_get_eht_iftype_cap(const struct ieee80211_supported_band *sband, return NULL; } +/** + * ieee80211_get_uhr_iftype_cap - return UHR capabilities for an sband's iftype + * @sband: the sband to search for the iftype on + * @iftype: enum nl80211_iftype + * + * Return: pointer to the struct ieee80211_sta_uhr_cap, or NULL is none found + */ +static inline const struct ieee80211_sta_uhr_cap * +ieee80211_get_uhr_iftype_cap(const struct ieee80211_supported_band *sband, + enum nl80211_iftype iftype) +{ + const struct ieee80211_sband_iftype_data *data = + ieee80211_get_sband_iftype_data(sband, iftype); + + if (data && data->uhr_cap.has_uhr) + return &data->uhr_cap; + + return NULL; +} + /** * wiphy_read_of_freq_limits - read frequency limits from device tree * @@ -1485,6 +1521,7 @@ struct cfg80211_s1g_short_beacon { * @he_cap: HE capabilities (or %NULL if HE isn't enabled) * @eht_cap: EHT capabilities (or %NULL if EHT isn't enabled) * @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled) + * @uhr_oper: UHR operation (or %NULL if UHR isn't enabled) * @ht_required: stations must support HT * @vht_required: stations must support VHT * @twt_responder: Enable Target Wait Time @@ -1524,6 +1561,7 @@ struct cfg80211_ap_settings { const struct ieee80211_he_operation *he_oper; const struct ieee80211_eht_cap_elem *eht_cap; const struct ieee80211_eht_operation *eht_oper; + const struct ieee80211_uhr_operation *uhr_oper; bool ht_required, vht_required, he_required, sae_h2e_required; bool twt_responder; u32 flags; @@ -1697,6 +1735,8 @@ struct sta_txpwr { * @eht_capa: EHT capabilities of station * @eht_capa_len: the length of the EHT capabilities * @s1g_capa: S1G capabilities of station + * @uhr_capa: UHR capabilities of the station + * @uhr_capa_len: the length of the UHR capabilities */ struct link_station_parameters { const u8 *mld_mac; @@ -1716,6 +1756,8 @@ struct link_station_parameters { const struct ieee80211_eht_cap_elem *eht_capa; u8 eht_capa_len; const struct ieee80211_s1g_cap *s1g_capa; + const struct ieee80211_uhr_cap *uhr_capa; + u8 uhr_capa_len; }; /** @@ -1784,6 +1826,7 @@ struct cfg80211_ttlm_params { * present/updated * @eml_cap: EML capabilities of this station * @link_sta_params: link related params. + * @epp_peer: EPP peer indication */ struct station_parameters { struct net_device *vlan; @@ -1810,6 +1853,7 @@ struct station_parameters { bool eml_cap_present; u16 eml_cap; struct link_station_parameters link_sta_params; + bool epp_peer; }; /** @@ -1895,6 +1939,11 @@ int cfg80211_check_station_change(struct wiphy *wiphy, * @RATE_INFO_FLAGS_EXTENDED_SC_DMG: 60GHz extended SC MCS * @RATE_INFO_FLAGS_EHT_MCS: EHT MCS information * @RATE_INFO_FLAGS_S1G_MCS: MCS field filled with S1G MCS + * @RATE_INFO_FLAGS_UHR_MCS: UHR MCS information + * @RATE_INFO_FLAGS_UHR_ELR_MCS: UHR ELR MCS was used + * (set together with @RATE_INFO_FLAGS_UHR_MCS) + * @RATE_INFO_FLAGS_UHR_IM: UHR Interference Mitigation + * was used */ enum rate_info_flags { RATE_INFO_FLAGS_MCS = BIT(0), @@ -1906,6 +1955,9 @@ enum rate_info_flags { RATE_INFO_FLAGS_EXTENDED_SC_DMG = BIT(6), RATE_INFO_FLAGS_EHT_MCS = BIT(7), RATE_INFO_FLAGS_S1G_MCS = BIT(8), + RATE_INFO_FLAGS_UHR_MCS = BIT(9), + RATE_INFO_FLAGS_UHR_ELR_MCS = BIT(10), + RATE_INFO_FLAGS_UHR_IM = BIT(11), }; /** @@ -1921,7 +1973,7 @@ enum rate_info_flags { * @RATE_INFO_BW_160: 160 MHz bandwidth * @RATE_INFO_BW_HE_RU: bandwidth determined by HE RU allocation * @RATE_INFO_BW_320: 320 MHz bandwidth - * @RATE_INFO_BW_EHT_RU: bandwidth determined by EHT RU allocation + * @RATE_INFO_BW_EHT_RU: bandwidth determined by EHT/UHR RU allocation * @RATE_INFO_BW_1: 1 MHz bandwidth * @RATE_INFO_BW_2: 2 MHz bandwidth * @RATE_INFO_BW_4: 4 MHz bandwidth @@ -1952,7 +2004,7 @@ enum rate_info_bw { * * @flags: bitflag of flags from &enum rate_info_flags * @legacy: bitrate in 100kbit/s for 802.11abg - * @mcs: mcs index if struct describes an HT/VHT/HE/EHT/S1G rate + * @mcs: mcs index if struct describes an HT/VHT/HE/EHT/S1G/UHR rate * @nss: number of streams (VHT & HE only) * @bw: bandwidth (from &enum rate_info_bw) * @he_gi: HE guard interval (from &enum nl80211_he_gi) @@ -3262,6 +3314,7 @@ struct cfg80211_ml_reconf_req { * Drivers shall disable MLO features for the current association if this * flag is not set. * @ASSOC_REQ_SPP_AMSDU: SPP A-MSDUs will be used on this connection (if any) + * @ASSOC_REQ_DISABLE_UHR: Disable UHR */ enum cfg80211_assoc_req_flags { ASSOC_REQ_DISABLE_HT = BIT(0), @@ -3272,6 +3325,7 @@ enum cfg80211_assoc_req_flags { ASSOC_REQ_DISABLE_EHT = BIT(5), CONNECT_REQ_MLO_SUPPORT = BIT(6), ASSOC_REQ_SPP_AMSDU = BIT(7), + ASSOC_REQ_DISABLE_UHR = BIT(8), }; /** @@ -3996,6 +4050,54 @@ struct cfg80211_nan_conf { u16 vendor_elems_len; }; +#define CFG80211_NAN_SCHED_NUM_TIME_SLOTS 32 + +/** + * struct cfg80211_nan_channel - NAN channel configuration + * + * This struct defines a NAN channel configuration + * + * @chandef: the channel definition + * @channel_entry: pointer to the Channel Entry blob as defined in Wi-Fi Aware + * (TM) 4.0 specification Table 100 (Channel Entry format for the NAN + * Availability attribute). + * @rx_nss: number of spatial streams supported on this channel + */ +struct cfg80211_nan_channel { + struct cfg80211_chan_def chandef; + const u8 *channel_entry; + u8 rx_nss; +}; + +/** + * struct cfg80211_nan_local_sched - NAN local schedule + * + * This struct defines NAN local schedule parameters + * + * @schedule: a mapping of time slots to chandef indexes in %nan_channels. + * An unscheduled slot will be set to %NL80211_NAN_SCHED_NOT_AVAIL_SLOT. + * @n_channels: number of channel definitions in %nan_channels. + * @nan_avail_blob: pointer to NAN Availability attribute blob. + * See %NL80211_ATTR_NAN_AVAIL_BLOB for more details. + * @nan_avail_blob_len: length of the @nan_avail_blob in bytes. + * @deferred: if true, the command containing this schedule configuration is a + * request from the device to perform an announced schedule update. This + * means that it needs to send the updated NAN availability to the peers, + * and do the actual switch on the right time (i.e. at the end of the slot + * after the slot in which the updated NAN Availability was sent). + * See %NL80211_ATTR_NAN_SCHED_DEFERRED for more details. + * If false, the schedule is applied immediately. + * @nan_channels: array of NAN channel definitions that can be scheduled. + */ +struct cfg80211_nan_local_sched { + u8 schedule[CFG80211_NAN_SCHED_NUM_TIME_SLOTS]; + u8 n_channels; + const u8 *nan_avail_blob; + u16 nan_avail_blob_len; + bool deferred; + struct cfg80211_nan_channel nan_channels[] __counted_by(n_channels); +}; + /** * enum cfg80211_nan_conf_changes - indicates changed fields in NAN * configuration @@ -4769,6 +4871,12 @@ struct mgmt_frame_regs { * @nan_change_conf: changes NAN configuration. The changed parameters must * be specified in @changes (using &enum cfg80211_nan_conf_changes); * All other parameters must be ignored. + * @nan_set_local_sched: configure the local schedule for NAN. The schedule + * consists of an array of %cfg80211_nan_channel and the schedule itself, + * in which each entry maps each time slot to the channel on which the + * radio should operate on. If the chandef of a NAN channel is not + * changed, the channel entry must also remain unchanged. It is the + * driver's responsibility to verify this. * * @set_multicast_to_unicast: configure multicast to unicast conversion for BSS * @@ -5146,7 +5254,9 @@ struct cfg80211_ops { struct wireless_dev *wdev, struct cfg80211_nan_conf *conf, u32 changes); - + int (*nan_set_local_sched)(struct wiphy *wiphy, + struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched); int (*set_multicast_to_unicast)(struct wiphy *wiphy, struct net_device *dev, const bool enabled); @@ -6740,7 +6850,7 @@ struct wireless_dev { struct list_head pmsr_list; spinlock_t pmsr_lock; - struct work_struct pmsr_free_wk; + struct wiphy_work pmsr_free_wk; unsigned long unprot_beacon_reported; @@ -6774,6 +6884,9 @@ struct wireless_dev { } ocb; struct { u8 cluster_id[ETH_ALEN] __aligned(2); + u8 n_channels; + struct cfg80211_chan_def *chandefs; + bool sched_update_pending; } nan; } u; @@ -8142,7 +8255,7 @@ void cfg80211_ibss_joined(struct net_device *dev, const u8 *bssid, * cfg80211 then sends a notification to userspace. */ void cfg80211_notify_new_peer_candidate(struct net_device *dev, - const u8 *macaddr, const u8 *ie, u8 ie_len, + const u8 *macaddr, const u8 *ie, size_t ie_len, int sig_dbm, gfp_t gfp); /** @@ -9900,6 +10013,18 @@ void cfg80211_nan_func_terminated(struct wireless_dev *wdev, enum nl80211_nan_func_term_reason reason, u64 cookie, gfp_t gfp); +/** + * cfg80211_nan_sched_update_done - notify deferred schedule update completion + * @wdev: the wireless device reporting the event + * @success: whether or not the schedule update was successful + * @gfp: allocation flags + * + * This function notifies user space that a deferred local NAN schedule update + * (requested with %NL80211_ATTR_NAN_SCHED_DEFERRED) has been completed. + */ +void cfg80211_nan_sched_update_done(struct wireless_dev *wdev, bool success, + gfp_t gfp); + /* ethtool helper */ void cfg80211_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info); @@ -10328,4 +10453,27 @@ cfg80211_s1g_get_primary_sibling(struct wiphy *wiphy, return ieee80211_get_channel_khz(wiphy, sibling_1mhz_khz); } + +/** + * cfg80211_incumbent_signal_notify - Notify userspace of incumbent signal detection + * @wiphy: the wiphy to use + * @chandef: channel definition in which the interference was detected + * @signal_interference_bitmap: bitmap indicating interference across 20 MHz segments + * @gfp: allocation context for message creation and multicast; pass GFP_ATOMIC + * if called from atomic context (e.g. firmware event handler), otherwise + * GFP_KERNEL + * + * Use this function to notify userspace when an incumbent signal is detected on + * the operating channel in the 6 GHz band. The notification includes the + * current channel definition and a bitmap representing interference across + * the operating bandwidth. Each bit in the bitmap corresponds to a 20 MHz + * segment, with the lowest bit representing the lowest frequency segment. + * Punctured sub-channels are included in the bitmap structure but are always + * set to zero since interference detection is not performed on them. + */ +void cfg80211_incumbent_signal_notify(struct wiphy *wiphy, + const struct cfg80211_chan_def *chandef, + u32 signal_interference_bitmap, + gfp_t gfp); + #endif /* __NET_CFG80211_H */ diff --git a/include/net/cfg802154.h b/include/net/cfg802154.h index 76d2cd2e2b309..2e960441ea49e 100644 --- a/include/net/cfg802154.h +++ b/include/net/cfg802154.h @@ -376,6 +376,7 @@ struct cfg802154_mac_pkt { struct list_head node; struct sk_buff *skb; struct ieee802154_sub_if_data *sdata; + netdevice_tracker dev_tracker; u8 page; u8 channel; }; diff --git a/include/net/devlink.h b/include/net/devlink.h index 9e824f61e40f6..f24e1332bcb24 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -129,6 +129,7 @@ struct devlink_rate { struct devlink_port { struct list_head list; struct list_head region_list; + struct list_head resource_list; struct devlink *devlink; const struct devlink_port_ops *ops; unsigned int index; @@ -1828,12 +1829,19 @@ int devl_resource_register(struct devlink *devlink, u64 resource_size, u64 resource_id, u64 parent_resource_id, - const struct devlink_resource_size_params *size_params); + const struct devlink_resource_size_params *params); void devl_resources_unregister(struct devlink *devlink); void devlink_resources_unregister(struct devlink *devlink); int devl_resource_size_get(struct devlink *devlink, u64 resource_id, u64 *p_resource_size); +int +devl_port_resource_register(struct devlink_port *devlink_port, + const char *resource_name, + u64 resource_size, u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *params); +void devl_port_resources_unregister(struct devlink_port *devlink_port); int devl_dpipe_table_resource_set(struct devlink *devlink, const char *table_name, u64 resource_id, u64 resource_units); diff --git a/include/net/dropreason-core.h b/include/net/dropreason-core.h index 58d91ccc56e0b..de74e537b4ede 100644 --- a/include/net/dropreason-core.h +++ b/include/net/dropreason-core.h @@ -129,6 +129,7 @@ FN(DUALPI2_STEP_DROP) \ FN(PSP_INPUT) \ FN(PSP_OUTPUT) \ + FN(RECURSION_LIMIT) \ FNe(MAX) /** @@ -616,6 +617,8 @@ enum skb_drop_reason { SKB_DROP_REASON_PSP_INPUT, /** @SKB_DROP_REASON_PSP_OUTPUT: PSP output checks failed */ SKB_DROP_REASON_PSP_OUTPUT, + /** @SKB_DROP_REASON_RECURSION_LIMIT: Dead loop on virtual device. */ + SKB_DROP_REASON_RECURSION_LIMIT, /** * @SKB_DROP_REASON_MAX: the maximum of core drop reasons, which * shouldn't be used as a real 'reason' - only for tracing code gen diff --git a/include/net/dst.h b/include/net/dst.h index f8aa1239b4db6..307073eae7f83 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -219,6 +219,12 @@ static inline u32 dst_mtu(const struct dst_entry *dst) return INDIRECT_CALL_INET(dst->ops->mtu, ip6_mtu, ipv4_mtu, dst); } +/* Variant of dst_mtu() for IPv4 users. */ +static inline u32 dst4_mtu(const struct dst_entry *dst) +{ + return INDIRECT_CALL_1(dst->ops->mtu, ipv4_mtu, dst); +} + /* RTT metrics are stored in milliseconds for user ABI, but used as jiffies */ static inline unsigned long dst_metric_rtt(const struct dst_entry *dst, int metric) { diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 238ad3349456a..795fb41b45f5d 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -88,8 +88,6 @@ struct ip6_sf_socklist { struct in6_addr sl_addr[] __counted_by(sl_max); }; -#define IP6_SFBLOCK 10 /* allocate this many at once */ - struct ipv6_mc_socklist { struct in6_addr addr; int ifindex; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index afec146b8bafd..d50e0afc3193f 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -58,7 +58,9 @@ struct inet_connection_sock_af_ops { * @icsk_accept_queue: FIFO of established children * @icsk_bind_hash: Bind node * @icsk_bind2_hash: Bind node in the bhash2 table - * @icsk_retransmit_timer: Resend (no ack) + * @icsk_delack_timer: Delayed ACK timer + * @icsk_keepalive_timer: Keepalive timer + * @mptcp_tout_timer: mptcp timer * @icsk_rto: Retransmit timeout * @icsk_pmtu_cookie Last pmtu seen by socket * @icsk_ca_ops Pluggable congestion control hook @@ -83,8 +85,11 @@ struct inet_connection_sock { struct request_sock_queue icsk_accept_queue; struct inet_bind_bucket *icsk_bind_hash; struct inet_bind2_bucket *icsk_bind2_hash; - struct timer_list icsk_retransmit_timer; - struct timer_list icsk_delack_timer; + struct timer_list icsk_delack_timer; + union { + struct timer_list icsk_keepalive_timer; + struct timer_list mptcp_tout_timer; + }; __u32 icsk_rto; __u32 icsk_rto_min; u32 icsk_rto_max; @@ -186,10 +191,9 @@ static inline void inet_csk_delack_init(struct sock *sk) memset(&inet_csk(sk)->icsk_ack, 0, sizeof(inet_csk(sk)->icsk_ack)); } -static inline unsigned long -icsk_timeout(const struct inet_connection_sock *icsk) +static inline unsigned long tcp_timeout_expires(const struct sock *sk) { - return READ_ONCE(icsk->icsk_retransmit_timer.expires); + return READ_ONCE(sk->tcp_retransmit_timer.expires); } static inline unsigned long @@ -205,7 +209,7 @@ static inline void inet_csk_clear_xmit_timer(struct sock *sk, const int what) if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0) { smp_store_release(&icsk->icsk_pending, 0); #ifdef INET_CSK_CLEAR_TIMERS - sk_stop_timer(sk, &icsk->icsk_retransmit_timer); + sk_stop_timer(sk, &sk->tcp_retransmit_timer); #endif } else if (what == ICSK_TIME_DACK) { smp_store_release(&icsk->icsk_ack.pending, 0); @@ -237,7 +241,7 @@ static inline void inet_csk_reset_xmit_timer(struct sock *sk, const int what, if (what == ICSK_TIME_RETRANS || what == ICSK_TIME_PROBE0 || what == ICSK_TIME_LOSS_PROBE || what == ICSK_TIME_REO_TIMEOUT) { smp_store_release(&icsk->icsk_pending, what); - sk_reset_timer(sk, &icsk->icsk_retransmit_timer, when); + sk_reset_timer(sk, &sk->tcp_retransmit_timer, when); } else if (what == ICSK_TIME_DACK) { smp_store_release(&icsk->icsk_ack.pending, icsk->icsk_ack.pending | ICSK_ACK_TIMER); diff --git a/include/net/inetpeer.h b/include/net/inetpeer.h index f475757daafba..414e9adf4c511 100644 --- a/include/net/inetpeer.h +++ b/include/net/inetpeer.h @@ -35,6 +35,7 @@ struct inetpeer_addr { struct inet_peer { struct rb_node rb_node; + u64 hash; struct inetpeer_addr daddr; u32 metrics[RTAX_MAX]; @@ -125,6 +126,9 @@ static inline int inetpeer_addr_cmp(const struct inetpeer_addr *a, { int i, n; + if (a->family != b->family) + return a->family < b->family ? -1 : 1; + if (a->family == AF_INET) n = sizeof(a->a4) / sizeof(u32); else diff --git a/include/net/ip.h b/include/net/ip.h index e94c59cecef01..25a71c46792bb 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -506,6 +506,31 @@ static inline unsigned int ip_dst_mtu_maybe_forward(const struct dst_entry *dst, return res; } +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip_dst_mtu_maybe_forward(), this deliberately ignores the + * ICMP-learned path MTU (rt->rt_pmtu). The advertised MSS bounds what the + * peer may send to us and must reflect our receive capability (the device or + * route-configured MTU), not a path MTU learned on the reverse (send) + * direction, which may not apply to the peer->us path and outlives the fnhe + * for the whole connection. See RFC 2923 section 2.3 and the comment above + * tcp_advertise_mss(). + */ +static inline unsigned int ip_dst_mtu_configured(const struct dst_entry *dst) +{ + unsigned int mtu, res; + + rcu_read_lock(); + mtu = dst_metric_raw(dst, RTAX_MTU); + if (!mtu) + mtu = READ_ONCE(dst_dev_rcu(dst)->mtu); + mtu = min_t(unsigned int, mtu, IP_MAX_MTU); + res = mtu - lwtunnel_headroom(dst->lwtstate, mtu); + rcu_read_unlock(); + + return res; +} + static inline unsigned int ip_skb_dst_mtu(struct sock *sk, const struct sk_buff *skb) { @@ -679,7 +704,8 @@ static inline void ip_ipgre_mc_map(__be32 naddr, const unsigned char *broadcast, static __inline__ void inet_reset_saddr(struct sock *sk) { - inet_sk(sk)->inet_rcv_saddr = inet_sk(sk)->inet_saddr = 0; + inet_sk(sk)->inet_saddr = 0; + WRITE_ONCE(inet_sk(sk)->inet_rcv_saddr, 0); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == PF_INET6) { struct ipv6_pinfo *np = inet6_sk(sk); diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 7c5512baa4b2b..2e8ca123d1465 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -266,6 +266,12 @@ static inline bool ipv6_anycast_destination(const struct dst_entry *dst, int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int (*output)(struct net *, struct sock *, struct sk_buff *)); +/* Variant of dst_mtu() for IPv6 users */ +static inline u32 dst6_mtu(const struct dst_entry *dst) +{ + return INDIRECT_CALL_1(dst->ops->mtu, ip6_mtu, dst); +} + static inline unsigned int ip6_skb_dst_mtu(const struct sk_buff *skb) { const struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? @@ -343,6 +349,45 @@ static inline unsigned int ip6_dst_mtu_maybe_forward(const struct dst_entry *dst rcu_read_unlock(); out: + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); + + return mtu - lwtunnel_headroom(dst->lwtstate, mtu); +} + +/* Configured/administrative MTU of a route, for advertising the TCP MSS. + * + * Unlike ip6_dst_mtu_maybe_forward(), this ignores any ICMPv6-learned path + * MTU (which is kept on the RTF_CACHE exception route) and returns the MTU of + * the underlying route (fib6_pmtu) or the egress device. The advertised MSS + * bounds what the peer may send to us and must reflect our receive + * capability, not a path MTU learned on the reverse (send) direction. See + * RFC 2923 section 2.3 and the comment above tcp_advertise_mss(). + */ +static inline unsigned int ip6_dst_mtu_configured(const struct dst_entry *dst) +{ + const struct rt6_info *rt = dst_rt6_info(dst); + const struct fib6_info *from; + struct inet6_dev *idev; + unsigned int mtu = 0; + + rcu_read_lock(); + /* IPv6 keeps the learned PMTU and the configured MTU in the same + * RTAX_MTU slot: the learned value sits on this (possibly RTF_CACHE) + * dst, the configured one on the underlying route. Reach the latter + * via ->from (fib6_pmtu), populated by ip6_route_info_create(). + */ + from = rcu_dereference(rt->from); + if (from) + mtu = from->fib6_pmtu; + if (!mtu) { + mtu = IPV6_MIN_MTU; + idev = __in6_dev_get(dst_dev_rcu(dst)); + if (idev) + mtu = max_t(unsigned int, mtu, READ_ONCE(idev->cnf.mtu6)); + } + rcu_read_unlock(); + + mtu = min_t(unsigned int, mtu, IP6_MAX_MTU); return mtu - lwtunnel_headroom(dst->lwtstate, mtu); } diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 359b595f1df93..b99805ee2fd14 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -162,7 +162,7 @@ static inline void ip6tunnel_xmit(struct sock *sk, struct sk_buff *skb, dev->name); DEV_STATS_INC(dev, tx_errors); } - kfree_skb(skb); + kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT); return; } diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index 24023c19c7b1f..040eed88fc672 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -628,6 +628,11 @@ static inline void fib_info_hold(struct fib_info *fi) refcount_inc(&fi->fib_clntref); } +static inline bool fib_info_hold_safe(struct fib_info *fi) +{ + return refcount_inc_not_zero(&fi->fib_clntref); +} + static inline void fib_info_put(struct fib_info *fi) { if (refcount_dec_and_test(&fi->fib_clntref)) diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index 253ed3930f6ef..85e3455cea259 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -629,8 +629,7 @@ struct metadata_dst *iptunnel_metadata_reply(struct metadata_dst *md, int skb_tunnel_check_pmtu(struct sk_buff *skb, struct dst_entry *encap_dst, int headroom, bool reply); -static inline void ip_tunnel_adj_headroom(struct net_device *dev, - unsigned int headroom) +static inline unsigned int ip_tunnel_limit_headroom(unsigned int headroom) { /* we must cap headroom to some upperlimit, else pskb_expand_head * will overflow header offsets in skb_headers_offset_update(). @@ -640,6 +639,14 @@ static inline void ip_tunnel_adj_headroom(struct net_device *dev, if (headroom > max_allowed) headroom = max_allowed; + return headroom; +} + +static inline void ip_tunnel_adj_headroom(struct net_device *dev, + unsigned int headroom) +{ + headroom = ip_tunnel_limit_headroom(headroom); + if (headroom > READ_ONCE(dev->needed_headroom)) WRITE_ONCE(dev->needed_headroom, headroom); } @@ -665,13 +672,29 @@ static inline int iptunnel_pull_offloads(struct sk_buff *skb) static inline void iptunnel_xmit_stats(struct net_device *dev, int pkt_len) { if (pkt_len > 0) { - struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); - - u64_stats_update_begin(&tstats->syncp); - u64_stats_add(&tstats->tx_bytes, pkt_len); - u64_stats_inc(&tstats->tx_packets); - u64_stats_update_end(&tstats->syncp); - put_cpu_ptr(tstats); + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { + struct pcpu_dstats *dstats = get_cpu_ptr(dev->dstats); + + u64_stats_update_begin(&dstats->syncp); + u64_stats_add(&dstats->tx_bytes, pkt_len); + u64_stats_inc(&dstats->tx_packets); + u64_stats_update_end(&dstats->syncp); + put_cpu_ptr(dstats); + return; + } + if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { + struct pcpu_sw_netstats *tstats = get_cpu_ptr(dev->tstats); + + u64_stats_update_begin(&tstats->syncp); + u64_stats_add(&tstats->tx_bytes, pkt_len); + u64_stats_inc(&tstats->tx_packets); + u64_stats_update_end(&tstats->syncp); + put_cpu_ptr(tstats); + return; + } + pr_err_once("iptunnel_xmit_stats pcpu_stat_type=%d\n", + dev->pcpu_stat_type); + WARN_ON_ONCE(1); return; } diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 2be04ca409d1f..94dfc3a265ee5 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -34,6 +34,12 @@ #define IP_VS_HDR_INVERSE 1 #define IP_VS_HDR_ICMP 2 +/* Destination Server Flags */ +#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ + +/* Destination Server Config Flags */ +#define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */ + /* Generic access of ipvs struct */ static inline struct netns_ipvs *net_ipvs(struct net* net) { @@ -725,6 +731,7 @@ struct ip_vs_dest { volatile unsigned int flags; /* dest status flags */ atomic_t conn_flags; /* flags to copy to conn */ atomic_t weight; /* server weight */ + unsigned long cflags; /* config flags */ atomic_t last_weight; /* server latest weight */ __u16 tun_type; /* tunnel type */ __be16 tun_port; /* tunnel port */ @@ -736,10 +743,11 @@ struct ip_vs_dest { /* connection counters and thresholds */ atomic_t activeconns; /* active connections */ - atomic_t inactconns; /* inactive connections */ + atomic_t totalconns; /* total connections */ atomic_t persistconns; /* persistent connections */ __u32 u_threshold; /* upper threshold */ __u32 l_threshold; /* lower threshold */ + __u32 l_threshold_val;/* used lower threshold */ /* for destination cache */ spinlock_t dst_lock; /* lock of dst_cache */ @@ -1580,6 +1588,8 @@ static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) kfree(dest); } +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode); + /* IPVS sync daemon data and function prototypes * (from ip_vs_sync.c) */ @@ -1637,8 +1647,8 @@ int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, - unsigned int hooknum, struct ip_vs_iphdr *iph); + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph); void ip_vs_dest_dst_rcu_free(struct rcu_head *head); #ifdef CONFIG_IP_VS_IPV6 @@ -1651,8 +1661,8 @@ int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, - unsigned int hooknum, struct ip_vs_iphdr *iph); + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph); #endif #ifdef CONFIG_SYSCTL @@ -1717,15 +1727,15 @@ static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) } void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int dir); + struct ip_vs_conn *cp, int dir, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph); #ifdef CONFIG_IP_VS_IPV6 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int dir); + struct ip_vs_conn *cp, int dir, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph); #endif -__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset); - static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) { __be32 diff[2] = { ~old, new }; @@ -1751,6 +1761,26 @@ static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) return csum_partial(diff, sizeof(diff), oldsum); } +static inline bool ip_vs_checksum_needed(struct sk_buff *skb) +{ + /* Checksum unnecessary or already validated? */ + if (skb_csum_unnecessary(skb)) + return false; + /* Locally generated ? */ + if (!skb->dev) + return false; + return true; +} + +static inline bool ip_vs_checksum_common_check(struct sk_buff *skb, + int offset, int proto, int af) +{ + if (!ip_vs_checksum_needed(skb)) + return true; + /* Validate csum even for FORWARD */ + return !nf_checksum(skb, NF_INET_LOCAL_IN, offset, proto, af); +} + /* Forget current conntrack (unconfirmed) and attach notrack entry */ static inline void ip_vs_notrack(struct sk_buff *skb) { @@ -1858,14 +1888,21 @@ void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); static inline int ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) { - /* We think the overhead of processing active connections is 256 + /* We think the overhead of processing active connections is 257 * times higher than that of inactive connections in average. (This - * 256 times might not be accurate, we will change it later) We + * 257 times might not be accurate, we will change it later) We * use the following formula to estimate the overhead now: - * dest->activeconns*256 + dest->inactconns + * dest->activeconns*256 + dest->totalconns */ return (atomic_read(&dest->activeconns) << 8) + - atomic_read(&dest->inactconns); + atomic_read(&dest->totalconns); +} + +static inline int +ip_vs_dest_inactconns(const struct ip_vs_dest *dest) +{ + return max(atomic_read(&dest->totalconns) - + atomic_read(&dest->activeconns), 0); } #ifdef CONFIG_IP_VS_PROTO_TCP diff --git a/include/net/ipv6.h b/include/net/ipv6.h index 087aa7f502d56..aa8b525908ea0 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -1147,11 +1147,11 @@ int ip6_local_out(struct net *net, struct sock *sk, struct sk_buff *skb); * Extension header (options) processing */ -void ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, - u8 *proto, struct in6_addr **daddr_p, - struct in6_addr *saddr); -void ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, - u8 *proto); +u8 ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, + u8 proto, struct in6_addr **daddr_p, + struct in6_addr *saddr); +u8 ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, + u8 proto); int ipv6_skip_exthdr(const struct sk_buff *, int start, u8 *nexthdrp, __be16 *frag_offp); diff --git a/include/net/libeth/xsk.h b/include/net/libeth/xsk.h index 481a7b28e6f24..3d77e72a8cc81 100644 --- a/include/net/libeth/xsk.h +++ b/include/net/libeth/xsk.h @@ -196,7 +196,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, struct libeth_xdp_tx_desc desc; struct xdp_desc_ctx ctx; - ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr); + ctx = xsk_buff_raw_get_ctx(sq->pool, xdesc->addr, xdesc->options); desc = (typeof(desc)){ .addr = ctx.dma, __libeth_xdp_tx_len(xdesc->len), @@ -205,7 +205,7 @@ __libeth_xsk_xmit_fill_buf_md(const struct xdp_desc *xdesc, BUILD_BUG_ON(!__builtin_constant_p(tmo == libeth_xsktmo)); tmo = tmo == libeth_xsktmo ? &__libeth_xsktmo : tmo; - xsk_tx_metadata_request(ctx.meta, tmo, &desc); + xsk_tx_metadata_request(sq->pool, &ctx.meta, tmo, &desc); return desc; } diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 4a30bd458c5a9..c13c758aca7e0 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -480,11 +480,15 @@ static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb) { - unsigned int seq, hh_alen; + unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN); + int err; + + err = skb_cow_head(skb, hh_alen); + if (err) + return err; do { seq = read_seqbegin(&hh->hh_lock); - hh_alen = HH_DATA_ALIGN(ETH_HLEN); memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN); } while (read_seqretry(&hh->hh_lock, seq)); return 0; diff --git a/include/net/netdev_lock.h b/include/net/netdev_lock.h index 3d3aef80beac1..849b91ab4e28b 100644 --- a/include/net/netdev_lock.h +++ b/include/net/netdev_lock.h @@ -80,6 +80,12 @@ netdev_ops_assert_locked_or_invisible(const struct net_device *dev) netdev_ops_assert_locked(dev); } +static inline void netdev_assert_locked_ops(const struct net_device *dev) +{ + if (netdev_need_ops_lock(dev)) + netdev_assert_locked(dev); +} + static inline void netdev_lock_ops_compat(struct net_device *dev) { if (netdev_need_ops_lock(dev)) diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h index cd00e0406cf46..feca25131930b 100644 --- a/include/net/netdev_queues.h +++ b/include/net/netdev_queues.h @@ -14,6 +14,10 @@ struct netdev_config { u8 hds_config; }; +struct netdev_queue_config { + u32 rx_page_size; +}; + /* See the netdev.yaml spec for definition of each statistic */ struct netdev_queue_stats_rx { u64 bytes; @@ -111,6 +115,11 @@ void netdev_stat_queue_sum(struct net_device *netdev, int tx_start, int tx_end, struct netdev_queue_stats_tx *tx_sum); +enum { + /* The queue checks and honours the page size qcfg parameter */ + QCFG_RX_PAGE_SIZE = 0x1, +}; + /** * struct netdev_queue_mgmt_ops - netdev ops for queue management * @@ -130,25 +139,35 @@ void netdev_stat_queue_sum(struct net_device *netdev, * @ndo_queue_get_dma_dev: Get dma device for zero-copy operations to be used * for this queue. Return NULL on error. * + * @ndo_default_qcfg: Populate queue config struct with defaults. Optional. + * + * @supported_params: Bitmask of supported parameters, see QCFG_*. + * * Note that @ndo_queue_mem_alloc and @ndo_queue_mem_free may be called while * the interface is closed. @ndo_queue_start and @ndo_queue_stop will only * be called for an interface which is open. */ struct netdev_queue_mgmt_ops { - size_t ndo_queue_mem_size; - int (*ndo_queue_mem_alloc)(struct net_device *dev, - void *per_queue_mem, - int idx); - void (*ndo_queue_mem_free)(struct net_device *dev, - void *per_queue_mem); - int (*ndo_queue_start)(struct net_device *dev, - void *per_queue_mem, - int idx); - int (*ndo_queue_stop)(struct net_device *dev, - void *per_queue_mem, - int idx); - struct device * (*ndo_queue_get_dma_dev)(struct net_device *dev, - int idx); + size_t ndo_queue_mem_size; + int (*ndo_queue_mem_alloc)(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_queue_mem, + int idx); + void (*ndo_queue_mem_free)(struct net_device *dev, + void *per_queue_mem); + int (*ndo_queue_start)(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *per_queue_mem, + int idx); + int (*ndo_queue_stop)(struct net_device *dev, + void *per_queue_mem, + int idx); + void (*ndo_default_qcfg)(struct net_device *dev, + struct netdev_queue_config *qcfg); + struct device * (*ndo_queue_get_dma_dev)(struct net_device *dev, + int idx); + + unsigned int supported_params; }; bool netif_rxq_has_unreadable_mp(struct net_device *dev, int idx); diff --git a/include/net/netdev_rx_queue.h b/include/net/netdev_rx_queue.h index 8cdcd138b33f2..cfa72c4853876 100644 --- a/include/net/netdev_rx_queue.h +++ b/include/net/netdev_rx_queue.h @@ -7,6 +7,7 @@ #include #include #include +#include /* This structure contains an instance of an RX queue. */ struct netdev_rx_queue { @@ -27,6 +28,7 @@ struct netdev_rx_queue { struct xsk_buff_pool *pool; #endif struct napi_struct *napi; + struct netdev_queue_config qcfg; struct pp_memory_provider_params mp_params; } ____cacheline_aligned_in_smp; diff --git a/include/net/netfilter/nf_conntrack_expect.h b/include/net/netfilter/nf_conntrack_expect.h index e9a8350e7ccfb..80f50fd0f7ad2 100644 --- a/include/net/netfilter/nf_conntrack_expect.h +++ b/include/net/netfilter/nf_conntrack_expect.h @@ -45,9 +45,12 @@ struct nf_conntrack_expect { void (*expectfn)(struct nf_conn *new, struct nf_conntrack_expect *this); - /* Helper to assign to new connection */ + /* Helper that created this expectation */ struct nf_conntrack_helper __rcu *helper; + /* Helper to assign to new connection */ + struct nf_conntrack_helper __rcu *assign_helper; + /* The conntrack of the master connection */ struct nf_conn *master; diff --git a/include/net/netfilter/nf_conntrack_timeout.h b/include/net/netfilter/nf_conntrack_timeout.h index 3a66d4abb6d68..d60aa86be0194 100644 --- a/include/net/netfilter/nf_conntrack_timeout.h +++ b/include/net/netfilter/nf_conntrack_timeout.h @@ -12,6 +12,7 @@ #define CTNL_TIMEOUT_NAME_MAX 32 struct nf_ct_timeout { + refcount_t refcnt; __u16 l3num; const struct nf_conntrack_l4proto *l4proto; struct rcu_head rcu; @@ -22,6 +23,22 @@ struct nf_conn_timeout { struct nf_ct_timeout __rcu *timeout; }; +static inline void nf_ct_timeout_put(const struct nf_conn *ct) +{ +#ifdef CONFIG_NF_CONNTRACK_TIMEOUT + struct nf_conn_timeout *timeout_ext; + struct nf_ct_timeout *timeout; + + timeout_ext = nf_ct_ext_find(ct, NF_CT_EXT_TIMEOUT); + if (!timeout_ext) + return; + + timeout = rcu_dereference(timeout_ext->timeout); + if (timeout && refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(timeout, rcu); +#endif +} + static inline unsigned int * nf_ct_timeout_data(const struct nf_conn_timeout *t) { @@ -56,8 +73,14 @@ struct nf_conn_timeout *nf_ct_timeout_ext_add(struct nf_conn *ct, #ifdef CONFIG_NF_CONNTRACK_TIMEOUT struct nf_conn_timeout *timeout_ext; + if (!timeout) + return NULL; + timeout_ext = nf_ct_ext_add(ct, NF_CT_EXT_TIMEOUT, gfp); - if (timeout_ext == NULL) + if (!timeout_ext || timeout_ext->timeout) + return NULL; + + if (!refcount_inc_not_zero(&timeout->refcnt)) return NULL; rcu_assign_pointer(timeout_ext->timeout, timeout); @@ -75,7 +98,7 @@ static inline unsigned int *nf_ct_timeout_lookup(const struct nf_conn *ct) struct nf_conn_timeout *timeout_ext; timeout_ext = nf_ct_timeout_find(ct); - if (timeout_ext) + if (timeout_ext && rcu_access_pointer(timeout_ext->timeout)) timeouts = nf_ct_timeout_data(timeout_ext); #endif return timeouts; diff --git a/include/net/netfilter/nf_flow_table.h b/include/net/netfilter/nf_flow_table.h index e9f72d2558e90..7c330caae52bf 100644 --- a/include/net/netfilter/nf_flow_table.h +++ b/include/net/netfilter/nf_flow_table.h @@ -141,6 +141,7 @@ struct flow_offload_tuple { union { struct { struct dst_entry *dst_cache; + u32 ifidx; u32 dst_cookie; }; struct { diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h index b35e8c02fadcd..e6b31aa9d6b7d 100644 --- a/include/net/netfilter/nf_tables.h +++ b/include/net/netfilter/nf_tables.h @@ -123,17 +123,6 @@ struct nft_regs { }; }; -struct nft_regs_track { - struct { - const struct nft_expr *selector; - const struct nft_expr *bitwise; - u8 num_reg; - } regs[NFT_REG32_NUM]; - - const struct nft_expr *cur; - const struct nft_expr *last; -}; - /* Store/load an u8, u16 or u64 integer to/from the u32 data register. * * Note, when using concatenations, register allocation happens at 32-bit @@ -433,8 +422,6 @@ int nft_expr_clone(struct nft_expr *dst, struct nft_expr *src, gfp_t gfp); void nft_expr_destroy(const struct nft_ctx *ctx, struct nft_expr *expr); int nft_expr_dump(struct sk_buff *skb, unsigned int attr, const struct nft_expr *expr, bool reset); -bool nft_expr_reduce_bitwise(struct nft_regs_track *track, - const struct nft_expr *expr); struct nft_set_ext; @@ -949,7 +936,6 @@ struct nft_offload_ctx; * @destroy_clone: destruction clone function * @dump: function to dump parameters * @validate: validate expression, called during loop detection - * @reduce: reduce expression * @gc: garbage collection expression * @offload: hardware offload expression * @offload_action: function to report true/false to allocate one slot or not in the flow @@ -983,8 +969,6 @@ struct nft_expr_ops { bool reset); int (*validate)(const struct nft_ctx *ctx, const struct nft_expr *expr); - bool (*reduce)(struct nft_regs_track *track, - const struct nft_expr *expr); bool (*gc)(struct net *net, const struct nft_expr *expr); int (*offload)(struct nft_offload_ctx *ctx, @@ -1304,6 +1288,7 @@ static inline void nft_use_inc_restore(u32 *use) * @sets: sets in the table * @objects: stateful objects in the table * @flowtables: flow tables in the table + * @objname_ht: hashtable for objects lookup by name * @hgenerator: handle generator state * @handle: table handle * @use: number of chain references to this table @@ -1323,6 +1308,7 @@ struct nft_table { struct list_head sets; struct list_head objects; struct list_head flowtables; + struct rhltable objname_ht; u64 hgenerator; u64 handle; u32 use; @@ -1410,7 +1396,7 @@ static inline void *nft_obj_data(const struct nft_object *obj) #define nft_expr_obj(expr) *((struct nft_object **)nft_expr_priv(expr)) struct nft_object *nft_obj_lookup(const struct net *net, - const struct nft_table *table, + struct nft_table *table, const struct nlattr *nla, u32 objtype, u8 genmask); @@ -1967,20 +1953,4 @@ static inline u64 nft_net_tstamp(const struct net *net) return nft_pernet(net)->tstamp; } -#define __NFT_REDUCE_READONLY 1UL -#define NFT_REDUCE_READONLY (void *)__NFT_REDUCE_READONLY - -void nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg, u8 len); -void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len); -void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg); - -static inline bool nft_reg_track_cmp(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg) -{ - return track->regs[dreg].selector && - track->regs[dreg].selector->ops == expr->ops && - track->regs[dreg].num_reg == 0; -} - #endif /* _NET_NF_TABLES_H */ diff --git a/include/net/netfilter/nft_fib.h b/include/net/netfilter/nft_fib.h index 7370fba844efc..e0422456f27bc 100644 --- a/include/net/netfilter/nft_fib.h +++ b/include/net/netfilter/nft_fib.h @@ -66,6 +66,4 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs, void nft_fib_store_result(void *reg, const struct nft_fib *priv, const struct net_device *dev); -bool nft_fib_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); #endif diff --git a/include/net/netfilter/nft_meta.h b/include/net/netfilter/nft_meta.h index fab3e6658e91c..6cf1d910bbf8d 100644 --- a/include/net/netfilter/nft_meta.h +++ b/include/net/netfilter/nft_meta.h @@ -45,9 +45,6 @@ int nft_meta_get_validate(const struct nft_ctx *ctx, int nft_meta_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr); -bool nft_meta_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr); - struct nft_inner_tun_ctx; void nft_meta_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt, diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h index 62166da045548..fa49a9598bbb0 100644 --- a/include/net/netns/ipv4.h +++ b/include/net/netns/ipv4.h @@ -271,7 +271,7 @@ struct netns_ipv4 { #ifdef CONFIG_IP_MROUTE #ifndef CONFIG_IP_MROUTE_MULTIPLE_TABLES - struct mr_table *mrt; + struct mr_table __rcu *mrt; #else struct list_head mr_tables; struct fib_rules_ops *mr_rules_ops; diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h index fb4f03ccd6156..cdd95477af7a2 100644 --- a/include/net/page_pool/types.h +++ b/include/net/page_pool/types.h @@ -161,6 +161,7 @@ struct memory_provider_ops; struct pp_memory_provider_params { void *mp_priv; const struct memory_provider_ops *mp_ops; + u32 rx_page_size; }; struct page_pool { diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h index 8a75c73fc5558..f3649ae8cab63 100644 --- a/include/net/pkt_sched.h +++ b/include/net/pkt_sched.h @@ -12,6 +12,7 @@ #define DEFAULT_TX_QUEUE_LEN 1000 #define STAB_SIZE_LOG_MAX 30 +#define QDISC_PKT_LEN_MAX (1 << 20) /* 1 MiB */ struct qdisc_walker { int stop; diff --git a/include/net/route.h b/include/net/route.h index f90106f383c56..45290177a33c4 100644 --- a/include/net/route.h +++ b/include/net/route.h @@ -276,6 +276,8 @@ int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb, u32 table_id, struct fib_info *fi, int *fa_index, int fa_start, unsigned int flags); +void fnhe_update_pmtu(struct fib_nh_exception *fnhe, u32 new, u32 orig); + static inline void ip_rt_put(struct rtable *rt) { /* dst_release() accepts a NULL parameter. diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 059eb6cb54f13..97c1b2828bcd5 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -101,6 +101,7 @@ struct Qdisc { struct hlist_node hash; u32 handle; u32 parent; + int depth; struct netdev_queue *dev_queue; @@ -754,7 +755,7 @@ void qdisc_offload_query_caps(struct net_device *dev, struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack); -void qdisc_free(struct Qdisc *qdisc); +void qdisc_free_rcu(struct Qdisc *qdisc); struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, u32 parentid, struct netlink_ext_ack *extack); @@ -966,12 +967,17 @@ static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) { - sch->qstats.drops += count; + WRITE_ONCE(sch->qstats.drops, sch->qstats.drops + count); } static inline void qstats_drop_inc(struct gnet_stats_queue *qstats) { - qstats->drops++; + WRITE_ONCE(qstats->drops, qstats->drops + 1); +} + +static inline void qstats_cpu_drop_inc(struct gnet_stats_queue __percpu *qstats) +{ + this_cpu_inc(qstats->drops); } static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats) diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 2ae390219efdd..e2191b158ac4c 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -316,7 +316,8 @@ struct sctp_cookie { __u8 auth_random[sizeof(struct sctp_paramhdr) + SCTP_AUTH_RANDOM_LENGTH]; - __u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2]; + __u8 auth_hmacs[sizeof(struct sctp_paramhdr) + + SCTP_AUTH_NUM_HMACS * sizeof(__u16)]; __u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS]; /* This is a shim for my peer's INIT packet, followed by @@ -2063,7 +2064,7 @@ struct sctp_association { force_delay:1; __u8 strreset_enable; - __u8 strreset_outstanding; /* request param count on the fly */ + __u8 strreset_outstanding; /* request param bitmask on the fly */ __u32 strreset_outseq; /* Update after receiving response */ __u32 strreset_inseq; /* Update after receiving request */ diff --git a/include/net/sock.h b/include/net/sock.h index 9637ee4f55ee1..9e13e1233e417 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -303,6 +303,8 @@ struct sk_filter; * @sk_txrehash: enable TX hash rethink * @sk_filter: socket filtering instructions * @sk_timer: sock cleanup timer + * @tcp_retransmit_timer: tcp retransmit timer + * @mptcp_retransmit_timer: mptcp retransmit timer * @sk_stamp: time stamp of last packet received * @sk_stamp_seq: lock for accessing sk_stamp on 32 bit architectures only * @sk_tsflags: SO_TIMESTAMPING flags @@ -480,8 +482,11 @@ struct sock { u32 sk_dst_pending_confirm; u32 sk_pacing_status; /* see enum sk_pacing */ struct page_frag sk_frag; - struct timer_list sk_timer; - + union { + struct timer_list sk_timer; + struct timer_list tcp_retransmit_timer; + struct timer_list mptcp_retransmit_timer; + }; unsigned long sk_pacing_rate; /* bytes per second */ atomic_t sk_zckey; atomic_t sk_tskey; @@ -2468,12 +2473,23 @@ int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue, struct sk_buff *skb)); int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb); -int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason *reason); +enum skb_drop_reason +sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb); static inline int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { - return sock_queue_rcv_skb_reason(sk, skb, NULL); + enum skb_drop_reason drop_reason = sock_queue_rcv_skb_reason(sk, skb); + + switch (drop_reason) { + case SKB_DROP_REASON_SOCKET_RCVBUFF: + return -ENOMEM; + case SKB_DROP_REASON_PROTO_MEM: + return -ENOBUFS; + case 0: + return 0; + default: + return -EPERM; + } } int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb); diff --git a/include/net/tcp.h b/include/net/tcp.h index 31ac90230b12d..12a9672f56eb5 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -731,8 +731,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue, void tcp_send_probe0(struct sock *); int tcp_write_wakeup(struct sock *, int mib); void tcp_send_fin(struct sock *sk); -void tcp_send_active_reset(struct sock *sk, gfp_t priority, - enum sk_rst_reason reason); +void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason); int tcp_send_synack(struct sock *); void tcp_push_one(struct sock *, unsigned int mss_now); void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags); @@ -1686,6 +1685,11 @@ static inline int tcp_full_space(const struct sock *sk) return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); } +static inline u32 tcp_dst_advmss(const struct dst_entry *dst) +{ + return max_t(u32, dst_metric_advmss(dst), TCP_MIN_MSS); +} + static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) { int unused_mem = sk_unused_reserved_mem(sk); @@ -1878,6 +1882,8 @@ static inline void tcp_fast_path_check(struct sock *sk) bool tcp_oow_rate_limited(struct net *net, const struct sk_buff *skb, int mib_idx, u32 *last_oow_ack_time); +void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb, + struct request_sock *req); static inline void tcp_mib_init(struct net *net) { diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h index 7c2bc46c67050..8bf17ba6c3bd7 100644 --- a/include/net/xdp_sock.h +++ b/include/net/xdp_sock.h @@ -80,6 +80,7 @@ struct xdp_sock { * call of __xsk_generic_xmit(). */ struct sk_buff *skb; + bool drain_cont; struct list_head map_list; /* Protects map_list */ @@ -139,45 +140,16 @@ void __xsk_map_flush(struct list_head *flush_list); static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, struct xsk_tx_metadata_compl *compl) { - if (!meta) - return; + compl->tx_timestamp = NULL; - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - compl->tx_timestamp = &meta->completion.tx_timestamp; - else - compl->tx_timestamp = NULL; -} - -/** - * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission - * and call appropriate xsk_tx_metadata_ops operation. - * @meta: pointer to AF_XDP metadata area - * @ops: pointer to struct xsk_tx_metadata_ops - * @priv: pointer to driver-private aread - * - * This function should be called by the networking device when - * it prepares AF_XDP egress packet. - */ -static inline void xsk_tx_metadata_request(const struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ if (!meta) return; - if (ops->tmo_request_launch_time) - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - ops->tmo_request_launch_time(meta->request.launch_time, - priv); - - if (ops->tmo_request_timestamp) - if (meta->flags & XDP_TXMD_FLAGS_TIMESTAMP) - ops->tmo_request_timestamp(priv); + /* we can only arrive here if the completion timestamp has been + * requested via XDP_TXMD_FLAGS_TIMESTAMP, see xsk_tx_metadata_request + */ - if (ops->tmo_request_checksum) - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) - ops->tmo_request_checksum(meta->request.csum_start, - meta->request.csum_offset, priv); + compl->tx_timestamp = &meta->completion.tx_timestamp; } /** @@ -223,12 +195,6 @@ static inline void xsk_tx_metadata_to_compl(struct xsk_tx_metadata *meta, { } -static inline void xsk_tx_metadata_request(struct xsk_tx_metadata *meta, - const struct xsk_tx_metadata_ops *ops, - void *priv) -{ -} - static inline void xsk_tx_metadata_complete(struct xsk_tx_metadata_compl *compl, const struct xsk_tx_metadata_ops *ops, void *priv) diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h index dd1d3a6e1b780..21c3b757bb883 100644 --- a/include/net/xdp_sock_drv.h +++ b/include/net/xdp_sock_drv.h @@ -236,17 +236,18 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) * xsk_buff_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Wrapper for xp_raw_get_ctx() to be used in drivers, see its kdoc for * details. * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { - return xp_raw_get_ctx(pool, addr); + return xp_raw_get_ctx(pool, addr, options); } #define XDP_TXMD_FLAGS_VALID ( \ @@ -256,30 +257,78 @@ xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) 0) static inline bool -xsk_buff_valid_tx_metadata(const struct xsk_tx_metadata *meta) +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { - return !(meta->flags & ~XDP_TXMD_FLAGS_VALID); + *flags = READ_ONCE(meta->flags); + if (*flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + if (pool->tx_metadata_len < + offsetofend(struct xsk_tx_metadata, request.launch_time)) + return false; + return !(*flags & ~XDP_TXMD_FLAGS_VALID); } -static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +/** + * xsk_tx_metadata_request - Evaluate AF_XDP TX metadata at submission + * and call appropriate xsk_tx_metadata_ops operation. + * @pool: pointer to AF_XDP buffer pool, used to validate the metadata + * @pmeta: pointer to pointer to AF_XDP metadata area + * @ops: pointer to struct xsk_tx_metadata_ops + * @priv: pointer to driver-private area + * + * This function should be called by the networking device when + * it prepares AF_XDP egress packet. + */ +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) { - struct xsk_tx_metadata *meta; + const struct xsk_tx_metadata *meta = *pmeta; + u64 flags; - if (!pool->tx_metadata_len) - return NULL; + if (!meta) + return; - meta = data - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) - return NULL; /* no way to signal the error to the user */ + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) { + *pmeta = NULL; + return; /* no way to signal the error to the user */ + } + + if (ops->tmo_request_launch_time) + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + ops->tmo_request_launch_time( + READ_ONCE(meta->request.launch_time), priv); + + if (ops->tmo_request_timestamp) + if (flags & XDP_TXMD_FLAGS_TIMESTAMP) + ops->tmo_request_timestamp(priv); - return meta; + if (ops->tmo_request_checksum) + if (flags & XDP_TXMD_FLAGS_CHECKSUM) + ops->tmo_request_checksum( + READ_ONCE(meta->request.csum_start), + READ_ONCE(meta->request.csum_offset), priv); + + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + *pmeta = NULL; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { - return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr)); + if (!pool->tx_metadata_len || !(options & XDP_TX_METADATA)) + return NULL; + + return data - pool->tx_metadata_len; +} + +static inline struct xsk_tx_metadata * +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) +{ + return __xsk_buff_get_metadata(pool, xp_raw_get_data(pool, addr), + options); } static inline void xsk_buff_dma_sync_for_cpu(struct xdp_buff *xdp) @@ -460,24 +509,34 @@ static inline void *xsk_buff_raw_get_data(struct xsk_buff_pool *pool, u64 addr) } static inline struct xdp_desc_ctx -xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +xsk_buff_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, u32 options) { return (struct xdp_desc_ctx){ }; } -static inline bool xsk_buff_valid_tx_metadata(struct xsk_tx_metadata *meta) +static inline bool +xsk_buff_valid_tx_metadata(const struct xsk_buff_pool *pool, + const struct xsk_tx_metadata *meta, u64 *flags) { return false; } +static inline void +xsk_tx_metadata_request(const struct xsk_buff_pool *pool, + struct xsk_tx_metadata **pmeta, + const struct xsk_tx_metadata_ops *ops, void *priv) +{ +} + static inline struct xsk_tx_metadata * -__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data) +__xsk_buff_get_metadata(const struct xsk_buff_pool *pool, void *data, + unsigned int options) { return NULL; } static inline struct xsk_tx_metadata * -xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr) +xsk_buff_get_metadata(struct xsk_buff_pool *pool, u64 addr, u32 options) { return NULL; } diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h index cac56e6b0869b..10e564d85a90c 100644 --- a/include/net/xsk_buff_pool.h +++ b/include/net/xsk_buff_pool.h @@ -85,11 +85,16 @@ struct xsk_buff_pool { bool unaligned; bool tx_sw_csum; void *addrs; - /* Mutual exclusion of the completion ring in the SKB mode. Two cases to protect: - * NAPI TX thread and sendmsg error paths in the SKB destructor callback and when - * sockets share a single cq when the same netdev and queue id is shared. + /* Mutual exclusion of the completion ring in the SKB mode. + * Protect: NAPI TX thread and sendmsg error paths in the SKB + * destructor callback. */ - spinlock_t cq_lock; + spinlock_t cq_prod_lock; + /* Mutual exclusion of the completion ring in the SKB mode. + * Protect: when sockets share a single cq when the same netdev + * and queue id is shared. + */ + spinlock_t cq_cached_prod_lock; struct xdp_buff_xsk *free_heads[]; }; @@ -149,7 +154,8 @@ struct xdp_desc_ctx { struct xsk_tx_metadata *meta; }; -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr); +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options); static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) { diff --git a/include/rdma/ib_umem.h b/include/rdma/ib_umem.h index 5122f78467675..8a74153332577 100644 --- a/include/rdma/ib_umem.h +++ b/include/rdma/ib_umem.h @@ -36,6 +36,7 @@ struct ib_umem_dmabuf { struct scatterlist *last_sg; unsigned long first_sg_offset; unsigned long last_sg_trim; + void (*pinned_revoke)(void *priv); void *private; u8 pinned : 1; u8 revoked : 1; @@ -84,7 +85,7 @@ int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, size_t length); unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt); + u64 virt); /** * ib_umem_find_best_pgoff - Find best HW page size @@ -120,16 +121,11 @@ static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, static inline bool ib_umem_is_contiguous(struct ib_umem *umem) { - dma_addr_t dma_addr; unsigned long pgsz; - /* - * Select the smallest aligned page that can contain the whole umem if - * it was contiguous. - */ - dma_addr = ib_umem_start_dma_addr(umem); - pgsz = roundup_pow_of_two((dma_addr ^ (umem->length - 1 + dma_addr)) + 1); - return !!ib_umem_find_best_pgoff(umem, pgsz, U64_MAX); + pgsz = ib_umem_find_best_pgsz(umem, ULONG_MAX, + ib_umem_start_dma_addr(umem)); + return pgsz && ib_umem_num_dma_blocks(umem, pgsz) == 1; } struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, @@ -141,6 +137,12 @@ struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, size_t size, int fd, int access); struct ib_umem_dmabuf * +ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, + unsigned long offset, size_t size, + int fd, int access); +void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, + void (*revoke)(void *priv), void *priv); +struct ib_umem_dmabuf * ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, struct device *dma_device, unsigned long offset, size_t size, @@ -171,7 +173,7 @@ static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offs } static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap, - unsigned long virt) + u64 virt) { return 0; } @@ -201,6 +203,18 @@ ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, return ERR_PTR(-EOPNOTSUPP); } +static inline struct ib_umem_dmabuf * +ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, + unsigned long offset, size_t size, + int fd, int access) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void +ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, + void (*revoke)(void *priv), void *priv) {} + static inline struct ib_umem_dmabuf * ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, struct device *dma_device, diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index eaeec00ef4c15..7f811e80b63ae 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -876,6 +876,20 @@ __attribute_const__ int ib_rate_to_mult(enum ib_rate rate); */ __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate); +struct ib_port_speed_info { + const char *str; + int rate; /* in deci-Gb/sec (100 MBps units) */ +}; + +/** + * ib_port_attr_to_speed_info - Convert port attributes to speed information + * @attr: Port attributes containing active_speed and active_width + * @speed_info: Speed information to return + * + * Returns 0 on success, -EINVAL on error. + */ +int ib_port_attr_to_speed_info(struct ib_port_attr *attr, + struct ib_port_speed_info *speed_info); /** * enum ib_mr_type - memory region type @@ -1870,6 +1884,11 @@ struct ib_dmah { struct ib_mr { struct ib_device *device; + /* + * Due to IB_MR_REREG_PD pd is not a fixed pointer and can change. For a + * user MR, this value should only be read from a system call that holds + * the uobject lock, or the driver should disable in-place REREG_PD. + */ struct ib_pd *pd; u32 lkey; u32 rkey; @@ -2402,6 +2421,8 @@ struct ib_device_ops { int comp_vector); int (*query_port)(struct ib_device *device, u32 port_num, struct ib_port_attr *port_attr); + int (*query_port_speed)(struct ib_device *device, u32 port_num, + u64 *speed); int (*modify_port)(struct ib_device *device, u32 port_num, int port_modify_mask, struct ib_port_modify *port_modify); diff --git a/include/scsi/libsas.h b/include/scsi/libsas.h index a0635b128d7ab..cc0a1f320f17d 100644 --- a/include/scsi/libsas.h +++ b/include/scsi/libsas.h @@ -684,7 +684,6 @@ extern int sas_register_ha(struct sas_ha_struct *); extern int sas_unregister_ha(struct sas_ha_struct *); extern void sas_prep_resume_ha(struct sas_ha_struct *sas_ha); extern void sas_resume_ha(struct sas_ha_struct *sas_ha); -extern void sas_resume_ha_no_sync(struct sas_ha_struct *sas_ha); extern void sas_suspend_ha(struct sas_ha_struct *sas_ha); int sas_phy_reset(struct sas_phy *phy, int hard_reset); diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h index f5a2432612360..936403c15e53d 100644 --- a/include/scsi/scsi_host.h +++ b/include/scsi/scsi_host.h @@ -714,6 +714,9 @@ struct Scsi_Host { */ struct device *dma_dev; + /* Used for an rcu-synchronizing eh wakeup */ + struct work_struct eh_work; + /* Delay for runtime autosuspend */ int rpm_autosuspend_delay; diff --git a/include/sound/core.h b/include/sound/core.h index 64327e9711222..618055761fe01 100644 --- a/include/sound/core.h +++ b/include/sound/core.h @@ -314,6 +314,8 @@ static inline void snd_card_unref(struct snd_card *card) put_device(&card->card_dev); } +DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T)) + #define snd_card_set_dev(card, devptr) ((card)->dev = (devptr)) /* device.c */ diff --git a/include/sound/tas2781.h b/include/sound/tas2781.h index 29d15ba65f04d..648fb55073282 100644 --- a/include/sound/tas2781.h +++ b/include/sound/tas2781.h @@ -59,9 +59,9 @@ #define TASDEVICE_CHECKSUM_REG TASDEVICE_REG(0x0, 0x0, 0x7e) /* XM_340 */ -#define TASDEVICE_XM_A1_REG TASDEVICE_REG(0x64, 0x63, 0x3c) +#define TASDEVICE_XM_A1_REG TASDEVICE_REG(0x64, 0x02, 0x4c) /* XM_341 */ -#define TASDEVICE_XM_A2_REG TASDEVICE_REG(0x64, 0x63, 0x38) +#define TASDEVICE_XM_A2_REG TASDEVICE_REG(0x64, 0x02, 0x64) /* Volume control */ #define TAS2563_DVC_LVL TASDEVICE_REG(0x00, 0x02, 0x0c) diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h index 3378ff9ee271c..7c85ca01c3f82 100644 --- a/include/target/target_core_fabric.h +++ b/include/target/target_core_fabric.h @@ -95,6 +95,8 @@ struct target_core_fabric_ops { struct se_lun *); void (*fabric_pre_unlink)(struct se_portal_group *, struct se_lun *); + void (*fabric_post_unlink)(struct se_portal_group *se_tpg, + struct se_lun *lun); struct se_tpg_np *(*fabric_make_np)(struct se_portal_group *, struct config_group *, const char *); void (*fabric_drop_np)(struct se_tpg_np *); diff --git a/include/trace/events/cachefiles.h b/include/trace/events/cachefiles.h index a743b2a35ea70..3ca9be3a21c56 100644 --- a/include/trace/events/cachefiles.h +++ b/include/trace/events/cachefiles.h @@ -380,7 +380,7 @@ TRACE_EVENT(cachefiles_rename, TRACE_EVENT(cachefiles_coherency, TP_PROTO(struct cachefiles_object *obj, ino_t ino, - u64 disk_aux, + const void *disk_aux, enum cachefiles_content content, enum cachefiles_coherency_trace why), @@ -397,12 +397,27 @@ TRACE_EVENT(cachefiles_coherency, ), TP_fast_assign( + union { + __be16 s[4]; + __be64 ll; + } x; + __entry->obj = obj->debug_id; __entry->why = why; __entry->content = content; __entry->ino = ino; __entry->aux = be64_to_cpup((__be64 *)obj->cookie->inline_aux); - __entry->disk_aux = disk_aux; + + /* cachefiles_xattr::data is 2-byte aligned but not 8-byte aligned. */ + if (disk_aux) { + x.s[0] = ((__be16 *)disk_aux)[0]; + x.s[1] = ((__be16 *)disk_aux)[1]; + x.s[2] = ((__be16 *)disk_aux)[2]; + x.s[3] = ((__be16 *)disk_aux)[3]; + __entry->disk_aux = be64_to_cpu(x.ll); + } else { + __entry->disk_aux = 0; + } ), TP_printk("o=%08x %s B=%llx c=%u aux=%llx dsk=%llx", diff --git a/include/trace/events/erofs.h b/include/trace/events/erofs.h index dad7360f42f95..59f3ef397bff3 100644 --- a/include/trace/events/erofs.h +++ b/include/trace/events/erofs.h @@ -26,10 +26,9 @@ struct erofs_map_blocks; #define show_mflags(flags) __print_flags(flags, "", \ { EROFS_MAP_MAPPED, "M" }, \ { EROFS_MAP_META, "I" }, \ - { EROFS_MAP_ENCODED, "E" }, \ - { EROFS_MAP_FULL_MAPPED, "F" }, \ - { EROFS_MAP_FRAGMENT, "R" }, \ - { EROFS_MAP_PARTIAL_REF, "P" }) + { EROFS_MAP_PARTIAL_MAPPED, "T" }, \ + { EROFS_MAP_PARTIAL_REF, "P" }, \ + { EROFS_MAP_FRAGMENT, "R" }) TRACE_EVENT(erofs_lookup, diff --git a/include/trace/events/icmp.h b/include/trace/events/icmp.h index 31559796949a7..acfcf2b132d60 100644 --- a/include/trace/events/icmp.h +++ b/include/trace/events/icmp.h @@ -27,17 +27,20 @@ TRACE_EVENT(icmp_send, TP_fast_assign( struct iphdr *iph = ip_hdr(skb); - struct udphdr *uh = udp_hdr(skb); - int proto_4 = iph->protocol; + struct udphdr _uh, *uh = NULL; __be32 *p32; __entry->skbaddr = skb; __entry->type = type; __entry->code = code; - if (proto_4 != IPPROTO_UDP || (u8 *)uh < skb->head || - (u8 *)uh + sizeof(struct udphdr) - > skb_tail_pointer(skb)) { + if (iph->protocol == IPPROTO_UDP) + uh = skb_header_pointer(skb, + skb_network_offset(skb) + + (iph->ihl << 2), + sizeof(_uh), &_uh); + + if (!uh) { __entry->sport = 0; __entry->dport = 0; __entry->ulen = 0; diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 406a42be429b2..adcfdc8d13441 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -1510,6 +1510,7 @@ struct drm_amdgpu_info_device { __u64 high_va_max; /* gfx10 pa_sc_tile_steering_override */ __u32 pa_sc_tile_steering_override; + __u32 pad; /* disabled TCCs */ __u64 tcc_disabled_mask; __u64 min_engine_clock; @@ -1534,7 +1535,6 @@ struct drm_amdgpu_info_device { __u32 csa_alignment; /* Userq IP mask (1 << AMDGPU_HW_IP_*) */ __u32 userq_ip_mask; - __u32 pad; }; struct drm_amdgpu_info_hw_ip { diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index 8e710bbb688e0..fb89c5c37eb04 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -597,7 +597,7 @@ struct btrfs_ioctl_search_args_v2 { __u64 buf_size; /* in - size of buffer * out - on EOVERFLOW: needed size * to store item */ - __u64 buf[]; /* out - found items */ + __u8 buf[]; /* out - found items */ }; /* With a @src_length of zero, the range from @src_offset->EOF is cloned! */ diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h index bcad11a787a55..157f11d3fb72f 100644 --- a/include/uapi/linux/devlink.h +++ b/include/uapi/linux/devlink.h @@ -181,6 +181,7 @@ enum devlink_sb_threshold_type { enum devlink_eswitch_mode { DEVLINK_ESWITCH_MODE_LEGACY, DEVLINK_ESWITCH_MODE_SWITCHDEV, + DEVLINK_ESWITCH_MODE_SWITCHDEV_INACTIVE, }; enum devlink_eswitch_inline_mode { diff --git a/include/uapi/linux/if_xdp.h b/include/uapi/linux/if_xdp.h index 23a0627814687..50d67df78911f 100644 --- a/include/uapi/linux/if_xdp.h +++ b/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/include/uapi/linux/ip_vs.h b/include/uapi/linux/ip_vs.h index 1ed234e7f2513..2c37c6ac7525a 100644 --- a/include/uapi/linux/ip_vs.h +++ b/include/uapi/linux/ip_vs.h @@ -28,12 +28,6 @@ #define IP_VS_SVC_F_SCHED_SH_FALLBACK IP_VS_SVC_F_SCHED1 /* SH fallback */ #define IP_VS_SVC_F_SCHED_SH_PORT IP_VS_SVC_F_SCHED2 /* SH use port */ -/* - * Destination Server Flags - */ -#define IP_VS_DEST_F_AVAILABLE 0x0001 /* server is available */ -#define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ - /* * IPVS sync daemon states */ diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h index f030adc462ee7..3a7debc21b52e 100644 --- a/include/uapi/linux/landlock.h +++ b/include/uapi/linux/landlock.h @@ -216,6 +216,23 @@ struct landlock_net_port_attr { * :manpage:`ftruncate(2)`, :manpage:`creat(2)`, or :manpage:`open(2)` with * ``O_TRUNC``. This access right is available since the third version of the * Landlock ABI. + * - %LANDLOCK_ACCESS_FS_IOCTL_DEV: Invoke :manpage:`ioctl(2)` commands on an opened + * character or block device. + * + * This access right applies to all `ioctl(2)` commands implemented by device + * drivers. However, the following common IOCTL commands continue to be + * invokable independent of the %LANDLOCK_ACCESS_FS_IOCTL_DEV right: + * + * * IOCTL commands targeting file descriptors (``FIOCLEX``, ``FIONCLEX``), + * * IOCTL commands targeting file descriptions (``FIONBIO``, ``FIOASYNC``), + * * IOCTL commands targeting file systems (``FIFREEZE``, ``FITHAW``, + * ``FIGETBSZ``, ``FS_IOC_GETFSUUID``, ``FS_IOC_GETFSSYSFSPATH``) + * * Some IOCTL commands which do not make sense when used with devices, but + * whose implementations are safe and return the right error codes + * (``FS_IOC_FIEMAP``, ``FICLONE``, ``FICLONERANGE``, ``FIDEDUPERANGE``) + * + * This access right is available since the fifth version of the Landlock + * ABI. * * Whether an opened file can be truncated with :manpage:`ftruncate(2)` or used * with `ioctl(2)` is determined during :manpage:`open(2)`, in the same way as @@ -237,6 +254,7 @@ struct landlock_net_port_attr { * device. * - %LANDLOCK_ACCESS_FS_MAKE_DIR: Create (or rename) a directory. * - %LANDLOCK_ACCESS_FS_MAKE_REG: Create (or rename or link) a regular file. + * This also guards the creation of whiteout objects as used in OverlayFS. * - %LANDLOCK_ACCESS_FS_MAKE_SOCK: Create (or rename or link) a UNIX domain * socket. * - %LANDLOCK_ACCESS_FS_MAKE_FIFO: Create (or rename or link) a named pipe. @@ -275,26 +293,6 @@ struct landlock_net_port_attr { * If multiple requirements are not met, the ``EACCES`` error code takes * precedence over ``EXDEV``. * - * The following access right applies both to files and directories: - * - * - %LANDLOCK_ACCESS_FS_IOCTL_DEV: Invoke :manpage:`ioctl(2)` commands on an opened - * character or block device. - * - * This access right applies to all `ioctl(2)` commands implemented by device - * drivers. However, the following common IOCTL commands continue to be - * invokable independent of the %LANDLOCK_ACCESS_FS_IOCTL_DEV right: - * - * * IOCTL commands targeting file descriptors (``FIOCLEX``, ``FIONCLEX``), - * * IOCTL commands targeting file descriptions (``FIONBIO``, ``FIOASYNC``), - * * IOCTL commands targeting file systems (``FIFREEZE``, ``FITHAW``, - * ``FIGETBSZ``, ``FS_IOC_GETFSUUID``, ``FS_IOC_GETFSSYSFSPATH``) - * * Some IOCTL commands which do not make sense when used with devices, but - * whose implementations are safe and return the right error codes - * (``FS_IOC_FIEMAP``, ``FICLONE``, ``FICLONERANGE``, ``FIDEDUPERANGE``) - * - * This access right is available since the fifth version of the Landlock - * ABI. - * * .. warning:: * * It is currently not possible to restrict some file-related actions diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h index 8134f10e4e6c0..857d1f66bf353 100644 --- a/include/uapi/linux/nl80211.h +++ b/include/uapi/linux/nl80211.h @@ -1361,6 +1361,26 @@ * user space that the NAN new cluster has been joined. The cluster ID is * indicated by %NL80211_ATTR_MAC. * + * @NL80211_CMD_INCUMBENT_SIGNAL_DETECT: Once any incumbent signal is detected + * on the operating channel in 6 GHz band, userspace is notified with the + * signal interference bitmap using + * %NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP. The current channel + * definition is also sent. + * + * @NL80211_CMD_NAN_SET_LOCAL_SCHED: Set the local NAN schedule. NAN must be + * operational (%NL80211_CMD_START_NAN was executed). Must contain + * %NL80211_ATTR_NAN_TIME_SLOTS and %NL80211_ATTR_NAN_AVAIL_BLOB, but + * %NL80211_ATTR_NAN_CHANNEL is optional (for example in case of a channel + * removal, that channel won't be provided). + * If %NL80211_ATTR_NAN_SCHED_DEFERRED is set, the command is a request + * from the device to perform an announced schedule update. See + * %NL80211_ATTR_NAN_SCHED_DEFERRED for more details. + * If not set, the schedule should be applied immediately. + * @NL80211_CMD_NAN_SCHED_UPDATE_DONE: Event sent to user space to notify that + * a deferred local NAN schedule update (requested with + * %NL80211_CMD_NAN_SET_LOCAL_SCHED and %NL80211_ATTR_NAN_SCHED_DEFERRED) + * has been completed. The presence of %NL80211_ATTR_NAN_SCHED_UPDATE_SUCCESS + * indicates that the update was successful. * @NL80211_CMD_MAX: highest used command number * @__NL80211_CMD_AFTER_LAST: internal use */ @@ -1624,6 +1644,12 @@ enum nl80211_commands { NL80211_CMD_NAN_NEXT_DW_NOTIFICATION, NL80211_CMD_NAN_CLUSTER_JOINED, + NL80211_CMD_INCUMBENT_SIGNAL_DETECT, + + NL80211_CMD_NAN_SET_LOCAL_SCHED, + + NL80211_CMD_NAN_SCHED_UPDATE_DONE, + /* add new commands above here */ /* used to define NL80211_CMD_MAX below */ @@ -2973,6 +2999,77 @@ enum nl80211_commands { * primary channel is 2 MHz wide, and the control channel designates * the 1 MHz primary subchannel within that 2 MHz primary. * + * @NL80211_ATTR_EPP_PEER: A flag attribute to indicate if the peer is an EPP + * STA. Used with %NL80211_CMD_NEW_STA and %NL80211_CMD_ADD_LINK_STA + * + * @NL80211_ATTR_UHR_CAPABILITY: UHR Capability information element (from + * association request when used with NL80211_CMD_NEW_STATION). Can be set + * only if HE/EHT are also available. + * @NL80211_ATTR_DISABLE_UHR: Force UHR capable interfaces to disable + * this feature during association. This is a flag attribute. + * Currently only supported in mac80211 drivers. + * @NL80211_ATTR_NAN_CHANNEL: This is a nested attribute. There can be multiple + * attributes of this type, each one represents a channel definition and + * consists of top-level attributes like %NL80211_ATTR_WIPHY_FREQ. Must + * contain %NL80211_ATTR_NAN_CHANNEL_ENTRY and + * %NL80211_ATTR_NAN_RX_NSS. + * This attribute is used with %NL80211_CMD_NAN_SET_LOCAL_SCHED to specify + * the channel definitions on which the radio needs to operate during + * specific time slots. All of the channel definitions should be mutually + * incompatible. The number of channels should fit the current + * configuration of channels and the possible interface combinations. + * If an existing NAN channel is changed but the chandef isn't, the + * channel entry must also remain unchanged. + * @NL80211_ATTR_NAN_CHANNEL_ENTRY: a byte array of 6 bytes. contains the + * Channel Entry as defined in Wi-Fi Aware (TM) 4.0 specification Table + * 100 (Channel Entry format for the NAN Availability attribute). + * @NL80211_ATTR_NAN_RX_NSS: (u8) RX NSS used for a NAN channel. This is + * used with %NL80211_ATTR_NAN_CHANNEL when configuring NAN channels with + * %NL80211_CMD_NAN_SET_LOCAL_SCHED. + * @NL80211_ATTR_NAN_TIME_SLOTS: an array of u8 values and 32 cells. each value + * maps a time slot to the chandef on which the radio should operate on in + * that time. %NL80211_NAN_SCHED_NOT_AVAIL_SLOT indicates unscheduled. + * The chandef is represented using its index, where the index is the + * sequential number of the %NL80211_ATTR_NAN_CHANNEL attribute within all + * the attributes of this type. + * Each slots spans over 16TUs, hence the entire schedule spans over + * 512TUs. Other slot durations and periods are currently not supported. + * @NL80211_ATTR_NAN_AVAIL_BLOB: (Binary) The NAN Availability attribute blob, + * including the attribute header, as defined in Wi-Fi Aware (TM) 4.0 + * specification Table 93 (NAN Availability attribute format). Required with + * %NL80211_CMD_NAN_SET_LOCAL_SCHED to provide the raw NAN Availability + * attribute. Used by the device to publish Schedule Update NAFs. + * @NL80211_ATTR_NAN_SCHED_DEFERRED: Flag attribute used with + * %NL80211_CMD_NAN_SET_LOCAL_SCHED. When present, the command is a + * request from the device to perform an announced schedule update. This + * means that it needs to send the updated NAN availability to the peers, + * and do the actual switch on the right time (i.e. at the end of the slot + * after the slot in which the updated NAN Availability was sent). Since + * the slots management is done in the device, the update to the peers + * needs to be sent by the device, so it knows the actual switch time. + * If the flag is not set, the schedule should be applied immediately. + * When this flag is set, the total number of NAN channels from both the + * old and new schedules must not exceed the allowed number of local NAN + * channels, because with deferred scheduling the old channels cannot be + * removed before adding the new ones to free up space. + * @NL80211_ATTR_NAN_SCHED_UPDATE_SUCCESS: flag attribute used with + * %NL80211_CMD_NAN_SCHED_UPDATE_DONE to indicate that the deferred + * schedule update completed successfully. If this flag is not present, + * the update failed. + * + * @NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP: u32 attribute specifying + * the signal interference bitmap detected on the operating bandwidth for + * %NL80211_CMD_INCUMBENT_SIGNAL_DETECT. Each bit represents a 20 MHz + * segment, lowest bit corresponds to the lowest 20 MHz segment, in the + * operating bandwidth where the interference is detected. Punctured + * sub-channels are included in the bitmap structure; however, since + * interference detection is not performed on these sub-channels, their + * corresponding bits are consistently set to zero. + * + * @NL80211_ATTR_UHR_OPERATION: Full UHR Operation element, as it appears in + * association response etc., since it's abridged in the beacon. Used + * for START_AP etc. + * * @NUM_NL80211_ATTR: total number of nl80211_attrs available * @NL80211_ATTR_MAX: highest attribute number currently defined * @__NL80211_ATTR_AFTER_LAST: internal use @@ -3541,6 +3638,23 @@ enum nl80211_attrs { NL80211_ATTR_S1G_PRIMARY_2MHZ, + NL80211_ATTR_EPP_PEER, + + NL80211_ATTR_UHR_CAPABILITY, + NL80211_ATTR_DISABLE_UHR, + + NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP, + + NL80211_ATTR_UHR_OPERATION, + + NL80211_ATTR_NAN_CHANNEL, + NL80211_ATTR_NAN_CHANNEL_ENTRY, + NL80211_ATTR_NAN_TIME_SLOTS, + NL80211_ATTR_NAN_RX_NSS, + NL80211_ATTR_NAN_AVAIL_BLOB, + NL80211_ATTR_NAN_SCHED_DEFERRED, + NL80211_ATTR_NAN_SCHED_UPDATE_SUCCESS, + /* add attributes here, update the policy in nl80211.c */ __NL80211_ATTR_AFTER_LAST, @@ -3893,6 +4007,12 @@ enum nl80211_eht_ru_alloc { * @NL80211_RATE_INFO_4_MHZ_WIDTH: 4 MHz S1G rate * @NL80211_RATE_INFO_8_MHZ_WIDTH: 8 MHz S1G rate * @NL80211_RATE_INFO_16_MHZ_WIDTH: 16 MHz S1G rate + * @NL80211_RATE_INFO_UHR_MCS: UHR MCS index (u8, 0-15, 17, 19, 20, 23) + * Note that the other EHT attributes (such as @NL80211_RATE_INFO_EHT_NSS) + * are used in conjunction with this where applicable + * @NL80211_RATE_INFO_UHR_ELR: UHR ELR flag, which restricts NSS to 1, + * MCS to 0 or 1, and GI to %NL80211_RATE_INFO_EHT_GI_1_6. + * @NL80211_RATE_INFO_UHR_IM: UHR Interference Mitigation flag * @__NL80211_RATE_INFO_AFTER_LAST: internal use */ enum nl80211_rate_info { @@ -3926,6 +4046,9 @@ enum nl80211_rate_info { NL80211_RATE_INFO_4_MHZ_WIDTH, NL80211_RATE_INFO_8_MHZ_WIDTH, NL80211_RATE_INFO_16_MHZ_WIDTH, + NL80211_RATE_INFO_UHR_MCS, + NL80211_RATE_INFO_UHR_ELR, + NL80211_RATE_INFO_UHR_IM, /* keep last */ __NL80211_RATE_INFO_AFTER_LAST, @@ -4248,6 +4371,10 @@ enum nl80211_mpath_info { * capabilities element * @NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE: EHT PPE thresholds information as * defined in EHT capabilities element + * @NL80211_BAND_IFTYPE_ATTR_UHR_CAP_MAC: UHR MAC capabilities as in UHR + * capabilities element + * @NL80211_BAND_IFTYPE_ATTR_UHR_CAP_PHY: UHR PHY capabilities as in UHR + * capabilities element * @__NL80211_BAND_IFTYPE_ATTR_AFTER_LAST: internal use * @NL80211_BAND_IFTYPE_ATTR_MAX: highest band attribute currently defined */ @@ -4265,6 +4392,8 @@ enum nl80211_band_iftype_attr { NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET, NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE, + NL80211_BAND_IFTYPE_ATTR_UHR_CAP_MAC, + NL80211_BAND_IFTYPE_ATTR_UHR_CAP_PHY, /* keep last */ __NL80211_BAND_IFTYPE_ATTR_AFTER_LAST, @@ -4444,6 +4573,11 @@ enum nl80211_wmm_rule { * channel in current regulatory domain. * @NL80211_FREQUENCY_ATTR_NO_16MHZ: 16 MHz operation is not allowed on this * channel in current regulatory domain. + * @NL80211_FREQUENCY_ATTR_S1G_NO_PRIMARY: Channel is not permitted for use + * as a primary channel. Does not prevent the channel from existing + * as a non-primary subchannel. Only applicable to S1G channels. + * @NL80211_FREQUENCY_ATTR_NO_UHR: UHR operation is not allowed on this channel + * in current regulatory domain. * @NL80211_FREQUENCY_ATTR_MAX: highest frequency attribute number * currently defined * @__NL80211_FREQUENCY_ATTR_AFTER_LAST: internal use @@ -4492,6 +4626,8 @@ enum nl80211_frequency_attr { NL80211_FREQUENCY_ATTR_NO_4MHZ, NL80211_FREQUENCY_ATTR_NO_8MHZ, NL80211_FREQUENCY_ATTR_NO_16MHZ, + NL80211_FREQUENCY_ATTR_S1G_NO_PRIMARY, + NL80211_FREQUENCY_ATTR_NO_UHR, /* keep last */ __NL80211_FREQUENCY_ATTR_AFTER_LAST, @@ -4705,6 +4841,7 @@ enum nl80211_sched_scan_match_attr { * despite NO_IR configuration. * @NL80211_RRF_ALLOW_20MHZ_ACTIVITY: Allow activity in 20 MHz bandwidth, * despite NO_IR configuration. + * @NL80211_RRF_NO_UHR: UHR operation not allowed */ enum nl80211_reg_rule_flags { NL80211_RRF_NO_OFDM = 1 << 0, @@ -4731,6 +4868,7 @@ enum nl80211_reg_rule_flags { NL80211_RRF_NO_6GHZ_AFC_CLIENT = 1 << 23, NL80211_RRF_ALLOW_6GHZ_VLP_AP = 1 << 24, NL80211_RRF_ALLOW_20MHZ_ACTIVITY = 1 << 25, + NL80211_RRF_NO_UHR = 1 << 26, }; #define NL80211_RRF_PASSIVE_SCAN NL80211_RRF_NO_IR @@ -8432,4 +8570,6 @@ enum nl80211_nan_capabilities { NL80211_NAN_CAPABILITIES_MAX = __NL80211_NAN_CAPABILITIES_LAST - 1, }; +#define NL80211_NAN_SCHED_NOT_AVAIL_SLOT 0xff + #endif /* __LINUX_NL80211_H */ diff --git a/include/uapi/linux/rkisp1-config.h b/include/uapi/linux/rkisp1-config.h index 3b060ea6eed71..810e874639ca0 100644 --- a/include/uapi/linux/rkisp1-config.h +++ b/include/uapi/linux/rkisp1-config.h @@ -1583,11 +1583,11 @@ struct rkisp1_ext_params_wdr_config { sizeof(struct rkisp1_ext_params_wdr_config)) /** - * enum rksip1_ext_param_buffer_version - RkISP1 extensible parameters version + * enum rkisp1_ext_param_buffer_version - RkISP1 extensible parameters version * * @RKISP1_EXT_PARAM_BUFFER_V1: First version of RkISP1 extensible parameters */ -enum rksip1_ext_param_buffer_version { +enum rkisp1_ext_param_buffer_version { RKISP1_EXT_PARAM_BUFFER_V1 = 1, }; @@ -1667,7 +1667,7 @@ enum rksip1_ext_param_buffer_version { * +---------------------------------------------------------------------+ * * @version: The RkISP1 extensible parameters buffer version, see - * :c:type:`rksip1_ext_param_buffer_version` + * :c:type:`rkisp1_ext_param_buffer_version` * @data_size: The RkISP1 configuration data effective size, excluding this * header * @data: The RkISP1 extensible configuration data blocks diff --git a/include/uapi/linux/time_types.h b/include/uapi/linux/time_types.h index bcc0002115d39..03a0d8aaadca5 100644 --- a/include/uapi/linux/time_types.h +++ b/include/uapi/linux/time_types.h @@ -30,7 +30,7 @@ struct __kernel_old_timeval { struct __kernel_old_timespec { __kernel_old_time_t tv_sec; /* seconds */ - long tv_nsec; /* nanoseconds */ + __kernel_long_t tv_nsec; /* nanoseconds */ }; struct __kernel_old_itimerval { diff --git a/include/uapi/misc/rp1_pio_if.h b/include/uapi/misc/rp1_pio_if.h index 8827b60eeb34a..ee9e7b85b2a03 100644 --- a/include/uapi/misc/rp1_pio_if.h +++ b/include/uapi/misc/rp1_pio_if.h @@ -183,6 +183,21 @@ struct rp1_pio_sm_config_xfer32_args { uint32_t buf_count; }; +#define RP1_PIO_SM_CONFIG_XFER_FL_DMA_PREFER_HEAVY 0 +#define RP1_PIO_SM_CONFIG_XFER_FL_DMA_PREFER_LIGHT 1 +#define RP1_PIO_SM_CONFIG_XFER_FL_DMA_FORCE_HEAVY 2 +#define RP1_PIO_SM_CONFIG_XFER_FL_DMA_FORCE_LIGHT 3 + +#define RP1_PIO_SM_CONFIG_XFER_FL_DMA_CYCLE (1 << 2) + +struct rp1_pio_sm_config_xfer_v2_args { + uint16_t sm; + uint16_t dir; + uint32_t flags; + uint32_t buf_size; + uint32_t buf_count; +}; + struct rp1_pio_sm_xfer_data_args { uint16_t sm; uint16_t dir; @@ -242,6 +257,7 @@ struct rp1_pio_interrupt_clear_args { #define PIO_IOC_SM_XFER_DATA _IOW(PIO_IOC_MAGIC, 1, struct rp1_pio_sm_xfer_data_args) #define PIO_IOC_SM_XFER_DATA32 _IOW(PIO_IOC_MAGIC, 2, struct rp1_pio_sm_xfer_data32_args) #define PIO_IOC_SM_CONFIG_XFER32 _IOW(PIO_IOC_MAGIC, 3, struct rp1_pio_sm_config_xfer32_args) +#define PIO_IOC_SM_CONFIG_XFER_V2 _IOW(PIO_IOC_MAGIC, 4, struct rp1_pio_sm_config_xfer_v2_args) #define PIO_IOC_READ_HW _IOW(PIO_IOC_MAGIC, 8, struct rp1_access_hw_args) #define PIO_IOC_WRITE_HW _IOW(PIO_IOC_MAGIC, 9, struct rp1_access_hw_args) diff --git a/include/uapi/rdma/ib_user_ioctl_cmds.h b/include/uapi/rdma/ib_user_ioctl_cmds.h index de6f5a94f1e33..35da4026f452b 100644 --- a/include/uapi/rdma/ib_user_ioctl_cmds.h +++ b/include/uapi/rdma/ib_user_ioctl_cmds.h @@ -73,6 +73,7 @@ enum uverbs_methods_device { UVERBS_METHOD_QUERY_CONTEXT, UVERBS_METHOD_QUERY_GID_TABLE, UVERBS_METHOD_QUERY_GID_ENTRY, + UVERBS_METHOD_QUERY_PORT_SPEED, }; enum uverbs_attrs_invoke_write_cmd_attr_ids { @@ -86,6 +87,11 @@ enum uverbs_attrs_query_port_cmd_attr_ids { UVERBS_ATTR_QUERY_PORT_RESP, }; +enum uverbs_attrs_query_port_speed_cmd_attr_ids { + UVERBS_ATTR_QUERY_PORT_SPEED_PORT_NUM, + UVERBS_ATTR_QUERY_PORT_SPEED_RESP, +}; + enum uverbs_attrs_get_context_attr_ids { UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT, diff --git a/include/uapi/sound/intel/avs/tokens.h b/include/uapi/sound/intel/avs/tokens.h index f3ff6aae09a9e..3ff6d9150822f 100644 --- a/include/uapi/sound/intel/avs/tokens.h +++ b/include/uapi/sound/intel/avs/tokens.h @@ -21,6 +21,7 @@ enum avs_tplg_token { AVS_TKN_MANIFEST_NUM_BINDINGS_U32 = 8, AVS_TKN_MANIFEST_NUM_CONDPATH_TMPLS_U32 = 9, AVS_TKN_MANIFEST_NUM_INIT_CONFIGS_U32 = 10, + AVS_TKN_MANIFEST_NUM_NHLT_CONFIGS_U32 = 11, /* struct avs_tplg_library */ AVS_TKN_LIBRARY_ID_U32 = 101, @@ -124,6 +125,7 @@ enum avs_tplg_token { AVS_TKN_MOD_KCONTROL_ID_U32 = 1707, AVS_TKN_MOD_INIT_CONFIG_NUM_IDS_U32 = 1708, AVS_TKN_MOD_INIT_CONFIG_ID_U32 = 1709, + AVS_TKN_MOD_NHLT_CONFIG_ID_U32 = 1710, /* struct avs_tplg_path_template */ AVS_TKN_PATH_TMPL_ID_U32 = 1801, @@ -160,6 +162,10 @@ enum avs_tplg_token { AVS_TKN_INIT_CONFIG_ID_U32 = 2401, AVS_TKN_INIT_CONFIG_PARAM_U8 = 2402, AVS_TKN_INIT_CONFIG_LENGTH_U32 = 2403, + + /* struct avs_tplg_nhlt_config */ + AVS_TKN_NHLT_CONFIG_ID_U32 = 2501, + AVS_TKN_NHLT_CONFIG_SIZE_U32 = 2502, }; #endif diff --git a/init/Kconfig b/init/Kconfig index 0c25e78e10f47..6ce315af48c15 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -162,6 +162,9 @@ config RUSTC_HAS_FILE_WITH_NUL config RUSTC_HAS_FILE_AS_C_STR def_bool RUSTC_VERSION >= 109100 +config RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS + def_bool RUSTC_VERSION >= 109800 + config PAHOLE_VERSION int default $(shell,$(srctree)/scripts/pahole-version.sh $(PAHOLE)) diff --git a/init/main.c b/init/main.c index 048a625382429..269b30038b9fa 100644 --- a/init/main.c +++ b/init/main.c @@ -104,6 +104,7 @@ #include #include #include +#include #include #include @@ -269,10 +270,11 @@ static void * __init get_boot_config_from_initrd(size_t *_size) { u32 size, csum; char *data; - u32 *hdr; + u8 *hdr; int i; - if (!initrd_end) + if (!initrd_end || initrd_end < initrd_start || + initrd_end - initrd_start < BOOTCONFIG_MAGIC_LEN + 8) return NULL; data = (char *)initrd_end - BOOTCONFIG_MAGIC_LEN; @@ -288,17 +290,27 @@ static void * __init get_boot_config_from_initrd(size_t *_size) return NULL; found: - hdr = (u32 *)(data - 8); - size = le32_to_cpu(hdr[0]); - csum = le32_to_cpu(hdr[1]); + hdr = (u8 *)(data - 8); + if ((unsigned long)hdr < initrd_start) + return NULL; - data = ((void *)hdr) - size; - if ((unsigned long)data < initrd_start) { - pr_err("bootconfig size %d is greater than initrd size %ld\n", + size = get_unaligned_le32(hdr); + csum = get_unaligned_le32(hdr + 4); + + if (size > XBC_DATA_MAX) { + pr_err("bootconfig size %u is greater than max size %d\n", + size, XBC_DATA_MAX); + return NULL; + } + + if (size > ((unsigned long)hdr - initrd_start)) { + pr_err("bootconfig size %u is greater than initrd size %lu\n", size, initrd_end - initrd_start); return NULL; } + data = ((void *)hdr) - size; + if (xbc_calc_checksum(data, size) != csum) { pr_err("bootconfig checksum failed\n"); return NULL; @@ -320,51 +332,6 @@ static void * __init get_boot_config_from_initrd(size_t *_size) #ifdef CONFIG_BOOT_CONFIG -static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; - -#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) - -static int __init xbc_snprint_cmdline(char *buf, size_t size, - struct xbc_node *root) -{ - struct xbc_node *knode, *vnode; - char *end = buf + size; - const char *val, *q; - int ret; - - xbc_node_for_each_key_value(root, knode, val) { - ret = xbc_node_compose_key_after(root, knode, - xbc_namebuf, XBC_KEYLEN_MAX); - if (ret < 0) - return ret; - - vnode = xbc_node_get_child(knode); - if (!vnode) { - ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf); - if (ret < 0) - return ret; - buf += ret; - continue; - } - xbc_array_for_each_value(vnode, val) { - /* - * For prettier and more readable /proc/cmdline, only - * quote the value when necessary, i.e. when it contains - * whitespace. - */ - q = strpbrk(val, " \t\r\n") ? "\"" : ""; - ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ", - xbc_namebuf, q, val, q); - if (ret < 0) - return ret; - buf += ret; - } - } - - return buf - (end - size); -} -#undef rest - /* Make an extra command line under given key word */ static char * __init xbc_make_cmdline(const char *key) { @@ -446,12 +413,6 @@ static void __init setup_boot_config(void) return; } - if (size >= XBC_DATA_MAX) { - pr_err("bootconfig size %ld greater than max size %d\n", - (long)size, XBC_DATA_MAX); - return; - } - ret = xbc_init(data, size, &msg, &pos); if (ret < 0) { if (pos < 0) @@ -1570,7 +1531,7 @@ static noinline void __init kernel_init_freeable(void) */ set_mems_allowed(node_states[N_MEMORY]); - cad_pid = get_pid(task_pid(current)); + rcu_assign_pointer(cad_pid, get_pid(task_pid(current))); smp_prepare_cpus(setup_max_cpus); diff --git a/io_uring/eventfd.c b/io_uring/eventfd.c index ab789e1ebe91e..7a57dd88304a6 100644 --- a/io_uring/eventfd.c +++ b/io_uring/eventfd.c @@ -50,9 +50,9 @@ static void io_eventfd_do_signal(struct rcu_head *rcu) /* * Returns true if the caller should put the ev_fd reference, false if not. */ -static bool __io_eventfd_signal(struct io_ev_fd *ev_fd) +static bool __io_eventfd_signal(struct io_ev_fd *ev_fd, bool defer) { - if (eventfd_signal_allowed()) { + if (!defer && eventfd_signal_allowed()) { eventfd_signal_mask(ev_fd->cq_ev_fd, EPOLL_URING_WAKE); return true; } @@ -72,7 +72,7 @@ static bool io_eventfd_trigger(struct io_ev_fd *ev_fd) return !ev_fd->eventfd_async || io_wq_current_is_worker(); } -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer) { bool skip = false; struct io_ev_fd *ev_fd; @@ -112,7 +112,7 @@ void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event) spin_unlock(&ctx->completion_lock); } - if (skip || __io_eventfd_signal(ev_fd)) + if (skip || __io_eventfd_signal(ev_fd, defer)) io_eventfd_put(ev_fd); } diff --git a/io_uring/eventfd.h b/io_uring/eventfd.h index e2f1985c2cf98..86c3c69eba1d7 100644 --- a/io_uring/eventfd.h +++ b/io_uring/eventfd.h @@ -4,4 +4,4 @@ int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int eventfd_async); int io_eventfd_unregister(struct io_ring_ctx *ctx); -void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event); +void io_eventfd_signal(struct io_ring_ctx *ctx, bool cqe_event, bool defer); diff --git a/io_uring/futex.c b/io_uring/futex.c index 64f3bd51c84c6..7f3b97c996926 100644 --- a/io_uring/futex.c +++ b/io_uring/futex.c @@ -41,24 +41,26 @@ void io_futex_cache_free(struct io_ring_ctx *ctx) io_alloc_cache_free(&ctx->futex_cache, kfree); } -static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) +static void __io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) { - hlist_del_init(&req->hash_node); - io_req_task_complete(req, tw); + hlist_del_init(&tw_req.req->hash_node); + io_req_task_complete(tw_req, tw); } -static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw) +static void io_futex_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_ring_ctx *ctx = req->ctx; io_tw_lock(ctx, tw); io_cache_free(&ctx->futex_cache, req->async_data); io_req_async_data_clear(req, 0); - __io_futex_complete(req, tw); + __io_futex_complete(tw_req, tw); } -static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw) +static void io_futexv_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); struct futex_vector *futexv = req->async_data; @@ -73,7 +75,7 @@ static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw) } io_req_async_data_free(req); - __io_futex_complete(req, tw); + __io_futex_complete(tw_req, tw); } static bool io_futexv_claim(struct io_futex *iof) @@ -144,8 +146,21 @@ int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) !futex_validate_input(iof->futex_flags, iof->futex_mask)) return -EINVAL; - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + return 0; +} + +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); + int ret; + + ret = io_futex_prep(req, sqe); + if (unlikely(ret)) + return ret; + + /* inflight tracking only needed for mm private hash */ + if (!(iof->futex_flags & FLAGS_SHARED)) + io_req_track_inflight(req); return 0; } @@ -161,13 +176,14 @@ static void io_futex_wakev_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futexv_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex); struct futex_vector *futexv; + unsigned int i; int ret; /* No flags or mask supported for waitv */ @@ -191,8 +207,14 @@ int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return ret; } - /* Mark as inflight, so file exit cancelation will find it */ - io_req_track_inflight(req); + /* inflight tracking only needed for mm private hash */ + for (i = 0; i < iof->futex_nr; i++) { + if (!(futexv[i].w.flags & FLAGS_SHARED)) { + io_req_track_inflight(req); + break; + } + } + iof->futexv_owned = 0; iof->futexv_unqueued = 0; req->flags |= REQ_F_ASYNC_DATA; @@ -210,7 +232,7 @@ static void io_futex_wake_fn(struct wake_q_head *wake_q, struct futex_q *q) io_req_set_res(req, 0, 0); req->io_task_work.func = io_futex_complete; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); } int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags) diff --git a/io_uring/futex.h b/io_uring/futex.h index d789fcf715e38..987db3f2c6d9a 100644 --- a/io_uring/futex.h +++ b/io_uring/futex.h @@ -3,6 +3,7 @@ #include "cancel.h" int io_futex_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); +int io_futex_wait_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futexv_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_futex_wait(struct io_kiocb *req, unsigned int issue_flags); int io_futexv_wait(struct io_kiocb *req, unsigned int issue_flags); diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index af00f942a75fa..e8b8d2c1cf7f8 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -210,9 +210,12 @@ static void io_worker_cancel_cb(struct io_worker *worker) struct io_wq *wq = worker->wq; atomic_dec(&acct->nr_running); - raw_spin_lock(&acct->workers_lock); - acct->nr_workers--; - raw_spin_unlock(&acct->workers_lock); + /* create_worker_cb() has not reserved a worker slot yet. */ + if (worker->create_work.func != create_worker_cb) { + raw_spin_lock(&acct->workers_lock); + acct->nr_workers--; + raw_spin_unlock(&acct->workers_lock); + } io_worker_ref_put(wq); clear_bit_unlock(0, &worker->create_state); io_worker_release(worker); diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index 7f398c4a3a6eb..fd64ba7a0927a 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -268,6 +268,20 @@ static __cold void io_ring_ctx_ref_free(struct percpu_ref *ref) complete(&ctx->ref_comp); } +/* + * Terminate the request if either of these conditions are true: + * + * 1) It's being executed by the original task, but that task is marked + * with PF_EXITING as it's exiting. + * 2) PF_KTHREAD is set, in which case the invoker of the task_work is + * our fallback task_work. + * 3) The ring has been closed and is going away. + */ +static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx) +{ + return (current->flags & (PF_EXITING | PF_KTHREAD)) || percpu_ref_is_dying(&ctx->refs); +} + static __cold void io_fallback_req_func(struct work_struct *work) { struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, @@ -278,8 +292,9 @@ static __cold void io_fallback_req_func(struct work_struct *work) percpu_ref_get(&ctx->refs); mutex_lock(&ctx->uring_lock); + ts.cancel = io_should_terminate_tw(ctx); llist_for_each_entry_safe(req, tmp, node, io_task_work.node) - req->io_task_work.func(req, ts); + req->io_task_work.func((struct io_tw_req){req}, ts); io_submit_flush_completions(ctx); mutex_unlock(&ctx->uring_lock); percpu_ref_put(&ctx->refs); @@ -527,9 +542,9 @@ static void io_queue_iowq(struct io_kiocb *req) io_wq_enqueue(tctx->io_wq, &req->work); } -static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw) +static void io_req_queue_iowq_tw(struct io_tw_req tw_req, io_tw_token_t tw) { - io_queue_iowq(req); + io_queue_iowq(tw_req.req); } void io_req_queue_iowq(struct io_kiocb *req) @@ -578,7 +593,7 @@ void __io_commit_cqring_flush(struct io_ring_ctx *ctx) if (ctx->off_timeout_used) io_flush_timeouts(ctx); if (ctx->has_evfd) - io_eventfd_signal(ctx, true); + io_eventfd_signal(ctx, true, false); } static inline void __io_cq_lock(struct io_ring_ctx *ctx) @@ -1152,10 +1167,11 @@ struct llist_node *io_handle_tw_list(struct llist_node *node, ctx = req->ctx; mutex_lock(&ctx->uring_lock); percpu_ref_get(&ctx->refs); + ts.cancel = io_should_terminate_tw(ctx); } INDIRECT_CALL_2(req->io_task_work.func, io_poll_task_func, io_req_rw_complete, - req, ts); + (struct io_tw_req){req}, ts); node = next; (*count)++; if (unlikely(need_resched())) { @@ -1210,11 +1226,6 @@ struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, { struct llist_node *node; - if (unlikely(current->flags & PF_EXITING)) { - io_fallback_tw(tctx, true); - return NULL; - } - node = llist_del_all(&tctx->task_list); if (node) { node = llist_reverse_order(node); @@ -1312,7 +1323,7 @@ static void io_req_local_work_add(struct io_kiocb *req, unsigned flags) if (!head) { io_ctx_mark_taskrun(ctx); if (ctx->has_evfd) - io_eventfd_signal(ctx, false); + io_eventfd_signal(ctx, false, flags & IOU_F_TWQ_IN_WAKE); } nr_wait = atomic_read(&ctx->cq_wait_nr); @@ -1411,7 +1422,7 @@ static int __io_run_local_work_loop(struct llist_node **node, io_task_work.node); INDIRECT_CALL_2(req->io_task_work.func, io_poll_task_func, io_req_rw_complete, - req, tw); + (struct io_tw_req){req}, tw); *node = next; if (++ret >= events) break; @@ -1432,6 +1443,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw, if (ctx->flags & IORING_SETUP_TASKRUN_FLAG) atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags); again: + tw.cancel = io_should_terminate_tw(ctx); min_events -= ret; ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events); if (ctx->retry_llist.first) @@ -1480,18 +1492,21 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events, return ret; } -static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw) +static void io_req_task_cancel(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; + io_tw_lock(req->ctx, tw); io_req_defer_failed(req, req->cqe.res); } -void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw) +void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_ring_ctx *ctx = req->ctx; io_tw_lock(ctx, tw); - if (unlikely(io_should_terminate_tw(ctx))) + if (unlikely(tw.cancel)) io_req_defer_failed(req, -EFAULT); else if (req->flags & REQ_F_FORCE_ASYNC) io_queue_iowq(req); @@ -1723,9 +1738,9 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, unsigned int min_events) return 0; } -void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw) +void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw) { - io_req_complete_defer(req); + io_req_complete_defer(tw_req.req); } /* @@ -3867,8 +3882,7 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p, static_branch_deferred_inc(&io_key_has_sqarray); if ((ctx->flags & IORING_SETUP_DEFER_TASKRUN) && - !(ctx->flags & IORING_SETUP_IOPOLL) && - !(ctx->flags & IORING_SETUP_SQPOLL)) + !(ctx->flags & IORING_SETUP_IOPOLL)) ctx->task_complete = true; if (ctx->task_complete || (ctx->flags & IORING_SETUP_IOPOLL)) diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index 05702288465b7..042bdd766571b 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -160,9 +160,9 @@ struct file *io_file_get_fixed(struct io_kiocb *req, int fd, void __io_req_task_work_add(struct io_kiocb *req, unsigned flags); void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags); void io_req_task_queue(struct io_kiocb *req); -void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw); +void io_req_task_complete(struct io_tw_req tw_req, io_tw_token_t tw); void io_req_task_queue_fail(struct io_kiocb *req, int ret); -void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw); +void io_req_task_submit(struct io_tw_req tw_req, io_tw_token_t tw); struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries); struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count); void tctx_task_work(struct callback_head *cb); @@ -582,19 +582,6 @@ static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) ctx->submitter_task == current); } -/* - * Terminate the request if either of these conditions are true: - * - * 1) It's being executed by the original task, but that task is marked - * with PF_EXITING as it's exiting. - * 2) PF_KTHREAD is set, in which case the invoker of the task_work is - * our fallback task_work. - */ -static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx) -{ - return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs); -} - static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) { io_req_set_res(req, res, 0); diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index bce74a8b64c6e..87b4d306cf1b6 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -70,8 +70,9 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx) return target_ctx->task_complete; } -static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw) +static void io_msg_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_ring_ctx *ctx = req->ctx; io_add_aux_cqe(ctx, req->cqe.user_data, req->cqe.res, req->cqe.flags); diff --git a/io_uring/net.c b/io_uring/net.c index 3ab2bfca1bd5d..455a191c9db41 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -433,6 +433,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) req->flags |= REQ_F_NOWAIT; if (req->flags & REQ_F_BUFFER_SELECT) sr->buf_group = req->buf_index; + sr->mshot_total_len = sr->mshot_len = 0; if (sr->flags & IORING_RECVSEND_BUNDLE) { if (req->opcode == IORING_OP_SENDMSG) return -EINVAL; @@ -851,7 +852,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static inline bool io_recv_finish(struct io_kiocb *req, struct io_async_msghdr *kmsg, struct io_br_sel *sel, bool mshot_finished, - unsigned issue_flags) + unsigned issue_flags, int consumed) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); unsigned int cflags = 0; @@ -875,7 +876,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, if (sr->flags & IORING_RECVSEND_BUNDLE) { size_t this_ret = sel->val - sr->done_io; - cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret)); + cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed)); if (sr->flags & IORING_RECV_RETRY) cflags = req->cqe.flags | (cflags & CQE_F_MASK); if (sr->mshot_len && sel->val >= sr->mshot_len) @@ -897,7 +898,7 @@ static inline bool io_recv_finish(struct io_kiocb *req, return false; } } else { - cflags |= io_put_kbuf(req, sel->val, sel->buf_list); + cflags |= io_put_kbuf(req, consumed, sel->buf_list); } /* @@ -1025,6 +1026,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) int ret, min_ret = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool mshot_finished = true; + int consumed = 0; + size_t len; sock = sock_from_file(req->file); if (unlikely(!sock)) @@ -1040,9 +1043,8 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: sel.buf_list = NULL; + len = sr->len; if (io_do_buffer_select(req)) { - size_t len = sr->len; - sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); if (!sel.addr) return -ENOBUFS; @@ -1063,6 +1065,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) if (req->flags & REQ_F_APOLL_MULTISHOT) { ret = io_recvmsg_multishot(sock, sr, kmsg, flags, &mshot_finished); + consumed = ret; } else { /* disable partial retry for recvmsg with cmsg attached */ if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) @@ -1070,6 +1073,15 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags); + /* + * With MSG_TRUNC, the net layer will return the full size of + * the packet, even if we only filled part of it in the buffers. + * Adjust the returned size to consume only the real part of the + * buffer. + */ + consumed = ret; + if (ret > 0) + consumed = min_t(size_t, ret, len); } if (ret < min_ret) { @@ -1096,7 +1108,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; - if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) goto retry_multishot; return sel.val; @@ -1106,6 +1118,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg struct io_br_sel *sel, unsigned int issue_flags) { struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); + size_t len; int ret; /* @@ -1151,13 +1164,14 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg /* special case 1 vec, can be a fast path */ if (ret == 1) { sr->buf = arg.iovs[0].iov_base; - sr->len = arg.iovs[0].iov_len; + len = sr->len = arg.iovs[0].iov_len; goto map_ubuf; } iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, - arg.out_len); + arg.out_len); + len = arg.out_len; } else { - size_t len = sel->val; + len = sel->val; *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); if (!sel->addr) @@ -1171,7 +1185,7 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg return ret; } - return 0; + return len; } int io_recv(struct io_kiocb *req, unsigned int issue_flags) @@ -1181,9 +1195,10 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) struct io_br_sel sel; struct socket *sock; unsigned flags; - int ret, min_ret = 0; + int ret, min_ret = 0, consumed = 0; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; bool mshot_finished; + size_t len = 0; if (!(req->flags & REQ_F_POLLED) && (sr->flags & IORING_RECVSEND_POLL_FIRST)) @@ -1199,6 +1214,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) retry_multishot: sel.buf_list = NULL; + len = sr->len; if (io_do_buffer_select(req)) { sel.val = sr->len; ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); @@ -1206,6 +1222,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) kmsg->msg.msg_inq = -1; goto out_free; } + len = ret; sr->buf = NULL; } @@ -1236,6 +1253,17 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) } mshot_finished = ret <= 0; + + /* + * With MSG_TRUNC, the net layer will return the full size of + * the packet, even if we only filled part of it in the buffers. + * Adjust the returned size to consume only the real part of the + * buffer. + */ + consumed = ret; + if (ret > 0) + consumed = min_t(size_t, ret, len); + if (ret > 0) ret += sr->done_io; else if (sr->done_io) @@ -1244,7 +1272,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) io_kbuf_recycle(req, sel.buf_list, issue_flags); sel.val = ret; - if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) + if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) goto retry_multishot; return sel.val; diff --git a/io_uring/notif.c b/io_uring/notif.c index d8ba1165c9494..9960bb2a32d5f 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -11,8 +11,9 @@ static const struct ubuf_info_ops io_ubuf_ops; -static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw) +static void io_notif_tw_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *notif = tw_req.req; struct io_notif_data *nd = io_notif_to_data(notif); struct io_ring_ctx *ctx = notif->ctx; @@ -34,7 +35,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw) } nd = nd->next; - io_req_task_complete(notif, tw); + io_req_task_complete((struct io_tw_req){notif}, tw); } while (nd); } diff --git a/io_uring/opdef.c b/io_uring/opdef.c index a57c820567f77..0a3b684d72d80 100644 --- a/io_uring/opdef.c +++ b/io_uring/opdef.c @@ -466,7 +466,7 @@ const struct io_issue_def io_issue_defs[] = { }, [IORING_OP_FUTEX_WAIT] = { #if defined(CONFIG_FUTEX) - .prep = io_futex_prep, + .prep = io_futex_wait_prep, .issue = io_futex_wait, #else .prep = io_eopnotsupp_prep, diff --git a/io_uring/poll.c b/io_uring/poll.c index a5e78747e63ab..f122a0070155b 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -190,9 +190,9 @@ enum { IOU_POLL_REQUEUE = 4, }; -static void __io_poll_execute(struct io_kiocb *req, int mask) +static void __io_poll_execute(struct io_kiocb *req, int mask, unsigned tw_flags) { - unsigned flags = 0; + unsigned flags = tw_flags; io_req_set_res(req, mask, 0); req->io_task_work.func = io_poll_task_func; @@ -200,14 +200,15 @@ static void __io_poll_execute(struct io_kiocb *req, int mask) trace_io_uring_task_add(req, mask); if (!(req->flags & REQ_F_POLL_NO_LAZY)) - flags = IOU_F_TWQ_LAZY_WAKE; + flags |= IOU_F_TWQ_LAZY_WAKE; __io_req_task_work_add(req, flags); } -static inline void io_poll_execute(struct io_kiocb *req, int res) +static inline void io_poll_execute(struct io_kiocb *req, int res, + unsigned tw_flags) { if (io_poll_get_ownership(req)) - __io_poll_execute(req, res); + __io_poll_execute(req, res, tw_flags); } /* @@ -224,7 +225,7 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) { int v; - if (unlikely(io_should_terminate_tw(req->ctx))) + if (unlikely(tw.cancel)) return -ECANCELED; do { @@ -315,15 +316,16 @@ static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw) return IOU_POLL_NO_ACTION; } -void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) +void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; int ret; ret = io_poll_check_events(req, tw); if (ret == IOU_POLL_NO_ACTION) { return; } else if (ret == IOU_POLL_REQUEUE) { - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); return; } io_poll_remove_entries(req); @@ -337,7 +339,7 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) poll = io_kiocb_to_cmd(req, struct io_poll); req->cqe.res = mangle_poll(req->cqe.res & poll->events); } else if (ret == IOU_POLL_REISSUE) { - io_req_task_submit(req, tw); + io_req_task_submit(tw_req, tw); return; } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) { req->cqe.res = ret; @@ -345,14 +347,14 @@ void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw) } io_req_set_res(req, req->cqe.res, 0); - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); } else { io_tw_lock(req->ctx, tw); if (ret == IOU_POLL_REMOVE_POLL_USE_RES) - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE) - io_req_task_submit(req, tw); + io_req_task_submit(tw_req, tw); else io_req_defer_failed(req, ret); } @@ -362,7 +364,7 @@ static void io_poll_cancel_req(struct io_kiocb *req) { io_poll_mark_cancelled(req); /* kick tw, which should complete the request */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, 0); } #define IO_ASYNC_POLL_COMMON (EPOLLONESHOT | EPOLLPRI) @@ -371,7 +373,7 @@ static __cold int io_pollfree_wake(struct io_kiocb *req, struct io_poll *poll) { io_poll_mark_cancelled(req); /* we have to kick tw in case it's not already */ - io_poll_execute(req, 0); + io_poll_execute(req, 0, IOU_F_TWQ_IN_WAKE); /* * If the waitqueue is being freed early but someone is already @@ -426,7 +428,7 @@ static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, else req->flags &= ~REQ_F_SINGLE_POLL; } - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, IOU_F_TWQ_IN_WAKE); } return 1; } @@ -614,7 +616,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, if (mask && (poll->events & EPOLLET) && io_poll_can_finish_inline(req, ipt)) { - __io_poll_execute(req, mask); + __io_poll_execute(req, mask, 0); return 0; } io_napi_add(req); @@ -625,7 +627,7 @@ static int __io_arm_poll_handler(struct io_kiocb *req, * poll was waken up, queue up a tw, it'll deal with it. */ if (atomic_cmpxchg(&req->poll_refs, 1, 0) != 1) - __io_poll_execute(req, 0); + __io_poll_execute(req, 0, 0); } return 0; } diff --git a/io_uring/poll.h b/io_uring/poll.h index c8438286dfa0a..5647c5138932b 100644 --- a/io_uring/poll.h +++ b/io_uring/poll.h @@ -46,4 +46,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags); bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all); -void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw); +void io_poll_task_func(struct io_tw_req tw_req, io_tw_token_t tw); diff --git a/io_uring/query.c b/io_uring/query.c index cf02893ba911b..e0b42e07eb080 100644 --- a/io_uring/query.c +++ b/io_uring/query.c @@ -36,6 +36,9 @@ static int io_handle_query_entry(struct io_ring_ctx *ctx, if (copy_from_user(&hdr, uhdr, sizeof(hdr))) return -EFAULT; + /* copy_struct_to_user() zeros up to usize bytes */ + if (hdr.size > PAGE_SIZE) + return -E2BIG; usize = hdr.size; hdr.size = min(hdr.size, IO_MAX_QUERY_SIZE); udata = u64_to_user_ptr(hdr.query_data); diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index dc87c6a86e346..8d28c6bfebb1e 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -94,20 +94,6 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen) return 0; } -static int io_buffer_validate(struct iovec *iov) -{ - /* - * Don't impose further limits on the size and buffer - * constraints here, we'll -EINVAL later when IO is - * submitted if they are wrong. - */ - if (!iov->iov_base) - return iov->iov_len ? -EFAULT : 0; - - return io_validate_user_buf_range((unsigned long)iov->iov_base, - iov->iov_len); -} - static void io_release_ubuf(void *priv) { struct io_mapped_ubuf *imu = priv; @@ -317,9 +303,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx, err = -EFAULT; break; } - err = io_buffer_validate(iov); - if (err) - break; node = io_sqe_buffer_register(ctx, iov, &last_hpage); if (IS_ERR(node)) { err = PTR_ERR(node); @@ -788,8 +771,17 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, struct io_imu_folio_data data; bool coalesced = false; - if (!iov->iov_base) + if (!iov->iov_base) { + if (iov->iov_len) + return ERR_PTR(-EFAULT); + /* remove the buffer without installing a new one */ return NULL; + } + + ret = io_validate_user_buf_range((unsigned long)iov->iov_base, + iov->iov_len); + if (ret) + return ERR_PTR(ret); node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); if (!node) @@ -895,9 +887,6 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, ret = PTR_ERR(iov); break; } - ret = io_buffer_validate(iov); - if (ret) - break; if (ctx->compat) arg += sizeof(struct compat_iovec); else @@ -1349,7 +1338,7 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter, struct iovec *iovec, unsigned nr_iovs, struct iou_vec *vec) { - unsigned long folio_size = 1 << imu->folio_shift; + unsigned long folio_size = 1UL << imu->folio_shift; unsigned long folio_mask = folio_size - 1; struct bio_vec *res_bvec = vec->bvec; size_t total_len = 0; diff --git a/io_uring/rw.c b/io_uring/rw.c index 1a5dee8fe619f..3666ff3d2e421 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -532,20 +532,15 @@ static void io_req_end_write(struct io_kiocb *req) } } -/* - * Trigger the notifications after having done some IO, and finish the write - * accounting, if any. - */ -static void io_req_io_end(struct io_kiocb *req) +/* Trigger the notifications after having done some IO. */ +static void io_req_io_notify(struct io_kiocb *req) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); - if (rw->kiocb.ki_flags & IOCB_WRITE) { - io_req_end_write(req); + if (rw->kiocb.ki_flags & IOCB_WRITE) fsnotify_modify(req->file); - } else { + else fsnotify_access(req->file); - } } static void __io_complete_rw_common(struct io_kiocb *req, long res) @@ -574,8 +569,9 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res) return res; } -void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) +void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct kiocb *kiocb = &rw->kiocb; @@ -585,13 +581,13 @@ void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw) io_req_set_res(req, io_fixup_rw_res(req, res), 0); } - io_req_io_end(req); + io_req_io_notify(req); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) req->cqe.flags |= io_put_kbuf(req, req->cqe.res, NULL); io_req_rw_cleanup(req, 0); - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); } static void io_complete_rw(struct kiocb *kiocb, long res) @@ -599,6 +595,10 @@ static void io_complete_rw(struct kiocb *kiocb, long res) struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); struct io_kiocb *req = cmd_to_io_kiocb(rw); + /* ring owner may block in freeze_super() before task_work runs */ + if (kiocb->ki_flags & IOCB_WRITE) + io_req_end_write(req); + if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) { __io_complete_rw_common(req, res); io_req_set_res(req, io_fixup_rw_res(req, res), 0); @@ -625,6 +625,24 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res) smp_store_release(&req->iopoll_completed, 1); } +static inline ssize_t io_fixup_restart_res(ssize_t ret) +{ + switch (ret) { + case -ERESTARTSYS: + case -ERESTARTNOINTR: + case -ERESTARTNOHAND: + case -ERESTART_RESTARTBLOCK: + /* + * We can't just restart the syscall, since previously + * submitted sqes may already be in progress. Just fail + * this IO with EINTR. + */ + return -EINTR; + default: + return ret; + } +} + static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) { struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); @@ -634,21 +652,8 @@ static inline void io_rw_done(struct io_kiocb *req, ssize_t ret) return; /* transform internal restart error codes */ - if (unlikely(ret < 0)) { - switch (ret) { - case -ERESTARTSYS: - case -ERESTARTNOINTR: - case -ERESTARTNOHAND: - case -ERESTART_RESTARTBLOCK: - /* - * We can't just restart the syscall, since previously - * submitted sqes may already be in progress. Just fail - * this IO with EINTR. - */ - ret = -EINTR; - break; - } - } + if (unlikely(ret < 0)) + ret = io_fixup_restart_res(ret); if (req->ctx->flags & IORING_SETUP_IOPOLL) io_complete_rw_iopoll(&rw->kiocb, ret); @@ -669,10 +674,12 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, __io_complete_rw_common(req, ret); /* - * Safe to call io_end from here as we're inline + * Safe to notify from here as we're inline * from the submission path. */ - io_req_io_end(req); + if (rw->kiocb.ki_flags & IOCB_WRITE) + io_req_end_write(req); + io_req_io_notify(req); if (sel) cflags = io_put_kbuf(req, ret, sel->buf_list); io_req_set_res(req, final_ret, cflags); @@ -1044,7 +1051,8 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) if (req->flags & REQ_F_BUFFERS_COMMIT) io_kbuf_recycle(req, sel.buf_list, issue_flags); - return ret; + + return io_fixup_restart_res(ret); } int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) @@ -1078,8 +1086,10 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags) return IOU_RETRY; } else if (ret <= 0) { io_kbuf_recycle(req, sel.buf_list, issue_flags); - if (ret < 0) + if (ret < 0) { + ret = io_fixup_restart_res(ret); req_set_fail(req); + } } else if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { cflags = io_put_kbuf(req, ret, sel.buf_list); } else { diff --git a/io_uring/rw.h b/io_uring/rw.h index 129a53fe54825..9bd7fbf70ea9b 100644 --- a/io_uring/rw.h +++ b/io_uring/rw.h @@ -46,7 +46,7 @@ int io_read_fixed(struct io_kiocb *req, unsigned int issue_flags); int io_write_fixed(struct io_kiocb *req, unsigned int issue_flags); void io_readv_writev_cleanup(struct io_kiocb *req); void io_rw_fail(struct io_kiocb *req); -void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw); +void io_req_rw_complete(struct io_tw_req tw_req, io_tw_token_t tw); int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe); int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags); void io_rw_cache_free(const void *entry); diff --git a/io_uring/timeout.c b/io_uring/timeout.c index dec1d13ca4207..3d03e178044bc 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -68,8 +68,9 @@ static inline bool io_timeout_finish(struct io_timeout *timeout, static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer); -static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw) +static void io_timeout_complete(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data = req->async_data; struct io_ring_ctx *ctx = req->ctx; @@ -85,7 +86,7 @@ static void io_timeout_complete(struct io_kiocb *req, io_tw_token_t tw) } } - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); } static __cold bool io_flush_killed_timeouts(struct list_head *list, int err) @@ -157,8 +158,10 @@ __cold void io_flush_timeouts(struct io_ring_ctx *ctx) io_flush_killed_timeouts(&list, 0); } -static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw) +static void io_req_tw_fail_links(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *link = tw_req.req; + io_tw_lock(link->ctx, tw); while (link) { struct io_kiocb *nxt = link->link; @@ -168,7 +171,7 @@ static void io_req_tw_fail_links(struct io_kiocb *link, io_tw_token_t tw) res = link->cqe.res; link->link = NULL; io_req_set_res(link, res, 0); - io_req_task_complete(link, tw); + io_req_task_complete((struct io_tw_req){link}, tw); link = nxt; } } @@ -317,14 +320,15 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) return 0; } -static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw) +static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_kiocb *prev = timeout->prev; int ret; if (prev) { - if (!io_should_terminate_tw(req->ctx)) { + if (!tw.cancel) { struct io_cancel_data cd = { .ctx = req->ctx, .data = prev->cqe.user_data, @@ -335,11 +339,11 @@ static void io_req_task_link_timeout(struct io_kiocb *req, io_tw_token_t tw) ret = -ECANCELED; } io_req_set_res(req, ret ?: -ETIME, 0); - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); io_put_req(prev); } else { io_req_set_res(req, -ETIME, 0); - io_req_task_complete(req, tw); + io_req_task_complete(tw_req, tw); } } diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index d1e3ba62ee8e8..d0fec7bb87d48 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -38,6 +38,8 @@ static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags) if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) { ioucmd->sqe = NULL; io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); + } else { + io_vec_free(&ac->vec); } } @@ -113,12 +115,13 @@ void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, } EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable); -static void io_uring_cmd_work(struct io_kiocb *req, io_tw_token_t tw) +static void io_uring_cmd_work(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); unsigned int flags = IO_URING_F_COMPLETE_DEFER; - if (io_should_terminate_tw(req->ctx)) + if (unlikely(tw.cancel)) flags |= IO_URING_F_TASK_DEAD; /* task_work executor checks the deffered list completion */ @@ -212,6 +215,8 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req); if (!ac) return -ENOMEM; + if (ac->vec.iovec) + req->flags |= REQ_F_NEED_CLEANUP; ioucmd->sqe = sqe; return 0; } @@ -261,10 +266,6 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) } ret = file->f_op->uring_cmd(ioucmd, issue_flags); - if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) { - if (ret >= 0) - return IOU_ISSUE_SKIP_COMPLETE; - } if (ret == -EAGAIN) { ioucmd->flags |= IORING_URING_CMD_REISSUE; return ret; diff --git a/io_uring/waitid.c b/io_uring/waitid.c index 921b4de3a31cb..2b756017423f0 100644 --- a/io_uring/waitid.c +++ b/io_uring/waitid.c @@ -16,7 +16,7 @@ #include "waitid.h" #include "../kernel/exit.h" -static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw); +static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw); #define IO_WAITID_CANCEL_FLAG BIT(31) #define IO_WAITID_REF_MASK GENMASK(30, 0) @@ -109,7 +109,23 @@ static int io_waitid_finish(struct io_kiocb *req, int ret) return ret; } -static void io_waitid_complete(struct io_kiocb *req, int ret) +static void io_waitid_remove_wq(struct io_kiocb *req) +{ + struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); + struct wait_queue_head *head; + + head = smp_load_acquire(&iw->head); + if (head) { + struct io_waitid_async *iwa = req->async_data; + + smp_store_release(&iw->head, NULL); + spin_lock_irq(&head->lock); + list_del_init(&iwa->wo.child_wait.entry); + spin_unlock_irq(&head->lock); + } +} + +static void io_waitid_complete(struct io_kiocb *req, int ret, bool copy_si) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); @@ -119,17 +135,22 @@ static void io_waitid_complete(struct io_kiocb *req, int ret) lockdep_assert_held(&req->ctx->uring_lock); hlist_del_init(&req->hash_node); + io_waitid_remove_wq(req); - ret = io_waitid_finish(req, ret); + if (copy_si) + ret = io_waitid_finish(req, ret); + else + io_waitid_free(req); if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); } -static bool __io_waitid_cancel(struct io_kiocb *req) +static bool __io_waitid_cancel(struct io_kiocb *req, bool copy_si) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); - struct io_waitid_async *iwa = req->async_data; + + lockdep_assert_held(&req->ctx->uring_lock); /* * Mark us canceled regardless of ownership. This will prevent a @@ -141,51 +162,65 @@ static bool __io_waitid_cancel(struct io_kiocb *req) if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return false; - spin_lock_irq(&iw->head->lock); - list_del_init(&iwa->wo.child_wait.entry); - spin_unlock_irq(&iw->head->lock); - io_waitid_complete(req, -ECANCELED); + io_waitid_complete(req, -ECANCELED, copy_si); io_req_queue_tw_complete(req, -ECANCELED); return true; } +static bool io_waitid_cancel_cb(struct io_kiocb *req) +{ + return __io_waitid_cancel(req, true); +} + +static bool io_waitid_cancel_nocopy_cb(struct io_kiocb *req) +{ + return __io_waitid_cancel(req, false); +} + int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned int issue_flags) { - return io_cancel_remove(ctx, cd, issue_flags, &ctx->waitid_list, __io_waitid_cancel); + return io_cancel_remove(ctx, cd, issue_flags, &ctx->waitid_list, io_waitid_cancel_cb); } bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { - return io_cancel_remove_all(ctx, tctx, &ctx->waitid_list, cancel_all, __io_waitid_cancel); + return io_cancel_remove_all(ctx, tctx, &ctx->waitid_list, cancel_all, + tctx ? io_waitid_cancel_cb : io_waitid_cancel_nocopy_cb); } static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); - struct io_waitid_async *iwa = req->async_data; if (!atomic_sub_return(1, &iw->refs)) return false; + io_waitid_remove_wq(req); + /* * Wakeup triggered, racing with us. It was prevented from * completing because of that, queue up the tw to do that. */ req->io_task_work.func = io_waitid_cb; io_req_task_work_add(req); - remove_wait_queue(iw->head, &iwa->wo.child_wait); return true; } -static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw) +static void io_waitid_cb(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_kiocb *req = tw_req.req; struct io_waitid_async *iwa = req->async_data; struct io_ring_ctx *ctx = req->ctx; int ret; io_tw_lock(ctx, tw); + if (unlikely(tw.cancel)) { + io_waitid_complete(req, -ECANCELED, false); + io_req_task_complete(tw_req, tw); + return; + } ret = __do_wait(&iwa->wo); @@ -209,13 +244,12 @@ static void io_waitid_cb(struct io_kiocb *req, io_tw_token_t tw) io_waitid_drop_issue_ref(req); return; } - - remove_wait_queue(iw->head, &iwa->wo.child_wait); + /* fall through to complete, will kill waitqueue */ } } - io_waitid_complete(req, ret); - io_req_task_complete(req, tw); + io_waitid_complete(req, ret, true); + io_req_task_complete(tw_req, tw); } static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, @@ -231,13 +265,14 @@ static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, return 0; list_del_init(&wait->entry); + smp_store_release(&iw->head, NULL); /* cancel is in progress */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return 1; req->io_task_work.func = io_waitid_cb; - io_req_task_work_add(req); + __io_req_task_work_add(req, IOU_F_TWQ_IN_WAKE); return 1; } @@ -257,6 +292,7 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) iw->which = READ_ONCE(sqe->len); iw->upid = READ_ONCE(sqe->fd); iw->options = READ_ONCE(sqe->file_index); + iw->head = NULL; iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2)); memset(&iw->info, 0, sizeof(iw->info)); return 0; @@ -288,11 +324,16 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags) * callback. */ io_ring_submit_lock(ctx, issue_flags); + + /* + * iw->head is valid under the ring lock, and as long as the request + * is on the waitid_list where cancelations may find it. + */ + iw->head = ¤t->signal->wait_chldexit; hlist_add_head(&req->hash_node, &ctx->waitid_list); init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait); iwa->wo.child_wait.private = req->tctx->task; - iw->head = ¤t->signal->wait_chldexit; add_wait_queue(iw->head, &iwa->wo.child_wait); ret = __do_wait(&iwa->wo); @@ -315,7 +356,7 @@ int io_waitid(struct io_kiocb *req, unsigned int issue_flags) } hlist_del_init(&req->hash_node); - remove_wait_queue(iw->head, &iwa->wo.child_wait); + io_waitid_remove_wq(req); ret = io_waitid_finish(req, ret); io_ring_submit_unlock(ctx, issue_flags); diff --git a/kernel/audit.c b/kernel/audit.c index db7868945a877..b2612f7ca6311 100644 --- a/kernel/audit.c +++ b/kernel/audit.c @@ -2030,7 +2030,7 @@ void audit_log_vformat(struct audit_buffer *ab, const char *fmt, va_list args) * here and AUDIT_BUFSIZ is at least 1024, then we can * log everything that printk could have logged. */ avail = audit_expand(ab, - max_t(unsigned, AUDIT_BUFSIZ, 1+len-avail)); + max_t(unsigned int, AUDIT_BUFSIZ, 1+len-avail)); if (!avail) goto out_va_end; len = vsnprintf(skb_tail_pointer(skb), avail, fmt, args2); @@ -2115,7 +2115,8 @@ void audit_log_n_hex(struct audit_buffer *ab, const unsigned char *buf, void audit_log_n_string(struct audit_buffer *ab, const char *string, size_t slen) { - int avail, new_len; + int avail; + size_t new_len; unsigned char *ptr; struct sk_buff *skb; @@ -2125,7 +2126,13 @@ void audit_log_n_string(struct audit_buffer *ab, const char *string, BUG_ON(!ab->skb); skb = ab->skb; avail = skb_tailroom(skb); - new_len = slen + 3; /* enclosing quotes + null terminator */ + + /* enclosing quotes + null terminator */ + if (check_add_overflow(slen, 3, &new_len)) { + audit_log_format(ab, "?"); + return; + } + if (new_len > avail) { avail = audit_expand(ab, new_len); if (!avail) diff --git a/kernel/audit.h b/kernel/audit.h index 0f05933a173be..08bd61f431b20 100644 --- a/kernel/audit.h +++ b/kernel/audit.h @@ -76,7 +76,7 @@ struct audit_names { int name_len; /* number of chars to log */ bool hidden; /* don't log this record */ - unsigned long ino; + u64 ino; dev_t dev; umode_t mode; kuid_t uid; @@ -225,15 +225,15 @@ extern int auditd_test_task(struct task_struct *task); #define AUDIT_INODE_BUCKETS 32 extern struct list_head audit_inode_hash[AUDIT_INODE_BUCKETS]; -static inline int audit_hash_ino(u32 ino) +static inline int audit_hash_ino(u64 ino) { - return (ino & (AUDIT_INODE_BUCKETS-1)); + return ((u32)ino & (AUDIT_INODE_BUCKETS-1)); } /* Indicates that audit should log the full pathname. */ #define AUDIT_NAME_FULL -1 -extern int audit_match_class(int class, unsigned syscall); +extern int audit_match_class(int class, unsigned int syscall); extern int audit_comparator(const u32 left, const u32 op, const u32 right); extern int audit_uid_comparator(kuid_t left, u32 op, kuid_t right); extern int audit_gid_comparator(kgid_t left, u32 op, kgid_t right); @@ -256,8 +256,13 @@ extern int audit_del_rule(struct audit_entry *entry); extern void audit_free_rule_rcu(struct rcu_head *head); extern struct list_head audit_filter_list[]; -extern struct audit_entry *audit_dupe_rule(struct audit_krule *old); +struct audit_watch_ctx { + struct inode *dir; + struct inode *child; +}; +extern struct audit_entry *audit_dupe_rule(struct audit_krule *old, + struct audit_watch_ctx *ctx); extern void audit_log_d_path_exe(struct audit_buffer *ab, struct mm_struct *mm); @@ -277,17 +282,18 @@ extern int audit_to_watch(struct audit_krule *krule, char *path, int len, extern int audit_add_watch(struct audit_krule *krule, struct list_head **list); extern void audit_remove_watch_rule(struct audit_krule *krule); extern char *audit_watch_path(struct audit_watch *watch); -extern int audit_watch_compare(struct audit_watch *watch, unsigned long ino, - dev_t dev); +extern int audit_watch_compare(struct audit_watch *watch, u64 ino, dev_t dev); extern struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, - char *pathname, int len); + char *pathname, int len, + struct audit_watch_ctx *ctx); extern char *audit_mark_path(struct audit_fsnotify_mark *mark); extern void audit_remove_mark(struct audit_fsnotify_mark *audit_mark); extern void audit_remove_mark_rule(struct audit_krule *krule); -extern int audit_mark_compare(struct audit_fsnotify_mark *mark, - unsigned long ino, dev_t dev); -extern int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old); +extern int audit_mark_compare(struct audit_fsnotify_mark *mark, u64 ino, + dev_t dev); +extern int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old, + struct audit_watch_ctx *ctx); extern int audit_exe_compare(struct task_struct *tsk, struct audit_fsnotify_mark *mark); @@ -318,13 +324,13 @@ extern struct list_head *audit_killed_trees(void); #define audit_watch_path(w) "" #define audit_watch_compare(w, i, d) 0 -#define audit_alloc_mark(k, p, l) (ERR_PTR(-EINVAL)) +#define audit_alloc_mark(k, p, l, c) (ERR_PTR(-EINVAL)) #define audit_mark_path(m) "" #define audit_remove_mark(m) do { } while (0) #define audit_remove_mark_rule(k) do { } while (0) #define audit_mark_compare(m, i, d) 0 #define audit_exe_compare(t, m) (-EINVAL) -#define audit_dupe_exe(n, o) (-EINVAL) +#define audit_dupe_exe(n, o, c) (-EINVAL) #define audit_remove_tree_rule(rule) BUG() #define audit_add_tree_rule(rule) -EINVAL diff --git a/kernel/audit_fsnotify.c b/kernel/audit_fsnotify.c index 0046f93ae9144..e62066bf0b0af 100644 --- a/kernel/audit_fsnotify.c +++ b/kernel/audit_fsnotify.c @@ -25,7 +25,7 @@ */ struct audit_fsnotify_mark { dev_t dev; /* associated superblock device */ - unsigned long ino; /* associated inode number */ + u64 ino; /* associated inode number */ char *path; /* insertion path */ struct fsnotify_mark mark; /* fsnotify mark on the inode */ struct audit_krule *rule; @@ -57,7 +57,7 @@ char *audit_mark_path(struct audit_fsnotify_mark *mark) return mark->path; } -int audit_mark_compare(struct audit_fsnotify_mark *mark, unsigned long ino, dev_t dev) +int audit_mark_compare(struct audit_fsnotify_mark *mark, u64 ino, dev_t dev) { if (mark->ino == AUDIT_INO_UNSET) return 0; @@ -71,19 +71,30 @@ static void audit_update_mark(struct audit_fsnotify_mark *audit_mark, audit_mark->ino = inode ? inode->i_ino : AUDIT_INO_UNSET; } -struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pathname, int len) +struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pathname, + int len, struct audit_watch_ctx *ctx) { struct audit_fsnotify_mark *audit_mark; struct path path; struct dentry *dentry; - int ret; + struct inode *dir, *child; + int ret, allow_dups; if (pathname[0] != '/' || pathname[len-1] == '/') return ERR_PTR(-EINVAL); - dentry = kern_path_parent(pathname, &path); - if (IS_ERR(dentry)) - return ERR_CAST(dentry); /* returning an error */ + if (!ctx) { + dentry = kern_path_parent(pathname, &path); + if (IS_ERR(dentry)) + return ERR_CAST(dentry); /* returning an error */ + dir = d_inode(path.dentry); + child = d_inode(dentry); + allow_dups = 0; + } else { + dir = ctx->dir; + child = ctx->child; + allow_dups = 1; + } audit_mark = kzalloc(sizeof(*audit_mark), GFP_KERNEL); if (unlikely(!audit_mark)) { @@ -94,18 +105,21 @@ struct audit_fsnotify_mark *audit_alloc_mark(struct audit_krule *krule, char *pa fsnotify_init_mark(&audit_mark->mark, audit_fsnotify_group); audit_mark->mark.mask = AUDIT_FS_EVENTS; audit_mark->path = pathname; - audit_update_mark(audit_mark, dentry->d_inode); audit_mark->rule = krule; - ret = fsnotify_add_inode_mark(&audit_mark->mark, path.dentry->d_inode, 0); + audit_update_mark(audit_mark, child); + ret = fsnotify_add_inode_mark(&audit_mark->mark, dir, allow_dups); + if (ret < 0) { audit_mark->path = NULL; fsnotify_put_mark(&audit_mark->mark); audit_mark = ERR_PTR(ret); } out: - dput(dentry); - path_put(&path); + if (!ctx) { + dput(dentry); + path_put(&path); + } return audit_mark; } diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index fda6beb041e05..f2c13a62bfb63 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -33,7 +33,7 @@ struct audit_chunk { struct audit_node { struct list_head list; struct audit_tree *owner; - unsigned index; /* index; upper bit indicates 'will prune' */ + unsigned int index; /* index; upper bit indicates 'will prune' */ } owners[] __counted_by(count); }; diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c index a700e3c8925ff..ec88f38ee685b 100644 --- a/kernel/audit_watch.c +++ b/kernel/audit_watch.c @@ -37,7 +37,7 @@ struct audit_watch { refcount_t count; /* reference count */ dev_t dev; /* associated superblock device */ char *path; /* insertion path */ - unsigned long ino; /* associated inode number */ + u64 ino; /* associated inode number */ struct audit_parent *parent; /* associated parent */ struct list_head wlist; /* entry in parent->watches list */ struct list_head rules; /* anchor for krule->rlist */ @@ -125,7 +125,7 @@ char *audit_watch_path(struct audit_watch *watch) return watch->path; } -int audit_watch_compare(struct audit_watch *watch, unsigned long ino, dev_t dev) +int audit_watch_compare(struct audit_watch *watch, u64 ino, dev_t dev) { return (watch->ino != AUDIT_INO_UNSET) && (watch->ino == ino) && @@ -244,7 +244,8 @@ static void audit_watch_log_rule_change(struct audit_krule *r, struct audit_watc /* Update inode info in audit rules based on filesystem event. */ static void audit_update_watch(struct audit_parent *parent, const struct qstr *dname, dev_t dev, - unsigned long ino, unsigned invalidating) + u64 ino, unsigned int invalidating, + struct audit_watch_ctx *ctx) { struct audit_watch *owatch, *nwatch, *nextw; struct audit_krule *r, *nextr; @@ -280,12 +281,12 @@ static void audit_update_watch(struct audit_parent *parent, list_del(&oentry->rule.rlist); list_del_rcu(&oentry->list); - nentry = audit_dupe_rule(&oentry->rule); + nentry = audit_dupe_rule(&oentry->rule, ctx); if (IS_ERR(nentry)) { list_del(&oentry->rule.list); audit_panic("error updating watch, removing"); } else { - int h = audit_hash_ino((u32)ino); + int h = audit_hash_ino(ino); /* * nentry->rule.watch == oentry->rule.watch so @@ -439,7 +440,7 @@ int audit_add_watch(struct audit_krule *krule, struct list_head **list) audit_add_to_parent(krule, parent); - h = audit_hash_ino((u32)watch->ino); + h = audit_hash_ino(watch->ino); *list = &audit_inode_hash[h]; error: path_put(&parent_path); @@ -479,10 +480,17 @@ static int audit_watch_handle_event(struct fsnotify_mark *inode_mark, u32 mask, if (WARN_ON_ONCE(inode_mark->group != audit_watch_group)) return 0; - if (mask & (FS_CREATE|FS_MOVED_TO) && inode) - audit_update_watch(parent, dname, inode->i_sb->s_dev, inode->i_ino, 0); - else if (mask & (FS_DELETE|FS_MOVED_FROM)) - audit_update_watch(parent, dname, AUDIT_DEV_UNSET, AUDIT_INO_UNSET, 1); + if (mask & (FS_CREATE|FS_MOVED_TO) && inode) { + struct audit_watch_ctx ctx = { .dir = dir, .child = inode }; + + audit_update_watch(parent, dname, inode->i_sb->s_dev, inode->i_ino, 0, + &ctx); + } else if (mask & (FS_DELETE|FS_MOVED_FROM)) { + struct audit_watch_ctx ctx = { .dir = dir, .child = NULL }; + + audit_update_watch(parent, dname, AUDIT_DEV_UNSET, AUDIT_INO_UNSET, 1, + &ctx); + } else if (mask & (FS_DELETE_SELF|FS_UNMOUNT|FS_MOVE_SELF)) audit_remove_parent_watches(parent); @@ -505,7 +513,8 @@ static int __init audit_watch_init(void) } device_initcall(audit_watch_init); -int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old) +int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old, + struct audit_watch_ctx *ctx) { struct audit_fsnotify_mark *audit_mark; char *pathname; @@ -514,7 +523,7 @@ int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old) if (!pathname) return -ENOMEM; - audit_mark = audit_alloc_mark(new, pathname, strlen(pathname)); + audit_mark = audit_alloc_mark(new, pathname, strlen(pathname), ctx); if (IS_ERR(audit_mark)) { kfree(pathname); return PTR_ERR(audit_mark); @@ -527,7 +536,7 @@ int audit_dupe_exe(struct audit_krule *new, struct audit_krule *old) int audit_exe_compare(struct task_struct *tsk, struct audit_fsnotify_mark *mark) { struct file *exe_file; - unsigned long ino; + u64 ino; dev_t dev; /* only do exe filtering if we are recording @current events/records */ diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c index c401082d9b250..d01903b9f9651 100644 --- a/kernel/auditfilter.c +++ b/kernel/auditfilter.c @@ -165,13 +165,13 @@ static inline int audit_to_inode(struct audit_krule *krule, static __u32 *classes[AUDIT_SYSCALL_CLASSES]; -int __init audit_register_class(int class, unsigned *list) +int __init audit_register_class(int class, unsigned int *list) { __u32 *p = kcalloc(AUDIT_BITMASK_SIZE, sizeof(__u32), GFP_KERNEL); if (!p) return -ENOMEM; while (*list != ~0U) { - unsigned n = *list++; + unsigned int n = *list++; if (n >= AUDIT_BITMASK_SIZE * 32 - AUDIT_SYSCALL_CLASSES) { kfree(p); return -EINVAL; @@ -186,7 +186,7 @@ int __init audit_register_class(int class, unsigned *list) return 0; } -int audit_match_class(int class, unsigned syscall) +int audit_match_class(int class, unsigned int syscall) { if (unlikely(syscall >= AUDIT_BITMASK_SIZE * 32)) return 0; @@ -237,7 +237,7 @@ static int audit_match_signal(struct audit_entry *entry) /* Common user-space to kernel rule translation. */ static inline struct audit_entry *audit_to_entry_common(struct audit_rule_data *rule) { - unsigned listnr; + unsigned int listnr; struct audit_entry *entry; int i, err; @@ -590,7 +590,7 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data, err = PTR_ERR(str); goto exit_free; } - audit_mark = audit_alloc_mark(&entry->rule, str, f_val); + audit_mark = audit_alloc_mark(&entry->rule, str, f_val, NULL); if (IS_ERR(audit_mark)) { kfree(str); err = PTR_ERR(audit_mark); @@ -818,7 +818,8 @@ static inline int audit_dupe_lsm_field(struct audit_field *df, * rule with the new rule in the filterlist, then free the old rule. * The rlist element is undefined; list manipulations are handled apart from * the initial copy. */ -struct audit_entry *audit_dupe_rule(struct audit_krule *old) +struct audit_entry *audit_dupe_rule(struct audit_krule *old, + struct audit_watch_ctx *ctx) { u32 fcount = old->field_count; struct audit_entry *entry; @@ -877,7 +878,7 @@ struct audit_entry *audit_dupe_rule(struct audit_krule *old) new->filterkey = fk; break; case AUDIT_EXE: - err = audit_dupe_exe(new, old); + err = audit_dupe_exe(new, old, ctx); break; } if (err) { @@ -1024,7 +1025,6 @@ static inline int audit_add_rule(struct audit_entry *entry) int audit_del_rule(struct audit_entry *entry) { struct audit_entry *e; - struct audit_tree *tree = entry->rule.tree; struct list_head *list; int ret = 0; #ifdef CONFIG_AUDITSYSCALL @@ -1046,6 +1046,10 @@ int audit_del_rule(struct audit_entry *entry) goto out; } + list_del_rcu(&e->list); + list_del(&e->rule.list); + synchronize_rcu(); + if (e->rule.watch) audit_remove_watch_rule(&e->rule); @@ -1063,16 +1067,11 @@ int audit_del_rule(struct audit_entry *entry) audit_signals--; #endif - list_del_rcu(&e->list); - list_del(&e->rule.list); call_rcu(&e->rcu, audit_free_rule_rcu); out: mutex_unlock(&audit_filter_mutex); - if (tree) - audit_put_tree(tree); /* that's the temporary one */ - return ret; } @@ -1157,6 +1156,8 @@ int audit_rule_change(int type, int seq, void *data, size_t datasz) } if (err || type == AUDIT_DEL_RULE) { + if (type == AUDIT_DEL_RULE && entry->rule.tree) + audit_put_tree(entry->rule.tree); if (entry->rule.exe) audit_remove_mark(entry->rule.exe); audit_free_rule(entry); @@ -1416,7 +1417,7 @@ static int update_lsm_rule(struct audit_krule *r) if (!security_audit_rule_known(r)) return 0; - nentry = audit_dupe_rule(r); + nentry = audit_dupe_rule(r, NULL); if (entry->rule.exe) audit_remove_mark(entry->rule.exe); if (IS_ERR(nentry)) { diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 3e42d32ebb285..c547add130284 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -150,7 +150,7 @@ static const struct audit_nfcfgop_tab audit_nfcfgs[] = { static int audit_match_perm(struct audit_context *ctx, int mask) { - unsigned n; + unsigned int n; if (unlikely(!ctx)) return 0; @@ -886,7 +886,7 @@ static int audit_filter_inode_name(struct task_struct *tsk, struct audit_names *n, struct audit_context *ctx) { - int h = audit_hash_ino((u32)n->ino); + int h = audit_hash_ino(n->ino); struct list_head *list = &audit_inode_hash[h]; return __audit_filter_op(tsk, ctx, list, n, ctx->major); @@ -1534,7 +1534,7 @@ static void audit_log_name(struct audit_context *context, struct audit_names *n, audit_log_format(ab, " name=(null)"); if (n->ino != AUDIT_INO_UNSET) - audit_log_format(ab, " inode=%lu dev=%02x:%02x mode=%#ho ouid=%u ogid=%u rdev=%02x:%02x", + audit_log_format(ab, " inode=%llu dev=%02x:%02x mode=%#ho ouid=%u ogid=%u rdev=%02x:%02x", n->ino, MAJOR(n->dev), MINOR(n->dev), diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c index dafa179da0c9c..51227c8d2b1e4 100644 --- a/kernel/bpf/arena.c +++ b/kernel/bpf/arena.c @@ -534,6 +534,8 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt) uaddr = (u32)uaddr; uaddr &= PAGE_MASK; full_uaddr = clear_lo32(arena->user_vm_start) + uaddr; + if (full_uaddr < arena->user_vm_start) + return; uaddr_end = min(arena->user_vm_end, full_uaddr + (page_cnt << PAGE_SHIFT)); if (full_uaddr >= uaddr_end) return; diff --git a/kernel/bpf/bloom_filter.c b/kernel/bpf/bloom_filter.c index 35e1ddca74d21..3f55f28fd5890 100644 --- a/kernel/bpf/bloom_filter.c +++ b/kernel/bpf/bloom_filter.c @@ -41,7 +41,7 @@ static long bloom_map_peek_elem(struct bpf_map *map, void *value) for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - if (!test_bit(h, bloom->bitset)) + if (!test_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h))) return -ENOENT; } @@ -57,9 +57,13 @@ static long bloom_map_push_elem(struct bpf_map *map, void *value, u64 flags) if (flags != BPF_ANY) return -EINVAL; + /* + * On 32-bit architectures, hashes larger than INT_MAX would be + * treated as negative by set_bit(). + */ for (i = 0; i < bloom->nr_hash_funcs; i++) { h = hash(bloom, value, map->value_size, i); - set_bit(h, bloom->bitset); + set_bit(h % BITS_PER_LONG, bloom->bitset + BIT_WORD(h)); } return 0; @@ -94,9 +98,10 @@ static int bloom_map_alloc_check(union bpf_attr *attr) static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) { - u32 bitset_bytes, bitset_mask, nr_hash_funcs, nr_bits; + u32 bitset_mask, nr_hash_funcs, nr_bits; int numa_node = bpf_map_attr_numa_node(attr); struct bpf_bloom_filter *bloom; + u64 bitset_bytes; if (attr->key_size != 0 || attr->value_size == 0 || attr->max_entries == 0 || @@ -127,22 +132,16 @@ static struct bpf_map *bloom_map_alloc(union bpf_attr *attr) if (check_mul_overflow(attr->max_entries, nr_hash_funcs, &nr_bits) || check_mul_overflow(nr_bits / 5, (u32)7, &nr_bits) || nr_bits > (1UL << 31)) { - /* The bit array size is 2^32 bits but to avoid overflowing the - * u32, we use U32_MAX, which will round up to the equivalent - * number of bytes - */ - bitset_bytes = BITS_TO_BYTES(U32_MAX); bitset_mask = U32_MAX; } else { if (nr_bits <= BITS_PER_LONG) nr_bits = BITS_PER_LONG; else nr_bits = roundup_pow_of_two(nr_bits); - bitset_bytes = BITS_TO_BYTES(nr_bits); bitset_mask = nr_bits - 1; } - bitset_bytes = roundup(bitset_bytes, sizeof(unsigned long)); + bitset_bytes = BITS_TO_LONGS((u64)bitset_mask + 1) * sizeof(unsigned long); bloom = bpf_map_area_alloc(sizeof(*bloom) + bitset_bytes, numa_node); if (!bloom) diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c index 6ac35430c5734..54596447fbca8 100644 --- a/kernel/bpf/bpf_iter.c +++ b/kernel/bpf/bpf_iter.c @@ -767,7 +767,7 @@ const struct bpf_func_proto bpf_loop_proto = { .func = bpf_loop, .gpl_only = false, .ret_type = RET_INTEGER, - .arg1_type = ARG_ANYTHING, + .arg1_type = ARG_SCALAR, .arg2_type = ARG_PTR_TO_FUNC, .arg3_type = ARG_PTR_TO_STACK_OR_NULL, .arg4_type = ARG_ANYTHING, diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c index 518c933fe944c..48876fe838ab9 100644 --- a/kernel/bpf/bpf_lsm.c +++ b/kernel/bpf/bpf_lsm.c @@ -293,7 +293,6 @@ BTF_ID(func, bpf_lsm_bpf_map_create) BTF_ID(func, bpf_lsm_bpf_map_free) BTF_ID(func, bpf_lsm_bpf_prog) BTF_ID(func, bpf_lsm_bpf_prog_load) -BTF_ID(func, bpf_lsm_bpf_prog_free) BTF_ID(func, bpf_lsm_bpf_token_create) BTF_ID(func, bpf_lsm_bpf_token_free) BTF_ID(func, bpf_lsm_bpf_token_cmd) diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 7a4618c054075..f92e1c16c3569 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -2795,14 +2795,29 @@ static void btf_modifier_show(const struct btf *btf, else t = btf_type_skip_modifiers(btf, type_id, NULL); - btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); + /* + * A modifier can resolve to void, which has no show op; print a + * placeholder rather than dereferencing NULL. + */ + if (!btf_type_ops(t)) + btf_df_show(btf, t, type_id, data, bits_offset, show); + else + btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } static void btf_var_show(const struct btf *btf, const struct btf_type *t, u32 type_id, void *data, u8 bits_offset, struct btf_show *show) { - t = btf_type_id_resolve(btf, &type_id); + /* + * btf_type_id_resolve() dereferences btf->resolved_ids, which is NULL + * for a base BTF (e.g. the vmlinux BTF that bpf_snprintf_btf() uses). + * Resolve the var's type directly in that case. + */ + if (btf->resolved_ids) + t = btf_type_id_resolve(btf, &type_id); + else + t = btf_type_skip_modifiers(btf, t->type, &type_id); btf_type_ops(t)->show(btf, t, type_id, data, bits_offset, show); } @@ -3504,7 +3519,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_ { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, { BPF_RB_NODE, "bpf_rb_node", false }, - { BPF_REFCOUNT, "bpf_refcount", false }, + { BPF_REFCOUNT, "bpf_refcount", true }, }; int type = 0, i; const char *name = __btf_name_by_offset(btf, var_type->name_off); @@ -3586,7 +3601,7 @@ static int btf_repeat_fields(struct btf_field_info *info, int info_cnt, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level); + u32 level, u32 *seen_mask); /* Find special fields in the struct type of a field. * @@ -3597,7 +3612,7 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_nested_struct(const struct btf *btf, const struct btf_type *t, u32 off, u32 nelems, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, err, i; @@ -3605,7 +3620,7 @@ static int btf_find_nested_struct(const struct btf *btf, const struct btf_type * if (level >= MAX_RESOLVE_DEPTH) return -E2BIG; - ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level); + ret = btf_find_struct_field(btf, t, field_mask, info, info_cnt, level, seen_mask); if (ret <= 0) return ret; @@ -3662,7 +3677,7 @@ static int btf_find_field_one(const struct btf *btf, if (expected_size && expected_size != sz * nelems) return 0; ret = btf_find_nested_struct(btf, var_type, off, nelems, field_mask, - &info[0], info_cnt, level); + &info[0], info_cnt, level, seen_mask); return ret; } @@ -3727,11 +3742,11 @@ static int btf_find_field_one(const struct btf *btf, static int btf_find_struct_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt, - u32 level) + u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_member *member; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_member(i, t, member) { const struct btf_type *member_type = btf_type_by_id(btf, @@ -3745,7 +3760,7 @@ static int btf_find_struct_field(const struct btf *btf, ret = btf_find_field_one(btf, t, member_type, i, off, 0, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) return ret; @@ -3756,11 +3771,11 @@ static int btf_find_struct_field(const struct btf *btf, static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, - int info_cnt, u32 level) + int info_cnt, u32 level, u32 *seen_mask) { int ret, idx = 0; const struct btf_var_secinfo *vsi; - u32 i, off, seen_mask = 0; + u32 i, off; for_each_vsi(i, t, vsi) { const struct btf_type *var = btf_type_by_id(btf, vsi->type); @@ -3768,7 +3783,7 @@ static int btf_find_datasec_var(const struct btf *btf, const struct btf_type *t, off = vsi->offset; ret = btf_find_field_one(btf, var, var_type, -1, off, vsi->size, - field_mask, &seen_mask, + field_mask, seen_mask, &info[idx], info_cnt - idx, level); if (ret < 0) @@ -3782,10 +3797,12 @@ static int btf_find_field(const struct btf *btf, const struct btf_type *t, u32 field_mask, struct btf_field_info *info, int info_cnt) { + u32 seen_mask = 0; + if (__btf_type_is_struct(t)) - return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0); + return btf_find_struct_field(btf, t, field_mask, info, info_cnt, 0, &seen_mask); else if (btf_type_is_datasec(t)) - return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0); + return btf_find_datasec_var(btf, t, field_mask, info, info_cnt, 0, &seen_mask); return -EINVAL; } @@ -4003,7 +4020,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type rec->spin_lock_off = rec->fields[i].offset; break; case BPF_RES_SPIN_LOCK: - WARN_ON_ONCE(rec->spin_lock_off >= 0); + WARN_ON_ONCE(rec->res_spin_lock_off >= 0); /* Cache offset for faster lookup at runtime */ rec->res_spin_lock_off = rec->fields[i].offset; break; @@ -6477,6 +6494,10 @@ struct bpf_raw_tp_null_args { static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { /* sched */ { "sched_pi_setprio", 0x10 }, + /* + * do_wait() passes NULL for wait4(-1) and waitid(P_ALL). + */ + { "sched_process_wait", 0x1 }, /* ... from sched_numa_pair_template event class */ { "sched_stick_numa", 0x100 }, { "sched_swap_numa", 0x100 }, @@ -6544,6 +6565,9 @@ static const struct bpf_raw_tp_null_args raw_tp_null_args[] = { { "rxrpc_resend", 0x10 }, { "rxrpc_tq", 0x10 }, { "rxrpc_client", 0x1 }, + /* signal */ + { "signal_generate", 0x20 }, + { "signal_deliver", 0x20 }, /* skb */ {"kfree_skb", 0x1000}, /* sunrpc */ @@ -8406,6 +8430,7 @@ BPF_CALL_4(bpf_btf_find_by_name_kind, char *, name, int, name_sz, u32, kind, int const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .func = bpf_btf_find_by_name_kind, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_MEM | MEM_RDONLY, .arg2_type = ARG_CONST_SIZE, diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c index fc2f44b60a551..c652ba01bc8bd 100644 --- a/kernel/bpf/cgroup.c +++ b/kernel/bpf/cgroup.c @@ -813,8 +813,10 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, struct bpf_prog *old_prog = NULL; struct bpf_cgroup_storage *storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_cgroup_storage *new_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; + struct bpf_cgroup_storage *old_storage[MAX_BPF_CGROUP_STORAGE_TYPE] = {}; struct bpf_prog *new_prog = prog ? : link->link.prog; enum cgroup_bpf_attach_type atype; + u32 old_flags, old_pl_flags; struct bpf_prog_list *pl; struct hlist_head *progs; int err; @@ -865,6 +867,8 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (pl) { old_prog = pl->prog; + old_pl_flags = pl->flags; + bpf_cgroup_storages_assign(old_storage, pl->storage); } else { pl = kmalloc(sizeof(*pl), GFP_KERNEL); if (!pl) { @@ -884,6 +888,7 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, pl->link = link; pl->flags = flags; bpf_cgroup_storages_assign(pl->storage, storage); + old_flags = cgrp->bpf.flags[atype]; cgrp->bpf.flags[atype] = saved_flags; if (type == BPF_LSM_CGROUP) { @@ -915,12 +920,15 @@ static int __cgroup_bpf_attach(struct cgroup *cgrp, if (old_prog) { pl->prog = old_prog; pl->link = NULL; + pl->flags = old_pl_flags; + bpf_cgroup_storages_assign(pl->storage, old_storage); } bpf_cgroup_storages_free(new_storage); if (!old_prog) { hlist_del(&pl->node); kfree(pl); } + cgrp->bpf.flags[atype] = old_flags; return err; } @@ -1018,6 +1026,20 @@ static void replace_effective_prog(struct cgroup *cgrp, } } +static bool cgroup_bpf_storages_compatible(struct bpf_prog *old_prog, + struct bpf_prog *new_prog) +{ + enum bpf_cgroup_storage_type stype; + + for_each_cgroup_storage_type(stype) { + if (old_prog->aux->cgroup_storage[stype] != + new_prog->aux->cgroup_storage[stype]) + return false; + } + + return true; +} + /** * __cgroup_bpf_replace() - Replace link's program and propagate the change * to descendants @@ -1056,6 +1078,9 @@ static int __cgroup_bpf_replace(struct cgroup *cgrp, if (!found) return -ENOENT; + if (!cgroup_bpf_storages_compatible(link->link.prog, new_prog)) + return -EINVAL; + cgrp->bpf.revisions[atype] += 1; old_prog = xchg(&link->link.prog, new_prog); replace_effective_prog(cgrp, atype, pl); @@ -2239,7 +2264,7 @@ int __cgroup_bpf_run_filter_getsockopt_kern(struct sock *sk, int level, if (ret < 0) return ret; - if (ctx.optlen > *optlen) + if (ctx.optlen > *optlen || ctx.optlen < 0) return -EFAULT; /* BPF programs can shrink the buffer, export the modifications. @@ -2384,6 +2409,7 @@ BPF_CALL_3(bpf_sysctl_set_new_value, struct bpf_sysctl_kern *, ctx, return -E2BIG; memcpy(ctx->new_val, buf, buf_len); + ((char *)ctx->new_val)[buf_len] = '\0'; ctx->new_len = buf_len; ctx->new_updated = 1; diff --git a/kernel/bpf/cpumask.c b/kernel/bpf/cpumask.c index 9876c5fe6c2a2..341f80bb72ff2 100644 --- a/kernel/bpf/cpumask.c +++ b/kernel/bpf/cpumask.c @@ -449,12 +449,12 @@ __bpf_kfunc u32 bpf_cpumask_weight(const struct cpumask *cpumask) * @src__sz: Length of the BPF memory region in bytes. * * Return: - * * 0 if the struct cpumask * instance was populated successfully. + * * 0 if the struct bpf_cpumask * instance was populated successfully. * * -EACCES if the memory region is too small to populate the cpumask. * * -EINVAL if the memory region is not aligned to the size of a long * and the architecture does not support efficient unaligned accesses. */ -__bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t src__sz) +__bpf_kfunc int bpf_cpumask_populate(struct bpf_cpumask *cpumask, void *src, size_t src__sz) { unsigned long source = (unsigned long)src; @@ -467,7 +467,7 @@ __bpf_kfunc int bpf_cpumask_populate(struct cpumask *cpumask, void *src, size_t !IS_ALIGNED(source, sizeof(long))) return -EINVAL; - bitmap_copy(cpumask_bits(cpumask), src, nr_cpu_ids); + bitmap_copy(cpumask_bits(&cpumask->cpumask), src, nr_cpu_ids); return 0; } diff --git a/kernel/bpf/disasm.c b/kernel/bpf/disasm.c index 20883c6b1546c..f52c174f2dbb1 100644 --- a/kernel/bpf/disasm.c +++ b/kernel/bpf/disasm.c @@ -323,7 +323,10 @@ void print_bpf_insn(const struct bpf_insn_cbs *cbs, */ u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; bool is_ptr = insn->src_reg == BPF_PSEUDO_MAP_FD || - insn->src_reg == BPF_PSEUDO_MAP_VALUE; + insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE || + insn->src_reg == BPF_PSEUDO_BTF_ID; char tmp[64]; if (is_ptr && !allow_ptr_leaks) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 5aed5659822f1..72757f7290ec2 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3960,7 +3960,7 @@ struct bpf_task_work_ctx { struct bpf_map *map; void *map_val; enum task_work_notify_mode mode; - bpf_task_work_callback_t callback_fn; + bpf_callback_t callback_fn; struct rcu_head rcu; } __aligned(8); @@ -4035,7 +4035,8 @@ static void bpf_task_work_callback(struct callback_head *cb) key = (void *)map_key_from_value(ctx->map, ctx->map_val, &idx); migrate_disable(); - ctx->callback_fn(ctx->map, key, ctx->map_val); + ctx->callback_fn((u64)(long)ctx->map, (u64)(long)key, + (u64)(long)ctx->map_val, 0, 0); migrate_enable(); bpf_task_work_ctx_reset(ctx); @@ -4148,7 +4149,7 @@ static struct bpf_task_work_ctx *bpf_task_work_acquire_ctx(struct bpf_task_work } static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work *tw, - struct bpf_map *map, bpf_task_work_callback_t callback_fn, + struct bpf_map *map, void *callback_fn, struct bpf_prog_aux *aux, enum task_work_notify_mode mode) { struct bpf_prog *prog; @@ -4173,7 +4174,7 @@ static int bpf_task_work_schedule(struct task_struct *task, struct bpf_task_work } ctx->task = task; - ctx->callback_fn = callback_fn; + ctx->callback_fn = (bpf_callback_t)callback_fn; ctx->prog = prog; ctx->mode = mode; ctx->map = map; diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c index f50533169cc34..a05ef527a0ad8 100644 --- a/kernel/bpf/log.c +++ b/kernel/bpf/log.c @@ -842,7 +842,7 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie mark_verifier_state_clean(env); } -static inline u32 vlog_alignment(u32 pos) +u32 bpf_vlog_alignment(u32 pos) { return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT), BPF_LOG_MIN_ALIGNMENT) - pos - 1; @@ -854,7 +854,7 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) { /* remove new line character */ bpf_vlog_reset(&env->log, env->prev_log_pos - 1); - verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_pos), ' '); + verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' '); } else { verbose(env, "%d:", env->insn_idx); } diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h index 5d18d7d85bef9..1834db20b8612 100644 --- a/kernel/bpf/mmap_unlock_work.h +++ b/kernel/bpf/mmap_unlock_work.h @@ -4,12 +4,15 @@ #ifndef __MMAP_UNLOCK_WORK_H__ #define __MMAP_UNLOCK_WORK_H__ +#include +#include #include /* irq_work to run mmap_read_unlock() in irq_work */ struct mmap_unlock_irq_work { struct irq_work irq_work; struct mm_struct *mm; + atomic_t active; }; DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); @@ -18,32 +21,36 @@ DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); * We cannot do mmap_read_unlock() when the irq is disabled, because of * risk to deadlock with rq_lock. To look up vma when the irqs are * disabled, we need to run mmap_read_unlock() in irq_work. We use a - * percpu variable to do the irq_work. If the irq_work is already used - * by another lookup, we fall over. + * percpu variable to do the irq_work. The active flag reserves the slot + * before mmap_read_trylock() and until the irq_work callback consumes mm. */ -static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +static inline struct mmap_unlock_irq_work *bpf_mmap_unlock_guard_get(void) { - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = false; + struct mmap_unlock_irq_work *work; - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&mmap_unlock_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } + if (!irqs_disabled()) + return NULL; + + /* + * PREEMPT_RT does not allow to trylock mmap sem in interrupt + * disabled context. Force the fallback code. + */ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) + return ERR_PTR(-EBUSY); + + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work) || + atomic_cmpxchg_acquire(&work->active, 0, 1)) + return ERR_PTR(-EBUSY); - *work_ptr = work; - return irq_work_busy; + return work; +} + +static inline void +bpf_mmap_unlock_guard_put(struct mmap_unlock_irq_work *work) +{ + if (work) + atomic_set_release(&work->active, 0); } static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) diff --git a/kernel/bpf/net_namespace.c b/kernel/bpf/net_namespace.c index 8e88201c98bfe..6599d771e4812 100644 --- a/kernel/bpf/net_namespace.c +++ b/kernel/bpf/net_namespace.c @@ -171,33 +171,28 @@ static int bpf_netns_link_update_prog(struct bpf_link *link, struct net *net; int idx, ret; + guard(mutex)(&netns_bpf_mutex); + if (old_prog && old_prog != link->prog) return -EPERM; if (new_prog->type != link->prog->type) return -EINVAL; - mutex_lock(&netns_bpf_mutex); - net = net_link->net; - if (!net || !check_net(net)) { + if (!net || !check_net(net)) /* Link auto-detached or netns dying */ - ret = -ENOLINK; - goto out_unlock; - } + return -ENOLINK; run_array = rcu_dereference_protected(net->bpf.run_array[type], lockdep_is_held(&netns_bpf_mutex)); idx = link_index(net, type, net_link); ret = bpf_prog_array_update_at(run_array, idx, new_prog); if (ret) - goto out_unlock; + return ret; old_prog = xchg(&link->prog, new_prog); bpf_prog_put(old_prog); - -out_unlock: - mutex_unlock(&netns_bpf_mutex); - return ret; + return 0; } static int bpf_netns_link_fill_info(const struct bpf_link *link, diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c index 42ae8d595c2c2..a9a036547c960 100644 --- a/kernel/bpf/offload.c +++ b/kernel/bpf/offload.c @@ -445,9 +445,8 @@ static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data) if (aux->offload) { args->info->ifindex = aux->offload->netdev->ifindex; - net = dev_net(aux->offload->netdev); - get_net(net); - ns = &net->ns; + net = maybe_get_net(dev_net(aux->offload->netdev)); + ns = net ? &net->ns : NULL; } else { args->info->ifindex = 0; ns = NULL; @@ -657,9 +656,8 @@ static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data) if (args->offmap->netdev) { args->info->ifindex = args->offmap->netdev->ifindex; - net = dev_net(args->offmap->netdev); - get_net(net); - ns = &net->ns; + net = maybe_get_net(dev_net(args->offmap->netdev)); + ns = net ? &net->ns : NULL; } else { args->info->ifindex = 0; ns = NULL; diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c index 632762b572998..06ce588d13a39 100644 --- a/kernel/bpf/percpu_freelist.c +++ b/kernel/bpf/percpu_freelist.c @@ -17,6 +17,8 @@ int pcpu_freelist_init(struct pcpu_freelist *s) raw_res_spin_lock_init(&head->lock); head->first = NULL; } + raw_res_spin_lock_init(&s->extralist.lock); + s->extralist.first = NULL; return 0; } @@ -46,22 +48,28 @@ void __pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { struct pcpu_freelist_head *head; - int cpu; + int cpu, this_cpu; if (___pcpu_freelist_push(this_cpu_ptr(s->freelist), node)) return; + this_cpu = raw_smp_processor_id(); while (true) { - for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { - if (cpu == raw_smp_processor_id()) + for_each_cpu_wrap(cpu, cpu_possible_mask, this_cpu) { + if (cpu == this_cpu) continue; + head = per_cpu_ptr(s->freelist, cpu); - if (raw_res_spin_lock(&head->lock)) - continue; - pcpu_freelist_push_node(head, node); - raw_res_spin_unlock(&head->lock); - return; + if (___pcpu_freelist_push(head, node)) + return; } + + /* + * Push cannot fail. Use the extra list when none of the + * per-CPU freelists can accept the node. + */ + if (___pcpu_freelist_push(&s->extralist, node)) + return; } } @@ -117,6 +125,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) } raw_res_spin_unlock(&head->lock); } + + /* Per-CPU lists are empty or unavailable, try the extra list. */ + head = &s->extralist; + if (!READ_ONCE(head->first)) + return NULL; + if (raw_res_spin_lock(&head->lock)) + return NULL; + node = head->first; + if (node) + WRITE_ONCE(head->first, node->next); + raw_res_spin_unlock(&head->lock); return node; } diff --git a/kernel/bpf/percpu_freelist.h b/kernel/bpf/percpu_freelist.h index 914798b749672..980cf2884fd27 100644 --- a/kernel/bpf/percpu_freelist.h +++ b/kernel/bpf/percpu_freelist.h @@ -14,6 +14,7 @@ struct pcpu_freelist_head { struct pcpu_freelist { struct pcpu_freelist_head __percpu *freelist; + struct pcpu_freelist_head extralist; }; struct pcpu_freelist_node { diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c index 9a5f94371e506..c1c9dee4dcdd0 100644 --- a/kernel/bpf/queue_stack_maps.c +++ b/kernel/bpf/queue_stack_maps.c @@ -99,8 +99,10 @@ static long __queue_map_get(struct bpf_map *map, void *value, bool delete) int err = 0; void *ptr; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); @@ -130,8 +132,10 @@ static long __stack_map_get(struct bpf_map *map, void *value, bool delete) void *ptr; u32 index; - if (raw_res_spin_lock_irqsave(&qs->lock, flags)) + if (raw_res_spin_lock_irqsave(&qs->lock, flags)) { + memset(value, 0, qs->map.value_size); return -EBUSY; + } if (queue_stack_map_is_empty(qs)) { memset(value, 0, qs->map.value_size); diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c index d706c4b7f532d..bdc524c8ed798 100644 --- a/kernel/bpf/ringbuf.c +++ b/kernel/bpf/ringbuf.c @@ -426,7 +426,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size) prod_pos = rb->producer_pos; new_prod_pos = prod_pos + len; - while (pend_pos < prod_pos) { + while (prod_pos - pend_pos > 0) { hdr = (void *)rb->data + (pend_pos & rb->mask); hdr_len = READ_ONCE(hdr->len); if (hdr_len & BPF_RINGBUF_BUSY_BIT) diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 2365541c81dd1..c479f649596f9 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "percpu_freelist.h" #include "mmap_unlock_work.h" @@ -152,6 +153,131 @@ static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, b : build_id_parse_nofault(vma, build_id, NULL); } +static inline void stack_map_build_id_set_ip(struct bpf_stack_build_id *id) +{ + id->status = BPF_STACK_BUILD_ID_IP; + memset(id->build_id, 0, BUILD_ID_SIZE_MAX); +} + +static inline u64 stack_map_build_id_offset(unsigned long vm_pgoff, + unsigned long vm_start, u64 ip) +{ + return (vm_pgoff << PAGE_SHIFT) + ip - vm_start; +} + +static inline void stack_map_build_id_set_valid(struct bpf_stack_build_id *id, + u64 offset, + const unsigned char *build_id) +{ + id->status = BPF_STACK_BUILD_ID_VALID; + id->offset = offset; + if (id->build_id != build_id) + memcpy(id->build_id, build_id, BUILD_ID_SIZE_MAX); +} + +struct stack_map_vma_lock { + struct vm_area_struct *vma; + struct mm_struct *mm; +}; + +/* + * Acquire a stable read-side reference on the VMA covering @ip. + * + * With CONFIG_PER_VMA_LOCK=y this returns a VMA with its per-VMA read + * lock held and mmap_lock dropped, so the caller may sleep. + * + * With CONFIG_PER_VMA_LOCK=n it returns a VMA with mmap_lock still + * held; the caller must snapshot any fields it needs and pin vm_file + * with get_file() before stack_map_unlock_vma() drops mmap_lock, as + * the VMA may be split, merged, or freed after that. + * + * Returns NULL on failure, in which case no lock is held. + */ +static struct vm_area_struct * +stack_map_lock_vma(struct stack_map_vma_lock *lock, unsigned long ip) +{ + struct mm_struct *mm = lock->mm; + struct vm_area_struct *vma; + + /* noop under !CONFIG_PER_VMA_LOCK */ + vma = lock_vma_under_rcu(mm, ip); + if (vma) { + lock->vma = vma; + return vma; + } + + /* + * Taking mmap_read_lock() is unsafe here, because the caller BPF + * program might already hold it, causing a deadlock. + */ + if (!mmap_read_trylock(mm)) + return NULL; + + vma = vma_lookup(mm, ip); + if (!vma) { + mmap_read_unlock(mm); + return NULL; + } + +#ifdef CONFIG_PER_VMA_LOCK + if (!vma_start_read_locked(vma)) { + mmap_read_unlock(mm); + return NULL; + } + mmap_read_unlock(mm); +#endif + + lock->vma = vma; + return vma; +} + +static void stack_map_unlock_vma(struct stack_map_vma_lock *lock) +{ +#ifdef CONFIG_PER_VMA_LOCK + vma_end_read(lock->vma); +#else + mmap_read_unlock(lock->mm); +#endif + lock->vma = NULL; +} + +static void stack_map_get_build_id_offset_sleepable(struct bpf_stack_build_id *id_offs, + u32 trace_nr) +{ + struct mm_struct *mm = current->mm; + struct stack_map_vma_lock lock = { .mm = mm }; + struct vm_area_struct *vma; + struct file *file; + u64 offset; + u64 ip; + + for (u32 i = 0; i < trace_nr; i++) { + ip = READ_ONCE(id_offs[i].ip); + + vma = stack_map_lock_vma(&lock, ip); + if (!vma) { + stack_map_build_id_set_ip(&id_offs[i]); + continue; + } + if (vma_is_anonymous(vma) || !vma->vm_file) { + stack_map_build_id_set_ip(&id_offs[i]); + stack_map_unlock_vma(&lock); + continue; + } + + file = get_file(vma->vm_file); + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_unlock_vma(&lock); + + /* build_id_parse_file() may block on filesystem reads */ + if (build_id_parse_file(file, id_offs[i].build_id, NULL)) + stack_map_build_id_set_ip(&id_offs[i]); + else + stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); + fput(file); + } +} + /* * Expects all id_offs[i].ip values to be set to correct initial IPs. * They will be subsequently: @@ -165,48 +291,60 @@ static int fetch_build_id(struct vm_area_struct *vma, unsigned char *build_id, b static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u32 trace_nr, bool user, bool may_fault) { - int i; - struct mmap_unlock_irq_work *work = NULL; - bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + struct mmap_unlock_irq_work *work; + bool has_user_ctx = user && current && current->mm; struct vm_area_struct *vma, *prev_vma = NULL; - const char *prev_build_id; + const unsigned char *prev_build_id = NULL; + int i; - /* If the irq_work is in use, fall back to report ips. Same - * fallback is used for kernel stack (!user) on a stackmap with - * build_id. - */ - if (!user || !current || !current->mm || irq_work_busy || - !mmap_read_trylock(current->mm)) { - /* cannot access current->mm, fall back to ips */ - for (i = 0; i < trace_nr; i++) { - id_offs[i].status = BPF_STACK_BUILD_ID_IP; - memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); - } + if (may_fault && has_user_ctx) { + stack_map_get_build_id_offset_sleepable(id_offs, trace_nr); return; } + if (!has_user_ctx) + goto fallback; + + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) + goto fallback; + + if (!mmap_read_trylock(current->mm)) { + bpf_mmap_unlock_guard_put(work); + goto fallback; + } + for (i = 0; i < trace_nr; i++) { u64 ip = READ_ONCE(id_offs[i].ip); + u64 offset; - if (range_in_vma(prev_vma, ip, ip)) { + if (prev_build_id && range_in_vma(prev_vma, ip, ip)) { vma = prev_vma; - memcpy(id_offs[i].build_id, prev_build_id, BUILD_ID_SIZE_MAX); - goto build_id_valid; + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_build_id_set_valid(&id_offs[i], offset, prev_build_id); + continue; } vma = find_vma(current->mm, ip); - if (!vma || fetch_build_id(vma, id_offs[i].build_id, may_fault)) { + if (!vma || vma_is_anonymous(vma) || + fetch_build_id(vma, id_offs[i].build_id, may_fault)) { /* per entry fall back to ips */ - id_offs[i].status = BPF_STACK_BUILD_ID_IP; - memset(id_offs[i].build_id, 0, BUILD_ID_SIZE_MAX); + stack_map_build_id_set_ip(&id_offs[i]); + prev_vma = vma; + prev_build_id = NULL; continue; } -build_id_valid: - id_offs[i].offset = (vma->vm_pgoff << PAGE_SHIFT) + ip - vma->vm_start; - id_offs[i].status = BPF_STACK_BUILD_ID_VALID; + offset = stack_map_build_id_offset(vma->vm_pgoff, vma->vm_start, ip); + stack_map_build_id_set_valid(&id_offs[i], offset, id_offs[i].build_id); prev_vma = vma; prev_build_id = id_offs[i].build_id; } bpf_mmap_unlock_mm(work, current->mm); + return; + +fallback: + /* cannot access current->mm, fall back to ips */ + for (i = 0; i < trace_nr; i++) + stack_map_build_id_set_ip(&id_offs[i]); } static struct perf_callchain_entry * @@ -246,78 +384,115 @@ get_callchain_entry_for_task(struct task_struct *task, u32 max_depth) #endif } -static long __bpf_get_stackid(struct bpf_map *map, - struct perf_callchain_entry *trace, u64 flags) +struct stackid { + struct stack_map_bucket *bucket; + u64 *ips; + u32 nr; + u32 len; + u32 hash; + u32 id; + bool hash_matches; +}; + +static int stackid_init(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) { struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); - struct stack_map_bucket *bucket, *new_bucket, *old_bucket; - u32 hash, id, trace_nr, trace_len, i, max_depth; u32 skip = flags & BPF_F_SKIP_FIELD_MASK; - bool user = flags & BPF_F_USER_STACK; - u64 *ips; - bool hash_matches; + u32 max_depth; if (trace->nr <= skip) /* skipping more than usable stack trace */ return -EFAULT; max_depth = stack_map_calculate_max_depth(map->value_size, stack_map_data_size(map), flags); - trace_nr = min_t(u32, trace->nr - skip, max_depth - skip); - trace_len = trace_nr * sizeof(u64); - ips = trace->ip + skip; - hash = jhash2((u32 *)ips, trace_len / sizeof(u32), 0); - id = hash & (smap->n_buckets - 1); - bucket = READ_ONCE(smap->buckets[id]); + stackid->nr = min_t(u32, trace->nr - skip, max_depth - skip); + stackid->len = stackid->nr * sizeof(u64); + stackid->ips = trace->ip + skip; + stackid->hash = jhash2((u32 *)stackid->ips, stackid->len / sizeof(u32), 0); + stackid->id = stackid->hash & (smap->n_buckets - 1); + stackid->bucket = READ_ONCE(smap->buckets[stackid->id]); + stackid->hash_matches = stackid->bucket && stackid->bucket->hash == stackid->hash; + return 0; +} + +static int stackid_fastpath(struct stackid *stackid, struct bpf_map *map, + struct perf_callchain_entry *trace, u64 flags) +{ + int err; + + err = stackid_init(stackid, map, trace, flags); + if (err) + return err; - hash_matches = bucket && bucket->hash == hash; /* fast cmp */ - if (hash_matches && flags & BPF_F_FAST_STACK_CMP) - return id; + if (stackid->hash_matches && flags & BPF_F_FAST_STACK_CMP) + return stackid->id; + + if (stack_map_use_build_id(map)) + return -ENOENT; + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, stackid->ips, stackid->len) == 0) + return stackid->id; + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) + return -EEXIST; + return -ENOENT; +} + +static struct stack_map_bucket * +stackid_new_bucket(struct stackid *stackid, struct bpf_map *map) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + struct bpf_stack_build_id *id_offs; + struct stack_map_bucket *bucket; + u32 i; + + bucket = (struct stack_map_bucket *) pcpu_freelist_pop(&smap->freelist); + if (unlikely(!bucket)) + return NULL; + + if (stack_map_use_build_id(map)) { + id_offs = (struct bpf_stack_build_id *)bucket->data; + for (i = 0; i < stackid->nr; i++) + id_offs[i].ip = stackid->ips[i]; + } else { + memcpy(bucket->data, stackid->ips, stackid->len); + } + + bucket->hash = stackid->hash; + bucket->nr = stackid->nr; + return bucket; +} + +static long stackid_install(struct stackid *stackid, struct bpf_map *map, + struct stack_map_bucket *new_bucket, u64 flags) +{ + struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); + bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *old_bucket; + u32 trace_len; if (stack_map_use_build_id(map)) { struct bpf_stack_build_id *id_offs; - /* for build_id+offset, pop a bucket before slow cmp */ - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - new_bucket->nr = trace_nr; id_offs = (struct bpf_stack_build_id *)new_bucket->data; - for (i = 0; i < trace_nr; i++) - id_offs[i].ip = ips[i]; - stack_map_get_build_id_offset(id_offs, trace_nr, user, false /* !may_fault */); - trace_len = trace_nr * sizeof(struct bpf_stack_build_id); - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, new_bucket->data, trace_len) == 0) { + stack_map_get_build_id_offset(id_offs, stackid->nr, user, false /* !may_fault */); + trace_len = stackid->nr * sizeof(struct bpf_stack_build_id); + if (stackid->hash_matches && stackid->bucket->nr == stackid->nr && + memcmp(stackid->bucket->data, new_bucket->data, trace_len) == 0) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); - return id; + return stackid->id; } - if (bucket && !(flags & BPF_F_REUSE_STACKID)) { + if (stackid->bucket && !(flags & BPF_F_REUSE_STACKID)) { pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); return -EEXIST; } - } else { - if (hash_matches && bucket->nr == trace_nr && - memcmp(bucket->data, ips, trace_len) == 0) - return id; - if (bucket && !(flags & BPF_F_REUSE_STACKID)) - return -EEXIST; - - new_bucket = (struct stack_map_bucket *) - pcpu_freelist_pop(&smap->freelist); - if (unlikely(!new_bucket)) - return -ENOMEM; - memcpy(new_bucket->data, ips, trace_len); } - new_bucket->hash = hash; - new_bucket->nr = trace_nr; - - old_bucket = xchg(&smap->buckets[id], new_bucket); + old_bucket = xchg(&smap->buckets[stackid->id], new_bucket); if (old_bucket) pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); - return id; + return stackid->id; } BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, @@ -325,23 +500,36 @@ BPF_CALL_3(bpf_get_stackid, struct pt_regs *, regs, struct bpf_map *, map, { u32 elem_size = stack_map_data_size(map); bool user = flags & BPF_F_USER_STACK; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel = !user; u32 max_depth; + int err; if (unlikely(flags & ~(BPF_F_SKIP_FIELD_MASK | BPF_F_USER_STACK | BPF_F_FAST_STACK_CMP | BPF_F_REUSE_STACKID))) return -EINVAL; max_depth = stack_map_calculate_max_depth(map->value_size, elem_size, flags); - trace = get_perf_callchain(regs, kernel, user, max_depth, - false, false); - if (unlikely(!trace)) - /* couldn't fetch the stack trace */ - return -EFAULT; + scoped_guard(preempt) { + trace = get_perf_callchain(regs, kernel, user, max_depth, + false, false); + if (unlikely(!trace)) + /* couldn't fetch the stack trace */ + return -EFAULT; + + err = stackid_fastpath(&stackid, map, trace, flags); + if (err != -ENOENT) + return err; + + new_bucket = stackid_new_bucket(&stackid, map); + if (!new_bucket) + return -ENOMEM; + } - return __bpf_get_stackid(map, trace, flags); + return stackid_install(&stackid, map, new_bucket, flags); } const struct bpf_func_proto bpf_get_stackid_proto = { @@ -369,7 +557,9 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, struct bpf_map *, map, u64, flags) { struct perf_event *event = ctx->event; + struct stack_map_bucket *new_bucket; struct perf_callchain_entry *trace; + struct stackid stackid; bool kernel, user; __u64 nr_kernel; int ret; @@ -395,7 +585,6 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, if (kernel) { trace->nr = nr_kernel; - ret = __bpf_get_stackid(map, trace, flags); } else { /* user */ u64 skip = flags & BPF_F_SKIP_FIELD_MASK; @@ -404,12 +593,22 @@ BPF_CALL_3(bpf_get_stackid_pe, struct bpf_perf_event_data_kern *, ctx, return -EFAULT; flags = (flags & ~BPF_F_SKIP_FIELD_MASK) | skip; - ret = __bpf_get_stackid(map, trace, flags); } + ret = stackid_fastpath(&stackid, map, trace, flags); + if (ret != -ENOENT) + goto out; + + new_bucket = stackid_new_bucket(&stackid, map); + if (new_bucket) { + trace->nr = nr; + return stackid_install(&stackid, map, new_bucket, flags); + } + ret = -ENOMEM; + +out: /* restore nr */ trace->nr = nr; - return ret; } @@ -460,6 +659,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, max_depth = stack_map_calculate_max_depth(size, elem_size, flags); + preempt_disable(); if (may_fault) rcu_read_lock(); /* need RCU for perf's callchain below */ @@ -476,6 +676,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, if (unlikely(!trace) || trace->nr < skip) { if (may_fault) rcu_read_unlock(); + preempt_enable(); goto err_fault; } @@ -496,6 +697,7 @@ static long __bpf_get_stack(struct pt_regs *regs, struct task_struct *task, /* trace/ips should not be dereferenced after this point */ if (may_fault) rcu_read_unlock(); + preempt_enable(); if (user_build_id) stack_map_get_build_id_offset(buf, trace_nr, user, may_fault); @@ -536,6 +738,7 @@ BPF_CALL_4(bpf_get_stack_sleepable, struct pt_regs *, regs, void *, buf, u32, si const struct bpf_func_proto bpf_get_stack_sleepable_proto = { .func = bpf_get_stack_sleepable, .gpl_only = true, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_PTR_TO_UNINIT_MEM, @@ -586,6 +789,7 @@ BPF_CALL_4(bpf_get_task_stack_sleepable, struct task_struct *, task, void *, buf const struct bpf_func_proto bpf_get_task_stack_sleepable_proto = { .func = bpf_get_task_stack_sleepable, .gpl_only = false, + .might_sleep = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index e92fd2bec98c7..bab472152d64d 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -1593,7 +1593,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr) err = security_bpf_map_create(map, attr, token, uattr.is_kernel); if (err) - goto free_map_sec; + goto free_map; err = bpf_map_alloc_id(map); if (err) @@ -3343,9 +3343,10 @@ static const char *bpf_link_type_strs[] = { static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) { const struct bpf_link *link = filp->private_data; - const struct bpf_prog *prog = link->prog; + const struct bpf_prog *prog; enum bpf_link_type type = link->type; char prog_tag[sizeof(prog->tag) * 2 + 1] = { }; + u32 prog_id = 0; if (type < ARRAY_SIZE(bpf_link_type_strs) && bpf_link_type_strs[type]) { if (link->type == BPF_LINK_TYPE_KPROBE_MULTI) @@ -3362,13 +3363,20 @@ static void bpf_link_show_fdinfo(struct seq_file *m, struct file *filp) } seq_printf(m, "link_id:\t%u\n", link->id); + rcu_read_lock(); + prog = READ_ONCE(link->prog); if (prog) { bin2hex(prog_tag, prog->tag, sizeof(prog->tag)); + prog_id = prog->aux->id; + } + rcu_read_unlock(); + + if (prog) { seq_printf(m, "prog_tag:\t%s\n" "prog_id:\t%u\n", prog_tag, - prog->aux->id); + prog_id); } if (link->ops->show_fdinfo) link->ops->show_fdinfo(link, m); @@ -5372,6 +5380,7 @@ static int bpf_link_get_info_by_fd(struct file *file, { struct bpf_link_info __user *uinfo = u64_to_user_ptr(attr->info.info); struct bpf_link_info info; + const struct bpf_prog *prog; u32 info_len = attr->info.info_len; int err; @@ -5386,8 +5395,12 @@ static int bpf_link_get_info_by_fd(struct file *file, info.type = link->type; info.id = link->id; - if (link->prog) - info.prog_id = link->prog->aux->id; + + rcu_read_lock(); + prog = READ_ONCE(link->prog); + if (prog) + info.prog_id = prog->aux->id; + rcu_read_unlock(); if (link->ops->fill_link_info) { err = link->ops->fill_link_info(link, &info); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index e791ae065c39b..13e1aabe6f886 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -753,9 +753,9 @@ static struct bpf_iter_reg task_vma_reg_info = { BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) { - struct mmap_unlock_irq_work *work = NULL; + struct mmap_unlock_irq_work *work; struct vm_area_struct *vma; - bool irq_work_busy = false; + bool __maybe_unused mmput_needed = false; struct mm_struct *mm; int ret = -ENOENT; @@ -765,14 +765,43 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, if (!task) return -ENOENT; - mm = task->mm; + if (task == current) { + mm = task->mm; + } else { + /* + * Foreign task: pin task->mm against a concurrent exit_mm(). + * Use trylock on alloc_lock instead of get_task_mm()'s + * blocking task_lock() to avoid deadlocking the target task. + */ + if (!IS_ENABLED(CONFIG_MMU)) + return -EOPNOTSUPP; + if (irqs_disabled()) + return -EBUSY; + if (!spin_trylock(&task->alloc_lock)) + return -EBUSY; + mm = task->mm; + if (mm && !(task->flags & PF_KTHREAD)) { + mmget(mm); + mmput_needed = true; + } else { + mm = NULL; + } + spin_unlock(&task->alloc_lock); + } if (!mm) return -ENOENT; - irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + work = bpf_mmap_unlock_guard_get(); + if (IS_ERR(work)) { + ret = PTR_ERR(work); + goto out; + } - if (irq_work_busy || !mmap_read_trylock(mm)) - return -EBUSY; + if (!mmap_read_trylock(mm)) { + bpf_mmap_unlock_guard_put(work); + ret = -EBUSY; + goto out; + } vma = find_vma(mm, start); @@ -782,6 +811,11 @@ BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, ret = 0; } bpf_mmap_unlock_mm(work, mm); +out: +#ifdef CONFIG_MMU + if (mmput_needed) + mmput_async(mm); +#endif return ret; } @@ -1161,6 +1195,8 @@ static void do_mmap_read_unlock(struct irq_work *entry) work = container_of(entry, struct mmap_unlock_irq_work, irq_work); mmap_read_unlock_non_owner(work->mm); + work->mm = NULL; + bpf_mmap_unlock_guard_put(work); } static int __init task_iter_init(void) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d1c51d812f3b6..9247549b5a772 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -431,7 +431,7 @@ static struct btf_record *reg_btf_record(const struct bpf_reg_state *reg) return rec; } -static bool subprog_is_global(const struct bpf_verifier_env *env, int subprog) +bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) { struct bpf_func_info_aux *aux = env->prog->aux->func_info_aux; @@ -611,6 +611,13 @@ static bool is_atomic_load_insn(const struct bpf_insn *insn) insn->imm == BPF_LOAD_ACQ; } +static bool is_atomic_fetch_insn(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + BPF_MODE(insn->code) == BPF_ATOMIC && + (insn->imm & BPF_FETCH); +} + static int __get_spi(s32 off) { return (-off - 1) / BPF_REG_SIZE; @@ -1719,6 +1726,8 @@ static int copy_func_state(struct bpf_func_state *dst, const struct bpf_func_state *src) { memcpy(dst, src, offsetof(struct bpf_func_state, stack)); + /* Instruction accounting is path-local, not part of verifier state. */ + dst->insns_subtotal = 0; return copy_stack_state(dst, src); } @@ -2110,7 +2119,7 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT); if (!elem) - return NULL; + return ERR_PTR(-ENOMEM); elem->insn_idx = insn_idx; elem->prev_insn_idx = prev_insn_idx; @@ -2120,12 +2129,12 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, env->stack_size++; err = copy_verifier_state(&elem->st, cur); if (err) - return NULL; + return ERR_PTR(-ENOMEM); elem->st.speculative |= speculative; if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) { verbose(env, "The sequence of %d jumps is too complex.\n", env->stack_size); - return NULL; + return ERR_PTR(-E2BIG); } if (elem->st.parent) { ++elem->st.parent->branches; @@ -2974,7 +2983,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL_ACCOUNT); if (!elem) - return NULL; + return ERR_PTR(-ENOMEM); elem->insn_idx = insn_idx; elem->prev_insn_idx = prev_insn_idx; @@ -2986,7 +2995,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, verbose(env, "The sequence of %d jumps is too complex for async cb.\n", env->stack_size); - return NULL; + return ERR_PTR(-E2BIG); } /* Unlike push_stack() do not copy_verifier_state(). * The caller state doesn't matter. @@ -2997,7 +3006,7 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, elem->st.in_sleepable = is_sleepable; frame = kzalloc(sizeof(*frame), GFP_KERNEL_ACCOUNT); if (!frame) - return NULL; + return ERR_PTR(-ENOMEM); init_func_state(env, frame, BPF_MAIN_FUNC /* callsite */, 0 /* frameno within this callchain */, @@ -4323,10 +4332,24 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, * dreg still needs precision before this insn */ } - } else if (class == BPF_LDX || is_atomic_load_insn(insn)) { - if (!bt_is_reg_set(bt, dreg)) + } else if (class == BPF_LDX || + is_atomic_load_insn(insn) || + is_atomic_fetch_insn(insn)) { + u32 load_reg = dreg; + + /* + * Atomic fetch operation writes the old value into + * a register (sreg or r0) and if it was tracked for + * precision, propagate to the stack slot like we do + * in regular ldx. + */ + if (is_atomic_fetch_insn(insn)) + load_reg = insn->imm == BPF_CMPXCHG ? + BPF_REG_0 : sreg; + + if (!bt_is_reg_set(bt, load_reg)) return 0; - bt_clear_reg(bt, dreg); + bt_clear_reg(bt, load_reg); /* scalars can only be spilled into stack w/o losing precision. * Load from any other memory can be zero extended. @@ -4369,7 +4392,7 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (subprog < 0) return -EFAULT; - if (subprog_is_global(env, subprog)) { + if (bpf_subprog_is_global(env, subprog)) { /* check that jump history doesn't have any * extra instructions from subprog; the next * instruction after call to global subprog @@ -5866,6 +5889,13 @@ static int map_kptr_match_type(struct bpf_verifier_env *env, if (type_flag(reg->type) & ~perm_flags) goto bad_type; + /* + * A BPF_KPTR_PERCPU field is read back as MEM_PERCPU, so the value + * stored in it must carry the same flag. + */ + if ((kptr_field->type == BPF_KPTR_PERCPU) != !!(reg->type & MEM_PERCPU)) + goto bad_type; + /* We need to verify reg->type and reg->btf, before accessing reg->btf */ reg_name = btf_type_name(reg->btf, reg->btf_id); @@ -6364,6 +6394,30 @@ static bool is_arena_reg(struct bpf_verifier_env *env, int regno) return reg->type == PTR_TO_ARENA; } +static bool is_load_acq_unsafe(struct bpf_verifier_env *env, int regno, + struct bpf_insn *insn) +{ + const struct bpf_reg_state *reg = reg_state(env, regno); + + /* + * A BPF_LOAD_ACQ is not rewritten to a BPF_PROBE_MEM load by the + * verifier, unlike a regular BPF_LDX. The JIT would emit a plain load + * with no exception table entry, so a fault (e.g. NULL deref) crashes + * the kernel instead of being handled. + * + * Reject the source pointer types that a BPF_LDX would have had that + * fault protection applied to, i.e. the ones bpf_convert_ctx_accesses() + * turns into BPF_PROBE_MEM: a bare PTR_TO_BTF_ID and any PTR_UNTRUSTED + * pointer (untrusted btf ids, untrusted MEM_ALLOC, rdonly untrusted + * memory). A PTR_TRUSTED pointer is not among them, is not converted, + * and stays allowed. Same for the other flagged PTR_TO_BTF_ID variants + * (MEM_ALLOC, MEM_RCU, ...), hence the exact match on the base type. + */ + return insn->imm == BPF_LOAD_ACQ && + (reg->type == PTR_TO_BTF_ID || + (type_flag(reg->type) & PTR_UNTRUSTED)); +} + /* Return false if @regno contains a pointer whose type isn't supported for * atomic instruction @insn. */ @@ -6380,7 +6434,8 @@ static bool atomic_ptr_type_ok(struct bpf_verifier_env *env, int regno, return false; if (is_arena_reg(env, regno)) return bpf_jit_supports_insn(insn, true); - + if (is_load_acq_unsafe(env, regno, insn)) + return false; return true; } @@ -6580,27 +6635,44 @@ static int round_up_stack_depth(struct bpf_verifier_env *env, int stack_depth) return round_up(max_t(u32, stack_depth, 1), 32); } +/* temporary state used for call frame depth calculation */ +struct bpf_subprog_call_depth_info { + int ret_insn; /* caller instruction where we return to. */ + int caller; /* caller subprogram idx */ + int frame; /* # of consecutive static call stack frames on top of stack */ +}; + /* starting from main bpf function walk all instructions of the function * and recursively walk all callees that given function can call. * Ignore jump and exit insns. - * Since recursion is prevented by check_cfg() this algorithm - * only needs a local stack of MAX_CALL_FRAMES to remember callsites */ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, + struct bpf_subprog_call_depth_info *dinfo, bool priv_stack_supported) { struct bpf_subprog_info *subprog = env->subprog_info; struct bpf_insn *insn = env->prog->insnsi; int depth = 0, frame = 0, i, subprog_end, subprog_depth; bool tail_call_reachable = false; - int ret_insn[MAX_CALL_FRAMES]; - int ret_prog[MAX_CALL_FRAMES]; - int j; + int total; + int tmp; + + /* no caller idx */ + dinfo[idx].caller = -1; i = subprog[idx].start; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; process_func: + if (subprog[idx].has_ld_abs) { + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_cb) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + } + /* protect against potential stack overflow that might happen when * bpf2bpf calls get combined with tailcalls. Limit the caller's stack * depth for such case down to 256 so that the worst case scenario @@ -6639,6 +6711,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, } if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) { + if (subprog_depth > env->max_stack_depth) + env->max_stack_depth = subprog_depth; if (subprog_depth > MAX_BPF_STACK) { verbose(env, "stack size of subprog %d is %d. Too large\n", idx, subprog_depth); @@ -6646,9 +6720,15 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, } } else { depth += subprog_depth; + if (depth > env->max_stack_depth) + env->max_stack_depth = depth; if (depth > MAX_BPF_STACK) { + total = 0; + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) + total++; + verbose(env, "combined stack size of %d calls is %d. Too large\n", - frame + 1, depth); + total, depth); return -EACCES; } } @@ -6662,10 +6742,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!is_bpf_throw_kfunc(insn + i)) continue; - if (subprog[idx].is_cb) - err = true; - for (int c = 0; c < frame && !err; c++) { - if (subprog[ret_prog[c]].is_cb) { + for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_cb) { err = true; break; } @@ -6681,8 +6759,6 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i)) continue; /* remember insn and function to return to */ - ret_insn[frame] = i + 1; - ret_prog[frame] = idx; /* find the callee */ next_insn = i + insn[i].imm + 1; @@ -6702,7 +6778,16 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, return -EINVAL; } } + + /* store caller info for after we return from callee */ + dinfo[idx].frame = frame; + dinfo[idx].ret_insn = i + 1; + + /* push caller idx into callee's dinfo */ + dinfo[sidx].caller = idx; + i = next_insn; + idx = sidx; if (!priv_stack_supported) subprog[idx].priv_stack_mode = NO_PRIV_STACK; @@ -6710,7 +6795,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, if (subprog[idx].has_tail_call) tail_call_reachable = true; - frame++; + frame = bpf_subprog_is_global(env, idx) ? 0 : frame + 1; if (frame >= MAX_CALL_FRAMES) { verbose(env, "the call stack of %d frames is too deep !\n", frame); @@ -6724,12 +6809,12 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, * tail call counter throughout bpf2bpf calls combined with tailcalls */ if (tail_call_reachable) - for (j = 0; j < frame; j++) { - if (subprog[ret_prog[j]].is_exception_cb) { + for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) { + if (subprog[tmp].is_exception_cb) { verbose(env, "cannot tail call within exception cb\n"); return -EINVAL; } - subprog[ret_prog[j]].tail_call_reachable = true; + subprog[tmp].tail_call_reachable = true; } if (subprog[0].tail_call_reachable) env->prog->aux->tail_call_reachable = true; @@ -6737,23 +6822,33 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx, /* end of for() loop means the last insn of the 'subprog' * was reached. Doesn't matter whether it was JA or EXIT */ - if (frame == 0) + if (frame == 0 && dinfo[idx].caller < 0) return 0; if (subprog[idx].priv_stack_mode != PRIV_STACK_ADAPTIVE) depth -= round_up_stack_depth(env, subprog[idx].stack_depth); - frame--; - i = ret_insn[frame]; - idx = ret_prog[frame]; + + /* pop caller idx from callee */ + idx = dinfo[idx].caller; + + /* retrieve caller state from its frame */ + frame = dinfo[idx].frame; + i = dinfo[idx].ret_insn; + goto continue_func; } static int check_max_stack_depth(struct bpf_verifier_env *env) { enum priv_stack_mode priv_stack_mode = PRIV_STACK_UNKNOWN; + struct bpf_subprog_call_depth_info *dinfo; struct bpf_subprog_info *si = env->subprog_info; bool priv_stack_supported; int ret; + dinfo = kvcalloc(env->subprog_cnt, sizeof(*dinfo), GFP_KERNEL_ACCOUNT); + if (!dinfo) + return -ENOMEM; + for (int i = 0; i < env->subprog_cnt; i++) { if (si[i].has_tail_call) { priv_stack_mode = NO_PRIV_STACK; @@ -6775,9 +6870,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) for (int i = env->subprog_cnt - 1; i >= 0; i--) { if (!i || si[i].is_async_cb) { priv_stack_supported = !i && priv_stack_mode == PRIV_STACK_ADAPTIVE; - ret = check_max_stack_depth_subprog(env, i, priv_stack_supported); - if (ret < 0) + ret = check_max_stack_depth_subprog(env, i, dinfo, + priv_stack_supported); + if (ret < 0) { + kvfree(dinfo); return ret; + } } } @@ -6788,6 +6886,8 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) } } + kvfree(dinfo); + return 0; } @@ -7954,7 +8054,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, { int err; - err = check_load_mem(env, insn, true, false, false, "atomic_load"); + err = check_reg_arg(env, insn->src_reg, SRC_OP); if (err) return err; @@ -7965,7 +8065,7 @@ static int check_atomic_load(struct bpf_verifier_env *env, return -EACCES; } - return 0; + return check_load_mem(env, insn, true, false, false, "atomic_load"); } static int check_atomic_store(struct bpf_verifier_env *env, @@ -9116,8 +9216,8 @@ static int process_iter_next_call(struct bpf_verifier_env *env, int insn_idx, prev_st = find_prev_entry(env, cur_st->parent, insn_idx); /* branch out active iter state */ queued_st = push_stack(env, insn_idx + 1, insn_idx, false); - if (!queued_st) - return -ENOMEM; + if (IS_ERR(queued_st)) + return PTR_ERR(queued_st); queued_iter = get_iter_from_state(queued_st, meta); queued_iter->iter.state = BPF_ITER_STATE_ACTIVE; @@ -9280,6 +9380,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = { [ARG_CONST_SIZE] = &scalar_types, [ARG_CONST_SIZE_OR_ZERO] = &scalar_types, [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types, + [ARG_SCALAR] = &scalar_types, [ARG_CONST_MAP_PTR] = &const_map_ptr_types, [ARG_PTR_TO_CTX] = &context_types, [ARG_PTR_TO_SOCK_COMMON] = &sock_types, @@ -10687,8 +10788,8 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins async_cb = push_async_cb(env, env->subprog_info[subprog].start, insn_idx, subprog, is_async_cb_sleepable(env, insn)); - if (!async_cb) - return -EFAULT; + if (IS_ERR(async_cb)) + return PTR_ERR(async_cb); callee = async_cb->frame[0]; callee->async_entry_cnt = caller->async_entry_cnt + 1; @@ -10704,8 +10805,8 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins * proceed with next instruction within current frame. */ callback_state = push_stack(env, env->subprog_info[subprog].start, insn_idx, false); - if (!callback_state) - return -ENOMEM; + if (IS_ERR(callback_state)) + return PTR_ERR(callback_state); err = setup_func_entry(env, subprog, insn_idx, set_callee_state_cb, callback_state); @@ -10735,7 +10836,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, err = btf_check_subprog_call(env, subprog, caller->regs); if (err == -EFAULT) return err; - if (subprog_is_global(env, subprog)) { + if (bpf_subprog_is_global(env, subprog)) { const char *sub_name = subprog_name(env, subprog); if (env->cur_state->active_locks) { @@ -11033,9 +11134,46 @@ static int set_task_work_schedule_callback_state(struct bpf_verifier_env *env, static bool is_rbtree_lock_required_kfunc(u32 btf_id); -/* Are we currently verifying the callback for a rbtree helper that must - * be called with lock held? If so, no need to complain about unreleased - * lock +static void account_processed_insn(struct bpf_verifier_env *env) +{ + struct bpf_func_state *frame = cur_func(env); + + env->insn_processed++; + frame->insns_subtotal++; + env->subprog_info[frame->subprogno].insns_self++; +} + +static void account_processed_insns(struct bpf_verifier_env *env, + struct bpf_func_state *callee, + struct bpf_func_state *caller) +{ + u32 insns; + + if (!callee) + return; + + insns = callee->insns_subtotal; + + env->subprog_info[callee->subprogno].insns_total += insns; + if (caller) + caller->insns_subtotal += insns; + callee->insns_subtotal = 0; +} + +static void account_current_path(struct bpf_verifier_env *env) +{ + struct bpf_verifier_state *state = env->cur_state; + int frame; + + for (frame = state->curframe; frame >= 0; frame--) + account_processed_insns(env, state->frame[frame], + frame ? state->frame[frame - 1] : NULL); +} + +/* + * Are we currently verifying the callback for an rbtree kfunc that must + * be called with a lock held, or one of that callback's subprogs? If so, + * no need to complain about an unreleased lock. */ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) { @@ -11043,17 +11181,19 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env) struct bpf_insn *insn = env->prog->insnsi; struct bpf_func_state *callee; int kfunc_btf_id; + u32 frame; - if (!state->curframe) - return false; - - callee = state->frame[state->curframe]; + for (frame = state->curframe; frame; frame--) { + callee = state->frame[frame]; + if (!callee->in_callback_fn) + continue; - if (!callee->in_callback_fn) - return false; + kfunc_btf_id = insn[callee->callsite].imm; + if (is_rbtree_lock_required_kfunc(kfunc_btf_id)) + return true; + } - kfunc_btf_id = insn[callee->callsite].imm; - return is_rbtree_lock_required_kfunc(kfunc_btf_id); + return false; } static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg, @@ -11134,6 +11274,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) verbose(env, "to caller at %d:\n", *insn_idx); print_verifier_state(env, state, caller->frameno, true); } + account_processed_insns(env, callee, caller); /* clear everything in the callee. In case of exceptional exits using * bpf_throw, this will be done by copy_verifier_state for extra frames. */ free_func_state(callee); @@ -12000,6 +12141,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn if (env->cur_state->curframe) { struct bpf_verifier_state *branch; + /* + * A taken tail call is modeled as a return from the current + * frame. A callback frame cannot be left that way because + * prepare_func_exit() would apply its return contract to the + * unknown R0 synthesized below. Stack-depth validation rejects + * this construct anyway. + */ + if (cur_func(env)->in_callback_fn) { + verbose(env, "cannot tail call within callback\n"); + return -EINVAL; + } mark_reg_scratched(env, BPF_REG_0); branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); if (IS_ERR(branch)) @@ -13669,6 +13821,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ { int flags = PROCESS_RES_LOCK; + if (in_rbtree_lock_required_cb(env)) { + verbose(env, "can't res_spin_{lock,unlock} in rbtree cb\n"); + return -EACCES; + } + if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) { verbose(env, "arg#%d doesn't point to map value or allocated object\n", i); return -EINVAL; @@ -13958,9 +14115,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_reg_state *regs; branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); - if (!branch) { + if (IS_ERR(branch)) { verbose(env, "failed to push state for failed lock acquisition\n"); - return -ENOMEM; + return PTR_ERR(branch); } regs = branch->frame[branch->curframe]->regs; @@ -14291,9 +14448,9 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, return 0; } -static bool check_reg_sane_offset(struct bpf_verifier_env *env, - const struct bpf_reg_state *reg, - enum bpf_reg_type type) +static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) { bool known = tnum_is_const(reg->var_off); s64 val = reg->var_off.value; @@ -14305,12 +14462,6 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, return false; } - if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { - verbose(env, "%s pointer offset %d is not allowed\n", - reg_type_str(env, type), reg->off); - return false; - } - if (smin == S64_MIN) { verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", reg_type_str(env, type)); @@ -14326,6 +14477,27 @@ static bool check_reg_sane_offset(struct bpf_verifier_env *env, return true; } +static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg, + enum bpf_reg_type type) +{ + s64 smin = reg->smin_value; + + if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { + verbose(env, "%s pointer offset %d is not allowed\n", + reg_type_str(env, type), reg->off); + return false; + } + + if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { + verbose(env, "%s pointer offset %lld is not allowed\n", + reg_type_str(env, type), smin); + return false; + } + + return true; +} + enum { REASON_BOUNDS = -1, REASON_TYPE = -2, @@ -14411,16 +14583,15 @@ struct bpf_sanitize_info { bool mask_to_left; }; -static struct bpf_verifier_state * -sanitize_speculative_path(struct bpf_verifier_env *env, - const struct bpf_insn *insn, - u32 next_idx, u32 curr_idx) +static int sanitize_speculative_path(struct bpf_verifier_env *env, + const struct bpf_insn *insn, + u32 next_idx, u32 curr_idx) { struct bpf_verifier_state *branch; struct bpf_reg_state *regs; branch = push_stack(env, next_idx, curr_idx, true); - if (branch && insn) { + if (!IS_ERR(branch) && insn) { regs = branch->frame[branch->curframe]->regs; if (BPF_SRC(insn->code) == BPF_K) { mark_reg_unknown(env, regs, insn->dst_reg); @@ -14429,7 +14600,7 @@ sanitize_speculative_path(struct bpf_verifier_env *env, mark_reg_unknown(env, regs, insn->src_reg); } } - return branch; + return PTR_ERR_OR_ZERO(branch); } static int sanitize_ptr_alu(struct bpf_verifier_env *env, @@ -14448,7 +14619,6 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, u8 opcode = BPF_OP(insn->code); u32 alu_state, alu_limit; struct bpf_reg_state tmp; - bool ret; int err; if (can_skip_alu_sanitation(env, insn)) @@ -14521,11 +14691,12 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env, tmp = *dst_reg; copy_register_state(dst_reg, ptr_reg); } - ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, - env->insn_idx); - if (!ptr_is_dst_reg && ret) + err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx); + if (err < 0) + return REASON_STACK; + if (!ptr_is_dst_reg) *dst_reg = tmp; - return !ret ? REASON_STACK : 0; + return 0; } static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) @@ -14698,13 +14869,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* - * Accesses to untrusted PTR_TO_MEM are done through probe - * instructions, hence no need to track offsets. - */ - if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) - return 0; - switch (base_type(ptr_reg->type)) { case PTR_TO_CTX: case PTR_TO_MAP_VALUE: @@ -14734,18 +14898,23 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. - * The id may be overwritten later if we create a new variable offset. + /* For 'scalar += pointer', dst_reg inherits the complete pointer + * register state. Individual fields may be adjusted later by pointer + * arithmetic. Callers guarantee that below does not overwrite off_reg. */ - dst_reg->type = ptr_reg->type; - dst_reg->id = ptr_reg->id; + if (dst_reg != ptr_reg) + *dst_reg = *ptr_reg; - if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || - !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) - return -EINVAL; + /* + * Accesses to untrusted PTR_TO_MEM are done through probe + * instructions, hence no need to track offsets. + */ + if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) + return 0; - /* pointer types do not carry 32-bit bounds at the moment. */ - __mark_reg32_unbounded(dst_reg); + if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || + !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) + return -EINVAL; if (sanitize_needed(opcode)) { ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, @@ -14754,6 +14923,14 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return sanitize_err(env, insn, ret, off_reg, dst_reg); } + /* + * Pointer types do not carry 32-bit bounds at the moment. Blank r32 + * only after sanitize_ptr_alu() may have snapshotted dst_reg into a + * speculative path: otherwise reg_bounds_sanity_check() might hit some + * constraints violations. + */ + __mark_reg32_unbounded(dst_reg); + switch (opcode) { case BPF_ADD: /* We can take a fixed offset as long as it doesn't overflow @@ -14800,7 +14977,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, } break; case BPF_SUB: - if (dst_reg == off_reg) { + if (dst_reg != ptr_reg) { /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); @@ -14870,7 +15047,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) + if (!check_reg_sane_offset_ptr(env, dst_reg, ptr_reg->type)) return -EINVAL; reg_bounds_sync(dst_reg); bounds_ret = sanitize_check_bounds(env, insn, dst_reg); @@ -15764,8 +15941,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, err = mark_chain_precision(env, insn->dst_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - src_reg, dst_reg); + off_reg = *dst_reg; + return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ @@ -16403,6 +16580,13 @@ static int is_branch_taken(struct bpf_reg_state *reg1, struct bpf_reg_state *reg if (__is_pointer_value(false, reg1) || __is_pointer_value(false, reg2)) { u64 val; + /* + * The low 32 bits of a valid pointer may well be zero, hence + * nothing below applies to a 32-bit comparison. + */ + if (is_jmp32) + return -1; + /* arrange that reg2 is a scalar, and reg1 is a pointer */ if (!is_reg_const(reg2, is_jmp32)) { opcode = flip_opcode(opcode); @@ -16992,8 +17176,8 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, /* branch out 'fallthrough' insn as a new state to explore */ queued_st = push_stack(env, idx + 1, idx, false); - if (!queued_st) - return -ENOMEM; + if (IS_ERR(queued_st)) + return PTR_ERR(queued_st); queued_st->may_goto_depth++; if (prev_st) @@ -17071,10 +17255,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * the fall-through branch for simulation under speculative * execution. */ - if (!env->bypass_spec_v1 && - !sanitize_speculative_path(env, insn, *insn_idx + 1, - *insn_idx)) - return -EFAULT; + if (!env->bypass_spec_v1) { + err = sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx); + if (err < 0) + return err; + } if (env->log.level & BPF_LOG_LEVEL) print_insn_state(env, this_branch, this_branch->curframe); *insn_idx += insn->off; @@ -17084,11 +17269,12 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, * program will go. If needed, push the goto branch for * simulation under speculative execution. */ - if (!env->bypass_spec_v1 && - !sanitize_speculative_path(env, insn, - *insn_idx + insn->off + 1, - *insn_idx)) - return -EFAULT; + if (!env->bypass_spec_v1) { + err = sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1, + *insn_idx); + if (err < 0) + return err; + } if (env->log.level & BPF_LOG_LEVEL) print_insn_state(env, this_branch, this_branch->curframe); return 0; @@ -17111,8 +17297,8 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env, other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false); - if (!other_branch) - return -EFAULT; + if (IS_ERR(other_branch)) + return PTR_ERR(other_branch); other_branch_regs = other_branch->frame[other_branch->curframe]->regs; if (BPF_SRC(insn->code) == BPF_X) { @@ -17278,6 +17464,15 @@ static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) verbose(env, "callback function not static\n"); return -EINVAL; } + /* + * When env->subprog_cnt == 1 this instruction won't be rewritten + * to hold a real function address. Assume that no usable program + * combines e.g. main and timer callback and just reject here. + */ + if (subprogno == 0) { + verbose(env, "callback function cannot be the main program\n"); + return -EINVAL; + } dst_reg->type = PTR_TO_FUNC; dst_reg->subprogno = subprogno; @@ -17337,6 +17532,7 @@ static bool may_access_skb(enum bpf_prog_type type) */ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) { + struct bpf_verifier_state *state = env->cur_state; struct bpf_reg_state *regs = cur_regs(env); static const int ctx_reg = BPF_REG_6; u8 mode = BPF_MODE(insn->code); @@ -17347,6 +17543,13 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return -EINVAL; } + for (i = state->curframe; i; i--) { + if (state->frame[i]->in_callback_fn) { + verbose(env, "cannot use BPF_LD_[ABS|IND] within callback\n"); + return -EINVAL; + } + } + if (!env->ops->gen_ld_abs) { verifier_bug(env, "gen_ld_abs is null"); return -EFAULT; @@ -17402,6 +17605,23 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) mark_reg_unknown(env, regs, BPF_REG_0); /* ld_abs load up to 32-bit skb data. */ regs[BPF_REG_0].subreg_def = env->insn_idx + 1; + /* + * See bpf_gen_ld_abs() which emits a hidden BPF_EXIT with r0=0 + * which must be explored by the verifier when in a subprog. + */ + if (env->cur_state->curframe) { + struct bpf_verifier_state *branch; + + mark_reg_scratched(env, BPF_REG_0); + branch = push_stack(env, env->insn_idx + 1, env->insn_idx, false); + if (IS_ERR(branch)) + return PTR_ERR(branch); + mark_reg_known_zero(env, regs, BPF_REG_0); + err = prepare_func_exit(env, &env->insn_idx); + if (err) + return err; + env->insn_idx--; + } return 0; } @@ -20323,7 +20543,9 @@ static int do_check(struct bpf_verifier_env *env) insn = &insns[env->insn_idx]; insn_aux = &env->insn_aux_data[env->insn_idx]; - if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { + account_processed_insn(env); + + if (env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed); @@ -20448,6 +20670,7 @@ static int do_check(struct bpf_verifier_env *env) "speculation barrier after jump instruction may not have the desired effect")) return -EFAULT; process_bpf_exit: + account_current_path(env); mark_verifier_state_scratched(env); err = update_branch_counts(env, env->cur_state); if (err) @@ -23602,6 +23825,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog) ret = do_check(env); out: + account_current_path(env); if (!ret && pop_log) bpf_vlog_reset(&env->log, 0); free_states(env); @@ -23644,7 +23868,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env) again: new_cnt = 0; for (i = 1; i < env->subprog_cnt; i++) { - if (!subprog_is_global(env, i)) + if (!bpf_subprog_is_global(env, i)) continue; sub_aux = subprog_aux(env, i); @@ -23692,19 +23916,20 @@ static int do_check_main(struct bpf_verifier_env *env) static void print_verification_stats(struct bpf_verifier_env *env) { - int i; + /* Skip over hidden subprogs which are not verified. */ + int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt; if (env->log.level & BPF_LOG_STATS) { verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, 1000)); - verbose(env, "stack depth "); - for (i = 0; i < env->subprog_cnt; i++) { - u32 depth = env->subprog_info[i].stack_depth; - - verbose(env, "%d", depth); - if (i + 1 < env->subprog_cnt) - verbose(env, "+"); - } + verbose(env, "stack depth %d", env->subprog_info[0].stack_depth); + for (i = 1; i < subprog_cnt; i++) + verbose(env, "+%d", env->subprog_info[i].stack_depth); + verbose(env, " max %d\n", env->max_stack_depth); + verbose(env, "insns processed %d", env->subprog_info[0].insns_total); + for (i = 1; i < subprog_cnt; i++) + if (bpf_subprog_is_global(env, i)) + verbose(env, "+%d", env->subprog_info[i].insns_total); verbose(env, "\n"); } verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " @@ -24351,13 +24576,25 @@ static int check_attach_btf_id(struct bpf_verifier_env *env) struct btf *bpf_get_btf_vmlinux(void) { - if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { + /* Pairs with the smp_store_release() on the parse path below. */ + struct btf *btf = smp_load_acquire(&btf_vmlinux); + + if (!btf && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) { mutex_lock(&bpf_verifier_lock); - if (!btf_vmlinux) - btf_vmlinux = btf_parse_vmlinux(); + btf = btf_vmlinux; + if (!btf) { + btf = btf_parse_vmlinux(); + /* + * Order the parsed BTF contents and the globals the + * parse populated (e.g. bpf_ctx_convert.t) before + * the pointer publication. Pairs with the acquire + * on the lockless fast path above. + */ + smp_store_release(&btf_vmlinux, btf); + } mutex_unlock(&bpf_verifier_lock); } - return btf_vmlinux; + return btf; } /* diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 67eb24b2323b7..5dbd9f39a27bb 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -1522,9 +1522,9 @@ static struct cgroup *current_cgns_cgroup_dfl(void) } else { /* * NOTE: This function may be called from bpf_cgroup_from_id() - * on a task which has already passed exit_task_namespaces() and - * nsproxy == NULL. Fall back to cgrp_dfl_root which will make all - * cgroups visible for lookups. + * on a task which has already passed exit_nsproxy_namespaces() + * and nsproxy == NULL. Fall back to cgrp_dfl_root which will + * make all cgroups visible for lookups. */ return &cgrp_dfl_root.cgrp; } @@ -4001,6 +4001,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, struct psi_trigger *new; struct cgroup *cgrp; struct psi_group *psi; + bool need_rtpoll_worker; ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); @@ -4020,12 +4021,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, } psi = cgroup_psi(cgrp); - new = psi_trigger_create(psi, buf, res, of->file, of); + new = psi_trigger_create(psi, buf, res, of->file, of, + &need_rtpoll_worker); if (IS_ERR(new)) { ret = PTR_ERR(new); goto out_unlock; } + /* + * The worker fork must run with neither cgroup_mutex nor the file's + * kernfs active reference held. The latter is broken since + * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so + * recheck before publishing @new. + */ + if (need_rtpoll_worker) { + cgroup_unlock(); + ret = psi_trigger_create_rtpoll_worker(psi); + cgroup_lock(); + + if (!ret && !of->priv) + ret = -ENODEV; + if (ret) { + psi_trigger_destroy(new); + goto out_unlock; + } + } + smp_store_release(&ctx->psi.trigger, new); out_unlock: diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index 337608f408ce0..0eb9ebd4fcb92 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -165,7 +165,7 @@ struct cpuset { * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ - int nr_deadline_tasks; + atomic_t nr_deadline_tasks; int nr_migrate_dl_tasks; u64 sum_migrate_dl_bw; diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 2386090351cd3..6ab677c002181 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -145,14 +145,14 @@ void inc_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks++; + atomic_inc(&cs->nr_deadline_tasks); } void dec_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks--; + atomic_dec(&cs->nr_deadline_tasks); } static inline bool is_partition_valid(const struct cpuset *cs) @@ -1033,7 +1033,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs) struct css_task_iter it; struct task_struct *task; - if (cs->nr_deadline_tasks == 0) + if (atomic_read(&cs->nr_deadline_tasks) == 0) return; css_task_iter_start(&cs->css, 0, &it); @@ -1410,7 +1410,46 @@ static bool partition_xcpus_del(int old_prs, struct cpuset *parent, return isolcpus_updated; } -static void update_unbound_workqueue_cpumask(bool isolcpus_updated) +/* + * isolated_cpus_can_update - check for isolated & nohz_full conflicts + * @add_cpus: cpu mask for cpus that are going to be isolated + * @del_cpus: cpu mask for cpus that are no longer isolated, can be NULL + * Return: false if there is conflict, true otherwise + * + * If nohz_full is enabled and we have isolated CPUs, their combination must + * still leave housekeeping CPUs. + * + * TBD: Should consider merging this function into + * prstate_housekeeping_conflict(). + */ +static bool isolated_cpus_can_update(struct cpumask *add_cpus, + struct cpumask *del_cpus) +{ + cpumask_var_t full_hk_cpus; + int res = true; + + if (!housekeeping_enabled(HK_TYPE_KERNEL_NOISE)) + return true; + + if (del_cpus && cpumask_weight_and(del_cpus, + housekeeping_cpumask(HK_TYPE_KERNEL_NOISE))) + return true; + + if (!alloc_cpumask_var(&full_hk_cpus, GFP_KERNEL)) + return false; + + cpumask_and(full_hk_cpus, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE), + housekeeping_cpumask(HK_TYPE_DOMAIN)); + cpumask_andnot(full_hk_cpus, full_hk_cpus, isolated_cpus); + cpumask_and(full_hk_cpus, full_hk_cpus, cpu_active_mask); + if (!cpumask_weight_andnot(full_hk_cpus, add_cpus)) + res = false; + + free_cpumask_var(full_hk_cpus); + return res; +} + +static void update_isolation_cpumasks(bool isolcpus_updated) { int ret; @@ -1568,15 +1607,18 @@ static int remote_partition_enable(struct cpuset *cs, int new_prs, if (!cpumask_intersects(tmp->new_cpus, cpu_active_mask) || cpumask_subset(top_cpuset.effective_cpus, tmp->new_cpus)) return PERR_INVCPUS; + if ((new_prs == PRS_ISOLATED) && + !isolated_cpus_can_update(tmp->new_cpus, NULL)) + return PERR_HKEEPING; spin_lock_irq(&callback_lock); isolcpus_updated = partition_xcpus_add(new_prs, NULL, tmp->new_cpus); list_add(&cs->remote_sibling, &remote_children); cpumask_copy(cs->effective_xcpus, tmp->new_cpus); spin_unlock_irq(&callback_lock); - update_unbound_workqueue_cpumask(isolcpus_updated); + update_isolation_cpumasks(isolcpus_updated); cpuset_force_rebuild(); - cs->prs_err = 0; + WRITE_ONCE(cs->prs_err, 0); /* * Propagate changes in top_cpuset's effective_cpus down the hierarchy. @@ -1622,7 +1664,7 @@ static void remote_partition_disable(struct cpuset *cs, struct tmpmasks *tmp) compute_excpus(cs, cs->effective_xcpus); reset_partition_data(cs); spin_unlock_irq(&callback_lock); - update_unbound_workqueue_cpumask(isolcpus_updated); + update_isolation_cpumasks(isolcpus_updated); cpuset_force_rebuild(); /* @@ -1655,7 +1697,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, WARN_ON_ONCE(!cpumask_subset(cs->effective_xcpus, subpartitions_cpus)); if (cpumask_empty(excpus)) { - cs->prs_err = PERR_CPUSEMPTY; + WRITE_ONCE(cs->prs_err, PERR_CPUSEMPTY); goto invalidate; } @@ -1670,10 +1712,13 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, if (adding) { WARN_ON_ONCE(cpumask_intersects(tmp->addmask, subpartitions_cpus)); if (!capable(CAP_SYS_ADMIN)) - cs->prs_err = PERR_ACCESS; + WRITE_ONCE(cs->prs_err, PERR_ACCESS); else if (cpumask_intersects(tmp->addmask, subpartitions_cpus) || cpumask_subset(top_cpuset.effective_cpus, tmp->addmask)) - cs->prs_err = PERR_NOCPUS; + WRITE_ONCE(cs->prs_err, PERR_NOCPUS); + else if ((prs == PRS_ISOLATED) && + !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) + WRITE_ONCE(cs->prs_err, PERR_HKEEPING); if (cs->prs_err) goto invalidate; } @@ -1691,7 +1736,7 @@ static void remote_cpus_update(struct cpuset *cs, struct cpumask *xcpus, if (xcpus) cpumask_copy(cs->exclusive_cpus, xcpus); spin_unlock_irq(&callback_lock); - update_unbound_workqueue_cpumask(isolcpus_updated); + update_isolation_cpumasks(isolcpus_updated); if (adding || deleting) cpuset_force_rebuild(); @@ -1775,6 +1820,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, int subparts_delta = 0; int isolcpus_updated = 0; struct cpumask *xcpus = user_xcpus(cs); + int parent_prs = parent->partition_root_state; bool nocpu; lockdep_assert_held(&cpuset_mutex); @@ -1839,6 +1885,10 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, if (prstate_housekeeping_conflict(new_prs, xcpus)) return PERR_HKEEPING; + if ((new_prs == PRS_ISOLATED) && (new_prs != parent_prs) && + !isolated_cpus_can_update(xcpus, NULL)) + return PERR_HKEEPING; + if (tasks_nocpu_error(parent, cs, xcpus)) return PERR_NOCPUS; @@ -1894,6 +1944,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, * * For invalid partition: * delmask = newmask & parent->effective_xcpus + * The partition may become valid soon. */ if (is_partition_invalid(cs)) { adding = false; @@ -1908,6 +1959,23 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, deleting = cpumask_and(tmp->delmask, tmp->delmask, parent->effective_xcpus); } + + /* + * TBD: Invalidate a currently valid child root partition may + * still break isolated_cpus_can_update() rule if parent is an + * isolated partition. + */ + if (is_partition_valid(cs) && (old_prs != parent_prs)) { + if ((parent_prs == PRS_ROOT) && + /* Adding to parent means removing isolated CPUs */ + !isolated_cpus_can_update(tmp->delmask, tmp->addmask)) + part_error = PERR_HKEEPING; + if ((parent_prs == PRS_ISOLATED) && + /* Adding to parent means adding isolated CPUs */ + !isolated_cpus_can_update(tmp->addmask, tmp->delmask)) + part_error = PERR_HKEEPING; + } + /* * The new CPUs to be removed from parent's effective CPUs * must be present. @@ -2049,7 +2117,7 @@ static int update_parent_effective_cpumask(struct cpuset *cs, int cmd, WARN_ON_ONCE(parent->nr_subparts < 0); } spin_unlock_irq(&callback_lock); - update_unbound_workqueue_cpumask(isolcpus_updated); + update_isolation_cpumasks(isolcpus_updated); if ((old_prs != new_prs) && (cmd == partcmd_update)) update_partition_exclusive_flag(cs, new_prs); @@ -2115,13 +2183,13 @@ static void compute_partition_effective_cpumask(struct cpuset *cs, * partition root. */ WARN_ON_ONCE(is_remote_partition(child)); - child->prs_err = 0; + WRITE_ONCE(child->prs_err, 0); if (!cpumask_subset(child->effective_xcpus, cs->effective_xcpus)) - child->prs_err = PERR_INVCPUS; + WRITE_ONCE(child->prs_err, PERR_INVCPUS); else if (populated && cpumask_subset(new_ecpus, child->effective_xcpus)) - child->prs_err = PERR_NOCPUS; + WRITE_ONCE(child->prs_err, PERR_NOCPUS); if (child->prs_err) { int old_prs = child->partition_root_state; @@ -2488,8 +2556,10 @@ static void partition_cpus_change(struct cpuset *cs, struct cpuset *trialcs, return; prs_err = validate_partition(cs, trialcs); - if (prs_err) - trialcs->prs_err = cs->prs_err = prs_err; + if (prs_err) { + WRITE_ONCE(cs->prs_err, prs_err); + trialcs->prs_err = prs_err; + } if (is_remote_partition(cs)) { if (trialcs->prs_err) @@ -3039,7 +3109,11 @@ static int update_prstate(struct cpuset *cs, int new_prs) * A change in load balance state only, no change in cpumasks. * Need to update isolated_cpus. */ - isolcpus_updated = true; + if ((new_prs == PRS_ISOLATED) && + !isolated_cpus_can_update(cs->effective_xcpus, NULL)) + err = PERR_HKEEPING; + else + isolcpus_updated = true; } else { /* * Switching back to member is always allowed even if it @@ -3074,7 +3148,7 @@ static int update_prstate(struct cpuset *cs, int new_prs) else if (isolcpus_updated) isolated_cpus_update(old_prs, new_prs, cs->effective_xcpus); spin_unlock_irq(&callback_lock); - update_unbound_workqueue_cpumask(isolcpus_updated); + update_isolation_cpumasks(isolcpus_updated); /* Force update if switching back to member & update effective_xcpus */ update_cpumasks_hier(cs, &tmpmask, !new_prs); @@ -3313,8 +3387,8 @@ static void cpuset_attach(struct cgroup_taskset *tset) cs->old_mems_allowed = cpuset_attach_nodemask_to; if (cs->nr_migrate_dl_tasks) { - cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; - oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + atomic_sub(cs->nr_migrate_dl_tasks, &oldcs->nr_deadline_tasks); reset_migrate_dl_data(cs); } @@ -3942,7 +4016,7 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) if (remote && (cpumask_empty(subpartitions_cpus) || (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) { - cs->prs_err = PERR_HOTPLUG; + WRITE_ONCE(cs->prs_err, PERR_HOTPLUG); remote_partition_disable(cs, tmp); compute_effective_cpumask(&new_cpus, cs, parent); remote = false; diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index b0f0d15085db1..7481fbb947d32 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -173,7 +173,7 @@ int cpu_cluster_pm_exit(void) EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit); #ifdef CONFIG_PM -static int cpu_pm_suspend(void) +static int cpu_pm_suspend(void *data) { int ret; @@ -185,20 +185,24 @@ static int cpu_pm_suspend(void) return ret; } -static void cpu_pm_resume(void) +static void cpu_pm_resume(void *data) { cpu_cluster_pm_exit(); cpu_pm_exit(); } -static struct syscore_ops cpu_pm_syscore_ops = { +static const struct syscore_ops cpu_pm_syscore_ops = { .suspend = cpu_pm_suspend, .resume = cpu_pm_resume, }; +static struct syscore cpu_pm_syscore = { + .ops = &cpu_pm_syscore_ops, +}; + static int cpu_pm_init(void) { - register_syscore_ops(&cpu_pm_syscore_ops); + register_syscore(&cpu_pm_syscore); return 0; } core_initcall(cpu_pm_init); diff --git a/kernel/crash_dump_dm_crypt.c b/kernel/crash_dump_dm_crypt.c index 9501b0704f19f..12e38922ca741 100644 --- a/kernel/crash_dump_dm_crypt.c +++ b/kernel/crash_dump_dm_crypt.c @@ -80,7 +80,6 @@ static int add_key_to_keyring(struct dm_crypt_key *dm_key, kexec_dprintk("Error when adding key"); } - key_ref_put(keyring_ref); return r; } @@ -103,6 +102,7 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) struct dm_crypt_key *key; size_t keys_header_size; key_ref_t keyring_ref; + int ret = 0; u64 addr; /* find the target keyring (which must be writable) */ @@ -117,7 +117,8 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) dm_crypt_keys_read((char *)&key_count, sizeof(key_count), &addr); if (key_count < 0 || key_count > KEY_NUM_MAX) { kexec_dprintk("Failed to read the number of dm-crypt keys\n"); - return -1; + ret = -1; + goto out; } kexec_dprintk("There are %u keys\n", key_count); @@ -125,8 +126,10 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) keys_header_size = get_keys_header_size(key_count); keys_header = kzalloc(keys_header_size, GFP_KERNEL); - if (!keys_header) - return -ENOMEM; + if (!keys_header) { + ret = -ENOMEM; + goto out; + } dm_crypt_keys_read((char *)keys_header, keys_header_size, &addr); @@ -136,7 +139,9 @@ static int restore_dm_crypt_keys_to_thread_keyring(void) add_key_to_keyring(key, keyring_ref); } - return 0; +out: + key_ref_put(keyring_ref); + return ret; } static int read_key_from_user_keying(struct dm_crypt_key *dm_key) diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c index fd483b558b504..cc334eb94b33d 100644 --- a/kernel/dma/direct.c +++ b/kernel/dma/direct.c @@ -164,22 +164,21 @@ static bool dma_direct_use_pool(struct device *dev, gfp_t gfp) return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev); } -static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, - dma_addr_t *dma_handle, gfp_t gfp) +static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size, + dma_addr_t *dma_handle, void **cpu_addr, gfp_t gfp) { struct page *page; u64 phys_limit; - void *ret; if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))) return NULL; gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); - page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok); + page = dma_alloc_from_pool(dev, size, cpu_addr, gfp, dma_coherent_ok); if (!page) return NULL; *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); - return ret; + return page; } static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, @@ -247,8 +246,11 @@ void *dma_direct_alloc(struct device *dev, size_t size, * the atomic pools instead if we aren't allowed block. */ if ((remap || force_dma_unencrypted(dev)) && - dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + dma_direct_use_pool(dev, gfp)) { + page = dma_direct_alloc_from_pool(dev, size, dma_handle, + &ret, gfp); + return page ? ret : NULL; + } /* we always manually zero the memory once we are done */ page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); @@ -357,7 +359,7 @@ struct page *dma_direct_alloc_pages(struct device *dev, size_t size, void *ret; if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) - return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); + return dma_direct_alloc_from_pool(dev, size, dma_handle, &ret, gfp); page = __dma_direct_alloc_pages(dev, size, gfp, false); if (!page) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index e27225f8aeb27..2945433166224 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -268,9 +268,9 @@ void __init swiotlb_update_mem_attributes(void) } static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, - unsigned long nslabs, bool late_alloc, unsigned int nareas) + void *vaddr, unsigned long nslabs, bool late_alloc, + unsigned int nareas) { - void *vaddr = phys_to_virt(start); unsigned long bytes = nslabs << IO_TLB_SHIFT, i; mem->nslabs = nslabs; @@ -411,7 +411,7 @@ void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, return; } - swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, false, nareas); + swiotlb_init_io_tlb_pool(mem, __pa(tlb), tlb, nslabs, false, nareas); add_mem_pool(&io_tlb_default_mem, mem); if (flags & SWIOTLB_VERBOSE) @@ -509,7 +509,7 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask, set_memory_decrypted((unsigned long)vstart, (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), nslabs, true, + swiotlb_init_io_tlb_pool(mem, virt_to_phys(vstart), vstart, nslabs, true, nareas); add_mem_pool(&io_tlb_default_mem, mem); @@ -607,25 +607,26 @@ static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes, u64 phys_limit) * @bytes: Size of the buffer. * @phys_limit: Maximum allowed physical address of the buffer. * @gfp: GFP flags for the allocation. + * @vaddr: Receives the virtual address for the allocated buffer. * * Return: Allocated pages, or %NULL on allocation failure. */ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, - u64 phys_limit, gfp_t gfp) + u64 phys_limit, gfp_t gfp, void **vaddr) { struct page *page; + *vaddr = NULL; + /* * Allocate from the atomic pools if memory is encrypted and * the allocation is atomic, because decrypting may block. */ if (!gfpflags_allow_blocking(gfp) && dev && force_dma_unencrypted(dev)) { - void *vaddr; - if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL)) return NULL; - return dma_alloc_from_pool(dev, bytes, &vaddr, gfp, + return dma_alloc_from_pool(dev, bytes, vaddr, gfp, dma_coherent_ok); } @@ -647,6 +648,8 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes, return NULL; } + if (page) + *vaddr = phys_to_virt(page_to_phys(page)); return page; } @@ -687,6 +690,7 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, { struct io_tlb_pool *pool; unsigned int slot_order; + void *tlb_vaddr; struct page *tlb; size_t pool_size; size_t tlb_size; @@ -703,7 +707,8 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, pool->areas = (void *)pool + sizeof(*pool); tlb_size = nslabs << IO_TLB_SHIFT; - while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp))) { + while (!(tlb = swiotlb_alloc_tlb(dev, tlb_size, phys_limit, gfp, + &tlb_vaddr))) { if (nslabs <= minslabs) goto error_tlb; nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE); @@ -717,11 +722,12 @@ static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev, if (!pool->slots) goto error_slots; - swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas); + swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), tlb_vaddr, nslabs, + true, nareas); return pool; error_slots: - swiotlb_free_tlb(page_address(tlb), tlb_size); + swiotlb_free_tlb(tlb_vaddr, tlb_size); error_tlb: kfree(pool); error: @@ -1849,7 +1855,8 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem, set_memory_decrypted((unsigned long)phys_to_virt(rmem->base), rmem->size >> PAGE_SHIFT); - swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs, + swiotlb_init_io_tlb_pool(pool, rmem->base, phys_to_virt(rmem->base), + nslabs, false, nareas); mem->force_bounce = true; mem->for_alloc = true; diff --git a/kernel/entry/syscall-common.c b/kernel/entry/syscall-common.c index 66e6ba7fa80c8..dbc5551935f78 100644 --- a/kernel/entry/syscall-common.c +++ b/kernel/entry/syscall-common.c @@ -37,6 +37,9 @@ long syscall_trace_enter(struct pt_regs *regs, long syscall, ret = ptrace_report_syscall_entry(regs); if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) return -1L; + + /* ptrace might have changed work flags */ + work = READ_ONCE(current_thread_info()->syscall_work); } /* Do seccomp after ptrace, to catch any tracer changes. */ diff --git a/kernel/events/core.c b/kernel/events/core.c index f3bd7c97c3aab..ad52e4bda8ff9 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -2248,6 +2248,34 @@ static inline struct list_head *get_event_list(struct perf_event *event) &event->pmu_ctx->flexible_active; } +/* @sibling must already be unlinked from its old leader's sibling_list. */ +static void perf_promote_sibling_to_leader(struct perf_event *sibling, + struct perf_event_context *ctx, + int group_caps) +{ + /* + * Events that have PERF_EV_CAP_SIBLING require being part of + * a group and cannot exist on their own, schedule them out + * and move them into the ERROR state. Also see + * _perf_event_enable(), it will not be able to recover this + * ERROR state. + */ + if (sibling->event_caps & PERF_EV_CAP_SIBLING) + __event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR); + + sibling->group_leader = sibling; + sibling->group_caps = group_caps; + + if (sibling->attach_state & PERF_ATTACH_CONTEXT) { + add_event_to_groups(sibling, ctx); + + if (sibling->state == PERF_EVENT_STATE_ACTIVE) + list_add_tail(&sibling->active_list, get_event_list(sibling)); + } + + perf_event__header_size(sibling); +} + static void perf_group_detach(struct perf_event *event) { struct perf_event *leader = event->group_leader; @@ -2271,8 +2299,9 @@ static void perf_group_detach(struct perf_event *event) */ if (leader != event) { list_del_init(&event->sibling_list); - event->group_leader->nr_siblings--; - event->group_leader->group_generation++; + leader->nr_siblings--; + leader->group_generation++; + perf_promote_sibling_to_leader(event, ctx, event->event_caps); goto out; } @@ -2282,32 +2311,14 @@ static void perf_group_detach(struct perf_event *event) * to whatever list we are on. */ list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { - - /* - * Events that have PERF_EV_CAP_SIBLING require being part of - * a group and cannot exist on their own, schedule them out - * and move them into the ERROR state. Also see - * _perf_event_enable(), it will not be able to recover this - * ERROR state. - */ - if (sibling->event_caps & PERF_EV_CAP_SIBLING) - __event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR); - - sibling->group_leader = sibling; list_del_init(&sibling->sibling_list); /* Inherit group flags from the previous leader */ - sibling->group_caps = event->group_caps; - - if (sibling->attach_state & PERF_ATTACH_CONTEXT) { - add_event_to_groups(sibling, event->ctx); - - if (sibling->state == PERF_EVENT_STATE_ACTIVE) - list_add_tail(&sibling->active_list, get_event_list(sibling)); - } + perf_promote_sibling_to_leader(sibling, ctx, event->group_caps); WARN_ON_ONCE(sibling->ctx != event->ctx); } + event->nr_siblings = 0; out: for_each_sibling_event(tmp, leader) @@ -2494,12 +2505,7 @@ __perf_remove_from_context(struct perf_event *event, if (flags & DETACH_DEAD) state = PERF_EVENT_STATE_DEAD; - event_sched_out(event, ctx); - - if (event->state > PERF_EVENT_STATE_OFF) - perf_cgroup_event_disable(event, ctx); - - perf_event_set_state(event, min(event->state, state)); + __event_disable(event, ctx, state); if (flags & DETACH_GROUP) perf_group_detach(event); @@ -2568,8 +2574,9 @@ static void __event_disable(struct perf_event *event, enum perf_event_state state) { event_sched_out(event, ctx); - perf_cgroup_event_disable(event, ctx); - perf_event_set_state(event, state); + if (event->state > PERF_EVENT_STATE_OFF) + perf_cgroup_event_disable(event, ctx); + perf_event_set_state(event, min(event->state, state)); } /* @@ -3809,13 +3816,13 @@ static void perf_pmu_sched_task(struct task_struct *prev, bool sched_in) { struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); - struct perf_cpu_pmu_context *cpc; + struct perf_cpu_pmu_context *cpc, *cpc2; /* cpuctx->task_ctx will be handled in perf_event_context_sched_in/out */ if (prev == next || cpuctx->task_ctx) return; - list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry) + list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); } @@ -5161,6 +5168,7 @@ static void free_event_rcu(struct rcu_head *head) if (event->ns) put_pid_ns(event->ns); perf_event_free_filter(event); + kfree(event->addr_filter_ranges); kmem_cache_free(perf_event_cache, event); } @@ -5607,8 +5615,6 @@ static void __free_event(struct perf_event *event) if (event->attach_state & PERF_ATTACH_CALLCHAIN) put_callchain_buffers(); - kfree(event->addr_filter_ranges); - if (event->attach_state & PERF_ATTACH_EXCLUSIVE) exclusive_event_destroy(event); @@ -6730,6 +6736,7 @@ static void perf_mmap_open(struct vm_area_struct *vma) } static void perf_pmu_output_stop(struct perf_event *event); +static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb); /* * A buffer can be mmap()ed multiple times; either directly through the same @@ -6745,9 +6752,6 @@ static void perf_mmap_close(struct vm_area_struct *vma) mapped_f unmapped = get_mapped(event, event_unmapped); struct perf_buffer *rb = ring_buffer_get(event); struct user_struct *mmap_user = rb->mmap_user; - int mmap_locked = rb->mmap_locked; - unsigned long size = perf_data_size(rb); - bool detach_rest = false; /* FIXIES vs perf_pmu_unregister() */ if (unmapped) @@ -6778,17 +6782,18 @@ static void perf_mmap_close(struct vm_area_struct *vma) mutex_unlock(&rb->aux_mutex); } - if (refcount_dec_and_test(&rb->mmap_count)) - detach_rest = true; - - if (!refcount_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) - goto out_put; - - ring_buffer_attach(event, NULL); - mutex_unlock(&event->mmap_mutex); + /* + * Drop references in reverse order of perf_mmap() to prevent + * rb revival after rb->mmap_count reaches zero. + */ + if (refcount_dec_and_mutex_lock(&event->mmap_count, + &event->mmap_mutex)) { + ring_buffer_attach(event, NULL); + mutex_unlock(&event->mmap_mutex); + } /* If there's still other mmap()s of this buffer, we're done. */ - if (!detach_rest) + if (!refcount_dec_and_test(&rb->mmap_count)) goto out_put; /* @@ -6841,11 +6846,7 @@ static void perf_mmap_close(struct vm_area_struct *vma) * Aside from that, this buffer is 'fully' detached and unmapped, * undo the VM accounting. */ - - atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked, - &mmap_user->locked_vm); - atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm); - free_uid(mmap_user); + perf_mmap_unaccount(vma, rb); out_put: ring_buffer_put(rb); /* could be last */ @@ -6987,6 +6988,15 @@ static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long atomic64_add(extra, &vma->vm_mm->pinned_vm); } +static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb) +{ + struct user_struct *user = rb->mmap_user; + + atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, + &user->locked_vm); + atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); +} + static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, unsigned long nr_pages) { @@ -7049,8 +7059,6 @@ static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, if (!rb) return -ENOMEM; - refcount_set(&rb->mmap_count, 1); - rb->mmap_user = get_current_user(); rb->mmap_locked = extra; ring_buffer_attach(event, rb); @@ -7200,16 +7208,54 @@ static int perf_mmap(struct file *file, struct vm_area_struct *vma) mapped(event, vma->vm_mm); /* - * Try to map it into the page table. On fail, invoke - * perf_mmap_close() to undo the above, as the callsite expects - * full cleanup in this case and therefore does not invoke - * vmops::close(). + * Try to map it into the page table. On fail undo the above, + * as the callsite expects full cleanup in this case and + * therefore does not invoke vmops::close(). */ ret = map_range(event->rb, vma); - if (ret) - perf_mmap_close(vma); + if (likely(!ret)) + return 0; + + /* Error path */ + + /* + * If this is the first mmap(), then event->mmap_count should + * be stable at 1. It is only modified by: + * perf_mmap_{open,close}() and perf_mmap(). + * + * The former are not possible because this mmap() hasn't been + * successful yet, and the latter is serialized by + * event->mmap_mutex which we still hold (note that mmap_lock + * is not strictly sufficient here, because the event fd can + * be passed to another process through trivial means like + * fork(), leading to concurrent mmap() from different mm). + * + * Make sure to remove event->rb before releasing + * event->mmap_mutex, such that any concurrent mmap() will not + * attempt use this failed buffer. + */ + if (refcount_read(&event->mmap_count) == 1) { + /* + * Minimal perf_mmap_close(); there can't be AUX or + * other events on account of this being the first. + */ + mapped = get_mapped(event, event_unmapped); + if (mapped) + mapped(event, vma->vm_mm); + perf_mmap_unaccount(vma, event->rb); + ring_buffer_attach(event, NULL); /* drops last rb->refcount */ + refcount_set(&event->mmap_count, 0); + return ret; + } + + /* + * Otherwise this is an already existing buffer, and there is + * no race vs first exposure, so fall-through and call + * perf_mmap_close(). + */ } + perf_mmap_close(vma); return ret; } @@ -13542,7 +13588,7 @@ SYSCALL_DEFINE5(perf_event_open, goto err_fd; } group_leader = fd_file(group)->private_data; - if (group_leader->state <= PERF_EVENT_STATE_REVOKED) { + if (group_leader->state <= PERF_EVENT_STATE_EXIT) { err = -ENODEV; goto err_fd; } @@ -13673,6 +13719,12 @@ SYSCALL_DEFINE5(perf_event_open, if (group_leader->ctx != ctx) goto err_locked; + /* Recheck under ctx::mutex to serialize against remove-on-exec. */ + if (group_leader->state <= PERF_EVENT_STATE_EXIT) { + err = -ENODEV; + goto err_locked; + } + /* * Only a group leader can be exclusive or pinned */ diff --git a/kernel/events/internal.h b/kernel/events/internal.h index d9cc570830918..c03c4f2eea571 100644 --- a/kernel/events/internal.h +++ b/kernel/events/internal.h @@ -67,6 +67,7 @@ static inline void rb_free_rcu(struct rcu_head *rcu_head) struct perf_buffer *rb; rb = container_of(rcu_head, struct perf_buffer, rcu_head); + free_uid(rb->mmap_user); rb_free(rb); } diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c index 20a9050237362..6d805a7168be6 100644 --- a/kernel/events/ring_buffer.c +++ b/kernel/events/ring_buffer.c @@ -340,6 +340,8 @@ ring_buffer_init(struct perf_buffer *rb, long watermark, int flags) rb->paused = 1; mutex_init(&rb->aux_mutex); + rb->mmap_user = get_current_user(); + refcount_set(&rb->mmap_count, 1); } void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags) @@ -507,7 +509,10 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) /* * Only send RECORD_AUX if we have something useful to communicate * - * Note: the OVERWRITE records by themselves are not considered + * PMU_FORMAT bits identify the PMU type rather than an AUX event + * has occurred, so ignore them for zero-sized records. + * + * The OVERWRITE records by themselves are not considered * useful, as they don't communicate any *new* information, * aside from the short-lived offset, that becomes history at * the next event sched-in and therefore isn't useful. @@ -516,7 +521,9 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size) * offset. So, from now on we don't output AUX records that * have *only* OVERWRITE flag set. */ - if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE)) + if (size || + (handle->aux_flags & ~(u64)(PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK | + PERF_AUX_FLAG_OVERWRITE))) perf_event_aux_event(handle->event, aux_head, size, handle->aux_flags); diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 725c5772429dc..a5b31443550b5 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -830,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get) if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) { /* We won the race, we are the ones to unlock SRCU */ __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); - return get ? get_uprobe(uprobe) : uprobe; + return get && uprobe ? get_uprobe(uprobe) : uprobe; } /* diff --git a/kernel/exit.c b/kernel/exit.c index c832946823f48..d60c1241923a9 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -212,7 +212,12 @@ static void __exit_signal(struct release_task_post *post, struct task_struct *ts __unhash_process(post, tsk, group_dead); write_sequnlock(&sig->stats_lock); - tsk->sighand = NULL; + /* + * Ensure that all preceeding state is visible. Pairs with + * the smp_acquire__after_ctrl_dep() in the sighand == NULL + * path of lock_task_sighand(). + */ + smp_store_release(&tsk->sighand, NULL); spin_unlock(&sighand->siglock); __cleanup_sighand(sighand); @@ -259,8 +264,11 @@ void release_task(struct task_struct *p) pidfs_exit(p); cgroup_release(p); - /* Retrieve @thread_pid before __unhash_process() may set it to NULL. */ - thread_pid = task_pid(p); + /* + * Pin @thread_pid before __unhash_process() clears it. The last + * PIDTYPE detach can otherwise free it before proc_flush_pid(). + */ + thread_pid = get_pid(task_pid(p)); write_lock_irq(&tasklist_lock); ptrace_release_task(p); @@ -289,8 +297,8 @@ void release_task(struct task_struct *p) } write_unlock_irq(&tasklist_lock); - /* @thread_pid can't go away until free_pids() below */ proc_flush_pid(thread_pid); + put_pid(thread_pid); add_device_randomness(&p->se.sum_exec_runtime, sizeof(p->se.sum_exec_runtime)); free_pids(post.pids); @@ -968,7 +976,7 @@ void __noreturn do_exit(long code) exit_fs(tsk); if (group_dead) disassociate_ctty(1); - exit_task_namespaces(tsk); + exit_nsproxy_namespaces(tsk); exit_task_work(tsk); exit_thread(tsk); diff --git a/kernel/fork.c b/kernel/fork.c index 91f1512a1d312..295cf8cf719ea 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -2455,7 +2455,7 @@ __latent_entropy struct task_struct *copy_process( if (p->io_context) exit_io_context(p); bad_fork_cleanup_namespaces: - exit_task_namespaces(p); + exit_nsproxy_namespaces(p); bad_fork_cleanup_mm: if (p->mm) { mm_clear_owner(p->mm, p); diff --git a/kernel/futex/core.c b/kernel/futex/core.c index 9e7dea6fc0ccd..69f340cddfd1f 100644 --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -44,6 +44,7 @@ #include #include #include +#include #include "futex.h" #include "../locking/rtmutex_common.h" @@ -124,7 +125,7 @@ late_initcall(fail_futex_debugfs); #endif /* CONFIG_FAIL_FUTEX */ static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph); +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p); #ifdef CONFIG_FUTEX_PRIVATE_HASH static bool futex_ref_get(struct futex_private_hash *fph); @@ -133,15 +134,6 @@ static bool futex_ref_is_dead(struct futex_private_hash *fph); enum { FR_PERCPU = 0, FR_ATOMIC }; -static inline bool futex_key_is_private(union futex_key *key) -{ - /* - * Relies on get_futex_key() to set either bit for shared - * futexes -- see comment with union futex_key. - */ - return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); -} - static bool futex_private_hash_get(struct futex_private_hash *fph) { return futex_ref_get(fph); @@ -149,33 +141,14 @@ static bool futex_private_hash_get(struct futex_private_hash *fph) void futex_private_hash_put(struct futex_private_hash *fph) { - if (futex_ref_put(fph)) - wake_up_var(fph->mm); -} - -/** - * futex_hash_get - Get an additional reference for the local hash. - * @hb: ptr to the private local hash. - * - * Obtain an additional reference for the already obtained hash bucket. The - * caller must already own an reference. - */ -void futex_hash_get(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; + struct mm_struct *mm; if (!fph) return; - WARN_ON_ONCE(!futex_private_hash_get(fph)); -} - -void futex_hash_put(struct futex_hash_bucket *hb) -{ - struct futex_private_hash *fph = hb->priv; - if (!fph) - return; - futex_private_hash_put(fph); + mm = fph->mm; + if (futex_ref_put(fph)) + wake_up_var(mm); } static struct futex_hash_bucket * @@ -183,14 +156,6 @@ __futex_hash_private(union futex_key *key, struct futex_private_hash *fph) { u32 hash; - if (!futex_key_is_private(key)) - return NULL; - - if (!fph) - fph = rcu_dereference(key->private.mm->futex_phash); - if (!fph || !fph->hash_mask) - return NULL; - hash = jhash2((void *)&key->private.address, sizeof(key->private.address) / 4, key->both.offset); @@ -211,13 +176,12 @@ static void futex_rehash_private(struct futex_private_hash *old, spin_lock(&hb_old->lock); plist_for_each_entry_safe(this, tmp, &hb_old->chain, list) { - plist_del(&this->list, &hb_old->chain); futex_hb_waiters_dec(hb_old); WARN_ON_ONCE(this->lock_ptr != &hb_old->lock); - hb_new = __futex_hash(&this->key, new); + hb_new = __futex_hash(&this->key, new, NULL); futex_hb_waiters_inc(hb_new); /* * The new pointer isn't published yet but an already @@ -271,9 +235,8 @@ static void futex_pivot_hash(struct mm_struct *mm) } } -struct futex_private_hash *futex_private_hash(void) +struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { - struct mm_struct *mm = current->mm; /* * Ideally we don't loop. If there is a replacement in progress * then a new private hash is already prepared and a reference can't be @@ -299,18 +262,17 @@ struct futex_private_hash *futex_private_hash(void) goto again; } -struct futex_hash_bucket *futex_hash(union futex_key *key) +struct futex_bucket_ref futex_hash(union futex_key *key) { - struct futex_private_hash *fph; - struct futex_hash_bucket *hb; - again: scoped_guard(rcu) { - hb = __futex_hash(key, NULL); - fph = hb->priv; + struct futex_private_hash *fph = NULL; + struct futex_hash_bucket *hb; + + hb = __futex_hash(key, NULL, &fph); if (!fph || futex_private_hash_get(fph)) - return hb; + return (struct futex_bucket_ref){ .hb = hb, .fph = fph }; } futex_pivot_hash(key->private.mm); goto again; @@ -318,15 +280,9 @@ struct futex_hash_bucket *futex_hash(union futex_key *key) #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static struct futex_hash_bucket * -__futex_hash_private(union futex_key *key, struct futex_private_hash *fph) +struct futex_bucket_ref futex_hash(union futex_key *key) { - return NULL; -} - -struct futex_hash_bucket *futex_hash(union futex_key *key) -{ - return __futex_hash(key, NULL); + return (struct futex_bucket_ref){ .hb = __futex_hash(key, NULL, NULL), .fph = NULL }; } #endif /* CONFIG_FUTEX_PRIVATE_HASH */ @@ -404,6 +360,8 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * __futex_hash - Return the hash bucket * @key: Pointer to the futex key for which the hash is calculated * @fph: Pointer to private hash if known + * @fph_p: Pointer to a private hash pointer; output for the private hash + * used when set. * * We hash on the keys returned from get_futex_key (see below) and return the * corresponding hash bucket. @@ -412,18 +370,22 @@ static int futex_mpol(struct mm_struct *mm, unsigned long addr) * global hash is returned. */ static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph) +__futex_hash(union futex_key *key, struct futex_private_hash *fph, struct futex_private_hash **fph_p) { int node = key->both.node; u32 hash; - if (node == FUTEX_NO_NODE) { - struct futex_hash_bucket *hb; - - hb = __futex_hash_private(key, fph); - if (hb) - return hb; +#ifdef CONFIG_FUTEX_PRIVATE_HASH + if (node == FUTEX_NO_NODE && futex_key_is_private(key)) { + if (!fph) + fph = rcu_dereference(key->private.mm->futex_phash); + if (fph && fph->hash_mask) { + if (fph_p) + *fph_p = fph; + return __futex_hash_private(key, fph); + } } +#endif hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(u32), @@ -1028,8 +990,11 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, return -1; /* - * Special case for regular (non PI) futexes. The unlock path in - * user space has two race scenarios: + * Special case for regular (non PI) futexes. Ordinarily, we do + * not perform any processing here unless the current thread was + * the owner of the futex (by the TID check below). + * + * However, the unlock path has three race scenarios: * * 1. The unlock path releases the user space futex value and * before it can execute the futex() syscall to wake up @@ -1038,42 +1003,70 @@ static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr, * 2. A woken up waiter is killed before it can acquire the * futex in user space. * - * In the second case, the wake up notification could be generated - * by the unlock path in user space after setting the futex value - * to zero or by the kernel after setting the OWNER_DIED bit below. + * 3. A woken up waiter is killed in user space after another + * thread has acquired the futex, but before it can set + * FUTEX_WAITERS. + * + * Note that, if userspace uses the FUTEX_ROBUST_UNLOCK flag, we + * will not see case 1 here. * - * In both cases the TID validation below prevents a wakeup of - * potential waiters which can cause these waiters to block - * forever. + * In the second and third case, the wake up notification could + * be generated from any of: * - * In both cases the following conditions are met: + * i. An ordinary futex wakeup after unlock (with or + * without FUTEX_ROBUST_UNLOCK) + * ii. A robust wakeup from another thread's death + * iii. A previous round through this special case * - * 1) task->robust_list->list_op_pending != NULL - * @pending_op == true - * 2) The owner part of user space futex value == 0 + * As a result, the futex world will be in one of four states: + * + * A. The futex word is 0 (unlocked) + * B. The futex word is owned by another thread + * (FUTEX_WAITERS is not set) + * C. The futex word is owned by another thread + * (FUTEX_WAITERS set) + * D. The futex's owner died and OWNER_DIED is set + * (the owner part of the word is 0) + * + * The key issue is that the kernel usually (at least from + * sources ii. and iii. or when so requested by userspace from + * source i.) only ever wakes *one* waiter at a time. If this + * waiter dies before acquiring the futex (or setting the + * FUTEX_WAITERS bit), the kernel *must* still wake the next + * waiter down the line to uphold the futex invariants and + * avoid lost wakeups. Note we do not need to handle state C, + * as it does not matter to us whether *we* successfully set + * the bit or a third thread did so in the meantime. + * + * Therefore, in these cases we must issue an additional + * futex_wake(). Note however that we *must not* set OWNER_DIED + * here. Our thread is *not* the owner of the futex. + * + * Thus to summarize, the conditions for needing the additional + * futex_wake() are: + * + * 1) @pending_op == true (the thread has not finished the + * mutex operation) + * 2) The futex word is in one of the states A, B or D * 3) Regular futex: @pi == false * - * If these conditions are met, it is safe to attempt waking up a - * potential waiter without touching the user space futex value and - * trying to set the OWNER_DIED bit. If the futex value is zero, - * the rest of the user space mutex state is consistent, so a woken - * waiter will just take over the uncontended futex. Setting the - * OWNER_DIED bit would create inconsistent state and malfunction - * of the user space owner died handling. Otherwise, the OWNER_DIED - * bit is already set, and the woken waiter is expected to deal with - * this. + * Note in particular that in all of the states A-D the owner + * portion of the futex word differs from our thread's TID + * (unless the actual owner has the same TID in another PID + * namespace, but we cannot currently distinguish that + * scenario), so this can be a special-case wakeup in the bail + * path of the ordinary TID check. */ owner = uval & FUTEX_TID_MASK; - if (pending_op && !pi && !owner) { - futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, - FUTEX_BITSET_MATCH_ANY); + if (owner != task_pid_vnr(curr)) { + if (pending_op && !pi && (!owner || !(uval & FUTEX_WAITERS))) { + futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, 1, + FUTEX_BITSET_MATCH_ANY); + } return 0; } - if (owner != task_pid_vnr(curr)) - return 0; - /* * Ok, this dying thread is truly holding a futex * of interest. Set the OWNER_DIED bit atomically @@ -1337,7 +1330,7 @@ static void exit_pi_state_list(struct task_struct *curr) * on the mutex. */ WARN_ON(curr != current); - guard(private_hash)(); + guard(private_hash)(current->mm); /* * We are a ZOMBIE and nobody can enqueue itself on * pi_state_list anymore, but we have to be careful @@ -1349,7 +1342,8 @@ static void exit_pi_state_list(struct task_struct *curr) pi_state = list_entry(next, struct futex_pi_state, list); key = pi_state->key; if (1) { - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* * We can race against put_pi_state() removing itself from the @@ -1474,13 +1468,11 @@ static void futex_cleanup_begin(struct task_struct *tsk) raw_spin_unlock_irq(&tsk->pi_lock); } -static void futex_cleanup_end(struct task_struct *tsk, int state) +static void futex_cleanup_end(struct task_struct *tsk) { - /* - * Lockless store. The only side effect is that an observer might - * take another loop until it becomes visible. - */ - tsk->futex_state = state; + scoped_guard(raw_spinlock_irq, &tsk->pi_lock) + tsk->futex_state = FUTEX_STATE_DEAD; + /* * Drop the exit protection. This unblocks waiters which observed * FUTEX_STATE_EXITING to reevaluate the state. @@ -1488,37 +1480,53 @@ static void futex_cleanup_end(struct task_struct *tsk, int state) mutex_unlock(&tsk->futex_exit_mutex); } -void futex_exec_release(struct task_struct *tsk) +void futex_exit_release(struct task_struct *tsk) { - /* - * The state handling is done for consistency, but in the case of - * exec() there is no way to prevent further damage as the PID stays - * the same. But for the unlikely and arguably buggy case that a - * futex is held on exec(), this provides at least as much state - * consistency protection which is possible. - */ futex_cleanup_begin(tsk); futex_cleanup(tsk); + futex_cleanup_end(tsk); +} + +void futex_exec_release(struct task_struct *tsk) +{ /* - * Reset the state to FUTEX_STATE_OK. The task is alive and about - * exec a new binary. + * exec() makes it interesting for futexes because the TID of the task + * stays the same, but from a futex perspective the task has to be + * treated like an exiting task. This is especially important for the + * sanity check for private futexes in attach_to_pi_owner() which + * compares the owner's mm with the waiter's mm. + * + * That check would give the wrong answer if futex_cleanup_end() would + * set the state to FUTEX_STATE_OK as long as the task still has the old + * mm. + * + * After the task has switched to the new mm it sets it to + * FUTEX_STATE_OK again in futex_exec_done(). */ - futex_cleanup_end(tsk, FUTEX_STATE_OK); + futex_exit_release(tsk); } -void futex_exit_release(struct task_struct *tsk) +/* + * exec() has switched to the new mm. Futex operations are safe again. + */ +void futex_exec_done(struct task_struct *tsk) { - futex_cleanup_begin(tsk); - futex_cleanup(tsk); - futex_cleanup_end(tsk, FUTEX_STATE_DEAD); + /* + * This store does not have to take tsk::futex::exit_mutex because the + * phase where waiters block on it during state FUTEX_STATE_EXITING has + * been finished when futex_cleanup_end() set the state to + * FUTEX_STATE_DEAD. + * + * This transitions back from FUTEX_STATE_DEAD to FUTEX_STATE_OK. The + * ordering guarantee required here is that the previous store to + * tsk::mm in the calling code cannot be reordered against this store. + */ + guard(raw_spinlock_irq)(&tsk->pi_lock); + tsk->futex_state = FUTEX_STATE_OK; } -static void futex_hash_bucket_init(struct futex_hash_bucket *fhb, - struct futex_private_hash *fph) +static void futex_hash_bucket_init(struct futex_hash_bucket *fhb) { -#ifdef CONFIG_FUTEX_PRIVATE_HASH - fhb->priv = fph; -#endif atomic_set(&fhb->waiters, 0); plist_head_init(&fhb->chain); spin_lock_init(&fhb->lock); @@ -1743,12 +1751,12 @@ static bool futex_pivot_pending(struct mm_struct *mm) { struct futex_private_hash *fph; - guard(rcu)(); + guard(mutex)(&mm->futex_hash_lock); if (!mm->futex_phash_new) return true; - fph = rcu_dereference(mm->futex_phash); + fph = rcu_dereference_raw(mm->futex_phash); return futex_ref_is_dead(fph); } @@ -1799,14 +1807,18 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) } if (!mm->futex_ref) { + unsigned int __percpu *ref = alloc_percpu(unsigned int); + + if (!ref) + return -ENOMEM; + /* - * This will always be allocated by the first thread and - * therefore requires no locking. + * Tasks sharing the mm can run this concurrently, so take the + * initial reference before publishing the counter. */ - mm->futex_ref = alloc_percpu(unsigned int); - if (!mm->futex_ref) - return -ENOMEM; - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*ref); /* 0 -> 1 */ + if (cmpxchg(&mm->futex_ref, NULL, ref)) + free_percpu(ref); } fph = kvzalloc(struct_size(fph, queues, hash_slots), @@ -1819,14 +1831,38 @@ static int futex_hash_allocate(unsigned int hash_slots, unsigned int flags) fph->mm = mm; for (i = 0; i < hash_slots; i++) - futex_hash_bucket_init(&fph->queues[i], fph); + futex_hash_bucket_init(&fph->queues[i]); if (custom) { + struct wait_bit_queue_entry __wbq_entry; + struct wait_queue_head *__wq_head; + /* * Only let prctl() wait / retry; don't unduly delay clone(). */ again: - wait_var_event(mm, futex_pivot_pending(mm)); + __wq_head = __var_waitqueue(mm); + init_wait_var_entry(&__wbq_entry, mm, 0); + __wbq_entry.wq_entry.func = woken_wake_bit_function; + add_wait_queue(__wq_head, &__wbq_entry.wq_entry); + + /* + * add_wait_queue() futex_ref_put() + * MB (this) MB (implied) + * futex_pivot_pending() wake_up_var() + * waitqueue_active() + * + * Notably, it must not be possible to see + * !futex_pivot_pending() && !waitqueue_active(). + */ + smp_mb(); + + while (!futex_pivot_pending(mm) && + wait_woken(&__wbq_entry.wq_entry, TASK_UNINTERRUPTIBLE, + MAX_SCHEDULE_TIMEOUT)) + /* empty */; + + remove_wait_queue(__wq_head, &__wbq_entry.wq_entry); } scoped_guard(mutex, &mm->futex_hash_lock) { @@ -1998,7 +2034,7 @@ static int __init futex_init(void) BUG_ON(!table); for (i = 0; i < hashsize; i++) - futex_hash_bucket_init(&table[i], NULL); + futex_hash_bucket_init(&table[i]); futex_queues[n] = table; } diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 2cd57096c38e1..17893fe1ce400 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -126,6 +126,15 @@ static inline bool should_fail_futex(bool fshared) } #endif +static inline bool futex_key_is_private(union futex_key *key) +{ + /* + * Relies on get_futex_key() to set either bit for shared + * futexes -- see comment with union futex_key. + */ + return !(key->both.offset & (FUT_OFF_INODE | FUT_OFF_MMSHARED)); +} + /* * Hash buckets are shared by all the futex_keys that hash to the same * location. Each key may have multiple futex_q structures, one for each task @@ -135,7 +144,6 @@ struct futex_hash_bucket { atomic_t waiters; spinlock_t lock; struct plist_head chain; - struct futex_private_hash *priv; } ____cacheline_aligned_in_smp; /* @@ -175,7 +183,7 @@ typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); * @requeue_pi_key: the requeue_pi target futex key * @bitset: bitset for the optional bitmasked wakeup * @requeue_state: State field for futex_requeue_pi() - * @drop_hb_ref: Waiter should drop the extra hash bucket reference if true + * @drop_fph: Waiter should drop the extra private hash reference when set * @requeue_wait: RCU wait for futex_requeue_pi() (RT only) * * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so @@ -202,7 +210,7 @@ struct futex_q { union futex_key *requeue_pi_key; u32 bitset; atomic_t requeue_state; - bool drop_hb_ref; + struct futex_private_hash *drop_fph; #ifdef CONFIG_PREEMPT_RT struct rcuwait requeue_wait; #endif @@ -222,28 +230,29 @@ extern struct hrtimer_sleeper * futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, int flags, u64 range_ns); -extern struct futex_hash_bucket *futex_hash(union futex_key *key); -#ifdef CONFIG_FUTEX_PRIVATE_HASH -extern void futex_hash_get(struct futex_hash_bucket *hb); -extern void futex_hash_put(struct futex_hash_bucket *hb); +struct futex_bucket_ref { + struct futex_hash_bucket *hb; + struct futex_private_hash *fph; +}; -extern struct futex_private_hash *futex_private_hash(void); +#ifdef CONFIG_FUTEX_PRIVATE_HASH +extern struct futex_private_hash *futex_private_hash(struct mm_struct *mm); extern void futex_private_hash_put(struct futex_private_hash *fph); #else /* !CONFIG_FUTEX_PRIVATE_HASH */ -static inline void futex_hash_get(struct futex_hash_bucket *hb) { } -static inline void futex_hash_put(struct futex_hash_bucket *hb) { } -static inline struct futex_private_hash *futex_private_hash(void) { return NULL; } +static inline struct futex_private_hash *futex_private_hash(struct mm_struct *mm) { return NULL; } static inline void futex_private_hash_put(struct futex_private_hash *fph) { } #endif -DEFINE_CLASS(hb, struct futex_hash_bucket *, - if (_T) futex_hash_put(_T), +extern struct futex_bucket_ref futex_hash(union futex_key *key); + +DEFINE_CLASS(hbr, struct futex_bucket_ref, + if (_T.fph) futex_private_hash_put(_T.fph), futex_hash(key), union futex_key *key); DEFINE_CLASS(private_hash, struct futex_private_hash *, if (_T) futex_private_hash_put(_T), - futex_private_hash(), void); + futex_private_hash(mm), struct mm_struct *mm); /** * futex_match - Check whether two futex keys are equal diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c index 64cb87d3a73e8..86f972c10fe8b 100644 --- a/kernel/futex/pi.c +++ b/kernel/futex/pi.c @@ -193,6 +193,58 @@ void put_pi_state(struct futex_pi_state *pi_state) * pi_mutex->wait_lock * p->pi_lock * + * Futex kernel state: + * + * The kernel tracks the task state in p::futex::state to protect against exit() + * and exec(). The states are: + * + * - FUTEX_STATE_OK when the task is alive and waiters can be attached + * + * - FUTEX_STATE_EXITING when the task cleans up the robust list and PI + * state. Concurrent waiters cannot attach anymore and have to wait until the + * cleanup is finished to re-evaluate the potential changes caused by the + * robust list and PI state cleanups. + * + * - FUTEX_STATE_DEAD when the task has cleaned up the robust list. This state + * is set independent of exit() or exec(). In the exit() case the task is + * gone. In the exec() case this ensures that nothing can attach to the task + * after cleaning up the robust list and PI state before it has switched to + * the new mm. From a futex point of view the task is dead until it sets the + * state to FUTEX_STATE_OK again after switching to the new mm. + * + * The valid state transitions for exit(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD + * + * The valid state transitions for exec(): + * + * FUTEX_STATE_OK -> FUTEX_STATE_EXITING -> FUTEX_STATE_DEAD -> FUTEX_STATE_OK + * + * The state has two related locks: + * + * 1) p::pi_lock + * + * p::pi_lock has to be taken by the waiter when evaluating the state to + * protect against a concurrent exit/exec cleanup by the owner. If the state + * is OK then the waiter can be attached to the owner while still holding + * pi_lock. + * + * The cleanup code has to hold it for all state transitions to ensure that + * the stores to the state cannot be reordered against previous stores on + * which the waiter correctness depends on. + * + * 2) p::futex::exit_mutex + * + * The mutex is acquired when the cleanup starts and released at the end. It + * obviously is not serializing the owner's cleanup against itself. It is + * used to avoid a live lock caused by a waiter preempting the owner's + * cleanup. Such a waiter would busy loop forever waiting for the owner to + * finish the cleanup. + * + * To prevent this, waiters have to drop all locks when observing + * FUTEX_STATE_EXITING and block on the mutex. When the owner releases the + * mutex after finishing the cleanup the waiters make progress and + * re-evaluate the situation. */ /* @@ -318,18 +370,10 @@ static int attach_to_pi_state(u32 __user *uaddr, u32 uval, return ret; } -static int handle_exit_race(u32 __user *uaddr, u32 uval, - struct task_struct *tsk) +static int handle_exit_race(u32 __user *uaddr, u32 uval) { u32 uval2; - /* - * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the - * caller that the alleged owner is busy. - */ - if (tsk && tsk->futex_state != FUTEX_STATE_DEAD) - return -EBUSY; - /* * Reread the user space value to handle the following situation: * @@ -426,7 +470,7 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, return -EAGAIN; p = find_get_task_by_vpid(pid); if (!p) - return handle_exit_race(uaddr, uval, NULL); + return handle_exit_race(uaddr, uval); if (unlikely(p->flags & PF_KTHREAD)) { put_task_struct(p); @@ -434,34 +478,55 @@ static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key, } /* - * We need to look at the task state to figure out, whether the - * task is exiting. To protect against the change of the task state - * in futex_exit_release(), we do this protected by p->pi_lock: + * We need to look at the task state to figure out whether the task is + * exiting. To protect against the change of the task state from + * FUTEX_STATE_OK to FUTEX_STATE_EXISTING in futex_cleanup_begin() it is + * required to do this protected by p->pi_lock, which prevents the owner + * from concurrently starting the exit cleanup. + * + * If the state is FUTEX_STATE_OK pi_lock must be held until the waiter + * is attached to protect against a concurrent exit()/exec(). */ raw_spin_lock_irq(&p->pi_lock); + + /* Validate that the task is ready for futex operations. */ if (unlikely(p->futex_state != FUTEX_STATE_OK)) { /* - * The task is on the way out. When the futex state is - * FUTEX_STATE_DEAD, we know that the task has finished - * the cleanup: + * The task is on the way out. When state is FUTEX_STATE_EXITING + * the cleanup is in progress. To avoid a live lock when the + * waiter preempted the owner, store the task pointer in + * @exiting and keep the reference on the task. The calling code + * will drop all locks, block on @p::futex::exit_mutex and wait + * for the owner to finish the cleanup. Once the owner released + * the mutex the waiter drops the reference count and + * re-evaluates the situation. */ - int ret = handle_exit_race(uaddr, uval, p); + if (p->futex_state == FUTEX_STATE_EXITING) { + raw_spin_unlock_irq(&p->pi_lock); + *exiting = p; + return -EBUSY; + } + + int ret = handle_exit_race(uaddr, uval); raw_spin_unlock_irq(&p->pi_lock); + put_task_struct(p); + return ret; + } + + if (IS_ENABLED(CONFIG_MMU) && futex_key_is_private(key)) { /* - * If the owner task is between FUTEX_STATE_EXITING and - * FUTEX_STATE_DEAD then store the task pointer and keep - * the reference on the task struct. The calling code will - * drop all locks, wait for the task to reach - * FUTEX_STATE_DEAD and then drop the refcount. This is - * required to prevent a live lock when the current task - * preempted the exiting task between the two states. + * A private futex key holds a pointer to the waiter's mm + * without holding a reference on it. So it must not be attached + * to an owner in a different address space. Otherwise that + * owner's exit cleanup could access the private hash after the + * key's mm is freed. */ - if (ret == -EBUSY) - *exiting = p; - else + if (unlikely(p->mm != key->private.mm)) { + raw_spin_unlock_irq(&p->pi_lock); put_task_struct(p); - return ret; + return -EPERM; + } } __attach_to_pi_owner(p, key, ps); @@ -940,7 +1005,8 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl retry_private: if (1) { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; futex_q_lock(&q, hb); @@ -998,17 +1064,11 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * Caution; releasing @hb in-scope. The hb->lock is still locked * while the reference is dropped. The reference can not be dropped * after the unlock because if a user initiated resize is in progress - * then we might need to wake him. This can not be done after the - * rt_mutex_pre_schedule() invocation. The hb will remain valid because - * the thread, performing resize, will block on hb->lock during - * the requeue. + * then we might need to wake him. The hb will remain valid + * because the thread, performing resize, will block on + * hb->lock during the requeue. */ - futex_hash_put(no_free_ptr(hb)); - /* - * Must be done before we enqueue the waiter, here is unfortunately - * under the hb lock, but that *should* work because it does nothing. - */ - rt_mutex_pre_schedule(); + futex_private_hash_put(no_free_ptr(hbr.fph)); rt_mutex_init_waiter(&rt_waiter); @@ -1074,10 +1134,6 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl * the */ futex_q_lockptr_lock(&q); - /* - * Waiter is unqueued. - */ - rt_mutex_post_schedule(); no_block: /* * Fixup the pi_state owner and possibly acquire the lock if we @@ -1093,11 +1149,9 @@ int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ktime_t *time, int tryl futex_unqueue_pi(&q); spin_unlock(q.lock_ptr); - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from futex_unlock_pi() */ - futex_hash_put(hb); - } + + /* Additional reference from futex_unlock_pi() */ + futex_private_hash_put(q.drop_fph); goto out; out_unlock_put_key: @@ -1153,7 +1207,8 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) if (ret) return ret; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; spin_lock(&hb->lock); retry_hb: @@ -1210,8 +1265,9 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags) * Acquire a reference for the leaving waiter to ensure * valid futex_q::lock_ptr. */ - futex_hash_get(hb); - top_waiter->drop_hb_ref = true; + if (futex_key_is_private(&key)) + top_waiter->drop_fph = futex_private_hash(key.private.mm); + __futex_unqueue(top_waiter); raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock); goto retry_hb; diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index b597cb3d17fc1..b3f4a4bccb126 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -154,8 +154,16 @@ static inline void futex_requeue_pi_complete(struct futex_q *q, int locked) } while (!atomic_try_cmpxchg(&q->requeue_state, &old, new)); #ifdef CONFIG_PREEMPT_RT - /* If the waiter interleaved with the requeue let it know */ - if (unlikely(old == Q_REQUEUE_PI_WAIT)) + /* + * The waiter in futex_requeue_pi_wakeup_sync() can interleave with the + * wake below: It will assign Q_REQUEUE_PI_IN_PROGRESS and here it will + * be updated to Q_REQUEUE_PI_LOCKED (locked = 1). The rcuwait_wait_event() + * will already read Q_REQUEUE_PI_LOCKED and skip the schedule() invocation, + * leading to an access of futex_q::requeue_wait after the waiter returned. + * In this case only we skip the wake here and rely on following wake in + * requeue_pi_wake_futex() to perform the wake if needed. + */ + if (unlikely(old == Q_REQUEUE_PI_WAIT) && new != Q_REQUEUE_PI_LOCKED) rcuwait_wake_up(&q->requeue_wait); #endif } @@ -241,8 +249,8 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key, * Acquire a reference for the waiter to ensure valid * futex_q::lock_ptr. */ - futex_hash_get(hb); - q->drop_hb_ref = true; + if (futex_key_is_private(key)) + q->drop_fph = futex_private_hash(key->private.mm); q->lock_ptr = &hb->lock; task = READ_ONCE(q->task); @@ -459,8 +467,10 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1, retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; futex_hb_waiters_inc(hb2); double_lock_hb(hb1, hb2); @@ -832,7 +842,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, switch (futex_requeue_pi_wakeup_sync(&q)) { case Q_REQUEUE_PI_IGNORE: { - CLASS(hb, hb)(&q.key); + CLASS(hbr, hbr)(&q.key); + auto hb = hbr.hb; /* The waiter is still on uaddr1 */ spin_lock(&hb->lock); ret = handle_early_requeue_pi_wakeup(hb, &q, to); @@ -902,11 +913,8 @@ int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, default: BUG(); } - if (q.drop_hb_ref) { - CLASS(hb, hb)(&q.key); - /* Additional reference from requeue_pi_wake_futex() */ - futex_hash_put(hb); - } + /* Additional reference from requeue_pi_wake_futex() */ + futex_private_hash_put(q.drop_fph); out: if (to) { diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index e2bbe5509ec27..67eeaabbeb3d9 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -169,7 +169,8 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) if ((flags & FLAGS_STRICT) && !nr_wake) return 0; - CLASS(hb, hb)(&key); + CLASS(hbr, hbr)(&key); + auto hb = hbr.hb; /* Make sure we really have tasks to wakeup */ if (!futex_hb_waiters_pending(hb)) @@ -266,8 +267,10 @@ int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2, retry_private: if (1) { - CLASS(hb, hb1)(&key1); - CLASS(hb, hb2)(&key2); + CLASS(hbr, hbr1)(&key1); + CLASS(hbr, hbr2)(&key2); + auto hb1 = hbr1.hb; + auto hb2 = hbr2.hb; double_lock_hb(hb1, hb2); op_ret = futex_atomic_op_inuser(op, uaddr2); @@ -409,7 +412,7 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) * Make sure to have a reference on the private_hash such that we * don't block on rehash after changing the task state below. */ - guard(private_hash)(); + guard(private_hash)(current->mm); /* * Enqueuing multiple futexes is tricky, because we need to enqueue @@ -446,7 +449,8 @@ int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) u32 val = vs[i].w.val; if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); ret = futex_get_value_locked(&uval, uaddr); @@ -620,7 +624,8 @@ int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, retry_private: if (1) { - CLASS(hb, hb)(&q->key); + CLASS(hbr, hbr)(&q->key); + auto hb = hbr.hb; futex_q_lock(q, hb); diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c index bf59e37d650ad..3cd0c40282c0d 100644 --- a/kernel/irq/generic-chip.c +++ b/kernel/irq/generic-chip.c @@ -650,7 +650,7 @@ static struct irq_data *irq_gc_get_irq_data(struct irq_chip_generic *gc) } #ifdef CONFIG_PM -static int irq_gc_suspend(void) +static int irq_gc_suspend(void *data) { struct irq_chip_generic *gc; @@ -670,7 +670,7 @@ static int irq_gc_suspend(void) return 0; } -static void irq_gc_resume(void) +static void irq_gc_resume(void *data) { struct irq_chip_generic *gc; @@ -693,7 +693,7 @@ static void irq_gc_resume(void) #define irq_gc_resume NULL #endif -static void irq_gc_shutdown(void) +static void irq_gc_shutdown(void *data) { struct irq_chip_generic *gc; @@ -709,15 +709,19 @@ static void irq_gc_shutdown(void) } } -static struct syscore_ops irq_gc_syscore_ops = { +static const struct syscore_ops irq_gc_syscore_ops = { .suspend = irq_gc_suspend, .resume = irq_gc_resume, .shutdown = irq_gc_shutdown, }; +static struct syscore irq_gc_syscore = { + .ops = &irq_gc_syscore_ops, +}; + static int __init irq_gc_init_ops(void) { - register_syscore_ops(&irq_gc_syscore_ops); + register_syscore(&irq_gc_syscore); return 0; } device_initcall(irq_gc_init_ops); diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index c09751b7a0c49..881d95c5af64d 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -1987,24 +1987,30 @@ const void *free_irq(unsigned int irq, void *dev_id) } EXPORT_SYMBOL(free_irq); -/* This function must be called with desc->lock held */ static const void *__cleanup_nmi(unsigned int irq, struct irq_desc *desc) { + struct irqaction *action = NULL; const char *devname = NULL; - desc->istate &= ~IRQS_NMI; + scoped_guard(raw_spinlock_irqsave, &desc->lock) { + irq_nmi_teardown(desc); - if (!WARN_ON(desc->action == NULL)) { - irq_pm_remove_action(desc, desc->action); - devname = desc->action->name; - unregister_handler_proc(irq, desc->action); + desc->istate &= ~IRQS_NMI; - kfree(desc->action); + if (!WARN_ON(desc->action == NULL)) { + action = desc->action; + irq_pm_remove_action(desc, action); + devname = action->name; + } desc->action = NULL; + + irq_settings_clr_disable_unlazy(desc); + irq_shutdown_and_deactivate(desc); } - irq_settings_clr_disable_unlazy(desc); - irq_shutdown_and_deactivate(desc); + if (action) + unregister_handler_proc(irq, action); + kfree(action); irq_release_resources(desc); @@ -2028,8 +2034,6 @@ const void *free_nmi(unsigned int irq, void *dev_id) if (WARN_ON(desc->depth == 0)) disable_nmi_nosync(irq); - guard(raw_spinlock_irqsave)(&desc->lock); - irq_nmi_teardown(desc); return __cleanup_nmi(irq, desc); } @@ -2279,13 +2283,14 @@ int request_nmi(unsigned int irq, irq_handler_t handler, /* Setup NMI state */ desc->istate |= IRQS_NMI; retval = irq_nmi_setup(desc); - if (retval) { - __cleanup_nmi(irq, desc); - return -EINVAL; - } - return 0; } + if (retval) { + __cleanup_nmi(irq, desc); + return -EINVAL; + } + return 0; + err_irq_setup: irq_chip_pm_put(&desc->irq_data); err_out: diff --git a/kernel/irq/pm.c b/kernel/irq/pm.c index f7394729cedc3..99ff65466d871 100644 --- a/kernel/irq/pm.c +++ b/kernel/irq/pm.c @@ -211,21 +211,26 @@ void rearm_wake_irq(unsigned int irq) /** * irq_pm_syscore_resume - enable interrupt lines early + * @data: syscore context * * Enable all interrupt lines with %IRQF_EARLY_RESUME set. */ -static void irq_pm_syscore_resume(void) +static void irq_pm_syscore_resume(void *data) { resume_irqs(true); } -static struct syscore_ops irq_pm_syscore_ops = { +static const struct syscore_ops irq_pm_syscore_ops = { .resume = irq_pm_syscore_resume, }; +static struct syscore irq_pm_syscore = { + .ops = &irq_pm_syscore_ops, +}; + static int __init irq_pm_init_ops(void) { - register_syscore_ops(&irq_pm_syscore_ops); + register_syscore(&irq_pm_syscore); return 0; } diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index 29c2404e743be..0537d330abb82 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -323,7 +324,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action) #undef MAX_NAMELEN -#define MAX_NAMELEN 10 +#define MAX_NAMELEN 11 void register_irq_proc(unsigned int irq, struct irq_desc *desc) { @@ -345,7 +346,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc) return; /* create /proc/irq/1234 */ - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); desc->dir = proc_mkdir(name, root_irq_dir); if (!desc->dir) return; @@ -398,7 +399,7 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc) #endif remove_proc_entry("spurious", desc->dir); - sprintf(name, "%u", irq); + snprintf(name, MAX_NAMELEN, "%u", irq); remove_proc_entry(name, root_irq_dir); } diff --git a/kernel/kcov.c b/kernel/kcov.c index ab8678b7b5e0c..c0d2357c9046e 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -86,17 +86,12 @@ struct kcov_remote { static DEFINE_SPINLOCK(kcov_remote_lock); static DEFINE_HASHTABLE(kcov_remote_map, 4); -static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas); +static struct list_head kcov_remote_areas[2] = { + LIST_HEAD_INIT(kcov_remote_areas[0]), LIST_HEAD_INIT(kcov_remote_areas[1]) +}; struct kcov_percpu_data { - void *irq_area; local_lock_t lock; - - unsigned int saved_mode; - unsigned int saved_size; - void *saved_area; - struct kcov *saved_kcov; - int saved_sequence; }; static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = { @@ -132,12 +127,13 @@ static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle) } /* Must be called with kcov_remote_lock locked. */ -static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) +static struct kcov_remote_area *kcov_remote_area_get(unsigned int size, bool irq) { struct kcov_remote_area *area; struct list_head *pos; + struct list_head *list = &kcov_remote_areas[irq]; - list_for_each(pos, &kcov_remote_areas) { + list_for_each(pos, list) { area = list_entry(pos, struct kcov_remote_area, list); if (area->size == size) { list_del(&area->list); @@ -149,11 +145,11 @@ static struct kcov_remote_area *kcov_remote_area_get(unsigned int size) /* Must be called with kcov_remote_lock locked. */ static void kcov_remote_area_put(struct kcov_remote_area *area, - unsigned int size) + unsigned int size, bool irq) { INIT_LIST_HEAD(&area->list); area->size = size; - list_add(&area->list, &kcov_remote_areas); + list_add(&area->list, &kcov_remote_areas[irq]); /* * KMSAN doesn't instrument this file, so it may not know area->list * is initialized. Unpoison it explicitly to avoid reports in @@ -388,6 +384,12 @@ void kcov_task_init(struct task_struct *t) { kcov_task_reset(t); t->kcov_handle = current->kcov_handle; + t->kcov_softirq = 0; + t->kcov_saved_mode = 0; + t->kcov_saved_size = 0; + t->kcov_saved_area = NULL; + t->kcov_saved_kcov = NULL; + t->kcov_saved_sequence = 0; } static void kcov_reset(struct kcov *kcov) @@ -815,34 +817,31 @@ static inline bool kcov_mode_enabled(unsigned int mode) static void kcov_remote_softirq_start(struct task_struct *t) { - struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); unsigned int mode; mode = READ_ONCE(t->kcov_mode); barrier(); if (kcov_mode_enabled(mode)) { - data->saved_mode = mode; - data->saved_size = t->kcov_size; - data->saved_area = t->kcov_area; - data->saved_sequence = t->kcov_sequence; - data->saved_kcov = t->kcov; + t->kcov_saved_mode = mode; + t->kcov_saved_size = t->kcov_size; + t->kcov_saved_area = t->kcov_area; + t->kcov_saved_sequence = t->kcov_sequence; + t->kcov_saved_kcov = t->kcov; kcov_stop(t); } } static void kcov_remote_softirq_stop(struct task_struct *t) { - struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); - - if (data->saved_kcov) { - kcov_start(t, data->saved_kcov, data->saved_size, - data->saved_area, data->saved_mode, - data->saved_sequence); - data->saved_mode = 0; - data->saved_size = 0; - data->saved_area = NULL; - data->saved_sequence = 0; - data->saved_kcov = NULL; + if (t->kcov_saved_kcov) { + kcov_start(t, t->kcov_saved_kcov, t->kcov_saved_size, + t->kcov_saved_area, t->kcov_saved_mode, + t->kcov_saved_sequence); + t->kcov_saved_mode = 0; + t->kcov_saved_size = 0; + t->kcov_saved_area = NULL; + t->kcov_saved_sequence = 0; + t->kcov_saved_kcov = NULL; } } @@ -903,17 +902,17 @@ void kcov_remote_start(u64 handle) sequence = kcov->sequence; if (in_task()) { size = kcov->remote_size; - area = kcov_remote_area_get(size); + area = kcov_remote_area_get(size, false); } else { size = CONFIG_KCOV_IRQ_AREA_SIZE; - area = this_cpu_ptr(&kcov_percpu_data)->irq_area; + area = kcov_remote_area_get(size, true); } spin_unlock(&kcov_remote_lock); - /* Can only happen when in_task(). */ + /* Allocate new buffer if we can sleep. */ if (!area) { local_unlock_irqrestore(&kcov_percpu_data.lock, flags); - area = vmalloc(size * sizeof(unsigned long)); + area = in_task() ? vmalloc(size * sizeof(unsigned long)) : NULL; if (!area) { kcov_put(kcov); return; @@ -1055,11 +1054,9 @@ void kcov_remote_stop(void) kcov_move_area(kcov->mode, kcov->area, kcov->size, area); spin_unlock(&kcov->lock); - if (in_task()) { - spin_lock(&kcov_remote_lock); - kcov_remote_area_put(area, size); - spin_unlock(&kcov_remote_lock); - } + spin_lock(&kcov_remote_lock); + kcov_remote_area_put(area, size, !in_task()); + spin_unlock(&kcov_remote_lock); local_unlock_irqrestore(&kcov_percpu_data.lock, flags); @@ -1105,14 +1102,21 @@ static void __init selftest(void) static int __init kcov_init(void) { - int cpu; + int cpu = num_possible_cpus(); + +#ifdef CONFIG_PREEMPT_RT + /* Allocate some extra buffers in order to prepare for softirq preemption. */ + cpu = cpu >= 4 ? cpu * 2 : cpu + 4; +#endif + while (cpu--) { + void *area = vmalloc(CONFIG_KCOV_IRQ_AREA_SIZE * sizeof(unsigned long)); + unsigned long flags; - for_each_possible_cpu(cpu) { - void *area = vmalloc_node(CONFIG_KCOV_IRQ_AREA_SIZE * - sizeof(unsigned long), cpu_to_node(cpu)); if (!area) return -ENOMEM; - per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area; + spin_lock_irqsave(&kcov_remote_lock, flags); + kcov_remote_area_put(area, CONFIG_KCOV_IRQ_AREA_SIZE, true); + spin_unlock_irqrestore(&kcov_remote_lock, flags); } /* diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c index 8a7baf4e332e3..2db82661cd60a 100644 --- a/kernel/kcsan/core.c +++ b/kernel/kcsan/core.c @@ -585,8 +585,14 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned * information is lost if dirtied by KCSAN. */ kcsan_save_irqtrace(current); - if (!interrupt_watcher) + if (!interrupt_watcher) { local_irq_save(irq_flags); + /* + * NMIs can still fire, disable checking for all interrupt + * contexts. + */ + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count++; + } watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); if (watchpoint == NULL) { @@ -699,8 +705,10 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); out_unlock: - if (!interrupt_watcher) + if (!interrupt_watcher) { + raw_cpu_ptr(&kcsan_cpu_ctx)->disable_count--; local_irq_restore(irq_flags); + } kcsan_restore_irqtrace(current); ctx->disable_scoped--; diff --git a/kernel/kexec_handover.c b/kernel/kexec_handover.c index c13b99f7c9891..e1a06b4fc17fc 100644 --- a/kernel/kexec_handover.c +++ b/kernel/kexec_handover.c @@ -562,20 +562,30 @@ early_param("kho_scratch", kho_parse_scratch_size); static void __init scratch_size_update(void) { - phys_addr_t size; + /* + * If fixed sizes are not provided via command line, calculate them + * now. + */ + if (scratch_scale) { + phys_addr_t size; - if (!scratch_scale) - return; + size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, + NUMA_NO_NODE); + size = size * scratch_scale / 100; + scratch_size_lowmem = size; - size = memblock_reserved_kern_size(ARCH_LOW_ADDRESS_LIMIT, - NUMA_NO_NODE); - size = size * scratch_scale / 100; - scratch_size_lowmem = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, + NUMA_NO_NODE); + size = size * scratch_scale / 100 - scratch_size_lowmem; + scratch_size_global = size; + } - size = memblock_reserved_kern_size(MEMBLOCK_ALLOC_ANYWHERE, - NUMA_NO_NODE); - size = size * scratch_scale / 100 - scratch_size_lowmem; - scratch_size_global = round_up(size, CMA_MIN_ALIGNMENT_BYTES); + /* + * Scratch areas are released as MIGRATE_CMA. Round them up to the right + * size. + */ + scratch_size_lowmem = round_up(scratch_size_lowmem, CMA_MIN_ALIGNMENT_BYTES); + scratch_size_global = round_up(scratch_size_global, CMA_MIN_ALIGNMENT_BYTES); } static phys_addr_t __init scratch_size_node(int nid) diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 87e6f4d61b95b..caf335bd5628d 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -1373,8 +1373,14 @@ static bool __within_kprobe_blacklist(unsigned long addr) /* * If 'kprobe_blacklist' is defined, check the address and * reject any probe registration in the prohibited area. + * Note: this can return true during transition period where + * (start_addr, end_addr) in the black list is shrinking + * but old entry has not been removed yet. This is acceptable + * because the worst case is that we reject more probes than + * we should. */ - list_for_each_entry(ent, &kprobe_blacklist, list) { + guard(rcu)(); + list_for_each_entry_rcu(ent, &kprobe_blacklist, list) { if (addr >= ent->start_addr && addr < ent->end_addr) return true; } @@ -2435,7 +2441,7 @@ int kprobe_add_ksym_blacklist(unsigned long entry) ent->start_addr = entry; ent->end_addr = entry + size; INIT_LIST_HEAD(&ent->list); - list_add_tail(&ent->list, &kprobe_blacklist); + list_add_tail_rcu(&ent->list, &kprobe_blacklist); return (int)size; } @@ -2529,8 +2535,8 @@ static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end) list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) { if (ent->start_addr < start || ent->start_addr >= end) continue; - list_del(&ent->list); - kfree(ent); + list_del_rcu(&ent->list); + kfree_rcu(ent, rcu); } } diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 2d4c5bab5af88..06f025abe8875 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -947,6 +947,34 @@ look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) return NULL; } +static __always_inline bool lock_class_cache_is_valid(const struct lockdep_map *lock, + const struct lock_class *class, + unsigned int subclass) +{ + unsigned int class_subclass; + + if (!class) + return false; + + if (unlikely(class < lock_classes || class >= lock_classes + MAX_LOCKDEP_KEYS)) + return false; + + if (unlikely(!arch_test_bit(class - lock_classes, lock_classes_in_use))) + return false; + + if (unlikely(!lock->key)) + return false; + + class_subclass = subclass ? subclass : class->subclass; + if (unlikely(class_subclass >= MAX_LOCKDEP_SUBCLASSES)) + return false; + + if (unlikely(READ_ONCE(class->key) != lock->key->subkeys + class_subclass)) + return false; + + return true; +} + /* * Static locks do not have their class-keys yet - for them the key is * the lock object itself. If the lock is in the per cpu area, the @@ -1379,9 +1407,9 @@ register_lock_class(struct lockdep_map *lock, unsigned int subclass, int force) out_set_class_cache: if (!subclass || force) - lock->class_cache[0] = class; + WRITE_ONCE(lock->class_cache[0], class); else if (subclass < NR_LOCKDEP_CACHING_CLASSES) - lock->class_cache[subclass] = class; + WRITE_ONCE(lock->class_cache[subclass], class); /* * Hash collision, did we smoke some? We found a class with a matching @@ -4941,7 +4969,7 @@ void lockdep_init_map_type(struct lockdep_map *lock, const char *name, int i; for (i = 0; i < NR_LOCKDEP_CACHING_CLASSES; i++) - lock->class_cache[i] = NULL; + WRITE_ONCE(lock->class_cache[i], NULL); #ifdef CONFIG_LOCK_STAT lock->cpu = raw_smp_processor_id(); @@ -5006,12 +5034,15 @@ EXPORT_SYMBOL_GPL(__lockdep_no_track__); void lockdep_set_lock_cmp_fn(struct lockdep_map *lock, lock_cmp_fn cmp_fn, lock_print_fn print_fn) { - struct lock_class *class = lock->class_cache[0]; + struct lock_class *class = READ_ONCE(lock->class_cache[0]); unsigned long flags; raw_local_irq_save(flags); lockdep_recursion_inc(); + if (!lock_class_cache_is_valid(lock, class, 0)) + class = NULL; + if (!class) class = register_lock_class(lock, 0, 0); @@ -5103,8 +5134,11 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, if (DEBUG_LOCKS_WARN_ON(subclass >= MAX_LOCKDEP_SUBCLASSES)) return 0; - if (subclass < NR_LOCKDEP_CACHING_CLASSES) - class = lock->class_cache[subclass]; + if (subclass < NR_LOCKDEP_CACHING_CLASSES) { + class = READ_ONCE(lock->class_cache[subclass]); + if (!lock_class_cache_is_valid(lock, class, subclass)) + class = NULL; + } /* * Not cached? */ @@ -5307,9 +5341,9 @@ static noinstr int match_held_lock(const struct held_lock *hlock, return 1; if (hlock->references) { - const struct lock_class *class = lock->class_cache[0]; + const struct lock_class *class = READ_ONCE(lock->class_cache[0]); - if (!class) + if (!lock_class_cache_is_valid(lock, class, 0)) class = look_up_lock_class(lock, 0); /* @@ -5437,6 +5471,8 @@ __lock_set_class(struct lockdep_map *lock, const char *name, lock->wait_type_outer, lock->lock_type); class = register_lock_class(lock, subclass, 0); + if (!class) + return 0; hlock->class_idx = class - lock_classes; curr->lockdep_depth = i; diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c index 73af81b99bf62..928860be7a4aa 100644 --- a/kernel/locking/rtmutex_api.c +++ b/kernel/locking/rtmutex_api.c @@ -398,6 +398,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, { int ret; + rt_mutex_futex_pre_schedule(); raw_spin_lock_irq(&lock->wait_lock); /* sleep on the mutex */ set_current_state(TASK_INTERRUPTIBLE); @@ -408,6 +409,7 @@ int __sched rt_mutex_wait_proxy_lock(struct rt_mutex_base *lock, */ fixup_rt_mutex_waiters(lock, true); raw_spin_unlock_irq(&lock->wait_lock); + rt_mutex_futex_post_schedule(); return ret; } diff --git a/kernel/module/dups.c b/kernel/module/dups.c index bd2149fbe1173..c21b675c239b7 100644 --- a/kernel/module/dups.c +++ b/kernel/module/dups.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "internal.h" @@ -38,27 +39,37 @@ static bool enable_dups_trace = IS_ENABLED(CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS_TRACE); module_param(enable_dups_trace, bool_enable_only, 0644); -/* - * Protects dup_kmod_reqs list, adds / removals with RCU. - */ +/* A mutex-protected list of active kmod requests. */ static DEFINE_MUTEX(kmod_dup_mutex); static LIST_HEAD(dup_kmod_reqs); struct kmod_dup_req { + refcount_t refcount; struct list_head list; char name[MODULE_NAME_LEN]; struct completion first_req_done; - struct work_struct complete_work; struct delayed_work delete_work; int dup_ret; }; +static void get_kmod_req(struct kmod_dup_req *kmod_req) +{ + refcount_inc(&kmod_req->refcount); +} + +static void put_kmod_req(struct kmod_dup_req *kmod_req) +{ + if (refcount_dec_and_test(&kmod_req->refcount)) + kfree(kmod_req); +} + static struct kmod_dup_req *kmod_dup_request_lookup(char *module_name) { struct kmod_dup_req *kmod_req; - list_for_each_entry_rcu(kmod_req, &dup_kmod_reqs, list, - lockdep_is_held(&kmod_dup_mutex)) { + lockdep_assert_held(&kmod_dup_mutex); + + list_for_each_entry(kmod_req, &dup_kmod_reqs, list) { if (strlen(kmod_req->name) == strlen(module_name) && !memcmp(kmod_req->name, module_name, strlen(module_name))) { return kmod_req; @@ -87,33 +98,10 @@ static void kmod_dup_request_delete(struct work_struct *work) * just returning 0. */ mutex_lock(&kmod_dup_mutex); - list_del_rcu(&kmod_req->list); - synchronize_rcu(); + list_del(&kmod_req->list); mutex_unlock(&kmod_dup_mutex); - kfree(kmod_req); -} -static void kmod_dup_request_complete(struct work_struct *work) -{ - struct kmod_dup_req *kmod_req; - - kmod_req = container_of(work, struct kmod_dup_req, complete_work); - - /* - * This will ensure that the kernel will let all the waiters get - * informed its time to check the return value. It's time to - * go home. - */ - complete_all(&kmod_req->first_req_done); - - /* - * Now that we have allowed prior request_module() calls to go on - * with life, let's schedule deleting this entry. We don't have - * to do it right away, but we *eventually* want to do it so to not - * let this linger forever as this is just a boot optimization for - * possible abuses of vmalloc() incurred by finit_module() thrashing. - */ - queue_delayed_work(system_wq, &kmod_req->delete_work, 60 * HZ); + put_kmod_req(kmod_req); } bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) @@ -129,8 +117,8 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) if (!new_kmod_req) return false; - memcpy(new_kmod_req->name, module_name, strlen(module_name)); - INIT_WORK(&new_kmod_req->complete_work, kmod_dup_request_complete); + refcount_set(&new_kmod_req->refcount, 1); + strscpy(new_kmod_req->name, module_name); INIT_DELAYED_WORK(&new_kmod_req->delete_work, kmod_dup_request_delete); init_completion(&new_kmod_req->first_req_done); @@ -161,10 +149,12 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * keep tab on duplicates later. */ pr_debug("New request_module() for %s\n", module_name); - list_add_rcu(&new_kmod_req->list, &dup_kmod_reqs); + list_add(&new_kmod_req->list, &dup_kmod_reqs); mutex_unlock(&kmod_dup_mutex); return false; } + + get_kmod_req(kmod_req); mutex_unlock(&kmod_dup_mutex); /* We are dealing with a duplicate request now */ @@ -194,7 +184,7 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) * calls bail out right away. */ *dup_ret = 0; - return true; + goto out; } /* @@ -209,12 +199,14 @@ bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) TASK_KILLABLE); if (ret) { *dup_ret = ret; - return true; + goto out; } /* Now the duplicate request has the same exact return value as the first request */ *dup_ret = kmod_req->dup_ret; +out: + put_kmod_req(kmod_req); return true; } @@ -224,24 +216,31 @@ void kmod_dup_request_announce(char *module_name, int ret) mutex_lock(&kmod_dup_mutex); + /* + * Look for a kmod_dup_req previously added in + * kmod_dup_request_exists_wait(). Note that a request_module_nowait() + * without its own kmod_dup_req entry can announce a result of + * a concurrent request_module() call. + */ kmod_req = kmod_dup_request_lookup(module_name); - if (!kmod_req) - goto out; + if (!kmod_req || completion_done(&kmod_req->first_req_done)) { + mutex_unlock(&kmod_dup_mutex); + return; + } kmod_req->dup_ret = ret; - /* - * If we complete() here we may allow duplicate threads - * to continue before the first one that submitted the - * request. We're in no rush also, given that each and - * every bounce back to userspace is slow we avoid that - * with a slight delay here. So queueue up the completion - * and let duplicates suffer, just wait a tad bit longer. - * There is no rush. But we also don't want to hold the - * caller up forever or introduce any boot delays. - */ - queue_work(system_wq, &kmod_req->complete_work); + /* Inform all duplicate waiters to check the return value. */ + complete_all(&kmod_req->first_req_done); -out: mutex_unlock(&kmod_dup_mutex); + + /* + * Now that we have allowed prior request_module() calls to go on + * with life, let's schedule deleting this entry. We don't have + * to do it right away, but we *eventually* want to do it so to not + * let this linger forever as this is just a boot optimization for + * possible abuses of vmalloc() incurred by finit_module() thrashing. + */ + queue_delayed_work(system_dfl_wq, &kmod_req->delete_work, 60 * HZ); } diff --git a/kernel/module/kallsyms.c b/kernel/module/kallsyms.c index 0fc11e45df9b9..f23126d804b25 100644 --- a/kernel/module/kallsyms.c +++ b/kernel/module/kallsyms.c @@ -258,17 +258,25 @@ static const char *find_kallsyms_symbol(struct module *mod, unsigned int i, best = 0; unsigned long nextval, bestval; struct mod_kallsyms *kallsyms = rcu_dereference(mod->kallsyms); - struct module_memory *mod_mem; + struct module_memory *mod_mem = NULL; - /* At worse, next value is at end of module */ - if (within_module_init(addr, mod)) - mod_mem = &mod->mem[MOD_INIT_TEXT]; - else - mod_mem = &mod->mem[MOD_TEXT]; + for_each_mod_mem_type(type) { +#ifndef CONFIG_KALLSYMS_ALL + if (!mod_mem_type_is_text(type)) + continue; +#endif + if (within_module_mem_type(addr, mod, type)) { + mod_mem = &mod->mem[type]; + break; + } + } - nextval = (unsigned long)mod_mem->base + mod_mem->size; + if (!mod_mem) + return NULL; - bestval = kallsyms_symbol_value(&kallsyms->symtab[best]); + /* Initialize bounds within memory region the address belongs to. */ + nextval = (unsigned long)mod_mem->base + mod_mem->size; + bestval = (unsigned long)mod_mem->base - 1; /* * Scan for closest preceding symbol, and next symbol. (ELF diff --git a/kernel/module/main.c b/kernel/module/main.c index 3745cd02c8479..d34187b02d14a 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -1958,6 +1958,7 @@ static int elf_validity_cache_sechdrs(struct load_info *info) * Specifically checks: * * * Section name table index is inbounds of section headers + * * Section name table type is SHT_STRTAB * * Section name table is not empty * * Section name table is NUL terminated * * All section name offsets are inbounds of the section @@ -1985,6 +1986,11 @@ static int elf_validity_cache_secstrings(struct load_info *info) strhdr = &info->sechdrs[info->hdr->e_shstrndx]; + if (strhdr->sh_type != SHT_STRTAB) { + pr_err("Invalid ELF section name table type: %u\n", strhdr->sh_type); + return -ENOEXEC; + } + /* * The section name table must be NUL-terminated, as required * by the spec. This makes strcmp and pr_* calls that access @@ -2151,7 +2157,7 @@ static int elf_validity_cache_index_sym(struct load_info *info) * Must have &load_info->index.sym populated. * * Looks at the symbol table's associated string table, makes sure it is - * in-bounds, and caches it. + * in-bounds and of type SHT_STRTAB, and caches it. * * Return: %0 if valid, %-ENOEXEC on failure. */ @@ -2165,6 +2171,12 @@ static int elf_validity_cache_index_str(struct load_info *info) return -ENOEXEC; } + if (info->sechdrs[str_idx].sh_type != SHT_STRTAB) { + pr_err("Invalid ELF symbol string table type: %u\n", + info->sechdrs[str_idx].sh_type); + return -ENOEXEC; + } + info->index.str = str_idx; return 0; } diff --git a/kernel/module/stats.c b/kernel/module/stats.c index 3ba0e98b3c910..2a4e2f6708965 100644 --- a/kernel/module/stats.c +++ b/kernel/module/stats.c @@ -253,7 +253,7 @@ int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason) mod_fail = kzalloc(sizeof(*mod_fail), GFP_KERNEL); if (!mod_fail) return -ENOMEM; - memcpy(mod_fail->name, name, strlen(name)); + strscpy(mod_fail->name, name); __set_bit(reason, &mod_fail->dup_fail_mask); atomic_long_inc(&mod_fail->count); list_add_rcu(&mod_fail->list, &dup_failed_modules); diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c index 19aa64ab08c83..6ce76a0278ab4 100644 --- a/kernel/nsproxy.c +++ b/kernel/nsproxy.c @@ -241,7 +241,7 @@ void switch_task_namespaces(struct task_struct *p, struct nsproxy *new) put_nsproxy(ns); } -void exit_task_namespaces(struct task_struct *p) +void exit_nsproxy_namespaces(struct task_struct *p) { switch_task_namespaces(p, NULL); } diff --git a/kernel/params.c b/kernel/params.c index 8942884e21c43..10ae7aac16d7b 100644 --- a/kernel/params.c +++ b/kernel/params.c @@ -261,6 +261,7 @@ EXPORT_SYMBOL_GPL(param_set_uint_minmax); int param_set_charp(const char *val, const struct kernel_param *kp) { + char *tmp; size_t len, maxlen = 1024; len = strnlen(val, maxlen + 1); @@ -269,19 +270,20 @@ int param_set_charp(const char *val, const struct kernel_param *kp) return -ENOSPC; } - maybe_kfree_parameter(*(char **)kp->arg); - /* * This is a hack. We can't kmalloc() in early boot, and we * don't need to; this mangled commandline is preserved. */ if (slab_is_available()) { - *(char **)kp->arg = kmalloc_parameter(len + 1); - if (!*(char **)kp->arg) + tmp = kmalloc_parameter(len + 1); + if (!tmp) return -ENOMEM; - strcpy(*(char **)kp->arg, val); + memcpy(tmp, val, len + 1); } else - *(const char **)kp->arg = val; + tmp = (char *)val; + + maybe_kfree_parameter(*(char **)kp->arg); + *(char **)kp->arg = tmp; return 0; } diff --git a/kernel/pid.c b/kernel/pid.c index 6a7769749ee68..9dfc327394f10 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -715,29 +715,6 @@ static struct ctl_table_root pid_table_root = { .set_ownership = pid_table_root_set_ownership, }; -static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos) -{ - struct pid *new_pid; - pid_t tmp_pid; - int r; - struct ctl_table tmp_table = *table; - - tmp_pid = pid_vnr(cad_pid); - tmp_table.data = &tmp_pid; - - r = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); - if (r || !write) - return r; - - new_pid = find_get_pid(tmp_pid); - if (!new_pid) - return -ESRCH; - - put_pid(xchg(&cad_pid, new_pid)); - return 0; -} - static const struct ctl_table pid_table[] = { { .procname = "pid_max", @@ -748,14 +725,6 @@ static const struct ctl_table pid_table[] = { .extra1 = &pid_max_min, .extra2 = &pid_max_max, }, -#ifdef CONFIG_PROC_SYSCTL - { - .procname = "cad_pid", - .maxlen = sizeof(int), - .mode = 0600, - .proc_handler = proc_do_cad_pid, - }, -#endif }; #endif diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c index e249e5786fbcd..b691e2a047b57 100644 --- a/kernel/power/snapshot.c +++ b/kernel/power/snapshot.c @@ -2798,9 +2798,10 @@ int snapshot_write_next(struct snapshot_handle *handle) return error; error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY); - if (error) + if (error) { + memory_bm_free(©_bm, PG_UNSAFE_CLEAR); return error; - + } nr_zero_pages = 0; hibernate_restore_protection_begin(); diff --git a/kernel/power/wakelock.c b/kernel/power/wakelock.c index 4e941999a53ba..5c2a248cf249a 100644 --- a/kernel/power/wakelock.c +++ b/kernel/power/wakelock.c @@ -63,7 +63,7 @@ static unsigned int number_of_wakelocks; static inline bool wakelocks_limit_exceeded(void) { - return number_of_wakelocks > CONFIG_PM_WAKELOCKS_LIMIT; + return number_of_wakelocks >= CONFIG_PM_WAKELOCKS_LIMIT; } static inline void increment_wakelocks_number(void) diff --git a/kernel/printk/nbcon.c b/kernel/printk/nbcon.c index 730d14f6cbc58..9c1e8ec04c556 100644 --- a/kernel/printk/nbcon.c +++ b/kernel/printk/nbcon.c @@ -1705,7 +1705,7 @@ bool nbcon_alloc(struct console *con) return false; rcuwait_init(&con->rcuwait); - init_irq_work(&con->irq_work, nbcon_irq_work); + con->irq_work = IRQ_WORK_INIT_LAZY(nbcon_irq_work); atomic_long_set(&ACCESS_PRIVATE(con, nbcon_prev_seq), -1UL); nbcon_state_set(con, &state); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index c27fc7fc64eb5..cc3d57be447d1 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -3155,6 +3155,106 @@ static inline void printk_kthreads_check_locked(void) { } #endif /* CONFIG_PRINTK */ + +/* + * Print out one record for each console. + * + * @do_cond_resched is set by the caller. It can be true only in schedulable + * context. + * + * @next_seq is set to the sequence number after the last available record. + * The value is valid only when there is at least one usable console and all + * usable consoles were flushed. + * + * @handover will be set to true if a printk waiter has taken over the + * console_lock, in which case the caller is no longer holding the + * console_lock. Otherwise it is set to false. + * + * @any_usable will be set to true if there are any usable consoles. + * + * Returns true when there was at least one usable console and a record was + * flushed. A returned false indicates there were no records to flush for any + * of the consoles. It may also indicate that there were no usable consoles, + * the context has been lost or there is a panic suitation. Regardless the + * reason, the caller should assume it is not useful to immediately try again. + * + * Requires the console_lock. + */ +static bool console_flush_one_record(bool do_cond_resched, u64 *next_seq, bool *handover, + bool *any_usable) +{ + struct console_flush_type ft; + bool any_progress = false; + struct console *con; + int cookie; + + printk_get_console_flush_type(&ft); + + cookie = console_srcu_read_lock(); + for_each_console_srcu(con) { + short flags = console_srcu_read_flags(con); + u64 printk_seq; + bool progress; + + /* + * console_flush_one_record() is only responsible for + * nbcon consoles when the nbcon consoles cannot print via + * their atomic or threaded flushing. + */ + if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload)) + continue; + + if (!console_is_usable(con, flags, !do_cond_resched)) + continue; + *any_usable = true; + + if (flags & CON_NBCON) { + progress = nbcon_legacy_emit_next_record(con, handover, cookie, + !do_cond_resched); + } else { + progress = console_emit_next_record(con, handover, cookie); + } + + /* + * If a handover has occurred, the SRCU read lock + * is already released. + */ + if (*handover) + return false; + + /* + * @con can be used here now that it is certain that this + * context is still holding the SRCU read lock. + */ + if (flags & CON_NBCON) + printk_seq = nbcon_seq_read(con); + else + printk_seq = con->seq; + + /* Track the next of the highest seq flushed. */ + if (printk_seq > *next_seq) + *next_seq = printk_seq; + + if (!progress) + continue; + any_progress = true; + + /* Allow panic_cpu to take over the consoles safely. */ + if (panic_on_other_cpu()) + goto abandon; + + if (do_cond_resched) + cond_resched(); + } + console_srcu_read_unlock(cookie); + + return any_progress; + +abandon: + console_srcu_read_unlock(cookie); + return false; +} + /* * Print out all remaining records to all consoles. * @@ -3180,77 +3280,24 @@ static inline void printk_kthreads_check_locked(void) { } */ static bool console_flush_all(bool do_cond_resched, u64 *next_seq, bool *handover) { - struct console_flush_type ft; bool any_usable = false; - struct console *con; bool any_progress; - int cookie; *next_seq = 0; *handover = false; do { - any_progress = false; - - printk_get_console_flush_type(&ft); - - cookie = console_srcu_read_lock(); - for_each_console_srcu(con) { - short flags = console_srcu_read_flags(con); - u64 printk_seq; - bool progress; - - /* - * console_flush_all() is only responsible for nbcon - * consoles when the nbcon consoles cannot print via - * their atomic or threaded flushing. - */ - if ((flags & CON_NBCON) && (ft.nbcon_atomic || ft.nbcon_offload)) - continue; - - if (!console_is_usable(con, flags, !do_cond_resched)) - continue; - any_usable = true; - - if (flags & CON_NBCON) { - progress = nbcon_legacy_emit_next_record(con, handover, cookie, - !do_cond_resched); - printk_seq = nbcon_seq_read(con); - } else { - progress = console_emit_next_record(con, handover, cookie); - printk_seq = con->seq; - } - - /* - * If a handover has occurred, the SRCU read lock - * is already released. - */ - if (*handover) - return false; + any_progress = console_flush_one_record(do_cond_resched, next_seq, handover, + &any_usable); - /* Track the next of the highest seq flushed. */ - if (printk_seq > *next_seq) - *next_seq = printk_seq; + if (*handover) + return false; - if (!progress) - continue; - any_progress = true; - - /* Allow panic_cpu to take over the consoles safely. */ - if (panic_on_other_cpu()) - goto abandon; - - if (do_cond_resched) - cond_resched(); - } - console_srcu_read_unlock(cookie); + if (panic_on_other_cpu()) + return false; } while (any_progress); return any_usable; - -abandon: - console_srcu_read_unlock(cookie); - return false; } static void __console_flush_and_unlock(void) @@ -3660,12 +3707,13 @@ static bool legacy_kthread_create(void) /** * printk_kthreads_shutdown - shutdown all threaded printers + * @data: syscore context * * On system shutdown all threaded printers are stopped. This allows printk * to transition back to atomic printing, thus providing a robust mechanism * for the final shutdown/reboot messages to be output. */ -static void printk_kthreads_shutdown(void) +static void printk_kthreads_shutdown(void *data) { struct console *con; @@ -3687,10 +3735,14 @@ static void printk_kthreads_shutdown(void) console_list_unlock(); } -static struct syscore_ops printk_syscore_ops = { +static const struct syscore_ops printk_syscore_ops = { .shutdown = printk_kthreads_shutdown, }; +static struct syscore printk_syscore = { + .ops = &printk_syscore_ops, +}; + /* * If appropriate, start nbcon kthreads and set @printk_kthreads_running. * If any kthreads fail to start, those consoles are unregistered. @@ -3758,7 +3810,7 @@ static void printk_kthreads_check_locked(void) static int __init printk_set_kthreads_ready(void) { - register_syscore_ops(&printk_syscore_ops); + register_syscore(&printk_syscore); console_list_lock(); printk_kthreads_ready = true; diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 2dc044fd126eb..2aa5589f0b8bf 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -377,7 +377,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, // Queuing callbacks before initialization not yet supported. if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) rcu_segcblist_init(&rtpcp->cblist); - needwake = (func == wakeme_after_rcu) || + needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || + (func == wakeme_after_rcu) || (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { if (rtp->lazy_jiffies) diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index cafb1cc8eff84..41e32b015a64e 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -973,7 +973,7 @@ void rcu_all_qs(void) { unsigned long flags; - if (!raw_cpu_read(rcu_data.rcu_urgent_qs)) + if (!READ_ONCE(*raw_cpu_ptr(&rcu_data.rcu_urgent_qs))) return; preempt_disable(); // For CONFIG_PREEMPT_COUNT=y kernels /* Load rcu_urgent_qs before other flags. */ @@ -981,8 +981,8 @@ void rcu_all_qs(void) preempt_enable(); return; } - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) { + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) { local_irq_save(flags); rcu_momentary_eqs(); local_irq_restore(flags); @@ -1002,8 +1002,8 @@ void rcu_note_context_switch(bool preempt) /* Load rcu_urgent_qs before other flags. */ if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) goto out; - this_cpu_write(rcu_data.rcu_urgent_qs, false); - if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) + WRITE_ONCE(*this_cpu_ptr(&rcu_data.rcu_urgent_qs), false); + if (unlikely(READ_ONCE(*this_cpu_ptr(&rcu_data.rcu_need_heavy_qs)))) rcu_momentary_eqs(); out: rcu_tasks_qs(current, preempt); diff --git a/kernel/reboot.c b/kernel/reboot.c index ec087827c85cd..61924a0560c42 100644 --- a/kernel/reboot.c +++ b/kernel/reboot.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -24,8 +26,7 @@ */ static int C_A_D = 1; -struct pid *cad_pid; -EXPORT_SYMBOL(cad_pid); +struct pid __rcu *cad_pid; #if defined(CONFIG_ARM) #define DEFAULT_REBOOT_MODE = REBOOT_HARD @@ -1366,6 +1367,39 @@ static struct attribute *reboot_attrs[] = { }; #ifdef CONFIG_SYSCTL +static int proc_do_cad_pid(const struct ctl_table *table, int write, void *buffer, + size_t *lenp, loff_t *ppos) +{ + struct ctl_table tmp_table = *table; + struct pid *new_pid; + struct pid *old_pid; + pid_t tmp_pid; + int r; + + rcu_read_lock(); + tmp_pid = pid_vnr(rcu_dereference(cad_pid)); + rcu_read_unlock(); + + tmp_table.data = &tmp_pid; + + r = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); + if (r || !write) + return r; + + new_pid = find_get_pid(tmp_pid); + if (!new_pid) + return -ESRCH; + + old_pid = unrcu_pointer(xchg(&cad_pid, RCU_INITIALIZER(new_pid))); + /* + * Wait for cad_pid readers before put_pid(). We cannot use + * call_rcu() here because free_pid() already owns pid->rcu. + */ + synchronize_rcu(); + put_pid(old_pid); + return 0; +} + static const struct ctl_table kern_reboot_table[] = { { .procname = "poweroff_cmd", @@ -1381,6 +1415,12 @@ static const struct ctl_table kern_reboot_table[] = { .mode = 0644, .proc_handler = proc_dointvec, }, + { + .procname = "cad_pid", + .maxlen = sizeof(int), + .mode = 0600, + .proc_handler = proc_do_cad_pid, + }, }; static void __init kernel_reboot_sysctls_init(void) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b5b5897b595b4..43f3ac16b0e8a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -434,6 +434,17 @@ static void __sched_core_flip(bool enabled) sched_core_lock(cpu, &flags); + /* + * A core-wide selection may have the shared rq lock temporarily + * released by a lock-dropping ->pick_task(). Flipping would + * rebind rq_lockp() under it. Wait it out. + */ + while (cpu_rq(cpu)->core->core_pick_in_flight) { + sched_core_unlock(cpu, &flags); + cpu_relax(); + sched_core_lock(cpu, &flags); + } + for_each_cpu(t, smt_mask) cpu_rq(t)->core_enabled = enabled; @@ -5587,8 +5598,8 @@ void sched_tick(void) { int cpu = smp_processor_id(); struct rq *rq = cpu_rq(cpu); - /* accounting goes to the donor task */ - struct task_struct *donor; + /* scheduler accounting goes to the donor task */ + struct task_struct *curr, *donor; struct rq_flags rf; unsigned long hw_pressure; u64 resched_latency; @@ -5599,6 +5610,7 @@ void sched_tick(void) sched_clock_tick(); rq_lock(rq, &rf); + curr = rq->curr; donor = rq->donor; psi_account_irqtime(rq, donor, NULL); @@ -5625,8 +5637,8 @@ void sched_tick(void) perf_event_task_tick(); - if (donor->flags & PF_WQ_WORKER) - wq_worker_tick(donor); + if (curr->flags & PF_WQ_WORKER) + wq_worker_tick(curr); if (!scx_switched_all()) { rq->idle_balance = idle_cpu(cpu); @@ -5923,10 +5935,9 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) schedstat_inc(this_rq()->sched_count); } -static void prev_balance(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) +static void prev_balance(struct rq *rq, struct rq_flags *rf) { - const struct sched_class *start_class = prev->sched_class; + const struct sched_class *start_class = rq->donor->sched_class; const struct sched_class *class; #ifdef CONFIG_SCHED_CLASS_EXT @@ -5951,7 +5962,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev, * a runnable task of @class priority or higher. */ for_active_class_range(class, start_class, &idle_sched_class) { - if (class->balance && class->balance(rq, prev, rf)) + if (class->balance && class->balance(rq, rf)) break; } } @@ -5960,7 +5971,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev, * Pick up the highest-prio task: */ static inline struct task_struct * -__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +__pick_next_task(struct rq *rq, struct rq_flags *rf) { const struct sched_class *class; struct task_struct *p; @@ -5976,34 +5987,34 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) * higher scheduling class, because otherwise those lose the * opportunity to pull in more work from other CPUs. */ - if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) && + if (likely(!sched_class_above(rq->donor->sched_class, &fair_sched_class) && rq->nr_running == rq->cfs.h_nr_queued)) { - p = pick_next_task_fair(rq, prev, rf); + p = pick_next_task_fair(rq, rq->donor, rf); if (unlikely(p == RETRY_TASK)) goto restart; /* Assume the next prioritized class is idle_sched_class */ if (!p) { p = pick_task_idle(rq); - put_prev_set_next_task(rq, prev, p); + put_prev_set_next_task(rq, rq->donor, p); } return p; } restart: - prev_balance(rq, prev, rf); + prev_balance(rq, rf); for_each_active_class(class) { if (class->pick_next_task) { - p = class->pick_next_task(rq, prev); + p = class->pick_next_task(rq, rq->donor); if (p) return p; } else { p = class->pick_task(rq); if (p) { - put_prev_set_next_task(rq, prev, p); + put_prev_set_next_task(rq, rq->donor, p); return p; } } @@ -6052,7 +6063,7 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f static void queue_core_balance(struct rq *rq); static struct task_struct * -pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +pick_next_task(struct rq *rq, struct rq_flags *rf) { struct task_struct *next, *p, *max = NULL; const struct cpumask *smt_mask; @@ -6064,7 +6075,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) bool need_sync; if (!sched_core_enabled(rq)) - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); cpu = cpu_of(rq); @@ -6077,9 +6088,11 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) */ rq->core_pick = NULL; rq->core_dl_server = NULL; - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); } + rq->core->core_pick_in_flight++; + /* * If there were no {en,de}queues since we picked (IOW, the task * pointers are all still valid), and we haven't scheduled the last @@ -6101,7 +6114,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) goto out_set_next; } - prev_balance(rq, prev, rf); + prev_balance(rq, rf); smt_mask = cpu_smt_mask(cpu); need_sync = !!rq->core->core_cookie; @@ -6274,7 +6287,8 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) } out_set_next: - put_prev_set_next_task(rq, prev, next); + rq->core->core_pick_in_flight--; + put_prev_set_next_task(rq, rq->donor, next); if (rq->core->core_forceidle_count && next == rq->idle) queue_core_balance(rq); @@ -6467,6 +6481,13 @@ static void sched_core_cpu_deactivate(unsigned int cpu) core_rq->core_forceidle_seq = rq->core_forceidle_seq; core_rq->core_forceidle_occupation = rq->core_forceidle_occupation; + /* + * A stale leftover would bias the count forever if this CPU later + * returns as its own leader. Move, don't copy. + */ + core_rq->core_pick_in_flight = rq->core_pick_in_flight; + rq->core_pick_in_flight = 0; + /* * Accounting edge for forced idle is handled in pick_next_task(). * Don't need another one here, since the hotplug thread shouldn't @@ -6496,9 +6517,9 @@ static inline void sched_core_cpu_deactivate(unsigned int cpu) {} static inline void sched_core_cpu_dying(unsigned int cpu) {} static struct task_struct * -pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +pick_next_task(struct rq *rq, struct rq_flags *rf) { - return __pick_next_task(rq, prev, rf); + return __pick_next_task(rq, rf); } #endif /* !CONFIG_SCHED_CORE */ @@ -6738,23 +6759,6 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) } #endif /* SCHED_PROXY_EXEC */ -static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner) -{ - if (!sched_proxy_exec()) - return; - /* - * pick_next_task() calls set_next_task() on the chosen task - * at some point, which ensures it is not push/pullable. - * However, the chosen/donor task *and* the mutex owner form an - * atomic pair wrt push/pull. - * - * Make sure owner we run is not pushable. Unfortunately we can - * only deal with that by means of a dequeue/enqueue cycle. :-/ - */ - dequeue_task(rq, owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE); - enqueue_task(rq, owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE); -} - /* * __schedule() is the main scheduler function. * @@ -6880,7 +6884,8 @@ static void __sched notrace __schedule(int sched_mode) } pick_again: - next = pick_next_task(rq, rq->donor, &rf); + assert_balance_callbacks_empty(rq); + next = pick_next_task(rq, &rf); rq_set_donor(rq, next); if (unlikely(task_is_blocked(next))) { next = find_proxy_task(rq, next, &rf); @@ -6904,9 +6909,6 @@ static void __sched notrace __schedule(int sched_mode) */ RCU_INIT_POINTER(rq->curr, next); - if (!task_current_donor(rq, next)) - proxy_tag_curr(rq, next); - /* * The membarrier system call requires each architecture * to have a full memory barrier after updating @@ -6941,10 +6943,6 @@ static void __sched notrace __schedule(int sched_mode) /* Also unlocks the rq: */ rq = context_switch(rq, prev, next, &rf); } else { - /* In case next was already curr but just got blocked_donor */ - if (!task_current_donor(rq, next)) - proxy_tag_curr(rq, next); - rq_unpin_lock(rq, &rf); __balance_callbacks(rq); raw_spin_rq_unlock_irq(rq); @@ -7312,6 +7310,17 @@ void rt_mutex_pre_schedule(void) sched_submit_work(current); } +/* + * Used within the futex syscall context, skips sched_submit_work() because none + * its work will be done. Asserts ensure that it is indeed the case. + */ +void rt_mutex_futex_pre_schedule(void) +{ + lockdep_assert(!(current->flags & (PF_WQ_WORKER | PF_IO_WORKER))); + lockdep_assert(!current->plug); + lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1)); +} + void rt_mutex_schedule(void) { lockdep_assert(current->sched_rt_mutex); @@ -7324,6 +7333,11 @@ void rt_mutex_post_schedule(void) lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); } +void rt_mutex_futex_post_schedule(void) +{ + lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0)); +} + /* * rt_mutex_setprio - set the current priority of a task * @p: task to boost @@ -8774,6 +8788,7 @@ void __init sched_init(void) rq->core_forceidle_count = 0; rq->core_forceidle_occupation = 0; rq->core_forceidle_start = 0; + rq->core_pick_in_flight = 0; rq->core_cookie = 0UL; #endif diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index e82bb019b9202..22a1fbccb929f 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -61,6 +61,17 @@ static DEFINE_PER_CPU(struct sugov_cpu, sugov_cpu); /************************ Governor internals ***********************/ +static void sugov_update_rate_limit_us(struct sugov_policy *sg_policy) +{ + /* + * Cast rate_limit_us before multiplication to force 64-bit arithmetic. + * Otherwise, on 32-bit platforms, both operands are converted to + * 32-bit unsigned long and the multiplication may overflow. + */ + sg_policy->freq_update_delay_ns = + (s64)sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; +} + static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; @@ -314,7 +325,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, * A CPU running a task which woken up after an IO operation can have its * utilization boosted to speed up the completion of those IO operations. * The IO boost value is increased each time a task wakes up from IO, in - * sugov_iowait_apply(), and it's instead decreased by this function, + * sugov_iowait_boost(), and it's instead decreased by this function, * each time an increase has not been requested (!iowait_boost_pending). * * A CPU which also appears to have been idle for at least one tick has also @@ -606,7 +617,7 @@ rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count tunables->rate_limit_us = rate_limit_us; list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook) - sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); return count; } @@ -848,7 +859,7 @@ static int sugov_start(struct cpufreq_policy *policy) void (*uu)(struct update_util_data *data, u64 time, unsigned int flags); unsigned int cpu; - sg_policy->freq_update_delay_ns = sg_policy->tunables->rate_limit_us * NSEC_PER_USEC; + sugov_update_rate_limit_us(sg_policy); sg_policy->last_freq_update_time = 0; sg_policy->next_freq = 0; sg_policy->work_in_progress = false; @@ -870,8 +881,19 @@ static int sugov_start(struct cpufreq_policy *policy) memset(sg_cpu, 0, sizeof(*sg_cpu)); sg_cpu->cpu = cpu; sg_cpu->sg_policy = sg_policy; + } + + /* + * Publish the hooks only after all per-CPU data is initialized, so a + * shared policy's sugov_update_shared() never reads an uninitialized + * sibling sugov_cpu. + */ + for_each_cpu(cpu, policy->cpus) { + struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu); + cpufreq_add_update_util_hook(cpu, &sg_cpu->update_util, uu); } + return 0; } diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index d5052f238adf7..0180758d90af8 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -1036,7 +1036,8 @@ static void update_dl_entity(struct sched_dl_entity *dl_se) if (dl_time_before(dl_se->deadline, rq_clock(rq)) || dl_entity_overflow(dl_se, rq_clock(rq))) { - if (unlikely((!dl_is_implicit(dl_se) || dl_se->dl_defer) && + if (unlikely((!dl_is_implicit(dl_se) || + (dl_se->dl_defer && dl_se->dl_defer_running)) && !dl_time_before(dl_se->deadline, rq_clock(rq)) && !is_dl_boosted(dl_se))) { update_dl_revised_wakeup(dl_se, rq); @@ -2473,8 +2474,14 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) resched_curr(rq); } -static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf) +static int balance_dl(struct rq *rq, struct rq_flags *rf) { + /* + * Note, rq->donor may change during rq lock drops, + * so don't re-use p across lock drops + */ + struct task_struct *p = rq->donor; + if (!on_dl_rq(&p->dl) && need_pull_dl_task(rq, p)) { /* * This is OK, because current is on_cpu, which avoids it being @@ -2761,12 +2768,26 @@ static int find_later_rq(struct task_struct *task) static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) { - struct task_struct *p; + struct task_struct *i, *p = NULL; + struct rb_node *next_node; if (!has_pushable_dl_tasks(rq)) return NULL; - p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root)); + next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root); + while (next_node) { + i = __node_2_pdl(next_node); + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { + p = i; + break; + } + + next_node = rb_next(next_node); + } + + if (!p) + return NULL; WARN_ON_ONCE(rq->cpu != task_cpu(p)); WARN_ON_ONCE(task_current(rq, p)); diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index d8280f8744331..812e0c34bb801 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2200,14 +2200,13 @@ static int balance_one(struct rq *rq, struct task_struct *prev) return true; } -static int balance_scx(struct rq *rq, struct task_struct *prev, - struct rq_flags *rf) +static int balance_scx(struct rq *rq, struct rq_flags *rf) { int ret; rq_unpin_lock(rq, rf); - ret = balance_one(rq, prev); + ret = balance_one(rq, rq->donor); #ifdef CONFIG_SCHED_SMT /* @@ -2395,9 +2394,14 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p, * sched_class, %SCX_OPS_ENQ_LAST must be set. Tell * ops.enqueue() that @p is the only one available for this cpu, * which should trigger an explicit follow-up scheduling event. + * + * Core scheduling can force this CPU idle while @p stays + * runnable. @p's cookie then won't match the core's, so skip + * the warning in that case. */ - if (sched_class_above(&ext_sched_class, next->sched_class)) { - WARN_ON_ONCE(!(sch->ops.flags & SCX_OPS_ENQ_LAST)); + if (next && sched_class_above(&ext_sched_class, next->sched_class)) { + WARN_ON_ONCE(sched_cpu_cookie_match(rq, p) && + !(sch->ops.flags & SCX_OPS_ENQ_LAST)); do_enqueue_task(rq, p, SCX_ENQ_LAST, -1); } else { do_enqueue_task(rq, p, 0, -1); @@ -2508,6 +2512,8 @@ static struct task_struct *pick_task_scx(struct rq *rq) * * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to * implement FIFO ordering within each local DSQ. See pick_task_scx(). + * + * Return: %true if @a should run after @b. */ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, bool in_fi) @@ -2515,6 +2521,10 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, struct scx_sched *sch = scx_root; /* + * scx_prio_less() returns whether @a should run after @b while + * ops.core_sched_before() returns whether its first argument should run + * before the second. Swap the arguments. + * * The const qualifiers are dropped from task_struct pointers when * calling ops.core_sched_before(). Accesses are controlled by the * verifier. @@ -2523,8 +2533,8 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, !scx_rq_bypassing(task_rq(a))) return SCX_CALL_OP_2TASKS_RET(sch, SCX_KF_REST, core_sched_before, task_rq(a), - (struct task_struct *)a, - (struct task_struct *)b); + (struct task_struct *)b, + (struct task_struct *)a); else return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); } @@ -2836,7 +2846,10 @@ static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork scx_set_task_state(p, SCX_TASK_INIT); if (p->scx.disallow) { - if (!fork) { + if (unlikely(fork)) { + scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", + p->comm, p->pid); + } else { struct rq *rq; struct rq_flags rf; @@ -2855,9 +2868,6 @@ static int scx_init_task(struct task_struct *p, struct task_group *tg, bool fork } task_rq_unlock(rq, p, &rf); - } else if (p->policy == SCHED_EXT) { - scx_error(sch, "ops.init_task() set task->scx.disallow for %s[%d] during fork", - p->comm, p->pid); } } @@ -3002,7 +3012,7 @@ void scx_post_fork(struct task_struct *p) void scx_cancel_fork(struct task_struct *p) { - if (scx_enabled()) { + if (scx_init_task_enabled) { struct rq *rq; struct rq_flags rf; @@ -3024,8 +3034,8 @@ void sched_ext_free(struct task_struct *p) raw_spin_unlock_irqrestore(&scx_tasks_lock, flags); /* - * @p is off scx_tasks and wholly ours. scx_enable()'s READY -> ENABLED - * transitions can't race us. Disable ops for @p. + * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY -> + * ENABLED transitions can't race us. Disable ops for @p. */ if (scx_get_task_state(p) != SCX_TASK_NONE) { struct rq_flags rf; @@ -3308,16 +3318,21 @@ void scx_group_set_bandwidth(struct task_group *tg, percpu_up_read(&scx_cgroup_ops_rwsem); } +/* + * cgroup_lock() must nest outside the rwsem write side: a writer waiting + * for cgroup_mutex deadlocks with cgroup teardown, which holds it while + * draining a set_* file write blocked on the rwsem behind the writer. + */ static void scx_cgroup_lock(void) { - percpu_down_write(&scx_cgroup_ops_rwsem); cgroup_lock(); + percpu_down_write(&scx_cgroup_ops_rwsem); } static void scx_cgroup_unlock(void) { - cgroup_unlock(); percpu_up_write(&scx_cgroup_ops_rwsem); + cgroup_unlock(); } #else /* CONFIG_EXT_GROUP_SCHED */ @@ -3957,24 +3972,12 @@ static void free_kick_pseqs(void) } } -static void scx_disable_workfn(struct kthread_work *work) +static void scx_root_disable(struct scx_sched *sch) { - struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); struct scx_exit_info *ei = sch->exit_info; struct scx_task_iter sti; struct task_struct *p; - int kind, cpu; - - kind = atomic_read(&sch->exit_kind); - while (true) { - if (kind == SCX_EXIT_DONE) /* already disabled? */ - return; - WARN_ON_ONCE(kind == SCX_EXIT_NONE); - if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) - break; - } - ei->kind = kind; - ei->reason = scx_exit_reason(ei->kind); + int cpu; /* guarantee forward progress by bypassing scx_ops */ scx_bypass(true); @@ -4119,6 +4122,9 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) lockdep_assert_preemption_disabled(); + if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) + kind = SCX_EXIT_ERROR; + if (!atomic_try_cmpxchg(&sch->exit_kind, &none, kind)) return false; @@ -4131,21 +4137,31 @@ static bool scx_claim_exit(struct scx_sched *sch, enum scx_exit_kind kind) return true; } -static void scx_disable(enum scx_exit_kind kind) +static void scx_disable_workfn(struct kthread_work *work) { - struct scx_sched *sch; + struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); + struct scx_exit_info *ei = sch->exit_info; + int kind; - if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)) - kind = SCX_EXIT_ERROR; + kind = atomic_read(&sch->exit_kind); + while (true) { + if (kind == SCX_EXIT_DONE) /* already disabled? */ + return; + WARN_ON_ONCE(kind == SCX_EXIT_NONE); + if (atomic_try_cmpxchg(&sch->exit_kind, &kind, SCX_EXIT_DONE)) + break; + } + ei->kind = kind; + ei->reason = scx_exit_reason(ei->kind); - rcu_read_lock(); - sch = rcu_dereference(scx_root); - if (sch) { - guard(preempt)(); - scx_claim_exit(sch, kind); + scx_root_disable(sch); +} + +static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) +{ + guard(preempt)(); + if (scx_claim_exit(sch, kind)) kthread_queue_work(sch->helper, &sch->disable_work); - } - rcu_read_unlock(); } static void dump_newline(struct seq_buf *s) @@ -4657,10 +4673,9 @@ struct scx_enable_cmd { int ret; }; -static void scx_enable_workfn(struct kthread_work *work) +static void scx_root_enable_workfn(struct kthread_work *work) { - struct scx_enable_cmd *cmd = - container_of(work, struct scx_enable_cmd, work); + struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work); struct sched_ext_ops *ops = cmd->ops; struct scx_sched *sch; struct scx_task_iter sti; @@ -4910,12 +4925,12 @@ static void scx_enable_workfn(struct kthread_work *work) * Flush scx_disable_work to ensure that error is reported before init * completion. sch's base reference will be put by bpf_scx_unreg(). */ - scx_error(sch, "scx_enable() failed (%d)", ret); + scx_error(sch, "scx_root_enable() failed (%d)", ret); kthread_flush_work(&sch->disable_work); cmd->ret = 0; } -static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) +static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) { static struct kthread_worker *helper; static DEFINE_MUTEX(helper_mutex); @@ -4941,7 +4956,7 @@ static int scx_enable(struct sched_ext_ops *ops, struct bpf_link *link) mutex_unlock(&helper_mutex); } - kthread_init_work(&cmd.work, scx_enable_workfn); + kthread_init_work(&cmd.work, scx_root_enable_workfn); cmd.ops = ops; kthread_queue_work(READ_ONCE(helper), &cmd.work); @@ -5084,7 +5099,7 @@ static void bpf_scx_unreg(void *kdata, struct bpf_link *link) struct sched_ext_ops *ops = kdata; struct scx_sched *sch = ops->priv; - scx_disable(SCX_EXIT_UNREG); + scx_disable(sch, SCX_EXIT_UNREG); kthread_flush_work(&sch->disable_work); kobject_put(&sch->kobj); } @@ -5212,7 +5227,15 @@ static struct bpf_struct_ops bpf_sched_ext_ops = { static void sysrq_handle_sched_ext_reset(u8 key) { - scx_disable(SCX_EXIT_SYSRQ); + struct scx_sched *sch; + + rcu_read_lock(); + sch = rcu_dereference(scx_root); + if (likely(sch)) + scx_disable(sch, SCX_EXIT_SYSRQ); + else + pr_info("sched_ext: BPF schedulers not loaded\n"); + rcu_read_unlock(); } static const struct sysrq_key_op sysrq_sched_ext_reset_op = { diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index ac5f08cd01a83..67e8c2065d827 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -764,17 +764,22 @@ static inline u64 cfs_rq_max_slice(struct cfs_rq *cfs_rq); * * -r_max < lag < max(r_max, q) */ -static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) +static s64 entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se, u64 avruntime) { u64 max_slice = cfs_rq_max_slice(cfs_rq) + TICK_NSEC; s64 vlag, limit; - WARN_ON_ONCE(!se->on_rq); - - vlag = avg_vruntime(cfs_rq) - se->vruntime; + vlag = avruntime - se->vruntime; limit = calc_delta_fair(max_slice, se); - se->vlag = clamp(vlag, -limit, limit); + return clamp(vlag, -limit, limit); +} + +static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se) +{ + WARN_ON_ONCE(!se->on_rq); + + se->vlag = entity_lag(cfs_rq, se, avg_vruntime(cfs_rq)); } /* @@ -878,6 +883,13 @@ static inline void __max_slice_update(struct sched_entity *se, struct rb_node *n } } +static inline void min_vruntime_copy(struct sched_entity *new, struct sched_entity *old) +{ + new->min_vruntime = old->min_vruntime; + new->min_slice = old->min_slice; + new->max_slice = old->max_slice; +} + /* * se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime) */ @@ -905,8 +917,9 @@ static inline bool min_vruntime_update(struct sched_entity *se, bool exit) se->max_slice == old_max_slice; } -RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity, - run_node, min_vruntime, min_vruntime_update); + +RB_DECLARE_CALLBACKS_MULTI(static, min_vruntime_cb, struct sched_entity, + run_node, min_vruntime_copy, min_vruntime_update); /* * Enqueue an entity into the rb-tree: @@ -916,6 +929,8 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) sum_w_vruntime_add(cfs_rq, se); se->min_vruntime = se->vruntime; se->min_slice = se->slice; + se->max_slice = se->slice; + rb_add_augmented_cached(&se->run_node, &cfs_rq->tasks_timeline, __entity_less, &min_vruntime_cb); } @@ -1235,7 +1250,6 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) se->exec_start = now; if (entity_is_task(se)) { - struct task_struct *donor = task_of(se); struct task_struct *running = rq->curr; /* * If se is a task, we account the time against the running @@ -1247,8 +1261,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se) trace_sched_stat_runtime(running, delta_exec); account_group_exec_runtime(running, delta_exec); - /* cgroup time is always accounted against the donor */ - cgroup_account_cputime(donor, delta_exec); + cgroup_account_cputime(running, delta_exec); } else { /* If not task, account the time against donor se */ se->sum_exec_runtime += delta_exec; @@ -3838,23 +3851,125 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) cfs_rq->avg.load_avg * PELT_MIN_DIVIDER); } -static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags); +static void +rescale_entity(struct sched_entity *se, unsigned long weight, bool rel_vprot) +{ + unsigned long old_weight = se->load.weight; + + /* + * VRUNTIME + * -------- + * + * COROLLARY #1: The virtual runtime of the entity needs to be + * adjusted if re-weight at !0-lag point. + * + * Proof: For contradiction assume this is not true, so we can + * re-weight without changing vruntime at !0-lag point. + * + * Weight VRuntime Avg-VRuntime + * before w v V + * after w' v' V' + * + * Since lag needs to be preserved through re-weight: + * + * lag = (V - v)*w = (V'- v')*w', where v = v' + * ==> V' = (V - v)*w/w' + v (1) + * + * Let W be the total weight of the entities before reweight, + * since V' is the new weighted average of entities: + * + * V' = (WV + w'v - wv) / (W + w' - w) (2) + * + * by using (1) & (2) we obtain: + * + * (WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v + * ==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v + * ==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v + * ==> (V - v)*W/(W + w' - w) = (V - v)*w/w' (3) + * + * Since we are doing at !0-lag point which means V != v, we + * can simplify (3): + * + * ==> W / (W + w' - w) = w / w' + * ==> Ww' = Ww + ww' - ww + * ==> W * (w' - w) = w * (w' - w) + * ==> W = w (re-weight indicates w' != w) + * + * So the cfs_rq contains only one entity, hence vruntime of + * the entity @v should always equal to the cfs_rq's weighted + * average vruntime @V, which means we will always re-weight + * at 0-lag point, thus breach assumption. Proof completed. + * + * + * COROLLARY #2: Re-weight does NOT affect weighted average + * vruntime of all the entities. + * + * Proof: According to corollary #1, Eq. (1) should be: + * + * (V - v)*w = (V' - v')*w' + * ==> v' = V' - (V - v)*w/w' (4) + * + * According to the weighted average formula, we have: + * + * V' = (WV - wv + w'v') / (W - w + w') + * = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w') + * = (WV - wv + w'V' - Vw + wv) / (W - w + w') + * = (WV + w'V' - Vw) / (W - w + w') + * + * ==> V'*(W - w + w') = WV + w'V' - Vw + * ==> V' * (W - w) = (W - w) * V (5) + * + * If the entity is the only one in the cfs_rq, then reweight + * always occurs at 0-lag point, so V won't change. Or else + * there are other entities, hence W != w, then Eq. (5) turns + * into V' = V. So V won't change in either case, proof done. + * + * + * So according to corollary #1 & #2, the effect of re-weight + * on vruntime should be: + * + * v' = V' - (V - v) * w / w' (4) + * = V - (V - v) * w / w' + * = V - vl * w / w' + * = V - vl' + */ + se->vlag = div64_long(se->vlag * old_weight, weight); + + /* + * DEADLINE + * -------- + * + * When the weight changes, the virtual time slope changes and + * we should adjust the relative virtual deadline accordingly. + * + * d' = v' + (d - v)*w/w' + * = V' - (V - v)*w/w' + (d - v)*w/w' + * = V - (V - v)*w/w' + (d - v)*w/w' + * = V + (d - V)*w/w' + */ + if (se->rel_deadline) + se->deadline = div64_long(se->deadline * old_weight, weight); + + if (rel_vprot) + se->vprot = div64_long(se->vprot * old_weight, weight); +} static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight) { bool curr = cfs_rq->curr == se; bool rel_vprot = false; - u64 vprot; + u64 avruntime = 0; if (se->on_rq) { /* commit outstanding execution time */ update_curr(cfs_rq); - update_entity_lag(cfs_rq, se); - se->deadline -= se->vruntime; + avruntime = avg_vruntime(cfs_rq); + se->vlag = entity_lag(cfs_rq, se, avruntime); + se->deadline -= avruntime; se->rel_deadline = 1; if (curr && protect_slice(se)) { - vprot = se->vprot - se->vruntime; + se->vprot -= avruntime; rel_vprot = true; } @@ -3865,30 +3980,23 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, } dequeue_load_avg(cfs_rq, se); - /* - * Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i), - * we need to scale se->vlag when w_i changes. - */ - se->vlag = div_s64(se->vlag * se->load.weight, weight); - if (se->rel_deadline) - se->deadline = div_s64(se->deadline * se->load.weight, weight); - - if (rel_vprot) - vprot = div_s64(vprot * se->load.weight, weight); + rescale_entity(se, weight, rel_vprot); update_load_set(&se->load, weight); do { u32 divider = get_pelt_divider(&se->avg); - se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider); } while (0); enqueue_load_avg(cfs_rq, se); if (se->on_rq) { - place_entity(cfs_rq, se, 0); if (rel_vprot) - se->vprot = se->vruntime + vprot; + se->vprot += avruntime; + se->deadline += avruntime; + se->rel_deadline = 0; + se->vruntime = avruntime - se->vlag; + update_load_add(&cfs_rq->load, se->load.weight); if (!curr) __enqueue_entity(cfs_rq, se); @@ -4363,7 +4471,8 @@ static inline void update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq) { long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg; - u32 new_sum, divider; + u64 new_sum; + u32 divider; /* Nothing to update */ if (!delta_avg) @@ -4377,7 +4486,7 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf /* Set new sched_entity's runnable */ se->avg.runnable_avg = gcfs_rq->avg.runnable_avg; - new_sum = se->avg.runnable_avg * divider; + new_sum = (u64)se->avg.runnable_avg * divider; delta_sum = (long)new_sum - (long)se->avg.runnable_sum; se->avg.runnable_sum = new_sum; @@ -5288,7 +5397,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) se->vruntime = vruntime - lag; - if (se->rel_deadline) { + if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) { se->deadline += se->vruntime; se->rel_deadline = 0; return; @@ -8779,7 +8888,7 @@ static void set_cpus_allowed_fair(struct task_struct *p, struct affinity_context } static int -balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +balance_fair(struct rq *rq, struct rq_flags *rf) { if (sched_fair_runnable(rq)) return 1; @@ -9100,6 +9209,8 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf idle: if (rf) { new_tasks = sched_balance_newidle(rq, rf); + /* The donor may have changed while the rq lock was dropped. */ + prev = rq->donor; /* * Because sched_balance_newidle() releases (and re-acquires) @@ -9395,6 +9506,7 @@ struct lb_env { int dst_cpu; struct rq *dst_rq; + bool dst_core_idle; struct cpumask *dst_grpmask; int new_dst_cpu; @@ -10577,7 +10689,9 @@ static inline void update_sg_lb_stats(struct lb_env *env, /* Check for a misfit task on the cpu */ if (sgs->group_misfit_task_load < rq->misfit_task_load) { sgs->group_misfit_task_load = rq->misfit_task_load; - *sg_overloaded = 1; + + if (balancing_at_rd) + *sg_overloaded = 1; } } else if (env->idle && sched_reduced_capacity(rq, env->sd)) { /* Check for a task running on a CPU with reduced capacity */ @@ -10636,13 +10750,30 @@ static bool update_sd_pick_busiest(struct lb_env *env, * We can use max_capacity here as reduction in capacity on some * CPUs in the group should either be possible to resolve * internally or be covered by avg_load imbalance (eventually). + * + * When SMT is active, only pull a misfit to dst_cpu if it is on a + * fully idle core; otherwise the effective capacity of the core is + * reduced and we may not actually provide more capacity than the + * source. */ if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && (sgs->group_type == group_misfit_task) && - (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || + (!env->dst_core_idle || + !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) || sds->local_stat.group_type != group_has_spare)) return false; + /* + * Candidate sg has no more than one task per CPU and has higher + * per-CPU capacity. Migrating tasks to less capable CPUs may harm + * throughput. Maximize throughput, power/energy consequences are not + * considered. + */ + if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && + (sgs->group_type <= group_fully_busy) && + (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) + return false; + if (sgs->group_type > busiest->group_type) return true; @@ -10745,17 +10876,6 @@ static bool update_sd_pick_busiest(struct lb_env *env, break; } - /* - * Candidate sg has no more than one task per CPU and has higher - * per-CPU capacity. Migrating tasks to less capable CPUs may harm - * throughput. Maximize throughput, power/energy consequences are not - * considered. - */ - if ((env->sd->flags & SD_ASYM_CPUCAPACITY) && - (sgs->group_type <= group_fully_busy) && - (capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu)))) - return false; - return true; } @@ -11206,6 +11326,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd unsigned long sum_util = 0; bool sg_overloaded = 0, sg_overutilized = 0; + env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu); + do { struct sg_lb_stats *sgs = &tmp_sgs; int local_group; diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index bf92ae29361ed..220fc9e7c22db 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -460,7 +460,7 @@ select_task_rq_idle(struct task_struct *p, int cpu, int flags) } static int -balance_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +balance_idle(struct rq *rq, struct rq_flags *rf) { return WARN_ON_ONCE(1); } diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 59fdb7ebbf22a..7650f6b99acd3 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup) return; cancel_delayed_work_sync(&cgroup->psi->avgs_work); + /* + * A psi_schedule_rtpoll_work() call racing the last trigger's + * destruction may have re-armed the timer after psi_trigger_destroy() + * deleted it. Spurious firing while the group is alive is harmless. + */ + timer_shutdown_sync(&cgroup->psi->rtpoll_timer); free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n"); @@ -1292,9 +1298,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) return 0; } +/* + * Create @group's rtpoll worker after psi_trigger_create() reported the need + * for one. kthread creation depends on the whole fork path and we don't want + * all of that nested inside cgroup_mutex, so the caller must drop it and any + * other lock that forks can wait behind. If two callers race, the loser stops + * its never-woken kthread. + */ +int psi_trigger_create_rtpoll_worker(struct psi_group *group) +{ + struct task_struct *task; + + task = kthread_create(psi_rtpoll_worker, group, "psimon"); + if (IS_ERR(task)) + return PTR_ERR(task); + + scoped_guard(mutex, &group->rtpoll_trigger_lock) { + if (!rcu_access_pointer(group->rtpoll_task)) { + atomic_set(&group->rtpoll_wakeup, 0); + wake_up_process(task); + rcu_assign_pointer(group->rtpoll_task, task); + + /* + * Poll once to catch up on scheduling attempts dropped + * while there was no rtpoll worker. + */ + psi_schedule_rtpoll_work(group, 1, true); + return 0; + } + } + + kthread_stop(task); + return 0; +} + struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of) + struct kernfs_open_file *of, + bool *need_rtpoll_worker) { struct psi_trigger *t; enum psi_states state; @@ -1302,6 +1343,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, bool privileged; u32 window_us; + *need_rtpoll_worker = false; + if (static_branch_likely(&psi_disabled)) return ERR_PTR(-EOPNOTSUPP); @@ -1362,26 +1405,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, if (privileged) { mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->rtpoll_task)) { - struct task_struct *task; - - task = kthread_create(psi_rtpoll_worker, group, "psimon"); - if (IS_ERR(task)) { - kfree(t); - mutex_unlock(&group->rtpoll_trigger_lock); - return ERR_CAST(task); - } - atomic_set(&group->rtpoll_wakeup, 0); - wake_up_process(task); - rcu_assign_pointer(group->rtpoll_task, task); - } - list_add(&t->node, &group->rtpoll_triggers); group->rtpoll_min_period = min(group->rtpoll_min_period, div_u64(t->win.size, UPDATES_PER_WINDOW)); group->rtpoll_nr_triggers[t->state]++; group->rtpoll_states |= (1 << t->state); + *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task); + mutex_unlock(&group->rtpoll_trigger_lock); } else { mutex_lock(&group->avgs_lock); @@ -1541,6 +1572,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, size_t buf_size; struct seq_file *seq; struct psi_trigger *new; + bool need_rtpoll_worker; + int ret; if (static_branch_likely(&psi_disabled)) return -EOPNOTSUPP; @@ -1565,12 +1598,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, res, file, NULL); + new = psi_trigger_create(&psi_system, buf, res, file, NULL, + &need_rtpoll_worker); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); } + if (need_rtpoll_worker) { + ret = psi_trigger_create_rtpoll_worker(&psi_system); + if (ret) { + psi_trigger_destroy(new); + mutex_unlock(&seq->lock); + return ret; + } + } + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c index 3e81b9d83d14d..fc9737decfb73 100644 --- a/kernel/sched/rt.c +++ b/kernel/sched/rt.c @@ -1591,8 +1591,14 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) resched_curr(rq); } -static int balance_rt(struct rq *rq, struct task_struct *p, struct rq_flags *rf) +static int balance_rt(struct rq *rq, struct rq_flags *rf) { + /* + * Note, rq->donor may change during rq lock drops, + * so don't re-use p across lock drops + */ + struct task_struct *p = rq->donor; + if (!on_rt_rq(&p->rt) && need_pull_rt_task(rq, p)) { /* * This is OK, because current is on_cpu, which avoids it being @@ -1847,13 +1853,22 @@ static int find_lowest_rq(struct task_struct *task) static struct task_struct *pick_next_pushable_task(struct rq *rq) { - struct task_struct *p; + struct plist_head *head = &rq->rt.pushable_tasks; + struct task_struct *i, *p = NULL; if (!has_pushable_tasks(rq)) return NULL; - p = plist_first_entry(&rq->rt.pushable_tasks, - struct task_struct, pushable_tasks); + plist_for_each_entry(i, head, pushable_tasks) { + /* skip tasks that cannot be migrated */ + if (!task_on_cpu(rq, i) && !is_migration_disabled(i)) { + p = i; + break; + } + } + + if (!p) + return NULL; BUG_ON(rq->cpu != task_cpu(p)); BUG_ON(task_current(rq, p)); diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 668841022dbf2..97dddc148d711 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1310,6 +1310,7 @@ struct rq { unsigned int core_forceidle_seq; unsigned int core_forceidle_occupation; u64 core_forceidle_start; + unsigned int core_pick_in_flight; #endif /* CONFIG_SCHED_CORE */ /* Scratch cpumask to be temporarily used under rq_lock */ @@ -1787,6 +1788,13 @@ static inline void scx_rq_clock_update(struct rq *rq, u64 clock) {} static inline void scx_rq_clock_invalidate(struct rq *rq) {} #endif /* !CONFIG_SCHED_CLASS_EXT */ +static inline void assert_balance_callbacks_empty(struct rq *rq) +{ + WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_LOCKING) && + rq->balance_callback && + rq->balance_callback != &balance_push_callback); +} + /* * Lockdep annotation that avoids accidental unlocks; it's like a * sticky/continuous lockdep_assert_held(). @@ -1803,7 +1811,7 @@ static inline void rq_pin_lock(struct rq *rq, struct rq_flags *rf) rq->clock_update_flags &= (RQCF_REQ_SKIP|RQCF_ACT_SKIP); rf->clock_update_flags = 0; - WARN_ON_ONCE(rq->balance_callback && rq->balance_callback != &balance_push_callback); + assert_balance_callbacks_empty(rq); } static inline void rq_unpin_lock(struct rq *rq, struct rq_flags *rf) @@ -2415,7 +2423,7 @@ struct sched_class { void (*wakeup_preempt)(struct rq *rq, struct task_struct *p, int flags); - int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf); + int (*balance)(struct rq *rq, struct rq_flags *rf); struct task_struct *(*pick_task)(struct rq *rq); /* * Optional! When implemented pick_next_task() should be equivalent to: diff --git a/kernel/sched/stop_task.c b/kernel/sched/stop_task.c index 2d4e279f05ee9..73aeb0743aa2e 100644 --- a/kernel/sched/stop_task.c +++ b/kernel/sched/stop_task.c @@ -16,7 +16,7 @@ select_task_rq_stop(struct task_struct *p, int cpu, int flags) } static int -balance_stop(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) +balance_stop(struct rq *rq, struct rq_flags *rf) { return sched_stop_runnable(rq); } diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 20f27e2cf7aec..d033f600f48c6 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -5,6 +5,7 @@ * (C) 2004 Nadia Yvette Chambers, Oracle */ #include "sched.h" +#include void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *key) { @@ -463,3 +464,17 @@ int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sy return default_wake_function(wq_entry, mode, sync, key); } EXPORT_SYMBOL(woken_wake_function); + +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) + return 0; + + /* Pairs with the smp_store_mb() in wait_woken(). */ + smp_mb(); /* C */ + wq_entry->flags |= WQ_FLAG_WOKEN; + + return default_wake_function(wq_entry, mode, sync, key); +} +EXPORT_SYMBOL(woken_wake_bit_function); diff --git a/kernel/sched/wait_bit.c b/kernel/sched/wait_bit.c index 1088d3b7012cf..348f7211b4aa4 100644 --- a/kernel/sched/wait_bit.c +++ b/kernel/sched/wait_bit.c @@ -167,9 +167,7 @@ wait_queue_head_t *__var_waitqueue(void *p) } EXPORT_SYMBOL(__var_waitqueue); -static int -var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, - int sync, void *arg) +struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg) { struct wait_bit_key *key = arg; struct wait_bit_queue_entry *wbq_entry = @@ -177,6 +175,16 @@ var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, if (wbq_entry->key.flags != key->flags || wbq_entry->key.bit_nr != key->bit_nr) + return NULL; + + return key; +} + +static int var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode, + int sync, void *arg) +{ + struct wait_bit_key *key = __var_wake_key(wq_entry, arg); + if (!key) return 0; return autoremove_wake_function(wq_entry, mode, sync, key); diff --git a/kernel/signal.c b/kernel/signal.c index 810098300ecdc..b18f07a5b62d6 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1183,6 +1183,7 @@ static inline bool has_si_pid_and_uid(struct kernel_siginfo *info) int send_signal_locked(int sig, struct kernel_siginfo *info, struct task_struct *t, enum pid_type type) { + struct kernel_siginfo rewritten; /* Should SIGKILL or SIGSTOP be received by a pid namespace init? */ bool force = false; @@ -1196,6 +1197,9 @@ int send_signal_locked(int sig, struct kernel_siginfo *info, /* SIGKILL and SIGSTOP is special or has ids */ struct user_namespace *t_user_ns; + rewritten = *info; + info = &rewritten; + rcu_read_lock(); t_user_ns = task_cred_xxx(t, user_ns); if (current_user_ns() != t_user_ns) { @@ -1364,8 +1368,16 @@ struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, rcu_read_lock(); for (;;) { sighand = rcu_dereference(tsk->sighand); - if (unlikely(sighand == NULL)) + if (unlikely(sighand == NULL)) { + /* + * Pairs with the smp_store_release() in + * __exit_signal(). It ensures that all state + * modifications to the task preceeding the store are + * visible to the callers of lock_task_sighand(). + */ + smp_acquire__after_ctrl_dep(); break; + } /* * This sighand can be already freed and even reused, but @@ -1893,6 +1905,18 @@ int kill_pid(struct pid *pid, int sig, int priv) } EXPORT_SYMBOL(kill_pid); +int kill_cad_pid(int sig, int priv) +{ + int ret; + + rcu_read_lock(); + ret = kill_pid(rcu_dereference(cad_pid), sig, priv); + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL(kill_cad_pid); + #ifdef CONFIG_POSIX_TIMERS /* * These functions handle POSIX timer signals. POSIX timers use diff --git a/kernel/sys.c b/kernel/sys.c index 8b58eece4e580..c8376d7a0741b 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -2189,7 +2189,7 @@ static int prctl_set_auxv(struct mm_struct *mm, unsigned long addr, BUILD_BUG_ON(sizeof(user_auxv) != sizeof(mm->saved_auxv)); task_lock(current); - memcpy(mm->saved_auxv, user_auxv, len); + memcpy(mm->saved_auxv, user_auxv, sizeof(user_auxv)); task_unlock(current); return 0; diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 2cd0172d05160..05a905e64fa1e 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -368,10 +368,9 @@ static int parse(struct nlattr *na, struct cpumask *mask) return -E2BIG; if (len < 1) return -EINVAL; - data = kmalloc(len, GFP_KERNEL); + data = nla_strdup(na, GFP_KERNEL); if (!data) return -ENOMEM; - nla_strscpy(data, na, len); ret = cpulist_parse(data, mask); kfree(data); return ret; diff --git a/kernel/time/Makefile b/kernel/time/Makefile index f7d52d9543cc7..662bccb3b7f9a 100644 --- a/kernel/time/Makefile +++ b/kernel/time/Makefile @@ -29,6 +29,6 @@ endif obj-$(CONFIG_GENERIC_GETTIMEOFDAY) += vsyscall.o obj-$(CONFIG_DEBUG_FS) += timekeeping_debug.o obj-$(CONFIG_TEST_UDELAY) += test_udelay.o -obj-$(CONFIG_TIME_NS) += namespace.o +obj-$(CONFIG_TIME_NS) += namespace.o namespace_vdso.o obj-$(CONFIG_TEST_CLOCKSOURCE_WATCHDOG) += clocksource-wdtest.o obj-$(CONFIG_TIME_KUNIT_TEST) += time_test.o diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c index 189ee8bbc3e65..525df334b15a7 100644 --- a/kernel/time/alarmtimer.c +++ b/kernel/time/alarmtimer.c @@ -521,12 +521,13 @@ static void alarm_handle_timer(struct alarm *alarm, ktime_t now) * alarm_timer_rearm - Posix timer callback for rearming timer * @timr: Pointer to the posixtimer data struct */ -static void alarm_timer_rearm(struct k_itimer *timr) +static bool alarm_timer_rearm(struct k_itimer *timr) { struct alarm *alarm = &timr->it.alarm.alarmtimer; timr->it_overrun += alarm_forward_now(alarm, timr->it_interval); alarm_start(alarm, alarm->node.expires); + return true; } /** @@ -582,7 +583,7 @@ static void alarm_timer_wait_running(struct k_itimer *timr) * @absolute: Expiry value is absolute time * @sigev_none: Posix timer does not deliver signals */ -static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, +static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct alarm *alarm = &timr->it.alarm.alarmtimer; @@ -594,6 +595,7 @@ static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires, alarm->node.expires = expires; else alarm_start(&timr->it.alarm.alarmtimer, expires); + return true; } /** diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index a59bc75ab7c5b..e6180545ea5b2 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -565,33 +565,40 @@ void clockevents_handle_noop(struct clock_event_device *dev) { } -/** - * clockevents_exchange_device - release and request clock devices - * @old: device to release (can be NULL) - * @new: device to request (can be NULL) - * - * Called from various tick functions with clockevents_lock held and - * interrupts disabled. - */ -void clockevents_exchange_device(struct clock_event_device *old, - struct clock_event_device *new) +void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) { /* * Caller releases a clock event device. We queue it into the * released list and do a notify add later. */ if (old) { - module_put(old->owner); clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED); list_move(&old->list, &clockevents_released); } if (new) { - BUG_ON(!clockevent_state_detached(new)); + WARN_ON(!clockevent_state_detached(new)); clockevents_shutdown(new); } } +/** + * clockevents_exchange_device - release and request clock devices + * @old: device to release (can be NULL) + * @new: device to request (can be NULL) + * + * Called from various tick functions with clockevents_lock held and + * interrupts disabled. + */ +void clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) +{ + __clockevents_exchange_device(old, new); + if (old) + module_put(old->owner); +} + /** * clockevents_suspend - suspend clock devices */ @@ -649,7 +656,7 @@ void tick_offline_cpu(unsigned int cpu) if (cpumask_test_cpu(cpu, dev->cpumask) && cpumask_weight(dev->cpumask) == 1 && !tick_is_broadcast_device(dev)) { - BUG_ON(!clockevent_state_detached(dev)); + WARN_ON(!clockevent_state_detached(dev)); list_del(&dev->list); } } diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index df71949616584..c2ce6cea24107 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -1482,8 +1482,12 @@ static int __init init_clocksource_sysfs(void) { int error = subsys_system_register(&clocksource_subsys, NULL); - if (!error) - error = device_register(&device_clocksource); + if (error) + return error; + + error = device_register(&device_clocksource); + if (error) + bus_unregister(&clocksource_subsys); return error; } diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c index 7c6110e964e7e..03a32dffc56cc 100644 --- a/kernel/time/itimer.c +++ b/kernel/time/itimer.c @@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value) static int put_itimerval(struct __kernel_old_itimerval __user *o, const struct itimerspec64 *i) { - struct __kernel_old_itimerval v; + struct __kernel_old_itimerval v = {}; v.it_interval.tv_sec = i->it_interval.tv_sec; v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC; diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c index 5b6997f4dc3da..51d2fcba7b5fb 100644 --- a/kernel/time/namespace.c +++ b/kernel/time/namespace.c @@ -18,8 +18,9 @@ #include #include #include +#include -#include +#include "namespace_internal.h" ktime_t do_timens_ktime_to_host(clockid_t clockid, ktime_t tim, struct timens_offsets *ns_offsets) @@ -138,117 +139,7 @@ struct time_namespace *copy_time_ns(u64 flags, return clone_time_ns(user_ns, old_ns); } -static struct timens_offset offset_from_ts(struct timespec64 off) -{ - struct timens_offset ret; - - ret.sec = off.tv_sec; - ret.nsec = off.tv_nsec; - - return ret; -} - -/* - * A time namespace VVAR page has the same layout as the VVAR page which - * contains the system wide VDSO data. - * - * For a normal task the VVAR pages are installed in the normal ordering: - * VVAR - * PVCLOCK - * HVCLOCK - * TIMENS <- Not really required - * - * Now for a timens task the pages are installed in the following order: - * TIMENS - * PVCLOCK - * HVCLOCK - * VVAR - * - * The check for vdso_clock->clock_mode is in the unlikely path of - * the seq begin magic. So for the non-timens case most of the time - * 'seq' is even, so the branch is not taken. - * - * If 'seq' is odd, i.e. a concurrent update is in progress, the extra check - * for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the - * update to finish and for 'seq' to become even anyway. - * - * Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which - * enforces the time namespace handling path. - */ -static void timens_setup_vdso_clock_data(struct vdso_clock *vc, - struct time_namespace *ns) -{ - struct timens_offset *offset = vc->offset; - struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic); - struct timens_offset boottime = offset_from_ts(ns->offsets.boottime); - - vc->seq = 1; - vc->clock_mode = VDSO_CLOCKMODE_TIMENS; - offset[CLOCK_MONOTONIC] = monotonic; - offset[CLOCK_MONOTONIC_RAW] = monotonic; - offset[CLOCK_MONOTONIC_COARSE] = monotonic; - offset[CLOCK_BOOTTIME] = boottime; - offset[CLOCK_BOOTTIME_ALARM] = boottime; -} - -struct page *find_timens_vvar_page(struct vm_area_struct *vma) -{ - if (likely(vma->vm_mm == current->mm)) - return current->nsproxy->time_ns->vvar_page; - - /* - * VM_PFNMAP | VM_IO protect .fault() handler from being called - * through interfaces like /proc/$pid/mem or - * process_vm_{readv,writev}() as long as there's no .access() - * in special_mapping_vmops(). - * For more details check_vma_flags() and __access_remote_vm() - */ - - WARN(1, "vvar_page accessed remotely"); - - return NULL; -} - -/* - * Protects possibly multiple offsets writers racing each other - * and tasks entering the namespace. - */ -static DEFINE_MUTEX(offset_lock); - -static void timens_set_vvar_page(struct task_struct *task, - struct time_namespace *ns) -{ - struct vdso_time_data *vdata; - struct vdso_clock *vc; - unsigned int i; - - if (ns == &init_time_ns) - return; - - /* Fast-path, taken by every task in namespace except the first. */ - if (likely(ns->frozen_offsets)) - return; - - mutex_lock(&offset_lock); - /* Nothing to-do: vvar_page has been already initialized. */ - if (ns->frozen_offsets) - goto out; - - ns->frozen_offsets = true; - vdata = page_address(ns->vvar_page); - vc = vdata->clock_data; - - for (i = 0; i < CS_BASES; i++) - timens_setup_vdso_clock_data(&vc[i], ns); - - if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) { - for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++) - timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns); - } - -out: - mutex_unlock(&offset_lock); -} +DEFINE_MUTEX(timens_offset_lock); void free_time_ns(struct time_namespace *ns) { @@ -298,12 +189,6 @@ static void timens_put(struct ns_common *ns) put_time_ns(to_time_ns(ns)); } -void timens_commit(struct task_struct *tsk, struct time_namespace *ns) -{ - timens_set_vvar_page(tsk, ns); - vdso_join_timens(tsk, ns); -} - static int timens_install(struct nsset *nsset, struct ns_common *new) { struct nsproxy *nsproxy = nsset->nsproxy; @@ -367,36 +252,33 @@ static void show_offset(struct seq_file *m, int clockid, struct timespec64 *ts) void proc_timens_show_offsets(struct task_struct *p, struct seq_file *m) { - struct ns_common *ns; - struct time_namespace *time_ns; + struct time_namespace *time_ns __free(time_ns) = NULL; + struct ns_common *ns = timens_for_children_get(p); - ns = timens_for_children_get(p); if (!ns) return; + time_ns = to_time_ns(ns); show_offset(m, CLOCK_MONOTONIC, &time_ns->offsets.monotonic); show_offset(m, CLOCK_BOOTTIME, &time_ns->offsets.boottime); - put_time_ns(time_ns); } int proc_timens_set_offset(struct file *file, struct task_struct *p, struct proc_timens_offset *offsets, int noffsets) { - struct ns_common *ns; - struct time_namespace *time_ns; + struct time_namespace *time_ns __free(time_ns) = NULL; + struct ns_common *ns = timens_for_children_get(p); struct timespec64 tp; int i, err; - ns = timens_for_children_get(p); if (!ns) return -ESRCH; + time_ns = to_time_ns(ns); - if (!file_ns_capable(file, time_ns->user_ns, CAP_SYS_TIME)) { - put_time_ns(time_ns); + if (!file_ns_capable(file, time_ns->user_ns, CAP_SYS_TIME)) return -EPERM; - } for (i = 0; i < noffsets; i++) { struct proc_timens_offset *off = &offsets[i]; @@ -409,15 +291,14 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, ktime_get_boottime_ts64(&tp); break; default: - err = -EINVAL; - goto out; + return -EINVAL; } - err = -ERANGE; + if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX) + return -ERANGE; - if (off->val.tv_sec > KTIME_SEC_MAX || - off->val.tv_sec < -KTIME_SEC_MAX) - goto out; + if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC) + return -EINVAL; tp = timespec64_add(tp, off->val); /* @@ -425,10 +306,10 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, * still unreachable. */ if (tp.tv_sec < 0 || tp.tv_sec > KTIME_SEC_MAX / 2) - goto out; + return -ERANGE; } - mutex_lock(&offset_lock); + mutex_lock(&timens_offset_lock); if (time_ns->frozen_offsets) { err = -EACCES; goto out_unlock; @@ -453,9 +334,7 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p, } out_unlock: - mutex_unlock(&offset_lock); -out: - put_time_ns(time_ns); + mutex_unlock(&timens_offset_lock); return err; } diff --git a/kernel/time/namespace_internal.h b/kernel/time/namespace_internal.h new file mode 100644 index 0000000000000..e85da11abb4d9 --- /dev/null +++ b/kernel/time/namespace_internal.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TIME_NAMESPACE_INTERNAL_H +#define _TIME_NAMESPACE_INTERNAL_H + +#include + +/* + * Protects possibly multiple offsets writers racing each other + * and tasks entering the namespace. + */ +extern struct mutex timens_offset_lock; + +#endif /* _TIME_NAMESPACE_INTERNAL_H */ diff --git a/kernel/time/namespace_vdso.c b/kernel/time/namespace_vdso.c new file mode 100644 index 0000000000000..0e154f9015012 --- /dev/null +++ b/kernel/time/namespace_vdso.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Author: Andrei Vagin + * Author: Dmitry Safonov + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "namespace_internal.h" + +static struct timens_offset offset_from_ts(struct timespec64 off) +{ + struct timens_offset ret; + + ret.sec = off.tv_sec; + ret.nsec = off.tv_nsec; + + return ret; +} + +/* + * A time namespace VVAR page has the same layout as the VVAR page which + * contains the system wide VDSO data. + * + * For a normal task the VVAR pages are installed in the normal ordering: + * VVAR + * PVCLOCK + * HVCLOCK + * TIMENS <- Not really required + * + * Now for a timens task the pages are installed in the following order: + * TIMENS + * PVCLOCK + * HVCLOCK + * VVAR + * + * The check for vdso_clock->clock_mode is in the unlikely path of + * the seq begin magic. So for the non-timens case most of the time + * 'seq' is even, so the branch is not taken. + * + * If 'seq' is odd, i.e. a concurrent update is in progress, the extra check + * for vdso_clock->clock_mode is a non-issue. The task is spin waiting for the + * update to finish and for 'seq' to become even anyway. + * + * Timens page has vdso_clock->clock_mode set to VDSO_CLOCKMODE_TIMENS which + * enforces the time namespace handling path. + */ +static void timens_setup_vdso_clock_data(struct vdso_clock *vc, + struct time_namespace *ns) +{ + struct timens_offset *offset = vc->offset; + struct timens_offset monotonic = offset_from_ts(ns->offsets.monotonic); + struct timens_offset boottime = offset_from_ts(ns->offsets.boottime); + + vc->seq = 1; + vc->clock_mode = VDSO_CLOCKMODE_TIMENS; + offset[CLOCK_MONOTONIC] = monotonic; + offset[CLOCK_MONOTONIC_RAW] = monotonic; + offset[CLOCK_MONOTONIC_COARSE] = monotonic; + offset[CLOCK_BOOTTIME] = boottime; + offset[CLOCK_BOOTTIME_ALARM] = boottime; +} + +struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + if (likely(vma->vm_mm == current->mm)) + return current->nsproxy->time_ns->vvar_page; + + /* + * VM_PFNMAP | VM_IO protect .fault() handler from being called + * through interfaces like /proc/$pid/mem or + * process_vm_{readv,writev}() as long as there's no .access() + * in special_mapping_vmops(). + * For more details check_vma_flags() and __access_remote_vm() + */ + + WARN(1, "vvar_page accessed remotely"); + + return NULL; +} + +static void timens_set_vvar_page(struct task_struct *task, + struct time_namespace *ns) +{ + struct vdso_time_data *vdata; + struct vdso_clock *vc; + unsigned int i; + + if (ns == &init_time_ns) + return; + + /* Fast-path, taken by every task in namespace except the first. */ + if (likely(ns->frozen_offsets)) + return; + + guard(mutex)(&timens_offset_lock); + /* Nothing to-do: vvar_page has been already initialized. */ + if (ns->frozen_offsets) + return; + + ns->frozen_offsets = true; + vdata = page_address(ns->vvar_page); + vc = vdata->clock_data; + + for (i = 0; i < CS_BASES; i++) + timens_setup_vdso_clock_data(&vc[i], ns); + + if (IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS)) { + for (i = 0; i < ARRAY_SIZE(vdata->aux_clock_data); i++) + timens_setup_vdso_clock_data(&vdata->aux_clock_data[i], ns); + } +} + +/* + * The vvar page layout depends on whether a task belongs to the root or + * non-root time namespace. Whenever a task changes its namespace, the VVAR + * page tables are cleared and then they will be re-faulted with a + * corresponding layout. + * See also the comment near timens_setup_vdso_clock_data() for details. + */ +static int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) +{ + struct mm_struct *mm = task->mm; + struct vm_area_struct *vma; + VMA_ITERATOR(vmi, mm, 0); + + guard(mmap_read_lock)(mm); + for_each_vma(vmi, vma) { + if (vma_is_special_mapping(vma, &vdso_vvar_mapping)) + zap_vma_pages(vma); + } + return 0; +} + +void timens_commit(struct task_struct *tsk, struct time_namespace *ns) +{ + timens_set_vvar_page(tsk, ns); + vdso_join_timens(tsk, ns); +} diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 99affd9e228ec..2d64972fd0804 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -19,7 +19,7 @@ #include "posix-timers.h" -static void posix_cpu_timer_rearm(struct k_itimer *timer); +static bool posix_cpu_timer_rearm(struct k_itimer *timer); void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit) { @@ -461,6 +461,109 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) trigger_base_recalc_expires(timer, p); } +/* + * Lookup the task via timer->it.cpu.pid and attempt to lock the task's sighand. + * + * This can race with the reaping of the task: + * + * CPU0 CPU1 + * + * // Finds task + * p = pid_task(pid, pid_type); __exit_signal(p) + * lock(p, sighand); + * posix_cpu_timers*_exit(); + * sighand = lock_task_sighand(p); unhash_task(p); + * p->sighand = NULL; + * unlock(sighand); + * + * In this case sighand is NULL, which means the task and the associated timer + * queue cannot be longer accessed safely. + * + * __exit_signal() invokes posix_cpu_timers_exit() and if the thread group is + * dead it also invokes posix_cpu_timers_group_exit(). These functions delete + * all pending timers from the related timer queues. The POSIX timers (k_itimer) + * themself are still accessible, but not longer connected to the task. + * + * exec() works slightly differently. The task which exec()'s terminates all + * other threads in the thread group and runs __exit_signal() on them. As the + * thread group is not dead they only clean up the per task timers via + * posix_cpu_timers_exit(). + * + * As the TGID on exec() stays the same per process timers stay queued, if they + * are armed. This works without a problem when exec() is done by the thread + * group leader. If a non-leader thread exec()'s this can end up in the + * following scenario: + * + * CPU0 CPU1 + * // Returns old leader + * p = pid_task(pid, pid_type); de_thread() + * switch_leader() + * release_task(old leader) + * __exit_signal() + * old_leader->sighand = NULL; + * // Returns NULL + * sighand = lock_task_sighand(p) + * + * That's problematic for several functions: + * + * - posix_cpu_timer_del(): If the timer is still enqueued on the task the + * underlying k_itimer will be freed which results in a UAF in + * run_posix_cpu_timers() or on timerqueue related add/delete operations. + * If the timer is not enqueued, the failure is harmless + * + * - posix_cpu_timer_set(): Independent of the enqueued state that results in a + * transient failure which is user space visible (-ESRCH) for regular posix + * timers. But for the use case in do_cpu_nanosleep() it's the same UAF + * problem just that the timer is allocated on the stack. + * + * - posix_cpu_timer_rearm(): Timer is not enqueued at that point, but this + * silently ignores the rearm request, which is a functional problem as the + * timer wont expire anymore. + */ +static struct task_struct *timer_lock_sighand(struct k_itimer *timer, unsigned long *flags) +{ + enum pid_type type = clock_pid_type(timer->it_clock); + struct cpu_timer *ctmr = &timer->it.cpu; + + guard(rcu)(); + + for (;;) { + struct task_struct *t = pid_task(timer->it.cpu.pid, type); + + /* Fail if the task cannot be found. */ + if (!t) + break; + + /* Try to lock the task's sighand */ + if (lock_task_sighand(t, flags)) + return t; + + /* + * The next PID lookup might either fail or return the new + * leader. This is correct for both exit() and exec(). + */ + } + + /* + * If the timer is still enqueued, warn. There is nothing safe to do + * here as there might be two timers in there which are removed in + * parallel and that will cause more damage than good. This should never + * happen! + * + * Ensure that the stores to the timer and timerqueue are visible: + * + * __exit_signal() + * posix_cpu_timers*_exit() + * write_seqlock(seqlock) + * smp_wmb(); <------- + * __unhash_process() | !pid_task() + * ----> smp_rmb(); + * WARN_ON_ONCE(...) + */ + smp_rmb(); + WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); + return NULL; +} /* * Clean up a CPU-clock timer that is about to be destroyed. @@ -470,29 +573,13 @@ static void disarm_timer(struct k_itimer *timer, struct task_struct *p) */ static int posix_cpu_timer_del(struct k_itimer *timer) { - struct cpu_timer *ctmr = &timer->it.cpu; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; + p = timer_lock_sighand(timer, &flags); - /* - * Protect against sighand release/switch in exit/exec and process/ - * thread timer list entry concurrent read/writes. - */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) { - /* - * This raced with the reaping of the task. The exit cleanup - * should have removed this timer from the timer queue. - */ - WARN_ON_ONCE(ctmr->head || timerqueue_node_queued(&ctmr->node)); - } else { + if (likely(p)) { if (timer->it.cpu.firing) { /* * Prevent signal delivery. The timer cannot be dequeued @@ -508,11 +595,8 @@ static int posix_cpu_timer_del(struct k_itimer *timer) unlock_task_sighand(p, &flags); } -out: - rcu_read_unlock(); - if (!ret) { - put_pid(ctmr->pid); + put_pid(timer->it.cpu.pid); timer->it_status = POSIX_TIMER_DISARMED; } return ret; @@ -626,21 +710,17 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct cpu_timer *ctmr = &timer->it.cpu; u64 old_expires, new_expires, now; - struct sighand_struct *sighand; struct task_struct *p; unsigned long flags; int ret = 0; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) { - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - rcu_read_unlock(); + p = timer_lock_sighand(timer, &flags); + /* + * If p has just been reaped, we can no longer get any information about + * it at all. + */ + if (!p) return -ESRCH; - } /* * Use the to_ktime conversion because that clamps the maximum @@ -648,20 +728,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value)); - /* - * Protect against sighand release/switch in exit/exec and p->cpu_timers - * and p->signal->cpu_timers read/write in arm_timer() - */ - sighand = lock_task_sighand(p, &flags); - /* - * If p has just been reaped, we can no - * longer get any information about it at all. - */ - if (unlikely(sighand == NULL)) { - rcu_read_unlock(); - return -ESRCH; - } - /* Retrieve the current expiry time before disarming the timer */ old_expires = cpu_timer_getexpires(ctmr); @@ -698,7 +764,7 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, /* Retry if the timer expiry is running concurrently */ if (unlikely(ret)) { unlock_task_sighand(p, &flags); - goto out; + return ret; } /* Convert relative expiry time to absolute */ @@ -733,8 +799,6 @@ static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags, */ if (!sigev_none && new_expires && now >= new_expires) cpu_timer_fire(timer); -out: - rcu_read_unlock(); return ret; } @@ -1011,24 +1075,20 @@ static void check_process_timers(struct task_struct *tsk, /* * This is called from the signal code (via posixtimer_rearm) * when the last timer signal was delivered and we have to reload the timer. + * + * Return true unconditionally so the core code assumes the timer to be + * armed. Otherwise it would requeue the signal. */ -static void posix_cpu_timer_rearm(struct k_itimer *timer) +static bool posix_cpu_timer_rearm(struct k_itimer *timer) { clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock); struct task_struct *p; - struct sighand_struct *sighand; unsigned long flags; u64 now; - rcu_read_lock(); - p = cpu_timer_task_rcu(timer); - if (!p) - goto out; - - /* Protect timer list r/w in arm_timer() */ - sighand = lock_task_sighand(p, &flags); - if (unlikely(sighand == NULL)) - goto out; + p = timer_lock_sighand(timer, &flags); + if (unlikely(!p)) + return true; /* * Fetch the current sample and update the timer's expiry time. @@ -1045,8 +1105,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer) */ arm_timer(timer, p); unlock_task_sighand(p, &flags); -out: - rcu_read_unlock(); + return true; } /** diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 56e17b625c72e..3082129d836c2 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -295,12 +295,13 @@ static inline int timer_overrun_to_int(struct k_itimer *timr) return (int)timr->it_overrun_last; } -static void common_hrtimer_rearm(struct k_itimer *timr) +static bool common_hrtimer_rearm(struct k_itimer *timr) { struct hrtimer *timer = &timr->it.real.timer; timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval); hrtimer_restart(timer); + return true; } static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr) @@ -802,7 +803,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id) return timer_overrun_to_int(scoped_timer); } -static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, +static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none) { struct hrtimer *timer = &timr->it.real.timer; @@ -829,6 +830,7 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires, if (!sigev_none) hrtimer_start_expires(timer, HRTIMER_MODE_ABS); + return true; } static int common_hrtimer_try_to_cancel(struct k_itimer *timr) diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h index 7f259e845d240..4ea9611dd7165 100644 --- a/kernel/time/posix-timers.h +++ b/kernel/time/posix-timers.h @@ -27,11 +27,11 @@ struct k_clock { int (*timer_del)(struct k_itimer *timr); void (*timer_get)(struct k_itimer *timr, struct itimerspec64 *cur_setting); - void (*timer_rearm)(struct k_itimer *timr); + bool (*timer_rearm)(struct k_itimer *timr); s64 (*timer_forward)(struct k_itimer *timr, ktime_t now); ktime_t (*timer_remaining)(struct k_itimer *timr, ktime_t now); int (*timer_try_to_cancel)(struct k_itimer *timr); - void (*timer_arm)(struct k_itimer *timr, ktime_t expires, + bool (*timer_arm)(struct k_itimer *timr, ktime_t expires, bool absolute, bool sigev_none); void (*timer_wait_running)(struct k_itimer *timr); }; diff --git a/kernel/time/sched_clock.c b/kernel/time/sched_clock.c index 425d429906d0d..f3aaef695b8cd 100644 --- a/kernel/time/sched_clock.c +++ b/kernel/time/sched_clock.c @@ -296,6 +296,11 @@ int sched_clock_suspend(void) return 0; } +static int sched_clock_syscore_suspend(void *data) +{ + return sched_clock_suspend(); +} + void sched_clock_resume(void) { struct clock_read_data *rd = &cd.read_data[0]; @@ -305,14 +310,23 @@ void sched_clock_resume(void) rd->read_sched_clock = cd.actual_read_sched_clock; } -static struct syscore_ops sched_clock_ops = { - .suspend = sched_clock_suspend, - .resume = sched_clock_resume, +static void sched_clock_syscore_resume(void *data) +{ + sched_clock_resume(); +} + +static const struct syscore_ops sched_clock_syscore_ops = { + .suspend = sched_clock_syscore_suspend, + .resume = sched_clock_syscore_resume, +}; + +static struct syscore sched_clock_syscore = { + .ops = &sched_clock_syscore_ops, }; static int __init sched_clock_syscore_init(void) { - register_syscore_ops(&sched_clock_ops); + register_syscore(&sched_clock_syscore); return 0; } diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 0207868c8b4d2..14eae39de4a76 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -162,23 +162,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, */ void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { - struct clock_event_device *cur = tick_broadcast_device.evtdev; + struct clock_event_device *cur; - if (tick_set_oneshot_wakeup_device(dev, cpu)) - return; + scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) { - if (!tick_check_broadcast_device(cur, dev)) - return; + if (tick_set_oneshot_wakeup_device(dev, cpu)) + return; - if (!try_module_get(dev->owner)) - return; + cur = tick_broadcast_device.evtdev; + if (!tick_check_broadcast_device(cur, dev)) + return; - clockevents_exchange_device(cur, dev); + if (!try_module_get(dev->owner)) + return; + + __clockevents_exchange_device(cur, dev); + if (cur) + cur->event_handler = clockevents_handle_noop; + WRITE_ONCE(tick_broadcast_device.evtdev, dev); + if (!cpumask_empty(tick_broadcast_mask)) + tick_broadcast_start_periodic(dev); + } + + /* Module release must be outside of the lock */ if (cur) - cur->event_handler = clockevents_handle_noop; - tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_broadcast_mask)) - tick_broadcast_start_periodic(dev); + module_put(cur->owner); if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) return; @@ -1211,7 +1219,7 @@ int tick_broadcast_oneshot_active(void) */ bool tick_broadcast_oneshot_available(void) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; } @@ -1219,7 +1227,7 @@ bool tick_broadcast_oneshot_available(void) #else int __tick_broadcast_oneshot_control(enum tick_broadcast_state state) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) return -EBUSY; diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 4e4f7bbe2a64b..c572ba9d6cc7e 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -54,6 +54,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev, } extern void clockevents_shutdown(struct clock_event_device *dev); +extern void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new); extern void clockevents_exchange_device(struct clock_event_device *old, struct clock_event_device *new); extern void clockevents_switch_state(struct clock_event_device *dev, diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 889a8de3fc204..377e29ca5a521 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -1995,6 +1995,11 @@ void timekeeping_resume(void) timerfd_resume(); } +static void timekeeping_syscore_resume(void *data) +{ + timekeeping_resume(); +} + int timekeeping_suspend(void) { struct timekeeper *tks = &tk_core.shadow_timekeeper; @@ -2062,15 +2067,24 @@ int timekeeping_suspend(void) return 0; } +static int timekeeping_syscore_suspend(void *data) +{ + return timekeeping_suspend(); +} + /* sysfs resume/suspend bits for timekeeping */ -static struct syscore_ops timekeeping_syscore_ops = { - .resume = timekeeping_resume, - .suspend = timekeeping_suspend, +static const struct syscore_ops timekeeping_syscore_ops = { + .resume = timekeeping_syscore_resume, + .suspend = timekeeping_syscore_suspend, +}; + +static struct syscore timekeeping_syscore = { + .ops = &timekeeping_syscore_ops, }; static int __init timekeeping_init_ops(void) { - register_syscore_ops(&timekeeping_syscore_ops); + register_syscore(&timekeeping_syscore); return 0; } device_initcall(timekeeping_init_ops); @@ -2140,6 +2154,11 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, * xtime_nsec_2 = xtime_nsec_1 - offset * Which simplifies to: * xtime_nsec -= offset + * + * When subtracting offset from xtime_nsec, the same amount + * (in appropriate units) has to be added to ntp_error, in + * order to correctly track the delta between the time + * reported in xtime_nsec, and the intended time. */ if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { /* NTP adjustment caused clocksource mult overflow */ @@ -2150,6 +2169,7 @@ static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, tk->tkr_mono.mult += mult_adj; tk->xtime_interval += interval; tk->tkr_mono.xtime_nsec -= offset; + tk->ntp_error += offset << tk->ntp_error_shift; } /* @@ -2693,10 +2713,12 @@ static int __do_adjtimex(struct tk_data *tkd, struct __kernel_timex *txc, return ret; add_device_randomness(txc, sizeof(*txc)); - if (!aux_clock) + if (!aux_clock) { ktime_get_real_ts64(&ts); - else - tk_get_aux_ts64(tkd->timekeeper.id, &ts); + } else { + if (!tk_get_aux_ts64(tkd->timekeeper.id, &ts)) + return -ENODEV; + } add_device_randomness(&ts, sizeof(ts)); @@ -3063,7 +3085,9 @@ static const struct attribute_group aux_clock_enable_attr_group = { static int __init tk_aux_sysfs_init(void) { struct kobject *auxo, *tko = kobject_create_and_add("time", kernel_kobj); + struct kobject *clks[MAX_AUX_CLOCKS]; int ret = -ENOMEM; + int i; if (!tko) return ret; @@ -3072,21 +3096,28 @@ static int __init tk_aux_sysfs_init(void) if (!auxo) goto err_clean; - for (int i = 0; i < MAX_AUX_CLOCKS; i++) { + for (i = 0; i < MAX_AUX_CLOCKS; i++) { char id[2] = { [0] = '0' + i, }; - struct kobject *clk = kobject_create_and_add(id, auxo); + clks[i] = kobject_create_and_add(id, auxo); - if (!clk) { + if (!clks[i]) { ret = -ENOMEM; - goto err_clean; + goto err_clks; } - ret = sysfs_create_group(clk, &aux_clock_enable_attr_group); + ret = sysfs_create_group(clks[i], &aux_clock_enable_attr_group); if (ret) - goto err_clean; + goto err_clk; } return 0; +err_clk: + kobject_put(clks[i]); +err_clks: + while (--i >= 0) { + sysfs_remove_group(clks[i], &aux_clock_enable_attr_group); + kobject_put(clks[i]); + } err_clean: kobject_put(auxo); kobject_put(tko); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index d5ebb1d927ea6..97a7bd5a8e3b1 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -2491,6 +2491,7 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h timer = hlist_entry(head->first, struct timer_list, entry); detach_timer(timer, false); timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu; + debug_timer_activate(timer); internal_add_timer(new_base, timer); } } diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c index 76d896a99d7b0..32f338052bc4f 100644 --- a/kernel/time/timer_migration.c +++ b/kernel/time/timer_migration.c @@ -1667,8 +1667,10 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node, } /* Assert single root without parent */ - if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) + if (WARN_ON_ONCE(i >= tmigr_hierarchy_levels)) { + kfree(stack); return -EINVAL; + } for (; i >= start_lvl; i--) { group = stack[i]; diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index dcb4e02afc5f4..36b3ed5251f46 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -31,9 +31,10 @@ ifdef CONFIG_GCOV_PROFILE_FTRACE GCOV_PROFILE := y endif -# Functions in this file could be invoked from early interrupt -# code and produce random code coverage. +# Functions in these files can run from IRQ entry before hardirq context +# is visible to KCOV, and produce coverage unrelated to syscall inputs. KCOV_INSTRUMENT_trace_preemptirq.o := n +KCOV_INSTRUMENT_trace_irqsoff.o := n CFLAGS_bpf_trace.o := -I$(src) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index 22d779ae345d0..e692d67907e49 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -3168,6 +3168,7 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr unsigned long __user *uoffsets; u64 __user *ucookies; void __user *upath; + unsigned long size; u32 flags, cnt, i; struct path path; char *name; @@ -3205,6 +3206,16 @@ int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *pr uref_ctr_offsets = u64_to_user_ptr(attr->link_create.uprobe_multi.ref_ctr_offsets); ucookies = u64_to_user_ptr(attr->link_create.uprobe_multi.cookies); + /* + * All uoffsets/uref_ctr_offsets/ucookies arrays have the same value + * size, we need to check their address range is safe for __get_user + * calls. + */ + size = sizeof(*uoffsets) * cnt; + if (!access_ok(uoffsets, size) || !access_ok(uref_ctr_offsets, size) || + !access_ok(ucookies, size)) + return -EFAULT; + name = strndup_user(upath, PATH_MAX); if (IS_ERR(name)) { err = PTR_ERR(name); diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 01f98f8a86e60..1c9ae332afad1 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -868,10 +868,8 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter return -ENOMEM; ret = get_ips_from_filter(filter, notfilter, addrs, mods, num); - if (ret < 0) - return ret; - - ret = register_fprobe_ips(fp, addrs, ret); + if (ret >= 0) + ret = register_fprobe_ips(fp, addrs, ret); for (int i = 0; i < num; i++) { if (mods[i]) diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 905f4d1679559..1bb2175ab1af0 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -76,6 +76,8 @@ .func_hash = &opsname.local_hash, \ .local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock), \ .subop_list = LIST_HEAD_INIT(opsname.subop_list), +/* Used only to synchronize the initialization of ftrace_ops */ +static DEFINE_MUTEX(ops_mutex); #else #define INIT_OPS_HASH(opsname) #endif @@ -160,11 +162,18 @@ const struct ftrace_ops ftrace_nop_ops = { static inline void ftrace_ops_init(struct ftrace_ops *ops) { #ifdef CONFIG_DYNAMIC_FTRACE - if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) { + unsigned long flags = smp_load_acquire(&ops->flags); + + if (!(flags & FTRACE_OPS_FL_INITIALIZED)) { + guard(mutex)(&ops_mutex); + /* Could have been initialized before lock taken */ + if (unlikely(ops->flags & FTRACE_OPS_FL_INITIALIZED)) + return; mutex_init(&ops->local_hash.regex_lock); INIT_LIST_HEAD(&ops->subop_list); ops->func_hash = &ops->local_hash; - ops->flags |= FTRACE_OPS_FL_INITIALIZED; + flags = ops->flags | FTRACE_OPS_FL_INITIALIZED; + smp_store_release(&ops->flags, flags); } #endif } @@ -1072,6 +1081,12 @@ struct ftrace_ops global_ops = { FTRACE_OPS_FL_PID, }; +/* + * parser_lock - Protects trace_parser state against concurrent operations. + * Held across trace_get_user() and subsequent buffer parsing to prevent races. + */ +static DEFINE_MUTEX(parser_lock); + /* * Used by the stack unwinder to know about dynamic ftrace trampolines. */ @@ -2588,7 +2603,8 @@ unsigned long ftrace_find_rec_direct(unsigned long ip) { struct ftrace_func_entry *entry; - entry = __ftrace_lookup_ip(direct_functions, ip); + guard(preempt_notrace)(); + entry = __ftrace_lookup_ip(rcu_dereference_sched(direct_functions), ip); if (!entry) return 0; @@ -4611,7 +4627,8 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) /** * ftrace_regex_open - initialize function tracer filter files - * @ops: The ftrace_ops that hold the hash filters + * @tr: The trace_array that holds the ftrace_ops [optional] + * @ops: The ftrace_ops that hold the hash filters [optional] * @flag: The type of filter to process * @inode: The inode, usually passed in to your open routine * @file: The file, usually passed in to your open routine @@ -4625,26 +4642,45 @@ ftrace_avail_addrs_open(struct inode *inode, struct file *file) * tracing_lseek() should be used as the lseek routine, and * release must call ftrace_regex_release(). * + * Note, If @tr is not NULL, its reference has to be taken before + * @ops may be referenced. + * If @ops is NULL and @tr is not, then @tr->ops is used. + * If @tr is NULL and @ops is not then @ops->private is uesd for @tr. + * If both @tr and @ops are NULL, then the &global_ops is + * to be used, and @tr will be the global_ops.private pointer. + * * Returns: 0 on success or a negative errno value on failure */ int -ftrace_regex_open(struct ftrace_ops *ops, int flag, +ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file) { - struct ftrace_iterator *iter; + struct ftrace_iterator *iter = NULL; struct ftrace_hash *hash; struct list_head *mod_head; - struct trace_array *tr = ops->private; - int ret = -ENOMEM; - - ftrace_ops_init(ops); + int ret = -ENODEV; if (unlikely(ftrace_disabled)) return -ENODEV; + if (!tr) { + if (!ops) + ops = &global_ops; + tr = ops->private; + } + if (tracing_check_open_get_tr(tr)) return -ENODEV; + if (!ops) + ops = tr->ops; + + if (WARN_ON_ONCE(!ops)) + goto out; + + ftrace_ops_init(ops); + + ret = -ENOMEM; iter = kzalloc(sizeof(*iter), GFP_KERNEL); if (!iter) goto out; @@ -4722,21 +4758,19 @@ ftrace_regex_open(struct ftrace_ops *ops, int flag, static int ftrace_filter_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, - FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, - inode, file); + return ftrace_regex_open(tr, NULL, + FTRACE_ITER_FILTER | FTRACE_ITER_DO_PROBES, + inode, file); } static int ftrace_notrace_open(struct inode *inode, struct file *file) { - struct ftrace_ops *ops = inode->i_private; + struct trace_array *tr = inode->i_private; - /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_NOTRACE, + return ftrace_regex_open(tr, NULL, FTRACE_ITER_NOTRACE, inode, file); } @@ -5783,6 +5817,8 @@ ftrace_regex_write(struct file *file, const char __user *ubuf, /* iter->hash is a local copy, so we don't need regex_lock */ parser = &iter->parser; + + guard(mutex)(&parser_lock); read = trace_get_user(parser, ubuf, cnt, ppos); if (read >= 0 && trace_parser_loaded(parser) && @@ -6551,12 +6587,14 @@ int ftrace_regex_release(struct inode *inode, struct file *file) iter = file->private_data; parser = &iter->parser; + mutex_lock(&parser_lock); if (trace_parser_loaded(parser)) { int enable = !(iter->flags & FTRACE_ITER_NOTRACE); ftrace_process_regex(iter, parser->buffer, parser->idx, enable); } + mutex_unlock(&parser_lock); trace_parser_put(parser); @@ -6888,10 +6926,12 @@ ftrace_graph_release(struct inode *inode, struct file *file) parser = &fgd->parser; + mutex_lock(&parser_lock); if (trace_parser_loaded((parser))) { ret = ftrace_graph_set_hash(fgd->new_hash, parser->buffer); } + mutex_unlock(&parser_lock); trace_parser_put(parser); @@ -7004,6 +7044,7 @@ ftrace_graph_write(struct file *file, const char __user *ubuf, parser = &fgd->parser; + guard(mutex)(&parser_lock); read = trace_get_user(parser, ubuf, cnt, ppos); if (read >= 0 && trace_parser_loaded(parser) && @@ -7037,15 +7078,15 @@ static const struct file_operations ftrace_graph_notrace_fops = { }; #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent) { trace_create_file("set_ftrace_filter", TRACE_MODE_WRITE, parent, - ops, &ftrace_filter_fops); + tr, &ftrace_filter_fops); trace_create_file("set_ftrace_notrace", TRACE_MODE_WRITE, parent, - ops, &ftrace_notrace_fops); + tr, &ftrace_notrace_fops); } /* @@ -7070,7 +7111,6 @@ void ftrace_destroy_filter_files(struct ftrace_ops *ops) static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) { - trace_create_file("available_filter_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_avail_fops); @@ -7083,7 +7123,7 @@ static __init int ftrace_init_dyn_tracefs(struct dentry *d_tracer) trace_create_file("touched_functions", TRACE_MODE_READ, d_tracer, NULL, &ftrace_touched_fops); - ftrace_create_filter_files(&global_ops, d_tracer); + ftrace_create_filter_files(NULL, d_tracer); #ifdef CONFIG_FUNCTION_GRAPH_TRACER trace_create_file("set_graph_function", TRACE_MODE_WRITE, d_tracer, @@ -7850,7 +7890,8 @@ static void add_to_clear_hash_list(struct list_head *clear_list, void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) { unsigned long start = (unsigned long)(start_ptr); - unsigned long end = (unsigned long)(end_ptr); + /* end is inclusive and end_ptr is exclusive */ + unsigned long end = (unsigned long)(end_ptr) - 1; struct ftrace_page **last_pg = &ftrace_pages_start; struct ftrace_page *tmp_page = NULL; struct ftrace_page *pg; @@ -7860,6 +7901,9 @@ void ftrace_free_mem(struct module *mod, void *start_ptr, void *end_ptr) struct ftrace_init_func *func, *func_next; LIST_HEAD(clear_hash); + if (start_ptr >= end_ptr) + return; + key.ip = start; key.flags = end; /* overload flags, as it is unsigned long */ diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index fefa16e51e0ef..008f6c407aac4 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -396,6 +396,17 @@ static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) return local_read(&bpage->page->commit); } +/** + * rb_page_capacity - Get the capacity of a buffer page + * @bpage: The buffer page + * + * Return: The maximum size available for events in the given buffer page. + */ +static __always_inline unsigned int rb_page_capacity(struct buffer_page *bpage) +{ + return (PAGE_SIZE << bpage->order) - BUF_PAGE_HDR_SIZE; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -404,6 +415,41 @@ static void free_buffer_page(struct buffer_page *bpage) kfree(bpage); } +/* + * For best performance, allocate cpu buffer data cache line sized + * and per CPU. + */ +#define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *) \ + kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu), \ + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + +#define alloc_cpu_page(cpu) (struct buffer_page *) \ + kzalloc_node(ALIGN(sizeof(struct buffer_page), \ + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + +static struct buffer_data_page *alloc_cpu_data(int cpu, int order) +{ + struct buffer_data_page *dpage; + struct page *page; + gfp_t mflags; + + /* + * __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails + * gracefully without invoking oom-killer and the system is not + * destabilized. + */ + mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_COMP | __GFP_ZERO; + + page = alloc_pages_node(cpu_to_node(cpu), mflags, order); + if (!page) + return NULL; + + dpage = page_address(page); + rb_init_page(dpage); + + return dpage; +} + /* * We need to fit the time_stamp delta into 27 bits. */ @@ -485,7 +531,7 @@ struct ring_buffer_per_cpu { raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; - struct buffer_data_page *free_page; + struct buffer_data_read_page free_page; unsigned long nr_pages; unsigned int current_context; struct list_head *pages; @@ -522,7 +568,7 @@ struct ring_buffer_per_cpu { unsigned int mapped; unsigned int user_mapped; /* user space mapping */ struct mutex mapping_lock; - unsigned long *subbuf_ids; /* ID to subbuf VA */ + struct buffer_page **subbuf_ids; /* ID to subbuf VA */ struct trace_buffer_meta *meta_page; struct ring_buffer_cpu_meta *ring_meta; @@ -537,7 +583,6 @@ struct ring_buffer_per_cpu { struct trace_buffer { unsigned flags; - int cpus; atomic_t record_disabled; atomic_t resizing; cpumask_var_t cpumask; @@ -560,11 +605,42 @@ struct trace_buffer { struct ring_buffer_meta *meta; - unsigned int subbuf_size; unsigned int subbuf_order; unsigned int max_data_size; }; +static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer) +{ + return PAGE_SIZE << buffer->subbuf_order; +} + +/** + * rb_subbuf_capacity - Get the capacity of a subbuffer + * @buffer: A trace buffer + * + * Unsafe to use without holding trace_buffer::mutex or with resizing enabled. + * Consider rb_page_capacity() instead. + * + * Return: The maximum size available for events in a trace buffer subbuffer. + */ +static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buffer) +{ + return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE; +} + +/** + * rb_subbuf_start - Get the start address of a subbuffer + * @buffer: A trace buffer + * @addr: An address of an event on a subbuffer + * + * Return: The start of the subbuffer for where @addr sits + */ +static __always_inline +unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr) +{ + return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1)); +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -604,7 +680,7 @@ int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), - (unsigned int)buffer->subbuf_size, + (unsigned int)rb_subbuf_capacity(buffer), (unsigned int)is_signed_type(char)); return !trace_seq_has_overflowed(s); @@ -799,14 +875,13 @@ static void rb_wake_up_waiters(struct irq_work *work) struct ring_buffer_per_cpu *cpu_buffer = container_of(rbwork, struct ring_buffer_per_cpu, irq_work); - /* Called from interrupt context */ - raw_spin_lock(&cpu_buffer->reader_lock); - rbwork->wakeup_full = false; - rbwork->full_waiters_pending = false; + scoped_guard(raw_spinlock_irqsave, &cpu_buffer->reader_lock) { + rbwork->wakeup_full = false; + rbwork->full_waiters_pending = false; - /* Waking up all waiters, they will reset the shortest full */ - cpu_buffer->shortest_full = 0; - raw_spin_unlock(&cpu_buffer->reader_lock); + /* Waking up all waiters, they will reset the shortest full */ + cpu_buffer->shortest_full = 0; + } wake_up_all(&rbwork->full_waiters); } @@ -1593,7 +1668,7 @@ rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) */ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) { - int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + int subbuf_size = rb_subbuf_size(buffer); struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; unsigned long ptr; @@ -2214,7 +2289,6 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_cpu_meta *meta = NULL; struct buffer_page *bpage, *tmp; bool user_thread = current->mm != NULL; - gfp_t mflags; long i; /* @@ -2228,13 +2302,6 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, if (i < nr_pages) return -ENOMEM; - /* - * __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails - * gracefully without invoking oom-killer and the system is not - * destabilized. - */ - mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL; - /* * If a user thread allocates too much, and si_mem_available() * reports there's enough memory, even though there is not. @@ -2251,10 +2318,8 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); for (i = 0; i < nr_pages; i++) { - struct page *page; - bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), - mflags, cpu_to_node(cpu_buffer->cpu)); + bpage = alloc_cpu_page(cpu_buffer->cpu); if (!bpage) goto free_pages; @@ -2277,13 +2342,10 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, bpage->range = 1; bpage->id = i + 1; } else { - page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu), - mflags | __GFP_COMP | __GFP_ZERO, - cpu_buffer->buffer->subbuf_order); - if (!page) + bpage->page = alloc_cpu_data(cpu_buffer->cpu, + cpu_buffer->buffer->subbuf_order); + if (!bpage->page) goto free_pages; - bpage->page = page_address(page); - rb_init_page(bpage->page); } bpage->order = cpu_buffer->buffer->subbuf_order; @@ -2334,14 +2396,12 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, static struct ring_buffer_per_cpu * rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) { - struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = NULL; + struct ring_buffer_per_cpu *cpu_buffer __free(kfree) = + alloc_cpu_buffer(cpu); struct ring_buffer_cpu_meta *meta; struct buffer_page *bpage; - struct page *page; int ret; - cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()), - GFP_KERNEL, cpu_to_node(cpu)); if (!cpu_buffer) return NULL; @@ -2357,10 +2417,10 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) init_waitqueue_head(&cpu_buffer->irq_work.full_waiters); mutex_init(&cpu_buffer->mapping_lock); - bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), - GFP_KERNEL, cpu_to_node(cpu)); + bpage = alloc_cpu_page(cpu); if (!bpage) return NULL; + bpage->order = cpu_buffer->buffer->subbuf_order; rb_check_bpage(cpu_buffer, bpage); @@ -2379,14 +2439,12 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) if (cpu_buffer->ring_meta->head_buffer) rb_meta_buffer_update(cpu_buffer, bpage); bpage->range = 1; + + atomic_inc(&cpu_buffer->resize_disabled); } else { - page = alloc_pages_node(cpu_to_node(cpu), - GFP_KERNEL | __GFP_COMP | __GFP_ZERO, - cpu_buffer->buffer->subbuf_order); - if (!page) + bpage->page = alloc_cpu_data(cpu, bpage->order); + if (!bpage->page) goto fail_free_reader; - bpage->page = page_address(page); - rb_init_page(bpage->page); } INIT_LIST_HEAD(&cpu_buffer->reader_page->list); @@ -2454,7 +2512,7 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) free_buffer_page(bpage); } - free_page((unsigned long)cpu_buffer->free_page); + free_pages((unsigned long)cpu_buffer->free_page.data, cpu_buffer->free_page.order); kfree(cpu_buffer); } @@ -2493,10 +2551,9 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, buffer->subbuf_order = order; subbuf_size = (PAGE_SIZE << order); - buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; /* Max payload is buffer page size - header (8bytes) */ - buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); + buffer->max_data_size = rb_subbuf_capacity(buffer) - (sizeof(u32) * 2); buffer->flags = flags; buffer->clock = trace_clock_local; @@ -2505,7 +2562,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&buffer->irq_work.waiters); - buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), @@ -2569,9 +2625,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, rb_range_meta_init(buffer, nr_pages, scratch_size); } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); if (nr_pages < 2) nr_pages = 2; } @@ -2944,7 +2999,7 @@ static void update_pages_handler(struct work_struct *work) * @size: the new size. * @cpu_id: the cpu buffer to resize * - * Minimum size is 2 * buffer->subbuf_size. + * Minimum size is 2 * rb_subbuf_capacity(buffer). * * Returns 0 on success and < 0 on failure. */ @@ -2966,12 +3021,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, !cpumask_test_cpu(cpu_id, buffer->cpumask)) return 0; - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; - /* * Keep CPUs from coming online while resizing to synchronize * with new per CPU buffers being created. @@ -2982,6 +3031,12 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + + /* we need a minimum of two pages */ + if (nr_pages < 2) + nr_pages = 2; + if (cpu_id == RING_BUFFER_ALL_CPUS) { /* * Don't succeed if resizing is disabled, as a reader might be @@ -3260,7 +3315,7 @@ rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event { unsigned long addr = (unsigned long)event; - addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; + addr &= (unsigned long)rb_subbuf_size(cpu_buffer->buffer) - 1; return addr - BUF_PAGE_HDR_SIZE; } @@ -3499,8 +3554,8 @@ static inline void rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, unsigned long tail, struct rb_event_info *info) { - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *tail_page = info->tail_page; + unsigned long bsize = rb_page_capacity(tail_page); struct ring_buffer_event *event; unsigned long length = info->length; @@ -3852,8 +3907,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, new_index = rb_event_index(cpu_buffer, event); old_index = new_index + rb_event_ts_length(event); - addr = (unsigned long)event; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, (unsigned long)event); bpage = READ_ONCE(cpu_buffer->tail_page); @@ -4500,7 +4554,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, tail = write - info->length; /* See if we shot pass the end of this buffer page */ - if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { + if (unlikely(write > rb_page_capacity(tail_page))) { check_buffer(cpu_buffer, info, CHECK_FULL_PAGE); return rb_move_tail(cpu_buffer, tail, info); } @@ -4745,7 +4799,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage = cpu_buffer->commit_page; struct buffer_page *start; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, addr); /* Do the likely case first */ if (likely(bpage->page == (void *)addr)) { @@ -5422,7 +5476,6 @@ static struct buffer_page * rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { struct buffer_page *reader = NULL; - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); unsigned long overwrite; unsigned long flags; int nr_loops = 0; @@ -5561,7 +5614,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) #define USECS_WAIT 1000000 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { /* If the write is past the end of page, a writer is still updating it */ - if (likely(!reader || rb_page_write(reader) <= bsize)) + if (likely(!reader || rb_page_write(reader) <= rb_page_capacity(reader))) break; udelay(1); @@ -5983,36 +6036,44 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); struct ring_buffer_iter * ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags) { + struct ring_buffer_iter *iter __free(kfree) = kzalloc_obj(*iter, flags); struct ring_buffer_per_cpu *cpu_buffer; - struct ring_buffer_iter *iter; - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return NULL; - iter = kzalloc(sizeof(*iter), flags); if (!iter) return NULL; - /* Holds the entire event: data and meta data */ - iter->event_size = buffer->subbuf_size; - iter->event = kmalloc(iter->event_size, flags); - if (!iter->event) { - kfree(iter); + if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - } cpu_buffer = buffer->buffers[cpu]; - iter->cpu_buffer = cpu_buffer; + /* + * Only KDB is using GFP_ATOMIC, for the others, lock the buffer to + * prevent concurrent resizing. + */ + if (gfpflags_allow_blocking(flags)) + mutex_lock(&buffer->mutex); atomic_inc(&cpu_buffer->resize_disabled); + if (gfpflags_allow_blocking(flags)) + mutex_unlock(&buffer->mutex); + + /* Holds the entire event: data and meta data. */ + iter->event_size = rb_page_capacity(READ_ONCE(cpu_buffer->reader_page)); + iter->event = kmalloc(iter->event_size, flags); + if (!iter->event) { + atomic_dec(&cpu_buffer->resize_disabled); + return NULL; + } + iter->cpu_buffer = cpu_buffer; + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); arch_spin_lock(&cpu_buffer->lock); rb_iter_reset(iter); arch_spin_unlock(&cpu_buffer->lock); - return iter; + return_ptr(iter); } EXPORT_SYMBOL_GPL(ring_buffer_read_start); @@ -6066,7 +6127,7 @@ unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; + return rb_subbuf_capacity(buffer) * buffer->buffers[cpu]->nr_pages; } EXPORT_SYMBOL_GPL(ring_buffer_size); @@ -6399,7 +6460,7 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, { struct ring_buffer_per_cpu *cpu_buffer_a; struct ring_buffer_per_cpu *cpu_buffer_b; - int ret = -EINVAL; + int ret = -EBUSY; if (!cpumask_test_cpu(cpu, buffer_a->cpumask) || !cpumask_test_cpu(cpu, buffer_b->cpumask)) @@ -6440,10 +6501,10 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, atomic_inc(&cpu_buffer_a->record_disabled); atomic_inc(&cpu_buffer_b->record_disabled); - ret = -EBUSY; - if (local_read(&cpu_buffer_a->committing)) + /* Do not swap if either buffer is in the process of writing */ + if (cpu_buffer_a->current_context) goto out_dec; - if (local_read(&cpu_buffer_b->committing)) + if (cpu_buffer_b->current_context) goto out_dec; /* @@ -6493,7 +6554,6 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) struct ring_buffer_per_cpu *cpu_buffer; struct buffer_data_read_page *bpage = NULL; unsigned long flags; - struct page *page; if (!cpumask_test_cpu(cpu, buffer->cpumask)) return ERR_PTR(-ENODEV); @@ -6507,30 +6567,24 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (cpu_buffer->free_page) { - bpage->data = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + if (cpu_buffer->free_page.data) { + *bpage = cpu_buffer->free_page; + cpu_buffer->free_page.data = NULL; } arch_spin_unlock(&cpu_buffer->lock); local_irq_restore(flags); - if (bpage->data) - goto out; - - page = alloc_pages_node(cpu_to_node(cpu), - GFP_KERNEL | __GFP_NORETRY | __GFP_COMP | __GFP_ZERO, - cpu_buffer->buffer->subbuf_order); - if (!page) { - kfree(bpage); - return ERR_PTR(-ENOMEM); + if (bpage->data) { + rb_init_page(bpage->data); + } else { + bpage->data = alloc_cpu_data(cpu, bpage->order); + if (!bpage->data) { + kfree(bpage); + return ERR_PTR(-ENOMEM); + } } - bpage->data = page_address(page); - - out: - rb_init_page(bpage->data); - return bpage; } EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page); @@ -6567,8 +6621,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (!cpu_buffer->free_page) { - cpu_buffer->free_page = bpage; + if (!cpu_buffer->free_page.data) { + cpu_buffer->free_page = *data_page; bpage = NULL; } @@ -6625,6 +6679,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, struct buffer_page *reader; unsigned long missed_events; unsigned int commit; + unsigned int size; unsigned int read; u64 save_timestamp; @@ -6643,15 +6698,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (!data_page || !data_page->data) return -1; - if (data_page->order != buffer->subbuf_order) - return -1; - bpage = data_page->data; if (!bpage) return -1; guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); + if (data_page->order != cpu_buffer->reader_page->order) + return -1; + reader = rb_get_reader_page(cpu_buffer); if (!reader) return -1; @@ -6659,7 +6714,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, event = rb_reader_event(cpu_buffer); read = reader->read; - commit = rb_page_size(reader); + commit = rb_page_commit(reader); + size = rb_page_size(reader); /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; @@ -6671,34 +6727,31 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * we must copy the data from the page to the buffer. * Otherwise, we can simply swap the page with the one passed in. */ - if (read || (len < (commit - read)) || + if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || cpu_buffer->mapped) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; - unsigned int size; + unsigned int event_size; + unsigned int flags = 0; - /* - * If a full page is expected, this can still be returned - * if there's been a previous partial read and the - * rest of the page can be read and the commit page is off - * the reader page. - */ - if (full && - (!read || (len < (commit - read)) || - cpu_buffer->reader_page == cpu_buffer->commit_page)) + /* If a full page is requested, it cannot be the commit page */ + if (full && cpu_buffer->reader_page == cpu_buffer->commit_page) return -1; - if (len > (commit - read)) - len = (commit - read); + if (len > (size - read)) + len = (size - read); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); + event_size = rb_event_ts_length(event); - if (len < size) + if (len < event_size) return -1; + if (commit & RB_MISSED_EVENTS) + flags = RB_MISSED_EVENTS; + /* save the current timestamp, since the user will need it */ save_timestamp = cpu_buffer->read_stamp; @@ -6710,25 +6763,25 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * one or two events. * We have already ensured there's enough space if this * is a time extend. */ - size = rb_event_length(event); - memcpy(bpage->data + pos, rpage->data + rpos, size); + event_size = rb_event_length(event); + memcpy(bpage->data + pos, rpage->data + rpos, event_size); - len -= size; + len -= event_size; rb_advance_reader(cpu_buffer); rpos = reader->read; - pos += size; + pos += event_size; - if (rpos >= commit) + if (rpos >= size) break; event = rb_reader_event(cpu_buffer); /* Always keep the time extend and data together */ - size = rb_event_ts_length(event); - } while (len >= size); + event_size = rb_event_ts_length(event); + } while (len >= event_size); /* update bpage */ - local_set(&bpage->commit, pos); + local_set(&bpage->commit, pos | flags); bpage->time_stamp = save_timestamp; /* we copied everything to the beginning */ @@ -6753,12 +6806,14 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * on the page. */ if (reader->real_end) - local_set(&bpage->commit, reader->real_end); + local_set(&bpage->commit, reader->real_end | + (commit & RB_MISSED_EVENTS)); } cpu_buffer->lost_events = 0; commit = local_read(&bpage->commit); + size = commit & ~RB_MISSED_MASK; /* * Set a flag in the commit field if we lost events */ @@ -6766,20 +6821,20 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* If there is room at the end of the page to save the * missed events, then record it there. */ - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { - memcpy(&bpage->data[commit], &missed_events, + if (rb_page_capacity(reader) - size >= sizeof(missed_events)) { + memcpy(&bpage->data[size], &missed_events, sizeof(missed_events)); - local_add(RB_MISSED_STORED, &bpage->commit); - commit += sizeof(missed_events); + commit |= RB_MISSED_STORED; + size += sizeof(missed_events); } - local_add(RB_MISSED_EVENTS, &bpage->commit); + local_set(&bpage->commit, commit | RB_MISSED_EVENTS); } /* * This page may be off to user land. Zero it out here. */ - if (commit < buffer->subbuf_size) - memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); + if (size < rb_page_capacity(reader)) + memset(&bpage->data[size], 0, rb_page_capacity(reader) - size); return read; } @@ -6805,7 +6860,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); */ int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) { - return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + return rb_subbuf_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get); @@ -6850,7 +6905,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage, *tmp; - int old_order, old_size; + unsigned int old_capacity; + int old_order; int nr_pages; int psize; int err; @@ -6859,9 +6915,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (!buffer || order < 0) return -EINVAL; - if (buffer->subbuf_order == order) - return 0; - psize = (1 << order) * PAGE_SIZE; if (psize <= BUF_PAGE_HDR_SIZE) return -EINVAL; @@ -6870,18 +6923,29 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; - old_order = buffer->subbuf_order; - old_size = buffer->subbuf_size; - /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); + + old_order = buffer->subbuf_order; + if (old_order == order) + return 0; + + old_capacity = rb_subbuf_capacity(buffer); + + /* The mmap fast path reads subbuf_order without buffer->mutex. */ + for_each_buffer_cpu(buffer, cpu) { + if (!cpumask_test_cpu(cpu, buffer->cpumask)) + continue; + if (atomic_read(&buffer->buffers[cpu]->resize_disabled)) + return -EBUSY; + } + atomic_inc(&buffer->record_disabled); /* Make sure all commits have finished */ synchronize_rcu(); buffer->subbuf_order = order; - buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { @@ -6891,14 +6955,14 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer = buffer->buffers[cpu]; - if (cpu_buffer->mapped) { + if (atomic_read(&cpu_buffer->resize_disabled)) { err = -EBUSY; goto error; } /* Update the number of pages to match the new size */ - nr_pages = old_size * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); + nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; + nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); /* we need a minimum of two pages */ if (nr_pages < 2) @@ -6920,7 +6984,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } for_each_buffer_cpu(buffer, cpu) { - struct buffer_data_page *old_free_data_page; + struct buffer_data_read_page old_free_data_page; struct list_head old_pages; unsigned long flags; @@ -6961,8 +7025,10 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update; cpu_buffer->nr_pages_to_update = 0; + arch_spin_lock(&cpu_buffer->lock); old_free_data_page = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + cpu_buffer->free_page.data = NULL; + arch_spin_unlock(&cpu_buffer->lock); rb_head_page_activate(cpu_buffer); @@ -6973,7 +7039,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) list_del_init(&bpage->list); free_buffer_page(bpage); } - free_pages((unsigned long)old_free_data_page, old_order); + free_pages((unsigned long)old_free_data_page.data, old_free_data_page.order); rb_check_pages(cpu_buffer); } @@ -6984,7 +7050,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) error: buffer->subbuf_order = old_order; - buffer->subbuf_size = old_size; atomic_dec(&buffer->record_disabled); @@ -7029,7 +7094,7 @@ static void rb_free_meta_page(struct ring_buffer_per_cpu *cpu_buffer) } static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, - unsigned long *subbuf_ids) + struct buffer_page **subbuf_ids) { struct trace_buffer_meta *meta = cpu_buffer->meta_page; unsigned int nr_subbufs = cpu_buffer->nr_pages + 1; @@ -7038,7 +7103,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, int id = 0; id = rb_page_id(cpu_buffer, cpu_buffer->reader_page, id); - subbuf_ids[id++] = (unsigned long)cpu_buffer->reader_page->page; + subbuf_ids[id++] = cpu_buffer->reader_page; cnt++; first_subbuf = subbuf = rb_set_head_page(cpu_buffer); @@ -7048,7 +7113,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, if (WARN_ON(id >= nr_subbufs)) break; - subbuf_ids[id] = (unsigned long)subbuf->page; + subbuf_ids[id] = subbuf; rb_inc_page(&subbuf); id++; @@ -7057,12 +7122,12 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, WARN_ON(cnt != nr_subbufs); - /* install subbuf ID to kern VA translation */ + /* install subbuf ID to bpage translation */ cpu_buffer->subbuf_ids = subbuf_ids; meta->meta_struct_len = sizeof(*meta); meta->nr_subbufs = nr_subbufs; - meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + meta->subbuf_size = rb_subbuf_size(cpu_buffer->buffer); meta->meta_page_size = meta->subbuf_size; rb_update_meta_page(cpu_buffer); @@ -7213,13 +7278,15 @@ static int __rb_map_vma(struct ring_buffer_per_cpu *cpu_buffer, } while (p < nr_pages) { + struct buffer_page *subbuf; struct page *page; int off = 0; if (WARN_ON_ONCE(s >= nr_subbufs)) return -EINVAL; - page = virt_to_page((void *)cpu_buffer->subbuf_ids[s]); + subbuf = cpu_buffer->subbuf_ids[s]; + page = virt_to_page((void *)subbuf->page); for (; off < (1 << (subbuf_order)); off++, page++) { if (p >= nr_pages) @@ -7246,7 +7313,8 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, struct vm_area_struct *vma) { struct ring_buffer_per_cpu *cpu_buffer; - unsigned long flags, *subbuf_ids; + struct buffer_page **subbuf_ids; + unsigned long flags; int err; if (!cpumask_test_cpu(cpu, buffer->cpumask)) @@ -7270,7 +7338,7 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, if (err) return err; - /* subbuf_ids include the reader while nr_pages does not */ + /* subbuf_ids includes the reader while nr_pages does not */ subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL); if (!subbuf_ids) { rb_free_meta_page(cpu_buffer); @@ -7421,7 +7489,7 @@ int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) * missed events, then record it there. */ commit = rb_page_size(reader); - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (rb_subbuf_capacity(buffer) - commit >= sizeof(missed_events)) { memcpy(&bpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &bpage->commit); @@ -7453,7 +7521,7 @@ int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) out: /* Some archs do not have data cache coherency between kernel and user-space */ flush_kernel_vmap_range(cpu_buffer->reader_page->page, - buffer->subbuf_size + BUF_PAGE_HDR_SIZE); + rb_subbuf_size(buffer)); rb_update_meta_page(cpu_buffer); @@ -7727,7 +7795,7 @@ static __init int test_ringbuffer(void) out_free: for_each_online_cpu(cpu) { - if (!rb_threads[cpu]) + if (IS_ERR_OR_NULL(rb_threads[cpu])) break; kthread_stop(rb_threads[cpu]); } diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 5b1693d58af06..7f25ae6330942 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -537,7 +537,7 @@ static struct trace_array global_trace = { .trace_flags = TRACE_DEFAULT_FLAGS, }; -static struct trace_array *printk_trace = &global_trace; +struct trace_array *printk_trace = &global_trace; /* List of trace_arrays interested in the top level trace_marker */ static LIST_HEAD(marker_copies); @@ -1904,10 +1904,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) unsigned long __read_mostly tracing_thresh; #ifdef CONFIG_TRACER_MAX_TRACE -static const struct file_operations tracing_max_lat_fops; - #ifdef LATENCY_FS_NOTIFY - static struct workqueue_struct *fsnotify_wq; static void latency_fsnotify_workfn(struct work_struct *work) @@ -1924,17 +1921,6 @@ static void latency_fsnotify_workfn_irq(struct irq_work *iwork) queue_work(fsnotify_wq, &tr->fsnotify_work); } -static void trace_create_maxlat_file(struct trace_array *tr, - struct dentry *d_tracer) -{ - INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); - init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); - tr->d_max_latency = trace_create_file("tracing_max_latency", - TRACE_MODE_WRITE, - d_tracer, tr, - &tracing_max_lat_fops); -} - __init static int latency_fsnotify_init(void) { fsnotify_wq = alloc_workqueue("tr_max_lat_wq", @@ -1959,14 +1945,22 @@ void latency_fsnotify(struct trace_array *tr) */ irq_work_queue(&tr->fsnotify_irqwork); } +#endif /* !LATENCY_FS_NOTIFY */ -#else /* !LATENCY_FS_NOTIFY */ - -#define trace_create_maxlat_file(tr, d_tracer) \ - trace_create_file("tracing_max_latency", TRACE_MODE_WRITE, \ - d_tracer, tr, &tracing_max_lat_fops) +static const struct file_operations tracing_max_lat_fops; +static void trace_create_maxlat_file(struct trace_array *tr, + struct dentry *d_tracer) +{ +#ifdef LATENCY_FS_NOTIFY + INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); + init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); #endif + tr->d_max_latency = trace_create_file("tracing_max_latency", + TRACE_MODE_WRITE, + d_tracer, tr, + &tracing_max_lat_fops); +} /* * Copy the new maximum trace into the separate maximum-trace @@ -2101,7 +2095,9 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) __update_max_tr(tr, tsk, cpu); arch_spin_unlock(&tr->max_lock); } - +#else /* !CONFIG_TRACER_MAX_TRACE */ +static inline void trace_create_maxlat_file(struct trace_array *tr, + struct dentry *d_tracer) { } #endif /* CONFIG_TRACER_MAX_TRACE */ struct pipe_wait { @@ -5894,7 +5890,7 @@ trace_event_update_with_eval_map(struct module *mod, map = start; - trace_event_update_all(map, len); + trace_event_update_all(map, len, mod); if (len <= 0) return; @@ -7325,7 +7321,7 @@ static char *trace_user_fault_read(struct trace_user_buf_info *tinfo, { int cpu = smp_processor_id(); char *buffer = per_cpu_ptr(tinfo->tbuf, cpu)->buf; - unsigned int cnt; + unsigned long long cnt; int trys = 0; int ret; @@ -8842,8 +8838,8 @@ static int tracing_buffers_mmap(struct file *filp, struct vm_area_struct *vma) struct trace_iterator *iter = &info->iter; int ret = 0; - /* A memmap'ed buffer is not supported for user space mmap */ - if (iter->tr->flags & TRACE_ARRAY_FL_MEMMAP) + /* A memmap'ed and backup buffers are not supported for user space mmap */ + if (iter->tr->flags & (TRACE_ARRAY_FL_MEMMAP | TRACE_ARRAY_FL_VMALLOC)) return -ENODEV; ret = get_snapshot_map(iter->tr); @@ -9397,11 +9393,72 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt, return cnt; } +/* + * The tr_index is the address of a trace_array->trace_flags_index[] + * element that holds the index of the trace flag. But since the + * trace_array reference has not been taken yet, it cannot be referenced + * as it could have been freed by a rmdir of the instance the trace_array + * represents. + * + * Search the list of trace_arrays and compare the tr_index to the + * address of the entire trace_array trace_flags_index array for each + * trace_array in the list. If one is matched, then take the reference + * and return it. If not, the trace_array no longer exits. + */ +static int trace_array_options_get(void *tr_index) +{ + struct trace_array *tr; + int ret; + + ret = security_locked_down(LOCKDOWN_TRACEFS); + if (ret) + return ret; + + if (tracing_disabled) + return -ENODEV; + + guard(mutex)(&trace_types_lock); + list_for_each_entry(tr, &ftrace_trace_arrays, list) { + if (tr_index >= (void *)&tr->trace_flags_index[0] && + tr_index < (void *)&tr->trace_flags_index[TRACE_FLAGS_MAX_SIZE]) { + tr->ref++; + return 0; + } + } + return -ENODEV; +} + +static int trace_options_open(struct inode *inode, struct file *filp) +{ + void *tr_index = inode->i_private; + + if (trace_array_options_get(tr_index) < 0) + return -ENODEV; + + filp->private_data = tr_index; + + return 0; +} + +static int trace_options_release(struct inode *inode, struct file *filp) +{ + void *tr_index = filp->private_data; + struct trace_array *tr; + unsigned int index; + + get_tr_index(tr_index, &tr, &index); + + trace_array_put(tr); + + return 0; +} + static const struct file_operations trace_options_core_fops = { - .open = tracing_open_generic, - .read = trace_options_core_read, - .write = trace_options_core_write, - .llseek = generic_file_llseek, + .open = trace_options_open, + .read = trace_options_core_read, + .write = trace_options_core_write, + .llseek = generic_file_llseek, + .release = trace_options_release, }; struct dentry *trace_create_file(const char *name, @@ -9699,6 +9756,8 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, /* Do not allow tracing while changing the order of the ring buffer */ tracing_stop_tr(tr); + trace_access_lock(RING_BUFFER_ALL_CPUS); + old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); if (old_order == order) goto out; @@ -9738,6 +9797,7 @@ buffer_subbuf_size_write(struct file *filp, const char __user *ubuf, #endif (*ppos)++; out: + trace_access_unlock(RING_BUFFER_ALL_CPUS); if (ret) cnt = ret; tracing_start_tr(tr); @@ -10258,6 +10318,8 @@ static int __remove_instance(struct trace_array *tr) reserve_mem_release_by_name(tr->range_name); kfree(tr->range_name); } + if (tr->flags & TRACE_ARRAY_FL_VMALLOC) + vfree((void *)tr->range_addr_start); for (i = 0; i < tr->nr_topts; i++) { kfree(tr->topts[i].topts); @@ -10389,9 +10451,7 @@ init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) create_trace_options_dir(tr); -#ifdef CONFIG_TRACER_MAX_TRACE trace_create_maxlat_file(tr, d_tracer); -#endif if (ftrace_create_function_files(tr, d_tracer)) MEM_FAIL(1, "Could not allocate function filter files"); @@ -11060,6 +11120,42 @@ __init static void do_allocate_snapshot(const char *name) static inline void do_allocate_snapshot(const char *name) { } #endif +__init static int backup_instance_area(const char *backup, + unsigned long *addr, phys_addr_t *size) +{ + struct trace_array *backup_tr; + void *allocated_vaddr = NULL; + + backup_tr = trace_array_get_by_name(backup, NULL); + if (!backup_tr) { + pr_warn("Tracing: Instance %s is not found.\n", backup); + return -ENOENT; + } + + if (!(backup_tr->flags & TRACE_ARRAY_FL_BOOT)) { + pr_warn("Tracing: Instance %s is not boot mapped.\n", backup); + trace_array_put(backup_tr); + return -EINVAL; + } + + *size = backup_tr->range_addr_size; + + allocated_vaddr = vzalloc(*size); + if (!allocated_vaddr) { + pr_warn("Tracing: Failed to allocate memory for copying instance %s (size 0x%lx)\n", + backup, (unsigned long)*size); + trace_array_put(backup_tr); + return -ENOMEM; + } + + memcpy(allocated_vaddr, + (void *)backup_tr->range_addr_start, (size_t)*size); + *addr = (unsigned long)allocated_vaddr; + + trace_array_put(backup_tr); + return 0; +} + __init static void enable_instances(void) { struct trace_array *tr; @@ -11082,11 +11178,15 @@ __init static void enable_instances(void) char *flag_delim; char *addr_delim; char *rname __free(kfree) = NULL; + char *backup; tok = strsep(&curr_str, ","); - flag_delim = strchr(tok, '^'); - addr_delim = strchr(tok, '@'); + name = strsep(&tok, "="); + backup = tok; + + flag_delim = strchr(name, '^'); + addr_delim = strchr(name, '@'); if (addr_delim) *addr_delim++ = '\0'; @@ -11094,7 +11194,15 @@ __init static void enable_instances(void) if (flag_delim) *flag_delim++ = '\0'; - name = tok; + if (trace_array_find(name)) { + pr_warn("Tracing: Instance %s already exists\n", name); + continue; + } + + if (backup) { + if (backup_instance_area(backup, &addr, &size) < 0) + continue; + } if (flag_delim) { char *flag; @@ -11172,7 +11280,8 @@ __init static void enable_instances(void) tr = trace_array_create_systems(name, NULL, addr, size); if (IS_ERR(tr)) { - pr_warn("Tracing: Failed to create instance buffer %s\n", curr_str); + pr_warn("Tracing: Failed to create instance buffer '%s' (%ld)\n", name, + PTR_ERR(tr)); continue; } @@ -11190,7 +11299,13 @@ __init static void enable_instances(void) tr->ref++; } - if (start) { + /* + * Backup buffers can be freed but need vfree(). + */ + if (backup) + tr->flags |= TRACE_ARRAY_FL_VMALLOC; + + if (start || backup) { tr->flags |= TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT; tr->range_name = no_free_ptr(rname); } diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 62d5a5f224c4a..ab88b87ac2ee2 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -346,8 +346,8 @@ struct trace_array { spinlock_t snapshot_trigger_lock; unsigned int snapshot; unsigned long max_latency; -#ifdef CONFIG_FSNOTIFY struct dentry *d_max_latency; +#ifdef CONFIG_FSNOTIFY struct work_struct fsnotify_work; struct irq_work fsnotify_irqwork; #endif @@ -451,6 +451,7 @@ enum { TRACE_ARRAY_FL_LAST_BOOT = BIT(2), TRACE_ARRAY_FL_MOD_INIT = BIT(3), TRACE_ARRAY_FL_MEMMAP = BIT(4), + TRACE_ARRAY_FL_VMALLOC = BIT(5), }; #ifdef CONFIG_MODULES @@ -479,6 +480,8 @@ extern bool trace_clock_in_ns(struct trace_array *tr); extern unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr); +extern struct trace_array *printk_trace; + /* * The global tracer (top) should be the first trace array added, * but we check the flag anyway. @@ -1253,7 +1256,7 @@ extern void clear_ftrace_function_probes(struct trace_array *tr); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); -void ftrace_create_filter_files(struct ftrace_ops *ops, +void ftrace_create_filter_files(struct trace_array *tr, struct dentry *parent); void ftrace_destroy_filter_files(struct ftrace_ops *ops); @@ -1276,11 +1279,11 @@ static inline void clear_ftrace_function_probes(struct trace_array *tr) { } +#define ftrace_create_filter_files(tr, parent) do { } while (0) /* * The ops parameter passed in is usually undefined. * This must be a macro. */ -#define ftrace_create_filter_files(ops, parent) do { } while (0) #define ftrace_destroy_filter_files(ops) do { } while (0) #endif /* CONFIG_FUNCTION_TRACER && CONFIG_DYNAMIC_FTRACE */ @@ -1755,13 +1758,13 @@ extern void clear_event_triggers(struct trace_array *tr); enum { EVENT_TRIGGER_FL_PROBE = BIT(0), + EVENT_TRIGGER_FL_COUNT = BIT(1), }; struct event_trigger_data { unsigned long count; int ref; int flags; - const struct event_trigger_ops *ops; struct event_command *cmd_ops; struct event_filter __rcu *filter; char *filter_str; @@ -1786,6 +1789,10 @@ struct enable_trigger_data { bool hist; }; +bool event_trigger_count(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event); + extern int event_enable_trigger_print(struct seq_file *m, struct event_trigger_data *data); extern void event_enable_trigger_free(struct event_trigger_data *data); @@ -1848,64 +1855,6 @@ extern void event_trigger_unregister(struct event_command *cmd_ops, extern void event_file_get(struct trace_event_file *file); extern void event_file_put(struct trace_event_file *file); -/** - * struct event_trigger_ops - callbacks for trace event triggers - * - * The methods in this structure provide per-event trigger hooks for - * various trigger operations. - * - * The @init and @free methods are used during trigger setup and - * teardown, typically called from an event_command's @parse() - * function implementation. - * - * The @print method is used to print the trigger spec. - * - * The @trigger method is the function that actually implements the - * trigger and is called in the context of the triggering event - * whenever that event occurs. - * - * All the methods below, except for @init() and @free(), must be - * implemented. - * - * @trigger: The trigger 'probe' function called when the triggering - * event occurs. The data passed into this callback is the data - * that was supplied to the event_command @reg() function that - * registered the trigger (see struct event_command) along with - * the trace record, rec. - * - * @init: An optional initialization function called for the trigger - * when the trigger is registered (via the event_command reg() - * function). This can be used to perform per-trigger - * initialization such as incrementing a per-trigger reference - * count, for instance. This is usually implemented by the - * generic utility function @event_trigger_init() (see - * trace_event_triggers.c). - * - * @free: An optional de-initialization function called for the - * trigger when the trigger is unregistered (via the - * event_command @reg() function). This can be used to perform - * per-trigger de-initialization such as decrementing a - * per-trigger reference count and freeing corresponding trigger - * data, for instance. This is usually implemented by the - * generic utility function @event_trigger_free() (see - * trace_event_triggers.c). - * - * @print: The callback function invoked to have the trigger print - * itself. This is usually implemented by a wrapper function - * that calls the generic utility function @event_trigger_print() - * (see trace_event_triggers.c). - */ -struct event_trigger_ops { - void (*trigger)(struct event_trigger_data *data, - struct trace_buffer *buffer, - void *rec, - struct ring_buffer_event *rbe); - int (*init)(struct event_trigger_data *data); - void (*free)(struct event_trigger_data *data); - int (*print)(struct seq_file *m, - struct event_trigger_data *data); -}; - /** * struct event_command - callbacks and data members for event commands * @@ -1955,7 +1904,7 @@ struct event_trigger_ops { * * @reg: Adds the trigger to the list of triggers associated with the * event, and enables the event trigger itself, after - * initializing it (via the event_trigger_ops @init() function). + * initializing it (via the event_command @init() function). * This is also where commands can use the @trigger_type value to * make the decision as to whether or not multiple instances of * the trigger should be allowed. This is usually implemented by @@ -1964,7 +1913,7 @@ struct event_trigger_ops { * * @unreg: Removes the trigger from the list of triggers associated * with the event, and disables the event trigger itself, after - * initializing it (via the event_trigger_ops @free() function). + * initializing it (via the event_command @free() function). * This is usually implemented by the generic utility function * @unregister_trigger() (see trace_event_triggers.c). * @@ -1978,12 +1927,41 @@ struct event_trigger_ops { * ignored. This is usually implemented by the generic utility * function @set_trigger_filter() (see trace_event_triggers.c). * - * @get_trigger_ops: The callback function invoked to retrieve the - * event_trigger_ops implementation associated with the command. - * This callback function allows a single event_command to - * support multiple trigger implementations via different sets of - * event_trigger_ops, depending on the value of the @param - * string. + * All the methods below, except for @init() and @free(), must be + * implemented. + * + * @trigger: The trigger 'probe' function called when the triggering + * event occurs. The data passed into this callback is the data + * that was supplied to the event_command @reg() function that + * registered the trigger (see struct event_command) along with + * the trace record, rec. + * + * @count_func: If defined and a numeric parameter is passed to the + * trigger, then this function will be called before @trigger + * is called. If this function returns false, then @trigger is not + * executed. + * + * @init: An optional initialization function called for the trigger + * when the trigger is registered (via the event_command reg() + * function). This can be used to perform per-trigger + * initialization such as incrementing a per-trigger reference + * count, for instance. This is usually implemented by the + * generic utility function @event_trigger_init() (see + * trace_event_triggers.c). + * + * @free: An optional de-initialization function called for the + * trigger when the trigger is unregistered (via the + * event_command @reg() function). This can be used to perform + * per-trigger de-initialization such as decrementing a + * per-trigger reference count and freeing corresponding trigger + * data, for instance. This is usually implemented by the + * generic utility function @event_trigger_free() (see + * trace_event_triggers.c). + * + * @print: The callback function invoked to have the trigger print + * itself. This is usually implemented by a wrapper function + * that calls the generic utility function @event_trigger_print() + * (see trace_event_triggers.c). */ struct event_command { struct list_head list; @@ -2004,7 +1982,18 @@ struct event_command { int (*set_filter)(char *filter_str, struct event_trigger_data *data, struct trace_event_file *file); - const struct event_trigger_ops *(*get_trigger_ops)(char *cmd, char *param); + void (*trigger)(struct event_trigger_data *data, + struct trace_buffer *buffer, + void *rec, + struct ring_buffer_event *rbe); + bool (*count_func)(struct event_trigger_data *data, + struct trace_buffer *buffer, + void *rec, + struct ring_buffer_event *rbe); + int (*init)(struct event_trigger_data *data); + void (*free)(struct event_trigger_data *data); + int (*print)(struct seq_file *m, + struct event_trigger_data *data); }; /** @@ -2025,7 +2014,7 @@ struct event_command { * either committed or discarded. At that point, if any commands * have deferred their triggers, those commands are finally * invoked following the close of the current event. In other - * words, if the event_trigger_ops @func() probe implementation + * words, if the event_command @func() probe implementation * itself logs to the trace buffer, this flag should be set, * otherwise it can be left unspecified. * @@ -2134,13 +2123,13 @@ static inline const char *get_syscall_name(int syscall) #ifdef CONFIG_EVENT_TRACING void trace_event_init(void); -void trace_event_update_all(struct trace_eval_map **map, int len); +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod); /* Used from boot time tracer */ extern int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); extern int trigger_process_regex(struct trace_event_file *file, char *buff); #else static inline void __init trace_event_init(void) { } -static inline void trace_event_update_all(struct trace_eval_map **map, int len) { } +static inline void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { } #endif #ifdef CONFIG_TRACER_SNAPSHOT diff --git a/kernel/trace/trace_btf.c b/kernel/trace/trace_btf.c index 5bbdbcbbde3cd..43e3798011776 100644 --- a/kernel/trace/trace_btf.c +++ b/kernel/trace/trace_btf.c @@ -61,47 +61,50 @@ struct btf_anon_stack { /* * Find a member of data structure/union by name and return it. - * Return NULL if not found, or -EINVAL if parameter is invalid. - * If the member is an member of anonymous union/structure, the offset - * of that anonymous union/structure is stored into @anon_offset. Caller - * can calculate the correct offset from the root data structure by - * adding anon_offset to the member's offset. + * Return NULL if not found, or ERR_PTR(-EINVAL) if parameter is invalid. + * If the member is a member of an anonymous union/structure, the bit offset + * of that anonymous union/structure is stored into @anon_offset. + * If @member_type is non-NULL, the actual containing structure/union type + * of the found member is stored into @member_type. */ const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset) + u32 *anon_offset, + const struct btf_type **member_type) { struct btf_anon_stack *anon_stack; const struct btf_member *member; + const struct btf_type *mtype; u32 tid, cur_offset = 0; const char *name; int i, top = 0; + if (!btf_type_is_struct(type)) + return ERR_PTR(-EINVAL); + anon_stack = kcalloc(BTF_ANON_STACK_MAX, sizeof(*anon_stack), GFP_KERNEL); if (!anon_stack) return ERR_PTR(-ENOMEM); retry: - if (!btf_type_is_struct(type)) { - member = ERR_PTR(-EINVAL); - goto out; - } - for_each_member(i, type, member) { if (!member->name_off) { /* Anonymous union/struct: push it for later use */ - if (btf_type_skip_modifiers(btf, member->type, &tid) && + mtype = btf_type_skip_modifiers(btf, member->type, &tid); + if (mtype && btf_type_is_struct(mtype) && top < BTF_ANON_STACK_MAX) { anon_stack[top].tid = tid; - anon_stack[top++].offset = - cur_offset + member->offset; + anon_stack[top++].offset = cur_offset + + __btf_member_bit_offset(type, member); } } else { name = btf_name_by_offset(btf, member->name_off); if (name && !strcmp(member_name, name)) { if (anon_offset) *anon_offset = cur_offset; + if (member_type) + *member_type = type; goto out; } } diff --git a/kernel/trace/trace_btf.h b/kernel/trace/trace_btf.h index 4bc44bc261e6f..4bd26bceae23a 100644 --- a/kernel/trace/trace_btf.h +++ b/kernel/trace/trace_btf.h @@ -8,4 +8,5 @@ const struct btf_param *btf_get_func_param(const struct btf_type *func_proto, const struct btf_member *btf_find_struct_member(struct btf *btf, const struct btf_type *type, const char *member_name, - u32 *anon_offset); + u32 *anon_offset, + const struct btf_type **member_type); diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index 7697ce22337b6..8296da33672f9 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -169,7 +169,8 @@ static bool eprobe_dyn_event_match(const char *system, const char *event, if (!slash) return false; - if (strncmp(ep->event_system, argv[0], slash - argv[0])) + if (strncmp(ep->event_system, argv[0], slash - argv[0]) || + ep->event_system[slash - argv[0]] != '\0') return false; if (strcmp(ep->event_name, slash + 1)) return false; @@ -484,13 +485,6 @@ static void eprobe_trigger_func(struct event_trigger_data *data, __eprobe_trace_func(edata, rec); } -static const struct event_trigger_ops eprobe_trigger_ops = { - .trigger = eprobe_trigger_func, - .print = eprobe_trigger_print, - .init = eprobe_trigger_init, - .free = eprobe_trigger_free, -}; - static int eprobe_trigger_cmd_parse(struct event_command *cmd_ops, struct trace_event_file *file, char *glob, char *cmd, @@ -513,12 +507,6 @@ static void eprobe_trigger_unreg_func(char *glob, } -static const struct event_trigger_ops *eprobe_trigger_get_ops(char *cmd, - char *param) -{ - return &eprobe_trigger_ops; -} - static struct event_command event_trigger_cmd = { .name = "eprobe", .trigger_type = ETT_EVENT_EPROBE, @@ -527,8 +515,11 @@ static struct event_command event_trigger_cmd = { .reg = eprobe_trigger_reg_func, .unreg = eprobe_trigger_unreg_func, .unreg_all = NULL, - .get_trigger_ops = eprobe_trigger_get_ops, .set_filter = NULL, + .trigger = eprobe_trigger_func, + .print = eprobe_trigger_print, + .init = eprobe_trigger_init, + .free = eprobe_trigger_free, }; static struct event_trigger_data * @@ -548,7 +539,6 @@ new_eprobe_trigger(struct trace_eprobe *ep, struct trace_event_file *file) trigger->flags = EVENT_TRIGGER_FL_PROBE; trigger->count = -1; - trigger->ops = &eprobe_trigger_ops; /* * EVENT PROBE triggers are not registered as commands with diff --git a/kernel/trace/trace_event_perf.c b/kernel/trace/trace_event_perf.c index a6bb7577e8c59..5b272856e5ab1 100644 --- a/kernel/trace/trace_event_perf.c +++ b/kernel/trace/trace_event_perf.c @@ -497,7 +497,17 @@ static int perf_ftrace_function_register(struct perf_event *event) static int perf_ftrace_function_unregister(struct perf_event *event) { struct ftrace_ops *ops = &event->ftrace_ops; - int ret = unregister_ftrace_function(ops); + int ret = 0; + + /* + * Perf will call this unconditionally even if the ops is not + * enabled. The unregister_ftrace_function() will warn if called + * when not enabled. Just bypass the unregistering if ops isn't + * enabled here. + */ + if (ops->flags & FTRACE_OPS_FL_ENABLED) + ret = unregister_ftrace_function(ops); + ftrace_free_filter(ops); return ret; } diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index b86f128ff99fb..45ae4ceb28c87 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -930,7 +930,7 @@ static int remove_cache_mod(struct trace_array *tr, const char *mod, if (strcmp(event_mod->module, mod) != 0) continue; - if (match && strcmp(event_mod->match, match) != 0) + if (match && (!event_mod->match || strcmp(event_mod->match, match) != 0)) continue; if (system && @@ -1333,7 +1333,9 @@ __ftrace_set_clr_event_nolock(struct trace_array *tr, const char *match, call = file->event_call; /* If a module is specified, skip events that are not that module */ - if (module && (!call->module || strcmp(module_name(call->module), module))) + if (module && + ((call->flags & TRACE_EVENT_FL_DYNAMIC) || + !call->module || strcmp(module_name(call->module), module))) continue; name = trace_event_name(call); @@ -1440,6 +1442,8 @@ int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set) /* Put back the colon to allow this to be called again */ if (buf) *(buf - 1) = ':'; + if (mod) + *(mod - 5) = ':'; return ret; } @@ -3400,7 +3404,7 @@ static void update_event_fields(struct trace_event_call *call, } /* Update all events for replacing eval and sanitizing */ -void trace_event_update_all(struct trace_eval_map **map, int len) +void trace_event_update_all(struct trace_eval_map **map, int len, struct module *mod) { struct trace_event_call *call, *p; const char *last_system = NULL; @@ -3409,8 +3413,13 @@ void trace_event_update_all(struct trace_eval_map **map, int len) int last_i; int i; + mutex_lock(&event_mutex); down_write(&trace_event_sem); list_for_each_entry_safe(call, p, &ftrace_events, list) { + + if (mod && call->module != mod) + continue; + /* events are usually grouped together with systems */ if (!last_system || call->class->system != last_system) { first = true; @@ -3447,6 +3456,7 @@ void trace_event_update_all(struct trace_eval_map **map, int len) cond_resched(); } up_write(&trace_event_sem); + mutex_unlock(&event_mutex); } static bool event_in_systems(struct trace_event_call *call, @@ -3769,8 +3779,8 @@ static void trace_module_add_events(struct module *mod) end = mod->trace_events + mod->num_trace_events; for_each_event(call, start, end) { - __register_event(*call, mod); - __add_event_to_tracers(*call); + if (!__register_event(*call, mod)) + __add_event_to_tracers(*call); } update_cache_events(mod); @@ -4725,6 +4735,8 @@ static __init void event_test_stuff(void) struct task_struct *test_thread; test_thread = kthread_run(event_test_thread, NULL, "test-events"); + if (WARN_ON(IS_ERR(test_thread))) + return; msleep(1); kthread_stop(test_thread); } diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index c6679caf2e9b6..9c63ea51c8bef 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -1027,6 +1027,9 @@ static int regex_match_full(char *str, struct regex *r, int len) if (!len) return strcmp(str, r->pattern) == 0; + if (len < r->len) + return 0; + return strncmp(str, r->pattern, len) == 0; } diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index cf2a4388845d6..9152f63cfb4cf 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -164,7 +164,6 @@ struct hist_field { struct hist_field *operands[HIST_FIELD_OPERANDS_MAX]; struct hist_trigger_data *hist_data; enum hist_field_fn fn_num; - unsigned int ref; unsigned int size; unsigned int offset; unsigned int is_signed; @@ -1905,16 +1904,8 @@ static int contains_operator(char *str, char **sep) return field_op; } -static void get_hist_field(struct hist_field *hist_field) -{ - hist_field->ref++; -} - static void __destroy_hist_field(struct hist_field *hist_field) { - if (--hist_field->ref > 1) - return; - kfree(hist_field->var.name); kfree(hist_field->name); @@ -1961,8 +1952,6 @@ static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data, if (!hist_field) return NULL; - hist_field->ref = 1; - hist_field->hist_data = hist_data; if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS) @@ -2217,10 +2206,8 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, for (i = 0; i < hist_data->n_var_refs; i++) { ref_field = hist_data->var_refs[i]; if (ref_field->var.idx == var_field->var.idx && - ref_field->var.hist_data == var_field->hist_data) { - get_hist_field(ref_field); + ref_field->var.hist_data == var_field->hist_data) return ref_field; - } } /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */ if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX) @@ -2228,7 +2215,7 @@ static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data, ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL); if (ref_field) { if (init_var_ref(ref_field, var_field, system, event_name)) { - destroy_hist_field(ref_field, 0); + __destroy_hist_field(ref_field); return NULL; } @@ -2324,6 +2311,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, struct ftrace_event_field *field = NULL; char *field_name, *modifier, *str; struct trace_array *tr = file->tr; + bool stack_modifier = false; modifier = str = kstrdup(field_str, GFP_KERNEL); if (!modifier) @@ -2346,9 +2334,10 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_EXECNAME; else if (strcmp(modifier, "syscall") == 0) *flags |= HIST_FIELD_FL_SYSCALL; - else if (strcmp(modifier, "stacktrace") == 0) + else if (strcmp(modifier, "stacktrace") == 0) { *flags |= HIST_FIELD_FL_STACKTRACE; - else if (strcmp(modifier, "log2") == 0) + stack_modifier = true; + } else if (strcmp(modifier, "log2") == 0) *flags |= HIST_FIELD_FL_LOG2; else if (strcmp(modifier, "usecs") == 0) *flags |= HIST_FIELD_FL_TIMESTAMP_USECS; @@ -2409,6 +2398,7 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, *flags |= HIST_FIELD_FL_CPU; } else if (field && field->filter_type == FILTER_STACKTRACE) { *flags |= HIST_FIELD_FL_STACKTRACE; + field = NULL; } else if (field && field->filter_type == FILTER_COMM) { *flags |= HIST_FIELD_FL_COMM | HIST_FIELD_FL_STRING; } else { @@ -2419,6 +2409,12 @@ parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file, } } } + + if (stack_modifier && + (!field || field->filter_type != FILTER_STACKTRACE)) { + hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + field = ERR_PTR(-EINVAL); + } out: kfree(str); @@ -3255,7 +3251,6 @@ static struct hist_field *create_var(struct hist_trigger_data *hist_data, goto out; } - var->ref = 1; var->flags = HIST_FIELD_FL_VAR; var->var.idx = idx; var->var.hist_data = var->hist_data = hist_data; @@ -4292,8 +4287,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, goto err; } else { /* Value */ - if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT | - HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | + if (hist_field->flags & (HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET | HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE)) goto err; @@ -4310,6 +4304,7 @@ static int __create_val_field(struct hist_trigger_data *hist_data, return ret; err: hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str)); + destroy_hist_field(hist_field, 0); return -EINVAL; } @@ -5669,7 +5664,7 @@ static int print_entries(struct seq_file *m, { struct tracing_map_sort_entry **sort_entries = NULL; struct tracing_map *map = hist_data->map; - int i, j, n_entries; + int i, j, n_entries, ret; struct hist_val_stat *stats = NULL; u64 val; @@ -5679,6 +5674,8 @@ static int print_entries(struct seq_file *m, if (n_entries < 0) return n_entries; + ret = n_entries; + /* Calculate the max and the total for each field if needed. */ for (j = 0; j < hist_data->n_vals; j++) { if (!(hist_data->fields[j]->flags & @@ -5688,7 +5685,7 @@ static int print_entries(struct seq_file *m, stats = kcalloc(hist_data->n_vals, sizeof(*stats), GFP_KERNEL); if (!stats) { - n_entries = -ENOMEM; + ret = -ENOMEM; goto out; } } @@ -5709,7 +5706,7 @@ static int print_entries(struct seq_file *m, out: tracing_map_destroy_sort_entries(sort_entries, n_entries); - return n_entries; + return ret; } static void hist_trigger_show(struct seq_file *m, @@ -5722,7 +5719,7 @@ static void hist_trigger_show(struct seq_file *m, seq_puts(m, "\n\n"); seq_puts(m, "# event histogram\n#\n# trigger info: "); - data->ops->print(m, data); + data->cmd_ops->print(m, data); seq_puts(m, "#\n\n"); hist_data = data->private_data; @@ -6044,7 +6041,7 @@ static void hist_trigger_debug_show(struct seq_file *m, seq_puts(m, "\n\n"); seq_puts(m, "# event histogram\n#\n# trigger info: "); - data->ops->print(m, data); + data->cmd_ops->print(m, data); seq_puts(m, "#\n\n"); hist_data = data->private_data; @@ -6354,20 +6351,22 @@ static void event_hist_trigger_free(struct event_trigger_data *data) free_hist_pad(); } -static const struct event_trigger_ops event_hist_trigger_ops = { - .trigger = event_hist_trigger, - .print = event_hist_trigger_print, - .init = event_hist_trigger_init, - .free = event_hist_trigger_free, -}; - static int event_hist_trigger_named_init(struct event_trigger_data *data) { + int ret; + + ret = event_hist_trigger_init(data->named_data); + if (ret < 0) { + kfree(data->cmd_ops); + data->cmd_ops = &trigger_hist_cmd; + return ret; + } + data->ref++; save_named_trigger(data->named_data->name, data); - return event_hist_trigger_init(data->named_data); + return 0; } static void event_hist_trigger_named_free(struct event_trigger_data *data) @@ -6379,24 +6378,14 @@ static void event_hist_trigger_named_free(struct event_trigger_data *data) data->ref--; if (!data->ref) { + struct event_command *cmd_ops = data->cmd_ops; + del_named_trigger(data); trigger_data_free(data); + kfree(cmd_ops); } } -static const struct event_trigger_ops event_hist_trigger_named_ops = { - .trigger = event_hist_trigger, - .print = event_hist_trigger_print, - .init = event_hist_trigger_named_init, - .free = event_hist_trigger_named_free, -}; - -static const struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd, - char *param) -{ - return &event_hist_trigger_ops; -} - static void hist_clear(struct event_trigger_data *data) { struct hist_trigger_data *hist_data = data->private_data; @@ -6590,15 +6579,20 @@ static int hist_register_trigger(char *glob, data->paused = true; if (named_data) { + struct event_command *cmd_ops; + data->private_data = named_data->private_data; set_named_trigger_data(data, named_data); - data->ops = &event_hist_trigger_named_ops; - } - - if (data->ops->init) { - ret = data->ops->init(data); - if (ret < 0) + /* Copy the command ops and update some of the functions */ + cmd_ops = kmalloc(sizeof(*cmd_ops), GFP_KERNEL); + if (!cmd_ops) { + ret = -ENOMEM; goto out; + } + *cmd_ops = *data->cmd_ops; + cmd_ops->init = event_hist_trigger_named_init; + cmd_ops->free = event_hist_trigger_named_free; + data->cmd_ops = cmd_ops; } if (hist_data->enable_timestamps) { @@ -6613,8 +6607,19 @@ static int hist_register_trigger(char *glob, tracing_set_filter_buffering(file->tr, true); } - if (named_data) + if (data->cmd_ops->init) { + ret = data->cmd_ops->init(data); + if (ret < 0) { + if (hist_data->enable_timestamps) + tracing_set_filter_buffering(file->tr, false); + goto out; + } + } + + if (named_data) { + remove_hist_vars(hist_data); destroy_hist_data(hist_data); + } out: return ret; } @@ -6628,11 +6633,12 @@ static int hist_trigger_enable(struct event_trigger_data *data, update_cond_flag(file); - if (trace_event_trigger_enable_disable(file, 1) < 0) { - list_del_rcu(&data->list); - update_cond_flag(file); + /* + * On failure the caller undoes the registration, and + * hist_unregister_trigger() can only find the trigger here. + */ + if (trace_event_trigger_enable_disable(file, 1) < 0) ret--; - } return ret; } @@ -6710,13 +6716,13 @@ static void hist_unregister_trigger(char *glob, } } - if (test && test->ops->free) - test->ops->free(test); - if (hist_data->enable_timestamps) { if (!hist_data->remove || test) tracing_set_filter_buffering(file->tr, false); } + + if (test && test->cmd_ops->free) + test->cmd_ops->free(test); } static bool hist_file_check_refs(struct trace_event_file *file) @@ -6763,8 +6769,8 @@ static void hist_unreg_all(struct trace_event_file *file) update_cond_flag(file); if (hist_data->enable_timestamps) tracing_set_filter_buffering(file->tr, false); - if (test->ops->free) - test->ops->free(test); + if (test->cmd_ops->free) + test->cmd_ops->free(test); } } } @@ -6921,6 +6927,8 @@ static int event_hist_trigger_parse(struct event_command *cmd_ops, return ret; out_unreg: event_trigger_unregister(cmd_ops, file, glob+1, trigger_data); + /* The unregister frees trigger_data, skip out_free */ + goto out; out_free: event_trigger_reset_filter(cmd_ops, trigger_data); @@ -6940,8 +6948,11 @@ static struct event_command trigger_hist_cmd = { .reg = hist_register_trigger, .unreg = hist_unregister_trigger, .unreg_all = hist_unreg_all, - .get_trigger_ops = event_hist_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = event_hist_trigger, + .print = event_hist_trigger_print, + .init = event_hist_trigger_init, + .free = event_hist_trigger_free, }; __init int register_trigger_hist_cmd(void) @@ -6973,66 +6984,6 @@ hist_enable_trigger(struct event_trigger_data *data, } } -static void -hist_enable_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) -{ - if (!data->count) - return; - - if (data->count != -1) - (data->count)--; - - hist_enable_trigger(data, buffer, rec, event); -} - -static const struct event_trigger_ops hist_enable_trigger_ops = { - .trigger = hist_enable_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops hist_enable_count_trigger_ops = { - .trigger = hist_enable_count_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops hist_disable_trigger_ops = { - .trigger = hist_enable_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops hist_disable_count_trigger_ops = { - .trigger = hist_enable_count_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops * -hist_enable_get_trigger_ops(char *cmd, char *param) -{ - const struct event_trigger_ops *ops; - bool enable; - - enable = (strcmp(cmd, ENABLE_HIST_STR) == 0); - - if (enable) - ops = param ? &hist_enable_count_trigger_ops : - &hist_enable_trigger_ops; - else - ops = param ? &hist_disable_count_trigger_ops : - &hist_disable_trigger_ops; - - return ops; -} - static void hist_enable_unreg_all(struct trace_event_file *file) { struct event_trigger_data *test, *n; @@ -7042,8 +6993,8 @@ static void hist_enable_unreg_all(struct trace_event_file *file) list_del_rcu(&test->list); update_cond_flag(file); trace_event_trigger_enable_disable(file, 0); - if (test->ops->free) - test->ops->free(test); + if (test->cmd_ops->free) + test->cmd_ops->free(test); } } } @@ -7055,8 +7006,12 @@ static struct event_command trigger_hist_enable_cmd = { .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, .unreg_all = hist_enable_unreg_all, - .get_trigger_ops = hist_enable_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = hist_enable_trigger, + .count_func = event_trigger_count, + .print = event_enable_trigger_print, + .init = event_trigger_init, + .free = event_enable_trigger_free, }; static struct event_command trigger_hist_disable_cmd = { @@ -7066,8 +7021,12 @@ static struct event_command trigger_hist_disable_cmd = { .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, .unreg_all = hist_enable_unreg_all, - .get_trigger_ops = hist_enable_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = hist_enable_trigger, + .count_func = event_trigger_count, + .print = event_enable_trigger_print, + .init = event_trigger_init, + .free = event_enable_trigger_free, }; static __init void unregister_trigger_hist_enable_disable_cmds(void) diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 98b8d5df15c79..5ec36a61e60cf 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -31,6 +31,20 @@ void trigger_data_free(struct event_trigger_data *data) kfree(data); } +static inline void data_ops_trigger(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event) +{ + const struct event_command *cmd_ops = data->cmd_ops; + + if (data->flags & EVENT_TRIGGER_FL_COUNT) { + if (!cmd_ops->count_func(data, buffer, rec, event)) + return; + } + + cmd_ops->trigger(data, buffer, rec, event); +} + /** * event_triggers_call - Call triggers associated with a trace event * @file: The trace_event_file associated with the event @@ -73,7 +87,7 @@ event_triggers_call(struct trace_event_file *file, if (data->paused) continue; if (!rec) { - data->ops->trigger(data, buffer, rec, event); + data_ops_trigger(data, buffer, rec, event); continue; } filter = rcu_dereference_sched(data->filter); @@ -83,7 +97,7 @@ event_triggers_call(struct trace_event_file *file, tt |= data->cmd_ops->trigger_type; continue; } - data->ops->trigger(data, buffer, rec, event); + data_ops_trigger(data, buffer, rec, event); } return tt; } @@ -125,7 +139,7 @@ event_triggers_post_call(struct trace_event_file *file, if (data->paused) continue; if (data->cmd_ops->trigger_type & tt) - data->ops->trigger(data, NULL, NULL, NULL); + data_ops_trigger(data, NULL, NULL, NULL); } } EXPORT_SYMBOL_GPL(event_triggers_post_call); @@ -194,7 +208,7 @@ static int trigger_show(struct seq_file *m, void *v) } data = list_entry(v, struct event_trigger_data, list); - data->ops->print(m, data); + data->cmd_ops->print(m, data); return 0; } @@ -381,7 +395,37 @@ __init int unregister_event_command(struct event_command *cmd) } /** - * event_trigger_print - Generic event_trigger_ops @print implementation + * event_trigger_count - Optional count function for event triggers + * @data: Trigger-specific data + * @buffer: The ring buffer that the event is being written to + * @rec: The trace entry for the event, NULL for unconditional invocation + * @event: The event meta data in the ring buffer + * + * For triggers that can take a count parameter that doesn't do anything + * special, they can use this function to assign to their .count_func + * field. + * + * This simply does a count down of the @data->count field. + * + * If the @data->count is greater than zero, it will decrement it. + * + * Returns false if @data->count is zero, otherwise true. + */ +bool event_trigger_count(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event) +{ + if (!data->count) + return false; + + if (data->count != -1) + (data->count)--; + + return true; +} + +/** + * event_trigger_print - Generic event_command @print implementation * @name: The name of the event trigger * @m: The seq_file being printed to * @data: Trigger-specific data @@ -416,7 +460,7 @@ event_trigger_print(const char *name, struct seq_file *m, } /** - * event_trigger_init - Generic event_trigger_ops @init implementation + * event_trigger_init - Generic event_command @init implementation * @data: Trigger-specific data * * Common implementation of event trigger initialization. @@ -433,7 +477,7 @@ int event_trigger_init(struct event_trigger_data *data) } /** - * event_trigger_free - Generic event_trigger_ops @free implementation + * event_trigger_free - Generic event_command @free implementation * @data: Trigger-specific data * * Common implementation of event trigger de-initialization. @@ -495,8 +539,8 @@ clear_event_triggers(struct trace_array *tr) list_for_each_entry_safe(data, n, &file->triggers, list) { trace_event_trigger_enable_disable(file, 0); list_del_rcu(&data->list); - if (data->ops->free) - data->ops->free(data); + if (data->cmd_ops->free) + data->cmd_ops->free(data); } } } @@ -559,8 +603,8 @@ static int register_trigger(char *glob, return -EEXIST; } - if (data->ops->init) { - ret = data->ops->init(data); + if (data->cmd_ops->init) { + ret = data->cmd_ops->init(data); if (ret < 0) return ret; } @@ -598,8 +642,8 @@ static bool try_unregister_trigger(char *glob, } if (data) { - if (data->ops->free) - data->ops->free(data); + if (data->cmd_ops->free) + data->cmd_ops->free(data); return true; } @@ -810,9 +854,13 @@ int event_trigger_separate_filter(char *param_and_filter, char **param, * @private_data: User data to associate with the event trigger * * Allocate an event_trigger_data instance and initialize it. The - * @cmd_ops are used along with the @cmd and @param to get the - * trigger_ops to assign to the event_trigger_data. @private_data can - * also be passed in and associated with the event_trigger_data. + * @cmd_ops defines how the trigger will operate. If @param is set, + * and @cmd_ops->trigger_ops->count_func is non NULL, then the + * data->count is set to @param and before the trigger is executed, the + * @cmd_ops->trigger_ops->count_func() is called. If that function returns + * false, the @cmd_ops->trigger_ops->trigger() function will not be called. + * @private_data can also be passed in and associated with the + * event_trigger_data. * * Use trigger_data_free() to free an event_trigger_data object. * @@ -824,18 +872,16 @@ struct event_trigger_data *trigger_data_alloc(struct event_command *cmd_ops, void *private_data) { struct event_trigger_data *trigger_data; - const struct event_trigger_ops *trigger_ops; - - trigger_ops = cmd_ops->get_trigger_ops(cmd, param); trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL); if (!trigger_data) return NULL; trigger_data->count = -1; - trigger_data->ops = trigger_ops; trigger_data->cmd_ops = cmd_ops; trigger_data->private_data = private_data; + if (param && cmd_ops->count_func) + trigger_data->flags |= EVENT_TRIGGER_FL_COUNT; INIT_LIST_HEAD(&trigger_data->list); INIT_LIST_HEAD(&trigger_data->named_list); @@ -1274,31 +1320,28 @@ traceon_trigger(struct event_trigger_data *data, tracing_on(); } -static void -traceon_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) +static bool +traceon_count_func(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event) { struct trace_event_file *file = data->private_data; if (file) { if (tracer_tracing_is_on(file->tr)) - return; + return false; } else { if (tracing_is_on()) - return; + return false; } if (!data->count) - return; + return false; if (data->count != -1) (data->count)--; - if (file) - tracer_tracing_on(file->tr); - else - tracing_on(); + return true; } static void @@ -1322,31 +1365,28 @@ traceoff_trigger(struct event_trigger_data *data, tracing_off(); } -static void -traceoff_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) +static bool +traceoff_count_func(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event) { struct trace_event_file *file = data->private_data; if (file) { if (!tracer_tracing_is_on(file->tr)) - return; + return false; } else { if (!tracing_is_on()) - return; + return false; } if (!data->count) - return; + return false; if (data->count != -1) (data->count)--; - if (file) - tracer_tracing_off(file->tr); - else - tracing_off(); + return true; } static int @@ -1363,58 +1403,18 @@ traceoff_trigger_print(struct seq_file *m, struct event_trigger_data *data) data->filter_str); } -static const struct event_trigger_ops traceon_trigger_ops = { - .trigger = traceon_trigger, - .print = traceon_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops traceon_count_trigger_ops = { - .trigger = traceon_count_trigger, - .print = traceon_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops traceoff_trigger_ops = { - .trigger = traceoff_trigger, - .print = traceoff_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops traceoff_count_trigger_ops = { - .trigger = traceoff_count_trigger, - .print = traceoff_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops * -onoff_get_trigger_ops(char *cmd, char *param) -{ - const struct event_trigger_ops *ops; - - /* we register both traceon and traceoff to this callback */ - if (strcmp(cmd, "traceon") == 0) - ops = param ? &traceon_count_trigger_ops : - &traceon_trigger_ops; - else - ops = param ? &traceoff_count_trigger_ops : - &traceoff_trigger_ops; - - return ops; -} - static struct event_command trigger_traceon_cmd = { .name = "traceon", .trigger_type = ETT_TRACE_ONOFF, .parse = event_trigger_parse, .reg = register_trigger, .unreg = unregister_trigger, - .get_trigger_ops = onoff_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = traceon_trigger, + .count_func = traceon_count_func, + .print = traceon_trigger_print, + .init = event_trigger_init, + .free = event_trigger_free, }; static struct event_command trigger_traceoff_cmd = { @@ -1424,8 +1424,12 @@ static struct event_command trigger_traceoff_cmd = { .parse = event_trigger_parse, .reg = register_trigger, .unreg = unregister_trigger, - .get_trigger_ops = onoff_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = traceoff_trigger, + .count_func = traceoff_count_func, + .print = traceoff_trigger_print, + .init = event_trigger_init, + .free = event_trigger_free, }; #ifdef CONFIG_TRACER_SNAPSHOT @@ -1442,20 +1446,6 @@ snapshot_trigger(struct event_trigger_data *data, tracing_snapshot(); } -static void -snapshot_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) -{ - if (!data->count) - return; - - if (data->count != -1) - (data->count)--; - - snapshot_trigger(data, buffer, rec, event); -} - static int register_snapshot_trigger(char *glob, struct event_trigger_data *data, @@ -1487,34 +1477,18 @@ snapshot_trigger_print(struct seq_file *m, struct event_trigger_data *data) data->filter_str); } -static const struct event_trigger_ops snapshot_trigger_ops = { - .trigger = snapshot_trigger, - .print = snapshot_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops snapshot_count_trigger_ops = { - .trigger = snapshot_count_trigger, - .print = snapshot_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops * -snapshot_get_trigger_ops(char *cmd, char *param) -{ - return param ? &snapshot_count_trigger_ops : &snapshot_trigger_ops; -} - static struct event_command trigger_snapshot_cmd = { .name = "snapshot", .trigger_type = ETT_SNAPSHOT, .parse = event_trigger_parse, .reg = register_snapshot_trigger, .unreg = unregister_snapshot_trigger, - .get_trigger_ops = snapshot_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = snapshot_trigger, + .count_func = event_trigger_count, + .print = snapshot_trigger_print, + .init = event_trigger_init, + .free = event_trigger_free, }; static __init int register_trigger_snapshot_cmd(void) @@ -1561,20 +1535,6 @@ stacktrace_trigger(struct event_trigger_data *data, trace_dump_stack(STACK_SKIP); } -static void -stacktrace_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) -{ - if (!data->count) - return; - - if (data->count != -1) - (data->count)--; - - stacktrace_trigger(data, buffer, rec, event); -} - static int stacktrace_trigger_print(struct seq_file *m, struct event_trigger_data *data) { @@ -1582,26 +1542,6 @@ stacktrace_trigger_print(struct seq_file *m, struct event_trigger_data *data) data->filter_str); } -static const struct event_trigger_ops stacktrace_trigger_ops = { - .trigger = stacktrace_trigger, - .print = stacktrace_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops stacktrace_count_trigger_ops = { - .trigger = stacktrace_count_trigger, - .print = stacktrace_trigger_print, - .init = event_trigger_init, - .free = event_trigger_free, -}; - -static const struct event_trigger_ops * -stacktrace_get_trigger_ops(char *cmd, char *param) -{ - return param ? &stacktrace_count_trigger_ops : &stacktrace_trigger_ops; -} - static struct event_command trigger_stacktrace_cmd = { .name = "stacktrace", .trigger_type = ETT_STACKTRACE, @@ -1609,8 +1549,12 @@ static struct event_command trigger_stacktrace_cmd = { .parse = event_trigger_parse, .reg = register_trigger, .unreg = unregister_trigger, - .get_trigger_ops = stacktrace_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = stacktrace_trigger, + .count_func = event_trigger_count, + .print = stacktrace_trigger_print, + .init = event_trigger_init, + .free = event_trigger_free, }; static __init int register_trigger_stacktrace_cmd(void) @@ -1645,24 +1589,24 @@ event_enable_trigger(struct event_trigger_data *data, set_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &enable_data->file->flags); } -static void -event_enable_count_trigger(struct event_trigger_data *data, - struct trace_buffer *buffer, void *rec, - struct ring_buffer_event *event) +static bool +event_enable_count_func(struct event_trigger_data *data, + struct trace_buffer *buffer, void *rec, + struct ring_buffer_event *event) { struct enable_trigger_data *enable_data = data->private_data; if (!data->count) - return; + return false; /* Skip if the event is in a state we want to switch to */ if (enable_data->enable == !(enable_data->file->flags & EVENT_FILE_FL_SOFT_DISABLED)) - return; + return false; if (data->count != -1) (data->count)--; - event_enable_trigger(data, buffer, rec, event); + return true; } int event_enable_trigger_print(struct seq_file *m, @@ -1707,34 +1651,6 @@ void event_enable_trigger_free(struct event_trigger_data *data) } } -static const struct event_trigger_ops event_enable_trigger_ops = { - .trigger = event_enable_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops event_enable_count_trigger_ops = { - .trigger = event_enable_count_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops event_disable_trigger_ops = { - .trigger = event_enable_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - -static const struct event_trigger_ops event_disable_count_trigger_ops = { - .trigger = event_enable_count_trigger, - .print = event_enable_trigger_print, - .init = event_trigger_init, - .free = event_enable_trigger_free, -}; - int event_enable_trigger_parse(struct event_command *cmd_ops, struct trace_event_file *file, char *glob, char *cmd, char *param_and_filter) @@ -1864,8 +1780,8 @@ int event_enable_register_trigger(char *glob, } } - if (data->ops->init) { - ret = data->ops->init(data); + if (data->cmd_ops->init) { + ret = data->cmd_ops->init(data); if (ret < 0) return ret; } @@ -1905,30 +1821,8 @@ void event_enable_unregister_trigger(char *glob, } } - if (data && data->ops->free) - data->ops->free(data); -} - -static const struct event_trigger_ops * -event_enable_get_trigger_ops(char *cmd, char *param) -{ - const struct event_trigger_ops *ops; - bool enable; - -#ifdef CONFIG_HIST_TRIGGERS - enable = ((strcmp(cmd, ENABLE_EVENT_STR) == 0) || - (strcmp(cmd, ENABLE_HIST_STR) == 0)); -#else - enable = strcmp(cmd, ENABLE_EVENT_STR) == 0; -#endif - if (enable) - ops = param ? &event_enable_count_trigger_ops : - &event_enable_trigger_ops; - else - ops = param ? &event_disable_count_trigger_ops : - &event_disable_trigger_ops; - - return ops; + if (data && data->cmd_ops->free) + data->cmd_ops->free(data); } static struct event_command trigger_enable_cmd = { @@ -1937,8 +1831,12 @@ static struct event_command trigger_enable_cmd = { .parse = event_enable_trigger_parse, .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, - .get_trigger_ops = event_enable_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = event_enable_trigger, + .count_func = event_enable_count_func, + .print = event_enable_trigger_print, + .init = event_trigger_init, + .free = event_enable_trigger_free, }; static struct event_command trigger_disable_cmd = { @@ -1947,8 +1845,12 @@ static struct event_command trigger_disable_cmd = { .parse = event_enable_trigger_parse, .reg = event_enable_register_trigger, .unreg = event_enable_unregister_trigger, - .get_trigger_ops = event_enable_get_trigger_ops, .set_filter = set_trigger_filter, + .trigger = event_enable_trigger, + .count_func = event_enable_count_func, + .print = event_enable_trigger_print, + .init = event_trigger_init, + .free = event_enable_trigger_free, }; static __init void unregister_trigger_enable_disable_cmds(void) diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index 8e839370f713e..5614a17510be5 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -868,6 +868,9 @@ void user_event_mm_dup(struct task_struct *t, struct user_event_mm *old_mm) struct user_event_mm *mm = user_event_mm_alloc(t); struct user_event_enabler *enabler; + /* On failure, do not free parent's copy */ + t->user_event_mm = NULL; + if (!mm) return; @@ -1119,10 +1122,9 @@ static void user_event_destroy_validators(struct user_event *user) } } -static void user_event_destroy_fields(struct user_event *user) +static void user_event_destroy_fields(struct list_head *head) { struct ftrace_event_field *field, *next; - struct list_head *head = &user->fields; list_for_each_entry_safe(field, next, head, link) { list_del(&field->link); @@ -1509,17 +1511,32 @@ static int user_event_set_call_visible(struct user_event *user, bool visible) static int destroy_user_event(struct user_event *user) { + LIST_HEAD(fields); int ret = 0; lockdep_assert_held(&event_mutex); - /* Must destroy fields before call removal */ - user_event_destroy_fields(user); + /* + * Detach the fields before removing the call. Removing the event + * frees the field list memory (trace_destroy_fields() is run on + * successful removal and kmem_cache_free()s the fields), but the + * fields here are allocated and owned by user_events. Destroy + * them separately once removal has succeeded. + */ + list_splice_init(&user->fields, &fields); ret = user_event_set_call_visible(user, false); - if (ret) + if (ret) { + /* + * Removal failed and the event stays registered, recover + * the fields so it is left in a consistent state. + */ + list_splice(&fields, &user->fields); return ret; + } + + user_event_destroy_fields(&fields); dyn_event_remove(&user->devent); hash_del(&user->node); @@ -2219,7 +2236,7 @@ static int user_event_parse(struct user_event_group *group, char *name, put_user_lock: mutex_unlock(&event_mutex); put_user: - user_event_destroy_fields(user); + user_event_destroy_fields(&user->fields); user_event_destroy_validators(user); kfree(user->call.print_fmt); diff --git a/kernel/trace/trace_fprobe.c b/kernel/trace/trace_fprobe.c index 8001dbf168919..f7eb875811bea 100644 --- a/kernel/trace/trace_fprobe.c +++ b/kernel/trace/trace_fprobe.c @@ -1481,11 +1481,21 @@ static int enable_trace_fprobe(struct trace_event_call *call, list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) { ret = __register_trace_fprobe(tf); if (ret < 0) - return ret; + goto err; } } return 0; + +err: + /* Failed to enable one of them. Roll back all */ + list_for_each_entry(tf, trace_probe_probe_list(tp), tp.list) + __unregister_trace_fprobe(tf); + if (file) + trace_probe_remove_file(tp, file); + else + trace_probe_clear_flag(tp, TP_FLAG_PROFILE); + return ret; } /* diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index d17c18934445f..a56994cfc64bb 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -101,7 +101,7 @@ int ftrace_create_function_files(struct trace_array *tr, return ret; } - ftrace_create_filter_files(tr->ops, parent); + ftrace_create_filter_files(tr, parent); return 0; } diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index 0e65d1f452657..c0513f03b6f9a 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -44,6 +44,7 @@ struct fgraph_data { }; struct ftrace_graph_ret_entry ret; int failed; + int ent_size; int cpu; }; @@ -1239,6 +1240,7 @@ print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s, if (s->full) { data->failed = 1; data->cpu = cpu; + data->ent_size = iter->ent_size; } else data->failed = 0; } @@ -1422,6 +1424,7 @@ print_graph_function_flags(struct trace_iterator *iter, u32 flags) if (data && data->failed) { field = &data->ent.ent; iter->cpu = data->cpu; + iter->ent_size = data->ent_size; ret = print_graph_entry(field, s, iter, flags); if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) { per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1; diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c index c706544be60cc..32b7e1d972a47 100644 --- a/kernel/trace/trace_mmiotrace.c +++ b/kernel/trace/trace_mmiotrace.c @@ -29,6 +29,7 @@ static void mmio_reset_data(struct trace_array *tr) { overrun_detected = false; prev_overruns = 0; + atomic_set(&dropped_count, 0); tracing_reset_online_cpus(&tr->array_buffer); } @@ -109,7 +110,6 @@ static void mmio_pipe_open(struct trace_iterator *iter) iter->private = hiter; } -/* XXX: This is not called when the pipe is closed! */ static void mmio_close(struct trace_iterator *iter) { struct header_iter *hiter = iter->private; @@ -146,7 +146,7 @@ static ssize_t mmio_read(struct trace_iterator *iter, struct file *filp, goto print_out; } - if (!hiter) + if (!hiter || !hiter->dev) return 0; mmio_print_pcidev(s, hiter->dev); @@ -279,6 +279,7 @@ static struct tracer mmio_tracer __read_mostly = .start = mmio_trace_start, .pipe_open = mmio_pipe_open, .close = mmio_close, + .pipe_close = mmio_close, .read = mmio_read, .print_line = mmio_print_line, .noboot = true, @@ -293,11 +294,15 @@ device_initcall(init_mmio_trace); static void __trace_mmiotrace_rw(struct trace_array *tr, struct mmiotrace_rw *rw) { - struct trace_buffer *buffer = tr->array_buffer.buffer; + struct trace_buffer *buffer; struct ring_buffer_event *event; struct trace_mmiotrace_rw *entry; unsigned int trace_ctx; + if (!tr) + return; + + buffer = tr->array_buffer.buffer; trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW, sizeof(*entry), trace_ctx); @@ -320,11 +325,15 @@ void mmio_trace_rw(struct mmiotrace_rw *rw) static void __trace_mmiotrace_map(struct trace_array *tr, struct mmiotrace_map *map) { - struct trace_buffer *buffer = tr->array_buffer.buffer; + struct trace_buffer *buffer; struct ring_buffer_event *event; struct trace_mmiotrace_map *entry; unsigned int trace_ctx; + if (!tr) + return; + + buffer = tr->array_buffer.buffer; trace_ctx = tracing_gen_ctx_flags(0); event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP, sizeof(*entry), trace_ctx); diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index 2333e5446c9fa..a0fe68cff4d33 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -187,7 +187,7 @@ void __trace_probe_log_err(int offset, int err_type) lockdep_assert_held(&dyn_event_ops_mutex); - if (!trace_probe_log.argv) + if (!trace_probe_log.argv || !trace_probe_log.argc) return; /* Recalculate the length and allocate buffer */ @@ -355,9 +355,8 @@ static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type) { const struct btf_type *real_type; u32 intdata; - s32 tid; - real_type = btf_type_skip_modifiers(btf, type->type, &tid); + real_type = btf_type_skip_modifiers(btf, type->type, NULL); if (!real_type) return false; @@ -374,14 +373,13 @@ static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type) const struct btf_type *real_type; const struct btf_array *array; u32 intdata; - s32 tid; if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY) return false; array = (const struct btf_array *)(type + 1); - real_type = btf_type_skip_modifiers(btf, array->type, &tid); + real_type = btf_type_skip_modifiers(btf, array->type, NULL); intdata = btf_type_int(real_type); return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) @@ -579,12 +577,12 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, { struct fetch_insn *code = *pcode; const struct btf_member *field; + const struct btf_type *mtype; u32 bitoffs, anon_offs; bool is_struct = ctx->struct_btf != NULL; struct btf *btf = ctx_btf(ctx); char *next; int is_ptr; - s32 tid; do { if (!is_struct) { @@ -595,7 +593,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, } /* Convert a struct pointer type to a struct type */ - type = btf_type_skip_modifiers(btf, type->type, &tid); + type = btf_type_skip_modifiers(btf, type->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -614,7 +612,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, anon_offs = 0; field = btf_find_struct_member(btf, type, fieldname, - &anon_offs); + &anon_offs, &mtype); if (IS_ERR(field)) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return PTR_ERR(field); @@ -627,7 +625,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, bitoffs += anon_offs; /* Accumulate the bit-offsets of the dot-connected fields */ - if (btf_type_kflag(type)) { + if (btf_type_kflag(mtype)) { bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset); ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset); } else { @@ -635,7 +633,7 @@ static int parse_btf_field(char *fieldname, const struct btf_type *type, ctx->last_bitsize = 0; } - type = btf_type_skip_modifiers(btf, field->type, &tid); + type = btf_type_skip_modifiers(btf, field->type, NULL); if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); return -EINVAL; @@ -754,7 +752,7 @@ static int parse_btf_arg(char *varname, return -ENOENT; found: - type = btf_type_skip_modifiers(ctx->btf, tid, &tid); + type = btf_type_skip_modifiers(ctx->btf, tid, NULL); found_type: if (!type) { trace_probe_log_err(ctx->offset, BAD_BTF_TID); @@ -1906,7 +1904,11 @@ const char **traceprobe_expand_meta_args(int argc, const char *argv[], trace_probe_log_err(0, BAD_VAR); return ERR_PTR(-ENOENT); } - /* Note: $argN starts from $arg1 */ + /* Note: $argN starts from $arg1, so $arg0 is invalid. */ + if (n == 0) { + trace_probe_log_err(0, BAD_ARG_NUM); + return ERR_PTR(-EINVAL); + } ret = sprint_nth_btf_arg(n - 1, type, buf + used, bufsize - used, ctx); if (ret < 0) @@ -2018,7 +2020,7 @@ int traceprobe_update_arg(struct probe_arg *arg) } /* When len=0, we just calculate the needed length */ -#define LEN_OR_ZERO (len ? len - pos : 0) +#define LEN_OR_ZERO (len > pos ? len - pos : 0) static int __set_print_fmt(struct trace_probe *tp, char *buf, int len, enum probe_print_type ptype) { @@ -2111,19 +2113,60 @@ int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype int traceprobe_define_arg_fields(struct trace_event_call *event_call, size_t offset, struct trace_probe *tp) { + struct trace_probe_event *tpe = trace_probe_event_from_call(event_call); int ret, i; + /* + * A field created by trace_define_field() only stores the name and + * type pointers, it does not copy the strings. Here they point into + * the probe_arg of @tp, which is freed when @tp is removed. For an + * event with multiple probes attached, the field list is defined + * once by the first probe but kept alive by the surviving siblings, + * so removing that first probe would leave the fields referencing + * freed memory. Duplicate the strings and anchor the copies on the + * trace_probe_event, which lives as long as the field list itself. + * + * event_define_fields() ignores the return value of this hook, so + * if a previous attempt failed before creating any field, it may + * call here again. Release duplicates left behind by such an + * attempt before starting over. + */ + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); + tpe->field_strings = NULL; + tpe->nr_field_strings = 0; + + if (tp->nr_args) { + tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *), + GFP_KERNEL); + if (!tpe->field_strings) + return -ENOMEM; + } + /* Set argument names as fields */ for (i = 0; i < tp->nr_args; i++) { struct probe_arg *parg = &tp->args[i]; const char *fmt = parg->type->fmttype; int size = parg->type->size; + char *name, *type; if (parg->fmt) fmt = parg->fmt; if (parg->count) size *= parg->count; - ret = trace_define_field(event_call, fmt, parg->name, + + name = kstrdup(parg->name, GFP_KERNEL); + type = kstrdup(fmt, GFP_KERNEL); + if (!name || !type) { + kfree(name); + kfree(type); + return -ENOMEM; + } + tpe->field_strings[tpe->nr_field_strings++] = name; + tpe->field_strings[tpe->nr_field_strings++] = type; + + ret = trace_define_field(event_call, type, name, offset + parg->offset, size, parg->type->is_signed, FILTER_OTHER); @@ -2135,6 +2178,11 @@ int traceprobe_define_arg_fields(struct trace_event_call *event_call, static void trace_probe_event_free(struct trace_probe_event *tpe) { + int i; + + for (i = 0; i < tpe->nr_field_strings; i++) + kfree(tpe->field_strings[i]); + kfree(tpe->field_strings); kfree(tpe->class.system); kfree(tpe->call.name); kfree(tpe->call.print_fmt); @@ -2344,16 +2392,17 @@ int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b) bool trace_probe_match_command_args(struct trace_probe *tp, int argc, const char **argv) { - char buf[MAX_ARGSTR_LEN + 1]; int i; if (tp->nr_args < argc) return false; for (i = 0; i < argc; i++) { - snprintf(buf, sizeof(buf), "%s=%s", - tp->args[i].name, tp->args[i].comm); - if (strcmp(buf, argv[i])) + int len = strlen(tp->args[i].name); + + if (strncmp(argv[i], tp->args[i].name, len) || + argv[i][len] != '=' || + strcmp(argv[i] + len + 1, tp->args[i].comm)) return false; } return true; diff --git a/kernel/trace/trace_probe.h b/kernel/trace/trace_probe.h index d5ea914e01b34..6ebf1f721f8f5 100644 --- a/kernel/trace/trace_probe.h +++ b/kernel/trace/trace_probe.h @@ -255,6 +255,8 @@ struct trace_probe_event { struct trace_event_call call; struct list_head files; struct list_head probes; + char **field_strings; + int nr_field_strings; struct trace_uprobe_filter filter[]; }; diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c index 0aa2514a6593b..e7f4e523587d0 100644 --- a/kernel/trace/trace_stack.c +++ b/kernel/trace/trace_stack.c @@ -499,7 +499,7 @@ stack_trace_filter_open(struct inode *inode, struct file *file) struct ftrace_ops *ops = inode->i_private; /* Checks for tracefs lockdown */ - return ftrace_regex_open(ops, FTRACE_ITER_FILTER, + return ftrace_regex_open(NULL, ops, FTRACE_ITER_FILTER, inode, file); } diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 20c79180fbfa9..7f469a3c9c175 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -7736,6 +7736,9 @@ static int wq_watchdog_param_set_thresh(const char *val, if (ret) return ret; + if (thresh > MAX_JIFFY_OFFSET / HZ) + return -ERANGE; + if (system_percpu_wq) wq_watchdog_set_thresh(thresh); else diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 21cd68084e468..b55f5c8e11f4c 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2157,10 +2157,11 @@ config KCOV_INSTRUMENT_ALL config KCOV_IRQ_AREA_SIZE hex "Size of interrupt coverage collection area in words" depends on KCOV + range 0x80 0x1000000 default 0x40000 help - KCOV uses preallocated per-cpu areas to collect coverage from - soft interrupts. This specifies the size of those areas in the + KCOV uses preallocated areas to collect coverage from soft + interrupts. This specifies the size of those areas in the number of unsigned long words. config KCOV_SELFTEST diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c index 3082d977efaae..7c6d3a086fdc4 100644 --- a/lib/alloc_tag.c +++ b/lib/alloc_tag.c @@ -26,6 +26,15 @@ static bool mem_profiling_support = true; static bool mem_profiling_support; #endif +/* + * Memory allocation profiling is permanently disabled and cannot be enabled. + * Must be called after setup_early_mem_profiling(). + */ +bool mem_alloc_profiling_permanently_disabled(void) +{ + return !mem_profiling_support; +} + static struct codetag_type *alloc_tag_cttype; #ifdef CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU diff --git a/lib/assoc_array.c b/lib/assoc_array.c index 388e656ac9743..01619e88a52c9 100644 --- a/lib/assoc_array.c +++ b/lib/assoc_array.c @@ -255,7 +255,8 @@ assoc_array_walk(const struct assoc_array *array, sc_segments = shortcut->index_key[sc_level >> ASSOC_ARRAY_KEY_CHUNK_SHIFT]; dissimilarity = segments ^ sc_segments; - if (round_up(sc_level, ASSOC_ARRAY_KEY_CHUNK_SIZE) > shortcut->skip_to_level) { + if (shortcut->skip_to_level < round_down(sc_level, + ASSOC_ARRAY_KEY_CHUNK_SIZE) + ASSOC_ARRAY_KEY_CHUNK_SIZE) { /* Trim segments that are beyond the shortcut */ int shift = shortcut->skip_to_level & ASSOC_ARRAY_KEY_CHUNK_MASK; dissimilarity &= ~(ULONG_MAX << shift); diff --git a/lib/bootconfig.c b/lib/bootconfig.c index 5d3802eba52a3..b4e8e17e29155 100644 --- a/lib/bootconfig.c +++ b/lib/bootconfig.c @@ -405,6 +405,71 @@ const char * __init xbc_node_find_next_key_value(struct xbc_node *root, return ""; /* No value key */ } +static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata; + +#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0) + +/** + * xbc_snprint_cmdline() - Render bootconfig keys under @root as a cmdline string + * @buf: Destination buffer (may be NULL when @size is 0 to query the length) + * @size: Size of @buf in bytes + * @root: Subtree root whose key=value pairs should be rendered + * + * Walk all key/value pairs under @root and emit them as a space-separated + * cmdline string into @buf. Values containing whitespace are quoted with + * double quotes. Returns the number of bytes that would be written if @buf + * were large enough (matching snprintf semantics), or a negative errno on + * failure. + */ +int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +{ + struct xbc_node *knode, *vnode; + const char *val, *q; + size_t len = 0; + int ret; + + /* + * Track the running written length rather than advancing @buf, so we + * never form "buf + size" or "buf += ret" while @buf is NULL (the + * size-probe call passes buf=NULL, size=0). NULL pointer arithmetic + * is undefined behavior and trips host UBSan / FORTIFY_SOURCE when + * this renderer runs at kernel build time. snprintf(NULL, 0, ...) + * itself is well defined and returns the would-be length. + */ + xbc_node_for_each_key_value(root, knode, val) { + ret = xbc_node_compose_key_after(root, knode, + xbc_namebuf, XBC_KEYLEN_MAX); + if (ret < 0) + return ret; + + vnode = xbc_node_get_child(knode); + if (!vnode) { + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s ", xbc_namebuf); + if (ret < 0) + return ret; + len += ret; + continue; + } + xbc_array_for_each_value(vnode, val) { + /* + * For prettier and more readable /proc/cmdline, only + * quote the value when necessary, i.e. when it contains + * whitespace. + */ + q = strpbrk(val, " \t\r\n") ? "\"" : ""; + ret = snprintf(buf ? buf + len : NULL, rest(len, size), + "%s=%s%s%s ", xbc_namebuf, q, val, q); + if (ret < 0) + return ret; + len += ret; + } + } + + return len; +} +#undef rest + /* XBC parse and tree build */ static int __init xbc_init_node(struct xbc_node *node, char *data, uint32_t flag) diff --git a/lib/compat_audit.c b/lib/compat_audit.c index 3d6b8996f027d..fee1dfccd116b 100644 --- a/lib/compat_audit.c +++ b/lib/compat_audit.c @@ -4,32 +4,32 @@ #include #include -unsigned compat_dir_class[] = { +unsigned int compat_dir_class[] = { #include ~0U }; -unsigned compat_read_class[] = { +unsigned int compat_read_class[] = { #include ~0U }; -unsigned compat_write_class[] = { +unsigned int compat_write_class[] = { #include ~0U }; -unsigned compat_chattr_class[] = { +unsigned int compat_chattr_class[] = { #include ~0U }; -unsigned compat_signal_class[] = { +unsigned int compat_signal_class[] = { #include ~0U }; -int audit_classify_compat_syscall(int abi, unsigned syscall) +int audit_classify_compat_syscall(int abi, unsigned int syscall) { switch (syscall) { #ifdef __NR_open diff --git a/lib/interval_tree_test.c b/lib/interval_tree_test.c index 5fd62656f42e3..c479d9176b759 100644 --- a/lib/interval_tree_test.c +++ b/lib/interval_tree_test.c @@ -139,13 +139,13 @@ static int intersection_range_check(void) intxn1 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn1) { - WARN_ON_ONCE("Failed to allocate intxn1\n"); + WARN_ONCE(1, "Failed to allocate intxn1\n"); return -ENOMEM; } intxn2 = bitmap_alloc(nnodes, GFP_KERNEL); if (!intxn2) { - WARN_ON_ONCE("Failed to allocate intxn2\n"); + WARN_ONCE(1, "Failed to allocate intxn2\n"); bitmap_free(intxn1); return -ENOMEM; } diff --git a/lib/maple_tree.c b/lib/maple_tree.c index 5aa4c95000188..cff42ec10f4bf 100644 --- a/lib/maple_tree.c +++ b/lib/maple_tree.c @@ -4026,35 +4026,40 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, void *entry, unsigned long range_lo, unsigned long range_hi, unsigned long *next, gfp_t gfp) { - unsigned long min = range_lo; - int ret = 0; - - range_lo = max(min, *next); - ret = mas_empty_area(mas, range_lo, range_hi, 1); - if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { - mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; - ret = 1; - } - if (ret < 0 && range_lo > min) { - mas_reset(mas); - ret = mas_empty_area(mas, min, range_hi, 1); - if (ret == 0) - ret = 1; - } - if (ret < 0) - return ret; + int ret; + unsigned long min; + min = range_lo; do { + range_lo = max(min, *next); + ret = mas_empty_area(mas, range_lo, range_hi, 1); + if (ret < 0 && range_lo > min) { + mas_reset(mas); + ret = mas_empty_area(mas, min, range_hi, 1); + if (ret == 0) + ret = 1; + } + if (ret < 0) + goto out; + mas_insert(mas, entry); } while (mas_nomem(mas, gfp)); - if (mas_is_err(mas)) - return xa_err(mas->node); + if (mas_is_err(mas)) { + ret = xa_err(mas->node); + goto out; + } + + if ((mas->tree->ma_flags & MT_FLAGS_ALLOC_WRAPPED) && ret == 0) { + mas->tree->ma_flags &= ~MT_FLAGS_ALLOC_WRAPPED; + ret = 1; + } *startp = mas->index; *next = *startp + 1; if (*next == 0) mas->tree->ma_flags |= MT_FLAGS_ALLOC_WRAPPED; +out: mas_destroy(mas); return ret; } diff --git a/lib/rhashtable.c b/lib/rhashtable.c index fde0f0e556f87..fcc7bd22d86ef 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -742,6 +742,7 @@ int rhashtable_walk_start_check(struct rhashtable_iter *iter) iter->walker.tbl = rht_dereference_rcu(ht->tbl, ht); iter->slot = 0; iter->skip = 0; + iter->p = NULL; return -EAGAIN; } diff --git a/lib/string.c b/lib/string.c index b632c71df1a50..01c9845024b85 100644 --- a/lib/string.c +++ b/lib/string.c @@ -839,7 +839,8 @@ void *memchr_inv(const void *start, int c, size_t bytes) { u8 value = c; u64 value64; - unsigned int words, prefix; + size_t words; + unsigned int prefix; if (bytes <= 16) return check_bytes8(start, value, bytes); diff --git a/lib/test_fortify/Makefile b/lib/test_fortify/Makefile index 399cae880e1dc..44cd5df41a810 100644 --- a/lib/test_fortify/Makefile +++ b/lib/test_fortify/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 ccflags-y := $(call cc-disable-warning,fortify-source) +ccflags-y += $(call cc-disable-warning,stringop-overread) quiet_cmd_test_fortify = TEST $@ cmd_test_fortify = $(CONFIG_SHELL) $(src)/test_fortify.sh \ diff --git a/lib/test_hmm.c b/lib/test_hmm.c index 00d34a6c6276b..ec1df4cd84807 100644 --- a/lib/test_hmm.c +++ b/lib/test_hmm.c @@ -392,7 +392,7 @@ static int dmirror_fault(struct dmirror *dmirror, unsigned long start, /* Since the mm is for the mirrored process, get a reference first. */ if (!mmget_not_zero(mm)) - return 0; + return -EFAULT; for (addr = start; addr < end; addr = range.end) { range.start = addr; diff --git a/lib/ucs2_string.c b/lib/ucs2_string.c index dfb4f2358cabf..ca5c615d5bef2 100644 --- a/lib/ucs2_string.c +++ b/lib/ucs2_string.c @@ -8,7 +8,7 @@ ucs2_strnlen(const ucs2_char_t *s, size_t maxlength) { unsigned long length = 0; - while (*s++ != 0 && length < maxlength) + while (length < maxlength && *s++ != 0) length++; return length; } diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c index a565c30c71a04..64d868dad8dd1 100644 --- a/lib/vdso/datastore.c +++ b/lib/vdso/datastore.c @@ -103,28 +103,3 @@ struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned VM_PFNMAP | VM_SEALED_SYSMAP, &vdso_vvar_mapping); } - -#ifdef CONFIG_TIME_NS -/* - * The vvar page layout depends on whether a task belongs to the root or - * non-root time namespace. Whenever a task changes its namespace, the VVAR - * page tables are cleared and then they will be re-faulted with a - * corresponding layout. - * See also the comment near timens_setup_vdso_clock_data() for details. - */ -int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) -{ - struct mm_struct *mm = task->mm; - struct vm_area_struct *vma; - VMA_ITERATOR(vmi, mm, 0); - - mmap_read_lock(mm); - for_each_vma(vmi, vma) { - if (vma_is_special_mapping(vma, &vdso_vvar_mapping)) - zap_vma_pages(vma); - } - mmap_read_unlock(mm); - - return 0; -} -#endif diff --git a/lib/xarray.c b/lib/xarray.c index 9a8b4916540cf..bfe7bef80f34e 100644 --- a/lib/xarray.c +++ b/lib/xarray.c @@ -1053,6 +1053,9 @@ void xas_split_alloc(struct xa_state *xas, void *entry, unsigned int order, if (xas->xa_shift + XA_CHUNK_SHIFT > order) return; + if (xas->xa->xa_flags & XA_FLAGS_ACCOUNT) + gfp |= __GFP_ACCOUNT; + do { struct xa_node *node; diff --git a/mm/compaction.c b/mm/compaction.c index 6769a9d6f16a8..7bb638df5db93 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -1364,12 +1364,44 @@ static bool suitable_migration_source(struct compact_control *cc, if (pageblock_skip_persistent(page)) return false; - if ((cc->mode != MIGRATE_ASYNC) || !cc->direct_compaction) + /* + * Background compaction produces blocks for the zone at + * large, with no particular allocation context. Allow all + * block types, including CMA. + */ + if (!cc->direct_compaction) return true; block_mt = get_pageblock_migratetype(page); - if (cc->migratetype == MIGRATE_MOVABLE) + /* + * CMA pages can only be taken by ALLOC_CMA requests. For anybody + * else, vacating a CMA block consumes free pages the caller + * could have used, and produces free pages it cannot. + */ + if (is_migrate_cma(block_mt) && !(cc->alloc_flags & ALLOC_CMA)) + return false; + + /* + * Per default, scans are restricted to blocks compatible with + * the request, to prevent cross-contamination. Once + * compaction priority escalates to synchronous scans, though, + * scan all blocks to try to make forward progress. For + * movable request, this likely helps little: there shouldn't + * be many migratable pages inside non-movable blocks besides + * allocator fallbacks. For non-movable requests, this helps a + * lot, as they can finally scan movable blocks. + */ + if (cc->mode != MIGRATE_ASYNC) + return true; + + /* + * Prevent migratetype == MIGRATE_MOVABLE || cc->order >= pageblock_order) return is_migrate_movable(block_mt); else return block_mt == cc->migratetype; @@ -1957,12 +1989,12 @@ static unsigned long fast_find_migrateblock(struct compact_control *cc) return pfn; /* - * Only allow kcompactd and direct requests for movable pages to - * quickly clear out a MOVABLE pageblock for allocation. This - * reduces the risk that a large movable pageblock is freed for - * an unmovable/reclaimable small allocation. + * Prevent direct_compaction && cc->migratetype != MIGRATE_MOVABLE) + if (cc->direct_compaction && cc->migratetype != MIGRATE_MOVABLE && + cc->order < pageblock_order) return pfn; /* @@ -2748,9 +2780,8 @@ compact_zone(struct compact_control *cc, struct capture_control *capc) static enum compact_result compact_zone_order(struct zone *zone, int order, gfp_t gfp_mask, enum compact_priority prio, unsigned int alloc_flags, int highest_zoneidx, - struct page **capture) + struct capture_control *capc) { - enum compact_result ret; struct compact_control cc = { .order = order, .search_order = order, @@ -2765,54 +2796,24 @@ static enum compact_result compact_zone_order(struct zone *zone, int order, .ignore_skip_hint = (prio == MIN_COMPACT_PRIORITY), .ignore_block_suitable = (prio == MIN_COMPACT_PRIORITY) }; - struct capture_control capc = { - .cc = &cc, - .page = NULL, - }; - /* - * Make sure the structs are really initialized before we expose the - * capture control, in case we are interrupted and the interrupt handler - * frees a page. - */ - barrier(); - WRITE_ONCE(current->capture_control, &capc); - - ret = compact_zone(&cc, &capc); - - /* - * Make sure we hide capture control first before we read the captured - * page pointer, otherwise an interrupt could free and capture a page - * and we would leak it. - */ - WRITE_ONCE(current->capture_control, NULL); - *capture = READ_ONCE(capc.page); - /* - * Technically, it is also possible that compaction is skipped but - * the page is still captured out of luck(IRQ came and freed the page). - * Returning COMPACT_SUCCESS in such cases helps in properly accounting - * the COMPACT[STALL|FAIL] when compaction is skipped. - */ - if (*capture) - ret = COMPACT_SUCCESS; - - return ret; + return compact_zone(&cc, capc); } /** * try_to_compact_pages - Direct compact to satisfy a high-order allocation * @gfp_mask: The GFP mask of the current allocation - * @order: The order of the current allocation + * @order: The order to try to make available * @alloc_flags: The allocation flags of the current allocation * @ac: The context of current allocation * @prio: Determines how hard direct compaction should try to succeed - * @capture: Pointer to free page created by compaction will be stored here + * @capc: Free page capture bypassing the freelist * * This is the main entry point for direct page compaction. */ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, - enum compact_priority prio, struct page **capture) + enum compact_priority prio, struct capture_control *capc) { struct zoneref *z; struct zone *zone; @@ -2839,8 +2840,17 @@ enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, continue; } + WRITE_ONCE(capc->zone, zone); + status = compact_zone_order(zone, order, gfp_mask, prio, - alloc_flags, ac->highest_zoneidx, capture); + alloc_flags, ac->highest_zoneidx, capc); + + WRITE_ONCE(capc->zone, NULL); + + /* Stop if a page has been captured */ + if (READ_ONCE(capc->page)) + status = COMPACT_SUCCESS; + rc = max(status, rc); /* The allocation should succeed, stop compacting */ diff --git a/mm/damon/core.c b/mm/damon/core.c index 3e221c888e302..6fa02025d2af1 100644 --- a/mm/damon/core.c +++ b/mm/damon/core.c @@ -213,8 +213,21 @@ int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, { struct damon_region *r, *next; unsigned int i; + unsigned long last_end; int err; + for (i = 0; i < nr_ranges; i++) { + unsigned long start, end; + + start = ALIGN_DOWN(ranges[i].start, min_sz_region); + end = ALIGN(ranges[i].end, min_sz_region); + if (start >= end) + return -EINVAL; + if (i > 0 && last_end > start) + return -EINVAL; + last_end = end; + } + /* Remove regions which are not in the new ranges */ damon_for_each_region_safe(r, next, t) { for (i = 0; i < nr_ranges; i++) { @@ -399,6 +412,7 @@ struct damos *damon_new_scheme(struct damos_access_pattern *pattern, INIT_LIST_HEAD(&scheme->filters); INIT_LIST_HEAD(&scheme->ops_filters); scheme->stat = (struct damos_stat){}; + scheme->last_applied = NULL; INIT_LIST_HEAD(&scheme->list); scheme->quota = *(damos_quota_init(quota)); @@ -2351,17 +2365,19 @@ static void damon_merge_two_regions(struct damon_target *t, * sz_limit size upper limit of each region */ static void damon_merge_regions_of(struct damon_target *t, unsigned int thres, - unsigned long sz_limit) + unsigned long sz_limit, bool count_age) { struct damon_region *r, *prev = NULL, *next; damon_for_each_region_safe(r, next, t) { - if (abs(r->nr_accesses - r->last_nr_accesses) > thres) - r->age = 0; - else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) - r->age = 0; - else - r->age++; + if (count_age) { + if (abs(r->nr_accesses - r->last_nr_accesses) > thres) + r->age = 0; + else if ((r->nr_accesses == 0) != (r->last_nr_accesses == 0)) + r->age = 0; + else + r->age++; + } if (prev && prev->ar.end == r->ar.start && abs(prev->nr_accesses - r->nr_accesses) <= thres && @@ -2395,18 +2411,26 @@ static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold, struct damon_target *t; unsigned int nr_regions; unsigned int max_thres; + bool count_age = true; max_thres = c->attrs.aggr_interval / (c->attrs.sample_interval ? c->attrs.sample_interval : 1); - do { + while (true) { nr_regions = 0; damon_for_each_target(t, c) { - damon_merge_regions_of(t, threshold, sz_limit); + damon_merge_regions_of(t, threshold, sz_limit, + count_age); nr_regions += damon_nr_regions(t); } - threshold = max(1, threshold * 2); - } while (nr_regions > c->attrs.max_nr_regions && - threshold / 2 < max_thres); + count_age = false; + if (nr_regions <= c->attrs.max_nr_regions || + max_thres <= threshold) + break; + if (threshold < max_thres / 2) + threshold = max(1, threshold * 2); + else + threshold = max_thres; + } } /* diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c index 512ea62e29abd..929aab600c745 100644 --- a/mm/damon/ops-common.c +++ b/mm/damon/ops-common.c @@ -101,8 +101,9 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r, unsigned int age_weight = s->quota.weight_age; int hotness; - freq_subscore = r->nr_accesses * DAMON_MAX_SUBSCORE / - damon_max_nr_accesses(&c->attrs); + freq_subscore = mult_frac(r->nr_accesses_bp / 10000, + DAMON_MAX_SUBSCORE, + damon_max_nr_accesses(&c->attrs)); age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000; for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec; @@ -368,6 +369,8 @@ static unsigned int damon_migrate_folio_list(struct list_head *folio_list, while (!list_empty(folio_list)) { folio = lru_to_folio(folio_list); list_del(&folio->lru); + node_stat_sub_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); folio_putback_lru(folio); } @@ -385,8 +388,17 @@ unsigned long damon_migrate_pages(struct list_head *folio_list, int target_nid) return nr_migrated; if (target_nid < 0 || target_nid >= MAX_NUMNODES || - !node_state(target_nid, N_MEMORY)) + !node_state(target_nid, N_MEMORY)) { + while (!list_empty(folio_list)) { + struct folio *folio = lru_to_folio(folio_list); + + list_del(&folio->lru); + node_stat_sub_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); + folio_putback_lru(folio); + } return nr_migrated; + } noreclaim_flag = memalloc_noreclaim_save(); diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c index 07a8aead439eb..1854b4bb6bd29 100644 --- a/mm/damon/paddr.c +++ b/mm/damon/paddr.c @@ -67,7 +67,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx) } } -static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) +static bool damon_pa_young(phys_addr_t paddr) { struct folio *folio = damon_get_folio(PHYS_PFN(paddr)); bool accessed; @@ -76,7 +76,6 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) return false; accessed = damon_folio_young(folio); - *folio_sz = folio_size(folio); folio_put(folio); return accessed; } @@ -84,23 +83,12 @@ static bool damon_pa_young(phys_addr_t paddr, unsigned long *folio_sz) static void __damon_pa_check_access(struct damon_region *r, struct damon_attrs *attrs, unsigned long addr_unit) { - static phys_addr_t last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; phys_addr_t sampling_addr = damon_pa_phys_addr( r->sampling_addr, addr_unit); - /* If the region is in the last checked page, reuse the result */ - if (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(sampling_addr, last_folio_sz)) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_pa_young(sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = sampling_addr; + accessed = damon_pa_young(sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx) @@ -279,6 +267,8 @@ static unsigned long damon_pa_migrate(struct damon_region *r, if (!folio_isolate_lru(folio)) goto put_folio; + node_stat_add_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); list_add(&folio->lru, &folio_list); put_folio: addr += folio_size(folio); diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c index adf60f672b8b8..37e0a8bef4b1f 100644 --- a/mm/damon/sysfs-schemes.c +++ b/mm/damon/sysfs-schemes.c @@ -166,6 +166,7 @@ static void damon_sysfs_scheme_regions_rm_dirs( list_for_each_entry_safe(r, next, ®ions->regions_list, list) { list_del(&r->list); + kobject_del(&r->kobj); kobject_put(&r->kobj); regions->nr_regions--; } @@ -704,8 +705,10 @@ static void damon_sysfs_scheme_filters_rm_dirs( struct damon_sysfs_scheme_filter **filters_arr = filters->filters_arr; int i; - for (i = 0; i < filters->nr; i++) + for (i = 0; i < filters->nr; i++) { + kobject_del(&filters_arr[i]->kobj); kobject_put(&filters_arr[i]->kobj); + } filters->nr = 0; kfree(filters_arr); filters->filters_arr = NULL; @@ -1191,8 +1194,10 @@ static void damos_sysfs_quota_goals_rm_dirs( struct damos_sysfs_quota_goal **goals_arr = goals->goals_arr; int i; - for (i = 0; i < goals->nr; i++) + for (i = 0; i < goals->nr; i++) { + kobject_del(&goals_arr[i]->kobj); kobject_put(&goals_arr[i]->kobj); + } goals->nr = 0; kfree(goals_arr); goals->goals_arr = NULL; @@ -1762,8 +1767,10 @@ static void damos_sysfs_dests_rm_dirs( struct damos_sysfs_dest **dests_arr = dests->dests_arr; int i; - for (i = 0; i < dests->nr; i++) + for (i = 0; i < dests->nr; i++) { + kobject_del(&dests_arr[i]->kobj); kobject_put(&dests_arr[i]->kobj); + } dests->nr = 0; kfree(dests_arr); dests->dests_arr = NULL; @@ -2303,6 +2310,7 @@ void damon_sysfs_schemes_rm_dirs(struct damon_sysfs_schemes *schemes) for (i = 0; i < schemes->nr; i++) { damon_sysfs_scheme_rm_dirs(schemes_arr[i]); + kobject_del(&schemes_arr[i]->kobj); kobject_put(&schemes_arr[i]->kobj); } schemes->nr = 0; @@ -2344,13 +2352,15 @@ static int damon_sysfs_schemes_add_dirs(struct damon_sysfs_schemes *schemes, goto out; err = damon_sysfs_scheme_add_dirs(scheme); if (err) - goto out; + goto del_out; schemes_arr[i] = scheme; schemes->nr++; } return 0; +del_out: + kobject_del(&scheme->kobj); out: damon_sysfs_schemes_rm_dirs(schemes); kobject_put(&scheme->kobj); diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c index a6c3e83a3dad5..7f52470be36e3 100644 --- a/mm/damon/sysfs.c +++ b/mm/damon/sysfs.c @@ -107,8 +107,10 @@ static void damon_sysfs_regions_rm_dirs(struct damon_sysfs_regions *regions) struct damon_sysfs_region **regions_arr = regions->regions_arr; int i; - for (i = 0; i < regions->nr; i++) + for (i = 0; i < regions->nr; i++) { + kobject_del(®ions_arr[i]->kobj); kobject_put(®ions_arr[i]->kobj); + } regions->nr = 0; kfree(regions_arr); regions->regions_arr = NULL; @@ -305,6 +307,7 @@ static void damon_sysfs_targets_rm_dirs(struct damon_sysfs_targets *targets) for (i = 0; i < targets->nr; i++) { damon_sysfs_target_rm_dirs(targets_arr[i]); + kobject_del(&targets_arr[i]->kobj); kobject_put(&targets_arr[i]->kobj); } targets->nr = 0; @@ -343,13 +346,15 @@ static int damon_sysfs_targets_add_dirs(struct damon_sysfs_targets *targets, err = damon_sysfs_target_add_dirs(target); if (err) - goto out; + goto del_out; targets_arr[i] = target; targets->nr++; } return 0; +del_out: + kobject_del(&target->kobj); out: damon_sysfs_targets_rm_dirs(targets); kobject_put(&target->kobj); @@ -1078,6 +1083,7 @@ static void damon_sysfs_contexts_rm_dirs(struct damon_sysfs_contexts *contexts) for (i = 0; i < contexts->nr; i++) { damon_sysfs_context_rm_dirs(contexts_arr[i]); + kobject_del(&contexts_arr[i]->kobj); kobject_put(&contexts_arr[i]->kobj); } contexts->nr = 0; @@ -1116,13 +1122,15 @@ static int damon_sysfs_contexts_add_dirs(struct damon_sysfs_contexts *contexts, err = damon_sysfs_context_add_dirs(context); if (err) - goto out; + goto del_out; contexts_arr[i] = context; contexts->nr++; } return 0; +del_out: + kobject_del(&context->kobj); out: damon_sysfs_contexts_rm_dirs(contexts); kobject_put(&context->kobj); @@ -1432,16 +1440,18 @@ static inline bool damon_sysfs_kdamond_running( static int damon_sysfs_apply_inputs(struct damon_ctx *ctx, struct damon_sysfs_context *sys_ctx) { + enum damon_ops_id ops_id; int err; - err = damon_select_ops(ctx, sys_ctx->ops_id); + ops_id = READ_ONCE(sys_ctx->ops_id); + err = damon_select_ops(ctx, ops_id); if (err) return err; - ctx->addr_unit = sys_ctx->addr_unit; + ctx->addr_unit = READ_ONCE(sys_ctx->addr_unit); /* addr_unit is respected by only DAMON_OPS_PADDR */ - if (sys_ctx->ops_id == DAMON_OPS_PADDR) + if (ops_id == DAMON_OPS_PADDR) ctx->min_sz_region = max( - DAMON_MIN_REGION / sys_ctx->addr_unit, 1); + DAMON_MIN_REGION / ctx->addr_unit, 1); err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs); if (err) return err; @@ -1867,6 +1877,7 @@ static void damon_sysfs_kdamonds_rm_dirs(struct damon_sysfs_kdamonds *kdamonds) for (i = 0; i < kdamonds->nr; i++) { damon_sysfs_kdamond_rm_dirs(kdamonds_arr[i]); + kobject_del(&kdamonds_arr[i]->kobj); kobject_put(&kdamonds_arr[i]->kobj); } kdamonds->nr = 0; @@ -1921,13 +1932,15 @@ static int damon_sysfs_kdamonds_add_dirs(struct damon_sysfs_kdamonds *kdamonds, err = damon_sysfs_kdamond_add_dirs(kdamond); if (err) - goto out; + goto del_out; kdamonds_arr[i] = kdamond; kdamonds->nr++; } return 0; +del_out: + kobject_del(&kdamond->kobj); out: damon_sysfs_kdamonds_rm_dirs(kdamonds); kobject_put(&kdamond->kobj); diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h index 6687778752728..0ce6481319432 100644 --- a/mm/damon/tests/core-kunit.h +++ b/mm/damon/tests/core-kunit.h @@ -166,6 +166,10 @@ static void damon_test_split_at(struct kunit *test) r->last_nr_accesses = 15; damon_add_region(r, t); damon_split_region_at(t, r, 25); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; + KUNIT_EXPECT_EQ(test, r->ar.start, 0ul); KUNIT_EXPECT_EQ(test, r->ar.end, 25ul); @@ -177,6 +181,7 @@ static void damon_test_split_at(struct kunit *test) KUNIT_EXPECT_EQ(test, r->nr_accesses, r_new->nr_accesses); KUNIT_EXPECT_EQ(test, r->last_nr_accesses, r_new->last_nr_accesses); +out: damon_free_target(t); damon_destroy_ctx(c); } @@ -261,14 +266,17 @@ static void damon_test_merge_regions_of(struct kunit *test) damon_add_region(r, t); } - damon_merge_regions_of(t, 9, 9999); + damon_merge_regions_of(t, 9, 9999, true); /* 0-112, 114-130, 130-156, 156-170 */ KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 5u); + if (damon_nr_regions(t) != 5) + goto out; for (i = 0; i < 5; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, saddrs[i]); KUNIT_EXPECT_EQ(test, r->ar.end, eaddrs[i]); } +out: damon_free_target(t); } @@ -588,6 +596,8 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 1); KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 2); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 2); KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 4); @@ -602,11 +612,14 @@ static void damos_test_filter_out(struct kunit *test) KUNIT_EXPECT_EQ(test, r->ar.start, DAMON_MIN_REGION * 2); KUNIT_EXPECT_EQ(test, r->ar.end, DAMON_MIN_REGION * 6); KUNIT_EXPECT_EQ(test, damon_nr_regions(t), 2); + if (damon_nr_regions(t) != 2) + goto out; r2 = damon_next_region(r); KUNIT_EXPECT_EQ(test, r2->ar.start, DAMON_MIN_REGION * 6); KUNIT_EXPECT_EQ(test, r2->ar.end, DAMON_MIN_REGION * 8); damon_destroy_region(r2, t); +out: damon_free_target(t); damos_free_filter(f); } diff --git a/mm/damon/tests/vaddr-kunit.h b/mm/damon/tests/vaddr-kunit.h index 30dc5459f1d2c..e2c36d4d8789e 100644 --- a/mm/damon/tests/vaddr-kunit.h +++ b/mm/damon/tests/vaddr-kunit.h @@ -149,12 +149,17 @@ static void damon_do_test_apply_three_regions(struct kunit *test, damon_set_regions(t, three_regions, 3, DAMON_MIN_REGION); + KUNIT_EXPECT_EQ(test, damon_nr_regions(t), nr_expected / 2); + if (damon_nr_regions(t) != nr_expected / 2) + goto out; + for (i = 0; i < nr_expected / 2; i++) { r = __nth_region_of(t, i); KUNIT_EXPECT_EQ(test, r->ar.start, expected[i * 2]); KUNIT_EXPECT_EQ(test, r->ar.end, expected[i * 2 + 1]); } +out: damon_destroy_target(t, NULL); } diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c index 7e834467b2d81..c49f494ec17cd 100644 --- a/mm/damon/vaddr.c +++ b/mm/damon/vaddr.c @@ -431,8 +431,6 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx) } struct damon_young_walk_private { - /* size of the folio for the access checked virtual memory address */ - unsigned long *folio_sz; bool young; }; @@ -468,7 +466,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = HPAGE_PMD_SIZE; folio_put(folio); huge_out: spin_unlock(ptl); @@ -490,7 +487,6 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr, if (pte_young(ptent) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = folio_size(folio); folio_put(folio); out: pte_unmap_unlock(pte, ptl); @@ -519,7 +515,6 @@ static int damon_young_hugetlb_entry(pte_t *pte, unsigned long hmask, if (pte_young(entry) || !folio_test_idle(folio) || mmu_notifier_test_young(walk->mm, addr)) priv->young = true; - *priv->folio_sz = huge_page_size(h); folio_put(folio); @@ -537,11 +532,9 @@ static const struct mm_walk_ops damon_young_ops = { .walk_lock = PGWALK_RDLOCK, }; -static bool damon_va_young(struct mm_struct *mm, unsigned long addr, - unsigned long *folio_sz) +static bool damon_va_young(struct mm_struct *mm, unsigned long addr) { struct damon_young_walk_private arg = { - .folio_sz = folio_sz, .young = false, }; @@ -558,29 +551,18 @@ static bool damon_va_young(struct mm_struct *mm, unsigned long addr, * r the region to be checked */ static void __damon_va_check_access(struct mm_struct *mm, - struct damon_region *r, bool same_target, + struct damon_region *r, struct damon_attrs *attrs) { - static unsigned long last_addr; - static unsigned long last_folio_sz = PAGE_SIZE; - static bool last_accessed; + bool accessed; if (!mm) { damon_update_region_access_rate(r, false, attrs); return; } - /* If the region is in the last checked page, reuse the result */ - if (same_target && (ALIGN_DOWN(last_addr, last_folio_sz) == - ALIGN_DOWN(r->sampling_addr, last_folio_sz))) { - damon_update_region_access_rate(r, last_accessed, attrs); - return; - } - - last_accessed = damon_va_young(mm, r->sampling_addr, &last_folio_sz); - damon_update_region_access_rate(r, last_accessed, attrs); - - last_addr = r->sampling_addr; + accessed = damon_va_young(mm, r->sampling_addr); + damon_update_region_access_rate(r, accessed, attrs); } static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) @@ -589,16 +571,12 @@ static unsigned int damon_va_check_accesses(struct damon_ctx *ctx) struct mm_struct *mm; struct damon_region *r; unsigned int max_nr_accesses = 0; - bool same_target; damon_for_each_target(t, ctx) { mm = damon_get_mm(t); - same_target = false; damon_for_each_region(r, t) { - __damon_va_check_access(mm, r, same_target, - &ctx->attrs); + __damon_va_check_access(mm, r, &ctx->attrs); max_nr_accesses = max(r->nr_accesses, max_nr_accesses); - same_target = true; } if (mm) mmput(mm); @@ -714,7 +692,8 @@ static void damos_va_migrate_dests_add(struct folio *folio, isolate: if (!folio_isolate_lru(folio)) return; - + node_stat_add_folio(folio, NR_ISOLATED_ANON + + folio_is_file_lru(folio)); list_add(&folio->lru, &migration_lists[i]); } diff --git a/mm/filemap.c b/mm/filemap.c index 7c26e3735f1bb..2d7533e965e6a 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -944,6 +944,12 @@ noinline int __filemap_add_folio(struct address_space *mapping, if (!xas_nomem(&xas, gfp)) break; + + /* + * Lock has been dropped: start again with the original index + * and order (but now with the memory reserved by xas_nomem()). + */ + xas_set_order(&xas, index, forder); } if (xas_error(&xas)) diff --git a/mm/gup.c b/mm/gup.c index 95d948c8e86c9..2264a36bc773f 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -2273,6 +2273,7 @@ static unsigned long collect_longterm_unpinnable_folios( for (folio = pofs_get_folio(pofs, i); folio; folio = pofs_next_folio(folio, pofs, &i)) { + const int pin_refs = folio_has_pincount(folio) ? 1 : GUP_PIN_COUNTING_BIAS; if (folio_is_longterm_pinnable(folio)) continue; @@ -2287,15 +2288,20 @@ static unsigned long collect_longterm_unpinnable_folios( continue; } + /* + * We drain not only to make the folio_isolate_lru() succeed, + * but also to remove any other folio references from LRU + * caches. + */ if (drained == 0 && folio_may_be_lru_cached(folio) && folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { + folio_expected_ref_count(folio) + pin_refs) { lru_add_drain(); drained = 1; } if (drained == 1 && folio_may_be_lru_cached(folio) && folio_ref_count(folio) != - folio_expected_ref_count(folio) + 1) { + folio_expected_ref_count(folio) + pin_refs) { lru_add_drain_all(); drained = 2; } diff --git a/mm/hmm.c b/mm/hmm.c index a56081d67ad69..df9b11e1d8ed0 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -407,7 +407,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp, if (pmd_none(pmd)) return hmm_vma_walk_hole(start, end, -1, walk); - if (thp_migration_supported() && is_pmd_migration_entry(pmd)) { + if (thp_migration_supported() && pmd_is_migration_entry(pmd)) { if (hmm_range_need_fault(hmm_vma_walk, hmm_pfns, npages, 0)) { hmm_vma_walk->last = addr; pmd_migration_entry_wait(walk->mm, pmdp); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index b90be9f16b49b..bedd9ba223fbb 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #include @@ -75,6 +76,7 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, static bool split_underused_thp = true; static atomic_t huge_zero_refcount; +static DEFINE_SPINLOCK(huge_zero_lock); struct folio *huge_zero_folio __read_mostly; unsigned long huge_zero_pfn __read_mostly = ~0UL; unsigned long huge_anon_orders_always __read_mostly; @@ -213,7 +215,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, static bool get_huge_zero_folio(void) { struct folio *zero_folio; -retry: + + /* Paired with atomic_set_release(). */ if (likely(atomic_inc_not_zero(&huge_zero_refcount))) return true; @@ -226,17 +229,22 @@ static bool get_huge_zero_folio(void) } /* Ensure zero folio won't have large_rmappable flag set. */ folio_clear_large_rmappable(zero_folio); - preempt_disable(); - if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { - preempt_enable(); + + /* Paired with critical section in shrink_huge_zero_folio_scan(). */ + spin_lock(&huge_zero_lock); + if (huge_zero_folio) { + /* Somebody else already installed it. */ + atomic_inc(&huge_zero_refcount); + spin_unlock(&huge_zero_lock); folio_put(zero_folio); - goto retry; + return true; } + WRITE_ONCE(huge_zero_folio, zero_folio); WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); + /* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */ + atomic_set_release(&huge_zero_refcount, 2); + spin_unlock(&huge_zero_lock); - /* We take additional reference here. It will be put back by shrinker */ - atomic_set(&huge_zero_refcount, 2); - preempt_enable(); count_vm_event(THP_ZERO_PAGE_ALLOC); return true; } @@ -286,15 +294,22 @@ static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, struct shrink_control *sc) { - if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { - struct folio *zero_folio = xchg(&huge_zero_folio, NULL); - BUG_ON(zero_folio == NULL); + struct folio *zero_folio; + + /* Paired with critical section in get_huge_zero_folio(). */ + scoped_guard(spinlock, &huge_zero_lock) { + /* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */ + if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1) + return 0; + + zero_folio = huge_zero_folio; + VM_WARN_ON_ONCE(!zero_folio); + WRITE_ONCE(huge_zero_folio, NULL); WRITE_ONCE(huge_zero_pfn, ~0UL); - folio_put(zero_folio); - return HPAGE_PMD_NR; } - return 0; + folio_put(zero_folio); + return HPAGE_PMD_NR; } static struct shrinker *huge_zero_folio_shrinker; @@ -1080,28 +1095,86 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) return pmd; } +static struct deferred_split *split_queue_node(int nid) +{ + struct pglist_data *pgdata = NODE_DATA(nid); + + return &pgdata->deferred_split_queue; +} + #ifdef CONFIG_MEMCG static inline -struct deferred_split *get_deferred_split_queue(struct folio *folio) +struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, + struct deferred_split *queue) { - struct mem_cgroup *memcg = folio_memcg(folio); - struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); + if (mem_cgroup_disabled()) + return NULL; + if (split_queue_node(folio_nid(folio)) == queue) + return NULL; + return container_of(queue, struct mem_cgroup, deferred_split_queue); +} - if (memcg) - return &memcg->deferred_split_queue; - else - return &pgdat->deferred_split_queue; +static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) +{ + return memcg ? &memcg->deferred_split_queue : split_queue_node(nid); } #else static inline -struct deferred_split *get_deferred_split_queue(struct folio *folio) +struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, + struct deferred_split *queue) { - struct pglist_data *pgdat = NODE_DATA(folio_nid(folio)); + return NULL; +} - return &pgdat->deferred_split_queue; +static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) +{ + return split_queue_node(nid); } #endif +static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg) +{ + struct deferred_split *queue; + + queue = memcg_split_queue(nid, memcg); + spin_lock(&queue->split_queue_lock); + + return queue; +} + +static struct deferred_split * +split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags) +{ + struct deferred_split *queue; + + queue = memcg_split_queue(nid, memcg); + spin_lock_irqsave(&queue->split_queue_lock, *flags); + + return queue; +} + +static struct deferred_split *folio_split_queue_lock(struct folio *folio) +{ + return split_queue_lock(folio_nid(folio), folio_memcg(folio)); +} + +static struct deferred_split * +folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags) +{ + return split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags); +} + +static inline void split_queue_unlock(struct deferred_split *queue) +{ + spin_unlock(&queue->split_queue_lock); +} + +static inline void split_queue_unlock_irqrestore(struct deferred_split *queue, + unsigned long flags) +{ + spin_unlock_irqrestore(&queue->split_queue_lock, flags); +} + static inline bool is_transparent_hugepage(const struct folio *folio) { if (!folio_test_large(folio)) @@ -1707,7 +1780,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, if (unlikely(is_swap_pmd(pmd))) { swp_entry_t entry = pmd_to_swp_entry(pmd); - VM_BUG_ON(!is_pmd_migration_entry(pmd)); + VM_BUG_ON(!pmd_is_migration_entry(pmd)); if (!is_readable_migration_entry(entry)) { entry = make_readable_migration_entry( swp_offset(entry)); @@ -2117,7 +2190,7 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (unlikely(!pmd_present(orig_pmd))) { VM_BUG_ON(thp_migration_supported() && - !is_pmd_migration_entry(orig_pmd)); + !pmd_is_migration_entry(orig_pmd)); goto out; } @@ -2218,7 +2291,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, } else if (thp_migration_supported()) { swp_entry_t entry; - VM_BUG_ON(!is_pmd_migration_entry(orig_pmd)); + VM_BUG_ON(!pmd_is_migration_entry(orig_pmd)); entry = pmd_to_swp_entry(orig_pmd); folio = pfn_swap_entry_folio(entry); flush_needed = 0; @@ -2238,6 +2311,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, * Use flush_needed to indicate whether the PMD entry * is present, instead of checking pmd_present() again. */ + if (flush_needed && pmd_dirty(orig_pmd)) + folio_mark_dirty(folio); if (flush_needed && pmd_young(orig_pmd) && likely(vma_has_recency(vma))) folio_mark_accessed(folio); @@ -2268,7 +2343,7 @@ static inline int pmd_move_must_withdraw(spinlock_t *new_pmd_ptl, static pmd_t move_soft_dirty_pmd(pmd_t pmd) { #ifdef CONFIG_MEM_SOFT_DIRTY - if (unlikely(is_pmd_migration_entry(pmd))) + if (unlikely(pmd_is_migration_entry(pmd))) pmd = pmd_swp_mksoft_dirty(pmd); else if (pmd_present(pmd)) pmd = pmd_mksoft_dirty(pmd); @@ -2278,9 +2353,11 @@ static pmd_t move_soft_dirty_pmd(pmd_t pmd) static pmd_t clear_uffd_wp_pmd(pmd_t pmd) { + if (pmd_none(pmd)) + return pmd; if (pmd_present(pmd)) pmd = pmd_clear_uffd_wp(pmd); - else if (is_swap_pmd(pmd)) + else pmd = pmd_swp_clear_uffd_wp(pmd); return pmd; @@ -2371,7 +2448,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, struct folio *folio = pfn_swap_entry_folio(entry); pmd_t newpmd; - VM_BUG_ON(!is_pmd_migration_entry(*pmd)); + VM_BUG_ON(!pmd_is_migration_entry(*pmd)); if (is_writable_migration_entry(entry)) { /* * A protection check is difficult so @@ -2559,8 +2636,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm if (!pmd_trans_huge(src_pmdval)) { spin_unlock(src_ptl); - if (is_pmd_migration_entry(src_pmdval)) { - pmd_migration_entry_wait(mm, &src_pmdval); + if (pmd_is_migration_entry(src_pmdval)) { + pmd_migration_entry_wait(mm, src_pmd); return -EAGAIN; } return -ENOENT; @@ -2848,7 +2925,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, VM_BUG_ON(haddr & ~HPAGE_PMD_MASK); VM_BUG_ON_VMA(vma->vm_start > haddr, vma); VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); - VM_BUG_ON(!is_pmd_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); + VM_BUG_ON(!pmd_is_migration_entry(*pmd) && !pmd_trans_huge(*pmd)); count_vm_event(THP_SPLIT_PMD); @@ -2862,7 +2939,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, zap_deposited_table(mm, pmd); if (!vma_is_dax(vma) && vma_is_special_huge(vma)) return; - if (unlikely(is_pmd_migration_entry(old_pmd))) { + if (unlikely(pmd_is_migration_entry(old_pmd))) { swp_entry_t entry; entry = pmd_to_swp_entry(old_pmd); @@ -2898,7 +2975,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, return __split_huge_zero_page_pmd(vma, haddr, pmd); } - pmd_migration = is_pmd_migration_entry(*pmd); + pmd_migration = pmd_is_migration_entry(*pmd); if (unlikely(pmd_migration)) { swp_entry_t entry; @@ -3058,7 +3135,7 @@ void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, bool freeze) { VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); - if (pmd_trans_huge(*pmd) || is_pmd_migration_entry(*pmd)) + if (pmd_trans_huge(*pmd) || pmd_is_migration_entry(*pmd)) __split_huge_pmd_locked(vma, pmd, address, freeze); } @@ -3223,7 +3300,7 @@ static void remap_page(struct folio *folio, unsigned long nr, int flags) if (!folio_test_anon(folio)) return; for (;;) { - remove_migration_ptes(folio, folio, RMP_LOCKED | flags); + remove_migration_ptes(folio, folio, TTU_RMAP_LOCKED | flags); i += folio_nr_pages(folio); if (i >= nr) break; @@ -3253,23 +3330,6 @@ static void lru_add_split_folio(struct folio *folio, struct folio *new_folio, } } -/* Racy check whether the huge page can be split */ -bool can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) -{ - int extra_pins; - - /* Additional pins from page cache */ - if (folio_test_anon(folio)) - extra_pins = folio_test_swapcache(folio) ? - folio_nr_pages(folio) : 0; - else - extra_pins = folio_nr_pages(folio); - if (pextra_pins) - *pextra_pins = extra_pins; - return folio_mapcount(folio) == folio_ref_count(folio) - extra_pins - - caller_pins; -} - static bool page_range_has_hwpoisoned(struct page *page, long nr_pages) { for (; nr_pages; page++, nr_pages--) @@ -3344,10 +3404,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order, (1L << PG_dirty) | LRU_GEN_MASK | LRU_REFS_MASK)); - if (handle_hwpoison && - page_range_has_hwpoisoned(new_head, new_nr_pages)) - folio_set_has_hwpoisoned(new_folio); - new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; @@ -3378,6 +3434,14 @@ static void __split_folio_to_order(struct folio *folio, int old_order, folio_set_large_rmappable(new_folio); } + /* + * PG_has_hwpoisoned is on the 2nd page, so set it after + * the compound head is prepped. + */ + if (handle_hwpoison && + page_range_has_hwpoisoned(new_head, new_nr_pages)) + folio_set_has_hwpoisoned(new_folio); + if (folio_test_young(folio)) folio_set_young(new_folio); if (folio_test_idle(folio)) @@ -3526,15 +3590,40 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, return ret; } -bool folio_split_supported(struct folio *folio, unsigned int new_order, - bool uniform_split, bool warns) +/** + * folio_check_splittable() - check if a folio can be split to a given order + * @folio: folio to be split + * @new_order: the smallest order of the after split folios (since buddy + * allocator like split generates folios with orders from @folio's + * order - 1 to new_order). + * @uniform_split: perform uniform split or not (non-uniform split) + * + * folio_check_splittable() checks if @folio can be split to @new_order using + * the requested split method. The truncated folio check must come first. + * + * Context: folio must be locked. + * + * Return: 0 - @folio can be split to @new_order, otherwise an error number is + * returned. + */ +int folio_check_splittable(struct folio *folio, unsigned int new_order, + bool uniform_split) { + VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); + /* + * Folios that just got truncated cannot get split. Signal to the + * caller that there was a race. + * + * TODO: this will also currently refuse folios without a mapping in the + * swapcache (shmem or to-be-anon folios). + */ + if (!folio->mapping && !folio_test_anon(folio)) + return -EBUSY; + if (folio_test_anon(folio)) { /* order-1 is not supported for anonymous THP. */ - VM_WARN_ONCE(warns && new_order == 1, - "Cannot split to order-1 folio"); if (new_order == 1) - return false; + return -EINVAL; } else if (!uniform_split || new_order) { if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !mapping_large_folio_support(folio->mapping)) { @@ -3555,9 +3644,7 @@ bool folio_split_supported(struct folio *folio, unsigned int new_order, * case, the mapping does not actually support large * folios properly. */ - VM_WARN_ONCE(warns, - "Cannot split file folio to non-0 order"); - return false; + return -EINVAL; } } @@ -3570,12 +3657,16 @@ bool folio_split_supported(struct folio *folio, unsigned int new_order, * here. */ if ((!uniform_split || new_order) && folio_test_swapcache(folio)) { - VM_WARN_ONCE(warns, - "Cannot split swapcache folio to non-0 order"); - return false; + return -EINVAL; } - return true; + if (is_huge_zero_folio(folio)) + return -EINVAL; + + if (folio_test_writeback(folio)) + return -EBUSY; + + return 0; } /* @@ -3603,50 +3694,45 @@ static int __folio_split(struct folio *folio, unsigned int new_order, struct page *split_at, struct page *lock_at, struct list_head *list, bool uniform_split) { - struct deferred_split *ds_queue = get_deferred_split_queue(folio); + struct deferred_split *ds_queue; XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); + struct mem_cgroup *memcg, *old_memcg; struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; - int order = folio_order(folio); + int old_order = folio_order(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; - int remap_flags = 0; - int extra_pins, ret; - pgoff_t end; - bool is_hzp; + enum ttu_flags ttu_flags = 0; + pgoff_t end = 0; + int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); - if (folio != page_folio(split_at) || folio != page_folio(lock_at)) - return -EINVAL; - - /* - * Folios that just got truncated cannot get split. Signal to the - * caller that there was a race. - * - * TODO: this will also currently refuse shmem folios that are in the - * swapcache. - */ - if (!is_anon && !folio->mapping) - return -EBUSY; - - if (new_order >= folio_order(folio)) - return -EINVAL; + if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { + ret = -EINVAL; + goto out_no_memcg; + } - if (!folio_split_supported(folio, new_order, uniform_split, /* warn = */ true)) - return -EINVAL; + if (new_order >= old_order) { + ret = -EINVAL; + goto out_no_memcg; + } - is_hzp = is_huge_zero_folio(folio); - if (is_hzp) { - pr_warn_ratelimited("Called split_huge_page for huge zero page\n"); - return -EBUSY; + ret = folio_check_splittable(folio, new_order, uniform_split); + if (ret) { + VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); + goto out_no_memcg; } - if (folio_test_writeback(folio)) - return -EBUSY; + /* + * switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); if (is_anon) { /* @@ -3711,7 +3797,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, * Racy check if we can split the page, before unmap_folio() will * split PMDs */ - if (!can_split_folio(folio, 1, &extra_pins)) { + if (folio_expected_ref_count(folio) != folio_ref_count(folio) - 1) { ret = -EAGAIN; goto out_unlock; } @@ -3734,29 +3820,33 @@ static int __folio_split(struct folio *folio, unsigned int new_order, } /* Prevent deferred_split_scan() touching ->_refcount */ - spin_lock(&ds_queue->split_queue_lock); - if (folio_ref_freeze(folio, 1 + extra_pins)) { + ds_queue = folio_split_queue_lock(folio); + if (folio_ref_freeze(folio, + (folio_test_anon(folio) && + !folio_test_swapcache(folio) ? + 0 : folio_nr_pages(folio)) + 1)) { struct swap_cluster_info *ci = NULL; struct lruvec *lruvec; int expected_refs; - if (folio_order(folio) > 1 && - !list_empty(&folio->_deferred_list)) { - ds_queue->split_queue_len--; + if (folio_order(folio) > 1) { + if (!list_empty(&folio->_deferred_list)) { + ds_queue->split_queue_len--; + /* + * Reinitialize page_deferred_list after removing the + * page from the split_queue, otherwise a subsequent + * split will see list corruption when checking the + * page_deferred_list. + */ + list_del_init(&folio->_deferred_list); + } if (folio_test_partially_mapped(folio)) { folio_clear_partially_mapped(folio); mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } - /* - * Reinitialize page_deferred_list after removing the - * page from the split_queue, otherwise a subsequent - * split will see list corruption when checking the - * page_deferred_list. - */ - list_del_init(&folio->_deferred_list); } - spin_unlock(&ds_queue->split_queue_lock); + split_queue_unlock(ds_queue); if (mapping) { int nr = folio_nr_pages(folio); @@ -3803,7 +3893,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order, next = folio_next(new_folio); - expected_refs = folio_expected_ref_count(new_folio) + 1; + expected_refs = (folio_test_anon(new_folio) && + !folio_test_swapcache(new_folio) ? + 0 : folio_nr_pages(new_folio)) + 1; folio_ref_unfreeze(new_folio, expected_refs); lru_add_split_folio(folio, new_folio, lruvec, list); @@ -3843,7 +3935,9 @@ static int __folio_split(struct folio *folio, unsigned int new_order, * Otherwise, a parallel folio_try_get() can grab @folio * and its caller can see stale page cache entries. */ - expected_refs = folio_expected_ref_count(folio) + 1; + expected_refs = (folio_test_anon(folio) && + !folio_test_swapcache(folio) ? + 0 : folio_nr_pages(folio)) + 1; folio_ref_unfreeze(folio, expected_refs); unlock_page_lruvec(lruvec); @@ -3851,7 +3945,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (ci) swap_cluster_unlock(ci); } else { - spin_unlock(&ds_queue->split_queue_lock); + split_queue_unlock(ds_queue); ret = -EAGAIN; } fail: @@ -3864,8 +3958,20 @@ static int __folio_split(struct folio *folio, unsigned int new_order, shmem_uncharge(mapping->host, nr_shmem_dropped); if (!ret && is_anon) - remap_flags = RMP_USE_SHARED_ZEROPAGE; - remap_page(folio, 1 << order, remap_flags); + ttu_flags = TTU_USE_SHARED_ZEROPAGE; + remap_page(folio, 1 << old_order, ttu_flags); + + /* + * Drop the mapping while the inode is still pinned. @folio stays + * locked and present in the page cache until the loop below, so + * eviction cannot free the inode yet; @lock_at is not enough, it may + * be a tail beyond EOF that the split already dropped from the page + * cache. Nothing past this point may touch the inode or the mapping. + */ + if (mapping) { + i_mmap_unlock_read(mapping); + mapping = NULL; + } /* * Unlock all after-split folios except the one containing @@ -3895,10 +4001,14 @@ static int __folio_split(struct folio *folio, unsigned int new_order, if (mapping) i_mmap_unlock_read(mapping); out: + /* restore to caller's old_memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +out_no_memcg: xas_destroy(&xas); - if (order == HPAGE_PMD_ORDER) + if (old_order == HPAGE_PMD_ORDER) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); - count_mthp_stat(order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); + count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); return ret; } @@ -4027,8 +4137,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio) WARN_ON_ONCE(folio_ref_count(folio)); WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg(folio)); - ds_queue = get_deferred_split_queue(folio); - spin_lock_irqsave(&ds_queue->split_queue_lock, flags); + ds_queue = folio_split_queue_lock_irqsave(folio, &flags); if (!list_empty(&folio->_deferred_list)) { ds_queue->split_queue_len--; if (folio_test_partially_mapped(folio)) { @@ -4039,7 +4148,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio) list_del_init(&folio->_deferred_list); unqueued = true; } - spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); + split_queue_unlock_irqrestore(ds_queue, flags); return unqueued; /* useful for debug warnings */ } @@ -4047,10 +4156,7 @@ bool __folio_unqueue_deferred_split(struct folio *folio) /* partially_mapped=false won't clear PG_partially_mapped folio flag */ void deferred_split_folio(struct folio *folio, bool partially_mapped) { - struct deferred_split *ds_queue = get_deferred_split_queue(folio); -#ifdef CONFIG_MEMCG - struct mem_cgroup *memcg = folio_memcg(folio); -#endif + struct deferred_split *ds_queue; unsigned long flags; /* @@ -4073,7 +4179,7 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) if (folio_test_swapcache(folio)) return; - spin_lock_irqsave(&ds_queue->split_queue_lock, flags); + ds_queue = folio_split_queue_lock_irqsave(folio, &flags); if (partially_mapped) { if (!folio_test_partially_mapped(folio)) { folio_set_partially_mapped(folio); @@ -4088,15 +4194,16 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); } if (list_empty(&folio->_deferred_list)) { + struct mem_cgroup *memcg; + + memcg = folio_split_queue_memcg(folio, ds_queue); list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); ds_queue->split_queue_len++; -#ifdef CONFIG_MEMCG if (memcg) set_shrinker_bit(memcg, folio_nid(folio), - deferred_split_shrinker->id); -#endif + shrinker_id(deferred_split_shrinker)); } - spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); + split_queue_unlock_irqrestore(ds_queue, flags); } static unsigned long deferred_split_count(struct shrinker *shrink, @@ -4142,43 +4249,42 @@ static bool thp_underused(struct folio *folio) static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc) { - struct pglist_data *pgdata = NODE_DATA(sc->nid); - struct deferred_split *ds_queue = &pgdata->deferred_split_queue; + struct deferred_split *ds_queue; unsigned long flags; - LIST_HEAD(list); - struct folio *folio, *next, *prev = NULL; - int split = 0, removed = 0; + struct folio *folio, *next; + int split = 0, i; + struct folio_batch fbatch; -#ifdef CONFIG_MEMCG - if (sc->memcg) - ds_queue = &sc->memcg->deferred_split_queue; -#endif + folio_batch_init(&fbatch); - spin_lock_irqsave(&ds_queue->split_queue_lock, flags); +retry: + ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags); /* Take pin on all head pages to avoid freeing them under us */ list_for_each_entry_safe(folio, next, &ds_queue->split_queue, _deferred_list) { if (folio_try_get(folio)) { - list_move(&folio->_deferred_list, &list); - } else { + folio_batch_add(&fbatch, folio); + } else if (folio_test_partially_mapped(folio)) { /* We lost race with folio_put() */ - if (folio_test_partially_mapped(folio)) { - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_del_init(&folio->_deferred_list); - ds_queue->split_queue_len--; + folio_clear_partially_mapped(folio); + mod_mthp_stat(folio_order(folio), + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } + list_del_init(&folio->_deferred_list); + ds_queue->split_queue_len--; if (!--sc->nr_to_scan) break; + if (!folio_batch_space(&fbatch)) + break; } - spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); + split_queue_unlock_irqrestore(ds_queue, flags); - list_for_each_entry_safe(folio, next, &list, _deferred_list) { + for (i = 0; i < folio_batch_count(&fbatch); i++) { bool did_split = false; bool underused = false; + struct deferred_split *fqueue; + folio = fbatch.folios[i]; if (!folio_test_partially_mapped(folio)) { /* * See try_to_map_unused_to_zeropage(): we cannot @@ -4201,38 +4307,27 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, } folio_unlock(folio); next: + if (did_split || !folio_test_partially_mapped(folio)) + continue; /* - * split_folio() removes folio from list on success. * Only add back to the queue if folio is partially mapped. * If thp_underused returns false, or if split_folio fails * in the case it was underused, then consider it used and * don't add it back to split_queue. */ - if (did_split) { - ; /* folio already removed from list */ - } else if (!folio_test_partially_mapped(folio)) { - list_del_init(&folio->_deferred_list); - removed++; - } else { - /* - * That unlocked list_del_init() above would be unsafe, - * unless its folio is separated from any earlier folios - * left on the list (which may be concurrently unqueued) - * by one safe folio with refcount still raised. - */ - swap(folio, prev); + fqueue = folio_split_queue_lock_irqsave(folio, &flags); + if (list_empty(&folio->_deferred_list)) { + list_add_tail(&folio->_deferred_list, &fqueue->split_queue); + fqueue->split_queue_len++; } - if (folio) - folio_put(folio); + split_queue_unlock_irqrestore(fqueue, flags); } + folios_put(&fbatch); - spin_lock_irqsave(&ds_queue->split_queue_lock, flags); - list_splice_tail(&list, &ds_queue->split_queue); - ds_queue->split_queue_len -= removed; - spin_unlock_irqrestore(&ds_queue->split_queue_lock, flags); - - if (prev) - folio_put(prev); + if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) { + cond_resched(); + goto retry; + } /* * Stop shrinker if we didn't split any page, but the queue is empty. @@ -4375,7 +4470,7 @@ static int split_huge_pages_pid(int pid, unsigned long vaddr_start, * can be split or not. So skip the check here. */ if (!folio_test_private(folio) && - !can_split_folio(folio, 0, NULL)) + folio_expected_ref_count(folio) != folio_ref_count(folio)) goto next; if (!folio_trylock(folio)) diff --git a/mm/hugetlb.c b/mm/hugetlb.c index e0b36a634eb62..d969c9ed91b16 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -705,7 +705,7 @@ static int allocate_file_region_entries(struct resv_map *resv, spin_lock(&resv->lock); - list_splice(&allocated_regions, &resv->region_cache); + list_splice_init(&allocated_regions, &resv->region_cache); resv->region_cache_count += to_allocate; } @@ -2140,7 +2140,8 @@ int dissolve_free_hugetlb_folio(struct folio *folio) if (h->surplus_huge_pages_node[folio_nid(folio)]) adjust_surplus = true; remove_hugetlb_folio(h, folio, adjust_surplus); - h->max_huge_pages--; + if (!adjust_surplus) + h->max_huge_pages--; spin_unlock_irq(&hugetlb_lock); /* @@ -2160,7 +2161,8 @@ int dissolve_free_hugetlb_folio(struct folio *folio) if (rc) { spin_lock_irq(&hugetlb_lock); add_hugetlb_folio(h, folio, adjust_surplus); - h->max_huge_pages++; + if (!adjust_surplus) + h->max_huge_pages++; goto out; } } else @@ -3451,7 +3453,7 @@ static void __init gather_bootmem_prealloc_parallel(unsigned long start, gather_bootmem_prealloc_node(nid); } -static void __init gather_bootmem_prealloc(void) +void __init hugetlb_bootmem_struct_page_init(void) { struct padata_mt_job job = { .thread_fn = gather_bootmem_prealloc_parallel, @@ -3648,7 +3650,7 @@ static unsigned long __init hugetlb_pages_alloc_boot(struct hstate *h) * - For gigantic pages, this is called early in the boot process and * pages are allocated from memblock allocated or something similar. * Gigantic pages are actually added to pools later with the routine - * gather_bootmem_prealloc. + * hugetlb_bootmem_struct_page_init. * - For non-gigantic pages, this is called later in the boot process after * all of mm is up and functional. Pages are allocated from buddy and * then added to hugetlb pools. @@ -4715,7 +4717,6 @@ static int __init hugetlb_init(void) hugetlb_cma_check(); hugetlb_init_hstates(); - gather_bootmem_prealloc(); report_hugepages(); hugetlb_sysfs_init(); @@ -5634,8 +5635,12 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, */ ; } else if (unlikely(is_hugetlb_entry_hwpoisoned(entry))) { - if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + /* + * A hwpoison entry never carries the uffd-wp bit: it is + * installed fresh by make_hwpoison_entry() and + * hugetlb_change_protection() leaves it untouched, so + * there is nothing to clear for the child. + */ set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(is_hugetlb_entry_migration(entry))) { swp_entry_t swp_entry = pte_to_swp_entry(entry); @@ -5654,7 +5659,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src, set_huge_pte_at(src, addr, src_pte, entry, sz); } if (!userfaultfd_wp(dst_vma)) - entry = huge_pte_clear_uffd_wp(entry); + entry = pte_swp_clear_uffd_wp(entry); set_huge_pte_at(dst, addr, dst_pte, entry, sz); } else if (unlikely(is_pte_marker(entry))) { pte_marker marker = copy_pte_marker( @@ -5872,6 +5877,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, bool adjust_reservation; unsigned long last_addr_mask; + i_mmap_assert_write_locked(vma->vm_file->f_mapping); WARN_ON(!is_vm_hugetlb_page(vma)); BUG_ON(start & ~huge_page_mask(h)); BUG_ON(end & ~huge_page_mask(h)); @@ -5963,7 +5969,10 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, /* * Restore the reservation for anonymous page, otherwise the - * backing page could be stolen by someone. + * backing page could be stolen by someone. Restore only on the + * last unmap, otherwise the owner could empty its resv map + * while the folio is still mapped by a child. Note that holding + * i_mmap_lock_write is needed to check the number of mappings. * If there we are freeing a surplus, do not set the restore * reservation bit. */ @@ -5971,7 +5980,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, spin_lock_irq(&hugetlb_lock); if (!h->surplus_huge_pages && __vma_private_lock(vma) && - folio_test_anon(folio)) { + !folio_mapped(folio) && folio_test_anon(folio)) { folio_set_hugetlb_restore_reserve(folio); /* Reservation to be adjusted after the spin lock */ adjust_reservation = true; @@ -7908,14 +7917,14 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re * There is no need to transfer the per-node surplus state * when we do not cross the node. */ - if (new_nid == old_nid) - return; - spin_lock_irq(&hugetlb_lock); - if (h->surplus_huge_pages_node[old_nid]) { - h->surplus_huge_pages_node[old_nid]--; - h->surplus_huge_pages_node[new_nid]++; + if (new_nid != old_nid) { + spin_lock_irq(&hugetlb_lock); + if (h->surplus_huge_pages_node[old_nid]) { + h->surplus_huge_pages_node[old_nid]--; + h->surplus_huge_pages_node[new_nid]++; + } + spin_unlock_irq(&hugetlb_lock); } - spin_unlock_irq(&hugetlb_lock); } /* diff --git a/mm/hugetlb_cgroup.c b/mm/hugetlb_cgroup.c index 58e895f3899a1..b6537a3e4f62c 100644 --- a/mm/hugetlb_cgroup.c +++ b/mm/hugetlb_cgroup.c @@ -104,6 +104,7 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, struct page_counter *fault, *fault_parent = NULL; struct page_counter *rsvd, *rsvd_parent = NULL; unsigned long limit; + int ret; if (parent_h_cgroup) { fault_parent = hugetlb_cgroup_counter_from_cgroup( @@ -125,8 +126,10 @@ static void hugetlb_cgroup_init(struct hugetlb_cgroup *h_cgroup, limit = round_down(PAGE_COUNTER_MAX, pages_per_huge_page(&hstates[idx])); - VM_BUG_ON(page_counter_set_max(fault, limit)); - VM_BUG_ON(page_counter_set_max(rsvd, limit)); + ret = page_counter_set_max(fault, limit); + VM_WARN_ON_ONCE(ret); + ret = page_counter_set_max(rsvd, limit); + VM_WARN_ON_ONCE(ret); } } diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 96ee2bd16ee15..795e7dbbca7d0 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -24,8 +24,8 @@ * * @remap_pte: called for each lowest-level entry (PTE). * @nr_walked: the number of walked pte. - * @reuse_page: the page which is reused for the tail vmemmap pages. - * @reuse_addr: the virtual address of the @reuse_page page. + * @vmemmap_head: the page to be installed as first in the vmemmap range + * @vmemmap_tail: the page to be installed as non-first in the vmemmap range * @vmemmap_pages: the list head of the vmemmap pages that can be freed * or is mapped from. * @flags: used to modify behavior in vmemmap page table walking @@ -34,17 +34,17 @@ struct vmemmap_remap_walk { void (*remap_pte)(pte_t *pte, unsigned long addr, struct vmemmap_remap_walk *walk); + unsigned long nr_walked; - struct page *reuse_page; - unsigned long reuse_addr; + struct page *vmemmap_head; + struct page *vmemmap_tail; struct list_head *vmemmap_pages; + /* Skip the TLB flush when we split the PMD */ #define VMEMMAP_SPLIT_NO_TLB_FLUSH BIT(0) /* Skip the TLB flush when we remap the PTE */ #define VMEMMAP_REMAP_NO_TLB_FLUSH BIT(1) -/* synchronize_rcu() to avoid writes from page_ref_add_unless() */ -#define VMEMMAP_SYNCHRONIZE_RCU BIT(2) unsigned long flags; }; @@ -140,14 +140,7 @@ static int vmemmap_pte_entry(pte_t *pte, unsigned long addr, { struct vmemmap_remap_walk *vmemmap_walk = walk->private; - /* - * The reuse_page is found 'first' in page table walking before - * starting remapping. - */ - if (!vmemmap_walk->reuse_page) - vmemmap_walk->reuse_page = pte_page(ptep_get(pte)); - else - vmemmap_walk->remap_pte(pte, addr, vmemmap_walk); + vmemmap_walk->remap_pte(pte, addr, vmemmap_walk); vmemmap_walk->nr_walked++; return 0; @@ -207,18 +200,14 @@ static void free_vmemmap_page_list(struct list_head *list) static void vmemmap_remap_pte(pte_t *pte, unsigned long addr, struct vmemmap_remap_walk *walk) { - /* - * Remap the tail pages as read-only to catch illegal write operation - * to the tail pages. - */ - pgprot_t pgprot = PAGE_KERNEL_RO; struct page *page = pte_page(ptep_get(pte)); pte_t entry; /* Remapping the head page requires r/w */ - if (unlikely(addr == walk->reuse_addr)) { - pgprot = PAGE_KERNEL; - list_del(&walk->reuse_page->lru); + if (unlikely(walk->nr_walked == 0 && walk->vmemmap_head)) { + VM_WARN_ON_ONCE(!PageHead((const struct page *)addr)); + + list_del(&walk->vmemmap_head->lru); /* * Makes sure that preceding stores to the page contents from @@ -226,53 +215,47 @@ static void vmemmap_remap_pte(pte_t *pte, unsigned long addr, * write. */ smp_wmb(); + + entry = mk_pte(walk->vmemmap_head, PAGE_KERNEL); + } else { + VM_WARN_ON_ONCE(!PageTail((const struct page *)addr)); + + /* + * Remap the tail pages as read-only to catch illegal write + * operation to the tail pages. + */ + entry = mk_pte(walk->vmemmap_tail, PAGE_KERNEL_RO); } - entry = mk_pte(walk->reuse_page, pgprot); list_add(&page->lru, walk->vmemmap_pages); set_pte_at(&init_mm, addr, pte, entry); } -/* - * How many struct page structs need to be reset. When we reuse the head - * struct page, the special metadata (e.g. page->flags or page->mapping) - * cannot copy to the tail struct page structs. The invalid value will be - * checked in the free_tail_page_prepare(). In order to avoid the message - * of "corrupted mapping in tail page". We need to reset at least 4 (one - * head struct page struct and three tail struct page structs) struct page - * structs. - */ -#define NR_RESET_STRUCT_PAGE 4 - -static inline void reset_struct_pages(struct page *start) -{ - struct page *from = start + NR_RESET_STRUCT_PAGE; - - BUILD_BUG_ON(NR_RESET_STRUCT_PAGE * 2 > PAGE_SIZE / sizeof(struct page)); - memcpy(start, from, sizeof(*from) * NR_RESET_STRUCT_PAGE); -} - static void vmemmap_restore_pte(pte_t *pte, unsigned long addr, struct vmemmap_remap_walk *walk) { - pgprot_t pgprot = PAGE_KERNEL; - struct page *page; - void *to; + struct page *src = pte_page(ptep_get(pte)), *dst; + + /* + * When rolling back vmemmap_remap_free(), keep the copied head page + * mapping and restore only PTEs currently pointing at the shared tail + * page. + */ + if (walk->vmemmap_tail && walk->vmemmap_tail != src) + return; - BUG_ON(pte_page(ptep_get(pte)) != walk->reuse_page); + VM_WARN_ON_ONCE(PageHead((const struct page *)addr)); - page = list_first_entry(walk->vmemmap_pages, struct page, lru); - list_del(&page->lru); - to = page_to_virt(page); - copy_page(to, (void *)walk->reuse_addr); - reset_struct_pages(to); + dst = list_first_entry(walk->vmemmap_pages, struct page, lru); + list_del(&dst->lru); + copy_page(page_to_virt(dst), page_to_virt(src)); /* * Makes sure that preceding stores to the page contents become visible * before the set_pte_at() write. */ smp_wmb(); - set_pte_at(&init_mm, addr, pte, mk_pte(page, pgprot)); + set_pte_at(&init_mm, addr, pte, mk_pte(dst, PAGE_KERNEL)); } /** @@ -282,33 +265,28 @@ static void vmemmap_restore_pte(pte_t *pte, unsigned long addr, * to remap. * @end: end address of the vmemmap virtual address range that we want to * remap. - * @reuse: reuse address. - * * Return: %0 on success, negative error code otherwise. */ -static int vmemmap_remap_split(unsigned long start, unsigned long end, - unsigned long reuse) +static int vmemmap_remap_split(unsigned long start, unsigned long end) { struct vmemmap_remap_walk walk = { .remap_pte = NULL, .flags = VMEMMAP_SPLIT_NO_TLB_FLUSH, }; - /* See the comment in the vmemmap_remap_free(). */ - BUG_ON(start - reuse != PAGE_SIZE); - - return vmemmap_remap_range(reuse, end, &walk); + return vmemmap_remap_range(start, end, &walk); } /** * vmemmap_remap_free - remap the vmemmap virtual address range [@start, @end) - * to the page which @reuse is mapped to, then free vmemmap - * which the range are mapped to. + * to use @vmemmap_head/tail, then free vmemmap which + * the range are mapped to. * @start: start address of the vmemmap virtual address range that we want * to remap. * @end: end address of the vmemmap virtual address range that we want to * remap. - * @reuse: reuse address. + * @vmemmap_head: the page to be installed as first in the vmemmap range + * @vmemmap_tail: the page to be installed as non-first in the vmemmap range * @vmemmap_pages: list to deposit vmemmap pages to be freed. It is callers * responsibility to free pages. * @flags: modifications to vmemmap_remap_walk flags @@ -316,69 +294,39 @@ static int vmemmap_remap_split(unsigned long start, unsigned long end, * Return: %0 on success, negative error code otherwise. */ static int vmemmap_remap_free(unsigned long start, unsigned long end, - unsigned long reuse, + struct page *vmemmap_head, + struct page *vmemmap_tail, struct list_head *vmemmap_pages, unsigned long flags) { int ret; struct vmemmap_remap_walk walk = { .remap_pte = vmemmap_remap_pte, - .reuse_addr = reuse, + .vmemmap_head = vmemmap_head, + .vmemmap_tail = vmemmap_tail, .vmemmap_pages = vmemmap_pages, .flags = flags, }; - int nid = page_to_nid((struct page *)reuse); - gfp_t gfp_mask = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; - /* - * Allocate a new head vmemmap page to avoid breaking a contiguous - * block of struct page memory when freeing it back to page allocator - * in free_vmemmap_page_list(). This will allow the likely contiguous - * struct page backing memory to be kept contiguous and allowing for - * more allocations of hugepages. Fallback to the currently - * mapped head page in case should it fail to allocate. - */ - walk.reuse_page = alloc_pages_node(nid, gfp_mask, 0); - if (walk.reuse_page) { - copy_page(page_to_virt(walk.reuse_page), - (void *)walk.reuse_addr); - list_add(&walk.reuse_page->lru, vmemmap_pages); - memmap_pages_add(1); - } + ret = vmemmap_remap_range(start, end, &walk); + if (!ret || !walk.nr_walked) + return ret; + + end = start + walk.nr_walked * PAGE_SIZE; /* - * In order to make remapping routine most efficient for the huge pages, - * the routine of vmemmap page table walking has the following rules - * (see more details from the vmemmap_pte_range()): - * - * - The range [@start, @end) and the range [@reuse, @reuse + PAGE_SIZE) - * should be continuous. - * - The @reuse address is part of the range [@reuse, @end) that we are - * walking which is passed to vmemmap_remap_range(). - * - The @reuse address is the first in the complete range. - * - * So we need to make sure that @start and @reuse meet the above rules. + * vmemmap_pages contains pages from the previous vmemmap_remap_range() + * call which failed. These are pages which were removed from + * the vmemmap. They will be restored in the following call. */ - BUG_ON(start - reuse != PAGE_SIZE); + walk = (struct vmemmap_remap_walk) { + .remap_pte = vmemmap_restore_pte, + .vmemmap_tail = vmemmap_tail, + .vmemmap_pages = vmemmap_pages, + .flags = 0, + }; - ret = vmemmap_remap_range(reuse, end, &walk); - if (ret && walk.nr_walked) { - end = reuse + walk.nr_walked * PAGE_SIZE; - /* - * vmemmap_pages contains pages from the previous - * vmemmap_remap_range call which failed. These - * are pages which were removed from the vmemmap. - * They will be restored in the following call. - */ - walk = (struct vmemmap_remap_walk) { - .remap_pte = vmemmap_restore_pte, - .reuse_addr = reuse, - .vmemmap_pages = vmemmap_pages, - .flags = 0, - }; - - vmemmap_remap_range(reuse, end, &walk); - } + vmemmap_remap_range(start, end, &walk); return ret; } @@ -415,29 +363,24 @@ static int alloc_vmemmap_page_list(unsigned long start, unsigned long end, * to remap. * @end: end address of the vmemmap virtual address range that we want to * remap. - * @reuse: reuse address. * @flags: modifications to vmemmap_remap_walk flags * * Return: %0 on success, negative error code otherwise. */ static int vmemmap_remap_alloc(unsigned long start, unsigned long end, - unsigned long reuse, unsigned long flags) + unsigned long flags) { LIST_HEAD(vmemmap_pages); struct vmemmap_remap_walk walk = { .remap_pte = vmemmap_restore_pte, - .reuse_addr = reuse, .vmemmap_pages = &vmemmap_pages, .flags = flags, }; - /* See the comment in the vmemmap_remap_free(). */ - BUG_ON(start - reuse != PAGE_SIZE); - if (alloc_vmemmap_page_list(start, end, &vmemmap_pages)) return -ENOMEM; - return vmemmap_remap_range(reuse, end, &walk); + return vmemmap_remap_range(start, end, &walk); } DEFINE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key); @@ -454,8 +397,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio, unsigned long flags) { int ret; - unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end; - unsigned long vmemmap_reuse; + unsigned long vmemmap_start, vmemmap_end; VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio); @@ -463,21 +405,18 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h, if (!folio_test_hugetlb_vmemmap_optimized(folio)) return 0; - if (flags & VMEMMAP_SYNCHRONIZE_RCU) - synchronize_rcu(); - + vmemmap_start = (unsigned long)&folio->page; vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h); - vmemmap_reuse = vmemmap_start; + vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE; /* * The pages which the vmemmap virtual address range [@vmemmap_start, - * @vmemmap_end) are mapped to are freed to the buddy allocator, and - * the range is mapped to the page which @vmemmap_reuse is mapped to. + * @vmemmap_end) are mapped to are freed to the buddy allocator. * When a HugeTLB page is freed to the buddy allocator, previously * discarded vmemmap pages must be allocated and remapping. */ - ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, vmemmap_reuse, flags); + ret = vmemmap_remap_alloc(vmemmap_start, vmemmap_end, flags); if (!ret) { folio_clear_hugetlb_vmemmap_optimized(folio); static_branch_dec(&hugetlb_optimize_vmemmap_key); @@ -498,7 +437,7 @@ static int __hugetlb_vmemmap_restore_folio(const struct hstate *h, */ int hugetlb_vmemmap_restore_folio(const struct hstate *h, struct folio *folio) { - return __hugetlb_vmemmap_restore_folio(h, folio, VMEMMAP_SYNCHRONIZE_RCU); + return __hugetlb_vmemmap_restore_folio(h, folio, 0); } /** @@ -521,14 +460,11 @@ long hugetlb_vmemmap_restore_folios(const struct hstate *h, struct folio *folio, *t_folio; long restored = 0; long ret = 0; - unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU; + unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; list_for_each_entry_safe(folio, t_folio, folio_list, lru) { if (folio_test_hugetlb_vmemmap_optimized(folio)) { ret = __hugetlb_vmemmap_restore_folio(h, folio, flags); - /* only need to synchronize_rcu() once for each batch */ - flags &= ~VMEMMAP_SYNCHRONIZE_RCU; - if (ret) break; restored++; @@ -565,9 +501,9 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct list_head *vmemmap_pages, unsigned long flags) { - int ret = 0; - unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end; - unsigned long vmemmap_reuse; + unsigned long vmemmap_start, vmemmap_end; + struct page *vmemmap_head, *vmemmap_tail; + int nid, ret = 0; VM_WARN_ON_ONCE_FOLIO(!folio_test_hugetlb(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_ref_count(folio), folio); @@ -577,8 +513,6 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, static_branch_inc(&hugetlb_optimize_vmemmap_key); - if (flags & VMEMMAP_SYNCHRONIZE_RCU) - synchronize_rcu(); /* * Very Subtle * If VMEMMAP_REMAP_NO_TLB_FLUSH is set, TLB flushing is not performed @@ -592,18 +526,30 @@ static int __hugetlb_vmemmap_optimize_folio(const struct hstate *h, */ folio_set_hugetlb_vmemmap_optimized(folio); + nid = folio_nid(folio); + vmemmap_head = alloc_pages_node(nid, GFP_KERNEL, 0); + if (!vmemmap_head) { + ret = -ENOMEM; + goto out; + } + + copy_page(page_to_virt(vmemmap_head), folio); + list_add(&vmemmap_head->lru, vmemmap_pages); + memmap_pages_add(1); + + vmemmap_tail = vmemmap_head; + vmemmap_start = (unsigned long)&folio->page; vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h); - vmemmap_reuse = vmemmap_start; - vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE; /* - * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end) - * to the page which @vmemmap_reuse is mapped to. Add pages previously - * mapping the range to vmemmap_pages list so that they can be freed by - * the caller. + * Remap the vmemmap virtual address range [@vmemmap_start, @vmemmap_end). + * Add pages previously mapping the range to vmemmap_pages list so that + * they can be freed by the caller. */ - ret = vmemmap_remap_free(vmemmap_start, vmemmap_end, vmemmap_reuse, + ret = vmemmap_remap_free(vmemmap_start, vmemmap_end, + vmemmap_head, vmemmap_tail, vmemmap_pages, flags); +out: if (ret) { static_branch_dec(&hugetlb_optimize_vmemmap_key); folio_clear_hugetlb_vmemmap_optimized(folio); @@ -626,27 +572,25 @@ void hugetlb_vmemmap_optimize_folio(const struct hstate *h, struct folio *folio) { LIST_HEAD(vmemmap_pages); - __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, VMEMMAP_SYNCHRONIZE_RCU); + __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, 0); free_vmemmap_page_list(&vmemmap_pages); } static int hugetlb_vmemmap_split_folio(const struct hstate *h, struct folio *folio) { - unsigned long vmemmap_start = (unsigned long)&folio->page, vmemmap_end; - unsigned long vmemmap_reuse; + unsigned long vmemmap_start, vmemmap_end; if (!vmemmap_should_optimize_folio(h, folio)) return 0; + vmemmap_start = (unsigned long)&folio->page; vmemmap_end = vmemmap_start + hugetlb_vmemmap_size(h); - vmemmap_reuse = vmemmap_start; - vmemmap_start += HUGETLB_VMEMMAP_RESERVE_SIZE; /* * Split PMDs on the vmemmap virtual address range [@vmemmap_start, * @vmemmap_end] */ - return vmemmap_remap_split(vmemmap_start, vmemmap_end, vmemmap_reuse); + return vmemmap_remap_split(vmemmap_start, vmemmap_end); } static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, @@ -656,7 +600,7 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, struct folio *folio; int nr_to_optimize; LIST_HEAD(vmemmap_pages); - unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH | VMEMMAP_SYNCHRONIZE_RCU; + unsigned long flags = VMEMMAP_REMAP_NO_TLB_FLUSH; nr_to_optimize = 0; list_for_each_entry(folio, folio_list, lru) { @@ -664,18 +608,19 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, unsigned long spfn, epfn; if (boot && folio_test_hugetlb_vmemmap_optimized(folio)) { + static_branch_inc(&hugetlb_optimize_vmemmap_key); + /* * Already optimized by pre-HVO, just map the * mirrored tail page structs RO. */ spfn = (unsigned long)&folio->page; - epfn = spfn + pages_per_huge_page(h); + epfn = spfn + hugetlb_vmemmap_size(h); vmemmap_wrprotect_hvo(spfn, epfn, folio_nid(folio), HUGETLB_VMEMMAP_RESERVE_SIZE); - register_page_bootmem_memmap(pfn_to_section_nr(spfn), + register_page_bootmem_memmap(pfn_to_section_nr(folio_pfn(folio)), &folio->page, - HUGETLB_VMEMMAP_RESERVE_SIZE); - static_branch_inc(&hugetlb_optimize_vmemmap_key); + HUGETLB_VMEMMAP_RESERVE_PAGES); continue; } @@ -709,8 +654,6 @@ static void __hugetlb_vmemmap_optimize_folios(struct hstate *h, int ret; ret = __hugetlb_vmemmap_optimize_folio(h, folio, &vmemmap_pages, flags); - /* only need to synchronize_rcu() once for each batch */ - flags &= ~VMEMMAP_SYNCHRONIZE_RCU; /* * Pages to be freed may have been accumulated. If we @@ -789,7 +732,6 @@ void __init hugetlb_vmemmap_init_early(int nid) { unsigned long psize, paddr, section_size; unsigned long ns, i, pnum, pfn, nr_pages; - unsigned long start, end; struct huge_bootmem_page *m = NULL; void *map; @@ -815,14 +757,6 @@ void __init hugetlb_vmemmap_init_early(int nid) paddr = virt_to_phys(m); pfn = PHYS_PFN(paddr); map = pfn_to_page(pfn); - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - if (vmemmap_populate_hvo(start, end, nid, - HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) - continue; - - memmap_boot_pages_add(HUGETLB_VMEMMAP_RESERVE_SIZE / PAGE_SIZE); pnum = pfn_to_section_nr(pfn); ns = psize / section_size; @@ -860,28 +794,36 @@ void __init hugetlb_vmemmap_init_late(int nid) h = m->hstate; pfn = PHYS_PFN(phys); nr_pages = pages_per_huge_page(h); + map = pfn_to_page(pfn); + start = (unsigned long)map; + end = start + nr_pages * sizeof(struct page); if (!hugetlb_bootmem_page_zones_valid(nid, m)) { /* * Oops, the hugetlb page spans multiple zones. - * Remove it from the list, and undo HVO. + * Remove it from the list, and populate it normally. */ list_del(&m->list); - map = pfn_to_page(pfn); - - start = (unsigned long)map; - end = start + nr_pages * sizeof(struct page); - - vmemmap_undo_hvo(start, end, nid, - HUGETLB_VMEMMAP_RESERVE_SIZE); - nr_mmap = end - start - HUGETLB_VMEMMAP_RESERVE_SIZE; + vmemmap_populate(start, end, nid, NULL); + nr_mmap = end - start; memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); memblock_phys_free(phys, huge_page_size(h)); continue; - } else + } + + if (vmemmap_populate_hvo(start, end, nid, + HUGETLB_VMEMMAP_RESERVE_SIZE) < 0) { + /* Fallback if HVO population fails */ + vmemmap_populate(start, end, nid, NULL); + nr_mmap = end - start; + } else { m->flags |= HUGE_BOOTMEM_ZONES_VALID; + nr_mmap = HUGETLB_VMEMMAP_RESERVE_SIZE; + } + + memmap_boot_pages_add(DIV_ROUND_UP(nr_mmap, PAGE_SIZE)); } } #endif diff --git a/mm/internal.h b/mm/internal.h index f5c0d927924d0..ef42eee87c7be 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -908,7 +908,15 @@ struct compact_control { * immediately when one is created during the free path. */ struct capture_control { - struct compact_control *cc; + struct zone *zone; + int migratetype; + /* + * Allocation request order. May differ from the compaction + * order: defrag_mode promotes sub-block allocations to + * pageblock-order compaction; capture still matches at the + * original allocation order so prep_new_page() is consistent. + */ + int order; struct page *page; }; diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index 6958aa713c67e..16f4e67beee82 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -355,7 +355,12 @@ void kasan_quarantine_remove_cache(struct kmem_cache *cache) */ on_each_cpu(per_cpu_remove_cache, cache, 1); - for_each_online_cpu(cpu) { + /* + * A CPU can go offline after on_each_cpu() returns, leaving cache + * objects on that CPU's shrink list. Scan all possible CPUs to + * drain those lists. + */ + for_each_possible_cpu(cpu) { sq = per_cpu_ptr(&shrink_qlist, cpu); raw_spin_lock_irqsave(&sq->lock, flags); qlist_move_cache(&sq->qlist, &to_free, cache); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 01a5d443ffc58..a11e56ef99236 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -353,12 +353,6 @@ int hugepage_madvise(struct vm_area_struct *vma, #endif *vm_flags &= ~VM_NOHUGEPAGE; *vm_flags |= VM_HUGEPAGE; - /* - * If the vma become good for khugepaged to scan, - * register it here without waiting a page fault that - * may not happen any time soon. - */ - khugepaged_enter_vma(vma, *vm_flags); break; case MADV_NOHUGEPAGE: *vm_flags &= ~VM_HUGEPAGE; @@ -939,7 +933,7 @@ static inline int check_pmd_state(pmd_t *pmd) * collapse it. Migration success or failure will eventually end * up with a present PMD mapping a folio again. */ - if (is_pmd_migration_entry(pmde)) + if (pmd_is_migration_entry(pmde)) return SCAN_PMD_MAPPED; if (!pmd_present(pmde)) return SCAN_PMD_NULL; diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 1ac56ceb29b6b..452f3714f337b 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -676,7 +676,7 @@ static struct kmemleak_object *__alloc_object(gfp_t gfp) atomic_set(&object->use_count, 1); object->excess_ref = 0; object->count = 0; /* white color initially */ - object->checksum = 0; + object->checksum = ~0; object->del_state = 0; /* task information */ @@ -972,7 +972,7 @@ static void reset_checksum(unsigned long ptr) } raw_spin_lock_irqsave(&object->lock, flags); - object->checksum = 0; + object->checksum = ~0; raw_spin_unlock_irqrestore(&object->lock, flags); put_object(object); } @@ -1401,7 +1401,8 @@ static bool update_checksum(struct kmemleak_object *object) for_each_possible_cpu(cpu) { void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); - object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); + object->checksum = crc32(object->checksum, + kasan_reset_tag((void *)ptr), object->size); } } else { object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); @@ -1688,6 +1689,42 @@ static void kmemleak_cond_resched(struct kmemleak_object *object) put_object(object); } +/* + * Scan all task kernel stacks, rescheduling between tasks. Each task is looked + * up and pinned within its own RCU read-side section, so no lock is held across + * the scan and the walk cannot trip the soft lockup watchdog. + */ +static void kmemleak_scan_task_stacks(void) +{ + struct pid *pid; + int nr = 1; + + do { + struct task_struct *p = NULL; + + rcu_read_lock(); + pid = find_ge_pid(nr, &init_pid_ns); + if (pid) { + nr = pid_nr(pid) + 1; + p = pid_task(pid, PIDTYPE_PID); + if (p) + get_task_struct(p); + } + rcu_read_unlock(); + + if (p) { + void *stack = try_get_task_stack(p); + + if (stack) { + scan_block(stack, stack + THREAD_SIZE, NULL); + put_task_stack(p); + } + put_task_struct(p); + } + cond_resched(); + } while (pid && !scan_should_stop()); +} + /* * Scan data sections and all the referenced memory blocks allocated via the * kernel's standard allocators. This function must be called with the @@ -1779,19 +1816,8 @@ static void kmemleak_scan(void) /* * Scanning the task stacks (may introduce false negatives). */ - if (kmemleak_stack_scan) { - struct task_struct *p, *g; - - rcu_read_lock(); - for_each_process_thread(g, p) { - void *stack = try_get_task_stack(p); - if (stack) { - scan_block(stack, stack + THREAD_SIZE, NULL); - put_task_stack(p); - } - } - rcu_read_unlock(); - } + if (kmemleak_stack_scan) + kmemleak_scan_task_stacks(); /* * Scan the objects already referenced from the sections scanned diff --git a/mm/madvise.c b/mm/madvise.c index 588a809dac11d..b8c25fea4fab9 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -177,6 +177,14 @@ static int madvise_update_vma(vm_flags_t new_flags, /* vm_flags is protected by the mmap_lock held in write mode. */ vma_start_write(vma); vm_flags_reset(vma, new_flags); + /* + * If the vma become good for khugepaged to scan, + * register it here without waiting a page fault that + * may not happen any time soon. + */ + if (new_flags & VM_HUGEPAGE) + khugepaged_enter_vma(vma, new_flags); + if (set_new_anon_name) return replace_anon_vma_name(vma, anon_name); @@ -391,7 +399,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (unlikely(!pmd_present(orig_pmd))) { VM_BUG_ON(thp_migration_supported() && - !is_pmd_migration_entry(orig_pmd)); + !pmd_is_migration_entry(orig_pmd)); goto huge_unlock; } diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c index 6eed14bff7426..af8c9289e8e01 100644 --- a/mm/memcontrol-v1.c +++ b/mm/memcontrol-v1.c @@ -95,7 +95,6 @@ enum { RES_LIMIT, RES_MAX_USAGE, RES_FAILCNT, - RES_SOFT_LIMIT, }; #ifdef CONFIG_LOCKDEP @@ -1139,13 +1138,13 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of, event->unregister_event = mem_cgroup_usage_unregister_event; } else if (!strcmp(name, "memory.oom_control")) { pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org" + "Please report your usecase to linux-mm@kvack.org" " if you depend on this functionality.\n"); event->register_event = mem_cgroup_oom_register_event; event->unregister_event = mem_cgroup_oom_unregister_event; } else if (!strcmp(name, "memory.pressure_level")) { pr_warn_once("pressure_level is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org " + "Please report your usecase to linux-mm@kvack.org " "if you depend on this functionality.\n"); event->register_event = vmpressure_register_event; event->unregister_event = vmpressure_unregister_event; @@ -1547,6 +1546,30 @@ static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css, return -EINVAL; } +static u64 mem_cgroup_soft_limit_read(struct cgroup_subsys_state *css, + struct cftype *cft) +{ + return (u64)PAGE_COUNTER_MAX * PAGE_SIZE; +} + +static ssize_t mem_cgroup_soft_limit_write(struct kernfs_open_file *of, + char *buf, size_t nbytes, loff_t off) +{ + unsigned long nr_pages; + int ret; + + ret = page_counter_memparse(strstrip(buf), "-1", &nr_pages); + if (ret) + return ret; + + pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. " + "Writing any value to this file has no effect. " + "Please report your usecase to linux-mm@kvack.org if you " + "depend on this functionality.\n"); + + return nbytes; +} + static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) { @@ -1583,8 +1606,6 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css, return (u64)counter->watermark * PAGE_SIZE; case RES_FAILCNT: return counter->failcnt; - case RES_SOFT_LIMIT: - return (u64)READ_ONCE(memcg->soft_limit) * PAGE_SIZE; default: BUG(); } @@ -1679,17 +1700,6 @@ static ssize_t mem_cgroup_write(struct kernfs_open_file *of, break; } break; - case RES_SOFT_LIMIT: - if (IS_ENABLED(CONFIG_PREEMPT_RT)) { - ret = -EOPNOTSUPP; - } else { - pr_warn_once("soft_limit_in_bytes is deprecated and will be removed. " - "Please report your usecase to linux-mm@kvack.org if you " - "depend on this functionality.\n"); - WRITE_ONCE(memcg->soft_limit, nr_pages); - ret = 0; - } - break; } return ret ?: nbytes; } @@ -1982,7 +1992,7 @@ static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css, struct mem_cgroup *memcg = mem_cgroup_from_css(css); pr_warn_once("oom_control is deprecated and will be removed. " - "Please report your usecase to linux-mm-@kvack.org if you " + "Please report your usecase to linux-mm@kvack.org if you " "depend on this functionality.\n"); /* cannot set to root cgroup and only 0 and 1 are allowed */ @@ -2027,9 +2037,8 @@ struct cftype mem_cgroup_legacy_files[] = { }, { .name = "soft_limit_in_bytes", - .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT), - .write = mem_cgroup_write, - .read_u64 = mem_cgroup_read_u64, + .write = mem_cgroup_soft_limit_write, + .read_u64 = mem_cgroup_soft_limit_read, }, { .name = "failcnt", diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h index 6358464bb4160..4cdbab2bffe69 100644 --- a/mm/memcontrol-v1.h +++ b/mm/memcontrol-v1.h @@ -103,7 +103,11 @@ static inline void memcg1_remove_from_trees(struct mem_cgroup *memcg) {} static inline void memcg1_soft_limit_reset(struct mem_cgroup *memcg) {} static inline void memcg1_css_offline(struct mem_cgroup *memcg) {} -static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked) { return true; } +static inline bool memcg1_oom_prepare(struct mem_cgroup *memcg, bool *locked) +{ + *locked = false; + return true; +} static inline void memcg1_oom_finish(struct mem_cgroup *memcg, bool locked) {} static inline void memcg1_oom_recover(struct mem_cgroup *memcg) {} diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 752d98fd39219..a4a3da6f241cc 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -2352,6 +2352,19 @@ static int try_charge_memcg(struct mem_cgroup *memcg, gfp_t gfp_mask, if (!gfpflags_allow_blocking(gfp_mask)) goto nomem; + /* + * OOM victim still needs to charge memory to exit. OOM reaper should + * help but it might fail on mmap_lock contention. If the victim is a + * large thread group then all exiting threads might compete on oom_lock + * just to learn that there is nothing really killable anymore. Bail + * out early and fail the charge to expedite their exit. They are + * considered fully reclaimed by the oom reaper and they shouldn't + * contribute further charges. + */ + if (tsk_is_oom_victim(current) && + mm_flags_test(MMF_OOM_SKIP, current->signal->oom_mm)) + goto nomem; + __memcg_memory_event(mem_over_limit, MEMCG_MAX, allow_spinning); raised_max_event = true; @@ -3809,9 +3822,11 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) page_counter_init(&memcg->swap, &parent->swap, false); #ifdef CONFIG_MEMCG_V1 memcg->memory.track_failcnt = !memcg_on_dfl; + memcg->memsw.track_failcnt = !memcg_on_dfl; WRITE_ONCE(memcg->oom_kill_disable, READ_ONCE(parent->oom_kill_disable)); page_counter_init(&memcg->kmem, &parent->kmem, false); page_counter_init(&memcg->tcpmem, &parent->tcpmem, false); + memcg->tcpmem.track_failcnt = !memcg_on_dfl; #endif } else { init_memcg_stats(); @@ -4028,6 +4043,7 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(MEMCG_KMEM); memcg->vmstats->state[index] += kmem; + memcg->vmstats->state_local[index] += kmem; if (parent) parent->vmstats->state_pending[index] += kmem; } @@ -4045,9 +4061,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_RECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4056,9 +4074,11 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent, int index = memcg_stats_index(NR_SLAB_UNRECLAIMABLE_B); lstats->state[index] += slab; + lstats->state_local[index] += slab; if (plstats) plstats->state_pending[index] += slab; memcg->vmstats->state[index] += slab; + memcg->vmstats->state_local[index] += slab; if (parent) parent->vmstats->state_pending[index] += slab; } @@ -4378,6 +4398,9 @@ static ssize_t memory_high_write(struct kernfs_open_file *of, unsigned long nr_pages = page_counter_read(&memcg->memory); unsigned long reclaimed; + if (high != READ_ONCE(memcg->memory.high)) + break; + if (nr_pages <= high) break; @@ -4429,6 +4452,9 @@ static ssize_t memory_max_write(struct kernfs_open_file *of, for (;;) { unsigned long nr_pages = page_counter_read(&memcg->memory); + if (max != READ_ONCE(memcg->memory.max)) + break; + if (nr_pages <= max) break; diff --git a/mm/memory.c b/mm/memory.c index 9b071e269d1ec..72793ec631072 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -6320,34 +6320,39 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, if (pmd_none(*vmf.pmd) && thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PMD_ORDER)) { ret = create_huge_pmd(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) + if (ret & VM_FAULT_FALLBACK) + goto fallback; + else return ret; - } else { - vmf.orig_pmd = pmdp_get_lockless(vmf.pmd); + } - if (unlikely(is_swap_pmd(vmf.orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !is_pmd_migration_entry(vmf.orig_pmd)); - if (is_pmd_migration_entry(vmf.orig_pmd)) - pmd_migration_entry_wait(mm, vmf.pmd); - return 0; - } - if (pmd_trans_huge(vmf.orig_pmd)) { - if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) - return do_huge_pmd_numa_page(&vmf); + vmf.orig_pmd = pmdp_get_lockless(vmf.pmd); + if (pmd_none(vmf.orig_pmd)) + goto fallback; - if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && - !pmd_write(vmf.orig_pmd)) { - ret = wp_huge_pmd(&vmf); - if (!(ret & VM_FAULT_FALLBACK)) - return ret; - } else { - huge_pmd_set_accessed(&vmf); - return 0; - } + if (unlikely(!pmd_present(vmf.orig_pmd))) { + VM_BUG_ON(thp_migration_supported() && + !pmd_is_migration_entry(vmf.orig_pmd)); + if (pmd_is_migration_entry(vmf.orig_pmd)) + pmd_migration_entry_wait(mm, vmf.pmd); + return 0; + } + if (pmd_trans_huge(vmf.orig_pmd)) { + if (pmd_protnone(vmf.orig_pmd) && vma_is_accessible(vma)) + return do_huge_pmd_numa_page(&vmf); + + if ((flags & (FAULT_FLAG_WRITE|FAULT_FLAG_UNSHARE)) && + !pmd_write(vmf.orig_pmd)) { + ret = wp_huge_pmd(&vmf); + if (!(ret & VM_FAULT_FALLBACK)) + return ret; + } else { + huge_pmd_set_accessed(&vmf); + return 0; } } +fallback: return handle_pte_fault(&vmf); } diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c index b7030bcd9b03e..9fdfb2d317654 100644 --- a/mm/memory_hotplug.c +++ b/mm/memory_hotplug.c @@ -221,7 +221,7 @@ void put_online_mems(void) bool movable_node_enabled = false; static int mhp_default_online_type = -1; -int mhp_get_default_online_type(void) +enum mmop mhp_get_default_online_type(void) { if (mhp_default_online_type >= 0) return mhp_default_online_type; @@ -240,7 +240,7 @@ int mhp_get_default_online_type(void) return mhp_default_online_type; } -void mhp_set_default_online_type(int online_type) +void mhp_set_default_online_type(enum mmop online_type) { mhp_default_online_type = online_type; } @@ -1046,7 +1046,7 @@ static inline struct zone *default_zone_for_pfn(int nid, unsigned long start_pfn return movable_node_enabled ? movable_zone : kernel_zone; } -struct zone *zone_for_pfn_range(int online_type, int nid, +struct zone *zone_for_pfn_range(enum mmop online_type, int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages) { @@ -1279,7 +1279,8 @@ static pg_data_t *hotadd_init_pgdat(int nid) pgdat = NODE_DATA(nid); /* init node's zones as empty zones, we don't have any present pages.*/ - free_area_init_core_hotplug(pgdat); + if (free_area_init_core_hotplug(pgdat)) + return NULL; /* * The node we allocated has no zone fallback lists. For avoiding @@ -2338,7 +2339,7 @@ EXPORT_SYMBOL_GPL(remove_memory); static int try_offline_memory_block(struct memory_block *mem, void *arg) { - uint8_t online_type = MMOP_ONLINE_KERNEL; + enum mmop online_type = MMOP_ONLINE_KERNEL; uint8_t **online_types = arg; struct page *page; int rc; @@ -2371,7 +2372,7 @@ static int try_reonline_memory_block(struct memory_block *mem, void *arg) int rc; if (**online_types != MMOP_OFFLINE) { - mem->online_type = **online_types; + mem->online_type = (enum mmop)**online_types; rc = device_online(&mem->dev); if (rc < 0) pr_warn("%s: Failed to re-online memory: %d", diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 08cfbe5e93e84..311b92799bb7d 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -660,7 +660,7 @@ static void queue_folios_pmd(pmd_t *pmd, struct mm_walk *walk) struct folio *folio; struct queue_pages *qp = walk->private; - if (unlikely(is_pmd_migration_entry(*pmd))) { + if (unlikely(pmd_is_migration_entry(*pmd))) { qp->nr_failed++; return; } @@ -2008,24 +2008,15 @@ struct mempolicy *get_vma_policy(struct vm_area_struct *vma, bool vma_policy_mof(struct vm_area_struct *vma) { struct mempolicy *pol; + pgoff_t ilx; + bool mof; - if (vma->vm_ops && vma->vm_ops->get_policy) { - bool ret = false; - pgoff_t ilx; /* ignored here */ - - pol = vma->vm_ops->get_policy(vma, vma->vm_start, &ilx); - if (pol && (pol->flags & MPOL_F_MOF)) - ret = true; - mpol_cond_put(pol); - - return ret; - } - - pol = vma->vm_policy; + pol = __get_vma_policy(vma, vma->vm_start, &ilx); if (!pol) pol = get_task_policy(current); - - return pol->flags & MPOL_F_MOF; + mof = pol->flags & MPOL_F_MOF; + mpol_cond_put(pol); + return mof; } bool apply_policy_zone(struct mempolicy *policy, enum zone_type zone) @@ -2695,7 +2686,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, prev_node = node; /* create a local copy of node weights to operate on outside rcu */ - weights = kzalloc(nr_node_ids, GFP_KERNEL); + weights = kmalloc(nr_node_ids, gfp & GFP_RECLAIM_MASK); if (!weights) return total_allocated; diff --git a/mm/migrate.c b/mm/migrate.c index ad7af66a4450a..3159be53a00d3 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -451,11 +451,12 @@ static bool remove_migration_pte(struct folio *folio, * Get rid of all migration entries and replace them by * references to the indicated page. */ -void remove_migration_ptes(struct folio *src, struct folio *dst, int flags) +void remove_migration_ptes(struct folio *src, struct folio *dst, + enum ttu_flags flags) { struct rmap_walk_arg rmap_walk_arg = { .folio = src, - .map_unused_to_zeropage = flags & RMP_USE_SHARED_ZEROPAGE, + .map_unused_to_zeropage = flags & TTU_USE_SHARED_ZEROPAGE, }; struct rmap_walk_control rwc = { @@ -463,9 +464,9 @@ void remove_migration_ptes(struct folio *src, struct folio *dst, int flags) .arg = &rmap_walk_arg, }; - VM_BUG_ON_FOLIO((flags & RMP_USE_SHARED_ZEROPAGE) && (src != dst), src); + VM_BUG_ON_FOLIO((flags & TTU_USE_SHARED_ZEROPAGE) && (src != dst), src); - if (flags & RMP_LOCKED) + if (flags & TTU_RMAP_LOCKED) rmap_walk_locked(dst, &rwc); else rmap_walk(dst, &rwc); @@ -542,7 +543,7 @@ void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) spinlock_t *ptl; ptl = pmd_lock(mm, pmd); - if (!is_pmd_migration_entry(*pmd)) + if (!pmd_is_migration_entry(*pmd)) goto unlock; migration_entry_wait_on_locked(pmd_to_swp_entry(*pmd), ptl); return; @@ -1530,8 +1531,7 @@ static int unmap_and_move_huge_page(new_folio_t get_new_folio, rc = move_to_new_folio(dst, src, mode); if (page_was_mapped) - remove_migration_ptes(src, !rc ? dst : src, - ttu ? RMP_LOCKED : 0); + remove_migration_ptes(src, !rc ? dst : src, ttu); if (ttu & TTU_RMAP_LOCKED) i_mmap_unlock_write(mapping); @@ -1821,7 +1821,7 @@ static int migrate_pages_batch(struct list_head *from, is_thp = folio_test_pmd_mappable(folio); nr_pages = folio_nr_pages(folio); - cond_resched(); + cond_resched_tasks_rcu_qs(); /* * The rare folio on the deferred split list should diff --git a/mm/migrate_device.c b/mm/migrate_device.c index abd9f6850db65..4abceeae55ab0 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -209,7 +209,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, bool anon_exclusive; pte_t swp_pte; - flush_cache_page(vma, addr, pte_pfn(pte)); + if (pte_present(pte)) + flush_cache_page(vma, addr, pte_pfn(pte)); anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); if (anon_exclusive) { @@ -230,7 +231,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, migrate->cpages++; /* Set the dirty flag on the folio now the pte is gone. */ - if (pte_dirty(pte)) + if (pte_present(pte) && pte_dirty(pte)) folio_mark_dirty(folio); /* Setup special migration page table entry */ diff --git a/mm/mm_init.c b/mm/mm_init.c index 1b977fa9e8bee..a27ba1c1a7192 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1351,7 +1351,7 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, unsigned long node_start_pfn, unsigned long node_end_pfn) { - unsigned long realtotalpages = 0, totalpages = 0; + unsigned long realtotalpages = 0; enum zone_type i; for (i = 0; i < MAX_NR_ZONES; i++) { @@ -1381,11 +1381,10 @@ static void __init calculate_node_totalpages(struct pglist_data *pgdat, zone->present_early_pages = real_size; #endif - totalpages += spanned; realtotalpages += real_size; } - pgdat->node_spanned_pages = totalpages; + pgdat->node_spanned_pages = node_end_pfn - node_start_pfn; pgdat->node_present_pages = realtotalpages; pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages); } @@ -1561,16 +1560,22 @@ void __init set_pageblock_order(void) * NOTE: this function is only called during memory hotplug */ #ifdef CONFIG_MEMORY_HOTPLUG -void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat) { int nid = pgdat->node_id; enum zone_type z; - int cpu; + int cpu, i; pgdat_init_internals(pgdat); - if (pgdat->per_cpu_nodestats == &boot_nodestats) - pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat); + if (pgdat->per_cpu_nodestats == &boot_nodestats) { + struct per_cpu_nodestat __percpu *p; + + p = alloc_percpu(struct per_cpu_nodestat); + if (!p) + return -ENOMEM; + pgdat->per_cpu_nodestats = p; + } /* * Reset the nr_zones, order and highest_zoneidx before reuse. @@ -1583,10 +1588,17 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) pgdat->node_start_pfn = 0; pgdat->node_present_pages = 0; - for_each_online_cpu(cpu) { - struct per_cpu_nodestat *p; + /* + * Hot-unplug can leave per-cpu vmstat deltas unfolded (folders skip + * offline nodes) - reconcile this at online. Foreign access to counters + * is safe: the node is not online yet and we hold the hotplug lock. + */ + for_each_possible_cpu(cpu) { + struct per_cpu_nodestat *p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); - p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu); + for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) + if (p->vm_node_stat_diff[i]) + node_page_state_add(p->vm_node_stat_diff[i], pgdat, i); memset(p, 0, sizeof(*p)); } @@ -1601,6 +1613,8 @@ void __ref free_area_init_core_hotplug(struct pglist_data *pgdat) zone->present_pages = 0; zone_init_internals(zone, z, nid, 0); } + + return 0; } #endif @@ -2231,10 +2245,13 @@ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) } /* - * There were no pages to initialize and free which means the zone's - * memory map is completely initialized. + * The loop only tests spfn before entering an iteration, so on exit it + * may point up to a section past the end of the zone. When it does, + * the rest of the zone has already been handed to + * deferred_init_memmap_chunk() and nothing is left to initialize. */ - pgdat->first_deferred_pfn = nr_pages ? spfn : ULONG_MAX; + pgdat->first_deferred_pfn = + spfn < zone_end_pfn(zone) ? spfn : ULONG_MAX; pgdat_resize_unlock(pgdat, &flags); @@ -2347,6 +2364,7 @@ void __init page_alloc_init_late(void) /* Reinit limits that are based on free pages after the kernel is up */ files_maxfiles_init(); #endif + hugetlb_bootmem_struct_page_init(); /* Accounting of total+free memory is stable at this point. */ mem_init_print_info(); diff --git a/mm/mremap.c b/mm/mremap.c index 419a0ea0a8708..bdf63ffaad3d5 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1247,18 +1247,29 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, { unsigned long start = vrm->addr; unsigned long end = vrm->addr + vrm->old_len; - unsigned long old_start = vrm->vma->vm_start; - unsigned long old_end = vrm->vma->vm_end; + struct vm_area_struct *vma = vrm->vma; + unsigned long old_start = vma->vm_start; + unsigned long old_end = vma->vm_end; /* We always clear VM_LOCKED[ONFAULT] on the old VMA. */ - vm_flags_clear(vrm->vma, VM_LOCKED_MASK); + vm_flags_clear(vma, VM_LOCKED_MASK); /* * anon_vma links of the old vma is no longer needed after its page * table has been moved. */ - if (new_vma != vrm->vma && start == old_start && end == old_end) - unlink_anon_vmas(vrm->vma); + if (new_vma != vma && start == old_start && end == old_end) { + const pgoff_t pgoff_unfaulted = vma->vm_start >> PAGE_SHIFT; + + unlink_anon_vmas(vma); + /* + * The VMA is now unfaulted and it is an invariant that + * unfaulted anonymous VMAs have page offset equal to + * vma->vm_start >> PAGE_SHIFT. + */ + if (vma_is_anonymous(vma) && !vma->vm_file) + vma->vm_pgoff = pgoff_unfaulted; + } /* Because we won't unmap we don't need to touch locked_vm. */ } diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 3c28eff76d523..ec831137d019c 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -784,14 +784,14 @@ static inline struct capture_control *task_capc(struct zone *zone) return unlikely(capc) && !(current->flags & PF_KTHREAD) && !capc->page && - capc->cc->zone == zone ? capc : NULL; + capc->zone == zone ? capc : NULL; } static inline bool compaction_capture(struct capture_control *capc, struct page *page, int order, int migratetype) { - if (!capc || order != capc->cc->order) + if (!capc || order != capc->order) return false; /* Do not accidentally pollute CMA or isolated regions*/ @@ -807,12 +807,12 @@ compaction_capture(struct capture_control *capc, struct page *page, * have trouble finding a high-order free page. */ if (order < pageblock_order && migratetype == MIGRATE_MOVABLE && - capc->cc->migratetype != MIGRATE_MOVABLE) + capc->migratetype != MIGRATE_MOVABLE) return false; - if (migratetype != capc->cc->migratetype) - trace_mm_page_alloc_extfrag(page, capc->cc->order, order, - capc->cc->migratetype, migratetype); + if (migratetype != capc->migratetype) + trace_mm_page_alloc_extfrag(page, capc->order, order, + capc->migratetype, migratetype); capc->page = page; return true; @@ -2987,8 +2987,10 @@ static void __free_frozen_pages(struct page *page, unsigned int order, migratetype = MIGRATE_MOVABLE; } - if (unlikely((fpi_flags & FPI_TRYLOCK) && IS_ENABLED(CONFIG_PREEMPT_RT) - && (in_nmi() || in_hardirq()))) { + if (unlikely((fpi_flags & FPI_TRYLOCK) && + ((IS_ENABLED(CONFIG_PREEMPT_RT) && + (in_nmi() || in_hardirq())) || + (!IS_ENABLED(CONFIG_SMP) && in_nmi())))) { add_page_to_zone_llist(zone, page, order); return; } @@ -4154,18 +4156,67 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; unsigned int noreclaim_flag; + struct capture_control capc = { + .zone = NULL, + .migratetype = ac->migratetype, + .order = order, + .page = NULL, + }; + int compact_order = order; - if (!order) + /* + * If fallbacks are not permitted (defrag_mode), we either + * need to reclaim space in a block of matching type, or clear + * out an entire block to allow __rmqueue_claim() to convert. + * + * Reclaim by itself is primarily freeing space in movable + * blocks, since that's where the LRU pages live. So this + * works for movable requests, but not for others. + * + * For those, promote the order to help make blocks, instead + * of spinning in reclaim alone unproductively. + */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + compact_order = max(order, pageblock_order); + + if (!compact_order) return NULL; psi_memstall_enter(&pflags); delayacct_compact_start(); + fs_reclaim_acquire(gfp_mask); noreclaim_flag = memalloc_noreclaim_save(); - *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, - prio, &page); + /* + * Make sure the structs are really initialized before we expose the + * capture control, in case we are interrupted and the interrupt handler + * frees a page. + */ + barrier(); + WRITE_ONCE(current->capture_control, &capc); + + *compact_result = try_to_compact_pages(gfp_mask, compact_order, + alloc_flags, ac, prio, &capc); + + /* + * Make sure we hide capture control first before we read the captured + * page pointer, otherwise an interrupt could free and capture a page + * and we would leak it. + */ + WRITE_ONCE(current->capture_control, NULL); + page = READ_ONCE(capc.page); + + /* + * Technically, it is also possible that compaction is skipped but + * the page is still captured out of luck(IRQ came and freed the page). + * Returning COMPACT_SUCCESS in such cases helps in properly accounting + * the COMPACT[STALL|FAIL] when compaction is skipped. + */ + if (page) + *compact_result = COMPACT_SUCCESS; memalloc_noreclaim_restore(noreclaim_flag); + fs_reclaim_release(gfp_mask); psi_memstall_leave(&pflags); delayacct_compact_end(); @@ -4189,7 +4240,7 @@ __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, struct zone *zone = page_zone(page); zone->compact_blockskip_flush = false; - compaction_defer_reset(zone, order, true); + compaction_defer_reset(zone, compact_order, true); count_vm_event(COMPACTSUCCESS); return page; } @@ -4426,9 +4477,14 @@ __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, struct page *page = NULL; unsigned long pflags; bool drained = false; + int reclaim_order = order; + + /* Match the slowpath compaction promotion in __alloc_pages_direct_compact */ + if ((alloc_flags & ALLOC_NOFRAGMENT) && ac->migratetype != MIGRATE_MOVABLE) + reclaim_order = max(order, pageblock_order); psi_memstall_enter(&pflags); - *did_some_progress = __perform_reclaim(gfp_mask, order, ac); + *did_some_progress = __perform_reclaim(gfp_mask, reclaim_order, ac); if (unlikely(!(*did_some_progress))) goto out; diff --git a/mm/page_reporting.c b/mm/page_reporting.c index e4c428e61d8c1..68c791d243b1e 100644 --- a/mm/page_reporting.c +++ b/mm/page_reporting.c @@ -80,7 +80,8 @@ __page_reporting_request(struct page_reporting_dev_info *prdev) * now we are limiting this to running no more than once every * couple of seconds. */ - schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY); + queue_delayed_work(system_freezable_wq, &prdev->work, + PAGE_REPORTING_DELAY); } /* notify prdev of free page reporting request */ @@ -343,7 +344,8 @@ static void page_reporting_process(struct work_struct *work) */ state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE); if (state == PAGE_REPORTING_REQUESTED) - schedule_delayed_work(&prdev->work, PAGE_REPORTING_DELAY); + queue_delayed_work(system_freezable_wq, &prdev->work, + PAGE_REPORTING_DELAY); } static DEFINE_MUTEX(page_reporting_mutex); diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 4eeca782b8887..91c4dacc41fb8 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -150,18 +150,29 @@ void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte) if (&init_mm == mm) return; - if (pte_user_accessible_page(pte)) { + if (pte_user_accessible_page(pte) && !pte_special(pte)) { page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT); } } EXPORT_SYMBOL(__page_table_check_pte_clear); +static inline bool page_table_check_huge_zero_pmd(pmd_t pmd) +{ + unsigned long pfn = pmd_pfn(pmd); + + if (!pfn_valid(pfn)) + return false; + + return is_huge_zero_folio(page_folio(pfn_to_page(pfn))); +} + void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd) { if (&init_mm == mm) return; - if (pmd_user_accessible_page(pmd)) { + if (pmd_user_accessible_page(pmd) && + !page_table_check_huge_zero_pmd(pmd)) { page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT); } } @@ -205,17 +216,21 @@ void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte, for (i = 0; i < nr; i++) __page_table_check_pte_clear(mm, ptep_get(ptep + i)); - if (pte_user_accessible_page(pte)) + if (pte_user_accessible_page(pte) && !pte_special(pte)) page_table_check_set(pte_pfn(pte), nr, pte_write(pte)); } EXPORT_SYMBOL(__page_table_check_ptes_set); static inline void page_table_check_pmd_flags(pmd_t pmd) { - if (pmd_present(pmd) && pmd_uffd_wp(pmd)) - WARN_ON_ONCE(pmd_write(pmd)); - else if (is_swap_pmd(pmd) && pmd_swp_uffd_wp(pmd)) - WARN_ON_ONCE(swap_cached_writable(pmd_to_swp_entry(pmd))); + if (pmd_present(pmd)) { + if (pmd_uffd_wp(pmd)) + WARN_ON_ONCE(pmd_write(pmd)); + } else if (pmd_swp_uffd_wp(pmd)) { + swp_entry_t entry = pmd_to_swp_entry(pmd); + + WARN_ON_ONCE(swap_cached_writable(entry)); + } } void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd, @@ -231,7 +246,8 @@ void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd, for (i = 0; i < nr; i++) __page_table_check_pmd_clear(mm, *(pmdp + i)); - if (pmd_user_accessible_page(pmd)) + if (pmd_user_accessible_page(pmd) && + !page_table_check_huge_zero_pmd(pmd)) page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd)); } EXPORT_SYMBOL(__page_table_check_pmds_set); diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index c498a91b6706f..41cf3192a8f62 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -104,7 +104,13 @@ static bool map_pte(struct page_vma_mapped_walk *pvmw, pmd_t *pmdvalp, static bool check_pte(struct page_vma_mapped_walk *pvmw, unsigned long pte_nr) { unsigned long pfn; - pte_t ptent = ptep_get(pvmw->pte); + pte_t ptent; + + if (is_vm_hugetlb_page(pvmw->vma)) + ptent = huge_ptep_get(pvmw->vma->vm_mm, pvmw->address, + pvmw->pte); + else + ptent = ptep_get(pvmw->pte); if (pvmw->flags & PVMW_MIGRATION) { swp_entry_t entry; @@ -246,7 +252,7 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) */ pmde = pmdp_get_lockless(pvmw->pmd); - if (pmd_trans_huge(pmde) || is_pmd_migration_entry(pmde)) { + if (pmd_trans_huge(pmde) || pmd_is_migration_entry(pmde)) { pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (!pmd_present(pmde)) { diff --git a/mm/pagewalk.c b/mm/pagewalk.c index 2d19d6a9b034d..47befc414adcb 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -127,6 +127,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, pmd = pmd_offset(pud, addr); do { again: + walk->action = ACTION_SUBTREE; next = pmd_addr_end(addr, end); if (pmd_none(*pmd)) { if (has_install) @@ -139,8 +140,6 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, continue; } - walk->action = ACTION_SUBTREE; - /* * This implies that each ->pmd_entry() handler * needs to know about pmd_trans_huge() pmds @@ -197,6 +196,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, pud = pud_offset(p4d, addr); do { again: + walk->action = ACTION_SUBTREE; next = pud_addr_end(addr, end); if (pud_none(*pud)) { if (has_install) @@ -209,8 +209,6 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, continue; } - walk->action = ACTION_SUBTREE; - if (ops->pud_entry) err = ops->pud_entry(pud, addr, next, walk); if (err) @@ -678,6 +676,8 @@ int walk_kernel_page_table_range_lockless(unsigned long start, unsigned long end * will also not lock the PTEs for the pte_entry() callback. * * This is for debugging purposes ONLY. + * + * The mmap write lock must be held. */ int walk_page_range_debug(struct mm_struct *mm, unsigned long start, unsigned long end, const struct mm_walk_ops *ops, @@ -691,24 +691,28 @@ int walk_page_range_debug(struct mm_struct *mm, unsigned long start, .no_vma = true }; - /* For convenience, we allow traversal of kernel mappings. */ - if (mm == &init_mm) - return walk_kernel_page_table_range(start, end, ops, - pgd, private); - if (start >= end || !walk.mm) - return -EINVAL; - if (!check_ops_valid(ops)) - return -EINVAL; - /* - * The mmap lock protects the page walker from changes to the page - * tables during the walk. However a read lock is insufficient to - * protect those areas which don't have a VMA as munmap() detaches - * the VMAs before downgrading to a read lock and actually tearing - * down PTEs/page tables. In which case, the mmap write lock should - * be held. + * When walking userland page tables, an mmap write lock must be held to + * account for munmap() downgrading to an mmap read lock when tearing + * down page tables. + * + * When walking kernel page tables, an mmap write lock must also be held + * to account for page table freeing on vmap huge page mapping. */ mmap_assert_write_locked(mm); + /* + * x86, arm64 ptdump allow walks of efi mm's and x86 ptdump allows walks + * of arbitrary mm's. + * + * However, they both must also hold the init_mm lock to account for + * concurrent kernel page table freeing. + */ + mmap_assert_write_locked(&init_mm); + + if (start >= end) + return -EINVAL; + if (!check_ops_valid(ops)) + return -EINVAL; return walk_pgd_range(start, end, &walk); } @@ -985,7 +989,7 @@ struct folio *folio_walk_start(struct folio_walk *fw, goto found; } } else if ((flags & FW_MIGRATION) && - is_pmd_migration_entry(pmd)) { + pmd_is_migration_entry(pmd)) { swp_entry_t entry = pmd_to_swp_entry(pmd); page = pfn_swap_entry_to_page(entry); diff --git a/mm/percpu-km.c b/mm/percpu-km.c index 4efa74a495cb6..dc096b5a6ce40 100644 --- a/mm/percpu-km.c +++ b/mm/percpu-km.c @@ -75,7 +75,7 @@ static struct pcpu_chunk *pcpu_create_chunk(gfp_t gfp) chunk->base_addr = page_address(pages); spin_lock_irqsave(&pcpu_lock, flags); - pcpu_chunk_populated(chunk, 0, nr_pages); + pcpu_chunk_populated(chunk, 0, chunk->nr_pages); spin_unlock_irqrestore(&pcpu_lock, flags); pcpu_stats_chunk_alloc(); diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index 8c22be79b7343..b7f347ed786e2 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -291,7 +291,7 @@ pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) if (pmdvalp) *pmdvalp = pmdval; - if (unlikely(pmd_none(pmdval) || is_pmd_migration_entry(pmdval))) + if (unlikely(pmd_none(pmdval) || pmd_is_migration_entry(pmdval))) goto nomap; if (unlikely(pmd_trans_huge(pmdval))) goto nomap; diff --git a/mm/ptdump.c b/mm/ptdump.c index b600c7f864b8b..217ebb6edc611 100644 --- a/mm/ptdump.c +++ b/mm/ptdump.c @@ -178,11 +178,18 @@ void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd) get_online_mems(); mmap_write_lock(mm); + /* To stabilise kernel page tables we must hold the init_mm lock too. */ + if (mm != &init_mm) + mmap_write_lock_nested(&init_mm, SINGLE_DEPTH_NESTING); + while (range->start != range->end) { walk_page_range_debug(mm, range->start, range->end, &ptdump_ops, pgd, st); range++; } + + if (mm != &init_mm) + mmap_write_unlock(&init_mm); mmap_write_unlock(mm); put_online_mems(); diff --git a/mm/rmap.c b/mm/rmap.c index 4c78c71099093..29f93ecf34800 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -1983,20 +1983,26 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) { + pteval = huge_ptep_get(mm, address, pvmw.pte); + } else { + pteval = ptep_get(pvmw.pte); + } if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { - pfn = swp_offset_pfn(pte_to_swp_entry(pteval)); + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ + const softleaf_t entry = softleaf_from_pte(pteval); + + pfn = softleaf_to_pfn(entry); VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); @@ -2378,20 +2384,23 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma, /* Unexpected PMD-mapped THP? */ VM_BUG_ON_FOLIO(!pvmw.pte, folio); - /* - * Handle PFN swap PTEs, such as device-exclusive ones, that - * actually map pages. - */ - pteval = ptep_get(pvmw.pte); + address = pvmw.address; + if (folio_test_hugetlb(folio)) + pteval = huge_ptep_get(mm, address, pvmw.pte); + else + pteval = ptep_get(pvmw.pte); if (likely(pte_present(pteval))) { pfn = pte_pfn(pteval); } else { + /* + * Handle PFN swap PTEs, such as device-exclusive ones, + * that actually map pages. + */ pfn = swp_offset_pfn(pte_to_swp_entry(pteval)); VM_WARN_ON_FOLIO(folio_test_hugetlb(folio), folio); } subpage = folio_page(folio, pfn - folio_pfn(folio)); - address = pvmw.address; anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(subpage); diff --git a/mm/secretmem.c b/mm/secretmem.c index 9b0f5d9ec6f4b..932fb32e63b62 100644 --- a/mm/secretmem.c +++ b/mm/secretmem.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include @@ -47,10 +49,69 @@ bool secretmem_active(void) return !!atomic_read(&secretmem_users); } +struct secretmem_inode_state { + struct user_struct *user; + atomic_long_t nr_pages_accounted; +}; + +static bool __secretmem_account_pages(struct user_struct *user, + unsigned long nr_pages) +{ + unsigned long page_limit, cur_pages, new_pages; + + if (!nr_pages) + return true; + + page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + + cur_pages = atomic_long_read(&user->locked_vm); + do { + new_pages = cur_pages + nr_pages; + if (new_pages > page_limit) + return false; + } while (!atomic_long_try_cmpxchg(&user->locked_vm, + &cur_pages, new_pages)); + return true; +} + +static bool secretmem_account_folio(struct secretmem_inode_state *state, + const struct folio *folio) +{ + const unsigned long nr_pages = folio_nr_pages(folio); + + if (!__secretmem_account_pages(state->user, nr_pages)) + return false; + + atomic_long_add(nr_pages, &state->nr_pages_accounted); + return true; +} + +static void __secretmem_unaccount_pages(struct secretmem_inode_state *state, + unsigned long nr_pages) +{ + atomic_long_sub(nr_pages, &state->user->locked_vm); + atomic_long_sub(nr_pages, &state->nr_pages_accounted); +} + +static void secretmem_unaccount_folio(struct secretmem_inode_state *state, + struct folio *folio) +{ + __secretmem_unaccount_pages(state, folio_nr_pages(folio)); +} + +static void secretmem_unaccount_all_folios(struct secretmem_inode_state *state) +{ + const unsigned long nr_pages_accounted = + atomic_long_read(&state->nr_pages_accounted); + + __secretmem_unaccount_pages(state, nr_pages_accounted); +} + static vm_fault_t secretmem_fault(struct vm_fault *vmf) { struct address_space *mapping = vmf->vma->vm_file->f_mapping; struct inode *inode = file_inode(vmf->vma->vm_file); + struct secretmem_inode_state *state = inode->i_private; pgoff_t offset = vmf->pgoff; gfp_t gfp = vmf->gfp_mask; unsigned long addr; @@ -72,8 +133,15 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) goto out; } + if (!secretmem_account_folio(state, folio)) { + folio_put(folio); + ret = VM_FAULT_SIGBUS; + goto out; + } + err = set_direct_map_invalid_noflush(folio_page(folio, 0)); if (err) { + secretmem_unaccount_folio(state, folio); folio_put(folio); ret = vmf_error(err); goto out; @@ -82,6 +150,7 @@ static vm_fault_t secretmem_fault(struct vm_fault *vmf) __folio_mark_uptodate(folio); err = filemap_add_folio(mapping, folio, offset, gfp); if (unlikely(err)) { + secretmem_unaccount_folio(state, folio); /* * If a split of large page was required, it * already happened when we marked the page invalid @@ -112,23 +181,30 @@ static const struct vm_operations_struct secretmem_vm_ops = { .fault = secretmem_fault, }; +static void secretmem_destroy_inode_priv(struct inode *inode) +{ + struct secretmem_inode_state *state = inode->i_private; + + secretmem_unaccount_all_folios(state); + free_uid(state->user); + kfree(state); + inode->i_private = NULL; +} + static int secretmem_release(struct inode *inode, struct file *file) { atomic_dec(&secretmem_users); + secretmem_destroy_inode_priv(inode); + return 0; } static int secretmem_mmap_prepare(struct vm_area_desc *desc) { - const unsigned long len = desc->end - desc->start; - if ((desc->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) return -EINVAL; - if (!mlock_future_ok(desc->mm, desc->vm_flags | VM_LOCKED, len)) - return -EAGAIN; - - desc->vm_flags |= VM_LOCKED | VM_DONTDUMP; + desc->vm_flags |= VM_DONTDUMP; desc->vm_ops = &secretmem_vm_ops; return 0; @@ -188,20 +264,40 @@ static const struct inode_operations secretmem_iops = { static struct vfsmount *secretmem_mnt; +static int secretmem_init_inode_priv(struct inode *inode) +{ + struct secretmem_inode_state *state; + + state = kzalloc_obj(*state); + if (!state) + return -ENOMEM; + + state->user = get_uid(current_user()); + inode->i_private = state; + return 0; +} + static struct file *secretmem_file_create(unsigned long flags) { struct file *file; struct inode *inode; const char *anon_name = "[secretmem]"; + int err; inode = anon_inode_make_secure_inode(secretmem_mnt->mnt_sb, anon_name, NULL); if (IS_ERR(inode)) return ERR_CAST(inode); + err = secretmem_init_inode_priv(inode); + if (err) + goto err_free_inode; + file = alloc_file_pseudo(inode, secretmem_mnt, "secretmem", O_RDWR | O_LARGEFILE, &secretmem_fops); - if (IS_ERR(file)) - goto err_free_inode; + if (IS_ERR(file)) { + err = PTR_ERR(file); + goto err_free_priv; + } mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER); mapping_set_unevictable(inode->i_mapping); @@ -216,10 +312,11 @@ static struct file *secretmem_file_create(unsigned long flags) atomic_inc(&secretmem_users); return file; - +err_free_priv: + secretmem_destroy_inode_priv(inode); err_free_inode: iput(inode); - return file; + return ERR_PTR(err); } SYSCALL_DEFINE1(memfd_secret, unsigned int, flags) diff --git a/mm/slab.h b/mm/slab.h index 36893299fa67c..0b49960bf07c9 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -299,9 +299,12 @@ struct kmem_cache { #define SLAB_SUPPORTS_SYSFS 1 void sysfs_slab_unlink(struct kmem_cache *s); void sysfs_slab_release(struct kmem_cache *s); +int sysfs_slab_alias(struct kmem_cache *s, const char *name); #else static inline void sysfs_slab_unlink(struct kmem_cache *s) { } static inline void sysfs_slab_release(struct kmem_cache *s) { } +static inline int sysfs_slab_alias(struct kmem_cache *s, const char *name) + { return 0; } #endif void *fixup_red_left(struct kmem_cache *s, void *p); @@ -394,9 +397,13 @@ static inline struct kmem_cache * kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, unsigned long caller) { unsigned int index; + enum kmalloc_cache_type type = kmalloc_type(flags, caller); + + if (flags & __GFP_NO_OBJ_EXT) + type = KMALLOC_NO_OBJ_EXT; if (!b) - b = &kmalloc_caches[kmalloc_type(flags, caller)]; + b = &kmalloc_caches[type]; if (size <= 192) index = kmalloc_size_index[size_index_elem(size)]; else @@ -418,11 +425,6 @@ extern void create_boot_cache(struct kmem_cache *, const char *name, unsigned int useroffset, unsigned int usersize); int slab_unmergeable(struct kmem_cache *s); -struct kmem_cache *find_mergeable(unsigned size, unsigned align, - slab_flags_t flags, const char *name, void (*ctor)(void *)); -struct kmem_cache * -__kmem_cache_alias(const char *name, unsigned int size, unsigned int align, - slab_flags_t flags, void (*ctor)(void *)); slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name); @@ -435,7 +437,8 @@ static inline bool is_kmalloc_normal(struct kmem_cache *s) { if (!is_kmalloc_cache(s)) return false; - return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT)); + + return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT|SLAB_NO_OBJ_EXT)); } bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj); @@ -520,6 +523,25 @@ bool slab_in_kunit_test(void); static inline bool slab_in_kunit_test(void) { return false; } #endif +/* + * Return true if KMALLOC_NORMAL caches may need obj_exts arrays. + * + * Memory allocation profiling requires obj_exts for all caches. + * Memcg usually doesn't need them for normal kmalloc caches, but kmalloc types + * with a priority higher than KMALLOC_CGROUP can be aliased with KMALLOC_NORMAL. + */ +static inline bool need_kmalloc_no_objext(void) +{ + if (!mem_alloc_profiling_permanently_disabled()) + return true; + + if (!mem_cgroup_kmem_disabled() && + (KMALLOC_NORMAL == KMALLOC_RECLAIM)) + return true; + + return false; +} + #ifdef CONFIG_SLAB_OBJ_EXT /* diff --git a/mm/slab_common.c b/mm/slab_common.c index 04583044a2bf0..edd49796be4c9 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -175,15 +175,22 @@ int slab_unmergeable(struct kmem_cache *s) return 0; } -struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, - slab_flags_t flags, const char *name, void (*ctor)(void *)) +static struct kmem_cache *find_mergeable(unsigned int size, slab_flags_t flags, + const char *name, struct kmem_cache_args *args) { struct kmem_cache *s; + unsigned int align; if (slab_nomerge) return NULL; - if (ctor) + if (args->ctor) + return NULL; + + if (IS_ENABLED(CONFIG_HARDENED_USERCOPY) && args->usersize) + return NULL; + + if (args->sheaf_capacity) return NULL; flags = kmem_cache_flags(flags, name); @@ -192,7 +199,7 @@ struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, return NULL; size = ALIGN(size, sizeof(void *)); - align = calculate_alignment(flags, align, size); + align = calculate_alignment(flags, args->align, size); size = ALIGN(size, align); list_for_each_entry_reverse(s, &slab_caches, list) { @@ -253,6 +260,31 @@ static struct kmem_cache *create_cache(const char *name, return ERR_PTR(err); } +static struct kmem_cache * +__kmem_cache_alias(const char *name, unsigned int size, slab_flags_t flags, + struct kmem_cache_args *args) +{ + struct kmem_cache *s; + + s = find_mergeable(size, flags, name, args); + if (s) { + if (sysfs_slab_alias(s, name)) + pr_err("SLUB: Unable to add cache alias %s to sysfs\n", + name); + + s->refcount++; + + /* + * Adjust the object sizes so that we clear + * the complete object on kzalloc. + */ + s->object_size = max(s->object_size, size); + s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); + } + + return s; +} + /** * __kmem_cache_create_args - Create a kmem cache. * @name: A string which is used in /proc/slabinfo to identify this cache. @@ -324,9 +356,7 @@ struct kmem_cache *__kmem_cache_create_args(const char *name, object_size - args->usersize < args->useroffset)) args->usersize = args->useroffset = 0; - if (!args->usersize && !args->sheaf_capacity) - s = __kmem_cache_alias(name, object_size, args->align, flags, - args->ctor); + s = __kmem_cache_alias(name, object_size, flags, args); if (s) goto out_unlock; @@ -803,6 +833,12 @@ EXPORT_SYMBOL(kmalloc_size_roundup); #define KMALLOC_RANDOM_NAME(N, sz) #endif +#ifdef CONFIG_SLAB_OBJ_EXT +#define KMALLOC_NO_OBJ_EXT_NAME(sz) .name[KMALLOC_NO_OBJ_EXT] = "kmalloc-no-objext-" #sz, +#else +#define KMALLOC_NO_OBJ_EXT_NAME(sz) +#endif + #define INIT_KMALLOC_INFO(__size, __short_size) \ { \ .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \ @@ -810,6 +846,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup); KMALLOC_CGROUP_NAME(__short_size) \ KMALLOC_DMA_NAME(__short_size) \ KMALLOC_RANDOM_NAME(RANDOM_KMALLOC_CACHES_NR, __short_size) \ + KMALLOC_NO_OBJ_EXT_NAME(__short_size) \ .size = __size, \ } @@ -917,6 +954,12 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type) return; } flags |= SLAB_ACCOUNT; + } else if (IS_ENABLED(CONFIG_SLAB_OBJ_EXT) && type == KMALLOC_NO_OBJ_EXT) { + if (!need_kmalloc_no_objext()) { + kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx]; + return; + } + flags |= SLAB_NO_OBJ_EXT | SLAB_NO_MERGE; } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) { flags |= SLAB_CACHE_DMA; } diff --git a/mm/slub.c b/mm/slub.c index 71e09c675047d..8fdbe531a0cba 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -351,17 +351,20 @@ enum track_item { TRACK_ALLOC, TRACK_FREE }; #ifdef SLAB_SUPPORTS_SYSFS static int sysfs_slab_add(struct kmem_cache *); -static int sysfs_slab_alias(struct kmem_cache *, const char *); +static int __init slab_kset_init(void); +static void __init slab_sysfs_process_aliases(void); #else static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } -static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) - { return 0; } +static inline int slab_kset_init(void) { return 0; } +static inline void slab_sysfs_process_aliases(void) { } #endif #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG) static void debugfs_slab_add(struct kmem_cache *); +static void __init slab_debugfs_root_init(void); #else static inline void debugfs_slab_add(struct kmem_cache *s) { } +static inline void slab_debugfs_root_init(void) { } #endif enum stat_item { @@ -2096,42 +2099,6 @@ static inline void init_slab_obj_exts(struct slab *slab) slab->obj_exts = 0; } -/* - * Calculate the allocation size for slabobj_ext array. - * - * When memory allocation profiling is enabled, the obj_exts array - * could be allocated from the same slab cache it's being allocated for. - * This would prevent the slab from ever being freed because it would - * always contain at least one allocated object (its own obj_exts array). - * - * To avoid this, increase the allocation size when we detect the array - * may come from the same cache, forcing it to use a different cache. - */ -static inline size_t obj_exts_alloc_size(struct kmem_cache *s, - struct slab *slab, gfp_t gfp) -{ - size_t sz = sizeof(struct slabobj_ext) * slab->objects; - struct kmem_cache *obj_exts_cache; - - if (sz > KMALLOC_MAX_CACHE_SIZE) - return sz; - - if (!is_kmalloc_normal(s)) - return sz; - - obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0); - /* - * We can't simply compare s with obj_exts_cache, because random kmalloc - * caches have multiple caches per size, selected by caller address. - * Since caller address may differ between kmalloc_slab() and actual - * allocation, bump size when sizes are equal. - */ - if (s->object_size == obj_exts_cache->object_size) - return obj_exts_cache->object_size + 1; - - return sz; -} - int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, gfp_t gfp, bool new_slab) { @@ -2140,13 +2107,17 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, unsigned long new_exts; unsigned long old_exts; struct slabobj_ext *vec; - size_t sz; + size_t sz = sizeof(struct slabobj_ext) * slab->objects; gfp &= ~OBJCGS_CLEAR_MASK; - /* Prevent recursive extension vector allocation */ - gfp |= __GFP_NO_OBJ_EXT; - sz = obj_exts_alloc_size(s, slab, gfp); + /* + * In most cases, obj_exts arrays are allocated from normal kmalloc. + * However, normal kmalloc caches must allocate them from + * KMALLOC_NO_OBJ_EXT caches to prevent recursion. + */ + if (is_kmalloc_normal(s)) + gfp |= __GFP_NO_OBJ_EXT; /* * Note that allow_spin may be false during early boot and its @@ -2155,7 +2126,7 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, * very early allocations on those. */ if (unlikely(!allow_spin)) - vec = kmalloc_nolock(sz, __GFP_ZERO | __GFP_NO_OBJ_EXT, + vec = kmalloc_nolock(sz, __GFP_ZERO | (gfp & __GFP_NO_OBJ_EXT), slab_nid(slab)); else vec = kmalloc_node(sz, gfp | __GFP_ZERO, slab_nid(slab)); @@ -2173,8 +2144,21 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, return -ENOMEM; } - VM_WARN_ON_ONCE(virt_to_slab(vec) != NULL && - virt_to_slab(vec)->slab_cache == s); + if (IS_ENABLED(CONFIG_DEBUG_VM)) { + struct kmem_cache *exts_cache; + struct slab *exts_slab; + + exts_slab = virt_to_slab(vec); + if (exts_slab) { + /* + * The vector must be allocated from either normal or + * KMALLOC_NO_OBJ_EXT kmalloc caches to avoid cycles. + */ + exts_cache = exts_slab->slab_cache; + WARN_ON_ONCE(!is_kmalloc_normal(exts_cache) && + !(exts_cache->flags & SLAB_NO_OBJ_EXT)); + } + } new_exts = (unsigned long)vec; if (unlikely(!allow_spin)) @@ -2198,7 +2182,6 @@ int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s, * assign slabobj_exts in parallel. In this case the existing * objcg vector should be reused. */ - mark_objexts_empty(vec); if (unlikely(!allow_spin)) kfree_nolock(vec); else @@ -2229,14 +2212,6 @@ static inline void free_slab_obj_exts(struct slab *slab) return; } - /* - * obj_exts was created with __GFP_NO_OBJ_EXT flag, therefore its - * corresponding extension will be NULL. alloc_tag_sub() will throw a - * warning if slab has extensions but the extension of an object is - * NULL, therefore replace NULL with CODETAG_EMPTY to indicate that - * the extension for obj_exts is expected to be NULL. - */ - mark_objexts_empty(obj_exts); if (unlikely(READ_ONCE(slab->obj_exts) & OBJEXTS_NOSPIN_ALLOC)) kfree_nolock(obj_exts); else @@ -8561,31 +8536,6 @@ void __init kmem_cache_init_late(void) #endif } -struct kmem_cache * -__kmem_cache_alias(const char *name, unsigned int size, unsigned int align, - slab_flags_t flags, void (*ctor)(void *)) -{ - struct kmem_cache *s; - - s = find_mergeable(size, align, flags, name, ctor); - if (s) { - if (sysfs_slab_alias(s, name)) - pr_err("SLUB: Unable to add cache alias %s to sysfs\n", - name); - - s->refcount++; - - /* - * Adjust the object sizes so that we clear - * the complete object on kzalloc. - */ - s->object_size = max(s->object_size, size); - s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); - } - - return s; -} - int do_kmem_cache_create(struct kmem_cache *s, const char *name, unsigned int size, struct kmem_cache_args *args, slab_flags_t flags) @@ -9818,7 +9768,7 @@ struct saved_alias { static struct saved_alias *alias_list; -static int sysfs_slab_alias(struct kmem_cache *s, const char *name) +int sysfs_slab_alias(struct kmem_cache *s, const char *name) { struct saved_alias *al; @@ -9847,28 +9797,20 @@ static int sysfs_slab_alias(struct kmem_cache *s, const char *name) return 0; } -static int __init slab_sysfs_init(void) +static int __init slab_kset_init(void) { - struct kmem_cache *s; - int err; - - mutex_lock(&slab_mutex); - slab_kset = kset_create_and_add("slab", NULL, kernel_kobj); if (!slab_kset) { - mutex_unlock(&slab_mutex); pr_err("Cannot register slab subsystem.\n"); return -ENOMEM; } - slab_state = FULL; + return 0; +} - list_for_each_entry(s, &slab_caches, list) { - err = sysfs_slab_add(s); - if (err) - pr_err("SLUB: Unable to add boot slab %s to sysfs\n", - s->name); - } +static void __init slab_sysfs_process_aliases(void) +{ + int err; while (alias_list) { struct saved_alias *al = alias_list; @@ -9880,12 +9822,41 @@ static int __init slab_sysfs_init(void) al->name); kfree(al); } +} +#endif /* SLAB_SUPPORTS_SYSFS */ +#if defined(SLAB_SUPPORTS_SYSFS) || \ + (defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)) +static int __init slab_late_init(void) +{ + struct kmem_cache *s; + int err; + + mutex_lock(&slab_mutex); + + err = slab_kset_init(); + if (err) + goto out; + + slab_debugfs_root_init(); + slab_state = FULL; + + list_for_each_entry(s, &slab_caches, list) { + if (sysfs_slab_add(s)) + pr_err("SLUB: Unable to add boot slab %s to sysfs\n", + s->name); + + if (s->flags & SLAB_STORE_USER) + debugfs_slab_add(s); + } + + slab_sysfs_process_aliases(); +out: mutex_unlock(&slab_mutex); - return 0; + return err; } -late_initcall(slab_sysfs_init); -#endif /* SLAB_SUPPORTS_SYSFS */ +late_initcall(slab_late_init); +#endif #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS) static int slab_debugfs_show(struct seq_file *seq, void *v) @@ -10077,23 +10048,16 @@ static void debugfs_slab_add(struct kmem_cache *s) void debugfs_slab_release(struct kmem_cache *s) { + if (unlikely(!slab_debugfs_root)) + return; + debugfs_lookup_and_remove(s->name, slab_debugfs_root); } -static int __init slab_debugfs_init(void) +static void __init slab_debugfs_root_init(void) { - struct kmem_cache *s; - slab_debugfs_root = debugfs_create_dir("slab", NULL); - - list_for_each_entry(s, &slab_caches, list) - if (s->flags & SLAB_STORE_USER) - debugfs_slab_add(s); - - return 0; - } -__initcall(slab_debugfs_init); #endif /* * The /proc/slabinfo ABI diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 37522d6cb3988..032a81450838b 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -302,59 +302,6 @@ int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end, return vmemmap_populate_range(start, end, node, altmap, -1, 0); } -/* - * Undo populate_hvo, and replace it with a normal base page mapping. - * Used in memory init in case a HVO mapping needs to be undone. - * - * This can happen when it is discovered that a memblock allocated - * hugetlb page spans multiple zones, which can only be verified - * after zones have been initialized. - * - * We know that: - * 1) The first @headsize / PAGE_SIZE vmemmap pages were individually - * allocated through memblock, and mapped. - * - * 2) The rest of the vmemmap pages are mirrors of the last head page. - */ -int __meminit vmemmap_undo_hvo(unsigned long addr, unsigned long end, - int node, unsigned long headsize) -{ - unsigned long maddr, pfn; - pte_t *pte; - int headpages; - - /* - * Should only be called early in boot, so nothing will - * be accessing these page structures. - */ - WARN_ON(!early_boot_irqs_disabled); - - headpages = headsize >> PAGE_SHIFT; - - /* - * Clear mirrored mappings for tail page structs. - */ - for (maddr = addr + headsize; maddr < end; maddr += PAGE_SIZE) { - pte = virt_to_kpte(maddr); - pte_clear(&init_mm, maddr, pte); - } - - /* - * Clear and free mappings for head page and first tail page - * structs. - */ - for (maddr = addr; headpages-- > 0; maddr += PAGE_SIZE) { - pte = virt_to_kpte(maddr); - pfn = pte_pfn(ptep_get(pte)); - pte_clear(&init_mm, maddr, pte); - memblock_phys_free(PFN_PHYS(pfn), PAGE_SIZE); - } - - flush_tlb_kernel_range(addr, end); - - return vmemmap_populate(addr, end, node, NULL); -} - /* * Write protect the mirrored tail page structs for HVO. This will be * called from the hugetlb code when gathering and initializing the diff --git a/mm/sparse.c b/mm/sparse.c index 17c50a6415c2d..bda92a084681a 100644 --- a/mm/sparse.c +++ b/mm/sparse.c @@ -670,7 +670,12 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); + struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap, + pgmap); + + memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); + + return page; } static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, @@ -679,13 +684,17 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long end = start + nr_pages * sizeof(struct page); + memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); vmemmap_free(start, end, altmap); } + static void free_map_bootmem(struct page *memmap) { unsigned long start = (unsigned long)memmap; unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION); + memmap_boot_pages_add(-1L * (DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), + PAGE_SIZE))); vmemmap_free(start, end, NULL); } @@ -742,13 +751,18 @@ static struct page * __meminit populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap) { - return kvmalloc_node(array_size(sizeof(struct page), - PAGES_PER_SECTION), GFP_KERNEL, nid); + struct page *page = kvmalloc_node(array_size(sizeof(struct page), + PAGES_PER_SECTION), GFP_KERNEL, nid); + + memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); + + return page; } static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap) { + memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); kvfree(pfn_to_page(pfn)); } @@ -761,6 +775,9 @@ static void free_map_bootmem(struct page *memmap) nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page)) >> PAGE_SHIFT; + memmap_boot_pages_add(-1L * (DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page), + PAGE_SIZE))); + for (i = 0; i < nr_pages; i++, page++) { type = bootmem_type(page); @@ -854,14 +871,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages, * The memmap of early sections is always fully populated. See * section_activate() and pfn_valid() . */ - if (!section_is_early) { - memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE))); + if (!section_is_early) depopulate_section_memmap(pfn, nr_pages, altmap); - } else if (memmap) { - memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), - PAGE_SIZE))); + else if (memmap) free_map_bootmem(memmap); - } if (empty) ms->section_mem_map = (unsigned long)NULL; @@ -906,7 +919,6 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn, section_deactivate(pfn, nr_pages, altmap); return ERR_PTR(-ENOMEM); } - memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)); return memmap; } diff --git a/mm/swapfile.c b/mm/swapfile.c index 28e522e72c3cb..cf33c3b2c31fb 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -1613,11 +1613,11 @@ struct swap_info_struct *get_swap_device(swp_entry_t entry) return si; bad_nofile: - pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_file, entry.val); out: return NULL; put_out: - pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val); + pr_err_ratelimited("%s: %s%08lx\n", __func__, Bad_offset, entry.val); percpu_ref_put(&si->users); return NULL; } @@ -3519,6 +3519,13 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags) error = -EBUSY; goto bad_swap_unlock_inode; } + if (IS_ENCRYPTED(inode)) { + pr_warn_once( + "Filesystem-level encrypted swapfile '%s' is unsupported. Create a loop device over it, or use dm-crypt\n", + name->name); + error = -EINVAL; + goto bad_swap_unlock_inode; + } /* * The swap subsystem needs a major overhaul to support this. @@ -3746,7 +3753,7 @@ static int __swap_duplicate(swp_entry_t entry, unsigned char usage, int nr) si = swap_entry_to_info(entry); if (WARN_ON_ONCE(!si)) { - pr_err("%s%08lx\n", Bad_file, entry.val); + pr_err_ratelimited("%s%08lx\n", Bad_file, entry.val); return -EINVAL; } diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 1b0fa239aa752..34798ce783e6c 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1956,8 +1956,10 @@ ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, } if (err) { - if (err == -EAGAIN) + if (err == -EAGAIN) { + err = 0; continue; + } break; } diff --git a/mm/util.c b/mm/util.c index 8989d57675286..31a59c038ba0a 100644 --- a/mm/util.c +++ b/mm/util.c @@ -1263,7 +1263,7 @@ void snapshot_page(struct page_snapshot *ps, const struct page *page) if (ps->idx < MAX_FOLIO_NR_PAGES) { memcpy(&ps->folio_snapshot, foliop, 2 * sizeof(struct page)); nr_pages = folio_nr_pages(&ps->folio_snapshot); - if (nr_pages > 1) + if (nr_pages > 2) memcpy(&ps->folio_snapshot.__page_2, &foliop->__page_2, sizeof(struct page)); set_ps_flags(ps, foliop, page); diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 021fc25268866..60025b682bc06 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -43,6 +43,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -155,10 +156,21 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0; - if (pmd_present(*pmd) && !pmd_free_pte_page(pmd, addr)) - return 0; + if (!pmd_present(*pmd)) + return pmd_set_huge(pmd, phys_addr, prot); - return pmd_set_huge(pmd, phys_addr, prot); + /* + * Acquire the mmap read lock to exclude ptdump, which walks + * kernel page tables it does not own under the mmap write lock. + * + * Concurrent read lock holders are safe: each exclusively owns + * the range it operates on and cannot reach this page table. + */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!pmd_free_pte_page(pmd, addr)) + return 0; + return pmd_set_huge(pmd, phys_addr, prot); + } } static int vmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, @@ -205,10 +217,15 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0; - if (pud_present(*pud) && !pud_free_pmd_page(pud, addr)) - return 0; + if (!pud_present(*pud)) + return pud_set_huge(pud, phys_addr, prot); - return pud_set_huge(pud, phys_addr, prot); + /* See comment in vmap_try_huge_pmd(). */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!pud_free_pmd_page(pud, addr)) + return 0; + return pud_set_huge(pud, phys_addr, prot); + } } static int vmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, @@ -256,10 +273,15 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; - if (p4d_present(*p4d) && !p4d_free_pud_page(p4d, addr)) - return 0; + if (!p4d_present(*p4d)) + return p4d_set_huge(p4d, phys_addr, prot); - return p4d_set_huge(p4d, phys_addr, prot); + /* See comment in vmap_try_huge_pmd(). */ + scoped_cond_guard(mmap_read_lock_try, return 0, &init_mm) { + if (!p4d_free_pud_page(p4d, addr)) + return 0; + return p4d_set_huge(p4d, phys_addr, prot); + } } static int vmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, diff --git a/mm/vmscan.c b/mm/vmscan.c index 51c6b3b7a0d7f..a941ff7444035 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -201,6 +201,13 @@ struct scan_control { */ int vm_swappiness = 60; +static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) +{ + if (sc->proactive && sc->proactive_swappiness) + return *sc->proactive_swappiness; + return mem_cgroup_swappiness(memcg); +} + #ifdef CONFIG_MEMCG /* Returns true for reclaim through cgroup limits or cgroup interfaces. */ @@ -241,13 +248,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) #endif return false; } - -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) -{ - if (sc->proactive && sc->proactive_swappiness) - return *sc->proactive_swappiness; - return mem_cgroup_swappiness(memcg); -} #else static bool cgroup_reclaim(struct scan_control *sc) { @@ -263,11 +263,6 @@ static bool writeback_throttling_sane(struct scan_control *sc) { return true; } - -static int sc_swappiness(struct scan_control *sc, struct mem_cgroup *memcg) -{ - return READ_ONCE(vm_swappiness); -} #endif /* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to @@ -1317,7 +1312,8 @@ static unsigned int shrink_folio_list(struct list_head *folio_list, goto keep_locked; if (folio_test_large(folio)) { /* cannot split folio, skip it */ - if (!can_split_folio(folio, 1, NULL)) + if (folio_expected_ref_count(folio) != + folio_ref_count(folio) - 1) goto activate_locked; /* * Split partially mapped folios right away. @@ -4473,8 +4469,6 @@ void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc, int tier_idx) { - bool success; - bool dirty, writeback; int gen = folio_lru_gen(folio); int type = folio_is_file_lru(folio); int zone = folio_zonenum(folio); @@ -4486,15 +4480,9 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c VM_WARN_ON_ONCE_FOLIO(gen >= MAX_NR_GENS, folio); - /* unevictable */ - if (!folio_evictable(folio)) { - success = lru_gen_del_folio(lruvec, folio, true); - VM_WARN_ON_ONCE_FOLIO(!success, folio); - folio_set_unevictable(folio); - lruvec_add_folio(lruvec, folio); - __count_vm_events(UNEVICTABLE_PGCULLED, delta); - return true; - } + /* unevictable: let it through and the generic path will cull it */ + if (!folio_evictable(folio)) + return false; /* promoted */ if (gen != lru_gen_from_seq(lrugen->min_seq[type])) { @@ -4524,21 +4512,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c return true; } - dirty = folio_test_dirty(folio); - writeback = folio_test_writeback(folio); - if (type == LRU_GEN_FILE && dirty) { - sc->nr.file_taken += delta; - if (!writeback) - sc->nr.unqueued_dirty += delta; - } - - /* waiting for writeback */ - if (writeback || (type == LRU_GEN_FILE && dirty)) { - gen = folio_inc_gen(lruvec, folio, true); - list_move(&folio->lru, &lrugen->folios[gen][type][zone]); - return true; - } - return false; } @@ -4566,9 +4539,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca if (!folio_test_referenced(folio)) set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0); - /* for shrink_folio_list() */ - folio_clear_reclaim(folio); - success = lru_gen_del_folio(lruvec, folio, true); VM_WARN_ON_ONCE_FOLIO(!success, folio); @@ -4762,11 +4732,9 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec, list_for_each_entry_safe_reverse(folio, next, &list, lru) { DEFINE_MIN_SEQ(lruvec); - if (!folio_evictable(folio)) { - list_del(&folio->lru); - folio_putback_lru(folio); + /* move_folios_to_lru() culls unevictable folios via folio_putback_lru() */ + if (!folio_evictable(folio)) continue; - } /* retry folios that may have missed folio_rotate_reclaimable() */ if (!skip_retry && !folio_test_active(folio) && !folio_mapped(folio) && @@ -5864,7 +5832,7 @@ static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc) } } - cond_resched(); + cond_resched_tasks_rcu_qs(); if (nr_reclaimed < nr_to_reclaim || proportional_reclaim) continue; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index bb1f1e124dc07..3564b37f1fce7 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -508,6 +508,11 @@ static int get_size_class_index(int size) return min_t(int, ZS_SIZE_CLASSES - 1, idx); } +static struct size_class *lookup_size_class(struct zs_pool *pool, size_t size) +{ + return pool->size_class[get_size_class_index(size + ZS_HANDLE_SIZE)]; +} + static inline void class_stat_add(struct size_class *class, int type, unsigned long cnt) { @@ -1052,7 +1057,7 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size) { struct size_class *class; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); return class->index; } @@ -1279,9 +1284,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, if (!handle) return (unsigned long)ERR_PTR(-ENOMEM); - /* extra space in chunk to keep the handle */ - size += ZS_HANDLE_SIZE; - class = pool->size_class[get_size_class_index(size)]; + class = lookup_size_class(pool, size); /* class->lock effectively protects the zpage migration */ spin_lock(&class->lock); diff --git a/mm/zswap.c b/mm/zswap.c index 12fb0e3954129..18249dd96ef86 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -1375,11 +1375,12 @@ static void shrink_worker(struct work_struct *w) } while (memcg && !mem_cgroup_tryget_online(memcg)); spin_unlock(&zswap_shrink_lock); - if (!memcg) { - /* - * Continue shrinking without incrementing failures if - * we found candidate memcgs in the last tree walk. - */ + /* + * A NULL memcg ends a full hierarchy pass (except when memcg is + * disabled, where it is always NULL: fall through to the root LRU). + * Count a failure only if the last pass found no candidates. + */ + if (!memcg && !mem_cgroup_disabled()) { if (!attempts && ++failures == MAX_RECLAIM_RETRIES) break; @@ -1398,7 +1399,7 @@ static void shrink_worker(struct work_struct *w) * and failures. */ if (ret == -ENOENT) - continue; + goto resched; ++attempts; if (ret && ++failures == MAX_RECLAIM_RETRIES) diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 2b74ed56eb166..306e588c1a694 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -323,10 +323,9 @@ static void vlan_transfer_features(struct net_device *dev, netif_inherit_tso_max(vlandev, dev); - if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) - vlandev->hard_header_len = dev->hard_header_len; - else - vlandev->hard_header_len = dev->hard_header_len + VLAN_HLEN; + vlandev->needed_headroom = dev->needed_headroom + VLAN_HLEN; + vlandev->needed_tailroom = dev->needed_tailroom; + vlandev->hard_header_len = dev->hard_header_len; #if IS_ENABLED(CONFIG_FCOE) vlandev->fcoe_ddp_xid = dev->fcoe_ddp_xid; diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index fbf296137b094..560521cf596e3 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -493,26 +493,6 @@ static const struct header_ops vlan_header_ops = { .parse_protocol = vlan_parse_protocol, }; -static int vlan_passthru_hard_header(struct sk_buff *skb, struct net_device *dev, - unsigned short type, - const void *daddr, const void *saddr, - unsigned int len) -{ - struct vlan_dev_priv *vlan = vlan_dev_priv(dev); - struct net_device *real_dev = vlan->real_dev; - - if (saddr == NULL) - saddr = dev->dev_addr; - - return dev_hard_header(skb, real_dev, type, daddr, saddr, len); -} - -static const struct header_ops vlan_passthru_header_ops = { - .create = vlan_passthru_hard_header, - .parse = eth_header_parse, - .parse_protocol = vlan_parse_protocol, -}; - static const struct device_type vlan_type = { .name = "vlan", }; @@ -571,14 +551,10 @@ static int vlan_dev_init(struct net_device *dev) dev->fcoe_ddp_xid = real_dev->fcoe_ddp_xid; #endif - dev->needed_headroom = real_dev->needed_headroom; - if (vlan_hw_offload_capable(real_dev->features, vlan->vlan_proto)) { - dev->header_ops = &vlan_passthru_header_ops; - dev->hard_header_len = real_dev->hard_header_len; - } else { - dev->header_ops = &vlan_header_ops; - dev->hard_header_len = real_dev->hard_header_len + VLAN_HLEN; - } + dev->needed_headroom = real_dev->needed_headroom + VLAN_HLEN; + dev->needed_tailroom = real_dev->needed_tailroom; + dev->header_ops = &vlan_header_ops; + dev->hard_header_len = real_dev->hard_header_len; dev->netdev_ops = &vlan_netdev_ops; diff --git a/net/9p/client.c b/net/9p/client.c index 08ae4c44d7305..75aac636b20e7 100644 --- a/net/9p/client.c +++ b/net/9p/client.c @@ -890,7 +890,7 @@ static void p9_fid_destroy(struct p9_fid *fid) spin_lock_irqsave(&clnt->lock, flags); idr_remove(&clnt->fids, fid->fid); spin_unlock_irqrestore(&clnt->lock, flags); - kfree(fid->rdir); + kvfree(fid->rdir); kfree(fid); } diff --git a/net/appletalk/aarp.c b/net/appletalk/aarp.c index 4744e3fd45447..4713c536eeca2 100644 --- a/net/appletalk/aarp.c +++ b/net/appletalk/aarp.c @@ -542,6 +542,11 @@ int aarp_send_ddp(struct net_device *dev, struct sk_buff *skb, struct ddpehdr *ddp = (struct ddpehdr *)skb->data; int ft = 2; + if (!at) { + kfree_skb(skb); + return NET_XMIT_DROP; + } + /* * Compressible ? * diff --git a/net/atm/common.c b/net/atm/common.c index f4c6e0131c921..f4be6cb220861 100644 --- a/net/atm/common.c +++ b/net/atm/common.c @@ -760,7 +760,7 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { struct atm_vcc *vcc; - unsigned long value; + int value; int error; if (__SO_LEVEL_MATCH(optname, level) && optlen != __SO_SIZE(optname)) @@ -772,8 +772,10 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, { struct atm_qos qos; - if (copy_from_sockptr(&qos, optval, sizeof(qos))) - return -EFAULT; + error = copy_safe_from_sockptr(&qos, sizeof(qos), optval, + optlen); + if (error) + return error; error = check_qos(&qos); if (error) return error; @@ -786,8 +788,10 @@ int vcc_setsockopt(struct socket *sock, int level, int optname, return 0; } case SO_SETCLP: - if (copy_from_sockptr(&value, optval, sizeof(value))) - return -EFAULT; + error = copy_safe_from_sockptr(&value, sizeof(value), optval, + optlen); + if (error) + return error; if (value) vcc->atm_options |= ATM_ATMOPT_CLP; else diff --git a/net/batman-adv/bridge_loop_avoidance.c b/net/batman-adv/bridge_loop_avoidance.c index 2c6e2b0d1ded4..5f2bb9eba3e1d 100644 --- a/net/batman-adv/bridge_loop_avoidance.c +++ b/net/batman-adv/bridge_loop_avoidance.c @@ -324,11 +324,6 @@ batadv_bla_del_backbone_claims(struct batadv_bla_backbone_gw *backbone_gw) } spin_unlock_bh(list_lock); } - - /* all claims gone, initialize CRC */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc = BATADV_BLA_CRC_INIT; - spin_unlock_bh(&backbone_gw->crc_lock); } /** @@ -699,12 +694,14 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, struct batadv_bla_backbone_gw *old_backbone_gw; struct batadv_bla_claim *claim; struct batadv_bla_claim search_claim; - bool remove_crc = false; int hash_added; + u16 claim_crc; + bool changed; ether_addr_copy(search_claim.addr, mac); search_claim.vid = vid; claim = batadv_claim_hash_find(bat_priv, &search_claim); + claim_crc = crc16(0, mac, ETH_ALEN); /* create a new claim entry if it does not exist yet. */ if (!claim) { @@ -736,43 +733,56 @@ static void batadv_bla_add_claim(struct batadv_priv *bat_priv, kfree(claim); return; } + + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); + + WRITE_ONCE(backbone_gw->lasttime, jiffies); + + batadv_claim_put(claim); + return; + } + + WRITE_ONCE(claim->lasttime, jiffies); + + /* replace backbone_gw atomically and adjust reference counters */ + spin_lock_bh(&claim->backbone_lock); + if (claim->backbone_gw != backbone_gw) { + changed = true; + + old_backbone_gw = claim->backbone_gw; + kref_get(&backbone_gw->refcount); + claim->backbone_gw = backbone_gw; } else { - WRITE_ONCE(claim->lasttime, jiffies); - if (claim->backbone_gw == backbone_gw) - /* no need to register a new backbone */ - goto claim_free_ref; + old_backbone_gw = NULL; + changed = false; + } + spin_unlock_bh(&claim->backbone_lock); + if (changed) { batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): changing ownership for %pM, vid %d to gw %pM\n", __func__, mac, batadv_print_vid(vid), backbone_gw->orig); - remove_crc = true; - } + /* add claim address to new backbone_gw */ + spin_lock_bh(&backbone_gw->crc_lock); + backbone_gw->crc ^= claim_crc; + spin_unlock_bh(&backbone_gw->crc_lock); - /* replace backbone_gw atomically and adjust reference counters */ - spin_lock_bh(&claim->backbone_lock); - old_backbone_gw = claim->backbone_gw; - kref_get(&backbone_gw->refcount); - claim->backbone_gw = backbone_gw; - spin_unlock_bh(&claim->backbone_lock); + WRITE_ONCE(backbone_gw->lasttime, jiffies); + } - if (remove_crc) { + if (old_backbone_gw) { /* remove claim address from old backbone_gw */ spin_lock_bh(&old_backbone_gw->crc_lock); - old_backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); + old_backbone_gw->crc ^= claim_crc; spin_unlock_bh(&old_backbone_gw->crc_lock); - } - batadv_backbone_gw_put(old_backbone_gw); - - /* add claim address to new backbone_gw */ - spin_lock_bh(&backbone_gw->crc_lock); - backbone_gw->crc ^= crc16(0, claim->addr, ETH_ALEN); - spin_unlock_bh(&backbone_gw->crc_lock); - WRITE_ONCE(backbone_gw->lasttime, jiffies); + batadv_backbone_gw_put(old_backbone_gw); + } -claim_free_ref: batadv_claim_put(claim); } @@ -942,26 +952,18 @@ static bool batadv_handle_unclaim(struct batadv_priv *bat_priv, const u8 *backbone_addr, const u8 *claim_addr, unsigned short vid) { - struct batadv_bla_backbone_gw *backbone_gw; - /* unclaim in any case if it is our own */ if (primary_if && batadv_compare_eth(backbone_addr, primary_if->net_dev->dev_addr)) batadv_bla_send_claim(bat_priv, claim_addr, vid, BATADV_CLAIM_TYPE_UNCLAIM); - backbone_gw = batadv_backbone_hash_find(bat_priv, backbone_addr, vid); - - if (!backbone_gw) - return true; - /* this must be an UNCLAIM frame */ batadv_dbg(BATADV_DBG_BLA, bat_priv, "%s(): UNCLAIM %pM on vid %d (sent by %pM)...\n", __func__, - claim_addr, batadv_print_vid(vid), backbone_gw->orig); + claim_addr, batadv_print_vid(vid), backbone_addr); batadv_bla_del_claim(bat_priv, claim_addr, vid); - batadv_backbone_gw_put(backbone_gw); return true; } diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c index c9ef48b5e69eb..72717e2952ab9 100644 --- a/net/batman-adv/distributed-arp-table.c +++ b/net/batman-adv/distributed-arp-table.c @@ -249,7 +249,10 @@ static u8 *batadv_arp_hw_src(struct sk_buff *skb, int hdr_size) */ static __be32 batadv_arp_ip_src(struct sk_buff *skb, int hdr_size) { - return *(__force __be32 *)(batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN); + u8 *src = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN; + __be32 *ip = (__force __be32 *)src; + + return get_unaligned(ip); } /** @@ -274,8 +277,9 @@ static u8 *batadv_arp_hw_dst(struct sk_buff *skb, int hdr_size) static __be32 batadv_arp_ip_dst(struct sk_buff *skb, int hdr_size) { u8 *dst = batadv_arp_hw_src(skb, hdr_size) + ETH_ALEN * 2 + 4; + __be32 *ip = (__force __be32 *)dst; - return *(__force __be32 *)dst; + return get_unaligned(ip); } /** @@ -371,18 +375,19 @@ batadv_dat_entry_hash_find(struct batadv_priv *bat_priv, __be32 ip, static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, u8 *mac_addr, unsigned short vid) { + u64 u64_mac = ether_addr_to_u64(mac_addr); struct batadv_dat_entry *dat_entry; int hash_added; dat_entry = batadv_dat_entry_hash_find(bat_priv, ip, vid); /* if this entry is already known, just update it */ if (dat_entry) { - if (!batadv_compare_eth(dat_entry->mac_addr, mac_addr)) - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); + dat_entry->last_update = jiffies; batadv_dbg(BATADV_DBG_DAT, bat_priv, "Entry updated: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); goto out; } @@ -393,7 +398,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, dat_entry->ip = ip; dat_entry->vid = vid; - ether_addr_copy(dat_entry->mac_addr, mac_addr); + atomic64_set(&dat_entry->mac_addr, u64_mac); dat_entry->last_update = jiffies; kref_init(&dat_entry->refcount); @@ -409,7 +414,7 @@ static void batadv_dat_entry_add(struct batadv_priv *bat_priv, __be32 ip, } batadv_dbg(BATADV_DBG_DAT, bat_priv, "New entry added: %pI4 %pM (vid: %d)\n", - &dat_entry->ip, dat_entry->mac_addr, batadv_print_vid(vid)); + &dat_entry->ip, mac_addr, batadv_print_vid(vid)); out: batadv_dat_entry_put(dat_entry); @@ -864,6 +869,8 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, struct netlink_callback *cb, struct batadv_dat_entry *dat_entry) { + u8 mac[ETH_ALEN]; + u64 u64_mac; int msecs; void *hdr; @@ -876,11 +883,12 @@ batadv_dat_cache_dump_entry(struct sk_buff *msg, u32 portid, genl_dump_check_consistent(cb, hdr); msecs = jiffies_to_msecs(jiffies - dat_entry->last_update); + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); if (nla_put_in_addr(msg, BATADV_ATTR_DAT_CACHE_IP4ADDRESS, dat_entry->ip) || - nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, - dat_entry->mac_addr) || + nla_put(msg, BATADV_ATTR_DAT_CACHE_HWADDRESS, ETH_ALEN, mac) || nla_put_u16(msg, BATADV_ATTR_DAT_CACHE_VID, dat_entry->vid) || nla_put_u32(msg, BATADV_ATTR_LAST_SEEN_MSECS, msecs)) { genlmsg_cancel(msg, hdr); @@ -1147,6 +1155,8 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, struct net_device *mesh_iface = bat_priv->mesh_iface; int hdr_size = 0; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!atomic_read(&bat_priv->distributed_arp_table)) goto out; @@ -1174,6 +1184,9 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_dst, vid); if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + /* If the ARP request is destined for a local client the local * client will answer itself. DAT would only generate a * duplicate packet. @@ -1182,7 +1195,7 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * additional DAT answer may trigger kernel warnings about * a packet coming from the wrong port. */ - if (batadv_is_my_client(bat_priv, dat_entry->mac_addr, vid)) { + if (batadv_is_my_client(bat_priv, mac, vid)) { ret = true; goto out; } @@ -1192,18 +1205,16 @@ bool batadv_dat_snoop_outgoing_arp_request(struct batadv_priv *bat_priv, * the backbone gws belonging to our backbone has claimed the * destination. */ - if (!batadv_bla_check_claim(bat_priv, - dat_entry->mac_addr, vid)) { + if (!batadv_bla_check_claim(bat_priv, mac, vid)) { batadv_dbg(BATADV_DBG_DAT, bat_priv, "Device %pM claimed by another backbone gw. Don't send ARP reply!", - dat_entry->mac_addr); + mac); ret = true; goto out; } skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, - hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1245,6 +1256,8 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, struct batadv_dat_entry *dat_entry = NULL; bool ret = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; int err; if (!atomic_read(&bat_priv->distributed_arp_table)) @@ -1272,8 +1285,11 @@ bool batadv_dat_snoop_incoming_arp_request(struct batadv_priv *bat_priv, if (!dat_entry) goto out; + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + skb_new = batadv_dat_arp_create_reply(bat_priv, ip_dst, ip_src, - dat_entry->mac_addr, hw_src, vid); + mac, hw_src, vid); if (!skb_new) goto out; @@ -1364,6 +1380,8 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, u8 *hw_src, *hw_dst; bool dropped = false; unsigned short vid; + u8 mac[ETH_ALEN]; + u64 u64_mac; if (!atomic_read(&bat_priv->distributed_arp_table)) goto out; @@ -1392,11 +1410,17 @@ bool batadv_dat_snoop_incoming_arp_reply(struct batadv_priv *bat_priv, * this frame would lead to doubled receive of an ARP reply. */ dat_entry = batadv_dat_entry_hash_find(bat_priv, ip_src, vid); - if (dat_entry && batadv_compare_eth(hw_src, dat_entry->mac_addr)) { - batadv_dbg(BATADV_DBG_DAT, bat_priv, "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", - hw_src, &ip_src, hw_dst, &ip_dst, - dat_entry->mac_addr, &dat_entry->ip); - dropped = true; + if (dat_entry) { + u64_mac = atomic64_read(&dat_entry->mac_addr); + u64_to_ether_addr(u64_mac, mac); + + if (batadv_compare_eth(hw_src, mac)) { + batadv_dbg(BATADV_DBG_DAT, bat_priv, + "Doubled ARP reply removed: ARP MSG = [src: %pM-%pI4 dst: %pM-%pI4]; dat_entry: %pM-%pI4\n", + hw_src, &ip_src, hw_dst, &ip_dst, + mac, &dat_entry->ip); + dropped = true; + } } /* Update our internal cache with both the IP addresses the node got diff --git a/net/batman-adv/main.c b/net/batman-adv/main.c index b9b0bef448030..f1b41078835b6 100644 --- a/net/batman-adv/main.c +++ b/net/batman-adv/main.c @@ -436,6 +436,10 @@ int batadv_batman_skb_recv(struct sk_buff *skb, struct net_device *dev, if (!skb) goto err_put; + /* Merged fragments re-enter here with reused skb metadata. */ + skb->dev = dev; + skb->skb_iif = dev->ifindex; + /* packet should hold at least type and version */ if (unlikely(!pskb_may_pull(skb, 2))) goto err_free; diff --git a/net/batman-adv/mesh-interface.c b/net/batman-adv/mesh-interface.c index 0d598bf39bfc2..3d49fd7ef3e62 100644 --- a/net/batman-adv/mesh-interface.c +++ b/net/batman-adv/mesh-interface.c @@ -258,6 +258,8 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, if (batadv_compare_eth(ethhdr->h_dest, ectp_addr)) goto dropped; + batadv_skb_set_priority(skb, 0); + gw_mode = atomic_read(&bat_priv->gw.mode); if (is_multicast_ether_addr(ethhdr->h_dest)) { /* if gw mode is off, broadcast every packet */ @@ -291,6 +293,9 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, send: if (do_bcast && !is_broadcast_ether_addr(ethhdr->h_dest)) { + /* WARNING batadv_mcast_forw_mode might add more headers + * in front of the skb. and might even reallocate the skb + */ forw_mode = batadv_mcast_forw_mode(bat_priv, skb, vid, &mcast_is_routable); switch (forw_mode) { @@ -308,8 +313,6 @@ static netdev_tx_t batadv_interface_tx(struct sk_buff *skb, } } - batadv_skb_set_priority(skb, 0); - /* ethernet packet should be broadcasted */ if (do_bcast) { primary_if = batadv_primary_if_get_selected(bat_priv); diff --git a/net/batman-adv/multicast_forw.c b/net/batman-adv/multicast_forw.c index 1404a3b7adfb1..c99579dc1ca98 100644 --- a/net/batman-adv/multicast_forw.c +++ b/net/batman-adv/multicast_forw.c @@ -1100,8 +1100,11 @@ static int batadv_mcast_forw_expand_head(struct batadv_priv *bat_priv, return -EINVAL; } - if (skb_headroom(skb) < hdr_size && - pskb_expand_head(skb, hdr_size, 0, GFP_ATOMIC) < 0) + if (skb_cow(skb, hdr_size) < 0) + return -ENOMEM; + + /* batadv_mcast_forw_scrape() + batadv_mcast_forw_packet() require linearized skb */ + if (skb_linearize(skb) < 0) return -ENOMEM; return 0; diff --git a/net/batman-adv/tvlv.c b/net/batman-adv/tvlv.c index a91f1891747c0..0c405c5814d66 100644 --- a/net/batman-adv/tvlv.c +++ b/net/batman-adv/tvlv.c @@ -420,8 +420,11 @@ static int batadv_tvlv_call_handler(struct batadv_priv *bat_priv, return NET_RX_SUCCESS; tvlv_offset = (unsigned char *)tvlv_value - skb->data; + if (!skb_set_transport_header_careful(skb, + tvlv_offset + tvlv_value_len)) + return -EINVAL; + skb_set_network_header(skb, tvlv_offset); - skb_set_transport_header(skb, tvlv_offset + tvlv_value_len); return tvlv_handler->mcast_handler(bat_priv, skb); } diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h index ac4494f1b8e2a..51d04f351fdd4 100644 --- a/net/batman-adv/types.h +++ b/net/batman-adv/types.h @@ -2127,7 +2127,7 @@ struct batadv_dat_entry { __be32 ip; /** @mac_addr: the MAC address associated to the stored IPv4 */ - u8 mac_addr[ETH_ALEN]; + atomic64_t mac_addr; /** @vid: the vlan ID associated to this entry */ unsigned short vid; diff --git a/net/bluetooth/eir.c b/net/bluetooth/eir.c index 1de5f9df6eec0..a55696820b227 100644 --- a/net/bluetooth/eir.c +++ b/net/bluetooth/eir.c @@ -369,6 +369,7 @@ u8 eir_create_scan_rsp(struct hci_dev *hdev, u8 instance, u8 *ptr) void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) { + const u8 *eir_end = eir + eir_len; size_t dlen; while ((eir = eir_get_data(eir, eir_len, EIR_SERVICE_DATA, &dlen))) { @@ -381,7 +382,7 @@ void *eir_get_service_data(u8 *eir, size_t eir_len, u16 uuid, size_t *len) } eir += dlen; - eir_len -= dlen; + eir_len = eir_end - eir; } return NULL; diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index ceeb19158a36c..6edc26fe6d2d9 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -286,8 +286,6 @@ static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data) struct hci_cp_enhanced_setup_sync_conn cp; const struct sco_param *param; - kfree(conn_handle); - if (!hci_conn_valid(hdev, conn)) return -ECANCELED; @@ -456,6 +454,15 @@ static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle) return true; } +static void hci_enhanced_setup_sync_destroy(struct hci_dev *hdev, void *data, + int err) +{ + struct conn_handle_t *conn_handle = data; + + hci_conn_put(conn_handle->conn); + kfree(conn_handle); +} + bool hci_setup_sync(struct hci_conn *conn, __u16 handle) { int result; @@ -467,12 +474,15 @@ bool hci_setup_sync(struct hci_conn *conn, __u16 handle) if (!conn_handle) return false; - conn_handle->conn = conn; + conn_handle->conn = hci_conn_get(conn); conn_handle->handle = handle; result = hci_cmd_sync_queue(conn->hdev, hci_enhanced_setup_sync, - conn_handle, NULL); - if (result < 0) + conn_handle, + hci_enhanced_setup_sync_destroy); + if (result < 0) { + hci_conn_put(conn); kfree(conn_handle); + } return result == 0; } @@ -1298,7 +1308,8 @@ static void hci_le_conn_failed(struct hci_conn *conn, u8 status) /* Enable advertising in case this was a failed connection * attempt as a peripheral. */ - hci_enable_advertising(hdev); + if (conn->role == HCI_ROLE_SLAVE) + hci_enable_advertising(hdev); } /* This function requires the caller holds hdev->lock */ @@ -2965,9 +2976,17 @@ static int abort_conn_sync(struct hci_dev *hdev, void *data) return hci_abort_conn_sync(hdev, conn, conn->abort_reason); } +static void abort_conn_destroy(struct hci_dev *hdev, void *data, int err) +{ + struct hci_conn *conn = data; + + hci_conn_put(conn); +} + int hci_abort_conn(struct hci_conn *conn, u8 reason) { struct hci_dev *hdev = conn->hdev; + int err; /* If abort_reason has already been set it means the connection is * already being aborted so don't attempt to overwrite it. @@ -2989,7 +3008,11 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) * as a result to MGMT_OP_DISCONNECT/MGMT_OP_UNPAIR which does * already queue its callback on cmd_sync_work. */ - return hci_cmd_sync_run_once(hdev, abort_conn_sync, conn, NULL); + err = hci_cmd_sync_run_once(hdev, abort_conn_sync, hci_conn_get(conn), + abort_conn_destroy); + if (err) + hci_conn_put(conn); + return (err == -EEXIST) ? 0 : err; } void hci_setup_tx_timestamp(struct sk_buff *skb, size_t key_offset, diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index a1d7e7985f76a..447a7ebcdf62f 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -2668,11 +2668,11 @@ int hci_register_dev(struct hci_dev *hdev) if (error) BT_WARN("register suspend notifier failed error:%d\n", error); - queue_work(hdev->req_workqueue, &hdev->power_on); - idr_init(&hdev->adv_monitors_idr); msft_register(hdev); + queue_work(hdev->req_workqueue, &hdev->power_on); + return id; err_wqueue: @@ -4130,7 +4130,7 @@ static int hci_send_cmd_sync(struct hci_dev *hdev, struct sk_buff *skb) if (READ_ONCE(hdev->req_status) == HCI_REQ_PEND && !hci_dev_test_and_set_flag(hdev, HCI_CMD_PENDING)) { kfree_skb(hdev->req_skb); - hdev->req_skb = skb_clone(hdev->sent_cmd, GFP_KERNEL); + hdev->req_skb = skb_get(hdev->sent_cmd); } return err; diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 396aafa609c9a..031ee997bb92f 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -293,8 +293,10 @@ static u8 hci_cc_reset(struct hci_dev *hdev, void *data, struct sk_buff *skb) hdev->ssp_debug_mode = 0; + hci_dev_lock(hdev); hci_bdaddr_list_clear(&hdev->le_accept_list); hci_bdaddr_list_clear(&hdev->le_resolv_list); + hci_dev_unlock(hdev); return rp->status; } @@ -3815,8 +3817,10 @@ static u8 hci_cc_le_set_cig_params(struct hci_dev *hdev, void *data, bt_dev_dbg(hdev, "status 0x%2.2x", rp->status); cp = hci_sent_cmd_data(hdev, HCI_OP_LE_SET_CIG_PARAMS); - if (!rp->status && (!cp || rp->num_handles != cp->num_cis || - rp->cig_id != cp->cig_id)) { + if (!rp->status && + (!cp || rp->num_handles != cp->num_cis || + rp->cig_id != cp->cig_id || + skb->len < array_size(rp->num_handles, sizeof(*rp->handle)))) { bt_dev_err(hdev, "unexpected Set CIG Parameters response data"); status = HCI_ERROR_UNSPECIFIED; } @@ -5661,10 +5665,11 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, hci_dev_lock(hdev); - /* All controllers implicitly stop advertising in the event of a - * connection, so ensure that the state bit is cleared. + /* Advertising stops when a connection is created. On a failed + * connection it keeps running, so leave the state bit alone. */ - hci_dev_clear_flag(hdev, HCI_LE_ADV); + if (!status) + hci_dev_clear_flag(hdev, HCI_LE_ADV); /* Check for existing connection: * @@ -5950,6 +5955,71 @@ static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, void *data, hci_dev_unlock(hdev); } +static int hci_le_pa_term_sync(struct hci_dev *hdev, __le16 handle) +{ + struct hci_cp_le_pa_term_sync cp; + + memset(&cp, 0, sizeof(cp)); + cp.handle = handle; + + return hci_send_cmd(hdev, HCI_OP_LE_PA_TERM_SYNC, sizeof(cp), &cp); +} + +static void hci_le_past_received_evt(struct hci_dev *hdev, void *data, + struct sk_buff *skb) +{ + struct hci_ev_le_past_received *ev = data; + int mask = hdev->link_mode; + __u8 flags = 0; + struct hci_conn *pa_sync, *conn; + + bt_dev_dbg(hdev, "status 0x%2.2x", ev->status); + + hci_dev_lock(hdev); + + hci_dev_clear_flag(hdev, HCI_PA_SYNC); + + conn = hci_conn_hash_lookup_create_pa_sync(hdev); + if (!conn) { + bt_dev_err(hdev, + "Unable to find connection for dst %pMR sid 0x%2.2x", + &ev->bdaddr, ev->sid); + goto unlock; + } + + conn->sync_handle = le16_to_cpu(ev->sync_handle); + conn->sid = HCI_SID_INVALID; + + mask |= hci_proto_connect_ind(hdev, &ev->bdaddr, PA_LINK, + &flags); + if (!(mask & HCI_LM_ACCEPT)) { + hci_le_pa_term_sync(hdev, ev->sync_handle); + goto unlock; + } + + if (!(flags & HCI_PROTO_DEFER)) + goto unlock; + + /* Add connection to indicate PA sync event */ + pa_sync = hci_conn_add_unset(hdev, PA_LINK, BDADDR_ANY, + HCI_ROLE_SLAVE); + + if (IS_ERR(pa_sync)) + goto unlock; + + pa_sync->sync_handle = le16_to_cpu(ev->sync_handle); + + if (ev->status) { + set_bit(HCI_CONN_PA_SYNC_FAILED, &pa_sync->flags); + + /* Notify iso layer */ + hci_connect_cfm(pa_sync, ev->status); + } + +unlock: + hci_dev_unlock(hdev); +} + static void hci_le_conn_update_complete_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -6426,16 +6496,6 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data, hci_dev_unlock(hdev); } -static int hci_le_pa_term_sync(struct hci_dev *hdev, __le16 handle) -{ - struct hci_cp_le_pa_term_sync cp; - - memset(&cp, 0, sizeof(cp)); - cp.handle = handle; - - return hci_send_cmd(hdev, HCI_OP_LE_PA_TERM_SYNC, sizeof(cp), &cp); -} - static void hci_le_pa_sync_established_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { @@ -7250,6 +7310,10 @@ static const struct hci_le_ev { /* [0x12 = HCI_EV_LE_EXT_ADV_SET_TERM] */ HCI_LE_EV(HCI_EV_LE_EXT_ADV_SET_TERM, hci_le_ext_adv_term_evt, sizeof(struct hci_evt_le_ext_adv_set_term)), + /* [0x18 = HCI_EVT_LE_PAST_RECEIVED] */ + HCI_LE_EV(HCI_EV_LE_PAST_RECEIVED, + hci_le_past_received_evt, + sizeof(struct hci_ev_le_past_received)), /* [0x19 = HCI_EVT_LE_CIS_ESTABLISHED] */ HCI_LE_EV(HCI_EVT_LE_CIS_ESTABLISHED, hci_le_cis_established_evt, sizeof(struct hci_evt_le_cis_established)), diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c index 855d46ebfd2bf..925b980de4f71 100644 --- a/net/bluetooth/hci_sync.c +++ b/net/bluetooth/hci_sync.c @@ -540,8 +540,6 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) { u8 instance = *(u8 *)data; - kfree(data); - hci_clear_adv_instance_sync(hdev, NULL, instance, false); if (list_empty(&hdev->adv_instances)) @@ -550,6 +548,12 @@ static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data) return 0; } +static void adv_timeout_expire_destroy(struct hci_dev *hdev, void *data, + int err) +{ + kfree(data); +} + static void adv_timeout_expire(struct work_struct *work) { u8 *inst_ptr; @@ -570,7 +574,9 @@ static void adv_timeout_expire(struct work_struct *work) goto unlock; *inst_ptr = hdev->cur_adv_instance; - hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL); + if (hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, + adv_timeout_expire_destroy) < 0) + kfree(inst_ptr); unlock: hci_dev_unlock(hdev); @@ -825,7 +831,7 @@ int hci_cmd_sync_run_once(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, void *data, hci_cmd_sync_work_destroy_t destroy) { if (hci_cmd_sync_lookup_entry(hdev, func, data, destroy)) - return 0; + return -EEXIST; return hci_cmd_sync_run(hdev, func, data, destroy); } @@ -929,12 +935,16 @@ int hci_update_eir_sync(struct hci_dev *hdev) memset(&cp, 0, sizeof(cp)); + hci_dev_lock(hdev); eir_create(hdev, cp.data); - if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) + if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) { + hci_dev_unlock(hdev); return 0; + } memcpy(hdev->eir, cp.data, sizeof(cp.data)); + hci_dev_unlock(hdev); return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp, HCI_CMD_TIMEOUT); @@ -966,6 +976,7 @@ int hci_update_class_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) return 0; + hci_dev_lock(hdev); cod[0] = hdev->minor_class; cod[1] = hdev->major_class; cod[2] = get_service_classes(hdev); @@ -973,8 +984,12 @@ int hci_update_class_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) cod[1] |= 0x20; - if (memcmp(cod, hdev->dev_class, 3) == 0) + if (memcmp(cod, hdev->dev_class, 3) == 0) { + hci_dev_unlock(hdev); return 0; + } + + hci_dev_unlock(hdev); return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CLASS_OF_DEV, sizeof(cod), cod, HCI_CMD_TIMEOUT); @@ -1054,14 +1069,19 @@ static int hci_set_random_addr_sync(struct hci_dev *hdev, bdaddr_t *rpa) * In this kind of scenario skip the update and let the random * address be updated at the next cycle. */ + rcu_read_lock(); + if (bacmp(&hdev->random_addr, BDADDR_ANY) && (hci_dev_test_flag(hdev, HCI_LE_ADV) || hci_lookup_le_connect(hdev))) { bt_dev_dbg(hdev, "Deferring random address update"); hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); + rcu_read_unlock(); return 0; } + rcu_read_unlock(); + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa, HCI_CMD_TIMEOUT); } @@ -1219,10 +1239,11 @@ static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, } static int -hci_set_ext_adv_params_sync(struct hci_dev *hdev, struct adv_info *adv, +hci_set_ext_adv_params_sync(struct hci_dev *hdev, u8 instance, const struct hci_cp_le_set_ext_adv_params *cp, struct hci_rp_le_set_ext_adv_params *rp) { + struct adv_info *adv; struct sk_buff *skb; skb = __hci_cmd_sync(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(*cp), @@ -1250,11 +1271,15 @@ hci_set_ext_adv_params_sync(struct hci_dev *hdev, struct adv_info *adv, if (!rp->status) { hdev->adv_addr_type = cp->own_addr_type; - if (!cp->handle) { + if (!instance) { /* Store in hdev for instance 0 */ hdev->adv_tx_power = rp->tx_power; - } else if (adv) { - adv->tx_power = rp->tx_power; + } else { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->tx_power = rp->tx_power; + hci_dev_unlock(hdev); } } @@ -1270,9 +1295,13 @@ static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) int err; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->adv_data_changed) + if (!adv || !adv->adv_data_changed) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_adv_data(hdev, instance, pdu->data, @@ -1283,16 +1312,27 @@ static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu->frag_pref = LE_SET_ADV_DATA_NO_FRAG; + if (adv) { + adv->adv_data_changed = false; + hci_dev_unlock(hdev); + } + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); - if (err) + if (err) { + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->adv_data_changed = true; + hci_dev_unlock(hdev); + } + return err; + } - /* Update data if the command succeed */ - if (adv) { - adv->adv_data_changed = false; - } else { + if (!instance) { memcpy(hdev->adv_data, pdu->data, len); hdev->adv_data_len = len; } @@ -1346,22 +1386,22 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) struct adv_info *adv; bool secondary_adv; - if (instance > 0) { - adv = hci_find_adv_instance(hdev, instance); - if (!adv) - return -EINVAL; - } else { - adv = NULL; - } - /* Updating parameters of an active instance will return a - * Command Disallowed error, so we must first disable the - * instance if it is active. + * Command Disallowed error, so disable it before taking a snapshot. */ - if (adv) { + if (instance > 0) { err = hci_disable_ext_adv_instance_sync(hdev, instance); if (err) return err; + + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (!adv) { + hci_dev_unlock(hdev); + return -EINVAL; + } + } else { + adv = NULL; } flags = hci_adv_instance_flags(hdev, instance); @@ -1372,8 +1412,11 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || mgmt_get_connectable(hdev); - if (!is_advertising_allowed(hdev, connectable)) + if (!is_advertising_allowed(hdev, connectable)) { + if (instance) + hci_dev_unlock(hdev); return -EPERM; + } /* Set require_privacy to true only when non-connectable * advertising is used and it is not periodic. @@ -1384,8 +1427,11 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) err = hci_get_random_address(hdev, require_privacy, adv_use_rpa(hdev, flags), adv, &own_addr_type, &random_addr); - if (err < 0) + if (err < 0) { + if (instance) + hci_dev_unlock(hdev); return err; + } memset(&cp, 0, sizeof(cp)); @@ -1436,6 +1482,9 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) cp.channel_map = hdev->le_adv_channel_map; cp.handle = adv ? adv->handle : instance; + if (instance) + hci_dev_unlock(hdev); + if (flags & MGMT_ADV_FLAG_SEC_2M) { cp.primary_phy = HCI_ADV_PHY_1M; cp.secondary_phy = HCI_ADV_PHY_2M; @@ -1448,12 +1497,12 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) cp.secondary_phy = HCI_ADV_PHY_1M; } - err = hci_set_ext_adv_params_sync(hdev, adv, &cp, &rp); + err = hci_set_ext_adv_params_sync(hdev, instance, &cp, &rp); if (err) return err; /* Update adv data as tx power is known now */ - err = hci_set_ext_adv_data_sync(hdev, cp.handle); + err = hci_set_ext_adv_data_sync(hdev, instance); if (err) return err; @@ -1461,9 +1510,14 @@ int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) && bacmp(&random_addr, BDADDR_ANY)) { /* Check if random address need to be updated */ - if (adv) { - if (!bacmp(&random_addr, &adv->random_addr)) + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (!adv || !bacmp(&random_addr, &adv->random_addr)) { + hci_dev_unlock(hdev); return 0; + } + hci_dev_unlock(hdev); } else { if (!bacmp(&random_addr, &hdev->random_addr)) return 0; @@ -1485,9 +1539,13 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) int err; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->scan_rsp_changed) + if (!adv || !adv->scan_rsp_changed) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_scan_rsp(hdev, instance, pdu->data); @@ -1497,15 +1555,27 @@ static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; pdu->frag_pref = LE_SET_ADV_DATA_NO_FRAG; + if (adv) { + adv->scan_rsp_changed = false; + hci_dev_unlock(hdev); + } + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); - if (err) + if (err) { + if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); + if (adv) + adv->scan_rsp_changed = true; + hci_dev_unlock(hdev); + } + return err; + } - if (adv) { - adv->scan_rsp_changed = false; - } else { + if (!instance) { memcpy(hdev->scan_rsp_data, pdu->data, len); hdev->scan_rsp_data_len = len; } @@ -1520,8 +1590,14 @@ static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) memset(&cp, 0, sizeof(cp)); + if (instance) + hci_dev_lock(hdev); + len = eir_create_scan_rsp(hdev, instance, cp.data); + if (instance) + hci_dev_unlock(hdev); + if (hdev->scan_rsp_data_len == len && !memcmp(cp.data, hdev->scan_rsp_data, len)) return 0; @@ -1656,9 +1732,13 @@ static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) struct adv_info *adv = NULL; if (instance) { + hci_dev_lock(hdev); + adv = hci_find_adv_instance(hdev, instance); - if (!adv || !adv->periodic) + if (!adv || !adv->periodic) { + hci_dev_unlock(hdev); return 0; + } } len = eir_create_per_adv_data(hdev, instance, pdu->data); @@ -1667,6 +1747,9 @@ static int hci_set_per_adv_data_sync(struct hci_dev *hdev, u8 instance) pdu->handle = adv ? adv->handle : instance; pdu->operation = LE_SET_ADV_DATA_OP_COMPLETE; + if (adv) + hci_dev_unlock(hdev); + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_PER_ADV_DATA, struct_size(pdu, data, len), pdu, HCI_CMD_TIMEOUT); @@ -2647,12 +2730,17 @@ static int hci_pause_addr_resolution(struct hci_dev *hdev) /* Cannot disable addr resolution if scanning is enabled or * when initiating an LE connection. */ + rcu_read_lock(); + if (hci_dev_test_flag(hdev, HCI_LE_SCAN) || hci_lookup_le_connect(hdev)) { + rcu_read_unlock(); bt_dev_err(hdev, "Command not allowed when scan/LE connect"); return -EPERM; } + rcu_read_unlock(); + /* Cannot disable addr resolution if advertising is enabled. */ err = hci_pause_advertising_sync(hdev); if (err) { @@ -2790,6 +2878,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) if (hci_dev_test_flag(hdev, HCI_PA_SYNC)) { struct hci_conn *conn; + rcu_read_lock(); + conn = hci_conn_hash_lookup_create_pa_sync(hdev); if (conn) { struct conn_params pa; @@ -2799,6 +2889,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) bacpy(&pa.addr, &conn->dst); pa.addr_type = conn->dst_type; + rcu_read_unlock(); + /* Clear first since there could be addresses left * behind. */ @@ -2808,6 +2900,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) err = hci_le_add_accept_list_sync(hdev, &pa, &num_entries); goto done; + } else { + rcu_read_unlock(); } } @@ -2818,10 +2912,13 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) * the controller. */ list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) { - if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) + rcu_read_lock(); + + if (hci_conn_hash_lookup_le(hdev, &b->bdaddr, b->bdaddr_type)) { + rcu_read_unlock(); continue; + } - /* Pointers not dereferenced, no locks needed */ pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns, &b->bdaddr, b->bdaddr_type); @@ -2829,6 +2926,8 @@ static u8 hci_update_accept_list_sync(struct hci_dev *hdev) &b->bdaddr, b->bdaddr_type); + rcu_read_unlock(); + /* If the device is not likely to connect or report, * remove it from the acceptlist. */ @@ -2955,6 +3054,8 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, if (sent) { struct hci_conn *conn; + rcu_read_lock(); + conn = hci_conn_hash_lookup_ba(hdev, PA_LINK, &sent->bdaddr); if (conn) { @@ -2979,8 +3080,12 @@ static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, phy++; } + rcu_read_unlock(); + if (num_phy) goto done; + } else { + rcu_read_unlock(); } } } @@ -3231,12 +3336,16 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev) /* If there is at least one pending LE connection, we should * keep the background scan running. */ + bool exists; /* If controller is connecting, we should not start scanning * since some controllers are not able to scan and connect at * the same time. */ - if (hci_lookup_le_connect(hdev)) + rcu_read_lock(); + exists = hci_lookup_le_connect(hdev); + rcu_read_unlock(); + if (exists) return 0; bt_dev_dbg(hdev, "start background scanning"); @@ -3451,6 +3560,7 @@ int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable) } static bool disconnected_accept_list_entries(struct hci_dev *hdev) + __must_hold(&hdev->lock) { struct bdaddr_list *b; @@ -3491,12 +3601,16 @@ int hci_update_scan_sync(struct hci_dev *hdev) if (hdev->scanning_paused) return 0; + hci_dev_lock(hdev); + if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) || disconnected_accept_list_entries(hdev)) scan = SCAN_PAGE; else scan = SCAN_DISABLED; + hci_dev_unlock(hdev); + if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) scan |= SCAN_INQUIRY; @@ -4405,6 +4519,9 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev) if (ext_adv_capable(hdev)) events[2] |= 0x02; /* LE Advertising Set Terminated */ + if (past_receiver_capable(hdev)) + events[2] |= 0x80; /* LE PAST Received */ + if (cis_capable(hdev)) { events[3] |= 0x01; /* LE CIS Established */ if (cis_peripheral_capable(hdev)) @@ -5192,6 +5309,7 @@ int hci_dev_open_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -5376,6 +5494,7 @@ int hci_dev_close_sync(struct hci_dev *hdev) if (hdev->req_skb) { kfree_skb(hdev->req_skb); hdev->req_skb = NULL; + hci_dev_clear_flag(hdev, HCI_CMD_PENDING); } clear_bit(HCI_RUNNING, &hdev->flags); @@ -6131,6 +6250,8 @@ static int hci_pause_discovery_sync(struct hci_dev *hdev) static int hci_update_event_filter_sync(struct hci_dev *hdev) { struct bdaddr_list_with_flags *b; + bdaddr_t *accept_list; + size_t i, num_entries = 0; u8 scan = SCAN_DISABLED; bool scanning = test_bit(HCI_PSCAN, &hdev->flags); int err; @@ -6147,23 +6268,49 @@ static int hci_update_event_filter_sync(struct hci_dev *hdev) /* Always clear event filter when starting */ hci_clear_event_filter_sync(hdev); - list_for_each_entry(b, &hdev->accept_list, list) { - if (!(b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP)) - continue; + hci_dev_lock(hdev); + + list_for_each_entry(b, &hdev->accept_list, list) + if (b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP) + num_entries++; + + if (!num_entries) { + hci_dev_unlock(hdev); + goto update_scan; + } - bt_dev_dbg(hdev, "Adding event filters for %pMR", &b->bdaddr); + accept_list = kmalloc_array(num_entries, sizeof(*accept_list), + GFP_KERNEL); + if (!accept_list) { + hci_dev_unlock(hdev); + return -ENOMEM; + } + + i = 0; + list_for_each_entry(b, &hdev->accept_list, list) + if (b->flags & HCI_CONN_FLAG_REMOTE_WAKEUP) + bacpy(&accept_list[i++], &b->bdaddr); - err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP, - HCI_CONN_SETUP_ALLOW_BDADDR, - &b->bdaddr, - HCI_CONN_SETUP_AUTO_ON); + hci_dev_unlock(hdev); + + for (i = 0; i < num_entries; i++) { + bt_dev_dbg(hdev, "Adding event filters for %pMR", + &accept_list[i]); + + err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP, + HCI_CONN_SETUP_ALLOW_BDADDR, + &accept_list[i], + HCI_CONN_SETUP_AUTO_ON); if (err) bt_dev_err(hdev, "Failed to set event filter for %pMR", - &b->bdaddr); + &accept_list[i]); else scan = SCAN_PAGE; } + kfree(accept_list); + +update_scan: if (scan && !scanning) hci_write_scan_enable_sync(hdev, scan); else if (!scan && scanning) @@ -6404,7 +6551,7 @@ static int hci_le_ext_directed_advertising_sync(struct hci_dev *hdev, if (err) return err; - err = hci_set_ext_adv_params_sync(hdev, NULL, &cp, &rp); + err = hci_set_ext_adv_params_sync(hdev, 0, &cp, &rp); if (err) return err; @@ -6575,7 +6722,9 @@ static int hci_le_create_conn_sync(struct hci_dev *hdev, void *data) if (hci_dev_test_flag(hdev, HCI_LE_SCAN) && hdev->le_scan_type == LE_SCAN_ACTIVE && !hci_dev_test_flag(hdev, HCI_LE_SIMULTANEOUS_ROLES)) { - hci_conn_del(conn); + conn->state = BT_OPEN; + hci_abort_conn_sync(hdev, conn, + HCI_ERROR_REJ_LIMITED_RESOURCES); hci_conn_put(conn); return -EBUSY; } diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c index 8957ce7c21b76..c2065abf753e0 100644 --- a/net/bluetooth/hci_sysfs.c +++ b/net/bluetooth/hci_sysfs.c @@ -13,7 +13,10 @@ static const struct class bt_class = { static void bt_link_release(struct device *dev) { struct hci_conn *conn = to_hci_conn(dev); + struct device *parent = dev->parent; + kfree(conn); + put_device(parent); } static const struct device_type bt_link = { @@ -21,6 +24,16 @@ static const struct device_type bt_link = { .release = bt_link_release, }; +/* + * The rfcomm tty device will possibly retain even when conn + * is down, and sysfs doesn't support move zombie device, + * so we should move the device before conn device is destroyed. + */ +static int __match_tty(struct device *dev, const void *data) +{ + return !strncmp(dev_name(dev), "rfcomm", 6); +} + void hci_conn_init_sysfs(struct hci_conn *conn) { struct hci_dev *hdev = conn->hdev; @@ -29,7 +42,7 @@ void hci_conn_init_sysfs(struct hci_conn *conn) conn->dev.type = &bt_link; conn->dev.class = &bt_class; - conn->dev.parent = &hdev->dev; + conn->dev.parent = get_device(&hdev->dev); device_initialize(&conn->dev); } @@ -69,7 +82,7 @@ void hci_conn_del_sysfs(struct hci_conn *conn) while (1) { struct device *dev; - dev = device_find_any_child(&conn->dev); + dev = device_find_child(&conn->dev, NULL, __match_tty); if (!dev) break; device_move(dev, NULL, DPM_ORDER_DEV_LAST); diff --git a/net/bluetooth/hidp/core.c b/net/bluetooth/hidp/core.c index 90d32ea8f9792..4c32191ec8302 100644 --- a/net/bluetooth/hidp/core.c +++ b/net/bluetooth/hidp/core.c @@ -546,9 +546,10 @@ static int hidp_process_data(struct hidp_session *session, struct sk_buff *skb, } if (test_bit(HIDP_WAITING_FOR_RETURN, &session->flags) && - param == session->waiting_report_type) { + param == session->waiting_report_type) { if (session->waiting_report_number < 0 || - session->waiting_report_number == skb->data[0]) { + (skb->len && + session->waiting_report_number == skb->data[0])) { /* hidp_get_raw_report() is waiting on this report. */ session->report_return = skb; done_with_skb = 0; @@ -563,16 +564,18 @@ static int hidp_process_data(struct hidp_session *session, struct sk_buff *skb, static void hidp_recv_ctrl_frame(struct hidp_session *session, struct sk_buff *skb) { - unsigned char hdr, type, param; + unsigned char type, param; + u8 *hdr; int free_skb = 1; BT_DBG("session %p skb %p len %u", session, skb, skb->len); - hdr = skb->data[0]; - skb_pull(skb, 1); + hdr = skb_pull_data(skb, 1); + if (!hdr) + goto free; - type = hdr & HIDP_HEADER_TRANS_MASK; - param = hdr & HIDP_HEADER_PARAM_MASK; + type = *hdr & HIDP_HEADER_TRANS_MASK; + param = *hdr & HIDP_HEADER_PARAM_MASK; switch (type) { case HIDP_TRANS_HANDSHAKE: @@ -593,6 +596,7 @@ static void hidp_recv_ctrl_frame(struct hidp_session *session, break; } +free: if (free_skb) kfree_skb(skb); } @@ -600,14 +604,15 @@ static void hidp_recv_ctrl_frame(struct hidp_session *session, static void hidp_recv_intr_frame(struct hidp_session *session, struct sk_buff *skb) { - unsigned char hdr; + u8 *hdr; BT_DBG("session %p skb %p len %u", session, skb, skb->len); - hdr = skb->data[0]; - skb_pull(skb, 1); + hdr = skb_pull_data(skb, 1); + if (!hdr) + goto free; - if (hdr == (HIDP_TRANS_DATA | HIDP_DATA_RTYPE_INPUT)) { + if (*hdr == (HIDP_TRANS_DATA | HIDP_DATA_RTYPE_INPUT)) { hidp_set_timer(session); if (session->input) @@ -619,9 +624,10 @@ static void hidp_recv_intr_frame(struct hidp_session *session, BT_DBG("report len %d", skb->len); } } else { - BT_DBG("Unsupported protocol header 0x%02x", hdr); + BT_DBG("Unsupported protocol header 0x%02x", *hdr); } +free: kfree_skb(skb); } diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c index f2ff10ae76e22..c30bf48d3858a 100644 --- a/net/bluetooth/iso.c +++ b/net/bluetooth/iso.c @@ -23,15 +23,19 @@ static struct bt_sock_list iso_sk_list = { }; /* ---- ISO connections ---- */ +enum { + ISO_CONN_DROPPED, + __ISO_CONN_NUM_FLAGS +}; + struct iso_conn { struct hci_conn *hcon; + DECLARE_BITMAP(flags, __ISO_CONN_NUM_FLAGS); /* @lock: spinlock protecting changes to iso_conn fields */ spinlock_t lock; struct sock *sk; - struct delayed_work timeout_work; - struct sk_buff *rx_skb; __u32 rx_len; __u16 tx_sn; @@ -55,6 +59,7 @@ static void iso_sock_kill(struct sock *sk); enum { BT_SK_BIG_SYNC, BT_SK_PA_SYNC, + BT_SK_KILLED, }; struct iso_pinfo { @@ -73,6 +78,7 @@ struct iso_pinfo { __u8 base_len; __u8 base[BASE_MAX_LENGTH]; struct iso_conn *conn; + struct delayed_work timeout_work; }; static struct bt_iso_qos default_qos; @@ -80,6 +86,7 @@ static struct bt_iso_qos default_qos; static bool check_ucast_qos(struct bt_iso_qos *qos); static bool check_bcast_qos(struct bt_iso_qos *qos); static bool iso_match_sid(struct sock *sk, void *data); +static bool iso_match_sid_past(struct sock *sk, void *data); static bool iso_match_sync_handle(struct sock *sk, void *data); static bool iso_match_sync_handle_pa_report(struct sock *sk, void *data); static void iso_sock_disconn(struct sock *sk); @@ -100,17 +107,12 @@ static void iso_conn_free(struct kref *ref) BT_DBG("conn %p", conn); - if (conn->sk) - iso_pi(conn->sk)->conn = NULL; - if (conn->hcon) { conn->hcon->iso_data = NULL; - hci_conn_drop(conn->hcon); + if (!test_and_set_bit(ISO_CONN_DROPPED, conn->flags)) + hci_conn_drop(conn->hcon); } - /* Ensure no more work items will run since hci_conn has been dropped */ - disable_delayed_work_sync(&conn->timeout_work); - kfree_skb(conn->rx_skb); kfree(conn); @@ -139,6 +141,14 @@ static struct iso_conn *iso_conn_hold_unless_zero(struct iso_conn *conn) return conn; } +static struct iso_conn *iso_conn_hold(struct iso_conn *conn) +{ + BT_DBG("conn %p refcnt %u", conn, kref_read(&conn->ref)); + + kref_get(&conn->ref); + return conn; +} + static struct sock *iso_sock_hold(struct iso_conn *conn) { if (!conn || !bt_sock_linked(&iso_sk_list, conn->sk)) @@ -151,48 +161,45 @@ static struct sock *iso_sock_hold(struct iso_conn *conn) static void iso_sock_timeout(struct work_struct *work) { - struct iso_conn *conn = container_of(work, struct iso_conn, - timeout_work.work); - struct sock *sk; - - conn = iso_conn_hold_unless_zero(conn); - if (!conn) - return; - - iso_conn_lock(conn); - sk = iso_sock_hold(conn); - iso_conn_unlock(conn); - iso_conn_put(conn); - - if (!sk) - return; + struct iso_pinfo *pi = container_of(work, struct iso_pinfo, + timeout_work.work); + struct sock *sk = &pi->bt.sk; BT_DBG("sock %p state %d", sk, sk->sk_state); lock_sock(sk); - sk->sk_err = ETIMEDOUT; - sk->sk_state_change(sk); + if (!sock_flag(sk, SOCK_ZAPPED)) { + sk->sk_err = ETIMEDOUT; + sk->sk_state_change(sk); + } release_sock(sk); - sock_put(sk); } static void iso_sock_set_timer(struct sock *sk, long timeout) { + lockdep_assert(lockdep_sock_is_held(sk)); + + cancel_delayed_work(&iso_pi(sk)->timeout_work); + if (!iso_pi(sk)->conn) return; BT_DBG("sock %p state %d timeout %ld", sk, sk->sk_state, timeout); - cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); - schedule_delayed_work(&iso_pi(sk)->conn->timeout_work, timeout); + schedule_delayed_work(&iso_pi(sk)->timeout_work, timeout); } static void iso_sock_clear_timer(struct sock *sk) { - if (!iso_pi(sk)->conn) - return; + BT_DBG("sock %p state %d", sk, sk->sk_state); + cancel_delayed_work(&iso_pi(sk)->timeout_work); +} + +static void iso_sock_disable_timer(struct sock *sk) +{ + lockdep_assert(!lockdep_sock_is_held(sk)); BT_DBG("sock %p state %d", sk, sk->sk_state); - cancel_delayed_work(&iso_pi(sk)->conn->timeout_work); + disable_delayed_work_sync(&iso_pi(sk)->timeout_work); } /* ---- ISO connections ---- */ @@ -207,7 +214,6 @@ static struct iso_conn *iso_conn_add(struct hci_conn *hcon) conn->hcon = hcon; iso_conn_unlock(conn); } - iso_conn_put(conn); return conn; } @@ -217,7 +223,6 @@ static struct iso_conn *iso_conn_add(struct hci_conn *hcon) kref_init(&conn->ref); spin_lock_init(&conn->lock); - INIT_DELAYED_WORK(&conn->timeout_work, iso_sock_timeout); hcon->iso_data = conn; conn->hcon = hcon; @@ -261,6 +266,7 @@ static void iso_chan_del(struct sock *sk, int err) } static void iso_conn_del(struct hci_conn *hcon, int err) + __must_hold(&hcon->hdev->lock) { struct iso_conn *conn = hcon->iso_data; struct sock *sk; @@ -275,18 +281,26 @@ static void iso_conn_del(struct hci_conn *hcon, int err) iso_conn_lock(conn); sk = iso_sock_hold(conn); iso_conn_unlock(conn); - iso_conn_put(conn); - if (!sk) { - iso_conn_put(conn); - return; - } + if (!sk) + goto done; + + iso_sock_disable_timer(sk); lock_sock(sk); - iso_sock_clear_timer(sk); iso_chan_del(sk, err); release_sock(sk); + iso_sock_kill(sk); sock_put(sk); + +done: + /* No sk access to conn->hcon any more (lock_sock + hdev->lock) */ + iso_conn_lock(conn); + conn->hcon = NULL; + hcon->iso_data = NULL; + iso_conn_unlock(conn); + + iso_conn_put(conn); } static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, @@ -302,8 +316,14 @@ static int __iso_chan_add(struct iso_conn *conn, struct sock *sk, return -EBUSY; } - iso_pi(sk)->conn = conn; + if (!conn->hcon) { + BT_ERR("conn->hcon missing"); + return -EIO; + } + + iso_pi(sk)->conn = iso_conn_hold(conn); conn->sk = sk; + clear_bit(ISO_CONN_DROPPED, conn->flags); if (parent) bt_accept_enqueue(parent, sk, true); @@ -354,6 +374,7 @@ static int iso_connect_bis(struct sock *sk) return -EHOSTUNREACH; hci_dev_lock(hdev); + lock_sock(sk); if (!bis_capable(hdev)) { err = -EOPNOTSUPP; @@ -406,13 +427,10 @@ static int iso_connect_bis(struct sock *sk) goto unlock; } - lock_sock(sk); - err = iso_chan_add(conn, sk, NULL); - if (err) { - release_sock(sk); + iso_conn_put(conn); + if (err) goto unlock; - } /* Update source addr of the socket */ bacpy(&iso_pi(sk)->src, &hcon->src); @@ -428,9 +446,8 @@ static int iso_connect_bis(struct sock *sk) iso_sock_set_timer(sk, READ_ONCE(sk->sk_sndtimeo)); } - release_sock(sk); - unlock: + release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); return err; @@ -458,6 +475,7 @@ static int iso_connect_cis(struct sock *sk) return -EHOSTUNREACH; hci_dev_lock(hdev); + lock_sock(sk); if (!cis_central_capable(hdev)) { err = -EOPNOTSUPP; @@ -512,13 +530,10 @@ static int iso_connect_cis(struct sock *sk) goto unlock; } - lock_sock(sk); - err = iso_chan_add(conn, sk, NULL); - if (err) { - release_sock(sk); + iso_conn_put(conn); + if (err) goto unlock; - } /* Update source addr of the socket */ bacpy(&iso_pi(sk)->src, &hcon->src); @@ -534,9 +549,8 @@ static int iso_connect_cis(struct sock *sk) iso_sock_set_timer(sk, READ_ONCE(sk->sk_sndtimeo)); } - release_sock(sk); - unlock: + release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); return err; @@ -779,9 +793,15 @@ static void iso_sock_cleanup_listen(struct sock *parent) */ static void iso_sock_kill(struct sock *sk) { + iso_sock_disable_timer(sk); + + lock_sock(sk); + if (!sock_flag(sk, SOCK_ZAPPED) || sk->sk_socket || - sock_flag(sk, SOCK_DEAD)) + test_bit(BT_SK_KILLED, &iso_pi(sk)->flags)) { + release_sock(sk); return; + } BT_DBG("sk %p state %d", sk, sk->sk_state); @@ -795,6 +815,9 @@ static void iso_sock_kill(struct sock *sk) /* Kill poor orphan */ bt_sock_unlink(&iso_sk_list, sk); sock_set_flag(sk, SOCK_DEAD); + set_bit(BT_SK_KILLED, &iso_pi(sk)->flags); + + release_sock(sk); sock_put(sk); } @@ -824,10 +847,8 @@ static void iso_sock_disconn(struct sock *sk) } sk->sk_state = BT_DISCONN; - iso_conn_lock(iso_pi(sk)->conn); - hci_conn_drop(iso_pi(sk)->conn->hcon); - iso_pi(sk)->conn->hcon = NULL; - iso_conn_unlock(iso_pi(sk)->conn); + if (!test_and_set_bit(ISO_CONN_DROPPED, iso_pi(sk)->conn->flags)) + hci_conn_drop(iso_pi(sk)->conn->hcon); } static void __iso_sock_close(struct sock *sk) @@ -869,11 +890,11 @@ static void __iso_sock_close(struct sock *sk) /* Must be called on unlocked socket. */ static void iso_sock_close(struct sock *sk) { + iso_sock_disable_timer(sk); + lock_sock(sk); - iso_sock_clear_timer(sk); __iso_sock_close(sk); release_sock(sk); - iso_sock_kill(sk); } static void iso_sock_init(struct sock *sk, struct sock *parent) @@ -940,6 +961,8 @@ static struct sock *iso_sock_alloc(struct net *net, struct socket *sock, iso_pi(sk)->qos = default_qos; iso_pi(sk)->sync_handle = -1; + INIT_DELAYED_WORK(&iso_pi(sk)->timeout_work, iso_sock_timeout); + bt_sock_link(&iso_sk_list, sk); return sk; } @@ -1028,15 +1051,15 @@ static int iso_sock_bind_pa_sk(struct sock *sk, struct sockaddr_iso *sa, goto done; } - iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis; - - for (int i = 0; i < iso_pi(sk)->bc_num_bis; i++) + for (int i = 0; i < sa->iso_bc->bc_num_bis; i++) if (sa->iso_bc->bc_bis[i] < 0x01 || sa->iso_bc->bc_bis[i] > 0x1f) { err = -EINVAL; goto done; } + iso_pi(sk)->bc_num_bis = sa->iso_bc->bc_num_bis; + memcpy(iso_pi(sk)->bc_bis, sa->iso_bc->bc_bis, iso_pi(sk)->bc_num_bis); @@ -1209,10 +1232,9 @@ static int iso_listen_bis(struct sock *sk) } err = iso_chan_add(conn, sk, NULL); - if (err) { - hci_conn_drop(hcon); + iso_conn_put(conn); + if (err) goto unlock; - } unlock: release_sock(sk); @@ -1391,6 +1413,9 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, BT_DBG("sock %p, sk %p", sock, sk); + lock_sock(sk); + + memset(sa, 0, sizeof(struct sockaddr_iso)); addr->sa_family = AF_BLUETOOTH; if (peer) { @@ -1401,6 +1426,7 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, sa->iso_bdaddr_type = iso_pi(sk)->dst_type; if (hcon && (hcon->type == BIS_LINK || hcon->type == PA_LINK)) { + memset(sa->iso_bc, 0, sizeof(struct sockaddr_iso_bc)); sa->iso_bc->bc_sid = iso_pi(sk)->bc_sid; sa->iso_bc->bc_num_bis = iso_pi(sk)->bc_num_bis; memcpy(sa->iso_bc->bc_bis, iso_pi(sk)->bc_bis, @@ -1412,6 +1438,8 @@ static int iso_sock_getname(struct socket *sock, struct sockaddr *addr, sa->iso_bdaddr_type = iso_pi(sk)->src_type; } + release_sock(sk); + return len; } @@ -1511,9 +1539,9 @@ static void iso_conn_defer_accept(struct hci_conn *conn) hci_send_cmd(hdev, HCI_OP_LE_ACCEPT_CIS, sizeof(cp), &cp); } -static void iso_conn_big_sync(struct sock *sk) +static int iso_conn_big_sync(struct sock *sk) { - int err; + int err = 0; struct hci_dev *hdev; struct iso_conn *conn; bdaddr_t src, dst; @@ -1528,7 +1556,7 @@ static void iso_conn_big_sync(struct sock *sk) hdev = hci_get_route(&dst, &src, src_type); if (!hdev) - return; + return -EHOSTUNREACH; /* hci_le_big_create_sync requires hdev lock to be held, since * it enqueues the HCI LE BIG Create Sync command via @@ -1544,8 +1572,10 @@ static void iso_conn_big_sync(struct sock *sk) * both before dereferencing conn->hcon. */ conn = iso_pi(sk)->conn; - if (!conn || !conn->hcon) + if (!conn || !conn->hcon) { + err = -ENOTCONN; goto unlock; + } if (!test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { err = hci_conn_big_create_sync(hdev, conn->hcon, @@ -1561,6 +1591,8 @@ static void iso_conn_big_sync(struct sock *sk) release_sock(sk); hci_dev_unlock(hdev); hci_dev_put(hdev); + + return err; } static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, @@ -1585,10 +1617,19 @@ static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, case BT_CONNECT2: if (test_bit(BT_SK_PA_SYNC, &pi->flags)) { release_sock(sk); - iso_conn_big_sync(sk); + err = iso_conn_big_sync(sk); lock_sock(sk); - sk->sk_state = BT_LISTEN; + /* The socket lock was dropped, so the + * connection may have been torn down + * meanwhile and iso_chan_del() may have + * already moved the socket to BT_CLOSED. + * Only move on to BT_LISTEN if the BIG sync + * was actually started and nothing else has + * changed the state. + */ + if (!err && sk->sk_state == BT_CONNECT2) + sk->sk_state = BT_LISTEN; } else { iso_conn_defer_accept(pi->conn->hcon); sk->sk_state = BT_CONFIG; @@ -1599,10 +1640,11 @@ static int iso_sock_recvmsg(struct socket *sock, struct msghdr *msg, case BT_CONNECTED: if (test_bit(BT_SK_PA_SYNC, &iso_pi(sk)->flags)) { release_sock(sk); - iso_conn_big_sync(sk); + err = iso_conn_big_sync(sk); lock_sock(sk); - sk->sk_state = BT_LISTEN; + if (!err && sk->sk_state == BT_CONNECTED) + sk->sk_state = BT_LISTEN; early_ret = true; } @@ -1711,7 +1753,7 @@ static bool check_bcast_qos(struct bt_iso_qos *qos) return false; if (!qos->bcast.timeout) - qos->bcast.sync_timeout = BT_ISO_SYNC_TIMEOUT; + qos->bcast.timeout = BT_ISO_SYNC_TIMEOUT; if (qos->bcast.timeout < 0x000a || qos->bcast.timeout > 0x4000) return false; @@ -1953,8 +1995,16 @@ static int iso_sock_release(struct socket *sock) release_sock(sk); } + /* Make sure sk is valid even if iso_conn_del() is concurrent */ + sock_hold(sk); + + lock_sock(sk); sock_orphan(sk); + release_sock(sk); + iso_sock_kill(sk); + + sock_put(sk); return err; } @@ -1962,14 +2012,17 @@ static void iso_sock_ready(struct sock *sk) { BT_DBG("sk %p", sk); - if (!sk) + lockdep_assert(lockdep_sock_is_held(sk)); + + switch (sk->sk_state) { + case BT_DISCONN: + case BT_CLOSED: return; + } - lock_sock(sk); iso_sock_clear_timer(sk); sk->sk_state = BT_CONNECTED; sk->sk_state_change(sk); - release_sock(sk); } static bool iso_match_big(struct sock *sk, void *data) @@ -1994,7 +2047,7 @@ static bool iso_match_pa_sync_flag(struct sock *sk, void *data) static void iso_conn_ready(struct iso_conn *conn) { struct sock *parent = NULL; - struct sock *sk = conn->sk; + struct sock *sk; struct hci_ev_le_big_sync_established *ev = NULL; struct hci_ev_le_pa_sync_established *ev2 = NULL; struct hci_ev_le_per_adv_report *ev3 = NULL; @@ -2002,8 +2055,43 @@ static void iso_conn_ready(struct iso_conn *conn) BT_DBG("conn %p", conn); + iso_conn_lock(conn); + sk = iso_sock_hold(conn); + iso_conn_unlock(conn); + if (sk) { - iso_sock_ready(conn->sk); + lock_sock(sk); + + /* conn->sk may have become NULL if racing with sk close, but + * due to held hdev->lock, it can't become different sk. + */ + if (!conn->sk) { + release_sock(sk); + sock_put(sk); + return; + } + + /* Attempt to update source address in case of BIS Sender if + * the advertisement is using a random address. + */ + if (conn->hcon->type == BIS_LINK && + conn->hcon->role == HCI_ROLE_MASTER && + !bacmp(&conn->hcon->dst, BDADDR_ANY)) { + struct hci_conn *bis = conn->hcon; + struct adv_info *adv; + + adv = hci_find_adv_instance(bis->hdev, + bis->iso_qos.bcast.bis); + if (adv && bacmp(&adv->random_addr, BDADDR_ANY)) { + iso_pi(sk)->src_type = BDADDR_LE_RANDOM; + bacpy(&iso_pi(sk)->src, &adv->random_addr); + } + } + + iso_sock_ready(sk); + + release_sock(sk); + sock_put(sk); } else { hcon = conn->hcon; if (!hcon) @@ -2060,6 +2148,14 @@ static void iso_conn_ready(struct iso_conn *conn) lock_sock(parent); + /* The listener may have been closed concurrently. */ + if (parent->sk_state != BT_LISTEN || + sock_flag(parent, SOCK_ZAPPED)) { + release_sock(parent); + sock_put(parent); + return; + } + sk = iso_sock_alloc(sock_net(parent), NULL, BTPROTO_ISO, GFP_ATOMIC, 0); if (!sk) { @@ -2142,6 +2238,16 @@ static bool iso_match_sid(struct sock *sk, void *data) return ev->sid == iso_pi(sk)->bc_sid; } +static bool iso_match_sid_past(struct sock *sk, void *data) +{ + struct hci_ev_le_past_received *ev = data; + + if (iso_pi(sk)->bc_sid == HCI_SID_INVALID) + return true; + + return ev->sid == iso_pi(sk)->bc_sid; +} + static bool iso_match_sync_handle(struct sock *sk, void *data) { struct hci_evt_le_big_info_adv_report *ev = data; @@ -2161,6 +2267,7 @@ static bool iso_match_sync_handle_pa_report(struct sock *sk, void *data) int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) { struct hci_ev_le_pa_sync_established *ev1; + struct hci_ev_le_past_received *ev1a; struct hci_evt_le_big_info_adv_report *ev2; struct hci_ev_le_per_adv_report *ev3; struct sock *sk; @@ -2174,6 +2281,7 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) * SID to listen to and once sync is established its handle needs to * be stored in iso_pi(sk)->sync_handle so it can be matched once * receiving the BIG Info. + * 1a. HCI_EV_LE_PAST_RECEIVED: alternative to 1. * 2. HCI_EVT_LE_BIG_INFO_ADV_REPORT: When connect_ind is triggered by a * a BIG Info it attempts to check if there any listening socket with * the same sync_handle and if it does then attempt to create a sync. @@ -2186,8 +2294,24 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) sk = iso_get_sock(&hdev->bdaddr, bdaddr, BT_LISTEN, iso_match_sid, ev1); if (sk && !ev1->status) { + lock_sock(sk); iso_pi(sk)->sync_handle = le16_to_cpu(ev1->handle); iso_pi(sk)->bc_sid = ev1->sid; + release_sock(sk); + } + + goto done; + } + + ev1a = hci_recv_event_data(hdev, HCI_EV_LE_PAST_RECEIVED); + if (ev1a) { + sk = iso_get_sock(&hdev->bdaddr, bdaddr, BT_LISTEN, + iso_match_sid_past, ev1a); + if (sk && !ev1a->status) { + lock_sock(sk); + iso_pi(sk)->sync_handle = le16_to_cpu(ev1a->sync_handle); + iso_pi(sk)->bc_sid = ev1a->sid; + release_sock(sk); } goto done; @@ -2214,27 +2338,35 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) ev2); if (sk) { - int err; - struct hci_conn *hcon = iso_pi(sk)->conn->hcon; + int err = 0; + bool big_sync; + struct hci_conn *hcon; + lock_sock(sk); + + hcon = iso_pi(sk)->conn ? iso_pi(sk)->conn->hcon : NULL; iso_pi(sk)->qos.bcast.encryption = ev2->encryption; if (ev2->num_bis < iso_pi(sk)->bc_num_bis) iso_pi(sk)->bc_num_bis = ev2->num_bis; - if (!test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags) && - !test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags)) { + big_sync = !test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags) && + !test_and_set_bit(BT_SK_BIG_SYNC, &iso_pi(sk)->flags); + + if (big_sync) err = hci_conn_big_create_sync(hdev, hcon, &iso_pi(sk)->qos, iso_pi(sk)->sync_handle, iso_pi(sk)->bc_num_bis, iso_pi(sk)->bc_bis); - if (err) { - bt_dev_err(hdev, "hci_le_big_create_sync: %d", - err); - sock_put(sk); - sk = NULL; - } + + release_sock(sk); + + if (big_sync && err) { + bt_dev_err(hdev, "hci_le_big_create_sync: %d", + err); + sock_put(sk); + sk = NULL; } } @@ -2252,9 +2384,11 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) if (!sk) goto done; - hcon = iso_pi(sk)->conn->hcon; + lock_sock(sk); + + hcon = iso_pi(sk)->conn ? iso_pi(sk)->conn->hcon : NULL; if (!hcon) - goto done; + goto release3; if (ev3->data_status == LE_PA_DATA_TRUNCATED) { /* The controller was unable to retrieve PA data. */ @@ -2262,12 +2396,12 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) HCI_MAX_PER_AD_TOT_LEN); hcon->le_per_adv_data_len = 0; hcon->le_per_adv_data_offset = 0; - goto done; + goto release3; } if (hcon->le_per_adv_data_offset + ev3->length > HCI_MAX_PER_AD_TOT_LEN) - goto done; + goto release3; memcpy(hcon->le_per_adv_data + hcon->le_per_adv_data_offset, ev3->data, ev3->length); @@ -2286,7 +2420,7 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) &base_len); if (!base || base_len > BASE_MAX_LENGTH) - goto done; + goto release3; memcpy(iso_pi(sk)->base, base, base_len); iso_pi(sk)->base_len = base_len; @@ -2296,6 +2430,9 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) */ hcon->le_per_adv_data_len = 0; } + +release3: + release_sock(sk); } else { sk = iso_get_sock(&hdev->bdaddr, BDADDR_ANY, BT_LISTEN, NULL, NULL); @@ -2314,6 +2451,7 @@ int iso_connect_ind(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 *flags) } static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) + __must_hold(&hcon->hdev->lock) { if (hcon->type != CIS_LINK && hcon->type != BIS_LINK && hcon->type != PA_LINK) { @@ -2325,8 +2463,10 @@ static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) struct hci_link *link, *t; list_for_each_entry_safe(link, t, &hcon->link_list, - list) + list) { + lockdep_assert_held(&link->conn->hdev->lock); iso_conn_del(link->conn, bt_to_errno(status)); + } return; } @@ -2348,14 +2488,17 @@ static void iso_connect_cfm(struct hci_conn *hcon, __u8 status) struct iso_conn *conn; conn = iso_conn_add(hcon); - if (conn) + if (conn) { iso_conn_ready(conn); + iso_conn_put(conn); + } } else { iso_conn_del(hcon, bt_to_errno(status)); } } static void iso_disconn_cfm(struct hci_conn *hcon, __u8 reason) + __must_hold(&hcon->hdev->lock) { if (hcon->type != CIS_LINK && hcon->type != BIS_LINK && hcon->type != PA_LINK) @@ -2400,7 +2543,7 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) switch (pb) { case ISO_START: case ISO_SINGLE: - if (conn->rx_len) { + if (conn->rx_skb || conn->rx_len) { BT_ERR("Unexpected start frame (len %d)", skb->len); kfree_skb(conn->rx_skb); conn->rx_skb = NULL; @@ -2481,12 +2624,14 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) break; case ISO_CONT: - BT_DBG("Cont: frag len %d (expecting %d)", skb->len, + case ISO_END: + BT_DBG("%s: frag len %d (expecting %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len, conn->rx_len); - if (!conn->rx_len) { - BT_ERR("Unexpected continuation frame (len %d)", - skb->len); + if (!conn->rx_skb) { + BT_ERR("Unexpected ISO %s frame (len %d)", + (pb == ISO_END) ? "End" : "Cont", skb->len); goto drop; } @@ -2502,17 +2647,9 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), skb->len); conn->rx_len -= skb->len; - break; - - case ISO_END: - if (!conn->rx_len) { - BT_ERR("Unexpected end frame (len %d)", skb->len); - goto drop; - } - skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len), - skb->len); - conn->rx_len -= skb->len; + if (pb == ISO_CONT) + break; if (!conn->rx_len) { struct sk_buff *rx_skb = conn->rx_skb; @@ -2523,6 +2660,13 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags) */ conn->rx_skb = NULL; iso_recv_frame(conn, rx_skb); + } else { + BT_ERR("ISO fragment incomplete (len %d, expected %d)", + skb->len, conn->rx_len); + kfree_skb(conn->rx_skb); + conn->rx_skb = NULL; + conn->rx_len = 0; + goto drop; } break; } diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c index e04948fb6832e..9365a98107f2d 100644 --- a/net/bluetooth/l2cap_core.c +++ b/net/bluetooth/l2cap_core.c @@ -1340,7 +1340,7 @@ static void l2cap_le_connect(struct l2cap_chan *chan) struct l2cap_ecred_conn_data { struct { struct l2cap_ecred_conn_req_hdr req; - __le16 scid[5]; + __le16 scid[L2CAP_ECRED_CONN_SCID_MAX]; } __packed pdu; struct l2cap_chan *chan; struct pid *pid; @@ -1355,7 +1355,7 @@ static void l2cap_ecred_defer_connect(struct l2cap_chan *chan, void *data) if (chan == conn->chan) return; - if (!test_and_clear_bit(FLAG_DEFER_SETUP, &chan->flags)) + if (!test_bit(FLAG_DEFER_SETUP, &chan->flags)) return; pid = chan->ops->get_peer_pid(chan); @@ -1365,9 +1365,16 @@ static void l2cap_ecred_defer_connect(struct l2cap_chan *chan, void *data) chan->mode != L2CAP_MODE_EXT_FLOWCTL || chan->state != BT_CONNECT) return; + if (!test_and_clear_bit(FLAG_DEFER_SETUP, &chan->flags)) + return; + if (test_and_set_bit(FLAG_ECRED_CONN_REQ_SENT, &chan->flags)) return; + /* Unreachable, checked in l2cap_connect (+timer drops it if reached) */ + if (WARN_ON_ONCE(conn->count >= ARRAY_SIZE(conn->pdu.scid))) + return; + l2cap_ecred_init(chan, 0); /* Set the same ident so we can match on the rsp */ @@ -3893,6 +3900,9 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data) struct l2cap_ecred_conn_rsp *rsp_flex = container_of(&rsp->pdu.rsp, struct l2cap_ecred_conn_rsp, hdr); + if (chan->mode != L2CAP_MODE_EXT_FLOWCTL) + return; + /* Check if channel for outgoing connection or if it wasn't deferred * since in those cases it must be skipped. */ @@ -3903,6 +3913,10 @@ static void l2cap_ecred_rsp_defer(struct l2cap_chan *chan, void *data) /* Reset ident so only one response is sent */ chan->ident = 0; + /* Unreachable, check in l2cap_ecred_conn_req. If reached, drop rest */ + if (WARN_ON_ONCE(rsp->count >= ARRAY_SIZE(rsp->pdu.scid))) + rsp->pdu.rsp.result = cpu_to_le16(L2CAP_CR_LE_NO_MEM); + /* Include all channels pending with the same ident */ if (!rsp->pdu.rsp.result) rsp_flex->dcid[rsp->count++] = cpu_to_le16(chan->scid); @@ -4831,6 +4845,10 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, if (!chan) return -EBADSLT; + chan = l2cap_chan_hold_unless_zero(chan); + if (!chan) + return -EBADSLT; + err = 0; l2cap_chan_lock(chan); @@ -4876,6 +4894,7 @@ static int l2cap_le_connect_rsp(struct l2cap_conn *conn, } l2cap_chan_unlock(chan); + l2cap_chan_put(chan); return err; } @@ -5065,6 +5084,7 @@ static int l2cap_le_connect_req(struct l2cap_conn *conn, __set_chan_timer(chan, chan->ops->get_sndtimeo(chan)); chan->ident = cmd->ident; + chan->mode = L2CAP_MODE_LE_FLOWCTL; if (test_bit(FLAG_DEFER_SETUP, &chan->flags)) { l2cap_state_change(chan, BT_CONNECT2); @@ -7112,6 +7132,11 @@ static void l2cap_recv_frame(struct l2cap_conn *conn, struct sk_buff *skb) break; case L2CAP_CID_CONN_LESS: + if (skb->len < L2CAP_PSMLEN_SIZE) { + kfree_skb(skb); + break; + } + psm = get_unaligned((__le16 *) skb->data); skb_pull(skb, L2CAP_PSMLEN_SIZE); l2cap_conless_channel(conn, psm, skb); @@ -7353,6 +7378,9 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid, goto done; } + mutex_lock(&conn->lock); + l2cap_chan_lock(chan); + if (chan->mode == L2CAP_MODE_EXT_FLOWCTL) { struct l2cap_chan_data data; @@ -7360,19 +7388,20 @@ int l2cap_chan_connect(struct l2cap_chan *chan, __le16 psm, u16 cid, data.pid = chan->ops->get_peer_pid(chan); data.count = 1; - l2cap_chan_list(conn, l2cap_chan_by_pid, &data); + __l2cap_chan_list(conn, l2cap_chan_by_pid, &data); + + /* Leave room for non-deferred channel that ends the group. */ + if (test_bit(FLAG_DEFER_SETUP, &chan->flags)) + data.count += 1; /* Check if there isn't too many channels being connected */ if (data.count > L2CAP_ECRED_CONN_SCID_MAX) { hci_conn_drop(hcon); err = -EPROTO; - goto done; + goto chan_unlock; } } - mutex_lock(&conn->lock); - l2cap_chan_lock(chan); - if (cid && __l2cap_get_chan_by_dcid(conn, cid)) { hci_conn_drop(hcon); err = -EBUSY; diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 1a916c0f40ccf..d332cd999d8c0 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -1284,7 +1284,12 @@ static void l2cap_sock_kill(struct sock *sk) BT_DBG("sk %p state %s", sk, state_to_string(sk->sk_state)); + /* Take lock to synchronize against access without owning sk->sk_socket, + * eg. in l2cap_sock_cleanup_listen(). proto_ops etc. don't need lock. + */ + lock_sock(sk); l2cap_sock_put_chan(sk); + release_sock(sk); /* Kill poor orphan */ sock_set_flag(sk, SOCK_DEAD); @@ -1487,14 +1492,10 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) * establish sk_lock -> conn->lock and invert the established * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock). * - * Instead, briefly take the child sk lock to fetch and pin its chan. - * l2cap_conn_del() reaches the chan free only via - * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes - * the child sk lock; holding it across l2cap_chan_hold_unless_zero() - * therefore guarantees the chan cannot be freed while we read and - * pin it (hold_unless_zero() additionally skips a chan already past - * its last reference). We then drop the sk lock before taking - * chan->lock, so sk and chan locks are never held together. + * Instead, briefly take the child sk lock to synchronize vs. + * l2cap_sock_kill that puts l2cap_pi(sk)->chan. We then drop the sk + * lock before taking chan->lock, so sk and chan locks are never held + * together. * * Since we cannot call l2cap_chan_close() without conn->lock, * schedule l2cap_chan_timeout to close the channel; it already @@ -1504,10 +1505,12 @@ static void l2cap_sock_cleanup_listen(struct sock *parent) struct l2cap_chan *chan; lock_sock_nested(sk, L2CAP_NESTING_NORMAL); - chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan); + chan = l2cap_pi(sk)->chan; + if (chan) + l2cap_chan_hold(chan); release_sock(sk); if (!chan) { - /* l2cap_conn_del() already tearing this child down */ + /* Already torn down */ sock_put(sk); continue; } @@ -1539,6 +1542,11 @@ static int l2cap_sock_new_connection_cb(struct l2cap_chan *chan, lock_sock(parent); + if (parent->sk_state != BT_LISTEN) { + release_sock(parent); + return -EINVAL; + } + /* Check for backlog size */ if (sk_acceptq_is_full(parent)) { BT_DBG("backlog full %d", parent->sk_ack_backlog); @@ -1703,10 +1711,14 @@ static void l2cap_sock_state_change_cb(struct l2cap_chan *chan, int state, if (!sk) return; + lock_sock(sk); + sk->sk_state = state; if (err) sk->sk_err = err; + + release_sock(sk); } static struct sk_buff *l2cap_sock_alloc_skb_cb(struct l2cap_chan *chan, @@ -1782,6 +1794,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) if (!sk) return; + lock_sock(sk); + if (test_and_clear_bit(FLAG_PENDING_SECURITY, &chan->flags)) { sk->sk_state = BT_CONNECTED; chan->state = BT_CONNECTED; @@ -1789,6 +1803,8 @@ static void l2cap_sock_resume_cb(struct l2cap_chan *chan) clear_bit(BT_SK_SUSPEND, &bt_sk(sk)->flags); sk->sk_state_change(sk); + + release_sock(sk); } static void l2cap_sock_set_shutdown_cb(struct l2cap_chan *chan) diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 410910f64c1c4..01f46a985e2b3 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -858,6 +858,12 @@ static u32 get_supported_settings(struct hci_dev *hdev) if (ll_privacy_capable(hdev)) settings |= MGMT_SETTING_LL_PRIVACY; + if (past_sender_capable(hdev)) + settings |= MGMT_SETTING_PAST_SENDER; + + if (past_receiver_capable(hdev)) + settings |= MGMT_SETTING_PAST_RECEIVER; + settings |= MGMT_SETTING_PHY_CONFIGURATION; return settings; @@ -943,6 +949,12 @@ static u32 get_current_settings(struct hci_dev *hdev) if (ll_privacy_enabled(hdev)) settings |= MGMT_SETTING_LL_PRIVACY; + if (past_sender_enabled(hdev)) + settings |= MGMT_SETTING_PAST_SENDER; + + if (past_receiver_enabled(hdev)) + settings |= MGMT_SETTING_PAST_RECEIVER; + return settings; } @@ -2422,11 +2434,15 @@ static int send_cancel(struct hci_dev *hdev, void *data) mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, 0, NULL, 0); - mgmt_pending_free(cmd); return 0; } +static void send_cancel_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2447,7 +2463,8 @@ static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev, if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_cancel, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_cancel, cmd, + send_cancel_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL, @@ -2633,7 +2650,7 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) if (IS_ERR(skb)) { mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, mgmt_status(PTR_ERR(skb))); - goto done; + return 0; } mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, 0, @@ -2641,12 +2658,14 @@ static int send_hci_cmd_sync(struct hci_dev *hdev, void *data) kfree_skb(skb); -done: - mgmt_pending_free(cmd); - return 0; } +static void send_hci_cmd_sync_destroy(struct hci_dev *hdev, void *data, int err) +{ + mgmt_pending_free(data); +} + static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -2659,12 +2678,21 @@ static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, MGMT_STATUS_INVALID_PARAMS); + /* The HCI command header carries the parameter length in a u8, a + * larger value would be truncated there while the parameters are + * still appended to the frame in full. + */ + if (le16_to_cpu(cp->params_len) > U8_MAX) + return mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, + MGMT_STATUS_INVALID_PARAMS); + hci_dev_lock(hdev); cmd = mgmt_pending_new(sk, MGMT_OP_HCI_CMD_SYNC, hdev, data, len); if (!cmd) err = -ENOMEM; else - err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, NULL); + err = hci_cmd_sync_queue(hdev, send_hci_cmd_sync, cmd, + send_hci_cmd_sync_destroy); if (err < 0) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_HCI_CMD_SYNC, @@ -2687,18 +2715,28 @@ static int mgmt_hci_cmd_sync(struct sock *sk, struct hci_dev *hdev, static bool pending_eir_or_class(struct hci_dev *hdev) { struct mgmt_pending_cmd *cmd; + bool pending = false; + + mutex_lock(&hdev->mgmt_pending_lock); list_for_each_entry(cmd, &hdev->mgmt_pending, list) { switch (cmd->opcode) { case MGMT_OP_ADD_UUID: case MGMT_OP_REMOVE_UUID: case MGMT_OP_SET_DEV_CLASS: + case MGMT_OP_SET_LOCAL_NAME: case MGMT_OP_SET_POWERED: - return true; + pending = true; + break; } + + if (pending) + break; } - return false; + mutex_unlock(&hdev->mgmt_pending_lock); + + return pending; } static const u8 bluetooth_base_uuid[] = { @@ -3082,6 +3120,8 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) struct mgmt_cp_unpair_device *cp = cmd->param; struct hci_conn *conn; + hci_dev_lock(hdev); + if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); @@ -3089,6 +3129,11 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); + if (conn) + hci_conn_get(conn); + + hci_dev_unlock(hdev); + if (!conn) return 0; @@ -3096,6 +3141,7 @@ static int unpair_device_sync(struct hci_dev *hdev, void *data) * will clean up the connection no matter the error. */ hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); return 0; } @@ -3243,6 +3289,8 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) struct mgmt_cp_disconnect *cp = cmd->param; struct hci_conn *conn; + hci_dev_lock(hdev); + if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); @@ -3250,6 +3298,11 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) conn = hci_conn_hash_lookup_le(hdev, &cp->addr.bdaddr, le_addr_type(cp->addr.type)); + if (conn) + hci_conn_get(conn); + + hci_dev_unlock(hdev); + if (!conn) return -ENOTCONN; @@ -3257,6 +3310,7 @@ static int disconnect_sync(struct hci_dev *hdev, void *data) * will clean up the connection no matter the error. */ hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); return 0; } @@ -3489,11 +3543,13 @@ static int set_io_capability(struct sock *sk, struct hci_dev *hdev, void *data, NULL, 0); } -static struct mgmt_pending_cmd *find_pairing(struct hci_conn *conn) +static struct mgmt_pending_cmd *remove_pairing(struct hci_conn *conn) { struct hci_dev *hdev = conn->hdev; struct mgmt_pending_cmd *cmd; + mutex_lock(&hdev->mgmt_pending_lock); + list_for_each_entry(cmd, &hdev->mgmt_pending, list) { if (cmd->opcode != MGMT_OP_PAIR_DEVICE) continue; @@ -3501,9 +3557,39 @@ static struct mgmt_pending_cmd *find_pairing(struct hci_conn *conn) if (cmd->user_data != conn) continue; + list_del(&cmd->list); + mutex_unlock(&hdev->mgmt_pending_lock); + return cmd; + } + + mutex_unlock(&hdev->mgmt_pending_lock); + + return NULL; +} + +static struct mgmt_pending_cmd *remove_pairing_by_addr(struct hci_dev *hdev, + bdaddr_t *bdaddr) +{ + struct mgmt_pending_cmd *cmd; + struct hci_conn *conn; + + mutex_lock(&hdev->mgmt_pending_lock); + + list_for_each_entry(cmd, &hdev->mgmt_pending, list) { + if (cmd->opcode != MGMT_OP_PAIR_DEVICE) + continue; + + conn = cmd->user_data; + if (bacmp(bdaddr, &conn->dst) != 0) + continue; + + list_del(&cmd->list); + mutex_unlock(&hdev->mgmt_pending_lock); return cmd; } + mutex_unlock(&hdev->mgmt_pending_lock); + return NULL; } @@ -3541,10 +3627,10 @@ void mgmt_smp_complete(struct hci_conn *conn, bool complete) u8 status = complete ? MGMT_STATUS_SUCCESS : MGMT_STATUS_FAILED; struct mgmt_pending_cmd *cmd; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (cmd) { cmd->cmd_complete(cmd, status); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } } @@ -3554,14 +3640,14 @@ static void pairing_complete_cb(struct hci_conn *conn, u8 status) BT_DBG("status %u", status); - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (!cmd) { BT_DBG("Unable to find a pending command"); return; } cmd->cmd_complete(cmd, mgmt_status(status)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } static void le_pairing_complete_cb(struct hci_conn *conn, u8 status) @@ -3573,14 +3659,14 @@ static void le_pairing_complete_cb(struct hci_conn *conn, u8 status) if (!status) return; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); if (!cmd) { BT_DBG("Unable to find a pending command"); return; } cmd->cmd_complete(cmd, mgmt_status(status)); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } static int pair_device(struct sock *sk, struct hci_dev *hdev, void *data, @@ -3737,23 +3823,17 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - cmd = pending_find(MGMT_OP_PAIR_DEVICE, hdev); + cmd = remove_pairing_by_addr(hdev, &addr->bdaddr); if (!cmd) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, MGMT_STATUS_INVALID_PARAMS); goto unlock; } - conn = cmd->user_data; - - if (bacmp(&addr->bdaddr, &conn->dst) != 0) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, - MGMT_STATUS_INVALID_PARAMS); - goto unlock; - } + conn = hci_conn_get(cmd->user_data); cmd->cmd_complete(cmd, MGMT_STATUS_CANCELLED); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_CANCEL_PAIR_DEVICE, 0, addr, sizeof(*addr)); @@ -3771,6 +3851,8 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data, if (conn->conn_reason == CONN_REASON_PAIR_DEVICE) hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM); + hci_conn_put(conn); + unlock: hci_dev_unlock(hdev); return err; @@ -4018,6 +4100,12 @@ static int set_local_name(struct sock *sk, struct hci_dev *hdev, void *data, goto failed; } + if (hdev_is_powered(hdev) && pending_eir_or_class(hdev)) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_LOCAL_NAME, + MGMT_STATUS_BUSY); + goto failed; + } + memcpy(hdev->short_name, cp->short_name, sizeof(hdev->short_name)); if (!hdev_is_powered(hdev)) { @@ -6008,6 +6096,7 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, struct mgmt_pending_cmd *cmd; const u16 max_uuid_count = ((U16_MAX - sizeof(*cp)) / 16); u16 uuid_count, expected_len; + u8 (*uuids)[16] = NULL; u8 status; int err; @@ -6084,12 +6173,10 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, hdev->discovery.result_filtering = true; hdev->discovery.type = cp->type; hdev->discovery.rssi = cp->rssi; - hdev->discovery.uuid_count = uuid_count; if (uuid_count > 0) { - hdev->discovery.uuids = kmemdup(cp->uuids, uuid_count * 16, - GFP_KERNEL); - if (!hdev->discovery.uuids) { + uuids = kmemdup(cp->uuids, uuid_count * sizeof(*uuids), GFP_KERNEL); + if (!uuids) { err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_START_SERVICE_DISCOVERY, MGMT_STATUS_FAILED, @@ -6099,6 +6186,11 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, } } + spin_lock(&hdev->discovery.lock); + hdev->discovery.uuids = uuids; + hdev->discovery.uuid_count = uuid_count; + spin_unlock(&hdev->discovery.lock); + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, start_discovery_complete); if (err < 0) { @@ -7309,6 +7401,9 @@ static void get_conn_info_complete(struct hci_dev *hdev, void *data, int err) rp.max_tx_power = HCI_TX_POWER_INVALID; } + if (conn) + hci_conn_put(conn); + mgmt_cmd_complete(cmd->sk, cmd->hdev->id, MGMT_OP_GET_CONN_INFO, status, &rp, sizeof(rp)); @@ -7323,6 +7418,8 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data) int err; __le16 handle; + hci_dev_lock(hdev); + /* Make sure we are still connected */ if (cp->addr.type == BDADDR_BREDR) conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, @@ -7330,12 +7427,16 @@ static int get_conn_info_sync(struct hci_dev *hdev, void *data) else conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, &cp->addr.bdaddr); - if (!conn || conn->state != BT_CONNECTED) + if (!conn || conn->state != BT_CONNECTED) { + hci_dev_unlock(hdev); return MGMT_STATUS_NOT_CONNECTED; + } - cmd->user_data = conn; + cmd->user_data = hci_conn_get(conn); handle = cpu_to_le16(conn->handle); + hci_dev_unlock(hdev); + /* Refresh RSSI each time */ err = hci_read_rssi_sync(hdev, handle); @@ -7469,6 +7570,9 @@ static void get_clock_info_complete(struct hci_dev *hdev, void *data, int err) } complete: + if (conn) + hci_conn_put(conn); + mgmt_cmd_complete(cmd->sk, cmd->hdev->id, cmd->opcode, status, &rp, sizeof(rp)); @@ -7485,15 +7589,21 @@ static int get_clock_info_sync(struct hci_dev *hdev, void *data) memset(&hci_cp, 0, sizeof(hci_cp)); hci_read_clock_sync(hdev, &hci_cp); + hci_dev_lock(hdev); + /* Make sure connection still exists */ conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &cp->addr.bdaddr); - if (!conn || conn->state != BT_CONNECTED) + if (!conn || conn->state != BT_CONNECTED) { + hci_dev_unlock(hdev); return MGMT_STATUS_NOT_CONNECTED; + } - cmd->user_data = conn; + cmd->user_data = hci_conn_get(conn); hci_cp.handle = cpu_to_le16(conn->handle); hci_cp.which = 0x01; /* Piconet clock */ + hci_dev_unlock(hdev); + return hci_read_clock_sync(hdev, &hci_cp); } @@ -7924,14 +8034,36 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev, static int conn_update_sync(struct hci_dev *hdev, void *data) { - struct hci_conn_params *params = data; - struct hci_conn *conn; + struct hci_conn *conn = data; + struct hci_conn_params *params; + struct hci_conn_params local = {}; - conn = hci_conn_hash_lookup_le(hdev, ¶ms->addr, params->addr_type); - if (!conn) - return -ECANCELED; + hci_dev_lock(hdev); + + if (!hci_conn_valid(hdev, conn) || conn->role != HCI_ROLE_MASTER) + goto cancel; + + params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type); + if (!params) + goto cancel; + + local.conn_min_interval = params->conn_min_interval; + local.conn_max_interval = params->conn_max_interval; + local.conn_latency = params->conn_latency; + local.supervision_timeout = params->supervision_timeout; + + hci_dev_unlock(hdev); + + return hci_le_conn_update_sync(hdev, conn, &local); - return hci_le_conn_update_sync(hdev, conn, params); +cancel: + hci_dev_unlock(hdev); + return -ECANCELED; +} + +static void conn_update_sync_destroy(struct hci_dev *hdev, void *data, int err) +{ + hci_conn_put(data); } static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data, @@ -8041,9 +8173,13 @@ static int load_conn_param(struct sock *sk, struct hci_dev *hdev, void *data, (conn->le_conn_min_interval != min || conn->le_conn_max_interval != max || conn->le_conn_latency != latency || - conn->le_supv_timeout != timeout)) - hci_cmd_sync_queue(hdev, conn_update_sync, - hci_param, NULL); + conn->le_supv_timeout != timeout)) { + hci_conn_get(conn); + if (hci_cmd_sync_queue(hdev, conn_update_sync, + conn, + conn_update_sync_destroy) < 0) + hci_conn_put(conn); + } } } @@ -10047,14 +10183,14 @@ void mgmt_auth_failed(struct hci_conn *conn, u8 hci_status) ev.addr.type = link_to_bdaddr(conn->type, conn->dst_type); ev.status = status; - cmd = find_pairing(conn); + cmd = remove_pairing(conn); mgmt_event(MGMT_EV_AUTH_FAILED, conn->hdev, &ev, sizeof(ev), cmd ? cmd->sk : NULL); if (cmd) { cmd->cmd_complete(cmd, status); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); } } @@ -10233,6 +10369,7 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, !hci_test_quirk(hdev, HCI_QUIRK_STRICT_DUPLICATE_FILTER)))) return false; + spin_lock(&hdev->discovery.lock); if (hdev->discovery.uuid_count != 0) { /* If a list of UUIDs is provided in filter, results with no * matching UUID should be dropped. @@ -10241,9 +10378,12 @@ static bool is_filter_match(struct hci_dev *hdev, s8 rssi, u8 *eir, hdev->discovery.uuids) && !eir_has_uuids(scan_rsp, scan_rsp_len, hdev->discovery.uuid_count, - hdev->discovery.uuids)) + hdev->discovery.uuids)) { + spin_unlock(&hdev->discovery.lock); return false; + } } + spin_unlock(&hdev->discovery.lock); /* If duplicate filtering does not report RSSI changes, then restart * scanning to ensure updated result with updated RSSI values. diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index dea81c8e8b201..e1b779014dde9 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -165,6 +165,11 @@ static bool read_supported_features(struct hci_dev *hdev, if (rp->sub_opcode != MSFT_OP_READ_SUPPORTED_FEATURES) goto failed; + if (skb->len < sizeof(*rp) + rp->evt_prefix_len) { + bt_dev_err(hdev, "MSFT event prefix length mismatch"); + goto failed; + } + if (rp->evt_prefix_len > 0) { msft->evt_prefix = kmemdup(rp->evt_prefix, rp->evt_prefix_len, GFP_KERNEL); @@ -764,8 +769,8 @@ void msft_register(struct hci_dev *hdev) INIT_LIST_HEAD(&msft->handle_map); INIT_LIST_HEAD(&msft->address_filters); - hdev->msft_data = msft; mutex_init(&msft->filter_lock); + hdev->msft_data = msft; } void msft_release(struct hci_dev *hdev) diff --git a/net/bluetooth/rfcomm/core.c b/net/bluetooth/rfcomm/core.c index dfade1933fa73..45f8cfebcc6a6 100644 --- a/net/bluetooth/rfcomm/core.c +++ b/net/bluetooth/rfcomm/core.c @@ -1031,6 +1031,23 @@ int rfcomm_send_rpn(struct rfcomm_session *s, int cr, u8 dlci, return rfcomm_send_frame(s, buf, ptr - buf); } +int rfcomm_dlc_send_rpn(struct rfcomm_dlc *d, u8 bit_rate, u8 data_bits, + u8 stop_bits, u8 parity, u8 flow_ctrl_settings, + u8 xon_char, u8 xoff_char, u16 param_mask) +{ + int err = -ENOTCONN; + + rfcomm_lock(); + if (d->session) + err = rfcomm_send_rpn(d->session, 1, d->dlci, bit_rate, + data_bits, stop_bits, parity, + flow_ctrl_settings, xon_char, xoff_char, + param_mask); + rfcomm_unlock(); + + return err; +} + static int rfcomm_send_rls(struct rfcomm_session *s, int cr, u8 dlci, u8 status) { struct rfcomm_hdr *hdr; @@ -1317,7 +1334,10 @@ static struct rfcomm_session *rfcomm_recv_disc(struct rfcomm_session *s, return s; } -void rfcomm_dlc_accept(struct rfcomm_dlc *d) +/* Must be called with rfcomm_mutex held, so that the session cannot be + * unlinked from under us. + */ +static void __rfcomm_dlc_accept(struct rfcomm_dlc *d) { struct sock *sk = d->session->sock->sk; struct l2cap_conn *conn = l2cap_pi(sk)->chan->conn; @@ -1339,6 +1359,21 @@ void rfcomm_dlc_accept(struct rfcomm_dlc *d) rfcomm_send_msc(d->session, 1, d->dlci, d->v24_sig); } +void rfcomm_dlc_accept(struct rfcomm_dlc *d) +{ + rfcomm_lock(); + + /* rfcomm_recv_disc() sets the dlc state to BT_CLOSED before calling + * __rfcomm_dlc_close(), so the RFCOMM_DEFER_SETUP handshake there is + * skipped and the session can already be unlinked by the time the + * deferred accept runs from rfcomm_sock_recvmsg(). + */ + if (d->session) + __rfcomm_dlc_accept(d); + + rfcomm_unlock(); +} + static void rfcomm_check_accept(struct rfcomm_dlc *d) { if (rfcomm_check_security(d)) { @@ -1351,7 +1386,7 @@ static void rfcomm_check_accept(struct rfcomm_dlc *d) d->state_change(d, 0); rfcomm_dlc_unlock(d); } else - rfcomm_dlc_accept(d); + __rfcomm_dlc_accept(d); } else { set_bit(RFCOMM_AUTH_PENDING, &d->flags); rfcomm_dlc_set_timer(d, RFCOMM_AUTH_TIMEOUT); @@ -1423,6 +1458,10 @@ static int rfcomm_apply_pn(struct rfcomm_dlc *d, int cr, struct rfcomm_pn *pn) d->mtu = __le16_to_cpu(pn->mtu); + /* MTU 0 causes an infinite loop when fragmenting in sendmsg */ + if (!d->mtu) + d->mtu = RFCOMM_DEFAULT_MTU; + if (cr && d->mtu > s->mtu) d->mtu = s->mtu; @@ -1778,6 +1817,11 @@ static struct rfcomm_session *rfcomm_recv_frame(struct rfcomm_session *s, return s; } + if (skb->len < sizeof(*hdr) + 1) { + kfree_skb(skb); + return s; + } + dlci = __get_dlci(hdr->addr); type = __get_type(hdr->ctrl); @@ -1936,7 +1980,7 @@ static void rfcomm_process_dlcs(struct rfcomm_session *s) d->state_change(d, 0); rfcomm_dlc_unlock(d); } else - rfcomm_dlc_accept(d); + __rfcomm_dlc_accept(d); } continue; } else if (test_and_clear_bit(RFCOMM_AUTH_REJECT, &d->flags)) { @@ -2173,9 +2217,13 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) BT_DBG("conn %p status 0x%02x encrypt 0x%02x", conn, status, encrypt); + rfcomm_lock(); + s = rfcomm_session_get(&conn->hdev->bdaddr, &conn->dst); - if (!s) + if (!s) { + rfcomm_unlock(); return; + } list_for_each_entry_safe(d, n, &s->dlcs, list) { if (test_and_clear_bit(RFCOMM_SEC_PENDING, &d->flags)) { @@ -2207,6 +2255,8 @@ static void rfcomm_security_cfm(struct hci_conn *conn, u8 status, u8 encrypt) set_bit(RFCOMM_AUTH_REJECT, &d->flags); } + rfcomm_unlock(); + rfcomm_schedule(); } diff --git a/net/bluetooth/rfcomm/tty.c b/net/bluetooth/rfcomm/tty.c index b783526ab5884..63f208abbd514 100644 --- a/net/bluetooth/rfcomm/tty.c +++ b/net/bluetooth/rfcomm/tty.c @@ -861,7 +861,7 @@ static void rfcomm_tty_set_termios(struct tty_struct *tty, BT_DBG("tty %p termios %p", tty, old); - if (!dev || !dev->dlc || !dev->dlc->session) + if (!dev || !dev->dlc) return; /* Handle turning off CRTSCTS */ @@ -982,9 +982,8 @@ static void rfcomm_tty_set_termios(struct tty_struct *tty, } if (changes) - rfcomm_send_rpn(dev->dlc->session, 1, dev->dlc->dlci, baud, - data_bits, stop_bits, parity, - RFCOMM_RPN_FLOW_NONE, x_on, x_off, changes); + rfcomm_dlc_send_rpn(dev->dlc, baud, data_bits, stop_bits, parity, + RFCOMM_RPN_FLOW_NONE, x_on, x_off, changes); } static void rfcomm_tty_throttle(struct tty_struct *tty) diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c index 285a98594ba43..a25b5f6263589 100644 --- a/net/bluetooth/sco.c +++ b/net/bluetooth/sco.c @@ -190,6 +190,9 @@ static void sco_sock_clear_timer(struct sock *sk) } /* ---- SCO connections ---- */ +/* Consumes a reference on @hcon, which the returned sco_conn owns until it is + * freed. On failure (NULL return) the reference is left for the caller to drop. + */ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) { struct sco_conn *conn = hcon->sco_data; @@ -200,6 +203,9 @@ static struct sco_conn *sco_conn_add(struct hci_conn *hcon) sco_conn_lock(conn); conn->hcon = hcon; sco_conn_unlock(conn); + } else { + /* conn already owns a reference on hcon */ + hci_conn_drop(hcon); } return conn; } @@ -267,10 +273,8 @@ static void sco_conn_del(struct hci_conn *hcon, int err) sco_conn_unlock(conn); sco_conn_put(conn); - if (!sk) { - sco_conn_put(conn); + if (!sk) return; - } /* Kill socket */ lock_sock(sk); @@ -285,7 +289,7 @@ static void __sco_chan_add(struct sco_conn *conn, struct sock *sk, { BT_DBG("conn %p", conn); - sco_pi(sk)->conn = conn; + sco_pi(sk)->conn = sco_conn_hold(conn); conn->sk = sk; if (parent) @@ -368,15 +372,15 @@ static int sco_connect(struct sock *sk) */ if (sk->sk_state != BT_OPEN && sk->sk_state != BT_BOUND) { release_sock(sk); - hci_conn_drop(hcon); + sco_conn_put(conn); err = -EBADFD; goto unlock; } err = sco_chan_add(conn, sk, NULL); + sco_conn_put(conn); if (err) { release_sock(sk); - hci_conn_drop(hcon); goto unlock; } @@ -1452,8 +1456,6 @@ static void sco_conn_ready(struct sco_conn *conn) bacpy(&sco_pi(sk)->src, &conn->hcon->src); bacpy(&sco_pi(sk)->dst, &conn->hcon->dst); - sco_conn_hold(conn); - hci_conn_hold(conn->hcon); __sco_chan_add(conn, sk, parent); if (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags)) @@ -1509,10 +1511,12 @@ static void sco_connect_cfm(struct hci_conn *hcon, __u8 status) if (!status) { struct sco_conn *conn; - conn = sco_conn_add(hcon); + conn = sco_conn_add(hci_conn_hold(hcon)); if (conn) { sco_conn_ready(conn); sco_conn_put(conn); + } else { + hci_conn_drop(hcon); } } else sco_conn_del(hcon, bt_to_errno(status)); diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c index 7ace0f4941bb6..4cd3470e5f56c 100644 --- a/net/bridge/br_arp_nd_proxy.c +++ b/net/bridge/br_arp_nd_proxy.c @@ -165,7 +165,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br, return; if (parp->ar_op != htons(ARPOP_RREQUEST) && parp->ar_op != htons(ARPOP_RREPLY) && - (ipv4_is_zeronet(sip) || sip == tip)) { + sip == tip) { /* prevent flooding to neigh suppress ports */ BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1; return; @@ -263,6 +263,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, int ns_olen; int i, len; u8 *daddr; + bool dad; u16 pvid; if (!dev || skb_linearize(request)) @@ -301,8 +302,13 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, } } + dad = ipv6_addr_any(&ipv6_hdr(request)->saddr); + /* Ethernet header */ - ether_addr_copy(eth_hdr(reply)->h_dest, daddr); + if (dad) + ipv6_eth_mc_map(&in6addr_linklocal_allnodes, eth_hdr(reply)->h_dest); + else + ether_addr_copy(eth_hdr(reply)->h_dest, daddr); ether_addr_copy(eth_hdr(reply)->h_source, n->ha); eth_hdr(reply)->h_proto = htons(ETH_P_IPV6); reply->protocol = htons(ETH_P_IPV6); @@ -318,7 +324,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, pip6->priority = ipv6_hdr(request)->priority; pip6->nexthdr = IPPROTO_ICMPV6; pip6->hop_limit = 255; - pip6->daddr = ipv6_hdr(request)->saddr; + pip6->daddr = dad ? in6addr_linklocal_allnodes : ipv6_hdr(request)->saddr; pip6->saddr = *(struct in6_addr *)n->primary_key; skb_pull(reply, sizeof(struct ipv6hdr)); @@ -331,7 +337,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p, na->icmph.icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT; na->icmph.icmp6_router = (n->flags & NTF_ROUTER) ? 1 : 0; na->icmph.icmp6_override = 1; - na->icmph.icmp6_solicited = 1; + na->icmph.icmp6_solicited = dad ? 0 : 1; na->target = ns->target; ether_addr_copy(&na->opt[2], n->ha); na->opt[0] = ND_OPT_TARGET_LL_ADDR; @@ -436,7 +442,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br, saddr = &iphdr->saddr; daddr = &iphdr->daddr; - if (ipv6_addr_any(saddr) || !ipv6_addr_cmp(saddr, daddr)) { + if (!ipv6_addr_cmp(saddr, daddr)) { /* prevent flooding to neigh suppress ports */ BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1; return; diff --git a/net/bridge/br_cfm.c b/net/bridge/br_cfm.c index e1de280bacc1b..85ca5b05852e3 100644 --- a/net/bridge/br_cfm.c +++ b/net/bridge/br_cfm.c @@ -367,7 +367,7 @@ static u32 ccm_tlv_extract(struct sk_buff *skb, u32 index, } /* note: already called with rcu_read_lock */ -static int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) +int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) { u32 mdlevel, interval, size, index, max; const struct br_cfm_common_hdr *hdr; @@ -489,11 +489,6 @@ static int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb) return 1; } -static struct br_frame_type cfm_frame_type __read_mostly = { - .type = cpu_to_be16(ETH_P_CFM), - .frame_handler = br_cfm_frame_rx, -}; - int br_cfm_mep_create(struct net_bridge *br, const u32 instance, struct br_cfm_mep_create *const create, @@ -559,7 +554,7 @@ int br_cfm_mep_create(struct net_bridge *br, INIT_DELAYED_WORK(&mep->ccm_tx_dwork, ccm_tx_work_expired); if (hlist_empty(&br->mep_list)) - br_add_frame(br, &cfm_frame_type); + br_opt_toggle(br, BROPT_CFM_ENABLED, true); hlist_add_tail_rcu(&mep->head, &br->mep_list); @@ -588,7 +583,7 @@ static void mep_delete_implementation(struct net_bridge *br, kfree_rcu(mep, rcu); if (hlist_empty(&br->mep_list)) - br_del_frame(br, &cfm_frame_type); + br_opt_toggle(br, BROPT_CFM_ENABLED, false); } int br_cfm_mep_delete(struct net_bridge *br, diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c index 525d4eccd194a..3055a72959cb6 100644 --- a/net/bridge/br_device.c +++ b/net/bridge/br_device.c @@ -107,7 +107,7 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev) else br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid); } else if ((dst = br_fdb_find_rcu(br, dest, vid)) != NULL) { - br_forward(dst->dst, skb, false, true); + br_forward(READ_ONCE(dst->dst), skb, false, true); } else { br_flood(br, skb, BR_PKT_UNICAST, false, true, vid); } @@ -503,7 +503,6 @@ void br_dev_setup(struct net_device *dev) spin_lock_init(&br->lock); INIT_LIST_HEAD(&br->port_list); INIT_HLIST_HEAD(&br->fdb_list); - INIT_HLIST_HEAD(&br->frame_type_list); #if IS_ENABLED(CONFIG_BRIDGE_MRP) INIT_HLIST_HEAD(&br->mrp_list); #endif diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c index 6eb3ab69a5140..fe85c8f197e67 100644 --- a/net/bridge/br_fdb.c +++ b/net/bridge/br_fdb.c @@ -470,7 +470,8 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr) spin_lock_bh(&br->hash_lock); vg = nbp_vlan_group(p); hlist_for_each_entry(f, &br->fdb_list, fdb_node) { - if (f->dst == p && test_bit(BR_FDB_LOCAL, &f->flags) && + if (READ_ONCE(f->dst) == p && + test_bit(BR_FDB_LOCAL, &f->flags) && !test_bit(BR_FDB_ADDED_BY_USER, &f->flags)) { /* delete old one */ fdb_delete_local(br, p, f); @@ -878,7 +879,7 @@ void br_fdb_delete_by_port(struct net_bridge *br, spin_lock_bh(&br->hash_lock); hlist_for_each_entry_safe(f, tmp, &br->fdb_list, fdb_node) { - if (f->dst != p) + if (READ_ONCE(f->dst) != p) continue; if (!do_all) @@ -1660,7 +1661,7 @@ void br_fdb_clear_offload(const struct net_device *dev, u16 vid) spin_lock_bh(&p->br->hash_lock); hlist_for_each_entry(f, &p->br->fdb_list, fdb_node) { - if (f->dst == p && f->key.vlan_id == vid) + if (READ_ONCE(f->dst) == p && f->key.vlan_id == vid) clear_bit(BR_FDB_OFFLOADED, &f->flags); } spin_unlock_bh(&p->br->hash_lock); diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c index 2cbae0f9ae1f0..ac0f56076e93b 100644 --- a/net/bridge/br_input.c +++ b/net/bridge/br_input.c @@ -223,7 +223,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb if (now != READ_ONCE(dst->used)) WRITE_ONCE(dst->used, now); - br_forward(dst->dst, skb, local_rcv, false); + br_forward(READ_ONCE(dst->dst), skb, local_rcv, false); } else { if (!mcast_hit) br_flood(br, skb, pkt_type, local_rcv, false, vid); @@ -317,17 +317,25 @@ static int nf_hook_bridge_pre(struct sk_buff *skb, struct sk_buff **pskb) return RX_HANDLER_CONSUMED; } +#define BR_CFM_MRP_OPTS \ + ((IS_ENABLED(CONFIG_BRIDGE_CFM) ? BIT(BROPT_CFM_ENABLED) : 0UL) | \ + (IS_ENABLED(CONFIG_BRIDGE_MRP) ? BIT(BROPT_MRP_ENABLED) : 0UL)) + /* Return 0 if the frame was not processed otherwise 1 * note: already called with rcu_read_lock */ static int br_process_frame_type(struct net_bridge_port *p, struct sk_buff *skb) { - struct br_frame_type *tmp; + struct net_bridge *br = p->br; + + if (skb->protocol == htons(ETH_P_CFM) && + br_opt_get(br, BROPT_CFM_ENABLED)) + return br_cfm_frame_rx(p, skb); - hlist_for_each_entry_rcu(tmp, &p->br->frame_type_list, list) - if (unlikely(tmp->type == skb->protocol)) - return tmp->frame_handler(p, skb); + if (skb->protocol == htons(ETH_P_MRP) && + br_opt_get(br, BROPT_MRP_ENABLED)) + return br_mrp_process(p, skb); return 0; } @@ -425,7 +433,8 @@ static rx_handler_result_t br_handle_frame(struct sk_buff **pskb) } } - if (unlikely(br_process_frame_type(p, skb))) + if (unlikely((READ_ONCE(p->br->options) & BR_CFM_MRP_OPTS) && + br_process_frame_type(p, skb))) return RX_HANDLER_PASS; forward: @@ -467,19 +476,3 @@ rx_handler_func_t *br_get_rx_handler(const struct net_device *dev) return br_handle_frame; } - -void br_add_frame(struct net_bridge *br, struct br_frame_type *ft) -{ - hlist_add_head_rcu(&ft->list, &br->frame_type_list); -} - -void br_del_frame(struct net_bridge *br, struct br_frame_type *ft) -{ - struct br_frame_type *tmp; - - hlist_for_each_entry(tmp, &br->frame_type_list, list) - if (ft == tmp) { - hlist_del_rcu(&ft->list); - return; - } -} diff --git a/net/bridge/br_mrp.c b/net/bridge/br_mrp.c index 3c36fa24bc05a..3613095a95fcf 100644 --- a/net/bridge/br_mrp.c +++ b/net/bridge/br_mrp.c @@ -6,13 +6,6 @@ static const u8 mrp_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x1 }; static const u8 mrp_in_test_dmac[ETH_ALEN] = { 0x1, 0x15, 0x4e, 0x0, 0x0, 0x3 }; -static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb); - -static struct br_frame_type mrp_frame_type __read_mostly = { - .type = cpu_to_be16(ETH_P_MRP), - .frame_handler = br_mrp_process, -}; - static bool br_mrp_is_ring_port(struct net_bridge_port *p_port, struct net_bridge_port *s_port, struct net_bridge_port *port) @@ -215,7 +208,7 @@ static struct sk_buff *br_mrp_alloc_test_skb(struct br_mrp *mrp, struct br_mrp_oui_hdr *oui = NULL; u8 length; - length = sizeof(*sub_opt) + sizeof(*sub_tlv) + sizeof(oui) + + length = sizeof(*sub_opt) + sizeof(*sub_tlv) + sizeof(*oui) + MRP_OPT_PADDING; br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_OPTION, length); @@ -224,11 +217,9 @@ static struct sk_buff *br_mrp_alloc_test_skb(struct br_mrp *mrp, sub_opt = skb_put(skb, sizeof(*sub_opt)); memset(sub_opt, 0x0, sizeof(*sub_opt)); - sub_tlv = skb_put(skb, sizeof(*sub_tlv)); - sub_tlv->type = BR_MRP_SUB_TLV_HEADER_TEST_AUTO_MGR; - /* 32 bit alligment shall be ensured therefore add 2 bytes */ - skb_put(skb, MRP_OPT_PADDING); + sub_tlv = skb_put_zero(skb, sizeof(*sub_tlv) + MRP_OPT_PADDING); + sub_tlv->type = BR_MRP_SUB_TLV_HEADER_TEST_AUTO_MGR; } br_mrp_skb_tlv(skb, BR_MRP_TLV_HEADER_END, 0x0); @@ -488,7 +479,7 @@ static void br_mrp_del_impl(struct net_bridge *br, struct br_mrp *mrp) kfree_rcu(mrp, rcu); if (hlist_empty(&br->mrp_list)) - br_del_frame(br, &mrp_frame_type); + br_opt_toggle(br, BROPT_MRP_ENABLED, false); } /* Adds a new MRP instance. @@ -538,7 +529,7 @@ int br_mrp_add(struct net_bridge *br, struct br_mrp_instance *instance) rcu_assign_pointer(mrp->s_port, p); if (hlist_empty(&br->mrp_list)) - br_add_frame(br, &mrp_frame_type); + br_opt_toggle(br, BROPT_MRP_ENABLED, true); INIT_DELAYED_WORK(&mrp->test_work, br_mrp_test_work_expired); INIT_DELAYED_WORK(&mrp->in_test_work, br_mrp_in_test_work_expired); @@ -1243,7 +1234,7 @@ static int br_mrp_rcv(struct net_bridge_port *p, * normal forwarding. * note: already called with rcu_read_lock */ -static int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) +int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) { /* If there is no MRP instance do normal forwarding */ if (likely(!(p->flags & BR_MRP_AWARE))) diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c index b77b3250afbd2..fda1f89ee22ae 100644 --- a/net/bridge/br_multicast.c +++ b/net/bridge/br_multicast.c @@ -1437,16 +1437,17 @@ struct net_bridge_port_group *br_multicast_new_port_group( goto free_out; } - rcu_assign_pointer(p->next, next); timer_setup(&p->timer, br_multicast_port_group_expired, 0); timer_setup(&p->rexmit_timer, br_multicast_port_group_rexmit, 0); - hlist_add_head(&p->mglist, &port->mglist); if (src) memcpy(p->eth_addr, src, ETH_ALEN); else eth_broadcast_addr(p->eth_addr); + RCU_INIT_POINTER(p->next, next); + hlist_add_head_rcu(&p->mglist, &port->mglist); + return p; free_out: @@ -1461,11 +1462,11 @@ void br_multicast_del_port_group(struct net_bridge_port_group *p) struct net_bridge_port *port = p->key.port; __u16 vid = p->key.addr.vid; - hlist_del_init(&p->mglist); + hlist_del_init_rcu(&p->mglist); if (!br_multicast_is_star_g(&p->key.addr)) rhashtable_remove_fast(&port->br->sg_port_tbl, &p->rhnode, br_sg_port_rht_params); - kfree(p); + kfree_rcu(p, rcu); br_multicast_port_ngroups_dec(port, vid); } @@ -3686,6 +3687,7 @@ br_multicast_leave_group(struct net_bridge_mcast *brmctx, p->flags |= MDB_PG_FLAGS_FAST_LEAVE; br_multicast_del_pg(mp, p, pp); + break; } goto out; } @@ -4310,8 +4312,8 @@ void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on) if (br_vlan_is_master(vlan)) { br = vlan->br; - if (!br_vlan_is_brentry(vlan) || - (on && + if (on && + (!br_vlan_is_brentry(vlan) || br_multicast_ctx_vlan_global_disabled(&vlan->br_mcast_ctx))) return; @@ -4455,8 +4457,6 @@ void br_multicast_dev_del(struct net_bridge *br) br_multicast_ctx_deinit(&br->multicast_ctx); br_multicast_gc(&deleted_head); cancel_work_sync(&br->mcast_gc_work); - - rcu_barrier(); } int br_multicast_set_router(struct net_bridge_mcast *brmctx, unsigned long val) diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c index 083e2fe96441d..6544c18febefe 100644 --- a/net/bridge/br_netfilter_hooks.c +++ b/net/bridge/br_netfilter_hooks.c @@ -296,7 +296,11 @@ int br_nf_pre_routing_finish_bridge(struct net *net, struct sock *sk, struct sk_ goto free_skb; } - neigh_hh_bridge(&neigh->hh, skb); + if (neigh_hh_bridge(&neigh->hh, skb)) { + neigh_release(neigh); + goto free_skb; + } + skb->dev = br_indev; ret = br_handle_frame_finish(net, sk, skb); diff --git a/net/bridge/br_netlink_tunnel.c b/net/bridge/br_netlink_tunnel.c index 71a12da30004c..e7eceab5b515d 100644 --- a/net/bridge/br_netlink_tunnel.c +++ b/net/bridge/br_netlink_tunnel.c @@ -271,7 +271,8 @@ static void __vlan_tunnel_handle_range(const struct net_bridge_port *p, if (!*v_start) goto out_init; - if (v && curr_change && br_vlan_can_enter_range(v, *v_end)) { + if (v && curr_change && + br_vlan_can_enter_range(v, *v_end, br_get_pvid(vg))) { *v_end = v; return; } @@ -301,7 +302,8 @@ int br_process_vlan_tunnel_info(const struct net_bridge *br, if (!(tinfo_last->flags & BRIDGE_VLAN_INFO_RANGE_BEGIN)) return -EINVAL; - if ((tinfo_curr->vid - tinfo_last->vid) != + if (tinfo_curr->vid < tinfo_last->vid || + (tinfo_curr->vid - tinfo_last->vid) != (tinfo_curr->tunid - tinfo_last->tunid)) return -EINVAL; t = tinfo_last->tunid; diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index b9b2981c48414..e4c8bc8576df6 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -489,12 +489,13 @@ enum net_bridge_opts { BROPT_MST_ENABLED, BROPT_MDB_OFFLOAD_FAIL_NOTIFICATION, BROPT_FDB_LOCAL_VLAN_0, + BROPT_CFM_ENABLED, + BROPT_MRP_ENABLED, }; struct net_bridge { spinlock_t lock; spinlock_t hash_lock; - struct hlist_head frame_type_list; struct net_device *dev; unsigned long options; /* These fields are accessed on each packet */ @@ -924,16 +925,6 @@ int nbp_backup_change(struct net_bridge_port *p, struct net_device *backup_dev); int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb); rx_handler_func_t *br_get_rx_handler(const struct net_device *dev); -struct br_frame_type { - __be16 type; - int (*frame_handler)(struct net_bridge_port *port, - struct sk_buff *skb); - struct hlist_node list; -}; - -void br_add_frame(struct net_bridge *br, struct br_frame_type *ft); -void br_del_frame(struct net_bridge *br, struct br_frame_type *ft); - static inline bool br_rx_handler_check_rcu(const struct net_device *dev) { return rcu_dereference(dev->rx_handler) == br_get_rx_handler(dev); @@ -1613,7 +1604,8 @@ void br_vlan_notify(const struct net_bridge *br, u16 vid, u16 vid_range, int cmd); bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end); + const struct net_bridge_vlan *range_end, + u16 pvid); void br_vlan_fill_forward_path_pvid(struct net_bridge *br, struct net_device_path_ctx *ctx, @@ -1860,7 +1852,8 @@ static inline void br_vlan_notify(const struct net_bridge *br, } static inline bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end) + const struct net_bridge_vlan *range_end, + u16 pvid) { return true; } @@ -2063,6 +2056,7 @@ int br_mrp_parse(struct net_bridge *br, struct net_bridge_port *p, bool br_mrp_enabled(struct net_bridge *br); void br_mrp_port_del(struct net_bridge *br, struct net_bridge_port *p); int br_mrp_fill_info(struct sk_buff *skb, struct net_bridge *br); +int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb); #else static inline int br_mrp_parse(struct net_bridge *br, struct net_bridge_port *p, struct nlattr *attr, int cmd, @@ -2086,6 +2080,11 @@ static inline int br_mrp_fill_info(struct sk_buff *skb, struct net_bridge *br) return 0; } +static inline int br_mrp_process(struct net_bridge_port *p, struct sk_buff *skb) +{ + return 0; +} + #endif /* br_cfm.c */ @@ -2094,6 +2093,7 @@ int br_cfm_parse(struct net_bridge *br, struct net_bridge_port *p, struct nlattr *attr, int cmd, struct netlink_ext_ack *extack); bool br_cfm_created(struct net_bridge *br); void br_cfm_port_del(struct net_bridge *br, struct net_bridge_port *p); +int br_cfm_frame_rx(struct net_bridge_port *port, struct sk_buff *skb); int br_cfm_config_fill_info(struct sk_buff *skb, struct net_bridge *br); int br_cfm_status_fill_info(struct sk_buff *skb, struct net_bridge *br, @@ -2118,6 +2118,12 @@ static inline void br_cfm_port_del(struct net_bridge *br, { } +static inline int br_cfm_frame_rx(struct net_bridge_port *port, + struct sk_buff *skb) +{ + return 0; +} + static inline int br_cfm_config_fill_info(struct sk_buff *skb, struct net_bridge *br) { return -EOPNOTSUPP; diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index ce72b837ff8ee..76da4f580d4c4 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -1131,7 +1131,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, if (err) goto out; - if (br_vlan_delete(br, old_pvid)) + if (!br_vlan_delete(br, old_pvid)) br_vlan_notify(br, NULL, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(br, NULL, pvid, 0, RTM_NEWVLAN); __set_bit(0, changed); @@ -1153,7 +1153,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, &vlchange, extack); if (err) goto err_port; - if (nbp_vlan_delete(p, old_pvid)) + if (!nbp_vlan_delete(p, old_pvid)) br_vlan_notify(br, p, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(p->br, p, pvid, 0, RTM_NEWVLAN); __set_bit(p->port_no, changed); @@ -1977,9 +1977,11 @@ void br_vlan_notify(const struct net_bridge *br, /* check if v_curr can enter a range ending in range_end */ bool br_vlan_can_enter_range(const struct net_bridge_vlan *v_curr, - const struct net_bridge_vlan *range_end) + const struct net_bridge_vlan *range_end, + u16 pvid) { - return v_curr->vid - range_end->vid == 1 && + return v_curr->vid != pvid && range_end->vid != pvid && + v_curr->vid - range_end->vid == 1 && range_end->flags == v_curr->flags && br_vlan_opts_eq_range(v_curr, range_end); } @@ -2061,8 +2063,8 @@ static int br_vlan_dump_dev(const struct net_device *dev, idx += range_end->vid - range_start->vid + 1; range_start = v; - } else if (dump_stats || v->vid == pvid || - !br_vlan_can_enter_range(v, range_end)) { + } else if (dump_stats || + !br_vlan_can_enter_range(v, range_end, pvid)) { u16 vlan_flags = br_vlan_flags(range_start, pvid); if (!br_vlan_fill_vids(skb, range_start->vid, diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c index 8fa89b04ee942..4a736e005bad7 100644 --- a/net/bridge/br_vlan_options.c +++ b/net/bridge/br_vlan_options.c @@ -310,8 +310,7 @@ int br_vlan_process_options(const struct net_bridge *br, continue; } - if (v->vid == pvid || - !br_vlan_can_enter_range(v, curr_end)) { + if (!br_vlan_can_enter_range(v, curr_end, pvid)) { br_vlan_notify(br, p, curr_start->vid, curr_end->vid, RTM_NEWVLAN); curr_start = v; diff --git a/net/bridge/netfilter/ebt_nflog.c b/net/bridge/netfilter/ebt_nflog.c index 61bf8f4465ab7..426f8adc912c7 100644 --- a/net/bridge/netfilter/ebt_nflog.c +++ b/net/bridge/netfilter/ebt_nflog.c @@ -41,11 +41,25 @@ ebt_nflog_tg(struct sk_buff *skb, const struct xt_action_param *par) static int ebt_nflog_tg_check(const struct xt_tgchk_param *par) { struct ebt_nflog_info *info = par->targinfo; + int ret; if (info->flags & ~EBT_NFLOG_MASK) return -EINVAL; info->prefix[EBT_NFLOG_PREFIX_SIZE - 1] = '\0'; - return 0; + + ret = nf_logger_find_get(par->family, NF_LOG_TYPE_ULOG); + if (ret != 0 && !par->nft_compat) { + request_module("%s", "nfnetlink_log"); + + ret = nf_logger_find_get(par->family, NF_LOG_TYPE_ULOG); + } + + return ret; +} + +static void ebt_nflog_tg_destroy(const struct xt_tgdtor_param *par) +{ + nf_logger_put(par->family, NF_LOG_TYPE_ULOG); } static struct xt_target ebt_nflog_tg_reg __read_mostly = { @@ -54,6 +68,7 @@ static struct xt_target ebt_nflog_tg_reg __read_mostly = { .family = NFPROTO_BRIDGE, .target = ebt_nflog_tg, .checkentry = ebt_nflog_tg_check, + .destroy = ebt_nflog_tg_destroy, .targetsize = sizeof(struct ebt_nflog_info), .me = THIS_MODULE, }; diff --git a/net/bridge/netfilter/nf_conntrack_bridge.c b/net/bridge/netfilter/nf_conntrack_bridge.c index 6482de4d87509..e4dae5fe0b0d1 100644 --- a/net/bridge/netfilter/nf_conntrack_bridge.c +++ b/net/bridge/netfilter/nf_conntrack_bridge.c @@ -281,6 +281,7 @@ static unsigned int nf_ct_bridge_pre(void *priv, struct sk_buff *skb, ret = nf_ct_br_defrag6(skb, &bridge_state); break; default: + nf_reset_ct(skb); nf_ct_set(skb, NULL, IP_CT_UNTRACKED); return NF_ACCEPT; } diff --git a/net/bridge/netfilter/nft_meta_bridge.c b/net/bridge/netfilter/nft_meta_bridge.c index a3d0c8a82d865..e4c9aa1f64e25 100644 --- a/net/bridge/netfilter/nft_meta_bridge.c +++ b/net/bridge/netfilter/nft_meta_bridge.c @@ -134,7 +134,6 @@ static const struct nft_expr_ops nft_meta_bridge_get_ops = { .init = nft_meta_bridge_get_init, .validate = nft_meta_bridge_get_validate, .dump = nft_meta_get_dump, - .reduce = nft_meta_get_reduce, }; static void nft_meta_bridge_set_eval(const struct nft_expr *expr, @@ -181,24 +180,6 @@ static int nft_meta_bridge_set_init(const struct nft_ctx *ctx, return 0; } -static bool nft_meta_bridge_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_meta_bridge_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static int nft_meta_bridge_set_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { @@ -223,7 +204,6 @@ static const struct nft_expr_ops nft_meta_bridge_set_ops = { .init = nft_meta_bridge_set_init, .destroy = nft_meta_set_destroy, .dump = nft_meta_set_dump, - .reduce = nft_meta_bridge_set_reduce, .validate = nft_meta_bridge_set_validate, }; diff --git a/net/bridge/netfilter/nft_reject_bridge.c b/net/bridge/netfilter/nft_reject_bridge.c index 1cb5c16e97b7f..cd2b04236a99c 100644 --- a/net/bridge/netfilter/nft_reject_bridge.c +++ b/net/bridge/netfilter/nft_reject_bridge.c @@ -184,7 +184,6 @@ static const struct nft_expr_ops nft_reject_bridge_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_bridge_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_bridge_type __read_mostly = { diff --git a/net/can/af_can.c b/net/can/af_can.c index a624c04ed5c63..7a026f3a70a0e 100644 --- a/net/can/af_can.c +++ b/net/can/af_can.c @@ -641,6 +641,16 @@ static int can_rcv_filter(struct can_dev_rcv_lists *dev_rcv_lists, struct sk_buf return matches; } +void can_set_skb_uid(struct sk_buff *skb) +{ + /* create non-zero unique skb identifier together with *skb */ + while (!(skb->hash)) + skb->hash = atomic_inc_return(&skbcounter); + + skb->sw_hash = 1; +} +EXPORT_SYMBOL(can_set_skb_uid); + static void can_receive(struct sk_buff *skb, struct net_device *dev) { struct can_dev_rcv_lists *dev_rcv_lists; @@ -652,9 +662,7 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev) atomic_long_inc(&pkg_stats->rx_frames); atomic_long_inc(&pkg_stats->rx_frames_delta); - /* create non-zero unique skb identifier together with *skb */ - while (!(can_skb_prv(skb)->skbcnt)) - can_skb_prv(skb)->skbcnt = atomic_inc_return(&skbcounter); + can_set_skb_uid(skb); rcu_read_lock(); diff --git a/net/can/bcm.c b/net/can/bcm.c index a40519638a262..67d5e3ecc04fb 100644 --- a/net/can/bcm.c +++ b/net/can/bcm.c @@ -111,11 +111,12 @@ struct bcm_op { int ifindex; canid_t can_id; u32 flags; - unsigned long frames_abs, frames_filtered; + atomic_long_t frames_abs, frames_filtered; struct bcm_timeval ival1, ival2; struct hrtimer timer, thrtimer; ktime_t rx_stamp, kt_ival1, kt_ival2, kt_lastmsg; int rx_ifindex; + int if_detected; /* first received ifindex in ANYDEV rx_op mode */ int cfsiz; u32 count; u32 nframes; @@ -127,7 +128,8 @@ struct bcm_op { struct canfd_frame last_sframe; struct sock *sk; struct net_device *rx_reg_dev; - spinlock_t bcm_tx_lock; /* protect currframe/count in runtime updates */ + spinlock_t bcm_tx_lock; /* protect tx data and timer updates */ + spinlock_t bcm_rx_update_lock; /* protect filter/timer data updates */ }; struct bcm_sock { @@ -227,10 +229,13 @@ static int bcm_proc_show(struct seq_file *m, void *v) list_for_each_entry_rcu(op, &bo->rx_ops, list) { - unsigned long reduction; + long reduction, frames_filtered, frames_abs; + + frames_filtered = atomic_long_read(&op->frames_filtered); + frames_abs = atomic_long_read(&op->frames_abs); /* print only active entries & prevent division by zero */ - if (!op->frames_abs) + if (!frames_abs) continue; seq_printf(m, "rx_op: %03X %-5s ", op->can_id, @@ -252,9 +257,9 @@ static int bcm_proc_show(struct seq_file *m, void *v) (long long)ktime_to_us(op->kt_ival2)); seq_printf(m, "# recv %ld (%ld) => reduction: ", - op->frames_filtered, op->frames_abs); + frames_filtered, frames_abs); - reduction = 100 - (op->frames_filtered * 100) / op->frames_abs; + reduction = 100 - (frames_filtered * 100) / frames_abs; seq_printf(m, "%s%ld%%\n", (reduction == 100) ? "near " : "", reduction); @@ -278,7 +283,8 @@ static int bcm_proc_show(struct seq_file *m, void *v) seq_printf(m, "t2=%lld ", (long long)ktime_to_us(op->kt_ival2)); - seq_printf(m, "# sent %ld\n", op->frames_abs); + seq_printf(m, "# sent %ld\n", + atomic_long_read(&op->frames_abs)); } seq_putc(m, '\n'); @@ -288,25 +294,49 @@ static int bcm_proc_show(struct seq_file *m, void *v) } #endif /* CONFIG_PROC_FS */ +static void bcm_update_rx_stats(struct bcm_op *op) +{ + /* prevent overflow of the reduction% calculation in bcm_proc_show() */ + if (atomic_long_inc_return(&op->frames_abs) > LONG_MAX / 100) { + atomic_long_set(&op->frames_filtered, 0); + atomic_long_set(&op->frames_abs, 0); + } +} + +static void bcm_update_tx_stats(struct bcm_op *op) +{ + /* tx_op has no reduction% calculation - use the full range and + * just keep the displayed counter non-negative on overflow + */ + if (atomic_long_inc_return(&op->frames_abs) == LONG_MAX) + atomic_long_set(&op->frames_abs, 0); +} + /* * bcm_can_tx - send the (next) CAN frame to the appropriate CAN interface * of the given bcm tx op */ -static void bcm_can_tx(struct bcm_op *op) +static void bcm_can_tx(struct bcm_op *op, struct canfd_frame *cf) { struct sk_buff *skb; struct net_device *dev; - struct canfd_frame *cf; + struct canfd_frame cframe; + bool cyclic = !cf; + unsigned int idx = 0; int err; /* no target device? => exit */ if (!op->ifindex) return; - /* read currframe under lock protection */ - spin_lock_bh(&op->bcm_tx_lock); - cf = op->frames + op->cfsiz * op->currframe; - spin_unlock_bh(&op->bcm_tx_lock); + if (cyclic) { + /* read currframe under lock protection */ + spin_lock_bh(&op->bcm_tx_lock); + idx = op->currframe; + memcpy(&cframe, op->frames + op->cfsiz * idx, op->cfsiz); + cf = &cframe; + spin_unlock_bh(&op->bcm_tx_lock); + } dev = dev_get_by_index(sock_net(op->sk), op->ifindex); if (!dev) { @@ -320,7 +350,6 @@ static void bcm_can_tx(struct bcm_op *op) can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; skb_put_data(skb, cf, op->cfsiz); @@ -333,16 +362,22 @@ static void bcm_can_tx(struct bcm_op *op) spin_lock_bh(&op->bcm_tx_lock); if (!err) - op->frames_abs++; + bcm_update_tx_stats(op); - op->currframe++; + /* only advance the cyclic sequence if nothing reset currframe while + * we were sending - a concurrent TX_RESET_MULTI_IDX means this + * frame's bookkeeping belongs to a sequence that no longer exists + */ + if (!cyclic || op->currframe == idx) { + op->currframe++; - /* reached last frame? */ - if (op->currframe >= op->nframes) - op->currframe = 0; + /* reached last frame? */ + if (op->currframe >= op->nframes) + op->currframe = 0; - if (op->count > 0) - op->count--; + if (op->count > 0) + op->count--; + } spin_unlock_bh(&op->bcm_tx_lock); out: @@ -361,7 +396,6 @@ static void bcm_send_to_user(struct bcm_op *op, struct bcm_msg_head *head, struct sockaddr_can *addr; struct sock *sk = op->sk; unsigned int datalen = head->nframes * op->cfsiz; - int err; unsigned int *pflags; enum skb_drop_reason reason; @@ -418,8 +452,8 @@ static void bcm_send_to_user(struct bcm_op *op, struct bcm_msg_head *head, addr->can_family = AF_CAN; addr->can_ifindex = op->rx_ifindex; - err = sock_queue_rcv_skb_reason(sk, skb, &reason); - if (err < 0) { + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) { struct bcm_sock *bo = bcm_sk(sk); sk_skb_reason_drop(sk, skb, reason); @@ -432,12 +466,18 @@ static bool bcm_tx_set_expiry(struct bcm_op *op, struct hrtimer *hrt) { ktime_t ival; + spin_lock_bh(&op->bcm_tx_lock); + if (op->kt_ival1 && op->count) ival = op->kt_ival1; - else if (op->kt_ival2) + else if (op->kt_ival2) { ival = op->kt_ival2; - else + } else { + spin_unlock_bh(&op->bcm_tx_lock); return false; + } + + spin_unlock_bh(&op->bcm_tx_lock); hrtimer_set_expires(hrt, ktime_add(ktime_get(), ival)); return true; @@ -454,26 +494,48 @@ static enum hrtimer_restart bcm_tx_timeout_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + bool tx_ival1, tx_ival2; + + /* snapshot kt_ival1/kt_ival2/count under lock to avoid torn + * ktime_t reads racing with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + tx_ival1 = op->kt_ival1 && (op->count > 0); + tx_ival2 = !!op->kt_ival2; + spin_unlock_bh(&op->bcm_tx_lock); - if (op->kt_ival1 && (op->count > 0)) { - bcm_can_tx(op); - if (!op->count && (op->flags & TX_COUNTEVT)) { + if (tx_ival1) { + u32 flags, count; + struct bcm_timeval ival1, ival2; + bcm_can_tx(op, NULL); + + /* snapshot variables under lock to avoid torn reads racing + * with concurrent bcm_tx_setup() updates + */ + spin_lock_bh(&op->bcm_tx_lock); + flags = op->flags; + count = op->count; + ival1 = op->ival1; + ival2 = op->ival2; + spin_unlock_bh(&op->bcm_tx_lock); + + if (!count && (flags & TX_COUNTEVT)) { /* create notification to user */ memset(&msg_head, 0, sizeof(msg_head)); msg_head.opcode = TX_EXPIRED; - msg_head.flags = op->flags; - msg_head.count = op->count; - msg_head.ival1 = op->ival1; - msg_head.ival2 = op->ival2; + msg_head.flags = flags; + msg_head.count = count; + msg_head.ival1 = ival1; + msg_head.ival2 = ival2; msg_head.can_id = op->can_id; msg_head.nframes = 0; bcm_send_to_user(op, &msg_head, NULL, 0); } - } else if (op->kt_ival2) { - bcm_can_tx(op); + } else if (tx_ival2) { + bcm_can_tx(op, NULL); } return bcm_tx_set_expiry(op, &op->timer) ? @@ -487,12 +549,9 @@ static void bcm_rx_changed(struct bcm_op *op, struct canfd_frame *data) { struct bcm_msg_head head; - /* update statistics */ - op->frames_filtered++; - - /* prevent statistics overflow */ - if (op->frames_filtered > ULONG_MAX/100) - op->frames_filtered = op->frames_abs = 0; + /* update statistics (frames_filtered <= frames_abs) */ + if (atomic_long_read(&op->frames_abs)) + atomic_long_inc(&op->frames_filtered); /* this element is not throttled anymore */ data->flags &= ~RX_THR; @@ -617,6 +676,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) struct bcm_op *op = container_of(hrtimer, struct bcm_op, timer); struct bcm_msg_head msg_head; + spin_lock_bh(&op->bcm_rx_update_lock); + /* if user wants to be informed, when cyclic CAN-Messages come back */ if ((op->flags & RX_ANNOUNCE_RESUME) && op->last_frames) { /* clear received CAN frames to indicate 'nothing received' */ @@ -633,6 +694,8 @@ static enum hrtimer_restart bcm_rx_timeout_handler(struct hrtimer *hrtimer) msg_head.can_id = op->can_id; msg_head.nframes = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + bcm_send_to_user(op, &msg_head, NULL, 0); return HRTIMER_NORESTART; @@ -681,15 +744,26 @@ static int bcm_rx_thr_flush(struct bcm_op *op) static enum hrtimer_restart bcm_rx_thr_handler(struct hrtimer *hrtimer) { struct bcm_op *op = container_of(hrtimer, struct bcm_op, thrtimer); + enum hrtimer_restart ret; - if (bcm_rx_thr_flush(op)) { + spin_lock_bh(&op->bcm_rx_update_lock); + + /* kt_ival2 may have been concurrently cleared by bcm_rx_setup() + * before it cancels this timer - never forward with a zero + * interval in that case. + */ + if (bcm_rx_thr_flush(op) && op->kt_ival2) { hrtimer_forward_now(hrtimer, op->kt_ival2); - return HRTIMER_RESTART; + ret = HRTIMER_RESTART; } else { /* rearm throttle handling */ op->kt_lastmsg = 0; - return HRTIMER_NORESTART; + ret = HRTIMER_NORESTART; } + + spin_unlock_bh(&op->bcm_rx_update_lock); + + return ret; } /* @@ -699,8 +773,10 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) { struct bcm_op *op = (struct bcm_op *)data; const struct canfd_frame *rxframe = (struct canfd_frame *)skb->data; + struct canfd_frame rtrframe; unsigned int i; unsigned char traffic_flags; + bool rtr_frame; if (op->can_id != rxframe->can_id) return; @@ -714,22 +790,59 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) return; } + /* An ANYDEV op with an active RX timeout and/or throttle timer + * tracks a single source interface: claim the first interface that + * delivers a matching frame and reject frames from any other one, + * before hrtimer_cancel() below can touch op->timer - this avoids + * racing bcm_rx_timeout_handler() across concurrent interfaces. + * RX_RTR_FRAME ops are excluded, as kt_ival1/kt_ival2 may briefly + * hold a stale value from an earlier non-RTR configuration. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + + if (!(op->flags & RX_RTR_FRAME) && + (op->kt_ival1 || op->kt_ival2)) { + /* don't claim to vanishing interface */ + if (!op->if_detected && + skb->dev->reg_state == NETREG_REGISTERED) + op->if_detected = skb->dev->ifindex; + + if (op->if_detected != skb->dev->ifindex) { + spin_unlock_bh(&op->bcm_rx_update_lock); + return; + } + } + + spin_unlock_bh(&op->bcm_rx_update_lock); + } + /* disable timeout */ hrtimer_cancel(&op->timer); - /* save rx timestamp */ - op->rx_stamp = skb->tstamp; - /* save originator for recvfrom() */ - op->rx_ifindex = skb->dev->ifindex; - /* update statistics */ - op->frames_abs++; + /* op->flags/op->frames may be updated concurrently by bcm_rx_setup() */ + spin_lock_bh(&op->bcm_rx_update_lock); + + rtr_frame = op->flags & RX_RTR_FRAME; + if (rtr_frame) { + bcm_update_rx_stats(op); + /* snapshot RTR content under lock */ + memcpy(&rtrframe, op->frames, op->cfsiz); + spin_unlock_bh(&op->bcm_rx_update_lock); - if (op->flags & RX_RTR_FRAME) { /* send reply for RTR-request (placed in op->frames[0]) */ - bcm_can_tx(op); + bcm_can_tx(op, &rtrframe); return; } + /* update statistics in the same critical section as bcm_rx_changed() + * below: frames_filtered must never be checked/incremented against a + * frames_abs snapshot from a concurrent bcm_rx_handler() call on + * another CPU for the same (wildcard) op, or frames_filtered can end + * up larger than frames_abs. + */ + bcm_update_rx_stats(op); + /* compute flags to distinguish between own/local/remote CAN traffic */ traffic_flags = 0; if (skb->sk) { @@ -738,6 +851,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) traffic_flags |= RX_OWN; } + /* save rx timestamp and originator for recvfrom() under lock: an + * ANYDEV op without an active timer can still run concurrently on + * different CPUs, so content and meta data must be bundled here. + */ + op->rx_stamp = skb->tstamp; + op->rx_ifindex = skb->dev->ifindex; + if (op->flags & RX_FILTER_ID) { /* the easiest case */ bcm_rx_update_and_send(op, op->last_frames, rxframe, @@ -773,6 +893,8 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data) rx_starttimer: bcm_rx_starttimer(op); + + spin_unlock_bh(&op->bcm_rx_update_lock); } /* @@ -838,6 +960,7 @@ static void bcm_rx_unreg(struct net_device *dev, struct bcm_op *op) /* mark as removed subscription */ op->rx_reg_dev = NULL; + dev_put(dev); } else printk(KERN_ERR "can-bcm: bcm_rx_unreg: registered device " "mismatch %p %p\n", op->rx_reg_dev, dev); @@ -868,17 +991,14 @@ static int bcm_delete_rx_op(struct list_head *ops, struct bcm_msg_head *mh, * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the dev. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(sock_net(op->sk), - op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(sock_net(op->sk), NULL, op->can_id, @@ -965,6 +1085,8 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->tx_ops, msg_head, ifindex); if (op) { + void *new_frames; + /* update existing BCM operation */ /* @@ -975,11 +1097,23 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes > op->nframes) return -E2BIG; - /* update CAN frames content */ + /* get new CAN frames content into a staging buffer before + * locking: validate and normalize the frames there so that + * bcm_can_tx() / bcm_tx_timeout_handler() never observe a + * partially updated or unvalidated frame in op->frames + */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + for (i = 0; i < msg_head->nframes; i++) { - cf = op->frames + op->cfsiz * i; + cf = new_frames + op->cfsiz * i; err = memcpy_from_msg((u8 *)cf, msg, op->cfsiz); + if (err < 0) { + kfree(new_frames); + return err; + } if (op->flags & CAN_FD_FRAME) { if (cf->len > 64) @@ -989,36 +1123,38 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, err = -EINVAL; } - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } if (msg_head->flags & TX_CP_CAN_ID) { /* copy can_id into frame */ cf->can_id = msg_head->can_id; } } + + spin_lock_bh(&op->bcm_tx_lock); + + /* update CAN frames content */ + memcpy(op->frames, new_frames, msg_head->nframes * op->cfsiz); + op->flags = msg_head->flags; - /* only lock for unlikely count/nframes/currframe changes */ if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX || - op->flags & SETTIMER) { - - spin_lock_bh(&op->bcm_tx_lock); + op->flags & TX_RESET_MULTI_IDX) { + /* potentially update changed nframes */ + op->nframes = msg_head->nframes; + /* restart multiple frame transmission */ + op->currframe = 0; + } - if (op->nframes != msg_head->nframes || - op->flags & TX_RESET_MULTI_IDX) { - /* potentially update changed nframes */ - op->nframes = msg_head->nframes; - /* restart multiple frame transmission */ - op->currframe = 0; - } + if (op->flags & SETTIMER) + op->count = msg_head->count; - if (op->flags & SETTIMER) - op->count = msg_head->count; + spin_unlock_bh(&op->bcm_tx_lock); - spin_unlock_bh(&op->bcm_tx_lock); - } + kfree(new_frames); } else { /* insert new BCM operation for the given can_id */ @@ -1094,10 +1230,12 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (op->flags & SETTIMER) { /* set timer values */ + spin_lock_bh(&op->bcm_tx_lock); op->ival1 = msg_head->ival1; op->ival2 = msg_head->ival2; op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + spin_unlock_bh(&op->bcm_tx_lock); /* disable an active timer due to zero values? */ if (!op->kt_ival1 && !op->kt_ival2) @@ -1115,7 +1253,7 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, list_add_rcu(&op->list, &bo->tx_ops); if (op->flags & TX_ANNOUNCE) - bcm_can_tx(op); + bcm_can_tx(op, NULL); if (op->flags & STARTTIMER) bcm_tx_start_timer(op); @@ -1129,6 +1267,39 @@ static int bcm_tx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return err; } +static int bcm_rx_setup_rtr_check(struct bcm_msg_head *msg_head, + struct bcm_op *op, void *new_frames) +{ + struct canfd_frame *frame0 = new_frames; + + if (!(msg_head->flags & RX_RTR_FRAME)) + return 0; + + /* this frame is sent out as-is by bcm_can_tx() whenever a matching + * remote request is received, so validate its length the same way + * bcm_tx_setup() validates TX_SETUP frames before installing it + */ + if (msg_head->flags & CAN_FD_FRAME) { + if (frame0->len > 64) + return -EINVAL; + } else { + if (frame0->len > 8) + return -EINVAL; + } + + /* funny feature in RX(!)_SETUP only for RTR-mode: + * copy can_id into frame BUT without RTR-flag to + * prevent a full-load-loopback-test ... ;-] + * normalize this on the staged buffer, before it is + * ever installed into op->frames. + */ + if ((msg_head->flags & TX_CP_CAN_ID) || + frame0->can_id == op->can_id) + frame0->can_id = op->can_id & ~CAN_RTR_FLAG; + + return 0; +} + /* * bcm_rx_setup - create or update a bcm rx op (for bcm_sendmsg) */ @@ -1138,6 +1309,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, struct bcm_sock *bo = bcm_sk(sk); struct bcm_op *op; int do_rx_register; + int new_op = 0; int err = 0; if ((msg_head->flags & RX_FILTER_ID) || (!(msg_head->nframes))) { @@ -1163,6 +1335,8 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* check the given can_id */ op = bcm_find_op(&bo->rx_ops, msg_head, ifindex); if (op) { + void *new_frames = NULL; + /* update existing BCM operation */ /* @@ -1174,21 +1348,62 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -E2BIG; if (msg_head->nframes) { - /* update CAN frames content */ - err = memcpy_from_msg(op->frames, msg, + /* get new CAN frames content before locking */ + new_frames = kmalloc(msg_head->nframes * op->cfsiz, + GFP_KERNEL); + if (!new_frames) + return -ENOMEM; + + err = memcpy_from_msg(new_frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) + if (err < 0) { + kfree(new_frames); return err; + } - /* clear last_frames to indicate 'nothing received' */ - memset(op->last_frames, 0, msg_head->nframes * op->cfsiz); + err = bcm_rx_setup_rtr_check(msg_head, op, new_frames); + if (err < 0) { + kfree(new_frames); + return err; + } } + spin_lock_bh(&op->bcm_rx_update_lock); op->nframes = msg_head->nframes; op->flags = msg_head->flags; - /* Only an update -> do not call can_rx_register() */ - do_rx_register = 0; + if (msg_head->nframes) { + /* update CAN frames content */ + memcpy(op->frames, new_frames, + msg_head->nframes * op->cfsiz); + + /* clear last_frames to indicate 'nothing received' */ + memset(op->last_frames, 0, + msg_head->nframes * op->cfsiz); + } + + if (msg_head->flags & SETTIMER) { + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + op->if_detected = 0; /* reclaim ifindex in ANYDEV mode */ + } + spin_unlock_bh(&op->bcm_rx_update_lock); + + /* free temporary frames / kfree(NULL) is safe */ + kfree(new_frames); + + /* Don't register a new CAN filter for the rx_op update unless + * a concurrent NETDEV_UNREGISTER notifier already tore down + * the previous registration. In this case the receiver needs + * to be re-registered here so that this update doesn't + * silently stop delivering frames for the given ifindex. + * Ops with ifindex = 0 (all CAN interfaces) never carry a + * tracked rx_reg_dev and stay registered as-is. + */ + do_rx_register = (ifindex && !op->rx_reg_dev) ? 1 : 0; } else { /* insert new BCM operation for the given can_id */ @@ -1197,6 +1412,7 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, return -ENOMEM; spin_lock_init(&op->bcm_tx_lock); + spin_lock_init(&op->bcm_rx_update_lock); op->can_id = msg_head->can_id; op->nframes = msg_head->nframes; op->cfsiz = CFSIZ(msg_head->flags); @@ -1230,14 +1446,12 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, if (msg_head->nframes) { err = memcpy_from_msg(op->frames, msg, msg_head->nframes * op->cfsiz); - if (err < 0) { - if (op->frames != &op->sframe) - kfree(op->frames); - if (op->last_frames != &op->last_sframe) - kfree(op->last_frames); - kfree(op); - return err; - } + if (err < 0) + goto free_op; + + err = bcm_rx_setup_rtr_check(msg_head, op, op->frames); + if (err < 0) + goto free_op; } /* bcm_can_tx / bcm_tx_timeout_handler needs this */ @@ -1256,35 +1470,29 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, /* call can_rx_register() */ do_rx_register = 1; + new_op = 1; } /* if ((op = bcm_find_op(&bo->rx_ops, msg_head->can_id, ifindex))) */ /* check flags */ if (op->flags & RX_RTR_FRAME) { - struct canfd_frame *frame0 = op->frames; - /* no timers in RTR-mode */ hrtimer_cancel(&op->thrtimer); hrtimer_cancel(&op->timer); - - /* - * funny feature in RX(!)_SETUP only for RTR-mode: - * copy can_id into frame BUT without RTR-flag to - * prevent a full-load-loopback-test ... ;-] - */ - if ((op->flags & TX_CP_CAN_ID) || - (frame0->can_id == op->can_id)) - frame0->can_id = op->can_id & ~CAN_RTR_FLAG; - } else { if (op->flags & SETTIMER) { - /* set timer value */ - op->ival1 = msg_head->ival1; - op->ival2 = msg_head->ival2; - op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); - op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + /* set timers (locked) for newly created op */ + if (new_op) { + spin_lock_bh(&op->bcm_rx_update_lock); + op->ival1 = msg_head->ival1; + op->ival2 = msg_head->ival2; + op->kt_ival1 = bcm_timeval_to_ktime(msg_head->ival1); + op->kt_ival2 = bcm_timeval_to_ktime(msg_head->ival2); + op->kt_lastmsg = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } /* disable an active timer due to zero value? */ if (!op->kt_ival1) @@ -1294,9 +1502,11 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, * In any case cancel the throttle timer, flush * potentially blocked msgs and reset throttle handling */ - op->kt_lastmsg = 0; hrtimer_cancel(&op->thrtimer); + + spin_lock_bh(&op->bcm_rx_update_lock); bcm_rx_thr_flush(op); + spin_unlock_bh(&op->bcm_rx_update_lock); } if ((op->flags & STARTTIMER) && op->kt_ival1) @@ -1304,7 +1514,10 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, HRTIMER_MODE_REL_SOFT); } - /* now we can register for can_ids, if we added a new bcm_op */ + /* now we can register for can_ids, if we added a new bcm_op + * or need to re-register after a NETDEV_UNREGISTER tore down + * the previous registration of an existing op + */ if (do_rx_register) { if (ifindex) { struct net_device *dev; @@ -1317,7 +1530,15 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, bcm_rx_handler, op, "bcm", sk); - op->rx_reg_dev = dev; + /* keep a reference so that a later + * unregister can safely reach the device even + * if a concurrent NETDEV_UNREGISTER has + * already unlisted it by ifindex + */ + if (!err) { + op->rx_reg_dev = dev; + dev_hold(dev); + } dev_put(dev); } else { /* the requested device is gone - do not @@ -1326,21 +1547,43 @@ static int bcm_rx_setup(struct bcm_msg_head *msg_head, struct msghdr *msg, err = -ENODEV; } - } else + } else { err = can_rx_register(sock_net(sk), NULL, op->can_id, REGMASK(op->can_id), bcm_rx_handler, op, "bcm", sk); + } + if (err) { - /* this bcm rx op is broken -> remove it */ - bcm_remove_op(op); + /* newly created bcm rx op is broken -> remove it */ + if (new_op) { + bcm_remove_op(op); + return err; + } + + /* an existing op just stays unregistered. + * Cancel op->timer and (defensively) op->thrtimer. + * Other settings can't be reached until the next + * successful RX_SETUP. + */ + hrtimer_cancel(&op->timer); + hrtimer_cancel(&op->thrtimer); return err; } - /* add this bcm_op to the list of the rx_ops */ - list_add_rcu(&op->list, &bo->rx_ops); + /* add a new bcm_op to the list of the rx_ops */ + if (new_op) + list_add_rcu(&op->list, &bo->rx_ops); } return msg_head->nframes * op->cfsiz + MHSIZ; + +free_op: + if (op->frames != &op->sframe) + kfree(op->frames); + if (op->last_frames != &op->last_sframe) + kfree(op->last_frames); + kfree(op); + return err; } /* @@ -1376,7 +1619,6 @@ static int bcm_tx_send(struct msghdr *msg, int ifindex, struct sock *sk, } can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; skb->dev = dev; can_skb_set_owner(skb, sk); err = can_send(skb, 1); /* send with loopback */ @@ -1545,11 +1787,30 @@ static void bcm_notify(struct bcm_sock *bo, unsigned long msg, case NETDEV_UNREGISTER: lock_sock(sk); - /* remove device specific receive entries */ - list_for_each_entry(op, &bo->rx_ops, list) + /* rx_ops: remove device specific receive entries */ + list_for_each_entry(op, &bo->rx_ops, list) { if (op->rx_reg_dev == dev) bcm_rx_unreg(dev, op); + /* release an ANYDEV op's claim (see bcm_rx_handler()) + * on this now confirmed-gone interface. + */ + if (!op->ifindex) { + spin_lock_bh(&op->bcm_rx_update_lock); + if (op->if_detected == dev->ifindex) + op->if_detected = 0; + spin_unlock_bh(&op->bcm_rx_update_lock); + } + } + + /* tx_ops: stop device specific cyclic transmissions on the + * vanishing ifindex. Cancelling the timer is enough to stop + * cyclic bcm_can_tx() calls as there is no re-arming. + */ + list_for_each_entry(op, &bo->tx_ops, list) + if (op->ifindex == dev->ifindex) + hrtimer_cancel(&op->timer); + /* remove device reference, if this is our bound device */ if (bo->bound && bo->ifindex == dev->ifindex) { #if IS_ENABLED(CONFIG_PROC_FS) @@ -1682,16 +1943,14 @@ static int bcm_release(struct socket *sock) * Only remove subscriptions that had not * been removed due to NETDEV_UNREGISTER * in bcm_notifier() + * + * op->rx_reg_dev is a tracked reference taken + * when the subscription was registered, so it + * stays valid here even if a concurrent + * NETDEV_UNREGISTER already unlisted the device. */ - if (op->rx_reg_dev) { - struct net_device *dev; - - dev = dev_get_by_index(net, op->ifindex); - if (dev) { - bcm_rx_unreg(dev, op); - dev_put(dev); - } - } + if (op->rx_reg_dev) + bcm_rx_unreg(op->rx_reg_dev, op); } else can_rx_unregister(net, NULL, op->can_id, REGMASK(op->can_id), diff --git a/net/can/isotp.c b/net/can/isotp.c index c21d816a52747..a77ac3954e410 100644 --- a/net/can/isotp.c +++ b/net/can/isotp.c @@ -126,6 +126,15 @@ MODULE_PARM_DESC(max_pdu_size, "maximum isotp pdu size (default " #define ISOTP_FC_TIMEOUT 1 /* 1 sec */ #define ISOTP_ECHO_TIMEOUT 2 /* 2 secs */ +/* so->tx_result[so->tx_gen % ISOTP_TX_RESULT_SLOTS] holds the packed value + * (err << ISOTP_TX_RESULT_GEN_BITS | gen) for each tx generation slot, so it + * can be handled with a single READ_ONCE()/WRITE_ONCE() access. + */ +#define ISOTP_TX_RESULT_SLOTS 4 +#define ISOTP_TX_RESULT_GEN_BITS 24 +#define ISOTP_TX_RESULT_GEN_MASK ((1U << ISOTP_TX_RESULT_GEN_BITS) - 1) +#define ISOTP_TX_RESULT_ERR_MASK 0xFF + enum { ISOTP_IDLE = 0, ISOTP_WAIT_FIRST_FC, @@ -165,7 +174,8 @@ struct isotp_sock { u32 force_tx_stmin; u32 force_rx_stmin; u32 cfecho; /* consecutive frame echo tag */ - u32 tx_gen; /* generation, bumped per new tx transfer */ + u32 tx_gen; /* transfer generation, increased per new tx transfer */ + u32 tx_result[ISOTP_TX_RESULT_SLOTS]; /* per-generation result slots */ struct tpcon rx, tx; struct list_head notifier; wait_queue_head_t wait; @@ -176,6 +186,65 @@ static LIST_HEAD(isotp_notifier_list); static DEFINE_SPINLOCK(isotp_notifier_lock); static struct isotp_sock *isotp_busy_notifier; +/* increase (24 bit) tx generation value */ +static u32 isotp_inc_tx_gen(u32 gen) +{ + return (gen + 1) & ISOTP_TX_RESULT_GEN_MASK; +} + +/* store 8 bit error and 24 bit tx generation values in packed u32 element */ +static u32 isotp_pack_tx_result(u32 gen, int err) +{ + return gen | ((u32)err << ISOTP_TX_RESULT_GEN_BITS); +} + +/* get the 24 bit tx generation value from the tx result */ +static u32 isotp_get_tx_gen(u32 gen_err) +{ + return gen_err & ISOTP_TX_RESULT_GEN_MASK; +} + +/* get the 8 bit error value from the tx result */ +static u32 isotp_get_tx_err(u32 gen_err) +{ + return (gen_err >> ISOTP_TX_RESULT_GEN_BITS) & ISOTP_TX_RESULT_ERR_MASK; +} + +/* store transfer result in per-generation%4 so->tx_result[] slot */ +static void isotp_set_tx_result(struct isotp_sock *so, u32 gen, int err) +{ + WRITE_ONCE(so->tx_result[gen % ISOTP_TX_RESULT_SLOTS], + isotp_pack_tx_result(gen, err)); +} + +/* fetch the result recorded for 'gen', as a (negative) errno (0 for success) */ +static int isotp_get_tx_result(struct isotp_sock *so, u32 gen) +{ + u32 result = READ_ONCE(so->tx_result[gen % ISOTP_TX_RESULT_SLOTS]); + + if (isotp_get_tx_gen(result) != gen) { + pr_notice_once("can-isotp: tx_result[] slot reused before read\n"); + + /* report failure rather than risk a false success */ + return -ECOMM; + } + + return -(isotp_get_tx_err(result)); +} + +/* true if done, shut down or superseded ('gen' is no longer the active + * transfer). Reads tx.state first (acquire) so tx_gen/tx_result reads + * below see at least what that state write published (common sequence). + */ +static bool isotp_tx_gen_done(struct isotp_sock *so, u32 gen) +{ + /* read tx.state first for the common sequence */ + u32 state = smp_load_acquire(&so->tx.state); + + return state == ISOTP_IDLE || state == ISOTP_SHUTDOWN || + READ_ONCE(so->tx_gen) != gen; +} + static inline struct isotp_sock *isotp_sk(const struct sock *sk) { return (struct isotp_sock *)sk; @@ -198,7 +267,7 @@ static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) rxtimer); struct sock *sk = &so->sk; - if (so->rx.state == ISOTP_WAIT_DATA) { + if (READ_ONCE(so->rx.state) == ISOTP_WAIT_DATA) { /* we did not get new data frames in time */ /* report 'connection timed out' */ @@ -207,7 +276,7 @@ static enum hrtimer_restart isotp_rx_timer_handler(struct hrtimer *hrtimer) sk_error_report(sk); /* reset rx state */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); } return HRTIMER_NORESTART; @@ -233,7 +302,6 @@ static int isotp_send_fc(struct sock *sk, int ae, u8 flowstatus) can_skb_reserve(nskb); can_skb_prv(nskb)->ifindex = dev->ifindex; - can_skb_prv(nskb)->skbcnt = 0; nskb->dev = dev; can_skb_set_owner(nskb, sk); @@ -289,7 +357,8 @@ static void isotp_rcv_skb(struct sk_buff *skb, struct sock *sk) addr->can_family = AF_CAN; addr->can_ifindex = skb->dev->ifindex; - if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) sk_skb_reason_drop(sk, skb, reason); } @@ -366,20 +435,19 @@ static void isotp_send_cframe(struct isotp_sock *so); static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) { struct sock *sk = &so->sk; + int tx_err = EBADMSG; /* default for unknown FC status */ - if (so->tx.state != ISOTP_WAIT_FC && - so->tx.state != ISOTP_WAIT_FIRST_FC) + if (READ_ONCE(so->tx.state) != ISOTP_WAIT_FC && + READ_ONCE(so->tx.state) != ISOTP_WAIT_FIRST_FC) return 0; hrtimer_cancel(&so->txtimer); /* isotp_tx_timeout() may have given up on this job while - * hrtimer_cancel() above waited for it to finish; so->rx_lock - * (held by our caller isotp_rcv()) rules out a concurrent claim, - * so a plain recheck is enough here. + * hrtimer_cancel() above waited for it to finish => recheck */ - if (so->tx.state != ISOTP_WAIT_FC && - so->tx.state != ISOTP_WAIT_FIRST_FC) + if (READ_ONCE(so->tx.state) != ISOTP_WAIT_FC && + READ_ONCE(so->tx.state) != ISOTP_WAIT_FIRST_FC) return 1; if ((cf->len < ae + FC_CONTENT_SZ) || @@ -390,13 +458,15 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) if (!sock_flag(sk, SOCK_DEAD)) sk_error_report(sk); - so->tx.state = ISOTP_IDLE; + isotp_set_tx_result(so, so->tx_gen, EBADMSG); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); return 1; } /* get static/dynamic communication params from first/every FC frame */ - if (so->tx.state == ISOTP_WAIT_FIRST_FC || + if (READ_ONCE(so->tx.state) == ISOTP_WAIT_FIRST_FC || so->opt.flags & CAN_ISOTP_DYN_FC_PARMS) { so->txfc.bs = cf->data[ae + 1]; so->txfc.stmin = cf->data[ae + 2]; @@ -420,13 +490,13 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) so->tx_gap = ktime_add_ns(so->tx_gap, (so->txfc.stmin - 0xF0) * 100000); - so->tx.state = ISOTP_WAIT_FC; + WRITE_ONCE(so->tx.state, ISOTP_WAIT_FC); } switch (cf->data[ae] & 0x0F) { case ISOTP_FC_CTS: so->tx.bs = 0; - so->tx.state = ISOTP_SENDING; + WRITE_ONCE(so->tx.state, ISOTP_SENDING); /* send CF frame and enable echo timeout handling */ hrtimer_start(&so->echotimer, ktime_set(ISOTP_ECHO_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); @@ -441,14 +511,19 @@ static int isotp_rcv_fc(struct isotp_sock *so, struct canfd_frame *cf, int ae) case ISOTP_FC_OVFLW: /* overflow on receiver side - report 'message too long' */ - sk->sk_err = EMSGSIZE; - if (!sock_flag(sk, SOCK_DEAD)) - sk_error_report(sk); + tx_err = EMSGSIZE; fallthrough; default: - /* stop this tx job */ - so->tx.state = ISOTP_IDLE; + /* reserved/unknown flow status (tx_err defaults to EBADMSG) */ + + sk->sk_err = tx_err; + if (!sock_flag(sk, SOCK_DEAD)) + sk_error_report(sk); + + isotp_set_tx_result(so, so->tx_gen, tx_err); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); } return 0; @@ -461,7 +536,7 @@ static int isotp_rcv_sf(struct sock *sk, struct canfd_frame *cf, int pcilen, struct sk_buff *nskb; hrtimer_cancel(&so->rxtimer); - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); if (!len || len > cf->len - pcilen) return 1; @@ -495,7 +570,7 @@ static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) int ff_pci_sz; hrtimer_cancel(&so->rxtimer); - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); /* get the used sender LL_DL from the (first) CAN frame data length */ so->rx.ll_dl = padlen(cf->len); @@ -549,7 +624,7 @@ static int isotp_rcv_ff(struct sock *sk, struct canfd_frame *cf, int ae) /* initial setup for this pdu reception */ so->rx.sn = 1; - so->rx.state = ISOTP_WAIT_DATA; + WRITE_ONCE(so->rx.state, ISOTP_WAIT_DATA); /* no creation of flow control frames */ if (so->opt.flags & CAN_ISOTP_LISTEN_MODE) @@ -567,7 +642,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, struct sk_buff *nskb; int i; - if (so->rx.state != ISOTP_WAIT_DATA) + if (READ_ONCE(so->rx.state) != ISOTP_WAIT_DATA) return 0; /* drop if timestamp gap is less than force_rx_stmin nano secs */ @@ -582,11 +657,9 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, hrtimer_cancel(&so->rxtimer); /* isotp_rx_timer_handler() may have raced us for so->rx.state - * while hrtimer_cancel() above waited for it to finish, already - * reporting ETIMEDOUT and resetting the reception; don't process - * this CF into a reassembly that has already been given up on. + * while hrtimer_cancel() above waited for it to finish => recheck */ - if (so->rx.state != ISOTP_WAIT_DATA) + if (READ_ONCE(so->rx.state) != ISOTP_WAIT_DATA) return 1; /* CFs are never longer than the FF */ @@ -607,7 +680,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, sk_error_report(sk); /* reset rx state */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); return 1; } so->rx.sn++; @@ -621,7 +694,7 @@ static int isotp_rcv_cf(struct sock *sk, struct canfd_frame *cf, int ae, if (so->rx.idx >= so->rx.len) { /* we are done */ - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); if ((so->opt.flags & ISOTP_CHECK_PADDING) && check_pad(so, cf, i + 1, so->opt.rxpad_content)) { @@ -692,8 +765,10 @@ static void isotp_rcv(struct sk_buff *skb, void *data) if (so->opt.flags & CAN_ISOTP_HALF_DUPLEX) { /* check rx/tx path half duplex expectations */ - if ((so->tx.state != ISOTP_IDLE && n_pci_type != N_PCI_FC) || - (so->rx.state != ISOTP_IDLE && n_pci_type == N_PCI_FC)) + if ((READ_ONCE(so->tx.state) != ISOTP_IDLE && + n_pci_type != N_PCI_FC) || + (READ_ONCE(so->rx.state) != ISOTP_IDLE && + n_pci_type == N_PCI_FC)) goto out_unlock; } @@ -787,6 +862,7 @@ static void isotp_send_cframe(struct isotp_sock *so) struct canfd_frame *cf; int can_send_ret; int ae = (so->opt.flags & CAN_ISOTP_EXTEND_ADDR) ? 1 : 0; + u32 old_cfecho; dev = dev_get_by_index(sock_net(sk), so->ifindex); if (!dev) @@ -800,7 +876,9 @@ static void isotp_send_cframe(struct isotp_sock *so) can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; + + /* set uid in tx skb to identify CF echo frames */ + can_set_skb_uid(skb); cf = (struct canfd_frame *)skb->data; skb_put_zero(skb, so->ll.mtu); @@ -818,12 +896,15 @@ static void isotp_send_cframe(struct isotp_sock *so) skb->dev = dev; can_skb_set_owner(skb, sk); - /* cfecho should have been zero'ed by init/isotp_rcv_echo() */ - if (so->cfecho) - pr_notice_once("can-isotp: cfecho is %08X != 0\n", so->cfecho); + /* zero'ed by init/isotp_rcv_echo(); reached lock-free via + * isotp_txfr_timer_handler() too, so use READ_ONCE()/WRITE_ONCE() + */ + old_cfecho = READ_ONCE(so->cfecho); + if (old_cfecho) + pr_notice_once("can-isotp: cfecho is %08X != 0\n", old_cfecho); /* set consecutive frame echo tag */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); /* send frame with local echo enabled */ can_send_ret = can_send(skb, 1); @@ -875,7 +956,6 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) { struct sock *sk = (struct sock *)data; struct isotp_sock *so = isotp_sk(sk); - struct canfd_frame *cf = (struct canfd_frame *)skb->data; /* only handle my own local echo CF/SF skb's (no FF!) */ if (skb->sk != sk) @@ -887,32 +967,35 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) spin_lock(&so->rx_lock); /* so->cfecho may since belong to a new transfer; recheck under lock */ - if (so->cfecho != *(u32 *)cf->data) + if (READ_ONCE(so->cfecho) != skb->hash) goto out_unlock; /* cancel local echo timeout */ hrtimer_cancel(&so->echotimer); /* local echo skb with consecutive frame has been consumed */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); /* claiming a transfer also takes so->rx_lock, so a plain recheck * is enough: so->tx.state can't have flipped to ISOTP_SENDING for * a new claim while we're still in here */ - if (so->tx.state != ISOTP_SENDING) + if (READ_ONCE(so->tx.state) != ISOTP_SENDING) goto out_unlock; if (so->tx.idx >= so->tx.len) { /* we are done */ - so->tx.state = ISOTP_IDLE; + + isotp_set_tx_result(so, so->tx_gen, 0); + /* set to IDLE after publishing tx_result */ + smp_store_release(&so->tx.state, ISOTP_IDLE); wake_up_interruptible(&so->wait); goto out_unlock; } if (so->txfc.bs && so->tx.bs >= so->txfc.bs) { /* stop and wait for FC with timeout */ - so->tx.state = ISOTP_WAIT_FC; + WRITE_ONCE(so->tx.state, ISOTP_WAIT_FC); hrtimer_start(&so->txtimer, ktime_set(ISOTP_FC_TIMEOUT, 0), HRTIMER_MODE_REL_SOFT); goto out_unlock; @@ -934,16 +1017,20 @@ static void isotp_rcv_echo(struct sk_buff *skb, void *data) spin_unlock(&so->rx_lock); } -/* shared by so->txtimer's and so->echotimer's callbacks. Both timers get - * cancelled under so->rx_lock elsewhere, so this must stay lock-free to - * avoid deadlocking with that; uses so->tx_gen instead to avoid tainting - * a new transfer with an error from the one that just timed out. +/* isotp_tx_timeout: we did not get any flow control or echo frame in time + * + * Shared by so->txtimer's and so->echotimer's callbacks. Both timers get + * cancelled under so->rx_lock elsewhere, so this must stay lock-free. + * + * tx.state is acquired before tx_gen. Common sequence in isotp_tx_gen_done(). + * cmpxchg() only orders itself, not the two preceding loads. */ static enum hrtimer_restart isotp_tx_timeout(struct isotp_sock *so) { struct sock *sk = &so->sk; + /* read tx.state first for the common sequence */ + u32 old_state = smp_load_acquire(&so->tx.state); u32 gen = READ_ONCE(so->tx_gen); - u32 old_state = READ_ONCE(so->tx.state); /* don't handle timeouts in IDLE or SHUTDOWN state */ if (old_state == ISOTP_IDLE || old_state == ISOTP_SHUTDOWN) @@ -953,14 +1040,14 @@ static enum hrtimer_restart isotp_tx_timeout(struct isotp_sock *so) if (cmpxchg(&so->tx.state, old_state, ISOTP_IDLE) != old_state) return HRTIMER_NORESTART; - /* we did not get any flow control or echo frame in time */ + /* detected timeout: report 'communication error on send' */ - if (READ_ONCE(so->tx_gen) == gen) { - /* report 'communication error on send' */ - sk->sk_err = ECOMM; - if (!sock_flag(sk, SOCK_DEAD)) - sk_error_report(sk); - } + /* a stale read of this slot by a waiter still falls back to ECOMM */ + isotp_set_tx_result(so, gen, ECOMM); + + sk->sk_err = ECOMM; + if (!sock_flag(sk, SOCK_DEAD)) + sk_error_report(sk); wake_up_interruptible(&so->wait); @@ -995,7 +1082,7 @@ static enum hrtimer_restart isotp_txfr_timer_handler(struct hrtimer *hrtimer) HRTIMER_MODE_REL_SOFT); /* cfecho should be consumed by isotp_rcv_echo() here */ - if (so->tx.state == ISOTP_SENDING && !so->cfecho) + if (READ_ONCE(so->tx.state) == ISOTP_SENDING && !READ_ONCE(so->cfecho)) isotp_send_cframe(so); return HRTIMER_NORESTART; @@ -1013,10 +1100,12 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) s64 hrtimer_sec = ISOTP_ECHO_TIMEOUT; struct hrtimer *tx_hrt = &so->echotimer; u32 new_state = ISOTP_SENDING; + u32 my_gen; + u32 old_cfecho; int off; int err; - if (!so->bound || so->tx.state == ISOTP_SHUTDOWN) + if (!so->bound || READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) return -EADDRNOTAVAIL; /* claim the socket under so->rx_lock: this serializes the claim @@ -1033,29 +1122,33 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) if (msg->msg_flags & MSG_DONTWAIT) return -EAGAIN; - if (so->tx.state == ISOTP_SHUTDOWN) + if (READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) return -EADDRNOTAVAIL; /* wait for complete transmission of current pdu */ err = wait_event_interruptible(so->wait, - so->tx.state == ISOTP_IDLE); + READ_ONCE(so->tx.state) == ISOTP_IDLE || + READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN); if (err) return err; } - /* new transfer: bump so->tx_gen and drain the old one's timers, - * still under the so->rx_lock we just claimed the socket with - */ - WRITE_ONCE(so->tx.state, ISOTP_SENDING); - WRITE_ONCE(so->tx_gen, READ_ONCE(so->tx_gen) + 1); + /* txfrtimer's callback re-arms echotimer lock-free: drain it first */ + hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); hrtimer_cancel(&so->echotimer); - hrtimer_cancel(&so->txfrtimer); - so->cfecho = 0; + + /* new transfer: increment so->tx_gen and set tx.state after barrier */ + my_gen = isotp_inc_tx_gen(READ_ONCE(so->tx_gen)); + isotp_set_tx_result(so, my_gen, ECOMM); /* prevent stale slot matching */ + WRITE_ONCE(so->tx_gen, my_gen); + smp_wmb(); /* see smp_load_acquire() in isotp_tx_[timeout|gen_done] */ + WRITE_ONCE(so->tx.state, ISOTP_SENDING); + WRITE_ONCE(so->cfecho, 0); spin_unlock_bh(&so->rx_lock); /* so->bound is only checked once above - a wakeup may have - * unbound/rebound the socket meanwhile, so re-validate it + * unbound/rebound the socket meanwhile => recheck */ if (!so->bound) { err = -EADDRNOTAVAIL; @@ -1106,7 +1199,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; + + /* set uid in tx skb to identify CF echo frames */ + can_set_skb_uid(skb); so->tx.len = size; so->tx.idx = 0; @@ -1115,8 +1210,9 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) skb_put_zero(skb, so->ll.mtu); /* cfecho should have been zero'ed by init / former isotp_rcv_echo() */ - if (so->cfecho) - pr_notice_once("can-isotp: uninit cfecho %08X\n", so->cfecho); + old_cfecho = READ_ONCE(so->cfecho); + if (old_cfecho) + pr_notice_once("can-isotp: uninit cfecho %08X\n", old_cfecho); /* check for single frame transmission depending on TX_DL */ if (size <= so->tx.ll_dl - SF_PCI_SZ4 - ae - off) { @@ -1144,7 +1240,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) cf->data[ae] |= size; /* set CF echo tag for isotp_rcv_echo() (SF-mode) */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); } else { /* send first frame */ @@ -1161,7 +1257,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) so->txfc.bs = 0; /* set CF echo tag for isotp_rcv_echo() (CF-mode) */ - so->cfecho = *(u32 *)cf->data; + WRITE_ONCE(so->cfecho, skb->hash); } else { /* standard flow control check */ new_state = ISOTP_WAIT_FIRST_FC; @@ -1171,12 +1267,12 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) tx_hrt = &so->txtimer; /* no CF echo tag for isotp_rcv_echo() (FF-mode) */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); } } spin_lock_bh(&so->rx_lock); - if (so->tx.state == ISOTP_SHUTDOWN) { + if (READ_ONCE(so->tx.state) == ISOTP_SHUTDOWN) { /* isotp_release() has since taken over and already drained * our timers - don't send into a socket that's going away */ @@ -1187,7 +1283,7 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) return -EADDRNOTAVAIL; } /* WAIT_FIRST_FC for standard FF, else stays ISOTP_SENDING */ - so->tx.state = new_state; + WRITE_ONCE(so->tx.state, new_state); hrtimer_start(tx_hrt, ktime_set(hrtimer_sec, 0), HRTIMER_MODE_REL_SOFT); spin_unlock_bh(&so->rx_lock); @@ -1204,20 +1300,49 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) __func__, ERR_PTR(err)); spin_lock_bh(&so->rx_lock); + + /* new transfer already claimed by a concurrent completion, + * timeout or sendmsg() while we were stuck in can_send()? + */ + if (READ_ONCE(so->tx_gen) != my_gen) { + /* don't touch timers and state of the new transfer */ + spin_unlock_bh(&so->rx_lock); + return err; + } + /* no transmission -> no timeout monitoring */ hrtimer_cancel(tx_hrt); goto err_out_drop_locked; } if (wait_tx_done) { - /* wait for complete transmission of current pdu */ - err = wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE); + /* wake up for: + * - concurrent sendmsg() claiming a new transfer + * - complete transmission of current PDU + * - shutdown state change in isotp_release() + * isotp_tx_gen_done() uses common tx.state/tx_gen read sequence + */ + err = wait_event_interruptible(so->wait, + isotp_tx_gen_done(so, my_gen)); if (err) goto err_event_drop; - err = sock_error(sk); - if (err) - return err; + /* still our claim, but isotp_release() force-shut it down */ + if (smp_load_acquire(&so->tx.state) == ISOTP_SHUTDOWN && + READ_ONCE(so->tx_gen) == my_gen) { + err = -EADDRNOTAVAIL; + goto err_event_drop; + } + + /* own completion, or tx_gen moved on - either way this is + * what isotp_get_tx_result() recorded for my_gen + */ + err = isotp_get_tx_result(so, my_gen); + + /* drain to avoid stale error for a later poll()/SO_ERROR */ + sock_error(sk); + + return err ? err : size; } return size; @@ -1227,15 +1352,26 @@ static int isotp_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) spin_lock_bh(&so->rx_lock); goto err_out_drop_locked; err_event_drop: - /* interrupted waiting on our own transfer - drain its timers */ + /* interrupted or shut down while waiting on our own transfer */ spin_lock_bh(&so->rx_lock); + + /* new transfer already started by concurrent sendmsg()? */ + if (READ_ONCE(so->tx_gen) != my_gen) { + /* don't touch timers and states of the new transfer */ + spin_unlock_bh(&so->rx_lock); + return err; + } + hrtimer_cancel(&so->txfrtimer); hrtimer_cancel(&so->txtimer); hrtimer_cancel(&so->echotimer); err_out_drop_locked: /* release the claim; so->rx_lock still held from above */ - so->cfecho = 0; - so->tx.state = ISOTP_IDLE; + WRITE_ONCE(so->cfecho, 0); + + /* only claim to IDLE if isotp_release() has not taken over */ + if (READ_ONCE(so->tx.state) != ISOTP_SHUTDOWN) + WRITE_ONCE(so->tx.state, ISOTP_IDLE); spin_unlock_bh(&so->rx_lock); wake_up_interruptible(&so->wait); @@ -1301,8 +1437,9 @@ static int isotp_release(struct socket *sock) /* best-effort: wait for a running pdu to finish, but don't block on * it forever - give up after the first signal */ - while (so->tx.state != ISOTP_IDLE && - wait_event_interruptible(so->wait, so->tx.state == ISOTP_IDLE) == 0) + while (READ_ONCE(so->tx.state) != ISOTP_IDLE && + wait_event_interruptible(so->wait, + READ_ONCE(so->tx.state) == ISOTP_IDLE) == 0) ; /* claim the socket under so->rx_lock like sendmsg() does, so its @@ -1310,9 +1447,12 @@ static int isotp_release(struct socket *sock) * unconditionally, even when a signal cut the wait above short */ spin_lock_bh(&so->rx_lock); - so->tx.state = ISOTP_SHUTDOWN; + WRITE_ONCE(so->tx.state, ISOTP_SHUTDOWN); spin_unlock_bh(&so->rx_lock); - so->rx.state = ISOTP_IDLE; + WRITE_ONCE(so->rx.state, ISOTP_IDLE); + + /* forced SHUTDOWN may have skipped IDLE (gave up on a signal) */ + wake_up_interruptible(&so->wait); spin_lock(&isotp_notifier_lock); while (isotp_busy_notifier == so) { @@ -1428,7 +1568,8 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len) * with so->bound in the same lock_sock() section above, so there is * no window in which a concurrent isotp_notify() could be missed. */ - if (so->tx.state != ISOTP_IDLE || so->rx.state != ISOTP_IDLE) { + if (READ_ONCE(so->tx.state) != ISOTP_IDLE || + READ_ONCE(so->rx.state) != ISOTP_IDLE) { err = -EAGAIN; goto out; } @@ -1462,7 +1603,7 @@ static int isotp_bind(struct socket *sock, struct sockaddr *uaddr, int len) isotp_rcv, sk, "isotp", sk); /* no consecutive frame echo skb in flight */ - so->cfecho = 0; + WRITE_ONCE(so->cfecho, 0); /* register for echo skb's */ can_rx_register(net, dev, tx_id, SINGLE_MASK(tx_id), @@ -1828,7 +1969,7 @@ static __poll_t isotp_poll(struct file *file, struct socket *sock, poll_table *w poll_wait(file, &so->wait, wait); /* Check for false positives due to TX state */ - if ((mask & EPOLLWRNORM) && (so->tx.state != ISOTP_IDLE)) + if ((mask & EPOLLWRNORM) && (READ_ONCE(so->tx.state) != ISOTP_IDLE)) mask &= ~(EPOLLOUT | EPOLLWRNORM); return mask; @@ -1888,13 +2029,18 @@ static __init int isotp_module_init(void) pr_info("can: isotp protocol (max_pdu_size %d)\n", max_pdu_size); + err = register_netdevice_notifier(&canisotp_notifier); + if (err) + return err; + err = can_proto_register(&isotp_can_proto); - if (err < 0) + if (err < 0) { pr_err("can: registration of isotp protocol failed %pe\n", ERR_PTR(err)); - else - register_netdevice_notifier(&canisotp_notifier); + unregister_netdevice_notifier(&canisotp_notifier); + return err; + } - return err; + return 0; } static __exit void isotp_module_exit(void) diff --git a/net/can/j1939/bus.c b/net/can/j1939/bus.c index 797719cb227ec..e6d81bcabc897 100644 --- a/net/can/j1939/bus.c +++ b/net/can/j1939/bus.c @@ -20,6 +20,7 @@ static void __j1939_ecu_release(struct kref *kref) struct j1939_priv *priv = ecu->priv; list_del(&ecu->list); + netdev_put(priv->ndev, &ecu->priv_dev_tracker); kfree(ecu); j1939_priv_put(priv); } @@ -155,6 +156,7 @@ struct j1939_ecu *j1939_ecu_create_locked(struct j1939_priv *priv, name_t name) if (!ecu) return ERR_PTR(-ENOMEM); kref_init(&ecu->kref); + netdev_hold(priv->ndev, &ecu->priv_dev_tracker, gfp_any()); ecu->addr = J1939_IDLE_ADDR; ecu->name = name; diff --git a/net/can/j1939/j1939-priv.h b/net/can/j1939/j1939-priv.h index 81f58924b4acd..cf26352d1d8ce 100644 --- a/net/can/j1939/j1939-priv.h +++ b/net/can/j1939/j1939-priv.h @@ -38,6 +38,7 @@ struct j1939_ecu { struct hrtimer ac_timer; struct kref kref; struct j1939_priv *priv; + netdevice_tracker priv_dev_tracker; /* count users, to help transport protocol decide for interaction */ int nusers; @@ -60,6 +61,7 @@ struct j1939_priv { rwlock_t lock; struct net_device *ndev; + netdevice_tracker dev_tracker; /* list of 256 ecu ptrs, that cache the claimed addresses. * also protected by the above lock @@ -230,6 +232,7 @@ enum j1939_session_state { struct j1939_session { struct j1939_priv *priv; + netdevice_tracker priv_dev_tracker; struct list_head active_session_list_entry; struct list_head sk_session_queue_entry; struct kref kref; diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c index a93af55df5fd5..21186932f8f61 100644 --- a/net/can/j1939/main.c +++ b/net/can/j1939/main.c @@ -137,7 +137,7 @@ static struct j1939_priv *j1939_priv_create(struct net_device *ndev) priv->ndev = ndev; kref_init(&priv->kref); kref_init(&priv->rx_kref); - dev_hold(ndev); + netdev_hold(ndev, &priv->dev_tracker, GFP_KERNEL); netdev_dbg(priv->ndev, "%s : 0x%p\n", __func__, priv); @@ -163,7 +163,7 @@ static void __j1939_priv_release(struct kref *kref) WARN_ON_ONCE(!list_empty(&priv->ecus)); WARN_ON_ONCE(!list_empty(&priv->j1939_socks)); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); } @@ -281,7 +281,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev) */ kref_get(&priv_new->rx_kref); mutex_unlock(&j1939_netdev_lock); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); return priv_new; } @@ -298,7 +298,7 @@ struct j1939_priv *j1939_netdev_start(struct net_device *ndev) j1939_priv_set(ndev, NULL); mutex_unlock(&j1939_netdev_lock); - dev_put(ndev); + netdev_put(ndev, &priv->dev_tracker); kfree(priv); return ERR_PTR(ret); diff --git a/net/can/j1939/socket.c b/net/can/j1939/socket.c index e3ba2e9fc0e9b..70a57afdef1a2 100644 --- a/net/can/j1939/socket.c +++ b/net/can/j1939/socket.c @@ -332,7 +332,8 @@ static void j1939_sk_recv_one(struct j1939_sock *jsk, struct sk_buff *oskb) if (skb->sk) skcb->msg_flags |= MSG_DONTROUTE; - if (sock_queue_rcv_skb_reason(&jsk->sk, skb, &reason) < 0) + reason = sock_queue_rcv_skb_reason(&jsk->sk, skb); + if (reason) sk_skb_reason_drop(&jsk->sk, skb, reason); } @@ -897,7 +898,6 @@ static struct sk_buff *j1939_sk_alloc_skb(struct net_device *ndev, can_skb_reserve(skb); can_skb_prv(skb)->ifindex = ndev->ifindex; - can_skb_prv(skb)->skbcnt = 0; skb_reserve(skb, offsetof(struct can_frame, data)); ret = memcpy_from_msg(skb_put(skb, size), msg, size); diff --git a/net/can/j1939/transport.c b/net/can/j1939/transport.c index 8656ab388c83e..9d9c342d26d58 100644 --- a/net/can/j1939/transport.c +++ b/net/can/j1939/transport.c @@ -282,6 +282,7 @@ static void j1939_session_destroy(struct j1939_session *session) kfree_skb(skb); } __j1939_session_drop(session); + netdev_put(session->priv->ndev, &session->priv_dev_tracker); j1939_priv_put(session->priv); kfree(session); } @@ -350,6 +351,18 @@ static void j1939_session_skb_drop_old(struct j1939_session *session) } } +static bool j1939_address_is_local(struct j1939_priv *priv, u8 addr) +{ + bool local = false; + + read_lock_bh(&priv->lock); + if (j1939_address_is_unicast(addr) && priv->ents[addr].nusers) + local = true; + read_unlock_bh(&priv->lock); + + return local; +} + void j1939_session_skb_queue(struct j1939_session *session, struct sk_buff *skb) { @@ -358,8 +371,7 @@ void j1939_session_skb_queue(struct j1939_session *session, j1939_ac_fixup(priv, skb); - if (j1939_address_is_unicast(skcb->addr.da) && - priv->ents[skcb->addr.da].nusers) + if (j1939_address_is_local(priv, skcb->addr.da)) skcb->flags |= J1939_ECU_LOCAL_DST; skcb->flags |= J1939_ECU_LOCAL_SRC; @@ -601,7 +613,6 @@ sk_buff *j1939_tp_tx_dat_new(struct j1939_priv *priv, skb->dev = priv->ndev; can_skb_reserve(skb); can_skb_prv(skb)->ifindex = priv->ndev->ifindex; - can_skb_prv(skb)->skbcnt = 0; /* reserve CAN header */ skb_reserve(skb, offsetof(struct can_frame, data)); @@ -1499,6 +1510,7 @@ static struct j1939_session *j1939_session_new(struct j1939_priv *priv, INIT_LIST_HEAD(&session->active_session_list_entry); INIT_LIST_HEAD(&session->sk_session_queue_entry); kref_init(&session->kref); + netdev_hold(priv->ndev, &session->priv_dev_tracker, gfp_any()); j1939_priv_get(priv); session->priv = priv; @@ -1536,7 +1548,6 @@ j1939_session *j1939_session_fresh_new(struct j1939_priv *priv, skb->dev = priv->ndev; can_skb_reserve(skb); can_skb_prv(skb)->ifindex = priv->ndev->ifindex; - can_skb_prv(skb)->skbcnt = 0; skcb = j1939_skb_to_cb(skb); memcpy(skcb, rel_skcb, sizeof(*skcb)); @@ -1547,7 +1558,7 @@ j1939_session *j1939_session_fresh_new(struct j1939_priv *priv, } /* alloc data area */ - skb_put(skb, size); + skb_put_zero(skb, size); /* skb is recounted in j1939_session_new() */ return session; } @@ -2017,8 +2028,7 @@ struct j1939_session *j1939_tp_send(struct j1939_priv *priv, return ERR_PTR(ret); /* fix DST flags, it may be used there soon */ - if (j1939_address_is_unicast(skcb->addr.da) && - priv->ents[skcb->addr.da].nusers) + if (j1939_address_is_local(priv, skcb->addr.da)) skcb->flags |= J1939_ECU_LOCAL_DST; /* src is always local, I'm sending ... */ diff --git a/net/can/raw.c b/net/can/raw.c index 263e7167d2f52..dc02defa560d6 100644 --- a/net/can/raw.c +++ b/net/can/raw.c @@ -76,7 +76,7 @@ MODULE_ALIAS("can-proto-1"); struct uniqframe { const struct sk_buff *skb; - int skbcnt; + u32 hash; unsigned int join_rx_count; }; @@ -164,7 +164,7 @@ static void raw_rcv(struct sk_buff *oskb, void *data) /* eliminate multiple filter matches for the same skb */ if (this_cpu_ptr(ro->uniq)->skb == oskb && - this_cpu_ptr(ro->uniq)->skbcnt == can_skb_prv(oskb)->skbcnt) { + this_cpu_ptr(ro->uniq)->hash == oskb->hash) { if (!ro->join_filters) return; @@ -174,7 +174,7 @@ static void raw_rcv(struct sk_buff *oskb, void *data) return; } else { this_cpu_ptr(ro->uniq)->skb = oskb; - this_cpu_ptr(ro->uniq)->skbcnt = can_skb_prv(oskb)->skbcnt; + this_cpu_ptr(ro->uniq)->hash = oskb->hash; this_cpu_ptr(ro->uniq)->join_rx_count = 1; /* drop first frame to check all enabled filters? */ if (ro->join_filters && ro->count > 1) @@ -206,7 +206,8 @@ static void raw_rcv(struct sk_buff *oskb, void *data) if (oskb->sk == sk) *pflags |= MSG_CONFIRM; - if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) sk_skb_reason_drop(sk, skb, reason); } @@ -560,8 +561,8 @@ static int raw_getname(struct socket *sock, struct sockaddr *uaddr, return RAW_MIN_NAMELEN; } -static int raw_setsockopt(struct socket *sock, int level, int optname, - sockptr_t optval, unsigned int optlen) +static int raw_setsockopt_locked(struct socket *sock, int optname, + sockptr_t optval, unsigned int optlen) { struct sock *sk = sock->sk; struct raw_sock *ro = raw_sk(sk); @@ -573,9 +574,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, int flag; int err = 0; - if (level != SOL_CAN_RAW) - return -EINVAL; - switch (optname) { case CAN_RAW_FILTER: if (optlen % sizeof(struct can_filter) != 0) @@ -596,17 +594,11 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, return -EFAULT; } - rtnl_lock(); - lock_sock(sk); - dev = ro->dev; - if (ro->bound && dev) { - if (dev->reg_state != NETREG_REGISTERED) { - if (count > 1) - kfree(filter); - err = -ENODEV; - goto out_fil; - } + if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED) { + if (count > 1) + kfree(filter); + return -ENODEV; } if (ro->bound) { @@ -620,7 +612,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, if (err) { if (count > 1) kfree(filter); - goto out_fil; + return err; } /* remove old filter registrations */ @@ -640,11 +632,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, } ro->filter = filter; ro->count = count; - - out_fil: - release_sock(sk); - rtnl_unlock(); - break; case CAN_RAW_ERR_FILTER: @@ -656,16 +643,9 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, err_mask &= CAN_ERR_MASK; - rtnl_lock(); - lock_sock(sk); - dev = ro->dev; - if (ro->bound && dev) { - if (dev->reg_state != NETREG_REGISTERED) { - err = -ENODEV; - goto out_err; - } - } + if (ro->bound && dev && dev->reg_state != NETREG_REGISTERED) + return -ENODEV; /* remove current error mask */ if (ro->bound) { @@ -674,7 +654,7 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, err_mask); if (err) - goto out_err; + return err; /* remove old err_mask registration */ raw_disable_errfilter(sock_net(sk), dev, sk, @@ -683,11 +663,6 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, /* link new err_mask to the socket */ ro->err_mask = err_mask; - - out_err: - release_sock(sk); - rtnl_unlock(); - break; case CAN_RAW_LOOPBACK: @@ -767,6 +742,26 @@ static int raw_setsockopt(struct socket *sock, int level, int optname, return err; } +static int raw_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct sock *sk = sock->sk; + int err; + + if (level != SOL_CAN_RAW) + return -EINVAL; + + rtnl_lock(); + lock_sock(sk); + + err = raw_setsockopt_locked(sock, optname, optval, optlen); + + release_sock(sk); + rtnl_unlock(); + + return err; +} + static int raw_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen) { @@ -960,7 +955,6 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) can_skb_reserve(skb); can_skb_prv(skb)->ifindex = dev->ifindex; - can_skb_prv(skb)->skbcnt = 0; /* fill the skb before testing for valid CAN frames */ err = memcpy_from_msg(skb_put(skb, size), msg, size); diff --git a/net/ceph/auth_x.c b/net/ceph/auth_x.c index a21c157daf7dd..c83559f015e46 100644 --- a/net/ceph/auth_x.c +++ b/net/ceph/auth_x.c @@ -781,9 +781,16 @@ static int ceph_x_update_authorizer( au = (struct ceph_x_authorizer *)auth->authorizer; if (au->secret_id < th->secret_id) { + int ret; + dout("ceph_x_update_authorizer service %u secret %llu < %llu\n", au->service, au->secret_id, th->secret_id); - return ceph_x_build_authorizer(ac, th, au); + ret = ceph_x_build_authorizer(ac, th, au); + if (ret) + return ret; + + auth->authorizer_buf = au->buf->vec.iov_base; + auth->authorizer_buf_len = au->buf->vec.iov_len; } return 0; } diff --git a/net/ceph/ceph_common.c b/net/ceph/ceph_common.c index e734e57be083a..5974c23347dc3 100644 --- a/net/ceph/ceph_common.c +++ b/net/ceph/ceph_common.c @@ -763,13 +763,13 @@ void ceph_destroy_client(struct ceph_client *client) atomic_set(&client->msgr.stopping, 1); + ceph_debugfs_client_cleanup(client); + /* unmount */ ceph_osdc_stop(&client->osdc); ceph_monc_stop(&client->monc); ceph_messenger_fini(&client->msgr); - ceph_debugfs_client_cleanup(client); - ceph_destroy_options(client->options); kfree(client); diff --git a/net/ceph/cls_lock_client.c b/net/ceph/cls_lock_client.c index 66136a4c1ce7f..a9655678be6fd 100644 --- a/net/ceph/cls_lock_client.c +++ b/net/ceph/cls_lock_client.c @@ -259,7 +259,8 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker) if (ret) return ret; - ceph_decode_copy(p, &locker->id.name, sizeof(locker->id.name)); + ceph_decode_copy_safe(p, end, &locker->id.name, + sizeof(locker->id.name), bad); s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO); if (IS_ERR(s)) return PTR_ERR(s); @@ -270,19 +271,23 @@ static int decode_locker(void **p, void *end, struct ceph_locker *locker) if (ret) return ret; - *p += sizeof(struct ceph_timespec); /* skip expiration */ + /* skip expiration */ + ceph_decode_skip_n(p, end, sizeof(struct ceph_timespec), bad); ret = ceph_decode_entity_addr(p, end, &locker->info.addr); if (ret) return ret; - len = ceph_decode_32(p); - *p += len; /* skip description */ + /* skip description */ + ceph_decode_skip_string(p, end, bad); dout("%s %s%llu cookie %s addr %s\n", __func__, ENTITY_NAME(locker->id.name), locker->id.cookie, ceph_pr_addr(&locker->info.addr)); return 0; + +bad: + return -EINVAL; } static int decode_lockers(void **p, void *end, u8 *type, char **tag, @@ -299,7 +304,7 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, if (ret) return ret; - *num_lockers = ceph_decode_32(p); + ceph_decode_32_safe(p, end, *num_lockers, err_inval); *lockers = kcalloc(*num_lockers, sizeof(**lockers), GFP_NOIO); if (!*lockers) return -ENOMEM; @@ -310,7 +315,8 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, goto err_free_lockers; } - *type = ceph_decode_8(p); + ret = -EINVAL; + ceph_decode_8_safe(p, end, *type, err_free_lockers); s = ceph_extract_encoded_string(p, end, NULL, GFP_NOIO); if (IS_ERR(s)) { ret = PTR_ERR(s); @@ -320,6 +326,9 @@ static int decode_lockers(void **p, void *end, u8 *type, char **tag, *tag = s; return 0; +err_inval: + return -EINVAL; + err_free_lockers: ceph_free_lockers(*lockers, *num_lockers); return ret; diff --git a/net/ceph/decode.c b/net/ceph/decode.c index bc109a1a4616f..2f21af38cd9a5 100644 --- a/net/ceph/decode.c +++ b/net/ceph/decode.c @@ -87,8 +87,9 @@ ceph_decode_entity_addr(void **p, void *end, struct ceph_entity_addr *addr) EXPORT_SYMBOL(ceph_decode_entity_addr); /* - * Return addr of desired type (MSGR2 or LEGACY) or error. - * Make sure there is only one match. + * Return addr of desired type (MSGR2 or LEGACY) or error. In case of + * multiple matches, use the first one for compatibility with userspace + * messenger. * * Assume encoding with MSG_ADDR2. */ @@ -121,14 +122,13 @@ int ceph_decode_entity_addrvec(void **p, void *end, bool msgr2, dout("%s i %d addr %s\n", __func__, i, ceph_pr_addr(&tmp_addr)); if (tmp_addr.type == my_type) { - if (found) { - pr_err("another match of type %d in addrvec\n", - le32_to_cpu(my_type)); - return -EINVAL; + if (!found) { + memcpy(addr, &tmp_addr, sizeof(*addr)); + found = true; + } else { + dout("%s skipping extra match of type %d in addrvec\n", + __func__, le32_to_cpu(my_type)); } - - memcpy(addr, &tmp_addr, sizeof(*addr)); - found = true; } } diff --git a/net/ceph/messenger_v2.c b/net/ceph/messenger_v2.c index 5c7435fc6483f..4bb238eb8606e 100644 --- a/net/ceph/messenger_v2.c +++ b/net/ceph/messenger_v2.c @@ -2144,6 +2144,11 @@ static int process_banner_prefix(struct ceph_connection *con) payload_len = ceph_decode_16(&p); dout("%s con %p payload_len %d\n", __func__, con, payload_len); + if (payload_len < sizeof(u64) + sizeof(u64)) { + con->error_msg = "protocol error, bad banner payload len"; + return -EINVAL; + } + return prepare_read_banner_payload(con, payload_len); } diff --git a/net/ceph/mon_client.c b/net/ceph/mon_client.c index ff4311d5764bd..96c9c4d0a375e 100644 --- a/net/ceph/mon_client.c +++ b/net/ceph/mon_client.c @@ -114,7 +114,7 @@ static struct ceph_monmap *ceph_monmap_decode(void **p, void *end, bool msgr2) dout("%s fsid %pU epoch %u num_mon %u\n", __func__, &fsid, epoch, num_mon); - if (num_mon > CEPH_MAX_MON) + if (num_mon == 0 || num_mon > CEPH_MAX_MON) goto e_inval; monmap = kmalloc(struct_size(monmap, mon_inst, num_mon), GFP_NOIO); @@ -821,7 +821,7 @@ static void handle_get_version_reply(struct ceph_mon_client *monc, struct ceph_mon_generic_request *req; u64 tid = le64_to_cpu(msg->hdr.tid); void *p = msg->front.iov_base; - void *end = p + msg->front_alloc_len; + void *const end = p + msg->front.iov_len; u64 handle; dout("%s msg %p tid %llu\n", __func__, msg, tid); diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 6d7d8c7d7d3fe..46b2c2ae77784 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -5033,7 +5034,7 @@ static int decode_watchers(void **p, void *end, if (ret) return ret; - *num_watchers = ceph_decode_32(p); + ceph_decode_32_safe(p, end, *num_watchers, bad); *watchers = kcalloc(*num_watchers, sizeof(**watchers), GFP_NOIO); if (!*watchers) return -ENOMEM; @@ -5047,6 +5048,9 @@ static int decode_watchers(void **p, void *end, } return 0; + +bad: + return -EINVAL; } /* @@ -5802,6 +5806,31 @@ static inline void convert_extent_map(struct ceph_sparse_read *sr) } #endif +static bool sparse_extent_map_valid(struct ceph_sparse_read *sr) +{ + u64 req_end, pos; + int i; + + if (check_add_overflow(sr->sr_req_off, sr->sr_req_len, &req_end)) + return false; + + pos = sr->sr_req_off; + for (i = 0; i < sr->sr_count; i++) { + struct ceph_sparse_extent *ext = &sr->sr_extent[i]; + u64 end; + + if (ext->off < pos) + return false; + if (check_add_overflow(ext->off, ext->len, &end)) + return false; + if (end > req_end) + return false; + pos = end; + } + + return true; +} + static int osd_sparse_read(struct ceph_connection *con, struct ceph_msg_data_cursor *cursor, char **pbuf) @@ -5853,6 +5882,10 @@ static int osd_sparse_read(struct ceph_connection *con, fallthrough; case CEPH_SPARSE_READ_DATA_LEN: convert_extent_map(sr); + if (!sparse_extent_map_valid(sr)) { + pr_warn_ratelimited("invalid sparse extent map\n"); + return -EREMOTEIO; + } ret = sizeof(sr->sr_datalen); *pbuf = (char *)&sr->sr_datalen; sr->sr_state = CEPH_SPARSE_READ_DATA_PRE; diff --git a/net/ceph/osdmap.c b/net/ceph/osdmap.c index c34a5bf86831b..68bd45852fc37 100644 --- a/net/ceph/osdmap.c +++ b/net/ceph/osdmap.c @@ -519,7 +519,11 @@ static struct crush_map *crush_decode(void *pbyval, void *end) ceph_decode_need(p, end, 4*sizeof(u32), bad); b->id = ceph_decode_32(p); + if (b->id != -1 - i) + goto bad; b->type = ceph_decode_16(p); + if (b->type == 0) + goto bad; b->alg = ceph_decode_8(p); if (b->alg != alg) { b->alg = 0; @@ -1438,7 +1442,7 @@ static struct ceph_pg_mapping *__decode_pg_temp(void **p, void *end, ceph_decode_32_safe(p, end, len, e_inval); if (len == 0 && incremental) return NULL; /* new_pg_temp: [] to remove */ - if (len > (SIZE_MAX - sizeof(*pg)) / sizeof(u32)) + if (len > CEPH_PG_MAX_SIZE) return ERR_PTR(-EINVAL); ceph_decode_need(p, end, len * sizeof(u32), e_inval); @@ -1619,7 +1623,7 @@ static struct ceph_pg_mapping *__decode_pg_upmap_items(void **p, void *end, u32 len, i; ceph_decode_32_safe(p, end, len, e_inval); - if (len > (SIZE_MAX - sizeof(*pg)) / (2 * sizeof(u32))) + if (len > CEPH_PG_MAX_SIZE) return ERR_PTR(-EINVAL); ceph_decode_need(p, end, 2 * len * sizeof(u32), e_inval); @@ -1844,6 +1848,8 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, void *new_up_client; void *new_state; void *new_weight_end; + const u32 new_state_item_size = + sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8)); u32 len; int ret; int i; @@ -1864,7 +1870,8 @@ static int decode_new_up_state_weight(void **p, void *end, u8 struct_v, new_state = *p; ceph_decode_32_safe(p, end, len, e_inval); - len *= sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8)); + if (check_mul_overflow(len, new_state_item_size, &len)) + goto e_inval; ceph_decode_need(p, end, len, e_inval); *p += len; @@ -2806,9 +2813,10 @@ static void get_temp_osds(struct ceph_osdmap *osdmap, } } - /* primary_temp? */ + /* primary_temp? (shouldn't ever be a nonexistent or down OSD) */ pg = lookup_pg_mapping(&osdmap->primary_temp, pgid); - if (pg) + if (pg && !WARN_ON_ONCE(ceph_osd_is_down(osdmap, + pg->primary_temp.osd))) temp->primary = pg->primary_temp.osd; } @@ -3057,8 +3065,11 @@ static int get_immediate_parent(struct crush_map *c, int id, if (b->items[j] != id) continue; - *parent_type_id = b->type; type_cn = lookup_crush_name(&c->type_names, b->type); + if (WARN_ON_ONCE(!type_cn)) + continue; + + *parent_type_id = b->type; parent_loc->cl_type_name = type_cn->cn_name; parent_loc->cl_name = cn->cn_name; return b->id; diff --git a/net/core/datagram.c b/net/core/datagram.c index c285c6465923e..173b5d97bd409 100644 --- a/net/core/datagram.c +++ b/net/core/datagram.c @@ -712,6 +712,9 @@ zerocopy_fill_skb_from_devmem(struct sk_buff *skb, struct iov_iter *from, size_t virt_addr, size, off; struct net_iov *niov; + if (i && skb_frags_readable(skb)) + return -EFAULT; + /* Devmem filling works by taking an IOVEC from the user where the * iov_addrs are interpreted as an offset in bytes into the dma-buf to * send from. We do not support other iter types. diff --git a/net/core/dev.c b/net/core/dev.c index 8dee703a0916e..48983bac5b539 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4005,6 +4005,9 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, return NULL; } +/* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) + * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). + */ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) { netdev_features_t features; @@ -4076,7 +4079,7 @@ struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *d skb->prev = skb; skb = validate_xmit_skb(skb, dev, again); - if (!skb) + if (IS_ERR_OR_NULL(skb)) continue; if (!head) @@ -4702,9 +4705,10 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) { struct net_device *dev = skb->dev; struct netdev_queue *txq = NULL; - struct Qdisc *q; - int rc = -ENOMEM; + enum skb_drop_reason reason; + int cpu, rc = -ENOMEM; bool again = false; + struct Qdisc *q; skb_reset_mac_header(skb); skb_assert_len(skb); @@ -4773,59 +4777,64 @@ int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) * Check this and shot the lock. It is not prone from deadlocks. *Either shot noqueue qdisc, it is even simpler 8) */ - if (dev->flags & IFF_UP) { - int cpu = smp_processor_id(); /* ok because BHs are off */ + if (unlikely(!(dev->flags & IFF_UP))) { + reason = SKB_DROP_REASON_DEV_READY; + goto drop; + } - if (!netif_tx_owned(txq, cpu)) { - bool is_list = false; + cpu = smp_processor_id(); /* ok because BHs are off */ - if (dev_xmit_recursion()) - goto recursion_alert; + if (likely(!netif_tx_owned(txq, cpu))) { + bool is_list = false; - skb = validate_xmit_skb(skb, dev, &again); - if (!skb) - goto out; + if (dev_xmit_recursion()) + goto recursion_alert; - HARD_TX_LOCK(dev, txq, cpu); + skb = validate_xmit_skb(skb, dev, &again); + if (IS_ERR_OR_NULL(skb)) { + if (PTR_ERR(skb) == -EINPROGRESS) + rc = NET_XMIT_SUCCESS; + goto out; + } - if (!netif_xmit_stopped(txq)) { - is_list = !!skb->next; + HARD_TX_LOCK(dev, txq, cpu); - dev_xmit_recursion_inc(); - skb = dev_hard_start_xmit(skb, dev, txq, &rc); - dev_xmit_recursion_dec(); + if (!netif_xmit_stopped(txq)) { + is_list = !!skb->next; - /* GSO segments a single SKB into - * a list of frames. TCP expects error - * to mean none of the data was sent. - */ - if (is_list) - rc = NETDEV_TX_OK; - } - HARD_TX_UNLOCK(dev, txq); - if (!skb) /* xmit completed */ - goto out; + dev_xmit_recursion_inc(); + skb = dev_hard_start_xmit(skb, dev, txq, &rc); + dev_xmit_recursion_dec(); - net_crit_ratelimited("Virtual device %s asks to queue packet!\n", - dev->name); - /* NETDEV_TX_BUSY or queue was stopped */ - if (!is_list) - rc = -ENETDOWN; - } else { - /* Recursion is detected! It is possible, - * unfortunately + /* GSO segments a single SKB into a list of frames. + * TCP expects error to mean none of the data was sent. */ -recursion_alert: - net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", - dev->name); - rc = -ENETDOWN; + if (is_list) + rc = NETDEV_TX_OK; } + HARD_TX_UNLOCK(dev, txq); + if (!skb) /* xmit completed */ + goto out; + + net_crit_ratelimited("Virtual device %s asks to queue packet!\n", + dev->name); + /* NETDEV_TX_BUSY or queue was stopped */ + if (!is_list) + rc = -ENETDOWN; + } else { + /* Recursion is detected! It is possible unfortunately. */ +recursion_alert: + net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", + dev->name); + rc = -ENETDOWN; } + reason = SKB_DROP_REASON_RECURSION_LIMIT; +drop: rcu_read_unlock_bh(); dev_core_stats_tx_dropped_inc(dev); - kfree_skb_list(skb); + kfree_skb_list_reason(skb, reason); return rc; out: rcu_read_unlock_bh(); @@ -5418,12 +5427,16 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, } /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers - * (e.g. bpf_xdp_adjust_tail), we need to update data_len here. + * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution + * from skb->len before updating data_len, then add the new one back. */ - if (xdp_buff_has_frags(xdp)) + skb->len -= skb->data_len; + if (xdp_buff_has_frags(xdp)) { skb->data_len = skb_shinfo(skb)->xdp_frags_size; - else + skb->len += skb->data_len; + } else { skb->data_len = 0; + } /* check if XDP changed eth hdr such SKB needs update */ eth = (struct ethhdr *)xdp->data; @@ -6778,22 +6791,6 @@ static void skb_defer_free_flush(void) #if defined(CONFIG_NET_RX_BUSY_POLL) -static void __busy_poll_stop(struct napi_struct *napi, unsigned long timeout) -{ - if (!timeout) { - gro_normal_list(&napi->gro); - __napi_schedule(napi); - return; - } - - /* Flush too old packets. If HZ < 1000, flush all packets */ - gro_flush_normal(&napi->gro, HZ >= 1000); - - clear_bit(NAPI_STATE_SCHED, &napi->state); - hrtimer_start(&napi->timer, ns_to_ktime(timeout), - HRTIMER_MODE_REL_PINNED); -} - enum { NAPI_F_PREFER_BUSY_POLL = 1, NAPI_F_END_ON_RESCHED = 2, @@ -6809,8 +6806,8 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, /* Busy polling means there is a high chance device driver hard irq * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was * set in napi_schedule_prep(). - * Since we are about to call napi->poll() once more, we can safely - * clear NAPI_STATE_MISSED. + * Since we either call napi->poll() once more or start the timer, + * we can safely clear NAPI_STATE_MISSED. * * Note: x86 could use a single "lock and ..." instruction * to perform these two clear_bit() @@ -6823,27 +6820,35 @@ static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, if (flags & NAPI_F_PREFER_BUSY_POLL) { napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); - if (napi->defer_hard_irqs_count) { - /* A short enough gro flush timeout and long enough - * poll can result in timer firing too early. - * Timer will be armed later if necessary. - */ + if (napi->defer_hard_irqs_count) timeout = napi_get_gro_flush_timeout(napi); + } + if (timeout) { + netpoll_poll_unlock(have_poll_lock); + + /* Drain aged GRO packets before clearing SCHED since the NAPI + * won't run again until after the timer fires. When HZ < 1000, + * GRO age comparison is too coarse, so flush everything. + */ + gro_flush_normal(&napi->gro, HZ >= 1000); + + clear_bit(NAPI_STATE_SCHED, &napi->state); + hrtimer_start(&napi->timer, ns_to_ktime(timeout), + HRTIMER_MODE_REL_PINNED); + } else { + /* Use driver poll to re-enable device interrupts. */ + rc = napi->poll(napi, budget); + /* Unless rc == budget we no longer own the NAPI instance, + * IRQ may fire on another CPU, poll this NAPI, and enter GRO. + */ + trace_napi_poll(napi, rc, budget); + netpoll_poll_unlock(have_poll_lock); + if (rc == budget) { + gro_normal_list(&napi->gro); + __napi_schedule(napi); } } - /* All we really want here is to re-enable device interrupts. - * Ideally, a new ndo_busy_poll_stop() could avoid another round. - */ - rc = napi->poll(napi, budget); - /* We can't gro_normal_list() here, because napi->poll() might have - * rearmed the napi (napi_complete_done()) in which case it could - * already be running on another CPU. - */ - trace_napi_poll(napi, rc, budget); - netpoll_poll_unlock(have_poll_lock); - if (rc == budget) - __busy_poll_stop(napi, timeout); bpf_net_ctx_clear(bpf_net_ctx); local_bh_enable(); } @@ -9846,7 +9851,7 @@ int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len) unsigned int orig_len = dev->tx_queue_len; int res; - if (new_len != (unsigned int)new_len) + if (new_len > S16_MAX) return -ERANGE; if (new_len != orig_len) { @@ -10228,6 +10233,37 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, netdev_ops_assert_locked(dev); + if (prog) { + enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB + ? XDP_MODE_DRV : XDP_MODE_SKB; + bool offload = mode == XDP_MODE_HW; + + if (!offload && dev_xdp_prog(dev, other_mode)) { + NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); + return -EEXIST; + } + if (!offload && bpf_prog_is_offloaded(prog->aux)) { + NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) { + NL_SET_ERR_MSG(extack, "Program bound to different device"); + return -EINVAL; + } + if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) { + NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_DEVMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); + return -EINVAL; + } + if (prog->expected_attach_type == BPF_XDP_CPUMAP) { + NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); + return -EINVAL; + } + } + if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && prog && !prog->aux->xdp_has_frags) { NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); @@ -10367,38 +10403,10 @@ static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack new_prog = link->link.prog; if (new_prog) { - bool offload = mode == XDP_MODE_HW; - enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB - ? XDP_MODE_DRV : XDP_MODE_SKB; - if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { NL_SET_ERR_MSG(extack, "XDP program already attached"); return -EBUSY; } - if (!offload && dev_xdp_prog(dev, other_mode)) { - NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); - return -EEXIST; - } - if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { - NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { - NL_SET_ERR_MSG(extack, "Program bound to different device"); - return -EINVAL; - } - if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { - NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); - return -EINVAL; - } - if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { - NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); - return -EINVAL; - } } /* don't call drivers if the effective program didn't change */ @@ -11214,6 +11222,21 @@ static void netdev_free_phy_link_topology(struct net_device *dev) } } +static void init_rx_queue_cfgs(struct net_device *dev) +{ + const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; + struct netdev_rx_queue *rxq; + int i; + + if (!qops || !qops->ndo_default_qcfg) + return; + + for (i = 0; i < dev->num_rx_queues; i++) { + rxq = __netif_get_rx_queue(dev, i); + qops->ndo_default_qcfg(dev, &rxq->qcfg); + } +} + /** * register_netdevice() - register a network device * @dev: device to register @@ -11259,6 +11282,8 @@ int register_netdevice(struct net_device *dev) if (!dev->name_node) goto out; + init_rx_queue_cfgs(dev); + /* Init, if this function is available */ if (dev->netdev_ops->ndo_init) { ret = dev->netdev_ops->ndo_init(dev); @@ -12049,6 +12074,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, free_percpu(dev->pcpu_refcnt); free_dev: #endif + ref_tracker_dir_exit(&dev->refcnt_tracker); kvfree(dev); return NULL; } @@ -12417,7 +12443,7 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat, int new_ifindex, struct netlink_ext_ack *extack) { - struct netdev_name_node *name_node; + struct netdev_name_node *name_node, *tmp; struct net *net_old = dev_net(dev); char new_name[IFNAMSIZ] = {}; int err, new_nsid; @@ -12463,13 +12489,19 @@ int __dev_change_net_namespace(struct net_device *dev, struct net *net, } /* Check that none of the altnames conflicts. */ err = -EEXIST; - netdev_for_each_altname(dev, name_node) { - if (netdev_name_in_use(net, name_node->name)) { - NL_SET_ERR_MSG_FMT(extack, - "An interface with the altname %s exists in the target netns", - name_node->name); - goto out; + netdev_for_each_altname_safe(dev, name_node, tmp) { + if (!netdev_name_in_use(net, name_node->name)) + continue; + + if (!check_net(net_old)) { + __netdev_name_node_alt_destroy(name_node); + continue; } + + NL_SET_ERR_MSG_FMT(extack, + "An interface with the altname %s exists in the target netns", + name_node->name); + goto out; } /* Check that new_ifindex isn't used yet. */ @@ -12920,7 +12952,6 @@ static struct pernet_operations __net_initdata netdev_net_ops = { static void __net_exit default_device_exit_net(struct net *net) { - struct netdev_name_node *name_node, *tmp; struct net_device *dev, *aux; /* * Push all migratable network devices back to the @@ -12944,10 +12975,6 @@ static void __net_exit default_device_exit_net(struct net *net) if (netdev_name_in_use(&init_net, fb_name)) snprintf(fb_name, IFNAMSIZ, "dev%%d"); - netdev_for_each_altname_safe(dev, name_node, tmp) - if (netdev_name_in_use(&init_net, name_node->name)) - __netdev_name_node_alt_destroy(name_node); - err = dev_change_net_namespace(dev, &init_net, fb_name); if (err) { pr_emerg("%s: failed to move %s to init_net: %d\n", diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 60d31c2feed3e..a5e75cd04d6ca 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -531,10 +531,10 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore, return; unlock_free: - spin_unlock_irqrestore(&data->drop_queue.lock, flags); u64_stats_update_begin(&data->stats.syncp); u64_stats_inc(&data->stats.dropped); u64_stats_update_end(&data->stats.syncp); + spin_unlock_irqrestore(&data->drop_queue.lock, flags); consume_skb(nskb); } @@ -567,13 +567,13 @@ static size_t net_dm_packet_report_size(size_t payload_len) /* NET_DM_ATTR_ORIGIN */ nla_total_size(sizeof(u16)) + /* NET_DM_ATTR_PC */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_SYMBOL */ nla_total_size(NET_DM_MAX_SYMBOL_LEN + 1) + /* NET_DM_ATTR_IN_PORT */ net_dm_in_port_size() + /* NET_DM_ATTR_TIMESTAMP */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_ORIG_LEN */ nla_total_size(sizeof(u32)) + /* NET_DM_ATTR_PROTO */ @@ -672,9 +672,7 @@ static int net_dm_packet_report_fill(struct sk_buff *msg, struct sk_buff *skb, if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol))) goto nla_put_failure; - attr = skb_put(msg, nla_total_size(payload_len)); - attr->nla_type = NET_DM_ATTR_PAYLOAD; - attr->nla_len = nla_attr_size(payload_len); + attr = __nla_reserve(msg, NET_DM_ATTR_PAYLOAD, payload_len); if (skb_copy_bits(skb, 0, nla_data(attr), payload_len)) goto nla_put_failure; @@ -769,7 +767,7 @@ net_dm_hw_packet_report_size(size_t payload_len, /* NET_DM_ATTR_FLOW_ACTION_COOKIE */ net_dm_flow_action_cookie_size(hw_metadata) + /* NET_DM_ATTR_TIMESTAMP */ - nla_total_size(sizeof(u64)) + + nla_total_size_64bit(sizeof(u64)) + /* NET_DM_ATTR_ORIG_LEN */ nla_total_size(sizeof(u32)) + /* NET_DM_ATTR_PROTO */ @@ -832,9 +830,7 @@ static int net_dm_hw_packet_report_fill(struct sk_buff *msg, if (nla_put_u16(msg, NET_DM_ATTR_PROTO, be16_to_cpu(skb->protocol))) goto nla_put_failure; - attr = skb_put(msg, nla_total_size(payload_len)); - attr->nla_type = NET_DM_ATTR_PAYLOAD; - attr->nla_len = nla_attr_size(payload_len); + attr = __nla_reserve(msg, NET_DM_ATTR_PAYLOAD, payload_len); if (skb_copy_bits(skb, 0, nla_data(attr), payload_len)) goto nla_put_failure; @@ -1002,10 +998,10 @@ net_dm_hw_trap_packet_probe(void *ignore, const struct devlink *devlink, return; unlock_free: - spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags); u64_stats_update_begin(&hw_data->stats.syncp); u64_stats_inc(&hw_data->stats.dropped); u64_stats_update_end(&hw_data->stats.syncp); + spin_unlock_irqrestore(&hw_data->drop_queue.lock, flags); net_dm_hw_metadata_free(n_hw_metadata); free: consume_skb(nskb); diff --git a/net/core/filter.c b/net/core/filter.c index 42385e7904313..35427793ddb7e 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -2551,11 +2551,13 @@ int skb_do_redirect(struct sk_buff *skb) BPF_CALL_2(bpf_redirect, u32, ifindex, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely(flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL))) + if (unlikely(!bpf_net_ctx_get() || + (flags & (~(BPF_F_INGRESS) | BPF_F_REDIRECT_INTERNAL)))) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = flags; ri->tgt_index = ifindex; @@ -2572,11 +2574,12 @@ static const struct bpf_func_proto bpf_redirect_proto = { BPF_CALL_2(bpf_redirect_peer, u32, ifindex, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely(flags)) + if (unlikely(!bpf_net_ctx_get() || flags)) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = BPF_F_PEER; ri->tgt_index = ifindex; @@ -2594,11 +2597,13 @@ static const struct bpf_func_proto bpf_redirect_peer_proto = { BPF_CALL_4(bpf_redirect_neigh, u32, ifindex, struct bpf_redir_neigh *, params, int, plen, u64, flags) { - struct bpf_redirect_info *ri = bpf_net_ctx_get_ri(); + struct bpf_redirect_info *ri; - if (unlikely((plen && plen < sizeof(*params)) || flags)) + if (unlikely((plen && plen < sizeof(*params)) || + !bpf_net_ctx_get() || flags)) return TC_ACT_SHOT; + ri = bpf_net_ctx_get_ri(); ri->flags = BPF_F_NEIGH | (plen ? BPF_F_NEXTHOP : 0); ri->tgt_index = ifindex; @@ -8189,10 +8194,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_setsockopt_proto; default: return NULL; @@ -8212,10 +8215,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) case BPF_CGROUP_UNIX_SENDMSG: case BPF_CGROUP_INET4_GETPEERNAME: case BPF_CGROUP_INET6_GETPEERNAME: - case BPF_CGROUP_UNIX_GETPEERNAME: case BPF_CGROUP_INET4_GETSOCKNAME: case BPF_CGROUP_INET6_GETSOCKNAME: - case BPF_CGROUP_UNIX_GETSOCKNAME: return &bpf_sock_addr_getsockopt_proto; default: return NULL; @@ -10539,10 +10540,11 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, si->dst_reg, si->dst_reg, \ offsetof(OBJ, OBJ_FIELD)); \ if (si->dst_reg == si->src_reg) { \ - *insn++ = BPF_JMP_A(1); \ + *insn++ = BPF_JMP_A(2); \ *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, \ temp)); \ + *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); \ } \ } while (0) @@ -10576,10 +10578,11 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, si->dst_reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, sk));\ if (si->dst_reg == si->src_reg) { \ - *insn++ = BPF_JMP_A(1); \ + *insn++ = BPF_JMP_A(2); \ *insn++ = BPF_LDX_MEM(BPF_DW, reg, si->src_reg, \ offsetof(struct bpf_sock_ops_kern, \ temp)); \ + *insn++ = BPF_MOV64_IMM(si->dst_reg, 0); \ } \ } while (0) diff --git a/net/core/gro.c b/net/core/gro.c index 9ec8a46b30bb9..252d7851d1aee 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -118,9 +118,12 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) if (unlikely(p->len + len >= GRO_LEGACY_MAX_SIZE)) { if (NAPI_GRO_CB(skb)->proto != IPPROTO_TCP || + NAPI_GRO_CB(skb)->encap_mark || + p->encapsulation || (p->protocol == htons(ETH_P_IPV6) && - skb_headroom(p) < sizeof(struct hop_jumbo_hdr)) || - p->encapsulation) + p->mac_header < sizeof(struct hop_jumbo_hdr)) || + (p->protocol != htons(ETH_P_IPV6) && + p->protocol != htons(ETH_P_IP))) return -E2BIG; } @@ -231,7 +234,9 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) { - if (unlikely(p->len + skb->len >= 65536)) + /* make sure to check flush flag and to not merge */ + if (unlikely(p->len + skb->len >= 65536 || + NAPI_GRO_CB(skb)->flush)) return -E2BIG; if (!pskb_may_pull(skb, skb_gro_offset(skb))) { diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c index a725d21159a6f..ca82da018be93 100644 --- a/net/core/gro_cells.c +++ b/net/core/gro_cells.c @@ -22,6 +22,8 @@ int gro_cells_receive(struct gro_cells *gcells, struct sk_buff *skb) if (unlikely(!(dev->flags & IFF_UP))) goto drop; + skb_unset_transport_header(skb); + if (!gcells->cells || skb_cloned(skb) || netif_elide_gro(dev)) { res = netif_rx(skb); goto unlock; diff --git a/net/core/lwt_bpf.c b/net/core/lwt_bpf.c index 2efcda93d1397..953237bca36b6 100644 --- a/net/core/lwt_bpf.c +++ b/net/core/lwt_bpf.c @@ -163,10 +163,10 @@ static int bpf_output(struct net *net, struct sock *sk, struct sk_buff *skb) return dst->lwtstate->orig_output(net, sk, skb); } -static int xmit_check_hhlen(struct sk_buff *skb, int hh_len) +static int xmit_check_headroom(struct sk_buff *skb, int hroom) { - if (skb_headroom(skb) < hh_len) { - int nhead = HH_DATA_ALIGN(hh_len - skb_headroom(skb)); + if (skb_headroom(skb) < hroom) { + int nhead = hroom - skb_headroom(skb); if (pskb_expand_head(skb, nhead, 0, GFP_ATOMIC)) return -ENOMEM; @@ -251,8 +251,10 @@ static int bpf_lwt_xmit_reroute(struct sk_buff *skb) * if there is enough header space in skb. */ err = skb_cow_head(skb, LL_RESERVED_SPACE(dst->dev)); - if (unlikely(err)) + if (unlikely(err)) { + dst_release(dst); goto err; + } skb_dst_drop(skb); skb_dst_set(skb, dst); @@ -276,7 +278,7 @@ static int bpf_xmit(struct sk_buff *skb) bpf = bpf_lwt_lwtunnel(dst->lwtstate); if (bpf->xmit.prog) { - int hh_len = dst->dev->hard_header_len; + int hroom = LL_RESERVED_SPACE(dst->dev); __be16 proto = skb->protocol; int ret; @@ -292,9 +294,12 @@ static int bpf_xmit(struct sk_buff *skb) return -EINVAL; } /* If the header was expanded, headroom might be too - * small for L2 header to come, expand as needed. + * small for the L2 header to come, expand as needed. + * neigh_hh_output() copies the cached header in + * HH_DATA_MOD aligned chunks, so match the reservation + * made before LWT xmit. */ - ret = xmit_check_hhlen(skb, hh_len); + ret = xmit_check_headroom(skb, hroom); if (unlikely(ret)) return ret; diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index b0e0f22d7b213..4bece5eda9d1a 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -398,14 +398,8 @@ static __net_init void preinit_net_sysctl(struct net *net) } /* init code that must occur even if setup_net() is not called. */ -static __net_init int preinit_net(struct net *net, struct user_namespace *user_ns) +static __net_init void preinit_net(struct net *net, struct user_namespace *user_ns) { - int ret; - - ret = ns_common_init(net); - if (ret) - return ret; - refcount_set(&net->passive, 1); ref_tracker_dir_init(&net->refcnt_tracker, 128, "net_refcnt"); ref_tracker_dir_init(&net->notrefcnt_tracker, 128, "net_notrefcnt"); @@ -426,7 +420,6 @@ static __net_init int preinit_net(struct net *net, struct user_namespace *user_n INIT_LIST_HEAD(&net->ptype_all); INIT_LIST_HEAD(&net->ptype_specific); preinit_net_sysctl(net); - return 0; } /* @@ -531,8 +524,12 @@ void net_passive_dec(struct net *net) if (refcount_dec_and_test(&net->passive)) { kfree(rcu_access_pointer(net->gen)); +#ifdef CONFIG_REF_TRACKER /* There should not be any trackers left there. */ ref_tracker_dir_exit(&net->notrefcnt_tracker); + if (!net->refcnt_tracker.dead) + ref_tracker_dir_exit(&net->refcnt_tracker); +#endif /* Wait for an extra rcu_barrier() before final free. */ llist_add(&net->defer_free_list, &defer_free_list); @@ -567,12 +564,14 @@ struct net *copy_net_ns(u64 flags, goto dec_ucounts; } - rv = preinit_net(net, user_ns); - if (rv < 0) - goto dec_ucounts; + preinit_net(net, user_ns); net->ucounts = ucounts; get_user_ns(user_ns); + rv = ns_common_init(net); + if (rv) + goto put_userns_no_common; + rv = down_read_killable(&pernet_ops_rwsem); if (rv < 0) goto put_userns; @@ -584,6 +583,7 @@ struct net *copy_net_ns(u64 flags, if (rv < 0) { put_userns: ns_common_free(net); +put_userns_no_common: #ifdef CONFIG_KEYS key_remove_domain(net->key_domain); #endif @@ -1281,7 +1281,8 @@ void __init net_ns_init(void) * This currently cannot fail as the initial network namespace * has a static inode number. */ - if (preinit_net(&init_net, &init_user_ns)) + preinit_net(&init_net, &init_user_ns); + if (ns_common_init(&init_net)) panic("Could not preinitialize the initial network namespace"); down_write(&pernet_ops_rwsem); diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c index c7d9341b76307..485bdb7447b99 100644 --- a/net/core/netdev_rx_queue.c +++ b/net/core/netdev_rx_queue.c @@ -22,6 +22,7 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) { struct netdev_rx_queue *rxq = __netif_get_rx_queue(dev, rxq_idx); const struct netdev_queue_mgmt_ops *qops = dev->queue_mgmt_ops; + struct netdev_queue_config qcfg; void *new_mem, *old_mem; int err; @@ -29,8 +30,21 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) !qops->ndo_queue_mem_alloc || !qops->ndo_queue_start) return -EOPNOTSUPP; + if (WARN_ON_ONCE(qops->supported_params && !qops->ndo_default_qcfg)) + return -EINVAL; + netdev_assert_locked(dev); + memset(&qcfg, 0, sizeof(qcfg)); + if (qops->ndo_default_qcfg) + qops->ndo_default_qcfg(dev, &qcfg); + + if (rxq->mp_params.rx_page_size) { + if (!(qops->supported_params & QCFG_RX_PAGE_SIZE)) + return -EOPNOTSUPP; + qcfg.rx_page_size = rxq->mp_params.rx_page_size; + } + new_mem = kvzalloc(qops->ndo_queue_mem_size, GFP_KERNEL); if (!new_mem) return -ENOMEM; @@ -41,7 +55,7 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) goto err_free_new_mem; } - err = qops->ndo_queue_mem_alloc(dev, new_mem, rxq_idx); + err = qops->ndo_queue_mem_alloc(dev, &qcfg, new_mem, rxq_idx); if (err) goto err_free_old_mem; @@ -54,7 +68,7 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) if (err) goto err_free_new_queue_mem; - err = qops->ndo_queue_start(dev, new_mem, rxq_idx); + err = qops->ndo_queue_start(dev, &qcfg, new_mem, rxq_idx); if (err) goto err_start_queue; } else { @@ -66,6 +80,7 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) kvfree(old_mem); kvfree(new_mem); + rxq->qcfg = qcfg; return 0; err_start_queue: @@ -76,7 +91,7 @@ int netdev_rx_queue_restart(struct net_device *dev, unsigned int rxq_idx) * WARN if we fail to recover the old rx queue, and at least free * old_mem so we don't also leak that. */ - if (qops->ndo_queue_start(dev, old_mem, rxq_idx)) { + if (qops->ndo_queue_start(dev, &rxq->qcfg, old_mem, rxq_idx)) { WARN(1, "Failed to restart old queue in error path. RX queue %d may be unhealthy.", rxq_idx); diff --git a/net/core/page_pool.c b/net/core/page_pool.c index b775b6305fb78..d1fa5640a9128 100644 --- a/net/core/page_pool.c +++ b/net/core/page_pool.c @@ -482,6 +482,13 @@ static int page_pool_register_dma_index(struct page_pool *pool, if (unlikely(!PP_DMA_INDEX_BITS)) goto out; + /* + * Drivers request GFP flags according to both the current context and + * the device constraints, but the XArray entry itself is by no mean + * used by the device, so remove zone/policy flags. + */ + gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM | __GFP_COMP); + if (in_softirq()) err = xa_alloc(&pool->dma_mapped, &id, netmem_to_page(netmem), PP_DMA_INDEX_LIMIT, gfp); @@ -498,29 +505,40 @@ static int page_pool_register_dma_index(struct page_pool *pool, return err; } -static int page_pool_release_dma_index(struct page_pool *pool, - netmem_ref netmem) +static void __page_pool_unmap_netmem_dma(struct page_pool *pool, + netmem_ref netmem) { struct page *old, *page = netmem_to_page(netmem); unsigned long id; + dma_addr_t dma; - if (unlikely(!PP_DMA_INDEX_BITS)) - return 0; - - id = netmem_get_dma_index(netmem); - if (!id) - return -1; + if (!pool->dma_map) + return; - if (in_softirq()) - old = xa_cmpxchg(&pool->dma_mapped, id, page, NULL, 0); - else - old = xa_cmpxchg_bh(&pool->dma_mapped, id, page, NULL, 0); - if (old != page) - return -1; + /* Cache dma_addr before xa_cmpxchg. The scrub path holds no page ref; + * the unref path calls put_page() regardless of cmpxchg outcome, so + * after the cmpxchg we cannot safely touch netmem fields. + */ + dma = page_pool_get_dma_addr_netmem(netmem); - netmem_set_dma_index(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) { + id = netmem_get_dma_index(netmem); + if (!id) + return; + + if (in_softirq()) + old = xa_cmpxchg(&pool->dma_mapped, + id, page, NULL, 0); + else + old = xa_cmpxchg_bh(&pool->dma_mapped, + id, page, NULL, 0); + if (old != page) + return; + } - return 0; + dma_unmap_page_attrs(pool->p.dev, dma, + PAGE_SIZE << pool->p.order, pool->p.dma_dir, + DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); } static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem, gfp_t gfp) @@ -726,24 +744,16 @@ void page_pool_clear_pp_info(netmem_ref netmem) static __always_inline void __page_pool_release_netmem_dma(struct page_pool *pool, netmem_ref netmem) { - dma_addr_t dma; - + /* Caller must hold a page ref: __page_pool_unmap_netmem_dma() is + * safe without a ref, but the field clears below require it. + */ if (!pool->dma_map) - /* Always account for inflight pages, even if we didn't - * map them - */ return; - if (page_pool_release_dma_index(pool, netmem)) - return; - - dma = page_pool_get_dma_addr_netmem(netmem); - - /* When page is unmapped, it cannot be returned to our pool */ - dma_unmap_page_attrs(pool->p.dev, dma, - PAGE_SIZE << pool->p.order, pool->p.dma_dir, - DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); + __page_pool_unmap_netmem_dma(pool, netmem); page_pool_set_dma_addr_netmem(netmem, 0); + if (likely(PP_DMA_INDEX_BITS)) + netmem_set_dma_index(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need @@ -1169,8 +1179,9 @@ static void page_pool_scrub(struct page_pool *pool) synchronize_net(); } + /* No page ref, dma-unmap only. */ xa_for_each(&pool->dma_mapped, id, ptr) - __page_pool_release_netmem_dma(pool, page_to_netmem((struct page *)ptr)); + __page_pool_unmap_netmem_dma(pool, page_to_netmem((struct page *)ptr)); } /* No more consumers should exist, but producers could still diff --git a/net/core/pktgen.c b/net/core/pktgen.c index d41b03fd1f636..a180c061a02d9 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -3971,6 +3971,7 @@ static void _rem_dev_from_if_list(struct pktgen_thread *t, struct pktgen_dev *p; if_lock(t); + proc_remove(pkt_dev->entry); list_for_each_safe(q, n, &t->if_list) { p = list_entry(q, struct pktgen_dev, list); if (p == pkt_dev) @@ -4000,9 +4001,6 @@ static int pktgen_remove_device(struct pktgen_thread *t, * list to determine if interface already exist, avoid race * with proc_create_data() */ - proc_remove(pkt_dev->entry); - - /* And update the thread if_list */ _rem_dev_from_if_list(t, pkt_dev); #ifdef CONFIG_XFRM diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 4909f20ff4d63..b0e6a6eff23ba 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -2214,6 +2214,11 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, return -EMSGSIZE; } +static const struct netlink_range_validation txqlen_range = { + .min = 0, + .max = S16_MAX, +}; + static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_UNSPEC] = { .strict_start_type = IFLA_DPLL_PIN }, [IFLA_IFNAME] = { .type = NLA_STRING, .len = IFNAMSIZ-1 }, @@ -2224,7 +2229,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = { [IFLA_LINK] = { .type = NLA_U32 }, [IFLA_MASTER] = { .type = NLA_U32 }, [IFLA_CARRIER] = { .type = NLA_U8 }, - [IFLA_TXQLEN] = { .type = NLA_U32 }, + [IFLA_TXQLEN] = NLA_POLICY_FULL_RANGE(NLA_U32, &txqlen_range), [IFLA_WEIGHT] = { .type = NLA_U32 }, [IFLA_OPERSTATE] = { .type = NLA_U8 }, [IFLA_LINKMODE] = { .type = NLA_U8 }, diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 43dca2c045766..0324292309921 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -895,6 +895,18 @@ static void skb_clone_fraglist(struct sk_buff *skb) skb_get(list); } +/** + * skb_pp_cow_data() - copy skb data into page-pool backed storage + * @pool: page pool to allocate from + * @pskb: pointer to skb pointer, replaced with the copied skb on success + * @headroom: headroom to reserve in the copied skb + * + * skb_copy_bits() handles both frags[] and frag_list input. If the copied + * skb remains non-linear, it uses frags[], which is the representation used + * by XDP multi-buffer. + * + * Return: 0 on success or a negative errno on failure. + */ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, unsigned int headroom) { @@ -904,12 +916,6 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, int err, i, head_off; void *data; - /* XDP does not support fraglist so we need to linearize - * the skb. - */ - if (skb_has_frag_list(skb)) - return -EOPNOTSUPP; - max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - headroom); if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE) return -ENOMEM; @@ -1367,10 +1373,13 @@ EXPORT_SYMBOL(skb_dump); * * Report xmit error if a device callback is tracking this skb. * skb must be freed afterwards. + * + * Does nothing for a cloned skb: the zerocopy state lives in + * skb_shinfo(), which the clones share. */ void skb_tx_error(struct sk_buff *skb) { - if (skb) { + if (skb && !skb_cloned(skb)) { skb_zcopy_downgrade_managed(skb); skb_zcopy_clear(skb, true); } @@ -1932,12 +1941,12 @@ int skb_copy_ubufs(struct sk_buff *skb, gfp_t gfp_mask) int i, order, psize, new_frags; u32 d_off; - if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) - return -EINVAL; - if (!skb_frags_readable(skb)) return -EFAULT; + if (skb_shared(skb) || skb_unclone(skb, gfp_mask)) + return -EINVAL; + if (!num_frags) goto release; @@ -3850,10 +3859,9 @@ skb_zerocopy(struct sk_buff *to, struct sk_buff *from, int len, int hlen) skb_len_add(to, len + plen); - if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) { - skb_tx_error(from); + if (unlikely(skb_orphan_frags(from, GFP_ATOMIC))) return -ENOMEM; - } + skb_zerocopy_clone(to, from, GFP_ATOMIC); for (i = 0; i < skb_shinfo(from)->nr_frags; i++) { @@ -6616,6 +6624,13 @@ int skb_mpls_pop(struct sk_buff *skb, __be16 next_proto, int mac_len, } skb->protocol = next_proto; + /* The last label is gone, so the inner header recorded by + * skb_mpls_push() no longer describes this packet. Drop it, or a + * later push keeps the stale offset. + */ + if (!eth_p_mpls(next_proto)) + skb->inner_protocol = 0; + return 0; } EXPORT_SYMBOL_GPL(skb_mpls_pop); diff --git a/net/core/sock.c b/net/core/sock.c index c52cec0bf942c..1b64ac35073db 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -520,32 +520,24 @@ int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) } EXPORT_SYMBOL(__sock_queue_rcv_skb); -int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason *reason) +enum skb_drop_reason +sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb) { enum skb_drop_reason drop_reason; int err; err = sk_filter_reason(sk, skb, &drop_reason); if (err) - goto out; + return drop_reason; err = __sock_queue_rcv_skb(sk, skb); switch (err) { case -ENOMEM: - drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF; - break; + return SKB_DROP_REASON_SOCKET_RCVBUFF; case -ENOBUFS: - drop_reason = SKB_DROP_REASON_PROTO_MEM; - break; - default: - drop_reason = SKB_NOT_DROPPED_YET; - break; + return SKB_DROP_REASON_PROTO_MEM; } -out: - if (reason) - *reason = drop_reason; - return err; + return SKB_NOT_DROPPED_YET; } EXPORT_SYMBOL(sock_queue_rcv_skb_reason); @@ -786,7 +778,6 @@ bool sk_mc_loop(const struct sock *sk) return inet6_test_bit(MC6_LOOP, sk); #endif } - WARN_ON_ONCE(1); return true; } EXPORT_SYMBOL(sk_mc_loop); @@ -2525,6 +2516,11 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) cgroup_sk_clone(&newsk->sk_cgrp_data); + RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); + + if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) + net_enable_timestamp(); + rcu_read_lock(); filter = rcu_dereference(sk->sk_filter); if (filter != NULL) @@ -2547,8 +2543,6 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) goto free; } - RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL); - if (bpf_sk_storage_clone(sk, newsk)) goto free; @@ -2575,9 +2569,6 @@ struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority) if (newsk->sk_prot->sockets_allocated) sk_sockets_allocated_inc(newsk); - - if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP) - net_enable_timestamp(); out: return newsk; free: diff --git a/net/core/sock_map.c b/net/core/sock_map.c index 5947b38e4f8b6..70bbc78fb079f 100644 --- a/net/core/sock_map.c +++ b/net/core/sock_map.c @@ -542,6 +542,8 @@ static bool sock_map_sk_state_allowed(const struct sock *sk) { if (sk_is_tcp(sk)) return (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_LISTEN); + if (sk_is_udp(sk)) + return sk_hashed(sk); if (sk_is_stream_unix(sk)) return (1 << sk->sk_state) & TCPF_ESTABLISHED; if (sk_is_vsock(sk) && diff --git a/net/core/xdp.c b/net/core/xdp.c index 9100e160113a9..7a28900c52c6c 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -585,7 +585,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdp->data - xdp->data_meta; totsize = xdp->data_end - xdp->data + metasize; - if (sizeof(*xdpf) + totsize > PAGE_SIZE) + if (sizeof(*xdpf) + totsize > SKB_WITH_OVERHEAD(PAGE_SIZE)) return NULL; page = dev_alloc_page(); @@ -602,7 +602,7 @@ struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp) xdpf->data = addr + metasize; xdpf->len = totsize - metasize; - xdpf->headroom = 0; + xdpf->headroom = metasize; xdpf->metasize = metasize; xdpf->frame_sz = PAGE_SIZE; xdpf->mem_type = MEM_TYPE_PAGE_ORDER0; @@ -871,7 +871,7 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf) headroom = xdpf->headroom + sizeof(*xdpf); totalsize = headroom + xdpf->len; - if (unlikely(totalsize > PAGE_SIZE)) + if (unlikely(totalsize > SKB_WITH_OVERHEAD(PAGE_SIZE))) return NULL; page = dev_alloc_page(); if (!page) diff --git a/net/devlink/dev.c b/net/devlink/dev.c index 02602704bdeaa..ed442c1bed8db 100644 --- a/net/devlink/dev.c +++ b/net/devlink/dev.c @@ -577,6 +577,7 @@ int devlink_nl_reload_doit(struct sk_buff *skb, struct genl_info *info) action != DEVLINK_RELOAD_ACTION_DRIVER_REINIT) { NL_SET_ERR_MSG_MOD(info->extack, "Changing namespace is only supported for reinit action"); + put_net(dest_net); return -EOPNOTSUPP; } } diff --git a/net/devlink/netlink_gen.c b/net/devlink/netlink_gen.c index 9fd00977d59e3..5ad435aee29de 100644 --- a/net/devlink/netlink_gen.c +++ b/net/devlink/netlink_gen.c @@ -229,7 +229,7 @@ static const struct nla_policy devlink_eswitch_get_nl_policy[DEVLINK_ATTR_DEV_NA static const struct nla_policy devlink_eswitch_set_nl_policy[DEVLINK_ATTR_ESWITCH_ENCAP_MODE + 1] = { [DEVLINK_ATTR_BUS_NAME] = { .type = NLA_NUL_STRING, }, [DEVLINK_ATTR_DEV_NAME] = { .type = NLA_NUL_STRING, }, - [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_MAX(NLA_U16, 1), + [DEVLINK_ATTR_ESWITCH_MODE] = NLA_POLICY_MAX(NLA_U16, 2), [DEVLINK_ATTR_ESWITCH_INLINE_MODE] = NLA_POLICY_MAX(NLA_U8, 3), [DEVLINK_ATTR_ESWITCH_ENCAP_MODE] = NLA_POLICY_MAX(NLA_U8, 1), }; diff --git a/net/devlink/port.c b/net/devlink/port.c index 93d8a25bb920c..21c72f1af9259 100644 --- a/net/devlink/port.c +++ b/net/devlink/port.c @@ -1024,6 +1024,7 @@ void devlink_port_init(struct devlink *devlink, return; devlink_port->devlink = devlink; INIT_LIST_HEAD(&devlink_port->region_list); + INIT_LIST_HEAD(&devlink_port->resource_list); devlink_port->initialized = true; } EXPORT_SYMBOL_GPL(devlink_port_init); @@ -1041,6 +1042,7 @@ EXPORT_SYMBOL_GPL(devlink_port_init); void devlink_port_fini(struct devlink_port *devlink_port) { WARN_ON(!list_empty(&devlink_port->region_list)); + WARN_ON(!list_empty(&devlink_port->resource_list)); } EXPORT_SYMBOL_GPL(devlink_port_fini); diff --git a/net/devlink/resource.c b/net/devlink/resource.c index 2d6324f3d91f2..dc9a55e443c21 100644 --- a/net/devlink/resource.c +++ b/net/devlink/resource.c @@ -36,15 +36,16 @@ struct devlink_resource { }; static struct devlink_resource * -devlink_resource_find(struct devlink *devlink, - struct devlink_resource *resource, u64 resource_id) +__devlink_resource_find(struct list_head *resource_list_head, + struct devlink_resource *resource, + u64 resource_id) { struct list_head *resource_list; if (resource) resource_list = &resource->resource_list; else - resource_list = &devlink->resource_list; + resource_list = resource_list_head; list_for_each_entry(resource, resource_list, list) { struct devlink_resource *child_resource; @@ -52,14 +53,23 @@ devlink_resource_find(struct devlink *devlink, if (resource->id == resource_id) return resource; - child_resource = devlink_resource_find(devlink, resource, - resource_id); + child_resource = __devlink_resource_find(resource_list_head, + resource, + resource_id); if (child_resource) return child_resource; } return NULL; } +static struct devlink_resource * +devlink_resource_find(struct devlink *devlink, + struct devlink_resource *resource, u64 resource_id) +{ + return __devlink_resource_find(&devlink->resource_list, + resource, resource_id); +} + static void devlink_resource_validate_children(struct devlink_resource *resource) { @@ -314,26 +324,12 @@ int devlink_resources_validate(struct devlink *devlink, return err; } -/** - * devl_resource_register - devlink resource register - * - * @devlink: devlink - * @resource_name: resource's name - * @resource_size: resource's size - * @resource_id: resource's id - * @parent_resource_id: resource's parent id - * @size_params: size parameters - * - * Generic resources should reuse the same names across drivers. - * Please see the generic resources list at: - * Documentation/networking/devlink/devlink-resource.rst - */ -int devl_resource_register(struct devlink *devlink, - const char *resource_name, - u64 resource_size, - u64 resource_id, - u64 parent_resource_id, - const struct devlink_resource_size_params *size_params) +static int +__devl_resource_register(struct devlink *devlink, + struct list_head *resource_list_head, + const char *resource_name, u64 resource_size, + u64 resource_id, u64 parent_resource_id, + const struct devlink_resource_size_params *params) { struct devlink_resource *resource; struct list_head *resource_list; @@ -343,7 +339,8 @@ int devl_resource_register(struct devlink *devlink, top_hierarchy = parent_resource_id == DEVLINK_RESOURCE_ID_PARENT_TOP; - resource = devlink_resource_find(devlink, NULL, resource_id); + resource = __devlink_resource_find(resource_list_head, NULL, + resource_id); if (resource) return -EEXIST; @@ -352,12 +349,13 @@ int devl_resource_register(struct devlink *devlink, return -ENOMEM; if (top_hierarchy) { - resource_list = &devlink->resource_list; + resource_list = resource_list_head; } else { struct devlink_resource *parent_resource; - parent_resource = devlink_resource_find(devlink, NULL, - parent_resource_id); + parent_resource = __devlink_resource_find(resource_list_head, + NULL, + parent_resource_id); if (parent_resource) { resource_list = &parent_resource->resource_list; resource->parent = parent_resource; @@ -372,46 +370,78 @@ int devl_resource_register(struct devlink *devlink, resource->size_new = resource_size; resource->id = resource_id; resource->size_valid = true; - memcpy(&resource->size_params, size_params, - sizeof(resource->size_params)); + memcpy(&resource->size_params, params, sizeof(resource->size_params)); INIT_LIST_HEAD(&resource->resource_list); list_add_tail(&resource->list, resource_list); return 0; } + +/** + * devl_resource_register - devlink resource register + * + * @devlink: devlink + * @resource_name: resource's name + * @resource_size: resource's size + * @resource_id: resource's id + * @parent_resource_id: resource's parent id + * @params: size parameters + * + * Generic resources should reuse the same names across drivers. + * Please see the generic resources list at: + * Documentation/networking/devlink/devlink-resource.rst + * + * Return: 0 on success, negative error code otherwise. + */ +int devl_resource_register(struct devlink *devlink, const char *resource_name, + u64 resource_size, u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *params) +{ + return __devl_resource_register(devlink, &devlink->resource_list, + resource_name, resource_size, + resource_id, parent_resource_id, + params); +} EXPORT_SYMBOL_GPL(devl_resource_register); -static void devlink_resource_unregister(struct devlink *devlink, - struct devlink_resource *resource) +static void devlink_resource_unregister(struct devlink_resource *resource) { struct devlink_resource *tmp, *child_resource; list_for_each_entry_safe(child_resource, tmp, &resource->resource_list, list) { - devlink_resource_unregister(devlink, child_resource); + devlink_resource_unregister(child_resource); list_del(&child_resource->list); kfree(child_resource); } } -/** - * devl_resources_unregister - free all resources - * - * @devlink: devlink - */ -void devl_resources_unregister(struct devlink *devlink) +static void +__devl_resources_unregister(struct devlink *devlink, + struct list_head *resource_list_head) { struct devlink_resource *tmp, *child_resource; lockdep_assert_held(&devlink->lock); - list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list, + list_for_each_entry_safe(child_resource, tmp, resource_list_head, list) { - devlink_resource_unregister(devlink, child_resource); + devlink_resource_unregister(child_resource); list_del(&child_resource->list); kfree(child_resource); } } + +/** + * devl_resources_unregister - free all resources + * + * @devlink: devlink + */ +void devl_resources_unregister(struct devlink *devlink) +{ + __devl_resources_unregister(devlink, &devlink->resource_list); +} EXPORT_SYMBOL_GPL(devl_resources_unregister); /** @@ -502,3 +532,46 @@ void devl_resource_occ_get_unregister(struct devlink *devlink, resource->occ_get_priv = NULL; } EXPORT_SYMBOL_GPL(devl_resource_occ_get_unregister); + +/** + * devl_port_resource_register - devlink port resource register + * + * @devlink_port: devlink port + * @resource_name: resource's name + * @resource_size: resource's size + * @resource_id: resource's id + * @parent_resource_id: resource's parent id + * @params: size parameters + * + * Generic resources should reuse the same names across drivers. + * Please see the generic resources list at: + * Documentation/networking/devlink/devlink-resource.rst + * + * Return: 0 on success, negative error code otherwise. + */ +int +devl_port_resource_register(struct devlink_port *devlink_port, + const char *resource_name, + u64 resource_size, u64 resource_id, + u64 parent_resource_id, + const struct devlink_resource_size_params *params) +{ + return __devl_resource_register(devlink_port->devlink, + &devlink_port->resource_list, + resource_name, resource_size, + resource_id, parent_resource_id, + params); +} +EXPORT_SYMBOL_GPL(devl_port_resource_register); + +/** + * devl_port_resources_unregister - unregister all devlink port resources + * + * @devlink_port: devlink port + */ +void devl_port_resources_unregister(struct devlink_port *devlink_port) +{ + __devl_resources_unregister(devlink_port->devlink, + &devlink_port->resource_list); +} +EXPORT_SYMBOL_GPL(devl_port_resources_unregister); diff --git a/net/dsa/tag_brcm.c b/net/dsa/tag_brcm.c index eadb358179ce3..9d2ea75dcb269 100644 --- a/net/dsa/tag_brcm.c +++ b/net/dsa/tag_brcm.c @@ -364,6 +364,7 @@ static const struct dsa_device_ops brcm_legacy_fcs_netdev_ops = { .xmit = brcm_leg_fcs_tag_xmit, .rcv = brcm_leg_tag_rcv, .needed_headroom = BRCM_LEG_TAG_LEN, + .needed_tailroom = ETH_FCS_LEN, }; DSA_TAG_DRIVER(brcm_legacy_fcs_netdev_ops); diff --git a/net/ethtool/cmis_cdb.c b/net/ethtool/cmis_cdb.c index fe156991d0bec..b39c7d47580da 100644 --- a/net/ethtool/cmis_cdb.c +++ b/net/ethtool/cmis_cdb.c @@ -2,6 +2,7 @@ #include #include +#include #include "common.h" #include "module_fw.h" @@ -179,6 +180,7 @@ cmis_cdb_validate_password(struct ethtool_cmis_cdb *cdb, pe_pl = *((struct cmis_password_entry_pl *)page_data.data); pe_pl.password = params->password; + netdev_assert_locked_ops(dev); err = ops->set_module_eeprom_by_page(dev, &page_data, &extack); if (err < 0) { if (extack._msg) @@ -546,6 +548,7 @@ __ethtool_cmis_cdb_execute_cmd(struct net_device *dev, if (!page_data->data) return -ENOMEM; + netdev_assert_locked_ops(dev); err = ops->set_module_eeprom_by_page(dev, page_data, &extack); if (err < 0) { if (extack._msg) diff --git a/net/ethtool/cmis_fw_update.c b/net/ethtool/cmis_fw_update.c index 291d04d2776a5..dff83807e975c 100644 --- a/net/ethtool/cmis_fw_update.c +++ b/net/ethtool/cmis_fw_update.c @@ -435,13 +435,9 @@ cmis_fw_update_commit_image(struct ethtool_cmis_cdb *cdb, static int cmis_fw_update_reset(struct net_device *dev) { __u32 reset_data = ETH_RESET_PHY; - int ret; - netdev_lock_ops(dev); - ret = dev->ethtool_ops->reset(dev, &reset_data); - netdev_unlock_ops(dev); - - return ret; + netdev_assert_locked_ops(dev); + return dev->ethtool_ops->reset(dev, &reset_data); } void diff --git a/net/ethtool/module.c b/net/ethtool/module.c index 9a11e7def0029..8a313c975e2f5 100644 --- a/net/ethtool/module.c +++ b/net/ethtool/module.c @@ -226,7 +226,9 @@ static void module_flash_fw_work(struct work_struct *work) module_fw = container_of(work, struct ethtool_module_fw_flash, work); dev = module_fw->fw_update.dev; + netdev_lock_ops(dev); ethtool_cmis_fw_update(&module_fw->fw_update); + netdev_unlock_ops(dev); module_flash_fw_work_list_del(&module_fw->list); diff --git a/net/handshake/handshake-test.c b/net/handshake/handshake-test.c index df3948e807a0f..9cc7a95f41207 100644 --- a/net/handshake/handshake-test.c +++ b/net/handshake/handshake-test.c @@ -375,6 +375,10 @@ static void handshake_req_cancel_test2(struct kunit *test) /* Pretend to accept this request */ next = handshake_req_next(hn, HANDSHAKE_HANDLER_CLASS_TLSHD); KUNIT_ASSERT_PTR_EQ(test, req, next); + /* Simulate FD_PREPARE() consuming the file reference handed + * off by handshake_req_next(); see handshake_nl_accept_doit(). + */ + fput(filp); /* Act */ result = handshake_req_cancel(sock->sk); @@ -417,6 +421,10 @@ static void handshake_req_cancel_test3(struct kunit *test) /* Pretend to accept this request */ next = handshake_req_next(hn, HANDSHAKE_HANDLER_CLASS_TLSHD); KUNIT_ASSERT_PTR_EQ(test, req, next); + /* Simulate FD_PREPARE() consuming the file reference handed + * off by handshake_req_next(); see handshake_nl_accept_doit(). + */ + fput(filp); /* Pretend to complete this request */ handshake_complete(next, -ETIMEDOUT, NULL); diff --git a/net/handshake/handshake.h b/net/handshake/handshake.h index 2289b0e274f40..da61cadd1ad3e 100644 --- a/net/handshake/handshake.h +++ b/net/handshake/handshake.h @@ -24,6 +24,7 @@ enum hn_flags_bits { HANDSHAKE_F_NET_DRAINING, }; +struct file; struct handshake_proto; /* One handshake request */ @@ -32,6 +33,7 @@ struct handshake_req { struct rhash_head hr_rhash; unsigned long hr_flags; const struct handshake_proto *hr_proto; + struct file *hr_file; struct sock *hr_sk; void (*hr_odestruct)(struct sock *sk); diff --git a/net/handshake/netlink.c b/net/handshake/netlink.c index d8211e0ba75c6..3fd4fef9bab1a 100644 --- a/net/handshake/netlink.c +++ b/net/handshake/netlink.c @@ -92,8 +92,7 @@ int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info) struct net *net = sock_net(skb->sk); struct handshake_net *hn = handshake_pernet(net); struct handshake_req *req = NULL; - struct socket *sock; - int class, fd, err; + int class, err; err = -EOPNOTSUPP; if (!hn) @@ -106,29 +105,26 @@ int handshake_nl_accept_doit(struct sk_buff *skb, struct genl_info *info) err = -EAGAIN; req = handshake_req_next(hn, class); - if (!req) - goto out_status; - - sock = req->hr_sk->sk_socket; - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - err = fd; - goto out_complete; - } - - err = req->hr_proto->hp_accept(req, info, fd); - if (err) { - put_unused_fd(fd); - goto out_complete; + if (req) { + FD_PREPARE(fdf, O_CLOEXEC, req->hr_file); + if (fdf.err) { + fput(req->hr_file); /* drop ref from handshake_req_next() */ + err = fdf.err; + goto out_complete; + } + + err = req->hr_proto->hp_accept(req, info, fd_prepare_fd(fdf)); + if (err) + goto out_complete; /* Automatic cleanup handles fput */ + + trace_handshake_cmd_accept(net, req, req->hr_sk, fd_prepare_fd(fdf)); + fd_publish(fdf); + return 0; } - fd_install(fd, get_file(sock->file)); - - trace_handshake_cmd_accept(net, req, req->hr_sk, fd); - return 0; - out_complete: - handshake_complete(req, -EIO, NULL); + if (req) + handshake_complete(req, -EIO, NULL); out_status: trace_handshake_cmd_accept_err(net, req, NULL, err); return err; @@ -205,19 +201,19 @@ static void __net_exit handshake_net_exit(struct net *net) */ spin_lock_bh(&hn->hn_lock); set_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags); - list_splice_init(&requests, &hn->hn_requests); + list_splice_init(&hn->hn_requests, &requests); + list_for_each_entry(req, &requests, hr_list) + get_file(req->hr_file); spin_unlock_bh(&hn->hn_lock); while (!list_empty(&requests)) { - req = list_first_entry(&requests, struct handshake_req, hr_list); - list_del(&req->hr_list); - - /* - * Requests on this list have not yet been - * accepted, so they do not have an fd to put. - */ + struct file *file; + req = list_first_entry(&requests, struct handshake_req, hr_list); + file = req->hr_file; + list_del_init(&req->hr_list); handshake_complete(req, -ETIMEDOUT, NULL); + fput(file); } } diff --git a/net/handshake/request.c b/net/handshake/request.c index 62efb7e32730e..d4a3a50f8a3b5 100644 --- a/net/handshake/request.c +++ b/net/handshake/request.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -162,13 +163,16 @@ static void __remove_pending_locked(struct handshake_net *hn, * otherwise %false. * * If @req was on a pending list, it has not yet been accepted. + * Returns %false when the net namespace is draining; the drain + * loop has taken ownership of the pending list. */ static bool remove_pending(struct handshake_net *hn, struct handshake_req *req) { bool ret = false; spin_lock_bh(&hn->hn_lock); - if (!list_empty(&req->hr_list)) { + if (!test_bit(HANDSHAKE_F_NET_DRAINING, &hn->hn_flags) && + !list_empty(&req->hr_list)) { __remove_pending_locked(hn, req); ret = true; } @@ -177,6 +181,17 @@ static bool remove_pending(struct handshake_net *hn, struct handshake_req *req) return ret; } +/** + * handshake_req_next - Return the next queued handshake request + * @hn: per-net handshake state + * @class: handler class to match + * + * On a non-NULL return, the caller owns an extra reference + * on @req->hr_file. FD_PREPARE() consumes it on success; on + * the FD_PREPARE() failure path the caller must fput() it. + * + * Return: pointer to a removed handshake_req, or NULL. + */ struct handshake_req *handshake_req_next(struct handshake_net *hn, int class) { struct handshake_req *req, *pos; @@ -187,6 +202,13 @@ struct handshake_req *handshake_req_next(struct handshake_net *hn, int class) if (pos->hr_proto->hp_handler_class != class) continue; __remove_pending_locked(hn, pos); + /* Hand off a file reference to the accept side under + * hn_lock. A concurrent handshake_req_cancel() can drop + * hr_file before accept reaches FD_PREPARE(); this extra + * reference keeps the file alive until FD_PREPARE() takes + * ownership. + */ + get_file(pos->hr_file); req = pos; break; } @@ -215,9 +237,16 @@ EXPORT_SYMBOL_IF_KUNIT(handshake_req_next); * A zero return value from handshake_req_submit() means that * exactly one subsequent completion callback is guaranteed. * - * A negative return value from handshake_req_submit() means that - * no completion callback will be done and that @req has been - * destroyed. + * A negative return value from handshake_req_submit() guarantees that + * no completion callback will occur and that @req is no longer owned by + * the caller. If cancellation wins the completion race after the request + * has been published, final destruction is deferred until socket teardown. + * + * The caller must hold a reference on @sock->file for the duration + * of this call. Once the request is published to the accept side, a + * concurrent completion or cancellation may release the request's pin on + * @sock->file; the caller's reference is what keeps @sock->sk valid until + * handshake_req_submit() returns. */ int handshake_req_submit(struct socket *sock, struct handshake_req *req, gfp_t flags) @@ -236,6 +265,14 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req, kfree(req); return -EINVAL; } + + /* + * Pin sock->file for the lifetime of the request so the + * accept side does not race a consumer that releases the + * socket while a handshake is pending. + */ + req->hr_file = get_file(sock->file); + req->hr_odestruct = req->hr_sk->sk_destruct; req->hr_sk->sk_destruct = handshake_sk_destruct; @@ -267,19 +304,19 @@ int handshake_req_submit(struct socket *sock, struct handshake_req *req, goto out_err; } - /* Prevent socket release while a handshake request is pending */ - sock_hold(req->hr_sk); - trace_handshake_submit(net, req, req->hr_sk); return 0; out_unlock: spin_unlock_bh(&hn->hn_lock); out_err: - /* Restore original destructor so socket teardown still runs on failure */ - req->hr_sk->sk_destruct = req->hr_odestruct; trace_handshake_submit_err(net, req, req->hr_sk, ret); - handshake_req_destroy(req); + if (!test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) { + /* Restore original destructor so socket teardown still runs. */ + req->hr_sk->sk_destruct = req->hr_odestruct; + fput(req->hr_file); + handshake_req_destroy(req); + } return ret; } EXPORT_SYMBOL(handshake_req_submit); @@ -291,11 +328,12 @@ void handshake_complete(struct handshake_req *req, int status, struct net *net = sock_net(sk); if (!test_and_set_bit(HANDSHAKE_F_REQ_COMPLETED, &req->hr_flags)) { + struct file *file = req->hr_file; + trace_handshake_complete(net, req, sk, status); req->hr_proto->hp_done(req, status, info); - /* Handshake request is no longer pending */ - sock_put(sk); + fput(file); } } EXPORT_SYMBOL_IF_KUNIT(handshake_complete); @@ -342,8 +380,7 @@ bool handshake_req_cancel(struct sock *sk) out_true: trace_handshake_cancel(net, req, sk); - /* Handshake request is no longer pending */ - sock_put(sk); + fput(req->hr_file); return true; } EXPORT_SYMBOL(handshake_req_cancel); diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index fd2fea25eff0d..82b9e94be12f1 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -820,6 +820,8 @@ int hsr_dev_finalize(struct net_device *hsr_dev, struct net_device *slave[2], hsr_del_ports(hsr); err_add_master: hsr_del_self_node(hsr); + hsr_del_nodes(&hsr->node_db); + hsr_del_nodes(&hsr->proxy_node_db); if (unregister) unregister_netdevice(hsr_dev); diff --git a/net/hsr/hsr_forward.c b/net/hsr/hsr_forward.c index 299de290ddaa5..901eb85c112d1 100644 --- a/net/hsr/hsr_forward.c +++ b/net/hsr/hsr_forward.c @@ -110,7 +110,7 @@ static bool is_supervision_frame(struct hsr_priv *hsr, struct sk_buff *skb) } /* end of tlvs must follow at the end */ - if (hsr_sup_tlv->HSR_TLV_type == HSR_TLV_EOT && + if (hsr_sup_tlv->HSR_TLV_type != HSR_TLV_EOT || hsr_sup_tlv->HSR_TLV_length != 0) return false; diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c index c96b63adf96ff..50a8104e864e2 100644 --- a/net/hsr/hsr_netlink.c +++ b/net/hsr/hsr_netlink.c @@ -239,7 +239,8 @@ void hsr_nl_ringerror(struct hsr_priv *hsr, unsigned char addr[ETH_ALEN], goto nla_put_failure; genlmsg_end(skb, msg_head); - genlmsg_multicast(&hsr_genl_family, skb, 0, 0, GFP_ATOMIC); + genlmsg_multicast_netns(&hsr_genl_family, dev_net(port->dev), + skb, 0, 0, GFP_ATOMIC); return; @@ -275,8 +276,12 @@ void hsr_nl_nodedown(struct hsr_priv *hsr, unsigned char addr[ETH_ALEN]) if (res < 0) goto nla_put_failure; + rcu_read_lock(); + master = hsr_port_get_hsr(hsr, HSR_PT_MASTER); genlmsg_end(skb, msg_head); - genlmsg_multicast(&hsr_genl_family, skb, 0, 0, GFP_ATOMIC); + genlmsg_multicast_netns(&hsr_genl_family, dev_net(master->dev), + skb, 0, 0, GFP_ATOMIC); + rcu_read_unlock(); return; diff --git a/net/hsr/hsr_slave.c b/net/hsr/hsr_slave.c index afe06ba00ea44..63df9ee5cacd3 100644 --- a/net/hsr/hsr_slave.c +++ b/net/hsr/hsr_slave.c @@ -149,9 +149,12 @@ static int hsr_portdev_setup(struct hsr_priv *hsr, struct net_device *dev, int res; /* Don't use promiscuous mode for offload since L2 frame forward - * happens at the offloaded hardware. + * happens at the offloaded hardware. The interlink port never + * gets forwarding offload (RedBox forwarding to/from it is done + * by this driver), so it still needs promiscuous mode to receive + * frames addressed to hsr_dev's MAC rather than its own. */ - if (!port->hsr->fwd_offloaded) { + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) { res = dev_set_promiscuity(dev, 1); if (res) return res; @@ -176,7 +179,7 @@ static int hsr_portdev_setup(struct hsr_priv *hsr, struct net_device *dev, fail_rx_handler: netdev_upper_dev_unlink(dev, hsr_dev); fail_upper_dev_link: - if (!port->hsr->fwd_offloaded) + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) dev_set_promiscuity(dev, -1); return res; @@ -240,8 +243,10 @@ void hsr_del_port(struct hsr_port *port) netdev_update_features(master->dev); dev_set_mtu(master->dev, hsr_get_max_mtu(hsr)); netdev_rx_handler_unregister(port->dev); - if (!port->hsr->fwd_offloaded) + if (!port->hsr->fwd_offloaded || port->type == HSR_PT_INTERLINK) dev_set_promiscuity(port->dev, -1); + if (port->type == HSR_PT_SLAVE_A || port->type == HSR_PT_SLAVE_B) + vlan_vids_del_by_dev(port->dev, master->dev); netdev_upper_dev_unlink(port->dev, master->dev); eth_hw_addr_set(port->dev, port->original_macaddress); } diff --git a/net/ieee802154/6lowpan/core.c b/net/ieee802154/6lowpan/core.c index 018929563c6bc..6a8d6852cb930 100644 --- a/net/ieee802154/6lowpan/core.c +++ b/net/ieee802154/6lowpan/core.c @@ -150,7 +150,7 @@ static int lowpan_newlink(struct net_device *ldev, wdev = dev_get_by_index(dev_net(ldev), nla_get_u32(tb[IFLA_LINK])); if (!wdev) return -ENODEV; - if (wdev->type != ARPHRD_IEEE802154) { + if (wdev->type != ARPHRD_IEEE802154 || !wdev->ieee802154_ptr) { dev_put(wdev); return -EINVAL; } diff --git a/net/ife/ife.c b/net/ife/ife.c index 7a75947a31e31..2ddf725d3389e 100644 --- a/net/ife/ife.c +++ b/net/ife/ife.c @@ -37,7 +37,7 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) * where ORIGDATA = original ethernet header ... */ int hdrm = metalen + IFE_METAHDRLEN; - int total_push = hdrm + skb->dev->hard_header_len; + int total_push = hdrm + ETH_HLEN; struct ifeheadr *ifehdr; struct ethhdr *iethh; /* inner ether header */ int skboff = 0; @@ -50,9 +50,9 @@ void *ife_encode(struct sk_buff *skb, u16 metalen) iethh = (struct ethhdr *) skb->data; __skb_push(skb, total_push); - memcpy(skb->data, iethh, skb->dev->hard_header_len); + memcpy(skb->data, iethh, ETH_HLEN); skb_reset_mac_header(skb); - skboff += skb->dev->hard_header_len; + skboff += ETH_HLEN; /* total metadata length */ ifehdr = (struct ifeheadr *) (skb->data + skboff); @@ -69,12 +69,12 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) int total_pull; u16 ifehdrln; - if (!pskb_may_pull(skb, skb->dev->hard_header_len + IFE_METAHDRLEN)) + if (!pskb_may_pull(skb, ETH_HLEN + IFE_METAHDRLEN)) return NULL; - ifehdr = (struct ifeheadr *) (skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); ifehdrln = ntohs(ifehdr->metalen); - total_pull = skb->dev->hard_header_len + ifehdrln; + total_pull = ETH_HLEN + ifehdrln; if (unlikely(ifehdrln < 2)) return NULL; @@ -82,7 +82,7 @@ void *ife_decode(struct sk_buff *skb, u16 *metalen) if (unlikely(!pskb_may_pull(skb, total_pull + ETH_HLEN))) return NULL; - ifehdr = (struct ifeheadr *)(skb->data + skb->dev->hard_header_len); + ifehdr = (struct ifeheadr *)(skb->data + ETH_HLEN); skb_set_mac_header(skb, total_pull); __skb_pull(skb, total_pull); *metalen = ifehdrln - IFE_METAHDRLEN; diff --git a/net/ipv4/datagram.c b/net/ipv4/datagram.c index c2b2cda1a7e50..7861615b34fc6 100644 --- a/net/ipv4/datagram.c +++ b/net/ipv4/datagram.c @@ -63,12 +63,12 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len } /* Update addresses before rehashing */ - inet->inet_daddr = fl4->daddr; + WRITE_ONCE(inet->inet_daddr, fl4->daddr); inet->inet_dport = usin->sin_port; if (!inet->inet_saddr) inet->inet_saddr = fl4->saddr; if (!inet->inet_rcv_saddr) { - inet->inet_rcv_saddr = fl4->saddr; + WRITE_ONCE(inet->inet_rcv_saddr, fl4->saddr); if (sk->sk_prot->rehash) sk->sk_prot->rehash(sk); } diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c index 51f0193092f03..0008d23aceb22 100644 --- a/net/ipv4/fib_rules.c +++ b/net/ipv4/fib_rules.c @@ -214,6 +214,8 @@ INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule, return 1; } +#define FIB_MAX_AUTO_TABLE_ID 4096 + static struct fib_table *fib_empty_table(struct net *net) { u32 id = 1; @@ -222,7 +224,7 @@ static struct fib_table *fib_empty_table(struct net *net) if (!fib_get_table(net, id)) return fib_new_table(net, id); - if (id++ == RT_TABLE_MAX) + if (id++ == FIB_MAX_AUTO_TABLE_ID) break; } return NULL; diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index 0caf38e44c738..2aba8c0b9b2b5 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -492,6 +492,34 @@ int ip_fib_check_default(__be32 gw, struct net_device *dev) return -1; } +static size_t fib_nexthop_nlmsg_size(const struct fib_nh_common *nhc, + bool skip_oif) +{ + size_t nhsize = 0; + + switch (nhc->nhc_gw_family) { + case AF_INET: + nhsize += nla_total_size(4); /* RTA_GATEWAY */ + break; + case AF_INET6: + nhsize += nla_total_size(sizeof(struct rtvia) + + sizeof(struct in6_addr)); + break; + } + + if (!skip_oif && nhc->nhc_dev) + nhsize += nla_total_size(4); /* RTA_OIF */ + + if (nhc->nhc_lwtstate) { + /* RTA_ENCAP */ + nhsize += lwtunnel_get_encap_size(nhc->nhc_lwtstate); + /* RTA_ENCAP_TYPE */ + nhsize += nla_total_size(2); + } + + return nhsize; +} + size_t fib_nlmsg_size(struct fib_info *fi) { size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg)) @@ -509,32 +537,35 @@ size_t fib_nlmsg_size(struct fib_info *fi) payload += nla_total_size(4); /* RTA_NH_ID */ if (nhs) { - size_t nh_encapsize = 0; - /* Also handles the special case nhs == 1 */ - - /* each nexthop is packed in an attribute */ - size_t nhsize = nla_total_size(sizeof(struct rtnexthop)); + size_t mpsize = 0; unsigned int i; - /* may contain flow and gateway attribute */ - nhsize += 2 * nla_total_size(4); - - /* grab encap info */ for (i = 0; i < fib_info_num_path(fi); i++) { struct fib_nh_common *nhc = fib_info_nhc(fi, i); + size_t nhsize; + + nhsize = fib_nexthop_nlmsg_size(nhc, nhs != 1); + + if (nhs != 1) + nhsize += NLA_ALIGN(sizeof(struct rtnexthop)); + +#ifdef CONFIG_IP_ROUTE_CLASSID + if (nhc->nhc_family == AF_INET) { + struct fib_nh *nh; - if (nhc->nhc_lwtstate) { - /* RTA_ENCAP_TYPE */ - nh_encapsize += lwtunnel_get_encap_size( - nhc->nhc_lwtstate); - /* RTA_ENCAP */ - nh_encapsize += nla_total_size(2); + nh = container_of(nhc, struct fib_nh, nh_common); + if (nh->nh_tclassid) + nhsize += nla_total_size(4); } +#endif + if (nhs == 1) + payload += nhsize; + else + mpsize += nhsize; } - /* all nexthops are packed in a nested attribute */ - payload += nla_total_size((nhs * nhsize) + nh_encapsize); - + if (nhs != 1) + payload += nla_total_size(mpsize); } return payload; @@ -1870,42 +1901,30 @@ static int call_fib_nh_notifiers(struct fib_nh *nh, return NOTIFY_DONE; } -/* Update the PMTU of exceptions when: - * - the new MTU of the first hop becomes smaller than the PMTU - * - the old MTU was the same as the PMTU, and it limited discovery of - * larger MTUs on the path. With that limit raised, we can now - * discover larger MTUs - * A special case is locked exceptions, for which the PMTU is smaller - * than the minimal accepted PMTU: - * - if the new MTU is greater than the PMTU, don't make any change - * - otherwise, unlock and set PMTU +/* Walk the exceptions of a nexthop after its first hop MTU changed. The + * chain is RCU protected here, while fnhe_update_pmtu() takes fnhe_lock + * for the update of each entry. */ void fib_nhc_update_mtu(struct fib_nh_common *nhc, u32 new, u32 orig) { struct fnhe_hash_bucket *bucket; int i; - bucket = rcu_dereference_protected(nhc->nhc_exceptions, 1); + rcu_read_lock(); + bucket = rcu_dereference(nhc->nhc_exceptions); if (!bucket) - return; + goto out; for (i = 0; i < FNHE_HASH_SIZE; i++) { struct fib_nh_exception *fnhe; - for (fnhe = rcu_dereference_protected(bucket[i].chain, 1); + for (fnhe = rcu_dereference(bucket[i].chain); fnhe; - fnhe = rcu_dereference_protected(fnhe->fnhe_next, 1)) { - if (fnhe->fnhe_mtu_locked) { - if (new <= fnhe->fnhe_pmtu) { - fnhe->fnhe_pmtu = new; - fnhe->fnhe_mtu_locked = false; - } - } else if (new < fnhe->fnhe_pmtu || - orig == fnhe->fnhe_pmtu) { - fnhe->fnhe_pmtu = new; - } - } + fnhe = rcu_dereference(fnhe->fnhe_next)) + fnhe_update_pmtu(fnhe, new, orig); } +out: + rcu_read_unlock(); } void fib_sync_mtu(struct net_device *dev, u32 orig_mtu) diff --git a/net/ipv4/fib_trie.c b/net/ipv4/fib_trie.c index 1308213791f19..9bf5550e2fb73 100644 --- a/net/ipv4/fib_trie.c +++ b/net/ipv4/fib_trie.c @@ -1385,7 +1385,7 @@ int fib_table_insert(struct net *net, struct fib_table *tb, out_remove_new_fa: fib_remove_alias(t, tp, l, new_fa); out_free_new_fa: - kmem_cache_free(fn_alias_kmem, new_fa); + alias_free_mem_rcu(new_fa); out: fib_release_info(fi); err: @@ -2172,10 +2172,14 @@ static int fib_leaf_notify(struct key_vector *l, struct fib_table *tb, if (fa->fa_slen == last_slen) continue; + if (!fib_info_hold_safe(fa->fa_info)) + continue; + last_slen = fa->fa_slen; err = call_fib_entry_notifier(nb, FIB_EVENT_ENTRY_REPLACE, l->key, KEYLENGTH - fa->fa_slen, fa, extack); + fib_info_put(fa->fa_info); if (err) return err; } diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c index ab8f309f8925d..d06c20deb175d 100644 --- a/net/ipv4/fou_core.c +++ b/net/ipv4/fou_core.c @@ -632,9 +632,9 @@ static int fou_create(struct net *net, struct fou_cfg *cfg, return 0; error: - kfree(fou); if (sock) udp_tunnel_sock_release(sock); + kfree_rcu(fou, rcu); return err; } diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c index fc0a93f43313d..a66e1a7175e90 100644 --- a/net/ipv4/icmp.c +++ b/net/ipv4/icmp.c @@ -539,11 +539,23 @@ static struct rtable *icmp_route_lookup(struct net *net, struct flowi4 *fl4, if (IS_ERR(rt2)) err = PTR_ERR(rt2); } else { - struct flowi4 fl4_2 = {}; + struct flowi4 fl4_2 = fl4_dec; unsigned long orefdst; - fl4_2.daddr = fl4_dec.saddr; - rt2 = ip_route_output_key(net, &fl4_2); + swap(fl4_2.daddr, fl4_2.saddr); + switch (fl4_2.flowi4_proto) { + case IPPROTO_TCP: + case IPPROTO_UDP: + case IPPROTO_SCTP: + case IPPROTO_DCCP: + swap(fl4_2.fl4_sport, fl4_2.fl4_dport); + break; + } + + fl4_2.flowi4_oif = l3mdev_master_ifindex(route_lookup_dev); + fl4_2.flowi4_flags |= FLOWI_FLAG_ANYSRC; + + rt2 = __ip_route_output_key(net, &fl4_2); if (IS_ERR(rt2)) { err = PTR_ERR(rt2); goto relookup_failed; @@ -778,7 +790,7 @@ void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info, /* RFC says return as much as we can without exceeding 576 bytes. */ - room = dst_mtu(&rt->dst); + room = dst4_mtu(&rt->dst); if (room > 576) room = 576; room -= sizeof(struct iphdr) + icmp_param.replyopts.opt.opt.optlen; diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index aca035eb58929..2846c2cee31cc 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -175,6 +175,10 @@ static void ip_ma_put(struct ip_mc_list *im) } } +#define pmc_dereference(e, pmc) \ + rcu_dereference_protected(e, lockdep_is_held(&(pmc)->lock) || \ + lockdep_is_held(&(pmc)->interface->mc_tomb_lock)) + #define for_each_pmc_rcu(in_dev, pmc) \ for (pmc = rcu_dereference(in_dev->mc_list); \ pmc != NULL; \ @@ -185,13 +189,28 @@ static void ip_ma_put(struct ip_mc_list *im) pmc != NULL; \ pmc = rtnl_dereference(pmc->next_rcu)) +#define for_each_psf_mclock(pmc, psf) \ + for (psf = pmc_dereference((pmc)->sources, pmc); \ + psf; \ + psf = pmc_dereference(psf->sf_next, pmc)) + +#define for_each_psf_rcu(im, psf) \ + for (psf = rcu_dereference((im)->sources); \ + psf; \ + psf = rcu_dereference(psf->sf_next)) + +#define for_each_psf_tomb(pmc, psf) \ + for (psf = pmc_dereference((pmc)->tomb, pmc); \ + psf; \ + psf = pmc_dereference(psf->sf_next, pmc)) + static void ip_sf_list_clear_all(struct ip_sf_list *psf) { struct ip_sf_list *next; while (psf) { - next = psf->sf_next; - kfree(psf); + next = rcu_dereference_protected(psf->sf_next, 1); + kfree_rcu(psf, rcu); psf = next; } } @@ -336,7 +355,7 @@ igmp_scount(struct ip_mc_list *pmc, int type, int gdeleted, int sdeleted) struct ip_sf_list *psf; int scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (!is_in(pmc, psf, type, gdeleted, sdeleted)) continue; scount++; @@ -481,7 +500,8 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, struct net *net = dev_net(dev); struct igmpv3_report *pih; struct igmpv3_grec *pgr = NULL; - struct ip_sf_list *psf, *psf_next, *psf_prev, **psf_list; + struct ip_sf_list *psf, *psf_next, *psf_prev; + struct ip_sf_list __rcu **psf_list; int scount, stotal, first, isquery, truncate; unsigned int mtu; @@ -504,7 +524,7 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, psf_list = sdeleted ? &pmc->tomb : &pmc->sources; - if (!*psf_list) + if (!rcu_access_pointer(*psf_list)) goto empty_source; pih = skb ? igmpv3_report_hdr(skb) : NULL; @@ -520,10 +540,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, } first = 1; psf_prev = NULL; - for (psf = *psf_list; psf; psf = psf_next) { + for (psf = pmc_dereference(*psf_list, pmc); + psf; + psf = psf_next) { __be32 *psrc; - psf_next = psf->sf_next; + psf_next = pmc_dereference(psf->sf_next, pmc); if (!is_in(pmc, psf, type, gdeleted, sdeleted)) { psf_prev = psf; @@ -570,10 +592,12 @@ static struct sk_buff *add_grec(struct sk_buff *skb, struct ip_mc_list *pmc, psf->sf_crcount--; if ((sdeleted || gdeleted) && psf->sf_crcount == 0) { if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, + psf_next); else - *psf_list = psf->sf_next; - kfree(psf); + rcu_assign_pointer(*psf_list, + psf_next); + kfree_rcu(psf, rcu); continue; } } @@ -642,28 +666,29 @@ static int igmpv3_send_report(struct in_device *in_dev, struct ip_mc_list *pmc) /* * remove zero-count source records from a source filter list */ -static void igmpv3_clear_zeros(struct ip_sf_list **ppsf) +static void igmpv3_clear_zeros(struct ip_sf_list __rcu **ppsf) { struct ip_sf_list *psf_prev, *psf_next, *psf; psf_prev = NULL; - for (psf = *ppsf; psf; psf = psf_next) { - psf_next = psf->sf_next; + for (psf = rcu_dereference_protected(*ppsf, 1); psf; psf = psf_next) { + psf_next = rcu_dereference_protected(psf->sf_next, 1); if (psf->sf_crcount == 0) { if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, psf_next); else - *ppsf = psf->sf_next; - kfree(psf); - } else + rcu_assign_pointer(*ppsf, psf_next); + kfree_rcu(psf, rcu); + } else { psf_prev = psf; + } } } static void kfree_pmc(struct ip_mc_list *pmc) { - ip_sf_list_clear_all(pmc->sources); - ip_sf_list_clear_all(pmc->tomb); + ip_sf_list_clear_all(rcu_dereference_protected(pmc->sources, 1)); + ip_sf_list_clear_all(rcu_dereference_protected(pmc->tomb, 1)); kfree(pmc); } @@ -697,7 +722,8 @@ static void igmpv3_send_cr(struct in_device *in_dev) igmpv3_clear_zeros(&pmc->sources); } } - if (pmc->crcount == 0 && !pmc->tomb && !pmc->sources) { + if (pmc->crcount == 0 && !rcu_access_pointer(pmc->tomb) && + !rcu_access_pointer(pmc->sources)) { if (pmc_prev) pmc_prev->next = pmc_next; else @@ -883,7 +909,7 @@ static int igmp_xmarksources(struct ip_mc_list *pmc, int nsrcs, __be32 *srcs) int i, scount; scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (scount == nsrcs) break; for (i = 0; i < nsrcs; i++) { @@ -914,7 +940,7 @@ static int igmp_marksources(struct ip_mc_list *pmc, int nsrcs, __be32 *srcs) /* mark INCLUDE-mode sources */ scount = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (scount == nsrcs) break; for (i = 0; i < nsrcs; i++) @@ -1214,11 +1240,12 @@ static void igmpv3_add_delrec(struct in_device *in_dev, struct ip_mc_list *im, if (pmc->sfmode == MCAST_INCLUDE) { struct ip_sf_list *psf; + for_each_psf_mclock(im, psf) + psf->sf_crcount = pmc->crcount; pmc->tomb = im->tomb; pmc->sources = im->sources; - im->tomb = im->sources = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) - psf->sf_crcount = pmc->crcount; + RCU_INIT_POINTER(im->tomb, NULL); + RCU_INIT_POINTER(im->sources, NULL); } spin_unlock_bh(&im->lock); @@ -1257,9 +1284,18 @@ static void igmpv3_del_delrec(struct in_device *in_dev, struct ip_mc_list *im) if (pmc) { im->interface = pmc->interface; if (im->sfmode == MCAST_INCLUDE) { - swap(im->tomb, pmc->tomb); - swap(im->sources, pmc->sources); - for (psf = im->sources; psf; psf = psf->sf_next) + struct ip_sf_list *sources, *tomb; + + tomb = rcu_replace_pointer(im->tomb, + rcu_dereference_protected(pmc->tomb, 1), + lockdep_is_held(&im->lock)); + rcu_assign_pointer(pmc->tomb, tomb); + + sources = rcu_replace_pointer(im->sources, + rcu_dereference_protected(pmc->sources, 1), + lockdep_is_held(&im->lock)); + rcu_assign_pointer(pmc->sources, sources); + for_each_psf_mclock(im, psf) psf->sf_crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); } else { @@ -1296,8 +1332,8 @@ static void igmpv3_clear_delrec(struct in_device *in_dev) struct ip_sf_list *psf; spin_lock_bh(&pmc->lock); - psf = pmc->tomb; - pmc->tomb = NULL; + psf = pmc_dereference(pmc->tomb, pmc); + RCU_INIT_POINTER(pmc->tomb, NULL); spin_unlock_bh(&pmc->lock); ip_sf_list_clear_all(psf); } @@ -1973,7 +2009,7 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, int rv = 0; psf_prev = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (psf->sf_inaddr == *psfsrc) break; psf_prev = psf; @@ -1982,7 +2018,7 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, /* source filter not found, or count wrong => bug */ return -ESRCH; } - psf->sf_count[sfmode]--; + WRITE_ONCE(psf->sf_count[sfmode], psf->sf_count[sfmode] - 1); if (psf->sf_count[sfmode] == 0) { ip_rt_multicast_event(pmc->interface); } @@ -1994,19 +2030,28 @@ static int ip_mc_del1_src(struct ip_mc_list *pmc, int sfmode, /* no more filters for this source */ if (psf_prev) - psf_prev->sf_next = psf->sf_next; + rcu_assign_pointer(psf_prev->sf_next, + pmc_dereference(psf->sf_next, pmc)); else - pmc->sources = psf->sf_next; + rcu_assign_pointer(pmc->sources, + pmc_dereference(psf->sf_next, pmc)); #ifdef CONFIG_IP_MULTICAST if (psf->sf_oldin && !IGMP_V1_SEEN(in_dev) && !IGMP_V2_SEEN(in_dev)) { - psf->sf_crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); - psf->sf_next = pmc->tomb; - pmc->tomb = psf; - rv = 1; - } else + struct ip_sf_list *dpsf = kmalloc_obj(*dpsf, GFP_ATOMIC); + + if (dpsf) { + *dpsf = *psf; + dpsf->sf_crcount = in_dev->mr_qrv ?: + READ_ONCE(net->ipv4.sysctl_igmp_qrv); + rcu_assign_pointer(dpsf->sf_next, + pmc_dereference(pmc->tomb, pmc)); + rcu_assign_pointer(pmc->tomb, dpsf); + rv = 1; + } + } #endif - kfree(psf); + kfree_rcu(psf, rcu); } return rv; } @@ -2043,7 +2088,7 @@ static int ip_mc_del_src(struct in_device *in_dev, __be32 *pmca, int sfmode, err = -EINVAL; if (!pmc->sfcount[sfmode]) goto out_unlock; - pmc->sfcount[sfmode]--; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] - 1); } err = 0; for (i = 0; i < sfcount; i++) { @@ -2066,7 +2111,7 @@ static int ip_mc_del_src(struct in_device *in_dev, __be32 *pmca, int sfmode, #ifdef CONFIG_IP_MULTICAST pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) psf->sf_crcount = 0; igmp_ifc_event(pmc->interface); } else if (sf_setstate(pmc) || changerec) { @@ -2087,7 +2132,7 @@ static int ip_mc_add1_src(struct ip_mc_list *pmc, int sfmode, struct ip_sf_list *psf, *psf_prev; psf_prev = NULL; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (psf->sf_inaddr == *psfsrc) break; psf_prev = psf; @@ -2097,12 +2142,12 @@ static int ip_mc_add1_src(struct ip_mc_list *pmc, int sfmode, if (!psf) return -ENOBUFS; psf->sf_inaddr = *psfsrc; - if (psf_prev) { - psf_prev->sf_next = psf; - } else - pmc->sources = psf; + if (psf_prev) + rcu_assign_pointer(psf_prev->sf_next, psf); + else + rcu_assign_pointer(pmc->sources, psf); } - psf->sf_count[sfmode]++; + WRITE_ONCE(psf->sf_count[sfmode], psf->sf_count[sfmode] + 1); if (psf->sf_count[sfmode] == 1) { ip_rt_multicast_event(pmc->interface); } @@ -2115,13 +2160,15 @@ static void sf_markstate(struct ip_mc_list *pmc) struct ip_sf_list *psf; int mca_xcount = pmc->sfcount[MCAST_EXCLUDE]; - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) { if (pmc->sfcount[MCAST_EXCLUDE]) { psf->sf_oldin = mca_xcount == psf->sf_count[MCAST_EXCLUDE] && !psf->sf_count[MCAST_INCLUDE]; - } else + } else { psf->sf_oldin = psf->sf_count[MCAST_INCLUDE] != 0; + } + } } static int sf_setstate(struct ip_mc_list *pmc) @@ -2132,27 +2179,31 @@ static int sf_setstate(struct ip_mc_list *pmc) int new_in, rv; rv = 0; - for (psf = pmc->sources; psf; psf = psf->sf_next) { + for_each_psf_mclock(pmc, psf) { if (pmc->sfcount[MCAST_EXCLUDE]) { new_in = mca_xcount == psf->sf_count[MCAST_EXCLUDE] && !psf->sf_count[MCAST_INCLUDE]; - } else + } else { new_in = psf->sf_count[MCAST_INCLUDE] != 0; + } if (new_in) { if (!psf->sf_oldin) { struct ip_sf_list *prev = NULL; - for (dpsf = pmc->tomb; dpsf; dpsf = dpsf->sf_next) { + for_each_psf_tomb(pmc, dpsf) { if (dpsf->sf_inaddr == psf->sf_inaddr) break; prev = dpsf; } if (dpsf) { + struct ip_sf_list *dpsf_next; + + dpsf_next = pmc_dereference(dpsf->sf_next, pmc); if (prev) - prev->sf_next = dpsf->sf_next; + rcu_assign_pointer(prev->sf_next, dpsf_next); else - pmc->tomb = dpsf->sf_next; - kfree(dpsf); + rcu_assign_pointer(pmc->tomb, dpsf_next); + kfree_rcu(dpsf, rcu); } psf->sf_crcount = qrv; rv++; @@ -2164,17 +2215,19 @@ static int sf_setstate(struct ip_mc_list *pmc) * add or update "delete" records if an active filter * is now inactive */ - for (dpsf = pmc->tomb; dpsf; dpsf = dpsf->sf_next) + for_each_psf_tomb(pmc, dpsf) { if (dpsf->sf_inaddr == psf->sf_inaddr) break; + } if (!dpsf) { dpsf = kmalloc(sizeof(*dpsf), GFP_ATOMIC); if (!dpsf) continue; *dpsf = *psf; /* pmc->lock held by callers */ - dpsf->sf_next = pmc->tomb; - pmc->tomb = dpsf; + rcu_assign_pointer(dpsf->sf_next, + pmc_dereference(pmc->tomb, pmc)); + rcu_assign_pointer(pmc->tomb, dpsf); } dpsf->sf_crcount = qrv; rv++; @@ -2214,7 +2267,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, #endif isexclude = pmc->sfmode == MCAST_EXCLUDE; if (!delta) - pmc->sfcount[sfmode]++; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] + 1); err = 0; for (i = 0; i < sfcount; i++) { err = ip_mc_add1_src(pmc, sfmode, &psfsrc[i]); @@ -2225,7 +2278,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, int j; if (!delta) - pmc->sfcount[sfmode]--; + WRITE_ONCE(pmc->sfcount[sfmode], pmc->sfcount[sfmode] - 1); for (j = 0; j < i; j++) (void) ip_mc_del1_src(pmc, sfmode, &psfsrc[j]); } else if (isexclude != (pmc->sfcount[MCAST_EXCLUDE] != 0)) { @@ -2245,7 +2298,7 @@ static int ip_mc_add_src(struct in_device *in_dev, __be32 *pmca, int sfmode, pmc->crcount = in_dev->mr_qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv); WRITE_ONCE(in_dev->mr_ifc_count, pmc->crcount); - for (psf = pmc->sources; psf; psf = psf->sf_next) + for_each_psf_mclock(pmc, psf) psf->sf_crcount = 0; igmp_ifc_event(in_dev); } else if (sf_setstate(pmc)) { @@ -2261,13 +2314,13 @@ static void ip_mc_clear_src(struct ip_mc_list *pmc) struct ip_sf_list *tomb, *sources; spin_lock_bh(&pmc->lock); - tomb = pmc->tomb; - pmc->tomb = NULL; - sources = pmc->sources; - pmc->sources = NULL; + tomb = pmc_dereference(pmc->tomb, pmc); + RCU_INIT_POINTER(pmc->tomb, NULL); + sources = pmc_dereference(pmc->sources, pmc); + RCU_INIT_POINTER(pmc->sources, NULL); pmc->sfmode = MCAST_EXCLUDE; - pmc->sfcount[MCAST_INCLUDE] = 0; - pmc->sfcount[MCAST_EXCLUDE] = 1; + WRITE_ONCE(pmc->sfcount[MCAST_INCLUDE], 0); + WRITE_ONCE(pmc->sfcount[MCAST_EXCLUDE], 1); spin_unlock_bh(&pmc->lock); ip_sf_list_clear_all(tomb); @@ -2849,20 +2902,19 @@ int ip_check_mc_rcu(struct in_device *in_dev, __be32 mc_addr, __be32 src_addr, u rv = 1; } else if (im) { if (src_addr) { - spin_lock_bh(&im->lock); - for (psf = im->sources; psf; psf = psf->sf_next) { + for_each_psf_rcu(im, psf) { if (psf->sf_inaddr == src_addr) break; } if (psf) - rv = psf->sf_count[MCAST_INCLUDE] || - psf->sf_count[MCAST_EXCLUDE] != - im->sfcount[MCAST_EXCLUDE]; + rv = READ_ONCE(psf->sf_count[MCAST_INCLUDE]) || + READ_ONCE(psf->sf_count[MCAST_EXCLUDE]) != + READ_ONCE(im->sfcount[MCAST_EXCLUDE]); else - rv = im->sfcount[MCAST_EXCLUDE] != 0; - spin_unlock_bh(&im->lock); - } else + rv = READ_ONCE(im->sfcount[MCAST_EXCLUDE]) != 0; + } else { rv = 1; /* unspecified source; tentatively allow */ + } } return rv; } @@ -3024,7 +3076,7 @@ static inline struct ip_sf_list *igmp_mcf_get_first(struct seq_file *seq) im = rcu_dereference(idev->mc_list); if (likely(im)) { spin_lock_bh(&im->lock); - psf = im->sources; + psf = pmc_dereference(im->sources, im); if (likely(psf)) { state->im = im; state->idev = idev; @@ -3040,7 +3092,7 @@ static struct ip_sf_list *igmp_mcf_get_next(struct seq_file *seq, struct ip_sf_l { struct igmp_mcf_iter_state *state = igmp_mcf_seq_private(seq); - psf = psf->sf_next; + psf = pmc_dereference(psf->sf_next, state->im); while (!psf) { spin_unlock_bh(&state->im->lock); state->im = state->im->next; @@ -3056,7 +3108,7 @@ static struct ip_sf_list *igmp_mcf_get_next(struct seq_file *seq, struct ip_sf_l state->im = rcu_dereference(state->idev->mc_list); } spin_lock_bh(&state->im->lock); - psf = state->im->sources; + psf = pmc_dereference(state->im->sources, state->im); } out: return psf; diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 1679bfefa560d..6193db63b2973 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -748,9 +748,9 @@ void inet_csk_init_xmit_timers(struct sock *sk, { struct inet_connection_sock *icsk = inet_csk(sk); - timer_setup(&icsk->icsk_retransmit_timer, retransmit_handler, 0); + timer_setup(&sk->tcp_retransmit_timer, retransmit_handler, 0); timer_setup(&icsk->icsk_delack_timer, delack_handler, 0); - timer_setup(&sk->sk_timer, keepalive_handler, 0); + timer_setup(&icsk->icsk_keepalive_timer, keepalive_handler, 0); icsk->icsk_pending = icsk->icsk_ack.pending = 0; } @@ -761,9 +761,9 @@ void inet_csk_clear_xmit_timers(struct sock *sk) smp_store_release(&icsk->icsk_pending, 0); smp_store_release(&icsk->icsk_ack.pending, 0); - sk_stop_timer(sk, &icsk->icsk_retransmit_timer); + sk_stop_timer(sk, &sk->tcp_retransmit_timer); sk_stop_timer(sk, &icsk->icsk_delack_timer); - sk_stop_timer(sk, &sk->sk_timer); + sk_stop_timer(sk, &icsk->icsk_keepalive_timer); } void inet_csk_clear_xmit_timers_sync(struct sock *sk) @@ -776,9 +776,9 @@ void inet_csk_clear_xmit_timers_sync(struct sock *sk) smp_store_release(&icsk->icsk_pending, 0); smp_store_release(&icsk->icsk_ack.pending, 0); - sk_stop_timer_sync(sk, &icsk->icsk_retransmit_timer); + sk_stop_timer_sync(sk, &sk->tcp_retransmit_timer); sk_stop_timer_sync(sk, &icsk->icsk_delack_timer); - sk_stop_timer_sync(sk, &sk->sk_timer); + sk_stop_timer_sync(sk, &icsk->icsk_keepalive_timer); } struct dst_entry *inet_csk_route_req(const struct sock *sk, @@ -964,11 +964,23 @@ static struct request_sock *inet_reqsk_clone(struct request_sock *req, nreq->rsk_listener = sk; - /* We need not acquire fastopenq->lock - * because the child socket is locked in inet_csk_listen_stop(). - */ - if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) + if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(nreq)->tfo_listener) { + struct fastopen_queue *fastopenq; + + /* reqsk_fastopen_remove() will uncharge nreq->rsk_listener, + * that is @sk, so charge it here. Unlike the listener + * being closed, @sk is live and needs its lock. + */ + fastopenq = &inet_csk(sk)->icsk_accept_queue.fastopenq; + spin_lock_bh(&fastopenq->lock); + fastopenq->qlen++; + spin_unlock_bh(&fastopenq->lock); + + /* We need not acquire fastopenq->lock + * because the child socket is locked in inet_csk_listen_stop(). + */ rcu_assign_pointer(tcp_sk(nreq->sk)->fastopen_rsk, nreq); + } return nreq; } diff --git a/net/ipv4/inet_diag.c b/net/ipv4/inet_diag.c index f0b6c5a411a20..3f5b1418a6109 100644 --- a/net/ipv4/inet_diag.c +++ b/net/ipv4/inet_diag.c @@ -287,17 +287,17 @@ int inet_sk_diag_fill(struct sock *sk, struct inet_connection_sock *icsk, r->idiag_timer = 1; r->idiag_retrans = READ_ONCE(icsk->icsk_retransmits); r->idiag_expires = - jiffies_delta_to_msecs(icsk_timeout(icsk) - jiffies); + jiffies_delta_to_msecs(tcp_timeout_expires(sk) - jiffies); } else if (icsk_pending == ICSK_TIME_PROBE0) { r->idiag_timer = 4; r->idiag_retrans = READ_ONCE(icsk->icsk_probes_out); r->idiag_expires = - jiffies_delta_to_msecs(icsk_timeout(icsk) - jiffies); - } else if (timer_pending(&sk->sk_timer)) { + jiffies_delta_to_msecs(tcp_timeout_expires(sk) - jiffies); + } else if (timer_pending(&icsk->icsk_keepalive_timer)) { r->idiag_timer = 2; r->idiag_retrans = READ_ONCE(icsk->icsk_probes_out); r->idiag_expires = - jiffies_delta_to_msecs(sk->sk_timer.expires - jiffies); + jiffies_delta_to_msecs(icsk->icsk_keepalive_timer.expires - jiffies); } if ((ext & (1 << (INET_DIAG_INFO - 1))) && handler->idiag_info_size) { diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c index 69838e44cae13..21f7bf9ea5ba2 100644 --- a/net/ipv4/inet_fragment.c +++ b/net/ipv4/inet_fragment.c @@ -237,6 +237,8 @@ void fqdir_pre_exit(struct fqdir *fqdir) rhashtable_walk_start(&hti); while ((fq = rhashtable_walk_next(&hti))) { + int refs = 0; + if (IS_ERR(fq)) { if (PTR_ERR(fq) != -EAGAIN) break; @@ -244,8 +246,12 @@ void fqdir_pre_exit(struct fqdir *fqdir) } spin_lock_bh(&fq->lock); if (!(fq->flags & INET_FRAG_COMPLETE)) + inet_frag_kill(fq, &refs); + + if (fq->flags & INET_FRAG_HASH_DEAD) inet_frag_queue_flush(fq, 0); spin_unlock_bh(&fq->lock); + inet_frag_putn(fq, refs); } rhashtable_walk_stop(&hti); @@ -395,8 +401,8 @@ static struct inet_frag_queue *inet_frag_create(struct fqdir *fqdir, *prev = ERR_PTR(-ENOMEM); return NULL; } - mod_timer(&q->timer, jiffies + fqdir->timeout); + spin_lock_bh(&q->lock); *prev = rhashtable_lookup_get_insert_key(&fqdir->rhashtable, &q->key, &q->node, f->rhash_params); if (*prev) { @@ -404,13 +410,13 @@ static struct inet_frag_queue *inet_frag_create(struct fqdir *fqdir, * we need to cancel what inet_frag_alloc() * anticipated. */ - int refs = 1; - q->flags |= INET_FRAG_COMPLETE; - inet_frag_kill(q, &refs); - inet_frag_putn(q, refs); + spin_unlock_bh(&q->lock); + inet_frag_putn(q, 2); return NULL; } + mod_timer(&q->timer, jiffies + fqdir->timeout); + spin_unlock_bh(&q->lock); return q; } @@ -437,6 +443,13 @@ int inet_frag_queue_insert(struct inet_frag_queue *q, struct sk_buff *skb, { struct sk_buff *last = q->fragments_tail; + /* An IP fragment is never a GSO packet, but an untrusted source + * (virtio_net_hdr) may have attached GSO metadata to it. Do not let + * that reach the reassembled skb, whose head keeps the first + * fragment's shinfo and whose frag_list is not GRO-shaped. + */ + skb_gso_reset(skb); + /* RFC5722, Section 4, amended by Errata ID : 3089 * When reassembling an IPv6 datagram, if * one or more its constituent fragments is determined to be an diff --git a/net/ipv4/inetpeer.c b/net/ipv4/inetpeer.c index 7b1e0a2d69066..17933ac5b4280 100644 --- a/net/ipv4/inetpeer.c +++ b/net/ipv4/inetpeer.c @@ -22,6 +22,7 @@ #include #include #include +#include /* * Theory of operations. @@ -53,6 +54,34 @@ */ static struct kmem_cache *peer_cachep __ro_after_init; +static siphash_aligned_key_t inetpeer_hash_key __read_mostly; + +static u64 inetpeer_addr_hash(const struct inetpeer_addr *a) +{ + net_get_random_once(&inetpeer_hash_key, sizeof(inetpeer_hash_key)); + + if (a->family == AF_INET) + return siphash_2u32((__force u32)a->a4.addr, a->a4.vif, + &inetpeer_hash_key); + + return siphash_4u32((__force u32)a->a6.s6_addr32[0], + (__force u32)a->a6.s6_addr32[1], + (__force u32)a->a6.s6_addr32[2], + (__force u32)a->a6.s6_addr32[3], + &inetpeer_hash_key); +} + +static int inetpeer_entry_cmp(u64 dhash, + const struct inetpeer_addr *daddr, + const struct inet_peer *p) +{ + if (dhash < p->hash) + return -1; + if (dhash > p->hash) + return 1; + + return inetpeer_addr_cmp(daddr, &p->daddr); +} void inet_peer_base_init(struct inet_peer_base *bp) { @@ -86,6 +115,7 @@ void __init inet_initpeers(void) /* Called with rcu_read_lock() or base->lock held */ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, + u64 dhash, struct inet_peer_base *base, unsigned int seq, struct inet_peer *gc_stack[], @@ -107,7 +137,7 @@ static struct inet_peer *lookup(const struct inetpeer_addr *daddr, break; parent = next; p = rb_entry(parent, struct inet_peer, rb_node); - cmp = inetpeer_addr_cmp(daddr, &p->daddr); + cmp = inetpeer_entry_cmp(dhash, daddr, p); if (cmp == 0) { now = jiffies; if (READ_ONCE(p->dtime) != now) @@ -172,6 +202,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, const struct inetpeer_addr *daddr) { struct inet_peer *p, *gc_stack[PEER_MAX_GC]; + u64 dhash = inetpeer_addr_hash(daddr); struct rb_node **pp, *parent; unsigned int gc_cnt, seq; @@ -179,7 +210,7 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, * Because of a concurrent writer, we might not find an existing entry. */ seq = read_seqbegin(&base->lock); - p = lookup(daddr, base, seq, NULL, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, NULL, &gc_cnt, &parent, &pp); if (p) return p; @@ -191,11 +222,12 @@ struct inet_peer *inet_getpeer(struct inet_peer_base *base, write_seqlock_bh(&base->lock); gc_cnt = 0; - p = lookup(daddr, base, seq, gc_stack, &gc_cnt, &parent, &pp); + p = lookup(daddr, dhash, base, seq, gc_stack, &gc_cnt, &parent, &pp); if (!p) { p = kmem_cache_alloc(peer_cachep, GFP_ATOMIC); if (p) { p->daddr = *daddr; + p->hash = dhash; p->dtime = (__u32)jiffies; refcount_set(&p->refcnt, 1); atomic_set(&p->rid, 0); diff --git a/net/ipv4/ip_gre.c b/net/ipv4/ip_gre.c index 879d37c557faf..300b802125db2 100644 --- a/net/ipv4/ip_gre.c +++ b/net/ipv4/ip_gre.c @@ -1017,6 +1017,8 @@ static void __gre_tunnel_init(struct net_device *dev) dev->features |= GRE_FEATURES; dev->hw_features |= GRE_FEATURES; + dev->lltx = true; + /* TCP offload with GRE SEQ is not supported, nor can we support 2 * levels of outer headers requiring an update. */ @@ -1028,8 +1030,6 @@ static void __gre_tunnel_init(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; - - dev->lltx = true; } static int ipgre_tunnel_init(struct net_device *dev) diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 7eaf35a6e24ba..4c4e9d1940f7d 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -791,6 +791,10 @@ int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, */ hlen = iph->ihl * 4; + if (mtu < hlen + 8) { + err = -EMSGSIZE; + goto fail; + } mtu = mtu - hlen; /* Size of data space */ IPCB(skb)->flags |= IPSKB_FRAG_COMPLETE; ll_rs = LL_RESERVED_SPACE(rt->dst.dev); @@ -1299,7 +1303,8 @@ static int ip_setup_cork(struct sock *sk, struct inet_cork *cork, return -EFAULT; cork->fragsize = ip_sk_use_pmtu(sk) ? - dst_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); + dst4_mtu(&rt->dst) : READ_ONCE(rt->dst.dev->mtu); + cork->fragsize = min(cork->fragsize, IP_MAX_MTU); if (!inetdev_valid_mtu(cork->fragsize)) return -ENETUNREACH; @@ -1438,7 +1443,7 @@ struct sk_buff *__ip_make_skb(struct sock *sk, pmtudisc = READ_ONCE(inet->pmtudisc); if (pmtudisc == IP_PMTUDISC_DO || pmtudisc == IP_PMTUDISC_PROBE || - (skb->len <= dst_mtu(&rt->dst) && + (skb->len <= dst4_mtu(&rt->dst) && ip_dont_fragment(sk, &rt->dst))) df = htons(IP_DF); diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 7d4f1face2e79..7af607f329ac3 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -1634,7 +1634,7 @@ int do_ip_getsockopt(struct sock *sk, int level, int optname, val = 0; dst = sk_dst_get(sk); if (dst) { - val = dst_mtu(dst); + val = dst4_mtu(dst); dst_release(dst); } if (!val) diff --git a/net/ipv4/ip_tunnel.c b/net/ipv4/ip_tunnel.c index 50d0f5fe4e4c6..e9da39010f8b3 100644 --- a/net/ipv4/ip_tunnel.c +++ b/net/ipv4/ip_tunnel.c @@ -317,7 +317,7 @@ static int ip_tunnel_bind_dev(struct net_device *dev) mtu = min(tdev->mtu, IP_MAX_MTU); } - dev->needed_headroom = t_hlen + hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(t_hlen + hlen); mtu -= t_hlen + (dev->type == ARPHRD_ETHER ? dev->hard_header_len : 0); if (mtu < IPV4_MIN_MTU) diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c index 4b5fd4b13722c..011db59ff61f8 100644 --- a/net/ipv4/ip_tunnel_core.c +++ b/net/ipv4/ip_tunnel_core.c @@ -65,7 +65,7 @@ void iptunnel_xmit(struct sock *sk, struct rtable *rt, struct sk_buff *skb, DEV_STATS_INC(dev, tx_errors); } ip_rt_put(rt); - kfree_skb(skb); + kfree_skb_reason(skb, SKB_DROP_REASON_RECURSION_LIMIT); return; } @@ -268,6 +268,9 @@ static int iptunnel_pmtud_build_icmp(struct sk_buff *skb, int mtu) eth_header(skb, skb->dev, ntohs(eh.h_proto), eh.h_source, eh.h_dest, 0); skb_reset_mac_header(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); + return skb->len; } @@ -371,6 +374,9 @@ static int iptunnel_pmtud_build_icmpv6(struct sk_buff *skb, int mtu) eth_header(skb, skb->dev, ntohs(eh.h_proto), eh.h_source, eh.h_dest, 0); skb_reset_mac_header(skb); + if (skb_valid_dst(skb)) + skb_dst_drop(skb); + return skb->len; } diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c index 94e974c201f0a..4c2447d770e45 100644 --- a/net/ipv4/ipip.c +++ b/net/ipv4/ipip.c @@ -248,7 +248,7 @@ static int ipip_tunnel_rcv(struct sk_buff *skb, u8 ipproto) tun_dst = ip_tun_rx_dst(skb, flags, 0, 0); if (!tun_dst) - return 0; + goto drop; ip_tunnel_md_udp_encap(skb, &tun_dst->u.tun_info); } skb_reset_mac_header(skb); diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index ca9eaee4c2ef5..b1282c01a77e0 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -102,7 +102,8 @@ static DEFINE_SPINLOCK(mfc_unres_lock); static struct kmem_cache *mrt_cachep __ro_after_init; static struct mr_table *ipmr_new_table(struct net *net, u32 id); -static void ipmr_free_table(struct mr_table *mrt); +static void ipmr_free_table(struct mr_table *mrt, + struct list_head *dev_kill_list); static void ip_mr_forward(struct net *net, struct mr_table *mrt, struct net_device *dev, struct sk_buff *skb, @@ -112,7 +113,8 @@ static int ipmr_cache_report(const struct mr_table *mrt, static void mroute_netlink_event(struct mr_table *mrt, struct mfc_cache *mfc, int cmd); static void igmpmsg_netlink_event(const struct mr_table *mrt, struct sk_buff *pkt); -static void mroute_clean_tables(struct mr_table *mrt, int flags); +static void mroute_clean_tables(struct mr_table *mrt, int flags, + struct list_head *dev_kill_list); static void ipmr_expire_process(struct timer_list *t); #ifdef CONFIG_IP_MROUTE_MULTIPLE_TABLES @@ -149,16 +151,6 @@ static struct mr_table *__ipmr_get_table(struct net *net, u32 id) return NULL; } -static struct mr_table *ipmr_get_table(struct net *net, u32 id) -{ - struct mr_table *mrt; - - rcu_read_lock(); - mrt = __ipmr_get_table(net, id); - rcu_read_unlock(); - return mrt; -} - static int ipmr_fib_lookup(struct net *net, struct flowi4 *flp4, struct mr_table **mrt) { @@ -250,6 +242,7 @@ static const struct fib_rules_ops __net_initconst ipmr_rules_ops_template = { static int __net_init ipmr_rules_init(struct net *net) { struct fib_rules_ops *ops; + LIST_HEAD(dev_kill_list); struct mr_table *mrt; int err; @@ -273,23 +266,22 @@ static int __net_init ipmr_rules_init(struct net *net) return 0; err2: - rtnl_lock(); - ipmr_free_table(mrt); - rtnl_unlock(); + ipmr_free_table(mrt, &dev_kill_list); err1: fib_rules_unregister(ops); return err; } -static void __net_exit ipmr_rules_exit(struct net *net) +static void __net_exit ipmr_rules_exit_rtnl(struct net *net, + struct list_head *dev_kill_list) { struct mr_table *mrt, *next; - ASSERT_RTNL(); list_for_each_entry_safe(mrt, next, &net->ipv4.mr_tables, list) { - list_del(&mrt->list); - ipmr_free_table(mrt); + list_del_rcu(&mrt->list); + ipmr_free_table(mrt, dev_kill_list); } + fib_rules_unregister(net->ipv4.mr_rules_ops); } @@ -310,28 +302,30 @@ bool ipmr_rule_default(const struct fib_rule *rule) } EXPORT_SYMBOL(ipmr_rule_default); #else -#define ipmr_for_each_table(mrt, net) \ - for (mrt = net->ipv4.mrt; mrt; mrt = NULL) - static struct mr_table *ipmr_mr_table_iter(struct net *net, struct mr_table *mrt) { if (!mrt) - return net->ipv4.mrt; + return rcu_dereference(net->ipv4.mrt); return NULL; } -static struct mr_table *ipmr_get_table(struct net *net, u32 id) +static struct mr_table *__ipmr_get_table(struct net *net, u32 id) { - return net->ipv4.mrt; + return rcu_dereference_check(net->ipv4.mrt, + lockdep_rtnl_is_held() || + !rcu_access_pointer(net->ipv4.mrt)); } -#define __ipmr_get_table ipmr_get_table +#define ipmr_for_each_table(mrt, net) \ + for (mrt = __ipmr_get_table(net, 0); mrt; mrt = NULL) static int ipmr_fib_lookup(struct net *net, struct flowi4 *flp4, struct mr_table **mrt) { - *mrt = net->ipv4.mrt; + *mrt = rcu_dereference(net->ipv4.mrt); + if (!*mrt) + return -EAGAIN; return 0; } @@ -342,15 +336,18 @@ static int __net_init ipmr_rules_init(struct net *net) mrt = ipmr_new_table(net, RT_TABLE_DEFAULT); if (IS_ERR(mrt)) return PTR_ERR(mrt); - net->ipv4.mrt = mrt; + + rcu_assign_pointer(net->ipv4.mrt, mrt); return 0; } -static void __net_exit ipmr_rules_exit(struct net *net) +static void __net_exit ipmr_rules_exit_rtnl(struct net *net, + struct list_head *dev_kill_list) { - ASSERT_RTNL(); - ipmr_free_table(net->ipv4.mrt); - net->ipv4.mrt = NULL; + struct mr_table *mrt = rcu_dereference_protected(net->ipv4.mrt, 1); + + RCU_INIT_POINTER(net->ipv4.mrt, NULL); + ipmr_free_table(mrt, dev_kill_list); } static int ipmr_rules_dump(struct net *net, struct notifier_block *nb, @@ -371,6 +368,17 @@ bool ipmr_rule_default(const struct fib_rule *rule) EXPORT_SYMBOL(ipmr_rule_default); #endif +static struct mr_table *ipmr_get_table(struct net *net, u32 id) +{ + struct mr_table *mrt; + + rcu_read_lock(); + mrt = __ipmr_get_table(net, id); + rcu_read_unlock(); + + return mrt; +} + static inline int ipmr_hash_cmp(struct rhashtable_compare_arg *arg, const void *ptr) { @@ -424,17 +432,21 @@ static struct mr_table *ipmr_new_table(struct net *net, u32 id) ipmr_expire_process, ipmr_new_table_set); } -static void ipmr_free_table(struct mr_table *mrt) +static void ipmr_free_table(struct mr_table *mrt, struct list_head *dev_kill_list) { struct net *net = read_pnet(&mrt->net); + LIST_HEAD(ipmr_dev_kill_list); WARN_ON_ONCE(!mr_can_free_table(net)); - timer_shutdown_sync(&mrt->ipmr_expire_timer); mroute_clean_tables(mrt, MRT_FLUSH_VIFS | MRT_FLUSH_VIFS_STATIC | - MRT_FLUSH_MFC | MRT_FLUSH_MFC_STATIC); - rhltable_destroy(&mrt->mfc_hash); - kfree(mrt); + MRT_FLUSH_MFC | MRT_FLUSH_MFC_STATIC, + &ipmr_dev_kill_list); + timer_shutdown_sync(&mrt->ipmr_expire_timer); + mr_table_free(mrt); + + WARN_ON_ONCE(!net_initialized(net) && !list_empty(&ipmr_dev_kill_list)); + list_splice(&ipmr_dev_kill_list, dev_kill_list); } /* Service routines creating virtual interfaces: DVMRP tunnels and PIMREG */ @@ -518,15 +530,16 @@ static netdev_tx_t reg_vif_xmit(struct sk_buff *skb, struct net_device *dev) }; int err; + rcu_read_lock(); err = ipmr_fib_lookup(net, &fl4, &mrt); if (err < 0) { + rcu_read_unlock(); kfree_skb(skb); return err; } DEV_STATS_ADD(dev, tx_bytes, skb->len); DEV_STATS_INC(dev, tx_packets); - rcu_read_lock(); /* Pairs with WRITE_ONCE() in vif_add() and vif_delete() */ ipmr_cache_report(mrt, skb, READ_ONCE(mrt->mroute_reg_vif_num), @@ -1120,12 +1133,19 @@ static int ipmr_cache_report(const struct mr_table *mrt, static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi, struct sk_buff *skb, struct net_device *dev) { + struct net *net = read_pnet(&mrt->net); const struct iphdr *iph = ip_hdr(skb); - struct mfc_cache *c; + struct mfc_cache *c = NULL; bool found = false; int err; spin_lock_bh(&mfc_unres_lock); + + if (!check_net(net)) { + err = -EINVAL; + goto err; + } + list_for_each_entry(c, &mrt->mfc_unres_queue, _c.list) { if (c->mfc_mcastgrp == iph->daddr && c->mfc_origin == iph->saddr) { @@ -1138,10 +1158,8 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi, /* Create a new entry if allowable */ c = ipmr_cache_alloc_unres(); if (!c) { - spin_unlock_bh(&mfc_unres_lock); - - kfree_skb(skb); - return -ENOBUFS; + err = -ENOBUFS; + goto err; } /* Fill in the new cache entry */ @@ -1151,17 +1169,8 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi, /* Reflect first query at mrouted. */ err = ipmr_cache_report(mrt, skb, vifi, IGMPMSG_NOCACHE); - - if (err < 0) { - /* If the report failed throw the cache entry - out - Brad Parker - */ - spin_unlock_bh(&mfc_unres_lock); - - ipmr_cache_free(c); - kfree_skb(skb); - return err; - } + if (err < 0) + goto err; atomic_inc(&mrt->cache_resolve_queue_len); list_add(&c->_c.list, &mrt->mfc_unres_queue); @@ -1174,18 +1183,26 @@ static int ipmr_cache_unresolved(struct mr_table *mrt, vifi_t vifi, /* See if we can append the packet */ if (c->_c.mfc_un.unres.unresolved.qlen > 3) { - kfree_skb(skb); + c = NULL; err = -ENOBUFS; - } else { - if (dev) { - skb->dev = dev; - skb->skb_iif = dev->ifindex; - } - skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb); - err = 0; + goto err; } + if (dev) { + skb->dev = dev; + skb->skb_iif = dev->ifindex; + } + + skb_queue_tail(&c->_c.mfc_un.unres.unresolved, skb); + spin_unlock_bh(&mfc_unres_lock); + return 0; + +err: + spin_unlock_bh(&mfc_unres_lock); + if (c) + ipmr_cache_free(c); + kfree_skb(skb); return err; } @@ -1293,12 +1310,12 @@ static int ipmr_mfc_add(struct net *net, struct mr_table *mrt, } /* Close the multicast socket, and clear the vif tables etc */ -static void mroute_clean_tables(struct mr_table *mrt, int flags) +static void mroute_clean_tables(struct mr_table *mrt, int flags, + struct list_head *dev_kill_list) { struct net *net = read_pnet(&mrt->net); - struct mr_mfc *c, *tmp; struct mfc_cache *cache; - LIST_HEAD(list); + struct mr_mfc *c, *tmp; int i; /* Shut down all active vif entries */ @@ -1308,9 +1325,8 @@ static void mroute_clean_tables(struct mr_table *mrt, int flags) !(flags & MRT_FLUSH_VIFS_STATIC)) || (!(mrt->vif_table[i].flags & VIFF_STATIC) && !(flags & MRT_FLUSH_VIFS))) continue; - vif_delete(mrt, i, 0, &list); + vif_delete(mrt, i, 0, dev_kill_list); } - unregister_netdevice_many(&list); } /* Wipe the cache */ @@ -1330,7 +1346,7 @@ static void mroute_clean_tables(struct mr_table *mrt, int flags) } if (flags & MRT_FLUSH_MFC) { - if (atomic_read(&mrt->cache_resolve_queue_len) != 0) { + if (atomic_read(&mrt->cache_resolve_queue_len) != 0 || !check_net(net)) { spin_lock_bh(&mfc_unres_lock); list_for_each_entry_safe(c, tmp, &mrt->mfc_unres_queue, list) { list_del(&c->list); @@ -1349,9 +1365,11 @@ static void mroute_clean_tables(struct mr_table *mrt, int flags) static void mrtsock_destruct(struct sock *sk) { struct net *net = sock_net(sk); + LIST_HEAD(dev_kill_list); struct mr_table *mrt; rtnl_lock(); + ipmr_for_each_table(mrt, net) { if (sk == rtnl_dereference(mrt->mroute_sk)) { IPV4_DEVCONF_ALL(net, MC_FORWARDING)--; @@ -1360,9 +1378,13 @@ static void mrtsock_destruct(struct sock *sk) NETCONFA_IFINDEX_ALL, net->ipv4.devconf_all); RCU_INIT_POINTER(mrt->mroute_sk, NULL); - mroute_clean_tables(mrt, MRT_FLUSH_VIFS | MRT_FLUSH_MFC); + mroute_clean_tables(mrt, MRT_FLUSH_VIFS | MRT_FLUSH_MFC, + &dev_kill_list); } } + + unregister_netdevice_many(&dev_kill_list); + rtnl_unlock(); } @@ -1485,7 +1507,9 @@ int ip_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, sk == rtnl_dereference(mrt->mroute_sk), parent); break; - case MRT_FLUSH: + case MRT_FLUSH: { + LIST_HEAD(dev_kill_list); + if (optlen != sizeof(val)) { ret = -EINVAL; break; @@ -1494,8 +1518,11 @@ int ip_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval, ret = -EFAULT; break; } - mroute_clean_tables(mrt, val); + + mroute_clean_tables(mrt, val, &dev_kill_list); + unregister_netdevice_many(&dev_kill_list); break; + } /* Control PIM assert. */ case MRT_ASSERT: if (optlen != sizeof(val)) { @@ -1895,7 +1922,7 @@ static int ipmr_prepare_xmit(struct net *net, struct mr_table *mrt, return -1; } - if (skb->len+encap > dst_mtu(&rt->dst) && (ntohs(iph->frag_off) & IP_DF)) { + if (skb->len+encap > dst4_mtu(&rt->dst) && (ntohs(iph->frag_off) & IP_DF)) { /* Do not fragment multicasts. Alas, IPv4 does not * allow to send ICMP, so that packets will disappear * to blackhole. @@ -2168,6 +2195,9 @@ int ip_mr_input(struct sk_buff *skb) if (IPCB(skb)->flags & IPSKB_FORWARDED) goto dont_forward; + if (!local) + skb_orphan(skb); + mrt = ipmr_rt_fib_lookup(net, skb); if (IS_ERR(mrt)) { kfree_skb(skb); @@ -3225,6 +3255,7 @@ static void __net_exit ipmr_notifier_exit(struct net *net) /* Setup for IP multicast routing */ static int __net_init ipmr_net_init(struct net *net) { + LIST_HEAD(dev_kill_list); int err; err = ipmr_notifier_init(net); @@ -3250,9 +3281,7 @@ static int __net_init ipmr_net_init(struct net *net) proc_cache_fail: remove_proc_entry("ip_mr_vif", net->proc_net); proc_vif_fail: - rtnl_lock(); - ipmr_rules_exit(net); - rtnl_unlock(); + ipmr_rules_exit_rtnl(net, &dev_kill_list); #endif ipmr_rules_fail: ipmr_notifier_exit(net); @@ -3269,20 +3298,16 @@ static void __net_exit ipmr_net_exit(struct net *net) ipmr_notifier_exit(net); } -static void __net_exit ipmr_net_exit_batch(struct list_head *net_list) +static void __net_exit ipmr_net_exit_rtnl(struct net *net, + struct list_head *dev_kill_list) { - struct net *net; - - rtnl_lock(); - list_for_each_entry(net, net_list, exit_list) - ipmr_rules_exit(net); - rtnl_unlock(); + ipmr_rules_exit_rtnl(net, dev_kill_list); } static struct pernet_operations ipmr_net_ops = { .init = ipmr_net_init, .exit = ipmr_net_exit, - .exit_batch = ipmr_net_exit_batch, + .exit_rtnl = ipmr_net_exit_rtnl, }; static const struct rtnl_msg_handler ipmr_rtnl_msg_handlers[] __initconst = { @@ -3300,7 +3325,8 @@ int __init ip_mr_init(void) { int err; - mrt_cachep = KMEM_CACHE(mfc_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + mrt_cachep = KMEM_CACHE(mfc_cache, + SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT); err = register_pernet_subsys(&ipmr_net_ops); if (err) diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c index 28d77d454d442..15f90658659a1 100644 --- a/net/ipv4/ipmr_base.c +++ b/net/ipv4/ipmr_base.c @@ -28,6 +28,20 @@ void vif_device_init(struct vif_device *v, } EXPORT_SYMBOL(vif_device_init); +static void __mr_free_table(struct work_struct *work) +{ + struct mr_table *mrt = container_of(to_rcu_work(work), + struct mr_table, work); + + rhltable_destroy(&mrt->mfc_hash); + kfree(mrt); +} + +void mr_table_free(struct mr_table *mrt) +{ + queue_rcu_work(system_unbound_wq, &mrt->work); +} + struct mr_table * mr_table_alloc(struct net *net, u32 id, struct mr_table_ops *ops, @@ -38,7 +52,7 @@ mr_table_alloc(struct net *net, u32 id, struct mr_table *mrt; int err; - mrt = kzalloc(sizeof(*mrt), GFP_KERNEL); + mrt = kzalloc(sizeof(*mrt), GFP_KERNEL_ACCOUNT); if (!mrt) return ERR_PTR(-ENOMEM); mrt->id = id; @@ -50,6 +64,8 @@ mr_table_alloc(struct net *net, u32 id, kfree(mrt); return ERR_PTR(err); } + + INIT_RCU_WORK(&mrt->work, __mr_free_table); INIT_LIST_HEAD(&mrt->mfc_cache_list); INIT_LIST_HEAD(&mrt->mfc_unres_queue); diff --git a/net/ipv4/netfilter/nf_reject_ipv4.c b/net/ipv4/netfilter/nf_reject_ipv4.c index c47d56576720f..4626dc46808f3 100644 --- a/net/ipv4/netfilter/nf_reject_ipv4.c +++ b/net/ipv4/netfilter/nf_reject_ipv4.c @@ -303,7 +303,7 @@ void nf_send_reset(struct net *net, struct sock *sk, struct sk_buff *oldskb, goto free_nskb; /* "Never happens" */ - if (nskb->len > dst_mtu(skb_dst(nskb))) + if (nskb->len > dst4_mtu(skb_dst(nskb))) goto free_nskb; nf_ct_attach(nskb, oldskb); diff --git a/net/ipv4/netfilter/nft_dup_ipv4.c b/net/ipv4/netfilter/nft_dup_ipv4.c index ef5dd88107ddb..d53a65ddbd7b7 100644 --- a/net/ipv4/netfilter/nft_dup_ipv4.c +++ b/net/ipv4/netfilter/nft_dup_ipv4.c @@ -76,7 +76,6 @@ static const struct nft_expr_ops nft_dup_ipv4_ops = { .eval = nft_dup_ipv4_eval, .init = nft_dup_ipv4_init, .dump = nft_dup_ipv4_dump, - .reduce = NFT_REDUCE_READONLY, }; static const struct nla_policy nft_dup_ipv4_policy[NFTA_DUP_MAX + 1] = { diff --git a/net/ipv4/netfilter/nft_fib_ipv4.c b/net/ipv4/netfilter/nft_fib_ipv4.c index e695283aeb2d0..177d738825b4c 100644 --- a/net/ipv4/netfilter/nft_fib_ipv4.c +++ b/net/ipv4/netfilter/nft_fib_ipv4.c @@ -163,7 +163,6 @@ static const struct nft_expr_ops nft_fib4_type_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops nft_fib4_ops = { @@ -173,7 +172,6 @@ static const struct nft_expr_ops nft_fib4_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops * diff --git a/net/ipv4/netfilter/nft_reject_ipv4.c b/net/ipv4/netfilter/nft_reject_ipv4.c index 6cb213bb7256a..55fc23a8f7a70 100644 --- a/net/ipv4/netfilter/nft_reject_ipv4.c +++ b/net/ipv4/netfilter/nft_reject_ipv4.c @@ -45,7 +45,6 @@ static const struct nft_expr_ops nft_reject_ipv4_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_ipv4_type __read_mostly = { diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 5a95b64b61c59..66d5c996866bf 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -1791,8 +1791,8 @@ static bool nh_res_bucket_migrate(struct nh_res_table *res_table, bool notify_nl, bool force) { struct nh_res_bucket *bucket = &res_table->nh_buckets[bucket_index]; + struct netlink_ext_ack extack = {}; struct nh_grp_entry *new_nhge; - struct netlink_ext_ack extack; int err; new_nhge = list_first_entry_or_null(&res_table->uw_nh_entries, @@ -2025,7 +2025,7 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge, { struct nh_grp_entry *nhges, *new_nhges; struct nexthop *nhp = nhge->nh_parent; - struct netlink_ext_ack extack; + struct netlink_ext_ack extack = {}; struct nexthop *nh = nhge->nh; struct nh_group *nhg, *newg; int i, j, err; diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 7f32753a16f41..21ced929cc9c1 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -944,7 +944,8 @@ static enum skb_drop_reason __ping_queue_rcv_skb(struct sock *sk, pr_debug("ping_queue_rcv_skb(sk=%p,sk->num=%d,skb=%p)\n", inet_sk(sk), inet_sk(sk)->inet_num, skb); - if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) { + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) { sk_skb_reason_drop(sk, skb, reason); pr_debug("ping_queue_rcv_skb -> failed\n"); return reason; diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index 26468a2ec63db..5735ae7cba01a 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -120,13 +120,21 @@ bool raw_v4_match(struct net *net, const struct sock *sk, unsigned short num, __be32 raddr, __be32 laddr, int dif, int sdif) { const struct inet_sock *inet = inet_sk(sk); + __be32 daddr, rcv_saddr; - if (net_eq(sock_net(sk), net) && inet->inet_num == num && - !(inet->inet_daddr && inet->inet_daddr != raddr) && - !(inet->inet_rcv_saddr && inet->inet_rcv_saddr != laddr) && - raw_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) - return true; - return false; + if (!net_eq(sock_net(sk), net) || inet->inet_num != num) + return false; + + daddr = READ_ONCE(inet->inet_daddr); + if (daddr && daddr != raddr) + return false; + + rcv_saddr = READ_ONCE(inet->inet_rcv_saddr); + if (rcv_saddr && rcv_saddr != laddr) + return false; + + return raw_sk_bound_dev_eq(net, READ_ONCE(sk->sk_bound_dev_if), + dif, sdif); } EXPORT_SYMBOL_GPL(raw_v4_match); @@ -300,7 +308,8 @@ static int raw_rcv_skb(struct sock *sk, struct sk_buff *skb) /* Charge it to the socket. */ ipv4_pktinfo_prepare(sk, skb, true); - if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) { + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) { sk_skb_reason_drop(sk, skb, reason); return NET_RX_DROP; } @@ -721,7 +730,8 @@ static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len) chk_addr_ret)) goto out; - inet->inet_rcv_saddr = inet->inet_saddr = addr->sin_addr.s_addr; + inet->inet_saddr = addr->sin_addr.s_addr; + WRITE_ONCE(inet->inet_rcv_saddr, addr->sin_addr.s_addr); if (chk_addr_ret == RTN_MULTICAST || chk_addr_ret == RTN_BROADCAST) inet->inet_saddr = 0; /* Use device */ sk_dst_reset(sk); diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 11d990703d31a..a25ab9d263ab7 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -741,6 +741,35 @@ static void update_or_create_fnhe(struct fib_nh_common *nhc, __be32 daddr, spin_unlock_bh(&fnhe_lock); } +/* Update the PMTU of an exception when: + * - the new MTU of the first hop becomes smaller than the PMTU + * - the old MTU was the same as the PMTU, and it limited discovery of + * larger MTUs on the path. With that limit raised, we can now + * discover larger MTUs + * A special case is locked exceptions, for which the PMTU is smaller + * than the minimal accepted PMTU: + * - if the new MTU is greater than the PMTU, don't make any change + * - otherwise, unlock and set PMTU + * + * fnhe_lock keeps fnhe_pmtu and fnhe_mtu_locked consistent against + * update_or_create_fnhe(), which sets both under the same lock. + */ +void fnhe_update_pmtu(struct fib_nh_exception *fnhe, u32 new, u32 orig) +{ + spin_lock_bh(&fnhe_lock); + + if (fnhe->fnhe_mtu_locked) { + if (new <= fnhe->fnhe_pmtu) { + fnhe->fnhe_pmtu = new; + fnhe->fnhe_mtu_locked = false; + } + } else if (new < fnhe->fnhe_pmtu || orig == fnhe->fnhe_pmtu) { + fnhe->fnhe_pmtu = new; + } + + spin_unlock_bh(&fnhe_lock); +} + static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4, bool kill_route) { @@ -892,8 +921,6 @@ void ip_rt_send_redirect(struct sk_buff *skb) peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif); if (!peer) { rcu_read_unlock(); - icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, - rt_nexthop(rt, ip_hdr(skb)->daddr)); return; } @@ -1333,8 +1360,8 @@ static unsigned int ipv4_default_advmss(const struct dst_entry *dst) rcu_read_lock(); net = dst_dev_net_rcu(dst); - advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size, - net->ipv4.ip_rt_min_advmss); + advmss = max_t(unsigned int, ip_dst_mtu_configured(dst) - header_size, + net->ipv4.ip_rt_min_advmss); rcu_read_unlock(); return min(advmss, IPV4_MAX_PMTU - header_size); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index db80b81a74cbb..8c0ffe50ae5d0 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -1132,8 +1132,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) zc = MSG_SPLICE_PAGES; } - if (!sockc_err && sockc.dmabuf_id && - (!(flags & MSG_ZEROCOPY) || !sock_flag(sk, SOCK_ZEROCOPY))) { + if (!sockc_err && sockc.dmabuf_id && (zc != MSG_ZEROCOPY || !binding)) { err = -EINVAL; goto out_err; } @@ -1203,7 +1202,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) trace_tcp_sendmsg_locked(sk, msg, skb, size_goal); - if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) { + if (copy <= 0 || !tcp_skb_can_collapse_to(skb) || + unlikely(skb_frags_readable(skb) != !binding)) { bool first_skb; new_segment: @@ -3166,8 +3166,7 @@ void __tcp_close(struct sock *sk, long timeout) /* Unread data was tossed, zap the connection. */ NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE); tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, sk->sk_allocation, - SK_RST_REASON_TCP_ABORT_ON_CLOSE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_CLOSE); } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) { /* Check zero linger _after_ checking for unread data. */ sk->sk_prot->disconnect(sk, 0); @@ -3241,7 +3240,7 @@ void __tcp_close(struct sock *sk, long timeout) struct tcp_sock *tp = tcp_sk(sk); if (READ_ONCE(tp->linger2) < 0) { tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_LINGER); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONLINGER); @@ -3260,7 +3259,7 @@ void __tcp_close(struct sock *sk, long timeout) if (sk->sk_state != TCP_CLOSE) { if (tcp_check_oom(sk, 0)) { tcp_set_state(sk, TCP_CLOSE); - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_MEMORY); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONMEMORY); @@ -3361,14 +3360,14 @@ int tcp_disconnect(struct sock *sk, int flags) } else if (unlikely(tp->repair)) { WRITE_ONCE(sk->sk_err, ECONNABORTED); } else if (tcp_need_reset(old_state)) { - tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (tp->snd_nxt != tp->write_seq && (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { /* The last check adjusts for discrepancy of Linux wrt. RFC * states */ - tcp_send_active_reset(sk, gfp_any(), + tcp_send_active_reset(sk, SK_RST_REASON_TCP_DISCONNECT_WITH_DATA); WRITE_ONCE(sk->sk_err, ECONNRESET); } else if (old_state == TCP_SYN_SENT) @@ -5088,8 +5087,7 @@ int tcp_abort(struct sock *sk, int err) bh_lock_sock(sk); if (tcp_need_reset(sk->sk_state)) - tcp_send_active_reset(sk, GFP_ATOMIC, - SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); tcp_done_with_error(sk, err); bh_unlock_sock(sk); diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c index a7b1fe194fb1e..77352f720696f 100644 --- a/net/ipv4/tcp_ao.c +++ b/net/ipv4/tcp_ao.c @@ -1120,6 +1120,15 @@ void tcp_ao_connect_init(struct sock *sk) l3index = l3mdev_master_ifindex_by_index(sock_net(sk), sk->sk_bound_dev_if); + hlist_for_each_entry(key, &ao_info->head, node) { + if (tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, + family, -1, -1)) { + /* pairs with tcp_inbound_ao_hash() */ + synchronize_rcu(); + break; + } + } + hlist_for_each_entry_safe(key, next, &ao_info->head, node) { if (!tcp_ao_key_cmp(key, l3index, addr, key->prefixlen, family, -1, -1)) continue; @@ -1147,12 +1156,7 @@ void tcp_ao_connect_init(struct sock *sk) ao_info->lisn = htonl(tp->write_seq); ao_info->snd_sne = 0; } else { - /* Can't happen: tcp_connect() verifies that there's - * at least one tcp-ao key that matches the remote peer. - */ - WARN_ON_ONCE(1); - rcu_assign_pointer(tp->ao_info, NULL); - kfree(ao_info); + tcp_ao_destroy_sock(sk, false); } } @@ -1854,6 +1858,9 @@ static int tcp_ao_del_cmd(struct sock *sk, unsigned short int family, if (cmd.ifindex && !(cmd.keyflags & TCP_AO_KEYF_IFINDEX)) return -EINVAL; + if (cmd.keyflags & TCP_AO_KEYF_IFINDEX) + l3index = cmd.ifindex; + ao_info = setsockopt_ao_info(sk); if (IS_ERR(ao_info)) return PTR_ERR(ao_info); diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index 7a3f0b244c8fc..ca51756d8aa47 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -454,6 +454,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, case __SK_REDIRECT: redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; + sock_hold(sk_redir); sk_msg_apply_bytes(psock, tosend); if (!psock->apply_bytes) { /* Clean up before releasing the sock lock. */ @@ -474,6 +475,7 @@ static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock, if (eval == __SK_REDIRECT) sock_put(sk_redir); + sock_put(sk_redir); lock_sock(sk); sk_mem_uncharge(sk, sent); @@ -589,7 +591,7 @@ static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) wait_for_memory: err = sk_stream_wait_memory(sk, &timeo); if (err) { - if (msg_tx && msg_tx != psock->cork) + if (msg_tx == &tmp) sk_msg_free(sk, msg_tx); goto out_err; } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index c650398e91998..05521a3509a96 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -252,7 +252,7 @@ static void tcp_measure_rcv_mss(struct sock *sk, const struct sk_buff *skb) struct tcp_sock *tp = tcp_sk(sk); val = tcp_win_from_space(sk, sk->sk_rcvbuf); - tcp_set_window_clamp(sk, val); + WRITE_ONCE(tp->window_clamp, val); if (tp->window_clamp < tp->rcvq_space.space) tp->rcvq_space.space = tp->window_clamp; @@ -1006,9 +1006,9 @@ static void tcp_event_data_recv(struct sock *sk, struct sk_buff *skb) /* The fastest case is the first. */ icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + TCP_ATO_MIN / 2; } else if (m < icsk->icsk_ack.ato) { - icsk->icsk_ack.ato = (icsk->icsk_ack.ato >> 1) + m; - if (icsk->icsk_ack.ato > icsk->icsk_rto) - icsk->icsk_ack.ato = icsk->icsk_rto; + icsk->icsk_ack.ato = min3((icsk->icsk_ack.ato >> 1) + (u32)m, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else if (m > icsk->icsk_rto) { /* Too long gap. Apparently sender failed to * restart window, so that we send ACKs quickly. @@ -3820,24 +3820,17 @@ static void tcp_send_ack_reflect_ect(struct sock *sk, bool accecn_reflector) __tcp_send_ack(sk, tp->rcv_nxt, flags); } -/* RFC 5961 7 [ACK Throttling] */ -static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) +/* Consume one slot from the per-netns RFC 5961 challenge ACK quota. + * Returns true if a challenge ACK may be sent. + */ +static bool tcp_challenge_ack_allowed(struct net *net) { - struct tcp_sock *tp = tcp_sk(sk); - struct net *net = sock_net(sk); u32 count, now, ack_limit; - /* First check our per-socket dupack rate limit. */ - if (__tcp_oow_rate_limited(net, - LINUX_MIB_TCPACKSKIPPEDCHALLENGE, - &tp->last_oow_ack_time)) - return; - ack_limit = READ_ONCE(net->ipv4.sysctl_tcp_challenge_ack_limit); if (ack_limit == INT_MAX) - goto send_ack; + return true; - /* Then check host-wide RFC 5961 rate limit. */ now = jiffies / HZ; if (now != READ_ONCE(net->ipv4.tcp_challenge_timestamp)) { u32 half = (ack_limit + 1) >> 1; @@ -3849,12 +3842,49 @@ static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) count = READ_ONCE(net->ipv4.tcp_challenge_count); if (count > 0) { WRITE_ONCE(net->ipv4.tcp_challenge_count, count - 1); -send_ack: + return true; + } + return false; +} + +/* RFC 5961 7 [ACK Throttling] */ +static void tcp_send_challenge_ack(struct sock *sk, bool accecn_reflector) +{ + struct tcp_sock *tp = tcp_sk(sk); + struct net *net = sock_net(sk); + + /* First check our per-socket dupack rate limit. */ + if (__tcp_oow_rate_limited(net, + LINUX_MIB_TCPACKSKIPPEDCHALLENGE, + &tp->last_oow_ack_time)) + return; + + /* Then check the per-netns RFC 5961 rate limit. */ + if (tcp_challenge_ack_allowed(net)) { NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK); tcp_send_ack_reflect_ect(sk, accecn_reflector); } } +/* Send a challenge ACK from a SYN-RECEIVED request socket. Uses + * __tcp_oow_rate_limited() directly so that an RST carrying payload + * cannot bypass the per-request rate limit. + */ +void tcp_reqsk_send_challenge_ack(struct sock *sk, struct sk_buff *skb, + struct request_sock *req) +{ + struct net *net = sock_net(sk); + + if (__tcp_oow_rate_limited(net, LINUX_MIB_TCPACKSKIPPEDCHALLENGE, + &tcp_rsk(req)->last_oow_ack_time)) + return; + + if (tcp_challenge_ack_allowed(net)) { + NET_INC_STATS(net, LINUX_MIB_TCPCHALLENGEACK); + req->rsk_ops->send_ack(sk, skb, req); + } +} + static void tcp_store_ts_recent(struct tcp_sock *tp) { tp->rx_opt.ts_recent = tp->rx_opt.rcv_tsval; diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index f0106c2be1cf2..caefe3db53a97 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -373,7 +373,7 @@ void tcp_v4_mtu_reduced(struct sock *sk) { struct inet_sock *inet = inet_sk(sk); struct dst_entry *dst; - u32 mtu; + u32 mtu, dmtu; if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) return; @@ -385,15 +385,14 @@ void tcp_v4_mtu_reduced(struct sock *sk) /* Something is about to be wrong... Remember soft error * for the case, if this connection will not able to recover. */ - if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst)) + dmtu = dst4_mtu(dst); + if (mtu < dmtu && ip_dont_fragment(sk, dst)) WRITE_ONCE(sk->sk_err_soft, EMSGSIZE); - mtu = dst_mtu(dst); - if (inet->pmtudisc != IP_PMTUDISC_DONT && ip_sk_accept_pmtu(sk) && - inet_csk(sk)->icsk_pmtu_cookie > mtu) { - tcp_sync_mss(sk, mtu); + inet_csk(sk)->icsk_pmtu_cookie > dmtu) { + tcp_sync_mss(sk, dmtu); /* Resend the TCP packet because it's * clear that the old packet has been @@ -998,6 +997,9 @@ static void tcp_v4_send_ack(const struct sock *sk, key->rcv_next); arg.iov[0].iov_len += tcp_ao_len_aligned(key->ao_key); rep.th.doff = arg.iov[0].iov_len / 4; + memset((u8 *)&rep.opt[offset] + tcp_ao_maclen(key->ao_key), + TCPOPT_NOP, tcp_ao_len_aligned(key->ao_key) - + tcp_ao_len(key->ao_key)); tcp_ao_hash_hdr(AF_INET, (char *)&rep.opt[offset], key->ao_key, key->traffic_key, @@ -1814,8 +1816,8 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, #endif tcp_ca_openreq_child(newsk, dst); - tcp_sync_mss(newsk, dst_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + tcp_sync_mss(newsk, dst4_mtu(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); @@ -2452,8 +2454,10 @@ int tcp_v4_rcv(struct sk_buff *skb) } drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb); - if (drop_reason) - break; + if (drop_reason) { + inet_twsk_put(inet_twsk(sk)); + goto discard_it; + } } /* to ACK */ fallthrough; @@ -2924,13 +2928,13 @@ static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i) icsk_pending == ICSK_TIME_REO_TIMEOUT || icsk_pending == ICSK_TIME_LOSS_PROBE) { timer_active = 1; - timer_expires = icsk_timeout(icsk); + timer_expires = tcp_timeout_expires(sk); } else if (icsk_pending == ICSK_TIME_PROBE0) { timer_active = 4; - timer_expires = icsk_timeout(icsk); - } else if (timer_pending(&sk->sk_timer)) { + timer_expires = tcp_timeout_expires(sk); + } else if (timer_pending(&icsk->icsk_keepalive_timer)) { timer_active = 2; - timer_expires = sk->sk_timer.expires; + timer_expires = icsk->icsk_keepalive_timer.expires; } else { timer_active = 0; timer_expires = jiffies; @@ -3214,24 +3218,24 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, { struct bpf_tcp_iter_state *iter = seq->private; struct hlist_nulls_node *node; - unsigned int expected = 1; - struct sock *sk; + struct sock *sk = *start_sk; + unsigned int expected = 0; - sock_hold(*start_sk); - iter->batch[iter->end_sk++].sk = *start_sk; - - sk = sk_nulls_next(*start_sk); *start_sk = NULL; sk_nulls_for_each_from(sk, node) { - if (seq_sk_match(seq, sk)) { - if (iter->end_sk < iter->max_sk) { - sock_hold(sk); - iter->batch[iter->end_sk++].sk = sk; - } else if (!*start_sk) { - /* Remember where we left off. */ - *start_sk = sk; - } - expected++; + if (!seq_sk_match(seq, sk)) + continue; + expected++; + if (iter->end_sk < iter->max_sk) { + /* reqsk_queue_hash_req() inserts with sk_refcnt == 0 + * and refcount_set()s it after the bucket lock drops. + */ + if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + continue; + iter->batch[iter->end_sk++].sk = sk; + } else if (!*start_sk) { + /* Remember where we left off. */ + *start_sk = sk; } } @@ -3269,12 +3273,13 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) struct sock *sk; int err; +again: sk = bpf_iter_tcp_resume(seq); if (!sk) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was too small. */ @@ -3282,15 +3287,18 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) bpf_iter_tcp_put_batch(iter); err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2, GFP_USER); - if (err) + if (err) { + iter->cur_sk = 0; + iter->end_sk = 0; return ERR_PTR(err); + } sk = bpf_iter_tcp_resume(seq); if (!sk) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was still too small. Hold onto the lock while we try @@ -3303,10 +3311,14 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) return ERR_PTR(err); } - expected = bpf_iter_fill_batch(seq, &sk); - WARN_ON_ONCE(iter->end_sk != expected); + bpf_iter_fill_batch(seq, &sk); + WARN_ON_ONCE(sk); done: bpf_iter_tcp_unlock_bucket(seq); + if (unlikely(!iter->end_sk)) { + ++iter->state.bucket; + goto again; + } return iter->batch[0].sk; } diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c index c70c29a3a0905..a496c26737fde 100644 --- a/net/ipv4/tcp_minisocks.c +++ b/net/ipv4/tcp_minisocks.c @@ -445,7 +445,7 @@ void tcp_openreq_init_rwin(struct request_sock *req, u32 rcv_wnd; int mss; - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); window_clamp = READ_ONCE(tp->window_clamp); /* Set this up on the first call only */ req->rsk_window_clamp = window_clamp ? : dst_metric(dst, RTAX_WINDOW); @@ -822,7 +822,7 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, * elsewhere and is checked directly against the child socket rather * than req because user data may have been sent out. */ - if ((flg & TCP_FLAG_ACK) && !fastopen && + if ((flg & TCP_FLAG_ACK) && !(flg & TCP_FLAG_RST) && !fastopen && (TCP_SKB_CB(skb)->ack_seq != tcp_rsk(req)->snt_isn + 1)) return sk; @@ -861,6 +861,16 @@ struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb, flg &= ~TCP_FLAG_SYN; } + /* RFC 5961 section 3.2, as clarified by RFC 9293 section + * 3.10.7.4, requires a challenge ACK for a non-exact + * in-window RST in SYN-RECEIVED. + */ + if ((flg & TCP_FLAG_RST) && + TCP_SKB_CB(skb)->seq != tcp_rsk(req)->rcv_nxt) { + tcp_reqsk_send_challenge_ack(sk, skb, req); + return NULL; + } + /* RFC793: "second check the RST bit" and * "fourth, check the SYN bit" */ diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index ed4bc3209c973..3ae6b4e0763c0 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -360,6 +360,7 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb, flush |= skb->ip_summed != p->ip_summed; flush |= skb->csum_level != p->csum_level; flush |= NAPI_GRO_CB(p)->count >= 64; + flush |= NAPI_GRO_CB(p)->is_flist != NAPI_GRO_CB(skb)->is_flist; skb_set_network_header(skb, skb_gro_receive_network_offset(skb)); if (flush || skb_gro_receive_list(p, skb)) @@ -423,12 +424,20 @@ static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, struct net *net; int iif, sdif; - if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST))) - return; - p = tcp_gro_lookup(head, th); if (p) { - NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + /* flist GRO applies to consecutive non-GSO skbs */ + if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) { + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + return; + } + + /* Fall back to the regular GRO path */ + if (NAPI_GRO_CB(p)->count == 1) + NAPI_GRO_CB(p)->is_flist = 0; + + NAPI_GRO_CB(skb)->is_flist = 0; + return; } @@ -438,7 +447,7 @@ static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, sk = __inet_lookup_established(net, iph->saddr, th->source, iph->daddr, ntohs(th->dest), iif, sdif); - NAPI_GRO_CB(skb)->is_flist = !sk; + NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb); if (sk) sock_gen_put(sk); } @@ -458,7 +467,8 @@ struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb) if (!th) goto flush; - tcp4_check_fraglist_gro(head, skb, th); + if (unlikely(skb->dev->features & NETIF_F_GRO_FRAGLIST)) + tcp4_check_fraglist_gro(head, skb, th); return tcp_gro_receive(head, skb, th); diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 479be2f242627..0cd46896e7983 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -129,7 +129,7 @@ static __u16 tcp_advertise_mss(struct sock *sk) int mss = tp->advmss; if (dst) { - unsigned int metric = dst_metric_advmss(dst); + unsigned int metric = tcp_dst_advmss(dst); if (metric < mss) { mss = metric; @@ -3519,7 +3519,7 @@ int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs) avail_wnd = cur_mss; } - len = cur_mss * segs; + len = cur_mss * (tcp_urg_mode(tp) ? 1 : segs); if (len > avail_wnd) { len = rounddown(avail_wnd, cur_mss); if (!len) @@ -3784,9 +3784,9 @@ void tcp_send_fin(struct sock *sk) * was unread data in the receive queue. This behavior is recommended * by RFC 2525, section 2.17. -DaveM */ -void tcp_send_active_reset(struct sock *sk, gfp_t priority, - enum sk_rst_reason reason) +void tcp_send_active_reset(struct sock *sk, enum sk_rst_reason reason) { + gfp_t priority = sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN); struct sk_buff *skb; TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS); @@ -3906,7 +3906,7 @@ struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst, } skb_dst_set(skb, dst); - mss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + mss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); memset(&opts, 0, sizeof(opts)); now = tcp_clock_ns(); @@ -4062,7 +4062,7 @@ static void tcp_connect_init(struct sock *sk) if (!tp->window_clamp) WRITE_ONCE(tp->window_clamp, dst_metric(dst, RTAX_WINDOW)); - tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst)); + tp->advmss = tcp_mss_clamp(tp, tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(sk); diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c index 1e6d7d90371a9..eb549fea8fdd9 100644 --- a/net/ipv4/tcp_timer.c +++ b/net/ipv4/tcp_timer.c @@ -125,7 +125,7 @@ static int tcp_out_of_resources(struct sock *sk, bool do_reset) (!tp->snd_wnd && !tp->packets_out)) do_reset = true; if (do_reset) - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_ABORT_ON_MEMORY); tcp_done(sk); __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONMEMORY); @@ -333,7 +333,9 @@ void tcp_delack_timer_handler(struct sock *sk) if (inet_csk_ack_scheduled(sk)) { if (!inet_csk_in_pingpong_mode(sk)) { /* Delayed ACK missed: inflate ATO. */ - icsk->icsk_ack.ato = min_t(u32, icsk->icsk_ack.ato << 1, icsk->icsk_rto); + icsk->icsk_ack.ato = min3((u32)icsk->icsk_ack.ato << 1, + icsk->icsk_rto, + (u32)TCP_DELACK_MAX); } else { /* Delayed ACK missed: leave pingpong mode and * deflate ATO. @@ -511,7 +513,7 @@ static bool tcp_rtx_probe0_timed_out(const struct sock *sk, * and tp->rcv_tstamp might very well have been written recently. * rcv_delta can thus be negative. */ - rcv_delta = icsk_timeout(icsk) - tp->rcv_tstamp; + rcv_delta = tcp_timeout_expires(sk) - tp->rcv_tstamp; if (rcv_delta <= timeout) return false; @@ -698,9 +700,9 @@ void tcp_write_timer_handler(struct sock *sk) !icsk->icsk_pending) return; - if (time_after(icsk_timeout(icsk), jiffies)) { - sk_reset_timer(sk, &icsk->icsk_retransmit_timer, - icsk_timeout(icsk)); + if (time_after(tcp_timeout_expires(sk), jiffies)) { + sk_reset_timer(sk, &sk->tcp_retransmit_timer, + tcp_timeout_expires(sk)); return; } tcp_mstamp_refresh(tcp_sk(sk)); @@ -726,12 +728,10 @@ void tcp_write_timer_handler(struct sock *sk) static void tcp_write_timer(struct timer_list *t) { - struct inet_connection_sock *icsk = - timer_container_of(icsk, t, icsk_retransmit_timer); - struct sock *sk = &icsk->icsk_inet.sk; + struct sock *sk = timer_container_of(sk, t, tcp_retransmit_timer); /* Avoid locking the socket when there is no pending event. */ - if (!smp_load_acquire(&icsk->icsk_pending)) + if (!smp_load_acquire(&inet_csk(sk)->icsk_pending)) goto out; bh_lock_sock(sk); @@ -757,12 +757,12 @@ EXPORT_IPV6_MOD(tcp_syn_ack_timeout); void tcp_reset_keepalive_timer(struct sock *sk, unsigned long len) { - sk_reset_timer(sk, &sk->sk_timer, jiffies + len); + sk_reset_timer(sk, &inet_csk(sk)->icsk_keepalive_timer, jiffies + len); } static void tcp_delete_keepalive_timer(struct sock *sk) { - sk_stop_timer(sk, &sk->sk_timer); + sk_stop_timer(sk, &inet_csk(sk)->icsk_keepalive_timer); } void tcp_set_keepalive(struct sock *sk, int val) @@ -779,8 +779,9 @@ EXPORT_IPV6_MOD_GPL(tcp_set_keepalive); static void tcp_keepalive_timer(struct timer_list *t) { - struct sock *sk = timer_container_of(sk, t, sk_timer); - struct inet_connection_sock *icsk = inet_csk(sk); + struct inet_connection_sock *icsk = + timer_container_of(icsk, t, icsk_keepalive_timer); + struct sock *sk = &icsk->icsk_inet.sk; struct tcp_sock *tp = tcp_sk(sk); u32 elapsed; @@ -807,7 +808,7 @@ static void tcp_keepalive_timer(struct timer_list *t) goto out; } } - tcp_send_active_reset(sk, GFP_ATOMIC, SK_RST_REASON_TCP_STATE); + tcp_send_active_reset(sk, SK_RST_REASON_TCP_STATE); goto death; } @@ -834,7 +835,7 @@ static void tcp_keepalive_timer(struct timer_list *t) icsk->icsk_probes_out > 0) || (user_timeout == 0 && icsk->icsk_probes_out >= keepalive_probes(tp))) { - tcp_send_active_reset(sk, GFP_ATOMIC, + tcp_send_active_reset(sk, SK_RST_REASON_TCP_KEEPALIVE_TIMEOUT); tcp_write_err(sk); goto out; diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 2f5c91ba0ce92..ea7585f6ec32a 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -2200,10 +2200,10 @@ int __udp_disconnect(struct sock *sk, int flags) */ sk->sk_state = TCP_CLOSE; - inet->inet_daddr = 0; + WRITE_ONCE(inet->inet_daddr, 0); inet->inet_dport = 0; sock_rps_reset_rxhash(sk); - sk->sk_bound_dev_if = 0; + WRITE_ONCE(sk->sk_bound_dev_if, 0); if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) { inet_reset_saddr(sk); if (sk->sk_prot->rehash && diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 589456bd8b5f1..4f7ae02df8d3a 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -178,17 +178,19 @@ static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb, int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb); bool remcsum, need_csum, offload_csum, gso_partial; struct sk_buff *segs = ERR_PTR(-EINVAL); - struct udphdr *uh = udp_hdr(skb); u16 mac_offset = skb->mac_header; __be16 protocol = skb->protocol; u16 mac_len = skb->mac_len; int udp_offset, outer_hlen; + struct udphdr *uh; __wsum partial; bool need_ipsec; if (unlikely(!pskb_may_pull(skb, tnl_hlen))) goto out; + uh = udp_hdr(skb); + /* Adjust partial header checksum to negate old length. * We cannot rely on the value contained in uh->len as it is * possible that the actual value exceeds the boundaries of the diff --git a/net/ipv4/udp_tunnel_nic.c b/net/ipv4/udp_tunnel_nic.c index b13e8f7092f46..4360d159ee8a7 100644 --- a/net/ipv4/udp_tunnel_nic.c +++ b/net/ipv4/udp_tunnel_nic.c @@ -32,13 +32,12 @@ struct udp_tunnel_nic_table_entry { * @lock: protects all fields * @need_sync: at least one port start changed * @need_replay: space was freed, we need a replay of all ports - * @work_pending: @work is currently scheduled * @n_tables: number of tables under @entries * @missed: bitmap of tables which overflown * @entries: table of tables of ports currently offloaded */ struct udp_tunnel_nic { - struct work_struct work; + struct delayed_work work; struct net_device *dev; @@ -46,7 +45,6 @@ struct udp_tunnel_nic { u8 need_sync:1; u8 need_replay:1; - u8 work_pending:1; unsigned int n_tables; unsigned long missed; @@ -301,11 +299,10 @@ __udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync(struct net_device *dev, struct udp_tunnel_nic *utn) { - if (!utn->need_sync || utn->work_pending) + if (!utn->need_sync) return; - queue_work(udp_tunnel_nic_workqueue, &utn->work); - utn->work_pending = 1; + queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 0); } static bool @@ -731,12 +728,17 @@ udp_tunnel_nic_replay(struct net_device *dev, struct udp_tunnel_nic *utn) static void udp_tunnel_nic_device_sync_work(struct work_struct *work) { struct udp_tunnel_nic *utn = - container_of(work, struct udp_tunnel_nic, work); + container_of(work, struct udp_tunnel_nic, work.work); - rtnl_lock(); + /* We cannot block on RTNL here, otherwise we would deadlock with + * udp_tunnel_nic_unregister() calling cancel_delayed_work_sync() + * while holding RTNL. Requeue with 1 jiffy delay if RTNL is contended. + */ + if (!rtnl_trylock()) { + queue_delayed_work(udp_tunnel_nic_workqueue, &utn->work, 1); + return; + } mutex_lock(&utn->lock); - - utn->work_pending = 0; __udp_tunnel_nic_device_sync(utn->dev, utn); if (utn->need_replay) @@ -757,7 +759,7 @@ udp_tunnel_nic_alloc(const struct udp_tunnel_nic_info *info, if (!utn) return NULL; utn->n_tables = n_tables; - INIT_WORK(&utn->work, udp_tunnel_nic_device_sync_work); + INIT_DELAYED_WORK(&utn->work, udp_tunnel_nic_device_sync_work); mutex_init(&utn->lock); for (i = 0; i < n_tables; i++) { @@ -901,11 +903,11 @@ udp_tunnel_nic_unregister(struct net_device *dev, struct udp_tunnel_nic *utn) udp_tunnel_nic_flush(dev, utn); udp_tunnel_nic_unlock(dev); - /* Wait for the work to be done using the state, netdev core will - * retry unregister until we give up our reference on this device. + /* Make sure no work is running or queued before freeing @utn. + * The work handler uses rtnl_trylock(), so it will not deadlock + * against the RTNL we are holding here. */ - if (utn->work_pending) - return; + cancel_delayed_work_sync(&utn->work); udp_tunnel_nic_free(utn); release_dev: diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index c98b1b919f18c..80706368a303c 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -1180,6 +1180,7 @@ ipv6_add_addr(struct inet6_dev *idev, struct ifa6_config *cfg, ipv6_link_dev_addr(idev, ifa); if (ifa->flags&IFA_F_TEMPORARY) { + /* manage_tempaddrs() relies on addresses being added to the head */ list_add(&ifa->tmp_list, &idev->tempaddr_list); in6_ifa_hold(ifa); } @@ -2610,8 +2611,10 @@ static void manage_tempaddrs(struct inet6_dev *idev, __u32 valid_lft, __u32 prefered_lft, bool create, unsigned long now) { - u32 flags; + u32 orig_prefered_lft = prefered_lft; struct inet6_ifaddr *ift; + bool reset_done = false; + u32 flags; read_lock_bh(&idev->lock); /* update all temporary addresses in the list */ @@ -2646,6 +2649,11 @@ static void manage_tempaddrs(struct inet6_dev *idev, prefered_lft = max_prefered; spin_lock(&ift->lock); + /* the first match is the most recent temp address */ + if (!reset_done && orig_prefered_lft > 0) { + ift->regen_count = 0; + reset_done = true; + } flags = ift->flags; ift->valid_lft = valid_lft; ift->prefered_lft = prefered_lft; diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index f8e1dc4f3227f..1c45e1d15506c 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -226,7 +226,7 @@ static int inet6_create(struct net *net, struct socket *sock, int protocol, np->pmtudisc = IPV6_PMTUDISC_WANT; inet6_assign_bit(REPFLOW, sk, READ_ONCE(net->ipv6.sysctl.flowlabel_reflect) & FLOWLABEL_REFLECT_ESTABLISHED); - sk->sk_ipv6only = net->ipv6.sysctl.bindv6only; + sk->sk_ipv6only = READ_ONCE(net->ipv6.sysctl.bindv6only); sk->sk_txrehash = READ_ONCE(net->core.sysctl_txrehash); /* Init the ipv4 part of the socket since we can have sockets diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index bf4e11614af25..188397b9a9db0 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -232,26 +232,28 @@ static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *des * Rearrange the destination address in @iph and the addresses in @rthdr * so that they appear in the order they will at the final destination. * See Appendix A2 of RFC 2402 for details. + * + * Return: 0 on success, -EINVAL if segments_left exceeds the number of + * addresses described by hdrlen. */ -static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) +static int ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) { - int segments, segments_left; + unsigned int segments, segments_left; struct in6_addr *addrs; struct in6_addr final_addr; segments_left = rthdr->segments_left; if (segments_left == 0) - return; - rthdr->segments_left = 0; + return 0; - /* The value of rthdr->hdrlen has been verified either by the system - * call if it is locally generated, or by ipv6_rthdr_rcv() for incoming - * packets. So we can assume that it is even and that segments is - * greater than or equal to segments_left. - * - * For the same reason we can assume that this option is of type 0. + /* Raw locally generated packets can reach AH6 without the invariant + * required by the rt0-style address rearrangement below. */ segments = rthdr->hdrlen >> 1; + if (segments_left > segments) + return -EINVAL; + + rthdr->segments_left = 0; addrs = ((struct rt0_hdr *)rthdr)->addr; final_addr = addrs[segments - 1]; @@ -261,6 +263,8 @@ static void ipv6_rearrange_rthdr(struct ipv6hdr *iph, struct ipv6_rt_hdr *rthdr) addrs[0] = iph->daddr; iph->daddr = final_addr; + + return 0; } static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) @@ -273,6 +277,7 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) } exthdr = { .iph = iph }; char *end = exthdr.raw + len; int nexthdr = iph->nexthdr; + int err; exthdr.iph++; @@ -292,7 +297,9 @@ static int ipv6_clear_mutable_options(struct ipv6hdr *iph, int len, int dir) break; case NEXTHDR_ROUTING: - ipv6_rearrange_rthdr(iph, exthdr.rth); + err = ipv6_rearrange_rthdr(iph, exthdr.rth); + if (err) + return err; break; default: diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index fbc0df2daed32..1cd9e800746bd 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -371,23 +371,16 @@ static void seg6_update_csum(struct sk_buff *skb) (__be32 *)addr); } -static int ipv6_srh_rcv(struct sk_buff *skb) +static int ipv6_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); struct ipv6_sr_hdr *hdr; - struct inet6_dev *idev; struct in6_addr *addr; int accept_seg6; hdr = (struct ipv6_sr_hdr *)skb_transport_header(skb); - idev = __in6_dev_get(skb->dev); - if (!idev) { - kfree_skb(skb); - return -1; - } - accept_seg6 = min(READ_ONCE(net->ipv6.devconf_all->seg6_enabled), READ_ONCE(idev->cnf.seg6_enabled)); @@ -455,7 +448,7 @@ static int ipv6_srh_rcv(struct sk_buff *skb) hdr->segments_left--; addr = hdr->segments + hdr->segments_left; - skb_push(skb, sizeof(struct ipv6hdr)); + skb_push(skb, -skb_network_offset(skb)); if (skb->ip_summed == CHECKSUM_COMPLETE) seg6_update_csum(skb); @@ -479,7 +472,7 @@ static int ipv6_srh_rcv(struct sk_buff *skb) } ipv6_hdr(skb)->hop_limit--; - skb_pull(skb, sizeof(struct ipv6hdr)); + skb_pull(skb, skb_transport_offset(skb)); goto looped_back; } @@ -488,12 +481,11 @@ static int ipv6_srh_rcv(struct sk_buff *skb) return -1; } -static int ipv6_rpl_srh_rcv(struct sk_buff *skb) +static int ipv6_rpl_srh_rcv(struct sk_buff *skb, struct inet6_dev *idev) { struct ipv6_rpl_sr_hdr *hdr, *ohdr, *chdr; struct inet6_skb_parm *opt = IP6CB(skb); struct net *net = dev_net(skb->dev); - struct inet6_dev *idev; struct ipv6hdr *oldhdr; unsigned int chdr_len; unsigned char *buf; @@ -502,8 +494,6 @@ static int ipv6_rpl_srh_rcv(struct sk_buff *skb) u64 n = 0; u32 r; - idev = __in6_dev_get(skb->dev); - accept_rpl_seg = min(READ_ONCE(net->ipv6.devconf_all->rpl_seg_enabled), READ_ONCE(idev->cnf.rpl_seg_enabled)); if (!accept_rpl_seg) { @@ -692,10 +682,14 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) switch (hdr->type) { case IPV6_SRCRT_TYPE_4: /* segment routing */ - return ipv6_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_srh_rcv(skb, idev); case IPV6_SRCRT_TYPE_3: /* rpl segment routing */ - return ipv6_rpl_srh_rcv(skb); + if (!idev) + goto disabled; + return ipv6_rpl_srh_rcv(skb, idev); default: break; } @@ -840,6 +834,10 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) icmpv6_param_prob(skb, ICMPV6_HDR_FIELD, (&hdr->type) - skb_network_header(skb)); return -1; + +disabled: + kfree_skb_reason(skb, SKB_DROP_REASON_IPV6DISABLED); + return -1; } static const struct inet6_protocol rthdr_protocol = { @@ -1101,9 +1099,9 @@ int ipv6_parse_hopopts(struct sk_buff *skb) * for headers. */ -static void ipv6_push_rthdr0(struct sk_buff *skb, u8 *proto, - struct ipv6_rt_hdr *opt, - struct in6_addr **addr_p, struct in6_addr *saddr) +static u8 ipv6_push_rthdr0(struct sk_buff *skb, u8 proto, + struct ipv6_rt_hdr *opt, + struct in6_addr **addr_p, struct in6_addr *saddr) { struct rt0_hdr *phdr, *ihdr; int hops; @@ -1122,13 +1120,13 @@ static void ipv6_push_rthdr0(struct sk_buff *skb, u8 *proto, phdr->addr[hops - 1] = **addr_p; *addr_p = ihdr->addr; - phdr->rt_hdr.nexthdr = *proto; - *proto = NEXTHDR_ROUTING; + phdr->rt_hdr.nexthdr = proto; + return NEXTHDR_ROUTING; } -static void ipv6_push_rthdr4(struct sk_buff *skb, u8 *proto, - struct ipv6_rt_hdr *opt, - struct in6_addr **addr_p, struct in6_addr *saddr) +static u8 ipv6_push_rthdr4(struct sk_buff *skb, u8 proto, + struct ipv6_rt_hdr *opt, + struct in6_addr **addr_p, struct in6_addr *saddr) { struct ipv6_sr_hdr *sr_phdr, *sr_ihdr; int plen, hops; @@ -1171,58 +1169,61 @@ static void ipv6_push_rthdr4(struct sk_buff *skb, u8 *proto, } #endif - sr_phdr->nexthdr = *proto; - *proto = NEXTHDR_ROUTING; + sr_phdr->nexthdr = proto; + return NEXTHDR_ROUTING; } -static void ipv6_push_rthdr(struct sk_buff *skb, u8 *proto, - struct ipv6_rt_hdr *opt, - struct in6_addr **addr_p, struct in6_addr *saddr) +static u8 ipv6_push_rthdr(struct sk_buff *skb, u8 proto, + struct ipv6_rt_hdr *opt, + struct in6_addr **addr_p, struct in6_addr *saddr) { switch (opt->type) { case IPV6_SRCRT_TYPE_0: case IPV6_SRCRT_STRICT: case IPV6_SRCRT_TYPE_2: - ipv6_push_rthdr0(skb, proto, opt, addr_p, saddr); + proto = ipv6_push_rthdr0(skb, proto, opt, addr_p, saddr); break; case IPV6_SRCRT_TYPE_4: - ipv6_push_rthdr4(skb, proto, opt, addr_p, saddr); + proto = ipv6_push_rthdr4(skb, proto, opt, addr_p, saddr); break; default: break; } + return proto; } -static void ipv6_push_exthdr(struct sk_buff *skb, u8 *proto, u8 type, struct ipv6_opt_hdr *opt) +static u8 ipv6_push_exthdr(struct sk_buff *skb, u8 proto, u8 type, struct ipv6_opt_hdr *opt) { struct ipv6_opt_hdr *h = skb_push(skb, ipv6_optlen(opt)); memcpy(h, opt, ipv6_optlen(opt)); - h->nexthdr = *proto; - *proto = type; + h->nexthdr = proto; + return type; } -void ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, - u8 *proto, - struct in6_addr **daddr, struct in6_addr *saddr) +u8 ipv6_push_nfrag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, + u8 proto, + struct in6_addr **daddr, struct in6_addr *saddr) { if (opt->srcrt) { - ipv6_push_rthdr(skb, proto, opt->srcrt, daddr, saddr); + proto = ipv6_push_rthdr(skb, proto, opt->srcrt, daddr, saddr); /* * IPV6_RTHDRDSTOPTS is ignored * unless IPV6_RTHDR is set (RFC3542). */ if (opt->dst0opt) - ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst0opt); + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst0opt); } if (opt->hopopt) - ipv6_push_exthdr(skb, proto, NEXTHDR_HOP, opt->hopopt); + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_HOP, opt->hopopt); + return proto; } -void ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, u8 *proto) +u8 ipv6_push_frag_opts(struct sk_buff *skb, struct ipv6_txoptions *opt, u8 proto) { if (opt->dst1opt) - ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst1opt); + proto = ipv6_push_exthdr(skb, proto, NEXTHDR_DEST, opt->dst1opt); + return proto; } EXPORT_SYMBOL(ipv6_push_frag_opts); diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c index fd5f7112a51ff..3024e77b3847b 100644 --- a/net/ipv6/fib6_rules.c +++ b/net/ipv6/fib6_rules.c @@ -305,6 +305,7 @@ INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule, suppress_route: ip6_rt_put_flags(rt, flags); + res->rt6 = NULL; return true; } diff --git a/net/ipv6/ila/ila_common.c b/net/ipv6/ila/ila_common.c index b8d43ed4689db..ca54a227fc2f6 100644 --- a/net/ipv6/ila/ila_common.c +++ b/net/ipv6/ila/ila_common.c @@ -84,6 +84,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, struct tcphdr *th = (struct tcphdr *) (skb_network_header(skb) + nhoff); + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&th->check, skb, diff, true, true); @@ -95,6 +96,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, (skb_network_header(skb) + nhoff); if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) { + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&uh->check, skb, diff, true, true); @@ -109,6 +111,7 @@ static void ila_csum_adjust_transport(struct sk_buff *skb, struct icmp6hdr *ih = (struct icmp6hdr *) (skb_network_header(skb) + nhoff); + ip6h = ipv6_hdr(skb); diff = get_csum_diff(ip6h, p); inet_proto_csum_replace_by_diff(&ih->icmp6_cksum, skb, diff, true, true); @@ -126,6 +129,15 @@ void ila_update_ipv6_locator(struct sk_buff *skb, struct ila_params *p, switch (p->csum_mode) { case ILA_CSUM_ADJUST_TRANSPORT: ila_csum_adjust_transport(skb, p); + /* + * ila_csum_adjust_transport() calls pskb_may_pull(), which can + * reallocate the skb head and leave ip6h (and the iaddr derived + * from it) dangling; reload both before the write below. The + * other csum modes do not pull, so their cached pointers stay + * valid. + */ + ip6h = ipv6_hdr(skb); + iaddr = ila_a2i(&ip6h->daddr); break; case ILA_CSUM_NEUTRAL_MAP: if (sir2ila) { diff --git a/net/ipv6/ip6_fib.c b/net/ipv6/ip6_fib.c index 940aae3d10be0..75875e4a1edea 100644 --- a/net/ipv6/ip6_fib.c +++ b/net/ipv6/ip6_fib.c @@ -85,7 +85,7 @@ static void fib6_walker_link(struct net *net, struct fib6_walker *w) static void fib6_walker_unlink(struct net *net, struct fib6_walker *w) { write_lock_bh(&net->ipv6.fib6_walker_lock); - list_del(&w->lh); + list_del_init(&w->lh); write_unlock_bh(&net->ipv6.fib6_walker_lock); } @@ -2757,7 +2757,7 @@ static void *ipv6_route_seq_start(struct seq_file *seq, loff_t *pos) static bool ipv6_route_iter_active(struct ipv6_route_iter *iter) { struct fib6_walker *w = &iter->w; - return w->node && !(w->state == FWS_U && w->node == w->root); + return w->node && !list_empty(&w->lh); } static void ipv6_route_native_seq_stop(struct seq_file *seq, void *v) diff --git a/net/ipv6/ip6_flowlabel.c b/net/ipv6/ip6_flowlabel.c index 8aa29b3d3daca..74de0f344f543 100644 --- a/net/ipv6/ip6_flowlabel.c +++ b/net/ipv6/ip6_flowlabel.c @@ -462,6 +462,21 @@ fl_create(struct net *net, struct sock *sk, struct in6_flowlabel_req *freq, return NULL; } +static bool fl_sock_at_lease_limit(const struct sock *sk) +{ + const struct ipv6_fl_socklist *sfl; + int count = 0; + + rcu_read_lock(); + for_each_sk_fl_rcu(sk, sfl) { + if (++count >= FL_MAX_PER_SOCK) + break; + } + rcu_read_unlock(); + + return count >= FL_MAX_PER_SOCK; +} + static int mem_check(struct sock *sk) { int room = FL_MAX_SIZE - atomic_read(&fl_size); @@ -671,6 +686,10 @@ static int ipv6_flowlabel_get(struct sock *sk, struct in6_flowlabel_req *freq, err = -ENOMEM; if (!sfl1) goto release; + err = -ENOBUFS; + if (fl_sock_at_lease_limit(sk) && + !capable(CAP_NET_ADMIN)) + goto release; if (fl->linger > fl1->linger) fl1->linger = fl->linger; if ((long)(fl->expires - fl1->expires) > 0) diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index 75961c4ebcdd2..81792c940d296 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -877,6 +877,7 @@ static int ip6gre_xmit_other(struct sk_buff *skb, struct net_device *dev) static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, struct net_device *dev) { + struct ip_tunnel_info *tun_info = NULL; struct ip6_tnl *t = netdev_priv(dev); __be16 payload_protocol; int ret; @@ -887,6 +888,9 @@ static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, if (!ip6_tnl_xmit_ctl(t, &t->parms.laddr, &t->parms.raddr)) goto tx_err; + if (t->parms.collect_md) + tun_info = skb_tunnel_info_txcheck(skb); + payload_protocol = skb_protocol(skb, true); switch (payload_protocol) { case htons(ETH_P_IP): @@ -906,7 +910,7 @@ static netdev_tx_t ip6gre_tunnel_xmit(struct sk_buff *skb, return NETDEV_TX_OK; tx_err: - if (!t->parms.collect_md || !IS_ERR(skb_tunnel_info_txcheck(skb))) + if (!IS_ERR(tun_info)) DEV_STATS_INC(dev, tx_errors); DEV_STATS_INC(dev, tx_dropped); kfree_skb(skb); @@ -1057,7 +1061,7 @@ static netdev_tx_t ip6erspan_tunnel_xmit(struct sk_buff *skb, /* TooBig packet may have updated dst->dev's mtu */ if (!t->parms.collect_md && dst) { mtu = READ_ONCE(dst_dev(dst)->mtu); - if (dst_mtu(dst) > mtu) + if (dst6_mtu(dst) > mtu) dst->ops->update_pmtu(dst, NULL, skb, mtu, false); } err = ip6_tnl_xmit(skb, dev, dsfield, &fl6, encap_limit, &mtu, @@ -1136,13 +1140,11 @@ static void ip6gre_tnl_link_config_route(struct ip6_tnl *t, int set_mtu, return; if (rt->dst.dev) { - unsigned short dst_len = rt->dst.dev->hard_header_len + - t_hlen; + unsigned int headroom; - if (t->dev->header_ops) - dev->hard_header_len = dst_len; - else - dev->needed_headroom = dst_len; + headroom = rt->dst.dev->hard_header_len + t_hlen; + headroom = ip_tunnel_limit_headroom(headroom); + dev->needed_headroom = headroom; if (set_mtu) { int mtu = rt->dst.dev->mtu - t_hlen; @@ -1170,8 +1172,8 @@ static int ip6gre_calc_hlen(struct ip6_tnl *tunnel) t_hlen = tunnel->hlen + sizeof(struct ipv6hdr); - if (tunnel->dev->header_ops) - tunnel->dev->hard_header_len = LL_MAX_HEADER + t_hlen; + if (tunnel->dev->header_ops && tunnel->dev->type == ARPHRD_IP6GRE) + tunnel->dev->hard_header_len = t_hlen; else tunnel->dev->needed_headroom = LL_MAX_HEADER + t_hlen; @@ -1454,6 +1456,8 @@ static void ip6gre_tnl_init_features(struct net_device *dev) dev->features |= GRE6_FEATURES; dev->hw_features |= GRE6_FEATURES; + dev->lltx = true; + /* TCP offload with GRE SEQ is not supported, nor can we support 2 * levels of outer headers requiring an update. */ @@ -1465,8 +1469,6 @@ static void ip6gre_tnl_init_features(struct net_device *dev) dev->features |= NETIF_F_GSO_SOFTWARE; dev->hw_features |= NETIF_F_GSO_SOFTWARE; - - dev->lltx = true; } static int ip6gre_tunnel_init_common(struct net_device *dev) diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index 168ec07e31cc3..b59b2b119a1cc 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -577,6 +577,7 @@ int ip6_mc_input(struct sk_buff *skb) if (deliver) { skb2 = skb_clone(skb, GFP_ATOMIC); } else { + skb_orphan(skb); skb2 = skb; skb = NULL; } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index 8f37c9cc868b0..7f17e700736a0 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -80,7 +80,7 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff * hdr = ipv6_hdr(skb); daddr = &hdr->daddr; - if (ipv6_addr_is_multicast(daddr)) { + if (unlikely(ipv6_addr_is_multicast(daddr))) { if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) && ((mroute6_is_socket(net, skb) && !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) || @@ -116,6 +116,8 @@ static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff * if (res != LWTUNNEL_XMIT_CONTINUE) return res; + hdr = ipv6_hdr(skb); + daddr = &hdr->daddr; } IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUT, skb->len); @@ -179,8 +181,8 @@ ip6_finish_output_gso_slowpath_drop(struct net *net, struct sock *sk, static int ip6_finish_output_gso(struct net *net, struct sock *sk, struct sk_buff *skb, unsigned int mtu) { - if (!(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) && - !skb_gso_validate_network_len(skb, mtu)) + if (unlikely(!(IP6CB(skb)->flags & IP6SKB_FAKEJUMBO) && + !skb_gso_validate_network_len(skb, mtu))) return ip6_finish_output_gso_slowpath_drop(net, sk, skb, mtu); return ip6_finish_output2(net, sk, skb); @@ -202,8 +204,8 @@ static int __ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff if (skb_is_gso(skb)) return ip6_finish_output_gso(net, sk, skb, mtu); - if (skb->len > mtu || - (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size)) + if (unlikely(skb->len > mtu || + (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))) return ip6_fragment(net, sk, skb, ip6_finish_output2); return ip6_finish_output2(net, sk, skb); @@ -301,15 +303,16 @@ int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, } } - if (opt) { + if (unlikely(opt)) { seg_len += opt->opt_nflen + opt->opt_flen; if (opt->opt_flen) - ipv6_push_frag_opts(skb, opt, &proto); + proto = ipv6_push_frag_opts(skb, opt, proto); if (opt->opt_nflen) - ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop, - &fl6->saddr); + proto = ipv6_push_nfrag_opts(skb, opt, proto, + &first_hop, + &fl6->saddr); } if (unlikely(seg_len > IPV6_MAXPLEN)) { @@ -352,8 +355,8 @@ int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6, skb->priority = priority; skb->mark = mark; - mtu = dst_mtu(dst); - if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) { + mtu = dst6_mtu(dst); + if (likely((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb))) { IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTREQUESTS); /* if egress device is enslaved to an L3 master device pass the @@ -630,7 +633,7 @@ int ip6_forward(struct sk_buff *skb) /* Limit redirects both by destination (here) and by source (inside ndisc_send_redirect) */ - if (inet_peer_xrlim_allow(peer, 1*HZ)) + if (peer && inet_peer_xrlim_allow(peer, 1*HZ)) ndisc_send_redirect(skb, target); rcu_read_unlock(); } else { @@ -653,7 +656,7 @@ int ip6_forward(struct sk_buff *skb) if (mtu < IPV6_MIN_MTU) mtu = IPV6_MIN_MTU; - if (ip6_pkt_too_big(skb, mtu)) { + if (unlikely(ip6_pkt_too_big(skb, mtu))) { /* Again, force OUTPUT device used as source address */ skb->dev = dev; icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); @@ -1367,7 +1370,7 @@ static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, /* * setup for corking */ - if (opt) { + if (unlikely(opt)) { if (WARN_ON(v6_cork->opt)) return -EINVAL; @@ -1402,15 +1405,17 @@ static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork, v6_cork->dontfrag = ipc6->dontfrag; if (rt->dst.flags & DST_XFRM_TUNNEL) mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? - READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst); + READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(&rt->dst); else mtu = READ_ONCE(np->pmtudisc) >= IPV6_PMTUDISC_PROBE ? - READ_ONCE(rt->dst.dev->mtu) : dst_mtu(xfrm_dst_path(&rt->dst)); + READ_ONCE(rt->dst.dev->mtu) : dst6_mtu(xfrm_dst_path(&rt->dst)); frag_size = READ_ONCE(np->frag_size); if (frag_size && frag_size < mtu) mtu = frag_size; + if (sk_is_udp(sk)) + mtu = min(mtu, IP6_MAX_MTU); cork->base.fragsize = mtu; cork->base.gso_size = ipc6->gso_size; cork->base.tx_flags = 0; @@ -1883,7 +1888,7 @@ static void ip6_cork_steal_dst(struct sk_buff *skb, struct inet_cork_full *cork) static void ip6_cork_release(struct inet_cork_full *cork, struct inet6_cork *v6_cork) { - if (v6_cork->opt) { + if (unlikely(v6_cork->opt)) { struct ipv6_txoptions *opt = v6_cork->opt; kfree(opt->dst0opt); @@ -1939,11 +1944,13 @@ struct sk_buff *__ip6_make_skb(struct sock *sk, __skb_pull(skb, skb_network_header_len(skb)); final_dst = &fl6->daddr; - if (opt && opt->opt_flen) - ipv6_push_frag_opts(skb, opt, &proto); - if (opt && opt->opt_nflen) - ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst, &fl6->saddr); - + if (unlikely(opt)) { + if (opt->opt_flen) + proto = ipv6_push_frag_opts(skb, opt, proto); + if (opt->opt_nflen) + proto = ipv6_push_nfrag_opts(skb, opt, proto, + &final_dst, &fl6->saddr); + } skb_push(skb, sizeof(struct ipv6hdr)); skb_reset_network_header(skb); hdr = ipv6_hdr(skb); @@ -1965,7 +1972,7 @@ struct sk_buff *__ip6_make_skb(struct sock *sk, ip6_cork_steal_dst(skb, cork); IP6_INC_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUTREQUESTS); - if (proto == IPPROTO_ICMPV6) { + if (unlikely(proto == IPPROTO_ICMPV6)) { struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb)); u8 icmp6_type; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index cbc984a32a29c..2560c0f8b2930 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -643,7 +643,7 @@ ip4ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, /* change mtu on this route */ if (rel_type == ICMP_DEST_UNREACH && rel_code == ICMP_FRAG_NEEDED) { - if (rel_info > dst_mtu(skb_dst(skb2))) + if (rel_info > dst6_mtu(skb_dst(skb2))) goto out; skb_dst_update_pmtu_no_confirm(skb2, rel_info); @@ -677,6 +677,9 @@ ip6ip6_err(struct sk_buff *skb, struct inet6_skb_parm *opt, if (!skb2) return 0; + /* Remove debris left by outer IPv6 stack. */ + memset(IP6CB(skb2), 0, sizeof(*IP6CB(skb2))); + skb_dst_drop(skb2); skb_pull(skb2, offset); skb_reset_network_header(skb2); @@ -1192,7 +1195,7 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, t->parms.name); goto tx_err_dst_release; } - mtu = dst_mtu(dst) - eth_hlen - psh_hlen - t->tun_hlen; + mtu = dst6_mtu(dst) - eth_hlen - psh_hlen - t->tun_hlen; if (encap_limit >= 0) { max_headroom += 8; mtu -= 8; @@ -1225,19 +1228,8 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, */ max_headroom += LL_RESERVED_SPACE(tdev); - if (skb_headroom(skb) < max_headroom || skb_shared(skb) || - (skb_cloned(skb) && !skb_clone_writable(skb, 0))) { - struct sk_buff *new_skb; - - new_skb = skb_realloc_headroom(skb, max_headroom); - if (!new_skb) - goto tx_err_dst_release; - - if (skb->sk) - skb_set_owner_w(new_skb, skb->sk); - consume_skb(skb); - skb = new_skb; - } + if (skb_cow_head(skb, max_headroom)) + goto tx_err_dst_release; if (t->parms.collect_md) { if (t->encap.type != TUNNEL_ENCAP_NONE) @@ -1270,7 +1262,7 @@ int ip6_tnl_xmit(struct sk_buff *skb, struct net_device *dev, __u8 dsfield, if (encap_limit >= 0) { init_tel_txopt(&opt, encap_limit); - ipv6_push_frag_opts(skb, &opt.ops, &proto); + proto = ipv6_push_frag_opts(skb, &opt.ops, proto); } skb_push(skb, sizeof(struct ipv6hdr)); @@ -1514,8 +1506,11 @@ static void ip6_tnl_link_config(struct ip6_tnl *t) tdev = __dev_get_by_index(t->net, p->link); if (tdev) { - dev->needed_headroom = tdev->hard_header_len + - tdev->needed_headroom + t_hlen; + unsigned int headroom; + + headroom = tdev->hard_header_len + tdev->needed_headroom; + headroom += t_hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(headroom); mtu = min_t(unsigned int, tdev->mtu, IP6_MAX_MTU); mtu = mtu - t_hlen; diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index e047a4680ab0e..d59d31ab960a5 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -1063,10 +1063,14 @@ static void ip6mr_cache_resolve(struct net *net, struct mr_table *mrt, static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, mifi_t mifi, int assert) { + enum skb_drop_reason reason; struct sock *mroute6_sk; struct sk_buff *skb; struct mrt6msg *msg; - int ret; + + mroute6_sk = rcu_dereference(mrt->mroute_sk); + if (!mroute6_sk) + return -EINVAL; #ifdef CONFIG_IPV6_PIMSM_V2 if (assert == MRT6MSG_WHOLEPKT || assert == MRT6MSG_WRMIFWHOLE) @@ -1136,23 +1140,17 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt, skb->ip_summed = CHECKSUM_UNNECESSARY; } - mroute6_sk = rcu_dereference(mrt->mroute_sk); - if (!mroute6_sk) { - kfree_skb(skb); - return -EINVAL; - } - mrt6msg_netlink_event(mrt, skb); /* Deliver to user space multicast routing algorithms */ - ret = sock_queue_rcv_skb(mroute6_sk, skb); + reason = sock_queue_rcv_skb_reason(mroute6_sk, skb); - if (ret < 0) { - net_warn_ratelimited("mroute6: pending queue full, dropping entries\n"); - kfree_skb(skb); + if (reason) { + sk_skb_reason_drop(mroute6_sk, skb, reason); + return -ENOMEM; } - return ret; + return 0; } /* Queue a packet for resolution. It gets locked cache entry! */ @@ -1395,7 +1393,7 @@ int __init ip6_mr_init(void) { int err; - mrt_cachep = KMEM_CACHE(mfc6_cache, SLAB_HWCACHE_ALIGN); + mrt_cachep = KMEM_CACHE(mfc6_cache, SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT); if (!mrt_cachep) return -ENOMEM; diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index a61e742794f9e..d784a8644ff20 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -1184,7 +1184,7 @@ int do_ipv6_getsockopt(struct sock *sk, int level, int optname, rcu_read_lock(); dst = __sk_dst_get(sk); if (dst) - val = dst_mtu(dst); + val = dst6_mtu(dst); rcu_read_unlock(); if (!val) return -ENOTCONN; @@ -1283,7 +1283,7 @@ int do_ipv6_getsockopt(struct sock *sk, int level, int optname, rcu_read_lock(); dst = __sk_dst_get(sk); if (dst) - mtuinfo.ip6m_mtu = dst_mtu(dst); + mtuinfo.ip6m_mtu = dst6_mtu(dst); rcu_read_unlock(); if (!mtuinfo.ip6m_mtu) return -ENOTCONN; diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c index c63f97c79c49b..c560f3e3b11fa 100644 --- a/net/ipv6/mcast.c +++ b/net/ipv6/mcast.c @@ -355,12 +355,12 @@ int ip6_mc_source(int add, int omode, struct sock *sk, { struct ipv6_pinfo *inet6 = inet6_sk(sk); struct in6_addr *source, *group; + struct ip6_sf_socklist *newpsl, *psl; struct net *net = sock_net(sk); struct ipv6_mc_socklist *pmc; - struct ip6_sf_socklist *psl; struct inet6_dev *idev; int leavegroup = 0; - int i, j, rv; + int i, j; int err; source = &((struct sockaddr_in6 *)&pgsr->gsr_source)->sin6_addr; @@ -409,13 +409,11 @@ int ip6_mc_source(int add, int omode, struct sock *sk, if (!add) { if (!psl) goto done; /* err = -EADDRNOTAVAIL */ - rv = !0; for (i = 0; i < psl->sl_count; i++) { - rv = !ipv6_addr_equal(&psl->sl_addr[i], source); - if (rv == 0) + if (ipv6_addr_equal(&psl->sl_addr[i], source)) break; } - if (rv) /* source not found */ + if (i == psl->sl_count) /* source not found */ goto done; /* err = -EADDRNOTAVAIL */ /* special case - (INCLUDE, empty) == LEAVE_GROUP */ @@ -424,58 +422,74 @@ int ip6_mc_source(int add, int omode, struct sock *sk, goto done; } + atomic_sub(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + + if (psl->sl_count == 1) { + newpsl = NULL; + } else { + newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, + psl->sl_count - 1), + GFP_KERNEL); + if (!newpsl) { + atomic_add(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + err = -ENOBUFS; + goto done; + } + newpsl->sl_max = psl->sl_count - 1; + newpsl->sl_count = psl->sl_count - 1; + for (j = 0; j < i; j++) + newpsl->sl_addr[j] = psl->sl_addr[j]; + for (j = i + 1; j < psl->sl_count; j++) + newpsl->sl_addr[j - 1] = psl->sl_addr[j]; + } + /* update the interface filter */ ip6_mc_del_src(idev, group, omode, 1, source, 1); - for (j = i+1; j < psl->sl_count; j++) - psl->sl_addr[j-1] = psl->sl_addr[j]; - psl->sl_count--; + rcu_assign_pointer(pmc->sflist, newpsl); + kfree_rcu(psl, rcu); err = 0; goto done; } /* else, add a new source to the filter */ - if (psl && psl->sl_count >= sysctl_mld_max_msf) { + if (psl && psl->sl_count >= READ_ONCE(sysctl_mld_max_msf)) { err = -ENOBUFS; goto done; } - if (!psl || psl->sl_count == psl->sl_max) { - struct ip6_sf_socklist *newpsl; - int count = IP6_SFBLOCK; - - if (psl) - count += psl->sl_max; - newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, count), - GFP_KERNEL); - if (!newpsl) { - err = -ENOBUFS; - goto done; - } - newpsl->sl_max = count; - newpsl->sl_count = count - IP6_SFBLOCK; - if (psl) { - for (i = 0; i < psl->sl_count; i++) - newpsl->sl_addr[i] = psl->sl_addr[i]; - atomic_sub(struct_size(psl, sl_addr, psl->sl_max), - &sk->sk_omem_alloc); + if (psl) { + for (i = 0; i < psl->sl_count; i++) { + if (ipv6_addr_equal(&psl->sl_addr[i], source)) + goto done; /* err = -EADDRNOTAVAIL */ } - rcu_assign_pointer(pmc->sflist, newpsl); - kfree_rcu(psl, rcu); - psl = newpsl; } - rv = 1; /* > 0 for insert logic below if sl_count is 0 */ - for (i = 0; i < psl->sl_count; i++) { - rv = !ipv6_addr_equal(&psl->sl_addr[i], source); - if (rv == 0) /* There is an error in the address. */ - goto done; + + i = psl ? psl->sl_count + 1 : 1; + newpsl = sock_kmalloc(sk, struct_size(newpsl, sl_addr, i), + GFP_KERNEL); + if (!newpsl) { + err = -ENOBUFS; + goto done; } - for (j = psl->sl_count-1; j >= i; j--) - psl->sl_addr[j+1] = psl->sl_addr[j]; - psl->sl_addr[i] = *source; - psl->sl_count++; - err = 0; + newpsl->sl_max = i; + newpsl->sl_count = i; + if (psl) { + for (j = 0; j < psl->sl_count; j++) + newpsl->sl_addr[j] = psl->sl_addr[j]; + } + newpsl->sl_addr[i - 1] = *source; + /* update the interface list */ ip6_mc_add_src(idev, group, omode, 1, source, 1); + + if (psl) + atomic_sub(struct_size(psl, sl_addr, psl->sl_max), + &sk->sk_omem_alloc); + rcu_assign_pointer(pmc->sflist, newpsl); + kfree_rcu(psl, rcu); + err = 0; done: mutex_unlock(&idev->mc_lock); in6_dev_put(idev); @@ -2350,14 +2364,18 @@ static int ip6_mc_del1_src(struct ifmcaddr6 *pmc, int sfmode, if (psf->sf_oldin && !(pmc->mca_flags & MAF_NOREPORT) && !mld_in_v1_mode(idev)) { - psf->sf_crcount = idev->mc_qrv; - rcu_assign_pointer(psf->sf_next, - mc_dereference(pmc->mca_tomb, idev)); - rcu_assign_pointer(pmc->mca_tomb, psf); - rv = 1; - } else { - kfree_rcu(psf, rcu); + struct ip6_sf_list *dpsf = kmalloc_obj(*dpsf); + + if (dpsf) { + *dpsf = *psf; + dpsf->sf_crcount = idev->mc_qrv; + rcu_assign_pointer(dpsf->sf_next, + mc_dereference(pmc->mca_tomb, idev)); + rcu_assign_pointer(pmc->mca_tomb, dpsf); + rv = 1; + } } + kfree_rcu(psf, rcu); } return rv; } diff --git a/net/ipv6/ndisc.c b/net/ipv6/ndisc.c index 9df1fba967c6e..7262356de10d9 100644 --- a/net/ipv6/ndisc.c +++ b/net/ipv6/ndisc.c @@ -1706,6 +1706,8 @@ void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target) } peer = inet_getpeer_v6(net->ipv6.peers, &ipv6_hdr(skb)->saddr); + if (!peer) + goto release; ret = inet_peer_xrlim_allow(peer, 1*HZ); if (!ret) diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c index 3586e636c66bd..7efc2a19662f9 100644 --- a/net/ipv6/netfilter/ip6_tables.c +++ b/net/ipv6/netfilter/ip6_tables.c @@ -647,6 +647,11 @@ check_entry_size_and_hooks(struct ip6t_entry *e, /* Clear counters and comefrom */ e->counters = ((struct xt_counters) { 0, 0 }); e->comefrom = 0; + + /* set F_PROTO, else ip6_packet_match won't do the right thing. */ + if (e->ipv6.proto) + e->ipv6.flags |= IP6T_F_PROTO; + return 0; } diff --git a/net/ipv6/netfilter/nft_dup_ipv6.c b/net/ipv6/netfilter/nft_dup_ipv6.c index 492a811828a71..95ec27b3971c1 100644 --- a/net/ipv6/netfilter/nft_dup_ipv6.c +++ b/net/ipv6/netfilter/nft_dup_ipv6.c @@ -74,7 +74,6 @@ static const struct nft_expr_ops nft_dup_ipv6_ops = { .eval = nft_dup_ipv6_eval, .init = nft_dup_ipv6_init, .dump = nft_dup_ipv6_dump, - .reduce = NFT_REDUCE_READONLY, }; static const struct nla_policy nft_dup_ipv6_policy[NFTA_DUP_MAX + 1] = { diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c index 3005dfbca6155..6c04d0a80b26a 100644 --- a/net/ipv6/netfilter/nft_fib_ipv6.c +++ b/net/ipv6/netfilter/nft_fib_ipv6.c @@ -225,7 +225,6 @@ static const struct nft_expr_ops nft_fib6_type_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops nft_fib6_ops = { @@ -235,7 +234,6 @@ static const struct nft_expr_ops nft_fib6_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static const struct nft_expr_ops * diff --git a/net/ipv6/netfilter/nft_reject_ipv6.c b/net/ipv6/netfilter/nft_reject_ipv6.c index 5c61294f410ee..ed69c768797ec 100644 --- a/net/ipv6/netfilter/nft_reject_ipv6.c +++ b/net/ipv6/netfilter/nft_reject_ipv6.c @@ -46,7 +46,6 @@ static const struct nft_expr_ops nft_reject_ipv6_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_ipv6_type __read_mostly = { diff --git a/net/ipv6/raw.c b/net/ipv6/raw.c index c534848933ee6..b796d84db5ce2 100644 --- a/net/ipv6/raw.c +++ b/net/ipv6/raw.c @@ -348,7 +348,7 @@ void raw6_icmp_error(struct sk_buff *skb, int nexthdr, const struct ipv6hdr *ip6h = (const struct ipv6hdr *)skb->data; if (!raw_v6_match(net, sk, nexthdr, &ip6h->saddr, &ip6h->daddr, - inet6_iif(skb), inet6_iif(skb))) + inet6_iif(skb), inet6_sdif(skb))) continue; rawv6_err(sk, skb, type, code, inner_offset, info); } @@ -368,7 +368,8 @@ static inline int rawv6_rcv_skb(struct sock *sk, struct sk_buff *skb) /* Charge it to the socket. */ skb_dst_drop(skb); - if (sock_queue_rcv_skb_reason(sk, skb, &reason) < 0) { + reason = sock_queue_rcv_skb_reason(sk, skb); + if (reason) { sk_skb_reason_drop(sk, skb, reason); return NET_RX_DROP; } diff --git a/net/ipv6/route.c b/net/ipv6/route.c index ef66a3c86febb..ddaf51eacc2d1 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -987,13 +987,13 @@ int rt6_route_rcv(struct net_device *dev, u8 *opt, int len, } else if (rinfo->prefix_len > 128) { return -EINVAL; } else if (rinfo->prefix_len > 64) { - if (rinfo->length < 2) { + /* RFC 4191: Length MUST be 3 when Prefix Length > 64 */ + if (rinfo->length < 3) return -EINVAL; - } } else if (rinfo->prefix_len > 0) { - if (rinfo->length < 1) { + /* RFC 4191: Length MUST be 2 or 3 when Prefix Length > 0 */ + if (rinfo->length < 2) return -EINVAL; - } } pref = rinfo->route_pref; @@ -2053,6 +2053,8 @@ static void rt6_update_exception_stamp_rt(struct rt6_info *rt) static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, struct rt6_info *rt, int mtu) { + u32 dmtu = dst6_mtu(&rt->dst); + /* If the new MTU is lower than the route PMTU, this new MTU will be the * lowest MTU in the path: always allow updating the route PMTU to * reflect PMTU decreases. @@ -2063,10 +2065,10 @@ static bool rt6_mtu_change_route_allowed(struct inet6_dev *idev, * handle this. */ - if (dst_mtu(&rt->dst) >= mtu) + if (dmtu >= mtu) return true; - if (dst_mtu(&rt->dst) == idev->cnf.mtu6) + if (dmtu == idev->cnf.mtu6) return true; return false; @@ -2936,7 +2938,7 @@ static void __ip6_rt_update_pmtu(struct dst_entry *dst, const struct sock *sk, if (mtu < IPV6_MIN_MTU) return; - if (mtu >= dst_mtu(dst)) + if (mtu >= dst6_mtu(dst)) return; if (!rt6_cache_allowed_for_pmtu(rt6)) { @@ -3252,7 +3254,7 @@ EXPORT_SYMBOL_GPL(ip6_sk_redirect); static unsigned int ip6_default_advmss(const struct dst_entry *dst) { - unsigned int mtu = dst_mtu(dst); + unsigned int mtu = ip6_dst_mtu_configured(dst); struct net *net; mtu -= sizeof(struct ipv6hdr) + sizeof(struct tcphdr); @@ -3999,6 +4001,7 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) struct net *net = info->nl_net; struct sk_buff *skb = NULL; struct fib6_table *table; + struct fib6_node *fn; int err = -ENOENT; if (rt == net->ipv6.fib6_null_entry) @@ -4006,12 +4009,16 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) table = rt->fib6_table; spin_lock_bh(&table->tb6_lock); + fn = rcu_dereference_protected(rt->fib6_node, + lockdep_is_held(&table->tb6_lock)); + if (!fn) + goto out_unlock; + if (rt->fib6_nsiblings && cfg->fc_delete_all_nh) { struct fib6_info *sibling, *next_sibling; - struct fib6_node *fn; /* prefer to send a single notification with all hops */ - skb = nlmsg_new(rt6_nlmsg_size(rt), gfp_any()); + skb = nlmsg_new(rt6_nlmsg_size(rt), GFP_ATOMIC); if (skb) { u32 seq = info->nlh ? info->nlh->nlmsg_seq : 0; @@ -4030,8 +4037,6 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) * and emit a replace or delete notification, respectively. */ info->skip_notify_kernel = 1; - fn = rcu_dereference_protected(rt->fib6_node, - lockdep_is_held(&table->tb6_lock)); if (rcu_access_pointer(fn->leaf) == rt) { struct fib6_info *last_sibling, *replace_rt; @@ -4067,7 +4072,7 @@ static int __ip6_del_rt_siblings(struct fib6_info *rt, struct fib6_config *cfg) if (skb) { rtnl_notify(skb, net, info->portid, RTNLGRP_IPV6_ROUTE, - info->nlh, gfp_any()); + info->nlh, GFP_ATOMIC); } return err; } @@ -4829,7 +4834,7 @@ static void rt6_upper_bound_set(struct fib6_info *rt, int *weight, int total) { int upper_bound = -1; - if (!rt6_is_dead(rt)) { + if (total && !rt6_is_dead(rt)) { *weight += rt->fib6_nh->fib_nh_weight; upper_bound = DIV_ROUND_CLOSEST_ULL((u64) (*weight) << 31, total) - 1; @@ -6017,7 +6022,7 @@ static int rt6_nh_dump_exceptions(struct fib6_nh *nh, void *arg) return 0; for (i = 0; i < FIB6_EXCEPTION_BUCKET_SIZE; i++) { - hlist_for_each_entry(rt6_ex, &bucket->chain, hlist) { + hlist_for_each_entry_rcu(rt6_ex, &bucket->chain, hlist) { if (w->skip) { w->skip--; continue; diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c index 2b41e4c0dddd1..7b52122201858 100644 --- a/net/ipv6/seg6_local.c +++ b/net/ipv6/seg6_local.c @@ -256,6 +256,22 @@ static bool decap_and_validate(struct sk_buff *skb, int proto) if (iptunnel_pull_offloads(skb)) return false; + if (proto == IPPROTO_IPIP) { + int iif = IP6CB(skb)->iif; + + memset(IPCB(skb), 0, sizeof(*IPCB(skb))); + IPCB(skb)->iif = iif; + } else if (proto == IPPROTO_IPV6) { + bool l3slave = ipv6_l3mdev_skb(IP6CB(skb)->flags); + int iif = IP6CB(skb)->iif; + + memset(IP6CB(skb), 0, sizeof(*IP6CB(skb))); + IP6CB(skb)->iif = iif; + IP6CB(skb)->nhoff = offsetof(struct ipv6hdr, nexthdr); + if (l3slave) + IP6CB(skb)->flags |= IP6SKB_L3SLAVE; + } + return true; } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index c03df1682a639..ae39e4c5d463a 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -963,7 +963,7 @@ static netdev_tx_t ipip6_tunnel_xmit(struct sk_buff *skb, iph6 = ipv6_hdr(skb); if (df) { - mtu = dst_mtu(&rt->dst) - t_hlen; + mtu = dst4_mtu(&rt->dst) - t_hlen; if (mtu < IPV4_MIN_MTU) { DEV_STATS_INC(dev, collisions); @@ -1128,7 +1128,7 @@ static void ipip6_tunnel_bind_dev(struct net_device *dev) WRITE_ONCE(dev->mtu, mtu); hlen = tdev->hard_header_len + tdev->needed_headroom; } - dev->needed_headroom = t_hlen + hlen; + dev->needed_headroom = ip_tunnel_limit_headroom(t_hlen + hlen); } static void ipip6_tunnel_update(struct ip_tunnel *t, diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 59b5900dd42bd..aca71fbb6c292 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -351,7 +351,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr, static void tcp_v6_mtu_reduced(struct sock *sk) { struct dst_entry *dst; - u32 mtu; + u32 mtu, dmtu; if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) return; @@ -368,8 +368,9 @@ static void tcp_v6_mtu_reduced(struct sock *sk) if (!dst) return; - if (inet_csk(sk)->icsk_pmtu_cookie > dst_mtu(dst)) { - tcp_sync_mss(sk, dst_mtu(dst)); + dmtu = dst6_mtu(dst); + if (inet_csk(sk)->icsk_pmtu_cookie > dmtu) { + tcp_sync_mss(sk, dmtu); tcp_simple_retransmit(sk); } } @@ -941,6 +942,8 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 (tcp_ao_len(key->ao_key) << 16) | (key->ao_key->sndid << 8) | (key->rcv_next)); + memset((u8 *)topt + tcp_ao_maclen(key->ao_key), TCPOPT_NOP, + tcp_ao_len_aligned(key->ao_key) - tcp_ao_len(key->ao_key)); tcp_ao_hash_hdr(AF_INET6, (char *)topt, key->ao_key, key->traffic_key, @@ -1497,8 +1500,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff * tcp_ca_openreq_child(newsk, dst); - tcp_sync_mss(newsk, dst_mtu(dst)); - newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst)); + tcp_sync_mss(newsk, dst6_mtu(dst)); + newtp->advmss = tcp_mss_clamp(tcp_sk(sk), tcp_dst_advmss(dst)); tcp_initialize_rcv_mss(newsk); @@ -1984,8 +1987,10 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) } drop_reason = psp_twsk_rx_policy_check(inet_twsk(sk), skb); - if (drop_reason) - break; + if (drop_reason) { + inet_twsk_put(inet_twsk(sk)); + goto discard_it; + } } /* to ACK */ fallthrough; @@ -2195,13 +2200,13 @@ static void get_tcp6_sock(struct seq_file *seq, struct sock *sp, int i) icsk_pending == ICSK_TIME_REO_TIMEOUT || icsk_pending == ICSK_TIME_LOSS_PROBE) { timer_active = 1; - timer_expires = icsk_timeout(icsk); + timer_expires = tcp_timeout_expires(sp); } else if (icsk_pending == ICSK_TIME_PROBE0) { timer_active = 4; - timer_expires = icsk_timeout(icsk); - } else if (timer_pending(&sp->sk_timer)) { + timer_expires = tcp_timeout_expires(sp); + } else if (timer_pending(&icsk->icsk_keepalive_timer)) { timer_active = 2; - timer_expires = sp->sk_timer.expires; + timer_expires = icsk->icsk_keepalive_timer.expires; } else { timer_active = 0; timer_expires = jiffies; diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c index 5670d32c27f80..798a63267ff24 100644 --- a/net/ipv6/tcpv6_offload.c +++ b/net/ipv6/tcpv6_offload.c @@ -29,7 +29,18 @@ static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, p = tcp_gro_lookup(head, th); if (p) { - NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + /* flist GRO applies to consecutive non-GSO skbs */ + if (!skb_is_gso(skb) || !NAPI_GRO_CB(p)->is_flist) { + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist; + return; + } + + /* Fall back to the regular GRO path */ + if (NAPI_GRO_CB(p)->count == 1) + NAPI_GRO_CB(p)->is_flist = 0; + + NAPI_GRO_CB(skb)->is_flist = 0; + return; } @@ -39,7 +50,7 @@ static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb, sk = __inet6_lookup_established(net, &hdr->saddr, th->source, &hdr->daddr, ntohs(th->dest), iif, sdif); - NAPI_GRO_CB(skb)->is_flist = !sk; + NAPI_GRO_CB(skb)->is_flist = !sk && !skb_is_gso(skb); if (sk) sock_gen_put(sk); #endif /* IS_ENABLED(CONFIG_IPV6) */ diff --git a/net/ipv6/xfrm6_input.c b/net/ipv6/xfrm6_input.c index 699a001ac1662..c9e208c57a6b5 100644 --- a/net/ipv6/xfrm6_input.c +++ b/net/ipv6/xfrm6_input.c @@ -249,7 +249,7 @@ int xfrm6_input_addr(struct sk_buff *skb, xfrm_address_t *daddr, goto drop; } - if (1 + sp->len == XFRM_MAX_DEPTH) { + if (sp->len >= XFRM_MAX_DEPTH) { XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR); goto drop; } diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index 125ea9a5b8a08..3b749475f6ed6 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -88,6 +88,7 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, xdst->u.rt6.rt6i_idev = in6_dev_get(dev); if (!xdst->u.rt6.rt6i_idev) { netdev_put(dev, &xdst->u.dst.dev_tracker); + xdst->u.dst.dev = NULL; return -ENODEV; } diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index c66b90c912e78..bd5838e65c317 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -335,6 +335,7 @@ static void iucv_sever_path(struct sock *sk, int with_user_data) unsigned char user_data[16]; struct iucv_sock *iucv = iucv_sk(sk); struct iucv_path *path = iucv->path; + struct sock_msg_q *p, *n; /* Whoever resets the path pointer, must sever and free it. */ if (xchg(&iucv->path, NULL)) { @@ -346,6 +347,19 @@ static void iucv_sever_path(struct sock *sk, int with_user_data) } else pr_iucv->path_sever(path, NULL); iucv_path_free(path); + + /* + * Message notifications queued on message_q still reference + * the now freed path; drop them, otherwise a later recvmsg() + * would pass the freed iucv_path to message_receive() via + * iucv_process_message_q(). + */ + spin_lock_bh(&iucv->message_q.lock); + list_for_each_entry_safe(p, n, &iucv->message_q.list, list) { + list_del(&p->list); + kfree(p); + } + spin_unlock_bh(&iucv->message_q.lock); } } @@ -1872,7 +1886,8 @@ static int afiucv_hs_callback_syn(struct sock *sk, struct sk_buff *skb) afiucv_swap_src_dest(skb); trans_hdr->flags = AF_IUCV_FLAG_SYN | AF_IUCV_FLAG_FIN; err = dev_queue_xmit(skb); - iucv_sock_kill(nsk); + if (nsk) + iucv_sock_kill(nsk); bh_unlock_sock(sk); goto out; } @@ -2064,6 +2079,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, sk = NULL; read_lock(&iucv_sk_list.lock); sk_for_each(sk, &iucv_sk_list.head) { + if (iucv_sk(sk)->hs_dev != dev) + continue; if (trans_hdr->flags == AF_IUCV_FLAG_SYN) { if ((!memcmp(&iucv_sk(sk)->src_name, trans_hdr->destAppName, 8)) && @@ -2089,6 +2106,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, } } } + if (sk) + sock_hold(sk); read_unlock(&iucv_sk_list.lock); if (!iucv) sk = NULL; @@ -2138,6 +2157,8 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, kfree_skb(skb); } + if (sk) + sock_put(sk); return err; } diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index dc126f7f030c4..0753af2a5128b 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -5,6 +5,7 @@ * Copyright (c) 2016 Tom Herbert */ +#include #include #include #include @@ -390,7 +391,9 @@ static int kcm_parse_func_strparser(struct strparser *strp, struct sk_buff *skb) struct bpf_prog *prog = psock->bpf_prog; int res; + rcu_read_lock(); res = bpf_prog_run_pin_on_cpu(prog, skb); + rcu_read_unlock(); return res; } diff --git a/net/l2tp/l2tp_debugfs.c b/net/l2tp/l2tp_debugfs.c index 2d0c8275a3a82..90fc0a59d50ac 100644 --- a/net/l2tp/l2tp_debugfs.c +++ b/net/l2tp/l2tp_debugfs.c @@ -306,6 +306,10 @@ static int l2tp_dfs_seq_release(struct inode *inode, struct file *file) seq = file->private_data; pd = seq->private; + if (pd->session) + l2tp_session_put(pd->session); + if (pd->tunnel) + l2tp_tunnel_put(pd->tunnel); if (pd->net) put_net_track(pd->net, &pd->ns_tracker); kfree(pd); diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c index 59457c0c14aab..fbdd3d055dbc7 100644 --- a/net/l2tp/l2tp_netlink.c +++ b/net/l2tp/l2tp_netlink.c @@ -248,8 +248,8 @@ static int l2tp_nl_cmd_tunnel_create(struct sk_buff *skb, struct genl_info *info kfree(tunnel); goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, - L2TP_CMD_TUNNEL_CREATE); + l2tp_tunnel_notify(&l2tp_nl_family, info, tunnel, + L2TP_CMD_TUNNEL_CREATE); l2tp_tunnel_put(tunnel); out: @@ -305,8 +305,8 @@ static int l2tp_nl_cmd_tunnel_modify(struct sk_buff *skb, struct genl_info *info goto out; } - ret = l2tp_tunnel_notify(&l2tp_nl_family, info, - tunnel, L2TP_CMD_TUNNEL_MODIFY); + l2tp_tunnel_notify(&l2tp_nl_family, info, + tunnel, L2TP_CMD_TUNNEL_MODIFY); l2tp_tunnel_put(tunnel); @@ -645,8 +645,8 @@ static int l2tp_nl_cmd_session_create(struct sk_buff *skb, struct genl_info *inf session = l2tp_session_get(net, tunnel->sock, tunnel->version, tunnel_id, session_id); if (session) { - ret = l2tp_session_notify(&l2tp_nl_family, info, session, - L2TP_CMD_SESSION_CREATE); + l2tp_session_notify(&l2tp_nl_family, info, session, + L2TP_CMD_SESSION_CREATE); l2tp_session_put(session); } } @@ -710,8 +710,8 @@ static int l2tp_nl_cmd_session_modify(struct sk_buff *skb, struct genl_info *inf if (info->attrs[L2TP_ATTR_RECV_TIMEOUT]) session->reorder_timeout = nla_get_msecs(info->attrs[L2TP_ATTR_RECV_TIMEOUT]); - ret = l2tp_session_notify(&l2tp_nl_family, info, - session, L2TP_CMD_SESSION_MODIFY); + l2tp_session_notify(&l2tp_nl_family, info, + session, L2TP_CMD_SESSION_MODIFY); l2tp_session_put(session); diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c index f388bf9abf378..c67ba1eab4769 100644 --- a/net/l2tp/l2tp_ppp.c +++ b/net/l2tp/l2tp_ppp.c @@ -1608,7 +1608,53 @@ static const struct seq_operations pppol2tp_seq_ops = { .stop = pppol2tp_seq_stop, .show = pppol2tp_seq_show, }; -#endif /* CONFIG_PROC_FS */ + +static int pppol2tp_proc_open(struct inode *inode, struct file *file) +{ + struct net *net = pde_data(inode); + struct pppol2tp_seq_data *pd; + + net = maybe_get_net(net); + if (!net) + return -ENXIO; + + pd = __seq_open_private(file, &pppol2tp_seq_ops, sizeof(*pd)); + if (!pd) { + put_net(net); + return -ENOMEM; + } + +#ifdef CONFIG_NET_NS + pd->p.net = net; + netns_tracker_alloc(net, &pd->p.ns_tracker, GFP_KERNEL); +#endif + return 0; +} + +static int pppol2tp_proc_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq = file->private_data; + struct pppol2tp_seq_data *pd = seq->private; + + if (pd->session) + l2tp_session_put(pd->session); + if (pd->tunnel) + l2tp_tunnel_put(pd->tunnel); + +#ifdef CONFIG_NET_NS + put_net_track(pd->p.net, &pd->p.ns_tracker); +#else + put_net(&init_net); +#endif + return seq_release_private(inode, file); +} + +static const struct proc_ops pppol2tp_proc_ops = { + .proc_open = pppol2tp_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = pppol2tp_proc_release, +}; /***************************************************************************** * Network namespace @@ -1619,8 +1665,8 @@ static __net_init int pppol2tp_init_net(struct net *net) struct proc_dir_entry *pde; int err = 0; - pde = proc_create_net("pppol2tp", 0444, net->proc_net, - &pppol2tp_seq_ops, sizeof(struct pppol2tp_seq_data)); + pde = proc_create_data("pppol2tp", 0444, net->proc_net, + &pppol2tp_proc_ops, net); if (!pde) { err = -ENOMEM; goto out; @@ -1635,9 +1681,13 @@ static __net_exit void pppol2tp_exit_net(struct net *net) remove_proc_entry("pppol2tp", net->proc_net); } +#endif /* CONFIG_PROC_FS */ + static struct pernet_operations pppol2tp_net_ops = { +#ifdef CONFIG_PROC_FS .init = pppol2tp_init_net, .exit = pppol2tp_exit_net, +#endif }; /***************************************************************************** diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index d32eacbb7517d..72c9d5aceae58 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -1124,9 +1124,6 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, fd->max_interval = params->max_interval; old = sdata_dereference(link->u.ap.fils_discovery, sdata); - if (old) - kfree_rcu(old, rcu_head); - if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); if (!new) @@ -1138,6 +1135,9 @@ static int ieee80211_set_fils_discovery(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.fils_discovery, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_FILS_DISCOVERY; return 0; } @@ -1157,8 +1157,6 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, link_conf->unsol_bcast_probe_resp_interval = params->interval; old = sdata_dereference(link->u.ap.unsol_bcast_probe_resp, sdata); - if (old) - kfree_rcu(old, rcu_head); if (params->tmpl && params->tmpl_len) { new = kzalloc(sizeof(*new) + params->tmpl_len, GFP_KERNEL); @@ -1171,6 +1169,9 @@ ieee80211_set_unsol_bcast_probe_resp(struct ieee80211_sub_if_data *sdata, RCU_INIT_POINTER(link->u.ap.unsol_bcast_probe_resp, NULL); } + if (old) + kfree_rcu(old, rcu_head); + *changed |= BSS_CHANGED_UNSOL_BCAST_PROBE_RESP; return 0; } @@ -1810,6 +1811,7 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, struct ieee80211_link_data *link = sdata_dereference(sdata->link[link_id], sdata); struct ieee80211_bss_conf *link_conf = link->conf; + u64 changes = BSS_CHANGED_BEACON_ENABLED; LIST_HEAD(keys); lockdep_assert_wiphy(local->hw.wiphy); @@ -1859,6 +1861,11 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, if (old_s1g_short_beacon) kfree_rcu(old_s1g_short_beacon, rcu_head); + if (link_conf->ftm_responder) { + link_conf->ftm_responder = false; + changes |= BSS_CHANGED_FTM_RESPONDER; + } + kfree(link_conf->ftmr_params); link_conf->ftmr_params = NULL; @@ -1880,8 +1887,7 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev, sdata->vif.cfg.ssid_len = 0; sdata->vif.cfg.s1g = false; clear_bit(SDATA_STATE_OFFCHANNEL_BEACON_STOPPED, &sdata->state); - ieee80211_link_info_change_notify(sdata, link, - BSS_CHANGED_BEACON_ENABLED); + ieee80211_link_info_change_notify(sdata, link, changes); ieee80211_remove_link_keys(link, &keys); if (!list_empty(&keys)) { diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c index a6895b40d4322..8403b5248a953 100644 --- a/net/mac80211/chan.c +++ b/net/mac80211/chan.c @@ -235,11 +235,12 @@ ieee80211_find_reservation_chanctx(struct ieee80211_local *local, return NULL; } -static enum nl80211_chan_width ieee80211_get_sta_bw(struct sta_info *sta, - unsigned int link_id) +static enum nl80211_chan_width +ieee80211_get_sta_bw(struct sta_info *sta, struct ieee80211_link_data *link) { enum ieee80211_sta_rx_bandwidth width; struct link_sta_info *link_sta; + int link_id = link->link_id; link_sta = wiphy_dereference(sta->local->hw.wiphy, sta->link[link_id]); @@ -254,7 +255,7 @@ static enum nl80211_chan_width ieee80211_get_sta_bw(struct sta_info *sta, * capabilities here. Calling it RX bandwidth capability is a bit * wrong though, since capabilities are in fact symmetric. */ - width = ieee80211_sta_cap_rx_bw(link_sta); + width = _ieee80211_sta_cap_rx_bw(link_sta, &link->conf->chanreq.oper); switch (width) { case IEEE80211_STA_RX_BW_20: @@ -289,7 +290,6 @@ static enum nl80211_chan_width ieee80211_get_max_required_bw(struct ieee80211_link_data *link) { struct ieee80211_sub_if_data *sdata = link->sdata; - unsigned int link_id = link->link_id; enum nl80211_chan_width max_bw = NL80211_CHAN_WIDTH_20_NOHT; struct sta_info *sta; @@ -300,7 +300,7 @@ ieee80211_get_max_required_bw(struct ieee80211_link_data *link) !(sta->sdata->bss && sta->sdata->bss == sdata->bss)) continue; - max_bw = max(max_bw, ieee80211_get_sta_bw(sta, link_id)); + max_bw = max(max_bw, ieee80211_get_sta_bw(sta, link)); } return max_bw; diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c index 6e36b09fe97f8..6ca2ff354e768 100644 --- a/net/mac80211/ibss.c +++ b/net/mac80211/ibss.c @@ -675,7 +675,9 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata) ifibss->state = IEEE80211_IBSS_MLME_SEARCH; - sta_info_flush(sdata, -1); + netif_carrier_off(sdata->dev); + if (!sta_info_flush(sdata, -1)) + synchronize_net(); spin_lock_bh(&ifibss->incomplete_lock); while (!list_empty(&ifibss->incomplete_stations)) { @@ -689,8 +691,6 @@ static void ieee80211_ibss_disconnect(struct ieee80211_sub_if_data *sdata) } spin_unlock_bh(&ifibss->incomplete_lock); - netif_carrier_off(sdata->dev); - sdata->vif.cfg.ibss_joined = false; sdata->vif.cfg.ibss_creator = false; sdata->vif.bss_conf.enable_beacon = false; @@ -717,7 +717,6 @@ static void ieee80211_csa_connection_drop_work(struct wiphy *wiphy, u.ibss.csa_connection_drop_work); ieee80211_ibss_disconnect(sdata); - synchronize_rcu(); skb_queue_purge(&sdata->skb_queue); /* trigger a scan to find another IBSS network to join */ @@ -1823,8 +1822,6 @@ int ieee80211_ibss_leave(struct ieee80211_sub_if_data *sdata) memset(&ifibss->ht_capa, 0, sizeof(ifibss->ht_capa)); memset(&ifibss->ht_capa_mask, 0, sizeof(ifibss->ht_capa_mask)); - synchronize_rcu(); - skb_queue_purge(&sdata->skb_queue); timer_delete_sync(&sdata->u.ibss.timer); diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 5c0c833fcf7a9..b0e64cf346e2b 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1864,6 +1864,15 @@ struct ieee802_11_elems { struct ieee80211_mle_per_sta_profile *prof; size_t sta_prof_len; + /* + * When parsing the beacon with MBSSID (from a transmitted BSS), this + * indicates that the profile the parser was instructed to look for + * (via the bss value in &struct ieee80211_elems_parse_params) couldn't + * be found (due to EMA, or perhaps broken AP) and the result cannot be + * considered complete. + */ + bool mbssid_nontx_profile_missing; + /* whether/which parse error occurred while retrieving these elements */ u8 parse_error; }; diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c index 72c129478da08..499126317a914 100644 --- a/net/mac80211/iface.c +++ b/net/mac80211/iface.c @@ -581,6 +581,7 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do WARN_ON(!list_empty(&sdata->u.ap.vlans)); } else if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) { /* remove all packets in parent bc_buf pointing to this dev */ + __skb_queue_head_init(&freeq); ps = &sdata->bss->ps; spin_lock_irqsave(&ps->bc_buf.lock, flags); @@ -588,10 +589,15 @@ static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_do if (skb->dev == sdata->dev) { __skb_unlink(skb, &ps->bc_buf); local->total_ps_buffered--; - ieee80211_free_txskb(&local->hw, skb); + __skb_queue_tail(&freeq, skb); } } spin_unlock_irqrestore(&ps->bc_buf.lock, flags); + + skb_queue_walk_safe(&freeq, skb, tmp) { + __skb_unlink(skb, &freeq); + ieee80211_free_txskb(&local->hw, skb); + } } if (going_down) diff --git a/net/mac80211/link.c b/net/mac80211/link.c index 05b0472bda40e..54415ca128d22 100644 --- a/net/mac80211/link.c +++ b/net/mac80211/link.c @@ -280,6 +280,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, u16 old_active = sdata->vif.active_links; unsigned long add = new_links & ~old_links; unsigned long rem = old_links & ~new_links; + unsigned long sta_rem = rem; unsigned int link_id; int ret; struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link; @@ -287,6 +288,7 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS]; bool use_deflink = old_links == 0; /* set for error case */ bool non_sta = sdata->vif.type != NL80211_IFTYPE_STATION; + struct sta_info *sta; lockdep_assert_wiphy(sdata->local->hw.wiphy); @@ -343,7 +345,8 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, link = links[link_id]; ieee80211_link_init(sdata, link_id, &link->data, &link->conf); ieee80211_link_setup(&link->data); - ieee80211_set_wmm_default(&link->data, true, non_sta); + if (sdata->vif.type != NL80211_IFTYPE_AP_VLAN) + ieee80211_set_wmm_default(&link->data, true, non_sta); } if (new_links == 0) @@ -382,11 +385,43 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata, memcpy(sdata->link, old_data, sizeof(old_data)); memcpy(sdata->vif.link_conf, old, sizeof(old)); ieee80211_set_vif_links_bitmaps(sdata, old_links, dormant_links); + for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) { + ieee80211_link_debugfs_remove(&links[link_id]->data); + ieee80211_link_stop(&links[link_id]->data); + } /* and free (only) the newly allocated links */ memset(to_free, 0, sizeof(links)); goto free; } + /* try to remove links that are now invalid from (MLO) stations */ + list_for_each_entry(sta, &sdata->local->sta_list, list) { + unsigned long rem_links = sta->sta.valid_links & sta_rem; + + if (sta->sdata != sdata) + continue; + + /* + * skip stations that would have no links left, + * those will be removed completely later + */ + if (sta->sta.valid_links == rem_links) + continue; + + for_each_set_bit(link_id, &rem_links, + IEEE80211_MLD_MAX_NUM_LINKS) + ieee80211_sta_remove_link(sta, link_id); + } + + /* + * Remove stations using any removed links. Note that due + * to the above station link removal, this only removes + * stations that were skipped above because they'd have no + * links left after link removal. + */ + for_each_set_bit(link_id, &sta_rem, IEEE80211_MLD_MAX_NUM_LINKS) + sta_info_flush(sdata, link_id); + /* use deflink/bss_conf again if and only if there are no more links */ use_deflink = new_links == 0; diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 8854e47b05672..0f4bd59a95ecc 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -5008,13 +5008,15 @@ static void ieee80211_rx_mgmt_deauth(struct ieee80211_sub_if_data *sdata, struct ieee80211_mgmt *mgmt, size_t len) { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; - u16 reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + u16 reason_code; lockdep_assert_wiphy(sdata->local->hw.wiphy); - if (len < 24 + 2) + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) return; + reason_code = le16_to_cpu(mgmt->u.deauth.reason_code); + if (!ether_addr_equal(mgmt->bssid, mgmt->sa)) { ieee80211_tdls_handle_disconnect(sdata, mgmt->sa, reason_code); return; @@ -6476,7 +6478,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, { struct ieee80211_if_managed *ifmgd = &sdata->u.mgd; struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data; - u16 capab_info, status_code, aid; + u16 capab_info, status_code, aid = 0; struct ieee80211_elems_parse_params parse_params = { .bss = NULL, .link_id = -1, @@ -6553,8 +6555,10 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata, if (elems->aid_resp) aid = le16_to_cpu(elems->aid_resp->aid); - else + else if (!assoc_data->s1g) aid = le16_to_cpu(mgmt->u.assoc_resp.aid); + else if (status_code == WLAN_STATUS_SUCCESS) + goto abandon_assoc; /* * The 5 MSB of the AID field are reserved for a non-S1G STA. For @@ -7301,7 +7305,7 @@ ieee80211_mgd_check_cross_link_csa(struct ieee80211_sub_if_data *sdata, prof = (void *)sta_profiles[link_id]; prof_elems = ieee802_11_parse_elems(prof->variable + (prof->sta_info_len - 1), - len - + len - sizeof(*prof) - (prof->sta_info_len - 1), false, NULL); @@ -7398,8 +7402,6 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, struct link_sta_info *link_sta; struct sta_info *sta; u64 changed = 0; - bool erp_valid; - u8 erp_value = 0; u32 ncrc = 0; u8 *bssid, *variable = mgmt->u.beacon.variable; u8 deauth_buf[IEEE80211_DEAUTH_FRAME_LEN]; @@ -7519,6 +7521,13 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, if (!elems) return; + /* + * Note: with MBSSID and an EMA (or broken) AP, we could fail to find + * the correct multi-BSSID profile for the non-transmitting AP we're + * connected to. The result's elems->mbssid_nontx_profile_missing is + * indicating that, but some things must happen regardless. + */ + if (rx_status->flag & RX_FLAG_DECRYPTED && ieee80211_mgd_ssid_mismatch(sdata, elems)) { sdata_info(sdata, "SSID mismatch for AP %pM, disconnect\n", @@ -7554,6 +7563,11 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, } } + /* + * P2P will almost certainly not have MBSSID, but this just + * assumes that it would at least always inherit NoA anyway + * since it's absent from the channel. + */ if (sdata->vif.p2p || sdata->vif.driver_flags & IEEE80211_VIF_GET_NOA_UPDATE) { struct ieee80211_p2p_noa_attr noa = {}; @@ -7611,23 +7625,17 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, ieee80211_rx_bss_info(link, mgmt, len, rx_status); + /* + * This assumes that all members of a multiple BSS set must be + * switching together, so we can parse channel switch elements + * from the transmitted BSS even if our non-transmitted one is + * not present in this beacon (due to EMA.) + */ ieee80211_sta_process_chanswitch(link, rx_status->mactime, rx_status->device_timestamp, elems, elems, IEEE80211_CSA_SOURCE_BEACON); - /* note that after this elems->ml_basic can no longer be used fully */ - ieee80211_mgd_check_cross_link_csa(sdata, rx_status->link_id, elems); - - ieee80211_mgd_update_bss_param_ch_cnt(sdata, bss_conf, elems); - - if (!sdata->u.mgd.epcs.enabled && - !link->u.mgd.disable_wmm_tracking && - ieee80211_sta_wmm_params(local, link, elems->wmm_param, - elems->wmm_param_len, - elems->mu_edca_param_set)) - changed |= BSS_CHANGED_QOS; - /* * If we haven't had a beacon before, tell the driver about the * DTIM period (and beacon timing if desired) now. @@ -7644,17 +7652,53 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, ieee80211_recalc_ps_vif(sdata); } - if (elems->erp_info) { - erp_valid = true; - erp_value = elems->erp_info[0]; - } else { - erp_valid = false; - } + /* RNR isn't inside an MBSSID profile */ + ieee80211_mgd_update_bss_param_ch_cnt(sdata, bss_conf, elems); + + /* assume ERP would be inherited anyway */ + if (!ieee80211_is_s1g_beacon(hdr->frame_control)) { + u8 erp_value = 0; + bool erp_valid; + + if (elems->erp_info) { + erp_valid = true; + erp_value = elems->erp_info[0]; + } else { + erp_valid = false; + } - if (!ieee80211_is_s1g_beacon(hdr->frame_control)) changed |= ieee80211_handle_bss_capability(link, le16_to_cpu(mgmt->u.beacon.capab_info), erp_valid, erp_value); + } + + /* + * There are some other things that we can only do when the + * real non-transmitted profile was actually parsed, so exit + * here before doing those. + */ + if (elems->mbssid_nontx_profile_missing) + goto apply; + + /* + * This requires multi-link element, which is from the MBSSID profile. + * Note that after this elems->ml_basic can no longer be used fully. + * + * Note also that currently the parsing is incorrect, so this will + * never actually do anything. + */ + ieee80211_mgd_check_cross_link_csa(sdata, rx_status->link_id, elems); + + /* + * EDCA parameters should be the same, but perhaps ACM can differ + * between BSSes in an MBSSID set. + */ + if (!sdata->u.mgd.epcs.enabled && + !link->u.mgd.disable_wmm_tracking && + ieee80211_sta_wmm_params(local, link, elems->wmm_param, + elems->wmm_param_len, + elems->mu_edca_param_set)) + changed |= BSS_CHANGED_QOS; sta = sta_info_get(sdata, sdata->vif.cfg.ap_addr); if (WARN_ON(!sta)) { @@ -7700,6 +7744,7 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link, ieee80211_process_adv_ttlm(sdata, elems, le64_to_cpu(mgmt->u.beacon.timestamp)); +apply: ieee80211_link_info_change_notify(sdata, link, changed); free: kfree(elems); @@ -10863,14 +10908,6 @@ int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata, goto err_free; } - for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; - link_id++) { - if (!(req->rem_links & BIT(link_id))) - continue; - - ieee80211_sta_remove_link(sta, link_id); - } - /* notify the driver and upper layers */ ieee80211_vif_cfg_change_notify(sdata, BSS_CHANGED_MLD_VALID_LINKS); diff --git a/net/mac80211/parse.c b/net/mac80211/parse.c index b9ec99f51851a..b117572f8cb17 100644 --- a/net/mac80211/parse.c +++ b/net/mac80211/parse.c @@ -1032,6 +1032,10 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params) sub.action = params->action; sub.link_id = params->link_id; + /* indicate to consumer whether or not profile was found */ + if (params->bss->transmitted_bss && !nontx_len) + elems->mbssid_nontx_profile_missing = true; + /* consume the space used for non-transmitted profile */ elems_parse->scratch_pos += nontx_len; diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c index 0a986607e6592..13b7432d732f8 100644 --- a/net/mac80211/rx.c +++ b/net/mac80211/rx.c @@ -1612,6 +1612,8 @@ static void sta_ps_start(struct sta_info *sta) else clear_bit(tid, &sta->txq_buffered_tids); } + + sta_info_recalc_tim(sta); } static void sta_ps_end(struct sta_info *sta) diff --git a/net/mac80211/s1g.c b/net/mac80211/s1g.c index 1f68df6e80670..1e2f07090a436 100644 --- a/net/mac80211/s1g.c +++ b/net/mac80211/s1g.c @@ -101,6 +101,10 @@ ieee80211_s1g_rx_twt_setup(struct ieee80211_sub_if_data *sdata, struct ieee80211_twt_setup *twt = (void *)mgmt->u.action.u.s1g.variable; struct ieee80211_twt_params *twt_agrt = (void *)twt->params; + if (!(twt->control & IEEE80211_TWT_CONTROL_NEG_TYPE_BROADCAST) && + twt->length < sizeof(twt->control) + sizeof(*twt_agrt)) + return; + twt_agrt->req_type &= cpu_to_le16(~IEEE80211_TWT_REQTYPE_REQUEST); /* broadcast TWT not supported yet */ @@ -143,7 +147,7 @@ ieee80211_s1g_tx_twt_setup_fail(struct ieee80211_sub_if_data *sdata, drv_twt_teardown_request(sdata->local, sdata, &sta->sta, flowid); - ieee80211_s1g_send_twt_teardown(sdata, mgmt->sa, sdata->vif.addr, + ieee80211_s1g_send_twt_teardown(sdata, mgmt->da, sdata->vif.addr, flowid); } diff --git a/net/mac80211/spectmgmt.c b/net/mac80211/spectmgmt.c index 7422888d36409..080235ca23fc5 100644 --- a/net/mac80211/spectmgmt.c +++ b/net/mac80211/spectmgmt.c @@ -227,7 +227,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, { enum nl80211_band new_band = current_band; int new_freq; - u8 new_chan_no = 0, new_op_class = 0; + int new_chan_no = -1, new_op_class = -1; struct ieee80211_channel *new_chan; struct cfg80211_chan_def new_chandef = {}; const struct ieee80211_sec_chan_offs_ie *sec_chan_offs; @@ -256,7 +256,7 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, new_op_class = ext_chansw_elem->new_operating_class; if (!ieee80211_operating_class_to_band(new_op_class, &new_band)) { - new_op_class = 0; + new_op_class = -1; if (!unprot_action) sdata_info(sdata, "cannot understand ECSA IE operating class, %d, ignoring\n", @@ -268,14 +268,14 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, } } - if (!new_op_class && elems->ch_switch_ie) { + if (new_op_class < 0 && elems->ch_switch_ie) { new_chan_no = elems->ch_switch_ie->new_ch_num; csa_ie->count = elems->ch_switch_ie->count; csa_ie->mode = elems->ch_switch_ie->mode; } /* nothing here we understand */ - if (!new_chan_no) + if (new_chan_no < 0) return 1; /* Mesh Channel Switch Parameters Element */ @@ -359,7 +359,8 @@ int ieee80211_parse_ch_switch_ie(struct ieee80211_sub_if_data *sdata, get_unaligned_le16(bwi->info.optional); } else if (!wide_bw_chansw_ie || !wbcs_elem_to_chandef(wide_bw_chansw_ie, &new_chandef)) { - if (!ieee80211_operating_class_to_chandef(new_op_class, new_chan, + if (new_op_class < 0 || + !ieee80211_operating_class_to_chandef(new_op_class, new_chan, &new_chandef)) new_chandef = csa_ie->chanreq.oper; } diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c index b0d9bb830f293..54a6b20a4c5cd 100644 --- a/net/mac80211/sta_info.c +++ b/net/mac80211/sta_info.c @@ -355,6 +355,15 @@ static void sta_info_free_link(struct link_sta_info *link_sta) free_percpu(link_sta->pcpu_rx_stats); } +static void sta_link_free_rcu(struct rcu_head *head) +{ + struct sta_link_alloc *alloc = + container_of(head, struct sta_link_alloc, rcu_head); + + sta_info_free_link(&alloc->info); + kfree(alloc); +} + static void sta_accumulate_removed_link_stats(struct sta_info *sta, int link_id) { struct link_sta_info *link_sta = wiphy_dereference(sta->local->hw.wiphy, @@ -427,10 +436,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id, RCU_INIT_POINTER(sta->link[link_id], NULL); RCU_INIT_POINTER(sta->sta.link[link_id], NULL); - if (alloc) { - sta_info_free_link(&alloc->info); - kfree_rcu(alloc, rcu_head); - } + if (alloc) + call_rcu(&alloc->rcu_head, sta_link_free_rcu); ieee80211_sta_recalc_aggregates(&sta->sta); } diff --git a/net/mac80211/status.c b/net/mac80211/status.c index 4b38aa0e902a8..8716eda8317d1 100644 --- a/net/mac80211/status.c +++ b/net/mac80211/status.c @@ -5,7 +5,7 @@ * Copyright 2006-2007 Jiri Benc * Copyright 2008-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH - * Copyright 2021-2025 Intel Corporation + * Copyright 2021-2026 Intel Corporation */ #include @@ -295,9 +295,10 @@ ieee80211_add_tx_radiotap_header(struct ieee80211_local *local, RATE_INFO_FLAGS_VHT_MCS | RATE_INFO_FLAGS_HE_MCS))) legacy_rate = status_rate->rate_idx.legacy; - } else if (info->status.rates[0].idx >= 0 && - !(info->status.rates[0].flags & (IEEE80211_TX_RC_MCS | - IEEE80211_TX_RC_VHT_MCS))) { + } else if (info->band < NUM_NL80211_BANDS && + info->status.rates[0].idx >= 0 && + !(info->status.rates[0].flags & (IEEE80211_TX_RC_MCS | + IEEE80211_TX_RC_VHT_MCS))) { struct ieee80211_supported_band *sband; sband = local->hw.wiphy->bands[info->band]; diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c index c125871adb62b..e7df1a0c85911 100644 --- a/net/mac80211/tx.c +++ b/net/mac80211/tx.c @@ -3895,7 +3895,7 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw, * injected frames or EAPOL frames from the local station. */ if (unlikely(!(info->flags & IEEE80211_TX_CTL_INJECTED) && - ieee80211_is_data(hdr->frame_control) && + ieee80211_is_data_present(hdr->frame_control) && !ieee80211_vif_is_mesh(&tx.sdata->vif) && tx.sdata->vif.type != NL80211_IFTYPE_OCB && !is_multicast_ether_addr(hdr->addr1) && @@ -5140,7 +5140,8 @@ static void ieee80211_set_beacon_cntdwn(struct ieee80211_sub_if_data *sdata, if (sdata->vif.type == NL80211_IFTYPE_AP && resp) { u16 *resp_offsets = resp->cntdwn_counter_offsets; - resp->data[resp_offsets[i]] = count; + if (resp_offsets[i]) + resp->data[resp_offsets[i]] = count; } } } diff --git a/net/mac80211/util.c b/net/mac80211/util.c index 62e1a37849d04..af99ac02953d9 100644 --- a/net/mac80211/util.c +++ b/net/mac80211/util.c @@ -2055,11 +2055,7 @@ int ieee80211_reconfig(struct ieee80211_local *local) } break; case NL80211_IFTYPE_NAN: - res = ieee80211_reconfig_nan(sdata); - if (res < 0) { - ieee80211_handle_reconfig_failure(local); - return res; - } + WARN_ON(ieee80211_reconfig_nan(sdata)); break; case NL80211_IFTYPE_AP_VLAN: case NL80211_IFTYPE_MONITOR: diff --git a/net/mac802154/ieee802154_i.h b/net/mac802154/ieee802154_i.h index 8f2bff268392b..279ff4e9b11ea 100644 --- a/net/mac802154/ieee802154_i.h +++ b/net/mac802154/ieee802154_i.h @@ -74,6 +74,10 @@ struct ieee802154_local { struct work_struct rx_beacon_work; struct list_head rx_mac_cmd_list; struct work_struct rx_mac_cmd_work; + /* Serializes rx_beacon_list and rx_mac_cmd_list against the RX + * softirq producer, the mac_wq workers and the teardown flush. + */ + spinlock_t rx_lock; /* Association */ struct ieee802154_pan_device *assoc_dev; @@ -300,6 +304,10 @@ static inline bool mac802154_is_beaconing(struct ieee802154_local *local) } void mac802154_rx_mac_cmd_worker(struct work_struct *work); +void mac802154_flush_list(struct list_head *list, + struct ieee802154_sub_if_data *sdata); +void mac802154_flush_queued_pkts(struct ieee802154_local *local, + struct ieee802154_sub_if_data *sdata); int mac802154_perform_association(struct ieee802154_sub_if_data *sdata, struct ieee802154_pan_device *coord, diff --git a/net/mac802154/iface.c b/net/mac802154/iface.c index b823720630e72..31353795fa240 100644 --- a/net/mac802154/iface.c +++ b/net/mac802154/iface.c @@ -694,6 +694,7 @@ void ieee802154_if_remove(struct ieee802154_sub_if_data *sdata) mutex_unlock(&sdata->local->iflist_mtx); synchronize_rcu(); + mac802154_flush_queued_pkts(sdata->local, sdata); unregister_netdevice(sdata->dev); } @@ -705,6 +706,11 @@ void ieee802154_remove_interfaces(struct ieee802154_local *local) list_for_each_entry_safe(sdata, tmp, &local->interfaces, list) { list_del_rcu(&sdata->list); + /* Best-effort: a frame the RX softirq queues for this sdata + * after the flush still pins the netdev, so the + * unregister_netdevice() below waits it out. + */ + mac802154_flush_queued_pkts(local, sdata); unregister_netdevice(sdata->dev); } mutex_unlock(&local->iflist_mtx); diff --git a/net/mac802154/llsec.c b/net/mac802154/llsec.c index 09a47104b5772..a55fa5017e930 100644 --- a/net/mac802154/llsec.c +++ b/net/mac802154/llsec.c @@ -891,6 +891,11 @@ llsec_do_decrypt_auth(struct sk_buff *skb, const struct mac802154_llsec *sec, data = skb_mac_header(skb) + skb->mac_len; datalen = skb_tail_pointer(skb) - data; + if (datalen < authlen) { + kfree_sensitive(req); + return -EBADMSG; + } + sg_init_one(&sg, skb_mac_header(skb), assoclen + datalen); if (!(hdr->sec.level & IEEE802154_SCF_SECLEVEL_ENC)) { diff --git a/net/mac802154/main.c b/net/mac802154/main.c index ea1efef3572ae..386c086c79d17 100644 --- a/net/mac802154/main.c +++ b/net/mac802154/main.c @@ -91,6 +91,7 @@ ieee802154_alloc_hw(size_t priv_data_len, const struct ieee802154_ops *ops) INIT_LIST_HEAD(&local->interfaces); INIT_LIST_HEAD(&local->rx_beacon_list); INIT_LIST_HEAD(&local->rx_mac_cmd_list); + spin_lock_init(&local->rx_lock); mutex_init(&local->iflist_mtx); tasklet_setup(&local->tasklet, ieee802154_tasklet_handler); diff --git a/net/mac802154/rx.c b/net/mac802154/rx.c index aac359b5c71df..21e9dbfba14a9 100644 --- a/net/mac802154/rx.c +++ b/net/mac802154/rx.c @@ -35,16 +35,23 @@ void mac802154_rx_beacon_worker(struct work_struct *work) container_of(work, struct ieee802154_local, rx_beacon_work); struct cfg802154_mac_pkt *mac_pkt; - mac_pkt = list_first_entry_or_null(&local->rx_beacon_list, - struct cfg802154_mac_pkt, node); - if (!mac_pkt) - return; + for (;;) { + spin_lock_bh(&local->rx_lock); + mac_pkt = list_first_entry_or_null(&local->rx_beacon_list, + struct cfg802154_mac_pkt, node); + if (mac_pkt) + list_del(&mac_pkt->node); + spin_unlock_bh(&local->rx_lock); + if (!mac_pkt) + break; - mac802154_process_beacon(local, mac_pkt->skb, mac_pkt->page, mac_pkt->channel); + mac802154_process_beacon(local, mac_pkt->skb, + mac_pkt->page, mac_pkt->channel); - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } } static bool mac802154_should_answer_beacon_req(struct ieee802154_local *local) @@ -68,22 +75,15 @@ static bool mac802154_should_answer_beacon_req(struct ieee802154_local *local) return interval == IEEE802154_ACTIVE_SCAN_DURATION; } -void mac802154_rx_mac_cmd_worker(struct work_struct *work) +static void mac802154_rx_mac_cmd(struct ieee802154_local *local, + struct cfg802154_mac_pkt *mac_pkt) { - struct ieee802154_local *local = - container_of(work, struct ieee802154_local, rx_mac_cmd_work); - struct cfg802154_mac_pkt *mac_pkt; u8 mac_cmd; int rc; - mac_pkt = list_first_entry_or_null(&local->rx_mac_cmd_list, - struct cfg802154_mac_pkt, node); - if (!mac_pkt) - return; - rc = ieee802154_get_mac_cmd(mac_pkt->skb, &mac_cmd); if (rc) - goto out; + return; switch (mac_cmd) { case IEEE802154_CMD_BEACON_REQ: @@ -121,11 +121,81 @@ void mac802154_rx_mac_cmd_worker(struct work_struct *work) default: break; } +} -out: - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); +void mac802154_rx_mac_cmd_worker(struct work_struct *work) +{ + struct ieee802154_local *local = + container_of(work, struct ieee802154_local, rx_mac_cmd_work); + struct cfg802154_mac_pkt *mac_pkt; + + for (;;) { + spin_lock_bh(&local->rx_lock); + mac_pkt = list_first_entry_or_null(&local->rx_mac_cmd_list, + struct cfg802154_mac_pkt, node); + if (mac_pkt) + list_del(&mac_pkt->node); + spin_unlock_bh(&local->rx_lock); + if (!mac_pkt) + break; + + /* A stopped interface cannot transmit; skipping avoids a + * needless association response (and the !netif_running() + * warning it would trip) during teardown. The beacon worker + * needs no such check as it never transmits. + */ + if (ieee802154_sdata_running(mac_pkt->sdata)) + mac802154_rx_mac_cmd(local, mac_pkt); + + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } +} + +/** + * mac802154_flush_list - free queued RX frames on @list + * @list: rx_beacon_list or rx_mac_cmd_list + * @sdata: only free frames received on this interface, or %NULL for all + * + * Each frame pins the net_device it was received on (via netdev_hold()), + * so release that reference as the frame is dropped. Caller must hold + * local->rx_lock. + */ +void mac802154_flush_list(struct list_head *list, + struct ieee802154_sub_if_data *sdata) +{ + struct cfg802154_mac_pkt *mac_pkt, *tmp; + + list_for_each_entry_safe(mac_pkt, tmp, list, node) { + if (sdata && mac_pkt->sdata != sdata) + continue; + list_del(&mac_pkt->node); + netdev_put(mac_pkt->sdata->dev, &mac_pkt->dev_tracker); + kfree_skb(mac_pkt->skb); + kfree(mac_pkt); + } +} + +/** + * mac802154_flush_queued_pkts - drop queued RX work referencing @sdata + * @local: the mac802154 device + * @sdata: interface being removed + * + * The workers dereference the queued frame's interface directly + * (mac_pkt->sdata) or through skb->dev in mac802154_process_beacon(). Drop + * the not-yet-started entries belonging to @sdata before it is unregistered + * so their netdev reference is released; an entry already dequeued by a + * running worker keeps its own reference until the worker completes, which + * unregister_netdevice() then waits out. + */ +void mac802154_flush_queued_pkts(struct ieee802154_local *local, + struct ieee802154_sub_if_data *sdata) +{ + spin_lock_bh(&local->rx_lock); + mac802154_flush_list(&local->rx_beacon_list, sdata); + mac802154_flush_list(&local->rx_mac_cmd_list, sdata); + spin_unlock_bh(&local->rx_lock); } static int @@ -221,7 +291,10 @@ ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata, mac_pkt->sdata = sdata; mac_pkt->page = sdata->local->scan_page; mac_pkt->channel = sdata->local->scan_channel; + netdev_hold(sdata->dev, &mac_pkt->dev_tracker, GFP_ATOMIC); + spin_lock(&sdata->local->rx_lock); list_add_tail(&mac_pkt->node, &sdata->local->rx_beacon_list); + spin_unlock(&sdata->local->rx_lock); queue_work(sdata->local->mac_wq, &sdata->local->rx_beacon_work); return NET_RX_SUCCESS; @@ -233,7 +306,10 @@ ieee802154_subif_frame(struct ieee802154_sub_if_data *sdata, mac_pkt->skb = skb_get(skb); mac_pkt->sdata = sdata; + netdev_hold(sdata->dev, &mac_pkt->dev_tracker, GFP_ATOMIC); + spin_lock(&sdata->local->rx_lock); list_add_tail(&mac_pkt->node, &sdata->local->rx_mac_cmd_list); + spin_unlock(&sdata->local->rx_lock); queue_work(sdata->local->mac_wq, &sdata->local->rx_mac_cmd_work); return NET_RX_SUCCESS; diff --git a/net/mac802154/scan.c b/net/mac802154/scan.c index 0eef3b4f5b9b3..e507a3ad35d0a 100644 --- a/net/mac802154/scan.c +++ b/net/mac802154/scan.c @@ -104,13 +104,9 @@ static unsigned int mac802154_scan_get_channel_time(u8 duration_order, static void mac802154_flush_queued_beacons(struct ieee802154_local *local) { - struct cfg802154_mac_pkt *mac_pkt, *tmp; - - list_for_each_entry_safe(mac_pkt, tmp, &local->rx_beacon_list, node) { - list_del(&mac_pkt->node); - kfree_skb(mac_pkt->skb); - kfree(mac_pkt); - } + spin_lock_bh(&local->rx_lock); + mac802154_flush_list(&local->rx_beacon_list, NULL); + spin_unlock_bh(&local->rx_lock); } static void @@ -179,6 +175,7 @@ void mac802154_scan_worker(struct work_struct *work) enum nl802154_scan_types scan_req_type; struct ieee802154_sub_if_data *sdata; unsigned int scan_duration = 0; + netdevice_tracker dev_tracker; struct wpan_phy *wpan_phy; u8 scan_req_duration; u8 page, channel; @@ -209,6 +206,14 @@ void mac802154_scan_worker(struct work_struct *work) return; } + /* + * sdata->dev is dereferenced below after rcu_read_unlock() and outside + * the rtnl, and a concurrent DEL_INTERFACE / PHY teardown can free it + * asynchronously from netdev_run_todo(). Pin it with a reference taken + * while the RCU read lock is still held, and drop it at every exit. + */ + netdev_hold(sdata->dev, &dev_tracker, GFP_ATOMIC); + wpan_phy = scan_req->wpan_phy; scan_req_type = scan_req->type; scan_req_duration = scan_req->duration; @@ -262,12 +267,14 @@ void mac802154_scan_worker(struct work_struct *work) "Scan page %u channel %u for %ums\n", page, channel, jiffies_to_msecs(scan_duration)); queue_delayed_work(local->mac_wq, &local->scan_work, scan_duration); + netdev_put(sdata->dev, &dev_tracker); return; end_scan: rtnl_lock(); mac802154_scan_cleanup_locked(local, sdata, false); rtnl_unlock(); + netdev_put(sdata->dev, &dev_tracker); } int mac802154_trigger_scan_locked(struct ieee802154_sub_if_data *sdata, @@ -404,6 +411,7 @@ void mac802154_beacon_worker(struct work_struct *work) container_of(work, struct ieee802154_local, beacon_work.work); struct cfg802154_beacon_request *beacon_req; struct ieee802154_sub_if_data *sdata; + netdevice_tracker dev_tracker; struct wpan_dev *wpan_dev; u8 interval; int ret; @@ -416,12 +424,14 @@ void mac802154_beacon_worker(struct work_struct *work) } sdata = IEEE802154_WPAN_DEV_TO_SUB_IF(beacon_req->wpan_dev); + netdev_hold(sdata->dev, &dev_tracker, GFP_ATOMIC); /* Wait an arbitrary amount of time in case we cannot use the device */ if (local->suspended || !ieee802154_sdata_running(sdata)) { rcu_read_unlock(); queue_delayed_work(local->mac_wq, &local->beacon_work, msecs_to_jiffies(1000)); + netdev_put(sdata->dev, &dev_tracker); return; } @@ -439,6 +449,7 @@ void mac802154_beacon_worker(struct work_struct *work) if (interval < IEEE802154_ACTIVE_SCAN_DURATION) queue_delayed_work(local->mac_wq, &local->beacon_work, local->beacon_interval); + netdev_put(sdata->dev, &dev_tracker); } int mac802154_stop_beacons_locked(struct ieee802154_local *local, diff --git a/net/mctp/device.c b/net/mctp/device.c index 04c5570bacff6..7861d83aaad26 100644 --- a/net/mctp/device.c +++ b/net/mctp/device.c @@ -536,7 +536,9 @@ int __init mctp_device_init(void) { int err; - register_netdevice_notifier(&mctp_dev_nb); + err = register_netdevice_notifier(&mctp_dev_nb); + if (err) + return err; err = rtnl_af_register(&mctp_af_ops); if (err) diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 1c70cb26e7ba1..608c43a4602d6 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -198,6 +198,7 @@ static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb) if (pskb_may_pull(skb, mpls_hdr_len + sizeof(struct iphdr))) { const struct iphdr *v4hdr; + hdr = mpls_hdr(skb) + label_index; v4hdr = (const struct iphdr *)(hdr + 1); if (v4hdr->version == 4) { hash = jhash_3words(ntohl(v4hdr->saddr), @@ -208,6 +209,7 @@ static u32 mpls_multipath_hash(struct mpls_route *rt, struct sk_buff *skb) sizeof(struct ipv6hdr))) { const struct ipv6hdr *v6hdr; + hdr = mpls_hdr(skb) + label_index; v6hdr = (const struct ipv6hdr *)(hdr + 1); hash = __ipv6_addr_jhash(&v6hdr->saddr, hash); hash = __ipv6_addr_jhash(&v6hdr->daddr, hash); @@ -2123,6 +2125,9 @@ static int mpls_valid_fib_dump_req(struct net *net, const struct nlmsghdr *nlh, int ifindex; if (i == RTA_OIF) { + if (!tb[i]) + continue; + ifindex = nla_get_u32(tb[i]); filter->dev = __dev_get_by_index(net, ifindex); if (!filter->dev) @@ -2465,6 +2470,7 @@ static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, r->rtm_family = AF_MPLS; r->rtm_dst_len = 20; r->rtm_src_len = 0; + r->rtm_tos = 0; r->rtm_table = RT_TABLE_MAIN; r->rtm_type = RTN_UNICAST; r->rtm_scope = RT_SCOPE_UNIVERSE; diff --git a/net/mptcp/fastopen.c b/net/mptcp/fastopen.c index 082c46c0f50ee..f717750906ffa 100644 --- a/net/mptcp/fastopen.c +++ b/net/mptcp/fastopen.c @@ -24,12 +24,13 @@ void mptcp_fastopen_subflow_synack_set_params(struct mptcp_subflow_context *subf sk = subflow->conn; tp = tcp_sk(ssk); - subflow->is_mptfo = 1; - + /* A valid TFO cookie does not guarantee SYN data. */ skb = skb_peek(&ssk->sk_receive_queue); - if (WARN_ON_ONCE(!skb)) + if (!skb) return; + subflow->is_mptfo = 1; + /* dequeue the skb from sk receive queue */ __skb_unlink(skb, &ssk->sk_receive_queue); skb_ext_reset(skb); diff --git a/net/mptcp/options.c b/net/mptcp/options.c index 845860311ea67..a7f048afc8440 100644 --- a/net/mptcp/options.c +++ b/net/mptcp/options.c @@ -50,6 +50,14 @@ static void mptcp_parse_option(const struct sk_buff *skb, } } + /* Only the MPC + ACK can be used with a RM_ADDR */ + if (subopt == OPTION_MPTCP_MPC_ACK) { + if ((mp_opt->suboptions & ~OPTION_MPTCP_RM_ADDR) != 0) + break; + } else if (mp_opt->suboptions != 0) { + break; + } + /* Cfr RFC 8684 Section 3.3.0: * If a checksum is present but its use had * not been negotiated in the MP_CAPABLE handshake, the receiver MUST @@ -85,7 +93,8 @@ static void mptcp_parse_option(const struct sk_buff *skb, * In other words, the only way for checksums not to be used * is if both hosts in their SYNs set A=0." */ - if (flags & MPTCP_CAP_CHECKSUM_REQD) + if ((flags & MPTCP_CAP_CHECKSUM_REQD) && + opsize < TCPOLEN_MPTCP_MPC_ACK_DATA) mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD; mp_opt->deny_join_id0 = !!(flags & MPTCP_CAP_DENY_JOIN_ID0); @@ -122,6 +131,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_JOIN: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + if (opsize == TCPOLEN_MPTCP_MPJ_SYN) { mp_opt->suboptions |= OPTION_MPTCP_MPJ_SYN; mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP; @@ -153,21 +167,23 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_DSS: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO | + OPTION_MPTCP_FASTCLOSE | + OPTION_MPTCP_FAIL)) != 0) + break; + pr_debug("DSS\n"); ptr++; flags = (*ptr++) & MPTCP_DSS_FLAG_MASK; - mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0; mp_opt->dsn64 = (flags & MPTCP_DSS_DSN64) != 0; mp_opt->use_map = (flags & MPTCP_DSS_HAS_MAP) != 0; mp_opt->ack64 = (flags & MPTCP_DSS_ACK64) != 0; mp_opt->use_ack = (flags & MPTCP_DSS_HAS_ACK); - pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n", - mp_opt->data_fin, mp_opt->dsn64, - mp_opt->use_map, mp_opt->ack64, - mp_opt->use_ack); - expected_opsize = TCPOLEN_MPTCP_DSS_BASE; if (mp_opt->use_ack) { @@ -178,18 +194,30 @@ static void mptcp_parse_option(const struct sk_buff *skb, } if (mp_opt->use_map) { + mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0; if (mp_opt->dsn64) expected_opsize += TCPOLEN_MPTCP_DSS_MAP64; else expected_opsize += TCPOLEN_MPTCP_DSS_MAP32; } + pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d\n", + mp_opt->data_fin, mp_opt->dsn64, + mp_opt->use_map, mp_opt->ack64, + mp_opt->use_ack); + /* Always parse any csum presence combination, we will enforce * RFC 8684 Section 3.3.0 checks later in subflow_data_ready */ if (opsize != expected_opsize && - opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) + opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) { + mp_opt->dsn64 = 0; + mp_opt->use_map = 0; + mp_opt->ack64 = 0; + mp_opt->use_ack = 0; + mp_opt->data_fin = 0; break; + } mp_opt->suboptions |= OPTION_MPTCP_DSS; if (mp_opt->use_ack) { @@ -234,6 +262,12 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_ADD_ADDR: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RM_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + mp_opt->echo = (*ptr++) & MPTCP_ADDR_ECHO; if (!mp_opt->echo) { if (opsize == TCPOLEN_MPTCP_ADD_ADDR || @@ -293,6 +327,14 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_RM_ADDR: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_MPC_ACK | + OPTIONS_MPTCP_MPJ | + OPTIONS_MPTCP_DSS | + OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_PRIO)) != 0) + break; + if (opsize < TCPOLEN_MPTCP_RM_ADDR_BASE + 1 || opsize > TCPOLEN_MPTCP_RM_ADDR_BASE + MPTCP_RM_IDS_MAX) break; @@ -307,6 +349,13 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_PRIO: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_MPJ | + OPTIONS_MPTCP_DSS | + OPTION_MPTCP_ADD_ADDR | + OPTION_MPTCP_RM_ADDR)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_PRIO) break; @@ -316,6 +365,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_FASTCLOSE: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RST)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_FASTCLOSE) break; @@ -327,6 +381,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_RST: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTION_MPTCP_FAIL | + OPTION_MPTCP_FASTCLOSE)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_RST) break; @@ -342,6 +401,11 @@ static void mptcp_parse_option(const struct sk_buff *skb, break; case MPTCPOPT_MP_FAIL: + /* Can be used with a restricted number of other options */ + if ((mp_opt->suboptions & ~(OPTIONS_MPTCP_DSS | + OPTION_MPTCP_RST)) != 0) + break; + if (opsize != TCPOLEN_MPTCP_FAIL) break; @@ -467,7 +531,7 @@ static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb, return false; /* MPC/MPJ needed only on 3rd ack packet, DATA_FIN and TCP shutdown take precedence */ - if (READ_ONCE(subflow->fully_established) || snd_data_fin_enable || + if (subflow->fully_established || snd_data_fin_enable || subflow->snd_isn != TCP_SKB_CB(skb)->seq || sk->sk_state != TCP_ESTABLISHED) return false; @@ -934,7 +998,7 @@ static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk, /* here we can process OoO, in-window pkts, only in-sequence 4th ack * will make the subflow fully established */ - if (likely(READ_ONCE(subflow->fully_established))) { + if (likely(subflow->fully_established)) { /* on passive sockets, check for 3rd ack retransmission * note that msk is always set by subflow_syn_recv_sock() * for mp_join subflows @@ -1196,7 +1260,7 @@ bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb) MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDR); } else { mptcp_pm_add_addr_echoed(msk, &mp_opt.addr); - mptcp_pm_del_add_timer(msk, &mp_opt.addr, true); + mptcp_pm_announced_del_timer(msk, &mp_opt.addr, true); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADD); } @@ -1416,7 +1480,7 @@ void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp, * RM | C | C | C | P |------|------|------|------| * PRIO | X | C | C | C | C |------|------|------| * FAIL | X | X | C | X | X | X |------|------| - * FC | X | X | X | X | X | X | X |------| + * FC | X | X | P | X | X | X | X |------| * RST | X | X | X | X | X | X | O | O | * ------|------|------|------|------|------|------|------|------| * diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c index 72a8f6a0d058d..6981fd48c63cc 100644 --- a/net/mptcp/pm.c +++ b/net/mptcp/pm.c @@ -12,7 +12,7 @@ #define ADD_ADDR_RETRANS_MAX 3 -struct mptcp_pm_add_entry { +struct mptcp_pm_add_addr { struct list_head list; struct mptcp_addr_info addr; u8 retrans_times; @@ -115,14 +115,14 @@ static bool mptcp_pm_is_init_remote_addr(struct mptcp_sock *msk, return mptcp_addresses_equal(&mpc_remote, remote, remote->port); } -bool mptcp_lookup_subflow_by_saddr(const struct list_head *list, - const struct mptcp_addr_info *saddr) +bool mptcp_pm_has_subflow_saddr(const struct mptcp_sock *msk, + const struct mptcp_addr_info *saddr) { struct mptcp_subflow_context *subflow; struct mptcp_addr_info cur; struct sock_common *skc; - list_for_each_entry(subflow, list, node) { + mptcp_for_each_subflow(msk, subflow) { skc = (struct sock_common *)mptcp_subflow_tcp_sock(subflow); mptcp_local_address(skc, &cur); @@ -133,11 +133,11 @@ bool mptcp_lookup_subflow_by_saddr(const struct list_head *list, return false; } -static struct mptcp_pm_add_entry * -mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, - const struct mptcp_addr_info *addr) +static struct mptcp_pm_add_addr * +mptcp_pm_announced_lookup(const struct mptcp_sock *msk, + const struct mptcp_addr_info *addr) { - struct mptcp_pm_add_entry *entry; + struct mptcp_pm_add_addr *entry; lockdep_assert_held(&msk->pm.lock); @@ -149,26 +149,26 @@ mptcp_lookup_anno_list_by_saddr(const struct mptcp_sock *msk, return NULL; } -bool mptcp_remove_anno_list_by_saddr(struct mptcp_sock *msk, - const struct mptcp_addr_info *addr) +bool mptcp_pm_announced_remove(struct mptcp_sock *msk, + const struct mptcp_addr_info *addr) { - struct mptcp_pm_add_entry *entry; + struct mptcp_pm_add_addr *entry; bool ret; - entry = mptcp_pm_del_add_timer(msk, addr, false); + entry = mptcp_pm_announced_del_timer(msk, addr, false); ret = entry; kfree_rcu(entry, rcu); return ret; } -bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk) +bool mptcp_pm_announced_has_ssk(struct mptcp_sock *msk, const struct sock *ssk) { - struct mptcp_pm_add_entry *entry; + struct mptcp_pm_add_addr *entry; struct mptcp_addr_info saddr; bool ret = false; - mptcp_local_address((struct sock_common *)sk, &saddr); + mptcp_local_address((struct sock_common *)ssk, &saddr); spin_lock_bh(&msk->pm.lock); list_for_each_entry(entry, &msk->pm.anno_list, list) { @@ -340,11 +340,12 @@ static unsigned int mptcp_adjust_add_addr_timeout(struct mptcp_sock *msk) static void mptcp_pm_add_timer(struct timer_list *timer) { - struct mptcp_pm_add_entry *entry = timer_container_of(entry, timer, - add_timer); + struct mptcp_pm_add_addr *entry = timer_container_of(entry, timer, + add_timer); struct mptcp_sock *msk = entry->sock; struct sock *sk = (struct sock *)msk; unsigned int timeout = 0; + bool retransmit; pr_debug("msk=%p\n", msk); @@ -364,7 +365,7 @@ static void mptcp_pm_add_timer(struct timer_list *timer) spin_lock_bh(&msk->pm.lock); - /* The cancel path (mptcp_pm_del_add_timer()) can race with this + /* The cancel path (mptcp_pm_announced_del_timer()) can race with this * callback. Once cancel updates retrans_times to MAX, suppress further * retransmissions here. If this callback acquires pm.lock first, one * final transmit attempt is still possible. @@ -377,14 +378,15 @@ static void mptcp_pm_add_timer(struct timer_list *timer) entry->retrans_times++; } - if (entry->retrans_times < ADD_ADDR_RETRANS_MAX) + retransmit = entry->retrans_times < ADD_ADDR_RETRANS_MAX; + if (retransmit) timeout <<= entry->retrans_times; else timeout = 0; spin_unlock_bh(&msk->pm.lock); - if (entry->retrans_times == ADD_ADDR_RETRANS_MAX) + if (!retransmit) mptcp_pm_subflow_established(msk); out: @@ -397,18 +399,18 @@ static void mptcp_pm_add_timer(struct timer_list *timer) sock_put(sk); } -struct mptcp_pm_add_entry * -mptcp_pm_del_add_timer(struct mptcp_sock *msk, - const struct mptcp_addr_info *addr, bool check_id) +struct mptcp_pm_add_addr * +mptcp_pm_announced_del_timer(struct mptcp_sock *msk, + const struct mptcp_addr_info *addr, bool check_id) { - struct mptcp_pm_add_entry *entry; struct sock *sk = (struct sock *)msk; + struct mptcp_pm_add_addr *entry; bool stop_timer = false; rcu_read_lock(); spin_lock_bh(&msk->pm.lock); - entry = mptcp_lookup_anno_list_by_saddr(msk, addr); + entry = mptcp_pm_announced_lookup(msk, addr); if (entry && (!check_id || entry->addr.id == addr->id)) { entry->retrans_times = ADD_ADDR_RETRANS_MAX; stop_timer = true; @@ -431,17 +433,19 @@ mptcp_pm_del_add_timer(struct mptcp_sock *msk, return entry; } -bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, +bool mptcp_pm_announced_alloc(struct mptcp_sock *msk, const struct mptcp_addr_info *addr) { - struct mptcp_pm_add_entry *add_entry = NULL; + struct mptcp_pm_add_addr *add_entry = NULL; struct sock *sk = (struct sock *)msk; unsigned int timeout; lockdep_assert_held(&msk->pm.lock); - add_entry = mptcp_lookup_anno_list_by_saddr(msk, addr); + if (msk->pm.status & BIT(MPTCP_PM_DESTROYING)) + return false; + add_entry = mptcp_pm_announced_lookup(msk, addr); if (add_entry) { if (WARN_ON_ONCE(mptcp_pm_is_kernel(msk))) return false; @@ -457,10 +461,10 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, add_entry->addr = *addr; add_entry->sock = msk; - add_entry->retrans_times = 0; timer_setup(&add_entry->add_timer, mptcp_pm_add_timer, 0); reset_timer: + add_entry->retrans_times = 0; add_entry->timer_done = false; timeout = mptcp_adjust_add_addr_timeout(msk); if (timeout) @@ -469,9 +473,9 @@ bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, return true; } -static void mptcp_pm_free_anno_list(struct mptcp_sock *msk) +static void mptcp_pm_free_announced_list(struct mptcp_sock *msk) { - struct mptcp_pm_add_entry *entry, *tmp; + struct mptcp_pm_add_addr *entry, *tmp; struct sock *sk = (struct sock *)msk; LIST_HEAD(free_list); @@ -736,7 +740,7 @@ void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk, spin_lock_bh(&pm->lock); - if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending)) + if (mptcp_pm_announced_lookup(msk, addr) && READ_ONCE(pm->work_pending)) mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED); spin_unlock_bh(&pm->lock); @@ -945,7 +949,7 @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, unsigned int opt_size, * let the PM state machine progress. */ if (skip_add_addr) { - mptcp_pm_del_add_timer(msk, addr, true); + mptcp_pm_announced_del_timer(msk, addr, true); mptcp_pm_subflow_established(msk); } return ret; @@ -1017,7 +1021,8 @@ bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc) return mptcp_pm_nl_is_backup(msk, &skc_local); } -static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) +static void +mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *iter, *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = (struct sock *)msk; @@ -1054,22 +1059,34 @@ static void mptcp_pm_subflows_chk_stale(const struct mptcp_sock *msk, struct soc } } -void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk) +void mptcp_pm_chk_stale(const struct mptcp_sock *msk) { - struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); - u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); - - /* keep track of rtx periods with no progress */ - if (!subflow->stale_count) { - subflow->stale_rcv_tstamp = rcv_tstamp; - subflow->stale_count++; - } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { - if (subflow->stale_count < U8_MAX) + struct mptcp_subflow_context *subflow; + + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + u32 rcv_tstamp; + + if (!__mptcp_subflow_active(subflow)) + continue; + + /* No data outstanding at TCP level? not stale */ + if (tcp_rtx_and_write_queues_empty(ssk)) + continue; + + /* keep track of rtx periods with no progress */ + rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp); + if (!subflow->stale_count) { + subflow->stale_rcv_tstamp = rcv_tstamp; subflow->stale_count++; - mptcp_pm_subflows_chk_stale(msk, ssk); - } else { - subflow->stale_count = 0; - mptcp_subflow_set_active(subflow); + } else if (subflow->stale_rcv_tstamp == rcv_tstamp) { + if (subflow->stale_count < U8_MAX) + subflow->stale_count++; + mptcp_pm_subflow_chk_stale(msk, ssk); + } else { + subflow->stale_count = 0; + mptcp_subflow_set_active(subflow); + } } } @@ -1100,10 +1117,16 @@ void mptcp_pm_worker(struct mptcp_sock *msk) void mptcp_pm_destroy(struct mptcp_sock *msk) { - mptcp_pm_free_anno_list(msk); + spin_lock_bh(&msk->pm.lock); + msk->pm.status |= BIT(MPTCP_PM_DESTROYING); + spin_unlock_bh(&msk->pm.lock); - if (mptcp_pm_is_userspace(msk)) - mptcp_userspace_pm_free_local_addr_list(msk); + mptcp_pm_free_announced_list(msk); + + /* Free the userspace local address list unconditionally: the socket + * can be reused (mptcp_disconnect()) and re-selected to a different PM + */ + mptcp_userspace_pm_free_local_addr_list(msk); } void mptcp_pm_data_reset(struct mptcp_sock *msk) diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c index a31a845e5cd42..fa34a333a3110 100644 --- a/net/mptcp/pm_kernel.c +++ b/net/mptcp/pm_kernel.c @@ -86,13 +86,13 @@ u8 mptcp_pm_get_limit_extra_subflows(const struct mptcp_sock *msk) } EXPORT_SYMBOL_GPL(mptcp_pm_get_limit_extra_subflows); -static bool lookup_subflow_by_daddr(const struct list_head *list, - const struct mptcp_addr_info *daddr) +static bool has_subflow_daddr(const struct mptcp_sock *msk, + const struct mptcp_addr_info *daddr) { struct mptcp_subflow_context *subflow; struct mptcp_addr_info cur; - list_for_each_entry(subflow, list, node) { + mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (!((1 << inet_sk_state_load(ssk)) & @@ -359,7 +359,7 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk) /* If the alloc fails, we are on memory pressure, not worth * continuing, and trying to create subflows. */ - if (!mptcp_pm_alloc_anno_list(msk, &local.addr)) + if (!mptcp_pm_announced_alloc(msk, &local.addr)) return; __clear_bit(endp_id, msk->pm.id_avail_bitmap); @@ -652,7 +652,7 @@ static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk) mptcp_pm_addr_send_ack(msk); mptcp_mpc_endpoint_setup(msk); - if (lookup_subflow_by_daddr(&msk->conn_list, &remote)) + if (has_subflow_daddr(msk, &remote)) return; /* pick id 0 port, if none is provided the remote address */ @@ -1028,7 +1028,7 @@ int mptcp_pm_nl_add_addr_doit(struct sk_buff *skb, struct genl_info *info) return ret; } -static bool mptcp_pm_remove_anno_addr(struct mptcp_sock *msk, +static bool mptcp_pm_remove_announced(struct mptcp_sock *msk, const struct mptcp_addr_info *addr, bool force) { @@ -1037,7 +1037,7 @@ static bool mptcp_pm_remove_anno_addr(struct mptcp_sock *msk, list.ids[list.nr++] = mptcp_endp_get_local_id(msk, addr); - ret = mptcp_remove_anno_list_by_saddr(msk, addr); + ret = mptcp_pm_announced_remove(msk, addr); if (ret || force) { spin_lock_bh(&msk->pm.lock); if (ret) @@ -1074,8 +1074,8 @@ static int mptcp_nl_remove_subflow_and_signal_addr(struct net *net, goto next; lock_sock(sk); - remove_subflow = mptcp_lookup_subflow_by_saddr(&msk->conn_list, addr); - mptcp_pm_remove_anno_addr(msk, addr, remove_subflow && + remove_subflow = mptcp_pm_has_subflow_saddr(msk, addr); + mptcp_pm_remove_announced(msk, addr, remove_subflow && !(entry->flags & MPTCP_PM_ADDR_FLAG_IMPLICIT)); list.ids[0] = mptcp_endp_get_local_id(msk, addr); @@ -1113,6 +1113,8 @@ static int mptcp_nl_remove_id_zero_address(struct net *net, while ((msk = mptcp_token_iter_next(net, &s_slot, &s_num)) != NULL) { struct sock *sk = (struct sock *)msk; struct mptcp_addr_info msk_local; + struct mptcp_addr_info anno_addr; + bool announced; if (list_empty(&msk->conn_list) || mptcp_pm_is_userspace(msk)) goto next; @@ -1122,7 +1124,13 @@ static int mptcp_nl_remove_id_zero_address(struct net *net, goto next; lock_sock(sk); + /* Drop a possibly pending ADD_ADDR for this address. */ + anno_addr = msk_local; + anno_addr.port = 0; + announced = mptcp_pm_announced_remove(msk, &anno_addr); spin_lock_bh(&msk->pm.lock); + if (announced) + msk->pm.add_addr_signaled--; mptcp_pm_remove_addr(msk, &list); mptcp_pm_rm_subflow(msk, &list); __mark_subflow_endp_available(msk, 0); @@ -1202,11 +1210,11 @@ static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk, list_for_each_entry(entry, rm_list, list) { if (slist.nr < MPTCP_RM_IDS_MAX && - mptcp_lookup_subflow_by_saddr(&msk->conn_list, &entry->addr)) + mptcp_pm_has_subflow_saddr(msk, &entry->addr)) slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr); if (alist.nr < MPTCP_RM_IDS_MAX && - mptcp_remove_anno_list_by_saddr(msk, &entry->addr)) + mptcp_pm_announced_remove(msk, &entry->addr)) alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr); } diff --git a/net/mptcp/pm_userspace.c b/net/mptcp/pm_userspace.c index 0d3a95e676f17..57c13293a5f89 100644 --- a/net/mptcp/pm_userspace.c +++ b/net/mptcp/pm_userspace.c @@ -54,6 +54,10 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, bitmap_zero(id_bitmap, MPTCP_PM_MAX_ADDR_ID + 1); spin_lock_bh(&msk->pm.lock); + if (msk->pm.status & BIT(MPTCP_PM_DESTROYING)) { + ret = -EINVAL; + goto append_err; + } mptcp_for_each_userspace_pm_addr(msk, e) { addr_match = mptcp_addresses_equal(&e->addr, &entry->addr, true); if (addr_match && entry->addr.id == 0 && needs_id) @@ -65,6 +69,19 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, } if (!addr_match && !id_match) { + unsigned int id; + + if (!entry->addr.id && needs_id) { + id = find_next_zero_bit(id_bitmap, + MPTCP_PM_MAX_ADDR_ID + 1, 1); + if (id > MPTCP_PM_MAX_ADDR_ID) { + ret = -ENOSPC; + goto append_err; + } + } else { + id = entry->addr.id; + } + /* Memory for the entry is allocated from the * sock option buffer. */ @@ -74,10 +91,7 @@ static int mptcp_userspace_pm_append_new_local_addr(struct mptcp_sock *msk, goto append_err; } - if (!e->addr.id && needs_id) - e->addr.id = find_next_zero_bit(id_bitmap, - MPTCP_PM_MAX_ADDR_ID + 1, - 1); + e->addr.id = id; list_add_tail_rcu(&e->list, &msk->pm.userspace_pm_local_addr_list); msk->pm.local_addr_used++; ret = e->addr.id; @@ -132,12 +146,15 @@ int mptcp_userspace_pm_get_local_id(struct mptcp_sock *msk, __be16 msk_sport = ((struct inet_sock *) inet_sk((struct sock *)msk))->inet_sport; struct mptcp_pm_addr_entry *entry; + int id; spin_lock_bh(&msk->pm.lock); entry = mptcp_userspace_pm_lookup_addr(msk, &skc->addr); + id = entry ? entry->addr.id : -1; spin_unlock_bh(&msk->pm.lock); - if (entry) - return entry->addr.id; + + if (id != -1) + return id; if (skc->addr.port == msk_sport) skc->addr.port = 0; @@ -228,7 +245,7 @@ int mptcp_pm_nl_announce_doit(struct sk_buff *skb, struct genl_info *info) lock_sock(sk); spin_lock_bh(&msk->pm.lock); - if (mptcp_pm_alloc_anno_list(msk, &addr_val.addr)) { + if (mptcp_pm_announced_alloc(msk, &addr_val.addr)) { msk->pm.add_addr_signaled++; mptcp_pm_announce_addr(msk, &addr_val.addr, false); mptcp_pm_addr_send_ack(msk); @@ -281,9 +298,9 @@ void mptcp_pm_remove_addr_entry(struct mptcp_sock *msk, int anno_nr = 0; /* only delete if either announced or matching a subflow */ - if (mptcp_remove_anno_list_by_saddr(msk, &entry->addr)) + if (mptcp_pm_announced_remove(msk, &entry->addr)) anno_nr++; - else if (!mptcp_lookup_subflow_by_saddr(&msk->conn_list, &entry->addr)) + else if (!mptcp_pm_has_subflow_saddr(msk, &entry->addr)) return; alist.ids[alist.nr++] = entry->addr.id; diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index 72a909bc0fca4..e9967d55b8ace 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -93,6 +93,7 @@ bool __mptcp_try_fallback(struct mptcp_sock *msk, int fb_mib) msk->allow_subflows = false; set_bit(MPTCP_FALLBACK_DONE, &msk->flags); + clear_bit(MPTCP_RTX_ENABLED, &msk->flags); __MPTCP_INC_STATS(net, fb_mib); spin_unlock_bh(&msk->fallback_lock); return true; @@ -147,6 +148,12 @@ struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) static void mptcp_drop(struct sock *sk, struct sk_buff *skb) { + /* The skb forward memory was already transferred to sk by + * mptcp_borrow_fwdmem(), even before setting the destructor. + */ + if (!skb->destructor) + sk_mem_reclaim(sk); + sk_drops_skbadd(sk, skb); __kfree_skb(skb); } @@ -419,9 +426,7 @@ static bool __mptcp_move_skb(struct sock *sk, struct sk_buff *skb) static void mptcp_stop_rtx_timer(struct sock *sk) { - struct inet_connection_sock *icsk = inet_csk(sk); - - sk_stop_timer(sk, &icsk->icsk_retransmit_timer); + sk_stop_timer(sk, &sk->mptcp_retransmit_timer); mptcp_sk(sk)->timer_ival = 0; } @@ -527,7 +532,7 @@ static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subfl const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? - icsk_timeout(inet_csk(ssk)) - jiffies : 0; + tcp_timeout_expires(ssk) - jiffies : 0; } static void mptcp_set_timeout(struct sock *sk) @@ -722,12 +727,12 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, mptcp_dss_corruption(msk, ssk); } } else { + sk_eat_skb(ssk, skb); + if (unlikely(!fin)) { DEBUG_NET_WARN_ON_ONCE(1); mptcp_dss_corruption(msk, ssk); } - - sk_eat_skb(ssk, skb); } WRITE_ONCE(tp->copied_seq, seq); @@ -930,20 +935,20 @@ static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list static bool mptcp_rtx_timer_pending(struct sock *sk) { - return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); + return timer_pending(&sk->mptcp_retransmit_timer); } static void mptcp_reset_rtx_timer(struct sock *sk) { - struct inet_connection_sock *icsk = inet_csk(sk); + struct mptcp_sock *msk = mptcp_sk(sk); unsigned long tout; - /* prevent rescheduling on close */ - if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) + /* Prevent rescheduling on close and in case of fallback. */ + if (!test_bit(MPTCP_RTX_ENABLED, &msk->flags)) return; - tout = mptcp_sk(sk)->timer_ival; - sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); + tout = msk->timer_ival; + sk_reset_timer(sk, &sk->mptcp_retransmit_timer, jiffies + tout); } bool mptcp_schedule_work(struct sock *sk) @@ -1510,7 +1515,9 @@ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) { - tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); + if (info->mss_now) + tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, + info->size_goal); release_sock(ssk); } @@ -1698,7 +1705,8 @@ static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool ret = __subflow_push_pending(sk, ssk, &info); if (ret <= 0) keep_pushing = false; - copied += ret; + else + copied += ret; } mptcp_for_each_subflow(msk, subflow) { @@ -2322,9 +2330,7 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, static void mptcp_retransmit_timer(struct timer_list *t) { - struct inet_connection_sock *icsk = timer_container_of(icsk, t, - icsk_retransmit_timer); - struct sock *sk = &icsk->icsk_inet.sk; + struct sock *sk = timer_container_of(sk, t, mptcp_retransmit_timer); struct mptcp_sock *msk = mptcp_sk(sk); bh_lock_sock(sk); @@ -2342,7 +2348,9 @@ static void mptcp_retransmit_timer(struct timer_list *t) static void mptcp_tout_timer(struct timer_list *t) { - struct sock *sk = timer_container_of(sk, t, sk_timer); + struct inet_connection_sock *icsk = + timer_container_of(icsk, t, mptcp_tout_timer); + struct sock *sk = &icsk->icsk_inet.sk; mptcp_schedule_work(sk); sock_put(sk); @@ -2367,7 +2375,6 @@ struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) /* still data outstanding at TCP level? skip this */ if (!tcp_rtx_and_write_queues_empty(ssk)) { - mptcp_pm_subflow_chk_stale(msk, ssk); min_stale_count = min_t(int, min_stale_count, subflow->stale_count); continue; } @@ -2680,9 +2687,10 @@ static void __mptcp_retrans(struct sock *sk) int ret, err; u16 len = 0; + mptcp_pm_chk_stale(msk); + mptcp_clean_una_wakeup(sk); - /* first check ssk: need to kick "stale" logic */ err = mptcp_sched_get_retrans(msk); dfrag = mptcp_rtx_head(sk); if (!dfrag) { @@ -2798,7 +2806,7 @@ void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) */ timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; - sk_reset_timer(sk, &sk->sk_timer, timeout); + sk_reset_timer(sk, &inet_csk(sk)->mptcp_tout_timer, timeout); } static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) @@ -2845,8 +2853,7 @@ static void mptcp_do_fastclose(struct sock *sk) */ inet_csk(ssk)->icsk_ack.rcv_mss = TCP_MIN_MSS; - tcp_send_active_reset(ssk, ssk->sk_allocation, - SK_RST_REASON_TCP_ABORT_ON_CLOSE); + tcp_send_active_reset(ssk, SK_RST_REASON_TCP_ABORT_ON_CLOSE); unlock: release_sock(ssk); } @@ -2929,8 +2936,8 @@ static void __mptcp_init_sock(struct sock *sk) spin_lock_init(&msk->fallback_lock); /* re-use the csk retrans timer for MPTCP-level retrans */ - timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); - timer_setup(&sk->sk_timer, mptcp_tout_timer, 0); + timer_setup(&sk->mptcp_retransmit_timer, mptcp_retransmit_timer, 0); + timer_setup(&msk->sk.mptcp_tout_timer, mptcp_tout_timer, 0); } static void mptcp_ca_reset(struct sock *sk) @@ -3048,6 +3055,9 @@ void mptcp_set_state(struct sock *sk, int state) * transition from TCP_SYN_RECV to TCP_CLOSE_WAIT. */ break; + case TCP_CLOSE: + clear_bit(MPTCP_RTX_ENABLED, &mptcp_sk(sk)->flags); + fallthrough; default: if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); @@ -3132,7 +3142,7 @@ static void __mptcp_destroy_sock(struct sock *sk) might_sleep(); mptcp_stop_rtx_timer(sk); - sk_stop_timer(sk, &sk->sk_timer); + sk_stop_timer(sk, &inet_csk(sk)->mptcp_tout_timer); msk->pm.status = 0; mptcp_release_sched(msk); @@ -3286,6 +3296,7 @@ static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) static int mptcp_disconnect(struct sock *sk, int flags) { + struct inet_connection_sock *icsk = inet_csk(sk); struct mptcp_sock *msk = mptcp_sk(sk); /* We are on the fastopen error path. We can't call straight into the @@ -3298,8 +3309,13 @@ static int mptcp_disconnect(struct sock *sk, int flags) mptcp_check_listen_stop(sk); mptcp_set_state(sk, TCP_CLOSE); - mptcp_stop_rtx_timer(sk); - mptcp_stop_tout_timer(sk); + /* The later subflow close can not kick again the tout timer, + * as the msk is already in closed status. + */ + msk->timer_ival = icsk->icsk_rto_min; + sk_stop_timer_sync(sk, &sk->mptcp_retransmit_timer); + icsk->icsk_mtup.probe_timestamp = 0; + sk_stop_timer_sync(sk, &icsk->mptcp_tout_timer); mptcp_pm_connection_closed(msk); @@ -3609,7 +3625,7 @@ static void schedule_3rdack_retransmission(struct sock *ssk) struct tcp_sock *tp = tcp_sk(ssk); unsigned long timeout; - if (READ_ONCE(mptcp_subflow_ctx(ssk)->fully_established)) + if (mptcp_subflow_ctx(ssk)->fully_established) return; /* reschedule with a timeout above RTT, as we must look only for drop */ @@ -3758,6 +3774,7 @@ bool mptcp_finish_join(struct sock *ssk) mptcp_data_unlock(parent); if (!ret) { + mptcp_pm_close_subflow(msk); err_prohibited: subflow->reset_reason = MPTCP_RST_EPROHIBIT; return false; @@ -3847,6 +3864,7 @@ static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) if (IS_ERR(ssk)) return PTR_ERR(ssk); + set_bit(MPTCP_RTX_ENABLED, &msk->flags); mptcp_set_state(sk, TCP_SYN_SENT); subflow = mptcp_subflow_ctx(ssk); #ifdef CONFIG_TCP_MD5SIG @@ -3994,6 +4012,7 @@ static int mptcp_listen(struct socket *sock, int backlog) goto unlock; } + set_bit(MPTCP_RTX_ENABLED, &msk->flags); mptcp_set_state(sk, TCP_LISTEN); sock_set_flag(sk, SOCK_RCU_FREE); diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index 3b8edd9b658b0..dc8920a6a90c3 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -37,6 +37,7 @@ OPTION_MPTCP_MPC_ACK) #define OPTIONS_MPTCP_MPJ (OPTION_MPTCP_MPJ_SYN | OPTION_MPTCP_MPJ_SYNACK | \ OPTION_MPTCP_MPJ_ACK) +#define OPTIONS_MPTCP_DSS (OPTION_MPTCP_DSS | OPTION_MPTCP_CSUMREQD) /* MPTCP option subtypes */ #define MPTCPOPT_MP_CAPABLE 0 @@ -115,6 +116,7 @@ #define MPTCP_WORK_RTX 1 #define MPTCP_FALLBACK_DONE 2 #define MPTCP_WORK_CLOSE_SUBFLOW 3 +#define MPTCP_RTX_ENABLED 4 /* MPTCP socket release cb flags */ #define MPTCP_PUSH_PENDING 1 @@ -190,9 +192,10 @@ enum mptcp_pm_status { MPTCP_PM_ESTABLISHED, MPTCP_PM_SUBFLOW_ESTABLISHED, MPTCP_PM_ALREADY_ESTABLISHED, /* persistent status, set after ESTABLISHED event */ - MPTCP_PM_MPC_ENDPOINT_ACCOUNTED /* persistent status, set after MPC local address is - * accounted int id_avail_bitmap - */ + MPTCP_PM_MPC_ENDPOINT_ACCOUNTED, /* persistent status, set after MPC local address is + * accounted int id_avail_bitmap + */ + MPTCP_PM_DESTROYING, /* To fence out PM list allocs */ }; enum mptcp_pm_type { @@ -538,11 +541,11 @@ struct mptcp_subflow_context { remote_key_valid : 1, /* received the peer key from */ disposable : 1, /* ctx can be free at ulp release time */ closing : 1, /* must not pass rx data to msk anymore */ - stale : 1, /* unable to snd/rcv data, do not use for xmit */ valid_csum_seen : 1, /* at least one csum validated */ is_mptfo : 1, /* subflow is doing TFO */ close_event_done : 1, /* has done the post-closed part */ mpc_drop : 1, /* the MPC option has been dropped in a rtx */ + resetting : 1, /* subflow is resetting */ __unused : 8; bool data_avail; bool scheduled; @@ -562,7 +565,11 @@ struct mptcp_subflow_context { u8 reset_seen:1; u8 reset_transient:1; u8 reset_reason:4; - u8 stale_count; + u8 stale_count; /* Protected by the msk socket lock */ + u8 stale; /* Protected by the msk socket lock, + * if set the subflow is unable to snd/rcv + * data, the schedule should skip it + */ u32 subflow_id; @@ -645,7 +652,7 @@ mptcp_send_active_reset_reason(struct sock *sk) enum sk_rst_reason reason; reason = sk_rst_convert_mptcp_reason(subflow->reset_reason); - tcp_send_active_reset(sk, GFP_ATOMIC, reason); + tcp_send_active_reset(sk, reason); } /* Made the fwd mem carried by the given skb available to the msk, @@ -878,7 +885,7 @@ static inline void mptcp_stop_tout_timer(struct sock *sk) if (!inet_csk(sk)->icsk_mtup.probe_timestamp) return; - sk_stop_timer(sk, &sk->sk_timer); + sk_stop_timer(sk, &inet_csk(sk)->mptcp_tout_timer); inet_csk(sk)->icsk_mtup.probe_timestamp = 0; } @@ -1051,7 +1058,7 @@ int mptcp_pm_parse_entry(struct nlattr *attr, struct genl_info *info, bool mptcp_pm_addr_families_match(const struct sock *sk, const struct mptcp_addr_info *loc, const struct mptcp_addr_info *rem); -void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk); +void mptcp_pm_chk_stale(const struct mptcp_sock *msk); void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side); void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk); bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk); @@ -1080,16 +1087,16 @@ int mptcp_pm_mp_prio_send_ack(struct mptcp_sock *msk, struct mptcp_addr_info *addr, struct mptcp_addr_info *rem, u8 bkup); -bool mptcp_pm_alloc_anno_list(struct mptcp_sock *msk, +bool mptcp_pm_announced_alloc(struct mptcp_sock *msk, const struct mptcp_addr_info *addr); -bool mptcp_pm_sport_in_anno_list(struct mptcp_sock *msk, const struct sock *sk); -struct mptcp_pm_add_entry * -mptcp_pm_del_add_timer(struct mptcp_sock *msk, - const struct mptcp_addr_info *addr, bool check_id); -bool mptcp_lookup_subflow_by_saddr(const struct list_head *list, - const struct mptcp_addr_info *saddr); -bool mptcp_remove_anno_list_by_saddr(struct mptcp_sock *msk, - const struct mptcp_addr_info *addr); +struct mptcp_pm_add_addr * +mptcp_pm_announced_del_timer(struct mptcp_sock *msk, + const struct mptcp_addr_info *addr, bool check_id); +bool mptcp_pm_announced_remove(struct mptcp_sock *msk, + const struct mptcp_addr_info *addr); +bool mptcp_pm_announced_has_ssk(struct mptcp_sock *msk, const struct sock *ssk); +bool mptcp_pm_has_subflow_saddr(const struct mptcp_sock *msk, + const struct mptcp_addr_info *saddr); int mptcp_pm_nl_set_flags(struct mptcp_pm_addr_entry *local, struct genl_info *info); int mptcp_userspace_pm_set_flags(struct mptcp_pm_addr_entry *local, diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 42fd759c51df4..1932fcf9f9252 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -235,7 +235,7 @@ static int subflow_check_req(struct request_sock *req, pr_debug("syn inet_sport=%d %d\n", ntohs(inet_sk(sk_listener)->inet_sport), ntohs(inet_sk((struct sock *)subflow_req->msk)->inet_sport)); - if (!mptcp_pm_sport_in_anno_list(subflow_req->msk, sk_listener)) { + if (!mptcp_pm_announced_has_ssk(subflow_req->msk, sk_listener)) { SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTSYNRX); subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); return -EPERM; @@ -441,6 +441,10 @@ void mptcp_subflow_reset(struct sock *ssk) /* must hold: tcp_done() could drop last reference on parent */ sock_hold(sk); + subflow->resetting = 1; + + /* No need to delay the actual close for to-be discarded data. */ + __skb_queue_purge(&ssk->sk_receive_queue); mptcp_send_active_reset_reason(ssk); tcp_done(ssk); if (!test_and_set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &mptcp_sk(sk)->flags)) @@ -928,7 +932,7 @@ static struct sock *subflow_syn_recv_sock(const struct sock *sk, pr_debug("ack inet_sport=%d %d\n", ntohs(inet_sk(sk)->inet_sport), ntohs(inet_sk((struct sock *)owner)->inet_sport)); - if (!mptcp_pm_sport_in_anno_list(owner, sk)) { + if (!mptcp_pm_announced_has_ssk(owner, sk)) { SUBFLOW_REQ_INC_STATS(req, MPTCP_MIB_MISMATCHPORTACKRX); subflow_add_reset_reason(skb, MPTCP_RST_EPROHIBIT); goto dispose_child; @@ -1864,6 +1868,13 @@ static void subflow_state_change(struct sock *sk) __subflow_state_change(sk); + /* Rx queue processing is unneeded, error reporting will take place at + * __mptcp_close_ssk() time and subflow reset can't happen in case of + * fallback: subflow_sched_work_if_closed() would be a no-op. + */ + if (subflow->resetting) + return; + /* as recvmsg() does not acquire the subflow socket for ssk selection * a fin packet carrying a DSS can be unnoticed if we don't trigger * the data available machinery here. @@ -2065,7 +2076,6 @@ static void subflow_ulp_clone(const struct request_sock *req, new_ctx->request_bkup = subflow_req->request_bkup; WRITE_ONCE(new_ctx->remote_id, subflow_req->remote_id); new_ctx->token = subflow_req->token; - new_ctx->thmac = subflow_req->thmac; /* the subflow req id is valid, fetched via subflow_check_req() * and subflow_token_join_request() diff --git a/net/mptcp/syncookies.c b/net/mptcp/syncookies.c index 7f22526346a7e..9474706641c11 100644 --- a/net/mptcp/syncookies.c +++ b/net/mptcp/syncookies.c @@ -26,7 +26,8 @@ struct join_entry { u32 local_nonce; u8 join_id; u8 local_id; - u8 backup; + u8 backup:1, + request_bkup:1; u8 valid; }; @@ -63,6 +64,7 @@ static void mptcp_join_store_state(struct join_entry *entry, entry->remote_nonce = subflow_req->remote_nonce; entry->local_nonce = subflow_req->local_nonce; entry->backup = subflow_req->backup; + entry->request_bkup = subflow_req->request_bkup; entry->join_id = subflow_req->remote_id; entry->local_id = subflow_req->local_id; entry->valid = 1; @@ -117,7 +119,9 @@ bool mptcp_token_join_cookie_init_state(struct mptcp_subflow_request_sock *subfl subflow_req->remote_nonce = e->remote_nonce; subflow_req->local_nonce = e->local_nonce; subflow_req->backup = e->backup; + subflow_req->request_bkup = e->request_bkup; subflow_req->remote_id = e->join_id; + subflow_req->local_id = e->local_id; subflow_req->token = e->token; subflow_req->msk = msk; spin_unlock_bh(&join_entry_locks[i]); diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c index 2f872d064396d..8cc538358f6a3 100644 --- a/net/ncsi/ncsi-netlink.c +++ b/net/ncsi/ncsi-netlink.c @@ -461,6 +461,10 @@ static int ncsi_send_cmd_nl(struct sk_buff *msg, struct genl_info *info) nca.req_flags = NCSI_REQ_FLAG_NETLINK_DRIVEN; nca.info = info; nca.payload = ntohs(hdr->length); + if (nca.payload > len - sizeof(*hdr)) { + ret = -EINVAL; + goto out_netlink; + } nca.data = data + sizeof(*hdr); ret = ncsi_xmit_cmd(&nca); diff --git a/net/netfilter/ipset/ip_set_bitmap_gen.h b/net/netfilter/ipset/ip_set_bitmap_gen.h index 798c7993635e6..ca1119f651efd 100644 --- a/net/netfilter/ipset/ip_set_bitmap_gen.h +++ b/net/netfilter/ipset/ip_set_bitmap_gen.h @@ -77,7 +77,7 @@ mtype_flush(struct ip_set *set) mtype_ext_cleanup(set); bitmap_zero(map->members, map->elements); set->elements = 0; - set->ext_size = 0; + DEBUG_NET_WARN_ON_ONCE(atomic64_read(&set->ext_size) > 0); } /* Calculate the actual memory size of the set data */ @@ -93,7 +93,7 @@ mtype_head(struct ip_set *set, struct sk_buff *skb) { const struct mtype *map = set->data; struct nlattr *nested; - size_t memsize = mtype_memsize(map, set->dsize) + set->ext_size; + size_t memsize = mtype_memsize(map, set->dsize) + atomic64_read(&set->ext_size); nested = nla_nest_start(skb, IPSET_ATTR_DATA); if (!nested) diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c index f51a1af31513c..e7c85b0af729b 100644 --- a/net/netfilter/ipset/ip_set_core.c +++ b/net/netfilter/ipset/ip_set_core.c @@ -350,7 +350,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, size_t len = ext->comment ? strlen(ext->comment) : 0; if (unlikely(c)) { - set->ext_size -= sizeof(*c) + strlen(c->str) + 1; + atomic64_sub(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, NULL); kfree_rcu(c, rcu); } @@ -362,7 +362,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment, if (unlikely(!c)) return; strscpy(c->str, ext->comment, len + 1); - set->ext_size += sizeof(*c) + strlen(c->str) + 1; + atomic64_add(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, c); } EXPORT_SYMBOL_GPL(ip_set_init_comment); @@ -392,7 +392,7 @@ ip_set_comment_free(struct ip_set *set, void *ptr) c = rcu_dereference_protected(comment->c, 1); if (unlikely(!c)) return; - set->ext_size -= sizeof(*c) + strlen(c->str) + 1; + atomic64_sub(sizeof(*c) + strlen(c->str) + 1, &set->ext_size); rcu_assign_pointer(comment->c, NULL); kfree_rcu(c, rcu); } @@ -679,11 +679,18 @@ __ip_set_get(struct ip_set *set) } static void -__ip_set_put(struct ip_set *set) +__ip_set_put_locked(struct ip_set *set) { - write_lock_bh(&ip_set_ref_lock); + lockdep_assert_held(&ip_set_ref_lock); BUG_ON(set->ref == 0); set->ref--; +} + +static void +__ip_set_put(struct ip_set *set) +{ + write_lock_bh(&ip_set_ref_lock); + __ip_set_put_locked(set); write_unlock_bh(&ip_set_ref_lock); } @@ -854,11 +861,11 @@ __ip_set_put_byindex(struct ip_set_net *inst, ip_set_id_t index) { struct ip_set *set; - rcu_read_lock(); - set = rcu_dereference(inst->ip_set_list)[index]; + write_lock_bh(&ip_set_ref_lock); + set = ip_set(inst, index); if (set) - __ip_set_put(set); - rcu_read_unlock(); + __ip_set_put_locked(set); + write_unlock_bh(&ip_set_ref_lock); } void diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h index 0426d012185db..c05c61fba9464 100644 --- a/net/netfilter/ipset/ip_set_hash_gen.h +++ b/net/netfilter/ipset/ip_set_hash_gen.h @@ -569,9 +569,10 @@ mtype_gc(struct work_struct *work) set = gc->set; h = set->data; - spin_lock_bh(&set->lock); - t = ipset_dereference_set(h->table, set); + rcu_read_lock_bh(); + t = rcu_dereference_bh(h->table); atomic_inc(&t->uref); + rcu_read_unlock_bh(); numof_locks = ahash_numof_locks(t->htable_bits); r = gc->region++; if (r >= numof_locks) { @@ -580,7 +581,6 @@ mtype_gc(struct work_struct *work) next_run = (IPSET_GC_PERIOD(set->timeout) * HZ) / numof_locks; if (next_run < HZ/10) next_run = HZ/10; - spin_unlock_bh(&set->lock); mtype_gc_do(set, h, t, r); @@ -860,15 +860,13 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, key = HKEY(value, h->initval, t->htable_bits); r = ahash_region(key); atomic_inc(&t->uref); + rcu_read_unlock_bh(); elements = t->hregion[r].elements; maxelem = t->maxelem; if (elements >= maxelem) { u32 e; - if (SET_WITH_TIMEOUT(set)) { - rcu_read_unlock_bh(); + if (SET_WITH_TIMEOUT(set)) mtype_gc_do(set, h, t, r); - rcu_read_lock_bh(); - } maxelem = h->maxelem; elements = 0; for (e = 0; e < ahash_numof_locks(t->htable_bits); e++) @@ -876,7 +874,6 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, if (elements >= maxelem && SET_WITH_FORCEADD(set)) forceadd = true; } - rcu_read_unlock_bh(); spin_lock_bh(&t->hregion[r].lock); n = rcu_dereference_bh(hbucket(t, key)); @@ -984,7 +981,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, #endif if (SET_WITH_COUNTER(set)) ip_set_init_counter(ext_counter(data, set), ext); - if (SET_WITH_COMMENT(set)) + if (SET_WITH_COMMENT(set) && !ext->target) ip_set_init_comment(set, ext_comment(data, set), ext); if (SET_WITH_SKBINFO(set)) ip_set_init_skbinfo(ext_skbinfo(data, set), ext); @@ -1289,7 +1286,7 @@ mtype_head(struct ip_set *set, struct sk_buff *skb) rcu_read_lock_bh(); t = rcu_dereference_bh(h->table); mtype_ext_size(set, &elements, &ext_size); - memsize = mtype_ahash_memsize(h, t) + ext_size + set->ext_size; + memsize = mtype_ahash_memsize(h, t) + ext_size + atomic64_read(&set->ext_size); htable_bits = t->htable_bits; rcu_read_unlock_bh(); diff --git a/net/netfilter/ipset/ip_set_list_set.c b/net/netfilter/ipset/ip_set_list_set.c index 34bb84d7b174c..2f11814bd512a 100644 --- a/net/netfilter/ipset/ip_set_list_set.c +++ b/net/netfilter/ipset/ip_set_list_set.c @@ -301,9 +301,12 @@ list_set_uadd(struct ip_set *set, void *value, const struct ip_set_ext *ext, e->set = set; INIT_LIST_HEAD(&e->list); list_set_init_extensions(set, ext, e); - if (n) + if (n) { list_set_replace(set, e, n); - else if (next) + return 0; + } + + if (next) list_add_tail_rcu(&e->list, &next->list); else if (prev) list_add_rcu(&e->list, &prev->list); @@ -420,8 +423,7 @@ list_set_flush(struct ip_set *set) list_for_each_entry_safe(e, n, &map->members, list) list_set_del(set, e); - set->elements = 0; - set->ext_size = 0; + DEBUG_NET_WARN_ON_ONCE(set->elements > 0); } static void @@ -455,7 +457,7 @@ list_set_head(struct ip_set *set, struct sk_buff *skb) { const struct list_set *map = set->data; struct nlattr *nested; - size_t memsize = list_set_memsize(map, set->dsize) + set->ext_size; + size_t memsize = list_set_memsize(map, set->dsize) + atomic64_read(&set->ext_size); nested = nla_nest_start(skb, IPSET_ATTR_DATA); if (!nested) diff --git a/net/netfilter/ipvs/ip_vs_app.c b/net/netfilter/ipvs/ip_vs_app.c index f9132b359f0c6..0c690a30a85dc 100644 --- a/net/netfilter/ipvs/ip_vs_app.c +++ b/net/netfilter/ipvs/ip_vs_app.c @@ -368,7 +368,7 @@ static inline int app_tcp_pkt_out(struct ip_vs_conn *cp, struct sk_buff *skb, if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len); + th = (struct tcphdr *)(skb->data + ipvsh->len); /* * Remember seq number in case this pkt gets resized @@ -444,7 +444,7 @@ static inline int app_tcp_pkt_in(struct ip_vs_conn *cp, struct sk_buff *skb, if (skb_ensure_writable(skb, ipvsh->len + sizeof(*th))) return 0; - th = (struct tcphdr *)(skb_network_header(skb) + ipvsh->len); + th = (struct tcphdr *)(skb->data + ipvsh->len); /* * Remember seq number in case this pkt gets resized diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index bc8ff5f15702b..7da11628073b9 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -570,12 +570,6 @@ static inline void ip_vs_bind_xmit_v6(struct ip_vs_conn *cp) #endif -static inline int ip_vs_dest_totalconns(struct ip_vs_dest *dest) -{ - return atomic_read(&dest->activeconns) - + atomic_read(&dest->inactconns); -} - /* * Bind a connection entry with a virtual service destination * Called just after a new connection entry is created. @@ -599,6 +593,9 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) flags = cp->flags; /* Bind with the destination and its corresponding transmitter */ if (flags & IP_VS_CONN_F_SYNC) { + /* Synced conns are hashed, so they can not get this flag */ + conn_flags &= ~IP_VS_CONN_F_ONE_PACKET; + /* if the connection is not template and is created * by sync, preserve the activity flag. */ @@ -624,23 +621,22 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) /* Update the connection counters */ if (!(flags & IP_VS_CONN_F_TEMPLATE)) { + int tc; + /* It is a normal connection, so modify the counters * according to the flags, later the protocol can * update them on state change */ if (!(flags & IP_VS_CONN_F_INACTIVE)) atomic_inc(&dest->activeconns); - else - atomic_inc(&dest->inactconns); + tc = atomic_inc_return(&dest->totalconns); + if (tc == READ_ONCE(dest->u_threshold)) + ip_vs_dest_update_overload(dest, 1); } else { /* It is a persistent connection/template, so increase the persistent connection counter */ atomic_inc(&dest->persistconns); } - - if (dest->u_threshold != 0 && - ip_vs_dest_totalconns(dest) >= dest->u_threshold) - dest->flags |= IP_VS_DEST_F_OVERLOAD; } @@ -721,30 +717,20 @@ static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp) /* Update the connection counters */ if (!(cp->flags & IP_VS_CONN_F_TEMPLATE)) { - /* It is a normal connection, so decrease the inactconns - or activeconns counter */ - if (cp->flags & IP_VS_CONN_F_INACTIVE) { - atomic_dec(&dest->inactconns); - } else { + int tc; + + /* It is a normal connection, so decrease the counters */ + if (!(cp->flags & IP_VS_CONN_F_INACTIVE)) atomic_dec(&dest->activeconns); - } + tc = atomic_fetch_dec(&dest->totalconns); + if (tc == READ_ONCE(dest->l_threshold_val)) + ip_vs_dest_update_overload(dest, -1); } else { /* It is a persistent connection/template, so decrease the persistent connection counter */ atomic_dec(&dest->persistconns); } - if (dest->l_threshold != 0) { - if (ip_vs_dest_totalconns(dest) < dest->l_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else if (dest->u_threshold != 0) { - if (ip_vs_dest_totalconns(dest) * 4 < dest->u_threshold * 3) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } else { - if (dest->flags & IP_VS_DEST_F_OVERLOAD) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - } - ip_vs_dest_put(dest); } @@ -773,7 +759,7 @@ int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest) * Checking the dest server status. */ if ((dest == NULL) || - !(dest->flags & IP_VS_DEST_F_AVAILABLE) || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || expire_quiescent_template(ipvs, dest) || (cdest && (dest != cdest))) { IP_VS_DBG_BUF(9, "check_template: dest not available for " @@ -1422,7 +1408,7 @@ void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs) continue; dest = cp->dest; - if (!dest || (dest->flags & IP_VS_DEST_F_AVAILABLE)) + if (!dest || (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) continue; if (atomic_read(&cp->n_control)) diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 90619453cb6f0..1f62ca2f73309 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -124,7 +124,7 @@ ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -160,7 +160,7 @@ ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb) struct ip_vs_dest *dest = cp->dest; struct netns_ipvs *ipvs = cp->ipvs; - if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (dest && (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { struct ip_vs_cpu_stats *s; struct ip_vs_service *svc; @@ -689,7 +689,7 @@ static int sysctl_nat_icmp_send(struct netns_ipvs *ipvs) { return 0; } #endif -__sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset) +static __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset) { return csum_fold(skb_checksum(skb, offset, skb->len - offset, 0)); } @@ -746,30 +746,28 @@ static int ip_vs_route_me_harder(struct netns_ipvs *ipvs, int af, * - inout: 1=in->out, 0=out->in */ void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int inout) + struct ip_vs_conn *cp, int inout, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph) { struct iphdr *iph = ip_hdr(skb); - unsigned int icmp_offset = iph->ihl*4; - struct icmphdr *icmph = (struct icmphdr *)(skb_network_header(skb) + - icmp_offset); - struct iphdr *ciph = (struct iphdr *)(icmph + 1); + struct icmphdr *icmph = (struct icmphdr *)(skb->data + toff); + struct iphdr *cih = (struct iphdr *)(icmph + 1); if (inout) { iph->saddr = cp->vaddr.ip; ip_send_check(iph); - ciph->daddr = cp->vaddr.ip; - ip_send_check(ciph); + cih->daddr = cp->vaddr.ip; + ip_send_check(cih); } else { iph->daddr = cp->daddr.ip; ip_send_check(iph); - ciph->saddr = cp->daddr.ip; - ip_send_check(ciph); + cih->saddr = cp->daddr.ip; + ip_send_check(cih); } /* the TCP/UDP/SCTP port */ - if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol || - IPPROTO_SCTP == ciph->protocol) { - __be16 *ports = (void *)ciph + ciph->ihl*4; + if (has_ports) { + __be16 *ports = (void *)(skb->data + ciph->len); if (inout) ports[1] = cp->vport; @@ -779,48 +777,40 @@ void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, /* And finally the ICMP checksum */ icmph->checksum = 0; - icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset); + icmph->checksum = ip_vs_checksum_complete(skb, toff); skb->ip_summed = CHECKSUM_UNNECESSARY; if (inout) - IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph, - "Forwarding altered outgoing ICMP"); + IP_VS_DBG_PKT(11, AF_INET, pp, skb, ciph->off, + "Forwarding altered outgoing ICMP"); else - IP_VS_DBG_PKT(11, AF_INET, pp, skb, (void *)ciph - (void *)iph, - "Forwarding altered incoming ICMP"); + IP_VS_DBG_PKT(11, AF_INET, pp, skb, ciph->off, + "Forwarding altered incoming ICMP"); } #ifdef CONFIG_IP_VS_IPV6 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, - struct ip_vs_conn *cp, int inout) + struct ip_vs_conn *cp, int inout, unsigned int toff, + bool has_ports, struct ip_vs_iphdr *ciph) { struct ipv6hdr *iph = ipv6_hdr(skb); - unsigned int icmp_offset = 0; - unsigned int offs = 0; /* header offset*/ - int protocol; struct icmp6hdr *icmph; - struct ipv6hdr *ciph; - unsigned short fragoffs; + struct ipv6hdr *cih; - ipv6_find_hdr(skb, &icmp_offset, IPPROTO_ICMPV6, &fragoffs, NULL); - icmph = (struct icmp6hdr *)(skb_network_header(skb) + icmp_offset); - offs = icmp_offset + sizeof(struct icmp6hdr); - ciph = (struct ipv6hdr *)(skb_network_header(skb) + offs); - - protocol = ipv6_find_hdr(skb, &offs, -1, &fragoffs, NULL); + icmph = (struct icmp6hdr *)(skb->data + toff); + cih = (struct ipv6hdr *)(skb->data + ciph->off); if (inout) { iph->saddr = cp->vaddr.in6; - ciph->daddr = cp->vaddr.in6; + cih->daddr = cp->vaddr.in6; } else { iph->daddr = cp->daddr.in6; - ciph->saddr = cp->daddr.in6; + cih->saddr = cp->daddr.in6; } /* the TCP/UDP/SCTP port */ - if (!fragoffs && (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol || - IPPROTO_SCTP == protocol)) { - __be16 *ports = (void *)(skb_network_header(skb) + offs); + if (has_ports) { + __be16 *ports = (void *)(skb->data + ciph->len); IP_VS_DBG(11, "%s() changed port %d to %d\n", __func__, ntohs(inout ? ports[1] : ports[0]), @@ -833,19 +823,17 @@ void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, /* And finally the ICMP checksum */ icmph->icmp6_cksum = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, - skb->len - icmp_offset, + skb->len - toff, IPPROTO_ICMPV6, 0); - skb->csum_start = skb_network_header(skb) - skb->head + icmp_offset; + skb->csum_start = skb_headroom(skb) + toff; skb->csum_offset = offsetof(struct icmp6hdr, icmp6_cksum); skb->ip_summed = CHECKSUM_PARTIAL; if (inout) - IP_VS_DBG_PKT(11, AF_INET6, pp, skb, - (void *)ciph - (void *)iph, + IP_VS_DBG_PKT(11, AF_INET6, pp, skb, ciph->off, "Forwarding altered outgoing ICMPv6"); else - IP_VS_DBG_PKT(11, AF_INET6, pp, skb, - (void *)ciph - (void *)iph, + IP_VS_DBG_PKT(11, AF_INET6, pp, skb, ciph->off, "Forwarding altered incoming ICMPv6"); } #endif @@ -855,36 +843,41 @@ void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, */ static int handle_response_icmp(int af, struct sk_buff *skb, union nf_inet_addr *snet, - __u8 protocol, struct ip_vs_conn *cp, + struct ip_vs_conn *cp, struct ip_vs_protocol *pp, - unsigned int offset, unsigned int ihl, - unsigned int hooknum) + struct ip_vs_iphdr *ciph, + unsigned int toff, unsigned int hooknum) { + int iproto = af == AF_INET6 ? IPPROTO_ICMPV6 : IPPROTO_ICMP; unsigned int verdict = NF_DROP; + unsigned int ctoff = ciph->len; + bool has_ports = false; if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) goto after_nat; /* Ensure the checksum is correct */ - if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) { + if (!ip_vs_checksum_common_check(skb, toff, iproto, af)) { /* Failed checksum! */ IP_VS_DBG_BUF(1, "Forward ICMP: failed checksum from %s!\n", IP_VS_DBG_ADDR(af, snet)); goto out; } - if (IPPROTO_TCP == protocol || IPPROTO_UDP == protocol || - IPPROTO_SCTP == protocol) - offset += 2 * sizeof(__u16); - if (skb_ensure_writable(skb, offset)) + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + ctoff += 2 * sizeof(__u16); + has_ports = true; + } + if (skb_ensure_writable(skb, ctoff)) goto out; #ifdef CONFIG_IP_VS_IPV6 if (af == AF_INET6) - ip_vs_nat_icmp_v6(skb, pp, cp, 1); + ip_vs_nat_icmp_v6(skb, pp, cp, 1, toff, has_ports, ciph); else #endif - ip_vs_nat_icmp(skb, pp, cp, 1); + ip_vs_nat_icmp(skb, pp, cp, 1, toff, has_ports, ciph); if (ip_vs_route_me_harder(cp->ipvs, af, skb, hooknum)) goto out; @@ -912,15 +905,15 @@ static int handle_response_icmp(int af, struct sk_buff *skb, * Currently handles error types - unreachable, quench, ttl exceeded. */ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, - int *related, unsigned int hooknum) + int *related, unsigned int hooknum, + struct ip_vs_iphdr *ipvsh) { - struct iphdr *iph; struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_iphdr ciph; struct ip_vs_conn *cp; struct ip_vs_protocol *pp; - unsigned int offset, ihl; + unsigned int offset; union nf_inet_addr snet; *related = 1; @@ -929,17 +922,18 @@ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, if (ip_is_fragment(ip_hdr(skb))) { if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum))) return NF_STOLEN; + if (!ip_vs_fill_iph_skb(AF_INET, skb, false, ipvsh)) + return NF_ACCEPT; } - iph = ip_hdr(skb); - offset = ihl = iph->ihl * 4; + offset = ipvsh->len; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); if (ic == NULL) return NF_DROP; IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %pI4->%pI4\n", ic->type, ntohs(icmp_id(ic)), - &iph->saddr, &iph->daddr); + &ipvsh->saddr.ip, &ipvsh->daddr.ip); /* * Work through seeing if this is for us. @@ -957,33 +951,34 @@ static int ip_vs_out_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, /* Now find the contained IP header */ offset += sizeof(_icmph); + if (!ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph)) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (cih == NULL) + if (!(cih && cih->version == 4 && + ciph.len - ciph.off >= sizeof(struct iphdr))) return NF_ACCEPT; /* The packet looks wrong, ignore */ - pp = ip_vs_proto_get(cih->protocol); + pp = ip_vs_proto_get(ciph.protocol); if (!pp) return NF_ACCEPT; /* Is the embedded protocol header present? */ - if (unlikely(cih->frag_off & htons(IP_OFFSET) && - pp->dont_defrag)) + if (unlikely(cih->frag_off & htons(IP_OFFSET) && !pp->dont_defrag)) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset, "Checking outgoing ICMP for"); - ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, true, &ciph); - /* The embedded headers contain source and dest in reverse order */ cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto, ipvs, AF_INET, skb, &ciph); if (!cp) return NF_ACCEPT; - snet.ip = iph->saddr; - return handle_response_icmp(AF_INET, skb, &snet, cih->protocol, cp, - pp, ciph.len, ihl, hooknum); + snet.ip = ipvsh->saddr.ip; + return handle_response_icmp(AF_INET, skb, &snet, cp, pp, &ciph, + ipvsh->len, hooknum); } #ifdef CONFIG_IP_VS_IPV6 @@ -996,7 +991,6 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, struct ip_vs_conn *cp; struct ip_vs_protocol *pp; union nf_inet_addr snet; - unsigned int offset; *related = 1; ic = frag_safe_skb_hp(skb, ipvsh->len, sizeof(_icmph), &_icmph); @@ -1032,6 +1026,10 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, if (!pp) return NF_ACCEPT; + /* Is the embedded protocol header present? */ + if (unlikely(ciph.fragoffs && !pp->dont_defrag)) + return NF_ACCEPT; + /* The embedded headers contain source and dest in reverse order */ cp = INDIRECT_CALL_1(pp->conn_out_get, ip_vs_conn_out_get_proto, ipvs, AF_INET6, skb, &ciph); @@ -1039,9 +1037,8 @@ static int ip_vs_out_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, return NF_ACCEPT; snet.in6 = ciph.saddr.in6; - offset = ciph.len; - return handle_response_icmp(AF_INET6, skb, &snet, ciph.protocol, cp, - pp, offset, ipvsh->len, hooknum); + return handle_response_icmp(AF_INET6, skb, &snet, cp, pp, &ciph, + ipvsh->len, hooknum); } #endif @@ -1367,7 +1364,8 @@ ip_vs_out_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *stat #endif if (unlikely(iph.protocol == IPPROTO_ICMP)) { int related; - int verdict = ip_vs_out_icmp(ipvs, skb, &related, hooknum); + int verdict = ip_vs_out_icmp(ipvs, skb, &related, + hooknum, &iph); if (related) return verdict; @@ -1575,9 +1573,8 @@ static int ipvs_gre_decap(struct netns_ipvs *ipvs, struct sk_buff *skb, */ static int ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, - unsigned int hooknum) + unsigned int hooknum, struct ip_vs_iphdr *iph) { - struct iphdr *iph; struct icmphdr _icmph, *ic; struct iphdr _ciph, *cih; /* The ip header contained within the ICMP */ struct ip_vs_iphdr ciph; @@ -1587,7 +1584,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, unsigned int offset, offset2, ihl, verdict; bool tunnel, new_cp = false; union nf_inet_addr *raddr; - char *outer_proto = "IPIP"; + char *outer_proto __maybe_unused = "IPIP"; unsigned int hlen_ipip; int ulen = 0; @@ -1597,17 +1594,19 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (ip_is_fragment(ip_hdr(skb))) { if (ip_vs_gather_frags(ipvs, skb, ip_vs_defrag_user(hooknum))) return NF_STOLEN; + if (!ip_vs_fill_iph_skb(AF_INET, skb, false, iph)) + return NF_ACCEPT; } - iph = ip_hdr(skb); - offset = ihl = iph->ihl * 4; + ihl = iph->len; + offset = iph->len; ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph); if (ic == NULL) return NF_DROP; IP_VS_DBG(12, "Incoming ICMP (%d,%d) %pI4->%pI4\n", ic->type, ntohs(icmp_id(ic)), - &iph->saddr, &iph->daddr); + &iph->saddr.ip, &iph->daddr.ip); /* * Work through seeing if this is for us. @@ -1626,10 +1625,12 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Now find the contained IP header */ offset += sizeof(_icmph); cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (!(cih && cih->version == 4 && cih->ihl >= 5)) + if (!cih) return NF_ACCEPT; /* The packet looks wrong, ignore */ - raddr = (union nf_inet_addr *)&cih->daddr; hlen_ipip = cih->ihl * 4; + if (!(cih->version == 4 && hlen_ipip >= sizeof(struct iphdr))) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + raddr = (union nf_inet_addr *)&cih->daddr; /* Special case for errors for IPIP/UDP/GRE tunnel packets */ tunnel = false; @@ -1646,9 +1647,6 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (!dest || dest->tun_type != IP_VS_CONN_F_TUNNEL_TYPE_IPIP) return NF_ACCEPT; offset += hlen_ipip; - cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); - if (!(cih && cih->version == 4 && cih->ihl >= 5)) - return NF_ACCEPT; /* The packet looks wrong, ignore */ tunnel = true; } else if ((cih->protocol == IPPROTO_UDP || /* Can be UDP encap */ cih->protocol == IPPROTO_GRE) && /* Can be GRE encap */ @@ -1673,33 +1671,32 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* Skip IP and UDP/GRE tunnel headers */ offset = offset2 + ulen; /* Now we should be at the original IP header */ - cih = skb_header_pointer(skb, offset, sizeof(_ciph), - &_ciph); - if (cih && cih->version == 4 && cih->ihl >= 5 && - iproto == IPPROTO_IPIP) + if (iproto == IPPROTO_IPIP) tunnel = true; else return NF_ACCEPT; } } - pd = ip_vs_proto_data_get(ipvs, cih->protocol); + if (!ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph)) + return NF_ACCEPT; + pd = ip_vs_proto_data_get(ipvs, ciph.protocol); if (!pd) return NF_ACCEPT; pp = pd->pp; + cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph); + if (!(cih && cih->version == 4 && + ciph.len - ciph.off >= sizeof(struct iphdr))) + return NF_ACCEPT; /* The packet looks wrong, ignore */ + /* Is the embedded protocol header present? */ - if (unlikely(cih->frag_off & htons(IP_OFFSET) && - pp->dont_defrag)) + if (unlikely(cih->frag_off & htons(IP_OFFSET) && !pp->dont_defrag)) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET, pp, skb, offset, "Checking incoming ICMP for"); - offset2 = offset; - ip_vs_fill_iph_skb_icmp(AF_INET, skb, offset, !tunnel, &ciph); - offset = ciph.len; - /* The embedded headers contain source and dest in reverse order. * For IPIP/UDP/GRE tunnel this is error for request, not for reply. */ @@ -1720,19 +1717,21 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, verdict = NF_DROP; /* Ensure the checksum is correct */ - if (!skb_csum_unnecessary(skb) && ip_vs_checksum_complete(skb, ihl)) { + if ((IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ || tunnel) && + !ip_vs_checksum_common_check(skb, ihl, IPPROTO_ICMP, AF_INET)) { /* Failed checksum! */ IP_VS_DBG(1, "Incoming ICMP: failed checksum from %pI4!\n", - &iph->saddr); + &iph->saddr.ip); goto out; } if (tunnel) { - unsigned int hlen_orig = cih->ihl * 4; + unsigned int hlen_orig = ciph.len - ciph.off; __be32 info = ic->un.gateway; __u8 type = ic->type; __u8 code = ic->code; + offset2 = offset; /* Update the MTU */ if (ic->type == ICMP_DEST_UNREACH && ic->code == ICMP_FRAG_NEEDED) { @@ -1773,6 +1772,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, if (pskb_pull(skb, offset2) == NULL) goto ignore_tunnel; skb_reset_network_header(skb); + memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt)); /* Ensure the IP header is present in headroom */ if (!pskb_may_pull(skb, hlen_orig)) goto ignore_tunnel; @@ -1791,10 +1791,7 @@ ip_vs_in_icmp(struct netns_ipvs *ipvs, struct sk_buff *skb, int *related, /* do the statistics and put it back */ ip_vs_in_stats(cp, skb); - if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol || - IPPROTO_SCTP == cih->protocol) - offset += 2 * sizeof(__u16); - verdict = ip_vs_icmp_xmit(skb, cp, pp, offset, hooknum, &ciph); + verdict = ip_vs_icmp_xmit(skb, cp, pp, iph->len, hooknum, &ciph); out: if (likely(!new_cp)) @@ -1854,8 +1851,8 @@ static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, return NF_ACCEPT; pp = pd->pp; - /* Cannot handle fragmented embedded protocol */ - if (ciph.fragoffs) + /* Is the embedded protocol header present? */ + if (ciph.fragoffs && !pp->dont_defrag) return NF_ACCEPT; IP_VS_DBG_PKT(11, AF_INET6, pp, skb, offset, @@ -1879,23 +1876,22 @@ static int ip_vs_in_icmp_v6(struct netns_ipvs *ipvs, struct sk_buff *skb, new_cp = true; } - /* VS/TUN, VS/DR and LOCALNODE just let it go */ - if ((hooknum == NF_INET_LOCAL_OUT) && - (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ)) { - verdict = NF_ACCEPT; + verdict = NF_DROP; + + /* Ensure the checksum is correct */ + if (IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && + !ip_vs_checksum_common_check(skb, iph->len, IPPROTO_ICMPV6, + AF_INET6)) { + /* Failed checksum! */ + IP_VS_DBG(1, "Incoming ICMPv6: failed checksum from %pI6c!\n", + &iph->saddr); goto out; } /* do the statistics and put it back */ ip_vs_in_stats(cp, skb); - /* Need to mangle contained IPv6 header in ICMPv6 packet */ - offset = ciph.len; - if (IPPROTO_TCP == ciph.protocol || IPPROTO_UDP == ciph.protocol || - IPPROTO_SCTP == ciph.protocol) - offset += 2 * sizeof(__u16); /* Also mangle ports */ - - verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, offset, hooknum, &ciph); + verdict = ip_vs_icmp_xmit_v6(skb, cp, pp, iph->len, hooknum, &ciph); out: if (likely(!new_cp)) @@ -1974,7 +1970,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state if (unlikely(iph.protocol == IPPROTO_ICMP)) { int related; int verdict = ip_vs_in_icmp(ipvs, skb, &related, - hooknum); + hooknum, &iph); if (related) return verdict; @@ -2034,7 +2030,7 @@ ip_vs_in_hook(void *priv, struct sk_buff *skb, const struct nf_hook_state *state } /* Check the server status */ - if (cp && cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) { + if (cp && cp->dest && !(cp->dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { /* the destination server is not available */ if (sysctl_expire_nodest_conn(ipvs)) { bool old_ct = ip_vs_conn_uses_old_conntrack(cp, skb); @@ -2110,6 +2106,7 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, const struct nf_hook_state *state) { struct netns_ipvs *ipvs = net_ipvs(state->net); + struct ip_vs_iphdr iphdr; int r; /* ipvs enabled in this netns ? */ @@ -2119,10 +2116,9 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, if (state->pf == NFPROTO_IPV4) { if (ip_hdr(skb)->protocol != IPPROTO_ICMP) return NF_ACCEPT; + ip_vs_fill_iph_skb(AF_INET, skb, false, &iphdr); #ifdef CONFIG_IP_VS_IPV6 } else { - struct ip_vs_iphdr iphdr; - ip_vs_fill_iph_skb(AF_INET6, skb, false, &iphdr); if (iphdr.protocol != IPPROTO_ICMPV6) @@ -2132,7 +2128,7 @@ ip_vs_forward_icmp(void *priv, struct sk_buff *skb, #endif } - return ip_vs_in_icmp(ipvs, skb, &r, state->hook); + return ip_vs_in_icmp(ipvs, skb, &r, state->hook, &iphdr); } static const struct nf_hook_ops ip_vs_ops4[] = { diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index a3b509908b8c0..c0257ef4d8b03 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -960,6 +960,40 @@ void ip_vs_stats_free(struct ip_vs_stats *stats) } } +/* Update overload flag based on number of dest conns and lower/upper + * connection thresholds: + * - conns reach u_threshold and exceed it: set the flag + * - conns go below l_threshold (or 75% of u_threshold): clear the flag + */ +static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + int conns; + u32 l, u; + + lockdep_assert_held(&dest->dst_lock); + u = READ_ONCE(dest->u_threshold); + if (!u) + goto unset; + l = READ_ONCE(dest->l_threshold_val); + conns = atomic_read(&dest->totalconns); + if (conns >= (mode > 0 ? l : u)) { + dest->flags |= IP_VS_DEST_F_OVERLOAD; + return; + } + if (conns >= (mode < 0 ? u : l)) + return; + +unset: + dest->flags &= ~IP_VS_DEST_F_OVERLOAD; +} + +void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode) +{ + spin_lock_bh(&dest->dst_lock); + __ip_vs_dest_update_overload(dest, mode); + spin_unlock_bh(&dest->dst_lock); +} + /* * Update a destination in the given service */ @@ -1024,12 +1058,21 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, } /* set the dest status flags */ - dest->flags |= IP_VS_DEST_F_AVAILABLE; - - if (udest->u_threshold == 0 || udest->u_threshold > dest->u_threshold) - dest->flags &= ~IP_VS_DEST_F_OVERLOAD; - dest->u_threshold = udest->u_threshold; - dest->l_threshold = udest->l_threshold; + dest->cflags |= IP_VS_DEST_CF_AVAILABLE; + + if (READ_ONCE(dest->u_threshold) != udest->u_threshold || + READ_ONCE(dest->l_threshold) != udest->l_threshold) { + spin_lock_bh(&dest->dst_lock); + WRITE_ONCE(dest->u_threshold, udest->u_threshold); + WRITE_ONCE(dest->l_threshold, udest->l_threshold); + /* Low threshold defaults to 75% of upper threshold */ + WRITE_ONCE(dest->l_threshold_val, + udest->l_threshold ? : + (udest->u_threshold - + (udest->u_threshold >> 2))); + __ip_vs_dest_update_overload(dest, 0); + spin_unlock_bh(&dest->dst_lock); + } dest->af = udest->af; @@ -1101,7 +1144,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) dest->port = udest->port; atomic_set(&dest->activeconns, 0); - atomic_set(&dest->inactconns, 0); + atomic_set(&dest->totalconns, 0); atomic_set(&dest->persistconns, 0); refcount_set(&dest->refcnt, 1); @@ -1142,6 +1185,9 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1212,6 +1258,9 @@ ip_vs_edit_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) return -ERANGE; } + if (udest->u_threshold > INT_MAX) + return -EINVAL; + if (udest->tun_type == IP_VS_CONN_F_TUNNEL_TYPE_GUE) { if (udest->tun_port == 0) { pr_err("%s(): tunnel port is zero\n", __func__); @@ -1276,7 +1325,7 @@ static void __ip_vs_unlink_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, int svcupd) { - dest->flags &= ~IP_VS_DEST_F_AVAILABLE; + dest->cflags &= ~IP_VS_DEST_CF_AVAILABLE; /* * Remove it from the d-linked destination list. @@ -2457,7 +2506,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); else #endif seq_printf(seq, @@ -2468,7 +2517,7 @@ static int ip_vs_info_seq_show(struct seq_file *seq, void *v) ip_vs_fwd_name(atomic_read(&dest->conn_flags)), atomic_read(&dest->weight), atomic_read(&dest->activeconns), - atomic_read(&dest->inactconns)); + ip_vs_dest_inactconns(dest)); } } @@ -2951,10 +3000,10 @@ __ip_vs_get_dest_entries(struct netns_ipvs *ipvs, const struct ip_vs_get_dests * entry.port = dest->port; entry.conn_flags = atomic_read(&dest->conn_flags); entry.weight = atomic_read(&dest->weight); - entry.u_threshold = dest->u_threshold; - entry.l_threshold = dest->l_threshold; + entry.u_threshold = READ_ONCE(dest->u_threshold); + entry.l_threshold = READ_ONCE(dest->l_threshold); entry.activeconns = atomic_read(&dest->activeconns); - entry.inactconns = atomic_read(&dest->inactconns); + entry.inactconns = ip_vs_dest_inactconns(dest); entry.persistconns = atomic_read(&dest->persistconns); ip_vs_copy_stats(&kstats, &dest->stats); ip_vs_export_stats_user(&entry.stats, &kstats); @@ -3553,12 +3602,14 @@ static int ip_vs_genl_fill_dest(struct sk_buff *skb, struct ip_vs_dest *dest) dest->tun_port) || nla_put_u16(skb, IPVS_DEST_ATTR_TUN_FLAGS, dest->tun_flags) || - nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, dest->u_threshold) || - nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, dest->l_threshold) || + nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, + READ_ONCE(dest->u_threshold)) || + nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, + READ_ONCE(dest->l_threshold)) || nla_put_u32(skb, IPVS_DEST_ATTR_ACTIVE_CONNS, atomic_read(&dest->activeconns)) || nla_put_u32(skb, IPVS_DEST_ATTR_INACT_CONNS, - atomic_read(&dest->inactconns)) || + ip_vs_dest_inactconns(dest)) || nla_put_u32(skb, IPVS_DEST_ATTR_PERSIST_CONNS, atomic_read(&dest->persistconns)) || nla_put_u16(skb, IPVS_DEST_ATTR_ADDR_FAMILY, dest->af)) diff --git a/net/netfilter/ipvs/ip_vs_dh.c b/net/netfilter/ipvs/ip_vs_dh.c index 75f4c231f4a02..f04d2530a093a 100644 --- a/net/netfilter/ipvs/ip_vs_dh.c +++ b/net/netfilter/ipvs/ip_vs_dh.c @@ -220,8 +220,8 @@ ip_vs_dh_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, s = (struct ip_vs_dh_state *) svc->sched_data; dest = ip_vs_dh_get(svc->af, s, &iph->daddr); - if (!dest - || !(dest->flags & IP_VS_DEST_F_AVAILABLE) + if (!dest || + !(dest->cflags & IP_VS_DEST_CF_AVAILABLE) || atomic_read(&dest->weight) <= 0 || is_overloaded(dest)) { ip_vs_scheduler_err(svc, "no destination available"); diff --git a/net/netfilter/ipvs/ip_vs_est.c b/net/netfilter/ipvs/ip_vs_est.c index 93a925f1ed9b8..5eec865588f92 100644 --- a/net/netfilter/ipvs/ip_vs_est.c +++ b/net/netfilter/ipvs/ip_vs_est.c @@ -187,8 +187,11 @@ static int ip_vs_estimation_kthread(void *data) } /* kthread 0 will handle the calc phase */ - if (ipvs->est_calc_phase) + if (ipvs->est_calc_phase) { ip_vs_est_calc_phase(ipvs); + if (kthread_should_stop() || !READ_ONCE(ipvs->enable)) + return 0; + } } while (1) { @@ -263,6 +266,7 @@ int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, kd->task = NULL; goto out; } + get_task_struct(kd->task); set_user_nice(kd->task, sysctl_est_nice(ipvs)); if (sysctl_est_preferred_cpulist(ipvs)) @@ -279,7 +283,7 @@ void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd) { if (kd->task) { pr_info("stopping estimator thread %d...\n", kd->id); - kthread_stop(kd->task); + kthread_stop_put(kd->task); kd->task = NULL; } } @@ -511,7 +515,7 @@ static void ip_vs_est_kthread_destroy(struct ip_vs_est_kt_data *kd) if (kd) { if (kd->task) { pr_info("stop unused estimator thread %d...\n", kd->id); - kthread_stop(kd->task); + kthread_stop_put(kd->task); } ip_vs_stats_free(kd->calc_stats); kfree(kd); diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index 206c6700e2006..d3be352a5ea8c 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -103,7 +103,7 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, char *s, c; unsigned char p[6]; char edelim; - __u16 hport; + __u32 hport; int i = 0; if (data_limit - data < plen) { @@ -145,7 +145,11 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, return -1; c = *data; if (isdigit(c)) { - p[i] = p[i]*10 + c - '0'; + unsigned int val = p[i] * 10 + c - '0'; + + if (val > 255) + return -1; + p[i] = val; } else if (c == ',' && i < 5) { i++; p[i] = 0; @@ -223,6 +227,8 @@ static int ip_vs_ftp_get_addrport(char *data, char *data_limit, if (!isdigit(*s)) break; hport = hport * 10 + *s - '0'; + if (hport > 65535) + return -1; } if (s == data_limit || !hport || *s != edelim) return -1; diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index 156181a3bacd7..9bb1e05063dc4 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -503,7 +503,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, */ dest = en->dest; - if ((dest->flags & IP_VS_DEST_F_AVAILABLE) && + if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) && atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc)) goto out; } diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index a021e6aba3d7b..1a53c1254fe57 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -170,8 +170,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) if (least->flags & IP_VS_DEST_F_OVERLOAD) continue; - if ((atomic_read(&least->weight) > 0) - && (least->flags & IP_VS_DEST_F_AVAILABLE)) { + if ((atomic_read(&least->weight) > 0) && + (least->cflags & IP_VS_DEST_CF_AVAILABLE)) { loh = ip_vs_dest_conn_overhead(least); goto nextstage; } @@ -187,8 +187,8 @@ static inline struct ip_vs_dest *ip_vs_dest_set_min(struct ip_vs_dest_set *set) doh = ip_vs_dest_conn_overhead(dest); if (((__s64)loh * atomic_read(&dest->weight) > - (__s64)doh * atomic_read(&least->weight)) - && (dest->flags & IP_VS_DEST_F_AVAILABLE)) { + (__s64)doh * atomic_read(&least->weight)) && + (dest->cflags & IP_VS_DEST_CF_AVAILABLE)) { least = dest; loh = doh; } diff --git a/net/netfilter/ipvs/ip_vs_lc.c b/net/netfilter/ipvs/ip_vs_lc.c index c2764505e380f..cd1a860ebaec4 100644 --- a/net/netfilter/ipvs/ip_vs_lc.c +++ b/net/netfilter/ipvs/ip_vs_lc.c @@ -31,7 +31,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, /* * Simply select the server with the least number of - * (activeconns<<5) + inactconns + * (activeconns*256) + totalconns * Except whose weight is equal to zero. * If the weight is equal to zero, it means that the server is * quiesced, the existing connections to the server still get @@ -57,7 +57,7 @@ ip_vs_lc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, IP_VS_DBG_ADDR(least->af, &least->addr), ntohs(least->port), atomic_read(&least->activeconns), - atomic_read(&least->inactconns)); + ip_vs_dest_inactconns(least)); return least; } diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index c67317be17dfa..fb8af6b15a399 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -11,7 +11,7 @@ static int sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int sctphoff); + struct ip_vs_iphdr *iph); static int sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -109,7 +109,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) + if (!sctp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -121,7 +121,7 @@ sctp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - sctph = (void *) skb_network_header(skb) + sctphoff; + sctph = (void *)skb->data + sctphoff; /* Only update csum if we really have to */ if (sctph->source != cp->vport || payload_csum || @@ -157,7 +157,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!sctp_csum_check(cp->af, skb, pp, sctphoff)) + if (!sctp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -169,7 +169,7 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - sctph = (void *) skb_network_header(skb) + sctphoff; + sctph = (void *)skb->data + sctphoff; /* Only update csum if we really have to */ if (sctph->dest != cp->dport || payload_csum || @@ -187,19 +187,22 @@ sctp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int sctp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int sctphoff) + struct ip_vs_iphdr *iph) { + unsigned int sctphoff = iph->len; struct sctphdr *sh; __le32 cmp, val; + if (!ip_vs_checksum_needed(skb)) + return 1; sh = (struct sctphdr *)(skb->data + sctphoff); cmp = sh->checksum; val = sctp_compute_cksum(skb, sctphoff); if (val != cmp) { /* CRC failure, dump it. */ - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); return 0; } return 1; @@ -443,12 +446,10 @@ set_sctp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state != IP_VS_SCTP_S_ESTABLISHED)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && (next_state == IP_VS_SCTP_S_ESTABLISHED)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index b382810156b2c..944efd34290cd 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -30,7 +30,7 @@ static int tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int tcphoff); + struct ip_vs_iphdr *iph); static int tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -167,7 +167,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) + if (!tcp_csum_check(cp->af, skb, pp, iph)) return 0; /* Call application helper if needed */ @@ -180,7 +180,7 @@ tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - tcph = (void *)skb_network_header(skb) + tcphoff; + tcph = (void *)skb->data + tcphoff; tcph->source = cp->vport; /* Adjust TCP checksums */ @@ -245,7 +245,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!tcp_csum_check(cp->af, skb, pp, tcphoff)) + if (!tcp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -261,7 +261,7 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - tcph = (void *)skb_network_header(skb) + tcphoff; + tcph = (void *)skb->data + tcphoff; tcph->dest = cp->dport; /* @@ -303,41 +303,13 @@ tcp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int tcp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int tcphoff) + struct ip_vs_iphdr *iph) { - switch (skb->ip_summed) { - case CHECKSUM_NONE: - skb->csum = skb_checksum(skb, tcphoff, skb->len - tcphoff, 0); - fallthrough; - case CHECKSUM_COMPLETE: -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) { - if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, - &ipv6_hdr(skb)->daddr, - skb->len - tcphoff, - IPPROTO_TCP, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - } else -#endif - if (csum_tcpudp_magic(ip_hdr(skb)->saddr, - ip_hdr(skb)->daddr, - skb->len - tcphoff, - ip_hdr(skb)->protocol, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - break; - default: - /* No need to checksum. */ - break; + if (!ip_vs_checksum_common_check(skb, iph->len, IPPROTO_TCP, af)) { + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); + return 0; } - return 1; } @@ -555,12 +527,10 @@ set_tcp_state(struct ip_vs_proto_data *pd, struct ip_vs_conn *cp, if (!(cp->flags & IP_VS_CONN_F_INACTIVE) && !tcp_state_active(new_state)) { atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); cp->flags |= IP_VS_CONN_F_INACTIVE; } else if ((cp->flags & IP_VS_CONN_F_INACTIVE) && tcp_state_active(new_state)) { atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); cp->flags &= ~IP_VS_CONN_F_INACTIVE; } } diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index dbd4155bb0752..96ac882df15c1 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -26,7 +26,7 @@ static int udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int udphoff); + struct ip_vs_iphdr *iph); static int udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, @@ -156,7 +156,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp, udphoff)) + if (!udp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -171,7 +171,7 @@ udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - udph = (void *)skb_network_header(skb) + udphoff; + udph = (void *)skb->data + udphoff; udph->source = cp->vport; /* @@ -239,7 +239,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, int ret; /* Some checks before mangling */ - if (!udp_csum_check(cp->af, skb, pp, udphoff)) + if (!udp_csum_check(cp->af, skb, pp, iph)) return 0; /* @@ -255,7 +255,7 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, payload_csum = true; } - udph = (void *)skb_network_header(skb) + udphoff; + udph = (void *)skb->data + udphoff; udph->dest = cp->dport; /* @@ -299,48 +299,20 @@ udp_dnat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, static int udp_csum_check(int af, struct sk_buff *skb, struct ip_vs_protocol *pp, - unsigned int udphoff) + struct ip_vs_iphdr *iph) { struct udphdr _udph, *uh; - uh = skb_header_pointer(skb, udphoff, sizeof(_udph), &_udph); + uh = skb_header_pointer(skb, iph->len, sizeof(_udph), &_udph); if (uh == NULL) return 0; - if (uh->check != 0) { - switch (skb->ip_summed) { - case CHECKSUM_NONE: - skb->csum = skb_checksum(skb, udphoff, - skb->len - udphoff, 0); - fallthrough; - case CHECKSUM_COMPLETE: -#ifdef CONFIG_IP_VS_IPV6 - if (af == AF_INET6) { - if (csum_ipv6_magic(&ipv6_hdr(skb)->saddr, - &ipv6_hdr(skb)->daddr, - skb->len - udphoff, - IPPROTO_UDP, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - } else -#endif - if (csum_tcpudp_magic(ip_hdr(skb)->saddr, - ip_hdr(skb)->daddr, - skb->len - udphoff, - ip_hdr(skb)->protocol, - skb->csum)) { - IP_VS_DBG_RL_PKT(0, af, pp, skb, 0, - "Failed checksum for"); - return 0; - } - break; - default: - /* No need to checksum. */ - break; - } + if (!uh->check) + return 1; + if (!ip_vs_checksum_common_check(skb, iph->len, IPPROTO_UDP, af)) { + IP_VS_DBG_RL_PKT(0, af, pp, skb, iph->off, + "Failed checksum for"); + return 0; } return 1; } diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 3402675bf5215..699651e84d832 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -748,9 +748,9 @@ void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) if (cp->flags & IP_VS_CONN_F_SEQ_MASK) { *(p++) = IPVS_OPT_SEQ_DATA; *(p++) = sizeof(struct ip_vs_sync_conn_options); - hton_seq((struct ip_vs_seq *)p, &cp->in_seq); + hton_seq(&cp->in_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); - hton_seq((struct ip_vs_seq *)p, &cp->out_seq); + hton_seq(&cp->out_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); } /* Handle pe data */ @@ -879,13 +879,10 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa spin_lock_bh(&cp->lock); if ((cp->flags ^ flags) & IP_VS_CONN_F_INACTIVE && !(flags & IP_VS_CONN_F_TEMPLATE) && dest) { - if (flags & IP_VS_CONN_F_INACTIVE) { + if (flags & IP_VS_CONN_F_INACTIVE) atomic_dec(&dest->activeconns); - atomic_inc(&dest->inactconns); - } else { + else atomic_inc(&dest->activeconns); - atomic_dec(&dest->inactconns); - } } flags &= IP_VS_CONN_F_BACKUP_UPD_MASK; flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK; @@ -1003,10 +1000,10 @@ static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer pp->name, state); continue; } - } else { - if (state >= IP_VS_CTPL_S_LAST) - IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n", - state); + } else if (state >= IP_VS_CTPL_S_LAST) { + IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n", + state); + continue; } ip_vs_conn_fill_param(ipvs, AF_INET, s->protocol, @@ -1163,10 +1160,10 @@ static inline int ip_vs_proc_sync_conn(struct netns_ipvs *ipvs, __u8 *p, __u8 *m retc = 40; goto out; } - } else { - if (state >= IP_VS_CTPL_S_LAST) - IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n", - state); + } else if (state >= IP_VS_CTPL_S_LAST) { + IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n", state); + retc = 40; + goto out; } if (ip_vs_conn_fill_param_sync(ipvs, af, s, ¶m, pe_data, pe_data_len, pe_name, pe_name_len)) { diff --git a/net/netfilter/ipvs/ip_vs_xmit.c b/net/netfilter/ipvs/ip_vs_xmit.c index f564b4197cee9..7db74b2ea358c 100644 --- a/net/netfilter/ipvs/ip_vs_xmit.c +++ b/net/netfilter/ipvs/ip_vs_xmit.c @@ -1499,20 +1499,22 @@ ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, */ int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, unsigned int hooknum, - struct ip_vs_iphdr *iph) + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph) { struct rtable *rt; /* Route to the other host */ int rc; int local; int rt_mode, was_input; + bool has_ports = false; + unsigned int wlen; /* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be forwarded directly here, because there is no need to translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { if (cp->packet_xmit) - rc = cp->packet_xmit(skb, cp, pp, iph); + rc = cp->packet_xmit(skb, cp, pp, ciph); else rc = NF_ACCEPT; /* do not touch skb anymore */ @@ -1530,7 +1532,7 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; local = __ip_vs_get_out_rt(cp->ipvs, cp->af, skb, cp->dest, cp->daddr.ip, rt_mode, - NULL, iph); + NULL, ciph); if (local < 0) goto tx_error; rt = skb_rtable(skb); @@ -1561,14 +1563,21 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, goto tx_error; } + wlen = ciph->len; + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + wlen += 2 * sizeof(__u16); /* Also mangle ports */ + has_ports = true; + } + /* copy-on-write the packet before mangling it */ - if (skb_ensure_writable(skb, offset)) + if (skb_ensure_writable(skb, wlen)) goto tx_error; if (skb_cow(skb, rt->dst.dev->hard_header_len)) goto tx_error; - ip_vs_nat_icmp(skb, pp, cp, 0); + ip_vs_nat_icmp(skb, pp, cp, 0, toff, has_ports, ciph); /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; @@ -1584,10 +1593,12 @@ ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, #ifdef CONFIG_IP_VS_IPV6 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, - struct ip_vs_protocol *pp, int offset, unsigned int hooknum, - struct ip_vs_iphdr *ipvsh) + struct ip_vs_protocol *pp, unsigned int toff, + unsigned int hooknum, struct ip_vs_iphdr *ciph) { + bool has_ports = false; struct rt6_info *rt; /* Route to the other host */ + unsigned int wlen; int rc; int local; int rt_mode; @@ -1597,7 +1608,7 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { if (cp->packet_xmit) - rc = cp->packet_xmit(skb, cp, pp, ipvsh); + rc = cp->packet_xmit(skb, cp, pp, ciph); else rc = NF_ACCEPT; /* do not touch skb anymore */ @@ -1614,7 +1625,7 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, IP_VS_RT_MODE_LOCAL | IP_VS_RT_MODE_NON_LOCAL | IP_VS_RT_MODE_RDR : IP_VS_RT_MODE_NON_LOCAL; local = __ip_vs_get_out_rt_v6(cp->ipvs, cp->af, skb, cp->dest, - &cp->daddr.in6, NULL, ipvsh, 0, rt_mode); + &cp->daddr.in6, NULL, ciph, 0, rt_mode); if (local < 0) goto tx_error; rt = dst_rt6_info(skb_dst(skb)); @@ -1645,14 +1656,21 @@ ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, goto tx_error; } + wlen = ciph->len; + if (ciph->protocol == IPPROTO_TCP || ciph->protocol == IPPROTO_UDP || + ciph->protocol == IPPROTO_SCTP) { + wlen += 2 * sizeof(__u16); /* Also mangle ports */ + has_ports = true; + } + /* copy-on-write the packet before mangling it */ - if (skb_ensure_writable(skb, offset)) + if (skb_ensure_writable(skb, wlen)) goto tx_error; if (skb_cow(skb, rt->dst.dev->hard_header_len)) goto tx_error; - ip_vs_nat_icmp_v6(skb, pp, cp, 0); + ip_vs_nat_icmp_v6(skb, pp, cp, 0, toff, has_ports, ciph); /* Another hack: avoid icmp_send in ip_fragment */ skb->ignore_df = 1; diff --git a/net/netfilter/nf_conntrack_broadcast.c b/net/netfilter/nf_conntrack_broadcast.c index 4f39bf7c843f2..75e53fde6b297 100644 --- a/net/netfilter/nf_conntrack_broadcast.c +++ b/net/netfilter/nf_conntrack_broadcast.c @@ -72,6 +72,7 @@ int nf_conntrack_broadcast_help(struct sk_buff *skb, exp->flags = NF_CT_EXPECT_PERMANENT; exp->class = NF_CT_EXPECT_CLASS_DEFAULT; rcu_assign_pointer(exp->helper, helper); + rcu_assign_pointer(exp->assign_helper, NULL); write_pnet(&exp->net, net); #ifdef CONFIG_NF_CONNTRACK_ZONES exp->zone = ct->zone; diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index a877f9ad1cf2d..201d2ef8a6f02 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -1690,16 +1690,18 @@ void nf_conntrack_free(struct nf_conn *ct) */ WARN_ON(refcount_read(&ct->ct_general.use) != 0); + rcu_read_lock(); if (ct->status & IPS_SRC_NAT_DONE) { const struct nf_nat_hook *nat_hook; - rcu_read_lock(); nat_hook = rcu_dereference(nf_nat_hook); if (nat_hook) nat_hook->remove_nat_bysrc(ct); - rcu_read_unlock(); } + nf_ct_timeout_put(ct); + rcu_read_unlock(); + kfree(ct->ext); kmem_cache_free(nf_conntrack_cachep, ct); cnet = nf_ct_pernet(net); @@ -1771,14 +1773,17 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, spin_lock_bh(&nf_conntrack_expect_lock); exp = nf_ct_find_expectation(net, zone, tuple, !tmpl || nf_ct_is_confirmed(tmpl)); if (exp) { + struct nf_conntrack_helper *assign_helper; + /* Welcome, Mr. Bond. We've been expecting you... */ __set_bit(IPS_EXPECTED_BIT, &ct->status); /* exp->master safe, refcnt bumped in nf_ct_find_expectation */ ct->master = exp->master; - if (exp->helper) { + assign_helper = rcu_dereference(exp->assign_helper); + if (assign_helper) { help = nf_ct_helper_ext_add(ct, GFP_ATOMIC); if (help) - rcu_assign_pointer(help->helper, exp->helper); + rcu_assign_pointer(help->helper, assign_helper); } #ifdef CONFIG_NF_CONNTRACK_MARK diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index 24d0576d84b7f..6266d4d2ffca4 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -304,7 +304,7 @@ struct nf_conntrack_expect *nf_ct_expect_alloc(struct nf_conn *me) { struct nf_conntrack_expect *new; - new = kmem_cache_alloc(nf_ct_expect_cachep, GFP_ATOMIC); + new = kmem_cache_zalloc(nf_ct_expect_cachep, GFP_ATOMIC); if (!new) return NULL; @@ -344,6 +344,7 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, helper = rcu_dereference(help->helper); rcu_assign_pointer(exp->helper, helper); + rcu_assign_pointer(exp->assign_helper, NULL); write_pnet(&exp->net, net); #ifdef CONFIG_NF_CONNTRACK_ZONES exp->zone = ct->zone; @@ -385,6 +386,7 @@ void nf_ct_expect_init(struct nf_conntrack_expect *exp, unsigned int class, #if IS_ENABLED(CONFIG_NF_NAT) memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); + exp->dir = 0; #endif } EXPORT_SYMBOL_GPL(nf_ct_expect_init); diff --git a/net/netfilter/nf_conntrack_h323_main.c b/net/netfilter/nf_conntrack_h323_main.c index bd7e9e13e4f68..a116c6629e2b8 100644 --- a/net/netfilter/nf_conntrack_h323_main.c +++ b/net/netfilter/nf_conntrack_h323_main.c @@ -642,7 +642,7 @@ static int expect_h245(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->helper, &nf_conntrack_helper_h245); + rcu_assign_pointer(exp->assign_helper, &nf_conntrack_helper_h245); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -766,7 +766,7 @@ static int expect_callforwarding(struct sk_buff *skb, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); nathook = rcu_dereference(nfct_h323_nat_hook); if (memcmp(&ct->tuplehash[dir].tuple.src.u3, @@ -1233,7 +1233,7 @@ static int expect_q931(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3 : NULL, &ct->tuplehash[!dir].tuple.dst.u3, IPPROTO_TCP, NULL, &port); - rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); exp->flags = NF_CT_EXPECT_PERMANENT; /* Accept multiple calls */ nathook = rcu_dereference(nfct_h323_nat_hook); @@ -1305,7 +1305,7 @@ static int process_gcf(struct sk_buff *skb, struct nf_conn *ct, nf_ct_expect_init(exp, NF_CT_EXPECT_CLASS_DEFAULT, nf_ct_l3num(ct), &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_UDP, NULL, &port); - rcu_assign_pointer(exp->helper, nf_conntrack_helper_ras); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_ras); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect RAS "); @@ -1522,7 +1522,7 @@ static int process_acf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); @@ -1576,7 +1576,7 @@ static int process_lcf(struct sk_buff *skb, struct nf_conn *ct, &ct->tuplehash[!dir].tuple.src.u3, &addr, IPPROTO_TCP, NULL, &port); exp->flags = NF_CT_EXPECT_PERMANENT; - rcu_assign_pointer(exp->helper, nf_conntrack_helper_q931); + rcu_assign_pointer(exp->assign_helper, nf_conntrack_helper_q931); if (nf_ct_expect_related(exp, 0) == 0) { pr_debug("nf_ct_ras: expect Q.931 "); diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c index 9150bcfd7ca83..ea0cdb7ec9151 100644 --- a/net/netfilter/nf_conntrack_helper.c +++ b/net/netfilter/nf_conntrack_helper.c @@ -419,6 +419,11 @@ static bool expect_iter_me(struct nf_conntrack_expect *exp, void *data) this = rcu_dereference_protected(exp->helper, lockdep_is_held(&nf_conntrack_expect_lock)); + if (this == me) + return true; + + this = rcu_dereference_protected(exp->assign_helper, + lockdep_is_held(&nf_conntrack_expect_lock)); return this == me; } diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index c8fe284b86db4..48cf3dad61bdb 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -1077,6 +1077,8 @@ static int ctnetlink_start(struct netlink_callback *cb) } cb->data = filter; + if (filter) + cb->answer_flags = NLM_F_DUMP_FILTERED; return 0; } @@ -2635,6 +2637,7 @@ static const struct nla_policy exp_nla_policy[CTA_EXPECT_MAX+1] = { static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr *const cda[], struct nf_conn *ct, + const struct nf_conntrack_helper *assign_helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask); @@ -2861,6 +2864,7 @@ static int ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, u32 portid, u32 report) { + struct nf_conntrack_helper *assign_helper = NULL; struct nlattr *cda[CTA_EXPECT_MAX+1]; struct nf_conntrack_tuple tuple, mask; struct nf_conntrack_expect *exp; @@ -2876,8 +2880,18 @@ ctnetlink_glue_attach_expect(const struct nlattr *attr, struct nf_conn *ct, if (err < 0) return err; + if (cda[CTA_EXPECT_HELP_NAME]) { + const char *helpname = nla_data(cda[CTA_EXPECT_HELP_NAME]); + + assign_helper = __nf_conntrack_helper_find(helpname, + nf_ct_l3num(ct), + tuple.dst.protonum); + if (!assign_helper) + return -EOPNOTSUPP; + } + exp = ctnetlink_alloc_expect((const struct nlattr * const *)cda, ct, - &tuple, &mask); + assign_helper, &tuple, &mask); if (IS_ERR(exp)) return PTR_ERR(exp); @@ -3516,6 +3530,7 @@ ctnetlink_parse_expect_nat(const struct nlattr *attr, static struct nf_conntrack_expect * ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, + const struct nf_conntrack_helper *assign_helper, struct nf_conntrack_tuple *tuple, struct nf_conntrack_tuple *mask) { @@ -3546,8 +3561,6 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, if (cda[CTA_EXPECT_FLAGS]) { exp->flags = ntohl(nla_get_be32(cda[CTA_EXPECT_FLAGS])); exp->flags &= ~NF_CT_EXPECT_USERSPACE; - } else { - exp->flags = 0; } if (cda[CTA_EXPECT_FN]) { const char *name = nla_data(cda[CTA_EXPECT_FN]); @@ -3559,8 +3572,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, goto err_out; } exp->expectfn = expfn->expectfn; - } else - exp->expectfn = NULL; + } exp->class = class; exp->master = ct; @@ -3569,6 +3581,7 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp->zone = ct->zone; #endif rcu_assign_pointer(exp->helper, helper); + rcu_assign_pointer(exp->assign_helper, assign_helper); exp->tuple = *tuple; exp->mask.src.u3 = mask->src.u3; exp->mask.src.u.all = mask->src.u.all; @@ -3578,12 +3591,6 @@ ctnetlink_alloc_expect(const struct nlattr * const cda[], struct nf_conn *ct, exp, nf_ct_l3num(ct)); if (err < 0) goto err_out; -#if IS_ENABLED(CONFIG_NF_NAT) - } else { - memset(&exp->saved_addr, 0, sizeof(exp->saved_addr)); - memset(&exp->saved_proto, 0, sizeof(exp->saved_proto)); - exp->dir = 0; -#endif } return exp; err_out: @@ -3624,7 +3631,7 @@ ctnetlink_create_expect(struct net *net, ct = nf_ct_tuplehash_to_ctrack(h); rcu_read_lock(); - exp = ctnetlink_alloc_expect(cda, ct, &tuple, &mask); + exp = ctnetlink_alloc_expect(cda, ct, NULL, &tuple, &mask); if (IS_ERR(exp)) { err = PTR_ERR(exp); goto err_rcu; diff --git a/net/netfilter/nf_conntrack_proto.c b/net/netfilter/nf_conntrack_proto.c index bc1d96686b9c5..554a661e56b36 100644 --- a/net/netfilter/nf_conntrack_proto.c +++ b/net/netfilter/nf_conntrack_proto.c @@ -79,6 +79,12 @@ void nf_ct_l4proto_log_invalid(const struct sk_buff *skb, struct net *net; va_list args; + /* nfnetlink_log may re-enter conntrack attribute dumping and try to + * take ct->lock again via helpers such as tcp_to_nlattr(), so invalid + * conntrack logs must only be emitted after dropping ct->lock. + */ + lockdep_assert_not_held(&ct->lock); + net = nf_ct_net(ct); if (likely(net->ct.sysctl_log_invalid == 0)) return; diff --git a/net/netfilter/nf_conntrack_proto_sctp.c b/net/netfilter/nf_conntrack_proto_sctp.c index 7e10fa65cbdd3..71cc920f68568 100644 --- a/net/netfilter/nf_conntrack_proto_sctp.c +++ b/net/netfilter/nf_conntrack_proto_sctp.c @@ -336,10 +336,12 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, struct sctphdr _sctph; const struct sctp_chunkhdr *sch; struct sctp_chunkhdr _sch; + bool log_invalid = false; u_int32_t offset, count; unsigned int *timeouts; unsigned long map[256 / sizeof(unsigned long)] = { 0 }; bool ignore = false; + u8 invalid_type = 0; if (sctp_error(skb, dataoff, state)) return -NF_ACCEPT; @@ -451,10 +453,8 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, /* Invalid */ if (new_state == SCTP_CONNTRACK_MAX) { - nf_ct_l4proto_log_invalid(skb, ct, state, - "Invalid, old_state %d, dir %d, type %d", - old_state, dir, sch->type); - + log_invalid = true; + invalid_type = sch->type; goto out_unlock; } @@ -529,6 +529,10 @@ int nf_conntrack_sctp_packet(struct nf_conn *ct, out_unlock: spin_unlock_bh(&ct->lock); + if (log_invalid) + nf_ct_l4proto_log_invalid(skb, ct, state, + "Invalid, old_state %d, dir %d, type %d", + old_state, dir, invalid_type); out: return -NF_ACCEPT; } diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c index e99ab1e88e9f8..91da35e2b0d8d 100644 --- a/net/netfilter/nf_conntrack_proto_tcp.c +++ b/net/netfilter/nf_conntrack_proto_tcp.c @@ -405,11 +405,11 @@ static void tcp_sack(const struct sk_buff *skb, unsigned int dataoff, return; /* Fast path for timestamp-only option */ - if (length == TCPOLEN_TSTAMP_ALIGNED - && *(__be32 *)ptr == htonl((TCPOPT_NOP << 24) - | (TCPOPT_NOP << 16) - | (TCPOPT_TIMESTAMP << 8) - | TCPOLEN_TIMESTAMP)) + if (length == TCPOLEN_TSTAMP_ALIGNED && + get_unaligned_be32(ptr) == ((TCPOPT_NOP << 24) | + (TCPOPT_NOP << 16) | + (TCPOPT_TIMESTAMP << 8) | + TCPOLEN_TIMESTAMP)) return; while (length > 0) { @@ -480,37 +480,81 @@ static void tcp_init_sender(struct ip_ct_tcp_state *sender, } } -__printf(6, 7) -static enum nf_ct_tcp_action nf_tcp_log_invalid(const struct sk_buff *skb, - const struct nf_conn *ct, - const struct nf_hook_state *state, - const struct ip_ct_tcp_state *sender, - enum nf_ct_tcp_action ret, - const char *fmt, ...) +enum nf_tcp_invalid_log_type { + NF_TCP_LOG_NONE, + NF_TCP_LOG_OVERSHOT, + NF_TCP_LOG_SEQ_OVER, + NF_TCP_LOG_ACK_OVER, + NF_TCP_LOG_SEQ_UNDER, + NF_TCP_LOG_ACK_UNDER, +}; + +struct nf_tcp_invalid_log { + enum nf_tcp_invalid_log_type type; + u32 value; +}; + +static enum nf_ct_tcp_action +nf_tcp_store_invalid(const struct nf_conn *ct, + const struct ip_ct_tcp_state *sender, + struct nf_tcp_invalid_log *log, + enum nf_ct_tcp_action ret, + enum nf_tcp_invalid_log_type type, + u32 value) { const struct nf_tcp_net *tn = nf_tcp_pernet(nf_ct_net(ct)); - struct va_format vaf; - va_list args; bool be_liberal; be_liberal = sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || tn->tcp_be_liberal; if (be_liberal) return NFCT_TCP_ACCEPT; - va_start(args, fmt); - vaf.fmt = fmt; - vaf.va = &args; - nf_ct_l4proto_log_invalid(skb, ct, state, "%pV", &vaf); - va_end(args); - + log->type = type; + log->value = value; return ret; } +static void nf_tcp_log_invalid(const struct sk_buff *skb, + const struct nf_conn *ct, + const struct nf_hook_state *state, + const struct nf_tcp_invalid_log *log) +{ + switch (log->type) { + case NF_TCP_LOG_OVERSHOT: + nf_ct_l4proto_log_invalid(skb, ct, state, + "%u bytes more than expected", + log->value); + break; + case NF_TCP_LOG_SEQ_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is over upper bound %u (over the window of the receiver)", + log->value); + break; + case NF_TCP_LOG_ACK_OVER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ACK is over upper bound %u (ACKed data not seen yet)", + log->value); + break; + case NF_TCP_LOG_SEQ_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "SEQ is under lower bound %u (already ACKed data retransmitted)", + log->value); + break; + case NF_TCP_LOG_ACK_UNDER: + nf_ct_l4proto_log_invalid(skb, ct, state, + "ignored ACK under lower bound %u (possible overly delayed)", + log->value); + break; + case NF_TCP_LOG_NONE: + break; + } +} + static enum nf_ct_tcp_action tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, unsigned int index, const struct sk_buff *skb, unsigned int dataoff, const struct tcphdr *tcph, - const struct nf_hook_state *hook_state) + struct nf_tcp_invalid_log *log) { struct ip_ct_tcp *state = &ct->proto.tcp; struct ip_ct_tcp_state *sender = &state->seen[dir]; @@ -640,31 +684,29 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, sender->td_end = end; sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED; - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "%u bytes more than expected", overshot); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_OVERSHOT, overshot); } - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "SEQ is over upper bound %u (over the window of the receiver)", - sender->td_maxend + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_SEQ_OVER, sender->td_maxend + 1); } if (!before(sack, receiver->td_end + 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID, - "ACK is over upper bound %u (ACKed data not seen yet)", - receiver->td_end + 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_INVALID, + NF_TCP_LOG_ACK_OVER, receiver->td_end + 1); /* Is the ending sequence in the receive window (if available)? */ in_recv_win = !receiver->td_maxwin || after(end, sender->td_end - receiver->td_maxwin - 1); if (!in_recv_win) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "SEQ is under lower bound %u (already ACKed data retransmitted)", - sender->td_end - receiver->td_maxwin - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_SEQ_UNDER, + sender->td_end - receiver->td_maxwin - 1); if (!after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) - return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE, - "ignored ACK under lower bound %u (possible overly delayed)", - receiver->td_end - MAXACKWINDOW(sender) - 1); + return nf_tcp_store_invalid(ct, sender, log, NFCT_TCP_IGNORE, + NF_TCP_LOG_ACK_UNDER, + receiver->td_end - MAXACKWINDOW(sender) - 1); /* Take into account window scaling (RFC 1323). */ if (!tcph->syn) @@ -719,11 +761,8 @@ tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir, return NFCT_TCP_ACCEPT; } -static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, - enum ip_conntrack_dir dir, - int index, - const struct sk_buff *skb, - const struct nf_hook_state *hook_state) +static bool __cold +nf_tcp_handle_invalid(struct nf_conn *ct, enum ip_conntrack_dir dir, int index) { const unsigned int *timeouts; const struct nf_tcp_net *tn; @@ -732,7 +771,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, if (!test_bit(IPS_ASSURED_BIT, &ct->status) || test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status)) - return; + return false; /* We don't want to have connections hanging around in ESTABLISHED * state for long time 'just because' conntrack deemed a FIN/RST @@ -747,7 +786,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, case TCP_FIN_SET: break; default: - return; + return false; } if (ct->proto.tcp.last_dir != dir && @@ -755,7 +794,7 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, ct->proto.tcp.last_index == TCP_RST_SET)) { expires = nf_ct_expires(ct); if (expires < 120 * HZ) - return; + return false; tn = nf_tcp_pernet(nf_ct_net(ct)); timeouts = nf_ct_timeout_lookup(ct); @@ -764,16 +803,15 @@ static void __cold nf_tcp_handle_invalid(struct nf_conn *ct, timeout = READ_ONCE(timeouts[TCP_CONNTRACK_UNACK]); if (expires > timeout) { - nf_ct_l4proto_log_invalid(skb, ct, hook_state, - "packet (index %d, dir %d) response for index %d lower timeout to %u", - index, dir, ct->proto.tcp.last_index, timeout); - WRITE_ONCE(ct->timeout, timeout + nfct_time_stamp); + return true; } } else { ct->proto.tcp.last_index = index; ct->proto.tcp.last_dir = dir; } + + return false; } /* table of valid flag combinations - PUSH, ECE and CWR are always valid */ @@ -969,7 +1007,9 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, struct net *net = nf_ct_net(ct); struct nf_tcp_net *tn = nf_tcp_pernet(net); enum tcp_conntrack new_state, old_state; + struct nf_tcp_invalid_log log = {}; unsigned int index, *timeouts; + bool lowered_timeout = false; enum nf_ct_tcp_action res; enum ip_conntrack_dir dir; const struct tcphdr *th; @@ -1252,14 +1292,18 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct, } res = tcp_in_window(ct, dir, index, - skb, dataoff, th, state); + skb, dataoff, th, &log); switch (res) { case NFCT_TCP_IGNORE: spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); return NF_ACCEPT; case NFCT_TCP_INVALID: - nf_tcp_handle_invalid(ct, dir, index, skb, state); + lowered_timeout = nf_tcp_handle_invalid(ct, dir, index); spin_unlock_bh(&ct->lock); + nf_tcp_log_invalid(skb, ct, state, &log); + if (lowered_timeout) + nf_ct_l4proto_log_invalid(skb, ct, state, "lowered timeout to UNACK"); return -NF_ACCEPT; case NFCT_TCP_ACCEPT: break; diff --git a/net/netfilter/nf_conntrack_sip.c b/net/netfilter/nf_conntrack_sip.c index 81534213f00f3..5b75e9bc99e9f 100644 --- a/net/netfilter/nf_conntrack_sip.c +++ b/net/netfilter/nf_conntrack_sip.c @@ -429,7 +429,7 @@ static const char *sip_skip_whitespace(const char *dptr, const char *limit) dptr = sip_follow_continuation(dptr, limit); break; } - return dptr; + return dptr < limit ? dptr : NULL; } /* Search within a SIP header value, dealing with continuation lines */ @@ -947,10 +947,8 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, return NF_ACCEPT; saddr = &ct->tuplehash[!dir].tuple.src.u3; } else if (sip_external_media) { - struct net_device *dev = skb_dst(skb)->dev; - struct net *net = dev_net(dev); - struct flowi fl; struct dst_entry *dst = NULL; + struct flowi fl; memset(&fl, 0, sizeof(fl)); @@ -970,7 +968,11 @@ static int set_expected_rtp_rtcp(struct sk_buff *skb, unsigned int protoff, * through the same interface as the signalling peer. */ if (dst) { - bool external_media = (dst->dev == dev); + const struct dst_entry *this_dst = skb_dst(skb); + bool external_media = false; + + if (this_dst && dst->dev == this_dst->dev) + external_media = true; dst_release(dst); if (external_media) @@ -1384,7 +1386,7 @@ static int process_register_request(struct sk_buff *skb, unsigned int protoff, nf_ct_expect_init(exp, SIP_EXPECT_SIGNALLING, nf_ct_l3num(ct), saddr, &daddr, proto, NULL, &port); exp->timeout.expires = sip_timeout * HZ; - rcu_assign_pointer(exp->helper, helper); + rcu_assign_pointer(exp->assign_helper, helper); exp->flags = NF_CT_EXPECT_PERMANENT | NF_CT_EXPECT_INACTIVE; hooks = rcu_dereference(nf_nat_sip_hooks); @@ -1627,7 +1629,7 @@ static int sip_help_tcp(struct sk_buff *skb, unsigned int protoff, unsigned int matchoff, matchlen; unsigned int msglen, origlen; const char *dptr, *end; - s16 diff, tdiff = 0; + s32 diff, tdiff = 0; int ret = NF_ACCEPT; unsigned long clen; bool term; diff --git a/net/netfilter/nf_conntrack_timeout.c b/net/netfilter/nf_conntrack_timeout.c index 0cc584d3dbb1d..c81becde2afa9 100644 --- a/net/netfilter/nf_conntrack_timeout.c +++ b/net/netfilter/nf_conntrack_timeout.c @@ -25,17 +25,32 @@ const struct nf_ct_timeout_hooks __rcu *nf_ct_timeout_hook __read_mostly; EXPORT_SYMBOL_GPL(nf_ct_timeout_hook); +/* nf_ct_iterate_cleanup() holds the conntrack lock. */ static int untimeout(struct nf_conn *ct, void *timeout) { struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct); if (timeout_ext) { - const struct nf_ct_timeout *t; + struct nf_ct_timeout *t; - t = rcu_access_pointer(timeout_ext->timeout); + rcu_read_lock(); + t = rcu_dereference(timeout_ext->timeout); + if (!t) { + rcu_read_unlock(); + return 0; + } - if (!timeout || t == timeout) + if (!timeout || t == timeout) { RCU_INIT_POINTER(timeout_ext->timeout, NULL); + + /* No race with nf_conntrack_free() which is called + * only after the conntrack has been removed from + * the hashes. + */ + if (refcount_dec_and_test(&t->refcnt)) + kfree_rcu(t, rcu); + } + rcu_read_unlock(); } /* We are not intended to delete this conntrack. */ @@ -70,6 +85,8 @@ int nf_ct_set_timeout(struct net *net, struct nf_conn *ct, const char *errmsg = NULL; int ret = 0; + WARN_ON_ONCE(!nf_ct_is_template(ct)); + rcu_read_lock(); h = rcu_dereference(nf_ct_timeout_hook); if (!h) { @@ -127,6 +144,8 @@ void nf_ct_destroy_timeout(struct nf_conn *ct) struct nf_conn_timeout *timeout_ext; const struct nf_ct_timeout_hooks *h; + WARN_ON_ONCE(!nf_ct_is_template(ct)); + rcu_read_lock(); h = rcu_dereference(nf_ct_timeout_hook); @@ -139,6 +158,8 @@ void nf_ct_destroy_timeout(struct nf_conn *ct) if (t) h->timeout_put(t); RCU_INIT_POINTER(timeout_ext->timeout, NULL); + if (t && refcount_dec_and_test(&t->refcnt)) + kfree_rcu(t, rcu); } } rcu_read_unlock(); diff --git a/net/netfilter/nf_flow_table_core.c b/net/netfilter/nf_flow_table_core.c index d3104816b7f71..7a93e22d049db 100644 --- a/net/netfilter/nf_flow_table_core.c +++ b/net/netfilter/nf_flow_table_core.c @@ -132,6 +132,7 @@ static int flow_offload_fill_route(struct flow_offload *flow, break; case FLOW_OFFLOAD_XMIT_XFRM: case FLOW_OFFLOAD_XMIT_NEIGH: + flow_tuple->ifidx = route->tuple[dir].out.ifindex; flow_tuple->dst_cache = dst; flow_tuple->dst_cookie = flow_offload_dst_cookie(flow_tuple); break; @@ -323,17 +324,18 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow) flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow); err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); if (err < 0) return err; + /* GC only iterates original-direction entries; publish original last. */ err = rhashtable_insert_fast(&flow_table->rhashtable, - &flow->tuplehash[1].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node, nf_flow_offload_rhash_params); if (err < 0) { rhashtable_remove_fast(&flow_table->rhashtable, - &flow->tuplehash[0].node, + &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node, nf_flow_offload_rhash_params); return err; } diff --git a/net/netfilter/nf_flow_table_ip.c b/net/netfilter/nf_flow_table_ip.c index 8cd4cf7ae2112..1be506a175af1 100644 --- a/net/netfilter/nf_flow_table_ip.c +++ b/net/netfilter/nf_flow_table_ip.c @@ -272,6 +272,7 @@ static unsigned int nf_flow_xmit_xfrm(struct sk_buff *skb, struct dst_entry *dst) { skb_orphan(skb); + skb_dst_drop(skb); skb_dst_set_noref(skb, dst); dst_output(state->net, state->sk, skb); return NF_STOLEN; @@ -333,19 +334,27 @@ static void nf_flow_encap_pop(struct sk_buff *skb, } } +struct nf_flow_xmit { + const void *dest; + const void *source; + struct net_device *outdev; +}; + static unsigned int nf_flow_queue_xmit(struct net *net, struct sk_buff *skb, - const struct flow_offload_tuple_rhash *tuplehash, - unsigned short type) + struct nf_flow_xmit *xmit) { - struct net_device *outdev; + struct net_device *dev = xmit->outdev; + unsigned int hh_len = LL_RESERVED_SPACE(dev); - outdev = dev_get_by_index_rcu(net, tuplehash->tuple.out.ifidx); - if (!outdev) - return NF_DROP; + if (unlikely(skb_headroom(skb) < hh_len && dev->header_ops)) { + skb = skb_expand_head(skb, hh_len); + if (!skb) + return NF_STOLEN; + } - skb->dev = outdev; - dev_hard_header(skb, skb->dev, type, tuplehash->tuple.out.h_dest, - tuplehash->tuple.out.h_source, skb->len); + skb->dev = dev; + dev_hard_header(skb, dev, ntohs(skb->protocol), + xmit->dest, xmit->source, skb->len); dev_queue_xmit(skb); return NF_STOLEN; @@ -424,10 +433,10 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, struct nf_flowtable_ctx ctx = { .in = state->in, }; + struct nf_flow_xmit xmit = {}; struct flow_offload *flow; - struct net_device *outdev; + struct neighbour *neigh; struct rtable *rt; - __be32 nexthop; int ret; tuplehash = nf_flow_offload_lookup(&ctx, flow_table, skb); @@ -454,25 +463,35 @@ nf_flow_offload_ip_hook(void *priv, struct sk_buff *skb, switch (tuplehash->tuple.xmit_type) { case FLOW_OFFLOAD_XMIT_NEIGH: rt = dst_rtable(tuplehash->tuple.dst_cache); - outdev = rt->dst.dev; - skb->dev = outdev; - nexthop = rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr); + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx); + if (!xmit.outdev) { + flow_offload_teardown(flow); + return NF_DROP; + } + neigh = ip_neigh_gw4(rt->dst.dev, rt_nexthop(rt, flow->tuplehash[!dir].tuple.src_v4.s_addr)); + if (IS_ERR(neigh)) { + flow_offload_teardown(flow); + return NF_DROP; + } + xmit.dest = neigh->ha; + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); - neigh_xmit(NEIGH_ARP_TABLE, outdev, &nexthop, skb); - ret = NF_STOLEN; break; case FLOW_OFFLOAD_XMIT_DIRECT: - ret = nf_flow_queue_xmit(state->net, skb, tuplehash, ETH_P_IP); - if (ret == NF_DROP) + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx); + if (!xmit.outdev) { flow_offload_teardown(flow); + return NF_DROP; + } + xmit.dest = tuplehash->tuple.out.h_dest; + xmit.source = tuplehash->tuple.out.h_source; break; default: WARN_ON_ONCE(1); - ret = NF_DROP; - break; + return NF_DROP; } - return ret; + return nf_flow_queue_xmit(state->net, skb, &xmit); } EXPORT_SYMBOL_GPL(nf_flow_offload_ip_hook); @@ -719,9 +738,9 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, struct nf_flowtable_ctx ctx = { .in = state->in, }; - const struct in6_addr *nexthop; + struct nf_flow_xmit xmit = {}; struct flow_offload *flow; - struct net_device *outdev; + struct neighbour *neigh; struct rt6_info *rt; int ret; @@ -749,24 +768,34 @@ nf_flow_offload_ipv6_hook(void *priv, struct sk_buff *skb, switch (tuplehash->tuple.xmit_type) { case FLOW_OFFLOAD_XMIT_NEIGH: rt = dst_rt6_info(tuplehash->tuple.dst_cache); - outdev = rt->dst.dev; - skb->dev = outdev; - nexthop = rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6); + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.ifidx); + if (!xmit.outdev) { + flow_offload_teardown(flow); + return NF_DROP; + } + neigh = ip_neigh_gw6(rt->dst.dev, rt6_nexthop(rt, &flow->tuplehash[!dir].tuple.src_v6)); + if (IS_ERR(neigh)) { + flow_offload_teardown(flow); + return NF_DROP; + } + xmit.dest = neigh->ha; + skb_dst_drop(skb); skb_dst_set_noref(skb, &rt->dst); - neigh_xmit(NEIGH_ND_TABLE, outdev, nexthop, skb); - ret = NF_STOLEN; break; case FLOW_OFFLOAD_XMIT_DIRECT: - ret = nf_flow_queue_xmit(state->net, skb, tuplehash, ETH_P_IPV6); - if (ret == NF_DROP) + xmit.outdev = dev_get_by_index_rcu(state->net, tuplehash->tuple.out.ifidx); + if (!xmit.outdev) { flow_offload_teardown(flow); + return NF_DROP; + } + xmit.dest = tuplehash->tuple.out.h_dest; + xmit.source = tuplehash->tuple.out.h_source; break; default: WARN_ON_ONCE(1); - ret = NF_DROP; - break; + return NF_DROP; } - return ret; + return nf_flow_queue_xmit(state->net, skb, &xmit); } EXPORT_SYMBOL_GPL(nf_flow_offload_ipv6_hook); diff --git a/net/netfilter/nf_flow_table_path.c b/net/netfilter/nf_flow_table_path.c index 76d4ff1454398..c5bb142f329bc 100644 --- a/net/netfilter/nf_flow_table_path.c +++ b/net/netfilter/nf_flow_table_path.c @@ -203,6 +203,9 @@ static void nft_dev_forward_path(struct nf_flow_route *route, if (nft_dev_fill_forward_path(route, dst, ct, dir, ha, &stack) >= 0) nft_dev_path_info(&stack, &info, ha, &ft->data); + if (info.outdev) + route->tuple[dir].out.ifindex = info.outdev->ifindex; + if (!info.indev || !nft_flowtable_find_dev(info.indev, ft)) return; @@ -217,7 +220,6 @@ static void nft_dev_forward_path(struct nf_flow_route *route, if (info.xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) { memcpy(route->tuple[dir].out.h_source, info.h_source, ETH_ALEN); memcpy(route->tuple[dir].out.h_dest, info.h_dest, ETH_ALEN); - route->tuple[dir].out.ifindex = info.outdev->ifindex; route->tuple[dir].out.hw_ifindex = info.hw_outdev->ifindex; route->tuple[dir].xmit_type = info.xmit_type; } @@ -265,11 +267,10 @@ int nft_flow_route(const struct nft_pktinfo *pkt, const struct nf_conn *ct, nft_default_forward_path(route, this_dst, dir); nft_default_forward_path(route, other_dst, !dir); - if (route->tuple[dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH && - route->tuple[!dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) { + if (route->tuple[dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) nft_dev_forward_path(route, ct, dir, ft); + if (route->tuple[!dir].xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) nft_dev_forward_path(route, ct, !dir, ft); - } return 0; } diff --git a/net/netfilter/nf_log_syslog.c b/net/netfilter/nf_log_syslog.c index 6fa0812cd79ca..6782c72bcfc26 100644 --- a/net/netfilter/nf_log_syslog.c +++ b/net/netfilter/nf_log_syslog.c @@ -1006,7 +1006,7 @@ static int __net_init nf_log_syslog_net_init(struct net *net) return ret; } -static void __net_exit nf_log_syslog_net_exit(struct net *net) +static void __net_exit nf_log_syslog_net_pre_exit(struct net *net) { nf_log_unset(net, &nf_ip_logger); nf_log_unset(net, &nf_arp_logger); @@ -1017,7 +1017,7 @@ static void __net_exit nf_log_syslog_net_exit(struct net *net) static struct pernet_operations nf_log_syslog_net_ops = { .init = nf_log_syslog_net_init, - .exit = nf_log_syslog_net_exit, + .pre_exit = nf_log_syslog_net_pre_exit, }; static int __init nf_log_syslog_init(void) @@ -1065,12 +1065,12 @@ static int __init nf_log_syslog_init(void) static void __exit nf_log_syslog_exit(void) { - unregister_pernet_subsys(&nf_log_syslog_net_ops); nf_log_unregister(&nf_ip_logger); nf_log_unregister(&nf_arp_logger); nf_log_unregister(&nf_ip6_logger); nf_log_unregister(&nf_netdev_logger); nf_log_unregister(&nf_bridge_logger); + unregister_pernet_subsys(&nf_log_syslog_net_ops); } module_init(nf_log_syslog_init); diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c index f7be30c69b5c8..6b00c81084fe2 100644 --- a/net/netfilter/nf_nat_sip.c +++ b/net/netfilter/nf_nat_sip.c @@ -267,12 +267,17 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, SIP_HDR_CONTACT, &in_header, &matchoff, &matchlen, &addr, &port) > 0) { + int old_len = skb->len, delta; + if (!map_addr(skb, protoff, dataoff, dptr, datalen, matchoff, matchlen, &addr, port)) { nf_ct_helper_log(skb, ct, "cannot mangle contact"); return NF_DROP; } + + delta = (int)skb->len - old_len; + coff += delta; } if (!map_sip_addr(skb, protoff, dataoff, dptr, datalen, SIP_HDR_FROM) || @@ -315,7 +320,7 @@ static unsigned int nf_nat_sip(struct sk_buff *skb, unsigned int protoff, } static void nf_nat_sip_seq_adjust(struct sk_buff *skb, unsigned int protoff, - s16 off) + s32 off) { enum ip_conntrack_info ctinfo; struct nf_conn *ct = nf_ct_get(skb, &ctinfo); diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c index 0842916c19f50..a3a66b6268cd6 100644 --- a/net/netfilter/nf_tables_api.c +++ b/net/netfilter/nf_tables_api.c @@ -44,8 +44,6 @@ enum { NFT_VALIDATE_DO, }; -static struct rhltable nft_objname_ht; - static u32 nft_chain_hash(const void *data, u32 len, u32 seed); static u32 nft_chain_hash_obj(const void *data, u32 len, u32 seed); static int nft_chain_hash_cmp(struct rhashtable_compare_arg *, const void *); @@ -939,58 +937,6 @@ static int nft_delflowtable(struct nft_ctx *ctx, return 0; } -static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg) -{ - int i; - - for (i = track->regs[dreg].num_reg; i > 0; i--) - __nft_reg_track_cancel(track, dreg - i); -} - -static void __nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, - u8 dreg, u8 num_reg) -{ - track->regs[dreg].selector = expr; - track->regs[dreg].bitwise = NULL; - track->regs[dreg].num_reg = num_reg; -} - -void nft_reg_track_update(struct nft_regs_track *track, - const struct nft_expr *expr, u8 dreg, u8 len) -{ - unsigned int regcount; - int i; - - __nft_reg_track_clobber(track, dreg); - - regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - __nft_reg_track_update(track, expr, dreg, i); -} -EXPORT_SYMBOL_GPL(nft_reg_track_update); - -void nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg, u8 len) -{ - unsigned int regcount; - int i; - - __nft_reg_track_clobber(track, dreg); - - regcount = DIV_ROUND_UP(len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - __nft_reg_track_cancel(track, dreg); -} -EXPORT_SYMBOL_GPL(nft_reg_track_cancel); - -void __nft_reg_track_cancel(struct nft_regs_track *track, u8 dreg) -{ - track->regs[dreg].selector = NULL; - track->regs[dreg].bitwise = NULL; - track->regs[dreg].num_reg = 0; -} -EXPORT_SYMBOL_GPL(__nft_reg_track_cancel); - /* * Tables */ @@ -1652,6 +1598,10 @@ static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info, if (err) goto err_chain_ht; + err = rhltable_init(&table->objname_ht, &nft_objname_ht_params); + if (err < 0) + goto err_obj_ht; + INIT_LIST_HEAD(&table->chains); INIT_LIST_HEAD(&table->sets); INIT_LIST_HEAD(&table->objects); @@ -1670,6 +1620,8 @@ static int nf_tables_newtable(struct sk_buff *skb, const struct nfnl_info *info, list_add_tail_rcu(&table->list, &nft_net->tables); return 0; err_trans: + rhltable_destroy(&table->objname_ht); +err_obj_ht: rhltable_destroy(&table->chains_ht); err_chain_ht: kfree(table->udata); @@ -1836,6 +1788,7 @@ static void nf_tables_table_destroy(struct nft_table *table) return; rhltable_destroy(&table->chains_ht); + rhltable_destroy(&table->objname_ht); kfree(table->name); kfree(table->udata); kfree(table); @@ -3908,23 +3861,6 @@ static int nf_tables_dump_rules(struct sk_buff *skb, return skb->len; } -static int nf_tables_dumpreset_rules(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - /* Mutex is held is to prevent that two concurrent dump-and-reset calls - * do not underrun counters and quotas. The commit_mutex is used for - * the lack a better lock, this is not transaction path. - */ - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_rules(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_rules_start(struct netlink_callback *cb) { struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; @@ -3944,16 +3880,10 @@ static int nf_tables_dump_rules_start(struct netlink_callback *cb) return -ENOMEM; } } - return 0; -} + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + ctx->reset = true; -static int nf_tables_dumpreset_rules_start(struct netlink_callback *cb) -{ - struct nft_rule_dump_ctx *ctx = (void *)cb->ctx; - - ctx->reset = true; - - return nf_tables_dump_rules_start(cb); + return 0; } static int nf_tables_dump_rules_done(struct netlink_callback *cb) @@ -4019,6 +3949,8 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; + char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -4032,47 +3964,16 @@ static int nf_tables_getrule(struct sk_buff *skb, const struct nfnl_info *info, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - skb2 = nf_tables_getrule_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, net, portid); -} - -static int nf_tables_getrule_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - u32 portid = NETLINK_CB(skb).portid; - struct net *net = info->net; - struct sk_buff *skb2; - char *buf; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start= nf_tables_dumpreset_rules_start, - .dump = nf_tables_dumpreset_rules, - .done = nf_tables_dump_rules_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getrule_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETRULE_RESET) + reset = true; + skb2 = nf_tables_getrule_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_RULE_TABLE]), (char *)nla_data(nla[NFTA_RULE_TABLE]), @@ -4179,6 +4080,8 @@ int nft_chain_validate(const struct nft_ctx *ctx, struct nft_chain *chain) if (err < 0) return err; } + + cond_resched(); } nft_chain_vstate_update(ctx, chain); @@ -4203,8 +4106,6 @@ static int nft_table_validate(struct net *net, const struct nft_table *table) err = nft_chain_validate(&ctx, chain); if (err < 0) goto err; - - cond_resched(); } err: @@ -6332,6 +6233,10 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) nla_nest_end(skb, nest); nlmsg_end(skb, nlh); + if (dump_ctx->reset && args.iter.count > args.iter.skip) + audit_log_nft_set_reset(table, cb->seq, + args.iter.count - args.iter.skip); + rcu_read_unlock(); if (args.iter.err && args.iter.err != -EMSGSIZE) @@ -6347,26 +6252,6 @@ static int nf_tables_dump_set(struct sk_buff *skb, struct netlink_callback *cb) return -ENOSPC; } -static int nf_tables_dumpreset_set(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - struct nft_set_dump_ctx *dump_ctx = cb->data; - int ret, skip = cb->args[0]; - - mutex_lock(&nft_net->commit_mutex); - - ret = nf_tables_dump_set(skb, cb); - - if (cb->args[0] > skip) - audit_log_nft_set_reset(dump_ctx->ctx.table, cb->seq, - cb->args[0] - skip); - - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_set_start(struct netlink_callback *cb) { struct nft_set_dump_ctx *dump_ctx = cb->data; @@ -6613,8 +6498,13 @@ static int nf_tables_getsetelem(struct sk_buff *skb, { struct netlink_ext_ack *extack = info->extack; struct nft_set_dump_ctx dump_ctx; + int rem, err = 0, nelems = 0; + struct net *net = info->net; struct nlattr *attr; - int rem, err = 0; + bool reset = false; + + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETSETELEM_RESET) + reset = true; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { @@ -6624,7 +6514,7 @@ static int nf_tables_getsetelem(struct sk_buff *skb, .module = THIS_MODULE, }; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; @@ -6635,75 +6525,21 @@ static int nf_tables_getsetelem(struct sk_buff *skb, if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) return -EINVAL; - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, false); + err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, reset); if (err) return err; nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, false); - if (err < 0) { - NL_SET_BAD_ATTR(extack, attr); - break; - } - } - - return err; -} - -static int nf_tables_getsetelem_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); - struct netlink_ext_ack *extack = info->extack; - struct nft_set_dump_ctx dump_ctx; - int rem, err = 0, nelems = 0; - struct nlattr *attr; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_set_start, - .dump = nf_tables_dumpreset_set, - .done = nf_tables_dump_set_done, - .module = THIS_MODULE, - }; - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - return err; - - c.data = &dump_ctx; - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - if (!nla[NFTA_SET_ELEM_LIST_ELEMENTS]) - return -EINVAL; - - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - rcu_read_lock(); - - err = nft_set_dump_ctx_init(&dump_ctx, skb, info, nla, true); - if (err) - goto out_unlock; - - nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) { - err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, true); + err = nft_get_set_elem(&dump_ctx.ctx, dump_ctx.set, attr, reset); if (err < 0) { NL_SET_BAD_ATTR(extack, attr); break; } nelems++; } - audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(info->net), nelems); - -out_unlock: - rcu_read_unlock(); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (reset) + audit_log_nft_set_reset(dump_ctx.ctx.table, nft_base_seq(net), + nelems); return err; } @@ -8126,7 +7962,7 @@ void nft_unregister_obj(struct nft_object_type *obj_type) EXPORT_SYMBOL_GPL(nft_unregister_obj); struct nft_object *nft_obj_lookup(const struct net *net, - const struct nft_table *table, + struct nft_table *table, const struct nlattr *nla, u32 objtype, u8 genmask) { @@ -8142,7 +7978,7 @@ struct nft_object *nft_obj_lookup(const struct net *net, !lockdep_commit_lock_is_held(net)); rcu_read_lock(); - list = rhltable_lookup(&nft_objname_ht, &k, nft_objname_ht_params); + list = rhltable_lookup(&table->objname_ht, &k, nft_objname_ht_params); if (!list) goto out; @@ -8421,7 +8257,7 @@ static int nf_tables_newobj(struct sk_buff *skb, const struct nfnl_info *info, if (err < 0) goto err_trans; - err = rhltable_insert(&nft_objname_ht, &obj->rhlhead, + err = rhltable_insert(&table->objname_ht, &obj->rhlhead, nft_objname_ht_params); if (err < 0) goto err_obj_ht; @@ -8567,19 +8403,6 @@ static int nf_tables_dump_obj(struct sk_buff *skb, struct netlink_callback *cb) return skb->len; } -static int nf_tables_dumpreset_obj(struct sk_buff *skb, - struct netlink_callback *cb) -{ - struct nftables_pernet *nft_net = nft_pernet(sock_net(skb->sk)); - int ret; - - mutex_lock(&nft_net->commit_mutex); - ret = nf_tables_dump_obj(skb, cb); - mutex_unlock(&nft_net->commit_mutex); - - return ret; -} - static int nf_tables_dump_obj_start(struct netlink_callback *cb) { struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; @@ -8596,16 +8419,10 @@ static int nf_tables_dump_obj_start(struct netlink_callback *cb) if (nla[NFTA_OBJ_TYPE]) ctx->type = ntohl(nla_get_be32(nla[NFTA_OBJ_TYPE])); - return 0; -} - -static int nf_tables_dumpreset_obj_start(struct netlink_callback *cb) -{ - struct nft_obj_dump_ctx *ctx = (void *)cb->ctx; + if (NFNL_MSG_TYPE(cb->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + ctx->reset = true; - ctx->reset = true; - - return nf_tables_dump_obj_start(cb); + return 0; } static int nf_tables_dump_obj_done(struct netlink_callback *cb) @@ -8625,8 +8442,8 @@ nf_tables_getobj_single(u32 portid, const struct nfnl_info *info, struct netlink_ext_ack *extack = info->extack; u8 genmask = nft_genmask_cur(info->net); u8 family = info->nfmsg->nfgen_family; - const struct nft_table *table; struct net *net = info->net; + struct nft_table *table; struct nft_object *obj; struct sk_buff *skb2; u32 objtype; @@ -8667,42 +8484,16 @@ nf_tables_getobj_single(u32 portid, const struct nfnl_info *info, static int nf_tables_getobj(struct sk_buff *skb, const struct nfnl_info *info, const struct nlattr * const nla[]) { - u32 portid = NETLINK_CB(skb).portid; - struct sk_buff *skb2; - - if (info->nlh->nlmsg_flags & NLM_F_DUMP) { - struct netlink_dump_control c = { - .start = nf_tables_dump_obj_start, - .dump = nf_tables_dump_obj, - .done = nf_tables_dump_obj_done, - .module = THIS_MODULE, - .data = (void *)nla, - }; - - return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); - } - - skb2 = nf_tables_getobj_single(portid, info, nla, false); - if (IS_ERR(skb2)) - return PTR_ERR(skb2); - - return nfnetlink_unicast(skb2, info->net, portid); -} - -static int nf_tables_getobj_reset(struct sk_buff *skb, - const struct nfnl_info *info, - const struct nlattr * const nla[]) -{ - struct nftables_pernet *nft_net = nft_pernet(info->net); u32 portid = NETLINK_CB(skb).portid; struct net *net = info->net; struct sk_buff *skb2; + bool reset = false; char *buf; if (info->nlh->nlmsg_flags & NLM_F_DUMP) { struct netlink_dump_control c = { - .start = nf_tables_dumpreset_obj_start, - .dump = nf_tables_dumpreset_obj, + .start = nf_tables_dump_obj_start, + .dump = nf_tables_dump_obj, .done = nf_tables_dump_obj_done, .module = THIS_MODULE, .data = (void *)nla, @@ -8711,18 +8502,16 @@ static int nf_tables_getobj_reset(struct sk_buff *skb, return nft_netlink_dump_start_rcu(info->sk, skb, info->nlh, &c); } - if (!try_module_get(THIS_MODULE)) - return -EINVAL; - rcu_read_unlock(); - mutex_lock(&nft_net->commit_mutex); - skb2 = nf_tables_getobj_single(portid, info, nla, true); - mutex_unlock(&nft_net->commit_mutex); - rcu_read_lock(); - module_put(THIS_MODULE); + if (NFNL_MSG_TYPE(info->nlh->nlmsg_type) == NFT_MSG_GETOBJ_RESET) + reset = true; + skb2 = nf_tables_getobj_single(portid, info, nla, reset); if (IS_ERR(skb2)) return PTR_ERR(skb2); + if (!reset) + return nfnetlink_unicast(skb2, net, NETLINK_CB(skb).portid); + buf = kasprintf(GFP_ATOMIC, "%.*s:%u", nla_len(nla[NFTA_OBJ_TABLE]), (char *)nla_data(nla[NFTA_OBJ_TABLE]), @@ -8796,18 +8585,17 @@ static int nf_tables_delobj(struct sk_buff *skb, const struct nfnl_info *info, return nft_delobj(&ctx, obj); } -static void -__nft_obj_notify(struct net *net, const struct nft_table *table, - struct nft_object *obj, u32 portid, u32 seq, int event, - u16 flags, int family, int report, gfp_t gfp) +static struct sk_buff * +nft_obj_notify_alloc(struct net *net, const struct nft_table *table, + struct nft_object *obj, u32 portid, u32 seq, int event, + u16 flags, int family, int report, gfp_t gfp) { - struct nftables_pernet *nft_net = nft_pernet(net); struct sk_buff *skb; int err; if (!report && !nfnetlink_has_listeners(net, NFNLGRP_NFTABLES)) - return; + return NULL; skb = nlmsg_new(NLMSG_GOODSIZE, gfp); if (skb == NULL) @@ -8821,10 +8609,10 @@ __nft_obj_notify(struct net *net, const struct nft_table *table, goto err; } - nft_notify_enqueue(skb, report, &nft_net->notify_list); - return; + return skb; err: nfnetlink_set_err(net, portid, NFNLGRP_NFTABLES, -ENOBUFS); + return NULL; } void nft_obj_notify(struct net *net, const struct nft_table *table, @@ -8833,6 +8621,7 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, { char *buf = kasprintf(gfp, "%s:%u", table->name, nft_base_seq(net)); + struct sk_buff *skb; audit_log_nfcfg(buf, family, @@ -8843,17 +8632,27 @@ void nft_obj_notify(struct net *net, const struct nft_table *table, gfp); kfree(buf); - __nft_obj_notify(net, table, obj, portid, seq, event, - flags, family, report, gfp); + /* Called from the packet path, holding no mutex: notify_list is + * serialised by commit_mutex, so send this notification directly. + */ + skb = nft_obj_notify_alloc(net, table, obj, portid, seq, event, + flags, family, report, gfp); + if (skb) + nfnetlink_send(skb, net, portid, NFNLGRP_NFTABLES, report, gfp); } EXPORT_SYMBOL_GPL(nft_obj_notify); static void nf_tables_obj_notify(const struct nft_ctx *ctx, struct nft_object *obj, int event) { - __nft_obj_notify(ctx->net, ctx->table, obj, ctx->portid, - ctx->seq, event, ctx->flags, ctx->family, - ctx->report, GFP_KERNEL); + struct nftables_pernet *nft_net = nft_pernet(ctx->net); + struct sk_buff *skb; + + skb = nft_obj_notify_alloc(ctx->net, ctx->table, obj, ctx->portid, + ctx->seq, event, ctx->flags, ctx->family, + ctx->report, GFP_KERNEL); + if (skb) + nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list); } /* @@ -10030,7 +9829,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_rule_policy, }, [NFT_MSG_GETRULE_RESET] = { - .call = nf_tables_getrule_reset, + .call = nf_tables_getrule, .type = NFNL_CB_RCU, .attr_count = NFTA_RULE_MAX, .policy = nft_rule_policy, @@ -10084,7 +9883,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_set_elem_list_policy, }, [NFT_MSG_GETSETELEM_RESET] = { - .call = nf_tables_getsetelem_reset, + .call = nf_tables_getsetelem, .type = NFNL_CB_RCU, .attr_count = NFTA_SET_ELEM_LIST_MAX, .policy = nft_set_elem_list_policy, @@ -10130,7 +9929,7 @@ static const struct nfnl_callback nf_tables_cb[NFT_MSG_MAX] = { .policy = nft_obj_policy, }, [NFT_MSG_GETOBJ_RESET] = { - .call = nf_tables_getobj_reset, + .call = nf_tables_getobj, .type = NFNL_CB_RCU, .attr_count = NFTA_OBJ_MAX, .policy = nft_obj_policy, @@ -10337,16 +10136,9 @@ void nf_tables_trans_destroy_flush_work(struct net *net) } EXPORT_SYMBOL_GPL(nf_tables_trans_destroy_flush_work); -static bool nft_expr_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - return false; -} - static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *chain) { const struct nft_expr *expr, *last; - struct nft_regs_track track = {}; unsigned int size, data_size; void *data, *data_boundary; struct nft_rule_dp *prule; @@ -10383,15 +10175,7 @@ static int nf_tables_commit_chain_prepare(struct net *net, struct nft_chain *cha return -ENOMEM; size = 0; - track.last = nft_expr_last(rule); nft_rule_for_each_expr(expr, last, rule) { - track.cur = expr; - - if (nft_expr_reduce(&track, expr)) { - expr = track.cur; - continue; - } - if (WARN_ON_ONCE(data + size + expr->ops->size > data_boundary)) return -ENOMEM; @@ -10501,9 +10285,9 @@ static void nf_tables_commit_chain(struct net *net, struct nft_chain *chain) nf_tables_commit_chain_free_rules_old(g0); } -static void nft_obj_del(struct nft_object *obj) +static void nft_obj_del(struct nft_table *table, struct nft_object *obj) { - rhltable_remove(&nft_objname_ht, &obj->rhlhead, nft_objname_ht_params); + rhltable_remove(&table->objname_ht, &obj->rhlhead, nft_objname_ht_params); list_del_rcu(&obj->list); } @@ -10990,10 +10774,6 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) return -EAGAIN; } - err = nft_flow_rule_offload_commit(net); - if (err < 0) - return err; - /* 1. Allocate space for next generation rules_gen_X[] */ list_for_each_entry_safe(trans, next, &nft_net->commit_list, list) { struct nft_table *table = trans->table; @@ -11018,6 +10798,16 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) } } + /* must be last, so audit and chain blob set up does not leave hardware + * in consistent state. + */ + err = nft_flow_rule_offload_commit(net); + if (err < 0) { + nf_tables_commit_chain_prepare_cancel(net); + nf_tables_commit_audit_free(&adl); + return err; + } + /* step 2. Make rules_gen_X visible to packet path */ list_for_each_entry(table, &nft_net->tables, list) { list_for_each_entry(chain, &table->chains, list) @@ -11191,7 +10981,7 @@ static int nf_tables_commit(struct net *net, struct sk_buff *skb) break; case NFT_MSG_DELOBJ: case NFT_MSG_DESTROYOBJ: - nft_obj_del(nft_trans_obj(trans)); + nft_obj_del(table, nft_trans_obj(trans)); nf_tables_obj_notify(&ctx, nft_trans_obj(trans), trans->msg_type); break; @@ -11482,7 +11272,7 @@ static int __nf_tables_abort(struct net *net, enum nfnl_abort_action action) nft_trans_destroy(trans); } else { nft_use_dec_restore(&table->use); - nft_obj_del(nft_trans_obj(trans)); + nft_obj_del(table, nft_trans_obj(trans)); } break; case NFT_MSG_DELOBJ: @@ -12108,7 +11898,7 @@ static void __nft_release_table(struct net *net, struct nft_table *table) nft_set_destroy(&ctx, set); } list_for_each_entry_safe(obj, ne, &table->objects, list) { - nft_obj_del(obj); + nft_obj_del(table, obj); nft_use_dec(&table->use); nft_obj_destroy(&ctx, obj); } @@ -12290,10 +12080,6 @@ static int __init nf_tables_module_init(void) if (err < 0) goto err_netdev_notifier; - err = rhltable_init(&nft_objname_ht, &nft_objname_ht_params); - if (err < 0) - goto err_rht_objname; - err = nft_offload_init(); if (err < 0) goto err_offload; @@ -12316,8 +12102,6 @@ static int __init nf_tables_module_init(void) err_netlink_notifier: nft_offload_exit(); err_offload: - rhltable_destroy(&nft_objname_ht); -err_rht_objname: unregister_netdevice_notifier(&nf_tables_flowtable_notifier); err_netdev_notifier: nf_tables_core_module_exit(); @@ -12339,7 +12123,6 @@ static void __exit nf_tables_module_exit(void) unregister_pernet_subsys(&nf_tables_net_ops); cancel_work_sync(&trans_gc_work); rcu_barrier(); - rhltable_destroy(&nft_objname_ht); nf_tables_core_module_exit(); } diff --git a/net/netfilter/nf_tables_offload.c b/net/netfilter/nf_tables_offload.c index fd30e205de849..933a17d0f2513 100644 --- a/net/netfilter/nf_tables_offload.c +++ b/net/netfilter/nf_tables_offload.c @@ -558,7 +558,7 @@ static void nft_flow_rule_offload_abort(struct net *net, break; } - if (WARN_ON_ONCE(err)) + if (WARN_ON_ONCE(err && err != -ENOMEM)) break; } } diff --git a/net/netfilter/nfnetlink_cttimeout.c b/net/netfilter/nfnetlink_cttimeout.c index 38d75484e531c..564e4642467dc 100644 --- a/net/netfilter/nfnetlink_cttimeout.c +++ b/net/netfilter/nfnetlink_cttimeout.c @@ -37,11 +37,8 @@ struct ctnl_timeout { struct list_head head; struct list_head free_head; struct rcu_head rcu_head; - refcount_t refcnt; char name[CTNL_TIMEOUT_NAME_MAX]; - - /* must be at the end */ - struct nf_ct_timeout timeout; + struct nf_ct_timeout *timeout; }; struct nfct_timeout_pernet { @@ -133,12 +130,12 @@ static int cttimeout_new_timeout(struct sk_buff *skb, /* You cannot replace one timeout policy by another of * different kind, sorry. */ - if (matching->timeout.l3num != l3num || - matching->timeout.l4proto->l4proto != l4num) + if (matching->timeout->l3num != l3num || + matching->timeout->l4proto->l4proto != l4num) return -EINVAL; - return ctnl_timeout_parse_policy(&matching->timeout.data, - matching->timeout.l4proto, + return ctnl_timeout_parse_policy(&matching->timeout->data, + matching->timeout->l4proto, info->net, cda[CTA_TIMEOUT_DATA]); } @@ -154,26 +151,35 @@ static int cttimeout_new_timeout(struct sk_buff *skb, goto err_proto_put; } - timeout = kzalloc(sizeof(struct ctnl_timeout) + - l4proto->ctnl_timeout.obj_size, GFP_KERNEL); + timeout = kzalloc(sizeof(*timeout), GFP_KERNEL); if (timeout == NULL) { ret = -ENOMEM; goto err_proto_put; } - ret = ctnl_timeout_parse_policy(&timeout->timeout.data, l4proto, + timeout->timeout = kzalloc(sizeof(*timeout->timeout) + + l4proto->ctnl_timeout.obj_size, GFP_KERNEL); + if (!timeout->timeout) { + ret = -ENOMEM; + goto err; + } + + ret = ctnl_timeout_parse_policy(&timeout->timeout->data, l4proto, info->net, cda[CTA_TIMEOUT_DATA]); if (ret < 0) - goto err; + goto err_free_timeout_policy; strcpy(timeout->name, nla_data(cda[CTA_TIMEOUT_NAME])); - timeout->timeout.l3num = l3num; - timeout->timeout.l4proto = l4proto; - refcount_set(&timeout->refcnt, 1); + timeout->timeout->l3num = l3num; + timeout->timeout->l4proto = l4proto; + refcount_set(&timeout->timeout->refcnt, 1); __module_get(THIS_MODULE); list_add_tail_rcu(&timeout->head, &pernet->nfct_timeout_list); return 0; + +err_free_timeout_policy: + kfree(timeout->timeout); err: kfree(timeout); err_proto_put: @@ -186,7 +192,7 @@ ctnl_timeout_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, { struct nlmsghdr *nlh; unsigned int flags = portid ? NLM_F_MULTI : 0; - const struct nf_conntrack_l4proto *l4proto = timeout->timeout.l4proto; + const struct nf_conntrack_l4proto *l4proto = timeout->timeout->l4proto; struct nlattr *nest_parms; int ret; @@ -198,17 +204,17 @@ ctnl_timeout_fill_info(struct sk_buff *skb, u32 portid, u32 seq, u32 type, if (nla_put_string(skb, CTA_TIMEOUT_NAME, timeout->name) || nla_put_be16(skb, CTA_TIMEOUT_L3PROTO, - htons(timeout->timeout.l3num)) || + htons(timeout->timeout->l3num)) || nla_put_u8(skb, CTA_TIMEOUT_L4PROTO, l4proto->l4proto) || nla_put_be32(skb, CTA_TIMEOUT_USE, - htonl(refcount_read(&timeout->refcnt)))) + htonl(refcount_read(&timeout->timeout->refcnt)))) goto nla_put_failure; nest_parms = nla_nest_start(skb, CTA_TIMEOUT_DATA); if (!nest_parms) goto nla_put_failure; - ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->timeout.data); + ret = l4proto->ctnl_timeout.obj_to_nlattr(skb, &timeout->timeout->data); if (ret < 0) goto nla_put_failure; @@ -308,23 +314,17 @@ static int cttimeout_get_timeout(struct sk_buff *skb, return ret; } -/* try to delete object, fail if it is still in use. */ -static int ctnl_timeout_try_del(struct net *net, struct ctnl_timeout *timeout) +static void ctnl_timeout_del(struct net *net, struct ctnl_timeout *timeout) { - int ret = 0; + /* We are protected by nfnl mutex. */ + list_del_rcu(&timeout->head); + nf_ct_untimeout(net, timeout->timeout); - /* We want to avoid races with ctnl_timeout_put. So only when the - * current refcnt is 1, we decrease it to 0. - */ - if (refcount_dec_if_one(&timeout->refcnt)) { - /* We are protected by nfnl mutex. */ - list_del_rcu(&timeout->head); - nf_ct_untimeout(net, &timeout->timeout); - kfree_rcu(timeout, rcu_head); - } else { - ret = -EBUSY; - } - return ret; + if (refcount_dec_and_test(&timeout->timeout->refcnt)) + kfree_rcu(timeout->timeout, rcu); + + kfree_rcu(timeout, rcu_head); + module_put(THIS_MODULE); } static int cttimeout_del_timeout(struct sk_buff *skb, @@ -339,7 +339,7 @@ static int cttimeout_del_timeout(struct sk_buff *skb, if (!cda[CTA_TIMEOUT_NAME]) { list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_list, head) - ctnl_timeout_try_del(info->net, cur); + ctnl_timeout_del(info->net, cur); return 0; } @@ -349,10 +349,8 @@ static int cttimeout_del_timeout(struct sk_buff *skb, if (strncmp(cur->name, name, CTNL_TIMEOUT_NAME_MAX) != 0) continue; - ret = ctnl_timeout_try_del(info->net, cur); - if (ret < 0) - return ret; - + ctnl_timeout_del(info->net, cur); + ret = 0; break; } return ret; @@ -513,24 +511,22 @@ static struct nf_ct_timeout *ctnl_timeout_find_get(struct net *net, if (strncmp(timeout->name, name, CTNL_TIMEOUT_NAME_MAX) != 0) continue; - if (!refcount_inc_not_zero(&timeout->refcnt)) + if (!refcount_inc_not_zero(&timeout->timeout->refcnt)) goto err; matching = timeout; + __module_get(THIS_MODULE); break; } err: - return matching ? &matching->timeout : NULL; + return matching ? matching->timeout : NULL; } -static void ctnl_timeout_put(struct nf_ct_timeout *t) +static void ctnl_timeout_put(struct nf_ct_timeout *timeout) { - struct ctnl_timeout *timeout = - container_of(t, struct ctnl_timeout, timeout); + if (refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(timeout, rcu); - if (refcount_dec_and_test(&timeout->refcnt)) { - kfree_rcu(timeout, rcu_head); - module_put(THIS_MODULE); - } + module_put(THIS_MODULE); } static const struct nfnl_callback cttimeout_cb[IPCTNL_MSG_TIMEOUT_MAX] = { @@ -611,8 +607,11 @@ static void __net_exit cttimeout_net_exit(struct net *net) list_for_each_entry_safe(cur, tmp, &pernet->nfct_timeout_freelist, free_head) { list_del(&cur->free_head); - if (refcount_dec_and_test(&cur->refcnt)) - kfree_rcu(cur, rcu_head); + if (refcount_dec_and_test(&cur->timeout->refcnt)) + kfree_rcu(cur->timeout, rcu); + + kfree_rcu(cur, rcu_head); + module_put(THIS_MODULE); } } @@ -651,24 +650,13 @@ static int __init cttimeout_init(void) return ret; } -static int untimeout(struct nf_conn *ct, void *timeout) -{ - struct nf_conn_timeout *timeout_ext = nf_ct_timeout_find(ct); - - if (timeout_ext) - RCU_INIT_POINTER(timeout_ext->timeout, NULL); - - return 0; -} - static void __exit cttimeout_exit(void) { nfnetlink_subsys_unregister(&cttimeout_subsys); - unregister_pernet_subsys(&cttimeout_ops); RCU_INIT_POINTER(nf_ct_timeout_hook, NULL); - - nf_ct_iterate_destroy(untimeout, NULL); + synchronize_net(); + unregister_pernet_subsys(&cttimeout_ops); } module_init(cttimeout_init); diff --git a/net/netfilter/nfnetlink_log.c b/net/netfilter/nfnetlink_log.c index a6989966d80d5..249f7e9e25cb5 100644 --- a/net/netfilter/nfnetlink_log.c +++ b/net/netfilter/nfnetlink_log.c @@ -228,13 +228,18 @@ static void __nfulnl_flush(struct nfulnl_instance *inst); static void __instance_destroy(struct nfulnl_instance *inst) { + spin_lock(&inst->lock); + if (inst->copy_mode == NFULNL_COPY_DISABLED) { + /* attempt to UNBIND a queue already pending + * destruction via netlink close event. Ignore. + */ + spin_unlock(&inst->lock); + return; + } + /* first pull it out of the global list */ hlist_del_rcu(&inst->hlist); - /* then flush all pending packets from skb */ - - spin_lock(&inst->lock); - /* lockless readers wont be able to use us */ inst->copy_mode = NFULNL_COPY_DISABLED; @@ -1160,21 +1165,26 @@ static int __net_init nfnl_log_net_init(struct net *net) return 0; } -static void __net_exit nfnl_log_net_exit(struct net *net) +static void __net_exit nfnl_log_net_pre_exit(struct net *net) { - struct nfnl_log_net *log = nfnl_log_pernet(net); - unsigned int i; - #ifdef CONFIG_PROC_FS remove_proc_entry("nfnetlink_log", net->nf.proc_netfilter); #endif nf_log_unset(net, &nfulnl_logger); +} + +static void __net_exit nfnl_log_net_exit(struct net *net) +{ + struct nfnl_log_net *log = nfnl_log_pernet(net); + unsigned int i; + for (i = 0; i < INSTANCE_BUCKETS; i++) WARN_ON_ONCE(!hlist_empty(&log->instance_table[i])); } static struct pernet_operations nfnl_log_net_ops = { .init = nfnl_log_net_init, + .pre_exit = nfnl_log_net_pre_exit, .exit = nfnl_log_net_exit, .id = &nfnl_log_net_id, .size = sizeof(struct nfnl_log_net), @@ -1218,8 +1228,8 @@ static void __exit nfnetlink_log_fini(void) { nfnetlink_subsys_unregister(&nfulnl_subsys); netlink_unregister_notifier(&nfulnl_rtnl_notifier); - unregister_pernet_subsys(&nfnl_log_net_ops); nf_log_unregister(&nfulnl_logger); + unregister_pernet_subsys(&nfnl_log_net_ops); } MODULE_DESCRIPTION("netfilter userspace logging"); diff --git a/net/netfilter/nft_bitwise.c b/net/netfilter/nft_bitwise.c index 1afb36fb5994d..e0a6703dc0e4a 100644 --- a/net/netfilter/nft_bitwise.c +++ b/net/netfilter/nft_bitwise.c @@ -402,61 +402,12 @@ static int nft_bitwise_offload(struct nft_offload_ctx *ctx, return 0; } -static bool nft_bitwise_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_bitwise *priv = nft_expr_priv(expr); - const struct nft_bitwise *bitwise; - unsigned int regcount; - u8 dreg; - int i; - - if (!track->regs[priv->sreg].selector) - return false; - - bitwise = nft_expr_priv(track->regs[priv->dreg].selector); - if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && - track->regs[priv->sreg].num_reg == 0 && - track->regs[priv->dreg].bitwise && - track->regs[priv->dreg].bitwise->ops == expr->ops && - priv->sreg == bitwise->sreg && - priv->sreg2 == bitwise->sreg2 && - priv->dreg == bitwise->dreg && - priv->op == bitwise->op && - priv->len == bitwise->len && - !memcmp(&priv->mask, &bitwise->mask, sizeof(priv->mask)) && - !memcmp(&priv->xor, &bitwise->xor, sizeof(priv->xor)) && - !memcmp(&priv->data, &bitwise->data, sizeof(priv->data))) { - track->cur = expr; - return true; - } - - if (track->regs[priv->sreg].bitwise || - track->regs[priv->sreg].num_reg != 0) { - nft_reg_track_cancel(track, priv->dreg, priv->len); - return false; - } - - if (priv->sreg != priv->dreg) { - nft_reg_track_update(track, track->regs[priv->sreg].selector, - priv->dreg, priv->len); - } - - dreg = priv->dreg; - regcount = DIV_ROUND_UP(priv->len, NFT_REG32_SIZE); - for (i = 0; i < regcount; i++, dreg++) - track->regs[dreg].bitwise = expr; - - return false; -} - static const struct nft_expr_ops nft_bitwise_ops = { .type = &nft_bitwise_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise)), .eval = nft_bitwise_eval, .init = nft_bitwise_init, .dump = nft_bitwise_dump, - .reduce = nft_bitwise_reduce, .offload = nft_bitwise_offload, }; @@ -559,48 +510,12 @@ static int nft_bitwise_fast_offload(struct nft_offload_ctx *ctx, return 0; } -static bool nft_bitwise_fast_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr); - const struct nft_bitwise_fast_expr *bitwise; - - if (!track->regs[priv->sreg].selector) - return false; - - bitwise = nft_expr_priv(track->regs[priv->dreg].selector); - if (track->regs[priv->sreg].selector == track->regs[priv->dreg].selector && - track->regs[priv->dreg].bitwise && - track->regs[priv->dreg].bitwise->ops == expr->ops && - priv->sreg == bitwise->sreg && - priv->dreg == bitwise->dreg && - priv->mask == bitwise->mask && - priv->xor == bitwise->xor) { - track->cur = expr; - return true; - } - - if (track->regs[priv->sreg].bitwise) { - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - return false; - } - - if (priv->sreg != priv->dreg) { - track->regs[priv->dreg].selector = - track->regs[priv->sreg].selector; - } - track->regs[priv->dreg].bitwise = expr; - - return false; -} - const struct nft_expr_ops nft_bitwise_fast_ops = { .type = &nft_bitwise_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_bitwise_fast_expr)), .eval = NULL, /* inlined */ .init = nft_bitwise_fast_init, .dump = nft_bitwise_fast_dump, - .reduce = nft_bitwise_fast_reduce, .offload = nft_bitwise_fast_offload, }; @@ -637,22 +552,3 @@ struct nft_expr_type nft_bitwise_type __read_mostly = { .maxattr = NFTA_BITWISE_MAX, .owner = THIS_MODULE, }; - -bool nft_expr_reduce_bitwise(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_expr *last = track->last; - const struct nft_expr *next; - - if (expr == last) - return false; - - next = nft_expr_next(expr); - if (next->ops == &nft_bitwise_ops) - return nft_bitwise_reduce(track, next); - else if (next->ops == &nft_bitwise_fast_ops) - return nft_bitwise_fast_reduce(track, next); - - return false; -} -EXPORT_SYMBOL_GPL(nft_expr_reduce_bitwise); diff --git a/net/netfilter/nft_byteorder.c b/net/netfilter/nft_byteorder.c index 5e7a7841b789b..74548b4c114b9 100644 --- a/net/netfilter/nft_byteorder.c +++ b/net/netfilter/nft_byteorder.c @@ -177,23 +177,12 @@ static int nft_byteorder_dump(struct sk_buff *skb, return -1; } -static bool nft_byteorder_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_byteorder *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, priv->len); - - return false; -} - static const struct nft_expr_ops nft_byteorder_ops = { .type = &nft_byteorder_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_byteorder)), .eval = nft_byteorder_eval, .init = nft_byteorder_init, .dump = nft_byteorder_dump, - .reduce = nft_byteorder_reduce, }; struct nft_expr_type nft_byteorder_type __read_mostly = { diff --git a/net/netfilter/nft_cmp.c b/net/netfilter/nft_cmp.c index 2605f43737bc9..b61dc9c3383ee 100644 --- a/net/netfilter/nft_cmp.c +++ b/net/netfilter/nft_cmp.c @@ -190,7 +190,6 @@ static const struct nft_expr_ops nft_cmp_ops = { .eval = nft_cmp_eval, .init = nft_cmp_init, .dump = nft_cmp_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp_offload, }; @@ -282,7 +281,6 @@ const struct nft_expr_ops nft_cmp_fast_ops = { .eval = NULL, /* inlined */ .init = nft_cmp_fast_init, .dump = nft_cmp_fast_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp_fast_offload, }; @@ -376,7 +374,6 @@ const struct nft_expr_ops nft_cmp16_fast_ops = { .eval = NULL, /* inlined */ .init = nft_cmp16_fast_init, .dump = nft_cmp16_fast_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_cmp16_fast_offload, }; diff --git a/net/netfilter/nft_compat.c b/net/netfilter/nft_compat.c index a77ffe2fcea75..ba4d304a30141 100644 --- a/net/netfilter/nft_compat.c +++ b/net/netfilter/nft_compat.c @@ -794,14 +794,6 @@ static const struct nfnetlink_subsystem nfnl_compat_subsys = { static struct nft_expr_type nft_match_type; -static bool nft_match_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct xt_match *match = expr->ops->data; - - return strcmp(match->name, "comment") == 0; -} - static const struct nft_expr_ops * nft_match_select_ops(const struct nft_ctx *ctx, const struct nlattr * const tb[]) @@ -844,7 +836,6 @@ nft_match_select_ops(const struct nft_ctx *ctx, ops->dump = nft_match_dump; ops->validate = nft_match_validate; ops->data = match; - ops->reduce = nft_match_reduce; matchsize = NFT_EXPR_SIZE(XT_ALIGN(match->matchsize)); if (matchsize > NFT_MATCH_LARGE_THRESH) { @@ -933,7 +924,6 @@ nft_target_select_ops(const struct nft_ctx *ctx, ops->destroy = nft_target_destroy; ops->dump = nft_target_dump; ops->data = target; - ops->reduce = NFT_REDUCE_READONLY; if (family == NFPROTO_BRIDGE) { ops->eval = nft_target_eval_bridge; diff --git a/net/netfilter/nft_connlimit.c b/net/netfilter/nft_connlimit.c index 714a594859357..99079afcc5baf 100644 --- a/net/netfilter/nft_connlimit.c +++ b/net/netfilter/nft_connlimit.c @@ -247,7 +247,6 @@ static const struct nft_expr_ops nft_connlimit_ops = { .destroy_clone = nft_connlimit_destroy_clone, .dump = nft_connlimit_dump, .gc = nft_connlimit_gc, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_connlimit_type __read_mostly = { diff --git a/net/netfilter/nft_counter.c b/net/netfilter/nft_counter.c index 0d70325280cc5..3fa6369790f4d 100644 --- a/net/netfilter/nft_counter.c +++ b/net/netfilter/nft_counter.c @@ -32,6 +32,9 @@ struct nft_counter_percpu_priv { static DEFINE_PER_CPU(struct u64_stats_sync, nft_counter_sync); +/* control plane only: sync fetch+reset */ +static DEFINE_SPINLOCK(nft_counter_lock); + static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) @@ -148,13 +151,25 @@ static void nft_counter_fetch(struct nft_counter_percpu_priv *priv, } } +static void nft_counter_fetch_and_reset(struct nft_counter_percpu_priv *priv, + struct nft_counter_tot *total) +{ + spin_lock(&nft_counter_lock); + nft_counter_fetch(priv, total); + nft_counter_reset(priv, total); + spin_unlock(&nft_counter_lock); +} + static int nft_counter_do_dump(struct sk_buff *skb, struct nft_counter_percpu_priv *priv, bool reset) { struct nft_counter_tot total; - nft_counter_fetch(priv, &total); + if (unlikely(reset)) + nft_counter_fetch_and_reset(priv, &total); + else + nft_counter_fetch(priv, &total); if (nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes), NFTA_COUNTER_PAD) || @@ -162,9 +177,6 @@ static int nft_counter_do_dump(struct sk_buff *skb, NFTA_COUNTER_PAD)) goto nla_put_failure; - if (reset) - nft_counter_reset(priv, &total); - return 0; nla_put_failure: @@ -301,7 +313,6 @@ static const struct nft_expr_ops nft_counter_ops = { .destroy_clone = nft_counter_destroy, .dump = nft_counter_dump, .clone = nft_counter_clone, - .reduce = NFT_REDUCE_READONLY, .offload = nft_counter_offload, .offload_stats = nft_counter_offload_stats, }; diff --git a/net/netfilter/nft_ct.c b/net/netfilter/nft_ct.c index c776eb38f1dbd..fa32bbb313065 100644 --- a/net/netfilter/nft_ct.c +++ b/net/netfilter/nft_ct.c @@ -698,29 +698,6 @@ static int nft_ct_get_dump(struct sk_buff *skb, return -1; } -static bool nft_ct_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ct *priv = nft_expr_priv(expr); - const struct nft_ct *ct; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - ct = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != ct->key) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static int nft_ct_set_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset) { @@ -755,27 +732,8 @@ static const struct nft_expr_ops nft_ct_get_ops = { .init = nft_ct_get_init, .destroy = nft_ct_get_destroy, .dump = nft_ct_get_dump, - .reduce = nft_ct_get_reduce, }; -static bool nft_ct_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_ct_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - #ifdef CONFIG_MITIGATION_RETPOLINE static const struct nft_expr_ops nft_ct_get_fast_ops = { .type = &nft_ct_type, @@ -784,7 +742,6 @@ static const struct nft_expr_ops nft_ct_get_fast_ops = { .init = nft_ct_get_init, .destroy = nft_ct_get_destroy, .dump = nft_ct_get_dump, - .reduce = nft_ct_set_reduce, }; #endif @@ -795,7 +752,6 @@ static const struct nft_expr_ops nft_ct_set_ops = { .init = nft_ct_set_init, .destroy = nft_ct_set_destroy, .dump = nft_ct_set_dump, - .reduce = nft_ct_set_reduce, }; #ifdef CONFIG_NF_CONNTRACK_ZONES @@ -806,7 +762,6 @@ static const struct nft_expr_ops nft_ct_set_zone_ops = { .init = nft_ct_set_init, .destroy = nft_ct_set_destroy, .dump = nft_ct_set_dump, - .reduce = nft_ct_set_reduce, }; #endif @@ -876,7 +831,6 @@ static const struct nft_expr_ops nft_notrack_ops = { .type = &nft_notrack_type, .size = NFT_EXPR_SIZE(0), .eval = nft_notrack_eval, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_notrack_type __read_mostly = { @@ -944,8 +898,6 @@ static void nft_ct_timeout_obj_eval(struct nft_object *obj, } } - rcu_assign_pointer(timeout->timeout, priv->timeout); - /* adjust the timeout as per 'new' state. ct is unconfirmed, * so the current timestamp must not be added. */ @@ -996,6 +948,7 @@ static int nft_ct_timeout_obj_init(const struct nft_ctx *ctx, timeout->l3num = l3num; timeout->l4proto = l4proto; + refcount_set(&timeout->refcnt, 1); ret = nf_ct_netns_get(ctx->net, ctx->family); if (ret < 0) @@ -1016,10 +969,10 @@ static void nft_ct_timeout_obj_destroy(const struct nft_ctx *ctx, struct nft_ct_timeout_obj *priv = nft_obj_data(obj); struct nf_ct_timeout *timeout = priv->timeout; - nf_queue_nf_hook_drop(ctx->net); nf_ct_untimeout(ctx->net, timeout); nf_ct_netns_put(ctx->net, ctx->family); - kfree_rcu(priv->timeout, rcu); + if (refcount_dec_and_test(&timeout->refcnt)) + kfree_rcu(priv->timeout, rcu); } static int nft_ct_timeout_obj_dump(struct sk_buff *skb, diff --git a/net/netfilter/nft_dup_netdev.c b/net/netfilter/nft_dup_netdev.c index 0573f96ce0791..06866799e9463 100644 --- a/net/netfilter/nft_dup_netdev.c +++ b/net/netfilter/nft_dup_netdev.c @@ -80,7 +80,6 @@ static const struct nft_expr_ops nft_dup_netdev_ops = { .eval = nft_dup_netdev_eval, .init = nft_dup_netdev_init, .dump = nft_dup_netdev_dump, - .reduce = NFT_REDUCE_READONLY, .offload = nft_dup_netdev_offload, .offload_action = nft_dup_netdev_offload_action, }; diff --git a/net/netfilter/nft_dynset.c b/net/netfilter/nft_dynset.c index 9123277be03ce..57bf94ae8724a 100644 --- a/net/netfilter/nft_dynset.c +++ b/net/netfilter/nft_dynset.c @@ -429,7 +429,6 @@ static const struct nft_expr_ops nft_dynset_ops = { .activate = nft_dynset_activate, .deactivate = nft_dynset_deactivate, .dump = nft_dynset_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_dynset_type __read_mostly = { diff --git a/net/netfilter/nft_exthdr.c b/net/netfilter/nft_exthdr.c index 9471328802d3b..cee93149dca7e 100644 --- a/net/netfilter/nft_exthdr.c +++ b/net/netfilter/nft_exthdr.c @@ -705,40 +705,12 @@ static int nft_exthdr_dump_strip(struct sk_buff *skb, return nft_exthdr_dump_common(skb, priv); } -static bool nft_exthdr_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_exthdr *priv = nft_expr_priv(expr); - const struct nft_exthdr *exthdr; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - exthdr = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->type != exthdr->type || - priv->op != exthdr->op || - priv->flags != exthdr->flags || - priv->offset != exthdr->offset || - priv->len != exthdr->len) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static const struct nft_expr_ops nft_exthdr_ipv6_ops = { .type = &nft_exthdr_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_exthdr)), .eval = nft_exthdr_ipv6_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_ipv4_ops = { @@ -747,7 +719,6 @@ static const struct nft_expr_ops nft_exthdr_ipv4_ops = { .eval = nft_exthdr_ipv4_eval, .init = nft_exthdr_ipv4_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_tcp_ops = { @@ -756,7 +727,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_ops = { .eval = nft_exthdr_tcp_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; static const struct nft_expr_ops nft_exthdr_tcp_set_ops = { @@ -765,7 +735,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_set_ops = { .eval = nft_exthdr_tcp_set_eval, .init = nft_exthdr_tcp_set_init, .dump = nft_exthdr_dump_set, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_exthdr_tcp_strip_ops = { @@ -774,7 +743,6 @@ static const struct nft_expr_ops nft_exthdr_tcp_strip_ops = { .eval = nft_exthdr_tcp_strip_eval, .init = nft_exthdr_tcp_strip_init, .dump = nft_exthdr_dump_strip, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_exthdr_sctp_ops = { @@ -783,7 +751,6 @@ static const struct nft_expr_ops nft_exthdr_sctp_ops = { .eval = nft_exthdr_sctp_eval, .init = nft_exthdr_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; #ifdef CONFIG_NFT_EXTHDR_DCCP @@ -793,7 +760,6 @@ static const struct nft_expr_ops nft_exthdr_dccp_ops = { .eval = nft_exthdr_dccp_eval, .init = nft_exthdr_dccp_init, .dump = nft_exthdr_dump, - .reduce = nft_exthdr_reduce, }; #endif diff --git a/net/netfilter/nft_fib.c b/net/netfilter/nft_fib.c index 22846136c754a..7b2a0a031c4b4 100644 --- a/net/netfilter/nft_fib.c +++ b/net/netfilter/nft_fib.c @@ -31,6 +31,15 @@ int nft_fib_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) const struct nft_fib *priv = nft_expr_priv(expr); unsigned int hooks; + switch (ctx->family) { + case NFPROTO_IPV4: + case NFPROTO_IPV6: + case NFPROTO_INET: + break; + default: + return -EOPNOTSUPP; + } + switch (priv->result) { case NFT_FIB_RESULT_OIF: case NFT_FIB_RESULT_OIFNAME: @@ -168,48 +177,6 @@ void nft_fib_store_result(void *reg, const struct nft_fib *priv, } EXPORT_SYMBOL_GPL(nft_fib_store_result); -bool nft_fib_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_fib *priv = nft_expr_priv(expr); - unsigned int len = NFT_REG32_SIZE; - const struct nft_fib *fib; - - switch (priv->result) { - case NFT_FIB_RESULT_OIF: - break; - case NFT_FIB_RESULT_OIFNAME: - if (priv->flags & NFTA_FIB_F_PRESENT) - len = NFT_REG32_SIZE; - else - len = IFNAMSIZ; - break; - case NFT_FIB_RESULT_ADDRTYPE: - break; - default: - WARN_ON_ONCE(1); - break; - } - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, len); - return false; - } - - fib = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->result != fib->result || - priv->flags != fib->flags) { - nft_reg_track_update(track, expr, priv->dreg, len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} -EXPORT_SYMBOL_GPL(nft_fib_reduce); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Query routing table from nftables"); MODULE_AUTHOR("Florian Westphal "); diff --git a/net/netfilter/nft_fib_inet.c b/net/netfilter/nft_fib_inet.c index 666a3741d20b6..a88d44e163d19 100644 --- a/net/netfilter/nft_fib_inet.c +++ b/net/netfilter/nft_fib_inet.c @@ -49,7 +49,6 @@ static const struct nft_expr_ops nft_fib_inet_ops = { .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, - .reduce = nft_fib_reduce, }; static struct nft_expr_type nft_fib_inet_type __read_mostly = { diff --git a/net/netfilter/nft_fib_netdev.c b/net/netfilter/nft_fib_netdev.c index 9121ec64e918f..5774a75440278 100644 --- a/net/netfilter/nft_fib_netdev.c +++ b/net/netfilter/nft_fib_netdev.c @@ -50,6 +50,33 @@ static void nft_fib_netdev_eval(const struct nft_expr *expr, regs->verdict.code = NFT_BREAK; } +static int nft_fib_netdev_validate(const struct nft_ctx *ctx, + const struct nft_expr *expr) +{ + const struct nft_fib *priv = nft_expr_priv(expr); + unsigned int hooks; + + switch (priv->result) { + case NFT_FIB_RESULT_OIF: + case NFT_FIB_RESULT_OIFNAME: + hooks = (1 << NF_NETDEV_INGRESS); + break; + case NFT_FIB_RESULT_ADDRTYPE: + if (priv->flags & NFTA_FIB_F_IIF) + hooks = (1 << NF_NETDEV_INGRESS); + else if (priv->flags & NFTA_FIB_F_OIF) + hooks = (1 << NF_NETDEV_EGRESS); + else + hooks = (1 << NF_NETDEV_INGRESS) | + (1 << NF_NETDEV_EGRESS); + break; + default: + return -EINVAL; + } + + return nft_chain_validate_hooks(ctx->chain, hooks); +} + static struct nft_expr_type nft_fib_netdev_type; static const struct nft_expr_ops nft_fib_netdev_ops = { .type = &nft_fib_netdev_type, @@ -57,8 +84,7 @@ static const struct nft_expr_ops nft_fib_netdev_ops = { .eval = nft_fib_netdev_eval, .init = nft_fib_init, .dump = nft_fib_dump, - .validate = nft_fib_validate, - .reduce = nft_fib_reduce, + .validate = nft_fib_netdev_validate, }; static struct nft_expr_type nft_fib_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_flow_offload.c b/net/netfilter/nft_flow_offload.c index b8f76c9057fda..3e1d6c6295b95 100644 --- a/net/netfilter/nft_flow_offload.c +++ b/net/netfilter/nft_flow_offload.c @@ -224,7 +224,6 @@ static const struct nft_expr_ops nft_flow_offload_ops = { .destroy = nft_flow_offload_destroy, .validate = nft_flow_offload_validate, .dump = nft_flow_offload_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_flow_offload_type __read_mostly = { diff --git a/net/netfilter/nft_fwd_netdev.c b/net/netfilter/nft_fwd_netdev.c index 256e832f1bb99..4bce36c3a6a07 100644 --- a/net/netfilter/nft_fwd_netdev.c +++ b/net/netfilter/nft_fwd_netdev.c @@ -228,7 +228,6 @@ static const struct nft_expr_ops nft_fwd_neigh_netdev_ops = { .init = nft_fwd_neigh_init, .dump = nft_fwd_neigh_dump, .validate = nft_fwd_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_fwd_netdev_ops = { @@ -238,7 +237,6 @@ static const struct nft_expr_ops nft_fwd_netdev_ops = { .init = nft_fwd_netdev_init, .dump = nft_fwd_netdev_dump, .validate = nft_fwd_validate, - .reduce = NFT_REDUCE_READONLY, .offload = nft_fwd_netdev_offload, .offload_action = nft_fwd_netdev_offload_action, }; diff --git a/net/netfilter/nft_hash.c b/net/netfilter/nft_hash.c index 5d034bbb6913c..1cf41e0a0e0c8 100644 --- a/net/netfilter/nft_hash.c +++ b/net/netfilter/nft_hash.c @@ -166,16 +166,6 @@ static int nft_jhash_dump(struct sk_buff *skb, return -1; } -static bool nft_jhash_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_jhash *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, sizeof(u32)); - - return false; -} - static int nft_symhash_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset) { @@ -196,30 +186,6 @@ static int nft_symhash_dump(struct sk_buff *skb, return -1; } -static bool nft_symhash_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_symhash *priv = nft_expr_priv(expr); - struct nft_symhash *symhash; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, sizeof(u32)); - return false; - } - - symhash = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->offset != symhash->offset || - priv->modulus != symhash->modulus) { - nft_reg_track_update(track, expr, priv->dreg, sizeof(u32)); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_hash_type; static const struct nft_expr_ops nft_jhash_ops = { .type = &nft_hash_type, @@ -227,7 +193,6 @@ static const struct nft_expr_ops nft_jhash_ops = { .eval = nft_jhash_eval, .init = nft_jhash_init, .dump = nft_jhash_dump, - .reduce = nft_jhash_reduce, }; static const struct nft_expr_ops nft_symhash_ops = { @@ -236,7 +201,6 @@ static const struct nft_expr_ops nft_symhash_ops = { .eval = nft_symhash_eval, .init = nft_symhash_init, .dump = nft_symhash_dump, - .reduce = nft_symhash_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_immediate.c b/net/netfilter/nft_immediate.c index 02ee5fb69871f..37c29947b380b 100644 --- a/net/netfilter/nft_immediate.c +++ b/net/netfilter/nft_immediate.c @@ -320,17 +320,6 @@ static bool nft_immediate_offload_action(const struct nft_expr *expr) return false; } -static bool nft_immediate_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_immediate_expr *priv = nft_expr_priv(expr); - - if (priv->dreg != NFT_REG_VERDICT) - nft_reg_track_cancel(track, priv->dreg, priv->dlen); - - return false; -} - static const struct nft_expr_ops nft_imm_ops = { .type = &nft_imm_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_immediate_expr)), @@ -341,7 +330,6 @@ static const struct nft_expr_ops nft_imm_ops = { .destroy = nft_immediate_destroy, .dump = nft_immediate_dump, .validate = nft_immediate_validate, - .reduce = nft_immediate_reduce, .offload = nft_immediate_offload, .offload_action = nft_immediate_offload_action, }; diff --git a/net/netfilter/nft_last.c b/net/netfilter/nft_last.c index de1b6066bfa85..e845779268d3a 100644 --- a/net/netfilter/nft_last.c +++ b/net/netfilter/nft_last.c @@ -125,7 +125,6 @@ static const struct nft_expr_ops nft_last_ops = { .destroy = nft_last_destroy, .clone = nft_last_clone, .dump = nft_last_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_last_type __read_mostly = { diff --git a/net/netfilter/nft_limit.c b/net/netfilter/nft_limit.c index 21d26b79b4607..0daeb0b23c205 100644 --- a/net/netfilter/nft_limit.c +++ b/net/netfilter/nft_limit.c @@ -243,7 +243,6 @@ static const struct nft_expr_ops nft_limit_pkts_ops = { .destroy = nft_limit_pkts_destroy, .clone = nft_limit_pkts_clone, .dump = nft_limit_pkts_dump, - .reduce = NFT_REDUCE_READONLY, }; static void nft_limit_bytes_eval(const struct nft_expr *expr, @@ -299,7 +298,6 @@ static const struct nft_expr_ops nft_limit_bytes_ops = { .dump = nft_limit_bytes_dump, .clone = nft_limit_bytes_clone, .destroy = nft_limit_bytes_destroy, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_log.c b/net/netfilter/nft_log.c index e355881379957..575021f6ee33c 100644 --- a/net/netfilter/nft_log.c +++ b/net/netfilter/nft_log.c @@ -291,7 +291,6 @@ static const struct nft_expr_ops nft_log_ops = { .init = nft_log_init, .destroy = nft_log_destroy, .dump = nft_log_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_log_type __read_mostly = { diff --git a/net/netfilter/nft_lookup.c b/net/netfilter/nft_lookup.c index beca9c7705e01..699254cb3ecd6 100644 --- a/net/netfilter/nft_lookup.c +++ b/net/netfilter/nft_lookup.c @@ -269,17 +269,6 @@ static int nft_lookup_validate(const struct nft_ctx *ctx, return 0; } -static bool nft_lookup_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_lookup *priv = nft_expr_priv(expr); - - if (priv->set->flags & NFT_SET_MAP) - nft_reg_track_cancel(track, priv->dreg, priv->set->dlen); - - return false; -} - static const struct nft_expr_ops nft_lookup_ops = { .type = &nft_lookup_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_lookup)), @@ -290,7 +279,6 @@ static const struct nft_expr_ops nft_lookup_ops = { .destroy = nft_lookup_destroy, .dump = nft_lookup_dump, .validate = nft_lookup_validate, - .reduce = nft_lookup_reduce, }; struct nft_expr_type nft_lookup_type __read_mostly = { diff --git a/net/netfilter/nft_masq.c b/net/netfilter/nft_masq.c index 868bd4d735555..2b01128737a3a 100644 --- a/net/netfilter/nft_masq.c +++ b/net/netfilter/nft_masq.c @@ -143,7 +143,6 @@ static const struct nft_expr_ops nft_masq_ipv4_ops = { .destroy = nft_masq_ipv4_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_ipv4_type __read_mostly = { @@ -171,7 +170,6 @@ static const struct nft_expr_ops nft_masq_ipv6_ops = { .destroy = nft_masq_ipv6_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_ipv6_type __read_mostly = { @@ -213,7 +211,6 @@ static const struct nft_expr_ops nft_masq_inet_ops = { .destroy = nft_masq_inet_destroy, .dump = nft_masq_dump, .validate = nft_masq_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_masq_inet_type __read_mostly = { diff --git a/net/netfilter/nft_meta.c b/net/netfilter/nft_meta.c index d3b0efec036aa..19e6d1c2436af 100644 --- a/net/netfilter/nft_meta.c +++ b/net/netfilter/nft_meta.c @@ -743,60 +743,16 @@ static int nft_meta_get_offload(struct nft_offload_ctx *ctx, return 0; } -bool nft_meta_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_meta *priv = nft_expr_priv(expr); - const struct nft_meta *meta; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - meta = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != meta->key || - priv->dreg != meta->dreg) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} -EXPORT_SYMBOL_GPL(nft_meta_get_reduce); - static const struct nft_expr_ops nft_meta_get_ops = { .type = &nft_meta_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), .eval = nft_meta_get_eval, .init = nft_meta_get_init, .dump = nft_meta_get_dump, - .reduce = nft_meta_get_reduce, .validate = nft_meta_get_validate, .offload = nft_meta_get_offload, }; -static bool nft_meta_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_meta_get_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static const struct nft_expr_ops nft_meta_set_ops = { .type = &nft_meta_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_meta)), @@ -804,7 +760,6 @@ static const struct nft_expr_ops nft_meta_set_ops = { .init = nft_meta_set_init, .destroy = nft_meta_set_destroy, .dump = nft_meta_set_dump, - .reduce = nft_meta_set_reduce, .validate = nft_meta_set_validate, }; diff --git a/net/netfilter/nft_nat.c b/net/netfilter/nft_nat.c index 6e21f72c5b574..e32cd9fbc7c2e 100644 --- a/net/netfilter/nft_nat.c +++ b/net/netfilter/nft_nat.c @@ -320,7 +320,6 @@ static const struct nft_expr_ops nft_nat_ops = { .destroy = nft_nat_destroy, .dump = nft_nat_dump, .validate = nft_nat_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_nat_type __read_mostly = { @@ -351,7 +350,6 @@ static const struct nft_expr_ops nft_nat_inet_ops = { .destroy = nft_nat_destroy, .dump = nft_nat_dump, .validate = nft_nat_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_inet_nat_type __read_mostly = { diff --git a/net/netfilter/nft_numgen.c b/net/netfilter/nft_numgen.c index bd058babfc820..06e87dfd76e74 100644 --- a/net/netfilter/nft_numgen.c +++ b/net/netfilter/nft_numgen.c @@ -84,16 +84,6 @@ static int nft_ng_inc_init(const struct nft_ctx *ctx, return err; } -static bool nft_ng_inc_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ng_inc *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - - return false; -} - static int nft_ng_dump(struct sk_buff *skb, enum nft_registers dreg, u32 modulus, enum nft_ng_types type, u32 offset) { @@ -178,16 +168,6 @@ static int nft_ng_random_dump(struct sk_buff *skb, priv->offset); } -static bool nft_ng_random_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_ng_random *priv = nft_expr_priv(expr); - - nft_reg_track_cancel(track, priv->dreg, NFT_REG32_SIZE); - - return false; -} - static struct nft_expr_type nft_ng_type; static const struct nft_expr_ops nft_ng_inc_ops = { .type = &nft_ng_type, @@ -196,7 +176,6 @@ static const struct nft_expr_ops nft_ng_inc_ops = { .init = nft_ng_inc_init, .destroy = nft_ng_inc_destroy, .dump = nft_ng_inc_dump, - .reduce = nft_ng_inc_reduce, }; static const struct nft_expr_ops nft_ng_random_ops = { @@ -205,7 +184,6 @@ static const struct nft_expr_ops nft_ng_random_ops = { .eval = nft_ng_random_eval, .init = nft_ng_random_init, .dump = nft_ng_random_dump, - .reduce = nft_ng_random_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_objref.c b/net/netfilter/nft_objref.c index 1a62e384766a7..633cce69568f1 100644 --- a/net/netfilter/nft_objref.c +++ b/net/netfilter/nft_objref.c @@ -123,7 +123,6 @@ static const struct nft_expr_ops nft_objref_ops = { .deactivate = nft_objref_deactivate, .dump = nft_objref_dump, .validate = nft_objref_validate, - .reduce = NFT_REDUCE_READONLY, }; struct nft_objref_map { @@ -245,7 +244,6 @@ static const struct nft_expr_ops nft_objref_map_ops = { .destroy = nft_objref_map_destroy, .dump = nft_objref_map_dump, .validate = nft_objref_map_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c index bdc2f6c90e2f7..8d84ef3fb487a 100644 --- a/net/netfilter/nft_osf.c +++ b/net/netfilter/nft_osf.c @@ -131,30 +131,6 @@ static int nft_osf_validate(const struct nft_ctx *ctx, return nft_chain_validate_hooks(ctx->chain, hooks); } -static bool nft_osf_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - struct nft_osf *priv = nft_expr_priv(expr); - struct nft_osf *osf; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, NFT_OSF_MAXGENRELEN); - return false; - } - - osf = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->flags != osf->flags || - priv->ttl != osf->ttl) { - nft_reg_track_update(track, expr, priv->dreg, NFT_OSF_MAXGENRELEN); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_osf_type; static const struct nft_expr_ops nft_osf_op = { .eval = nft_osf_eval, @@ -163,7 +139,6 @@ static const struct nft_expr_ops nft_osf_op = { .dump = nft_osf_dump, .type = &nft_osf_type, .validate = nft_osf_validate, - .reduce = nft_osf_reduce, }; static struct nft_expr_type nft_osf_type __read_mostly = { diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c index da1c6bf0dc572..e07888aaf1475 100644 --- a/net/netfilter/nft_payload.c +++ b/net/netfilter/nft_payload.c @@ -256,37 +256,10 @@ static int nft_payload_dump(struct sk_buff *skb, return -1; } -static bool nft_payload_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_payload *priv = nft_expr_priv(expr); - const struct nft_payload *payload; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - payload = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->base != payload->base || - priv->offset != payload->offset || - priv->len != payload->len) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static bool nft_payload_offload_mask(struct nft_offload_reg *reg, u32 priv_len, u32 field_len) { - unsigned int remainder, delta, k; struct nft_data mask = {}; - __be32 remainder_mask; if (priv_len == field_len) { memset(®->mask, 0xff, priv_len); @@ -295,15 +268,7 @@ static bool nft_payload_offload_mask(struct nft_offload_reg *reg, return false; } - memset(&mask, 0xff, field_len); - remainder = priv_len % sizeof(u32); - if (remainder) { - k = priv_len / sizeof(u32); - delta = field_len - priv_len; - remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1)); - mask.data[k] = (__force u32)remainder_mask; - } - + memset(&mask, 0xff, priv_len); memcpy(®->mask, &mask, field_len); return true; @@ -584,7 +549,6 @@ static const struct nft_expr_ops nft_payload_ops = { .eval = nft_payload_eval, .init = nft_payload_init, .dump = nft_payload_dump, - .reduce = nft_payload_reduce, .offload = nft_payload_offload, }; @@ -594,7 +558,6 @@ const struct nft_expr_ops nft_payload_fast_ops = { .eval = nft_payload_eval, .init = nft_payload_init, .dump = nft_payload_dump, - .reduce = nft_payload_reduce, .offload = nft_payload_offload, }; @@ -1022,32 +985,12 @@ static int nft_payload_set_dump(struct sk_buff *skb, return -1; } -static bool nft_payload_set_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - int i; - - for (i = 0; i < NFT_REG32_NUM; i++) { - if (!track->regs[i].selector) - continue; - - if (track->regs[i].selector->ops != &nft_payload_ops && - track->regs[i].selector->ops != &nft_payload_fast_ops) - continue; - - __nft_reg_track_cancel(track, i); - } - - return false; -} - static const struct nft_expr_ops nft_payload_set_ops = { .type = &nft_payload_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)), .eval = nft_payload_set_eval, .init = nft_payload_set_init, .dump = nft_payload_set_dump, - .reduce = nft_payload_set_reduce, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_queue.c b/net/netfilter/nft_queue.c index 344fe311878fe..8eb13a02942ed 100644 --- a/net/netfilter/nft_queue.c +++ b/net/netfilter/nft_queue.c @@ -191,7 +191,6 @@ static const struct nft_expr_ops nft_queue_ops = { .init = nft_queue_init, .dump = nft_queue_dump, .validate = nft_queue_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops nft_queue_sreg_ops = { @@ -201,7 +200,6 @@ static const struct nft_expr_ops nft_queue_sreg_ops = { .init = nft_queue_sreg_init, .dump = nft_queue_sreg_dump, .validate = nft_queue_validate, - .reduce = NFT_REDUCE_READONLY, }; static const struct nft_expr_ops * diff --git a/net/netfilter/nft_quota.c b/net/netfilter/nft_quota.c index df0798da2329b..bb3cf3d16e79b 100644 --- a/net/netfilter/nft_quota.c +++ b/net/netfilter/nft_quota.c @@ -140,11 +140,16 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, u64 consumed, consumed_cap, quota; u32 flags = priv->flags; - /* Since we inconditionally increment consumed quota for each packet + /* Since we unconditionally increment consumed quota for each packet * that we see, don't go over the quota boundary in what we send to * userspace. */ - consumed = atomic64_read(priv->consumed); + if (reset) { + consumed = atomic64_xchg(priv->consumed, 0); + clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); + } else { + consumed = atomic64_read(priv->consumed); + } quota = atomic64_read(&priv->quota); if (consumed >= quota) { consumed_cap = quota; @@ -160,10 +165,6 @@ static int nft_quota_do_dump(struct sk_buff *skb, struct nft_quota *priv, nla_put_be32(skb, NFTA_QUOTA_FLAGS, htonl(flags))) goto nla_put_failure; - if (reset) { - atomic64_sub(consumed, priv->consumed); - clear_bit(NFT_QUOTA_DEPLETED_BIT, &priv->flags); - } return 0; nla_put_failure: @@ -265,7 +266,6 @@ static const struct nft_expr_ops nft_quota_ops = { .destroy = nft_quota_destroy, .clone = nft_quota_clone, .dump = nft_quota_dump, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_quota_type __read_mostly = { diff --git a/net/netfilter/nft_range.c b/net/netfilter/nft_range.c index ea382f7bbd78d..cbb02644b8362 100644 --- a/net/netfilter/nft_range.c +++ b/net/netfilter/nft_range.c @@ -138,7 +138,6 @@ static const struct nft_expr_ops nft_range_ops = { .eval = nft_range_eval, .init = nft_range_init, .dump = nft_range_dump, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_range_type __read_mostly = { diff --git a/net/netfilter/nft_redir.c b/net/netfilter/nft_redir.c index 95eedad85c835..58ae802db8f52 100644 --- a/net/netfilter/nft_redir.c +++ b/net/netfilter/nft_redir.c @@ -146,7 +146,6 @@ static const struct nft_expr_ops nft_redir_ipv4_ops = { .destroy = nft_redir_ipv4_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_ipv4_type __read_mostly = { @@ -174,7 +173,6 @@ static const struct nft_expr_ops nft_redir_ipv6_ops = { .destroy = nft_redir_ipv6_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_ipv6_type __read_mostly = { @@ -203,7 +201,6 @@ static const struct nft_expr_ops nft_redir_inet_ops = { .destroy = nft_redir_inet_destroy, .dump = nft_redir_dump, .validate = nft_redir_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_redir_inet_type __read_mostly = { diff --git a/net/netfilter/nft_reject_inet.c b/net/netfilter/nft_reject_inet.c index 49020e67304ad..dcae83ddc32e5 100644 --- a/net/netfilter/nft_reject_inet.c +++ b/net/netfilter/nft_reject_inet.c @@ -79,7 +79,6 @@ static const struct nft_expr_ops nft_reject_inet_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_inet_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_inet_type __read_mostly = { diff --git a/net/netfilter/nft_reject_netdev.c b/net/netfilter/nft_reject_netdev.c index 2558ce1505d98..b53e81e4ca75d 100644 --- a/net/netfilter/nft_reject_netdev.c +++ b/net/netfilter/nft_reject_netdev.c @@ -158,7 +158,6 @@ static const struct nft_expr_ops nft_reject_netdev_ops = { .init = nft_reject_init, .dump = nft_reject_dump, .validate = nft_reject_netdev_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_reject_netdev_type __read_mostly = { diff --git a/net/netfilter/nft_rt.c b/net/netfilter/nft_rt.c index dc50b9a5bd688..ad527f3596c03 100644 --- a/net/netfilter/nft_rt.c +++ b/net/netfilter/nft_rt.c @@ -195,7 +195,6 @@ static const struct nft_expr_ops nft_rt_get_ops = { .init = nft_rt_get_init, .dump = nft_rt_get_dump, .validate = nft_rt_validate, - .reduce = NFT_REDUCE_READONLY, }; struct nft_expr_type nft_rt_type __read_mostly = { diff --git a/net/netfilter/nft_set_pipapo_avx2.c b/net/netfilter/nft_set_pipapo_avx2.c index 6395982e4d95c..29a548595857a 100644 --- a/net/netfilter/nft_set_pipapo_avx2.c +++ b/net/netfilter/nft_set_pipapo_avx2.c @@ -1156,6 +1156,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, struct nft_pipapo_scratch *scratch; const struct nft_pipapo_field *f; unsigned long *res, *fill, *map; + struct nft_pipapo_elem *e; bool map_index; int i; @@ -1228,14 +1229,11 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, next_match: if (ret < 0) { scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + e = NULL; + goto out; } if (last) { - struct nft_pipapo_elem *e; - e = f->mt[ret].e; if (unlikely(__nft_set_elem_expired(&e->ext, tstamp) || !nft_set_elem_active(&e->ext, genmask))) { @@ -1245,9 +1243,7 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, } scratch->map_index = map_index; - kernel_fpu_end(); - __local_unlock_nested_bh(&scratch->bh_lock); - return e; + goto out; } map_index = !map_index; @@ -1255,9 +1251,12 @@ struct nft_pipapo_elem *pipapo_get_avx2(const struct nft_pipapo_match *m, data += NFT_PIPAPO_GROUPS_PADDED_SIZE(f); } + e = NULL; +out: + asm volatile("vzeroupper"); kernel_fpu_end(); __local_unlock_nested_bh(&scratch->bh_lock); - return NULL; + return e; } /** diff --git a/net/netfilter/nft_socket.c b/net/netfilter/nft_socket.c index 36affbb697c2f..c55a1310226a4 100644 --- a/net/netfilter/nft_socket.c +++ b/net/netfilter/nft_socket.c @@ -249,31 +249,6 @@ static int nft_socket_dump(struct sk_buff *skb, return 0; } -static bool nft_socket_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_socket *priv = nft_expr_priv(expr); - const struct nft_socket *socket; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - socket = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != socket->key || - priv->dreg != socket->dreg || - priv->level != socket->level) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static int nft_socket_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { @@ -296,7 +271,6 @@ static const struct nft_expr_ops nft_socket_ops = { .init = nft_socket_init, .dump = nft_socket_dump, .validate = nft_socket_validate, - .reduce = nft_socket_reduce, }; static struct nft_expr_type nft_socket_type __read_mostly = { diff --git a/net/netfilter/nft_synproxy.c b/net/netfilter/nft_synproxy.c index ad3e7d27f02e9..a80bdce38546f 100644 --- a/net/netfilter/nft_synproxy.c +++ b/net/netfilter/nft_synproxy.c @@ -290,7 +290,6 @@ static const struct nft_expr_ops nft_synproxy_ops = { .dump = nft_synproxy_dump, .type = &nft_synproxy_type, .validate = nft_synproxy_validate, - .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_synproxy_type __read_mostly = { diff --git a/net/netfilter/nft_tproxy.c b/net/netfilter/nft_tproxy.c index 50481280abd26..f2101af8c867f 100644 --- a/net/netfilter/nft_tproxy.c +++ b/net/netfilter/nft_tproxy.c @@ -331,7 +331,6 @@ static const struct nft_expr_ops nft_tproxy_ops = { .init = nft_tproxy_init, .destroy = nft_tproxy_destroy, .dump = nft_tproxy_dump, - .reduce = NFT_REDUCE_READONLY, .validate = nft_tproxy_validate, }; diff --git a/net/netfilter/nft_tunnel.c b/net/netfilter/nft_tunnel.c index db183e43941ac..78d47f5503551 100644 --- a/net/netfilter/nft_tunnel.c +++ b/net/netfilter/nft_tunnel.c @@ -124,31 +124,6 @@ static int nft_tunnel_get_dump(struct sk_buff *skb, return -1; } -static bool nft_tunnel_get_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_tunnel *priv = nft_expr_priv(expr); - const struct nft_tunnel *tunnel; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - tunnel = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != tunnel->key || - priv->dreg != tunnel->dreg || - priv->mode != tunnel->mode) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return false; -} - static struct nft_expr_type nft_tunnel_type; static const struct nft_expr_ops nft_tunnel_get_ops = { .type = &nft_tunnel_type, @@ -156,7 +131,6 @@ static const struct nft_expr_ops nft_tunnel_get_ops = { .eval = nft_tunnel_get_eval, .init = nft_tunnel_get_init, .dump = nft_tunnel_get_dump, - .reduce = nft_tunnel_get_reduce, }; static struct nft_expr_type nft_tunnel_type __read_mostly = { diff --git a/net/netfilter/nft_xfrm.c b/net/netfilter/nft_xfrm.c index 3210cfc966ab2..7ffe6a2690d13 100644 --- a/net/netfilter/nft_xfrm.c +++ b/net/netfilter/nft_xfrm.c @@ -259,32 +259,6 @@ static int nft_xfrm_validate(const struct nft_ctx *ctx, const struct nft_expr *e return nft_chain_validate_hooks(ctx->chain, hooks); } -static bool nft_xfrm_reduce(struct nft_regs_track *track, - const struct nft_expr *expr) -{ - const struct nft_xfrm *priv = nft_expr_priv(expr); - const struct nft_xfrm *xfrm; - - if (!nft_reg_track_cmp(track, expr, priv->dreg)) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - xfrm = nft_expr_priv(track->regs[priv->dreg].selector); - if (priv->key != xfrm->key || - priv->dreg != xfrm->dreg || - priv->dir != xfrm->dir || - priv->spnum != xfrm->spnum) { - nft_reg_track_update(track, expr, priv->dreg, priv->len); - return false; - } - - if (!track->regs[priv->dreg].bitwise) - return true; - - return nft_expr_reduce_bitwise(track, expr); -} - static struct nft_expr_type nft_xfrm_type; static const struct nft_expr_ops nft_xfrm_get_ops = { .type = &nft_xfrm_type, @@ -293,7 +267,6 @@ static const struct nft_expr_ops nft_xfrm_get_ops = { .init = nft_xfrm_get_init, .dump = nft_xfrm_get_dump, .validate = nft_xfrm_validate, - .reduce = nft_xfrm_reduce, }; static struct nft_expr_type nft_xfrm_type __read_mostly = { diff --git a/net/netfilter/xt_CT.c b/net/netfilter/xt_CT.c index 498f5871c84a0..90a295cf5ec6b 100644 --- a/net/netfilter/xt_CT.c +++ b/net/netfilter/xt_CT.c @@ -284,7 +284,7 @@ static void xt_ct_tg_destroy(const struct xt_tgdtor_param *par, struct nf_conn_help *help; if (ct) { - if (info->helper[0] || info->timeout[0]) + if (info->helper[0]) nf_queue_nf_hook_drop(par->net); help = nfct_help(ct); diff --git a/net/netfilter/xt_cgroup.c b/net/netfilter/xt_cgroup.c index 43d2ae2be628d..28e6cd51b2fb0 100644 --- a/net/netfilter/xt_cgroup.c +++ b/net/netfilter/xt_cgroup.c @@ -23,7 +23,7 @@ MODULE_DESCRIPTION("Xtables: process control group matching"); MODULE_ALIAS("ipt_cgroup"); MODULE_ALIAS("ip6t_cgroup"); -#define NET_CLS_CLASSID_INVALID_MSG "xt_cgroup: classid invalid without net_cls cgroups\n" +#define NET_CLS_CLASSID_INVALID_MSG "classid invalid without net_cls cgroups\n" static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) { @@ -33,7 +33,7 @@ static int cgroup_mt_check_v0(const struct xt_mtchk_param *par) return -EINVAL; if (!IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -49,7 +49,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -59,7 +59,7 @@ static int cgroup_mt_check_v1(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } @@ -89,7 +89,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) return -EINVAL; if (!info->has_path && !info->has_classid) { - pr_info("xt_cgroup: no path or classid specified\n"); + pr_info_ratelimited("no path or classid specified\n"); return -EINVAL; } @@ -99,7 +99,7 @@ static int cgroup_mt_check_v2(const struct xt_mtchk_param *par) } if (info->has_classid && !IS_ENABLED(CONFIG_CGROUP_NET_CLASSID)) { - pr_info(NET_CLS_CLASSID_INVALID_MSG); + pr_info_ratelimited(NET_CLS_CLASSID_INVALID_MSG); return -EINVAL; } diff --git a/net/netfilter/xt_hashlimit.c b/net/netfilter/xt_hashlimit.c index 3b507694e81e5..b3f15bbf259e2 100644 --- a/net/netfilter/xt_hashlimit.c +++ b/net/netfilter/xt_hashlimit.c @@ -117,6 +117,7 @@ struct xt_hashlimit_htable { refcount_t use; u_int8_t family; bool rnd_initialized; + bool ratematch; struct hashlimit_cfg3 cfg; /* config */ @@ -323,6 +324,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg, kvfree(hinfo); return -ENOMEM; } + hinfo->ratematch = !!(cfg->mode & XT_HASHLIMIT_RATE_MATCH); spin_lock_init(&hinfo->lock); switch (revision) { @@ -870,7 +872,10 @@ static int hashlimit_mt_check_common(const struct xt_mtchk_param *par, } /* Check for overflow. */ - if (revision >= 3 && cfg->mode & XT_HASHLIMIT_RATE_MATCH) { + if (cfg->mode & XT_HASHLIMIT_RATE_MATCH) { + if (revision < 3) + return -EINVAL; + if (cfg->avg == 0 || cfg->avg > U32_MAX) { pr_info_ratelimited("invalid rate\n"); return -ERANGE; @@ -903,6 +908,15 @@ static int hashlimit_mt_check_common(const struct xt_mtchk_param *par, mutex_unlock(&hashlimit_mutex); return ret; } + } else { + if ((cfg->mode & XT_HASHLIMIT_RATE_MATCH && + !(*hinfo)->ratematch) || + (!(cfg->mode & XT_HASHLIMIT_RATE_MATCH) && + (*hinfo)->ratematch)) { + mutex_unlock(&hashlimit_mutex); + htable_put(*hinfo); + return -EINVAL; + } } mutex_unlock(&hashlimit_mutex); diff --git a/net/netfilter/xt_hl.c b/net/netfilter/xt_hl.c index c1a70f8f04417..59e93d97b5072 100644 --- a/net/netfilter/xt_hl.c +++ b/net/netfilter/xt_hl.c @@ -6,6 +6,7 @@ * Hop Limit matching module * (C) 2001-2002 Maciej Soltysiak */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include #include @@ -22,6 +23,18 @@ MODULE_LICENSE("GPL"); MODULE_ALIAS("ipt_ttl"); MODULE_ALIAS("ip6t_hl"); +static int ttl_mt_check(const struct xt_mtchk_param *par) +{ + const struct ipt_ttl_info *info = par->matchinfo; + + if (info->mode > IPT_TTL_GT) { + pr_info_ratelimited("Unknown TTL match mode: %d\n", info->mode); + return -EINVAL; + } + + return 0; +} + static bool ttl_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct ipt_ttl_info *info = par->matchinfo; @@ -41,6 +54,18 @@ static bool ttl_mt(const struct sk_buff *skb, struct xt_action_param *par) return false; } +static int hl_mt6_check(const struct xt_mtchk_param *par) +{ + const struct ip6t_hl_info *info = par->matchinfo; + + if (info->mode > IP6T_HL_GT) { + pr_info_ratelimited("Unknown Hop Limit match mode: %d\n", info->mode); + return -EINVAL; + } + + return 0; +} + static bool hl_mt6(const struct sk_buff *skb, struct xt_action_param *par) { const struct ip6t_hl_info *info = par->matchinfo; @@ -65,6 +90,7 @@ static struct xt_match hl_mt_reg[] __read_mostly = { .name = "ttl", .revision = 0, .family = NFPROTO_IPV4, + .checkentry = ttl_mt_check, .match = ttl_mt, .matchsize = sizeof(struct ipt_ttl_info), .me = THIS_MODULE, @@ -73,6 +99,7 @@ static struct xt_match hl_mt_reg[] __read_mostly = { .name = "hl", .revision = 0, .family = NFPROTO_IPV6, + .checkentry = hl_mt6_check, .match = hl_mt6, .matchsize = sizeof(struct ip6t_hl_info), .me = THIS_MODULE, diff --git a/net/netlabel/netlabel_unlabeled.c b/net/netlabel/netlabel_unlabeled.c index 2237a5261dd2a..0dfbb63d513ce 100644 --- a/net/netlabel/netlabel_unlabeled.c +++ b/net/netlabel/netlabel_unlabeled.c @@ -295,7 +295,7 @@ static int netlbl_unlhsh_add_addr6(struct netlbl_unlhsh_iface *iface, if (ret_val != 0) kfree(entry); - return 0; + return ret_val; } #endif /* IPv6 */ diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c index 978c129c60950..81b4494e2db74 100644 --- a/net/netlink/genetlink.c +++ b/net/netlink/genetlink.c @@ -1514,6 +1514,7 @@ struct ctrl_dump_policy_ctx { struct netlink_policy_dump_state *state; const struct genl_family *rt; struct genl_op_iter *op_iter; + struct module *owner; u32 op; u16 fam_id; u8 dump_map:1, @@ -1556,6 +1557,9 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) return -ENOENT; ctx->rt = rt; + ctx->owner = rt->module; + if (!try_module_get(ctx->owner)) + return -ENOENT; if (tb[CTRL_ATTR_OP]) { struct genl_split_ops doit, dump; @@ -1566,7 +1570,7 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) err = genl_get_cmd_both(ctx->op, rt, &doit, &dump); if (err) { NL_SET_BAD_ATTR(cb->extack, tb[CTRL_ATTR_OP]); - return err; + goto err_put_owner; } if (doit.policy) { @@ -1584,16 +1588,20 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) goto err_free_state; } - if (!ctx->state) - return -ENODATA; + if (!ctx->state) { + err = -ENODATA; + goto err_put_owner; + } ctx->dump_map = 1; return 0; } ctx->op_iter = kmalloc(sizeof(*ctx->op_iter), GFP_KERNEL); - if (!ctx->op_iter) - return -ENOMEM; + if (!ctx->op_iter) { + err = -ENOMEM; + goto err_put_owner; + } genl_op_iter_init(rt, ctx->op_iter); ctx->dump_map = genl_op_iter_next(ctx->op_iter); @@ -1625,6 +1633,8 @@ static int ctrl_dumppolicy_start(struct netlink_callback *cb) netlink_policy_dump_free(ctx->state); err_free_op_iter: kfree(ctx->op_iter); +err_put_owner: + module_put(ctx->owner); return err; } @@ -1761,6 +1771,7 @@ static int ctrl_dumppolicy_done(struct netlink_callback *cb) kfree(ctx->op_iter); netlink_policy_dump_free(ctx->state); + module_put(ctx->owner); return 0; } diff --git a/net/nfc/digital_core.c b/net/nfc/digital_core.c index dae378f1d52b6..b180212532243 100644 --- a/net/nfc/digital_core.c +++ b/net/nfc/digital_core.c @@ -127,7 +127,7 @@ static void digital_wq_cmd_complete(struct work_struct *work) mutex_unlock(&ddev->cmd_lock); - if (!IS_ERR(cmd->resp)) + if (!IS_ERR_OR_NULL(cmd->resp)) print_hex_dump_debug("DIGITAL RX: ", DUMP_PREFIX_NONE, 16, 1, cmd->resp->data, cmd->resp->len, false); diff --git a/net/nfc/digital_technology.c b/net/nfc/digital_technology.c index e29dd10f280ed..3c2d225d694a5 100644 --- a/net/nfc/digital_technology.c +++ b/net/nfc/digital_technology.c @@ -778,6 +778,8 @@ static void digital_in_recv_sensf_res(struct nfc_digital_dev *ddev, void *arg, sensf_res = (struct digital_sensf_res *)resp->data; + resp->len = min_t(unsigned int, resp->len, NFC_SENSF_RES_MAXSIZE); + memcpy(target.sensf_res, sensf_res, resp->len); target.sensf_res_len = resp->len; diff --git a/net/nfc/llcp_commands.c b/net/nfc/llcp_commands.c index b652323bc2c12..a93bf0b43d504 100644 --- a/net/nfc/llcp_commands.c +++ b/net/nfc/llcp_commands.c @@ -193,7 +193,8 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -201,9 +202,15 @@ int nfc_llcp_parse_gb_tlv(struct nfc_llcp_local *local, return -ENODEV; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { @@ -243,7 +250,8 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, const u8 *tlv_array, u16 tlv_array_len) { const u8 *tlv = tlv_array; - u8 type, length, offset = 0; + u8 type, length; + u16 offset = 0; pr_debug("TLV array length %d\n", tlv_array_len); @@ -251,9 +259,15 @@ int nfc_llcp_parse_connection_tlv(struct nfc_llcp_sock *sock, return -ENOTCONN; while (offset < tlv_array_len) { + if (offset + 2 > tlv_array_len) + return -EINVAL; + type = tlv[0]; length = tlv[1]; + if (offset + 2 + length > tlv_array_len) + return -EINVAL; + pr_debug("type 0x%x length %d\n", type, length); switch (type) { diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c index c83a00e429852..41f8e2f7875a0 100644 --- a/net/nfc/llcp_core.c +++ b/net/nfc/llcp_core.c @@ -849,13 +849,16 @@ static struct nfc_llcp_sock *nfc_llcp_sock_get_sn(struct nfc_llcp_local *local, static const u8 *nfc_llcp_connect_sn(const struct sk_buff *skb, size_t *sn_len) { u8 type, length; - const u8 *tlv = &skb->data[2]; - size_t tlv_array_len = skb->len - LLCP_HEADER_SIZE, offset = 0; + const u8 *tlv = &skb->data[LLCP_HEADER_SIZE]; + const u8 *tlv_end = skb_tail_pointer(skb); - while (offset < tlv_array_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + pr_debug("type 0x%x length %d\n", type, length); if (type == LLCP_TLV_SN) { @@ -863,7 +866,6 @@ static const u8 *nfc_llcp_connect_sn(const struct sk_buff *skb, size_t *sn_len) return &tlv[2]; } - offset += length + 2; tlv += length + 2; } @@ -1286,10 +1288,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, { struct nfc_llcp_sock *llcp_sock; u8 dsap, ssap, type, length, tid, sap; - const u8 *tlv; - u16 tlv_len, offset; + const u8 *tlv, *tlv_end; const char *service_name; - size_t service_name_len; + int service_name_len; struct nfc_llcp_sdp_tlv *sdp; HLIST_HEAD(llc_sdres_list); size_t sdres_tlvs_len; @@ -1305,22 +1306,34 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, return; } + /* + * Walk the SNL TLV list in the linear part of the skb only, + * bounded by skb_tail_pointer(). Each TLV needs a two-byte + * header (type, length) and its declared length must fit before + * the end; this also keeps the walk safe for very short frames. + */ tlv = &skb->data[LLCP_HEADER_SIZE]; - tlv_len = skb->len - LLCP_HEADER_SIZE; - offset = 0; + tlv_end = skb_tail_pointer(skb); sdres_tlvs_len = 0; - while (offset < tlv_len) { + while (tlv + 2 < tlv_end) { type = tlv[0]; length = tlv[1]; + if (tlv + 2 + length > tlv_end) + break; + switch (type) { case LLCP_TLV_SDREQ: + if (length < 1) + break; + tid = tlv[2]; service_name = (char *) &tlv[3]; service_name_len = length - 1; - pr_debug("Looking for %.16s\n", service_name); + pr_debug("Looking for %.*s\n", service_name_len, + service_name); if (service_name_len == strlen("urn:nfc:sn:sdp") && !strncmp(service_name, "urn:nfc:sn:sdp", @@ -1380,6 +1393,9 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; case LLCP_TLV_SDRES: + if (length != 2) + break; + mutex_lock(&local->sdreq_lock); pr_debug("LLCP_TLV_SDRES: searching tid %d\n", tlv[2]); @@ -1408,7 +1424,6 @@ static void nfc_llcp_recv_snl(struct nfc_llcp_local *local, break; } - offset += length + 2; tlv += length + 2; } @@ -1552,6 +1567,11 @@ static void nfc_llcp_rx_work(struct work_struct *work) static void __nfc_llcp_recv(struct nfc_llcp_local *local, struct sk_buff *skb) { + if (!pskb_may_pull(skb, LLCP_HEADER_SIZE)) { + kfree_skb(skb); + return; + } + local->rx_pending = skb; timer_delete(&local->link_timer); schedule_work(&local->rx_work); diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c index 915929cd724f9..507447bbceea4 100644 --- a/net/nfc/llcp_sock.c +++ b/net/nfc/llcp_sock.c @@ -319,14 +319,22 @@ static int nfc_llcp_getsockopt(struct socket *sock, int level, int optname, if (get_user(len, optlen)) return -EFAULT; - local = llcp_sock->local; - if (!local) - return -ENODEV; + if (len < 0) + return -EINVAL; + + if (len < sizeof(u32)) + return -EINVAL; len = min_t(u32, len, sizeof(u32)); lock_sock(sk); + local = llcp_sock->local; + if (!local) { + release_sock(sk); + return -ENODEV; + } + switch (optname) { case NFC_LLCP_RW: rw = llcp_sock->rw > LLCP_MAX_RW ? local->rw : llcp_sock->rw; diff --git a/net/nfc/nci/data.c b/net/nfc/nci/data.c index 5f98c73db5afd..4253edea5c8db 100644 --- a/net/nfc/nci/data.c +++ b/net/nfc/nci/data.c @@ -46,11 +46,11 @@ void nci_data_exchange_complete(struct nci_dev *ndev, struct sk_buff *skb, timer_delete_sync(&ndev->data_timer); clear_bit(NCI_DATA_EXCHANGE_TO, &ndev->flags); - /* Mark the exchange as done before calling the callback. - * The callback (e.g. rawsock_data_exchange_complete) may - * want to immediately queue another data exchange. - */ - clear_bit(NCI_DATA_EXCHANGE, &ndev->flags); + /* Claim completion atomically -- both close and rx_work may race here */ + if (!test_and_clear_bit(NCI_DATA_EXCHANGE, &ndev->flags)) { + kfree_skb(skb); + return; + } if (cb) { /* forward skb to nfc core */ diff --git a/net/nfc/nci/ntf.c b/net/nfc/nci/ntf.c index c96512bb86531..f5c9a8ab7ec10 100644 --- a/net/nfc/nci/ntf.c +++ b/net/nfc/nci/ntf.c @@ -440,7 +440,7 @@ void nci_clear_target_list(struct nci_dev *ndev) static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, const struct sk_buff *skb) { - struct nci_rf_discover_ntf ntf; + struct nci_rf_discover_ntf ntf = {}; const __u8 *data; bool add_target = true; @@ -525,15 +525,19 @@ static int nci_rf_discover_ntf_packet(struct nci_dev *ndev, static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_nfca_poll_iso_dep *nfca_poll; struct activation_params_nfcb_poll_iso_dep *nfcb_poll; switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfca_poll = &ntf->activation_params.nfca_poll_iso_dep; nfca_poll->rats_res_len = min_t(__u8, *data++, NFC_ATS_MAXSIZE); + data_len--; + nfca_poll->rats_res_len = min_t(__u8, nfca_poll->rats_res_len, data_len); pr_debug("rats_res_len %d\n", nfca_poll->rats_res_len); if (nfca_poll->rats_res_len > 0) { memcpy(nfca_poll->rats_res, @@ -542,8 +546,12 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, break; case NCI_NFC_B_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; nfcb_poll = &ntf->activation_params.nfcb_poll_iso_dep; nfcb_poll->attrib_res_len = min_t(__u8, *data++, 50); + data_len--; + nfcb_poll->attrib_res_len = min_t(__u8, nfcb_poll->attrib_res_len, data_len); pr_debug("attrib_res_len %d\n", nfcb_poll->attrib_res_len); if (nfcb_poll->attrib_res_len > 0) { memcpy(nfcb_poll->attrib_res, @@ -562,7 +570,7 @@ static int nci_extract_activation_params_iso_dep(struct nci_dev *ndev, static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, struct nci_rf_intf_activated_ntf *ntf, - const __u8 *data) + const __u8 *data, __u8 data_len) { struct activation_params_poll_nfc_dep *poll; struct activation_params_listen_nfc_dep *listen; @@ -570,9 +578,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, switch (ntf->activation_rf_tech_and_mode) { case NCI_NFC_A_PASSIVE_POLL_MODE: case NCI_NFC_F_PASSIVE_POLL_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; poll = &ntf->activation_params.poll_nfc_dep; poll->atr_res_len = min_t(__u8, *data++, NFC_ATR_RES_MAXSIZE - 2); + data_len--; + poll->atr_res_len = min_t(__u8, poll->atr_res_len, data_len); pr_debug("atr_res_len %d\n", poll->atr_res_len); if (poll->atr_res_len > 0) memcpy(poll->atr_res, data, poll->atr_res_len); @@ -580,9 +592,13 @@ static int nci_extract_activation_params_nfc_dep(struct nci_dev *ndev, case NCI_NFC_A_PASSIVE_LISTEN_MODE: case NCI_NFC_F_PASSIVE_LISTEN_MODE: + if (data_len < 1) + return NCI_STATUS_RF_PROTOCOL_ERROR; listen = &ntf->activation_params.listen_nfc_dep; listen->atr_req_len = min_t(__u8, *data++, NFC_ATR_REQ_MAXSIZE - 2); + data_len--; + listen->atr_req_len = min_t(__u8, listen->atr_req_len, data_len); pr_debug("atr_req_len %d\n", listen->atr_req_len); if (listen->atr_req_len > 0) memcpy(listen->atr_req, data, listen->atr_req_len); @@ -603,6 +619,12 @@ static void nci_target_auto_activated(struct nci_dev *ndev, struct nfc_target *target; int rc; + /* This is a new target, check if we've enough room */ + if (ndev->n_targets == NCI_MAX_DISCOVERED_TARGETS) { + pr_debug("not enough room, ignoring new target...\n"); + return; + } + target = &ndev->targets[ndev->n_targets]; rc = nci_add_new_protocol(ndev, target, ntf->rf_protocol, @@ -688,7 +710,7 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, const struct sk_buff *skb) { struct nci_conn_info *conn_info; - struct nci_rf_intf_activated_ntf ntf; + struct nci_rf_intf_activated_ntf ntf = {}; const __u8 *data; int err = NCI_STATUS_OK; @@ -806,12 +828,14 @@ static int nci_rf_intf_activated_ntf_packet(struct nci_dev *ndev, switch (ntf.rf_interface) { case NCI_RF_INTERFACE_ISO_DEP: err = nci_extract_activation_params_iso_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_NFC_DEP: err = nci_extract_activation_params_nfc_dep(ndev, - &ntf, data); + &ntf, data, + ntf.activation_params_len); break; case NCI_RF_INTERFACE_FRAME: diff --git a/net/nfc/nci/rsp.c b/net/nfc/nci/rsp.c index 9eeb862825c5f..b0ab4f5acbce8 100644 --- a/net/nfc/nci/rsp.c +++ b/net/nfc/nci/rsp.c @@ -50,11 +50,27 @@ static u8 nci_core_init_rsp_packet_v1(struct nci_dev *ndev, const struct nci_core_init_rsp_1 *rsp_1 = (void *)skb->data; const struct nci_core_init_rsp_2 *rsp_2; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status 0x%x\n", rsp_1->status); if (rsp_1->status != NCI_STATUS_OK) return rsp_1->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp_1)) + return NCI_STATUS_SYNTAX_ERROR; + + /* Ensure the variable-length rf_interfaces array and trailing + * rsp_2 structure are fully contained within the skb. + */ + if (skb_headlen(skb) < sizeof(*rsp_1) + + rsp_1->num_supported_rf_interfaces + + sizeof(*rsp_2)) + return NCI_STATUS_SYNTAX_ERROR; + ndev->nfcc_features = __le32_to_cpu(rsp_1->nfcc_features); ndev->num_supported_rf_interfaces = rsp_1->num_supported_rf_interfaces; @@ -87,15 +103,25 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, const struct sk_buff *skb) { const struct nci_core_init_rsp_nci_ver2 *rsp = (void *)skb->data; - const u8 *supported_rf_interface = rsp->supported_rf_interfaces; + const u8 *supported_rf_interface; u8 rf_interface_idx = 0; u8 rf_extension_cnt = 0; + /* Ensure that the status field can be accessed. */ + if (skb_headlen(skb) < 1) + return NCI_STATUS_SYNTAX_ERROR; + pr_debug("status %x\n", rsp->status); if (rsp->status != NCI_STATUS_OK) return rsp->status; + /* Success response must contain the full fixed-size header */ + if (skb_headlen(skb) < sizeof(*rsp)) + return NCI_STATUS_SYNTAX_ERROR; + + supported_rf_interface = rsp->supported_rf_interfaces; + ndev->nfcc_features = __le32_to_cpu(rsp->nfcc_features); ndev->num_supported_rf_interfaces = rsp->num_supported_rf_interfaces; @@ -104,13 +130,22 @@ static u8 nci_core_init_rsp_packet_v2(struct nci_dev *ndev, NCI_MAX_SUPPORTED_RF_INTERFACES); while (rf_interface_idx < ndev->num_supported_rf_interfaces) { - ndev->supported_rf_interfaces[rf_interface_idx++] = *supported_rf_interface++; + /* Each entry: [rf_interface_type (1B)] [ext_count (1B)] [ext...] */ + if (supported_rf_interface + 2 > skb_tail_pointer(skb)) + break; + ndev->supported_rf_interfaces[rf_interface_idx] = *supported_rf_interface++; - /* skip rf extension parameters */ rf_extension_cnt = *supported_rf_interface++; + if (supported_rf_interface + rf_extension_cnt > skb_tail_pointer(skb)) + break; + + /* Only count the entry after full validation */ + rf_interface_idx++; supported_rf_interface += rf_extension_cnt; } + ndev->num_supported_rf_interfaces = rf_interface_idx; + ndev->max_logical_connections = rsp->max_logical_connections; ndev->max_routing_table_size = __le16_to_cpu(rsp->max_routing_table_size); @@ -336,6 +371,7 @@ static void nci_core_conn_close_rsp_packet(struct nci_dev *ndev, list_del(&conn_info->list); if (conn_info == ndev->rf_conn_info) ndev->rf_conn_info = NULL; + devm_kfree(&ndev->nfc_dev->dev, conn_info->dest_params); devm_kfree(&ndev->nfc_dev->dev, conn_info); } } diff --git a/net/openvswitch/actions.c b/net/openvswitch/actions.c index 792ca44a461da..5d6a4c6b7ea6f 100644 --- a/net/openvswitch/actions.c +++ b/net/openvswitch/actions.c @@ -836,12 +836,8 @@ static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port, u16 mru = OVS_CB(skb)->mru; u32 cutlen = OVS_CB(skb)->cutlen; - if (unlikely(cutlen > 0)) { - if (skb->len - cutlen > ovs_mac_header_len(key)) - pskb_trim(skb, skb->len - cutlen); - else - pskb_trim(skb, ovs_mac_header_len(key)); - } + if (unlikely(cutlen < skb->len)) + pskb_trim(skb, max(cutlen, ovs_mac_header_len(key))); if (likely(!mru || (skb->len <= mru + vport->dev->hard_header_len))) { @@ -1111,6 +1107,10 @@ static int execute_masked_set_action(struct sk_buff *skb, return err; } +/* When 'last' is true, recirc() should always consume the 'skb'. + * Otherwise, recirc() should keep 'skb' intact regardless what + * actions are executed on recirculation. + */ static int execute_recirc(struct datapath *dp, struct sk_buff *skb, struct sw_flow_key *key, const struct nlattr *a, bool last) @@ -1121,8 +1121,12 @@ static int execute_recirc(struct datapath *dp, struct sk_buff *skb, int err; err = ovs_flow_key_update(skb, key); - if (err) + if (err) { + if (last) + ovs_kfree_skb_reason(skb, + OVS_DROP_ACTION_ERROR); return err; + } } BUG_ON(!is_flow_key_valid(key)); @@ -1233,7 +1237,7 @@ static void execute_psample(struct datapath *dp, struct sk_buff *skb, psample_group.net = ovs_dp_get_net(dp); md.in_ifindex = OVS_CB(skb)->input_vport->dev->ifindex; - md.trunc_size = skb->len - OVS_CB(skb)->cutlen; + md.trunc_size = min(skb->len, OVS_CB(skb)->cutlen); md.rate_as_probability = 1; rate = OVS_CB(skb)->probability ? OVS_CB(skb)->probability : U32_MAX; @@ -1283,22 +1287,21 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, clone = skb_clone(skb, GFP_ATOMIC); if (clone) do_output(dp, clone, port, key); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; break; } case OVS_ACTION_ATTR_TRUNC: { struct ovs_action_trunc *trunc = nla_data(a); - if (skb->len > trunc->max_len) - OVS_CB(skb)->cutlen = skb->len - trunc->max_len; + OVS_CB(skb)->cutlen = trunc->max_len; break; } case OVS_ACTION_ATTR_USERSPACE: output_userspace(dp, skb, key, a, attr, len, OVS_CB(skb)->cutlen); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; if (nla_is_last(a, rem)) { consume_skb(skb); return 0; @@ -1376,7 +1379,7 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, if (!is_flow_key_valid(key)) { err = ovs_flow_key_update(skb, key); if (err) - return err; + break; } err = ovs_ct_execute(ovs_dp_get_net(dp), skb, key, @@ -1452,7 +1455,7 @@ static int do_execute_actions(struct datapath *dp, struct sk_buff *skb, case OVS_ACTION_ATTR_PSAMPLE: execute_psample(dp, skb, a); - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; if (nla_is_last(a, rem)) { consume_skb(skb); return 0; diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c index f63c8ac74e4a8..3f51f19686541 100644 --- a/net/openvswitch/conntrack.c +++ b/net/openvswitch/conntrack.c @@ -932,10 +932,14 @@ static int ovs_ct_check_limit(struct net *net, const struct ovs_conntrack_info *info) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - const struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + const struct ovs_ct_limit_info *ct_limit_info; u32 per_zone_limit, connections; u32 conncount_key; + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); + if (!ct_limit_info) + return 0; + conncount_key = info->zone.id; per_zone_limit = ct_limit_get(ct_limit_info, info->zone.id); @@ -1584,42 +1588,56 @@ static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info) #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) static int ovs_ct_limit_init(struct net *net, struct ovs_net *ovs_net) { + struct ovs_ct_limit_info *info; int i, err; - ovs_net->ct_limit_info = kmalloc(sizeof(*ovs_net->ct_limit_info), - GFP_KERNEL); - if (!ovs_net->ct_limit_info) + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) return -ENOMEM; - ovs_net->ct_limit_info->default_limit = OVS_CT_LIMIT_DEFAULT; - ovs_net->ct_limit_info->limits = + info->default_limit = OVS_CT_LIMIT_DEFAULT; + info->limits = kmalloc_array(CT_LIMIT_HASH_BUCKETS, sizeof(struct hlist_head), GFP_KERNEL); - if (!ovs_net->ct_limit_info->limits) { - kfree(ovs_net->ct_limit_info); + if (!info->limits) { + kfree(info); return -ENOMEM; } for (i = 0; i < CT_LIMIT_HASH_BUCKETS; i++) - INIT_HLIST_HEAD(&ovs_net->ct_limit_info->limits[i]); + INIT_HLIST_HEAD(&info->limits[i]); - ovs_net->ct_limit_info->data = nf_conncount_init(net, sizeof(u32)); + info->data = nf_conncount_init(net, sizeof(u32)); - if (IS_ERR(ovs_net->ct_limit_info->data)) { - err = PTR_ERR(ovs_net->ct_limit_info->data); - kfree(ovs_net->ct_limit_info->limits); - kfree(ovs_net->ct_limit_info); + if (IS_ERR(info->data)) { + err = PTR_ERR(info->data); + kfree(info->limits); + kfree(info); pr_err("openvswitch: failed to init nf_conncount %d\n", err); return err; } + rcu_assign_pointer(ovs_net->ct_limit_info, info); return 0; } -static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) +static void *ovs_ct_limit_exit_start(struct ovs_net *ovs_net) { - const struct ovs_ct_limit_info *info = ovs_net->ct_limit_info; + return rcu_replace_pointer(ovs_net->ct_limit_info, NULL, + lockdep_ovsl_is_held()); +} + +/* The CT limit state must be detached by ovs_ct_limit_exit_start() and an + * RCU grace period must elapse before this function runs. The pernet core + * guarantees the grace period between the .pre_exit and .exit callbacks. + */ +static void ovs_ct_limit_exit_finish(struct net *net, void *data) +{ + const struct ovs_ct_limit_info *info = data; int i; + if (!info) + return; + nf_conncount_destroy(net, info->data); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { struct hlist_head *head = &info->limits[i]; @@ -1627,7 +1645,7 @@ static void ovs_ct_limit_exit(struct net *net, struct ovs_net *ovs_net) struct hlist_node *next; hlist_for_each_entry_safe(ct_limit, next, head, hlist_node) - kfree_rcu(ct_limit, rcu); + kfree(ct_limit); } kfree(info->limits); kfree(info); @@ -1666,12 +1684,13 @@ static bool check_zone_id(int zone_id, u16 *pzone) return false; } -static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_set_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1680,6 +1699,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = zone_limit->limit; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1697,6 +1717,7 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, ct_limit->limit = zone_limit->limit; ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_set(info, ct_limit); ovs_unlock(); } @@ -1711,12 +1732,13 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit, return 0; } -static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, - struct ovs_ct_limit_info *info) +static int ovs_ct_limit_del_zone_limit(struct ovs_net *ovs_net, + struct nlattr *nla_zone_limit) { struct ovs_zone_limit *zone_limit; - int rem; + struct ovs_ct_limit_info *info; u16 zone; + int rem; rem = NLA_ALIGN(nla_len(nla_zone_limit)); zone_limit = (struct ovs_zone_limit *)nla_data(nla_zone_limit); @@ -1725,6 +1747,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, if (unlikely(zone_limit->zone_id == OVS_ZONE_LIMIT_DEFAULT_ZONE)) { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); info->default_limit = OVS_CT_LIMIT_DEFAULT; ovs_unlock(); } else if (unlikely(!check_zone_id( @@ -1732,6 +1755,7 @@ static int ovs_ct_limit_del_zone_limit(struct nlattr *nla_zone_limit, OVS_NLERR(true, "zone id is out of range"); } else { ovs_lock(); + info = ovsl_dereference(ovs_net->ct_limit_info); ct_limit_del(info, zone); ovs_unlock(); } @@ -1775,6 +1799,7 @@ static int __ovs_ct_limit_get_zone_limit(struct net *net, return nla_put_nohdr(reply, sizeof(zone_limit), &zone_limit); } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_zone_limit(struct net *net, struct nlattr *nla_zone_limit, struct ovs_ct_limit_info *info, @@ -1798,12 +1823,10 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, &zone))) { OVS_NLERR(true, "zone id is out of range"); } else { - rcu_read_lock(); limit = ct_limit_get(info, zone); err = __ovs_ct_limit_get_zone_limit( net, info->data, zone, limit, reply); - rcu_read_unlock(); if (err) return err; } @@ -1818,6 +1841,7 @@ static int ovs_ct_limit_get_zone_limit(struct net *net, return 0; } +/* Called with RCU read lock held. */ static int ovs_ct_limit_get_all_zone_limit(struct net *net, struct ovs_ct_limit_info *info, struct sk_buff *reply) @@ -1830,19 +1854,16 @@ static int ovs_ct_limit_get_all_zone_limit(struct net *net, if (err) return err; - rcu_read_lock(); for (i = 0; i < CT_LIMIT_HASH_BUCKETS; ++i) { head = &info->limits[i]; hlist_for_each_entry_rcu(ct_limit, head, hlist_node) { err = __ovs_ct_limit_get_zone_limit(net, info->data, ct_limit->zone, ct_limit->limit, reply); if (err) - goto exit_err; + return err; } } -exit_err: - rcu_read_unlock(); return err; } @@ -1852,7 +1873,6 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_SET, @@ -1865,8 +1885,8 @@ static int ovs_ct_limit_cmd_set(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_set_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_set_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1886,7 +1906,6 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) struct sk_buff *reply; struct ovs_header *ovs_reply_header; struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_DEL, @@ -1899,8 +1918,8 @@ static int ovs_ct_limit_cmd_del(struct sk_buff *skb, struct genl_info *info) goto exit_err; } - err = ovs_ct_limit_del_zone_limit(a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], - ct_limit_info); + err = ovs_ct_limit_del_zone_limit(ovs_net, + a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]); if (err) goto exit_err; @@ -1920,7 +1939,7 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) struct ovs_header *ovs_reply_header; struct net *net = sock_net(skb->sk); struct ovs_net *ovs_net = net_generic(net, ovs_net_id); - struct ovs_ct_limit_info *ct_limit_info = ovs_net->ct_limit_info; + struct ovs_ct_limit_info *ct_limit_info; int err; reply = ovs_ct_limit_cmd_reply_start(info, OVS_CT_LIMIT_CMD_GET, @@ -1934,18 +1953,19 @@ static int ovs_ct_limit_cmd_get(struct sk_buff *skb, struct genl_info *info) goto exit_err; } + rcu_read_lock(); + ct_limit_info = rcu_dereference(ovs_net->ct_limit_info); if (a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT]) { err = ovs_ct_limit_get_zone_limit( net, a[OVS_CT_LIMIT_ATTR_ZONE_LIMIT], ct_limit_info, reply); - if (err) - goto exit_err; } else { err = ovs_ct_limit_get_all_zone_limit(net, ct_limit_info, reply); - if (err) - goto exit_err; } + rcu_read_unlock(); + if (err) + goto exit_err; nla_nest_end(reply, nla_reply); genlmsg_end(reply, ovs_reply_header); @@ -2003,6 +2023,7 @@ int ovs_ct_init(struct net *net) { unsigned int n_bits = sizeof(struct ovs_key_ct_labels) * BITS_PER_BYTE; struct ovs_net *ovs_net = net_generic(net, ovs_net_id); + int err = 0; if (nf_connlabels_get(net, n_bits - 1)) { ovs_net->xt_label = false; @@ -2012,18 +2033,36 @@ int ovs_ct_init(struct net *net) } #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - return ovs_ct_limit_init(net, ovs_net); -#else - return 0; + err = ovs_ct_limit_init(net, ovs_net); + if (err && ovs_net->xt_label) + nf_connlabels_put(net); #endif + return err; } -void ovs_ct_exit(struct net *net) +/* Must be called with ovs_mutex held. Detaches the RCU-protected + * ct_limit_info and stores it in ovs_net->ct_limit_exit_data for + * ovs_ct_exit_finish() to complete the teardown after an RCU grace period. + */ +void ovs_ct_exit_start(struct net *net __maybe_unused) +{ +#if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) + struct ovs_net *ovs_net = net_generic(net, ovs_net_id); + + ovs_net->ct_limit_exit_data = ovs_ct_limit_exit_start(ovs_net); +#endif +} + +/* Completes the CT limit teardown. The pernet core guarantees an RCU + * grace period between detaching the state in ovs_ct_exit_start() and + * this call, so no RCU readers remain. + */ +void ovs_ct_exit_finish(struct net *net) { struct ovs_net *ovs_net = net_generic(net, ovs_net_id); #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - ovs_ct_limit_exit(net, ovs_net); + ovs_ct_limit_exit_finish(net, ovs_net->ct_limit_exit_data); #endif if (ovs_net->xt_label) diff --git a/net/openvswitch/conntrack.h b/net/openvswitch/conntrack.h index 317e525c8a113..ab21d032fde3d 100644 --- a/net/openvswitch/conntrack.h +++ b/net/openvswitch/conntrack.h @@ -14,7 +14,8 @@ enum ovs_key_attr; #if IS_ENABLED(CONFIG_NF_CONNTRACK) int ovs_ct_init(struct net *); -void ovs_ct_exit(struct net *); +void ovs_ct_exit_start(struct net *net); +void ovs_ct_exit_finish(struct net *net); bool ovs_ct_verify(struct net *, enum ovs_key_attr attr); int ovs_ct_copy_action(struct net *, const struct nlattr *, const struct sw_flow_key *, struct sw_flow_actions **, @@ -40,7 +41,8 @@ void ovs_ct_free_action(const struct nlattr *a); static inline int ovs_ct_init(struct net *net) { return 0; } -static inline void ovs_ct_exit(struct net *net) { } +static inline void ovs_ct_exit_start(struct net *net) { } +static inline void ovs_ct_exit_finish(struct net *net) { } static inline bool ovs_ct_verify(struct net *net, int attr) { diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c index 56c744e1e14c5..c7991b9db6a76 100644 --- a/net/openvswitch/datapath.c +++ b/net/openvswitch/datapath.c @@ -276,7 +276,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) upcall.portid = ovs_vport_find_upcall_portid(p, skb); upcall.mru = OVS_CB(skb)->mru; - error = ovs_dp_upcall(dp, skb, key, &upcall, 0); + error = ovs_dp_upcall(dp, skb, key, &upcall, U32_MAX); switch (error) { case 0: case -EAGAIN: @@ -285,6 +285,7 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key) consume_skb(skb); break; default: + skb_tx_error(skb); kfree_skb(skb); break; } @@ -457,7 +458,8 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, struct sk_buff *nskb = NULL; struct sk_buff *user_skb = NULL; /* to be queued to userspace */ struct nlattr *nla; - size_t len; + size_t msg_size; + size_t skb_len; unsigned int hlen; int err, dp_ifindex; u64 hash; @@ -478,7 +480,8 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, skb = nskb; } - if (nla_attr_size(skb->len) > USHRT_MAX) { + skb_len = min(skb->len, cutlen); + if (nla_attr_size(skb_len) > USHRT_MAX) { err = -EFBIG; goto out; } @@ -493,13 +496,13 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, * padding logic. Only perform zerocopy if padding is not required. */ if (dp->user_features & OVS_DP_F_UNALIGNED) - hlen = skb_zerocopy_headlen(skb); + hlen = min(skb_zerocopy_headlen(skb), cutlen); else - hlen = skb->len; + hlen = skb_len; - len = upcall_msg_size(upcall_info, hlen - cutlen, - OVS_CB(skb)->acts_origlen); - user_skb = genlmsg_new(len, GFP_ATOMIC); + msg_size = upcall_msg_size(upcall_info, hlen, + OVS_CB(skb)->acts_origlen); + user_skb = genlmsg_new(msg_size, GFP_ATOMIC); if (!user_skb) { err = -ENOMEM; goto out; @@ -560,7 +563,7 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, } /* Add OVS_PACKET_ATTR_LEN when packet is truncated */ - if (cutlen > 0 && + if (skb_len < skb->len && nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) { err = -ENOBUFS; goto out; @@ -585,9 +588,9 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, err = -ENOBUFS; goto out; } - nla->nla_len = nla_attr_size(skb->len - cutlen); + nla->nla_len = nla_attr_size(skb_len); - err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen); + err = skb_zerocopy(user_skb, skb, skb_len, hlen); if (err) goto out; @@ -599,8 +602,6 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb, err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid); user_skb = NULL; out: - if (err) - skb_tx_error(skb); consume_skb(user_skb); consume_skb(nskb); @@ -644,6 +645,7 @@ static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info) packet->ignore_df = 1; } OVS_CB(packet)->mru = mru; + OVS_CB(packet)->cutlen = U32_MAX; if (a[OVS_PACKET_ATTR_HASH]) { hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]); @@ -1110,9 +1112,8 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) error = -EEXIST; goto err_unlock_ovs; } - /* The flow identifier has to be the same for flow updates. - * Look for any overlapping flow. - */ + + /* Look for any overlapping flow. */ if (unlikely(!ovs_flow_cmp(flow, &match))) { if (ovs_identifier_is_key(&flow->id)) flow = ovs_flow_tbl_lookup_exact(&dp->table, @@ -1124,6 +1125,30 @@ static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info) goto err_unlock_ovs; } } + + if (unlikely(reply)) { + size_t cur, req; + + cur = ovs_flow_cmd_msg_size(acts, &new_flow->id, + ufid_flags); + req = ovs_flow_cmd_msg_size(acts, &flow->id, + ufid_flags); + if (cur < req) { + struct sk_buff *resized; + + resized = ovs_flow_cmd_alloc_info(acts, + &flow->id, + info, false, + ufid_flags); + if (IS_ERR(resized)) { + error = PTR_ERR(resized); + goto err_unlock_ovs; + } + kfree_skb(reply); + reply = resized; + } + } + /* Update actions. */ old_acts = ovsl_dereference(flow->sf_acts); rcu_assign_pointer(flow->sf_acts, acts); @@ -1447,33 +1472,34 @@ static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info) goto unlock; } - ovs_flow_tbl_remove(&dp->table, flow); - ovs_unlock(); - - reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts, + reply = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts), &flow->id, info, false, ufid_flags); - if (likely(reply)) { - if (!IS_ERR(reply)) { - rcu_read_lock(); /*To keep RCU checker happy. */ - err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, - reply, info->snd_portid, - info->snd_seq, 0, - OVS_FLOW_CMD_DEL, - ufid_flags); - rcu_read_unlock(); - if (WARN_ON_ONCE(err < 0)) { - kfree_skb(reply); - goto out_free; - } + if (IS_ERR(reply)) { + netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, + PTR_ERR(reply)); + reply = NULL; + } - ovs_notify(&dp_flow_genl_family, reply, info); - } else { - netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, - PTR_ERR(reply)); + if (likely(reply)) { + err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, + reply, info->snd_portid, + info->snd_seq, 0, + OVS_FLOW_CMD_DEL, ufid_flags); + if (WARN_ON_ONCE(err < 0)) { + kfree_skb(reply); + reply = NULL; } } + /* Removal has to happen after ovs_flow_cmd_fill_info(), as it uses + * the flow->mask that can be scheduled to be freed by the + * ovs_flow_tbl_remove() and we're not holding the RCU read lock. + */ + ovs_flow_tbl_remove(&dp->table, flow); + ovs_unlock(); + + if (likely(reply)) + ovs_notify(&dp_flow_genl_family, reply, info); -out_free: ovs_flow_free(flow, true); return 0; unlock: @@ -2732,6 +2758,13 @@ static void __net_exit list_vports_from_net(struct net *net, struct net *dnet, } } +static void __net_exit ovs_pre_exit_net(struct net *dnet) +{ + ovs_lock(); + ovs_ct_exit_start(dnet); + ovs_unlock(); +} + static void __net_exit ovs_exit_net(struct net *dnet) { struct datapath *dp, *dp_next; @@ -2742,7 +2775,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) ovs_lock(); - ovs_ct_exit(dnet); + ovs_ct_exit_finish(dnet); list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node) __dp_destroy(dp); @@ -2766,6 +2799,7 @@ static void __net_exit ovs_exit_net(struct net *dnet) static struct pernet_operations ovs_net_ops = { .init = ovs_init_net, + .pre_exit = ovs_pre_exit_net, .exit = ovs_exit_net, .id = &ovs_net_id, .size = sizeof(struct ovs_net), diff --git a/net/openvswitch/datapath.h b/net/openvswitch/datapath.h index db0c3e69d66ca..b2c2b8da12d40 100644 --- a/net/openvswitch/datapath.h +++ b/net/openvswitch/datapath.h @@ -118,7 +118,7 @@ struct datapath { * @mru: The maximum received fragement size; 0 if the packet is not * fragmented. * @acts_origlen: The netlink size of the flow actions applied to this skb. - * @cutlen: The number of bytes from the packet end to be removed. + * @cutlen: The number of bytes in the packet to preserve on output. * @probability: The sampling probability that was applied to this skb; 0 means * no sampling has occurred; U32_MAX means 100% probability. * @upcall_pid: Netlink socket PID to use for sending this packet to userspace; @@ -164,7 +164,10 @@ struct dp_upcall_info { * Protected by genl_mutex. * @dp_notify_work: A work notifier to handle port unregistering. * @masks_rebalance: A work to periodically optimize flow table caches. - * @ct_limit_info: A hash table of conntrack zone connection limits. + * @ct_limit_info: Hash table of conntrack zone connection limits. Protected + * by RCU; updates and teardown are serialized by ovs_mutex. May be NULL during + * netns teardown. + * @ct_limit_exit_data: CT limit state detached at .pre_exit, freed at .exit. * @xt_label: Whether connlables are configured for the network or not. */ struct ovs_net { @@ -172,7 +175,8 @@ struct ovs_net { struct work_struct dp_notify_work; struct delayed_work masks_rebalance; #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT) - struct ovs_ct_limit_info *ct_limit_info; + struct ovs_ct_limit_info __rcu *ct_limit_info; + struct ovs_ct_limit_info *ct_limit_exit_data; #endif bool xt_label; }; diff --git a/net/openvswitch/flow.c b/net/openvswitch/flow.c index 66366982f6044..868d9fdf3afdf 100644 --- a/net/openvswitch/flow.c +++ b/net/openvswitch/flow.c @@ -288,7 +288,7 @@ static void get_ipv6_ext_hdrs(struct sk_buff *skb, struct ipv6hdr *nh, if (*ext_hdrs & OFPIEH12_ESP) *ext_hdrs |= OFPIEH12_UNREP; if ((*ext_hdrs & ~(OFPIEH12_HOP | OFPIEH12_DEST | - OFPIEH12_ROUTER | IPPROTO_FRAGMENT | + OFPIEH12_ROUTER | OFPIEH12_FRAG | OFPIEH12_AUTH | OFPIEH12_UNREP)) || dest_options_header_count >= 2) { *ext_hdrs |= OFPIEH12_UNSEQ; @@ -301,7 +301,7 @@ static void get_ipv6_ext_hdrs(struct sk_buff *skb, struct ipv6hdr *nh, *ext_hdrs |= OFPIEH12_UNREP; if ((*ext_hdrs & ~(OFPIEH12_HOP | OFPIEH12_DEST | OFPIEH12_ROUTER | - IPPROTO_FRAGMENT | OFPIEH12_UNREP)) || + OFPIEH12_FRAG | OFPIEH12_UNREP)) || dest_options_header_count >= 2) { *ext_hdrs |= OFPIEH12_UNSEQ; } @@ -889,8 +889,6 @@ static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key) * Ethernet header * @key: output flow key * - * The caller must ensure that skb->len >= ETH_HLEN. - * * Initializes @skb header fields as follows: * * - skb->mac_header: the L2 header. @@ -910,8 +908,6 @@ static int key_extract_l3l4(struct sk_buff *skb, struct sw_flow_key *key) */ static int key_extract(struct sk_buff *skb, struct sw_flow_key *key) { - struct ethhdr *eth; - /* Flags are always used as part of stats */ key->tp.flags = 0; @@ -926,6 +922,13 @@ static int key_extract(struct sk_buff *skb, struct sw_flow_key *key) skb_reset_network_header(skb); key->eth.type = skb->protocol; } else { + struct ethhdr *eth; + int err; + + err = check_header(skb, ETH_HLEN); + if (unlikely(err)) + return err; + eth = eth_hdr(skb); ether_addr_copy(key->eth.src, eth->h_source); ether_addr_copy(key->eth.dst, eth->h_dest); diff --git a/net/openvswitch/flow_table.c b/net/openvswitch/flow_table.c index ffc72a741a50d..46bc08fc3d441 100644 --- a/net/openvswitch/flow_table.c +++ b/net/openvswitch/flow_table.c @@ -258,11 +258,13 @@ static int tbl_mask_array_realloc(struct flow_table *tbl, int size) if (ovsl_dereference(old->masks[i])) new->masks[new->count++] = old->masks[i]; } - call_rcu(&old->rcu, mask_array_rcu_cb); } rcu_assign_pointer(tbl->mask_array, new); + if (old) + call_rcu(&old->rcu, mask_array_rcu_cb); + return 0; } diff --git a/net/openvswitch/meter.c b/net/openvswitch/meter.c index cc08e0403909a..671984edfe1eb 100644 --- a/net/openvswitch/meter.c +++ b/net/openvswitch/meter.c @@ -133,18 +133,10 @@ static void dp_meter_instance_remove(struct dp_meter_instance *ti, static int attach_meter(struct dp_meter_table *tbl, struct dp_meter *meter) { - struct dp_meter_instance *ti = rcu_dereference_ovsl(tbl->ti); - u32 hash = meter_hash(ti, meter->id); + struct dp_meter_instance *ti; + u32 hash; int err; - /* In generally, slots selected should be empty, because - * OvS uses id-pool to fetch a available id. - */ - if (unlikely(rcu_dereference_ovsl(ti->dp_meters[hash]))) - return -EBUSY; - - dp_meter_instance_insert(ti, meter); - /* That function is thread-safe. */ tbl->count++; if (tbl->count >= tbl->max_meters_allowed) { @@ -152,16 +144,29 @@ static int attach_meter(struct dp_meter_table *tbl, struct dp_meter *meter) goto attach_err; } - if (tbl->count >= ti->n_meters && - dp_meter_instance_realloc(tbl, ti->n_meters * 2)) { - err = -ENOMEM; + ti = rcu_dereference_ovsl(tbl->ti); + if (tbl->count >= ti->n_meters) { + err = dp_meter_instance_realloc(tbl, ti->n_meters * 2); + if (err) + goto attach_err; + + ti = rcu_dereference_ovsl(tbl->ti); + } + + hash = meter_hash(ti, meter->id); + + /* In general, selected slots should be empty, because + * OvS uses id-pool to fetch available ids. + */ + if (unlikely(rcu_dereference_ovsl(ti->dp_meters[hash]))) { + err = -EBUSY; goto attach_err; } + dp_meter_instance_insert(ti, meter); return 0; attach_err: - dp_meter_instance_remove(ti, meter); tbl->count--; return err; } diff --git a/net/openvswitch/vport.c b/net/openvswitch/vport.c index 1f0c86a9f43b0..7470f43d31ea8 100644 --- a/net/openvswitch/vport.c +++ b/net/openvswitch/vport.c @@ -503,7 +503,7 @@ int ovs_vport_receive(struct vport *vport, struct sk_buff *skb, OVS_CB(skb)->input_vport = vport; OVS_CB(skb)->mru = 0; - OVS_CB(skb)->cutlen = 0; + OVS_CB(skb)->cutlen = U32_MAX; OVS_CB(skb)->probability = 0; OVS_CB(skb)->upcall_pid = 0; if (unlikely(dev_net(skb->dev) != ovs_dp_get_net(vport->dp))) { diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 8b3961fba750b..772ecfa13cd78 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -87,6 +87,7 @@ #include #include #include +#include #ifdef CONFIG_INET #include #endif @@ -1313,13 +1314,25 @@ static int packet_rcv_has_room(struct packet_sock *po, struct sk_buff *skb) return ret; } -static void packet_rcv_try_clear_pressure(struct packet_sock *po) +static void __packet_rcv_try_clear_pressure(struct packet_sock *po) { if (packet_sock_flag(po, PACKET_SOCK_PRESSURE) && __packet_rcv_has_room(po, NULL) == ROOM_NORMAL) packet_sock_flag_set(po, PACKET_SOCK_PRESSURE, false); } +static void packet_rcv_try_clear_pressure(struct packet_sock *po) +{ + struct sock *sk = &po->sk; + + if (!packet_sock_flag(po, PACKET_SOCK_PRESSURE)) + return; + + spin_lock_bh(&sk->sk_receive_queue.lock); + __packet_rcv_try_clear_pressure(po); + spin_unlock_bh(&sk->sk_receive_queue.lock); +} + static void packet_sock_destruct(struct sock *sk) { skb_queue_purge(&sk->sk_error_queue); @@ -1327,6 +1340,8 @@ static void packet_sock_destruct(struct sock *sk) WARN_ON(atomic_read(&sk->sk_rmem_alloc)); WARN_ON(refcount_read(&sk->sk_wmem_alloc)); + packet_free_pending(pkt_sk(sk)); + if (!sock_flag(sk, SOCK_DEAD)) { pr_err("Attempt to release alive packet socket: %p\n", sk); return; @@ -1923,19 +1938,20 @@ static void packet_parse_headers(struct sk_buff *skb, struct socket *sock) { int depth; + /* On TX skb->data is the L2 header; anchor it for all socket types. */ + skb_reset_mac_header(skb); + if ((!skb->protocol || skb->protocol == htons(ETH_P_ALL)) && - sock->type == SOCK_RAW) { - skb_reset_mac_header(skb); + sock->type == SOCK_RAW) skb->protocol = dev_parse_header_protocol(skb); - } + + skb_probe_transport_header(skb); /* Move network header to the right position for VLAN tagged packets */ if (likely(skb->dev->type == ARPHRD_ETHER) && eth_type_vlan(skb->protocol) && vlan_get_protocol_and_depth(skb, skb->protocol, &depth) != 0) skb_set_network_header(skb, depth); - - skb_probe_transport_header(skb); } /* @@ -1952,8 +1968,9 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg, struct net_device *dev; struct sockcm_cookie sockc; __be16 proto = 0; - int err; + int hard_header_len; int extra_len = 0; + int err; /* * Get and verify the address. @@ -1996,14 +2013,18 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg, extra_len = 4; /* We're doing our own CRC */ } + /* Keep the allocation-time header length across retry. */ + if (!skb) + hard_header_len = READ_ONCE(dev->hard_header_len); + err = -EMSGSIZE; - if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN + extra_len) + if (len > dev->mtu + hard_header_len + VLAN_HLEN + extra_len) goto out_unlock; if (!skb) { - size_t reserved = LL_RESERVED_SPACE(dev); + size_t reserved = LL_RESERVED_SPACE_EX(dev, hard_header_len); int tlen = dev->needed_tailroom; - unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0; + unsigned int hhlen = dev->header_ops ? hard_header_len : 0; rcu_read_unlock(); skb = sock_wmalloc(sk, len + reserved + tlen, 0, GFP_KERNEL); @@ -2033,7 +2054,7 @@ static int packet_sendmsg_spkt(struct socket *sock, struct msghdr *msg, err = -EINVAL; goto out_unlock; } - if (len > (dev->mtu + dev->hard_header_len + extra_len) && + if (len > (dev->mtu + hard_header_len + extra_len) && !packet_extra_vlan_len_allowed(dev, skb)) { err = -EMSGSIZE; goto out_unlock; @@ -2515,11 +2536,11 @@ static void tpacket_destruct_skb(struct sk_buff *skb) __u32 ts; ph = skb_zcopy_get_nouarg(skb); - packet_dec_pending(&po->tx_ring); ts = __packet_set_timestamp(po, ph, skb); __packet_set_status(po, ph, TP_STATUS_AVAILABLE | ts); + packet_dec_pending(&po->tx_ring); complete(&po->skb_completion); } @@ -2568,6 +2589,7 @@ static int packet_snd_vnet_parse(struct msghdr *msg, size_t *len, static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, void *frame, struct net_device *dev, void *data, int tp_len, __be16 proto, unsigned char *addr, int hlen, int copylen, + int hard_header_len, const struct sockcm_cookie *sockc) { union tpacket_uhdr ph; @@ -2599,8 +2621,8 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, } else if (copylen) { int hdrlen = min_t(int, copylen, tp_len); - skb_push(skb, dev->hard_header_len); - skb_put(skb, copylen - dev->hard_header_len); + skb_push(skb, hard_header_len); + skb_put(skb, copylen - hard_header_len); err = skb_store_bits(skb, 0, data, hdrlen); if (unlikely(err)) return err; @@ -2640,6 +2662,9 @@ static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb, len = ((to_write > len_max) ? len_max : to_write); } + if (unlikely(!skb->len)) + return -EINVAL; + packet_parse_headers(skb, sock); return tp_len; @@ -2649,7 +2674,8 @@ static int tpacket_parse_header(struct packet_sock *po, void *frame, int size_max, void **data) { union tpacket_uhdr ph; - int tp_len, off; + u32 tp_len; + int off; ph.raw = frame; @@ -2669,7 +2695,7 @@ static int tpacket_parse_header(struct packet_sock *po, void *frame, break; } if (unlikely(tp_len > size_max)) { - pr_err("packet size is too long (%d > %d)\n", tp_len, size_max); + pr_err("packet size is too long (%u > %d)\n", tp_len, size_max); return -EMSGSIZE; } @@ -2731,7 +2757,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) void *data; int len_sum = 0; int status = TP_STATUS_AVAILABLE; - int hlen, tlen, copylen = 0; + int hard_header_len, hlen, tlen, copylen = 0; long timeo; mutex_lock(&po->pg_vec_lock); @@ -2778,8 +2804,9 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto out_put; } + hard_header_len = READ_ONCE(dev->hard_header_len); if (po->sk.sk_socket->type == SOCK_RAW) - reserve = dev->hard_header_len; + reserve = hard_header_len; size_max = po->tx_ring.frame_size - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); @@ -2816,7 +2843,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto tpacket_error; status = TP_STATUS_SEND_REQUEST; - hlen = LL_RESERVED_SPACE(dev); + hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len); tlen = dev->needed_tailroom; if (vnet_hdr_sz) { data += vnet_hdr_sz; @@ -2834,10 +2861,10 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) vnet_hdr.hdr_len); has_vnet_hdr = true; } - copylen = max_t(int, copylen, dev->hard_header_len); + copylen = max_t(int, copylen, hard_header_len); skb = sock_alloc_send_skb(&po->sk, hlen + tlen + sizeof(struct sockaddr_ll) + - (copylen - dev->hard_header_len), + (copylen - hard_header_len), !need_wait, &err); if (unlikely(skb == NULL)) { @@ -2847,7 +2874,8 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg) goto out_status; } tp_len = tpacket_fill_skb(po, skb, ph, dev, data, tp_len, proto, - addr, hlen, copylen, &sockc); + addr, hlen, copylen, hard_header_len, + &sockc); if (likely(tp_len >= 0) && tp_len > dev->mtu + reserve && !vnet_hdr_sz && @@ -2955,7 +2983,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) int offset = 0; struct packet_sock *po = pkt_sk(sk); int vnet_hdr_sz = READ_ONCE(po->vnet_hdr_sz); - int hlen, tlen, linear; + int hard_header_len, hlen, tlen, linear; int extra_len = 0; /* @@ -2995,8 +3023,9 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) goto out_unlock; } + hard_header_len = READ_ONCE(dev->hard_header_len); if (sock->type == SOCK_RAW) - reserve = dev->hard_header_len; + reserve = hard_header_len; if (vnet_hdr_sz) { err = packet_snd_vnet_parse(msg, &len, &vnet_hdr, vnet_hdr_sz); if (err) @@ -3017,10 +3046,10 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) goto out_unlock; err = -ENOBUFS; - hlen = LL_RESERVED_SPACE(dev); + hlen = LL_RESERVED_SPACE_EX(dev, hard_header_len); tlen = dev->needed_tailroom; linear = __virtio16_to_cpu(vio_le(), vnet_hdr.hdr_len); - linear = max(linear, min_t(int, len, dev->hard_header_len)); + linear = max(linear, min_t(int, len, hard_header_len)); skb = packet_alloc_skb(sk, hlen + tlen, hlen, len, linear, msg->msg_flags & MSG_DONTWAIT, &err); if (skb == NULL) @@ -3036,7 +3065,7 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len) } else if (reserve) { skb_reserve(skb, -reserve); if (len < reserve + sizeof(struct ipv6hdr) && - dev->min_header_len != dev->hard_header_len) + dev->min_header_len != hard_header_len) skb_reset_network_header(skb); } @@ -3181,7 +3210,6 @@ static int packet_release(struct socket *sock) /* Purge queues */ skb_queue_purge(&sk->sk_receive_queue); - packet_free_pending(po); sock_put(sk); return 0; @@ -4304,7 +4332,7 @@ static __poll_t packet_poll(struct file *file, struct socket *sock, TP_STATUS_KERNEL)) mask |= EPOLLIN | EPOLLRDNORM; } - packet_rcv_try_clear_pressure(po); + __packet_rcv_try_clear_pressure(po); spin_unlock_bh(&sk->sk_receive_queue.lock); spin_lock_bh(&sk->sk_write_queue.lock); if (po->tx_ring.pg_vec) { @@ -4345,11 +4373,26 @@ static const struct vm_operations_struct packet_mmap_ops = { .close = packet_mm_close, }; +struct packet_pg_vec { + struct packet_pg_vec_free *deferred; + unsigned int order; + unsigned int len; + struct pgv pg_vec[] __counted_by(len); +}; + +struct packet_pg_vec_free { + struct delayed_work work; + struct sock *sk; + struct packet_pg_vec *vec; +}; + static void free_pg_vec(struct pgv *pg_vec, unsigned int order, unsigned int len) { + struct packet_pg_vec *vec; int i; + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); for (i = 0; i < len; i++) { if (likely(pg_vec[i].buffer)) { if (is_vmalloc_addr(pg_vec[i].buffer)) @@ -4360,7 +4403,46 @@ static void free_pg_vec(struct pgv *pg_vec, unsigned int order, pg_vec[i].buffer = NULL; } } - kfree(pg_vec); + kfree(vec->deferred); + kfree(vec); +} + +static void packet_free_pg_vec_work(struct work_struct *work) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + struct sock *sk; + + deferred = container_of_const(to_delayed_work(work), + struct packet_pg_vec_free, work); + vec = deferred->vec; + sk = deferred->sk; + if (sk_wmem_alloc_get(sk)) { + queue_delayed_work(system_long_wq, &deferred->work, 1); + return; + } + + free_pg_vec(vec->pg_vec, vec->order, vec->len); + sock_put(sk); +} + +static void packet_free_tx_ring(struct sock *sk, struct pgv *pg_vec, + unsigned int order, unsigned int len) +{ + struct packet_pg_vec_free *deferred; + struct packet_pg_vec *vec; + + vec = container_of_const(pg_vec, struct packet_pg_vec, pg_vec[0]); + deferred = vec->deferred; + if (!deferred || !sk_wmem_alloc_get(sk)) { + free_pg_vec(pg_vec, order, len); + return; + } + + /* A detached ring's pending count can miss late skb destructors. */ + deferred->sk = sk; + sock_hold(sk); + queue_delayed_work(system_long_wq, &deferred->work, 0); } static char *alloc_one_pg_vec_page(unsigned long order) @@ -4388,20 +4470,35 @@ static char *alloc_one_pg_vec_page(unsigned long order) return NULL; } -static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order) +static struct pgv *alloc_pg_vec(struct tpacket_req *req, int order, bool tx_ring) { unsigned int block_nr = req->tp_block_nr; + struct packet_pg_vec *vec; struct pgv *pg_vec; int i; - pg_vec = kcalloc(block_nr, sizeof(struct pgv), GFP_KERNEL | __GFP_NOWARN); - if (unlikely(!pg_vec)) - goto out; + vec = kzalloc_flex(*vec, pg_vec, block_nr, GFP_KERNEL | __GFP_NOWARN); + if (unlikely(!vec)) + return NULL; + vec->order = order; + vec->len = block_nr; + pg_vec = vec->pg_vec; for (i = 0; i < block_nr; i++) { pg_vec[i].buffer = alloc_one_pg_vec_page(order); if (unlikely(!pg_vec[i].buffer)) goto out_free_pgvec; + + if (tx_ring && !vec->deferred && + is_vmalloc_addr(pg_vec[i].buffer)) { + vec->deferred = kzalloc_obj(*vec->deferred, + GFP_KERNEL | __GFP_NOWARN); + if (!vec->deferred) + goto out_free_pgvec; + vec->deferred->vec = vec; + INIT_DELAYED_WORK(&vec->deferred->work, + packet_free_pg_vec_work); + } } out: @@ -4484,7 +4581,7 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -ENOMEM; order = get_order(req->tp_block_size); - pg_vec = alloc_pg_vec(req, order); + pg_vec = alloc_pg_vec(req, order, tx_ring); if (unlikely(!pg_vec)) goto out; switch (po->tp_version) { @@ -4536,6 +4633,9 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, err = -EBUSY; mutex_lock(&po->pg_vec_lock); if (closing || atomic_long_read(&po->mapped) == 0) { + if (tx_ring && !closing && packet_read_pending(rb)) + goto out_unlock; + err = 0; spin_lock_bh(&rb_queue->lock); swap(rb->pg_vec, pg_vec); @@ -4544,24 +4644,29 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, rb->frame_max = (req->tp_frame_nr - 1); rb->head = 0; rb->frame_size = req->tp_frame_size; + po->prot_hook.func = (po->rx_ring.pg_vec) ? + tpacket_rcv : packet_rcv; spin_unlock_bh(&rb_queue->lock); swap(rb->pg_vec_order, order); swap(rb->pg_vec_len, req->tp_block_nr); rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; - po->prot_hook.func = (po->rx_ring.pg_vec) ? - tpacket_rcv : packet_rcv; skb_queue_purge(rb_queue); if (atomic_long_read(&po->mapped)) pr_err("packet_mmap: vma is busy: %ld\n", atomic_long_read(&po->mapped)); } +out_unlock: mutex_unlock(&po->pg_vec_lock); spin_lock(&po->bind_lock); WRITE_ONCE(po->num, num); - if (was_running) + /* + * NETDEV_UNREGISTER may have invalidated the binding while bind_lock + * was dropped above. Do not re-add a fanout hook to a dead device. + */ + if (was_running && READ_ONCE(po->ifindex) != -1) register_prot_hook(sk); spin_unlock(&po->bind_lock); @@ -4574,7 +4679,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, out_free_pg_vec: if (pg_vec) { bitmap_free(rx_owner_map); - free_pg_vec(pg_vec, order, req->tp_block_nr); + if (tx_ring && closing) + packet_free_tx_ring(sk, pg_vec, order, req->tp_block_nr); + else + free_pg_vec(pg_vec, order, req->tp_block_nr); } out: return err; diff --git a/net/phonet/pep.c b/net/phonet/pep.c index 7c695f3afc285..36c85811fd08b 100644 --- a/net/phonet/pep.c +++ b/net/phonet/pep.c @@ -55,6 +55,8 @@ static unsigned char *pep_get_sb(struct sk_buff *skb, u8 *ptype, u8 *plen, ph = skb_header_pointer(skb, 0, 2, &h); if (ph == NULL || ph->sb_len < 2 || !pskb_may_pull(skb, ph->sb_len)) return NULL; + /* pskb_may_pull() may have reallocated the head; refetch ph. */ + ph = skb_header_pointer(skb, 0, 2, &h); ph->sb_len -= 2; *ptype = ph->sb_type; *plen = ph->sb_len; @@ -1114,7 +1116,7 @@ static int pep_getsockopt(struct sock *sk, int level, int optname, len = min_t(unsigned int, sizeof(int), len); if (put_user(len, optlen)) return -EFAULT; - if (put_user(val, (int __user *) optval)) + if (copy_to_user(optval, &val, len)) return -EFAULT; return 0; } diff --git a/net/phonet/pn_dev.c b/net/phonet/pn_dev.c index ec9363c337a9f..4bf8f63d45a9a 100644 --- a/net/phonet/pn_dev.c +++ b/net/phonet/pn_dev.c @@ -350,16 +350,34 @@ static struct pernet_operations phonet_net_ops = { /* Initialize Phonet devices list */ int __init phonet_device_init(void) { - int err = register_pernet_subsys(&phonet_net_ops); + int err; + + err = register_pernet_subsys(&phonet_net_ops); if (err) return err; - proc_create_net("pnresource", 0, init_net.proc_net, &pn_res_seq_ops, - sizeof(struct seq_net_private)); - register_netdevice_notifier(&phonet_device_notifier); + if (!proc_create_net("pnresource", 0, init_net.proc_net, + &pn_res_seq_ops, sizeof(struct seq_net_private))) { + err = -ENOMEM; + goto err_pernet; + } + + err = register_netdevice_notifier(&phonet_device_notifier); + if (err) + goto err_proc; + err = phonet_netlink_register(); if (err) - phonet_device_exit(); + goto err_notifier; + + return 0; + +err_notifier: + unregister_netdevice_notifier(&phonet_device_notifier); +err_proc: + remove_proc_entry("pnresource", init_net.proc_net); +err_pernet: + unregister_pernet_subsys(&phonet_net_ops); return err; } @@ -367,8 +385,8 @@ void phonet_device_exit(void) { rtnl_unregister_all(PF_PHONET); unregister_netdevice_notifier(&phonet_device_notifier); - unregister_pernet_subsys(&phonet_net_ops); remove_proc_entry("pnresource", init_net.proc_net); + unregister_pernet_subsys(&phonet_net_ops); } int phonet_route_add(struct net_device *dev, u8 daddr) diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c index e35c31977479c..afda9cb660275 100644 --- a/net/psp/psp_main.c +++ b/net/psp/psp_main.c @@ -294,6 +294,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) if (proto == htons(ETH_P_IP)) { struct iphdr *iph = (struct iphdr *)(skb->data + l2_hlen); + if (unlikely(iph->ihl < 5)) + return -EINVAL; + is_udp = iph->protocol == IPPROTO_UDP; l3_hlen = iph->ihl * 4; if (l3_hlen != sizeof(struct iphdr) && @@ -347,6 +350,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) if (proto == htons(ETH_P_IP)) { struct iphdr *iph = (struct iphdr *)(skb->data + l2_hlen); + if (unlikely(ntohs(iph->tot_len) < l3_hlen + encap)) + return -EINVAL; + iph->protocol = psph->nexthdr; iph->tot_len = htons(ntohs(iph->tot_len) - encap); iph->check = 0; @@ -354,6 +360,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv) } else { struct ipv6hdr *ipv6h = (struct ipv6hdr *)(skb->data + l2_hlen); + if (unlikely(ntohs(ipv6h->payload_len) < encap)) + return -EINVAL; + ipv6h->nexthdr = psph->nexthdr; ipv6h->payload_len = htons(ntohs(ipv6h->payload_len) - encap); } diff --git a/net/qrtr/af_qrtr.c b/net/qrtr/af_qrtr.c index 305523cebe3ba..1f40496d84a33 100644 --- a/net/qrtr/af_qrtr.c +++ b/net/qrtr/af_qrtr.c @@ -1009,8 +1009,10 @@ static int qrtr_send_resume_tx(struct qrtr_cb *cb) return -EINVAL; skb = qrtr_alloc_ctrl_packet(&pkt, GFP_KERNEL); - if (!skb) + if (!skb) { + qrtr_node_release(node); return -ENOMEM; + } pkt->cmd = cpu_to_le32(QRTR_TYPE_RESUME_TX); pkt->client.node = cpu_to_le32(cb->dst_node); @@ -1261,6 +1263,14 @@ static int qrtr_create(struct net *net, struct socket *sock, if (sock->type != SOCK_DGRAM) return -EPROTOTYPE; + /* QRTR keeps its port and node state in module-global variables that + * are not partitioned per network namespace, and the in-kernel name + * service only operates in init_net. Confine the family to init_net so + * a socket in another namespace cannot reach the global control plane. + */ + if (!net_eq(net, &init_net)) + return -EAFNOSUPPORT; + sk = sk_alloc(net, AF_QIPCRTR, GFP_KERNEL, &qrtr_proto, kern); if (!sk) return -ENOMEM; diff --git a/net/qrtr/ns.c b/net/qrtr/ns.c index a10e8de4f7e10..e8e3ba1b3b823 100644 --- a/net/qrtr/ns.c +++ b/net/qrtr/ns.c @@ -85,9 +85,9 @@ struct qrtr_node { /* Max nodes limit is chosen based on the current platform requirements. * If the requirement changes in the future, this value can be increased. */ -#define QRTR_NS_MAX_NODES 64 +#define QRTR_NS_MAX_NODES 512 -static u8 node_count; +static u16 node_count; static struct qrtr_node *node_get(unsigned int node_id) { diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c index 7a0f5150e9103..a9445937d225b 100644 --- a/net/rds/af_rds.c +++ b/net/rds/af_rds.c @@ -43,7 +43,6 @@ /* this is just used for stats gathering :/ */ static DEFINE_SPINLOCK(rds_sock_lock); -static unsigned long rds_sock_count; static LIST_HEAD(rds_sock_list); DECLARE_WAIT_QUEUE_HEAD(rds_poll_waitq); @@ -82,7 +81,6 @@ static int rds_release(struct socket *sock) spin_lock_bh(&rds_sock_lock); list_del_init(&rs->rs_item); - rds_sock_count--; spin_unlock_bh(&rds_sock_lock); rds_trans_put(rs->rs_transport); @@ -219,7 +217,7 @@ static __poll_t rds_poll(struct file *file, struct socket *sock, poll_wait(file, sk_sleep(sk), wait); - if (rs->rs_seen_congestion) + if (READ_ONCE(rs->rs_seen_congestion)) poll_wait(file, &rds_poll_waitq, wait); read_lock_irqsave(&rs->rs_recv_lock, flags); @@ -247,7 +245,7 @@ static __poll_t rds_poll(struct file *file, struct socket *sock, /* clear state any time we wake a seen-congested socket */ if (mask) - rs->rs_seen_congestion = 0; + WRITE_ONCE(rs->rs_seen_congestion, 0); return mask; } @@ -694,7 +692,6 @@ static int __rds_create(struct socket *sock, struct sock *sk, int protocol) spin_lock_bh(&rds_sock_lock); list_add_tail(&rs->rs_item, &rds_sock_list); - rds_sock_count++; spin_unlock_bh(&rds_sock_lock); return 0; @@ -735,6 +732,7 @@ static void rds_sock_inc_info(struct socket *sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(sock->sk); struct rds_sock *rs; struct rds_incoming *inc; unsigned int total = 0; @@ -744,6 +742,9 @@ static void rds_sock_inc_info(struct socket *sock, unsigned int len, spin_lock_bh(&rds_sock_lock); list_for_each_entry(rs, &rds_sock_list, rs_item) { + /* Only show sockets in the caller's netns. */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; /* This option only supports IPv4 sockets. */ if (!ipv6_addr_v4mapped(&rs->rs_bound_addr)) continue; @@ -774,6 +775,7 @@ static void rds6_sock_inc_info(struct socket *sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(sock->sk); struct rds_incoming *inc; unsigned int total = 0; struct rds_sock *rs; @@ -783,6 +785,9 @@ static void rds6_sock_inc_info(struct socket *sock, unsigned int len, spin_lock_bh(&rds_sock_lock); list_for_each_entry(rs, &rds_sock_list, rs_item) { + /* Only show sockets in the caller's netns. */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; read_lock(&rs->rs_recv_lock); list_for_each_entry(inc, &rs->rs_recv_queue, i_item) { @@ -806,7 +811,9 @@ static void rds_sock_info(struct socket *sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(sock->sk); struct rds_info_socket sinfo; + unsigned int copied = 0; unsigned int cnt = 0; struct rds_sock *rs; @@ -814,12 +821,24 @@ static void rds_sock_info(struct socket *sock, unsigned int len, spin_lock_bh(&rds_sock_lock); - if (len < rds_sock_count) { - cnt = rds_sock_count; - goto out; + /* First pass: count entries visible in the caller's netns. */ + list_for_each_entry(rs, &rds_sock_list, rs_item) { + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; + if (!ipv6_addr_v4mapped(&rs->rs_bound_addr)) + continue; + cnt++; } + if (len < cnt) + goto out; + list_for_each_entry(rs, &rds_sock_list, rs_item) { + if (copied >= cnt) + break; + /* Only show sockets in the caller's netns. */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; /* This option only supports IPv4 sockets. */ if (!ipv6_addr_v4mapped(&rs->rs_bound_addr)) continue; @@ -832,8 +851,13 @@ static void rds_sock_info(struct socket *sock, unsigned int len, sinfo.inum = sock_i_ino(rds_rs_to_sk(rs)); rds_info_copy(iter, &sinfo, sizeof(sinfo)); - cnt++; + copied++; } + /* A concurrent rds_bind() can change rs_bound_addr between the + * two passes without holding rds_sock_lock, so copied may be + * less than cnt. Report what was actually copied. + */ + cnt = copied; out: lens->nr = cnt; @@ -847,17 +871,32 @@ static void rds6_sock_info(struct socket *sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(sock->sk); struct rds6_info_socket sinfo6; + unsigned int copied = 0; + unsigned int cnt = 0; struct rds_sock *rs; len /= sizeof(struct rds6_info_socket); spin_lock_bh(&rds_sock_lock); - if (len < rds_sock_count) + /* First pass: count entries visible in the caller's netns. */ + list_for_each_entry(rs, &rds_sock_list, rs_item) { + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; + cnt++; + } + + if (len < cnt) goto out; list_for_each_entry(rs, &rds_sock_list, rs_item) { + if (copied >= cnt) + break; + /* Only show sockets in the caller's netns. */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), net)) + continue; sinfo6.sndbuf = rds_sk_sndbuf(rs); sinfo6.rcvbuf = rds_sk_rcvbuf(rs); sinfo6.bound_addr = rs->rs_bound_addr; @@ -867,10 +906,12 @@ static void rds6_sock_info(struct socket *sock, unsigned int len, sinfo6.inum = sock_i_ino(rds_rs_to_sk(rs)); rds_info_copy(iter, &sinfo6, sizeof(sinfo6)); + copied++; } + cnt = copied; out: - lens->nr = rds_sock_count; + lens->nr = cnt; lens->each = sizeof(struct rds6_info_socket); spin_unlock_bh(&rds_sock_lock); diff --git a/net/rds/cong.c b/net/rds/cong.c index 8b689ebbd5b52..c85af56e42e73 100644 --- a/net/rds/cong.c +++ b/net/rds/cong.c @@ -256,9 +256,9 @@ void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask) map, &map->m_addr); rds_stats_inc(s_cong_update_received); atomic_inc(&rds_cong_generation); - if (waitqueue_active(&map->m_waitq)) + if (wq_has_sleeper(&map->m_waitq)) wake_up(&map->m_waitq); - if (waitqueue_active(&rds_poll_waitq)) + if (wq_has_sleeper(&rds_poll_waitq)) wake_up_all(&rds_poll_waitq); if (portmask && !list_empty(&rds_cong_monitor)) { diff --git a/net/rds/connection.c b/net/rds/connection.c index 4764628fe12a3..e9c6a7dd83037 100644 --- a/net/rds/connection.c +++ b/net/rds/connection.c @@ -106,10 +106,12 @@ static struct rds_connection *rds_conn_lookup(struct net *net, } /* - * This is called by transports as they're bringing down a connection. - * It clears partial message state so that the transport can start sending - * and receiving over this connection again in the future. It is up to - * the transport to have serialized this call with its send and recv. + * This is called by rds_conn_shutdown() once the transport has brought + * a path down. It clears partial message state so that the transport + * can start sending and receiving over this path again in the future. + * The caller owns RDS_IN_XMIT and RDS_RECV_REFILL across this call, + * which is what serializes it against the send and receive-refill + * paths. */ static void rds_conn_path_reset(struct rds_conn_path *cp) { @@ -120,7 +122,16 @@ static void rds_conn_path_reset(struct rds_conn_path *cp) rds_stats_inc(s_conn_reset); rds_send_path_reset(cp); - cp->cp_flags = 0; + + /* Clear the bits the reset is responsible for individually: a + * blanket cp_flags = 0 is a plain store that can clobber a + * concurrent atomic read-modify-write on the same word. + * RDS_IN_XMIT and RDS_RECV_REFILL are held as locks by the + * caller, rds_conn_shutdown(), which releases them once the + * teardown is complete. + */ + clear_bit(RDS_LL_SEND_FULL, &cp->cp_flags); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); /* Do not clear next_rx_seq here, else we cannot distinguish * retransmitted packets from new packets, and will hand all @@ -393,28 +404,70 @@ void rds_conn_shutdown(struct rds_conn_path *cp) } mutex_unlock(&cp->cp_cm_lock); + /* Quiesce the transmit and receive-refill paths by + * acquiring their bit locks, not merely waiting for + * them to be released: with a plain wait, either path + * can re-take its lock the instant after we sample it + * clear and then run concurrently with the transport + * shutdown and the path reset below. Holding both + * locks across the teardown makes that structurally + * impossible. + */ wait_event(cp->cp_waitq, - !test_bit(RDS_IN_XMIT, &cp->cp_flags)); + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); wait_event(cp->cp_waitq, - !test_bit(RDS_RECV_REFILL, &cp->cp_flags)); + !test_and_set_bit(RDS_RECV_REFILL, &cp->cp_flags)); conn->c_trans->conn_path_shutdown(cp); rds_conn_path_reset(cp); + /* Release the two locks and wake any waiter (e.g. + * rds_tcp_reset_callbacks()) that blocked on them while + * we held them. The unlock orders the transport's ring + * re-initialization and the path reset above before + * either bit is seen clear. rds_conn_path_reset() leaves + * both bits alone: ownership ends here, not inside the + * reset. + */ + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + clear_bit_unlock(RDS_RECV_REFILL, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); + if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING, - RDS_CONN_DOWN) && - !rds_conn_path_transition(cp, RDS_CONN_ERROR, RDS_CONN_DOWN)) { - /* This can happen - eg when we're in the middle of tearing - * down the connection, and someone unloads the rds module. - * Quite reproducible with loopback connections. - * Mostly harmless. + /* The path was dropped again while we tore it + * down: by a socket state-change callback in + * irq context on receipt of a FIN, or by an + * accept that claimed the path just before a + * drop put it back to RDS_CONN_ERROR and then + * installed a fresh socket on it. Unless a + * pending destroy suppressed it, the drop also + * queued another shutdown pass, and that pass + * must run, because it is what tears down + * whatever attached to the path after the + * transport shutdown above sampled its state. + * Consuming the RDS_CONN_ERROR here would turn + * that pass into a no-op: leave the state + * alone, and let the pass finish the job. * - * Note that this also happens with rds-tcp because - * we could have triggered rds_conn_path_drop in irq - * mode from rds_tcp_state change on the receipt of - * a FIN, thus we need to recheck for RDS_CONN_ERROR - * here. + * Quiesce the reconnect timer before bailing + * out, though. When a pending destroy did + * suppress the queue, no later pass runs, and + * rds_conn_path_destroy() is about to flush + * cp_down_w and free the path: it must not + * find cp_conn_w still armed. A successor + * pass, when there is one, re-arms the + * reconnect from its own tail. + */ + cancel_delayed_work_sync(&cp->cp_conn_w); + clear_bit(RDS_RECONNECT_PENDING, &cp->cp_flags); + + if (rds_conn_path_state(cp) == RDS_CONN_ERROR) + return; + /* No current cp_state writer leaves a + * DISCONNECTING path in any state but + * RDS_CONN_ERROR; report loudly if one ever + * does. */ rds_conn_path_error(cp, "%s: failed to transition " "to state DOWN, current state " @@ -541,6 +594,7 @@ static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len, struct rds_info_lengths *lens, int want_send, bool isv6) { + struct net *net = sock_net(sock->sk); struct hlist_head *head; struct list_head *list; struct rds_connection *conn; @@ -563,6 +617,9 @@ static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len, struct rds_conn_path *cp; int npaths; + /* Only show connections in the caller's netns. */ + if (!net_eq(rds_conn_net(conn), net)) + continue; if (!isv6 && conn->c_isv6) continue; @@ -661,6 +718,7 @@ void rds_for_each_conn_info(struct socket *sock, unsigned int len, u64 *buffer, size_t item_len) { + struct net *net = sock_net(sock->sk); struct hlist_head *head; struct rds_connection *conn; size_t i; @@ -673,6 +731,9 @@ void rds_for_each_conn_info(struct socket *sock, unsigned int len, for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash); i++, head++) { hlist_for_each_entry_rcu(conn, head, c_hash_node) { + /* Only show connections in the caller's netns. */ + if (!net_eq(rds_conn_net(conn), net)) + continue; /* Zero the per-item buffer before handing it to the * visitor so any field the visitor does not write - @@ -706,6 +767,7 @@ static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, u64 *buffer, size_t item_len) { + struct net *net = sock_net(sock->sk); struct hlist_head *head; struct rds_connection *conn; size_t i; @@ -720,6 +782,10 @@ static void rds_walk_conn_path_info(struct socket *sock, unsigned int len, hlist_for_each_entry_rcu(conn, head, c_hash_node) { struct rds_conn_path *cp; + /* Only show connections in the caller's netns. */ + if (!net_eq(rds_conn_net(conn), net)) + continue; + /* XXX We only copy the information from the first * path for now. The problem is that if there are * more than one underlying paths, we cannot report diff --git a/net/rds/ib.c b/net/rds/ib.c index ce5be43c5fbac..1061bcf7d1315 100644 --- a/net/rds/ib.c +++ b/net/rds/ib.c @@ -431,6 +431,10 @@ static int rds_ib_laddr_check_cm(struct net *net, const struct in6_addr *addr, sa = (struct sockaddr *)&sin; } else { #if IS_ENABLED(CONFIG_IPV6) + if (!ipv6_mod_enabled()) { + ret = -EADDRNOTAVAIL; + goto out; + } memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; sin6.sin6_addr = *addr; diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c index 5289afbb61aa7..d9b6c9d2f6791 100644 --- a/net/rds/ib_cm.c +++ b/net/rds/ib_cm.c @@ -810,6 +810,10 @@ int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id, dp = event->param.conn.private_data; if (isv6) { #if IS_ENABLED(CONFIG_IPV6) + if (!ipv6_mod_enabled()) { + err = -EOPNOTSUPP; + goto out; + } dp_cmn = &dp->ricp_v6.dp_cmn; saddr6 = &dp->ricp_v6.dp_saddr; daddr6 = &dp->ricp_v6.dp_daddr; @@ -1039,6 +1043,19 @@ int rds_ib_conn_path_connect(struct rds_conn_path *cp) return ret; } +static unsigned long rds_ib_conn_path_shutdown_check_wait(struct rds_conn_path *cp) +{ + struct rds_connection *conn = cp->cp_conn; + struct rds_ib_connection *ic = conn->c_transport_data; + + return (!ic->i_cm_id || + (rds_ib_ring_empty(&ic->i_recv_ring) && + (atomic_read(&ic->i_signaled_sends) == 0) && + (atomic_read(&ic->i_fastreg_inuse_count)) == 0 && + (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR))) ? 0 + : msecs_to_jiffies(1000); +} + /* * This is so careful about only cleaning up resources that were built up * so that it can be called at any point during startup. In fact it @@ -1079,11 +1096,13 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp) * sends to complete we're ensured that there will be no * more tx processing. */ - wait_event(rds_ib_ring_empty_wait, - rds_ib_ring_empty(&ic->i_recv_ring) && - (atomic_read(&ic->i_signaled_sends) == 0) && - (atomic_read(&ic->i_fastreg_inuse_count) == 0) && - (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR)); + while (!wait_event_timeout(rds_ib_ring_empty_wait, + rds_ib_conn_path_shutdown_check_wait(cp) == 0, + msecs_to_jiffies(1000))) { + tasklet_schedule(&ic->i_send_tasklet); + tasklet_schedule(&ic->i_recv_tasklet); + } + tasklet_kill(&ic->i_send_tasklet); tasklet_kill(&ic->i_recv_tasklet); diff --git a/net/rds/ib_recv.c b/net/rds/ib_recv.c index 4248dfa816ebf..0b52e060988d5 100644 --- a/net/rds/ib_recv.c +++ b/net/rds/ib_recv.c @@ -363,15 +363,14 @@ static int acquire_refill(struct rds_connection *conn) static void release_refill(struct rds_connection *conn) { - clear_bit(RDS_RECV_REFILL, &conn->c_flags); - smp_mb__after_atomic(); + clear_bit_unlock(RDS_RECV_REFILL, &conn->c_flags); /* We don't use wait_on_bit()/wake_up_bit() because our waking is in a * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. */ - if (waitqueue_active(&conn->c_waitq)) + if (wq_has_sleeper(&conn->c_waitq)) wake_up_all(&conn->c_waitq); } @@ -392,7 +391,9 @@ void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp) /* the goal here is to just make sure that someone, somewhere * is posting buffers. If we can't get the refill lock, - * let them do their thing + * let them do their thing. The holder may also be + * rds_conn_shutdown() tearing the path down, in which case + * there is nothing to post. */ if (!acquire_refill(conn)) return; diff --git a/net/rds/message.c b/net/rds/message.c index 064e507f0ccf2..19202aa33f6f4 100644 --- a/net/rds/message.c +++ b/net/rds/message.c @@ -365,7 +365,9 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in for (i = 0; i < rm->data.op_nents; ++i) { sg_set_page(&rm->data.op_sg[i], virt_to_page((void *)page_addrs[i]), - PAGE_SIZE, 0); + i == rm->data.op_nents - 1 + ? total_len - (i * PAGE_SIZE) + : PAGE_SIZE, 0); } return rm; diff --git a/net/rds/recv.c b/net/rds/recv.c index 66205d6924bf3..4b12fa4f1e786 100644 --- a/net/rds/recv.c +++ b/net/rds/recv.c @@ -366,6 +366,21 @@ void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr, goto out; } + /* + * rds_find_bound() uses a global (netns-agnostic) hash table. + * An RDS connection created in netns A can match a socket bound + * in the init netns, delivering inc cross-netns with inc->i_conn + * pointing into netns A. When cleanup_net() then frees that conn, + * any subsequent dereference of inc->i_conn is a use-after-free. + * Drop the inc if the receiving socket lives in a different netns. + */ + if (!net_eq(sock_net(rds_rs_to_sk(rs)), rds_conn_net(conn))) { + rds_stats_inc(s_recv_drop_no_sock); + rds_sock_put(rs); + rs = NULL; + goto out; + } + /* Process extension headers */ rds_recv_incoming_exthdrs(inc, rs); diff --git a/net/rds/send.c b/net/rds/send.c index 071c5dca969a2..5d09c7654f136 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -114,8 +114,13 @@ static void release_in_xmit(struct rds_conn_path *cp) * hot path and finding waiters is very rare. We don't want to walk * the system-wide hashed waitqueue buckets in the fast path only to * almost never find waiters. + * + * wq_has_sleeper() supplies the full barrier that orders the wait + * queue read after the bit clear; clear_bit_unlock() alone is only + * a release and would let this check read a stale empty queue, + * losing the wake-up. */ - if (waitqueue_active(&cp->cp_waitq)) + if (wq_has_sleeper(&cp->cp_waitq)) wake_up_all(&cp->cp_waitq); } @@ -180,8 +185,11 @@ int rds_send_xmit(struct rds_conn_path *cp) WRITE_ONCE(cp->cp_send_gen, send_gen); /* - * rds_conn_shutdown() sets the conn state and then tests RDS_IN_XMIT, - * we do the opposite to avoid races. + * rds_conn_shutdown() sets the conn state and then acquires + * RDS_IN_XMIT; we take the lock first and then check the state. + * Ownership is decided by the atomic RMW on the cp_flags word: + * if the teardown won the bit we back off here, and if we won + * it the teardown waits until we release it. */ if (!rds_conn_path_up(cp)) { release_in_xmit(cp); @@ -1339,7 +1347,7 @@ int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len) ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs); if (ret) { - rs->rs_seen_congestion = 1; + WRITE_ONCE(rs->rs_seen_congestion, 1); goto out; } while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, diff --git a/net/rds/tcp.c b/net/rds/tcp.c index b66dfcc3efaa0..f6d243c4e52cb 100644 --- a/net/rds/tcp.c +++ b/net/rds/tcp.c @@ -46,14 +46,6 @@ static DEFINE_SPINLOCK(rds_tcp_tc_list_lock); static LIST_HEAD(rds_tcp_tc_list); -/* rds_tcp_tc_count counts only IPv4 connections. - * rds6_tcp_tc_count counts both IPv4 and IPv6 connections. - */ -static unsigned int rds_tcp_tc_count; -#if IS_ENABLED(CONFIG_IPV6) -static unsigned int rds6_tcp_tc_count; -#endif - /* Track rds_tcp_connection structs so they can be cleaned up */ static DEFINE_SPINLOCK(rds_tcp_conn_lock); static LIST_HEAD(rds_tcp_conn_list); @@ -110,11 +102,6 @@ void rds_tcp_restore_callbacks(struct socket *sock, /* done under the callback_lock to serialize with write_space */ spin_lock(&rds_tcp_tc_list_lock); list_del_init(&tc->t_list_item); -#if IS_ENABLED(CONFIG_IPV6) - rds6_tcp_tc_count--; -#endif - if (!tc->t_cpath->cp_conn->c_isv6) - rds_tcp_tc_count--; spin_unlock(&rds_tcp_tc_list_lock); tc->t_sock = NULL; @@ -128,46 +115,90 @@ void rds_tcp_restore_callbacks(struct socket *sock, } /* - * rds_tcp_reset_callbacks() switches the to the new sock and - * returns the existing tc->t_sock. + * rds_tcp_reset_callbacks() switches a path to a new socket and + * releases the old one it finds in tc->t_sock, resolving a duelling + * SYN. * - * The only functions that set tc->t_sock are rds_tcp_set_callbacks - * and rds_tcp_reset_callbacks. Send and receive trust that - * it is set. The absence of RDS_CONN_UP bit protects those paths - * from being called while it isn't set. + * tc->t_sock is set by rds_tcp_set_callbacks() and cleared by + * rds_tcp_restore_callbacks(). Four paths write it: the active + * connect in rds_tcp_conn_path_connect(), which sets it and clears it + * again on failure; the accept path in rds_tcp_accept_one(), which + * sets it for a path with no socket yet; the teardown in + * rds_tcp_conn_path_shutdown(), which clears it; and the swap done + * here, which does both. The connect and accept paths are serialized + * against each other by t_conn_path_lock. Send and receive trust + * that it is set: the absence of RDS_CONN_UP protects those paths + * from being called while it isn't, and the swap done here runs under + * RDS_IN_XMIT so that it cannot interleave with a sender already + * inside rds_send_xmit(). */ void rds_tcp_reset_callbacks(struct socket *sock, struct rds_conn_path *cp) { struct rds_tcp_connection *tc = cp->cp_transport_data; - struct socket *osock = tc->t_sock; - - if (!osock) - goto newsock; + struct socket *osock; /* Need to resolve a duelling SYN between peers. * We have an outstanding SYN to this peer, which may * potentially have transitioned to the RDS_CONN_UP state, * so we must quiesce any send threads before resetting - * cp_transport_data. We quiesce these threads by setting - * cp_state to something other than RDS_CONN_UP, and then - * waiting for any existing threads in rds_send_xmit to - * complete release_in_xmit(). (Subsequent threads entering - * rds_send_xmit() will bail on !rds_conn_up(). + * cp_transport_data. Setting cp_state to something other + * than RDS_CONN_UP stops new senders, and owning RDS_IN_XMIT + * excludes any thread already inside rds_send_xmit() - or a + * teardown in rds_conn_shutdown(), which holds the same lock + * for the duration of the transport shutdown - for the whole + * socket swap and the rds_send_path_reset() below. * - * However an incoming syn-ack at this point would end up - * marking the conn as RDS_CONN_UP, and would again permit - * rds_send_xmi() threads through, so ideally we would - * synchronize on RDS_CONN_UP after lock_sock(), but cannot - * do that: waiting on !RDS_IN_XMIT after lock_sock() may - * end up deadlocking with tcp_sendmsg(), and the RDS_IN_XMIT - * would not get set. As a result, we set c_state to - * RDS_CONN_RESETTTING, to ensure that rds_tcp_state_change - * cannot mark rds_conn_path_up() in the window before lock_sock() + * An incoming syn-ack at this point would end up marking the + * conn as RDS_CONN_UP, and would again permit rds_send_xmit() + * threads through, so ideally we would synchronize on + * RDS_CONN_UP after lock_sock(), but cannot do that: acquiring + * RDS_IN_XMIT after lock_sock() may end up deadlocking with + * tcp_sendmsg(), which takes the socket lock while holding + * RDS_IN_XMIT. As a result, we set c_state to + * RDS_CONN_RESETTING, to ensure that rds_tcp_state_change + * cannot mark rds_conn_path_up() in the window before + * lock_sock(). + * + * Only make that transition if the path is still connecting + * (or already resetting from an earlier duel). A path in any + * other state - typically RDS_CONN_DISCONNECTING or + * RDS_CONN_ERROR with a shutdown in flight - is dropped + * instead. That still replaces its state, with RDS_CONN_ERROR, + * and, unless a pending destroy is about to reap the whole + * connection anyway, queues one more shutdown pass. A shutdown + * already in flight leaves that RDS_CONN_ERROR alone when it + * finishes; the queued pass then completes the transition to + * RDS_CONN_DOWN and tears down anything that attached to the + * path in the meantime. + */ + if (!rds_conn_path_transition(cp, RDS_CONN_CONNECTING, + RDS_CONN_RESETTING) && + !rds_conn_path_transition(cp, RDS_CONN_RESETTING, + RDS_CONN_RESETTING)) + rds_conn_path_drop(cp, 0); + wait_event(cp->cp_waitq, + !test_and_set_bit_lock(RDS_IN_XMIT, &cp->cp_flags)); + + /* Read t_sock only while owning RDS_IN_XMIT, never before the + * wait: the teardown in rds_conn_shutdown() releases the old + * socket and clears t_sock, so a pointer sampled earlier can + * be stale by the time we wake up. The teardown holds the + * same lock while it does so, so what we read here cannot + * change under us until we release it. + */ + osock = tc->t_sock; + if (!osock) + goto newsock; + + /* reset receive side state for rds_tcp_data_recv() for osock. + * + * The sync cancels while owning RDS_IN_XMIT rely on cp_wq + * being ordered: a teardown blocked on the bit occupies + * cp_wq's only execution slot, so cp_send_w and cp_recv_w are + * pending at most and the cancels never flush. Nothing here + * may flush or wait on cp_wq itself. */ - atomic_set(&cp->cp_state, RDS_CONN_RESETTING); - wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags)); - /* reset receive side state for rds_tcp_data_recv() for osock */ cancel_delayed_work_sync(&cp->cp_send_w); cancel_delayed_work_sync(&cp->cp_recv_w); lock_sock(osock->sk); @@ -185,6 +216,9 @@ void rds_tcp_reset_callbacks(struct socket *sock, lock_sock(sock->sk); rds_tcp_set_callbacks(sock, cp); release_sock(sock->sk); + + clear_bit_unlock(RDS_IN_XMIT, &cp->cp_flags); + wake_up_all(&cp->cp_waitq); } /* Add tc to rds_tcp_tc_list and set tc->t_sock. See comments @@ -201,11 +235,6 @@ void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp) /* done under the callback_lock to serialize with write_space */ spin_lock(&rds_tcp_tc_list_lock); list_add_tail(&tc->t_list_item, &rds_tcp_tc_list); -#if IS_ENABLED(CONFIG_IPV6) - rds6_tcp_tc_count++; -#endif - if (!tc->t_cpath->cp_conn->c_isv6) - rds_tcp_tc_count++; spin_unlock(&rds_tcp_tc_list_lock); /* accepted sockets need our listen data ready undone */ @@ -233,20 +262,37 @@ static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(rds_sock->sk); struct rds_info_tcp_socket tsinfo; struct rds_tcp_connection *tc; + unsigned int copied = 0; + unsigned int cnt = 0; unsigned long flags; spin_lock_irqsave(&rds_tcp_tc_list_lock, flags); - if (len / sizeof(tsinfo) < rds_tcp_tc_count) + /* First pass: count entries visible in the caller's netns. */ + list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { + if (tc->t_cpath->cp_conn->c_isv6) + continue; + if (!net_eq(rds_conn_net(tc->t_cpath->cp_conn), net)) + continue; + cnt++; + } + + if (len / sizeof(tsinfo) < cnt) goto out; list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { struct inet_sock *inet = inet_sk(tc->t_sock->sk); + if (copied >= cnt) + break; if (tc->t_cpath->cp_conn->c_isv6) continue; + /* Only show connections in the caller's netns. */ + if (!net_eq(rds_conn_net(tc->t_cpath->cp_conn), net)) + continue; tsinfo.local_addr = inet->inet_saddr; tsinfo.local_port = inet->inet_sport; @@ -261,10 +307,12 @@ static void rds_tcp_tc_info(struct socket *rds_sock, unsigned int len, tsinfo.tos = tc->t_cpath->cp_conn->c_tos; rds_info_copy(iter, &tsinfo, sizeof(tsinfo)); + copied++; } + cnt = copied; out: - lens->nr = rds_tcp_tc_count; + lens->nr = cnt; lens->each = sizeof(tsinfo); spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags); @@ -279,19 +327,35 @@ static void rds6_tcp_tc_info(struct socket *sock, unsigned int len, struct rds_info_iterator *iter, struct rds_info_lengths *lens) { + struct net *net = sock_net(sock->sk); struct rds6_info_tcp_socket tsinfo6; struct rds_tcp_connection *tc; + unsigned int copied = 0; + unsigned int cnt = 0; unsigned long flags; spin_lock_irqsave(&rds_tcp_tc_list_lock, flags); - if (len / sizeof(tsinfo6) < rds6_tcp_tc_count) + /* First pass: count entries visible in the caller's netns. */ + list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { + if (!net_eq(rds_conn_net(tc->t_cpath->cp_conn), net)) + continue; + cnt++; + } + + if (len / sizeof(tsinfo6) < cnt) goto out; list_for_each_entry(tc, &rds_tcp_tc_list, t_list_item) { struct sock *sk = tc->t_sock->sk; struct inet_sock *inet = inet_sk(sk); + if (copied >= cnt) + break; + /* Only show connections in the caller's netns. */ + if (!net_eq(rds_conn_net(tc->t_cpath->cp_conn), net)) + continue; + tsinfo6.local_addr = sk->sk_v6_rcv_saddr; tsinfo6.local_port = inet->inet_sport; tsinfo6.peer_addr = sk->sk_v6_daddr; @@ -304,10 +368,12 @@ static void rds6_tcp_tc_info(struct socket *sock, unsigned int len, tsinfo6.last_seen_una = tc->t_last_seen_una; rds_info_copy(iter, &tsinfo6, sizeof(tsinfo6)); + copied++; } + cnt = copied; out: - lens->nr = rds6_tcp_tc_count; + lens->nr = cnt; lens->each = sizeof(tsinfo6); spin_unlock_irqrestore(&rds_tcp_tc_list_lock, flags); @@ -331,21 +397,25 @@ int rds_tcp_laddr_check(struct net *net, const struct in6_addr *addr, /* If the scope_id is specified, check only those addresses * hosted on the specified interface. */ + rcu_read_lock(); if (scope_id != 0) { - rcu_read_lock(); dev = dev_get_by_index_rcu(net, scope_id); /* scope_id is not valid... */ if (!dev) { rcu_read_unlock(); return -EADDRNOTAVAIL; } - rcu_read_unlock(); } #if IS_ENABLED(CONFIG_IPV6) - ret = ipv6_chk_addr(net, addr, dev, 0); - if (ret) - return 0; + if (ipv6_mod_enabled()) { + ret = ipv6_chk_addr(net, addr, dev, 0); + if (ret) { + rcu_read_unlock(); + return 0; + } + } #endif + rcu_read_unlock(); return -EADDRNOTAVAIL; } @@ -637,13 +707,13 @@ static void __net_exit rds_tcp_exit_net(struct net *net) { struct rds_tcp_net *rtn = net_generic(net, rds_tcp_netid); - rds_tcp_kill_sock(net); - if (rtn->rds_tcp_sysctl) unregister_net_sysctl_table(rtn->rds_tcp_sysctl); if (net != &init_net) kfree(rtn->ctl_table); + + rds_tcp_kill_sock(net); } static struct pernet_operations rds_tcp_net_ops = { diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c index 65c5425a02de4..93e8ee10bc8d7 100644 --- a/net/rds/tcp_listen.c +++ b/net/rds/tcp_listen.c @@ -181,7 +181,11 @@ int rds_tcp_accept_one(struct socket *sock) if (rs_tcp->t_sock) { /* Duelling SYN has been handled in rds_tcp_accept_one() */ rds_tcp_reset_callbacks(new_sock, cp); - /* rds_connect_path_complete() marks RDS_CONN_UP */ + /* rds_connect_path_complete() marks RDS_CONN_UP, or, + * if a concurrent shutdown won the duel, drops the + * path again and the pass that drop queues reaps the + * socket installed above. + */ rds_connect_path_complete(cp, RDS_CONN_RESETTING); } else { rds_tcp_set_callbacks(new_sock, cp); diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h index d2b31d15851b9..38b0997ce0cb3 100644 --- a/net/rxrpc/ar-internal.h +++ b/net/rxrpc/ar-internal.h @@ -1266,9 +1266,11 @@ int rxrpc_io_thread(void *data); void rxrpc_post_response(struct rxrpc_connection *conn, struct sk_buff *skb); static inline void rxrpc_wake_up_io_thread(struct rxrpc_local *local) { - if (!local->io_thread) + struct task_struct *io_thread = READ_ONCE(local->io_thread); + + if (!io_thread) return; - wake_up_process(READ_ONCE(local->io_thread)); + wake_up_process(io_thread); } static inline bool rxrpc_protocol_error(struct sk_buff *skb, enum rxrpc_abort_reason why) diff --git a/net/sched/act_api.c b/net/sched/act_api.c index f948646985dde..46c7d84413819 100644 --- a/net/sched/act_api.c +++ b/net/sched/act_api.c @@ -41,11 +41,9 @@ int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)) } EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); -static void tcf_action_goto_chain_exec(const struct tc_action *a, +static void tcf_action_goto_chain_exec(const struct tcf_chain *chain, struct tcf_result *res) { - const struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); - res->goto_tp = rcu_dereference_bh(chain->filter_chain); } @@ -445,16 +443,28 @@ static size_t tcf_action_shared_attrs_size(const struct tc_action *act) + nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */ + cookie_len /* TCA_ACT_COOKIE */ + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */ + /* TCA_ACT_USED_HW_STATS */ + + nla_total_size(sizeof(struct nla_bitfield32)) + + nla_total_size(sizeof(u32)) /* TCA_ACT_IN_HW_COUNT */ + nla_total_size(0) /* TCA_ACT_STATS nested */ + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */ /* TCA_STATS_BASIC */ + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) - /* TCA_STATS_PKT64 */ - + nla_total_size_64bit(sizeof(u64)) + /* TCA_STATS_BASIC_HW */ + + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) + /* TCA_STATS_PKT64, emitted by both of the basic copies above */ + + 2 * nla_total_size_64bit(sizeof(u64)) + /* TCA_STATS_RATE_EST */ + + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est)) + /* TCA_STATS_RATE_EST64 */ + + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est64)) /* TCA_STATS_QUEUE */ + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ - + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ + /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), + * which may emit an extra NLA_PAD attribute. + */ + + nla_total_size_64bit(sizeof(struct tcf_t)); } static size_t tcf_action_full_attrs_size(size_t sz) @@ -1170,12 +1180,14 @@ int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions, return TC_ACT_OK; } } else if (TC_ACT_EXT_CMP(ret, TC_ACT_GOTO_CHAIN)) { - if (unlikely(!rcu_access_pointer(a->goto_chain))) { + struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); + + if (unlikely(!chain)) { tcf_set_drop_reason(skb, SKB_DROP_REASON_TC_CHAIN_NOTFOUND); return TC_ACT_SHOT; } - tcf_action_goto_chain_exec(a, res); + tcf_action_goto_chain_exec(chain, res); } if (ret != TC_ACT_PIPE) @@ -1188,18 +1200,13 @@ EXPORT_SYMBOL(tcf_action_exec); int tcf_action_destroy(struct tc_action *actions[], int bind) { - const struct tc_action_ops *ops; struct tc_action *a; int ret = 0, i; tcf_act_for_each_action(i, a, actions) { actions[i] = NULL; - ops = a->ops; - ret = __tcf_idr_release(a, bind, true); - if (ret == ACT_P_DELETED) - module_put(ops->owner); - else if (ret < 0) - return ret; + /* Drop our reference even if the action is still bound to a filter. */ + ret = tcf_idr_release(a, bind); } return ret; } @@ -1578,7 +1585,7 @@ void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, if (a->cpu_bstats) { _bstats_update(this_cpu_ptr(a->cpu_bstats), bytes, packets); - this_cpu_ptr(a->cpu_qstats)->drops += drops; + this_cpu_add(a->cpu_qstats->drops, drops); if (hw) _bstats_update(this_cpu_ptr(a->cpu_bstats_hw), @@ -1685,12 +1692,12 @@ static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[], static int tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n, - struct tc_action *actions[], int event, + struct tc_action *actions[], size_t attr_size, int event, struct netlink_ext_ack *extack) { struct sk_buff *skb; - skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); + skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); if (!skb) return -ENOBUFS; if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, @@ -1855,11 +1862,13 @@ static int tcf_action_delete(struct net *net, struct tc_action *actions[]) static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, struct tc_action *action) { - size_t attr_size = tcf_action_fill_size(action); struct tc_action *actions[TCA_ACT_MAX_PRIO] = { [0] = action, }; struct sk_buff *skb; + size_t attr_size; + + attr_size = tcf_action_full_attrs_size(tcf_action_fill_size(action)); skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); if (!skb) @@ -1876,15 +1885,18 @@ static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action) { const struct tc_action_ops *ops = action->ops; - struct sk_buff *skb; + struct sk_buff *skb = NULL; int ret; - if (!rtnl_notify_needed(net, 0, RTNLGRP_TC)) { - skb = NULL; - } else { + if (rtnl_notify_needed(net, 0, RTNLGRP_TC)) { skb = tcf_reoffload_del_notify_msg(net, action); + /* The action has already lost its hardware instance and is + * skip_sw, so it must be released whether or not the + * notification can be built. Drop the notification rather + * than leave an action behind that processes no packets. + */ if (IS_ERR(skb)) - return PTR_ERR(skb); + skb = NULL; } ret = tcf_idr_release_unsafe(action); @@ -2041,7 +2053,8 @@ tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, attr_size = tcf_action_full_attrs_size(attr_size); if (event == RTM_GETACTION) - ret = tcf_get_notify(net, portid, n, actions, event, extack); + ret = tcf_get_notify(net, portid, n, actions, attr_size, event, + extack); else { /* delete */ ret = tcf_del_notify(net, n, actions, portid, attr_size, extack); if (ret) diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c index 284800b7d3c55..06d8f78b73683 100644 --- a/net/sched/act_bpf.c +++ b/net/sched/act_bpf.c @@ -76,7 +76,7 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb, break; case TC_ACT_SHOT: action = filter_res; - qstats_drop_inc(this_cpu_ptr(prog->common.cpu_qstats)); + qstats_cpu_drop_inc(prog->common.cpu_qstats); break; case TC_ACT_UNSPEC: action = prog->tcf_action; @@ -389,6 +389,31 @@ static void tcf_bpf_cleanup(struct tc_action *act) tcf_bpf_cfg_cleanup(&tmp); } +static size_t tcf_bpf_get_fill_size(const struct tc_action *act) +{ + struct tcf_bpf *prog = to_bpf(act); + size_t size = nla_total_size(sizeof(struct tc_act_bpf)); + + /* bpf_ops and bpf_num_ops are published as separate stores under + * tcf_lock, so take it here as tcf_bpf_dump() does. + */ + spin_lock_bh(&prog->tcf_lock); + if (tcf_bpf_is_ebpf(prog)) { + /* TCA_ACT_BPF_NAME */ + size += nla_total_size(ACT_BPF_NAME_LEN + 1); + size += nla_total_size(sizeof(u32)); /* TCA_ACT_BPF_ID */ + size += nla_total_size(BPF_TAG_SIZE); /* TCA_ACT_BPF_TAG */ + } else { + size += nla_total_size(sizeof(u16)); /* TCA_ACT_BPF_OPS_LEN */ + /* TCA_ACT_BPF_OPS */ + size += nla_total_size(prog->bpf_num_ops * + sizeof(struct sock_filter)); + } + spin_unlock_bh(&prog->tcf_lock); + + return size; +} + static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .id = TCA_ID_BPF, @@ -397,6 +422,7 @@ static struct tc_action_ops act_bpf_ops __read_mostly = { .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, + .get_fill_size = tcf_bpf_get_fill_size, .size = sizeof(struct tcf_bpf), }; MODULE_ALIAS_NET_ACT("bpf"); diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c index 3a377604ad343..b6873f393aa33 100644 --- a/net/sched/act_csum.c +++ b/net/sched/act_csum.c @@ -259,7 +259,9 @@ static int tcf_csum_ipv4_udp(struct sk_buff *skb, unsigned int ihl, const struct iphdr *iph; u16 ul; - if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP) + if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & + (SKB_GSO_UDP | SKB_GSO_UDP_L4 | + SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)) return 1; /* @@ -315,7 +317,9 @@ static int tcf_csum_ipv6_udp(struct sk_buff *skb, unsigned int ihl, const struct ipv6hdr *ip6h; u16 ul; - if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & SKB_GSO_UDP) + if (skb_is_gso(skb) && skb_shinfo(skb)->gso_type & + (SKB_GSO_UDP | SKB_GSO_UDP_L4 | + SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM)) return 1; /* diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 27d9a7181a0d3..cbac27ba76afc 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -838,8 +838,15 @@ static int tcf_ct_ipv6_is_fragment(struct sk_buff *skb, bool *frag) return 0; } +/* On error, tells the caller whether it still owns @skb and must free it + * itself. @skb is ours only when the header checks below reject the packet + * before it is handed to the defragmentation engine; once nf_ct_handle_ + * fragments() has been called the skb is either queued (-EINPROGRESS) or has + * already been freed by it. + */ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, - u8 family, u16 zone, bool *defrag) + u8 family, u16 zone, bool *defrag, + bool *skb_is_ours) { enum ip_conntrack_info ctinfo; struct tc_skb_cb cb; @@ -857,8 +864,12 @@ static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb, err = tcf_ct_ipv4_is_fragment(skb, &frag); else err = tcf_ct_ipv6_is_fragment(skb, &frag); - if (err || !frag) + if (err) { + *skb_is_ours = true; return err; + } + if (!frag) + return 0; cb = *tc_skb_cb(skb); err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &cb.mru); @@ -975,6 +986,7 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, int nh_ofs, err, retval; struct tcf_ct_params *p; bool add_helper = false; + bool skb_is_ours = false; bool skip_add = false; bool defrag = false; struct nf_conn *ct; @@ -1010,9 +1022,18 @@ TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a, */ nh_ofs = skb_network_offset(skb); skb_pull_rcsum(skb, nh_ofs); - err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag); - if (err) + err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag, + &skb_is_ours); + if (err) { + /* The skb is still ours only when the header checks rejected + * it; returning TC_ACT_CONSUMED for such a packet would leak + * it, since no caller frees an skb it was told it no longer + * owns. + */ + if (skb_is_ours) + goto drop; goto out_frag; + } err = nf_ct_skb_network_trim(skb, family); if (err) @@ -1634,6 +1655,51 @@ static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_ct_get_fill_size(const struct tc_action *act) +{ + const struct tcf_ct_params *p; + size_t size; + + size = nla_total_size(sizeof(struct tc_ct)) /* TCA_CT_PARMS */ + + nla_total_size(sizeof(u16)); /* TCA_CT_ACTION */ + + rcu_read_lock(); + p = rcu_dereference(to_ct(act)->params); + + if (p->ct_action & TCA_CT_ACT_CLEAR) + goto out; + + /* TCA_CT_MARK, TCA_CT_MARK_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) + size += nla_total_size(sizeof(p->mark)) + + nla_total_size(sizeof(p->mark_mask)); + + /* TCA_CT_LABELS, TCA_CT_LABELS_MASK */ + if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) + size += nla_total_size(sizeof(p->labels)) + + nla_total_size(sizeof(p->labels_mask)); + + if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) + size += nla_total_size(sizeof(p->zone)); /* TCA_CT_ZONE */ + + if (p->ct_action & TCA_CT_ACT_NAT) + /* TCA_CT_NAT_IPV6_{MIN,MAX}, the larger of the two address + * variants, plus TCA_CT_NAT_PORT_{MIN,MAX}. + */ + size += 2 * nla_total_size(sizeof(struct in6_addr)) + + 2 * nla_total_size(sizeof(__be16)); + + /* TCA_CT_HELPER_{NAME,FAMILY,PROTO} */ + if (p->helper) + size += nla_total_size(NF_CT_HELPER_NAME_LEN) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); +out: + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ct_ops = { .kind = "ct", .id = TCA_ID_CT, @@ -1643,6 +1709,7 @@ static struct tc_action_ops act_ct_ops = { .init = tcf_ct_init, .cleanup = tcf_ct_cleanup, .stats_update = tcf_stats_update, + .get_fill_size = tcf_ct_get_fill_size, .offload_act_setup = tcf_ct_offload_act_setup, .size = sizeof(struct tcf_ct), }; diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c index 71efe04d00b5c..0517225c144e4 100644 --- a/net/sched/act_ctinfo.c +++ b/net/sched/act_ctinfo.c @@ -355,6 +355,16 @@ static void tcf_ctinfo_cleanup(struct tc_action *a) kfree_rcu(cp, rcu); } +static size_t tcf_ctinfo_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_ctinfo)) /* TCA_CTINFO_ACT */ + + nla_total_size(sizeof(u16)) /* TCA_CTINFO_ZONE */ + /* TCA_CTINFO_PARMS_{DSCP_MASK,DSCP_STATEMASK,CPMARK_MASK} */ + + 3 * nla_total_size(sizeof(u32)) + /* TCA_CTINFO_STATS_{DSCP_SET,DSCP_ERROR,CPMARK_SET} */ + + 3 * nla_total_size_64bit(sizeof(u64)); +} + static struct tc_action_ops act_ctinfo_ops = { .kind = "ctinfo", .id = TCA_ID_CTINFO, @@ -363,6 +373,7 @@ static struct tc_action_ops act_ctinfo_ops = { .dump = tcf_ctinfo_dump, .init = tcf_ctinfo_init, .cleanup= tcf_ctinfo_cleanup, + .get_fill_size = tcf_ctinfo_get_fill_size, .size = sizeof(struct tcf_ctinfo), }; MODULE_ALIAS_NET_ACT("ctinfo"); diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c index e949280eb800d..565860cccba6d 100644 --- a/net/sched/act_gact.c +++ b/net/sched/act_gact.c @@ -89,6 +89,11 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla, p_parm = nla_data(tb[TCA_GACT_PROB]); if (p_parm->ptype >= MAX_RAND) return -EINVAL; + if (!tcf_action_valid(p_parm->paction)) { + NL_SET_ERR_MSG(extack, + "invalid fallback control action"); + return -EINVAL; + } if (TC_ACT_EXT_CMP(p_parm->paction, TC_ACT_GOTO_CHAIN)) { NL_SET_ERR_MSG(extack, "goto chain not allowed on fallback"); diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c index 4ad01d4e820db..b0d66ba80baad 100644 --- a/net/sched/act_ife.c +++ b/net/sched/act_ife.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -723,11 +724,11 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, tcf_lastuse_update(&ife->tcf_tm); if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); tlv_data = ife_decode(skb, &metalen); if (unlikely(!tlv_data)) { - qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); + qstats_cpu_drop_inc(ife->common.cpu_qstats); return TC_ACT_SHOT; } @@ -740,7 +741,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype, &dlen, NULL); if (!curr_data) { - qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); + qstats_cpu_drop_inc(ife->common.cpu_qstats); return TC_ACT_SHOT; } @@ -755,7 +756,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a, } if (WARN_ON(tlv_data != ifehdr_end)) { - qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); + qstats_cpu_drop_inc(ife->common.cpu_qstats); return TC_ACT_SHOT; } @@ -795,7 +796,7 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, where ORIGDATA = original ethernet header ... */ u16 metalen = ife_get_sz(skb, p); - int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN; + int hdrm = metalen + ETH_HLEN + IFE_METAHDRLEN; unsigned int skboff = 0; int new_len = skb->len + hdrm; bool exceed_mtu = false; @@ -821,12 +822,12 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, * so lets be conservative.. */ if ((action == TC_ACT_SHOT) || exceed_mtu) { drop: - qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats)); + qstats_cpu_drop_inc(ife->common.cpu_qstats); return TC_ACT_SHOT; } if (skb_at_tc_ingress(skb)) - skb_push(skb, skb->dev->hard_header_len); + skb_push(skb, ETH_HLEN); ife_meta = ife_encode(skb, metalen); if (!ife_meta) @@ -856,11 +857,27 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a, oethh->h_proto = htons(p->eth_type); if (skb_at_tc_ingress(skb)) - skb_pull(skb, skb->dev->hard_header_len); + skb_pull(skb, ETH_HLEN); return action; } +/* IFE encapsulates the original Ethernet header and, on decode, expects to + * find one, so it can only ever work on skbs that carry one. Loopback carries + * Ethernet header as well, so it qualifies here. + * At ingress, also verify that the L2 header about to be pushed back really + * is an Ethernet header because the skb could've been redirected with mirred + * from a non-Ethernet device. + */ +static bool tcf_ife_is_eth_skb(const struct sk_buff *skb) +{ + if (skb->dev->type != ARPHRD_ETHER && + skb->dev->type != ARPHRD_LOOPBACK) + return false; + + return !skb_at_tc_ingress(skb) || skb->mac_len == ETH_HLEN; +} + TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) @@ -869,6 +886,13 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, struct tcf_ife_params *p; int ret; + if (unlikely(!tcf_ife_is_eth_skb(skb))) { + bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb); + tcf_lastuse_update(&ife->tcf_tm); + qstats_cpu_drop_inc(ife->common.cpu_qstats); + return TC_ACT_SHOT; + } + p = rcu_dereference_bh(ife->params); if (p->flags & IFE_ENCODE) { ret = tcf_ife_encode(skb, a, res, p); @@ -878,6 +902,28 @@ TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb, return tcf_ife_decode(skb, a, res); } +static size_t tcf_ife_get_fill_size(const struct tc_action *act) +{ + struct tcf_ife_info *ife = to_ife(act); + const struct tcf_ife_params *p; + struct tcf_meta_info *e; + size_t size = nla_total_size(sizeof(struct tc_ife)) /* TCA_IFE_PARMS */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_DMAC */ + + nla_total_size(ETH_ALEN) /* TCA_IFE_SMAC */ + + nla_total_size(2) /* TCA_IFE_TYPE */ + + nla_total_size(0); /* TCA_IFE_METALST */ + + rcu_read_lock(); + p = rcu_dereference(ife->params); + if (p) { + list_for_each_entry_rcu(e, &p->metalist, metalist) + size += nla_total_size(sizeof(u32)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_ife_ops = { .kind = "ife", .id = TCA_ID_IFE, @@ -886,6 +932,7 @@ static struct tc_action_ops act_ife_ops = { .dump = tcf_ife_dump, .cleanup = tcf_ife_cleanup, .init = tcf_ife_init, + .get_fill_size = tcf_ife_get_fill_size, .size = sizeof(struct tcf_ife_info), }; MODULE_ALIAS_NET_ACT("ife"); diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c index 6654011dcd2ba..758a13f53213f 100644 --- a/net/sched/act_mpls.c +++ b/net/sched/act_mpls.c @@ -123,7 +123,7 @@ TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb, return p->action; drop: - qstats_drop_inc(this_cpu_ptr(m->common.cpu_qstats)); + qstats_cpu_drop_inc(m->common.cpu_qstats); return TC_ACT_SHOT; } diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c index 5aa91fb145a27..7ce50e303575e 100644 --- a/net/sched/act_pedit.c +++ b/net/sched/act_pedit.c @@ -622,6 +622,29 @@ static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_pedit_get_fill_size(const struct tc_action *act) +{ + const struct tcf_pedit_parms *parms; + size_t size; + + rcu_read_lock(); + parms = rcu_dereference(to_pedit(act)->parms); + size = nla_total_size(struct_size_t(struct tc_pedit, keys, + parms->tcfp_nkeys)); + if (parms->tcfp_keys_ex) { + /* TCA_PEDIT_KEYS_EX, holding one TCA_PEDIT_KEY_EX nest with a + * HTYPE and a CMD attribute per key. + */ + size += nla_total_size(0) + + parms->tcfp_nkeys * (nla_total_size(0) + + nla_total_size(sizeof(u16)) + + nla_total_size(sizeof(u16))); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_pedit_ops = { .kind = "pedit", .id = TCA_ID_PEDIT, @@ -631,6 +654,7 @@ static struct tc_action_ops act_pedit_ops = { .dump = tcf_pedit_dump, .cleanup = tcf_pedit_cleanup, .init = tcf_pedit_init, + .get_fill_size = tcf_pedit_get_fill_size, .offload_act_setup = tcf_pedit_offload_act_setup, .size = sizeof(struct tcf_pedit), }; diff --git a/net/sched/act_police.c b/net/sched/act_police.c index 0e1c611833790..e5fb45ad76bfe 100644 --- a/net/sched/act_police.c +++ b/net/sched/act_police.c @@ -128,6 +128,12 @@ static int tcf_police_init(struct net *net, struct nlattr *nla, if (tb[TCA_POLICE_RESULT]) { tcfp_result = nla_get_u32(tb[TCA_POLICE_RESULT]); + if (!tcf_action_valid(tcfp_result)) { + NL_SET_ERR_MSG(extack, + "invalid fallback control action"); + err = -EINVAL; + goto failure; + } if (TC_ACT_EXT_CMP(tcfp_result, TC_ACT_GOTO_CHAIN)) { NL_SET_ERR_MSG(extack, "goto chain not allowed on fallback"); @@ -310,7 +316,7 @@ TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb, qstats_overlimit_inc(this_cpu_ptr(police->common.cpu_qstats)); inc_drops: if (ret == TC_ACT_SHOT) - qstats_drop_inc(this_cpu_ptr(police->common.cpu_qstats)); + qstats_cpu_drop_inc(police->common.cpu_qstats); end: return ret; } @@ -484,6 +490,17 @@ static int tcf_police_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_police_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_police)) /* TCA_POLICE_TBF */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_RATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PEAKRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTRATE64 */ + + nla_total_size_64bit(sizeof(u64)) /* TCA_POLICE_PKTBURST64 */ + + nla_total_size(sizeof(u32)) /* TCA_POLICE_RESULT */ + + nla_total_size(sizeof(u32)); /* TCA_POLICE_AVRATE */ +} + MODULE_AUTHOR("Alexey Kuznetsov"); MODULE_DESCRIPTION("Policing actions"); MODULE_LICENSE("GPL"); @@ -497,6 +514,7 @@ static struct tc_action_ops act_police_ops = { .dump = tcf_police_dump, .init = tcf_police_init, .cleanup = tcf_police_cleanup, + .get_fill_size = tcf_police_get_fill_size, .offload_act_setup = tcf_police_offload_act_setup, .size = sizeof(struct tcf_police), }; diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c index 2ceb4d141b713..44319a159b55d 100644 --- a/net/sched/act_sample.c +++ b/net/sched/act_sample.c @@ -315,6 +315,14 @@ static int tcf_sample_offload_act_setup(struct tc_action *act, void *entry_data, return 0; } +static size_t tcf_sample_get_fill_size(const struct tc_action *act) +{ + return nla_total_size(sizeof(struct tc_sample)) /* TCA_SAMPLE_PARMS */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_RATE */ + + nla_total_size(sizeof(u32)) /* TCA_SAMPLE_TRUNC_SIZE */ + + nla_total_size(sizeof(u32)); /* TCA_SAMPLE_PSAMPLE_GROUP */ +} + static struct tc_action_ops act_sample_ops = { .kind = "sample", .id = TCA_ID_SAMPLE, @@ -324,6 +332,7 @@ static struct tc_action_ops act_sample_ops = { .dump = tcf_sample_dump, .init = tcf_sample_init, .cleanup = tcf_sample_cleanup, + .get_fill_size = tcf_sample_get_fill_size, .get_psample_group = tcf_sample_get_group, .offload_act_setup = tcf_sample_offload_act_setup, .size = sizeof(struct tcf_sample), diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c index 5450c1293eb50..8691fe1e67da8 100644 --- a/net/sched/act_skbedit.c +++ b/net/sched/act_skbedit.c @@ -86,7 +86,7 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb, return params->action; err: - qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats)); + qstats_cpu_drop_inc(d->common.cpu_qstats); return TC_ACT_SHOT; } diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c index a9e0c1326e2ab..caa3cdfd2d3dd 100644 --- a/net/sched/act_skbmod.c +++ b/net/sched/act_skbmod.c @@ -38,7 +38,6 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (unlikely(p->action == TC_ACT_SHOT)) goto drop; - max_edit_len = skb_mac_header_len(skb); flags = p->flags; /* tcf_skbmod_init() guarantees "flags" to be one of the following: @@ -51,14 +50,19 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb, if (flags == SKBMOD_F_ECN) { switch (skb_protocol(skb, true)) { case cpu_to_be16(ETH_P_IP): + max_edit_len = sizeof(struct iphdr); + break; case cpu_to_be16(ETH_P_IPV6): - max_edit_len += skb_network_header_len(skb); + max_edit_len = sizeof(struct ipv6hdr); break; default: goto out; } - } else if (!skb->dev || skb->dev->type != ARPHRD_ETHER) { - goto out; + max_edit_len += skb_network_offset(skb); + } else { + if (!skb->dev || skb->dev->type != ARPHRD_ETHER) + goto out; + max_edit_len = ETH_HLEN; } err = skb_ensure_writable(skb, max_edit_len); diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c index 876b30c5709e1..ff401ace4f3da 100644 --- a/net/sched/act_tunnel_key.c +++ b/net/sched/act_tunnel_key.c @@ -342,14 +342,20 @@ static const struct nla_policy tunnel_key_policy[TCA_TUNNEL_KEY_MAX + 1] = { [TCA_TUNNEL_KEY_ENC_TTL] = { .type = NLA_U8 }, }; -static void tunnel_key_release_params(struct tcf_tunnel_key_params *p) +static void tunnel_key_release_params_rcu(struct rcu_head *head) { - if (!p) - return; + struct tcf_tunnel_key_params *p = container_of(head, typeof(*p), rcu); + if (p->tcft_action == TCA_TUNNEL_KEY_ACT_SET) dst_release(&p->tcft_enc_metadata->dst); + kfree(p); +} - kfree_rcu(p, rcu); +static void tunnel_key_release_params(struct tcf_tunnel_key_params *p) +{ + if (!p) + return; + call_rcu(&p->rcu, tunnel_key_release_params_rcu); } static int tunnel_key_init(struct net *net, struct nlattr *nla, @@ -829,6 +835,85 @@ static int tcf_tunnel_key_offload_act_setup(struct tc_action *act, return 0; } +static size_t +tunnel_key_geneve_opts_fill_size(const struct ip_tunnel_info *info) +{ + const u8 *src = ip_tunnel_info_opts(info); + int len = info->options_len; + size_t size = 0; + + while (len > 0) { + const struct geneve_opt *opt = (const struct geneve_opt *)src; + + /* TCA_TUNNEL_KEY_ENC_OPT_GENEVE_{CLASS,TYPE,DATA} */ + size += nla_total_size(2) + + nla_total_size(1) + + nla_total_size(opt->length * 4); + + len -= sizeof(struct geneve_opt) + opt->length * 4; + src += sizeof(struct geneve_opt) + opt->length * 4; + } + + return size; +} + +static size_t tunnel_key_opts_fill_size(const struct ip_tunnel_info *info) +{ + size_t size; + + if (!info->options_len) + return 0; + + /* TCA_TUNNEL_KEY_ENC_OPTS and the per-protocol nest inside it */ + size = nla_total_size(0) + nla_total_size(0); + + if (test_bit(IP_TUNNEL_GENEVE_OPT_BIT, info->key.tun_flags)) { + size += tunnel_key_geneve_opts_fill_size(info); + } else if (test_bit(IP_TUNNEL_VXLAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_VXLAN_GBP */ + size += nla_total_size(sizeof(u32)); + } else if (test_bit(IP_TUNNEL_ERSPAN_OPT_BIT, info->key.tun_flags)) { + /* TCA_TUNNEL_KEY_ENC_OPT_ERSPAN_{VER,INDEX,DIR,HWID} */ + size += nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(__be32)) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + + return size; +} + +static size_t tunnel_key_get_fill_size(const struct tc_action *act) +{ + struct tcf_tunnel_key *t = to_tunnel_key(act); + const struct tcf_tunnel_key_params *params; + /* TCA_TUNNEL_KEY_PARMS */ + size_t size = nla_total_size(sizeof(struct tc_tunnel_key)); + + rcu_read_lock(); + params = rcu_dereference(t->params); + if (params->tcft_action == TCA_TUNNEL_KEY_ACT_SET) { + const struct ip_tunnel_info *info = + ¶ms->tcft_enc_metadata->u.tun_info; + + /* In dump order: TCA_TUNNEL_KEY_ENC_KEY_ID, the IPv6 address + * pair (larger than the IPv4 one), ..._ENC_DST_PORT, + * ..._NO_CSUM, ..._NO_FRAG, the options and ..._ENC_{TOS,TTL}. + */ + size += nla_total_size(sizeof(__be32)) + + 2 * nla_total_size(sizeof(struct in6_addr)) + + nla_total_size(sizeof(__be16)) + + nla_total_size(sizeof(u8)) + + nla_total_size(0) + + tunnel_key_opts_fill_size(info) + + nla_total_size(sizeof(u8)) + + nla_total_size(sizeof(u8)); + } + rcu_read_unlock(); + + return size; +} + static struct tc_action_ops act_tunnel_key_ops = { .kind = "tunnel_key", .id = TCA_ID_TUNNEL_KEY, @@ -837,6 +922,7 @@ static struct tc_action_ops act_tunnel_key_ops = { .dump = tunnel_key_dump, .init = tunnel_key_init, .cleanup = tunnel_key_release, + .get_fill_size = tunnel_key_get_fill_size, .offload_act_setup = tcf_tunnel_key_offload_act_setup, .size = sizeof(struct tcf_tunnel_key), }; diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c index 3b0a438977cd5..8a881048e71ec 100644 --- a/net/sched/cls_api.c +++ b/net/sched/cls_api.c @@ -443,7 +443,22 @@ static void tcf_chain_put(struct tcf_chain *chain); static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, bool sig_destroy, struct netlink_ext_ack *extack) { - tp->ops->destroy(tp, rtnl_held, extack); + /* A locked classifier's destroy callback (e.g. u32_destroy) uses + * rtnl_dereference() and mutates shared structures (e.g. the + * tc_u_common hash list) that are only safe under rtnl_lock. When an + * unlocked classifier's request (e.g. flower on ingress) loses the + * tcf_chain_tp_insert_unique() race and ends up dropping the last + * reference on a locked classifier's proto, destroy() would run + * without rtnl held. Take it here in that case. + */ + bool not_lockless = !rtnl_held && + !(tp->ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); + + if (not_lockless) + rtnl_lock(); + tp->ops->destroy(tp, rtnl_held || not_lockless, extack); + if (not_lockless) + rtnl_unlock(); tcf_proto_count_usesw(tp, false); if (sig_destroy) tcf_proto_signal_destroyed(tp->chain, tp); @@ -2233,6 +2248,12 @@ static bool is_ingress_or_clsact(struct tcf_block *block, struct Qdisc *q) return tcf_block_shared(block) || (q && !!(q->flags & TCQ_F_INGRESS)); } +enum tcf_tp_insert_state { + TP_NOT_CREATED = 0, /* did not create and insert a new tp */ + TP_CREATED, /* created and inserted a new tp */ + TP_NOT_OWNED, /* created a proto but failed to insert */ +}; + static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, struct netlink_ext_ack *extack) { @@ -2253,12 +2274,12 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, unsigned long cl; void *fh; int err; - int tp_created; + enum tcf_tp_insert_state tp_state; bool rtnl_held = false; u32 flags; replay: - tp_created = 0; + tp_state = TP_NOT_CREATED; err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); @@ -2380,13 +2401,15 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, goto errout_tp; } - tp_created = 1; + tp_state = TP_CREATED; tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, rtnl_held); if (IS_ERR(tp)) { err = PTR_ERR(tp); goto errout_tp; } + if (tp != tp_new) + tp_state = TP_NOT_OWNED; } else { mutex_unlock(&chain->filter_chain_lock); } @@ -2440,13 +2463,13 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, } errout: - if (err && tp_created) + if (err && tp_state == TP_CREATED) tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); errout_tp: if (chain) { if (tp && !IS_ERR(tp)) tcf_proto_put(tp, rtnl_held, NULL); - if (!tp_created) + if (tp_state == TP_NOT_CREATED) tcf_chain_put(chain); } tcf_block_release(q, block, rtnl_held); @@ -3231,7 +3254,7 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, tcf_chain_put(chain); errout_block: tcf_block_release(q, block, true); - if (err == -EAGAIN) + if (err == -EAGAIN && n->nlmsg_type == RTM_NEWCHAIN) /* Replay the request. */ goto replay; return err; @@ -4047,6 +4070,8 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru fl = rcu_dereference_bh(qe->filter_chain); switch (tcf_classify(skb, NULL, fl, &cl_res, false)) { + case TC_ACT_REDIRECT: + fallthrough; case TC_ACT_SHOT: qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); @@ -4058,10 +4083,6 @@ struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, stru __qdisc_drop(skb, to_free); *ret = __NET_XMIT_STOLEN; return NULL; - case TC_ACT_REDIRECT: - skb_do_redirect(skb); - *ret = __NET_XMIT_STOLEN; - return NULL; case TC_ACT_CONSUMED: *ret = __NET_XMIT_STOLEN; return NULL; diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c index e888d0aa9f21a..a6003fa35c7c5 100644 --- a/net/sched/cls_bpf.c +++ b/net/sched/cls_bpf.c @@ -374,7 +374,8 @@ static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog) } static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, - u32 gen_flags, const struct tcf_proto *tp) + u32 gen_flags, const struct tcf_proto *tp, + struct netlink_ext_ack *extack) { struct bpf_prog *fp; char *name = NULL; @@ -388,6 +389,19 @@ static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog, if (IS_ERR(fp)) return PTR_ERR(fp); + if (bpf_prog_is_dev_bound(fp->aux)) { + struct tcf_block *block = tp->chain->block; + struct net_device *dev; + + dev = block->q ? qdisc_dev(block->q) : NULL; + if (!dev || !bpf_offload_dev_match(fp, dev)) { + NL_SET_ERR_MSG(extack, + "Program is bound to a different device"); + bpf_prog_put(fp); + return -EINVAL; + } + } + if (tb[TCA_BPF_NAME]) { name = nla_memdup(tb[TCA_BPF_NAME], GFP_KERNEL); if (!name) { @@ -492,7 +506,7 @@ static int cls_bpf_change(struct net *net, struct sk_buff *in_skb, prog->gen_flags = gen_flags; ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog) : - cls_bpf_prog_from_efd(tb, prog, gen_flags, tp); + cls_bpf_prog_from_efd(tb, prog, gen_flags, tp, extack); if (ret < 0) goto errout_idr; diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c index b9c58c040c305..5a0ddcf3e2b40 100644 --- a/net/sched/cls_route.c +++ b/net/sched/cls_route.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,7 @@ struct route4_head { struct route4_bucket { /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */ struct route4_filter __rcu *ht[16 + 16 + 1]; + refcount_t filters_ref; struct rcu_head rcu; }; @@ -52,6 +54,7 @@ struct route4_filter { struct tcf_result res; struct tcf_exts exts; u32 handle; + bool dying; struct route4_bucket *bkt; struct tcf_proto *tp; struct rcu_work rwork; @@ -66,9 +69,11 @@ static inline int route4_fastmap_hash(u32 id, int iif) static DEFINE_SPINLOCK(fastmap_lock); static void -route4_reset_fastmap(struct route4_head *head) +route4_reset_fastmap(struct route4_head *head, struct route4_filter *f) { spin_lock_bh(&fastmap_lock); + if (f) + f->dying = true; memset(head->fastmap, 0, sizeof(head->fastmap)); spin_unlock_bh(&fastmap_lock); } @@ -81,9 +86,11 @@ route4_set_fastmap(struct route4_head *head, u32 id, int iif, /* fastmap updates must look atomic to aling id, iff, filter */ spin_lock_bh(&fastmap_lock); - head->fastmap[h].id = id; - head->fastmap[h].iif = iif; - head->fastmap[h].filter = f; + if (f == ROUTE4_FAILURE || !f->dying) { + head->fastmap[h].id = id; + head->fastmap[h].iif = iif; + head->fastmap[h].filter = f; + } spin_unlock_bh(&fastmap_lock); } @@ -297,6 +304,13 @@ static void route4_destroy(struct tcf_proto *tp, bool rtnl_held, next = rtnl_dereference(f->next); RCU_INIT_POINTER(b->ht[h2], next); tcf_unbind_filter(tp, &f->res); + /* Mark the filter dying under fastmap_lock so + * any in-flight reader that still holds it + * will skip the republish in route4_set_fastmap(). + */ + spin_lock_bh(&fastmap_lock); + f->dying = true; + spin_unlock_bh(&fastmap_lock); if (tcf_exts_get_net(&f->exts)) route4_queue_work(f); else @@ -307,6 +321,11 @@ static void route4_destroy(struct tcf_proto *tp, bool rtnl_held, kfree_rcu(b, rcu); } } + + /* All filters are unlinked and marked dying, so no in-flight + * reader can republish a stale entry after this reset. + */ + route4_reset_fastmap(head, NULL); kfree_rcu(head, rcu); } @@ -319,7 +338,7 @@ static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, struct route4_filter *nf; struct route4_bucket *b; unsigned int h = 0; - int i, h1; + int h1; if (!head || !f) return -EINVAL; @@ -334,34 +353,25 @@ static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, /* unlink it */ RCU_INIT_POINTER(*fp, rtnl_dereference(f->next)); - /* Remove any fastmap lookups that might ref filter - * notice we unlink'd the filter so we can't get it - * back in the fastmap. + /* Clear any fastmap entries that may ref this filter and + * mark it dying so in-flight readers can't republish it + * after the reset. */ - route4_reset_fastmap(head); + route4_reset_fastmap(head, f); /* Delete it */ tcf_unbind_filter(tp, &f->res); tcf_exts_get_net(&f->exts); tcf_queue_work(&f->rwork, route4_delete_filter_work); - /* Strip RTNL protected tree */ - for (i = 0; i <= 32; i++) { - struct route4_filter *rt; - - rt = rtnl_dereference(b->ht[i]); - if (rt) - goto out; + if (refcount_dec_and_test(&b->filters_ref)) { + RCU_INIT_POINTER(head->table[to_hash(h)], NULL); + kfree_rcu(b, rcu); } - - /* OK, session has no flows */ - RCU_INIT_POINTER(head->table[to_hash(h)], NULL); - kfree_rcu(b, rcu); break; } } -out: *last = true; for (h1 = 0; h1 <= 256; h1++) { if (rcu_access_pointer(head->table[h1])) { @@ -383,8 +393,9 @@ static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = { static int route4_set_parms(struct net *net, struct tcf_proto *tp, unsigned long base, struct route4_filter *f, u32 handle, struct route4_head *head, - struct nlattr **tb, struct nlattr *est, int new, - u32 flags, struct netlink_ext_ack *extack) + struct nlattr **tb, struct nlattr *est, + struct route4_filter *fold, u32 flags, + struct netlink_ext_ack *extack) { u32 id = 0, to = 0, nhandle = 0x8000; struct route4_filter *fp; @@ -397,7 +408,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, return err; if (tb[TCA_ROUTE4_TO]) { - if (new && handle & 0x8000) { + if (!fold && handle & 0x8000) { NL_SET_ERR_MSG(extack, "Invalid handle"); return -EINVAL; } @@ -420,14 +431,14 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, } else nhandle |= 0xFFFF << 16; - if (handle && new) { + if (handle && (!fold || nhandle == (handle & ~0x7F00))) nhandle |= handle & 0x7F00; - if (nhandle != handle) { - NL_SET_ERR_MSG_FMT(extack, - "Handle mismatch constructed: %x (expected: %x)", - handle, nhandle); - return -EINVAL; - } + + if (handle && !fold && nhandle != handle) { + NL_SET_ERR_MSG_FMT(extack, + "Handle mismatch constructed: %x (expected: %x)", + handle, nhandle); + return -EINVAL; } if (!nhandle) { @@ -442,6 +453,7 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, if (b == NULL) return -ENOBUFS; + refcount_set(&b->filters_ref, 1); rcu_assign_pointer(head->table[h1], b); } else { unsigned int h2 = from_hash(nhandle >> 16); @@ -449,8 +461,14 @@ static int route4_set_parms(struct net *net, struct tcf_proto *tp, for (fp = rtnl_dereference(b->ht[h2]); fp; fp = rtnl_dereference(fp->next)) - if (fp->handle == f->handle) + if (fp != fold && fp->handle == nhandle) { + NL_SET_ERR_MSG_FMT(extack, + "Handle %x is already in use", + nhandle); return -EEXIST; + } + + refcount_inc(&b->filters_ref); } if (tb[TCA_ROUTE4_TO]) @@ -483,9 +501,8 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, struct route4_filter *fold, *f1, *pfp, *f = NULL; struct route4_bucket *b; struct nlattr *tb[TCA_ROUTE4_MAX + 1]; - unsigned int h, th; + unsigned int h; int err; - bool new = true; if (!handle) { NL_SET_ERR_MSG(extack, "Creating with handle of 0 is invalid"); @@ -522,11 +539,10 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, f->tp = fold->tp; f->bkt = fold->bkt; - new = false; } err = route4_set_parms(net, tp, base, f, handle, head, tb, - tca[TCA_RATE], new, flags, extack); + tca[TCA_RATE], fold, flags, extack); if (err < 0) goto errout; @@ -543,22 +559,25 @@ static int route4_change(struct net *net, struct sk_buff *in_skb, rcu_assign_pointer(*fp, f); if (fold) { - th = to_hash(fold->handle); + b = fold->bkt; h = from_hash(fold->handle >> 16); - b = rtnl_dereference(head->table[th]); - if (b) { - fp = &b->ht[h]; - for (pfp = rtnl_dereference(*fp); pfp; - fp = &pfp->next, pfp = rtnl_dereference(*fp)) { - if (pfp == fold) { - rcu_assign_pointer(*fp, fold->next); - break; + fp = &b->ht[h]; + for (pfp = rtnl_dereference(*fp); pfp; + fp = &pfp->next, pfp = rtnl_dereference(*fp)) { + if (pfp == fold) { + rcu_assign_pointer(*fp, fold->next); + if (refcount_dec_and_test(&b->filters_ref)) { + unsigned int th = to_hash(fold->handle); + + RCU_INIT_POINTER(head->table[th], NULL); + kfree_rcu(b, rcu); } + break; } } } - route4_reset_fastmap(head); + route4_reset_fastmap(head, fold); *arg = f; if (fold) { tcf_unbind_filter(tp, &fold->res); diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c index 58e849c0acf41..83a60698cc009 100644 --- a/net/sched/cls_u32.c +++ b/net/sched/cls_u32.c @@ -370,6 +370,10 @@ static int u32_init(struct tcf_proto *tp) refcount_set(&root_ht->refcnt, 1); root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : id2handle(0); + if (root_ht->handle == 0) { + kfree(root_ht); + return -ENOMEM; + } root_ht->prio = tp->prio; root_ht->is_root = true; idr_init(&root_ht->handle_idr); @@ -695,21 +699,33 @@ static int u32_delete(struct tcf_proto *tp, void *arg, bool *last, return ret; } -static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid) +static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid, int *err) { u32 index = htid | 0x800; u32 max = htid | 0xFFF; + *err = 0; + if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, GFP_KERNEL)) { index = htid + 1; - if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, - GFP_KERNEL)) - index = max; + *err = idr_alloc_u32(&ht->handle_idr, NULL, &index, max, + GFP_KERNEL); + if (*err) + return 0; } return index; } +static int u32_kid_extack(int err, struct netlink_ext_ack *extack) +{ + if (err == -ENOSPC) + NL_SET_ERR_MSG_MOD(extack, "Hash table node ID pool exhausted"); + else + NL_SET_ERR_MSG_MOD(extack, "Failed to allocate node ID"); + return err; +} + static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = { [TCA_U32_CLASSID] = { .type = NLA_U32 }, [TCA_U32_HASH] = { .type = NLA_U32 }, @@ -1076,7 +1092,9 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, * handle which is used to uniquely identify the match entry. */ if (!TC_U32_NODE(handle)) { - handle = gen_new_kid(ht, htid); + handle = gen_new_kid(ht, htid, &err); + if (err) + return u32_kid_extack(err, extack); } else { handle = htid | TC_U32_NODE(handle); err = idr_alloc_u32(&ht->handle_idr, NULL, &handle, @@ -1088,7 +1106,9 @@ static int u32_change(struct net *net, struct sk_buff *in_skb, /* The user did not give us a handle; lets just generate one * from the table's pool of nodeids. */ - handle = gen_new_kid(ht, htid); + handle = gen_new_kid(ht, htid, &err); + if (err) + return u32_kid_extack(err, extack); } if (tb[TCA_U32_SEL] == NULL) { @@ -1336,6 +1356,9 @@ static void u32_bind_class(void *fh, u32 classid, unsigned long cl, void *q, { struct tc_u_knode *n = fh; + if (TC_U32_KEY(n->handle) == 0) + return; + tc_cls_bind_class(classid, cl, q, &n->res, base); } diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index f56b18c8aebf7..df271cbc2d9f2 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -415,12 +415,13 @@ static __u8 __detect_linklayer(struct tc_ratespec *r, __u32 *rtab) } static struct qdisc_rate_table *qdisc_rtab_list; +static DEFINE_SPINLOCK(qdisc_rtab_lock); struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, struct nlattr *tab, struct netlink_ext_ack *extack) { - struct qdisc_rate_table *rtab; + struct qdisc_rate_table *rtab, *new_rtab; if (tab == NULL || r->rate == 0 || r->cell_log == 0 || r->cell_log >= 32 || @@ -429,15 +430,20 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, return NULL; } + new_rtab = kmalloc_obj(*new_rtab); + + spin_lock(&qdisc_rtab_lock); for (rtab = qdisc_rtab_list; rtab; rtab = rtab->next) { if (!memcmp(&rtab->rate, r, sizeof(struct tc_ratespec)) && !memcmp(&rtab->data, nla_data(tab), TC_RTAB_SIZE)) { rtab->refcnt++; + spin_unlock(&qdisc_rtab_lock); + kfree(new_rtab); return rtab; } } - rtab = kmalloc(sizeof(*rtab), GFP_KERNEL); + rtab = new_rtab; if (rtab) { rtab->rate = *r; rtab->refcnt = 1; @@ -449,6 +455,7 @@ struct qdisc_rate_table *qdisc_get_rtab(struct tc_ratespec *r, } else { NL_SET_ERR_MSG(extack, "Failed to allocate new qdisc rate table"); } + spin_unlock(&qdisc_rtab_lock); return rtab; } EXPORT_SYMBOL(qdisc_get_rtab); @@ -457,18 +464,25 @@ void qdisc_put_rtab(struct qdisc_rate_table *tab) { struct qdisc_rate_table *rtab, **rtabp; - if (!tab || --tab->refcnt) + if (!tab) return; + spin_lock(&qdisc_rtab_lock); + if (--tab->refcnt) { + spin_unlock(&qdisc_rtab_lock); + return; + } + for (rtabp = &qdisc_rtab_list; (rtab = *rtabp) != NULL; rtabp = &rtab->next) { if (rtab == tab) { *rtabp = rtab->next; - kfree(rtab); - return; + break; } } + spin_unlock(&qdisc_rtab_lock); + kfree(tab); } EXPORT_SYMBOL(qdisc_put_rtab); @@ -596,8 +610,11 @@ void __qdisc_calculate_pkt_len(struct sk_buff *skb, pkt_len <<= stab->szopts.size_log; out: - if (unlikely(pkt_len < 1)) - pkt_len = 1; + /* A size table can inflate qdisc_pkt_len() beyond any real packet + * (via overhead, the data table, or size_log); cap it so deficit + * schedulers such as DRR/ETS terminate their refill loops. + */ + pkt_len = clamp_t(int, pkt_len, 1, QDISC_PKT_LEN_MAX); qdisc_skb_cb(skb)->pkt_len = pkt_len; } @@ -1096,6 +1113,9 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, unsigned int i, num_q, ingress; struct netdev_queue *dev_queue; + if (new) + new->depth = 0; + ingress = 0; num_q = dev->num_tx_queues; if ((q && q->flags & TCQ_F_INGRESS) || @@ -1193,9 +1213,15 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, NL_SET_ERR_MSG(extack, "STAB not supported on a non root"); return -EINVAL; } + if (new && parent->depth >= 7) { + NL_SET_ERR_MSG(extack, "Qdisc hierarchy is too deep"); + return -E2BIG; + } err = cops->graft(parent, cl, new, &old, extack); if (err) return err; + if (new) + new->depth = parent->depth + 1; notify_and_destroy(net, skb, n, classid, old, new, extack); } return 0; @@ -1355,7 +1381,7 @@ static struct Qdisc *qdisc_create(struct net_device *dev, err_out3: lockdep_unregister_key(&sch->root_lock_key); netdev_put(dev, &sch->dev_tracker); - qdisc_free(sch); + qdisc_free_rcu(sch); err_out2: bpf_module_put(ops, ops->owner); err_out: diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c index c19eeed7e7cfb..8ed8ac120693d 100644 --- a/net/sched/sch_cake.c +++ b/net/sched/sch_cake.c @@ -1268,7 +1268,6 @@ static struct sk_buff *cake_ack_filter(struct cake_sched_data *q, seglen = ntohs(ipv6h_check->payload_len); } else { - WARN_ON(1); /* shouldn't happen */ continue; } @@ -1830,7 +1829,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (ack) { WRITE_ONCE(b->ack_drops, b->ack_drops + 1); - sch->qstats.drops++; + qdisc_qstats_drop(sch); ack_pkt_len = qdisc_pkt_len(ack); b->bytes += ack_pkt_len; q->buffer_used += skb->truesize - ack->truesize; diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c index b284f8eda8753..1d2caed1bf5b8 100644 --- a/net/sched/sch_codel.c +++ b/net/sched/sch_codel.c @@ -206,7 +206,7 @@ static int codel_init(struct Qdisc *sch, struct nlattr *opt, codel_params_init(&q->params); codel_vars_init(&q->vars); codel_stats_init(&q->stats); - q->params.mtu = psched_mtu(qdisc_dev(sch)); + q->params.mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (opt) { int err = codel_change(sch, opt, extack); diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c index 44686fd1c2749..ba802db279932 100644 --- a/net/sched/sch_drr.c +++ b/net/sched/sch_drr.c @@ -82,8 +82,9 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid, NL_SET_ERR_MSG(extack, "Specified DRR quantum cannot be zero"); return -EINVAL; } + quantum = clamp_t(u32, quantum, 256, 1 << 20); } else - quantum = psched_mtu(qdisc_dev(sch)); + quantum = clamp_t(u32, (u32)psched_mtu(qdisc_dev(sch)), 256, 1 << 20); if (cl != NULL) { if (tca[TCA_RATE]) { @@ -97,10 +98,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid, } } - sch_tree_lock(sch); if (tb[TCA_DRR_QUANTUM]) - cl->quantum = quantum; - sch_tree_unlock(sch); + WRITE_ONCE(cl->quantum, quantum); return 0; } @@ -251,7 +250,7 @@ static int drr_dump_class(struct Qdisc *sch, unsigned long arg, nest = nla_nest_start_noflag(skb, TCA_OPTIONS); if (nest == NULL) goto nla_put_failure; - if (nla_put_u32(skb, TCA_DRR_QUANTUM, cl->quantum)) + if (nla_put_u32(skb, TCA_DRR_QUANTUM, READ_ONCE(cl->quantum))) goto nla_put_failure; return nla_nest_end(skb, nest); @@ -362,7 +361,7 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch, if (!cl_is_active(cl)) { list_add_tail(&cl->alist, &q->active); - WRITE_ONCE(cl->deficit, cl->quantum); + WRITE_ONCE(cl->deficit, READ_ONCE(cl->quantum)); } sch->qstats.backlog += len; @@ -403,7 +402,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch) return skb; } - WRITE_ONCE(cl->deficit, cl->deficit + cl->quantum); + WRITE_ONCE(cl->deficit, cl->deficit + READ_ONCE(cl->quantum)); list_move_tail(&cl->alist, &q->active); } out: diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c index f20a798e8f36a..4ee841f7a94d4 100644 --- a/net/sched/sch_dualpi2.c +++ b/net/sched/sch_dualpi2.c @@ -208,9 +208,11 @@ static void dualpi2_reset_c_protection(struct dualpi2_sched_data *q) static void dualpi2_calculate_c_protection(struct Qdisc *sch, struct dualpi2_sched_data *q, u32 wc) { + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->c_protection_wc = wc; q->c_protection_wl = MAX_WC - wc; - q->c_protection_init = (s32)psched_mtu(qdisc_dev(sch)) * + q->c_protection_init = (s32)mtu * ((int)q->c_protection_wc - (int)q->c_protection_wl); dualpi2_reset_c_protection(q); } @@ -285,8 +287,9 @@ static bool must_drop(struct Qdisc *sch, struct dualpi2_sched_data *q, u64 local_l_prob; bool overload; u32 prob; + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); - if (sch->qstats.backlog < 2 * psched_mtu(qdisc_dev(sch))) + if (sch->qstats.backlog < 2 * mtu) return false; prob = READ_ONCE(q->pi2_prob); @@ -715,7 +718,8 @@ static u32 get_memory_limit(struct Qdisc *sch, u32 limit) /* Apply rule of thumb, i.e., doubling the packet length, * to further include per packet overhead in memory_limit. */ - u64 memlim = mul_u32_u32(limit, 2 * psched_mtu(qdisc_dev(sch))); + u64 memlim = mul_u32_u32(limit, 2 * clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 1, 1 << 20)); if (upper_32_bits(memlim)) return U32_MAX; diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c index a4b07b661b775..27e549646dee1 100644 --- a/net/sched/sch_ets.c +++ b/net/sched/sch_ets.c @@ -83,11 +83,7 @@ static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr, unsigned int *quantum, struct netlink_ext_ack *extack) { - *quantum = nla_get_u32(attr); - if (!*quantum) { - NL_SET_ERR_MSG(extack, "ETS quantum cannot be zero"); - return -EINVAL; - } + *quantum = clamp_t(u32, nla_get_u32(attr), 256, 1 << 20); return 0; } @@ -634,11 +630,13 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt, return err; } /* If there are more bands than strict + quanta provided, the remaining - * ones are ETS with quantum of MTU. Initialize the missing values here. + * ones are ETS with quantum of max(MTU, 256). Initialize the missing + * values here. */ for (i = nstrict; i < nbands; i++) { if (!quanta[i]) - quanta[i] = psched_mtu(qdisc_dev(sch)); + quanta[i] = clamp_t(u32, (u32)psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); } /* Before commit, make sure we can allocate all new qdiscs */ diff --git a/net/sched/sch_fifo.c b/net/sched/sch_fifo.c index e6bfd39ff3396..1b6388d50967f 100644 --- a/net/sched/sch_fifo.c +++ b/net/sched/sch_fifo.c @@ -19,7 +19,7 @@ static int bfifo_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) { - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= READ_ONCE(sch->limit))) return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c index 5e41930079948..7599a4f7a3e2a 100644 --- a/net/sched/sch_fq.c +++ b/net/sched/sch_fq.c @@ -771,8 +771,12 @@ static struct sk_buff *fq_dequeue(struct Qdisc *sch) * f->time_next_packet was set when prior packet was sent, * and current time (@now) can be too late by tens of us. */ - if (f->time_next_packet) - len -= min(len/2, now - f->time_next_packet); + if (f->time_next_packet) { + s64 drift = now - f->time_next_packet; + + if (drift > 0) + len -= min_t(u64, len / 2, drift); + } f->time_next_packet = now + len; } out: @@ -920,7 +924,7 @@ static int fq_resize(struct Qdisc *sch, u32 log) } static const struct netlink_range_validation iq_range = { - .max = INT_MAX, + .max = 1 << 20, }; static const struct nla_policy fq_policy[TCA_FQ_MAX + 1] = { @@ -1046,14 +1050,10 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt, nla_get_u32(tb[TCA_FQ_FLOW_PLIMIT])); if (tb[TCA_FQ_QUANTUM]) { - u32 quantum = nla_get_u32(tb[TCA_FQ_QUANTUM]); + u32 quantum = clamp_t(u32, nla_get_u32(tb[TCA_FQ_QUANTUM]), + 256, 1 << 20); - if (quantum > 0 && quantum <= (1 << 20)) { - WRITE_ONCE(q->quantum, quantum); - } else { - NL_SET_ERR_MSG_MOD(extack, "invalid quantum"); - err = -EINVAL; - } + WRITE_ONCE(q->quantum, quantum); } if (tb[TCA_FQ_INITIAL_QUANTUM]) @@ -1166,12 +1166,14 @@ static int fq_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_sched_data *q = qdisc_priv(sch); + u32 mtu; int i, err; sch->limit = 10000; q->flow_plimit = 100; - q->quantum = 2 * psched_mtu(qdisc_dev(sch)); - q->initial_quantum = 10 * psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); + q->quantum = clamp_t(u32, 2 * mtu, 256, 1 << 20); + q->initial_quantum = min_t(u32, 10 * mtu, 1 << 20); q->flow_refill_delay = msecs_to_jiffies(40); q->flow_max_rate = ~0UL; q->time_next_delayed_flow = ~0ULL; diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c index 3548985bbd422..3b3401dc5c6f1 100644 --- a/net/sched/sch_fq_codel.c +++ b/net/sched/sch_fq_codel.c @@ -176,7 +176,7 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, flow->cvars.count += i; q->backlogs[idx] -= len; q->memory_usage -= mem; - sch->qstats.drops += i; + __qdisc_qstats_drop(sch, i); sch->qstats.backlog -= len; sch->q.qlen -= i; return idx; @@ -507,6 +507,7 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { struct fq_codel_sched_data *q = qdisc_priv(sch); + u32 mtu; int i; int err; @@ -514,13 +515,14 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, q->flows_cnt = 1024; q->memory_limit = 32 << 20; /* 32 MBytes */ q->drop_batch_size = 64; - q->quantum = psched_mtu(qdisc_dev(sch)); + mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, FQ_CODEL_QUANTUM_MAX); + q->quantum = mtu; INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); codel_params_init(&q->cparams); codel_stats_init(&q->cstats); q->cparams.ecn = true; - q->cparams.mtu = psched_mtu(qdisc_dev(sch)); + q->cparams.mtu = mtu; if (opt) { err = fq_codel_change(sch, opt, extack); diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c index 0c7b2095f8655..ff26999afe4c6 100644 --- a/net/sched/sch_fq_pie.c +++ b/net/sched/sch_fq_pie.c @@ -341,7 +341,8 @@ static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt, nla_get_u32(tb[TCA_FQ_PIE_BETA])); if (tb[TCA_FQ_PIE_QUANTUM]) - WRITE_ONCE(q->quantum, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM])); + WRITE_ONCE(q->quantum, + max(256U, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]))); if (tb[TCA_FQ_PIE_MEMORY_LIMIT]) WRITE_ONCE(q->memory_limit, @@ -427,7 +428,8 @@ static int fq_pie_init(struct Qdisc *sch, struct nlattr *opt, pie_params_init(&q->p_params); sch->limit = 10 * 1024; q->p_params.limit = sch->limit; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->sch = sch; q->ecn_prob = 10; q->flows_cnt = 1024; diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 34573c9b8a5eb..bf8bd129ce911 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -1067,21 +1067,21 @@ void qdisc_reset(struct Qdisc *qdisc) } EXPORT_SYMBOL(qdisc_reset); -void qdisc_free(struct Qdisc *qdisc) +static void qdisc_free_cb(struct rcu_head *head) { - if (qdisc_is_percpu_stats(qdisc)) { - free_percpu(qdisc->cpu_bstats); - free_percpu(qdisc->cpu_qstats); + struct Qdisc *q = container_of(head, struct Qdisc, rcu); + + if (qdisc_is_percpu_stats(q)) { + free_percpu(q->cpu_bstats); + free_percpu(q->cpu_qstats); } - kfree(qdisc); + kfree(q); } -static void qdisc_free_cb(struct rcu_head *head) +void qdisc_free_rcu(struct Qdisc *qdisc) { - struct Qdisc *q = container_of(head, struct Qdisc, rcu); - - qdisc_free(q); + call_rcu(&qdisc->rcu, qdisc_free_cb); } static void __qdisc_destroy(struct Qdisc *qdisc) @@ -1108,7 +1108,7 @@ static void __qdisc_destroy(struct Qdisc *qdisc) trace_qdisc_destroy(qdisc); - call_rcu(&qdisc->rcu, qdisc_free_cb); + qdisc_free_rcu(qdisc); } void qdisc_destroy(struct Qdisc *qdisc) @@ -1259,7 +1259,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - WRITE_ONCE(dev_queue->trans_start, 0); + WRITE_ONCE(dev_queue->trans_start, jiffies); *need_watchdog_p = 1; } } diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c index 532fde548b88f..9185b361dd78e 100644 --- a/net/sched/sch_gred.c +++ b/net/sched/sch_gred.c @@ -179,7 +179,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, * if no default DP has been configured. This * allows for DP flows to be left untouched. */ - if (likely(sch->qstats.backlog + qdisc_pkt_len(skb) <= + if (likely((u64)sch->qstats.backlog + qdisc_pkt_len(skb) <= sch->limit)) return qdisc_enqueue_tail(skb, sch); else @@ -244,7 +244,7 @@ static int gred_enqueue(struct sk_buff *skb, struct Qdisc *sch, break; } - if (gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { + if ((u64)gred_backlog(t, q, sch) + qdisc_pkt_len(skb) <= q->limit) { q->backlog += qdisc_pkt_len(skb); return qdisc_enqueue_tail(skb, sch); } @@ -389,7 +389,7 @@ static int gred_offload_dump_stats(struct Qdisc *sch) packets += u64_stats_read(&hw_stats->stats.bstats[i].packets); sch->qstats.qlen += hw_stats->stats.qstats[i].qlen; sch->qstats.backlog += hw_stats->stats.qstats[i].backlog; - sch->qstats.drops += hw_stats->stats.qstats[i].drops; + __qdisc_qstats_drop(sch, hw_stats->stats.qstats[i].drops); sch->qstats.requeues += hw_stats->stats.qstats[i].requeues; sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits; } diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c index e1c9d1edc8a99..6c5ffc0c4e8d4 100644 --- a/net/sched/sch_hhf.c +++ b/net/sched/sch_hhf.c @@ -550,7 +550,7 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt, return err; if (tb[TCA_HHF_QUANTUM]) - new_quantum = nla_get_u32(tb[TCA_HHF_QUANTUM]); + new_quantum = max(256U, nla_get_u32(tb[TCA_HHF_QUANTUM])); if (tb[TCA_HHF_NON_HH_WEIGHT]) new_hhf_non_hh_weight = nla_get_u32(tb[TCA_HHF_NON_HH_WEIGHT]); @@ -612,7 +612,7 @@ static int hhf_init(struct Qdisc *sch, struct nlattr *opt, int i; sch->limit = 1000; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 256, 1 << 20); get_random_bytes(&q->perturbation, sizeof(q->perturbation)); INIT_LIST_HEAD(&q->new_buckets); INIT_LIST_HEAD(&q->old_buckets); diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c index db97fdb403515..fc8c540393433 100644 --- a/net/sched/sch_htb.c +++ b/net/sched/sch_htb.c @@ -223,6 +223,7 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, struct htb_class *cl; struct tcf_result res; struct tcf_proto *tcf; + unsigned int hops = 0; int result; /* allow to select class by setting skb->priority to valid classid; @@ -265,6 +266,10 @@ static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch, if (!cl->level) return cl; /* we hit leaf; return it */ + if (++hops > TC_HTB_MAXDEPTH) { + pr_warn_ratelimited("htb: classify loop detected, dropping packet\n"); + return NULL; + } /* we have got inner class; apply inner filter chain */ tcf = rcu_dereference_bh(cl->filter_list); } @@ -632,13 +637,11 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch, } else { return qdisc_drop(skb, sch, to_free); } -#ifdef CONFIG_NET_CLS_ACT } else if (!cl) { if (ret & __NET_XMIT_BYPASS) qdisc_qstats_drop(sch); __qdisc_drop(skb, to_free); return ret; -#endif } else if ((ret = qdisc_enqueue(skb, cl->leaf.q, to_free)) != NET_XMIT_SUCCESS) { if (net_xmit_drop_count(ret)) { diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c index 40149edecbd5a..b85726b2896e1 100644 --- a/net/sched/sch_pie.c +++ b/net/sched/sch_pie.c @@ -35,7 +35,7 @@ bool pie_drop_early(struct Qdisc *sch, struct pie_params *params, { u64 rnd; u64 local_prob = vars->prob; - u32 mtu = psched_mtu(qdisc_dev(sch)); + u32 mtu = clamp_t(u32, psched_mtu(qdisc_dev(sch)), 1, 1 << 20); /* If there is still burst allowance left skip random early drop */ if (vars->burst_time > 0) diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c index cefb65201e178..b60ddfee6a68a 100644 --- a/net/sched/sch_plug.c +++ b/net/sched/sch_plug.c @@ -89,7 +89,7 @@ static int plug_enqueue(struct sk_buff *skb, struct Qdisc *sch, { struct plug_sched_data *q = qdisc_priv(sch); - if (likely(sch->qstats.backlog + skb->len <= q->limit)) { + if (likely((u64)sch->qstats.backlog + skb->len <= q->limit)) { if (!q->unplug_indefinite) q->pkts_current_epoch++; return qdisc_enqueue_tail(skb, sch); diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c index 96eb2f122973a..d11b0b192e2c2 100644 --- a/net/sched/sch_sfq.c +++ b/net/sched/sch_sfq.c @@ -657,6 +657,11 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, return -EINVAL; } + if (ctl->quantum > 1 << 20) { + NL_SET_ERR_MSG_MOD(extack, "quantum too large"); + return -EINVAL; + } + if (ctl->perturb_period < 0 || ctl->perturb_period > INT_MAX / HZ) { NL_SET_ERR_MSG_MOD(extack, "invalid perturb period"); @@ -685,7 +690,7 @@ static int sfq_change(struct Qdisc *sch, struct nlattr *opt, /* update and validate configuration */ if (ctl->quantum) - quantum = ctl->quantum; + quantum = max(256U, ctl->quantum); if (ctl->flows) maxflows = min_t(u32, ctl->flows, SFQ_MAX_FLOWS); if (ctl->divisor) { @@ -796,7 +801,8 @@ static int sfq_init(struct Qdisc *sch, struct nlattr *opt, q->tail = NULL; q->divisor = SFQ_DEFAULT_HASH_DIVISOR; q->maxflows = SFQ_DEFAULT_FLOWS; - q->quantum = psched_mtu(qdisc_dev(sch)); + q->quantum = clamp_t(u32, psched_mtu(qdisc_dev(sch)), + 256, 1 << 20); q->perturb_period = 0; get_random_bytes(&q->perturbation, sizeof(q->perturbation)); diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c index 4d463ba5ce725..eb424b20027e5 100644 --- a/net/sched/sch_teql.c +++ b/net/sched/sch_teql.c @@ -377,6 +377,7 @@ static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) nores = 1; break; } + skb->dev = dev; __skb_pull(skb, skb_network_offset(skb)); } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); diff --git a/net/sctp/associola.c b/net/sctp/associola.c index 5793d71852b83..febb8a076f803 100644 --- a/net/sctp/associola.c +++ b/net/sctp/associola.c @@ -543,6 +543,9 @@ void sctp_assoc_rm_peer(struct sctp_association *asoc, asoc->addip_last_asconf->transport == peer) asoc->addip_last_asconf->transport = NULL; + if (asoc->new_transport == peer) + asoc->new_transport = NULL; + /* If we have something on the transmitted list, we have to * save it off. The best place is the active path. */ @@ -573,6 +576,10 @@ void sctp_assoc_rm_peer(struct sctp_association *asoc, if (ch->transport == peer) ch->transport = NULL; + list_for_each_entry(ch, &asoc->outqueue.control_chunk_list, list) + if (ch->transport == peer) + ch->transport = NULL; + asoc->peer.transport_count--; sctp_ulpevent_notify_peer_addr_change(peer, SCTP_ADDR_REMOVED, 0); @@ -614,6 +621,9 @@ struct sctp_transport *sctp_assoc_add_peer(struct sctp_association *asoc, return peer; } + if (asoc->peer.transport_count == U16_MAX) + return NULL; + peer = sctp_transport_new(asoc->base.net, addr, gfp); if (!peer) return NULL; @@ -1706,6 +1716,8 @@ void sctp_asconf_queue_teardown(struct sctp_association *asoc) sctp_assoc_free_asconf_queue(asoc); /* Free any cached ASCONF chunk. */ - if (asoc->addip_last_asconf) + if (asoc->addip_last_asconf) { sctp_chunk_free(asoc->addip_last_asconf); + asoc->addip_last_asconf = NULL; + } } diff --git a/net/sctp/auth.c b/net/sctp/auth.c index 82aad477590e2..fbbfa0c279552 100644 --- a/net/sctp/auth.c +++ b/net/sctp/auth.c @@ -672,7 +672,7 @@ int sctp_auth_ep_add_chunkid(struct sctp_endpoint *ep, __u8 chunk_id) /* Check if we can add this chunk to the array */ param_len = ntohs(p->param_hdr.length); nchunks = param_len - sizeof(struct sctp_paramhdr); - if (nchunks == SCTP_NUM_CHUNK_TYPES) + if (nchunks == SCTP_AUTH_MAX_CHUNKS) return -EINVAL; p->chunks[nchunks] = chunk_id; diff --git a/net/sctp/inqueue.c b/net/sctp/inqueue.c index a024c08432471..5f988b3a8814f 100644 --- a/net/sctp/inqueue.c +++ b/net/sctp/inqueue.c @@ -71,8 +71,11 @@ void sctp_inq_free(struct sctp_inq *queue) */ void sctp_inq_push(struct sctp_inq *q, struct sctp_chunk *chunk) { - /* Directly call the packet handling routine. */ - if (chunk->rcvr->dead) { + /* Directly call the packet handling routine. Drop the chunk if the + * receiver or the transport it was looked up on is gone. + */ + if (chunk->rcvr->dead || + (chunk->transport && chunk->transport->dead)) { sctp_chunk_free(chunk); return; } diff --git a/net/sctp/ipv6.c b/net/sctp/ipv6.c index 7434309785cc5..922b19982d4fe 100644 --- a/net/sctp/ipv6.c +++ b/net/sctp/ipv6.c @@ -1225,11 +1225,17 @@ void sctp_v6_protosw_exit(void) /* Register with inet6 layer. */ int sctp_v6_add_protocol(void) { + int ret; + /* Register notifier for inet6 address additions/deletions. */ - register_inet6addr_notifier(&sctp_inet6addr_notifier); + ret = register_inet6addr_notifier(&sctp_inet6addr_notifier); + if (ret) + return ret; - if (inet6_add_protocol(&sctpv6_protocol, IPPROTO_SCTP) < 0) + if (inet6_add_protocol(&sctpv6_protocol, IPPROTO_SCTP) < 0) { + unregister_inet6addr_notifier(&sctp_inet6addr_notifier); return -EAGAIN; + } return 0; } diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index f6b8c13dafa4a..e92fb9da4647a 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -650,6 +650,7 @@ static int __sctp_outq_flush_rtx(struct sctp_outq *q, struct sctp_packet *pkt, if (chunk->tsn_gap_acked) { list_move_tail(&chunk->transmitted_list, &transport->transmitted); + chunk->transport = transport; continue; } diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c index bbeff25185b17..c718dc066f3a9 100644 --- a/net/sctp/protocol.c +++ b/net/sctp/protocol.c @@ -1297,12 +1297,18 @@ static void sctp_v4_protosw_exit(void) static int sctp_v4_add_protocol(void) { + int ret; + /* Register notifier for inet address additions/deletions. */ - register_inetaddr_notifier(&sctp_inetaddr_notifier); + ret = register_inetaddr_notifier(&sctp_inetaddr_notifier); + if (ret) + return ret; /* Register SCTP with inet layer. */ - if (inet_add_protocol(&sctp_protocol, IPPROTO_SCTP) < 0) + if (inet_add_protocol(&sctp_protocol, IPPROTO_SCTP) < 0) { + unregister_inetaddr_notifier(&sctp_inetaddr_notifier); return -EAGAIN; + } return 0; } @@ -1410,10 +1416,6 @@ static int __net_init sctp_defaults_init(struct net *net) net->sctp.l3mdev_accept = 1; #endif - status = sctp_sysctl_net_register(net); - if (status) - goto err_sysctl_register; - /* Allocate and initialise sctp mibs. */ status = init_sctp_mibs(net); if (status) @@ -1447,8 +1449,6 @@ static int __net_init sctp_defaults_init(struct net *net) cleanup_sctp_mibs(net); #endif err_init_mibs: - sctp_sysctl_net_unregister(net); -err_sysctl_register: return status; } @@ -1463,7 +1463,6 @@ static void __net_exit sctp_defaults_exit(struct net *net) net->sctp.proc_net_sctp = NULL; #endif cleanup_sctp_mibs(net); - sctp_sysctl_net_unregister(net); } static struct pernet_operations sctp_defaults_ops = { @@ -1477,16 +1476,28 @@ static int __net_init sctp_ctrlsock_init(struct net *net) /* Initialize the control inode/socket for handling OOTB packets. */ status = sctp_ctl_sock_init(net); - if (status) + if (status) { pr_err("Failed to initialize the SCTP control sock\n"); + return status; + } + + status = sctp_sysctl_net_register(net); + if (status) { + inet_ctl_sock_destroy(net->sctp.ctl_sock); + net->sctp.ctl_sock = NULL; + } return status; } static void __net_exit sctp_ctrlsock_exit(struct net *net) { + sctp_sysctl_net_unregister(net); + sctp_udp_sock_stop(net); + /* Free the control endpoint. */ inet_ctl_sock_destroy(net->sctp.ctl_sock); + net->sctp.ctl_sock = NULL; } static struct pernet_operations sctp_ctrlsock_ops = { diff --git a/net/sctp/sm_make_chunk.c b/net/sctp/sm_make_chunk.c index a7804e47d3827..04716666b29ba 100644 --- a/net/sctp/sm_make_chunk.c +++ b/net/sctp/sm_make_chunk.c @@ -1802,9 +1802,9 @@ struct sctp_association *sctp_unpack_cookie( goto fail; } - /* Check to see if the cookie is stale. If there is already - * an association, there is no need to check cookie's expiration - * for init collision case of lost COOKIE ACK. + /* Check to see if the cookie is stale. RFC 9260 Section 5.2.4 + * exempts an expired cookie only when both Verification Tags match + * the current association. * If skb has been timestamped, then use the stamp, otherwise * use current time. This introduces a small possibility that * a cookie may be considered expired, but this would only slow @@ -1815,7 +1815,10 @@ struct sctp_association *sctp_unpack_cookie( else kt = ktime_get_real(); - if (!asoc && ktime_before(bear_cookie->expiration, kt)) { + if ((!asoc || + asoc->c.my_vtag != bear_cookie->my_vtag || + asoc->c.peer_vtag != bear_cookie->peer_vtag) && + ktime_before(bear_cookie->expiration, kt)) { suseconds_t usecs = ktime_to_us(ktime_sub(kt, bear_cookie->expiration)); __be32 n = htonl(usecs); @@ -2168,7 +2171,13 @@ static enum sctp_ierror sctp_verify_param(struct net *net, case SCTP_PARAM_HEARTBEAT_INFO: case SCTP_PARAM_UNRECOGNIZED_PARAMETERS: case SCTP_PARAM_ECN_CAPABLE: + break; case SCTP_PARAM_ADAPTATION_LAYER_IND: + if (ntohs(param.p->length) != sizeof(*param.aind)) { + sctp_process_inv_paramlength(asoc, param.p, + chunk, err_chunk); + retval = SCTP_IERROR_ABORT; + } break; case SCTP_PARAM_SUPPORTED_EXT: @@ -3153,6 +3162,12 @@ static __be16 sctp_process_asconf_param(struct sctp_association *asoc, if (!peer) return SCTP_ERROR_DNS_FAILED; + /* Don't free asconf->transport; a later wildcard DEL-IP + * parameter reuses it. + */ + if (peer == asconf->transport) + return SCTP_ERROR_REQ_REFUSED; + sctp_assoc_rm_peer(asoc, peer); break; case SCTP_PARAM_SET_PRIMARY: @@ -3197,6 +3212,9 @@ bool sctp_verify_asconf(const struct sctp_association *asoc, *errp = param.p; switch (param.p->type) { case SCTP_PARAM_ERR_CAUSE: + if (length < sizeof(struct sctp_addip_param) + + sizeof(struct sctp_errhdr)) + return false; break; case SCTP_PARAM_IPV4_ADDRESS: if (length != sizeof(struct sctp_ipv4addr_param)) @@ -3321,12 +3339,11 @@ struct sctp_chunk *sctp_process_asconf(struct sctp_association *asoc, goto done; } done: - asoc->peer.addip_serial++; - /* If we are sending a new ASCONF_ACK hold a reference to it in assoc * after freeing the reference to old asconf ack if any. */ if (asconf_ack) { + asoc->peer.addip_serial++; sctp_chunk_hold(asconf_ack); list_add_tail(&asconf_ack->transmitted_list, &asoc->asconf_ack_list); @@ -3431,20 +3448,15 @@ static __be16 sctp_get_asconf_response(struct sctp_chunk *asconf_ack, case SCTP_PARAM_ERR_CAUSE: length = sizeof(*asconf_ack_param); err_param = (void *)asconf_ack_param + length; - asconf_ack_len -= length; - if (asconf_ack_len > 0) - return err_param->cause; - else - return SCTP_ERROR_INV_PARAM; - break; + return err_param->cause; default: return SCTP_ERROR_INV_PARAM; } } length = ntohs(asconf_ack_param->param_hdr.length); - asconf_ack_param = (void *)asconf_ack_param + length; - asconf_ack_len -= length; + asconf_ack_param = (void *)asconf_ack_param + SCTP_PAD4(length); + asconf_ack_len -= SCTP_PAD4(length); } return err_code; diff --git a/net/sctp/sm_sideeffect.c b/net/sctp/sm_sideeffect.c index 424f10a6fdba9..0d99b7e8c082f 100644 --- a/net/sctp/sm_sideeffect.c +++ b/net/sctp/sm_sideeffect.c @@ -1332,6 +1332,10 @@ static int sctp_cmd_interpreter(enum sctp_event_type event_type, sctp_outq_uncork(&asoc->outqueue, gfp); local_cork = 0; } + /* No chunk left in this packet may use this asoc. */ + if (event_type == SCTP_EVENT_T_CHUNK && + chunk->asoc == asoc) + chunk->pdiscard = 1; /* Delete the current association. */ sctp_cmd_delete_tcb(commands, asoc); asoc = NULL; @@ -1541,17 +1545,8 @@ static int sctp_cmd_interpreter(enum sctp_event_type event_type, timeout = asoc->timeouts[cmd->obj.to]; BUG_ON(!timeout); - /* - * SCTP has a hard time with timer starts. Because we process - * timer starts as side effects, it can be hard to tell if we - * have already started a timer or not, which leads to BUG - * halts when we call add_timer. So here, instead of just starting - * a timer, if the timer is already started, and just mod - * the timer with the shorter of the two expiration times - */ - if (!timer_pending(timer)) + if (!timer_reduce(timer, jiffies + timeout)) sctp_association_hold(asoc); - timer_reduce(timer, jiffies + timeout); break; case SCTP_CMD_TIMER_RESTART: diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index abfd72d1f20c6..3a8e16b29660c 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -416,6 +416,8 @@ enum sctp_disposition sctp_sf_do_5_1B_init(struct net *net, /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -641,7 +643,7 @@ static bool sctp_auth_chunk_verify(struct net *net, struct sctp_chunk *chunk, struct sctp_chunk auth; if (!chunk->auth_chunk) - return true; + return !sctp_auth_recv_cid(chunk->chunk_hdr->type, asoc); /* SCTP-AUTH: auth_chunk pointer is only set when the cookie-echo * is supposed to be authenticated and we have to do delayed @@ -1620,6 +1622,8 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( /* Update socket peer label if first association. */ if (security_sctp_assoc_request(new_asoc, chunk->skb)) { sctp_association_free(new_asoc); + if (err_chunk) + sctp_chunk_free(err_chunk); return sctp_sf_pdiscard(net, ep, asoc, type, arg, commands); } @@ -1685,6 +1689,7 @@ static enum sctp_disposition sctp_sf_do_unexpected_init( * parameter type. */ sctp_addto_chunk(repl, len, unk_param); + sctp_chunk_free(err_chunk); } sctp_add_cmd_sf(commands, SCTP_CMD_NEW_ASOC, SCTP_ASOC(new_asoc)); @@ -6140,8 +6145,12 @@ enum sctp_disposition sctp_sf_t4_timer_expire( struct sctp_cmd_seq *commands) { struct sctp_chunk *chunk = asoc->addip_last_asconf; - struct sctp_transport *transport = chunk->transport; + struct sctp_transport *transport; + + if (!chunk) + return SCTP_DISPOSITION_CONSUME; + transport = chunk->transport; SCTP_INC_STATS(net, SCTP_MIB_T4_RTO_EXPIREDS); /* ADDIP 4.1 B1) Increment the error counters and perform path failure diff --git a/net/sctp/stream.c b/net/sctp/stream.c index 39b8f5e8ce353..8826dd60e59e7 100644 --- a/net/sctp/stream.c +++ b/net/sctp/stream.c @@ -22,6 +22,15 @@ #include #include +#define SCTP_STRRESET_MASK(type) \ + BIT(ntohs(type) - ntohs(SCTP_PARAM_RESET_OUT_REQUEST)) +#define SCTP_STRRESET_TEST(asoc, type) \ + ((asoc)->strreset_outstanding & SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_SET(asoc, type) \ + ((asoc)->strreset_outstanding |= SCTP_STRRESET_MASK(type)) +#define SCTP_STRRESET_CLEAR(asoc, type) \ + ((asoc)->strreset_outstanding &= ~SCTP_STRRESET_MASK(type)) + static void sctp_stream_shrink_out(struct sctp_stream *stream, __u16 outcnt) { struct sctp_association *asoc; @@ -308,7 +317,8 @@ int sctp_send_reset_streams(struct sctp_association *asoc, goto out; param_len += str_nums * sizeof(__u16) + - sizeof(struct sctp_strreset_inreq); + (out ? sizeof(struct sctp_strreset_inreq) + : sizeof(struct sctp_strreset_outreq)); } if (param_len > SCTP_MAX_CHUNK_LEN - @@ -371,7 +381,10 @@ int sctp_send_reset_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = out + in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_IN_REQUEST); out: return retval; @@ -416,7 +429,7 @@ int sctp_send_reset_assoc(struct sctp_association *asoc) return retval; } - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_TSN_REQUEST); return 0; } @@ -473,7 +486,10 @@ int sctp_send_add_streams(struct sctp_association *asoc, goto out; } - asoc->strreset_outstanding = !!out + !!in; + if (out) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); + if (in) + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); out: return retval; @@ -481,13 +497,13 @@ int sctp_send_add_streams(struct sctp_association *asoc, static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( struct sctp_association *asoc, __be32 resp_seq, - __be16 type) + __be16 type, bool match_seq) { struct sctp_chunk *chunk = asoc->strreset_chunk; struct sctp_reconf_chunk *hdr; union sctp_params param; - if (!chunk) + if (!chunk || !chunk->transport) return NULL; hdr = (struct sctp_reconf_chunk *)chunk->chunk_hdr; @@ -498,7 +514,7 @@ static struct sctp_paramhdr *sctp_chunk_lookup_strreset_param( */ struct sctp_strreset_tsnreq *req = param.v; - if ((!resp_seq || req->request_seq == resp_seq) && + if ((!match_seq || req->request_seq == resp_seq) && (!type || type == req->param_hdr.type)) return param.v; } @@ -563,13 +579,14 @@ struct sctp_chunk *sctp_process_strreset_outreq( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( asoc, outreq->response_seq, - SCTP_PARAM_RESET_IN_REQUEST)) { + SCTP_PARAM_RESET_IN_REQUEST, true) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_IN_REQUEST)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_IN_REQUEST); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -639,6 +656,9 @@ struct sctp_chunk *sctp_process_strreset_inreq( nums = (ntohs(param.p->length) - sizeof(*inreq)) / sizeof(__u16); str_p = inreq->list_of_streams; + if (nums * sizeof(__u16) + sizeof(struct sctp_strreset_outreq) > + SCTP_MAX_CHUNK_LEN - sizeof(struct sctp_reconf_chunk)) + goto out; for (i = 0; i < nums; i++) { if (ntohs(str_p[i]) >= stream->outcnt) { result = SCTP_STRRESET_ERR_WRONG_SSN; @@ -665,7 +685,7 @@ struct sctp_chunk *sctp_process_strreset_inreq( SCTP_SO(stream, i)->state = SCTP_STREAM_CLOSED; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_OUT_REQUEST); sctp_chunk_hold(asoc->strreset_chunk); result = SCTP_STRRESET_PERFORMED; @@ -812,13 +832,14 @@ struct sctp_chunk *sctp_process_strreset_addstrm_out( if (asoc->strreset_chunk) { if (!sctp_chunk_lookup_strreset_param( - asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { + asoc, 0, SCTP_PARAM_RESET_ADD_IN_STREAMS, false) || + !SCTP_STRRESET_TEST(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS)) { /* same process with outstanding isn't 0 */ result = SCTP_STRRESET_ERR_IN_PROGRESS; goto out; } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, SCTP_PARAM_RESET_ADD_IN_STREAMS); asoc->strreset_outseq++; if (!asoc->strreset_outstanding) { @@ -895,7 +916,7 @@ struct sctp_chunk *sctp_process_strreset_addstrm_in( goto out; asoc->strreset_chunk = chunk; - asoc->strreset_outstanding = 1; + SCTP_STRRESET_SET(asoc, SCTP_PARAM_RESET_ADD_OUT_STREAMS); sctp_chunk_hold(asoc->strreset_chunk); stream->outcnt = outcnt; @@ -923,8 +944,9 @@ struct sctp_chunk *sctp_process_strreset_resp( struct sctp_paramhdr *req; __u32 result; - req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0); - if (!req) + req = sctp_chunk_lookup_strreset_param(asoc, resp->response_seq, 0, + true); + if (!req || !SCTP_STRRESET_TEST(asoc, req->type)) return NULL; result = ntohl(resp->result); @@ -1074,7 +1096,7 @@ struct sctp_chunk *sctp_process_strreset_resp( nums, 0, GFP_ATOMIC); } - asoc->strreset_outstanding--; + SCTP_STRRESET_CLEAR(asoc, req->type); asoc->strreset_outseq++; /* remove everything for this reconf request */ diff --git a/net/sctp/sysctl.c b/net/sctp/sysctl.c index 15e7db9a3ab2e..fca840484ebf7 100644 --- a/net/sctp/sysctl.c +++ b/net/sctp/sysctl.c @@ -615,11 +615,16 @@ int sctp_sysctl_net_register(struct net *net) void sctp_sysctl_net_unregister(struct net *net) { + struct ctl_table_header *header = net->sctp.sysctl_header; const struct ctl_table *table; - table = net->sctp.sysctl_header->ctl_table_arg; - unregister_net_sysctl_table(net->sctp.sysctl_header); + if (!header) + return; + + table = header->ctl_table_arg; + unregister_net_sysctl_table(header); kfree(table); + net->sctp.sysctl_header = NULL; } static struct ctl_table_header *sctp_sysctl_header; diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 47963eda478f6..e23390d9326b1 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -409,13 +409,13 @@ void smc_sk_init(struct net *net, struct sock *sk, int protocol) "sk_lock-AF_SMC", &smc_key); spin_lock_init(&smc->accept_q_lock); spin_lock_init(&smc->conn.send_lock); - sk->sk_prot->hash(sk); mutex_init(&smc->clcsock_release_lock); smc_init_saved_callbacks(smc); smc->limit_smc_hs = net->smc.limit_smc_hs; smc->use_fallback = false; /* assume rdma capability first */ smc->fallback_rsn = 0; smc_close_init(smc); + sk->sk_prot->hash(sk); } static struct sock *smc_sock_alloc(struct net *net, struct socket *sock, @@ -1935,11 +1935,12 @@ static void smc_listen_out(struct smc_sock *new_smc) atomic_dec(&lsmc->queued_smc_hs); release_sock(newsmcsk); /* lock in smc_listen_work() */ + lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); if (lsmc->sk.sk_state == SMC_LISTEN) { - lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING); smc_accept_enqueue(&lsmc->sk, newsmcsk); release_sock(&lsmc->sk); } else { /* no longer listening */ + release_sock(&lsmc->sk); smc_close_non_accepted(newsmcsk); } @@ -3236,7 +3237,8 @@ int smc_ioctl(struct socket *sock, unsigned int cmd, return -EINVAL; } if (smc->sk.sk_state == SMC_INIT || - smc->sk.sk_state == SMC_CLOSED) + smc->sk.sk_state == SMC_CLOSED || + !READ_ONCE(smc->conn.sndbuf_desc)) answ = 0; else answ = smc->conn.sndbuf_desc->len - diff --git a/net/smc/smc.h b/net/smc/smc.h index ea45467c11409..1425fd756d343 100644 --- a/net/smc/smc.h +++ b/net/smc/smc.h @@ -277,9 +277,9 @@ struct smc_connection { * 0 for SMC-R, 32 for SMC-D */ u64 peer_token; /* SMC-D token of peer */ - u8 killed : 1; /* abnormal termination */ - u8 freed : 1; /* normal termination */ - u8 out_of_sync : 1; /* out of sync with peer */ + u8 killed; /* abnormal termination */ + u8 freed; /* normal termination */ + u8 out_of_sync; /* out of sync with peer */ }; struct smc_sock { /* smc sock container */ diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index be0c2da83d2bf..34fa809e95047 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1134,13 +1134,13 @@ struct smc_link *smc_switch_conns(struct smc_link_group *lgr, read_unlock_bh(&lgr->conns_lock); /* pre-fetch buffer outside of send_lock, might sleep */ rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend); - if (rc) - goto err_out; - /* avoid race with smcr_tx_sndbuf_nonempty() */ - spin_lock_bh(&conn->send_lock); - smc_switch_link_and_count(conn, to_lnk); - rc = smc_switch_cursor(smc, pend, wr_buf); - spin_unlock_bh(&conn->send_lock); + if (!rc) { + /* avoid race with smcr_tx_sndbuf_nonempty() */ + spin_lock_bh(&conn->send_lock); + smc_switch_link_and_count(conn, to_lnk); + rc = smc_switch_cursor(smc, pend, wr_buf); + spin_unlock_bh(&conn->send_lock); + } sock_put(&smc->sk); if (rc) goto err_out; @@ -1195,14 +1195,16 @@ static void smcd_buf_detach(struct smc_connection *conn) { struct smcd_dev *smcd = conn->lgr->smcd; u64 peer_token = conn->peer_token; + struct smc_buf_desc *buf_desc; if (!conn->sndbuf_desc) return; smc_ism_detach_dmb(smcd, peer_token); - kfree(conn->sndbuf_desc); + buf_desc = conn->sndbuf_desc; conn->sndbuf_desc = NULL; + kfree(buf_desc); } static void smc_buf_unuse(struct smc_connection *conn, @@ -1254,11 +1256,10 @@ void smc_conn_free(struct smc_connection *conn) goto lgr_put; if (lgr->is_smcd) { - if (!list_empty(&lgr->list)) - smc_ism_unset_conn(conn); + smc_ism_unset_conn(conn); + tasklet_kill(&conn->rx_tsklet); if (smc_ism_support_dmb_nocopy(lgr->smcd)) smcd_buf_detach(conn); - tasklet_kill(&conn->rx_tsklet); } else { smc_cdc_wait_pend_tx_wr(conn); if (current_work() != &conn->abort_work) @@ -1511,12 +1512,12 @@ static void smc_conn_kill(struct smc_connection *conn, bool soft) smc_sk_wake_ups(smc); if (conn->lgr->is_smcd) { smc_ism_unset_conn(conn); - if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) - smcd_buf_detach(conn); if (soft) tasklet_kill(&conn->rx_tsklet); else tasklet_unlock_wait(&conn->rx_tsklet); + if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) + smcd_buf_detach(conn); } else { smc_cdc_wait_pend_tx_wr(conn); } @@ -1558,10 +1559,10 @@ static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft) read_lock_bh(&lgr->conns_lock); node = rb_first(&lgr->conns_all); while (node) { - read_unlock_bh(&lgr->conns_lock); conn = rb_entry(node, struct smc_connection, alert_node); smc = container_of(conn, struct smc_sock, conn); sock_hold(&smc->sk); /* sock_put below */ + read_unlock_bh(&lgr->conns_lock); lock_sock(&smc->sk); smc_conn_kill(conn, soft); release_sock(&smc->sk); diff --git a/net/smc/smc_inet.c b/net/smc/smc_inet.c index a94084b4a498e..520b666fdd8fa 100644 --- a/net/smc/smc_inet.c +++ b/net/smc/smc_inet.c @@ -15,13 +15,16 @@ #include "smc_inet.h" #include "smc.h" +#include "smc_close.h" static int smc_inet_init_sock(struct sock *sk); +static void smc_inet_destroy_sock(struct sock *sk); static struct proto smc_inet_prot = { .name = "INET_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -68,6 +71,7 @@ static struct proto smc_inet6_prot = { .name = "INET6_SMC", .owner = THIS_MODULE, .init = smc_inet_init_sock, + .destroy = smc_inet_destroy_sock, .hash = smc_hash_sk, .unhash = smc_unhash_sk, .release_cb = smc_release_cb, @@ -116,6 +120,18 @@ static int smc_inet_init_sock(struct sock *sk) return smc_create_clcsk(net, sk, sk->sk_family); } +static void smc_inet_destroy_sock(struct sock *sk) +{ + /* The sock is hashed and smc_diag dumps dereference smc->clcsock + * without clcsock_release_lock, while sk_common_release() calls + * .destroy before .unhash. Unhash first, as __smc_release() does, + * so no dump can observe the clcsock being released; the second + * unhash is a no-op. + */ + sk->sk_prot->unhash(sk); + smc_clcsock_release(smc_sk(sk)); +} + int __init smc_inet_init(void) { int rc; diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c index f865c58c3aa77..65cf07bb30d8d 100644 --- a/net/smc/smc_llc.c +++ b/net/smc/smc_llc.c @@ -157,6 +157,7 @@ struct smc_llc_msg_confirm_rkey { /* type 0x06 */ }; #define SMC_LLC_DEL_RKEY_MAX 8 +#define SMC_LLC_DEL_RKEY_V2_INLINE 9 #define SMC_LLC_FLAG_RKEY_RETRY 0x10 #define SMC_LLC_FLAG_RKEY_NEG 0x20 @@ -177,6 +178,15 @@ struct smc_llc_msg_delete_rkey_v2 { /* type 0x29 */ __be32 rkey[]; }; +/* the leading rkeys of a DELETE_RKEY_V2 fit into union smc_llc_msg */ +struct smc_llc_msg_delete_rkey_v2_inline { /* type 0x29 */ + struct smc_llc_hdr hd; + u8 num_rkeys; + u8 num_inval_rkeys; + u8 reserved[2]; + __be32 rkey[SMC_LLC_DEL_RKEY_V2_INLINE]; +}; + union smc_llc_msg { struct smc_llc_msg_confirm_link confirm_link; struct smc_llc_msg_add_link add_link; @@ -186,6 +196,7 @@ union smc_llc_msg { struct smc_llc_msg_confirm_rkey confirm_rkey; struct smc_llc_msg_delete_rkey delete_rkey; + struct smc_llc_msg_delete_rkey_v2_inline delete_rkey_v2; struct smc_llc_msg_test_link test_link; struct { @@ -194,15 +205,25 @@ union smc_llc_msg { } raw; }; +static_assert(SMC_LLC_DEL_RKEY_V2_INLINE == + (sizeof(union smc_llc_msg) - + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)) / + sizeof(__be32)); +static_assert(offsetof(struct smc_llc_msg_delete_rkey_v2_inline, rkey) == + offsetof(struct smc_llc_msg_delete_rkey_v2, rkey)); + #define SMC_LLC_FLAG_RESP 0x80 struct smc_llc_qentry { struct list_head list; struct smc_link *link; + u16 body_len; union smc_llc_msg msg; + u8 body[] __counted_by(body_len); }; -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len); struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) { @@ -320,6 +341,7 @@ int smc_llc_flow_initiate(struct smc_link_group *lgr, void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) { spin_lock_bh(&lgr->llc_flow_lock); + smc_llc_flow_qentry_del(flow); memset(flow, 0, sizeof(*flow)); flow->type = SMC_LLC_FLOW_NONE; spin_unlock_bh(&lgr->llc_flow_lock); @@ -998,15 +1020,20 @@ static int smc_llc_cli_conf_link(struct smc_link *link, static void smc_llc_save_add_link_rkeys(struct smc_link *link, struct smc_link *link_new, - u8 *llc_msg) + struct smc_llc_qentry *qentry) { + const u32 rt_off = offsetof(struct smc_llc_msg_add_link_v2_ext, rt); struct smc_llc_msg_add_link_v2_ext *ext; struct smc_link_group *lgr = link->lgr; int max, i; - ext = (struct smc_llc_msg_add_link_v2_ext *)(llc_msg + - SMC_WR_TX_SIZE); + /* the rkey count itself is only there if enough bytes arrived */ + if (qentry->body_len < rt_off) + return; + ext = (struct smc_llc_msg_add_link_v2_ext *)qentry->body; max = min_t(u8, ext->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, (qentry->body_len - rt_off) / + sizeof(ext->rt[0])); down_write(&lgr->rmbs_lock); for (i = 0; i < max; i++) { smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, @@ -1099,9 +1126,7 @@ int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) if (rc) goto out_clear_lnk; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; - smc_llc_save_add_link_rkeys(link, lnk_new, llc_msg); + smc_llc_save_add_link_rkeys(link, lnk_new, qentry); } else { rc = smc_llc_cli_rkey_exchange(link, lnk_new); if (rc) { @@ -1481,7 +1506,7 @@ int smc_llc_srv_add_link(struct smc_link *link, } add_llc = &qentry->msg.add_link; if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = -ENOLINK; goto out_err; } @@ -1492,7 +1517,8 @@ int smc_llc_srv_add_link(struct smc_link *link, lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; } smc_llc_save_add_link_info(link_new, add_llc); - smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); + /* add_llc still points into qentry, so only detach it here */ + smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); rc = smc_ib_ready_link(link_new); if (rc) @@ -1501,9 +1527,7 @@ int smc_llc_srv_add_link(struct smc_link *link, if (rc) goto out_err; if (lgr->smc_version == SMC_V2) { - u8 *llc_msg = smc_link_shared_v2_rxbuf(link) ? - (u8 *)lgr->wr_rx_buf_v2 : (u8 *)add_llc; - smc_llc_save_add_link_rkeys(link, link_new, llc_msg); + smc_llc_save_add_link_rkeys(link, link_new, qentry); } else { rc = smc_llc_srv_rkey_exchange(link, link_new); if (rc) @@ -1512,14 +1536,14 @@ int smc_llc_srv_add_link(struct smc_link *link, rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); if (rc) goto out_err; - kfree(ini); - return 0; + goto out; out_err: if (link_new) { link_new->state = SMC_LNK_INACTIVE; smcr_link_clear(link_new, false); } out: + kfree(qentry); kfree(ini); if (send_req_add_link_resp) smc_llc_send_req_add_link_response(req_qentry); @@ -1552,7 +1576,8 @@ void smc_llc_add_link_local(struct smc_link *link) add_llc.hd.common.llc_type = SMC_LLC_ADD_LINK; smc_llc_init_msg_hdr(&add_llc.hd, link->lgr, sizeof(add_llc)); /* no dev and port needed */ - smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc, + sizeof(union smc_llc_msg)); } /* worker to process an add link message */ @@ -1588,7 +1613,8 @@ void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) del_llc.link_num = del_link_id; del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; - smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); + smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc, + sizeof(union smc_llc_msg)); } static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) @@ -1810,19 +1836,28 @@ static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) link = qentry->link; if (lgr->smc_version == SMC_V2) { - struct smc_llc_msg_delete_rkey_v2 *llcv2; + struct smc_llc_msg_delete_rkey_v2_inline *llcv2; - if (smc_link_shared_v2_rxbuf(link)) { - memcpy(lgr->wr_rx_buf_v2, llc, sizeof(*llc)); - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)lgr->wr_rx_buf_v2; - } else { - llcv2 = (struct smc_llc_msg_delete_rkey_v2 *)llc; - } + /* The leading SMC_LLC_DEL_RKEY_V2_INLINE rkeys are declared in + * the message itself, any further ones were received into + * qentry->body. + */ + llcv2 = &qentry->msg.delete_rkey_v2; llcv2->num_inval_rkeys = 0; max = min_t(u8, llcv2->num_rkeys, SMC_LLC_RKEYS_PER_MSG_V2); + max = min_t(u32, max, SMC_LLC_DEL_RKEY_V2_INLINE + + qentry->body_len / sizeof(__be32)); for (i = 0; i < max; i++) { - if (smc_rtoken_delete(link, llcv2->rkey[i])) + __be32 rkey; + + if (i < SMC_LLC_DEL_RKEY_V2_INLINE) + rkey = llcv2->rkey[i]; + else + memcpy(&rkey, qentry->body + + (i - SMC_LLC_DEL_RKEY_V2_INLINE) * + sizeof(rkey), sizeof(rkey)); + if (smc_rtoken_delete(link, rkey)) llcv2->num_inval_rkeys++; } memset(&llc->rkey[0], 0, sizeof(llc->rkey)); @@ -1911,6 +1946,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_REQ_ADD_LINK) { /* server started add_link processing */ + /* free any qentry stashed in REQ_ADD_LINK state */ + smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); lgr->llc_flow_lcl.type = SMC_LLC_FLOW_ADD_LINK; smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); @@ -1927,7 +1964,8 @@ static void smc_llc_event_handler(struct smc_llc_qentry *qentry) return; case SMC_LLC_CONFIRM_LINK: case SMC_LLC_ADD_LINK_CONT: - if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { + if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE && + !lgr->llc_flow_lcl.qentry) { /* a flow is waiting for this message */ smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); wake_up(&lgr->llc_msg_waiter); @@ -2063,18 +2101,52 @@ static void smc_llc_rx_response(struct smc_link *link, wake_up(&link->lgr->llc_msg_waiter); } -static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) +/* the longest tail either reader of qentry->body can use */ +static u32 smc_llc_max_body_len(union smc_llc_msg *llc) +{ + switch (llc->raw.hdr.common.llc_type) { + case SMC_LLC_ADD_LINK: + return offsetof(struct smc_llc_msg_add_link_v2_ext, rt) + + SMC_LLC_RKEYS_PER_MSG_V2 * + sizeof(struct smc_llc_msg_add_link_cont_rt); + case SMC_LLC_DELETE_RKEY: + return (SMC_LLC_RKEYS_PER_MSG_V2 - + SMC_LLC_DEL_RKEY_V2_INLINE) * sizeof(__be32); + default: + return 0; + } +} + +static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc, + u32 byte_len) { struct smc_link_group *lgr = link->lgr; struct smc_llc_qentry *qentry; unsigned long flags; + u16 body_len = 0; + + /* V2 messages can be longer than the inline union smc_llc_msg. Carry + * the remainder in the qentry itself, so that its lifetime and its + * length match the message the peer actually sent. + */ + if (lgr->smc_version == SMC_V2 && byte_len > SMC_WR_TX_SIZE) + body_len = min_t(u32, byte_len, SMC_WR_BUF_V2_SIZE) - + SMC_WR_TX_SIZE; + body_len = min_t(u32, body_len, smc_llc_max_body_len(llc)); - qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); + qentry = kmalloc_flex(*qentry, body, body_len, GFP_ATOMIC); if (!qentry) return; + qentry->body_len = body_len; qentry->link = link; INIT_LIST_HEAD(&qentry->list); memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); + if (body_len) { + u8 *src = smc_link_shared_v2_rxbuf(link) ? + (u8 *)lgr->wr_rx_buf_v2 : (u8 *)llc; + + memcpy(qentry->body, src + SMC_WR_TX_SIZE, body_len); + } /* process responses immediately */ if ((llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) && @@ -2106,7 +2178,7 @@ static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) return; /* invalid message */ } - smc_llc_enqueue(link, llc); + smc_llc_enqueue(link, llc, wc->byte_len); } /***************************** worker, utils *********************************/ diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c index 4a3d7b405132e..24524dfd55180 100644 --- a/net/smc/smc_rx.c +++ b/net/smc/smc_rx.c @@ -115,16 +115,15 @@ static void smc_rx_pipe_buf_release(struct pipe_inode_info *pipe, struct pipe_buffer *buf) { struct smc_spd_priv *priv = (struct smc_spd_priv *)buf->private; + struct smc_connection *conn = &priv->smc->conn; struct smc_sock *smc = priv->smc; - struct smc_connection *conn; struct sock *sk = &smc->sk; - if (sk->sk_state == SMC_CLOSED || - sk->sk_state == SMC_PEERFINCLOSEWAIT || - sk->sk_state == SMC_APPFINCLOSEWAIT) - goto out; - conn = &smc->conn; lock_sock(sk); + if (conn->freed) { + release_sock(sk); + goto out; + } smc_rx_update_cons(smc, priv->len); release_sock(sk); if (atomic_sub_and_test(priv->len, &conn->splice_pending)) @@ -150,7 +149,12 @@ static const struct pipe_buf_operations smc_pipe_ops = { static void smc_rx_spd_release(struct splice_pipe_desc *spd, unsigned int i) { + struct smc_spd_priv *priv = (struct smc_spd_priv *)spd->partial[i].private; + struct sock *sk = &priv->smc->sk; + + kfree(priv); put_page(spd->pages[i]); + sock_put(sk); } static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, @@ -209,6 +213,10 @@ static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, offset = 0; } } + for (i = 0; i < nr_pages; i++) { + get_page(pages[i]); + sock_hold(&smc->sk); + } spd.nr_pages_max = nr_pages; spd.nr_pages = nr_pages; spd.pages = pages; @@ -217,16 +225,8 @@ static int smc_rx_splice(struct pipe_inode_info *pipe, char *src, size_t len, spd.spd_release = smc_rx_spd_release; bytes = splice_to_pipe(pipe, &spd); - if (bytes > 0) { - sock_hold(&smc->sk); - if (!lgr->is_smcd && smc->conn.rmb_desc->is_vm) { - for (i = 0; i < PAGE_ALIGN(bytes + offset) / PAGE_SIZE; i++) - get_page(pages[i]); - } else { - get_page(smc->conn.rmb_desc->pages); - } + if (bytes > 0) atomic_add(bytes, &smc->conn.splice_pending); - } kfree(priv); kfree(partial); kfree(pages); diff --git a/net/smc/smc_tx.h b/net/smc/smc_tx.h index a59f370b8b432..610a945aefd69 100644 --- a/net/smc/smc_tx.h +++ b/net/smc/smc_tx.h @@ -20,11 +20,15 @@ static inline int smc_tx_prepared_sends(struct smc_connection *conn) { + struct smc_buf_desc *sndbuf_desc = READ_ONCE(conn->sndbuf_desc); union smc_host_cursor sent, prep; + if (!sndbuf_desc) + return 0; + smc_curs_copy(&sent, &conn->tx_curs_sent, conn); smc_curs_copy(&prep, &conn->tx_curs_prep, conn); - return smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep); + return smc_curs_diff(sndbuf_desc->len, &sent, &prep); } void smc_tx_pending(struct smc_connection *conn); diff --git a/net/smc/smc_wr.c b/net/smc/smc_wr.c index b04a21b8c5111..4624817137be8 100644 --- a/net/smc/smc_wr.c +++ b/net/smc/smc_wr.c @@ -602,9 +602,9 @@ static void smc_wr_init_sge(struct smc_link *lnk) /* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE. * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer - * and the same buffer for all sges. When a larger message arrived then - * the content of the first small sge is copied to the beginning of - * the larger spillover buffer, allowing easy data mapping. + * and the same buffer for all sges. The spillover sge starts at + * SMC_WR_TX_SIZE, so the leading bytes of that buffer are never + * written. */ for (i = 0; i < lnk->wr_rx_cnt; i++) { int x = i * lnk->wr_rx_sge_cnt; diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index bb3c3db2713b1..6e40544fe2c09 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -2074,7 +2074,11 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, goto unwrap_failed; opaque_len = be32_to_cpup(p++); offset = (u8 *)(p) - (u8 *)head->iov_base; - if (offset + opaque_len > rcv_buf->len) + if (offset > rcv_buf->len) + goto unwrap_failed; + if (opaque_len > rcv_buf->len - offset) + goto unwrap_failed; + if (opaque_len <= GSS_KRB5_TOK_HDR_LEN) goto unwrap_failed; maj_stat = gss_unwrap(ctx->gc_gss_ctx, offset, diff --git a/net/sunrpc/auth_gss/gss_krb5_wrap.c b/net/sunrpc/auth_gss/gss_krb5_wrap.c index b3e1738ff6bfa..71b406d00472b 100644 --- a/net/sunrpc/auth_gss/gss_krb5_wrap.c +++ b/net/sunrpc/auth_gss/gss_krb5_wrap.c @@ -74,6 +74,8 @@ static void _rotate_left(struct xdr_buf *buf, unsigned int shift) int shifted = 0; int this_shift; + if (!buf->len) + return; shift %= buf->len; while (shifted < shift) { this_shift = min(shift - shifted, LOCAL_BUF_LEN); @@ -86,6 +88,8 @@ static void rotate_left(u32 base, struct xdr_buf *buf, unsigned int shift) { struct xdr_buf subbuf; + if (buf->len <= base) + return; xdr_buf_subsegment(buf, &subbuf, base, buf->len - base); _rotate_left(&subbuf, shift); } @@ -155,6 +159,9 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, dprintk("RPC: %s\n", __func__); + if (len - offset <= GSS_KRB5_TOK_HDR_LEN) + return GSS_S_DEFECTIVE_TOKEN; + ptr = buf->head[0].iov_base + offset; if (be16_to_cpu(*((__be16 *)ptr)) != KG2_TOK_WRAP) @@ -221,14 +228,16 @@ gss_krb5_unwrap_v2(struct krb5_ctx *kctx, int offset, int len, * head buffer space rather than that actually occupied. */ movelen = min_t(unsigned int, buf->head[0].iov_len, len); + if (movelen < offset + GSS_KRB5_TOK_HDR_LEN + headskip) + return GSS_S_DEFECTIVE_TOKEN; movelen -= offset + GSS_KRB5_TOK_HDR_LEN + headskip; - BUG_ON(offset + GSS_KRB5_TOK_HDR_LEN + headskip + movelen > - buf->head[0].iov_len); memmove(ptr, ptr + GSS_KRB5_TOK_HDR_LEN + headskip, movelen); buf->head[0].iov_len -= GSS_KRB5_TOK_HDR_LEN + headskip; buf->len = len - (GSS_KRB5_TOK_HDR_LEN + headskip); /* Trim off the trailing "extra count" and checksum blob */ + if (ec + GSS_KRB5_TOK_HDR_LEN + tailskip > buf->len - offset) + return GSS_S_DEFECTIVE_TOKEN; xdr_buf_trim(buf, ec + GSS_KRB5_TOK_HDR_LEN + tailskip); *align = XDR_QUADLEN(GSS_KRB5_TOK_HDR_LEN + headskip); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c index f549e4c05defc..711857ac688ae 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.c +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c @@ -121,12 +121,6 @@ static int gssp_rpc_create(struct net *net, struct rpc_clnt **_clnt) return result; } -void init_gssp_clnt(struct sunrpc_net *sn) -{ - mutex_init(&sn->gssp_lock); - sn->gssp_clnt = NULL; -} - int set_gssp_clnt(struct net *net) { struct sunrpc_net *sn = net_generic(net, sunrpc_net_id); diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.h b/net/sunrpc/auth_gss/gss_rpc_upcall.h index 31e96344167e3..b3c2b2b907985 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.h +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.h @@ -29,7 +29,6 @@ int gssp_accept_sec_context_upcall(struct net *net, struct gssp_upcall_data *data); void gssp_free_upcall_data(struct gssp_upcall_data *data); -void init_gssp_clnt(struct sunrpc_net *); int set_gssp_clnt(struct net *); void clear_gssp_clnt(struct sunrpc_net *); diff --git a/net/sunrpc/auth_gss/gss_rpc_xdr.c b/net/sunrpc/auth_gss/gss_rpc_xdr.c index f320c0a8e6049..752c4e8ebd44a 100644 --- a/net/sunrpc/auth_gss/gss_rpc_xdr.c +++ b/net/sunrpc/auth_gss/gss_rpc_xdr.c @@ -222,7 +222,8 @@ static int gssx_dec_linux_creds(struct xdr_stream *xdr, return 0; out_free_groups: - groups_free(creds->cr_group_info); + put_group_info(creds->cr_group_info); + creds->cr_group_info = NULL; return err; } @@ -230,6 +231,7 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, struct gssx_option_array *oa) { struct svc_cred *creds; + bool creds_decoded = false; u32 count, i; __be32 *p; int err; @@ -242,12 +244,12 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; /* we recognize only 1 currently: CREDS_VALUE */ - oa->count = 1; - oa->data = kmalloc(sizeof(struct gssx_option), GFP_KERNEL); if (!oa->data) return -ENOMEM; + oa->count = 1; + creds = kzalloc(sizeof(struct svc_cred), GFP_KERNEL); if (!creds) { err = -ENOMEM; @@ -280,9 +282,14 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, if (length == sizeof(CREDS_VALUE) && memcmp(p, CREDS_VALUE, sizeof(CREDS_VALUE)) == 0) { /* We have creds here. parse them */ + if (creds_decoded) { + err = -EINVAL; + goto free_creds; + } err = gssx_dec_linux_creds(xdr, creds); if (err) goto free_creds; + creds_decoded = true; oa->data[0].value.len = 1; /* presence */ } else { /* consume uninteresting buffer */ @@ -294,8 +301,10 @@ static int gssx_dec_option_array(struct xdr_stream *xdr, return 0; free_creds: + free_svc_cred(creds); kfree(creds); free_oa: + oa->count = 0; kfree(oa->data); oa->data = NULL; return err; diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index e2f0df8cdaa6a..f7f885ab0a4b9 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -949,6 +949,8 @@ svcauth_gss_unwrap_priv(struct svc_rqst *rqstp, u32 seq, struct gss_ctx *ctx) } if (len > xdr_stream_remaining(xdr)) goto unwrap_failed; + if (len <= GSS_KRB5_TOK_HDR_LEN) + goto unwrap_failed; offset = xdr_stream_pos(xdr); saved_len = buf->len; @@ -1466,7 +1468,6 @@ static int create_use_gss_proxy_proc_entry(struct net *net) &use_gss_proxy_proc_ops, net); if (!*p) return -ENOMEM; - init_gssp_clnt(sn); return 0; } @@ -1573,6 +1574,9 @@ svcauth_gss_decode_credbody(struct xdr_stream *xdr, u32 body_len; __be32 *p; + /* Early-return paths leave deterministic state, not stale residue. */ + memset(gc, 0, sizeof(*gc)); + p = xdr_inline_decode(xdr, XDR_UNIT); if (!p) return false; @@ -1942,6 +1946,8 @@ svcauth_gss_release(struct svc_rqst *rqstp) if (!gsd) goto out; + if (rqstp->rq_auth_stat != rpc_auth_ok) + goto out; gc = &gsd->clcred; if (gc->gc_proc != RPC_GSS_PROC_DATA) goto out; diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c index 6f6e0d4928afd..fda19596566fe 100644 --- a/net/sunrpc/cache.c +++ b/net/sunrpc/cache.c @@ -423,10 +423,9 @@ void sunrpc_destroy_cache_detail(struct cache_detail *cd) list_del_init(&cd->others); spin_unlock(&cd->hash_lock); spin_unlock(&cache_list_lock); - if (list_empty(&cache_list)) { - /* module must be being unloaded so its safe to kill the worker */ - cancel_delayed_work_sync(&cache_cleaner); - } + cancel_delayed_work_sync(&cache_cleaner); + if (!list_empty(&cache_list)) + queue_delayed_work(system_power_efficient_wq, &cache_cleaner, 0); } EXPORT_SYMBOL_GPL(sunrpc_destroy_cache_detail); diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index a142e20ce7e7c..07de0efaf37db 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -96,7 +96,10 @@ static void rpc_unregister_client(struct rpc_clnt *clnt) static void __rpc_clnt_remove_pipedir(struct rpc_clnt *clnt) { - rpc_remove_client_dir(clnt); + if (clnt->pipefs_sb) { + rpc_remove_client_dir(clnt); + clnt->pipefs_sb = NULL; + } } static void rpc_clnt_remove_pipedir(struct rpc_clnt *clnt) @@ -177,19 +180,28 @@ static int rpc_clnt_skip_event(struct rpc_clnt *clnt, unsigned long event) } static int __rpc_clnt_handle_event(struct rpc_clnt *clnt, unsigned long event, - struct super_block *sb) + struct super_block *sb) { + int err = 0; + switch (event) { case RPC_PIPEFS_MOUNT: - return rpc_setup_pipedir_sb(sb, clnt); + clnt->pipefs_sb = sb; + err = rpc_setup_pipedir_sb(sb, clnt); + if (err) + clnt->pipefs_sb = NULL; + break; case RPC_PIPEFS_UMOUNT: - __rpc_clnt_remove_pipedir(clnt); + if (clnt->pipefs_sb == sb) { + __rpc_clnt_remove_pipedir(clnt); + clnt->pipefs_sb = NULL; + } break; default: printk(KERN_ERR "%s: unknown event: %ld\n", __func__, event); return -ENOTSUPP; } - return 0; + return err; } static int __rpc_pipefs_event(struct rpc_clnt *clnt, unsigned long event, diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 53bcca365fb1c..2d74c660de666 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -490,6 +490,8 @@ static int rpcb_register_inet4(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { @@ -516,6 +518,8 @@ static int rpcb_register_inet6(struct sunrpc_net *sn, int result; map->r_addr = rpc_sockaddr2uaddr(sap, GFP_KERNEL); + if (!map->r_addr) + return -ENOMEM; msg->rpc_proc = &rpcb_procedures4[RPCBPROC_UNSET]; if (port != 0) { diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index bab6cab294052..04109278fda64 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -54,6 +54,7 @@ static __net_init int sunrpc_init_net(struct net *net) INIT_LIST_HEAD(&sn->all_clients); spin_lock_init(&sn->rpc_client_lock); spin_lock_init(&sn->rpcb_clnt_lock); + mutex_init(&sn->gssp_lock); return 0; err_pipefs: diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 1f4bf5b290e4a..1c7410488236e 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -352,7 +352,7 @@ static int svc_pool_map_get_node(unsigned int pidx) if (m->mode == SVC_POOL_PERNODE) return m->pool_to[pidx]; } - return numa_mem_id(); + return NUMA_NO_NODE; } /* * Set the given thread's cpus_allowed mask so that it @@ -402,6 +402,7 @@ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) struct svc_pool_map *m = &svc_pool_map; int cpu = raw_smp_processor_id(); unsigned int pidx = 0; + unsigned int i; if (serv->sv_nrpools <= 1) return serv->sv_pools; @@ -414,8 +415,34 @@ struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv) pidx = m->to_pool[cpu_to_node(cpu)]; break; } + pidx %= serv->sv_nrpools; + + /* + * It's possible to have a pool with no threads. Userland can just set + * things up this way directly. Also, when threads are autodistributed + * they are spread evenly across the pools, but when there are fewer + * threads than pools some pools can end up with none. + * + * A transport enqueued on a threadless pool would never be picked up, + * since each thread only services its own pool. Fall back to the next + * populated pool, trading NUMA locality for a guarantee that the + * transport is serviced. + */ + for (i = 0; i < serv->sv_nrpools; i++) { + struct svc_pool *pool = &serv->sv_pools[pidx]; + + /* This is set under the service mutex and rarely ever + * changes. A data race here is harmless. + */ + if (data_race(pool->sp_nrthreads)) + return pool; + + if (++pidx >= serv->sv_nrpools) + pidx = 0; + } - return &serv->sv_pools[pidx % serv->sv_nrpools]; + /* No pool has any threads; nothing can service the transport. */ + return &serv->sv_pools[pidx]; } static int svc_rpcb_setup(struct svc_serv *serv, struct net *net) @@ -476,6 +503,35 @@ __svc_init_bc(struct svc_serv *serv) } #endif +static int svc_pool_init_counters(struct svc_pool *pool) +{ + int err; + + err = percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); + if (err) + return err; + err = percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); + if (err) + goto err_sockets; + err = percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (err) + goto err_threads; + return 0; + +err_threads: + percpu_counter_destroy(&pool->sp_sockets_queued); +err_sockets: + percpu_counter_destroy(&pool->sp_messages_arrived); + return err; +} + +static void svc_pool_destroy_counters(struct svc_pool *pool) +{ + percpu_counter_destroy(&pool->sp_messages_arrived); + percpu_counter_destroy(&pool->sp_sockets_queued); + percpu_counter_destroy(&pool->sp_threads_woken); +} + /* * Create an RPC service */ @@ -541,12 +597,18 @@ __svc_create(struct svc_program *prog, int nprogs, struct svc_stat *stats, INIT_LIST_HEAD(&pool->sp_all_threads); init_llist_head(&pool->sp_idle_threads); - percpu_counter_init(&pool->sp_messages_arrived, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_sockets_queued, 0, GFP_KERNEL); - percpu_counter_init(&pool->sp_threads_woken, 0, GFP_KERNEL); + if (svc_pool_init_counters(pool)) + goto out_err; } return serv; + +out_err: + while (i--) + svc_pool_destroy_counters(&serv->sv_pools[i]); + kfree(serv->sv_pools); + kfree(serv); + return NULL; } /** @@ -625,9 +687,7 @@ svc_destroy(struct svc_serv **servp) for (i = 0; i < serv->sv_nrpools; i++) { struct svc_pool *pool = &serv->sv_pools[i]; - percpu_counter_destroy(&pool->sp_messages_arrived); - percpu_counter_destroy(&pool->sp_sockets_queued); - percpu_counter_destroy(&pool->sp_threads_woken); + svc_pool_destroy_counters(pool); } kfree(serv->sv_pools); kfree(serv); @@ -663,6 +723,15 @@ svc_release_buffer(struct svc_rqst *rqstp) kfree(rqstp->rq_pages); } +static void svc_rqst_free_rcu(struct rcu_head *head) +{ + struct svc_rqst *rqstp = container_of(head, struct svc_rqst, rq_rcu_head); + + kfree(rqstp->rq_resp); + kfree(rqstp->rq_argp); + kfree(rqstp); +} + static void svc_rqst_free(struct svc_rqst *rqstp) { @@ -671,10 +740,8 @@ svc_rqst_free(struct svc_rqst *rqstp) svc_release_buffer(rqstp); if (rqstp->rq_scratch_folio) folio_put(rqstp->rq_scratch_folio); - kfree(rqstp->rq_resp); - kfree(rqstp->rq_argp); kfree(rqstp->rq_auth_data); - kfree_rcu(rqstp, rq_rcu_head); + call_rcu(&rqstp->rq_rcu_head, svc_rqst_free_rcu); } static struct svc_rqst * @@ -691,7 +758,9 @@ svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node) rqstp->rq_server = serv; rqstp->rq_pool = pool; - rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, node); + rqstp->rq_scratch_folio = __folio_alloc_node(GFP_KERNEL, 0, + node == NUMA_NO_NODE ? + numa_mem_id() : node); if (!rqstp->rq_scratch_folio) goto out_enomem; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 9c14558bc552b..0ed8d56374714 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -68,6 +68,17 @@ #define RPCDBG_FACILITY RPCDBG_SVCXPRT +/* + * For UDP: + * 1 for header page + * enough pages for RPCSVC_MAXPAYLOAD_UDP + * 1 in case payload is not aligned + * 1 for tail page + */ +enum { + SUNRPC_MAX_UDP_SENDPAGES = 1 + RPCSVC_MAXPAYLOAD_UDP / PAGE_SIZE + 1 + 1 +}; + /* To-do: to avoid tying up an nfsd thread while waiting for a * handshake request, the request could instead be deferred. */ @@ -737,7 +748,7 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) .msg_flags = MSG_SPLICE_PAGES, .msg_controllen = sizeof(buffer), }; - unsigned int count; + int count; int err; svc_udp_release_ctxt(xprt, rqstp->rq_xprt_ctxt); @@ -750,18 +761,23 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) if (svc_xprt_is_dead(xprt)) goto out_notconn; - count = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, xdr); + count = xdr_buf_to_bvec(svsk->sk_bvec, SUNRPC_MAX_UDP_SENDPAGES, xdr); + if (count < 0) { + err = count; + goto out_trace; + } - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, count, rqstp->rq_res.len); err = sock_sendmsg(svsk->sk_sock, &msg); if (err == -ECONNREFUSED) { /* ICMP error on earlier request. */ - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, count, rqstp->rq_res.len); err = sock_sendmsg(svsk->sk_sock, &msg); } +out_trace: trace_svcsock_udp_send(xprt, err); mutex_unlock(&xprt->xpt_mutex); @@ -1240,26 +1256,31 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp, struct msghdr msg = { .msg_flags = MSG_SPLICE_PAGES, }; - unsigned int count; + int count; void *buf; int ret; /* The stream record marker is copied into a temporary page - * fragment buffer so that it can be included in rq_bvec. + * fragment buffer so that it can be included in sk_bvec. */ buf = page_frag_alloc(&svsk->sk_frag_cache, sizeof(marker), GFP_KERNEL); if (!buf) return -ENOMEM; memcpy(buf, &marker, sizeof(marker)); - bvec_set_virt(rqstp->rq_bvec, buf, sizeof(marker)); + bvec_set_virt(svsk->sk_bvec, buf, sizeof(marker)); - count = xdr_buf_to_bvec(rqstp->rq_bvec + 1, rqstp->rq_maxpages, + count = xdr_buf_to_bvec(svsk->sk_bvec + 1, rqstp->rq_maxpages, &rqstp->rq_res); + if (count < 0) { + ret = count; + goto out; + } - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, 1 + count, sizeof(marker) + rqstp->rq_res.len); ret = sock_sendmsg(svsk->sk_sock, &msg); +out: page_frag_free(buf); return ret; } @@ -1402,6 +1423,20 @@ void svc_sock_update_bufs(struct svc_serv *serv) spin_unlock_bh(&serv->sv_lock); } +static int svc_sock_sendpages(struct svc_serv *serv, struct socket *sock, int flags) +{ + switch (sock->type) { + case SOCK_STREAM: + /* +1 for TCP record marker */ + if (flags & SVC_SOCK_TEMPORARY) + return svc_serv_maxpages(serv) + 1; + return 0; + case SOCK_DGRAM: + return SUNRPC_MAX_UDP_SENDPAGES; + } + return -EINVAL; +} + /* * Initialize socket for RPC use and create svc_sock struct */ @@ -1412,12 +1447,26 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, struct svc_sock *svsk; struct sock *inet; int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); + int sendpages; unsigned long pages; + sendpages = svc_sock_sendpages(serv, sock, flags); + if (sendpages < 0) + return ERR_PTR(sendpages); + pages = svc_serv_maxpages(serv); svsk = kzalloc(struct_size(svsk, sk_pages, pages), GFP_KERNEL); if (!svsk) return ERR_PTR(-ENOMEM); + + if (sendpages) { + svsk->sk_bvec = kcalloc(sendpages, sizeof(*svsk->sk_bvec), GFP_KERNEL); + if (!svsk->sk_bvec) { + kfree(svsk); + return ERR_PTR(-ENOMEM); + } + } + svsk->sk_maxpages = pages; inet = sock->sk; @@ -1429,6 +1478,7 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, inet->sk_protocol, ntohs(inet_sk(inet)->inet_sport)); if (err < 0) { + kfree(svsk->sk_bvec); kfree(svsk); return ERR_PTR(err); } @@ -1646,5 +1696,6 @@ static void svc_sock_free(struct svc_xprt *xprt) sock_release(sock); page_frag_cache_drain(&svsk->sk_frag_cache); + kfree(svsk->sk_bvec); kfree(svsk); } diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index fe17a6e8372b4..d3cd543ff80ad 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -139,13 +139,14 @@ xdr_free_bvec(struct xdr_buf *buf) /** * xdr_buf_to_bvec - Copy components of an xdr_buf into a bio_vec array * @bvec: bio_vec array to populate - * @bvec_size: element count of @bio_vec + * @bvec_size: element count of @bvec * @xdr: xdr_buf to be copied * - * Returns the number of entries consumed in @bvec. + * Returns the number of entries consumed in @bvec on success, or + * -ESERVERFAULT when @xdr does not fit within @bvec_size entries. */ -unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, - const struct xdr_buf *xdr) +int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, + const struct xdr_buf *xdr) { const struct kvec *head = xdr->head; const struct kvec *tail = xdr->tail; @@ -187,10 +188,209 @@ unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, bvec_overflow: pr_warn_once("%s: bio_vec array overflow\n", __func__); - return count; + return -ESERVERFAULT; } EXPORT_SYMBOL_GPL(xdr_buf_to_bvec); +/** + * xdr_buf_to_sg - Populate a scatterlist from an xdr_buf range + * @buf: xdr_buf to map + * @offset: starting byte offset within @buf + * @len: number of bytes to cover + * @sg: scatterlist array initialized with sg_init_table() + * @nsg: number of entries available in @sg + * + * @sg is traversed with sg_next(), so callers may pass a list + * assembled with sg_chain(). + * + * Return: on success, the number of scatterlist entries used; the + * last used entry is marked with sg_mark_end(). On failure, a + * negative errno. + */ +int xdr_buf_to_sg(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg, unsigned int nsg) +{ + unsigned int page_len, thislen, page_offset; + struct scatterlist *cur = sg, *prev = NULL; + int nents = 0; + int i; + + if (len == 0) + return 0; + + if (offset >= buf->head[0].iov_len) { + offset -= buf->head[0].iov_len; + } else { + thislen = min_t(unsigned int, + buf->head[0].iov_len - offset, len); + if (nents >= nsg) + return -ENOSPC; + sg_set_buf(cur, buf->head[0].iov_base + offset, + thislen); + prev = cur; + cur = sg_next(cur); + nents++; + len -= thislen; + offset = 0; + } + if (len == 0) + goto done; + + if (offset >= buf->page_len) { + offset -= buf->page_len; + } else { + page_len = min(buf->page_len - offset, len); + len -= page_len; + page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1); + i = (offset + buf->page_base) >> PAGE_SHIFT; + thislen = PAGE_SIZE - page_offset; + do { + if (thislen > page_len) + thislen = page_len; + if (nents >= nsg) + return -ENOSPC; + sg_set_page(cur, buf->pages[i], + thislen, page_offset); + prev = cur; + cur = sg_next(cur); + nents++; + page_len -= thislen; + i++; + page_offset = 0; + thislen = PAGE_SIZE; + } while (page_len != 0); + offset = 0; + } + if (len == 0) + goto done; + + if (offset < buf->tail[0].iov_len) { + thislen = min_t(unsigned int, + buf->tail[0].iov_len - offset, len); + if (nents >= nsg) + return -ENOSPC; + sg_set_buf(cur, buf->tail[0].iov_base + offset, + thislen); + prev = cur; + nents++; + len -= thislen; + } + if (len != 0) + return -EINVAL; + +done: + if (prev) + sg_mark_end(prev); + return nents; +} +EXPORT_SYMBOL_GPL(xdr_buf_to_sg); + +/* + * Count the scatterlist entries needed to cover [offset, offset + len) + * within @buf. Mirrors the walk in xdr_buf_to_sg() so the caller can + * size an allocation that matches the requested sub-range rather than + * the full xdr_buf. + */ +static unsigned int xdr_buf_sg_nents(const struct xdr_buf *buf, + unsigned int offset, unsigned int len) +{ + unsigned int nsg = 0, thislen, page_offset; + + if (len == 0) + return 0; + + if (offset < buf->head[0].iov_len) { + thislen = min_t(unsigned int, + buf->head[0].iov_len - offset, len); + nsg++; + len -= thislen; + offset = 0; + } else { + offset -= buf->head[0].iov_len; + } + if (len == 0) + return nsg; + + if (offset < buf->page_len) { + thislen = min(buf->page_len - offset, len); + page_offset = (offset + buf->page_base) & (PAGE_SIZE - 1); + nsg += DIV_ROUND_UP(page_offset + thislen, PAGE_SIZE); + len -= thislen; + offset = 0; + } else { + offset -= buf->page_len; + } + if (len == 0) + return nsg; + + if (offset < buf->tail[0].iov_len) + nsg++; + return nsg; +} + +/** + * xdr_buf_to_sg_alloc - Populate a scatterlist for an xdr_buf range + * @buf: xdr_buf to map + * @offset: starting byte offset within @buf + * @len: number of bytes to cover + * @sg_head: caller-provided scatterlist array (typically stack-allocated) + * @sg_head_nents: number of entries in @sg_head + * @sg_overflow: OUT: chained extension, or NULL when @sg_head sufficed + * @gfp: memory allocation flags for overflow + * + * Populates @sg_head directly when the xdr_buf fits. When more + * entries are needed, an overflow scatterlist is allocated and + * chained from @sg_head so that the result is traversable with + * sg_next(). + * + * Return: on success, the number of populated scatterlist entries + * (counting only data entries, not chain entries). @sg_head is + * the head of the resulting list. Caller must kfree @sg_overflow + * when done. On failure, a negative errno. + */ +int xdr_buf_to_sg_alloc(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg_head, + unsigned int sg_head_nents, + struct scatterlist **sg_overflow, gfp_t gfp) +{ + unsigned int nsg; + int ret; + + *sg_overflow = NULL; + if (len == 0) + return 0; + + nsg = xdr_buf_sg_nents(buf, offset, len); + if (nsg == 0) + return -EINVAL; + + if (nsg <= sg_head_nents) { + sg_init_table(sg_head, nsg); + } else { + /* +1 replaces the slot sg_chain() consumes as the link. */ + unsigned int overflow_nents = nsg - sg_head_nents + 1; + struct scatterlist *overflow; + + overflow = kmalloc_array(overflow_nents, sizeof(*overflow), + gfp); + if (!overflow) + return -ENOMEM; + + sg_init_table(sg_head, sg_head_nents); + sg_init_table(overflow, overflow_nents); + sg_chain(sg_head, sg_head_nents, overflow); + *sg_overflow = overflow; + } + + ret = xdr_buf_to_sg(buf, offset, len, sg_head, nsg); + if (ret < 0) { + kfree(*sg_overflow); + *sg_overflow = NULL; + } + return ret; +} +EXPORT_SYMBOL_GPL(xdr_buf_to_sg_alloc); + /** * xdr_inline_pages - Prepare receive buffer for a large reply * @xdr: xdr_buf into which reply will be placed @@ -1849,7 +2049,7 @@ void xdr_buf_trim(struct xdr_buf *buf, unsigned int len) trim -= cur; } fix_len: - buf->len -= (len - trim); + buf->len -= min_t(unsigned int, buf->len, len - trim); } EXPORT_SYMBOL_GPL(xdr_buf_trim); diff --git a/net/sunrpc/xprtrdma/ib_client.c b/net/sunrpc/xprtrdma/ib_client.c index 28c68b5f68238..888638318b6dd 100644 --- a/net/sunrpc/xprtrdma/ib_client.c +++ b/net/sunrpc/xprtrdma/ib_client.c @@ -51,7 +51,11 @@ static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device) * to be invoked when the device is removed, unless this notification * is unregistered first. * - * On failure, a negative errno is returned. + * On failure, a negative errno is returned. rn->rn_done is left + * NULL on every failure path (it is armed before xa_alloc but + * cleared again if xa_alloc fails), so the @rn may safely be + * passed to rpcrdma_rn_unregister() without a separate + * registered/unregistered flag in the caller. */ int rpcrdma_rn_register(struct ib_device *device, struct rpcrdma_notification *rn, @@ -62,10 +66,21 @@ int rpcrdma_rn_register(struct ib_device *device, if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags)) return -ENETUNREACH; - if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) + /* + * Arm rn_done before xa_alloc() publishes @rn: once @rn is + * visible in rd_xa, a concurrent rpcrdma_remove_one() can + * call rn->rn_done(), so the pointer must already be set. + * + * Restore NULL if xa_alloc() fails. rn_done doubles as the + * registration sentinel for rpcrdma_rn_unregister(); a stale + * value would unregister an @rn that was never inserted. + */ + rn->rn_done = done; + if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) { + rn->rn_done = NULL; return -ENOMEM; + } kref_get(&rd->rd_kref); - rn->rn_done = done; trace_rpcrdma_client_register(device, rn); return 0; } @@ -83,6 +98,10 @@ static void rpcrdma_rn_release(struct kref *kref) * rpcrdma_rn_unregister - stop device removal notifications * @device: monitored device * @rn: notification object that no longer wishes to be notified + * + * It is safe to call this on an @rn whose registration never + * completed or failed; rn_done == NULL is treated as + * never-registered and the call is a no-op. */ void rpcrdma_rn_unregister(struct ib_device *device, struct rpcrdma_notification *rn) @@ -92,6 +111,21 @@ void rpcrdma_rn_unregister(struct ib_device *device, if (!rd) return; + /* + * rn_done is the registration sentinel: rpcrdma_rn_register + * leaves it NULL on every failure path, clearing it again if + * xa_alloc fails, so a non-NULL rn_done marks a completed + * registration. A NULL rn_done means this notification was + * never registered (or its registration failed) or has + * already been unregistered, and the call is a no-op. + * Without this guard, rn_index == 0 from a kzalloc'd + * parent would erase another caller's slot 0 and underflow + * rd_kref. + */ + if (!rn->rn_done) + return; + rn->rn_done = NULL; + trace_rpcrdma_client_unregister(device, rn); xa_erase(&rd->rd_xa, rn->rn_index); kref_put(&rd->rd_kref, rpcrdma_rn_release); diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 5d1f96bffd749..e2694fb03ba1b 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -542,6 +542,7 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) rpcrdma_sendctx_dma_unmap(sc); sc->sc_req = NULL; + req->rl_sendctx = NULL; rpcrdma_req_put(req); } @@ -550,8 +551,11 @@ void rpcrdma_sendctx_unmap(struct rpcrdma_sendctx *sc) */ static void rpcrdma_sendctx_cancel(struct rpcrdma_sendctx *sc) { + struct rpcrdma_req *req = sc->sc_req; + rpcrdma_sendctx_dma_unmap(sc); sc->sc_req = NULL; + req->rl_sendctx = NULL; } /* Prepare an SGE for the RPC-over-RDMA transport header. diff --git a/net/sunrpc/xprtrdma/svc_rdma_pcl.c b/net/sunrpc/xprtrdma/svc_rdma_pcl.c index b63cfeaa29234..a1c61311714da 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_pcl.c +++ b/net/sunrpc/xprtrdma/svc_rdma_pcl.c @@ -213,7 +213,6 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk = pcl_alloc_chunk(segcount, 0); if (!chunk) return false; - list_add_tail(&chunk->ch_list, &pcl->cl_chunks); for (j = 0; j < segcount; j++) { segment = &chunk->ch_segments[j]; @@ -225,6 +224,7 @@ bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, chunk->ch_length += segment->rs_length; chunk->ch_segcount++; } + list_add_tail(&chunk->ch_list, &pcl->cl_chunks); } return true; } diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index e7e4a39ca6c65..1075bfd9fed61 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -430,11 +430,14 @@ static void svc_rdma_build_arg_xdr(struct svc_rqst *rqstp, * to the first byte past the Read list. rc_read_pcl and * rc_call_pcl cl_count fields are set to the number of * Read segments in the list. - * %false: Read list is corrupt. @rctxt's xdr_stream is left in an - * unknown state. + * %false: Read list is corrupt or exceeds the page budget. @rctxt's + * xdr_stream is left in an unknown state. */ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) { + unsigned int maxlen = rctxt->rc_maxpages << PAGE_SHIFT; + unsigned int total_len = 0; + rctxt->rc_call_pcl.cl_count = 0; rctxt->rc_read_pcl.cl_count = 0; while (xdr_item_is_present(p)) { @@ -448,6 +451,11 @@ static bool xdr_count_read_segments(struct svc_rdma_recv_ctxt *rctxt, __be32 *p) xdr_decode_read_segment(p, &position, &handle, &length, &offset); + if (length > maxlen) + return false; + total_len += length; + if (PAGE_ALIGN(total_len) > maxlen) + return false; if (position) { if (position & 3) return false; @@ -498,10 +506,13 @@ static bool xdr_check_write_chunk(struct svc_rdma_recv_ctxt *rctxt) return false; /* Before trusting the segcount value enough to use it in - * a computation, perform a simple range check. This is an - * arbitrary but sensible limit (ie, not architectural). + * a computation, perform a simple range check. A zero + * segcount describes no remote buffer and is rejected so + * downstream consumers never see a degenerate ch_segcount==0 + * chunk. The upper bound is an arbitrary but sensible limit + * (ie, not architectural). */ - if (unlikely(segcount > rctxt->rc_maxpages)) + if (segcount == 0 || unlikely(segcount > rctxt->rc_maxpages)) return false; p = xdr_inline_decode(&rctxt->rc_stream, diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 310de7a80be52..626e79a317678 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -766,7 +766,7 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, len -= seg_len; if (len && ((head->rc_curpage + 1) > rqstp->rq_maxpages)) - goto out_overrun; + goto out_put; } ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset, @@ -779,7 +779,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp, cc->cc_sqecount += ret; return 0; -out_overrun: +out_put: + svc_rdma_put_rw_ctxt(rdma, ctxt); trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage); return -EINVAL; } @@ -817,7 +818,7 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp, * svc_rdma_copy_inline_range - Copy part of the inline content into pages * @rqstp: RPC transaction context * @head: context for ongoing I/O - * @offset: offset into the Receive buffer of region to copy + * @offset: offset into the inline content of region to copy * @remaining: length of region to copy * * Take a page at a time from rqstp->rq_pages and copy the inline @@ -834,9 +835,13 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp, unsigned int offset, unsigned int remaining) { - unsigned char *dst, *src = head->rc_recv_buf; + unsigned char *dst, *src = head->rc_saved_arg.head[0].iov_base; + unsigned int inline_len = head->rc_saved_arg.head[0].iov_len; unsigned int page_no, numpages; + if (offset > inline_len || remaining > inline_len - offset) + return -EINVAL; + numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT; for (page_no = 0; page_no < numpages; page_no++) { unsigned int page_len; @@ -887,9 +892,10 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, { const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int inline_len, start, length; int ret; + inline_len = head->rc_saved_arg.head[0].iov_len; start = 0; chunk = pcl_first_chunk(pcl); length = chunk->ch_position; @@ -907,6 +913,8 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, break; start += length; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_copy_inline_range(rqstp, head, start, length); if (ret < 0) @@ -914,7 +922,9 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, } start += length; - length = head->rc_byte_len - start; + if (start > inline_len) + return -EINVAL; + length = inline_len - start; return svc_rdma_copy_inline_range(rqstp, head, start, length); } @@ -939,8 +949,12 @@ svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp, static int svc_rdma_read_data_item(struct svc_rqst *rqstp, struct svc_rdma_recv_ctxt *head) { - return svc_rdma_build_read_chunk(rqstp, head, - pcl_first_chunk(&head->rc_read_pcl)); + struct svc_rdma_chunk *chunk = pcl_first_chunk(&head->rc_read_pcl); + + if (chunk->ch_position > head->rc_saved_arg.head[0].iov_len) + return -EINVAL; + + return svc_rdma_build_read_chunk(rqstp, head, chunk); } /** @@ -966,17 +980,20 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, const struct svc_rdma_segment *segment; int ret; + if (!length) + return 0; + ret = -EINVAL; pcl_for_each_segment(segment, chunk) { struct svc_rdma_segment dummy; - if (offset > segment->rs_length) { + if (offset >= segment->rs_length) { offset -= segment->rs_length; continue; } dummy.rs_handle = segment->rs_handle; - dummy.rs_length = min_t(u32, length, segment->rs_length) - offset; + dummy.rs_length = min_t(u32, length, segment->rs_length - offset); dummy.rs_offset = segment->rs_offset + offset; ret = svc_rdma_build_read_segment(rqstp, head, &dummy); @@ -985,6 +1002,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp, head->rc_readbytes += dummy.rs_length; length -= dummy.rs_length; + if (!length) + break; offset = 0; } return ret; @@ -1009,14 +1028,17 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, pcl_first_chunk(&head->rc_call_pcl); const struct svc_rdma_pcl *pcl = &head->rc_read_pcl; struct svc_rdma_chunk *chunk, *next; - unsigned int start, length; + unsigned int call_len, start, length; int ret; if (pcl_is_empty(pcl)) return svc_rdma_build_read_chunk(rqstp, head, call_chunk); + call_len = call_chunk->ch_length; start = 0; chunk = pcl_first_chunk(pcl); + if (chunk->ch_position > call_len) + return -EINVAL; length = chunk->ch_position; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1033,6 +1055,10 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, break; start += length; + if (next->ch_position > call_len) + return -EINVAL; + if (head->rc_readbytes > next->ch_position) + return -EINVAL; length = next->ch_position - head->rc_readbytes; ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); @@ -1041,7 +1067,9 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp, } start += length; - length = call_chunk->ch_length - start; + if (start > call_len) + return -EINVAL; + length = call_len - start; return svc_rdma_read_chunk_range(rqstp, head, call_chunk, start, length); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index 914cd263c2f17..082db5cea2134 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -700,20 +700,21 @@ static int svc_rdma_xb_count_sges(const struct xdr_buf *xdr, } /** - * svc_rdma_pull_up_needed - Determine whether to use pull-up + * svc_rdma_check_pull_up - Determine whether to use pull-up * @rdma: controlling transport * @sctxt: send_ctxt for the Send WR * @write_pcl: Write chunk list provided by client * @xdr: xdr_buf containing RPC message to transmit * * Returns: - * %true if pull-up must be used - * %false otherwise + * %1 if pull-up must be used + * %0 if pull-up is not needed + * %-E2BIG if the reply is too large to be pulled up */ -static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, - const struct svc_rdma_send_ctxt *sctxt, - const struct svc_rdma_pcl *write_pcl, - const struct xdr_buf *xdr) +static int svc_rdma_check_pull_up(const struct svcxprt_rdma *rdma, + const struct svc_rdma_send_ctxt *sctxt, + const struct svc_rdma_pcl *write_pcl, + const struct xdr_buf *xdr) { /* Resources needed for the transport header */ struct svc_rdma_pullup_data args = { @@ -725,11 +726,22 @@ static bool svc_rdma_pull_up_needed(const struct svcxprt_rdma *rdma, ret = pcl_process_nonpayloads(write_pcl, xdr, svc_rdma_xb_count_sges, &args); if (ret < 0) - return false; + return 0; if (args.pd_length < RPCRDMA_PULLUP_THRESH) - return true; - return args.pd_num_sges >= rdma->sc_max_send_sges; + return 1; + if (args.pd_num_sges < rdma->sc_max_send_sges) + return 0; + + /* + * The reply has too many SGEs to Send inline, so it has to be + * linearized into sc_xprt_buf. That buffer holds only + * sc_max_req_size bytes, so a larger reply cannot be pulled up. + * RFC 8166 Section 4.5.3 requires responding with ERR_CHUNK. + */ + if (args.pd_length > rdma->sc_max_req_size) + return -E2BIG; + return 1; } /** @@ -785,7 +797,7 @@ static int svc_rdma_xb_linearize(const struct xdr_buf *xdr, * Assemble the elements of @xdr into the transport header buffer. * * Assumptions: - * pull_up_needed has determined that @xdr will fit in the buffer. + * check_pull_up has determined that @xdr will fit in the buffer. * * Returns: * %0 if pull-up was successful @@ -820,6 +832,7 @@ static int svc_rdma_pull_up_reply_msg(const struct svcxprt_rdma *rdma, * * Returns: * %0 if DMA mapping was successful. + * %-E2BIG if the reply is too large to be pulled up * %-EMSGSIZE if a buffer manipulation problem occurred * %-EIO if DMA mapping failed * @@ -835,6 +848,7 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, .md_rdma = rdma, .md_ctxt = sctxt, }; + int ret; /* Set up the (persistently-mapped) transport header SGE. */ sctxt->sc_send_wr.num_sge = 1; @@ -849,7 +863,10 @@ int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma, /* For pull-up, svc_rdma_send() will sync the transport header. * No additional DMA mapping is necessary. */ - if (svc_rdma_pull_up_needed(rdma, sctxt, write_pcl, xdr)) + ret = svc_rdma_check_pull_up(rdma, sctxt, write_pcl, xdr); + if (ret < 0) + return ret; + if (ret) return svc_rdma_pull_up_reply_msg(rdma, sctxt, write_pcl, xdr); return pcl_process_nonpayloads(write_pcl, xdr, @@ -1033,7 +1050,7 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) &rctxt->rc_reply_pcl, sctxt, &rqstp->rq_res); if (ret < 0) - goto reply_chunk; + goto send_err; rc_size = ret; } @@ -1054,10 +1071,10 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) ret = svc_rdma_send_reply_msg(rdma, sctxt, rctxt, rqstp); if (ret < 0) - goto put_ctxt; + goto send_err; return 0; -reply_chunk: +send_err: if (ret != -E2BIG && ret != -EINVAL) goto put_ctxt; diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 12857381e8610..55f810a22c6aa 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -43,6 +43,7 @@ */ #include +#include #include #include #include @@ -226,12 +227,16 @@ svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt, * structure for the listening endpoint. * * This function creates a new xprt for the new connection and enqueues it on - * the accept queue for the listent xprt. When the listen thread is kicked, it + * the accept queue for the listen xprt. When the listen thread is kicked, it * will call the recvfrom method on the listen xprt which will accept the new * connection. + * + * Return values: + * %0: Do not destroy @new_cma_id + * %1: Destroy @new_cma_id (allocation failure) */ -static void handle_connect_req(struct rdma_cm_id *new_cma_id, - struct rdma_conn_param *param) +static int handle_connect_req(struct rdma_cm_id *new_cma_id, + struct rdma_conn_param *param) { struct svcxprt_rdma *listen_xprt = new_cma_id->context; struct svcxprt_rdma *newxprt; @@ -241,7 +246,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, listen_xprt->sc_xprt.xpt_net, ibdev_to_node(new_cma_id->device)); if (!newxprt) - return; + return 1; newxprt->sc_cm_id = new_cma_id; new_cma_id->context = newxprt; svc_rdma_parse_connect_private(newxprt, param); @@ -275,6 +280,7 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags); svc_xprt_enqueue(&listen_xprt->sc_xprt); + return 0; } /** @@ -298,8 +304,7 @@ static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id, switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: - handle_connect_req(cma_id, &event->param.conn); - break; + return handle_connect_req(cma_id, &event->param.conn); case RDMA_CM_EVENT_ADDR_CHANGE: listen_id = svc_rdma_create_listen_id(cma_rdma->xpt_net, sap, cma_xprt); @@ -576,13 +581,26 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) return &newxprt->sc_xprt; errout: - /* Take a reference in case the DTO handler runs */ - svc_xprt_get(&newxprt->sc_xprt); - if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp)) - ib_destroy_qp(newxprt->sc_qp); - rdma_destroy_id(newxprt->sc_cm_id); - rpcrdma_rn_unregister(dev, &newxprt->sc_rn); - /* This call to put will destroy the transport */ + /* + * Drop the kref_init birth reference. svc_xprt_free will + * dispatch xpo_free = svc_rdma_free, which tears down sc_qp, + * sc_sq_cq, sc_rq_cq, and sc_pd under existing IS_ERR/NULL + * guards, and sc_rn under the rn_done sentinel guard inside + * rpcrdma_rn_unregister. + * + * sc_cm_id is destroyed unconditionally by svc_rdma_free; that + * is safe here because sc_cm_id is non-NULL by caller invariant + * on every path that reaches this errout: handle_connect_req + * installs newxprt->sc_cm_id before queueing the new xprt for + * accept, and svc_rdma_accept has already dereferenced it above + * the first goto errout. + * + * svc_handle_xprt() drops its pre-acquired module reference when + * ->xpo_accept() returns NULL. Take a replacement reference before + * freeing @newxprt, because svc_xprt_free() drops the module + * reference associated with @newxprt. + */ + __module_get(newxprt->sc_xprt.xpt_class->xcl_owner); svc_xprt_put(&newxprt->sc_xprt); return NULL; } @@ -595,11 +613,22 @@ static void svc_rdma_detach(struct svc_xprt *xprt) rdma_disconnect(rdma->sc_cm_id); } -static void __svc_rdma_free(struct work_struct *work) +/** + * svc_rdma_free - Release class-specific transport resources + * @xprt: Generic svc transport object + */ +static void svc_rdma_free(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = - container_of(work, struct svcxprt_rdma, sc_work); - struct ib_device *device = rdma->sc_cm_id->device; + container_of(xprt, struct svcxprt_rdma, sc_xprt); + struct ib_device *device; + + might_sleep(); + + if (!rdma->sc_cm_id) + goto out_free; + + device = rdma->sc_cm_id->device; /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) @@ -625,21 +654,14 @@ static void __svc_rdma_free(struct work_struct *work) if (rdma->sc_pd && !IS_ERR(rdma->sc_pd)) ib_dealloc_pd(rdma->sc_pd); - /* Destroy the CM ID */ - rdma_destroy_id(rdma->sc_cm_id); - if (!test_bit(XPT_LISTENER, &rdma->sc_xprt.xpt_flags)) rpcrdma_rn_unregister(device, &rdma->sc_rn); - kfree(rdma); -} -static void svc_rdma_free(struct svc_xprt *xprt) -{ - struct svcxprt_rdma *rdma = - container_of(xprt, struct svcxprt_rdma, sc_xprt); + /* Destroy the CM ID */ + rdma_destroy_id(rdma->sc_cm_id); - INIT_WORK(&rdma->sc_work, __svc_rdma_free); - schedule_work(&rdma->sc_work); +out_free: + kfree(rdma); } static int svc_rdma_has_wspace(struct svc_xprt *xprt) diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index efd91ed249fab..8449107b2a095 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -1080,9 +1080,15 @@ static struct rpcrdma_rep *rpcrdma_rep_get_locked(struct rpcrdma_buffer *buf) * @buf: buffer pool * @rep: rep to release * + * The rep's transient association with an rpc_rqst, established + * by rpcrdma_reply_handler() and torn down here, must not survive + * onto rb_free_reps: rpcrdma_post_recvs() pulls reps from the free + * list to re-post them, and a non-NULL rr_rqst on a free-listed rep + * would imply the rep is still referenced by a req. */ void rpcrdma_rep_put(struct rpcrdma_buffer *buf, struct rpcrdma_rep *rep) { + rep->rr_rqst = NULL; llist_add(&rep->rr_node, &buf->rb_free_reps); } @@ -1117,6 +1123,22 @@ static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) spin_unlock(&buf->rb_lock); } +static unsigned int rpcrdma_req_pool_slack(unsigned int max_reqs) +{ + /* The sendctx ring can hold up to one Send-signaling batch + * (re_send_batch, set by frwr_open() to re_max_requests >> 3) + * of unfinished Sends. Each pins its req until a signaled Send + * completion releases the sendctx. Size the pool above max_reqs + * by that batch so the recycle delay does not stall a slot + * allocation that the RPC/RDMA credit window would admit. + * + * Round up: re_max_requests >> 3 is zero when max_reqs < 8, but + * a single unsignaled Send is still enough to pin one req. One + * slack slot covers that case. + */ + return DIV_ROUND_UP(max_reqs, 8); +} + /** * rpcrdma_buffer_create - Create initial set of req/rep objects * @r_xprt: transport instance to (re)initialize @@ -1126,6 +1148,7 @@ static void rpcrdma_reps_destroy(struct rpcrdma_buffer *buf) int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) { struct rpcrdma_buffer *buf = &r_xprt->rx_buf; + unsigned int max_reqs; int i, rc; buf->rb_bc_srv_max_requests = 0; @@ -1139,7 +1162,9 @@ int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) INIT_LIST_HEAD(&buf->rb_all_reps); rc = -ENOMEM; - for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) { + max_reqs = r_xprt->rx_xprt.max_reqs; + max_reqs += rpcrdma_req_pool_slack(max_reqs); + for (i = 0; i < max_reqs; i++) { struct rpcrdma_req *req; req = rpcrdma_req_create(r_xprt, @@ -1265,9 +1290,11 @@ rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt) */ void rpcrdma_reply_put(struct rpcrdma_buffer *buffers, struct rpcrdma_req *req) { - if (req->rl_reply) { - rpcrdma_rep_put(buffers, req->rl_reply); + struct rpcrdma_rep *rep = req->rl_reply; + + if (rep) { req->rl_reply = NULL; + rpcrdma_rep_put(buffers, rep); } /* I2: rl_reply NULL after the put closes the * 'rep on rb_free_reps still referenced by req' window. diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 6b64c6c494941..2946f6b562fbc 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1202,9 +1202,9 @@ static void xs_save_old_callbacks(struct sock_xprt *transport, struct sock *sk) static void xs_restore_old_callbacks(struct sock_xprt *transport, struct sock *sk) { - sk->sk_data_ready = transport->old_data_ready; + WRITE_ONCE(sk->sk_data_ready, transport->old_data_ready); sk->sk_state_change = transport->old_state_change; - sk->sk_write_space = transport->old_write_space; + WRITE_ONCE(sk->sk_write_space, transport->old_write_space); sk->sk_error_report = transport->old_error_report; } @@ -1664,6 +1664,7 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) { struct sock_xprt *transport = container_of(xprt, struct sock_xprt, xprt); struct sock *sk = transport->inet; + void (*write_space)(struct sock *sock); if (transport->rcvsize) { sk->sk_userlocks |= SOCK_RCVBUF_LOCK; @@ -1672,7 +1673,8 @@ static void xs_udp_do_set_buffer_size(struct rpc_xprt *xprt) if (transport->sndsize) { sk->sk_userlocks |= SOCK_SNDBUF_LOCK; sk->sk_sndbuf = transport->sndsize * xprt->max_reqs * 2; - sk->sk_write_space(sk); + write_space = READ_ONCE(sk->sk_write_space); + write_space(sk); } } @@ -1988,8 +1990,8 @@ static int xs_local_finish_connecting(struct rpc_xprt *xprt, xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_state_change = xs_local_state_change; sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; @@ -2191,8 +2193,8 @@ static void xs_udp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; - sk->sk_write_space = xs_udp_write_space; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); + WRITE_ONCE(sk->sk_write_space, xs_udp_write_space); sk->sk_use_task_frag = false; xprt_set_connected(xprt); @@ -2378,9 +2380,9 @@ static int xs_tcp_finish_connecting(struct rpc_xprt *xprt, struct socket *sock) xs_save_old_callbacks(transport, sk); sk->sk_user_data = xprt; - sk->sk_data_ready = xs_data_ready; + WRITE_ONCE(sk->sk_data_ready, xs_data_ready); sk->sk_state_change = xs_tcp_state_change; - sk->sk_write_space = xs_tcp_write_space; + WRITE_ONCE(sk->sk_write_space, xs_tcp_write_space); sk->sk_error_report = xs_error_report; sk->sk_use_task_frag = false; @@ -2647,7 +2649,17 @@ static int xs_tls_handshake_sync(struct rpc_xprt *lower_xprt, struct xprtsec_par rc = wait_for_completion_interruptible_timeout(&lower_transport->handshake_done, XS_TLS_HANDSHAKE_TO); if (rc <= 0) { - tls_handshake_cancel(sk); + if (!tls_handshake_cancel(sk)) { + /* + * Cancellation lost to handshake_complete(): the + * callback still owns its xprt reference and is in + * flight. Wait for it to finish before returning. + */ + wait_for_completion(&lower_transport->handshake_done); + if (rc == 0) + rc = -ETIMEDOUT; + goto out; + } if (rc == 0) rc = -ETIMEDOUT; goto out_put_xprt; diff --git a/net/tipc/link.c b/net/tipc/link.c index 931f55f781a13..da9e3b1b0ce21 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -504,7 +504,7 @@ bool tipc_link_create(struct net *net, char *if_name, int bearer_id, snprintf(l->name, sizeof(l->name), "%s:%s-%s:unknown", self_str, if_name, peer_str); - strcpy(l->if_name, if_name); + strscpy(l->if_name, if_name); l->addr = peer; l->peer_caps = peer_caps; l->net = net; @@ -574,7 +574,7 @@ bool tipc_link_bc_create(struct net *net, u32 ownnode, u32 peer, u8 *peer_id, snprintf(l->name, sizeof(l->name), "%s:%s", tipc_bclink_name, peer_str); } else { - strcpy(l->name, tipc_bclink_name); + strscpy(l->name, tipc_bclink_name); } trace_tipc_link_reset(l, TIPC_DUMP_ALL, "bclink created!"); tipc_link_reset(l); @@ -1898,7 +1898,7 @@ static void tipc_link_build_proto_msg(struct tipc_link *l, int mtyp, bool probe, msg_set_dest_session(hdr, l->peer_session); } msg_set_max_pkt(hdr, l->advertised_mtu); - strcpy(data, l->if_name); + memcpy(data, l->if_name, TIPC_MAX_IF_NAME); msg_set_size(hdr, INT_H_SIZE + TIPC_MAX_IF_NAME); skb_trim(skb, INT_H_SIZE + TIPC_MAX_IF_NAME); } diff --git a/net/tipc/name_table.c b/net/tipc/name_table.c index 7f42fb6a8481f..cc99a9324837a 100644 --- a/net/tipc/name_table.c +++ b/net/tipc/name_table.c @@ -763,21 +763,40 @@ struct publication *tipc_nametbl_publish(struct net *net, struct tipc_uaddr *ua, struct tipc_socket_addr *sk, u32 key) { struct name_table *nt = tipc_name_table(net); + u32 max_user_pub = TIPC_MAX_PUBL - 1; struct tipc_net *tn = tipc_net(net); struct publication *p = NULL; struct sk_buff *skb = NULL; + bool protocol_type = false; u32 rc_dests; + if (ua->sr.type == TIPC_NODE_STATE || ua->sr.type == TIPC_LINK_STATE || + ua->sr.type == TIPC_TOP_SRV) + protocol_type = true; + spin_lock_bh(&tn->nametbl_lock); + if (protocol_type) + goto insert; - if (nt->local_publ_count >= TIPC_MAX_PUBL) { - pr_warn("Bind failed, max limit %u reached\n", TIPC_MAX_PUBL); + /* Reserve one entry for node state service type because it has cluster + * scope and it is distributed in bulk. So, the maximum number of user's + * publications is (TIPC_MAX_PUBL - 1). + */ + if (nt->local_publ_count >= max_user_pub) { + pr_warn("Bind failed, max limit %u reached\n", max_user_pub); goto exit; } +insert: p = tipc_nametbl_insert_publ(net, ua, sk, key); if (p) { - nt->local_publ_count++; + /* Not count node state, link state and topology server types + * so that maximum nt->local_publ_count does not prevent + * protocol service types from being inserted into the name + * table. + */ + if (!protocol_type) + nt->local_publ_count++; skb = tipc_named_publish(net, p); } rc_dests = nt->rc_dests; @@ -810,7 +829,10 @@ void tipc_nametbl_withdraw(struct net *net, struct tipc_uaddr *ua, p = tipc_nametbl_remove_publ(net, ua, sk, key); if (p) { - nt->local_publ_count--; + if (p->sr.type != TIPC_NODE_STATE && + p->sr.type != TIPC_LINK_STATE && + p->sr.type != TIPC_TOP_SRV) + nt->local_publ_count--; skb = tipc_named_withdraw(net, p); list_del_init(&p->binding_sock); kfree_rcu(p, rcu); diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c index 8336a9664703f..1307dd1a96134 100644 --- a/net/tipc/netlink.c +++ b/net/tipc/netlink.c @@ -113,12 +113,16 @@ const struct nla_policy tipc_nl_node_policy[TIPC_NLA_NODE_MAX + 1] = { }; /* Properties valid for media, bearer and link */ +static const struct netlink_range_validation tipc_nl_mtu_range = { + .max = U16_MAX, +}; + const struct nla_policy tipc_nl_prop_policy[TIPC_NLA_PROP_MAX + 1] = { [TIPC_NLA_PROP_UNSPEC] = { .type = NLA_UNSPEC }, [TIPC_NLA_PROP_PRIO] = { .type = NLA_U32 }, [TIPC_NLA_PROP_TOL] = { .type = NLA_U32 }, [TIPC_NLA_PROP_WIN] = { .type = NLA_U32 }, - [TIPC_NLA_PROP_MTU] = { .type = NLA_U32 }, + [TIPC_NLA_PROP_MTU] = NLA_POLICY_FULL_RANGE(NLA_U32, &tipc_nl_mtu_range), [TIPC_NLA_PROP_BROADCAST] = { .type = NLA_U32 }, [TIPC_NLA_PROP_BROADCAST_RATIO] = { .type = NLA_U32 } }; diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c index 079aebb16ed8f..30e37c05a3f33 100644 --- a/net/tipc/netlink_compat.c +++ b/net/tipc/netlink_compat.c @@ -222,6 +222,10 @@ static int __tipc_nl_compat_dumpit(struct tipc_nl_compat_cmd_dump *cmd, int rem; len = (*cmd->dumpit)(buf, &cb); + if (len < 0) { + err = len; + goto err_out; + } nlmsg_for_each_msg(nlmsg, nlmsg_hdr(buf), len, rem) { err = nlmsg_parse_deprecated(nlmsg, GENL_HDRLEN, diff --git a/net/tipc/node.c b/net/tipc/node.c index 8dca9b1f4168a..6eeb8bef3a430 100644 --- a/net/tipc/node.c +++ b/net/tipc/node.c @@ -1061,18 +1061,23 @@ static void __tipc_node_link_down(struct tipc_node *n, int *bearer_id, static void tipc_node_link_down(struct tipc_node *n, int bearer_id, bool delete) { - struct tipc_link_entry *le = &n->links[bearer_id]; struct tipc_media_addr *maddr = NULL; - struct tipc_link *l = le->link; int old_bearer_id = bearer_id; + struct tipc_link_entry *le; struct sk_buff_head xmitq; - - if (!l) - return; + struct tipc_link *l; __skb_queue_head_init(&xmitq); + /* Synchronize the link lookup with bearer teardown. */ tipc_node_write_lock(n); + le = &n->links[bearer_id]; + l = le->link; + if (!l) { + tipc_node_write_unlock_fast(n); + return; + } + if (!tipc_link_is_establishing(l)) { __tipc_node_link_down(n, &bearer_id, &xmitq, &maddr); } else { diff --git a/net/tipc/socket.c b/net/tipc/socket.c index 677e17ec2939b..af14b50933f5c 100644 --- a/net/tipc/socket.c +++ b/net/tipc/socket.c @@ -501,6 +501,7 @@ static int tipc_sk_create(struct net *net, struct socket *sock, tipc_set_sk_state(sk, TIPC_OPEN); if (tipc_sk_insert(tsk)) { sk_free(sk); + sock->sk = NULL; pr_warn("Socket create failed; port number exhausted\n"); return -EINVAL; } @@ -794,7 +795,7 @@ static __poll_t tipc_poll(struct file *file, struct socket *sock, __poll_t revents = 0; sock_poll_wait(file, sock, wait); - trace_tipc_sk_poll(sk, NULL, TIPC_DUMP_ALL, " "); + trace_tipc_sk_poll(sk, NULL, TIPC_DUMP_NONE, " "); if (sk->sk_shutdown & RCV_SHUTDOWN) revents |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM; @@ -1934,7 +1935,7 @@ static int tipc_recvmsg(struct socket *sock, struct msghdr *m, if (likely(!err)) { int offset = skb_cb->bytes_read; - copy = min_t(int, dlen - offset, buflen); + copy = min_t(size_t, dlen - offset, buflen); rc = skb_copy_datagram_msg(skb, hlen + offset, m, copy); if (unlikely(rc)) goto exit; @@ -2066,7 +2067,7 @@ static int tipc_recvstream(struct socket *sock, struct msghdr *m, /* Copy data if msg ok, otherwise return error/partial data */ if (likely(!err)) { offset = skb_cb->bytes_read; - copy = min_t(int, dlen - offset, buflen - copied); + copy = min_t(size_t, dlen - offset, buflen - copied); rc = skb_copy_datagram_msg(skb, hlen + offset, m, copy); if (unlikely(rc)) break; diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 71734411ff4c3..577fca23fe100 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -530,7 +530,8 @@ static int tls_push_data(struct sock *sk, if (!size) { last_record: tls_push_record_flags = flags; - if (flags & MSG_MORE) { + if ((flags & MSG_MORE) && + record->num_frags < MAX_SKB_FRAGS - 1) { more = true; break; } @@ -594,13 +595,15 @@ void tls_device_splice_eof(struct socket *sock) struct tls_context *tls_ctx = tls_get_ctx(sk); struct iov_iter iter = {}; - if (!tls_is_partially_sent_record(tls_ctx)) + if (!tls_is_partially_sent_record(tls_ctx) && + !tls_is_pending_open_record(tls_ctx)) return; mutex_lock(&tls_ctx->tx_lock); lock_sock(sk); - if (tls_is_partially_sent_record(tls_ctx)) { + if (tls_is_partially_sent_record(tls_ctx) || + tls_is_pending_open_record(tls_ctx)) { iov_iter_bvec(&iter, ITER_SOURCE, NULL, 0, 0); tls_push_data(sk, &iter, 0, 0, TLS_RECORD_TYPE_DATA); } diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c index 39a2ab47fe720..ab30f5de2c658 100644 --- a/net/tls/tls_main.c +++ b/net/tls/tls_main.c @@ -620,6 +620,17 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, int rc = 0; int conf; + /* TLS and sockmap are mutually exclusive. A socket already in a + * sockmap (i.e. with a psock attached) cannot be upgraded to TLS. + * sockmap rejects TLS sockets already (see sk_psock_init()). + */ + rcu_read_lock(); + if (sk_psock(sk)) { + rcu_read_unlock(); + return -EINVAL; + } + rcu_read_unlock(); + if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info))) return -EINVAL; diff --git a/net/tls/tls_strp.c b/net/tls/tls_strp.c index c72e883176273..c0611c4327769 100644 --- a/net/tls/tls_strp.c +++ b/net/tls/tls_strp.c @@ -431,9 +431,10 @@ static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort) return 0; } -static bool tls_strp_check_queue_ok(struct tls_strparser *strp) +static bool tls_strp_check_queue_ok(struct tls_strparser *strp, + unsigned int len) { - unsigned int len = strp->stm.offset + strp->stm.full_len; + unsigned int remaining = strp->stm.offset + len; struct sk_buff *first, *skb; u32 seq; @@ -444,9 +445,9 @@ static bool tls_strp_check_queue_ok(struct tls_strparser *strp) /* Make sure there's no duplicate data in the queue, * and the decrypted status matches. */ - while (skb->len < len) { + while (skb->len < remaining) { seq += skb->len; - len -= skb->len; + remaining -= skb->len; skb = skb->next; if (TCP_SKB_CB(skb)->seq != seq) @@ -525,6 +526,11 @@ static int tls_strp_read_sock(struct tls_strparser *strp) tls_strp_load_anchor_with_queue(strp, inq); if (!strp->stm.full_len) { + if (inq < TLS_HEADER_SIZE) + return tls_strp_read_copy(strp, true); + if (!tls_strp_check_queue_ok(strp, TLS_HEADER_SIZE)) + return tls_strp_read_copy(strp, false); + sz = tls_rx_msg_size(strp, strp->anchor); if (sz < 0) return sz; @@ -535,7 +541,7 @@ static int tls_strp_read_sock(struct tls_strparser *strp) return tls_strp_read_copy(strp, true); } - if (!tls_strp_check_queue_ok(strp)) + if (!tls_strp_check_queue_ok(strp, strp->stm.full_len)) return tls_strp_read_copy(strp, false); WRITE_ONCE(strp->msg_ready, 1); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index f19b504efa5d0..ea78e2ce754fb 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -458,7 +458,7 @@ int tls_tx_records(struct sock *sk, int flags) } tx_err: - if (rc < 0 && rc != -EAGAIN) + if (rc < 0 && rc != -EAGAIN && rc != -EINTR && rc != -ERESTARTSYS) tls_err_abort(sk, rc); return rc; @@ -1116,6 +1116,14 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, if (!sk_stream_memory_free(sk)) goto wait_for_sndbuf; + /* open record may be full if we couldn't push it in the last sendmsg call */ + if (sk_msg_full(msg_pl)) { + full_record = true; + sk_msg_trim(sk, msg_en, + msg_pl->sg.size + prot->overhead_size); + goto copied; + } + alloc_encrypted: ret = tls_alloc_encrypted_msg(sk, required_size); if (ret) { @@ -1216,6 +1224,12 @@ static int tls_sw_sendmsg_locked(struct sock *sk, struct msghdr *msg, msg_pl, try_to_copy); if (ret < 0) goto trim_sgl; + + if (sk_msg_full(msg_pl)) { + full_record = true; + sk_msg_trim(sk, msg_en, + msg_pl->sg.size + prot->overhead_size); + } } /* Open records defined only if successfully copied, otherwise @@ -1404,12 +1418,24 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock, if (ret < 0) return ret; + if (sk_flush_backlog(sk)) + released = true; if (!skb_queue_empty(&sk->sk_receive_queue)) { tls_strp_check_rcv(&ctx->strp); if (tls_strp_msg_ready(ctx)) break; } + /* sk_flush_backlog() can run tcp_reset(), which sets + * sk_err and then sk_shutdown via tcp_done(). Recheck + * sk_err here so a connection abort surfaces as the + * actual error rather than a clean EOF. + */ + if (sk->sk_err) { + if (has_copied) + return -READ_ONCE(sk->sk_err); + return sock_error(sk); + } if (sk->sk_shutdown & RCV_SHUTDOWN) return 0; @@ -1731,6 +1757,8 @@ tls_decrypt_sw(struct sock *sk, struct tls_context *tls_ctx, /* If opportunistic TLS 1.3 ZC failed retry without ZC */ if (unlikely(darg->zc && prot->version == TLS_1_3_VERSION && darg->tail != TLS_RECORD_TYPE_DATA)) { + iov_iter_revert(&msg->msg_iter, strp_msg(darg->skb)->full_len - + prot->overhead_size); darg->zc = false; if (!darg->tail) TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXNOPADVIOL); @@ -2303,6 +2331,11 @@ ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos, if (err < 0) return err; + /* If crypto failed the connection is broken */ + err = ctx->async_wait.err; + if (err) + goto splice_read_end; + if (!skb_queue_empty(&ctx->rx_list)) { skb = __skb_dequeue(&ctx->rx_list); } else { diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index b65479a738f4c..a1e832e531bf3 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -835,6 +835,7 @@ static int unix_listen(struct socket *sock, int backlog) if (err) goto out; unix_state_lock(sk); + err = -EINVAL; if (sk->sk_state != TCP_CLOSE && sk->sk_state != TCP_LISTEN) goto out_unlock; if (backlog > sk->sk_max_ack_backlog) @@ -2830,8 +2831,8 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state) return 1; } -static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, - int flags, int copied) +static struct sk_buff *manage_oob(struct sk_buff *skb, struct sk_buff **last, + struct sock *sk, int flags, int copied) { struct sk_buff *read_skb = NULL, *unread_skb = NULL; struct unix_sock *u = unix_sk(sk); @@ -2845,11 +2846,13 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, if (copied && (!u->oob_skb || skb == u->oob_skb)) { skb = NULL; } else if (flags & MSG_PEEK) { + *last = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); } else { read_skb = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); __skb_unlink(read_skb, &sk->sk_receive_queue); + *last = skb; } if (!skb) @@ -2868,8 +2871,10 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk, __skb_unlink(skb, &sk->sk_receive_queue); unread_skb = skb; skb = skb_peek(&sk->sk_receive_queue); + *last = skb; } } else if (!sock_flag(sk, SOCK_URGINLINE)) { + *last = skb; skb = skb_peek_next(skb, &sk->sk_receive_queue); } @@ -2989,8 +2994,8 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state, again: #if IS_ENABLED(CONFIG_AF_UNIX_OOB) if (skb) { - skb = manage_oob(skb, sk, flags, copied); - if (!skb && copied) { + skb = manage_oob(skb, &last, sk, flags, copied); + if (!skb && (copied || !state->size)) { unix_state_unlock(sk); break; } diff --git a/net/unix/garbage.c b/net/unix/garbage.c index 3986717090266..cad17f639b01b 100644 --- a/net/unix/garbage.c +++ b/net/unix/garbage.c @@ -186,6 +186,7 @@ static void unix_del_edge(struct scm_fp_list *fpl, struct unix_edge *edge) if (!vertex->out_degree) { edge->predecessor->vertex = NULL; list_move_tail(&vertex->entry, &fpl->vertices); + list_del(&vertex->scc_entry); } } diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c index d007ce89d19b8..4ebf1cab2d971 100644 --- a/net/vmw_vsock/af_vsock.c +++ b/net/vmw_vsock/af_vsock.c @@ -1716,12 +1716,10 @@ static int vsock_connect(struct socket *sock, struct sockaddr *addr, prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); } - if (sk->sk_err) { - err = -sk->sk_err; + err = sock_error(sk); + if (err) { sk->sk_state = TCP_CLOSE; sock->state = SS_UNCONNECTED; - } else { - err = 0; } out_wait: @@ -1760,10 +1758,10 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, * created upon connection establishment. */ timeout = sock_rcvtimeo(listener, arg->flags & O_NONBLOCK); - prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); while ((connected = vsock_dequeue_accept(listener)) == NULL && - listener->sk_err == 0) { + timeout != 0) { + prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); release_sock(listener); timeout = schedule_timeout(timeout); finish_wait(sk_sleep(listener), &wait); @@ -1772,19 +1770,12 @@ static int vsock_accept(struct socket *sock, struct socket *newsock, if (signal_pending(current)) { err = sock_intr_errno(timeout); goto out; - } else if (timeout == 0) { - err = -EAGAIN; - goto out; } - - prepare_to_wait(sk_sleep(listener), &wait, TASK_INTERRUPTIBLE); } - finish_wait(sk_sleep(listener), &wait); - - if (listener->sk_err) - err = -listener->sk_err; - if (connected) { + if (!connected) { + err = -EAGAIN; + } else { sk_acceptq_removed(listener); lock_sock_nested(connected, SINGLE_DEPTH_NESTING); diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 4da752b47b116..54c7f51f9145d 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -323,7 +323,7 @@ static void hvs_open_connection(struct vmbus_channel *chan) goto out; if (conn_from_host) { - if (sk->sk_ack_backlog >= sk->sk_max_ack_backlog) + if (sk_acceptq_is_full(sk)) goto out; new = vsock_create_connected(sk); diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c index 8c867023a2e52..8b56ff4bdf4db 100644 --- a/net/vmw_vsock/virtio_transport.c +++ b/net/vmw_vsock/virtio_transport.c @@ -343,12 +343,13 @@ static void virtio_transport_tx_work(struct work_struct *work) struct virtqueue *vq; bool added = false; - vq = vsock->vqs[VSOCK_VQ_TX]; mutex_lock(&vsock->tx_lock); if (!vsock->tx_run) goto out; + vq = vsock->vqs[VSOCK_VQ_TX]; + do { struct sk_buff *skb; unsigned int len; @@ -448,13 +449,13 @@ static void virtio_transport_event_work(struct work_struct *work) container_of(work, struct virtio_vsock, event_work); struct virtqueue *vq; - vq = vsock->vqs[VSOCK_VQ_EVENT]; - mutex_lock(&vsock->event_lock); if (!vsock->event_run) goto out; + vq = vsock->vqs[VSOCK_VQ_EVENT]; + do { struct virtio_vsock_event *event; unsigned int len; @@ -614,12 +615,12 @@ static void virtio_transport_rx_work(struct work_struct *work) container_of(work, struct virtio_vsock, rx_work); struct virtqueue *vq; - vq = vsock->vqs[VSOCK_VQ_RX]; - mutex_lock(&vsock->rx_lock); if (!vsock->rx_run) - goto out; + goto out_nofill; + + vq = vsock->vqs[VSOCK_VQ_RX]; do { virtqueue_disable_cb(vq); @@ -667,6 +668,7 @@ static void virtio_transport_rx_work(struct work_struct *work) out: if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2) virtio_vsock_rx_fill(vsock); +out_nofill: mutex_unlock(&vsock->rx_lock); } @@ -846,10 +848,10 @@ static void virtio_vsock_remove(struct virtio_device *vdev) /* Other works can be queued before 'config->del_vqs()', so we flush * all works before to free the vsock object to avoid use after free. */ - flush_work(&vsock->rx_work); flush_work(&vsock->tx_work); flush_work(&vsock->event_work); flush_work(&vsock->send_pkt_work); + flush_work(&vsock->rx_work); mutex_unlock(&the_virtio_vsock_mutex); diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c index d93735dbe3a25..518ef9fca33d0 100644 --- a/net/vmw_vsock/virtio_transport_common.c +++ b/net/vmw_vsock/virtio_transport_common.c @@ -26,6 +26,13 @@ /* Threshold for detecting small packets to copy */ #define GOOD_COPY_LEN 128 +/* Max payload that can be collapsed into a single linear skb, using the same + * allocation threshold as virtio_vsock_alloc_skb() to avoid adding pressure + * on the page allocator. + */ +#define MAX_COLLAPSE_LEN \ + SKB_MAX_ORDER(VIRTIO_VSOCK_SKB_HEADROOM, PAGE_ALLOC_COSTLY_ORDER) + static void virtio_transport_cancel_close_work(struct vsock_sock *vsk, bool cancel_timeout); static s64 virtio_transport_has_space(struct virtio_vsock_sock *vvs); @@ -422,6 +429,145 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, return ret; } +static bool virtio_transport_can_collapse(struct sk_buff *skb) +{ + /* skbs that are partially consumed, mark a SEQPACKET message boundary, + * or are already large enough should not be collapsed: they either + * need special accounting, carry protocol state, or already have a + * good data-to-overhead ratio. + */ + if (VIRTIO_VSOCK_SKB_CB(skb)->offset) + return false; + if (le32_to_cpu(virtio_vsock_hdr(skb)->flags) & VIRTIO_VSOCK_SEQ_EOM) + return false; + if (skb->len >= MAX_COLLAPSE_LEN) + return false; + return true; +} + +/* Iterate through the packets in the queue starting from the current skb to + * count the number of bytes we can collapse. + */ +static unsigned int +virtio_transport_collapse_size(struct sk_buff *skb, struct sk_buff_head *queue) +{ + unsigned int target = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset; + + while ((skb = skb_peek_next(skb, queue)) && + virtio_transport_can_collapse(skb)) { + unsigned int len = skb->len - VIRTIO_VSOCK_SKB_CB(skb)->offset; + + if (len > MAX_COLLAPSE_LEN - target) + return target; + + target += len; + } + + return target; +} + +/* Called under lock_sock to compact the receive queue by merging small skbs. + * @min_to_free: minimum number of skbs to eliminate from the queue. May free + * more to fill each collapsed skb to capacity. + */ +static void +virtio_transport_collapse_rx_queue(struct virtio_vsock_sock *vvs, + u32 min_to_free) +{ + struct sk_buff *skb, *next_skb, *new_skb = NULL; + struct sk_buff_head new_queue; + u32 saved = 0; + + __skb_queue_head_init(&new_queue); + + skb_queue_walk_safe(&vvs->rx_queue, skb, next_skb) { + struct virtio_vsock_hdr *hdr = virtio_vsock_hdr(skb); + u32 src_off = VIRTIO_VSOCK_SKB_CB(skb)->offset; + u32 src_len = skb->len - src_off; + bool keep; + + keep = !virtio_transport_can_collapse(skb); + if (keep) { + /* Finalize pending collapsed skb to preserve packet + * ordering. + */ + if (new_skb) { + __skb_queue_tail(&new_queue, new_skb); + new_skb = NULL; + saved--; + } + goto next; + } + + /* Finalize if this packet won't fit in the remaining tailroom, + * so we can allocate a right-sized new_skb. + */ + if (new_skb && src_len > skb_tailroom(new_skb)) { + __skb_queue_tail(&new_queue, new_skb); + new_skb = NULL; + saved--; + } + + if (!new_skb) { + unsigned int alloc_size; + + /* Check after finalizing to opportunistically fill + * each collapsed skb to capacity, merging more skbs + * than strictly required. + */ + if (saved >= min_to_free) + break; + + alloc_size = virtio_transport_collapse_size(skb, &vvs->rx_queue); + + /* Only this skb's data is eligible, nothing to merge + * with. Keep as-is. + */ + if (alloc_size <= src_len) { + keep = true; + goto next; + } + + new_skb = virtio_vsock_alloc_linear_skb(alloc_size + + VIRTIO_VSOCK_SKB_HEADROOM, GFP_KERNEL); + if (!new_skb) + break; + + memcpy(virtio_vsock_hdr(new_skb), hdr, + sizeof(struct virtio_vsock_hdr)); + virtio_vsock_hdr(new_skb)->len = 0; + } + + /* Cannot fail since src_off/src_len are within bounds, but if + * it does, discard new_skb to avoid queuing corrupted data. + */ + if (WARN_ON_ONCE(skb_copy_bits(skb, src_off, + skb_put(new_skb, src_len), + src_len))) { + kfree_skb(new_skb); + new_skb = NULL; + break; + } + + le32_add_cpu(&virtio_vsock_hdr(new_skb)->len, src_len); + virtio_vsock_hdr(new_skb)->flags |= hdr->flags; + +next: + __skb_unlink(skb, &vvs->rx_queue); + if (keep) { + __skb_queue_tail(&new_queue, skb); + } else { + consume_skb(skb); + saved++; + } + } + + if (new_skb) + __skb_queue_tail(&new_queue, new_skb); + + skb_queue_splice(&new_queue, &vvs->rx_queue); +} + static bool virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, u32 len) { @@ -1341,12 +1487,29 @@ virtio_transport_recv_enqueue(struct vsock_sock *vsk, { struct virtio_vsock_sock *vvs = vsk->trans; bool can_enqueue, free_pkt = false; + u32 len, queue_max, queue_len; struct virtio_vsock_hdr *hdr; - u32 len; hdr = virtio_vsock_hdr(skb); len = le32_to_cpu(hdr->len); + /* virtio_transport_inc_rx_pkt() rejects packets when the per-skb + * overhead (skb_queue_len * SKB_TRUESIZE(0)) exceeds buf_alloc. + * Proactively collapse the queue before that happens. + * No rx_lock needed: lock_sock is held by caller, preventing + * concurrent enqueue or dequeue. + */ + queue_max = vvs->buf_alloc / SKB_TRUESIZE(0); + queue_len = skb_queue_len(&vvs->rx_queue); + if (queue_len >= queue_max) { + /* Walking a large queue may take a significant amount of time + * and cache misses, causing traffic burstiness. Limit the + * collapse to freeing room for this packet and the next one. + * It may free more to fill each collapsed skb to capacity. + */ + virtio_transport_collapse_rx_queue(vvs, queue_len + 2 - queue_max); + } + spin_lock_bh(&vvs->rx_lock); can_enqueue = virtio_transport_inc_rx_pkt(vvs, len); diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c index 75bf643ff6faf..4f02a6f269cdb 100644 --- a/net/vmw_vsock/vmci_transport.c +++ b/net/vmw_vsock/vmci_transport.c @@ -1002,7 +1002,7 @@ static int vmci_transport_recv_listen(struct sock *sk, * reset. Otherwise we create and initialize a child socket and reply * with a connection negotiation. */ - if (sk->sk_ack_backlog >= sk->sk_max_ack_backlog) { + if (sk_acceptq_is_full(sk)) { vmci_transport_reply_reset(pkt); return -ECONNREFUSED; } diff --git a/net/wireless/core.c b/net/wireless/core.c index 6b204ca91a140..b565f777e5561 100644 --- a/net/wireless/core.c +++ b/net/wireless/core.c @@ -5,7 +5,7 @@ * Copyright 2006-2010 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2015-2017 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -234,6 +234,7 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); rdev_stop_p2p_device(rdev, wdev); wdev->is_running = false; @@ -251,6 +252,8 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, void cfg80211_stop_nan(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev) { + struct cfg80211_nan_local_sched empty_sched = {}; + lockdep_assert_held(&rdev->wiphy.mtx); if (WARN_ON(wdev->iftype != NL80211_IFTYPE_NAN)) @@ -259,12 +262,64 @@ void cfg80211_stop_nan(struct cfg80211_registered_device *rdev, if (!wdev_running(wdev)) return; + cfg80211_pmsr_wdev_down(wdev); + + /* + * If there is a scheduled update pending, mark it as canceled, so the + * empty schedule will be accepted + */ + wdev->u.nan.sched_update_pending = false; + + /* Unschedule all */ + cfg80211_nan_set_local_schedule(rdev, wdev, &empty_sched); + rdev_stop_nan(rdev, wdev); wdev->is_running = false; rdev->opencount--; } +int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev, + struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched) +{ + int ret; + + lockdep_assert_held(&rdev->wiphy.mtx); + + if (wdev->iftype != NL80211_IFTYPE_NAN || !wdev_running(wdev)) + return -EINVAL; + + if (wdev->u.nan.sched_update_pending) + return -EBUSY; + + ret = rdev_nan_set_local_sched(rdev, wdev, sched); + if (ret) + return ret; + + wdev->u.nan.sched_update_pending = sched->deferred; + + kfree(wdev->u.nan.chandefs); + wdev->u.nan.chandefs = NULL; + wdev->u.nan.n_channels = 0; + + if (!sched->n_channels) + return 0; + + wdev->u.nan.chandefs = kcalloc(sched->n_channels, + sizeof(*wdev->u.nan.chandefs), + GFP_KERNEL); + if (!wdev->u.nan.chandefs) + return -ENOMEM; + + for (int i = 0; i < sched->n_channels; i++) + wdev->u.nan.chandefs[i] = sched->nan_channels[i].chandef; + + wdev->u.nan.n_channels = sched->n_channels; + + return 0; +} + void cfg80211_shutdown_all_interfaces(struct wiphy *wiphy) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); @@ -726,6 +781,24 @@ static int wiphy_verify_combinations(struct wiphy *wiphy) return ret; } +static bool wiphy_cipher_suites_valid(const struct wiphy *wiphy) +{ + int i, j; + + if (wiphy->n_cipher_suites && !wiphy->cipher_suites) + return false; + + for (i = 0; i < wiphy->n_cipher_suites; i++) { + for (j = 0; j < i; j++) { + if (wiphy->cipher_suites[i] == + wiphy->cipher_suites[j]) + return false; + } + } + + return true; +} + int wiphy_register(struct wiphy *wiphy) { struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); @@ -858,6 +931,9 @@ int wiphy_register(struct wiphy *wiphy) if (res) return res; + if (!wiphy_cipher_suites_valid(wiphy)) + return -EINVAL; + /* sanity check supported bands/channels */ for (band = 0; band < NUM_NL80211_BANDS; band++) { const struct ieee80211_sband_iftype_data *iftd; @@ -1195,6 +1271,7 @@ void wiphy_unregister(struct wiphy *wiphy) /* this has nothing to do now but make sure it's gone */ cancel_work_sync(&rdev->wiphy_work); + cancel_work_sync(&rdev->sched_scan_res_wk); cancel_work_sync(&rdev->rfkill_block); cancel_work_sync(&rdev->conn_work); flush_work(&rdev->event_work); @@ -1443,7 +1520,7 @@ void cfg80211_init_wdev(struct wireless_dev *wdev) INIT_LIST_HEAD(&wdev->mgmt_registrations); INIT_LIST_HEAD(&wdev->pmsr_list); spin_lock_init(&wdev->pmsr_lock); - INIT_WORK(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); + wiphy_work_init(&wdev->pmsr_free_wk, cfg80211_pmsr_free_wk); #ifdef CONFIG_CFG80211_WEXT wdev->wext.default_key = -1; @@ -1578,7 +1655,6 @@ static int cfg80211_netdev_notifier_call(struct notifier_block *nb, } /* since we just did cfg80211_leave() nothing to do there */ cancel_work_sync(&wdev->disconnect_wk); - cancel_work_sync(&wdev->pmsr_free_wk); break; case NETDEV_DOWN: wiphy_lock(&rdev->wiphy); diff --git a/net/wireless/core.h b/net/wireless/core.h index d5d78752227af..1086aa1a44f42 100644 --- a/net/wireless/core.h +++ b/net/wireless/core.h @@ -545,6 +545,10 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev, void cfg80211_stop_nan(struct cfg80211_registered_device *rdev, struct wireless_dev *wdev); +int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev, + struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched); + struct cfg80211_internal_bss * cfg80211_bss_update(struct cfg80211_registered_device *rdev, struct cfg80211_internal_bss *tmp, @@ -566,7 +570,7 @@ cfg80211_get_6ghz_power_type(const u8 *elems, size_t elems_len); void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid); void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev); -void cfg80211_pmsr_free_wk(struct work_struct *work); +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work); void cfg80211_remove_link(struct wireless_dev *wdev, unsigned int link_id); void cfg80211_remove_links(struct wireless_dev *wdev); diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c index 3fc175f9f8686..a304c0e8736f6 100644 --- a/net/wireless/mlme.c +++ b/net/wireless/mlme.c @@ -32,14 +32,10 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, .timeout_reason = NL80211_TIMEOUT_UNSPECIFIED, .req_ie = data->req_ies, .req_ie_len = data->req_ies_len, - .resp_ie = mgmt->u.assoc_resp.variable, - .resp_ie_len = data->len - - offsetof(struct ieee80211_mgmt, - u.assoc_resp.variable), - .status = le16_to_cpu(mgmt->u.assoc_resp.status_code), .ap_mld_addr = data->ap_mld_addr, }; unsigned int link_id; + bool is_s1g = false; for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { cr.links[link_id].status = data->links[link_id].status; @@ -60,16 +56,32 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, if (cr.links[link_id].bss->channel->band == NL80211_BAND_S1GHZ) { WARN_ON(link_id); - cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable; - cr.resp_ie_len = data->len - - offsetof(struct ieee80211_mgmt, - u.s1g_assoc_resp.variable); + is_s1g = true; } if (cr.ap_mld_addr) cr.valid_links |= BIT(link_id); } + if (is_s1g) { + if (data->len < offsetof(struct ieee80211_mgmt, + u.s1g_assoc_resp.variable)) + goto free_bss; + cr.resp_ie = (u8 *)&mgmt->u.s1g_assoc_resp.variable; + cr.resp_ie_len = data->len - + offsetof(struct ieee80211_mgmt, + u.s1g_assoc_resp.variable); + } else { + if (data->len < offsetof(struct ieee80211_mgmt, + u.assoc_resp.variable)) + goto free_bss; + cr.resp_ie = mgmt->u.assoc_resp.variable; + cr.resp_ie_len = data->len - + offsetof(struct ieee80211_mgmt, + u.assoc_resp.variable); + } + cr.status = le16_to_cpu(mgmt->u.assoc_resp.status_code); + trace_cfg80211_send_rx_assoc(dev, data); /* @@ -78,22 +90,24 @@ void cfg80211_rx_assoc_resp(struct net_device *dev, * and got a reject -- we only try again with an assoc * frame instead of reassoc. */ - if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) { - for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { - struct cfg80211_bss *bss = data->links[link_id].bss; - - if (!bss) - continue; - - cfg80211_unhold_bss(bss_from_pub(bss)); - cfg80211_put_bss(wiphy, bss); - } - return; - } + if (cfg80211_sme_rx_assoc_resp(wdev, cr.status)) + goto free_bss; nl80211_send_rx_assoc(rdev, dev, data); /* update current_bss etc., consumes the bss reference */ __cfg80211_connect_result(dev, &cr, cr.status == WLAN_STATUS_SUCCESS); + return; + +free_bss: + for (link_id = 0; link_id < ARRAY_SIZE(data->links); link_id++) { + struct cfg80211_bss *bss = data->links[link_id].bss; + + if (!bss) + continue; + + cfg80211_unhold_bss(bss_from_pub(bss)); + cfg80211_put_bss(wiphy, bss); + } } EXPORT_SYMBOL(cfg80211_rx_assoc_resp); @@ -150,19 +164,35 @@ void cfg80211_rx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len) { struct wireless_dev *wdev = dev->ieee80211_ptr; struct ieee80211_mgmt *mgmt = (void *)buf; + __le16 fc; lockdep_assert_wiphy(wdev->wiphy); - trace_cfg80211_rx_mlme_mgmt(dev, buf, len); + if (len < sizeof(fc)) + return; + + fc = mgmt->frame_control; - if (WARN_ON(len < 2)) + if (ieee80211_is_auth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.auth.status_code)) + return; + } else if (ieee80211_is_deauth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) + return; + } else if (ieee80211_is_disassoc(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code)) + return; + } else { return; + } + + trace_cfg80211_rx_mlme_mgmt(dev, buf, len); - if (ieee80211_is_auth(mgmt->frame_control)) + if (ieee80211_is_auth(fc)) cfg80211_process_auth(wdev, buf, len); - else if (ieee80211_is_deauth(mgmt->frame_control)) + else if (ieee80211_is_deauth(fc)) cfg80211_process_deauth(wdev, buf, len, false); - else if (ieee80211_is_disassoc(mgmt->frame_control)) + else cfg80211_process_disassoc(wdev, buf, len, false); } EXPORT_SYMBOL(cfg80211_rx_mlme_mgmt); @@ -215,15 +245,28 @@ void cfg80211_tx_mlme_mgmt(struct net_device *dev, const u8 *buf, size_t len, { struct wireless_dev *wdev = dev->ieee80211_ptr; struct ieee80211_mgmt *mgmt = (void *)buf; + __le16 fc; lockdep_assert_wiphy(wdev->wiphy); - trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect); + if (len < sizeof(fc)) + return; - if (WARN_ON(len < 2)) + fc = mgmt->frame_control; + + if (ieee80211_is_deauth(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.deauth.reason_code)) + return; + } else if (ieee80211_is_disassoc(fc)) { + if (len < offsetofend(struct ieee80211_mgmt, u.disassoc.reason_code)) + return; + } else { return; + } + + trace_cfg80211_tx_mlme_mgmt(dev, buf, len, reconnect); - if (ieee80211_is_deauth(mgmt->frame_control)) + if (ieee80211_is_deauth(fc)) cfg80211_process_deauth(wdev, buf, len, reconnect); else cfg80211_process_disassoc(wdev, buf, len, reconnect); diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c index 6e41a4269e1c5..28b9cd017ae38 100644 --- a/net/wireless/nl80211.c +++ b/net/wireless/nl80211.c @@ -332,6 +332,62 @@ static int validate_nan_cluster_id(const struct nlattr *attr, return 0; } +static int validate_nan_avail_blob(const struct nlattr *attr, + struct netlink_ext_ack *extack) +{ + const u8 *data = nla_data(attr); + unsigned int len = nla_len(attr); + u16 attr_len; + + /* Need at least: Attr ID (1) + Length (2) */ + if (len < 3) { + NL_SET_ERR_MSG_FMT(extack, + "NAN Availability: Too short (need at least 3 bytes, have %u)", + len); + return -EINVAL; + } + + if (data[0] != 0x12) { + NL_SET_ERR_MSG_FMT(extack, + "NAN Availability: Invalid Attribute ID 0x%02x (expected 0x12)", + data[0]); + return -EINVAL; + } + + attr_len = get_unaligned_le16(&data[1]); + + if (attr_len != len - 3) { + NL_SET_ERR_MSG_FMT(extack, + "NAN Availability: Length field (%u) doesn't match data length (%u)", + attr_len, len - 3); + return -EINVAL; + } + + return 0; +} + +static int validate_uhr_capa(const struct nlattr *attr, + struct netlink_ext_ack *extack) +{ + const u8 *data = nla_data(attr); + unsigned int len = nla_len(attr); + + if (!ieee80211_uhr_capa_size_ok(data, len, false)) + return -EINVAL; + return 0; +} + +static int validate_uhr_operation(const struct nlattr *attr, + struct netlink_ext_ack *extack) +{ + const u8 *data = nla_data(attr); + unsigned int len = nla_len(attr); + + if (!ieee80211_uhr_oper_size_ok(data, len, false)) + return -EINVAL; + return 0; +} + /* policy for the attributes */ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR]; @@ -347,7 +403,9 @@ nl80211_ftm_responder_policy[NL80211_FTM_RESP_ATTR_MAX + 1] = { static const struct nla_policy nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = { [NL80211_PMSR_FTM_REQ_ATTR_ASAP] = { .type = NLA_FLAG }, - [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = { .type = NLA_U32 }, + [NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE] = + NLA_POLICY_RANGE(NLA_U32, NL80211_PREAMBLE_LEGACY, + NL80211_PREAMBLE_HE), [NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP] = NLA_POLICY_MAX(NLA_U8, 15), [NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD] = { .type = NLA_U16 }, @@ -501,7 +559,7 @@ nl80211_mbssid_config_policy[NL80211_MBSSID_CONFIG_ATTR_MAX + 1] = { [NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX] = { .type = NLA_U32 }, [NL80211_MBSSID_CONFIG_ATTR_EMA] = { .type = NLA_FLAG }, [NL80211_MBSSID_CONFIG_ATTR_TX_LINK_ID] = - NLA_POLICY_MAX(NLA_U8, IEEE80211_MLD_MAX_NUM_LINKS), + NLA_POLICY_RANGE(NLA_U8, 0, IEEE80211_MLD_MAX_NUM_LINKS - 1), }; static const struct nla_policy @@ -932,6 +990,20 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = { NLA_POLICY_NESTED(nl80211_s1g_short_beacon), [NL80211_ATTR_BSS_PARAM] = { .type = NLA_FLAG }, [NL80211_ATTR_S1G_PRIMARY_2MHZ] = { .type = NLA_FLAG }, + [NL80211_ATTR_EPP_PEER] = { .type = NLA_FLAG }, + [NL80211_ATTR_UHR_CAPABILITY] = + NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_uhr_capa, 255), + [NL80211_ATTR_DISABLE_UHR] = { .type = NLA_FLAG }, + [NL80211_ATTR_UHR_OPERATION] = + NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_uhr_operation), + [NL80211_ATTR_NAN_CHANNEL] = NLA_POLICY_NESTED(nl80211_policy), + [NL80211_ATTR_NAN_CHANNEL_ENTRY] = NLA_POLICY_EXACT_LEN(6), + [NL80211_ATTR_NAN_RX_NSS] = { .type = NLA_U8 }, + [NL80211_ATTR_NAN_TIME_SLOTS] = + NLA_POLICY_EXACT_LEN(CFG80211_NAN_SCHED_NUM_TIME_SLOTS), + [NL80211_ATTR_NAN_AVAIL_BLOB] = + NLA_POLICY_VALIDATE_FN(NLA_BINARY, validate_nan_avail_blob), + [NL80211_ATTR_NAN_SCHED_DEFERRED] = { .type = NLA_FLAG }, }; /* policy for the key attributes */ @@ -1314,6 +1386,12 @@ static int nl80211_msg_put_channel(struct sk_buff *msg, struct wiphy *wiphy, if ((chan->flags & IEEE80211_CHAN_NO_16MHZ) && nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_16MHZ)) goto nla_put_failure; + if ((chan->flags & IEEE80211_CHAN_S1G_NO_PRIMARY) && + nla_put_flag(msg, NL80211_FREQUENCY_ATTR_S1G_NO_PRIMARY)) + goto nla_put_failure; + if ((chan->flags & IEEE80211_CHAN_NO_UHR) && + nla_put_flag(msg, NL80211_FREQUENCY_ATTR_NO_UHR)) + goto nla_put_failure; } if (nla_put_u32(msg, NL80211_FREQUENCY_ATTR_MAX_TX_POWER, @@ -1947,6 +2025,7 @@ nl80211_send_iftype_data(struct sk_buff *msg, { const struct ieee80211_sta_he_cap *he_cap = &iftdata->he_cap; const struct ieee80211_sta_eht_cap *eht_cap = &iftdata->eht_cap; + const struct ieee80211_sta_uhr_cap *uhr_cap = &iftdata->uhr_cap; if (nl80211_put_iftypes(msg, NL80211_BAND_IFTYPE_ATTR_IFTYPES, iftdata->types_mask)) @@ -1998,6 +2077,14 @@ nl80211_send_iftype_data(struct sk_buff *msg, return -ENOBUFS; } + if (uhr_cap->has_uhr) { + if (nla_put(msg, NL80211_BAND_IFTYPE_ATTR_UHR_CAP_MAC, + sizeof(uhr_cap->mac), &uhr_cap->mac) || + nla_put(msg, NL80211_BAND_IFTYPE_ATTR_UHR_CAP_PHY, + sizeof(uhr_cap->phy), &uhr_cap->phy)) + return -ENOBUFS; + } + if (sband->band == NL80211_BAND_6GHZ && nla_put(msg, NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA, sizeof(iftdata->he_6ghz_capa), @@ -3511,11 +3598,10 @@ static bool nl80211_can_set_dev_channel(struct wireless_dev *wdev) } static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev, - struct genl_info *info, bool monitor, + struct netlink_ext_ack *extack, + struct nlattr **attrs, bool monitor, struct cfg80211_chan_def *chandef) { - struct netlink_ext_ack *extack = info->extack; - struct nlattr **attrs = info->attrs; u32 control_freq; if (!attrs[NL80211_ATTR_WIPHY_FREQ]) { @@ -3525,10 +3611,10 @@ static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev, } control_freq = MHZ_TO_KHZ( - nla_get_u32(info->attrs[NL80211_ATTR_WIPHY_FREQ])); - if (info->attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET]) + nla_get_u32(attrs[NL80211_ATTR_WIPHY_FREQ])); + if (attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET]) control_freq += - nla_get_u32(info->attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET]); + nla_get_u32(attrs[NL80211_ATTR_WIPHY_FREQ_OFFSET]); memset(chandef, 0, sizeof(*chandef)); chandef->chan = ieee80211_get_channel_khz(&rdev->wiphy, control_freq); @@ -3596,40 +3682,43 @@ static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev, attrs[NL80211_ATTR_S1G_PRIMARY_2MHZ]); } - if (info->attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]) { + if (attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]) { chandef->edmg.channels = - nla_get_u8(info->attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]); + nla_get_u8(attrs[NL80211_ATTR_WIPHY_EDMG_CHANNELS]); - if (info->attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG]) + if (attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG]) chandef->edmg.bw_config = - nla_get_u8(info->attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG]); + nla_get_u8(attrs[NL80211_ATTR_WIPHY_EDMG_BW_CONFIG]); } else { chandef->edmg.bw_config = 0; chandef->edmg.channels = 0; } - if (info->attrs[NL80211_ATTR_PUNCT_BITMAP]) { + if (attrs[NL80211_ATTR_PUNCT_BITMAP]) { chandef->punctured = - nla_get_u32(info->attrs[NL80211_ATTR_PUNCT_BITMAP]); + nla_get_u32(attrs[NL80211_ATTR_PUNCT_BITMAP]); if (chandef->punctured && !wiphy_ext_feature_isset(&rdev->wiphy, NL80211_EXT_FEATURE_PUNCT)) { - NL_SET_ERR_MSG(extack, - "driver doesn't support puncturing"); + NL_SET_ERR_MSG_ATTR(extack, + attrs[NL80211_ATTR_WIPHY_FREQ], + "driver doesn't support puncturing"); return -EINVAL; } } if (!cfg80211_chandef_valid(chandef)) { - NL_SET_ERR_MSG(extack, "invalid channel definition"); + NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ], + "invalid channel definition"); return -EINVAL; } if (!_cfg80211_chandef_usable(&rdev->wiphy, chandef, IEEE80211_CHAN_DISABLED, monitor ? IEEE80211_CHAN_CAN_MONITOR : 0)) { - NL_SET_ERR_MSG(extack, "(extension) channel is disabled"); + NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ], + "(extension) channel is disabled"); return -EINVAL; } @@ -3644,10 +3733,11 @@ static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev, } int nl80211_parse_chandef(struct cfg80211_registered_device *rdev, - struct genl_info *info, + struct netlink_ext_ack *extack, + struct nlattr **attrs, struct cfg80211_chan_def *chandef) { - return _nl80211_parse_chandef(rdev, info, false, chandef); + return _nl80211_parse_chandef(rdev, extack, attrs, false, chandef); } static int __nl80211_set_channel(struct cfg80211_registered_device *rdev, @@ -3674,7 +3764,7 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev, link_id = 0; } - result = _nl80211_parse_chandef(rdev, info, + result = _nl80211_parse_chandef(rdev, info->extack, info->attrs, iftype == NL80211_IFTYPE_MONITOR, &chandef); if (result) @@ -6034,7 +6124,8 @@ static int nl80211_parse_mbssid_config(struct wiphy *wiphy, } static struct cfg80211_mbssid_elems * -nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) +nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs, + struct netlink_ext_ack *extack) { struct nlattr *nl_elems; struct cfg80211_mbssid_elems *elems; @@ -6045,6 +6136,12 @@ nl80211_parse_mbssid_elems(struct wiphy *wiphy, struct nlattr *attrs) return ERR_PTR(-EINVAL); nla_for_each_nested(nl_elems, attrs, rem_elems) { + int ret; + + ret = validate_ie_attr(nl_elems, extack); + if (ret) + return ERR_PTR(ret); + if (num_elems >= 255) return ERR_PTR(-EINVAL); num_elems++; @@ -6216,7 +6313,8 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (attrs[NL80211_ATTR_MBSSID_ELEMS]) { struct cfg80211_mbssid_elems *mbssid = nl80211_parse_mbssid_elems(&rdev->wiphy, - attrs[NL80211_ATTR_MBSSID_ELEMS]); + attrs[NL80211_ATTR_MBSSID_ELEMS], + extack); if (IS_ERR(mbssid)) return PTR_ERR(mbssid); @@ -6232,8 +6330,10 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev, if (IS_ERR(rnr)) return PTR_ERR(rnr); - if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) + if (rnr && rnr->cnt < bcn->mbssid_ies->cnt) { + kfree(rnr); return -EINVAL; + } bcn->rnr_ies = rnr; } @@ -6405,8 +6505,12 @@ static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params) if (cap && cap->datalen >= sizeof(*params->he_cap) + 1) params->he_cap = (void *)(cap->data + 1); cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, ies_len); - if (cap && cap->datalen >= sizeof(*params->he_oper) + 1) + if (cap && cap->datalen >= sizeof(*params->he_oper) + 1) { params->he_oper = (void *)(cap->data + 1); + /* takes extension ID into account */ + if (cap->datalen < ieee80211_he_oper_size((void *)params->he_oper)) + return -EINVAL; + } cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_CAPABILITY, ies, ies_len); if (cap) { if (!cap->datalen) @@ -6426,6 +6530,7 @@ static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params) cap->datalen - 1)) return -EINVAL; } + return 0; } @@ -6549,6 +6654,9 @@ static int nl80211_validate_ap_phy_operation(struct cfg80211_ap_settings *params (channel->flags & IEEE80211_CHAN_NO_EHT)) return -EOPNOTSUPP; + if (params->uhr_oper && (channel->flags & IEEE80211_CHAN_NO_UHR)) + return -EOPNOTSUPP; + return 0; } @@ -6730,7 +6838,8 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) } if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) { - err = nl80211_parse_chandef(rdev, info, ¶ms->chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + ¶ms->chandef); if (err) goto out; } else if (wdev->valid_links) { @@ -6847,6 +6956,9 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info) if (err) goto out; + if (info->attrs[NL80211_ATTR_UHR_OPERATION]) + params->uhr_oper = nla_data(info->attrs[NL80211_ATTR_UHR_OPERATION]); + err = nl80211_validate_ap_phy_operation(params); if (err) goto out; @@ -7131,7 +7243,8 @@ bool nl80211_put_sta_rate(struct sk_buff *msg, struct rate_info *info, int attr) break; case RATE_INFO_BW_EHT_RU: rate_flg = 0; - WARN_ON(!(info->flags & RATE_INFO_FLAGS_EHT_MCS)); + WARN_ON(!(info->flags & RATE_INFO_FLAGS_EHT_MCS) && + !(info->flags & RATE_INFO_FLAGS_UHR_MCS)); break; } @@ -7184,6 +7297,23 @@ bool nl80211_put_sta_rate(struct sk_buff *msg, struct rate_info *info, int attr) nla_put_u8(msg, NL80211_RATE_INFO_EHT_RU_ALLOC, info->eht_ru_alloc)) return false; + } else if (info->flags & RATE_INFO_FLAGS_UHR_MCS) { + if (nla_put_u8(msg, NL80211_RATE_INFO_UHR_MCS, info->mcs)) + return false; + if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_NSS, info->nss)) + return false; + if (nla_put_u8(msg, NL80211_RATE_INFO_EHT_GI, info->eht_gi)) + return false; + if (info->bw == RATE_INFO_BW_EHT_RU && + nla_put_u8(msg, NL80211_RATE_INFO_EHT_RU_ALLOC, + info->eht_ru_alloc)) + return false; + if (info->flags & RATE_INFO_FLAGS_UHR_ELR_MCS && + nla_put_flag(msg, NL80211_RATE_INFO_UHR_ELR)) + return false; + if (info->flags & RATE_INFO_FLAGS_UHR_IM && + nla_put_flag(msg, NL80211_RATE_INFO_UHR_IM)) + return false; } nla_nest_end(msg, rate); @@ -8057,7 +8187,8 @@ int cfg80211_check_station_change(struct wiphy *wiphy, if (params->ext_capab || params->link_sta_params.ht_capa || params->link_sta_params.vht_capa || params->link_sta_params.he_capa || - params->link_sta_params.eht_capa) + params->link_sta_params.eht_capa || + params->link_sta_params.uhr_capa) return -EINVAL; if (params->sta_flags_mask & BIT(NL80211_STA_FLAG_SPP_AMSDU)) return -EINVAL; @@ -8277,6 +8408,16 @@ static int nl80211_set_station_tdls(struct genl_info *info, } } + if (info->attrs[NL80211_ATTR_UHR_CAPABILITY]) { + if (!params->link_sta_params.eht_capa) + return -EINVAL; + + params->link_sta_params.uhr_capa = + nla_data(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + params->link_sta_params.uhr_capa_len = + nla_len(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + } + if (info->attrs[NL80211_ATTR_S1G_CAPABILITY]) params->link_sta_params.s1g_capa = nla_data(info->attrs[NL80211_ATTR_S1G_CAPABILITY]); @@ -8597,6 +8738,16 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info) } } + if (info->attrs[NL80211_ATTR_UHR_CAPABILITY]) { + if (!params.link_sta_params.eht_capa) + return -EINVAL; + + params.link_sta_params.uhr_capa = + nla_data(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + params.link_sta_params.uhr_capa_len = + nla_len(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + } + if (info->attrs[NL80211_ATTR_EML_CAPABILITY]) { params.eml_cap_present = true; params.eml_cap = @@ -8656,10 +8807,11 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info) params.link_sta_params.ht_capa = NULL; params.link_sta_params.vht_capa = NULL; - /* HE and EHT require WME */ + /* HE, EHT and UHR require WME */ if (params.link_sta_params.he_capa_len || params.link_sta_params.he_6ghz_capa || - params.link_sta_params.eht_capa_len) + params.link_sta_params.eht_capa_len || + params.link_sta_params.uhr_capa_len) return -EINVAL; } @@ -8776,6 +8928,10 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info) goto out; } } + + params.epp_peer = + nla_get_flag(info->attrs[NL80211_ATTR_EPP_PEER]); + err = rdev_add_station(rdev, dev, mac_addr, ¶ms); out: dev_put(params.vlan); @@ -11162,7 +11318,7 @@ static int nl80211_start_radar_detection(struct sk_buff *skb, if (dfs_region == NL80211_DFS_UNSET) return -EINVAL; - err = nl80211_parse_chandef(rdev, info, &chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef); if (err) return err; @@ -11250,7 +11406,7 @@ static int nl80211_notify_radar_detection(struct sk_buff *skb, return -EINVAL; } - err = nl80211_parse_chandef(rdev, info, &chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef); if (err) { GENL_SET_ERR_MSG(info, "Unable to extract chandef info"); return err; @@ -11436,7 +11592,8 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info) goto free; skip_beacons: - err = nl80211_parse_chandef(rdev, info, ¶ms.chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + ¶ms.chandef); if (err) goto free; @@ -12329,6 +12486,9 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info) if (nla_get_flag(info->attrs[NL80211_ATTR_DISABLE_EHT])) req.flags |= ASSOC_REQ_DISABLE_EHT; + if (nla_get_flag(info->attrs[NL80211_ATTR_DISABLE_UHR])) + req.flags |= ASSOC_REQ_DISABLE_UHR; + if (info->attrs[NL80211_ATTR_VHT_CAPABILITY_MASK]) memcpy(&req.vht_capa_mask, nla_data(info->attrs[NL80211_ATTR_VHT_CAPABILITY_MASK]), @@ -12662,7 +12822,8 @@ static int nl80211_join_ibss(struct sk_buff *skb, struct genl_info *info) ibss.ie_len = nla_len(info->attrs[NL80211_ATTR_IE]); } - err = nl80211_parse_chandef(rdev, info, &ibss.chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + &ibss.chandef); if (err) return err; @@ -13208,6 +13369,9 @@ static int nl80211_connect(struct sk_buff *skb, struct genl_info *info) if (nla_get_flag(info->attrs[NL80211_ATTR_DISABLE_EHT])) connect.flags |= ASSOC_REQ_DISABLE_EHT; + if (nla_get_flag(info->attrs[NL80211_ATTR_DISABLE_UHR])) + connect.flags |= ASSOC_REQ_DISABLE_UHR; + if (info->attrs[NL80211_ATTR_VHT_CAPABILITY_MASK]) memcpy(&connect.vht_capa_mask, nla_data(info->attrs[NL80211_ATTR_VHT_CAPABILITY_MASK]), @@ -13658,7 +13822,7 @@ static int nl80211_remain_on_channel(struct sk_buff *skb, duration > rdev->wiphy.max_remain_on_channel_duration) return -EINVAL; - err = nl80211_parse_chandef(rdev, info, &chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef); if (err) return err; @@ -13874,7 +14038,8 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info) */ chandef.chan = NULL; if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) { - err = nl80211_parse_chandef(rdev, info, &chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + &chandef); if (err) return err; } @@ -14277,7 +14442,8 @@ static int nl80211_join_ocb(struct sk_buff *skb, struct genl_info *info) struct ocb_setup setup = {}; int err; - err = nl80211_parse_chandef(rdev, info, &setup.chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + &setup.chandef); if (err) return err; @@ -14352,7 +14518,8 @@ static int nl80211_join_mesh(struct sk_buff *skb, struct genl_info *info) cfg.auto_open_plinks = false; if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) { - err = nl80211_parse_chandef(rdev, info, &setup.chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + &setup.chandef); if (err) return err; } else { @@ -16234,6 +16401,224 @@ void cfg80211_nan_func_terminated(struct wireless_dev *wdev, } EXPORT_SYMBOL(cfg80211_nan_func_terminated); +void cfg80211_nan_sched_update_done(struct wireless_dev *wdev, bool success, + gfp_t gfp) +{ + struct wiphy *wiphy = wdev->wiphy; + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); + struct sk_buff *msg; + void *hdr; + + trace_cfg80211_nan_sched_update_done(wiphy, wdev, success); + + /* Can happen if we stopped NAN */ + if (!wdev->u.nan.sched_update_pending) + return; + + wdev->u.nan.sched_update_pending = false; + + if (!wdev->owner_nlportid) + return; + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, gfp); + if (!msg) + return; + + hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_NAN_SCHED_UPDATE_DONE); + if (!hdr) + goto nla_put_failure; + + if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx) || + nla_put_u64_64bit(msg, NL80211_ATTR_WDEV, wdev_id(wdev), + NL80211_ATTR_PAD) || + (success && + nla_put_flag(msg, NL80211_ATTR_NAN_SCHED_UPDATE_SUCCESS))) + goto nla_put_failure; + + genlmsg_end(msg, hdr); + + genlmsg_unicast(wiphy_net(wiphy), msg, wdev->owner_nlportid); + + return; + +nla_put_failure: + nlmsg_free(msg); +} +EXPORT_SYMBOL(cfg80211_nan_sched_update_done); + +static int nl80211_parse_nan_channel(struct cfg80211_registered_device *rdev, + struct nlattr *channel, + struct genl_info *info, + struct cfg80211_nan_local_sched *sched, + u8 index) +{ + struct nlattr **channel_parsed __free(kfree) = NULL; + struct cfg80211_chan_def chandef; + u8 n_rx_nss; + int ret; + + channel_parsed = kcalloc(NL80211_ATTR_MAX + 1, sizeof(*channel_parsed), + GFP_KERNEL); + if (!channel_parsed) + return -ENOMEM; + + ret = nla_parse_nested(channel_parsed, NL80211_ATTR_MAX, channel, NULL, + info->extack); + if (ret) + return ret; + + ret = nl80211_parse_chandef(rdev, info->extack, channel_parsed, + &chandef); + if (ret) + return ret; + + if (chandef.chan->band == NL80211_BAND_6GHZ) { + NL_SET_ERR_MSG(info->extack, + "6 GHz band is not supported"); + return -EOPNOTSUPP; + } + + if (!cfg80211_reg_can_beacon(&rdev->wiphy, &chandef, + NL80211_IFTYPE_NAN)) { + NL_SET_ERR_MSG_ATTR(info->extack, channel, + "Channel in NAN schedule is not allowed for NAN operation"); + return -EINVAL; + } + + for (int i = 0; i < index; i++) { + if (cfg80211_chandef_compatible(&sched->nan_channels[i].chandef, + &chandef)) { + NL_SET_ERR_MSG_ATTR(info->extack, channel, + "Channels in NAN schedule must be mutually incompatible"); + return -EINVAL; + } + } + + if (!channel_parsed[NL80211_ATTR_NAN_CHANNEL_ENTRY]) + return -EINVAL; + + sched->nan_channels[index].channel_entry = + nla_data(channel_parsed[NL80211_ATTR_NAN_CHANNEL_ENTRY]); + + if (!channel_parsed[NL80211_ATTR_NAN_RX_NSS]) + return -EINVAL; + + sched->nan_channels[index].rx_nss = + nla_get_u8(channel_parsed[NL80211_ATTR_NAN_RX_NSS]); + + n_rx_nss = u8_get_bits(rdev->wiphy.nan_capa.n_antennas, 0x03); + if (sched->nan_channels[index].rx_nss > n_rx_nss || + !sched->nan_channels[index].rx_nss) { + NL_SET_ERR_MSG_ATTR(info->extack, channel, + "Invalid RX NSS in NAN channel definition"); + return -EINVAL; + } + + sched->nan_channels[index].chandef = chandef; + + return 0; +} + +static bool nl80211_nan_is_sched_empty(struct cfg80211_nan_local_sched *sched) +{ + if (!sched->n_channels) + return true; + + for (int i = 0; i < ARRAY_SIZE(sched->schedule); i++) { + if (sched->schedule[i] != NL80211_NAN_SCHED_NOT_AVAIL_SLOT) + return false; + } + + return true; +} + +static int nl80211_nan_set_local_sched(struct sk_buff *skb, + struct genl_info *info) +{ + struct cfg80211_registered_device *rdev = info->user_ptr[0]; + struct cfg80211_nan_local_sched *sched __free(kfree) = NULL; + struct wireless_dev *wdev = info->user_ptr[1]; + int rem, i = 0, n_channels = 0; + struct nlattr *channel; + bool sched_empty; + + if (wdev->iftype != NL80211_IFTYPE_NAN) + return -EOPNOTSUPP; + + if (!wdev_running(wdev)) + return -ENOTCONN; + + if (!info->attrs[NL80211_ATTR_NAN_TIME_SLOTS]) + return -EINVAL; + + /* First count how many channel attributes we got */ + nlmsg_for_each_attr_type(channel, NL80211_ATTR_NAN_CHANNEL, + info->nlhdr, GENL_HDRLEN, rem) + n_channels++; + + sched = kzalloc(struct_size(sched, nan_channels, n_channels), + GFP_KERNEL); + if (!sched) + return -ENOMEM; + + sched->n_channels = n_channels; + + nlmsg_for_each_attr_type(channel, NL80211_ATTR_NAN_CHANNEL, + info->nlhdr, GENL_HDRLEN, rem) { + int ret = nl80211_parse_nan_channel(rdev, channel, info, sched, + i); + + if (ret) + return ret; + i++; + } + + memcpy(sched->schedule, + nla_data(info->attrs[NL80211_ATTR_NAN_TIME_SLOTS]), + nla_len(info->attrs[NL80211_ATTR_NAN_TIME_SLOTS])); + + for (int slot = 0; slot < ARRAY_SIZE(sched->schedule); slot++) { + if (sched->schedule[slot] != NL80211_NAN_SCHED_NOT_AVAIL_SLOT && + sched->schedule[slot] >= sched->n_channels) { + NL_SET_ERR_MSG(info->extack, + "Invalid time slot in NAN schedule"); + return -EINVAL; + } + } + + sched_empty = nl80211_nan_is_sched_empty(sched); + + sched->deferred = + nla_get_flag(info->attrs[NL80211_ATTR_NAN_SCHED_DEFERRED]); + + if (sched_empty) { + if (sched->deferred) { + NL_SET_ERR_MSG(info->extack, + "Schedule cannot be deferred if all time slots are unavailable"); + return -EINVAL; + } + + if (info->attrs[NL80211_ATTR_NAN_AVAIL_BLOB]) { + NL_SET_ERR_MSG(info->extack, + "NAN Availability blob must be empty if all time slots are unavailable"); + return -EINVAL; + } + } else { + if (!info->attrs[NL80211_ATTR_NAN_AVAIL_BLOB]) { + NL_SET_ERR_MSG(info->extack, + "NAN Availability blob attribute is required"); + return -EINVAL; + } + + sched->nan_avail_blob = + nla_data(info->attrs[NL80211_ATTR_NAN_AVAIL_BLOB]); + sched->nan_avail_blob_len = + nla_len(info->attrs[NL80211_ATTR_NAN_AVAIL_BLOB]); + } + + return cfg80211_nan_set_local_schedule(rdev, wdev, sched); +} + static int nl80211_get_protocol_features(struct sk_buff *skb, struct genl_info *info) { @@ -16824,7 +17209,7 @@ static int nl80211_tdls_channel_switch(struct sk_buff *skb, !info->attrs[NL80211_ATTR_OPER_CLASS]) return -EINVAL; - err = nl80211_parse_chandef(rdev, info, &chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef); if (err) return err; @@ -17398,6 +17783,10 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info) params.count = nla_get_u8(info->attrs[NL80211_ATTR_COLOR_CHANGE_COUNT]); params.color = nla_get_u8(info->attrs[NL80211_ATTR_COLOR_CHANGE_COLOR]); + params.link_id = nl80211_link_id(info->attrs); + if (!wdev->links[params.link_id].ap.beacon_interval) + return -EINVAL; + err = nl80211_parse_beacon(rdev, info->attrs, ¶ms.beacon_next, info->extack); if (err) @@ -17471,7 +17860,6 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info) goto out; } - params.link_id = nl80211_link_id(info->attrs); err = rdev_color_change(rdev, dev, ¶ms); out: @@ -17633,6 +18021,16 @@ nl80211_add_mod_link_station(struct sk_buff *skb, struct genl_info *info, } } + if (info->attrs[NL80211_ATTR_UHR_CAPABILITY]) { + if (!params.eht_capa) + return -EINVAL; + + params.uhr_capa = + nla_data(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + params.uhr_capa_len = + nla_len(info->attrs[NL80211_ATTR_UHR_CAPABILITY]); + } + if (info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]) params.he_6ghz_capa = nla_data(info->attrs[NL80211_ATTR_HE_6GHZ_CAPABILITY]); @@ -19028,6 +19426,12 @@ static const struct genl_small_ops nl80211_small_ops[] = { .flags = GENL_UNS_ADMIN_PERM, .internal_flags = IFLAGS(NL80211_FLAG_NEED_NETDEV_UP), }, + { + .cmd = NL80211_CMD_NAN_SET_LOCAL_SCHED, + .doit = nl80211_nan_set_local_sched, + .flags = GENL_ADMIN_PERM, + .internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP), + }, }; static struct genl_family nl80211_fam __ro_after_init = { @@ -19975,7 +20379,7 @@ void nl80211_send_ibss_bssid(struct cfg80211_registered_device *rdev, } void cfg80211_notify_new_peer_candidate(struct net_device *dev, const u8 *addr, - const u8 *ie, u8 ie_len, + const u8 *ie, size_t ie_len, int sig_dbm, gfp_t gfp) { struct wireless_dev *wdev = dev->ieee80211_ptr; @@ -20988,6 +21392,46 @@ void cfg80211_ch_switch_notify(struct net_device *dev, } EXPORT_SYMBOL(cfg80211_ch_switch_notify); +void cfg80211_incumbent_signal_notify(struct wiphy *wiphy, + const struct cfg80211_chan_def *chandef, + u32 signal_interference_bitmap, + gfp_t gfp) +{ + struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy); + struct sk_buff *msg; + void *hdr; + + trace_cfg80211_incumbent_signal_notify(wiphy, chandef, signal_interference_bitmap); + + msg = nlmsg_new(NLMSG_DEFAULT_SIZE, gfp); + if (!msg) + return; + + hdr = nl80211hdr_put(msg, 0, 0, 0, NL80211_CMD_INCUMBENT_SIGNAL_DETECT); + if (!hdr) + goto nla_put_failure; + + if (nla_put_u32(msg, NL80211_ATTR_WIPHY, rdev->wiphy_idx)) + goto nla_put_failure; + + if (nl80211_send_chandef(msg, chandef)) + goto nla_put_failure; + + if (nla_put_u32(msg, NL80211_ATTR_INCUMBENT_SIGNAL_INTERFERENCE_BITMAP, + signal_interference_bitmap)) + goto nla_put_failure; + + genlmsg_end(msg, hdr); + + genlmsg_multicast_netns(&nl80211_fam, wiphy_net(&rdev->wiphy), msg, 0, + NL80211_MCGRP_MLME, gfp); + return; + +nla_put_failure: + nlmsg_free(msg); +} +EXPORT_SYMBOL(cfg80211_incumbent_signal_notify); + void cfg80211_ch_switch_started_notify(struct net_device *dev, struct cfg80211_chan_def *chandef, unsigned int link_id, u8 count, diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h index 5e25782af1e07..048ba92c3e429 100644 --- a/net/wireless/nl80211.h +++ b/net/wireless/nl80211.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * Portions of this file - * Copyright (C) 2018, 2020-2024 Intel Corporation + * Copyright (C) 2018, 2020-2025 Intel Corporation */ #ifndef __NET_WIRELESS_NL80211_H #define __NET_WIRELESS_NL80211_H @@ -23,7 +23,8 @@ static inline u64 wdev_id(struct wireless_dev *wdev) } int nl80211_parse_chandef(struct cfg80211_registered_device *rdev, - struct genl_info *info, + struct netlink_ext_ack *extack, + struct nlattr **attrs, struct cfg80211_chan_def *chandef); int nl80211_parse_random_mac(struct nlattr **attrs, u8 *mac_addr, u8 *mac_addr_mask); diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c index 13801cf35e9fc..6882c39b73a46 100644 --- a/net/wireless/pmsr.c +++ b/net/wireless/pmsr.c @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Copyright (C) 2018 - 2021, 2023 - 2024 Intel Corporation + * Copyright (C) 2018 - 2021, 2023 - 2026 Intel Corporation */ #include #include "core.h" @@ -114,6 +114,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI], "FTM: LCI request not supported"); + return -EOPNOTSUPP; } out->ftm.request_civicloc = @@ -122,6 +123,7 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev, NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC], "FTM: civic location request not supported"); + return -EOPNOTSUPP; } out->ftm.trigger_based = @@ -196,6 +198,7 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, { struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1]; struct nlattr *req[NL80211_PMSR_REQ_ATTR_MAX + 1]; + bool have_measurement_type = false; struct nlattr *treq; int err, rem; @@ -221,7 +224,8 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, if (err) return err; - err = nl80211_parse_chandef(rdev, info, &out->chandef); + err = nl80211_parse_chandef(rdev, info->extack, info->attrs, + &out->chandef); if (err) return err; @@ -248,6 +252,14 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, } nla_for_each_nested(treq, req[NL80211_PMSR_REQ_ATTR_DATA], rem) { + if (have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, treq, + "multiple measurement types in request data"); + return -EINVAL; + } + + have_measurement_type = true; + switch (nla_type(treq)) { case NL80211_PMSR_TYPE_FTM: err = pmsr_parse_ftm(rdev, treq, out, info); @@ -257,10 +269,16 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev, "unsupported measurement type"); err = -EINVAL; } + if (err) + return err; } - if (err) - return err; + if (!have_measurement_type) { + NL_SET_ERR_MSG_ATTR(info->extack, + req[NL80211_PMSR_REQ_ATTR_DATA], + "missing measurement type in request data"); + return -EINVAL; + } return 0; } @@ -625,13 +643,11 @@ static void cfg80211_pmsr_process_abort(struct wireless_dev *wdev) } } -void cfg80211_pmsr_free_wk(struct work_struct *work) +void cfg80211_pmsr_free_wk(struct wiphy *wiphy, struct wiphy_work *work) { struct wireless_dev *wdev = container_of(work, struct wireless_dev, pmsr_free_wk); - guard(wiphy)(wdev->wiphy); - cfg80211_pmsr_process_abort(wdev); } @@ -647,7 +663,7 @@ void cfg80211_pmsr_wdev_down(struct wireless_dev *wdev) } spin_unlock_bh(&wdev->pmsr_lock); - cancel_work_sync(&wdev->pmsr_free_wk); + wiphy_work_cancel(wdev->wiphy, &wdev->pmsr_free_wk); if (found) cfg80211_pmsr_process_abort(wdev); @@ -662,7 +678,7 @@ void cfg80211_release_pmsr(struct wireless_dev *wdev, u32 portid) list_for_each_entry(req, &wdev->pmsr_list, list) { if (req->nl_portid == portid) { req->nl_portid = 0; - schedule_work(&wdev->pmsr_free_wk); + wiphy_work_queue(wdev->wiphy, &wdev->pmsr_free_wk); } } spin_unlock_bh(&wdev->pmsr_lock); diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h index ac6884bacf3fa..f4c06282b2e15 100644 --- a/net/wireless/rdev-ops.h +++ b/net/wireless/rdev-ops.h @@ -1060,6 +1060,22 @@ rdev_nan_change_conf(struct cfg80211_registered_device *rdev, return ret; } +static inline int +rdev_nan_set_local_sched(struct cfg80211_registered_device *rdev, + struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched) +{ + int ret; + + trace_rdev_nan_set_local_sched(&rdev->wiphy, wdev, sched); + if (rdev->ops->nan_set_local_sched) + ret = rdev->ops->nan_set_local_sched(&rdev->wiphy, wdev, sched); + else + ret = -EOPNOTSUPP; + trace_rdev_return_int(&rdev->wiphy, ret); + return ret; +} + static inline int rdev_set_mac_acl(struct cfg80211_registered_device *rdev, struct net_device *dev, struct cfg80211_acl_data *params) diff --git a/net/wireless/reg.c b/net/wireless/reg.c index 73cab51f63790..15e56dfad00f7 100644 --- a/net/wireless/reg.c +++ b/net/wireless/reg.c @@ -5,7 +5,7 @@ * Copyright 2008-2011 Luis R. Rodriguez * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2017 Intel Deutschland GmbH - * Copyright (C) 2018 - 2025 Intel Corporation + * Copyright (C) 2018 - 2026 Intel Corporation * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above @@ -1605,6 +1605,8 @@ static u32 map_regdom_flags(u32 rd_flags) channel_flags |= IEEE80211_CHAN_ALLOW_6GHZ_VLP_AP; if (rd_flags & NL80211_RRF_ALLOW_20MHZ_ACTIVITY) channel_flags |= IEEE80211_CHAN_ALLOW_20MHZ_ACTIVITY; + if (rd_flags & NL80211_RRF_NO_UHR) + channel_flags |= IEEE80211_CHAN_NO_UHR; return channel_flags; } diff --git a/net/wireless/scan.c b/net/wireless/scan.c index 2f2104bb0d52f..9648e24181fb0 100644 --- a/net/wireless/scan.c +++ b/net/wireless/scan.c @@ -5,7 +5,7 @@ * Copyright 2008 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2016 Intel Deutschland GmbH - * Copyright (C) 2018-2025 Intel Corporation + * Copyright (C) 2018-2026 Intel Corporation */ #include #include @@ -205,7 +205,7 @@ bool cfg80211_is_element_inherited(const struct element *elem, return true; if (elem->id == WLAN_EID_EXTENSION) { - if (!ext_id_len) + if (!ext_id_len || !elem->datalen) return true; loop_len = ext_id_len; list = &non_inherit_elem->data[3 + id_len]; @@ -2626,7 +2626,9 @@ ssize_t cfg80211_defragment_element(const struct element *elem, const u8 *ies, ssize_t copied; u8 elem_datalen; - if (!elem) + if (!elem || (const u8 *)elem < ies || + (const u8 *)elem + sizeof(*elem) > ies + ieslen || + (const u8 *)elem + sizeof(*elem) + elem->datalen > ies + ieslen) return -EINVAL; /* elem might be invalid after the memmove */ @@ -3326,14 +3328,15 @@ cfg80211_inform_bss_frame_data(struct wiphy *wiphy, bssid = ext->u.s1g_beacon.sa; capability = le16_to_cpu(compat->compat_info); beacon_interval = le16_to_cpu(compat->beacon_int); + tsf = le32_to_cpu(ext->u.s1g_beacon.timestamp); + tsf |= (u64)le32_to_cpu(compat->tsf_completion) << 32; } else { bssid = mgmt->bssid; beacon_interval = le16_to_cpu(mgmt->u.probe_resp.beacon_int); capability = le16_to_cpu(mgmt->u.probe_resp.capab_info); + tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); } - tsf = le64_to_cpu(mgmt->u.probe_resp.timestamp); - if (ieee80211_is_probe_resp(mgmt->frame_control)) ftype = CFG80211_BSS_FTYPE_PRESP; else if (ext) @@ -3627,8 +3630,10 @@ int cfg80211_wext_siwscan(struct net_device *dev, /* translate "Scan for SSID" request */ if (wreq) { if (wrqu->data.flags & IW_SCAN_THIS_ESSID) { - if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) - return -EINVAL; + if (wreq->essid_len > IEEE80211_MAX_SSID_LEN) { + err = -EINVAL; + goto out; + } memcpy(creq->req.ssids[0].ssid, wreq->essid, wreq->essid_len); creq->req.ssids[0].ssid_len = wreq->essid_len; diff --git a/net/wireless/trace.h b/net/wireless/trace.h index 2b71f1d867a08..30d3619f702cd 100644 --- a/net/wireless/trace.h +++ b/net/wireless/trace.h @@ -2393,6 +2393,27 @@ TRACE_EVENT(rdev_del_nan_func, WIPHY_PR_ARG, WDEV_PR_ARG, __entry->cookie) ); +TRACE_EVENT(rdev_nan_set_local_sched, + TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, + struct cfg80211_nan_local_sched *sched), + TP_ARGS(wiphy, wdev, sched), + TP_STRUCT__entry( + WIPHY_ENTRY + WDEV_ENTRY + __array(u8, schedule, CFG80211_NAN_SCHED_NUM_TIME_SLOTS) + ), + TP_fast_assign( + WIPHY_ASSIGN; + WDEV_ASSIGN; + memcpy(__entry->schedule, sched->schedule, + CFG80211_NAN_SCHED_NUM_TIME_SLOTS); + ), + TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", schedule: %s", + WIPHY_PR_ARG, WDEV_PR_ARG, + __print_array(__entry->schedule, + CFG80211_NAN_SCHED_NUM_TIME_SLOTS, 1)) +); + TRACE_EVENT(rdev_set_mac_acl, TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, struct cfg80211_acl_data *params), @@ -4222,6 +4243,42 @@ TRACE_EVENT(cfg80211_nan_cluster_joined, WDEV_PR_ARG, __entry->cluster_id, __entry->new_cluster ? " [new]" : "") ); + +TRACE_EVENT(cfg80211_incumbent_signal_notify, + TP_PROTO(struct wiphy *wiphy, + const struct cfg80211_chan_def *chandef, + u32 signal_interference_bitmap), + TP_ARGS(wiphy, chandef, signal_interference_bitmap), + TP_STRUCT__entry( + WIPHY_ENTRY + CHAN_DEF_ENTRY + __field(u32, signal_interference_bitmap) + ), + TP_fast_assign( + WIPHY_ASSIGN; + CHAN_DEF_ASSIGN(chandef); + __entry->signal_interference_bitmap = signal_interference_bitmap; + ), + TP_printk(WIPHY_PR_FMT ", " CHAN_DEF_PR_FMT ", signal_interference_bitmap=0x%x", + WIPHY_PR_ARG, CHAN_DEF_PR_ARG, __entry->signal_interference_bitmap) +); + +TRACE_EVENT(cfg80211_nan_sched_update_done, + TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev, bool success), + TP_ARGS(wiphy, wdev, success), + TP_STRUCT__entry( + WIPHY_ENTRY + WDEV_ENTRY + __field(bool, success) + ), + TP_fast_assign( + WIPHY_ASSIGN; + WDEV_ASSIGN; + __entry->success = success; + ), + TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT " success=%d", + WIPHY_PR_ARG, WDEV_PR_ARG, __entry->success) +); #endif /* !__RDEV_OPS_TRACE || TRACE_HEADER_MULTI_READ */ #undef TRACE_INCLUDE_PATH diff --git a/net/wireless/util.c b/net/wireless/util.c index 81d6d27d273cc..918d8f8468bdb 100644 --- a/net/wireless/util.c +++ b/net/wireless/util.c @@ -5,7 +5,7 @@ * Copyright 2007-2009 Johannes Berg * Copyright 2013-2014 Intel Mobile Communications GmbH * Copyright 2017 Intel Deutschland GmbH - * Copyright (C) 2018-2023, 2025 Intel Corporation + * Copyright (C) 2018-2023, 2025-2026 Intel Corporation */ #include #include @@ -1573,26 +1573,30 @@ static u32 cfg80211_calculate_bitrate_he(struct rate_info *rate) return result / 10000; } -static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) +static u32 _cfg80211_calculate_bitrate_eht_uhr(struct rate_info *rate) { #define SCALE 6144 - static const u32 mcs_divisors[16] = { - 102399, /* 16.666666... */ - 51201, /* 8.333333... */ - 34134, /* 5.555555... */ - 25599, /* 4.166666... */ - 17067, /* 2.777777... */ - 12801, /* 2.083333... */ - 11377, /* 1.851725... */ - 10239, /* 1.666666... */ - 8532, /* 1.388888... */ - 7680, /* 1.250000... */ - 6828, /* 1.111111... */ - 6144, /* 1.000000... */ - 5690, /* 0.926106... */ - 5120, /* 0.833333... */ - 409600, /* 66.666666... */ - 204800, /* 33.333333... */ + static const u32 mcs_divisors[] = { + [ 0] = 102399, /* 16.666666... */ + [ 1] = 51201, /* 8.333333... */ + [ 2] = 34134, /* 5.555555... */ + [ 3] = 25599, /* 4.166666... */ + [ 4] = 17067, /* 2.777777... */ + [ 5] = 12801, /* 2.083333... */ + [ 6] = 11377, /* 1.851725... */ + [ 7] = 10239, /* 1.666666... */ + [ 8] = 8532, /* 1.388888... */ + [ 9] = 7680, /* 1.250000... */ + [10] = 6828, /* 1.111111... */ + [11] = 6144, /* 1.000000... */ + [12] = 5690, /* 0.926106... */ + [13] = 5120, /* 0.833333... */ + [14] = 409600, /* 66.666666... */ + [15] = 204800, /* 33.333333... */ + [17] = 38400, /* 6.250180... */ + [19] = 19200, /* 3.125090... */ + [20] = 15360, /* 2.500000... */ + [23] = 9600, /* 1.562545... */ }; static const u32 rates_996[3] = { 480388888, 453700000, 408333333 }; static const u32 rates_484[3] = { 229411111, 216666666, 195000000 }; @@ -1603,8 +1607,6 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) u64 tmp; u32 result; - if (WARN_ON_ONCE(rate->mcs > 15)) - return 0; if (WARN_ON_ONCE(rate->eht_gi > NL80211_RATE_INFO_EHT_GI_3_2)) return 0; if (WARN_ON_ONCE(rate->eht_ru_alloc > @@ -1685,7 +1687,7 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) rate->eht_ru_alloc == NL80211_RATE_INFO_EHT_RU_ALLOC_26) result = rates_26[rate->eht_gi]; else { - WARN(1, "invalid EHT MCS: bw:%d, ru:%d\n", + WARN(1, "invalid EHT or UHR MCS: bw:%d, ru:%d\n", rate->bw, rate->eht_ru_alloc); return 0; } @@ -1699,11 +1701,64 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) tmp *= rate->nss; do_div(tmp, 8); + /* and handle interference mitigation - 0.9x */ + if (rate->flags & RATE_INFO_FLAGS_UHR_IM) { + if (WARN(rate->nss != 1 || rate->mcs == 15, + "invalid NSS or MCS for UHR IM\n")) + return 0; + tmp *= 9000; + do_div(tmp, 10000); + } + result = tmp; return result / 10000; } +static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate) +{ + if (WARN_ONCE(rate->mcs > 15, "bad EHT MCS %d\n", rate->mcs)) + return 0; + + if (WARN_ONCE(rate->flags & (RATE_INFO_FLAGS_UHR_ELR_MCS | + RATE_INFO_FLAGS_UHR_IM), + "bad EHT MCS flags 0x%x\n", rate->flags)) + return 0; + + return _cfg80211_calculate_bitrate_eht_uhr(rate); +} + +static u32 cfg80211_calculate_bitrate_uhr(struct rate_info *rate) +{ + if (rate->flags & RATE_INFO_FLAGS_UHR_ELR_MCS) { + WARN_ONCE(rate->eht_gi != NL80211_RATE_INFO_EHT_GI_1_6, + "bad UHR ELR guard interval %d\n", + rate->eht_gi); + WARN_ONCE(rate->mcs > 1, "bad UHR ELR MCS %d\n", rate->mcs); + WARN_ONCE(rate->nss != 1, "bad UHR ELR NSS %d\n", rate->nss); + WARN_ONCE(rate->bw != RATE_INFO_BW_20, + "bad UHR ELR bandwidth %d\n", + rate->bw); + WARN_ONCE(rate->flags & RATE_INFO_FLAGS_UHR_IM, + "bad UHR MCS flags 0x%x\n", rate->flags); + if (rate->mcs == 0) + return 17; + return 33; + } + + switch (rate->mcs) { + case 0 ... 15: + case 17: + case 19: + case 20: + case 23: + return _cfg80211_calculate_bitrate_eht_uhr(rate); + } + + WARN_ONCE(1, "bad UHR MCS %d\n", rate->mcs); + return 0; +} + static u32 cfg80211_calculate_bitrate_s1g(struct rate_info *rate) { /* For 1, 2, 4, 8 and 16 MHz channels */ @@ -1828,6 +1883,8 @@ u32 cfg80211_calculate_bitrate(struct rate_info *rate) return cfg80211_calculate_bitrate_he(rate); if (rate->flags & RATE_INFO_FLAGS_EHT_MCS) return cfg80211_calculate_bitrate_eht(rate); + if (rate->flags & RATE_INFO_FLAGS_UHR_MCS) + return cfg80211_calculate_bitrate_uhr(rate); if (rate->flags & RATE_INFO_FLAGS_S1G_MCS) return cfg80211_calculate_bitrate_s1g(rate); diff --git a/net/wireless/wext-sme.c b/net/wireless/wext-sme.c index 573b6b15a446e..b5914f3658db4 100644 --- a/net/wireless/wext-sme.c +++ b/net/wireless/wext-sme.c @@ -319,6 +319,15 @@ int cfg80211_wext_siwgenie(struct net_device *dev, return 0; if (ie_len) { + const struct element *elem; + + for_each_element(elem, extra, ie_len) { + /* nothing */ + } + + if (!for_each_element_completed(elem, extra, ie_len)) + return -EINVAL; + ie = kmemdup(extra, ie_len, GFP_KERNEL); if (!ie) return -ENOMEM; diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c index 655d1e0ae25f7..13ba76effe5d6 100644 --- a/net/x25/af_x25.c +++ b/net/x25/af_x25.c @@ -362,6 +362,7 @@ static void x25_destroy_timer(struct timer_list *t) struct sock *sk = timer_container_of(sk, t, sk_timer); x25_destroy_socket_from_timer(sk); + sock_put(sk); } /* @@ -397,9 +398,8 @@ static void __x25_destroy_socket(struct sock *sk) if (sk_has_allocations(sk)) { /* Defer: outstanding buffers */ - sk->sk_timer.expires = jiffies + 10 * HZ; sk->sk_timer.function = x25_destroy_timer; - add_timer(&sk->sk_timer); + sk_reset_timer(sk, &sk->sk_timer, jiffies + 10 * HZ); } else { /* drop last reference so sock_put will free */ __sock_put(sk); @@ -1772,15 +1772,19 @@ void x25_kill_by_neigh(struct x25_neigh *nb) { struct sock *s; +again: write_lock_bh(&x25_list_lock); sk_for_each(s, &x25_list) { if (x25_sk(s)->neighbour == nb) { + sock_hold(s); write_unlock_bh(&x25_list_lock); lock_sock(s); - x25_disconnect(s, ENETUNREACH, 0, 0); + if (x25_sk(s)->neighbour == nb) + x25_disconnect(s, ENETUNREACH, 0, 0); release_sock(s); - write_lock_bh(&x25_list_lock); + sock_put(s); + goto again; } } write_unlock_bh(&x25_list_lock); diff --git a/net/x25/x25_timer.c b/net/x25/x25_timer.c index 2ec63a1f4c6d4..7896cd43f1cc1 100644 --- a/net/x25/x25_timer.c +++ b/net/x25/x25_timer.c @@ -36,45 +36,45 @@ void x25_init_timers(struct sock *sk) void x25_start_heartbeat(struct sock *sk) { - mod_timer(&sk->sk_timer, jiffies + 5 * HZ); + sk_reset_timer(sk, &sk->sk_timer, jiffies + 5 * HZ); } void x25_stop_heartbeat(struct sock *sk) { - timer_delete(&sk->sk_timer); + sk_stop_timer(sk, &sk->sk_timer); } void x25_start_t2timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t2); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t2); } void x25_start_t21timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t21); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t21); } void x25_start_t22timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t22); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t22); } void x25_start_t23timer(struct sock *sk) { struct x25_sock *x25 = x25_sk(sk); - mod_timer(&x25->timer, jiffies + x25->t23); + sk_reset_timer(sk, &x25->timer, jiffies + x25->t23); } void x25_stop_timer(struct sock *sk) { - timer_delete(&x25_sk(sk)->timer); + sk_stop_timer(sk, &x25_sk(sk)->timer); } unsigned long x25_display_timer(struct sock *sk) @@ -108,7 +108,7 @@ static void x25_heartbeat_expiry(struct timer_list *t) sock_flag(sk, SOCK_DEAD))) { bh_unlock_sock(sk); x25_destroy_socket_from_timer(sk); - return; + goto out; } break; @@ -120,8 +120,14 @@ static void x25_heartbeat_expiry(struct timer_list *t) break; } restart_heartbeat: - x25_start_heartbeat(sk); + /* Do not rearm once __x25_destroy_socket() has unlinked the socket: + * it is past its cancel point and owns the teardown from there on. + */ + if (sk_hashed(sk)) + x25_start_heartbeat(sk); bh_unlock_sock(sk); +out: + sock_put(sk); } /* @@ -166,4 +172,5 @@ static void x25_timer_expiry(struct timer_list *t) } else x25_do_timer_expiry(sk); bh_unlock_sock(sk); + sock_put(sk); } diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c index 9ec7bd948acc7..81ffda5501078 100644 --- a/net/xdp/xdp_umem.c +++ b/net/xdp/xdp_umem.c @@ -209,6 +209,8 @@ static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr) if (mr->flags & XDP_UMEM_TX_METADATA_LEN) { if (mr->tx_metadata_len >= 256 || mr->tx_metadata_len % 8) return -EINVAL; + if (mr->tx_metadata_len < 16) + return -EINVAL; umem->tx_metadata_len = mr->tx_metadata_len; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 9e0a486d54fb3..2e41770a9988b 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -160,6 +160,17 @@ static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len, return 0; } +static void __xsk_rcv_zc_safe(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, + u32 len, u32 flags) +{ + u64 addr; + + addr = xp_get_handle(xskb, xskb->pool); + __xskq_prod_reserve_desc(xs->rx, addr, len, flags); + + xp_release(xskb); +} + static int xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) { struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); @@ -242,9 +253,11 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) u32 frame_size = __xsk_pool_get_rx_frame_size(xs->pool); void *copy_from = xsk_copy_xdp_start(xdp), *copy_to; u32 from_len, meta_len, rem, num_desc; - struct xdp_buff_xsk *xskb; + struct xdp_buff_xsk *xskb, *tmp; struct xdp_buff *xsk_xdp; + LIST_HEAD(xsk_buffs); skb_frag_t *frag; + u32 i; from_len = xdp->data_end - copy_from; meta_len = xdp->data - copy_from; @@ -287,22 +300,45 @@ static int __xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, u32 len) frag = &sinfo->frags[0]; } + for (i = 0; i < num_desc; i++) { + xsk_xdp = xsk_buff_alloc(xs->pool); + if (!xsk_xdp) + goto err_alloc; + + xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); + if (unlikely(!list_empty(&xskb->list_node))) + goto err_alloc; + + list_add_tail(&xskb->list_node, &xsk_buffs); + } + do { u32 to_len = frame_size + meta_len; u32 copied; - xsk_xdp = xsk_buff_alloc(xs->pool); + xskb = list_first_entry(&xsk_buffs, struct xdp_buff_xsk, + list_node); + list_del_init(&xskb->list_node); + xsk_xdp = &xskb->xdp; copy_to = xsk_xdp->data - meta_len; copied = xsk_copy_xdp(copy_to, ©_from, to_len, &from_len, &frag, rem); rem -= copied; - xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); - __xsk_rcv_zc(xs, xskb, copied - meta_len, rem ? XDP_PKT_CONTD : 0); + __xsk_rcv_zc_safe(xs, xskb, copied - meta_len, + rem ? XDP_PKT_CONTD : 0); meta_len = 0; } while (rem); return 0; + +err_alloc: + list_for_each_entry_safe(xskb, tmp, &xsk_buffs, list_node) { + list_del_init(&xskb->list_node); + xsk_buff_free(&xskb->xdp); + } + xs->rx_dropped++; + return -ENOMEM; } static bool xsk_tx_writeable(struct xdp_sock *xs) @@ -547,12 +583,11 @@ static int xsk_wakeup(struct xdp_sock *xs, u8 flags) static int xsk_cq_reserve_locked(struct xsk_buff_pool *pool) { - unsigned long flags; int ret; - spin_lock_irqsave(&pool->cq_lock, flags); + spin_lock(&pool->cq_cached_prod_lock); ret = xskq_prod_reserve(pool->cq); - spin_unlock_irqrestore(&pool->cq_lock, flags); + spin_unlock(&pool->cq_cached_prod_lock); return ret; } @@ -603,7 +638,7 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool, unsigned long flags; u32 idx, i; - spin_lock_irqsave(&pool->cq_lock, flags); + spin_lock_irqsave(&pool->cq_prod_lock, flags); idx = xskq_get_prod(pool->cq); if (unlikely(num_descs > 1)) { @@ -621,16 +656,27 @@ static void xsk_cq_submit_addr_locked(struct xsk_buff_pool *pool, descs_processed++; } xskq_prod_submit_n(pool->cq, descs_processed); - spin_unlock_irqrestore(&pool->cq_lock, flags); + spin_unlock_irqrestore(&pool->cq_prod_lock, flags); } -static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n) +static void xsk_cq_submit_addr_single_locked(struct xsk_buff_pool *pool, + struct xdp_desc *desc) { unsigned long flags; + u32 idx; + + spin_lock_irqsave(&pool->cq_prod_lock, flags); + idx = xskq_get_prod(pool->cq); + xskq_prod_write_addr(pool->cq, idx, desc->addr); + xskq_prod_submit_n(pool->cq, 1); + spin_unlock_irqrestore(&pool->cq_prod_lock, flags); +} - spin_lock_irqsave(&pool->cq_lock, flags); +static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n) +{ + spin_lock(&pool->cq_cached_prod_lock); xskq_prod_cancel_n(pool->cq, n); - spin_unlock_irqrestore(&pool->cq_lock, flags); + spin_unlock(&pool->cq_cached_prod_lock); } static void xsk_destruct_skb(struct sk_buff *skb) @@ -676,8 +722,11 @@ static void xsk_consume_skb(struct sk_buff *skb) static void xsk_drop_skb(struct sk_buff *skb) { - xdp_sk(skb->sk)->tx->invalid_descs += xsk_get_num_desc(skb); - xsk_consume_skb(skb); + struct xdp_sock *xs = xdp_sk(skb->sk); + + xs->tx->invalid_descs += xsk_get_num_desc(skb); + consume_skb(skb); + xs->skb = NULL; } static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, @@ -686,15 +735,16 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, { struct xsk_tx_metadata *meta = NULL; u16 csum_start, csum_offset; + u64 flags; if (unlikely(pool->tx_metadata_len == 0)) return -EINVAL; meta = buffer - pool->tx_metadata_len; - if (unlikely(!xsk_buff_valid_tx_metadata(meta))) + if (unlikely(!xsk_buff_valid_tx_metadata(pool, meta, &flags))) return -EINVAL; - if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) { + if (flags & XDP_TXMD_FLAGS_CHECKSUM) { csum_start = READ_ONCE(meta->request.csum_start); csum_offset = READ_ONCE(meta->request.csum_offset); @@ -715,8 +765,10 @@ static int xsk_skb_metadata(struct sk_buff *skb, void *buffer, } } - if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME) - skb->skb_mstamp_ns = meta->request.launch_time; + if (flags & XDP_TXMD_FLAGS_LAUNCH_TIME) + skb->skb_mstamp_ns = READ_ONCE(meta->request.launch_time); + if (!(flags & XDP_TXMD_FLAGS_TIMESTAMP)) + meta = NULL; xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); return 0; @@ -768,7 +820,7 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, } /* in case of -EOVERFLOW that could happen below, - * xsk_consume_skb() will release this node as whole skb + * xsk_drop_skb() will release this node as whole skb * would be dropped, which implies freeing all list elements */ xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; @@ -867,6 +919,8 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, xsk_addr = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg; } + xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; + if (unlikely(nr_frags == (MAX_SKB_FRAGS - 1) && xp_mb_desc(desc))) { err = -EOVERFLOW; goto free_err; @@ -884,8 +938,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb_add_rx_frag(skb, nr_frags, page, 0, len, PAGE_SIZE); refcount_add(PAGE_SIZE, &xs->sk.sk_wmem_alloc); - - xsk_addr->addrs[xsk_addr->num_descs] = desc->addr; } } @@ -913,13 +965,14 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, static int __xsk_generic_xmit(struct sock *sk) { struct xdp_sock *xs = xdp_sk(sk); - bool sent_frame = false; struct xdp_desc desc; struct sk_buff *skb; + u32 cached_cons; u32 max_batch; int err = 0; mutex_lock(&xs->mutex); + cached_cons = xs->tx->cached_cons; /* Since we dropped the RCU read lock, the socket state might have changed. */ if (unlikely(!xsk_is_bound(xs))) { @@ -948,11 +1001,21 @@ static int __xsk_generic_xmit(struct sock *sk) goto out; } + if (unlikely(xs->drain_cont)) { + xsk_cq_submit_addr_single_locked(xs->pool, &desc); + xs->tx->invalid_descs++; + xskq_cons_release(xs->tx); + xs->drain_cont = xp_mb_desc(&desc); + continue; + } + skb = xsk_build_skb(xs, &desc); if (IS_ERR(skb)) { err = PTR_ERR(skb); if (err != -EOVERFLOW) goto out; + if (xp_mb_desc(&desc)) + xs->drain_cont = true; err = 0; continue; } @@ -981,18 +1044,33 @@ static int __xsk_generic_xmit(struct sock *sk) goto out; } - sent_frame = true; xs->skb = NULL; } if (xskq_has_descs(xs->tx)) { + bool drain = xs->skb || xs->drain_cont || xp_mb_desc(&desc); + + err = xsk_cq_reserve_locked(xs->pool); + if (err) { + xs->tx->invalid_descs--; + if (xs->skb) + xsk_drop_skb(xs->skb); + xs->drain_cont = drain; + err = -EAGAIN; + goto out; + } + if (xs->skb) xsk_drop_skb(xs->skb); + + xsk_cq_submit_addr_single_locked(xs->pool, &desc); + xskq_cons_release(xs->tx); + xs->drain_cont = xp_mb_desc(&desc); } out: - if (sent_frame) + if (xs->tx->cached_cons != cached_cons) __xsk_tx_release(xs); mutex_unlock(&xs->mutex); diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c index a129ce6f1c25f..6e9c51b6b4302 100644 --- a/net/xdp/xsk_buff_pool.c +++ b/net/xdp/xsk_buff_pool.c @@ -96,7 +96,8 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs, INIT_LIST_HEAD(&pool->xskb_list); INIT_LIST_HEAD(&pool->xsk_tx_list); spin_lock_init(&pool->xsk_tx_list_lock); - spin_lock_init(&pool->cq_lock); + spin_lock_init(&pool->cq_prod_lock); + spin_lock_init(&pool->cq_cached_prod_lock); refcount_set(&pool->users, 1); pool->fq = xs->fq_tmp; @@ -766,22 +767,25 @@ EXPORT_SYMBOL(xp_raw_get_dma); * xp_raw_get_ctx - get &xdp_desc context * @pool: XSk buff pool desc address belongs to * @addr: desc address (from userspace) + * @options: desc options (from userspace) * * Helper for getting desc's DMA address and metadata pointer, if present. - * Saves one call on hotpath, double calculation of the actual address, - * and inline checks for metadata presence and sanity. + * Saves one call on hotpath and double calculation of the actual address. + * Metadata is validated later by xsk_tx_metadata_request(). * * Return: new &xdp_desc_ctx struct containing desc's DMA address and metadata - * pointer, if it is present and valid (initialized to %NULL otherwise). + * pointer, if it is present (initialized to %NULL otherwise). */ -struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr) +struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr, + u32 options) { struct xdp_desc_ctx ret; addr = __xp_raw_get_addr(pool, addr); ret.dma = __xp_raw_get_dma(pool, addr); - ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr)); + ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr), + options); return ret; } diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 1eb8d9f8b1041..4f764b5748d2d 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -440,20 +440,26 @@ static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_de q->cached_prod = cached_prod; } -static inline int xskq_prod_reserve_desc(struct xsk_queue *q, - u64 addr, u32 len, u32 flags) +static inline void __xskq_prod_reserve_desc(struct xsk_queue *q, + u64 addr, u32 len, u32 flags) { struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; u32 idx; - if (xskq_prod_is_full(q)) - return -ENOBUFS; - /* A, matches D */ idx = q->cached_prod++ & q->ring_mask; ring->desc[idx].addr = addr; ring->desc[idx].len = len; ring->desc[idx].options = flags; +} + +static inline int xskq_prod_reserve_desc(struct xsk_queue *q, + u64 addr, u32 len, u32 flags) +{ + if (xskq_prod_is_full(q)) + return -ENOBUFS; + + __xskq_prod_reserve_desc(q, addr, len, flags); return 0; } diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c index dd7a7dc027855..fb47c2f731040 100644 --- a/net/xfrm/espintcp.c +++ b/net/xfrm/espintcp.c @@ -40,6 +40,11 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) rcu_read_lock(); skb->dev = dev_get_by_index_rcu(sock_net(sk), skb->skb_iif); + if (!skb->dev) { + XFRM_INC_STATS(sock_net(sk), LINUX_MIB_XFRMINERROR); + kfree_skb(skb); + goto out; + } local_bh_disable(); #if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == AF_INET6) @@ -48,6 +53,7 @@ static void handle_esp(struct sk_buff *skb, struct sock *sk) #endif xfrm4_rcv_encap(skb, IPPROTO_ESP, 0, TCP_ENCAP_ESPINTCP); local_bh_enable(); +out: rcu_read_unlock(); } @@ -518,7 +524,8 @@ static void espintcp_close(struct sock *sk, long timeout) strp_stop(&ctx->strp); sk->sk_prot = &tcp_prot; - barrier(); + + synchronize_rcu(); disable_work_sync(&ctx->work); strp_done(&ctx->strp); diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c index 550457e4c4f01..4719aba331cd6 100644 --- a/net/xfrm/xfrm_device.c +++ b/net/xfrm/xfrm_device.c @@ -182,7 +182,7 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur err = x->type_offload->xmit(x, skb, esp_features); if (err) { if (err == -EINPROGRESS) - return NULL; + return ERR_PTR(-EINPROGRESS); XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR); kfree_skb(skb); @@ -224,7 +224,15 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur pskb = skb2; } - return skb; + /* skb_gso_segment() set skb->prev to the last segment, but async + * crypto may have stolen it above without updating ->prev. Repoint + * it at the last retained segment so validate_xmit_skb_list() does + * not chain onto a segment now owned by the crypto engine. + */ + if (skb) + skb->prev = pskb; + + return skb ? skb : ERR_PTR(-EINPROGRESS); } EXPORT_SYMBOL_GPL(validate_xmit_xfrm); diff --git a/net/xfrm/xfrm_iptfs.c b/net/xfrm/xfrm_iptfs.c index fe8e4f21b3281..d39253b06c4ef 100644 --- a/net/xfrm/xfrm_iptfs.c +++ b/net/xfrm/xfrm_iptfs.c @@ -480,6 +480,7 @@ static int iptfs_skb_add_frags(struct sk_buff *skb, } __skb_frag_ref(tofrag); shinfo->nr_frags++; + shinfo->flags |= SKBFL_SHARED_FRAG; /* see if we are done */ fraglen = tofrag->len; diff --git a/net/xfrm/xfrm_nat_keepalive.c b/net/xfrm/xfrm_nat_keepalive.c index f50b1f48f2ed9..22ca3dc1508b6 100644 --- a/net/xfrm/xfrm_nat_keepalive.c +++ b/net/xfrm/xfrm_nat_keepalive.c @@ -155,25 +155,50 @@ static void nat_keepalive_send(struct nat_keepalive *ka) } } +enum { + NAT_KEEPALIVE_BATCH_SIZE = 16, + NAT_KEEPALIVE_BATCH_FULL = 1, +}; + struct nat_keepalive_work_ctx { + struct xfrm_state *batch[NAT_KEEPALIVE_BATCH_SIZE]; + unsigned int nr; time64_t next_run; time64_t now; }; -static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) +static int nat_keepalive_work_collect(struct xfrm_state *x, int count, void *ptr) { struct nat_keepalive_work_ctx *ctx = ptr; + + if (!READ_ONCE(x->nat_keepalive_interval)) + return 0; + + if (ctx->nr == ARRAY_SIZE(ctx->batch)) + return NAT_KEEPALIVE_BATCH_FULL; + + xfrm_state_hold(x); + ctx->batch[ctx->nr++] = x; + return 0; +} + +static void nat_keepalive_work_single(struct xfrm_state *x, + struct nat_keepalive_work_ctx *ctx) +{ bool send_keepalive = false; struct nat_keepalive ka; - time64_t next_run; + time64_t next_run = 0; u32 interval; int delta; + spin_lock_bh(&x->lock); + + if (x->km.state == XFRM_STATE_DEAD) + goto out; + interval = x->nat_keepalive_interval; if (!interval) - return 0; - - spin_lock(&x->lock); + goto out; delta = (int)(ctx->now - x->lastused); if (delta < interval) { @@ -187,14 +212,14 @@ static int nat_keepalive_work_single(struct xfrm_state *x, int count, void *ptr) send_keepalive = true; } - spin_unlock(&x->lock); +out: + spin_unlock_bh(&x->lock); if (send_keepalive) nat_keepalive_send(&ka); - if (!ctx->next_run || next_run < ctx->next_run) + if (next_run && (!ctx->next_run || next_run < ctx->next_run)) ctx->next_run = next_run; - return 0; } static void nat_keepalive_work(struct work_struct *work) @@ -202,13 +227,23 @@ static void nat_keepalive_work(struct work_struct *work) struct nat_keepalive_work_ctx ctx; struct xfrm_state_walk walk; struct net *net; + int err, i; ctx.next_run = 0; ctx.now = ktime_get_real_seconds(); net = container_of(work, struct net, xfrm.nat_keepalive_work.work); xfrm_state_walk_init(&walk, IPPROTO_ESP, NULL); - xfrm_state_walk(net, &walk, nat_keepalive_work_single, &ctx); + do { + ctx.nr = 0; + err = xfrm_state_walk(net, &walk, nat_keepalive_work_collect, &ctx); + local_bh_disable(); + for (i = 0; i < ctx.nr; i++) { + nat_keepalive_work_single(ctx.batch[i], &ctx); + xfrm_state_put(ctx.batch[i]); + } + local_bh_enable(); + } while (err == NAT_KEEPALIVE_BATCH_FULL); xfrm_state_walk_done(&walk, net); if (ctx.next_run) schedule_delayed_work(&net->xfrm.nat_keepalive_work, diff --git a/net/xfrm/xfrm_output.c b/net/xfrm/xfrm_output.c index 54222fcbd7fd8..b152edf50feca 100644 --- a/net/xfrm/xfrm_output.c +++ b/net/xfrm/xfrm_output.c @@ -629,10 +629,8 @@ static int xfrm_dev_direct_output(struct sock *sk, struct xfrm_state *x, nf_reset_ct(skb); err = skb_dst(skb)->ops->local_out(net, sk, skb); - if (unlikely(err != 1)) { - kfree_skb(skb); + if (unlikely(err != 1)) return err; - } /* In transport mode, network destination is * directly reachable, while in tunnel mode, diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index 4dad1f8172286..25f9cbd3ce6a4 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -1329,8 +1329,8 @@ static void xfrm_hash_rebuild(struct work_struct *work) } } - if (policy->selector.prefixlen_d < dbits || - policy->selector.prefixlen_s < sbits) + if (policy->selector.prefixlen_d >= dbits && + policy->selector.prefixlen_s >= sbits) continue; bin = xfrm_policy_inexact_alloc_bin(policy, dir); diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 89daad5db10c9..f463c7bd5321d 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -2070,8 +2070,11 @@ static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig, x->mode_cbs = orig->mode_cbs; if (x->mode_cbs && x->mode_cbs->clone_state) { - if (x->mode_cbs->clone_state(x, orig)) + if (x->mode_cbs->clone_state(x, orig)) { + if (!x->mode_data) + x->mode_cbs = NULL; goto error; + } } @@ -2972,7 +2975,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (sockptr_is_null(optval) && !optlen) { xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL); xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL); - __sk_dst_reset(sk); + sk_dst_reset(sk); return 0; } @@ -3012,7 +3015,7 @@ int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen) if (err >= 0) { xfrm_sk_policy_insert(sk, err, pol); xfrm_pol_put(pol); - __sk_dst_reset(sk); + sk_dst_reset(sk); err = 0; } @@ -3253,6 +3256,8 @@ int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack) if (x->mode_cbs->init_state) err = x->mode_cbs->init_state(x); module_put(x->mode_cbs->owner); + if (err && !x->mode_data) + x->mode_cbs = NULL; } error: return err; diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c index 9ad6619888c73..0f0384b6a11d0 100644 --- a/net/xfrm/xfrm_user.c +++ b/net/xfrm/xfrm_user.c @@ -916,7 +916,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net, if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH_TRUNC], extack))) goto error; - if (!x->props.aalgo) { + if (!x->aalg) { if ((err = attach_auth(&x->aalg, &x->props.aalgo, attrs[XFRMA_ALG_AUTH], extack))) goto error; @@ -2068,13 +2068,12 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family, switch (ut[i].mode) { case XFRM_MODE_TUNNEL: case XFRM_MODE_BEET: + case XFRM_MODE_IPTFS: if (ut[i].optional && dir == XFRM_POLICY_OUT) { NL_SET_ERR_MSG(extack, "Mode in optional template not allowed in outbound policy"); return -EINVAL; } break; - case XFRM_MODE_IPTFS: - break; default: if (ut[i].family != prev_family) { NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change"); diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs index 0c57cf9b4004f..8305adbdf8cbf 100644 --- a/rust/bindings/lib.rs +++ b/rust/bindings/lib.rs @@ -25,11 +25,17 @@ )] #[allow(dead_code)] +#[allow(clippy::as_underscore)] #[allow(clippy::cast_lossless)] #[allow(clippy::ptr_as_ptr)] #[allow(clippy::ref_as_ptr)] #[allow(clippy::undocumented_unsafe_blocks)] -#[cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#[cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#[allow(unnecessary_transmutes)] +#[cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] mod bindings_raw { use pin_init::{MaybeZeroable, Zeroable}; diff --git a/rust/kernel/bug.rs b/rust/kernel/bug.rs index 36aef43e5ebec..53dbdaec3560b 100644 --- a/rust/kernel/bug.rs +++ b/rust/kernel/bug.rs @@ -8,6 +8,7 @@ #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(CONFIG_DEBUG_BUGVERBOSE)] macro_rules! warn_flags { @@ -47,6 +48,7 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, not(CONFIG_UML), not(CONFIG_LOONGARCH), not(CONFIG_ARM)))] #[cfg(not(CONFIG_DEBUG_BUGVERBOSE))] macro_rules! warn_flags { @@ -73,6 +75,7 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, CONFIG_UML))] macro_rules! warn_flags { ($flags:expr) => { @@ -91,9 +94,14 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] +#[cfg(not(testlib))] #[cfg(all(CONFIG_BUG, any(CONFIG_LOONGARCH, CONFIG_ARM)))] macro_rules! warn_flags { ($flags:expr) => { + if false { + _ = $flags; + } + // SAFETY: It is always safe to call `WARN_ON()`. unsafe { $crate::bindings::WARN_ON(true) } }; @@ -101,9 +109,13 @@ macro_rules! warn_flags { #[macro_export] #[doc(hidden)] -#[cfg(not(CONFIG_BUG))] +#[cfg(any(testlib, not(CONFIG_BUG)))] macro_rules! warn_flags { - ($flags:expr) => {}; + ($flags:expr) => { + if false { + _ = $flags; + } + }; } #[doc(hidden)] diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs index a509d09ca4523..1c378350f7ea1 100644 --- a/rust/kernel/cpufreq.rs +++ b/rust/kernel/cpufreq.rs @@ -361,23 +361,28 @@ impl TableBuilder { } } - /// Adds a new entry to the table. - pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + /// Adds a raw frequency-table entry. + fn push(&mut self, frequency: u32, flags: u32, driver_data: u32) -> Result { // Adds the new entry at the end of the vector. Ok(self.entries.push( bindings::cpufreq_frequency_table { flags, driver_data, - frequency: freq.as_khz() as u32, + frequency, }, GFP_KERNEL, )?) } + /// Adds a new entry to the table. + pub fn add(&mut self, freq: Hertz, flags: u32, driver_data: u32) -> Result { + self.push(freq.as_khz() as u32, flags, driver_data) + } + /// Consumes the [`TableBuilder`] and returns [`TableBox`]. pub fn to_table(mut self) -> Result { // Add last entry to the table. - self.add(Hertz(c_ulong::MAX), 0, 0)?; + self.push(bindings::CPUFREQ_TABLE_END as u32, 0, 0)?; TableBox::new(self.entries) } @@ -816,7 +821,9 @@ pub trait Driver { } /// Driver's `bios_limit` callback. - fn bios_limit(_policy: &mut Policy, _limit: &mut u32) -> Result { + /// + /// Returns HW/BIOS max frequency limitations for the CPU. + fn bios_limit(_policy: &mut Policy) -> Result { build_error!(VTABLE_DEFAULT_ERROR) } @@ -1351,9 +1358,12 @@ impl Registration { from_result(|| { let mut policy = PolicyCpu::from_cpu(cpu_id)?; - + let val = T::bios_limit(&mut policy)?; // SAFETY: `limit` is guaranteed by the C code to be valid. - T::bios_limit(&mut policy, &mut (unsafe { *limit })).map(|()| 0) + unsafe { + *limit = val; + } + Ok(0) }) } diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs index 176531f54ed30..a82b1bde1fa8a 100644 --- a/rust/kernel/device.rs +++ b/rust/kernel/device.rs @@ -625,7 +625,7 @@ macro_rules! dev_printk { /// ``` #[macro_export] macro_rules! dev_emerg { - ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_emerg, $($f)*) } } /// Prints an alert-level message (level 1) prefixed with device information. @@ -651,7 +651,7 @@ macro_rules! dev_emerg { /// ``` #[macro_export] macro_rules! dev_alert { - ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_alert, $($f)*) } } /// Prints a critical-level message (level 2) prefixed with device information. @@ -677,7 +677,7 @@ macro_rules! dev_alert { /// ``` #[macro_export] macro_rules! dev_crit { - ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_crit, $($f)*) } } /// Prints an error-level message (level 3) prefixed with device information. @@ -703,7 +703,7 @@ macro_rules! dev_crit { /// ``` #[macro_export] macro_rules! dev_err { - ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_err, $($f)*) } } /// Prints a warning-level message (level 4) prefixed with device information. @@ -729,7 +729,7 @@ macro_rules! dev_err { /// ``` #[macro_export] macro_rules! dev_warn { - ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_warn, $($f)*) } } /// Prints a notice-level message (level 5) prefixed with device information. @@ -755,7 +755,7 @@ macro_rules! dev_warn { /// ``` #[macro_export] macro_rules! dev_notice { - ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_notice, $($f)*) } } /// Prints an info-level message (level 6) prefixed with device information. @@ -781,7 +781,7 @@ macro_rules! dev_notice { /// ``` #[macro_export] macro_rules! dev_info { - ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_info, $($f)*) } } /// Prints a debug-level message (level 7) prefixed with device information. @@ -807,5 +807,5 @@ macro_rules! dev_info { /// ``` #[macro_export] macro_rules! dev_dbg { - ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*); } + ($($f:tt)*) => { $crate::dev_printk!(pr_dbg, $($f)*) } } diff --git a/rust/kernel/drm/ioctl.rs b/rust/kernel/drm/ioctl.rs index 69efbdb4c85ae..0e038355b825c 100644 --- a/rust/kernel/drm/ioctl.rs +++ b/rust/kernel/drm/ioctl.rs @@ -135,6 +135,12 @@ macro_rules! declare_drm_ioctls { // dev/file match the current driver these ioctls are being declared // for, and it's not clear how to enforce this within the type system. let dev = $crate::drm::device::Device::from_raw(raw_dev); + + // Enforce that the handler accepts higher-ranked + // lifetimes, preventing it from requiring 'static + // references that could escape this scope. + let _: for<'a> fn(&'a _, &'a mut _, &'a _) -> _ = $func; + // SAFETY: The ioctl argument has size `_IOC_SIZE(cmd)`, which we // asserted above matches the size of this type, and all bit patterns of // UAPI structs must be valid. diff --git a/rust/kernel/list.rs b/rust/kernel/list.rs index 7355bbac16a7f..fb4ed721d1f62 100644 --- a/rust/kernel/list.rs +++ b/rust/kernel/list.rs @@ -233,7 +233,7 @@ pub use self::arc_field::{define_list_arc_field_getter, ListArcField}; /// assert_eq!(list.iter().count(), 3); /// } /// -/// // Pop the items from the list using `pop_front()` and verify the content. +/// // Pop the items from the list using `pop_back()` and verify the content. /// { /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 15)); /// assert_eq!(list.pop_back().ok_or(EINVAL)?.value.foo(), ("a", 32)); diff --git a/rust/kernel/time.rs b/rust/kernel/time.rs index 6ea98dfcd0278..0c944d89f0bc2 100644 --- a/rust/kernel/time.rs +++ b/rust/kernel/time.rs @@ -421,15 +421,25 @@ impl Delta { /// to the value in the [`Delta`]. #[inline] pub fn as_micros_ceil(self) -> i64 { + // Only positive values need to be rounded up: truncating division already + // rounds towards zero, i.e. up, for negative values. + // + // The usual `(nanos + d - 1) / d` is not used because the addition overflows + // once `nanos` exceeds `i64::MAX - (d - 1)`; saturating the addition instead + // would drop the rounding bias and return a result one unit too small. + let n = self.as_nanos(); + + let (n, add) = if n > 0 { (n - 1, 1) } else { (n, 0) }; + #[cfg(CONFIG_64BIT)] { - self.as_nanos().saturating_add(NSEC_PER_USEC - 1) / NSEC_PER_USEC + n / NSEC_PER_USEC + add } #[cfg(not(CONFIG_64BIT))] // SAFETY: It is always safe to call `ktime_to_us()` with any value. unsafe { - bindings::ktime_to_us(self.as_nanos().saturating_add(NSEC_PER_USEC - 1)) + bindings::ktime_to_us(n) + add } } diff --git a/rust/uapi/lib.rs b/rust/uapi/lib.rs index 1d5fd9efb93e9..1817a67e75e90 100644 --- a/rust/uapi/lib.rs +++ b/rust/uapi/lib.rs @@ -14,6 +14,7 @@ #![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))] #![allow( clippy::all, + clippy::as_underscore, clippy::cast_lossless, clippy::ptr_as_ptr, clippy::ref_as_ptr, @@ -27,7 +28,12 @@ unreachable_pub, unsafe_op_in_unsafe_fn )] -#![cfg_attr(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES, allow(unnecessary_transmutes))] +#![cfg_attr(not(CONFIG_RUSTC_HAS_UNNECESSARY_TRANSMUTES), allow(unknown_lints))] +#![allow(unnecessary_transmutes)] +#![cfg_attr( + CONFIG_RUSTC_HAS_SUSPICIOUS_RUNTIME_SYMBOL_DEFINITIONS, + allow(suspicious_runtime_symbol_definitions) +)] // Manual definition of blocklisted types. type __kernel_size_t = usize; diff --git a/rust/uapi/uapi_helper.h b/rust/uapi/uapi_helper.h index 06d7d1a2e8dab..1c4aa4292dcec 100644 --- a/rust/uapi/uapi_helper.h +++ b/rust/uapi/uapi_helper.h @@ -6,11 +6,11 @@ * Sorted alphabetically. */ -#include #include #include #include #include +#include #include #include #include diff --git a/samples/damon/mtier.c b/samples/damon/mtier.c index dab8fb10b617a..1a3c4ab3d383f 100644 --- a/samples/damon/mtier.c +++ b/samples/damon/mtier.c @@ -153,6 +153,9 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) if (!scheme) goto free_out; damon_set_schemes(ctx, &scheme, 1); + /* zero target value causes division by zero in damos_quota_store() */ + if (!node0_mem_used_bp || !node0_mem_free_bp) + goto free_out; quota_goal = damos_new_quota_goal( promote ? DAMOS_QUOTA_NODE_MEM_USED_BP : DAMOS_QUOTA_NODE_MEM_FREE_BP, @@ -174,6 +177,7 @@ static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote) static int damon_sample_mtier_start(void) { struct damon_ctx *ctx; + int err; ctx = damon_sample_mtier_build_ctx(true); if (!ctx) @@ -185,12 +189,21 @@ static int damon_sample_mtier_start(void) return -ENOMEM; } ctxs[1] = ctx; - return damon_start(ctxs, 2, true); + err = damon_start(ctxs, 2, true); + if (!err) + return 0; + + if (damon_is_running(ctxs[0])) + damon_stop(ctxs, 1); + damon_destroy_ctx(ctxs[0]); + damon_destroy_ctx(ctxs[1]); + return err; } static void damon_sample_mtier_stop(void) { - damon_stop(ctxs, 2); + damon_stop(ctxs, 1); + damon_stop(&ctxs[1], 1); damon_destroy_ctx(ctxs[0]); damon_destroy_ctx(ctxs[1]); } diff --git a/samples/damon/prcl.c b/samples/damon/prcl.c index b7c50f2656ce7..edeae145c4a8a 100644 --- a/samples/damon/prcl.c +++ b/samples/damon/prcl.c @@ -106,11 +106,18 @@ static int damon_sample_prcl_start(void) damon_set_schemes(ctx, &scheme, 1); err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_prcl_stop(void) diff --git a/samples/damon/wsse.c b/samples/damon/wsse.c index 799ad44439434..ff5e8a890f448 100644 --- a/samples/damon/wsse.c +++ b/samples/damon/wsse.c @@ -87,10 +87,17 @@ static int damon_sample_wsse_start(void) target->pid = target_pidp; err = damon_start(&ctx, 1, true); - if (err) + if (err) { + damon_destroy_ctx(ctx); return err; + } repeat_call_control.data = ctx; - return damon_call(ctx, &repeat_call_control); + err = damon_call(ctx, &repeat_call_control); + if (err) { + damon_stop(&ctx, 1); + damon_destroy_ctx(ctx); + } + return err; } static void damon_sample_wsse_stop(void) diff --git a/samples/ftrace/ftrace-direct-modify.c b/samples/ftrace/ftrace-direct-modify.c index 1ba1927b548ee..164d9dd6fd921 100644 --- a/samples/ftrace/ftrace-direct-modify.c +++ b/samples/ftrace/ftrace-direct-modify.c @@ -320,9 +320,15 @@ static int __init ftrace_direct_init(void) ftrace_set_filter_ip(&direct, (unsigned long) my_ip, 0, 0); ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_exit(void) diff --git a/samples/ftrace/ftrace-direct-multi-modify.c b/samples/ftrace/ftrace-direct-multi-modify.c index 7a7822dfeb50a..b03766c6217bc 100644 --- a/samples/ftrace/ftrace-direct-multi-modify.c +++ b/samples/ftrace/ftrace-direct-multi-modify.c @@ -364,9 +364,15 @@ static int __init ftrace_direct_multi_init(void) ret = register_ftrace_direct(&direct, my_tramp); - if (!ret) - simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); - return ret; + if (ret) + return ret; + simple_tsk = kthread_run(simple_thread, NULL, "event-sample-fn"); + if (IS_ERR(simple_tsk)) { + unregister_ftrace_direct(&direct, my_tramp, true); + return PTR_ERR(simple_tsk); + } + + return 0; } static void __exit ftrace_direct_multi_exit(void) diff --git a/scripts/mksysmap b/scripts/mksysmap index c4531eacde202..856b26ba2ac02 100755 --- a/scripts/mksysmap +++ b/scripts/mksysmap @@ -35,7 +35,7 @@ / __efistub_/d # arm64 local symbols in PIE namespace -/ __pi_\\$/d +/ __pi_\$/d / __pi_\.L/d # arm64 local symbols in non-VHE KVM namespace @@ -83,7 +83,7 @@ / _SDA2_BASE_$/d # MODULE_INFO() -/ __UNIQUE_ID_modinfo[0-9]*$/d +/ __UNIQUE_ID_modinfo_[0-9]*$/d # --------------------------------------------------------------------------- # Ignored patterns diff --git a/scripts/mod/modpost.c b/scripts/mod/modpost.c index 010c398f6a705..1bc11585e41b3 100644 --- a/scripts/mod/modpost.c +++ b/scripts/mod/modpost.c @@ -1570,14 +1570,14 @@ static void read_symbols(const char *modname) struct elf_info info = { }; Elf_Sym *sym; - if (!parse_elf(&info, modname)) - return; - if (!strends(modname, ".o")) { error("%s: filename must be suffixed with .o\n", modname); return; } + if (!parse_elf(&info, modname)) + return; + /* strip trailing .o */ mod = new_module(modname, strlen(modname) - strlen(".o")); @@ -1679,8 +1679,17 @@ void __attribute__((format(printf, 2, 3))) buf_printf(struct buffer *buf, va_start(ap, fmt); len = vsnprintf(tmp, SZ, fmt, ap); - buf_write(buf, tmp, len); va_end(ap); + + if (len < 0) { + perror("vsnprintf failed"); + exit(1); + } + if (len >= SZ) + fatal("buf_printf output truncated for string %s: %d bytes needed, %d available\n", + tmp, len + 1, SZ); + + buf_write(buf, tmp, len); } void buf_write(struct buffer *buf, const char *s, int len) diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh index d2323de0692c1..82be88d44c184 100755 --- a/scripts/rust_is_available.sh +++ b/scripts/rust_is_available.sh @@ -242,6 +242,20 @@ if [ "$bindgen_libclang_cversion" -ge 1900100 ] && fi fi +if [ "$bindgen_libclang_cversion" -ge 2200000 ] && + [ "$rust_bindings_generator_cversion" -lt 7201 ]; then + # Distributions may have patched the issue. + if ! "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang_22.h | grep -q 'pub foo'; then + echo >&2 "***" + echo >&2 "*** Rust bindings generator '$BINDGEN' < 0.72.1 together with libclang >= 22" + echo >&2 "*** may not work due to a bug (https://github.com/rust-lang/rust-bindgen/pull/3278)." + echo >&2 "*** Your bindgen version: $rust_bindings_generator_version" + echo >&2 "*** Your libclang version: $bindgen_libclang_version" + echo >&2 "***" + warning=1 + fi +fi + # If the C compiler is Clang, then we can also check whether its version # matches the `libclang` version used by the Rust bindings generator. # diff --git a/scripts/rust_is_available_bindgen_libclang_22.h b/scripts/rust_is_available_bindgen_libclang_22.h new file mode 100644 index 0000000000000..6b33544c14a81 --- /dev/null +++ b/scripts/rust_is_available_bindgen_libclang_22.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +struct S; +struct S { + int foo; +}; diff --git a/scripts/rust_is_available_test.py b/scripts/rust_is_available_test.py index 4fcc319dea84e..a5fadef98132f 100755 --- a/scripts/rust_is_available_test.py +++ b/scripts/rust_is_available_test.py @@ -54,7 +54,7 @@ def generate_rustc(cls, stdout): """) @classmethod - def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched=False, libclang_concat_patched=False): + def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched=False, libclang_concat_patched=False, libclang_22_patched=False): if libclang_stderr is None: libclang_case = f"raise SystemExit({cls.bindgen_default_bindgen_libclang_failure_exit_code})" else: @@ -70,6 +70,11 @@ def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched= else: libclang_concat_case = "pass" + if libclang_22_patched: + libclang_22_case = "print('pub foo: ::std::os::raw::c_int,')" + else: + libclang_22_case = "pass" + return cls.generate_executable(f"""#!/usr/bin/env python3 import sys if "rust_is_available_bindgen_libclang.h" in " ".join(sys.argv): @@ -78,6 +83,8 @@ def generate_bindgen(cls, version_stdout, libclang_stderr, version_0_66_patched= {version_0_66_case} elif "rust_is_available_bindgen_libclang_concat.h" in " ".join(sys.argv): {libclang_concat_case} +elif "rust_is_available_bindgen_libclang_22.h" in " ".join(sys.argv): + {libclang_22_case} else: print({repr(version_stdout)}) """) @@ -300,6 +307,27 @@ def test_bindgen_bad_libclang_concat(self): bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_concat_patched=True) result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) + def test_bindgen_bad_libclang_22(self): + for (bindgen_version, libclang_version, expected_not_patched) in ( + ("0.71.1", "21.1.0", self.Expected.SUCCESS), + ("0.71.1", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + ("0.71.1", "22.1.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.0", "22.0.0", self.Expected.SUCCESS_WITH_WARNINGS), + + ("0.72.1", "22.0.0", self.Expected.SUCCESS), + ): + with self.subTest(bindgen_version=bindgen_version, libclang_version=libclang_version): + cc = self.generate_clang(f"clang version {libclang_version}") + libclang_stderr = f"scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version {libclang_version} [-W#pragma-messages], err: false" + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr) + result = self.run_script(expected_not_patched, { "BINDGEN": bindgen, "CC": cc }) + if expected_not_patched == self.Expected.SUCCESS_WITH_WARNINGS: + self.assertIn(f"Rust bindings generator '{bindgen}' < 0.72.1 together with libclang >= 22", result.stderr) + + bindgen = self.generate_bindgen(f"bindgen {bindgen_version}", libclang_stderr, libclang_22_patched=True) + result = self.run_script(self.Expected.SUCCESS, { "BINDGEN": bindgen, "CC": cc }) + def test_clang_matches_bindgen_libclang_different_bindgen(self): bindgen = self.generate_bindgen_libclang("scripts/rust_is_available_bindgen_libclang.h:2:9: warning: clang version 999.0.0 [-W#pragma-messages], err: false") result = self.run_script(self.Expected.SUCCESS_WITH_WARNINGS, { "BINDGEN": bindgen }) diff --git a/scripts/sorttable.c b/scripts/sorttable.c index deed676bfe384..8ab635070e29d 100644 --- a/scripts/sorttable.c +++ b/scripts/sorttable.c @@ -449,7 +449,7 @@ static inline void *get_index(void *start, int entsize, int index) } static int extable_ent_size; -static int long_size; +static int long_size __maybe_unused; #define ERRSTR_MAXSZ 256 diff --git a/scripts/tags.sh b/scripts/tags.sh index 99ce427d9a69d..173f5d23c4dc0 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -100,7 +100,7 @@ all_compiled_sources() { echo include/generated/autoconf.h find $ignore -name "*.cmd" -exec \ - grep -Poh '(?<=^ )\S+|(?<== )\S+[^\\](?=$)' {} \+ | + grep -Poh '(?<=^ )\S+\.([chS]|rs)(?=\s)|(?<== )\S+\.(?1)(?=$)' {} \+ | awk '!a[$0]++' } | xargs realpath -esq $([ -z "$KBUILD_ABS_SRCTREE" ] && echo --relative-to=.) | sort -u diff --git a/security/apparmor/af_unix.c b/security/apparmor/af_unix.c index 1e061345c0b16..834a3b1c2f0af 100644 --- a/security/apparmor/af_unix.c +++ b/security/apparmor/af_unix.c @@ -416,7 +416,7 @@ static int profile_peer_perm(struct aa_profile *profile, u32 request, unix_sk(sk), peer_addr, peer_addrlen, &p, &ad->info); - return fn_for_each_in_ns(peer_label, peerp, + return fn_for_each_in_scope(peer_label, peerp, match_label(profile, rules, state, request, peerp, p, ad)); } diff --git a/security/apparmor/apparmorfs.c b/security/apparmor/apparmorfs.c index 515cc80064c60..4dfca0c6e938d 100644 --- a/security/apparmor/apparmorfs.c +++ b/security/apparmor/apparmorfs.c @@ -877,7 +877,7 @@ static ssize_t query_label(char *buf, size_t buf_len, perms = allperms; if (view_only) { - label_for_each_in_ns(i, labels_ns(label), label, profile) { + label_for_each_in_scope(i, labels_ns(label), label, profile) { profile_query_cb(profile, &perms, match_str, match_len); } } else { diff --git a/security/apparmor/domain.c b/security/apparmor/domain.c index cf9feed1306de..8e478ac99ff6d 100644 --- a/security/apparmor/domain.c +++ b/security/apparmor/domain.c @@ -116,7 +116,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @label: label to check access permissions for * @stack: whether this is a stacking request * @state: state to start match in - * @subns: whether to do permission checks on components in a subns + * @inview: whether to match labels in view or only in scope * @request: permissions to request * @perms: perms struct to set * @@ -128,7 +128,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, */ static int label_compound_match(struct aa_profile *profile, struct aa_label *label, bool stack, - aa_state_t state, bool subns, u32 request, + aa_state_t state, bool inview, u32 request, struct aa_perms *perms) { struct aa_ruleset *rules = profile->label.rules[0]; @@ -136,9 +136,9 @@ static int label_compound_match(struct aa_profile *profile, struct label_it i; struct path_cond cond = { }; - /* find first subcomponent that is visible */ + /* find first subcomponent that is in view and going to be interated with */ label_for_each(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, tp, stack, state); if (!state) @@ -152,7 +152,7 @@ static int label_compound_match(struct aa_profile *profile, next: label_for_each_cont(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = aa_dfa_match(rules->file->dfa, state, "//&"); state = match_component(profile, tp, false, state); @@ -178,7 +178,7 @@ static int label_compound_match(struct aa_profile *profile, * @label: label to check access permissions for * @stack: whether this is a stacking request * @start: state to start match in - * @subns: whether to do permission checks on components in a subns + * @inview: whether to match labels in view or only in scope * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * @@ -190,7 +190,7 @@ static int label_compound_match(struct aa_profile *profile, */ static int label_components_match(struct aa_profile *profile, struct aa_label *label, bool stack, - aa_state_t start, bool subns, u32 request, + aa_state_t start, bool inview, u32 request, struct aa_perms *perms) { struct aa_ruleset *rules = profile->label.rules[0]; @@ -202,7 +202,7 @@ static int label_components_match(struct aa_profile *profile, /* find first subcomponent to test */ label_for_each(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, tp, stack, start); if (!state) @@ -219,7 +219,7 @@ static int label_components_match(struct aa_profile *profile, aa_apply_modes_to_perms(profile, &tmp); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, tp, stack, start); if (!state) @@ -246,26 +246,26 @@ static int label_components_match(struct aa_profile *profile, * @label: label to match (NOT NULL) * @stack: whether this is a stacking request * @state: state to start in - * @subns: whether to match subns components + * @inview: whether to match labels in view or only in scope * @request: permission request * @perms: Returns computed perms (NOT NULL) * * Returns: the state the match finished in, may be the none matching state */ static int label_match(struct aa_profile *profile, struct aa_label *label, - bool stack, aa_state_t state, bool subns, u32 request, + bool stack, aa_state_t state, bool inview, u32 request, struct aa_perms *perms) { int error; *perms = nullperms; - error = label_compound_match(profile, label, stack, state, subns, + error = label_compound_match(profile, label, stack, state, inview, request, perms); if (!error) return error; *perms = allperms; - return label_components_match(profile, label, stack, state, subns, + return label_components_match(profile, label, stack, state, inview, request, perms); } @@ -864,6 +864,61 @@ static int profile_onexec(const struct cred *subj_cred, } /* ensure none ns domain transitions are correctly applied with onexec */ +static struct aa_label *label_merge_wrap(struct aa_label *a, struct aa_label *b, + gfp_t gfp) +{ + struct aa_label *label = aa_label_merge(a, b, gfp); + + if (!label) + return ERR_PTR(-ENOMEM); + return label; +} + +static bool is_profile_priv_restricted_to_stack(const struct cred *subj_cred, + struct aa_profile *profile) +{ + if (profile_unconfined(profile) && profile == profile->ns->unconfined && + aa_unprivileged_unconfined_restricted && + /* cap_capable returns false (0) if true, hence true here means + * doesn't have capability and the stack will be restricted + */ + cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, + CAP_OPT_NOAUDIT)) + return true; + return false; +} + +static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; + +static struct aa_label *priv_restricted_transition(const struct cred *subj_cred, + struct aa_profile *profile, + const char *op, u32 request, + const char *name, + struct aa_label *transition, + gfp_t gfp) +{ + if (!is_profile_priv_restricted_to_stack(subj_cred, profile)) + return aa_get_newest_label(transition); + + /* transition allowed but only via stack */ + struct aa_label *target = label_merge_wrap(&profile->label, + transition, gfp); + if (IS_ERR_OR_NULL(target)) + return target; + + /* doing this here is less than optimal but good enough until the + * fs mediation rework lands + */ + struct aa_perms perms = { + .allow = request, + .audit = request, + }; + aa_audit_file(subj_cred, profile, &perms, op, + request, name, NULL, target, + subj_cred->euid, stack_msg, 0); + + return target; +} static struct aa_label *handle_onexec(const struct cred *subj_cred, struct aa_label *label, @@ -881,29 +936,36 @@ static struct aa_label *handle_onexec(const struct cred *subj_cred, AA_BUG(!bprm); AA_BUG(!buffer); - /* TODO: determine how much we want to loosen this */ - error = fn_for_each_in_ns(label, profile, + /* TODO: determine how much we want to loosen this + * only check profiles in scope for permission to change at exec + */ + error = fn_for_each_in_scope(label, profile, profile_onexec(subj_cred, profile, onexec, stack, bprm, buffer, cond, unsafe)); if (error) return ERR_PTR(error); - new = fn_label_build_in_ns(label, profile, GFP_KERNEL, - stack ? aa_label_merge(&profile->label, onexec, - GFP_KERNEL) - : aa_get_newest_label(onexec), + new = fn_label_build_in_scope(label, profile, GFP_KERNEL, + stack ? label_merge_wrap(&profile->label, onexec, + GFP_KERNEL) + : priv_restricted_transition(subj_cred, profile, + OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, + bprm->filename, onexec, + GFP_KERNEL), profile_transition(subj_cred, profile, bprm, buffer, cond, unsafe)); - if (new) + AA_BUG(!new); + if (!IS_ERR(new)) return new; /* TODO: get rid of GLOBAL_ROOT_UID */ - error = fn_for_each_in_ns(label, profile, + error = fn_for_each_in_scope(label, profile, aa_audit_file(subj_cred, profile, &nullperms, OP_CHANGE_ONEXEC, AA_MAY_ONEXEC, bprm->filename, NULL, onexec, GLOBAL_ROOT_UID, - "failed to build target label", -ENOMEM)); + "failed to build target label", + PTR_ERR(new))); return ERR_PTR(error); } @@ -966,14 +1028,10 @@ int apparmor_bprm_creds_for_exec(struct linux_binprm *bprm) profile_transition(subj_cred, profile, bprm, buffer, &cond, &unsafe)); - AA_BUG(!new); if (IS_ERR(new)) { error = PTR_ERR(new); goto done; - } else if (!new) { - error = -ENOMEM; - goto done; } /* Policy has specified a domain transitions. If no_new_privs and @@ -1075,8 +1133,8 @@ static struct aa_label *build_change_hat(const struct cred *subj_cred, if (!hat) { error = -ENOENT; if (COMPLAIN_MODE(profile)) { - hat = aa_new_learning_profile(profile, true, name, - GFP_KERNEL); + hat = __aa_new_learning_profile(profile, true, name, + GFP_KERNEL); if (!hat) { info = "failed null profile create"; error = -ENOMEM; @@ -1114,6 +1172,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, bool sibling = false; const char *name, *info = NULL; int i, error; + bool needput = false; AA_BUG(!label); AA_BUG(!hats); @@ -1126,7 +1185,6 @@ static struct aa_label *change_hat(const struct cred *subj_cred, * the profiles and label, we can rely on the namespaces being live * and avoid incrementing their refcounts while grabbing the lock. */ - label = aa_get_label(label); ns = labels_ns(label); retry: @@ -1134,15 +1192,19 @@ static struct aa_label *change_hat(const struct cred *subj_cred, if (label_is_stale(label)) { new = aa_get_newest_label(label); new_ns = labels_ns(new); + + if (needput) + /* aa_put_label() is safe to call when under lock */ + aa_put_label(label); + label = new; + needput = true; + /* check if replaced with label in parent ns, and lock there */ if (new_ns != ns) { - aa_put_label(new); mutex_unlock(&ns->lock); ns = new_ns; - label = new; + /* retry will bottom out at the root of the tree */ goto retry; } - aa_put_label(label); - label = new; } if (PROFILE_IS_HAT(labels_profile(label))) @@ -1151,9 +1213,10 @@ static struct aa_label *change_hat(const struct cred *subj_cred, /*find first matching hat */ for (i = 0; i < count && !hat; i++) { name = hats[i]; - label_for_each_in_ns(it, labels_ns(label), label, profile) { + label_for_each_in_scope(it, labels_ns(label), label, profile) { if (sibling && PROFILE_IS_HAT(profile)) { - root = aa_get_profile(profile->parent); + root = aa_get_profile(rcu_dereference_protected(profile->parent, + mutex_is_locked(&ns->lock))); } else if (!sibling && !PROFILE_IS_HAT(profile)) { root = aa_get_profile(profile); } else { /* conflicting change type */ @@ -1187,7 +1250,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, * change_hat. */ name = NULL; - label_for_each_in_ns(it, labels_ns(label), label, profile) { + label_for_each_in_scope(it, labels_ns(label), label, profile) { if (!list_empty(&profile->base.profiles)) { info = "hat not found"; error = -ENOENT; @@ -1198,7 +1261,7 @@ static struct aa_label *change_hat(const struct cred *subj_cred, error = -ECHILD; fail: - label_for_each_in_ns(it, labels_ns(label), label, profile) { + label_for_each_in_scope(it, labels_ns(label), label, profile) { /* * no target as it has failed to be found or built * @@ -1214,19 +1277,20 @@ static struct aa_label *change_hat(const struct cred *subj_cred, } } mutex_unlock(&ns->lock); + if (needput) + aa_put_label(label); return ERR_PTR(error); build: - new = fn_label_build_in_ns(label, profile, GFP_KERNEL, + new = fn_label_build_in_scope(label, profile, GFP_KERNEL, build_change_hat(subj_cred, profile, name, sibling), aa_get_label(&profile->label)); - if (!new) { - info = "label build failed"; - error = -ENOMEM; - goto fail; - } /* else if (IS_ERR) build_change_hat has logged error so return new */ mutex_unlock(&ns->lock); + AA_BUG(!new); + /* return new label or error ptr */ + if (needput) + aa_put_label(label); return new; } @@ -1280,7 +1344,7 @@ int aa_change_hat(const char *hats[], int count, u64 token, int flags) bool empty = true; rcu_read_lock(); - label_for_each_in_ns(i, labels_ns(label), label, profile) { + label_for_each_in_scope(i, labels_ns(label), label, profile) { empty &= list_empty(&profile->base.profiles); } rcu_read_unlock(); @@ -1367,7 +1431,7 @@ int aa_change_hat(const char *hats[], int count, u64 token, int flags) perms.kill = AA_MAY_CHANGEHAT; fail: - fn_for_each_in_ns(label, profile, + fn_for_each_in_scope(label, profile, aa_audit_file(subj_cred, profile, &perms, OP_CHANGE_HAT, AA_MAY_CHANGEHAT, NULL, NULL, target, GLOBAL_ROOT_UID, info, error)); @@ -1399,8 +1463,6 @@ static int change_profile_perms_wrapper(const char *op, const char *name, return error; } -static const char *stack_msg = "change_profile unprivileged unconfined converted to stacking"; - /** * aa_change_profile - perform a one-way profile transition * @fqname: name of profile may include namespace (NOT NULL) @@ -1460,28 +1522,6 @@ int aa_change_profile(const char *fqname, int flags) op = OP_CHANGE_PROFILE; } - /* This should move to a per profile test. Requires pushing build - * into callback - */ - if (!stack && unconfined(label) && - label == &labels_ns(label)->unconfined->label && - aa_unprivileged_unconfined_restricted && - /* TODO: refactor so this check is a fn */ - cap_capable(current_cred(), &init_user_ns, CAP_MAC_OVERRIDE, - CAP_OPT_NOAUDIT)) { - /* regardless of the request in this case apparmor - * stacks against unconfined so admin set policy can't be - * by-passed - */ - stack = true; - perms.audit = request; - (void) fn_for_each_in_ns(label, profile, - aa_audit_file(subj_cred, profile, &perms, op, - request, auditname, NULL, target, - GLOBAL_ROOT_UID, stack_msg, 0)); - perms.audit = 0; - } - if (*fqname == '&') { stack = true; /* don't have label_parse() do stacking */ @@ -1521,7 +1561,7 @@ int aa_change_profile(const char *fqname, int flags) * * if (!stack) { */ - error = fn_for_each_in_ns(label, profile, + error = fn_for_each_in_scope(label, profile, change_profile_perms_wrapper(op, auditname, subj_cred, profile, target, stack, @@ -1535,7 +1575,7 @@ int aa_change_profile(const char *fqname, int flags) check: /* check if tracing task is allowed to trace target domain */ error = may_change_ptraced_domain(subj_cred, target, &info); - if (error && !fn_for_each_in_ns(label, profile, + if (error && !fn_for_each_in_scope(label, profile, COMPLAIN_MODE(profile))) goto audit; @@ -1551,10 +1591,14 @@ int aa_change_profile(const char *fqname, int flags) /* stacking is always a subset, so only check the nonstack case */ if (!stack) { - new = fn_label_build_in_ns(label, profile, GFP_KERNEL, - aa_get_label(target), + new = fn_label_build_in_scope(label, profile, GFP_KERNEL, + priv_restricted_transition(subj_cred, profile, + op, request, + auditname, target, + GFP_KERNEL), aa_get_label(&profile->label)); - if (IS_ERR_OR_NULL(new)) + AA_BUG(!new); + if (IS_ERR(new)) goto build_fail; /* * no new privs prevents domain transitions that would @@ -1578,10 +1622,9 @@ int aa_change_profile(const char *fqname, int flags) goto build_fail; error = aa_replace_current_label(new); } else { - if (new) { - aa_put_label(new); - new = NULL; - } + /* new will be recomputed so at exec time. So discard */ + aa_put_label(new); + new = NULL; /* full transition will be built in exec path */ aa_set_current_onexec(target, stack); @@ -1599,7 +1642,7 @@ int aa_change_profile(const char *fqname, int flags) perms.allow = 0; audit: - error = fn_for_each_in_ns(label, profile, + error = fn_for_each_in_scope(label, profile, aa_audit_file(subj_cred, profile, &perms, op, request, auditname, NULL, new ? new : target, diff --git a/security/apparmor/include/cred.h b/security/apparmor/include/cred.h index b028e4c13b6f7..d37f33f0eec89 100644 --- a/security/apparmor/include/cred.h +++ b/security/apparmor/include/cred.h @@ -184,13 +184,9 @@ static inline struct aa_label *begin_current_label_crit_section(void) { struct aa_label *label = aa_current_raw_label(); - might_sleep(); - if (label_is_stale(label)) { label = aa_get_newest_label(label); - if (aa_replace_current_label(label) == 0) - /* task cred will keep the reference */ - aa_put_label(label); + aa_schedule_stale_label_replacement(); } return label; diff --git a/security/apparmor/include/label.h b/security/apparmor/include/label.h index 335f21930702a..574be681f83e9 100644 --- a/security/apparmor/include/label.h +++ b/security/apparmor/include/label.h @@ -23,7 +23,7 @@ struct aa_ruleset; #define LOCAL_VEC_ENTRIES 8 #define DEFINE_VEC(T, V) \ - struct aa_ ## T *(_ ## V ## _localtmp)[LOCAL_VEC_ENTRIES]; \ + struct aa_ ## T *(_ ## V ## _localtmp)[LOCAL_VEC_ENTRIES + 1]; \ struct aa_ ## T **(V) #define vec_setup(T, V, N, GFP) \ @@ -31,10 +31,10 @@ struct aa_ruleset; if ((N) <= LOCAL_VEC_ENTRIES) { \ typeof(N) i; \ (V) = (_ ## V ## _localtmp); \ - for (i = 0; i < (N); i++) \ + for (i = 0; i <= (N); i++) \ (V)[i] = NULL; \ } else \ - (V) = kzalloc(sizeof(struct aa_ ## T *) * (N), (GFP)); \ + (V) = kzalloc_objs(struct aa_ ## T *, (N) + 1, (GFP)); \ (V) ? 0 : -ENOMEM; \ }) diff --git a/security/apparmor/include/lib.h b/security/apparmor/include/lib.h index 26df19c1df4f8..1721782c6f923 100644 --- a/security/apparmor/include/lib.h +++ b/security/apparmor/include/lib.h @@ -76,6 +76,19 @@ int aa_print_debug_params(char *buffer); /* Flag indicating whether initialization completed */ extern int apparmor_initialized; +/* semantic split of scope and view */ +#define aa_in_scope(SUBJ, OBJ) \ + aa_ns_visible(SUBJ, OBJ, false) + +#define aa_in_view(SUBJ, OBJ) \ + aa_ns_visible(SUBJ, OBJ, true) + +#define label_for_each_in_scope(I, NS, L, P) \ + label_for_each_in_ns(I, NS, L, P) + +#define fn_for_each_in_scope(L, P, FN) \ + fn_for_each_in_ns(L, P, FN) + /* fn's in lib */ const char *skipn_spaces(const char *str, size_t n); const char *aa_splitn_fqname(const char *fqname, size_t n, const char **ns_name, @@ -259,7 +272,6 @@ void aa_policy_destroy(struct aa_policy *policy); * * Returns: new label on success * ERR_PTR if build @FN fails - * NULL if label_build fails due to low memory conditions * * @FN must return a label or ERR_PTR on failure. NULL is not allowed */ @@ -275,7 +287,7 @@ void aa_policy_destroy(struct aa_policy *policy); DEFINE_VEC(label, __lvec); \ DEFINE_VEC(profile, __pvec); \ if (vec_setup(label, __lvec, (L)->size, (GFP))) { \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ goto __done; \ } \ __j = 0; \ @@ -297,28 +309,29 @@ void aa_policy_destroy(struct aa_policy *policy); if (__count > 1) { \ __new_ = aa_vec_find_or_create_label(__pvec,\ __count, (GFP)); \ - /* only fails if out of Mem */ \ if (!__new_) \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ } else \ __new_ = aa_get_label(&__pvec[0]->label); \ vec_cleanup(profile, __pvec, __count); \ } else \ - __new_ = NULL; \ + __new_ = ERR_PTR(-ENOMEM); \ __do_cleanup: \ vec_cleanup(label, __lvec, (L)->size); \ } else { \ (P) = labels_profile(L); \ __new_ = (FN); \ + AA_BUG(!__new_); \ } \ __done: \ - if (!__new_) \ + if (PTR_ERR(__new_)) \ AA_DEBUG(DEBUG_LABEL, "label build failed\n"); \ + AA_BUG(!__new_); \ (__new_); \ }) -#define __fn_build_in_ns(NS, P, NS_FN, OTHER_FN) \ +#define __fn_build_in_scope(NS, P, NS_FN, OTHER_FN) \ ({ \ struct aa_label *__new; \ if ((P)->ns != (NS)) \ @@ -328,10 +341,10 @@ __done: \ (__new); \ }) -#define fn_label_build_in_ns(L, P, GFP, NS_FN, OTHER_FN) \ +#define fn_label_build_in_scope(L, P, GFP, NS_FN, OTHER_FN) \ ({ \ fn_label_build((L), (P), (GFP), \ - __fn_build_in_ns(labels_ns(L), (P), (NS_FN), (OTHER_FN))); \ + __fn_build_in_scope(labels_ns(L), (P), (NS_FN), (OTHER_FN))); \ }) #endif /* __AA_LIB_H */ diff --git a/security/apparmor/include/policy.h b/security/apparmor/include/policy.h index bf105ae9019d3..2eaa1bee31bc0 100644 --- a/security/apparmor/include/policy.h +++ b/security/apparmor/include/policy.h @@ -271,6 +271,9 @@ struct aa_profile *aa_alloc_profile(const char *name, struct aa_proxy *proxy, gfp_t gfp); struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, gfp_t gfp); +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp); struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, const char *base, gfp_t gfp); void aa_free_profile(struct aa_profile *profile); diff --git a/security/apparmor/include/task.h b/security/apparmor/include/task.h index b1aaaf60fa8b8..6f26758ca10f0 100644 --- a/security/apparmor/include/task.h +++ b/security/apparmor/include/task.h @@ -21,15 +21,22 @@ static inline struct aa_task_ctx *task_ctx(struct task_struct *task) * @onexec: profile to transition to on next exec (MAY BE NULL) * @previous: profile the task may return to (MAY BE NULL) * @token: magic value the task must know for returning to @previous_profile + * @label_replacement_tw: for aa_schedule_stale_label_replacement() + * @label_replacement_pending: is @label_replacement_tw pending? + * + * When changing this, check if aa_dup_task_ctx() needs to be updated. */ struct aa_task_ctx { struct aa_label *nnp; struct aa_label *onexec; struct aa_label *previous; u64 token; + struct callback_head label_replacement_tw; + bool label_replacement_pending; }; int aa_replace_current_label(struct aa_label *label); +void aa_schedule_stale_label_replacement(void); void aa_set_current_onexec(struct aa_label *label, bool stack); int aa_set_current_hat(struct aa_label *label, u64 token); int aa_restore_previous_label(u64 cookie); @@ -56,10 +63,10 @@ static inline void aa_free_task_ctx(struct aa_task_ctx *ctx) static inline void aa_dup_task_ctx(struct aa_task_ctx *new, const struct aa_task_ctx *old) { - *new = *old; - aa_get_label(new->nnp); - aa_get_label(new->previous); - aa_get_label(new->onexec); + new->nnp = aa_get_label(old->nnp); + new->onexec = aa_get_label(old->onexec); + new->previous = aa_get_label(old->previous); + new->token = old->token; } /** diff --git a/security/apparmor/label.c b/security/apparmor/label.c index d382ac772ca1d..5c42cd01ea473 100644 --- a/security/apparmor/label.c +++ b/security/apparmor/label.c @@ -1278,7 +1278,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, * @rules: ruleset to search * @label: label to check access permissions for * @state: state to start match in - * @subns: whether to do permission checks on components in a subns + * @inview: whether to match labels in view or only in scope * @request: permissions to request * @perms: perms struct to set * @@ -1291,7 +1291,7 @@ static inline aa_state_t match_component(struct aa_profile *profile, static int label_compound_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, - aa_state_t state, bool subns, u32 request, + aa_state_t state, bool inview, u32 request, struct aa_perms *perms) { struct aa_profile *tp; @@ -1299,7 +1299,7 @@ static int label_compound_match(struct aa_profile *profile, /* find first subcomponent that is visible */ label_for_each(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, rules, tp, state); if (!state) @@ -1313,7 +1313,7 @@ static int label_compound_match(struct aa_profile *profile, next: label_for_each_cont(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = aa_dfa_match(rules->policy->dfa, state, "//&"); state = match_component(profile, rules, tp, state); @@ -1334,7 +1334,7 @@ static int label_compound_match(struct aa_profile *profile, * @rules: ruleset to search * @label: label to check access permissions for * @start: state to start match in - * @subns: whether to do permission checks on components in a subns + * @inview: whether to match labels in view or only in scope * @request: permissions to request * @perms: an initialized perms struct to add accumulation to * @@ -1347,7 +1347,7 @@ static int label_compound_match(struct aa_profile *profile, static int label_components_match(struct aa_profile *profile, struct aa_ruleset *rules, struct aa_label *label, aa_state_t start, - bool subns, u32 request, + bool inview, u32 request, struct aa_perms *perms) { struct aa_profile *tp; @@ -1357,7 +1357,7 @@ static int label_components_match(struct aa_profile *profile, /* find first subcomponent to test */ label_for_each(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, rules, tp, start); if (!state) @@ -1372,7 +1372,7 @@ static int label_components_match(struct aa_profile *profile, tmp = *aa_lookup_perms(rules->policy, state); aa_perms_accum(perms, &tmp); label_for_each_cont(i, label, tp) { - if (!aa_ns_visible(profile->ns, tp->ns, subns)) + if (!aa_ns_visible(profile->ns, tp->ns, inview)) continue; state = match_component(profile, rules, tp, start); if (!state) @@ -1397,24 +1397,24 @@ static int label_components_match(struct aa_profile *profile, * @rules: ruleset to search * @label: label to match (NOT NULL) * @state: state to start in - * @subns: whether to match subns components + * @inview: whether to match labels in view or only in scope * @request: permission request * @perms: Returns computed perms (NOT NULL) * * Returns: the state the match finished in, may be the none matching state */ int aa_label_match(struct aa_profile *profile, struct aa_ruleset *rules, - struct aa_label *label, aa_state_t state, bool subns, + struct aa_label *label, aa_state_t state, bool inview, u32 request, struct aa_perms *perms) { - aa_state_t tmp = label_compound_match(profile, rules, label, state, subns, - request, perms); + aa_state_t tmp = label_compound_match(profile, rules, label, state, + inview, request, perms); if ((perms->allow & request) == request) return tmp; /* failed compound_match try component matches */ *perms = allperms; - return label_components_match(profile, rules, label, state, subns, + return label_components_match(profile, rules, label, state, inview, request, perms); } diff --git a/security/apparmor/lib.c b/security/apparmor/lib.c index 82dbb97ad4065..5d53c15e46a9e 100644 --- a/security/apparmor/lib.c +++ b/security/apparmor/lib.c @@ -478,24 +478,22 @@ bool aa_policy_init(struct aa_policy *policy, const char *prefix, const char *name, gfp_t gfp) { char *hname; + size_t hname_sz; + INIT_LIST_HEAD(&policy->list); + INIT_LIST_HEAD(&policy->profiles); + hname_sz = (prefix ? strlen(prefix) + 2 : 0) + strlen(name) + 1; /* freed by policy_free */ - if (prefix) { - hname = aa_str_alloc(strlen(prefix) + strlen(name) + 3, gfp); - if (hname) - sprintf(hname, "%s//%s", prefix, name); - } else { - hname = aa_str_alloc(strlen(name) + 1, gfp); - if (hname) - strcpy(hname, name); - } + hname = aa_str_alloc(hname_sz, gfp); if (!hname) return false; + if (prefix) + scnprintf(hname, hname_sz, "%s//%s", prefix, name); + else + strscpy(hname, name, hname_sz); policy->hname = hname; /* base.name is a substring of fqname */ policy->name = basename(policy->hname); - INIT_LIST_HEAD(&policy->list); - INIT_LIST_HEAD(&policy->profiles); return true; } diff --git a/security/apparmor/match.c b/security/apparmor/match.c index 0de249725efbf..b2a4bee4e3939 100644 --- a/security/apparmor/match.c +++ b/security/apparmor/match.c @@ -27,13 +27,13 @@ * @blob: data to unpack (NOT NULL) * @bsize: size of blob * - * Returns: pointer to table else NULL on failure + * Returns: pointer to table else ERR_PTR on failure * * NOTE: must be freed by kvfree (not kfree) */ static struct table_header *unpack_table(char *blob, size_t bsize) { - struct table_header *table = NULL; + struct table_header *table = ERR_PTR(-EPROTO); struct table_header th; size_t tsize; @@ -74,20 +74,21 @@ static struct table_header *unpack_table(char *blob, size_t bsize) else if (th.td_flags == YYTD_DATA32) UNPACK_ARRAY(table->td_data, blob, th.td_lolen, u32, __be32, get_unaligned_be32); - else - goto fail; + else { + kvfree(table); + table = ERR_PTR(-EPROTO); + goto out; + } /* if table was vmalloced make sure the page tables are synced * before it is used, as it goes live to all cpus. */ if (is_vmalloc_addr(table)) vm_unmap_aliases(); - } + } else + table = ERR_PTR(-ENOMEM); out: return table; -fail: - kvfree(table); - return NULL; } /** @@ -359,8 +360,11 @@ struct aa_dfa *aa_dfa_unpack(void *blob, size_t size, int flags) while (size > 0) { table = unpack_table(data, size); - if (!table) + if (IS_ERR(table)) { + error = PTR_ERR(table); + table = NULL; goto fail; + } switch (table->td_id) { case YYTD_ID_ACCEPT: diff --git a/security/apparmor/mount.c b/security/apparmor/mount.c index 523570aa1a5a1..2f5d918832c10 100644 --- a/security/apparmor/mount.c +++ b/security/apparmor/mount.c @@ -735,17 +735,11 @@ int aa_pivotroot(const struct cred *subj_cred, struct aa_label *label, build_pivotroot(subj_cred, profile, new_path, new_buffer, old_path, old_buffer)); - if (!target) { - info = "label build failed"; - error = -ENOMEM; - goto fail; - } else if (!IS_ERR(target)) { + AA_BUG(!target); + if (!IS_ERR(target)) { error = aa_replace_current_label(target); - if (error) { - /* TODO: audit target */ - aa_put_label(target); - goto out; - } + if (error) + goto fail; aa_put_label(target); } else /* already audited error */ @@ -763,7 +757,8 @@ int aa_pivotroot(const struct cred *subj_cred, struct aa_label *label, NULL /*new_name */, NULL /* old_name */, NULL, NULL, - 0, NULL, AA_MAY_PIVOTROOT, &nullperms, info, + 0, target->hname, AA_MAY_PIVOTROOT, &nullperms, info, error)); + aa_put_label(target); goto out; } diff --git a/security/apparmor/policy.c b/security/apparmor/policy.c index b0203b30db606..70d6315a059af 100644 --- a/security/apparmor/policy.c +++ b/security/apparmor/policy.c @@ -698,7 +698,7 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, } /** - * aa_new_learning_profile - create or find a null-X learning profile + * __aa_new_learning_profile - create or find a null-X learning profile * @parent: profile that caused this profile to be created (NOT NULL) * @hat: true if the null- learning profile is a hat * @base: name to base the null profile off of @@ -715,14 +715,16 @@ struct aa_profile *aa_alloc_null(struct aa_profile *parent, const char *name, * * Returns: new refcounted profile else NULL on failure */ -struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, - const char *base, gfp_t gfp) +struct aa_profile *__aa_new_learning_profile(struct aa_profile *parent, + bool hat, const char *base, + gfp_t gfp) { struct aa_profile *p, *profile; const char *bname; char *name = NULL; AA_BUG(!parent); + AA_BUG(!mutex_is_locked(&parent->ns->lock)); if (base) { name = kmalloc(strlen(parent->base.hname) + 8 + strlen(base), @@ -754,7 +756,6 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, if (hat) profile->label.flags |= FLAG_HAT; - mutex_lock_nested(&profile->ns->lock, profile->ns->level); p = __find_child(&parent->base.profiles, bname); if (p) { aa_free_profile(profile); @@ -762,7 +763,6 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, } else { __add_profile(&parent->base.profiles, profile); } - mutex_unlock(&profile->ns->lock); /* refcount released by caller */ out: @@ -776,6 +776,18 @@ struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, return NULL; } +struct aa_profile *aa_new_learning_profile(struct aa_profile *parent, bool hat, + const char *base, gfp_t gfp) +{ + struct aa_profile *profile; + + mutex_lock_nested(&parent->ns->lock, parent->ns->level); + profile = __aa_new_learning_profile(parent, hat, base, gfp); + mutex_unlock(&parent->ns->lock); + + return profile; +} + /** * replacement_allowed - test to see if replacement is allowed * @profile: profile to test if it can be replaced (MAYBE NULL) diff --git a/security/apparmor/task.c b/security/apparmor/task.c index c9bc9cc694750..f9519baa4e60f 100644 --- a/security/apparmor/task.c +++ b/security/apparmor/task.c @@ -14,6 +14,7 @@ #include #include +#include #include "include/audit.h" #include "include/cred.h" @@ -88,6 +89,32 @@ int aa_replace_current_label(struct aa_label *label) return 0; } +static void aa_replace_stale_label_tw_func(struct callback_head *tw) +{ + struct aa_task_ctx *ctx = task_ctx(current); + struct aa_label *label; + + ctx->label_replacement_pending = false; + label = aa_current_raw_label(); + if (!label_is_stale(label)) + return; + label = aa_get_newest_label(label); + aa_replace_current_label(label); + aa_put_label(label); +} + +/* replace the current task's stale label on syscall return */ +void aa_schedule_stale_label_replacement(void) +{ + struct aa_task_ctx *ctx = task_ctx(current); + + if (ctx->label_replacement_pending) + return; + init_task_work(&ctx->label_replacement_tw, aa_replace_stale_label_tw_func); + if (task_work_add(current, &ctx->label_replacement_tw, TWA_RESUME) == 0) + ctx->label_replacement_pending = true; +} + /** * aa_set_current_onexec - set the tasks change_profile to happen onexec diff --git a/security/integrity/Makefile b/security/integrity/Makefile index 92b63039c654c..45dfdedbdad48 100644 --- a/security/integrity/Makefile +++ b/security/integrity/Makefile @@ -18,6 +18,7 @@ integrity-$(CONFIG_LOAD_IPL_KEYS) += platform_certs/load_ipl_s390.o integrity-$(CONFIG_LOAD_PPC_KEYS) += platform_certs/efi_parser.o \ platform_certs/load_powerpc.o \ platform_certs/keyring_handler.o +integrity-$(CONFIG_EFI) += efi_secureboot.o # The relative order of the 'ima' and 'evm' LSMs depends on the order below. obj-$(CONFIG_IMA) += ima/ obj-$(CONFIG_EVM) += evm/ diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c index 457c0a396caf3..17219424a34dd 100644 --- a/security/integrity/digsig_asymmetric.c +++ b/security/integrity/digsig_asymmetric.c @@ -108,6 +108,10 @@ int asymmetric_verify(struct key *keyring, const char *sig, pks.hash_algo = hash_algo_name[hdr->hash_algo]; pk = asymmetric_key_public_key(key); + if (!pk) { + ret = -ENOKEY; + goto out; + } pks.pkey_algo = pk->pkey_algo; if (!strcmp(pk->pkey_algo, "rsa")) { pks.encoding = "pkcs1"; diff --git a/security/integrity/efi_secureboot.c b/security/integrity/efi_secureboot.c new file mode 100644 index 0000000000000..bfd4260a83a33 --- /dev/null +++ b/security/integrity/efi_secureboot.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-1.0+ +/* + * Copyright (C) 2018 IBM Corporation + */ +#include +#include +#include + +#ifndef arch_efi_boot_mode +#define arch_efi_boot_mode efi_secureboot_mode_unset +#endif + +static enum efi_secureboot_mode get_sb_mode(void) +{ + enum efi_secureboot_mode mode; + + if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE)) { + pr_info("integrity: secureboot mode unknown, no efi\n"); + return efi_secureboot_mode_unknown; + } + + mode = efi_get_secureboot_mode(efi.get_variable); + if (mode == efi_secureboot_mode_disabled) + pr_info("integrity: secureboot mode disabled\n"); + else if (mode == efi_secureboot_mode_unknown) + pr_info("integrity: secureboot mode unknown\n"); + else + pr_info("integrity: secureboot mode enabled\n"); + return mode; +} + +/* + * Query secure boot status + * + * Note don't call this function too early e.g. in __setup hook otherwise the + * kernel may hang when calling efi_get_secureboot_mode. + * + */ +bool arch_get_secureboot(void) +{ + static enum efi_secureboot_mode sb_mode; + static bool initialized; + + if (!initialized && efi_enabled(EFI_BOOT)) { + sb_mode = arch_efi_boot_mode; + + if (sb_mode == efi_secureboot_mode_unset) + sb_mode = get_sb_mode(); + initialized = true; + } + + if (sb_mode == efi_secureboot_mode_enabled) + return true; + else + return false; +} diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 5149ff4fd50d2..a87ef7c05f11e 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -27,7 +27,7 @@ core_param(ima_appraise, ima_appraise_cmdline_default, charp, 0); void __init ima_appraise_parse_cmdline(void) { const char *str = ima_appraise_cmdline_default; - bool sb_state = arch_ima_get_secureboot(); + bool sb_state = arch_get_secureboot(); int appraisal_state = ima_appraise; if (!str) @@ -90,6 +90,11 @@ static int ima_fix_xattr(struct dentry *dentry, struct ima_iint_cache *iint) int rc, offset; u8 algo = iint->ima_hash->algo; + if (IS_RDONLY(d_inode(dentry))) + return -EROFS; + if (IS_IMMUTABLE(d_inode(dentry))) + return -EPERM; + if (algo <= HASH_ALGO_SHA1) { offset = 1; iint->ima_hash->xattr.sha1.type = IMA_XATTR_DIGEST; @@ -303,8 +308,13 @@ static int xattr_verify(enum ima_hooks func, struct ima_iint_cache *iint, } else { set_bit(IMA_DIGSIG, &iint->atomic_flags); } - if (xattr_len - sizeof(xattr_value->type) - hash_start >= - iint->ima_hash->length) + /* + * Use addition, not subtraction: sizeof() forces unsigned + * math and a short xattr_len would wrap around, bypassing + * this bounds check. + */ + if (xattr_len >= (int)sizeof(xattr_value->type) + hash_start + + (int)iint->ima_hash->length) /* * xattr length may be longer. md5 hash in previous * version occupied 20 bytes in xattr, instead of 16 @@ -772,6 +782,8 @@ static int validate_hash_algo(struct dentry *dentry, return -EACCES; path = dentry_path(dentry, pathbuf, PATH_MAX); + if (IS_ERR(path)) + path = NULL; integrity_audit_msg(AUDIT_INTEGRITY_DATA, d_inode(dentry), path, "set_data", errmsg, -EACCES, 0); diff --git a/security/integrity/ima/ima_efi.c b/security/integrity/ima/ima_efi.c index 138029bfcce1e..78191879dd98f 100644 --- a/security/integrity/ima/ima_efi.c +++ b/security/integrity/ima/ima_efi.c @@ -2,52 +2,9 @@ /* * Copyright (C) 2018 IBM Corporation */ -#include #include #include -#include - -#ifndef arch_ima_efi_boot_mode -#define arch_ima_efi_boot_mode efi_secureboot_mode_unset -#endif - -static enum efi_secureboot_mode get_sb_mode(void) -{ - enum efi_secureboot_mode mode; - - if (!efi_rt_services_supported(EFI_RT_SUPPORTED_GET_VARIABLE)) { - pr_info("ima: secureboot mode unknown, no efi\n"); - return efi_secureboot_mode_unknown; - } - - mode = efi_get_secureboot_mode(efi.get_variable); - if (mode == efi_secureboot_mode_disabled) - pr_info("ima: secureboot mode disabled\n"); - else if (mode == efi_secureboot_mode_unknown) - pr_info("ima: secureboot mode unknown\n"); - else - pr_info("ima: secureboot mode enabled\n"); - return mode; -} - -bool arch_ima_get_secureboot(void) -{ - static enum efi_secureboot_mode sb_mode; - static bool initialized; - - if (!initialized && efi_enabled(EFI_BOOT)) { - sb_mode = arch_ima_efi_boot_mode; - - if (sb_mode == efi_secureboot_mode_unset) - sb_mode = get_sb_mode(); - initialized = true; - } - - if (sb_mode == efi_secureboot_mode_enabled) - return true; - else - return false; -} +#include /* secureboot arch rules */ static const char * const sb_arch_rules[] = { @@ -67,7 +24,7 @@ static const char * const sb_arch_rules[] = { const char * const *arch_get_ima_policy(void) { - if (IS_ENABLED(CONFIG_IMA_ARCH_POLICY) && arch_ima_get_secureboot()) { + if (IS_ENABLED(CONFIG_IMA_ARCH_POLICY) && arch_get_secureboot()) { if (IS_ENABLED(CONFIG_MODULE_SIG)) set_module_sig_enforced(); if (IS_ENABLED(CONFIG_KEXEC_SIG)) diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index ebaebccfbe9ab..e51b53b7789b1 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -658,6 +658,43 @@ static int ima_file_check(struct file *file, int mask) MAY_APPEND), FILE_CHECK); } +/* + * ima_reset_action_flags - invalidate action flags after a content change + * @inode: inode of the file whose content is about to be truncated + * + * Clear IMA_DONE_MASK so the file is re-collected, re-measured, + * re-audited, and re-appraised on next access. + */ +static void ima_reset_action_flags(struct inode *inode) +{ + struct ima_iint_cache *iint; + + if (!ima_policy_flag || !S_ISREG(inode->i_mode)) + return; + + iint = ima_iint_find(inode); + if (!iint) + return; + + mutex_lock(&iint->mutex); + iint->flags &= ~IMA_DONE_MASK; + iint->measured_pcrs = 0; + mutex_unlock(&iint->mutex); + return; +} + +static int ima_path_truncate(const struct path *path) +{ + ima_reset_action_flags(path->dentry->d_inode); + return 0; +} + +static int ima_file_truncate(struct file *file) +{ + ima_reset_action_flags(file_inode(file)); + return 0; +} + static int __ima_inode_hash(struct inode *inode, struct file *file, char *buf, size_t buf_size) { @@ -941,8 +978,7 @@ static int ima_load_data(enum kernel_load_data_id id, bool contents) switch (id) { case LOADING_KEXEC_IMAGE: - if (IS_ENABLED(CONFIG_KEXEC_SIG) - && arch_ima_get_secureboot()) { + if (IS_ENABLED(CONFIG_KEXEC_SIG) && arch_get_secureboot()) { pr_err("impossible to appraise a kernel image without a file descriptor; try using kexec_file_load syscall.\n"); return -EACCES; } @@ -1271,11 +1307,13 @@ static struct security_hook_list ima_hooks[] __ro_after_init = { LSM_HOOK_INIT(file_release, ima_file_free), LSM_HOOK_INIT(mmap_file, ima_file_mmap), LSM_HOOK_INIT(file_mprotect, ima_file_mprotect), + LSM_HOOK_INIT(file_truncate, ima_file_truncate), LSM_HOOK_INIT(kernel_load_data, ima_load_data), LSM_HOOK_INIT(kernel_post_load_data, ima_post_load_data), LSM_HOOK_INIT(kernel_read_file, ima_read_file), LSM_HOOK_INIT(kernel_post_read_file, ima_post_read_file), LSM_HOOK_INIT(path_post_mknod, ima_post_path_mknod), + LSM_HOOK_INIT(path_truncate, ima_path_truncate), #ifdef CONFIG_IMA_MEASURE_ASYMMETRIC_KEYS LSM_HOOK_INIT(key_post_create_or_update, ima_post_key_create_or_update), #endif diff --git a/security/integrity/integrity.h b/security/integrity/integrity.h index c2c2da6911233..17b037b3fd096 100644 --- a/security/integrity/integrity.h +++ b/security/integrity/integrity.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include diff --git a/security/integrity/platform_certs/load_uefi.c b/security/integrity/platform_certs/load_uefi.c index d1fdd113450a6..c0d6948446c30 100644 --- a/security/integrity/platform_certs/load_uefi.c +++ b/security/integrity/platform_certs/load_uefi.c @@ -212,7 +212,7 @@ static int __init load_uefi_certs(void) } /* the MOK/MOKx can not be trusted when secure boot is disabled */ - if (!arch_ima_get_secureboot()) + if (!arch_get_secureboot()) return 0; mokx = get_cert_list(L"MokListXRT", &mok_var, &mokxsize, &status); diff --git a/security/keys/keyring.c b/security/keys/keyring.c index e105349794f23..fd95a0eb7a466 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -271,6 +271,7 @@ static unsigned long keyring_get_key_chunk(const void *data, int level) unsigned long chunk = 0; const u8 *d; int desc_len = index_key->desc_len, n = sizeof(chunk); + unsigned int offset; level /= ASSOC_ARRAY_KEY_CHUNK_SIZE; switch (level) { @@ -284,17 +285,18 @@ static unsigned long keyring_get_key_chunk(const void *data, int level) return (unsigned long)index_key->domain_tag; default: level -= 4; - if (desc_len <= sizeof(index_key->desc)) + offset = sizeof(index_key->desc) + level * sizeof(long); + if (desc_len <= offset) return 0; - d = index_key->description + sizeof(index_key->desc); - d += level * sizeof(long); - desc_len -= sizeof(index_key->desc); + d = index_key->description + offset; + desc_len -= offset; if (desc_len > n) desc_len = n; + d += desc_len; do { chunk <<= 8; - chunk |= *d++; + chunk |= *--d; } while (--desc_len > 0); return chunk; } @@ -375,7 +377,7 @@ static int keyring_diff_objects(const void *object, const void *data) return -1; differ_plus_i: - level += i; + level += i - (int)sizeof(a->desc); differ: i = level * 8 + __ffs(seg_a ^ seg_b); return i; diff --git a/security/keys/trusted-keys/trusted_dcp.c b/security/keys/trusted-keys/trusted_dcp.c index 7b6eb655df0cb..c078adebe190e 100644 --- a/security/keys/trusted-keys/trusted_dcp.c +++ b/security/keys/trusted-keys/trusted_dcp.c @@ -69,7 +69,7 @@ static bool skip_zk_test; module_param_named(dcp_skip_zk_test, skip_zk_test, bool, 0); MODULE_PARM_DESC(dcp_skip_zk_test, "Don't test whether device keys are zero'ed"); -static unsigned int calc_blob_len(unsigned int payload_len) +static size_t calc_blob_len(unsigned int payload_len) { return sizeof(struct dcp_blob_fmt) + payload_len + DCP_BLOB_AUTHLEN; } @@ -200,7 +200,8 @@ static int encrypt_blob_key(u8 *plain_key, u8 *encrypted_key) static int trusted_dcp_seal(struct trusted_key_payload *p, char *datablob) { struct dcp_blob_fmt *b = (struct dcp_blob_fmt *)p->blob; - int blen, ret; + size_t blen; + int ret; u8 *plain_blob_key; blen = calc_blob_len(p->key_len); @@ -242,7 +243,8 @@ static int trusted_dcp_seal(struct trusted_key_payload *p, char *datablob) static int trusted_dcp_unseal(struct trusted_key_payload *p, char *datablob) { struct dcp_blob_fmt *b = (struct dcp_blob_fmt *)p->blob; - int blen, ret; + size_t blen; + int ret; u8 *plain_blob_key = NULL; if (b->fmt_version != DCP_BLOB_VERSION) { @@ -253,9 +255,14 @@ static int trusted_dcp_unseal(struct trusted_key_payload *p, char *datablob) } p->key_len = le32_to_cpu(b->payload_len); + if (p->key_len < MIN_KEY_SIZE || p->key_len > MAX_KEY_SIZE) { + ret = -EINVAL; + goto out; + } + blen = calc_blob_len(p->key_len); if (blen != p->blob_len) { - pr_err("DCP blob has bad length: %i != %i\n", blen, + pr_err("DCP blob has bad length: %zu != %u\n", blen, p->blob_len); ret = -EINVAL; goto out; diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c index 636acb66a4f69..ea6717da7cbad 100644 --- a/security/keys/trusted-keys/trusted_tpm1.c +++ b/security/keys/trusted-keys/trusted_tpm1.c @@ -981,9 +981,9 @@ static int __init trusted_tpm_init(void) static void trusted_tpm_exit(void) { if (chip) { + unregister_key_type(&key_type_trusted); put_device(&chip->dev); kfree(digests); - unregister_key_type(&key_type_trusted); } } diff --git a/security/landlock/access.h b/security/landlock/access.h index 7961c6630a2d7..42c95747d7bd6 100644 --- a/security/landlock/access.h +++ b/security/landlock/access.h @@ -61,14 +61,30 @@ union access_masks_all { static_assert(sizeof(typeof_member(union access_masks_all, masks)) == sizeof(typeof_member(union access_masks_all, all))); -typedef u16 layer_mask_t; - -/* Makes sure all layers can be checked. */ -static_assert(BITS_PER_TYPE(layer_mask_t) >= LANDLOCK_MAX_NUM_LAYERS); +/** + * struct layer_access_masks - A boolean matrix of layers and access rights + * + * This has a bit for each combination of layer numbers and access rights. + * During access checks, it is used to represent the access rights for each + * layer which still need to be fulfilled. When all bits are 0, the access + * request is considered to be fulfilled. + */ +struct layer_access_masks { + /** + * @access: The unfulfilled access rights for each layer. + */ + access_mask_t access[LANDLOCK_MAX_NUM_LAYERS]; +}; /* - * Tracks domains responsible of a denied access. This is required to avoid - * storing in each object the full layer_masks[] required by update_request(). + * Tracks domains responsible of a denied access. This avoids storing in each + * object the full matrix of per-layer unfulfilled access rights, which is + * required by update_request(). + * + * Each nibble represents the layer index of the newest layer which denied a + * certain access right. For file system access rights, the upper four bits are + * the index of the layer which denies LANDLOCK_ACCESS_FS_IOCTL_DEV and the + * lower nibble represents LANDLOCK_ACCESS_FS_TRUNCATE. */ typedef u8 deny_masks_t; @@ -97,4 +113,11 @@ landlock_upgrade_handled_access_masks(struct access_masks access_masks) return access_masks; } +/* Checks the subset relation between access masks. */ +static inline bool access_mask_subset(access_mask_t subset, + access_mask_t superset) +{ + return (subset | superset) == superset; +} + #endif /* _SECURITY_LANDLOCK_ACCESS_H */ diff --git a/security/landlock/audit.c b/security/landlock/audit.c index e899995f1fd59..60ff217ab95b5 100644 --- a/security/landlock/audit.c +++ b/security/landlock/audit.c @@ -180,38 +180,21 @@ static void test_get_hierarchy(struct kunit *const test) #endif /* CONFIG_SECURITY_LANDLOCK_KUNIT_TEST */ +/* Get the youngest layer that denied the access_request. */ static size_t get_denied_layer(const struct landlock_ruleset *const domain, access_mask_t *const access_request, - const layer_mask_t (*const layer_masks)[], - const size_t layer_masks_size) + const struct layer_access_masks *masks) { - const unsigned long access_req = *access_request; - unsigned long access_bit; - access_mask_t missing = 0; - long youngest_layer = -1; - - for_each_set_bit(access_bit, &access_req, layer_masks_size) { - const layer_mask_t mask = (*layer_masks)[access_bit]; - long layer; - - if (!mask) - continue; - - /* __fls(1) == 0 */ - layer = __fls(mask); - if (layer > youngest_layer) { - youngest_layer = layer; - missing = BIT(access_bit); - } else if (layer == youngest_layer) { - missing |= BIT(access_bit); + for (ssize_t i = ARRAY_SIZE(masks->access) - 1; i >= 0; i--) { + if (masks->access[i] & *access_request) { + *access_request &= masks->access[i]; + return i; } } - *access_request = missing; - if (youngest_layer == -1) - return domain->num_layers - 1; - - return youngest_layer; + /* Not found - fall back to default values */ + *access_request = 0; + return domain->num_layers - 1; } #ifdef CONFIG_SECURITY_LANDLOCK_KUNIT_TEST @@ -221,50 +204,39 @@ static void test_get_denied_layer(struct kunit *const test) const struct landlock_ruleset dom = { .num_layers = 5, }; - const layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_READ_FILE)] = BIT(1), - [BIT_INDEX(LANDLOCK_ACCESS_FS_READ_DIR)] = BIT(1) | BIT(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_REMOVE_DIR)] = BIT(2), + const struct layer_access_masks masks = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_READ_DIR, + .access[1] = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_READ_DIR, + .access[2] = LANDLOCK_ACCESS_FS_REMOVE_DIR, }; access_mask_t access; access = LANDLOCK_ACCESS_FS_EXECUTE; - KUNIT_EXPECT_EQ(test, 0, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 0, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, LANDLOCK_ACCESS_FS_EXECUTE); access = LANDLOCK_ACCESS_FS_READ_FILE; - KUNIT_EXPECT_EQ(test, 1, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 1, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, LANDLOCK_ACCESS_FS_READ_FILE); access = LANDLOCK_ACCESS_FS_READ_DIR; - KUNIT_EXPECT_EQ(test, 1, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 1, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, LANDLOCK_ACCESS_FS_READ_DIR); access = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR; - KUNIT_EXPECT_EQ(test, 1, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 1, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR); access = LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_READ_DIR; - KUNIT_EXPECT_EQ(test, 1, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 1, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, LANDLOCK_ACCESS_FS_READ_DIR); access = LANDLOCK_ACCESS_FS_WRITE_FILE; - KUNIT_EXPECT_EQ(test, 4, - get_denied_layer(&dom, &access, &layer_masks, - sizeof(layer_masks))); + KUNIT_EXPECT_EQ(test, 4, get_denied_layer(&dom, &access, &masks)); KUNIT_EXPECT_EQ(test, access, 0); } @@ -370,9 +342,6 @@ static bool is_valid_request(const struct landlock_request *const request) return false; } - if (WARN_ON_ONCE(!!request->layer_masks ^ !!request->layer_masks_size)) - return false; - if (request->deny_masks) { if (WARN_ON_ONCE(!request->all_existing_optional_access)) return false; @@ -406,12 +375,12 @@ void landlock_log_denial(const struct landlock_cred_security *const subject, if (missing) { /* Gets the nearest domain that denies the request. */ if (request->layer_masks) { - youngest_layer = get_denied_layer( - subject->domain, &missing, request->layer_masks, - request->layer_masks_size); + youngest_layer = get_denied_layer(subject->domain, + &missing, + request->layer_masks); } else { youngest_layer = get_layer_from_deny_masks( - &missing, request->all_existing_optional_access, + &missing, _LANDLOCK_ACCESS_FS_OPTIONAL, request->deny_masks); } youngest_denied = diff --git a/security/landlock/audit.h b/security/landlock/audit.h index 92428b7fc4d80..56778331b58c7 100644 --- a/security/landlock/audit.h +++ b/security/landlock/audit.h @@ -43,8 +43,7 @@ struct landlock_request { access_mask_t access; /* Required fields for requests with layer masks. */ - const layer_mask_t (*layer_masks)[]; - size_t layer_masks_size; + const struct layer_access_masks *layer_masks; /* Required fields for requests with deny masks. */ const access_mask_t all_existing_optional_access; diff --git a/security/landlock/domain.c b/security/landlock/domain.c index a647b68e8d060..0c27e01f4996a 100644 --- a/security/landlock/domain.c +++ b/security/landlock/domain.c @@ -90,12 +90,12 @@ static struct landlock_details *get_current_details(void) return ERR_CAST(buffer); /* - * Create the new details according to the path's length. Do not - * allocate with GFP_KERNEL_ACCOUNT because it is independent from the - * caller. + * Create the new details according to the path's length. Account to + * the calling task's memcg, like the other Landlock per-domain + * allocations, even if it may not control the related size. */ details = - kzalloc(struct_size(details, exe_path, path_size), GFP_KERNEL); + kzalloc_flex(*details, exe_path, path_size, GFP_KERNEL_ACCOUNT); if (!details) return ERR_PTR(-ENOMEM); @@ -182,32 +182,36 @@ static void test_get_layer_deny_mask(struct kunit *const test) deny_masks_t landlock_get_deny_masks(const access_mask_t all_existing_optional_access, const access_mask_t optional_access, - const layer_mask_t (*const layer_masks)[], - const size_t layer_masks_size) + const struct layer_access_masks *const masks) { const unsigned long access_opt = optional_access; unsigned long access_bit; deny_masks_t deny_masks = 0; + access_mask_t all_denied = 0; /* This may require change with new object types. */ - WARN_ON_ONCE(access_opt != - (optional_access & all_existing_optional_access)); + WARN_ON_ONCE(!access_mask_subset(optional_access, + all_existing_optional_access)); - if (WARN_ON_ONCE(!layer_masks)) + if (WARN_ON_ONCE(!masks)) return 0; if (WARN_ON_ONCE(!access_opt)) return 0; - for_each_set_bit(access_bit, &access_opt, layer_masks_size) { - const layer_mask_t mask = (*layer_masks)[access_bit]; + for (ssize_t i = ARRAY_SIZE(masks->access) - 1; i >= 0; i--) { + const access_mask_t denied = masks->access[i] & optional_access; + const unsigned long newly_denied = denied & ~all_denied; - if (!mask) + if (!newly_denied) continue; - /* __fls(1) == 0 */ - deny_masks |= get_layer_deny_mask(all_existing_optional_access, - access_bit, __fls(mask)); + for_each_set_bit(access_bit, &newly_denied, + 8 * sizeof(access_mask_t)) { + deny_masks |= get_layer_deny_mask( + all_existing_optional_access, access_bit, i); + } + all_denied |= denied; } return deny_masks; } @@ -216,28 +220,28 @@ landlock_get_deny_masks(const access_mask_t all_existing_optional_access, static void test_landlock_get_deny_masks(struct kunit *const test) { - const layer_mask_t layers1[BITS_PER_TYPE(access_mask_t)] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0) | - BIT_ULL(9), - [BIT_INDEX(LANDLOCK_ACCESS_FS_TRUNCATE)] = BIT_ULL(1), - [BIT_INDEX(LANDLOCK_ACCESS_FS_IOCTL_DEV)] = BIT_ULL(2) | - BIT_ULL(0), + const struct layer_access_masks layers1 = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_IOCTL_DEV, + .access[1] = LANDLOCK_ACCESS_FS_TRUNCATE, + .access[2] = LANDLOCK_ACCESS_FS_IOCTL_DEV, + .access[9] = LANDLOCK_ACCESS_FS_EXECUTE, }; KUNIT_EXPECT_EQ(test, 0x1, landlock_get_deny_masks(_LANDLOCK_ACCESS_FS_OPTIONAL, LANDLOCK_ACCESS_FS_TRUNCATE, - &layers1, ARRAY_SIZE(layers1))); + &layers1)); KUNIT_EXPECT_EQ(test, 0x20, landlock_get_deny_masks(_LANDLOCK_ACCESS_FS_OPTIONAL, LANDLOCK_ACCESS_FS_IOCTL_DEV, - &layers1, ARRAY_SIZE(layers1))); + &layers1)); KUNIT_EXPECT_EQ( test, 0x21, landlock_get_deny_masks(_LANDLOCK_ACCESS_FS_OPTIONAL, LANDLOCK_ACCESS_FS_TRUNCATE | LANDLOCK_ACCESS_FS_IOCTL_DEV, - &layers1, ARRAY_SIZE(layers1))); + &layers1)); } #endif /* CONFIG_SECURITY_LANDLOCK_KUNIT_TEST */ diff --git a/security/landlock/domain.h b/security/landlock/domain.h index 7fb70b25f85a1..3460186263ff3 100644 --- a/security/landlock/domain.h +++ b/security/landlock/domain.h @@ -33,10 +33,7 @@ enum landlock_log_status { * Rarely accessed, mainly when logging the first domain's denial. * * The contained pointers are initialized at the domain creation time and never - * changed again. Contrary to most other Landlock object types, this one is - * not allocated with GFP_KERNEL_ACCOUNT because its size may not be under the - * caller's control (e.g. unknown exe_path) and the data is not explicitly - * requested nor used by tasks. + * changed again. */ struct landlock_details { /** @@ -122,8 +119,7 @@ struct landlock_hierarchy { deny_masks_t landlock_get_deny_masks(const access_mask_t all_existing_optional_access, const access_mask_t optional_access, - const layer_mask_t (*const layer_masks)[], - size_t layer_masks_size); + const struct layer_access_masks *const masks); int landlock_init_hierarchy_log(struct landlock_hierarchy *const hierarchy); diff --git a/security/landlock/errata/abi-1.h b/security/landlock/errata/abi-1.h index e8a2bff2e5b6a..80930957e84d2 100644 --- a/security/landlock/errata/abi-1.h +++ b/security/landlock/errata/abi-1.h @@ -14,3 +14,26 @@ * actions. */ LANDLOCK_ERRATUM(3) + +/** + * DOC: erratum_4 + * + * Erratum 4: Creation of whiteout objects + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * This fix changes the access rights required for the creation of whiteout + * objects through :manpage:`mknod(2)`, :manpage:`renameat2(2)`, or + * :manpage:`link(2)`. Creating whiteout objects is now guarded by + * ``LANDLOCK_ACCESS_FS_MAKE_REG`` instead of ``LANDLOCK_ACCESS_FS_MAKE_CHAR``. + * + * Whiteout objects are used in OverlayFS to mark the absence of a file in an + * upper file system. Despite being created with ``S_IFCHR``, whiteout objects + * do not count as character devices. + * + * Impact: + * + * Sandboxed programs that create OverlayFS whiteouts (such as fuse-overlayfs) + * now require ``LANDLOCK_ACCESS_FS_MAKE_REG`` instead of + * ``LANDLOCK_ACCESS_FS_MAKE_CHAR``. + */ +LANDLOCK_ERRATUM(4) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index 4f8dfeadbbe22..a32658c18880f 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -331,7 +332,7 @@ int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, /* Files only get access rights that make sense. */ if (!d_is_dir(path->dentry) && - (access_rights | ACCESS_FILE) != ACCESS_FILE) + !access_mask_subset(access_rights, ACCESS_FILE)) return -EINVAL; if (WARN_ON_ONCE(ruleset->num_layers != 1)) return -EINVAL; @@ -402,51 +403,43 @@ static const struct access_masks any_fs = { * Check that a destination file hierarchy has more restrictions than a source * file hierarchy. This is only used for link and rename actions. * - * @layer_masks_child2: Optional child masks. + * Returns: true if child1 may be moved from parent1 to parent2 without + * increasing its access rights. If child2 is set, an additional condition is + * that child2 may be used from parent2 to parent1 without increasing its access + * rights. */ -static bool no_more_access( - const layer_mask_t (*const layer_masks_parent1)[LANDLOCK_NUM_ACCESS_FS], - const layer_mask_t (*const layer_masks_child1)[LANDLOCK_NUM_ACCESS_FS], - const bool child1_is_directory, - const layer_mask_t (*const layer_masks_parent2)[LANDLOCK_NUM_ACCESS_FS], - const layer_mask_t (*const layer_masks_child2)[LANDLOCK_NUM_ACCESS_FS], - const bool child2_is_directory) +static bool no_more_access(const struct layer_access_masks *const parent1, + const struct layer_access_masks *const child1, + const bool child1_is_dir, + const struct layer_access_masks *const parent2, + const struct layer_access_masks *const child2, + const bool child2_is_dir) { - unsigned long access_bit; + for (size_t i = 0; i < ARRAY_SIZE(parent2->access); i++) { + access_mask_t child_access = parent1->access[i] & + child1->access[i]; + access_mask_t parent_access = parent2->access[i]; - for (access_bit = 0; access_bit < ARRAY_SIZE(*layer_masks_parent2); - access_bit++) { /* Ignores accesses that only make sense for directories. */ - const bool is_file_access = - !!(BIT_ULL(access_bit) & ACCESS_FILE); - - if (child1_is_directory || is_file_access) { - /* - * Checks if the destination restrictions are a - * superset of the source ones (i.e. inherited access - * rights without child exceptions): - * restrictions(parent2) >= restrictions(child1) - */ - if ((((*layer_masks_parent1)[access_bit] & - (*layer_masks_child1)[access_bit]) | - (*layer_masks_parent2)[access_bit]) != - (*layer_masks_parent2)[access_bit]) - return false; + if (!child1_is_dir) { + child_access &= ACCESS_FILE; + parent_access &= ACCESS_FILE; } + if (!access_mask_subset(child_access, parent_access)) + return false; - if (!layer_masks_child2) + if (!child2) continue; - if (child2_is_directory || is_file_access) { - /* - * Checks inverted restrictions for RENAME_EXCHANGE: - * restrictions(parent1) >= restrictions(child2) - */ - if ((((*layer_masks_parent2)[access_bit] & - (*layer_masks_child2)[access_bit]) | - (*layer_masks_parent1)[access_bit]) != - (*layer_masks_parent1)[access_bit]) - return false; + + /* Checks inverted restrictions for RENAME_EXCHANGE. */ + child_access = parent2->access[i] & child2->access[i]; + parent_access = parent1->access[i]; + if (!child2_is_dir) { + child_access &= ACCESS_FILE; + parent_access &= ACCESS_FILE; } + if (!access_mask_subset(child_access, parent_access)) + return false; } return true; } @@ -458,25 +451,25 @@ static bool no_more_access( static void test_no_more_access(struct kunit *const test) { - const layer_mask_t rx0[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_READ_FILE)] = BIT_ULL(0), + const struct layer_access_masks rx0 = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_READ_FILE, }; - const layer_mask_t mx0[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_MAKE_REG)] = BIT_ULL(0), + const struct layer_access_masks mx0 = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, }; - const layer_mask_t x0[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0), + const struct layer_access_masks x0 = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE, }; - const layer_mask_t x1[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(1), + const struct layer_access_masks x1 = { + .access[1] = LANDLOCK_ACCESS_FS_EXECUTE, }; - const layer_mask_t x01[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0) | - BIT_ULL(1), + const struct layer_access_masks x01 = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE, + .access[1] = LANDLOCK_ACCESS_FS_EXECUTE, }; - const layer_mask_t allows_all[LANDLOCK_NUM_ACCESS_FS] = {}; + const struct layer_access_masks allows_all = {}; /* Checks without restriction. */ NMA_TRUE(&x0, &allows_all, false, &allows_all, NULL, false); @@ -564,31 +557,30 @@ static void test_no_more_access(struct kunit *const test) #undef NMA_TRUE #undef NMA_FALSE -static bool is_layer_masks_allowed( - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +static bool is_layer_masks_allowed(const struct layer_access_masks *masks) { - return !memchr_inv(layer_masks, 0, sizeof(*layer_masks)); + return mem_is_zero(&masks->access, sizeof(masks->access)); } /* - * Removes @layer_masks accesses that are not requested. + * Removes @masks accesses that are not requested. * * Returns true if the request is allowed, false otherwise. */ -static bool -scope_to_request(const access_mask_t access_request, - layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS]) +static bool scope_to_request(const access_mask_t access_request, + struct layer_access_masks *masks) { - const unsigned long access_req = access_request; - unsigned long access_bit; + bool saw_unfulfilled_access = false; - if (WARN_ON_ONCE(!layer_masks)) + if (WARN_ON_ONCE(!masks)) return true; - for_each_clear_bit(access_bit, &access_req, ARRAY_SIZE(*layer_masks)) - (*layer_masks)[access_bit] = 0; - - return is_layer_masks_allowed(layer_masks); + for (size_t i = 0; i < ARRAY_SIZE(masks->access); i++) { + masks->access[i] &= access_request; + if (masks->access[i]) + saw_unfulfilled_access = true; + } + return !saw_unfulfilled_access; } #ifdef CONFIG_SECURITY_LANDLOCK_KUNIT_TEST @@ -596,48 +588,41 @@ scope_to_request(const access_mask_t access_request, static void test_scope_to_request_with_exec_none(struct kunit *const test) { /* Allows everything. */ - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; + struct layer_access_masks masks = {}; /* Checks and scopes with execute. */ - KUNIT_EXPECT_TRUE(test, scope_to_request(LANDLOCK_ACCESS_FS_EXECUTE, - &layer_masks)); - KUNIT_EXPECT_EQ(test, 0, - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)]); - KUNIT_EXPECT_EQ(test, 0, - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)]); + KUNIT_EXPECT_TRUE(test, + scope_to_request(LANDLOCK_ACCESS_FS_EXECUTE, &masks)); + KUNIT_EXPECT_EQ(test, 0, masks.access[0]); } static void test_scope_to_request_with_exec_some(struct kunit *const test) { /* Denies execute and write. */ - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)] = BIT_ULL(1), + struct layer_access_masks masks = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE, + .access[1] = LANDLOCK_ACCESS_FS_WRITE_FILE, }; /* Checks and scopes with execute. */ KUNIT_EXPECT_FALSE(test, scope_to_request(LANDLOCK_ACCESS_FS_EXECUTE, - &layer_masks)); - KUNIT_EXPECT_EQ(test, BIT_ULL(0), - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)]); - KUNIT_EXPECT_EQ(test, 0, - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)]); + &masks)); + KUNIT_EXPECT_EQ(test, LANDLOCK_ACCESS_FS_EXECUTE, masks.access[0]); + KUNIT_EXPECT_EQ(test, 0, masks.access[1]); } static void test_scope_to_request_without_access(struct kunit *const test) { /* Denies execute and write. */ - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)] = BIT_ULL(0), - [BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)] = BIT_ULL(1), + struct layer_access_masks masks = { + .access[0] = LANDLOCK_ACCESS_FS_EXECUTE, + .access[1] = LANDLOCK_ACCESS_FS_WRITE_FILE, }; /* Checks and scopes without access request. */ - KUNIT_EXPECT_TRUE(test, scope_to_request(0, &layer_masks)); - KUNIT_EXPECT_EQ(test, 0, - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_EXECUTE)]); - KUNIT_EXPECT_EQ(test, 0, - layer_masks[BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)]); + KUNIT_EXPECT_TRUE(test, scope_to_request(0, &masks)); + KUNIT_EXPECT_EQ(test, 0, masks.access[0]); + KUNIT_EXPECT_EQ(test, 0, masks.access[1]); } #endif /* CONFIG_SECURITY_LANDLOCK_KUNIT_TEST */ @@ -646,20 +631,16 @@ static void test_scope_to_request_without_access(struct kunit *const test) * Returns true if there is at least one access right different than * LANDLOCK_ACCESS_FS_REFER. */ -static bool -is_eacces(const layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS], - const access_mask_t access_request) +static bool is_eacces(const struct layer_access_masks *masks, + const access_mask_t access_request) { - unsigned long access_bit; - /* LANDLOCK_ACCESS_FS_REFER alone must return -EXDEV. */ - const unsigned long access_check = access_request & - ~LANDLOCK_ACCESS_FS_REFER; - - if (!layer_masks) + if (!masks) return false; - for_each_set_bit(access_bit, &access_check, ARRAY_SIZE(*layer_masks)) { - if ((*layer_masks)[access_bit]) + for (size_t i = 0; i < ARRAY_SIZE(masks->access); i++) { + /* LANDLOCK_ACCESS_FS_REFER alone must return -EXDEV. */ + if (masks->access[i] & access_request & + ~LANDLOCK_ACCESS_FS_REFER) return true; } return false; @@ -672,37 +653,37 @@ is_eacces(const layer_mask_t (*const layer_masks)[LANDLOCK_NUM_ACCESS_FS], static void test_is_eacces_with_none(struct kunit *const test) { - const layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; + const struct layer_access_masks masks = {}; - IE_FALSE(&layer_masks, 0); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_REFER); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_EXECUTE); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_WRITE_FILE); + IE_FALSE(&masks, 0); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_REFER); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_EXECUTE); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_WRITE_FILE); } static void test_is_eacces_with_refer(struct kunit *const test) { - const layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_REFER)] = BIT_ULL(0), + const struct layer_access_masks masks = { + .access[0] = LANDLOCK_ACCESS_FS_REFER, }; - IE_FALSE(&layer_masks, 0); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_REFER); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_EXECUTE); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_WRITE_FILE); + IE_FALSE(&masks, 0); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_REFER); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_EXECUTE); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_WRITE_FILE); } static void test_is_eacces_with_write(struct kunit *const test) { - const layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = { - [BIT_INDEX(LANDLOCK_ACCESS_FS_WRITE_FILE)] = BIT_ULL(0), + const struct layer_access_masks masks = { + .access[0] = LANDLOCK_ACCESS_FS_WRITE_FILE, }; - IE_FALSE(&layer_masks, 0); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_REFER); - IE_FALSE(&layer_masks, LANDLOCK_ACCESS_FS_EXECUTE); + IE_FALSE(&masks, 0); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_REFER); + IE_FALSE(&masks, LANDLOCK_ACCESS_FS_EXECUTE); - IE_TRUE(&layer_masks, LANDLOCK_ACCESS_FS_WRITE_FILE); + IE_TRUE(&masks, LANDLOCK_ACCESS_FS_WRITE_FILE); } #endif /* CONFIG_SECURITY_LANDLOCK_KUNIT_TEST */ @@ -751,26 +732,25 @@ static void test_is_eacces_with_write(struct kunit *const test) * - true if the access request is granted; * - false otherwise. */ -static bool is_access_to_paths_allowed( - const struct landlock_ruleset *const domain, - const struct path *const path, - const access_mask_t access_request_parent1, - layer_mask_t (*const layer_masks_parent1)[LANDLOCK_NUM_ACCESS_FS], - struct landlock_request *const log_request_parent1, - struct dentry *const dentry_child1, - const access_mask_t access_request_parent2, - layer_mask_t (*const layer_masks_parent2)[LANDLOCK_NUM_ACCESS_FS], - struct landlock_request *const log_request_parent2, - struct dentry *const dentry_child2) +static bool +is_access_to_paths_allowed(const struct landlock_ruleset *const domain, + const struct path *const path, + const access_mask_t access_request_parent1, + struct layer_access_masks *layer_masks_parent1, + struct landlock_request *const log_request_parent1, + struct dentry *const dentry_child1, + const access_mask_t access_request_parent2, + struct layer_access_masks *layer_masks_parent2, + struct landlock_request *const log_request_parent2, + struct dentry *const dentry_child2) { bool allowed_parent1 = false, allowed_parent2 = false, is_dom_check, child1_is_directory = true, child2_is_directory = true; struct path walker_path; access_mask_t access_masked_parent1, access_masked_parent2; - layer_mask_t _layer_masks_child1[LANDLOCK_NUM_ACCESS_FS], - _layer_masks_child2[LANDLOCK_NUM_ACCESS_FS]; - layer_mask_t(*layer_masks_child1)[LANDLOCK_NUM_ACCESS_FS] = NULL, - (*layer_masks_child2)[LANDLOCK_NUM_ACCESS_FS] = NULL; + struct layer_access_masks _layer_masks_child1, _layer_masks_child2; + struct layer_access_masks *layer_masks_child1 = NULL, + *layer_masks_child2 = NULL; if (!access_request_parent1 && !access_request_parent2) return true; @@ -810,22 +790,20 @@ static bool is_access_to_paths_allowed( } if (unlikely(dentry_child1)) { - landlock_unmask_layers( - find_rule(domain, dentry_child1), - landlock_init_layer_masks( - domain, LANDLOCK_MASK_ACCESS_FS, - &_layer_masks_child1, LANDLOCK_KEY_INODE), - &_layer_masks_child1, ARRAY_SIZE(_layer_masks_child1)); + if (landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + &_layer_masks_child1, + LANDLOCK_KEY_INODE)) + landlock_unmask_layers(find_rule(domain, dentry_child1), + &_layer_masks_child1); layer_masks_child1 = &_layer_masks_child1; child1_is_directory = d_is_dir(dentry_child1); } if (unlikely(dentry_child2)) { - landlock_unmask_layers( - find_rule(domain, dentry_child2), - landlock_init_layer_masks( - domain, LANDLOCK_MASK_ACCESS_FS, - &_layer_masks_child2, LANDLOCK_KEY_INODE), - &_layer_masks_child2, ARRAY_SIZE(_layer_masks_child2)); + if (landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + &_layer_masks_child2, + LANDLOCK_KEY_INODE)) + landlock_unmask_layers(find_rule(domain, dentry_child2), + &_layer_masks_child2); layer_masks_child2 = &_layer_masks_child2; child2_is_directory = d_is_dir(dentry_child2); } @@ -881,16 +859,12 @@ static bool is_access_to_paths_allowed( } rule = find_rule(domain, walker_path.dentry); - allowed_parent1 = allowed_parent1 || - landlock_unmask_layers( - rule, access_masked_parent1, - layer_masks_parent1, - ARRAY_SIZE(*layer_masks_parent1)); - allowed_parent2 = allowed_parent2 || - landlock_unmask_layers( - rule, access_masked_parent2, - layer_masks_parent2, - ARRAY_SIZE(*layer_masks_parent2)); + allowed_parent1 = + allowed_parent1 || + landlock_unmask_layers(rule, layer_masks_parent1); + allowed_parent2 = + allowed_parent2 || + landlock_unmask_layers(rule, layer_masks_parent2); /* Stops when a rule from each layer grants access. */ if (allowed_parent1 && allowed_parent2) @@ -943,8 +917,6 @@ static bool is_access_to_paths_allowed( log_request_parent1->audit.u.path = *path; log_request_parent1->access = access_masked_parent1; log_request_parent1->layer_masks = layer_masks_parent1; - log_request_parent1->layer_masks_size = - ARRAY_SIZE(*layer_masks_parent1); } if (!allowed_parent2) { @@ -953,8 +925,6 @@ static bool is_access_to_paths_allowed( log_request_parent2->audit.u.path = *path; log_request_parent2->access = access_masked_parent2; log_request_parent2->layer_masks = layer_masks_parent2; - log_request_parent2->layer_masks_size = - ARRAY_SIZE(*layer_masks_parent2); } return allowed_parent1 && allowed_parent2; } @@ -967,7 +937,7 @@ static int current_check_access_path(const struct path *const path, }; const struct landlock_cred_security *const subject = landlock_get_applicable_subject(current_cred(), masks, NULL); - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; + struct layer_access_masks layer_masks; struct landlock_request request = {}; if (!subject) @@ -985,7 +955,8 @@ static int current_check_access_path(const struct path *const path, return -EACCES; } -static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) +static __attribute_const__ access_mask_t get_mode_access(const umode_t mode, + const dev_t dev) { switch (mode & S_IFMT) { case S_IFLNK: @@ -993,6 +964,9 @@ static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) case S_IFDIR: return LANDLOCK_ACCESS_FS_MAKE_DIR; case S_IFCHR: + /* Whiteout objects are guarded with MAKE_REG. */ + if (dev == WHITEOUT_DEV) + return LANDLOCK_ACCESS_FS_MAKE_REG; return LANDLOCK_ACCESS_FS_MAKE_CHAR; case S_IFBLK: return LANDLOCK_ACCESS_FS_MAKE_BLOCK; @@ -1009,6 +983,13 @@ static __attribute_const__ access_mask_t get_mode_access(const umode_t mode) } } +static access_mask_t get_dentry_access(const struct dentry *const dentry) +{ + const struct inode *const inode = d_backing_inode(dentry); + + return get_mode_access(inode->i_mode, inode->i_rdev); +} + static access_mask_t maybe_remove(const struct dentry *const dentry) { if (d_is_negative(dentry)) @@ -1042,12 +1023,11 @@ static access_mask_t maybe_remove(const struct dentry *const dentry) * - true if all the domain access rights are allowed for @dir; * - false if the walk reached @mnt_root. */ -static bool collect_domain_accesses( - const struct landlock_ruleset *const domain, - const struct dentry *const mnt_root, struct dentry *dir, - layer_mask_t (*const layer_masks_dom)[LANDLOCK_NUM_ACCESS_FS]) +static bool collect_domain_accesses(const struct landlock_ruleset *const domain, + const struct dentry *const mnt_root, + struct dentry *dir, + struct layer_access_masks *layer_masks_dom) { - unsigned long access_dom; bool ret = false; if (WARN_ON_ONCE(!domain || !mnt_root || !dir || !layer_masks_dom)) @@ -1055,18 +1035,17 @@ static bool collect_domain_accesses( if (is_nouser_or_private(dir)) return true; - access_dom = landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, - layer_masks_dom, - LANDLOCK_KEY_INODE); + if (!landlock_init_layer_masks(domain, LANDLOCK_MASK_ACCESS_FS, + layer_masks_dom, LANDLOCK_KEY_INODE)) + return true; dget(dir); while (true) { struct dentry *parent_dentry; /* Gets all layers allowing all domain accesses. */ - if (landlock_unmask_layers(find_rule(domain, dir), access_dom, - layer_masks_dom, - ARRAY_SIZE(*layer_masks_dom))) { + if (landlock_unmask_layers(find_rule(domain, dir), + layer_masks_dom)) { /* * Stops when all handled accesses are allowed by at * least one rule in each layer. @@ -1098,6 +1077,7 @@ static bool collect_domain_accesses( * @new_dentry: Destination file or directory. * @removable: Sets to true if it is a rename operation. * @exchange: Sets to true if it is a rename operation with RENAME_EXCHANGE. + * @whiteout: Sets to true if it is a rename operation with RENAME_WHITEOUT. * * Because of its unprivileged constraints, Landlock relies on file hierarchies * (and not only inodes) to tie access rights to files. Being able to link or @@ -1146,7 +1126,8 @@ static bool collect_domain_accesses( static int current_check_refer_path(struct dentry *const old_dentry, const struct path *const new_dir, struct dentry *const new_dentry, - const bool removable, const bool exchange) + const bool removable, const bool exchange, + const bool whiteout) { const struct landlock_cred_security *const subject = landlock_get_applicable_subject(current_cred(), any_fs, NULL); @@ -1154,8 +1135,8 @@ static int current_check_refer_path(struct dentry *const old_dentry, access_mask_t access_request_parent1, access_request_parent2; struct path mnt_dir; struct dentry *old_parent; - layer_mask_t layer_masks_parent1[LANDLOCK_NUM_ACCESS_FS] = {}, - layer_masks_parent2[LANDLOCK_NUM_ACCESS_FS] = {}; + struct layer_access_masks layer_masks_parent1 = {}, + layer_masks_parent2 = {}; struct landlock_request request1 = {}, request2 = {}; if (!subject) @@ -1166,18 +1147,25 @@ static int current_check_refer_path(struct dentry *const old_dentry, if (exchange) { if (unlikely(d_is_negative(new_dentry))) return -ENOENT; - access_request_parent1 = - get_mode_access(d_backing_inode(new_dentry)->i_mode); + access_request_parent1 = get_dentry_access(new_dentry); } else { access_request_parent1 = 0; } - access_request_parent2 = - get_mode_access(d_backing_inode(old_dentry)->i_mode); + access_request_parent2 = get_dentry_access(old_dentry); if (removable) { access_request_parent1 |= maybe_remove(old_dentry); access_request_parent2 |= maybe_remove(new_dentry); } + /* + * In case of renameat2(2) with RENAME_WHITEOUT, a whiteout object is + * created in the source location, so we require an additional access + * right there. + */ + if (whiteout) + access_request_parent1 |= + get_mode_access(S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); + /* The mount points are the same for old and new paths, cf. EXDEV. */ if (old_dentry->d_parent == new_dir->dentry) { /* @@ -1208,11 +1196,12 @@ static int current_check_refer_path(struct dentry *const old_dentry, /* * old_dentry may be the root of the common mount point and * !IS_ROOT(old_dentry) at the same time (e.g. with open_tree() and - * OPEN_TREE_CLONE). We do not need to call dget(old_parent) because - * we keep a reference to old_dentry. + * OPEN_TREE_CLONE). Pin the dentry used as old_parent in either case. + * Otherwise, dget_parent() safely fetches and pins the current parent + * against a concurrent rename(2). */ - old_parent = (old_dentry == mnt_dir.dentry) ? old_dentry : - old_dentry->d_parent; + old_parent = (old_dentry == mnt_dir.dentry) ? dget(old_dentry) : + dget_parent(old_dentry); /* new_dir->dentry is equal to new_dentry->d_parent */ allow_parent1 = collect_domain_accesses(subject->domain, mnt_dir.dentry, @@ -1221,9 +1210,10 @@ static int current_check_refer_path(struct dentry *const old_dentry, allow_parent2 = collect_domain_accesses(subject->domain, mnt_dir.dentry, new_dir->dentry, &layer_masks_parent2); - - if (allow_parent1 && allow_parent2) + if (allow_parent1 && allow_parent2) { + dput(old_parent); return 0; + } /* * To be able to compare source and destination domain access rights, @@ -1235,8 +1225,10 @@ static int current_check_refer_path(struct dentry *const old_dentry, subject->domain, &mnt_dir, access_request_parent1, &layer_masks_parent1, &request1, old_dentry, access_request_parent2, &layer_masks_parent2, &request2, - exchange ? new_dentry : NULL)) + exchange ? new_dentry : NULL)) { + dput(old_parent); return 0; + } if (request1.access) { request1.audit.u.path.dentry = old_parent; @@ -1246,6 +1238,7 @@ static int current_check_refer_path(struct dentry *const old_dentry, request2.audit.u.path.dentry = new_dir->dentry; landlock_log_denial(subject, &request2); } + dput(old_parent); /* * This prioritizes EACCES over EXDEV for all actions, including @@ -1527,7 +1520,7 @@ static int hook_path_link(struct dentry *const old_dentry, struct dentry *const new_dentry) { return current_check_refer_path(old_dentry, new_dir, new_dentry, false, - false); + false, false); } static int hook_path_rename(const struct path *const old_dir, @@ -1538,7 +1531,8 @@ static int hook_path_rename(const struct path *const old_dir, { /* old_dir refers to old_dentry->d_parent and new_dir->mnt */ return current_check_refer_path(old_dentry, new_dir, new_dentry, true, - !!(flags & RENAME_EXCHANGE)); + !!(flags & RENAME_EXCHANGE), + !!(flags & RENAME_WHITEOUT)); } static int hook_path_mkdir(const struct path *const dir, @@ -1551,7 +1545,8 @@ static int hook_path_mknod(const struct path *const dir, struct dentry *const dentry, const umode_t mode, const unsigned int dev) { - return current_check_access_path(dir, get_mode_access(mode)); + return current_check_access_path( + dir, get_mode_access(mode, new_decode_dev(dev))); } static int hook_path_symlink(const struct path *const dir, @@ -1630,7 +1625,7 @@ static bool is_device(const struct file *const file) static int hook_file_open(struct file *const file) { - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_FS] = {}; + struct layer_access_masks layer_masks = {}; access_mask_t open_access_request, full_access_request, allowed_access, optional_access; const struct landlock_cred_security *const subject = @@ -1665,20 +1660,14 @@ static int hook_file_open(struct file *const file) &layer_masks, &request, NULL, 0, NULL, NULL, NULL)) { allowed_access = full_access_request; } else { - unsigned long access_bit; - const unsigned long access_req = full_access_request; - /* * Calculate the actual allowed access rights from layer_masks. - * Add each access right to allowed_access which has not been - * vetoed by any layer. + * Remove the access rights from the full access request which + * are still unfulfilled in any of the layers. */ - allowed_access = 0; - for_each_set_bit(access_bit, &access_req, - ARRAY_SIZE(layer_masks)) { - if (!layer_masks[access_bit]) - allowed_access |= BIT_ULL(access_bit); - } + allowed_access = full_access_request; + for (size_t i = 0; i < ARRAY_SIZE(layer_masks.access); i++) + allowed_access &= ~layer_masks.access[i]; } /* @@ -1690,11 +1679,10 @@ static int hook_file_open(struct file *const file) landlock_file(file)->allowed_access = allowed_access; #ifdef CONFIG_AUDIT landlock_file(file)->deny_masks = landlock_get_deny_masks( - _LANDLOCK_ACCESS_FS_OPTIONAL, optional_access, &layer_masks, - ARRAY_SIZE(layer_masks)); + _LANDLOCK_ACCESS_FS_OPTIONAL, optional_access, &layer_masks); #endif /* CONFIG_AUDIT */ - if ((open_access_request & allowed_access) == open_access_request) + if (access_mask_subset(open_access_request, allowed_access)) return 0; /* Sets access to reflect the actual request. */ diff --git a/security/landlock/net.c b/security/landlock/net.c index 8bce78cf3d6d0..7ae97dec04342 100644 --- a/security/landlock/net.c +++ b/security/landlock/net.c @@ -48,7 +48,7 @@ static int current_check_access_socket(struct socket *const sock, { unsigned short sock_family; __be16 port; - layer_mask_t layer_masks[LANDLOCK_NUM_ACCESS_NET] = {}; + struct layer_access_masks layer_masks = {}; const struct landlock_rule *rule; struct landlock_id id = { .type = LANDLOCK_KEY_NET_PORT, @@ -201,8 +201,10 @@ static int current_check_access_socket(struct socket *const sock, access_request = landlock_init_layer_masks(subject->domain, access_request, &layer_masks, LANDLOCK_KEY_NET_PORT); - if (landlock_unmask_layers(rule, access_request, &layer_masks, - ARRAY_SIZE(layer_masks))) + if (!access_request) + return 0; + + if (landlock_unmask_layers(rule, &layer_masks)) return 0; audit_net.family = address->sa_family; @@ -214,7 +216,6 @@ static int current_check_access_socket(struct socket *const sock, .audit.u.net = &audit_net, .access = access_request, .layer_masks = &layer_masks, - .layer_masks_size = ARRAY_SIZE(layer_masks), }); return -EACCES; } diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index ce7940efea516..7573778dbdf24 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -33,9 +33,8 @@ static struct landlock_ruleset *create_ruleset(const u32 num_layers) { struct landlock_ruleset *new_ruleset; - new_ruleset = - kzalloc(struct_size(new_ruleset, access_masks, num_layers), - GFP_KERNEL_ACCOUNT); + new_ruleset = kzalloc_flex(*new_ruleset, access_masks, num_layers, + GFP_KERNEL_ACCOUNT); if (!new_ruleset) return ERR_PTR(-ENOMEM); refcount_set(&new_ruleset->usage, 1); @@ -553,7 +552,7 @@ landlock_merge_ruleset(struct landlock_ruleset *const parent, return new_dom; new_dom->hierarchy = - kzalloc(sizeof(*new_dom->hierarchy), GFP_KERNEL_ACCOUNT); + kzalloc_obj(*new_dom->hierarchy, GFP_KERNEL_ACCOUNT); if (!new_dom->hierarchy) return ERR_PTR(-ENOMEM); @@ -605,22 +604,24 @@ landlock_find_rule(const struct landlock_ruleset *const ruleset, return NULL; } -/* - * @layer_masks is read and may be updated according to the access request and - * the matching rule. - * @masks_array_size must be equal to ARRAY_SIZE(*layer_masks). +/** + * landlock_unmask_layers - Remove the access rights in @masks + * which are granted in @rule + * + * Updates the set of (per-layer) unfulfilled access rights @masks + * so that all the access rights granted in @rule are removed from it + * (because they are now fulfilled). + * + * @rule: A rule that grants a set of access rights for each layer + * @masks: A matrix of unfulfilled access rights for each layer * - * Returns true if the request is allowed (i.e. relevant layer masks for the - * request are empty). + * Returns true if the request is allowed (i.e. the access rights granted all + * remaining unfulfilled access rights and masks has no leftover set bits). */ bool landlock_unmask_layers(const struct landlock_rule *const rule, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[], - const size_t masks_array_size) + struct layer_access_masks *masks) { - size_t layer_level; - - if (!access_request || !layer_masks) + if (!masks) return true; if (!rule) return false; @@ -635,28 +636,18 @@ bool landlock_unmask_layers(const struct landlock_rule *const rule, * by only one rule, but by the union (binary OR) of multiple rules. * E.g. /a/b + /a => /a/b */ - for (layer_level = 0; layer_level < rule->num_layers; layer_level++) { - const struct landlock_layer *const layer = - &rule->layers[layer_level]; - const layer_mask_t layer_bit = BIT_ULL(layer->level - 1); - const unsigned long access_req = access_request; - unsigned long access_bit; - bool is_empty; + for (size_t i = 0; i < rule->num_layers; i++) { + const struct landlock_layer *const layer = &rule->layers[i]; - /* - * Records in @layer_masks which layer grants access to each - * requested access. - */ - is_empty = true; - for_each_set_bit(access_bit, &access_req, masks_array_size) { - if (layer->access & BIT_ULL(access_bit)) - (*layer_masks)[access_bit] &= ~layer_bit; - is_empty = is_empty && !(*layer_masks)[access_bit]; - } - if (is_empty) - return true; + /* Clear the bits where the layer in the rule grants access. */ + masks->access[layer->level - 1] &= ~layer->access; + } + + for (size_t i = 0; i < ARRAY_SIZE(masks->access); i++) { + if (masks->access[i]) + return false; } - return false; + return true; } typedef access_mask_t @@ -666,13 +657,12 @@ get_access_mask_t(const struct landlock_ruleset *const ruleset, /** * landlock_init_layer_masks - Initialize layer masks from an access request * - * Populates @layer_masks such that for each access right in @access_request, + * Populates @masks such that for each access right in @access_request, * the bits for all the layers are set where this access right is handled. * * @domain: The domain that defines the current restrictions. * @access_request: The requested access rights to check. - * @layer_masks: It must contain %LANDLOCK_NUM_ACCESS_FS or - * %LANDLOCK_NUM_ACCESS_NET elements according to @key_type. + * @masks: Layer access masks to populate. * @key_type: The key type to switch between access masks of different types. * * Returns: An access mask where each access right bit is set which is handled @@ -681,23 +671,20 @@ get_access_mask_t(const struct landlock_ruleset *const ruleset, access_mask_t landlock_init_layer_masks(const struct landlock_ruleset *const domain, const access_mask_t access_request, - layer_mask_t (*const layer_masks)[], + struct layer_access_masks *const masks, const enum landlock_key_type key_type) { access_mask_t handled_accesses = 0; - size_t layer_level, num_access; get_access_mask_t *get_access_mask; switch (key_type) { case LANDLOCK_KEY_INODE: get_access_mask = landlock_get_fs_access_mask; - num_access = LANDLOCK_NUM_ACCESS_FS; break; #if IS_ENABLED(CONFIG_INET) case LANDLOCK_KEY_NET_PORT: get_access_mask = landlock_get_net_access_mask; - num_access = LANDLOCK_NUM_ACCESS_NET; break; #endif /* IS_ENABLED(CONFIG_INET) */ @@ -706,27 +693,18 @@ landlock_init_layer_masks(const struct landlock_ruleset *const domain, return 0; } - memset(layer_masks, 0, - array_size(sizeof((*layer_masks)[0]), num_access)); - /* An empty access request can happen because of O_WRONLY | O_RDWR. */ if (!access_request) return 0; - /* Saves all handled accesses per layer. */ - for (layer_level = 0; layer_level < domain->num_layers; layer_level++) { - const unsigned long access_req = access_request; - const access_mask_t access_mask = - get_access_mask(domain, layer_level); - unsigned long access_bit; - - for_each_set_bit(access_bit, &access_req, num_access) { - if (BIT_ULL(access_bit) & access_mask) { - (*layer_masks)[access_bit] |= - BIT_ULL(layer_level); - handled_accesses |= BIT_ULL(access_bit); - } - } + for (size_t i = 0; i < domain->num_layers; i++) { + const access_mask_t handled = get_access_mask(domain, i); + + masks->access[i] = access_request & handled; + handled_accesses |= masks->access[i]; } + for (size_t i = domain->num_layers; i < ARRAY_SIZE(masks->access); i++) + masks->access[i] = 0; + return handled_accesses; } diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 5da9a64f5af7d..e8bd30df56af1 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -302,14 +302,12 @@ landlock_get_scope_mask(const struct landlock_ruleset *const ruleset, } bool landlock_unmask_layers(const struct landlock_rule *const rule, - const access_mask_t access_request, - layer_mask_t (*const layer_masks)[], - const size_t masks_array_size); + struct layer_access_masks *masks); access_mask_t landlock_init_layer_masks(const struct landlock_ruleset *const domain, const access_mask_t access_request, - layer_mask_t (*const layer_masks)[], + struct layer_access_masks *masks, const enum landlock_key_type key_type); #endif /* _SECURITY_LANDLOCK_RULESET_H */ diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h index 30e3fa962f6fa..f698ff7962282 100644 --- a/security/selinux/include/security.h +++ b/security/selinux/include/security.h @@ -225,6 +225,7 @@ int security_read_policy(void **data, size_t *len); int security_read_state_kernel(void **data, size_t *len); int security_policycap_supported(unsigned int req_cap); +/* Maximum supported number of permissions per class */ #define SEL_VEC_MAX 32 struct av_decision { u32 allowed; diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c index c2c31521cace1..33556922f15ed 100644 --- a/security/selinux/ss/avtab.c +++ b/security/selinux/ss/avtab.c @@ -349,7 +349,7 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po struct avtab_extended_perms xperms; __le32 buf32[ARRAY_SIZE(xperms.perms.p)]; int rc; - unsigned int set, vers = pol->policyvers; + unsigned int vers = pol->policyvers; memset(&key, 0, sizeof(struct avtab_key)); memset(&datum, 0, sizeof(struct avtab_datum)); @@ -360,9 +360,12 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po pr_err("SELinux: avtab: truncated entry\n"); return rc; } + /* Read five or more items: source type, target type, + * target class, AV type, and at least one datum. + */ items2 = le32_to_cpu(buf32[0]); - if (items2 > ARRAY_SIZE(buf32)) { - pr_err("SELinux: avtab: entry overflow\n"); + if (items2 < 5 || items2 > ARRAY_SIZE(buf32)) { + pr_err("SELinux: avtab: invalid item count\n"); return -EINVAL; } rc = next_entry(buf32, fp, sizeof(u32) * items2); @@ -391,6 +394,13 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po return -EINVAL; } + if (!policydb_type_isvalid(pol, key.source_type) || + !policydb_type_isvalid(pol, key.target_type) || + !policydb_class_isvalid(pol, key.target_class)) { + pr_err("SELinux: avtab: invalid type or class\n"); + return -EINVAL; + } + val = le32_to_cpu(buf32[items++]); enabled = (val & AVTAB_ENABLED_OLD) ? AVTAB_ENABLED : 0; @@ -409,6 +419,11 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po for (i = 0; i < ARRAY_SIZE(spec_order); i++) { if (val & spec_order[i]) { + if (items >= items2) { + pr_err("SELinux: avtab: entry has too many items (%d/%d)\n", + items + 1, items2); + return -EINVAL; + } key.specified = spec_order[i] | enabled; datum.u.data = le32_to_cpu(buf32[items++]); rc = insertf(a, &key, &datum, p); @@ -444,9 +459,13 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po return -EINVAL; } - set = hweight16(key.specified & (AVTAB_XPERMS | AVTAB_TYPE | AVTAB_AV)); - if (!set || set > 1) { - pr_err("SELinux: avtab: more than one specifier\n"); + if (hweight16(key.specified & ~AVTAB_ENABLED) != 1) { + pr_err("SELinux: avtab: not exactly one specifier\n"); + return -EINVAL; + } + + if (key.specified & ~AVTAB_SPECIFIER_MASK) { + pr_err("SELinux: avtab: invalid specifier\n"); return -EINVAL; } @@ -471,6 +490,10 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po pr_err("SELinux: avtab: truncated entry\n"); return rc; } + if (!avtab_is_valid_xperm_specified(xperms.specified)) + pr_warn_once_policyload(pol, + "SELinux: avtab: unsupported xperm specifier %#x\n", + xperms.specified); rc = next_entry(&xperms.driver, fp, sizeof(u8)); if (rc) { pr_err("SELinux: avtab: truncated entry\n"); diff --git a/security/selinux/ss/avtab.h b/security/selinux/ss/avtab.h index 850b3453f2598..1de4cce288a76 100644 --- a/security/selinux/ss/avtab.h +++ b/security/selinux/ss/avtab.h @@ -44,6 +44,7 @@ struct avtab_key { AVTAB_XPERMS_DONTAUDIT) #define AVTAB_ENABLED_OLD 0x80000000 /* reserved for used in cond_avtab */ #define AVTAB_ENABLED 0x8000 /* reserved for used in cond_avtab */ +#define AVTAB_SPECIFIER_MASK (AVTAB_AV | AVTAB_TYPE | AVTAB_XPERMS | AVTAB_ENABLED) u16 specified; /* what field is specified */ }; @@ -68,6 +69,18 @@ struct avtab_extended_perms { struct extended_perms_data perms; }; +static inline bool avtab_is_valid_xperm_specified(u8 specified) +{ + switch (specified) { + case AVTAB_XPERMS_IOCTLFUNCTION: + case AVTAB_XPERMS_IOCTLDRIVER: + case AVTAB_XPERMS_NLMSG: + return true; + default: + return false; + } +} + struct avtab_datum { union { u32 data; /* access vector or type value */ diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c index 1bebfcb9c6a12..280dc57df8d39 100644 --- a/security/selinux/ss/conditional.c +++ b/security/selinux/ss/conditional.c @@ -165,7 +165,7 @@ void cond_policydb_destroy(struct policydb *p) int cond_init_bool_indexes(struct policydb *p) { kfree(p->bool_val_to_struct); - p->bool_val_to_struct = kmalloc_array( + p->bool_val_to_struct = kcalloc( p->p_bools.nprim, sizeof(*p->bool_val_to_struct), GFP_KERNEL); if (!p->bool_val_to_struct) return -ENOMEM; @@ -199,19 +199,12 @@ int cond_index_bool(void *key, void *datum, void *datap) return 0; } -static int bool_isvalid(struct cond_bool_datum *b) -{ - if (!(b->state == 0 || b->state == 1)) - return 0; - return 1; -} - int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp) { char *key = NULL; struct cond_bool_datum *booldatum; __le32 buf[3]; - u32 len; + u32 len, val; int rc; booldatum = kzalloc(sizeof(*booldatum), GFP_KERNEL); @@ -223,11 +216,12 @@ int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp) goto err; booldatum->value = le32_to_cpu(buf[0]); - booldatum->state = le32_to_cpu(buf[1]); + val = le32_to_cpu(buf[1]); rc = -EINVAL; - if (!bool_isvalid(booldatum)) + if (!val_is_boolean(val)) goto err; + booldatum->state = (int)val; len = le32_to_cpu(buf[2]); @@ -241,6 +235,7 @@ int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp) return 0; err: + pr_err("SELinux: conditional: failed to read boolean\n"); cond_destroy_bool(key, booldatum, NULL); return rc; } @@ -362,7 +357,8 @@ static int expr_node_isvalid(struct policydb *p, struct cond_expr_node *expr) return 0; } - if (expr->boolean > p->p_bools.nprim) { + if (expr->expr_type == COND_BOOL && + (expr->boolean == 0 || expr->boolean > p->p_bools.nprim)) { pr_err("SELinux: conditional expressions uses unknown bool.\n"); return 0; } @@ -710,9 +706,8 @@ static int duplicate_policydb_bools(struct policydb *newdb, struct cond_bool_datum **cond_bool_array; int rc; - cond_bool_array = kmalloc_array(orig->p_bools.nprim, - sizeof(*orig->bool_val_to_struct), - GFP_KERNEL); + cond_bool_array = kcalloc(orig->p_bools.nprim, + sizeof(*orig->bool_val_to_struct), GFP_KERNEL); if (!cond_bool_array) return -ENOMEM; diff --git a/security/selinux/ss/constraint.h b/security/selinux/ss/constraint.h index 203033cfad672..1d75a8a044df9 100644 --- a/security/selinux/ss/constraint.h +++ b/security/selinux/ss/constraint.h @@ -50,6 +50,7 @@ struct constraint_expr { u32 op; /* operator */ struct ebitmap names; /* names */ + /* internally unused, only forwarded via policydb_write() */ struct type_set *type_names; struct constraint_expr *next; /* next expression */ diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c index 91df3db6a88cd..483d7d54642a3 100644 --- a/security/selinux/ss/policydb.c +++ b/security/selinux/ss/policydb.c @@ -638,13 +638,11 @@ static int sens_index(void *key, void *datum, void *datap) levdatum = datum; p = datap; - if (!levdatum->isalias) { - if (!levdatum->level.sens || - levdatum->level.sens > p->p_levels.nprim) - return -EINVAL; + if (!levdatum->level.sens || levdatum->level.sens > p->p_levels.nprim) + return -EINVAL; + if (!levdatum->isalias) p->sym_val_to_name[SYM_LEVELS][levdatum->level.sens - 1] = key; - } return 0; } @@ -657,12 +655,11 @@ static int cat_index(void *key, void *datum, void *datap) catdatum = datum; p = datap; - if (!catdatum->isalias) { - if (!catdatum->value || catdatum->value > p->p_cats.nprim) - return -EINVAL; + if (!catdatum->value || catdatum->value > p->p_cats.nprim) + return -EINVAL; + if (!catdatum->isalias) p->sym_val_to_name[SYM_CATS][catdatum->value - 1] = key; - } return 0; } @@ -721,6 +718,7 @@ static inline void symtab_hash_eval(struct symtab *s) static int policydb_index(struct policydb *p) { int i, rc; + u32 v; if (p->mls_enabled) pr_debug( @@ -773,6 +771,24 @@ static int policydb_index(struct policydb *p) if (rc) goto out; } + + /* + * A sparse class value is absorbed by policydb_class_isvalid() and + * its siblings, but no such predicate exists for booleans: every + * user of bool_val_to_struct[] walks it by index and dereferences + * each entry -- cond_evaluate_expr(), the two getters and + * security_set_bools() -- so an unclaimed one has no consumer that + * can tolerate it. + */ + for (v = 0; v < p->p_bools.nprim; v++) { + if (!p->bool_val_to_struct[v]) { + pr_err("SELinux: boolean %u is declared but not defined\n", + v + 1); + rc = -EINVAL; + goto out; + } + } + rc = 0; out: return rc; @@ -932,7 +948,7 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s) return 0; } -int policydb_class_isvalid(struct policydb *p, unsigned int class) +int policydb_class_isvalid(struct policydb *p, u16 class) { if (!class || class > p->p_classes.nprim) return 0; @@ -1123,7 +1139,18 @@ int str_read(char **strp, gfp_t flags, struct policy_file *fp, u32 len) return 0; } -static int perm_read(struct policydb *p, struct symtab *s, struct policy_file *fp) +/* + * Bitmap of the permission values a symtab has claimed. Values are 1-based + * and bounded by SEL_VEC_MAX, the width of an access vector, so the whole set + * fits in a u32 and the callers reject an nprim past that width. + */ +static u32 perm_claimed_mask(u32 nprim) +{ + return nprim ? U32_MAX >> (SEL_VEC_MAX - nprim) : 0; +} + +static int perm_read(struct policydb *p, struct symtab *s, + struct policy_file *fp, u32 *claimed) { char *key = NULL; struct perm_datum *perdatum; @@ -1141,6 +1168,16 @@ static int perm_read(struct policydb *p, struct symtab *s, struct policy_file *f len = le32_to_cpu(buf[0]); perdatum->value = le32_to_cpu(buf[1]); + rc = -EINVAL; + if (perdatum->value < 1 || perdatum->value > SEL_VEC_MAX) + goto bad; + /* indexes an nprim-sized array in security_get_permissions() */ + if (perdatum->value > s->nprim) + goto bad; + /* two permissions cannot share one slot of that array */ + if (*claimed & (1U << (perdatum->value - 1))) + goto bad; + *claimed |= 1U << (perdatum->value - 1); rc = str_read(&key, GFP_KERNEL, fp, len); if (rc) @@ -1161,7 +1198,7 @@ static int common_read(struct policydb *p, struct symtab *s, struct policy_file char *key = NULL; struct common_datum *comdatum; __le32 buf[4]; - u32 i, len, nel; + u32 i, len, nel, claimed = 0; int rc; comdatum = kzalloc(sizeof(*comdatum), GFP_KERNEL); @@ -1175,22 +1212,36 @@ static int common_read(struct policydb *p, struct symtab *s, struct policy_file len = le32_to_cpu(buf[0]); comdatum->value = le32_to_cpu(buf[1]); nel = le32_to_cpu(buf[3]); + rc = -EINVAL; + if (nel > SEL_VEC_MAX) + goto bad; rc = symtab_init(&comdatum->permissions, nel); if (rc) goto bad; comdatum->permissions.nprim = le32_to_cpu(buf[2]); + /* no permission value can reach a slot past SEL_VEC_MAX */ + rc = -EINVAL; + if (comdatum->permissions.nprim > SEL_VEC_MAX) + goto bad; rc = str_read(&key, GFP_KERNEL, fp, len); if (rc) goto bad; for (i = 0; i < nel; i++) { - rc = perm_read(p, &comdatum->permissions, fp); + rc = perm_read(p, &comdatum->permissions, fp, &claimed); if (rc) goto bad; } + rc = -EINVAL; + if (claimed != perm_claimed_mask(comdatum->permissions.nprim)) { + pr_err("SELinux: common %s does not define every permission it declares\n", + key); + goto bad; + } + hash_eval(&comdatum->permissions.table, "common_permissions", key); rc = symtab_insert(s, key, comdatum); @@ -1326,7 +1377,7 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file * char *key = NULL; struct class_datum *cladatum; __le32 buf[6]; - u32 i, len, len2, ncons, nel; + u32 i, len, len2, ncons, nel, val, claimed = 0, inherited = 0; int rc; cladatum = kzalloc(sizeof(*cladatum), GFP_KERNEL); @@ -1339,13 +1390,25 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file * len = le32_to_cpu(buf[0]); len2 = le32_to_cpu(buf[1]); - cladatum->value = le32_to_cpu(buf[2]); nel = le32_to_cpu(buf[4]); + rc = -EINVAL; + if (nel > SEL_VEC_MAX) + goto bad; + + val = le32_to_cpu(buf[2]); + rc = -EINVAL; + if (val > U16_MAX) + goto bad; + cladatum->value = val; rc = symtab_init(&cladatum->permissions, nel); if (rc) goto bad; cladatum->permissions.nprim = le32_to_cpu(buf[3]); + /* no permission value can reach a slot past SEL_VEC_MAX */ + rc = -EINVAL; + if (cladatum->permissions.nprim > SEL_VEC_MAX) + goto bad; ncons = le32_to_cpu(buf[5]); @@ -1366,13 +1429,36 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file * cladatum->comkey); goto bad; } + + /* + * security_get_permissions() maps the common's permissions + * into an array sized by this class's nprim, so a class must + * declare at least as many as the common it inherits. + */ + if (cladatum->permissions.nprim < + cladatum->comdatum->permissions.nprim) { + pr_err("SELinux: class %s has fewer permissions than common %s\n", + key, cladatum->comkey); + goto bad; + } } for (i = 0; i < nel; i++) { - rc = perm_read(p, &cladatum->permissions, fp); + rc = perm_read(p, &cladatum->permissions, fp, &claimed); if (rc) goto bad; } + /* the class's own permissions must claim the slots the common leaves */ + if (cladatum->comdatum) + inherited = cladatum->comdatum->permissions.nprim; + rc = -EINVAL; + if (claimed != (perm_claimed_mask(cladatum->permissions.nprim) & + ~perm_claimed_mask(inherited))) { + pr_err("SELinux: class %s does not define every permission it declares\n", + key); + goto bad; + } + hash_eval(&cladatum->permissions.table, "class_permissions", key); rc = read_cons_helper(p, &cladatum->constraints, ncons, 0, fp); @@ -1396,16 +1482,59 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file * if (rc) goto bad; - cladatum->default_user = le32_to_cpu(buf[0]); - cladatum->default_role = le32_to_cpu(buf[1]); - cladatum->default_range = le32_to_cpu(buf[2]); + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + switch (val) { + case 0: + case DEFAULT_SOURCE: + case DEFAULT_TARGET: + cladatum->default_user = val; + break; + default: + goto bad; + } + val = le32_to_cpu(buf[1]); + switch (val) { + case 0: + case DEFAULT_SOURCE: + case DEFAULT_TARGET: + cladatum->default_role = val; + break; + default: + goto bad; + } + val = le32_to_cpu(buf[2]); + switch (val) { + case 0: + case DEFAULT_SOURCE_LOW: + case DEFAULT_SOURCE_HIGH: + case DEFAULT_SOURCE_LOW_HIGH: + case DEFAULT_TARGET_LOW: + case DEFAULT_TARGET_HIGH: + case DEFAULT_TARGET_LOW_HIGH: + case DEFAULT_GLBLUB: + cladatum->default_range = val; + break; + default: + goto bad; + } } if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) { rc = next_entry(buf, fp, sizeof(u32) * 1); if (rc) goto bad; - cladatum->default_type = le32_to_cpu(buf[0]); + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + switch (val) { + case 0: + case DEFAULT_TARGET: + case DEFAULT_SOURCE: + cladatum->default_type = val; + break; + default: + goto bad; + } } rc = symtab_insert(s, key, cladatum); @@ -1415,6 +1544,8 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file * return 0; bad: cls_destroy(key, cladatum, NULL); + if (rc) + pr_err("SELinux: invalid class\n"); return rc; } @@ -1606,7 +1737,7 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f struct level_datum *levdatum; int rc; __le32 buf[2]; - u32 len; + u32 len, val; levdatum = kzalloc(sizeof(*levdatum), GFP_KERNEL); if (!levdatum) @@ -1617,7 +1748,11 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f goto bad; len = le32_to_cpu(buf[0]); - levdatum->isalias = le32_to_cpu(buf[1]); + val = le32_to_cpu(buf[1]); + rc = -EINVAL; + if (!val_is_boolean(val)) + goto bad; + levdatum->isalias = val; rc = str_read(&key, GFP_KERNEL, fp, len); if (rc) @@ -1633,6 +1768,8 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f return 0; bad: sens_destroy(key, levdatum, NULL); + if (rc) + pr_err("SELinux: invalid sensitivity\n"); return rc; } @@ -1642,7 +1779,7 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp struct cat_datum *catdatum; int rc; __le32 buf[3]; - u32 len; + u32 len, val; catdatum = kzalloc(sizeof(*catdatum), GFP_KERNEL); if (!catdatum) @@ -1654,7 +1791,11 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp len = le32_to_cpu(buf[0]); catdatum->value = le32_to_cpu(buf[1]); - catdatum->isalias = le32_to_cpu(buf[2]); + val = le32_to_cpu(buf[2]); + rc = -EINVAL; + if (!val_is_boolean(val)) + goto bad; + catdatum->isalias = val; rc = str_read(&key, GFP_KERNEL, fp, len); if (rc) @@ -1666,6 +1807,8 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp return 0; bad: cat_destroy(key, catdatum, NULL); + if (rc) + pr_err("SELinux: invalid category\n"); return rc; } @@ -1847,7 +1990,7 @@ static int range_read(struct policydb *p, struct policy_file *fp) struct mls_range *r = NULL; int rc; __le32 buf[2]; - u32 i, nel; + u32 i, nel, val; if (p->policyvers < POLICYDB_VERSION_MLS) return 0; @@ -1878,7 +2021,11 @@ static int range_read(struct policydb *p, struct policy_file *fp) rc = next_entry(buf, fp, sizeof(u32)); if (rc) goto out; - rt->target_class = le32_to_cpu(buf[0]); + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + if (val > U16_MAX) + goto out; + rt->target_class = val; } else rt->target_class = p->process_class; @@ -1915,6 +2062,8 @@ static int range_read(struct policydb *p, struct policy_file *fp) out: kfree(rt); kfree(r); + if (rc) + pr_err("SELinux: invalid range\n"); return rc; } @@ -1923,7 +2072,7 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f struct filename_trans_key key, *ft = NULL; struct filename_trans_datum *last, *datum = NULL; char *name = NULL; - u32 len, stype, otype; + u32 len, stype, otype, val; __le32 buf[4]; int rc; @@ -1942,9 +2091,17 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f if (rc) goto out; + rc = -EINVAL; stype = le32_to_cpu(buf[0]); + if (!policydb_type_isvalid(p, stype)) + goto out; key.ttype = le32_to_cpu(buf[1]); - key.tclass = le32_to_cpu(buf[2]); + if (!policydb_type_isvalid(p, key.ttype)) + goto out; + val = le32_to_cpu(buf[2]); + if (val > U16_MAX || !policydb_class_isvalid(p, val)) + goto out; + key.tclass = val; key.name = name; otype = le32_to_cpu(buf[3]); @@ -2000,6 +2157,9 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f kfree(ft); kfree(name); kfree(datum); + + if (rc) + pr_err("SELinux: invalid compat filename transition\n"); return rc; } @@ -2008,7 +2168,8 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp struct filename_trans_key *ft = NULL; struct filename_trans_datum **dst, *datum, *first = NULL; char *name = NULL; - u32 len, ttype, tclass, ndatum, i; + u32 len, ttype, ndatum, i, val; + u16 tclass; __le32 buf[3]; int rc; @@ -2027,8 +2188,15 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp if (rc) goto out; + rc = -EINVAL; ttype = le32_to_cpu(buf[0]); - tclass = le32_to_cpu(buf[1]); + if (!policydb_type_isvalid(p, ttype)) + goto out; + val = le32_to_cpu(buf[1]); + rc = -EINVAL; + if (val > U16_MAX || !policydb_class_isvalid(p, val)) + goto out; + tclass = val; ndatum = le32_to_cpu(buf[2]); if (ndatum == 0) { @@ -2058,6 +2226,10 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp datum->otype = le32_to_cpu(buf[0]); + rc = -EINVAL; + if (!policydb_type_isvalid(p, datum->otype)) + goto out; + dst = &datum->next; } @@ -2089,6 +2261,9 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp ebitmap_destroy(&datum->stypes); kfree(datum); } + + if (rc) + pr_err("SELinux: invalid filename transition\n"); return rc; } @@ -2136,7 +2311,7 @@ static int filename_trans_read(struct policydb *p, struct policy_file *fp) static int genfs_read(struct policydb *p, struct policy_file *fp) { int rc; - u32 i, j, nel, nel2, len, len2; + u32 i, j, nel, nel2, len, len2, val; __le32 buf[1]; struct ocontext *l, *c; struct ocontext *newc = NULL; @@ -2206,7 +2381,11 @@ static int genfs_read(struct policydb *p, struct policy_file *fp) if (rc) goto out; - newc->v.sclass = le32_to_cpu(buf[0]); + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + if (val > U16_MAX || (val != 0 && !policydb_class_isvalid(p, val))) + goto out; + newc->v.sclass = val; rc = context_read_and_validate(&newc->context[0], p, fp); if (rc) @@ -2243,6 +2422,9 @@ static int genfs_read(struct policydb *p, struct policy_file *fp) } ocontext_destroy(newc, OCON_FSUSE); + if (rc) + pr_err("SELinux: invalid genfs\n"); + return rc; } @@ -2251,7 +2433,7 @@ static int ocontext_read(struct policydb *p, { int rc; unsigned int i; - u32 j, nel, len; + u32 j, nel, len, val; __be64 prefixbuf[1]; __le32 buf[3]; struct ocontext *l, *c; @@ -2315,11 +2497,25 @@ static int ocontext_read(struct policydb *p, rc = next_entry(buf, fp, sizeof(u32) * 3); if (rc) goto out; - c->u.port.protocol = le32_to_cpu(buf[0]); - c->u.port.low_port = le32_to_cpu(buf[1]); - c->u.port.high_port = le32_to_cpu(buf[2]); - rc = context_read_and_validate(&c->context[0], - p, fp); + + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + if (val > U8_MAX) + goto out; + c->u.port.protocol = val; + val = le32_to_cpu(buf[1]); + if (val > U16_MAX) + goto out; + c->u.port.low_port = val; + val = le32_to_cpu(buf[2]); + if (val > U16_MAX) + goto out; + c->u.port.high_port = val; + if (c->u.port.low_port == 0 || + c->u.port.low_port > c->u.port.high_port) + goto out; + + rc = context_read_and_validate(&c->context[0], p, fp); if (rc) goto out; break; @@ -2437,6 +2633,8 @@ static int ocontext_read(struct policydb *p, } rc = 0; out: + if (rc) + pr_err("SELinux: invalid ocon\n"); return rc; } @@ -2451,7 +2649,7 @@ int policydb_read(struct policydb *p, struct policy_file *fp) struct role_trans_datum *rtd = NULL; int rc; __le32 buf[4]; - u32 i, j, len, nprim, nel, perm; + u32 i, j, len, nprim, nel, perm, val; char *policydb_str; const struct policydb_compat_info *info; @@ -2643,7 +2841,11 @@ int policydb_read(struct policydb *p, struct policy_file *fp) rc = next_entry(buf, fp, sizeof(u32)); if (rc) goto bad; - rtk->tclass = le32_to_cpu(buf[0]); + rc = -EINVAL; + val = le32_to_cpu(buf[0]); + if (val > U16_MAX) + goto bad; + rtk->tclass = val; } else rtk->tclass = p->process_class; diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h index 89a180b1742fb..20b834581106a 100644 --- a/security/selinux/ss/policydb.h +++ b/security/selinux/ss/policydb.h @@ -48,7 +48,7 @@ struct common_datum { /* Class attributes */ struct class_datum { - u32 value; /* class value */ + u16 value; /* class value */ char *comkey; /* common name */ struct common_datum *comdatum; /* common datum */ struct symtab permissions; /* class-specific permission symbol table */ @@ -74,7 +74,7 @@ struct class_datum { /* Role attributes */ struct role_datum { u32 value; /* internal role value */ - u32 bounds; /* boundary of role */ + u32 bounds; /* boundary of role, 0 for none */ struct ebitmap dominates; /* set of roles dominated by this role */ struct ebitmap types; /* set of authorized types for role */ }; @@ -82,7 +82,7 @@ struct role_datum { struct role_trans_key { u32 role; /* current role */ u32 type; /* program executable type, or new object type */ - u32 tclass; /* process class, or new object class */ + u16 tclass; /* process class, or new object class */ }; struct role_trans_datum { @@ -110,7 +110,8 @@ struct role_allow { /* Type attributes */ struct type_datum { u32 value; /* internal type value */ - u32 bounds; /* boundary of type */ + u32 bounds; /* boundary of type, 0 for none */ + /* internally unused, only forwarded via policydb_write() */ unsigned char primary; /* primary name? */ unsigned char attribute; /* attribute ?*/ }; @@ -118,7 +119,7 @@ struct type_datum { /* User attributes */ struct user_datum { u32 value; /* internal user value */ - u32 bounds; /* bounds of user */ + u32 bounds; /* bounds of user, 0 for none */ struct ebitmap roles; /* set of authorized roles for user */ struct mls_range range; /* MLS range (min - max) for user */ struct mls_level dfltlevel; /* default login MLS level for user */ @@ -139,7 +140,7 @@ struct cat_datum { struct range_trans { u32 source_type; u32 target_type; - u32 target_class; + u16 target_class; }; /* Boolean data type */ @@ -195,7 +196,7 @@ struct ocontext { } ibendport; } u; union { - u32 sclass; /* security class for genfs */ + u16 sclass; /* security class for genfs (can be 0 for wildcard) */ u32 behavior; /* labeling behavior for fs_use */ } v; struct context context[2]; /* security context(s) */ @@ -322,7 +323,7 @@ struct policy_file { extern void policydb_destroy(struct policydb *p); extern int policydb_load_isids(struct policydb *p, struct sidtab *s); extern int policydb_context_isvalid(struct policydb *p, struct context *c); -extern int policydb_class_isvalid(struct policydb *p, unsigned int class); +extern int policydb_class_isvalid(struct policydb *p, u16 class); extern int policydb_type_isvalid(struct policydb *p, unsigned int type); extern int policydb_role_isvalid(struct policydb *p, unsigned int role); extern int policydb_read(struct policydb *p, struct policy_file *fp); @@ -388,9 +389,23 @@ static inline char *sym_name(struct policydb *p, unsigned int sym_num, return p->sym_val_to_name[sym_num][element_nr]; } +static inline bool val_is_boolean(u32 value) +{ + return value == 0 || value == 1; +} + extern int str_read(char **strp, gfp_t flags, struct policy_file *fp, u32 len); extern u16 string_to_security_class(struct policydb *p, const char *name); extern u32 string_to_av_perm(struct policydb *p, u16 tclass, const char *name); +#define pr_warn_once_policyload(policy, fmt, ...) \ + do { \ + static const void *prev_policy__; \ + if (prev_policy__ != policy) { \ + pr_warn(fmt, ##__VA_ARGS__); \ + prev_policy__ = policy; \ + } \ + } while (0) + #endif /* _SS_POLICYDB_H_ */ diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c index 1f5b05a59f849..fb150025359b0 100644 --- a/security/selinux/ss/services.c +++ b/security/selinux/ss/services.c @@ -446,8 +446,6 @@ static int dump_masked_av_helper(void *k, void *d, void *args) struct perm_datum *pdatum = d; char **permission_names = args; - BUG_ON(pdatum->value < 1 || pdatum->value > 32); - permission_names[pdatum->value - 1] = (char *)k; return 0; @@ -466,7 +464,7 @@ static void security_dump_masked_av(struct policydb *policydb, char *tclass_name; char *scontext_name = NULL; char *tcontext_name = NULL; - char *permission_names[32]; + char *permission_names[SEL_VEC_MAX]; int index; u32 length; bool need_comma = false; @@ -507,7 +505,7 @@ static void security_dump_masked_av(struct policydb *policydb, "scontext=%s tcontext=%s tclass=%s perms=", reason, scontext_name, tcontext_name, tclass_name); - for (index = 0; index < 32; index++) { + for (index = 0; index < SEL_VEC_MAX; index++) { u32 mask = (1 << index); if ((mask & permissions) == 0) @@ -2220,7 +2218,9 @@ void selinux_policy_cancel(struct selinux_load_state *load_state) oldpolicy = rcu_dereference_protected(state->policy, lockdep_is_held(&state->policy_mutex)); - sidtab_cancel_convert(oldpolicy->sidtab); + /* a first load has no outgoing policy and converted nothing */ + if (oldpolicy) + sidtab_cancel_convert(oldpolicy->sidtab); selinux_policy_free(load_state->policy); kfree(load_state->convert_data); } @@ -3288,7 +3288,7 @@ static int get_classes_callback(void *k, void *d, void *args) { struct class_datum *datum = d; char *name = k, **classes = args; - u32 value = datum->value - 1; + u16 value = datum->value - 1; classes[value] = kstrdup(name, GFP_ATOMIC); if (!classes[value]) @@ -3301,6 +3301,7 @@ int security_get_classes(struct selinux_policy *policy, char ***classes, u32 *nclasses) { struct policydb *policydb; + u32 i; int rc; policydb = &policy->policydb; @@ -3313,16 +3314,29 @@ int security_get_classes(struct selinux_policy *policy, rc = hashtab_map(&policydb->p_classes.table, get_classes_callback, *classes); - if (rc) { - u32 i; + if (rc) + goto err; - for (i = 0; i < *nclasses; i++) - kfree((*classes)[i]); - kfree(*classes); + /* + * The class symtab may be sparse, which policydb_class_isvalid() exists + * to absorb; the callback fills this array by value, so an unclaimed + * one leaves a NULL that sel_make_classes() hands to sel_make_dir(). + */ + for (i = 0; i < *nclasses; i++) { + if (!(*classes)[i]) { + rc = -EINVAL; + goto err; + } } out: return rc; + +err: + for (i = 0; i < *nclasses; i++) + kfree((*classes)[i]); + kfree(*classes); + return rc; } static int get_permissions_callback(void *k, void *d, void *args) diff --git a/security/smack/smack.h b/security/smack/smack.h index 759343a6bbaeb..4f7f4760e432a 100644 --- a/security/smack/smack.h +++ b/security/smack/smack.h @@ -303,8 +303,9 @@ int smack_populate_secattr(struct smack_known *skp); * Shared data. */ extern int smack_enabled __initdata; -extern int smack_cipso_direct; -extern int smack_cipso_mapped; +extern u8 smack_cipso_auto_level[2]; +#define smack_cipso_direct (+smack_cipso_auto_level[0]) +#define smack_cipso_mapped (+smack_cipso_auto_level[1]) extern struct smack_known *smack_net_ambient; extern struct smack_known *smack_syslog_label; #ifdef CONFIG_SECURITY_SMACK_BRINGUP diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index 7fd8a60b8be90..217304cd20c3b 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -130,12 +130,13 @@ static int smk_bu_note(char *note, struct smack_known *sskp, #define smk_bu_note(note, sskp, oskp, mode, RC) (RC) #endif -#ifdef CONFIG_SECURITY_SMACK_BRINGUP -static int smk_bu_current(char *note, struct smack_known *oskp, - int mode, int rc) +static int +smk_bu_tsk_to_obj(struct task_struct *tsk, const struct task_smack *tsp, + char *note, struct smack_known *oskp, int mode, int rc) { - struct task_smack *tsp = smack_cred(current_cred()); +#ifdef CONFIG_SECURITY_SMACK_BRINGUP char acc[SMK_NUM_ACCESS_TYPE + 1]; + char comm[TASK_COMM_LEN]; if (rc <= 0) return rc; @@ -143,14 +144,22 @@ static int smk_bu_current(char *note, struct smack_known *oskp, rc = 0; smk_bu_mode(mode, acc); + pr_info("Smack %s: (%s %s %s) %s %s\n", smk_bu_mess[rc], - tsp->smk_task->smk_known, oskp->smk_known, - acc, current->comm, note); + smk_of_task(tsp)->smk_known, oskp->smk_known, + acc, get_task_comm(comm, tsk), note); return 0; -} #else -#define smk_bu_current(note, oskp, mode, RC) (RC) + return rc; #endif +} + +static int smk_bu_current(char *note, struct smack_known *oskp, + int mode, int rc) +{ + return smk_bu_tsk_to_obj(current, smack_cred(current_cred()), + note, oskp, mode, rc); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP static int smk_bu_task(struct task_struct *otp, int mode, int rc) @@ -1312,7 +1321,7 @@ static int smack_inode_getattr(const struct path *path) */ static int smack_inode_xattr_skipcap(const char *name) { - if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX))) + if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0) return 0; if (strcmp(name, XATTR_NAME_SMACK) == 0 || @@ -1969,7 +1978,7 @@ static int smack_file_send_sigiotask(struct task_struct *tsk, { struct smack_known **blob; struct smack_known *skp; - struct smack_known *tkp = smk_of_task(smack_cred(tsk->cred)); + struct smack_known *tkp = smk_of_task_struct_obj(tsk); const struct cred *tcred; struct file *file; int rc; @@ -3353,14 +3362,20 @@ static int smack_sem_semop(struct kern_ipc_perm *isp, struct sembuf *sops, } /** - * smk_curacc_msq : helper to check if current has access on msq - * @isp : the msq + * smk_tskacc_msq : helper to check if tsk has access on msq + * @tsk: the task that requests access + * @isp : the sysv msg queue permissions * @access : access requested * - * return 0 if current has access, error otherwise + * return 0 if tsk has access, error otherwise */ -static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +static int +smk_tskacc_msq(struct task_struct *tsk, struct kern_ipc_perm *isp, int access) { + const bool tsk_is_current = (tsk == current); + const struct cred * const tsk_cred = + (tsk_is_current ? current_cred() : get_task_cred(tsk)); + struct task_smack * const tsp = smack_cred(tsk_cred); struct smack_known *msp = smack_of_ipc(isp); struct smk_audit_info ad; int rc; @@ -3369,11 +3384,25 @@ static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_IPC); ad.a.u.ipc_id = isp->id; #endif - rc = smk_curacc(msp, access, &ad); - rc = smk_bu_current("msq", msp, access, rc); + rc = smk_tskacc(tsp, msp, access, &ad); + rc = smk_bu_tsk_to_obj(tsk, tsp, "msq", msp, access, rc); + if (!tsk_is_current) + put_cred(tsk_cred); return rc; } +/** + * smk_curacc_msq : helper to check if current has access on msq + * @isp : the sysv msg queue permissions + * @access : access requested + * + * return 0 if current has access, error otherwise + */ +static int smk_curacc_msq(struct kern_ipc_perm *isp, int access) +{ + return smk_tskacc_msq(current, isp, access); +} + /** * smack_msg_queue_associate - Smack access check for msg_queue * @isp: the object @@ -3441,21 +3470,21 @@ static int smack_msg_queue_msgsnd(struct kern_ipc_perm *isp, struct msg_msg *msg } /** - * smack_msg_queue_msgrcv - Smack access check for msg_queue + * smack_msg_queue_msgrcv - check it target has r/w access to msg_queue * @isp: the object * @msg: unused - * @target: unused + * @target: the task that msgrcv() from the queue * @type: unused * @mode: unused * - * Returns 0 if current has read and write access, error code otherwise + * Returns 0 if target has read and write access, error code otherwise */ static int smack_msg_queue_msgrcv(struct kern_ipc_perm *isp, struct msg_msg *msg, struct task_struct *target, long type, int mode) { - return smk_curacc_msq(isp, MAY_READWRITE); + return smk_tskacc_msq(target, isp, MAY_READWRITE); } /** diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c index d27d9140dda2f..32db00102d1d1 100644 --- a/security/smack/smackfs.c +++ b/security/smack/smackfs.c @@ -83,18 +83,27 @@ static DEFINE_MUTEX(smk_net6addr_lock); struct smack_known *smack_net_ambient; /* - * This is the level in a CIPSO header that indicates a + * Sensitivity levels for automatically created CIPSO labels. + * See smack_access.c`smack_populate_secattr() + * + * [0] "direct" labeling, label length < SMK_CIPSOLEN(24): * smack label is contained directly in the category set. * It can be reset via smackfs/direct - */ -int smack_cipso_direct = SMACK_CIPSO_DIRECT_DEFAULT; - -/* - * This is the level in a CIPSO header that indicates a + * + * [1] "mapped" labeling, label length >= SMK_CIPSOLEN(24): * secid is contained directly in the category set. * It can be reset via smackfs/mapped */ -int smack_cipso_mapped = SMACK_CIPSO_MAPPED_DEFAULT; +u8 smack_cipso_auto_level[2] = { + SMACK_CIPSO_DIRECT_DEFAULT, + SMACK_CIPSO_MAPPED_DEFAULT, +}; + +static int +smk_cipso_auto_level_idx(const struct file *file) +{ + return (file_inode(file)->i_ino != SMK_DIRECT); +} #ifdef CONFIG_SECURITY_SMACK_BRINGUP /* @@ -1598,24 +1607,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf, static ssize_t smk_write_doi(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - unsigned long u; + int ret; + u32 u; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; + ret = kstrtou32_from_user(buf, count, 10, &u); + if (unlikely(ret)) + return ret; - temp[count] = '\0'; - - if (kstrtoul(temp, 10, &u)) - return -EINVAL; - - if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX) + if (u == CIPSO_V4_DOI_UNKNOWN) return -EINVAL; return smk_cipso_doi(u, GFP_KERNEL) ? : count; @@ -1628,158 +1630,80 @@ static const struct file_operations smk_doi_ops = { }; /** - * smk_read_direct - read() for /smack/direct - * @filp: file pointer, not actually used + * smk_read_cipso_auto_level - read() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to put the result * @count: maximum to send along * @ppos: where to start * * Returns number of bytes read or error code, as appropriate */ -static ssize_t smk_read_direct(struct file *filp, char __user *buf, +static ssize_t smk_read_cipso_auto_level(struct file *filp, char __user *buf, size_t count, loff_t *ppos) { - char temp[80]; - ssize_t rc; + char temp[sizeof "255"]; + int n; if (*ppos != 0) return 0; - sprintf(temp, "%d", smack_cipso_direct); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; + n = sprintf(temp, "%u", (unsigned int)smack_cipso_auto_level[ + smk_cipso_auto_level_idx(filp)]); + return simple_read_from_buffer(buf, count, ppos, temp, n); } /** - * smk_write_direct - write() for /smack/direct - * @file: file pointer, not actually used + * smk_write_cipso_auto_level - write() for smackfs/direct and smackfs/mapped + * @filp: file pointer * @buf: where to get the data from * @count: bytes sent * @ppos: where to start * * Returns number of bytes written or error code, as appropriate */ -static ssize_t smk_write_direct(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) +static ssize_t +smk_write_cipso_auto_level(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) { - struct smack_known *skp; - char temp[80]; - int i; + int ret, idx; + u8 i, old_lvl; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; - /* - * Don't do anything if the value hasn't actually changed. - * If it is changing reset the level on entries that were - * set up to be direct when they were created. + * draft-ietf-cipso-ipsecurity-01 (CIPSO 2.2), 3.4.2.4: + * "Sensitivity Level is 1 octet in length. Its value is from 0 to 255" */ - if (smack_cipso_direct != i) { - mutex_lock(&smack_known_lock); - list_for_each_entry_rcu(skp, &smack_known_list, list) - if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_direct) - skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_direct = i; - mutex_unlock(&smack_known_lock); - } - - return count; -} - -static const struct file_operations smk_direct_ops = { - .read = smk_read_direct, - .write = smk_write_direct, - .llseek = default_llseek, -}; - -/** - * smk_read_mapped - read() for /smack/mapped - * @filp: file pointer, not actually used - * @buf: where to put the result - * @count: maximum to send along - * @ppos: where to start - * - * Returns number of bytes read or error code, as appropriate - */ -static ssize_t smk_read_mapped(struct file *filp, char __user *buf, - size_t count, loff_t *ppos) -{ - char temp[80]; - ssize_t rc; - - if (*ppos != 0) - return 0; - - sprintf(temp, "%d", smack_cipso_mapped); - rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp)); - - return rc; -} - -/** - * smk_write_mapped - write() for /smack/mapped - * @file: file pointer, not actually used - * @buf: where to get the data from - * @count: bytes sent - * @ppos: where to start - * - * Returns number of bytes written or error code, as appropriate - */ -static ssize_t smk_write_mapped(struct file *file, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct smack_known *skp; - char temp[80]; - int i; - - if (!smack_privileged(CAP_MAC_ADMIN)) - return -EPERM; - - if (count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; + ret = kstrtou8_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; /* * Don't do anything if the value hasn't actually changed. * If it is changing reset the level on entries that were - * set up to be mapped when they were created. + * set up to be "auto" level when they were created. */ - if (smack_cipso_mapped != i) { + idx = smk_cipso_auto_level_idx(filp); + old_lvl = smack_cipso_auto_level[idx]; + + if (old_lvl != i) { + struct smack_known *skp; mutex_lock(&smack_known_lock); list_for_each_entry_rcu(skp, &smack_known_list, list) if (skp->smk_netlabel.attr.mls.lvl == - smack_cipso_mapped) + old_lvl) skp->smk_netlabel.attr.mls.lvl = i; - smack_cipso_mapped = i; + smack_cipso_auto_level[idx] = i; mutex_unlock(&smack_known_lock); } return count; } -static const struct file_operations smk_mapped_ops = { - .read = smk_read_mapped, - .write = smk_write_mapped, +static const struct file_operations +smk_cipso_auto_level_ops = { + .read = smk_read_cipso_auto_level, + .write = smk_write_cipso_auto_level, .llseek = default_llseek, }; @@ -2179,22 +2103,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf, static ssize_t smk_write_logging(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (count >= sizeof(temp) || count == 0) - return -EINVAL; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; - - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < 0 || i > 3) return -EINVAL; log_policy = i; @@ -2838,22 +2755,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf, static ssize_t smk_write_ptrace(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - char temp[32]; - int i; + int i, ret; if (!smack_privileged(CAP_MAC_ADMIN)) return -EPERM; - if (*ppos != 0 || count >= sizeof(temp) || count == 0) - return -EINVAL; - - if (copy_from_user(temp, buf, count) != 0) - return -EFAULT; - - temp[count] = '\0'; + ret = kstrtos32_from_user(buf, count, 10, &i); + if (unlikely(ret)) + return ret; - if (sscanf(temp, "%d", &i) != 1) - return -EINVAL; if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX) return -EINVAL; smack_ptrace_rule = i; @@ -2888,7 +2798,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_DOI] = { "doi", &smk_doi_ops, S_IRUGO|S_IWUSR}, [SMK_DIRECT] = { - "direct", &smk_direct_ops, S_IRUGO|S_IWUSR}, + "direct", &smk_cipso_auto_level_ops, 0644}, [SMK_AMBIENT] = { "ambient", &smk_ambient_ops, S_IRUGO|S_IWUSR}, [SMK_NET4ADDR] = { @@ -2904,7 +2814,7 @@ static int smk_fill_super(struct super_block *sb, struct fs_context *fc) [SMK_ACCESSES] = { "access", &smk_access_ops, S_IRUGO|S_IWUGO}, [SMK_MAPPED] = { - "mapped", &smk_mapped_ops, S_IRUGO|S_IWUSR}, + "mapped", &smk_cipso_auto_level_ops, 0644}, [SMK_LOAD2] = { "load2", &smk_load2_ops, S_IRUGO|S_IWUSR}, [SMK_LOAD_SELF2] = { diff --git a/sound/core/control.c b/sound/core/control.c index c714f2e5596b3..652c09e110f6a 100644 --- a/sound/core/control.c +++ b/sound/core/control.c @@ -2219,7 +2219,6 @@ EXPORT_SYMBOL_GPL(snd_ctl_request_layer); */ void snd_ctl_register_layer(struct snd_ctl_layer_ops *lops) { - struct snd_card *card; int card_number; scoped_guard(rwsem_write, &snd_ctl_layer_rwsem) { @@ -2227,11 +2226,12 @@ void snd_ctl_register_layer(struct snd_ctl_layer_ops *lops) snd_ctl_layer = lops; } for (card_number = 0; card_number < SNDRV_CARDS; card_number++) { - card = snd_card_ref(card_number); + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); + if (card) { scoped_guard(rwsem_read, &card->controls_rwsem) lops->lregister(card); - snd_card_unref(card); } } } diff --git a/sound/core/control_led.c b/sound/core/control_led.c index e33dfcf863cf1..ae903af93dd32 100644 --- a/sound/core/control_led.c +++ b/sound/core/control_led.c @@ -240,8 +240,6 @@ static void snd_ctl_led_notify(struct snd_card *card, unsigned int mask, } } -DEFINE_FREE(snd_card_unref, struct snd_card *, if (_T) snd_card_unref(_T)) - static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, unsigned int group, bool set) { @@ -257,6 +255,8 @@ static int snd_ctl_led_set_id(int card_number, struct snd_ctl_elem_id *id, kctl = snd_ctl_find_id(card, id); if (!kctl) return -ENOENT; + if (!kctl->info || !kctl->get) + return -EINVAL; ioff = snd_ctl_get_ioff(kctl, id); vd = &kctl->vd[ioff]; access = vd->access & SNDRV_CTL_ELEM_ACCESS_LED_MASK; @@ -758,18 +758,17 @@ static int __init snd_ctl_led_init(void) static void __exit snd_ctl_led_exit(void) { struct snd_ctl_led *led; - struct snd_card *card; unsigned int group, card_number; snd_ctl_disconnect_layer(&snd_ctl_led_lops); for (card_number = 0; card_number < SNDRV_CARDS; card_number++) { if (!snd_ctl_led_card_valid[card_number]) continue; - card = snd_card_ref(card_number); - if (card) { + struct snd_card *card __free(snd_card_unref) = + snd_card_ref(card_number); + + if (card) snd_ctl_led_sysfs_remove(card); - snd_card_unref(card); - } } for (group = 0; group < MAX_LED; group++) { led = &snd_ctl_leds[group]; diff --git a/sound/core/init.c b/sound/core/init.c index c8f992cff4a43..bc5182c5ce12b 100644 --- a/sound/core/init.c +++ b/sound/core/init.c @@ -579,6 +579,8 @@ EXPORT_SYMBOL_GPL(snd_card_disconnect_sync); static int snd_card_do_free(struct snd_card *card) { + bool managed = card->managed; + card->releasing = true; #if IS_ENABLED(CONFIG_SND_MIXER_OSS) if (snd_mixer_oss_notify_callback) @@ -593,7 +595,7 @@ static int snd_card_do_free(struct snd_card *card) } if (card->release_completion) complete(card->release_completion); - if (!card->managed) + if (!managed) kfree(card); return 0; } diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 0ae95a4670891..304b7e4965e3f 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -1441,6 +1441,8 @@ static int snd_pcm_pre_start(struct snd_pcm_substream *substream, struct snd_pcm_runtime *runtime = substream->runtime; if (runtime->state != SNDRV_PCM_STATE_PREPARED) return -EBADFD; + if (atomic_read(&runtime->buffer_accessing) < 0) + return -EBADFD; /* during hw_params, hw_free or prepare */ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && !snd_pcm_playback_data(substream)) return -EPIPE; @@ -2334,6 +2336,7 @@ static void relink_to_local(struct snd_pcm_substream *substream) static int snd_pcm_unlink(struct snd_pcm_substream *substream) { + struct snd_pcm_substream *s; struct snd_pcm_group *group; bool nonatomic = substream->pcm->nonatomic; bool do_free = false; @@ -2346,6 +2349,12 @@ static int snd_pcm_unlink(struct snd_pcm_substream *substream) group = substream->group; snd_pcm_group_lock_irq(group, nonatomic); + /* release drain waiters before changing membership, else snd_pcm_drain() + * leaves its on-stack wait entry queued on a member's sleep list + */ + snd_pcm_group_for_each_entry(s, substream) + wake_up(&s->runtime->sleep); + relink_to_local(substream); refcount_dec(&group->refs); @@ -3982,20 +3991,33 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file, return -EINVAL; } runtime = substream->runtime; - if (runtime->state == SNDRV_PCM_STATE_OPEN) - return -EBADFD; - if (!(runtime->info & SNDRV_PCM_INFO_MMAP)) - return -ENXIO; + /* don't race with buffer reallocation in hw_params/hw_free */ + if (!atomic_inc_unless_negative(&runtime->buffer_accessing)) + return -EBUSY; + if (runtime->state == SNDRV_PCM_STATE_OPEN) { + err = -EBADFD; + goto out; + } + if (!(runtime->info & SNDRV_PCM_INFO_MMAP)) { + err = -ENXIO; + goto out; + } if (runtime->access == SNDRV_PCM_ACCESS_RW_INTERLEAVED || - runtime->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) - return -EINVAL; + runtime->access == SNDRV_PCM_ACCESS_RW_NONINTERLEAVED) { + err = -EINVAL; + goto out; + } size = area->vm_end - area->vm_start; offset = area->vm_pgoff << PAGE_SHIFT; dma_bytes = PAGE_ALIGN(runtime->dma_bytes); - if ((size_t)size > dma_bytes) - return -EINVAL; - if (offset > dma_bytes - size) - return -EINVAL; + if ((size_t)size > dma_bytes) { + err = -EINVAL; + goto out; + } + if (offset > dma_bytes - size) { + err = -EINVAL; + goto out; + } area->vm_ops = &snd_pcm_vm_ops_data; area->vm_private_data = substream; @@ -4005,6 +4027,8 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file, err = snd_pcm_lib_default_mmap(substream, area); if (!err) atomic_inc(&substream->mmap_count); +out: + atomic_dec(&runtime->buffer_accessing); return err; } EXPORT_SYMBOL(snd_pcm_mmap_data); diff --git a/sound/core/rawmidi.c b/sound/core/rawmidi.c index 20d36a346ccab..7508d99c1e441 100644 --- a/sound/core/rawmidi.c +++ b/sound/core/rawmidi.c @@ -782,7 +782,7 @@ int snd_rawmidi_input_params(struct snd_rawmidi_substream *substream, substream->framing = framing; substream->clock_type = clock_type; } - return 0; + return err; } EXPORT_SYMBOL(snd_rawmidi_input_params); diff --git a/sound/core/seq/oss/seq_oss_rw.c b/sound/core/seq/oss/seq_oss_rw.c index 111c792bc72ca..b7147ac78ee8b 100644 --- a/sound/core/seq/oss/seq_oss_rw.c +++ b/sound/core/seq/oss/seq_oss_rw.c @@ -57,7 +57,8 @@ snd_seq_oss_read(struct seq_oss_devinfo *dp, char __user *buf, int count) break; } ev_len = ev_length(&rec); - if (ev_len < count) { + if (count < ev_len) { + err = -EINVAL; snd_seq_oss_readq_unlock(readq, flags); break; } diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c index 0c31b19dd948b..e73c1becb6e15 100644 --- a/sound/core/seq/seq_clientmgr.c +++ b/sound/core/seq/seq_clientmgr.c @@ -528,7 +528,7 @@ static int bounce_error_event(struct snd_seq_client *client, struct snd_seq_event *event, int err, int atomic, int hop) { - struct snd_seq_event bounce_ev; + struct snd_seq_event bounce_ev, quoted; int result; if (client == NULL || @@ -544,15 +544,19 @@ static int bounce_error_event(struct snd_seq_client *client, * For user clients, send SNDRV_SEQ_EVENT_BOUNCE with the * original event embedded as variable-length data. This * avoids exposing data.quote.event (a kernel pointer) to - * userspace. The variable-length path in snd_seq_event_dup() - * copies the event data from data.ext.ptr into chained cells, - * and snd_seq_expand_var_event() copies only the data content - * -- never the pointer -- to userspace. + * userspace. Sanitise the embedded copy too - a queued + * variable-length event carries the address of its own + * extension cell, and the payload goes out verbatim. */ + quoted = *event; + if (snd_seq_ev_is_variable("ed)) { + quoted.data.ext.len &= ~SNDRV_SEQ_EXT_MASK; + quoted.data.ext.ptr = NULL; + } bounce_ev.type = SNDRV_SEQ_EVENT_BOUNCE; bounce_ev.flags = SNDRV_SEQ_EVENT_LENGTH_VARIABLE; bounce_ev.data.ext.len = sizeof(struct snd_seq_event); - bounce_ev.data.ext.ptr = (char *)event; + bounce_ev.data.ext.ptr = (char *)"ed; } else { /* * For kernel clients, quote the event pointer directly. diff --git a/sound/core/seq/seq_midi.c b/sound/core/seq/seq_midi.c index 24d485caeb0a2..f61f30b911803 100644 --- a/sound/core/seq/seq_midi.c +++ b/sound/core/seq/seq_midi.c @@ -42,9 +42,11 @@ struct seq_midisynth { struct snd_rawmidi *rmidi; int device; int subdevice; + struct snd_rawmidi_substream __rcu *input_substream; + snd_use_lock_t input_use_lock; /* in-flight event_input users */ struct snd_rawmidi_file input_rfile; - spinlock_t output_lock; /* protects output_rfile publication */ snd_use_lock_t output_use_lock; /* in-flight event_input users */ + struct snd_rawmidi_substream __rcu *output_substream; struct snd_rawmidi_file output_rfile; int seq_client; int seq_port; @@ -76,6 +78,14 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) msynth = runtime->private_data; if (msynth == NULL) return; + + scoped_guard(rcu) { + if (rcu_dereference(msynth->input_substream) != substream) + return; + + snd_use_lock_use(&msynth->input_use_lock); + } + memset(&ev, 0, sizeof(ev)); while (runtime->avail > 0) { res = snd_rawmidi_kernel_read(substream, buf, sizeof(buf)); @@ -95,6 +105,8 @@ static void snd_midi_input_event(struct snd_rawmidi_substream *substream) memset(&ev, 0, sizeof(ev)); } } + + snd_use_lock_free(&msynth->input_use_lock); } static int dump_midi(struct snd_rawmidi_substream *substream, const char *buf, int count) @@ -134,8 +146,8 @@ static int event_process_midi(struct snd_seq_event *ev, int direct, if (snd_BUG_ON(!msynth)) return -EINVAL; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - substream = msynth->output_rfile.output; + scoped_guard(rcu) { + substream = rcu_dereference(msynth->output_substream); if (!substream) return -ENODEV; snd_use_lock_use(&msynth->output_use_lock); @@ -177,7 +189,7 @@ static int snd_seq_midisynth_new(struct seq_midisynth *msynth, msynth->card = card; msynth->device = device; msynth->subdevice = subdevice; - spin_lock_init(&msynth->output_lock); + snd_use_lock_init(&msynth->input_use_lock); snd_use_lock_init(&msynth->output_use_lock); return 0; } @@ -188,28 +200,31 @@ static int midisynth_subscribe(void *private_data, struct snd_seq_port_subscribe int err; struct seq_midisynth *msynth = private_data; struct snd_rawmidi_runtime *runtime; + struct snd_rawmidi_file rfile = {}; struct snd_rawmidi_params params; /* open midi port */ err = snd_rawmidi_kernel_open(msynth->rmidi, msynth->subdevice, SNDRV_RAWMIDI_LFLG_INPUT, - &msynth->input_rfile); + &rfile); if (err < 0) { pr_debug("ALSA: seq_midi: midi input open failed!!!\n"); return err; } - runtime = msynth->input_rfile.input->runtime; + runtime = rfile.input->runtime; memset(¶ms, 0, sizeof(params)); params.avail_min = 1; params.buffer_size = input_buffer_size; - err = snd_rawmidi_input_params(msynth->input_rfile.input, ¶ms); + err = snd_rawmidi_input_params(rfile.input, ¶ms); if (err < 0) { - snd_rawmidi_kernel_release(&msynth->input_rfile); + snd_rawmidi_kernel_release(&rfile); return err; } snd_midi_event_reset_encode(msynth->parser); runtime->event = snd_midi_input_event; runtime->private_data = msynth; + msynth->input_rfile = rfile; + rcu_assign_pointer(msynth->input_substream, rfile.input); snd_rawmidi_kernel_read(msynth->input_rfile.input, NULL, 0); return 0; } @@ -219,10 +234,19 @@ static int midisynth_unsubscribe(void *private_data, struct snd_seq_port_subscri { int err; struct seq_midisynth *msynth = private_data; + struct snd_rawmidi_file rfile; - if (snd_BUG_ON(!msynth->input_rfile.input)) + rcu_assign_pointer(msynth->input_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->input_use_lock); + + rfile = msynth->input_rfile; + msynth->input_rfile = (struct snd_rawmidi_file){}; + + if (snd_BUG_ON(!rfile.input)) return -EINVAL; - err = snd_rawmidi_kernel_release(&msynth->input_rfile); + + err = snd_rawmidi_kernel_release(&rfile); return err; } @@ -252,8 +276,8 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info return err; } snd_midi_event_reset_decode(msynth->parser); - scoped_guard(spinlock_irqsave, &msynth->output_lock) - msynth->output_rfile = rfile; + msynth->output_rfile = rfile; + rcu_assign_pointer(msynth->output_substream, rfile.output); return 0; } @@ -261,17 +285,16 @@ static int midisynth_use(void *private_data, struct snd_seq_port_subscribe *info static int midisynth_unuse(void *private_data, struct snd_seq_port_subscribe *info) { struct seq_midisynth *msynth = private_data; - struct snd_rawmidi_file rfile = {}; + struct snd_rawmidi_file rfile; - scoped_guard(spinlock_irqsave, &msynth->output_lock) { - rfile = msynth->output_rfile; - msynth->output_rfile = (struct snd_rawmidi_file){}; - } + rcu_assign_pointer(msynth->output_substream, NULL); + synchronize_rcu(); + snd_use_lock_sync(&msynth->output_use_lock); + rfile = msynth->output_rfile; + msynth->output_rfile = (struct snd_rawmidi_file){}; if (snd_BUG_ON(!rfile.output)) return -EINVAL; - - snd_use_lock_sync(&msynth->output_use_lock); snd_rawmidi_drain_output(rfile.output); return snd_rawmidi_kernel_release(&rfile); } diff --git a/sound/core/seq/seq_prioq.c b/sound/core/seq/seq_prioq.c index e649485a87722..8c9a293cb5b51 100644 --- a/sound/core/seq/seq_prioq.c +++ b/sound/core/seq/seq_prioq.c @@ -132,7 +132,7 @@ int snd_seq_prioq_cell_in(struct snd_seq_prioq * f, struct snd_seq_event_cell * cell) { struct snd_seq_event_cell *cur, *prev; - int count; + int remaining; int prior; if (snd_BUG_ON(!f || !cell)) @@ -162,10 +162,16 @@ int snd_seq_prioq_cell_in(struct snd_seq_prioq * f, prev = NULL; /* previous cell */ cur = f->head; /* cursor */ - count = 10000; /* FIXME: enough big, isn't it? */ + remaining = f->cells; while (cur != NULL) { /* compare timestamps */ int rel = compare_timestamp_rel(&cell->event, &cur->event); + + if (remaining-- <= 0) { + pr_err("ALSA: seq: inconsistent prioq cell count\n"); + return -EINVAL; + } + if (rel < 0) /* new cell has earlier schedule time, */ break; @@ -176,10 +182,6 @@ int snd_seq_prioq_cell_in(struct snd_seq_prioq * f, /* move cursor to next cell */ prev = cur; cur = cur->next; - if (! --count) { - pr_err("ALSA: seq: cannot find a pointer.. infinite loop?\n"); - return -EINVAL; - } } /* insert it before cursor */ diff --git a/sound/core/seq/seq_timer.c b/sound/core/seq/seq_timer.c index 29b018a212fc3..a89451f47872e 100644 --- a/sound/core/seq/seq_timer.c +++ b/sound/core/seq/seq_timer.c @@ -61,12 +61,23 @@ struct snd_seq_timer *snd_seq_timer_new(void) void snd_seq_timer_delete(struct snd_seq_timer **tmr) { struct snd_seq_timer *t = *tmr; - *tmr = NULL; + struct snd_timer_instance *ti; if (t == NULL) { pr_debug("ALSA: seq: snd_seq_timer_delete() called with NULL timer\n"); return; } + + scoped_guard(spinlock_irq, &t->lock) { + ti = t->timeri; + t->timeri = NULL; + } + if (ti) { + snd_timer_close(ti); + snd_timer_instance_free(ti); + } + + *tmr = NULL; t->running = 0; /* reset time */ @@ -351,11 +362,10 @@ static int initialize_timer(struct snd_seq_timer *tmr) tmr->ticks = 1; if (!(t->hw.flags & SNDRV_TIMER_HW_SLAVE)) { unsigned long r = snd_timer_resolution(tmr->timeri); - if (r) { - tmr->ticks = (unsigned int)(1000000000uL / (r * freq)); - if (! tmr->ticks) - tmr->ticks = 1; - } + unsigned long den; + + if (r && !check_mul_overflow(r, freq, &den)) + tmr->ticks = max(1U, (unsigned int)(1000000000uL / den)); } tmr->initialized = 1; return 0; diff --git a/sound/core/timer.c b/sound/core/timer.c index 35651c273b2f1..c5f0e895ea024 100644 --- a/sound/core/timer.c +++ b/sound/core/timer.c @@ -409,6 +409,8 @@ static void remove_slave_links(struct snd_timer_instance *timeri, list_del_init(&slave->ack_list); list_del_init(&slave->active_list); } + /* the close is done; a reopen must not see the mark */ + timeri->flags &= ~SNDRV_TIMER_IFLG_DEAD; } /* @@ -437,10 +439,20 @@ static void snd_timer_close_locked(struct snd_timer_instance *timeri, snd_timer_stop(timeri); if (timer) { + struct snd_timer_instance *slave; + bool busy; + timer->num_instances--; - /* wait, until the active callback is finished */ + /* unqueue then drain the slaves' callbacks before remove_slave_links() severs them */ spin_lock_irq(&timer->lock); - while (timeri->flags & SNDRV_TIMER_IFLG_CALLBACK) { + list_for_each_entry(slave, &timeri->slave_list_head, open_list) + list_del_init(&slave->ack_list); + for (;;) { + busy = timeri->flags & SNDRV_TIMER_IFLG_CALLBACK; + list_for_each_entry(slave, &timeri->slave_list_head, open_list) + busy |= slave->flags & SNDRV_TIMER_IFLG_CALLBACK; + if (!busy) + break; spin_unlock_irq(&timer->lock); udelay(10); spin_lock_irq(&timer->lock); @@ -873,12 +885,15 @@ void snd_timer_interrupt(struct snd_timer * timer, unsigned long ticks_left) ack_list_head = &timer->ack_list_head; else ack_list_head = &timer->sack_list_head; - if (list_empty(&ti->ack_list)) + /* don't requeue an instance whose callback is still running */ + if (list_empty(&ti->ack_list) && + !(ti->flags & SNDRV_TIMER_IFLG_CALLBACK)) list_add_tail(&ti->ack_list, ack_list_head); list_for_each_entry(ts, &ti->slave_active_head, active_list) { ts->pticks = ti->pticks; ts->resolution = resolution; - if (list_empty(&ts->ack_list)) + if (list_empty(&ts->ack_list) && + !(ts->flags & SNDRV_TIMER_IFLG_CALLBACK)) list_add_tail(&ts->ack_list, ack_list_head); } } diff --git a/sound/core/ump.c b/sound/core/ump.c index 8d8681a42ca5f..0e6a7d438d75d 100644 --- a/sound/core/ump.c +++ b/sound/core/ump.c @@ -1333,6 +1333,8 @@ static void update_legacy_names(struct snd_ump_endpoint *ump) { struct snd_rawmidi *rmidi = ump->legacy_rmidi; + if (!rmidi) + return; update_legacy_substreams(ump, rmidi, SNDRV_RAWMIDI_STREAM_INPUT); update_legacy_substreams(ump, rmidi, SNDRV_RAWMIDI_STREAM_OUTPUT); } @@ -1341,6 +1343,8 @@ static void ump_legacy_set_rawmidi_name(struct snd_ump_endpoint *ump) { struct snd_rawmidi *rmidi = ump->legacy_rmidi; + if (!rmidi) + return; snprintf(rmidi->name, sizeof(rmidi->name), "%.68s (MIDI 1.0)", ump->core.name); } @@ -1366,6 +1370,7 @@ int snd_ump_attach_legacy_rawmidi(struct snd_ump_endpoint *ump, &rmidi); if (err < 0) { kfree(ump->out_cvts); + ump->out_cvts = NULL; return err; } diff --git a/sound/core/ump_convert.c b/sound/core/ump_convert.c index 0fe13d0316568..85cc67de63306 100644 --- a/sound/core/ump_convert.c +++ b/sound/core/ump_convert.c @@ -258,6 +258,7 @@ static int cvt_legacy_sysex_to_ump(struct ump_cvt_to_ump *cvt, else status = UMP_SYSEX_STATUS_CONTINUE; *data = ump_compose(UMP_MSG_TYPE_DATA, group, status, cvt->len); + data[1] = 0; offset = 8; for (i = 0; i < cvt->len; i++) { *data |= cvt->buf[i] << offset; diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c index 3f8488716a082..d08027681c373 100644 --- a/sound/drivers/aloop.c +++ b/sound/drivers/aloop.c @@ -1805,6 +1805,12 @@ static int loopback_probe(struct platform_device *devptr) int dev = devptr->id; int err; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct loopback), &card); if (err < 0) diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c index 1860ff75fe159..b871fa34bf1bd 100644 --- a/sound/drivers/dummy.c +++ b/sound/drivers/dummy.c @@ -782,7 +782,7 @@ static int snd_dummy_capsrc_put(struct snd_kcontrol *kcontrol, struct snd_ctl_el left = ucontrol->value.integer.value[0] & 1; right = ucontrol->value.integer.value[1] & 1; guard(spinlock_irq)(&dummy->mixer_lock); - change = dummy->capture_source[addr][0] != left && + change = dummy->capture_source[addr][0] != left || dummy->capture_source[addr][1] != right; dummy->capture_source[addr][0] = left; dummy->capture_source[addr][1] = right; @@ -1017,6 +1017,12 @@ static int snd_dummy_probe(struct platform_device *devptr) int idx, err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct snd_dummy), &card); if (err < 0) diff --git a/sound/drivers/mpu401/mpu401.c b/sound/drivers/mpu401/mpu401.c index d3f9424088d4f..e1550e02d264e 100644 --- a/sound/drivers/mpu401/mpu401.c +++ b/sound/drivers/mpu401/mpu401.c @@ -89,6 +89,12 @@ static int snd_mpu401_probe(struct platform_device *devptr) int err; struct snd_card *card; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (port[dev] == SNDRV_AUTO_PORT) { dev_err(&devptr->dev, "specify port\n"); return -EINVAL; diff --git a/sound/drivers/mtpav.c b/sound/drivers/mtpav.c index d31eadf4be5f8..1ed6d4f0cb625 100644 --- a/sound/drivers/mtpav.c +++ b/sound/drivers/mtpav.c @@ -642,9 +642,7 @@ static void snd_mtpav_free(struct snd_card *card) { struct mtpav *crd = card->private_data; - guard(spinlock_irqsave)(&crd->spinlock); - if (crd->istimer > 0) - snd_mtpav_remove_output_timer(crd); + timer_shutdown_sync(&crd->timer); } /* diff --git a/sound/drivers/mts64.c b/sound/drivers/mts64.c index fe50b48c10e77..017e478bd21f3 100644 --- a/sound/drivers/mts64.c +++ b/sound/drivers/mts64.c @@ -900,6 +900,12 @@ static int snd_mts64_probe(struct platform_device *pdev) p = platform_get_drvdata(pdev); platform_set_drvdata(pdev, NULL); + if (dev < 0) { + dev_warn(&pdev->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (dev >= SNDRV_CARDS) return -ENODEV; if (!enable[dev]) diff --git a/sound/drivers/portman2x4.c b/sound/drivers/portman2x4.c index b903a138fc2a4..fc7aa7ceb5a5f 100644 --- a/sound/drivers/portman2x4.c +++ b/sound/drivers/portman2x4.c @@ -697,6 +697,12 @@ static int snd_portman_probe(struct platform_device *pdev) p = platform_get_drvdata(pdev); platform_set_drvdata(pdev, NULL); + if (dev < 0) { + dev_warn(&pdev->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + if (dev >= SNDRV_CARDS) return -ENODEV; if (!enable[dev]) diff --git a/sound/drivers/serial-u16550.c b/sound/drivers/serial-u16550.c index 3c28961091b12..cc9c325df910b 100644 --- a/sound/drivers/serial-u16550.c +++ b/sound/drivers/serial-u16550.c @@ -846,6 +846,12 @@ static int snd_serial_probe(struct platform_device *devptr) int err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + switch (adaptor[dev]) { case SNDRV_SERIAL_SOUNDCANVAS: ins[dev] = 1; diff --git a/sound/drivers/virmidi.c b/sound/drivers/virmidi.c index a204f42d10265..70e16105f1c4a 100644 --- a/sound/drivers/virmidi.c +++ b/sound/drivers/virmidi.c @@ -75,6 +75,12 @@ static int snd_virmidi_probe(struct platform_device *devptr) int idx, err; int dev = devptr->id; + if (dev < 0 || dev >= SNDRV_CARDS) { + dev_warn(&devptr->dev, + "Invalid card index %d, using default 0\n", dev); + dev = 0; + } + err = snd_devm_card_new(&devptr->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct snd_card_virmidi), &card); if (err < 0) diff --git a/sound/hda/codecs/Kconfig b/sound/hda/codecs/Kconfig index addbc94243365..dcf340e5a0c1a 100644 --- a/sound/hda/codecs/Kconfig +++ b/sound/hda/codecs/Kconfig @@ -69,6 +69,7 @@ comment "Set to Y if you want auto-loading the codec driver" config SND_HDA_CODEC_CA0132 tristate "Build Creative CA0132 codec support" + select SND_HDA_GENERIC help Say Y or M here to include Creative CA0132 codec support in snd-hda-intel driver. diff --git a/sound/hda/codecs/ca0132.c b/sound/hda/codecs/ca0132.c index dd054aedd501c..92fc93fb209a9 100644 --- a/sound/hda/codecs/ca0132.c +++ b/sound/hda/codecs/ca0132.c @@ -24,6 +24,7 @@ #include "hda_local.h" #include "hda_auto_parser.h" #include "hda_jack.h" +#include "generic.h" #include "ca0132_regs.h" @@ -1060,6 +1061,8 @@ enum dsp_download_state { */ struct ca0132_spec { + struct hda_gen_spec gen; + const struct snd_kcontrol_new *mixers[5]; unsigned int num_mixers; const struct hda_verb *base_init_verbs; @@ -1174,6 +1177,7 @@ enum { QUIRK_R3D, QUIRK_AE5, QUIRK_AE7, + QUIRK_GENERIC, QUIRK_NONE = HDA_FIXUP_ID_NOT_SET, }; @@ -1292,6 +1296,20 @@ static const struct hda_pintbl ae7_pincfgs[] = { {} }; +static const struct hda_pintbl ca0132_generic_pincfgs[] = { + { 0x0b, 0x41014111 }, + { 0x0c, 0x414520f0 }, /* SPDIF out */ + { 0x0d, 0x01014010 }, /* lineout */ + { 0x0e, 0x41c501f0 }, + { 0x0f, 0x411111f0 }, /* disabled */ + { 0x10, 0x411111f0 }, /* disabled */ + { 0x11, 0x41012014 }, + { 0x12, 0x37a790f0 }, /* mic */ + { 0x13, 0x77a701f0 }, + { 0x18, 0x500000f0 }, + {} +}; + static const struct hda_quirk ca0132_quirks[] = { SND_PCI_QUIRK(0x1028, 0x057b, "Alienware M17x R4", QUIRK_ALIENWARE_M17XR4), SND_PCI_QUIRK(0x1028, 0x0685, "Alienware 15 2015", QUIRK_ALIENWARE), @@ -1304,6 +1322,7 @@ static const struct hda_quirk ca0132_quirks[] = { SND_PCI_QUIRK(0x1458, 0xA016, "Recon3Di", QUIRK_R3DI), SND_PCI_QUIRK(0x1458, 0xA026, "Gigabyte G1.Sniper Z97", QUIRK_R3DI), SND_PCI_QUIRK(0x1458, 0xA036, "Gigabyte GA-Z170X-Gaming 7", QUIRK_R3DI), + SND_PCI_QUIRK(0x1458, 0xA046, "Gigabyte GA-Z170X-Gaming G1", QUIRK_GENERIC), SND_PCI_QUIRK(0x3842, 0x1038, "EVGA X99 Classified", QUIRK_R3DI), SND_PCI_QUIRK(0x3842, 0x104b, "EVGA X299 Dark", QUIRK_R3DI), SND_PCI_QUIRK(0x3842, 0x1055, "EVGA Z390 DARK", QUIRK_R3DI), @@ -1325,6 +1344,7 @@ static const struct hda_model_fixup ca0132_quirk_models[] = { { .id = QUIRK_R3D, .name = "r3d" }, { .id = QUIRK_AE5, .name = "ae5" }, { .id = QUIRK_AE7, .name = "ae7" }, + { .id = QUIRK_GENERIC, .name = "generic" }, {} }; @@ -9882,14 +9902,57 @@ static void sbz_detect_quirk(struct hda_codec *codec) } } +static void ca0132_generic_init_hook(struct hda_codec *codec) +{ + struct ca0132_spec *spec = codec->spec; + + snd_hda_sequence_write(codec, spec->spec_init_verbs); +} + +static int ca0132_generic_probe(struct hda_codec *codec) +{ + struct ca0132_spec *spec = codec->spec; + struct auto_pin_cfg *cfg = &spec->gen.autocfg; + int err; + + snd_hda_gen_spec_init(&spec->gen); + + snd_hda_apply_pincfgs(codec, ca0132_generic_pincfgs); + + ca0132_init_chip(codec); + + err = ca0132_prepare_verbs(codec); + if (err < 0) + return err; + + err = snd_hda_parse_pin_def_config(codec, cfg, NULL); + if (err < 0) + return err; + err = snd_hda_gen_parse_auto_config(codec, cfg); + if (err < 0) + return err; + + spec->gen.init_hook = ca0132_generic_init_hook; + spec->gen.automute_speaker = 0; + spec->gen.automute_lo = 0; + + snd_hda_sequence_write(codec, spec->spec_init_verbs); + return 0; +} + static void ca0132_codec_remove(struct hda_codec *codec) { struct ca0132_spec *spec = codec->spec; - if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO) + switch (ca0132_quirk(spec)) { + case QUIRK_GENERIC: + snd_hda_gen_remove(codec); + return; + case QUIRK_ZXR_DBPRO: return dbpro_free(codec); - else + default: return ca0132_free(codec); + } } static int ca0132_codec_probe(struct hda_codec *codec, @@ -9906,14 +9969,21 @@ static int ca0132_codec_probe(struct hda_codec *codec, codec->spec = spec; spec->codec = codec; - /* Detect codec quirk */ - snd_hda_pick_fixup(codec, ca0132_quirk_models, ca0132_quirks, NULL); - if (ca0132_quirk(spec) == QUIRK_SBZ) - sbz_detect_quirk(codec); - + /* These must be set before any path is taken */ codec->pcm_format_first = 1; codec->no_sticky_stream = 1; + /* Detect codec quirk */ + snd_hda_pick_fixup(codec, ca0132_quirk_models, ca0132_quirks, NULL); + switch (ca0132_quirk(spec)) { + case QUIRK_SBZ: + sbz_detect_quirk(codec); + break; + case QUIRK_GENERIC: + return ca0132_generic_probe(codec); + default: + break; + } spec->dsp_state = DSP_DOWNLOAD_INIT; spec->num_mixers = 1; @@ -10014,36 +10084,51 @@ static int ca0132_codec_build_controls(struct hda_codec *codec) { struct ca0132_spec *spec = codec->spec; - if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO) + switch (ca0132_quirk(spec)) { + case QUIRK_GENERIC: + return snd_hda_gen_build_controls(codec); + case QUIRK_ZXR_DBPRO: return dbpro_build_controls(codec); - else + default: return ca0132_build_controls(codec); + } } static int ca0132_codec_build_pcms(struct hda_codec *codec) { struct ca0132_spec *spec = codec->spec; - if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO) + switch (ca0132_quirk(spec)) { + case QUIRK_GENERIC: + return snd_hda_gen_build_pcms(codec); + case QUIRK_ZXR_DBPRO: return dbpro_build_pcms(codec); - else + default: return ca0132_build_pcms(codec); + } } static int ca0132_codec_init(struct hda_codec *codec) { struct ca0132_spec *spec = codec->spec; - if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO) + switch (ca0132_quirk(spec)) { + case QUIRK_GENERIC: + return snd_hda_gen_init(codec); + case QUIRK_ZXR_DBPRO: return dbpro_init(codec); - else + default: return ca0132_init(codec); + } } static int ca0132_codec_suspend(struct hda_codec *codec) { struct ca0132_spec *spec = codec->spec; + if (ca0132_quirk(spec) == QUIRK_GENERIC) + return 0; + cancel_delayed_work_sync(&spec->unsol_hp_work); return 0; } diff --git a/sound/hda/codecs/conexant.c b/sound/hda/codecs/conexant.c index c3c8326d0b77a..fe7445fabb887 100644 --- a/sound/hda/codecs/conexant.c +++ b/sound/hda/codecs/conexant.c @@ -294,6 +294,7 @@ enum { CXT_FIXUP_HEADSET_MIC, CXT_FIXUP_HP_MIC_NO_PRESENCE, CXT_PINCFG_SWS_JS201D, + CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10, CXT_PINCFG_TOP_SPEAKER, CXT_FIXUP_HP_A_U, CXT_FIXUP_ACER_SWIFT_HP, @@ -831,6 +832,12 @@ static const struct hda_pintbl cxt_pincfg_lemote[] = { {} }; +/* Lenovo IdeaPad Slim 5 16AKP10 with SN6140 */ +static const struct hda_pintbl cxt_pincfg_lenovo_ideapad_slim5_16akp10[] = { + { 0x1a, 0x95a60130 }, /* Internal mic, fixed/always-connected */ + {} +}; + /* SuoWoSi/South-holding JS201D with sn6140 */ static const struct hda_pintbl cxt_pincfg_sws_js201d[] = { { 0x16, 0x03211040 }, /* hp out */ @@ -1011,6 +1018,10 @@ static const struct hda_fixup cxt_fixups[] = { .type = HDA_FIXUP_PINS, .v.pins = cxt_pincfg_sws_js201d, }, + [CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10] = { + .type = HDA_FIXUP_PINS, + .v.pins = cxt_pincfg_lenovo_ideapad_slim5_16akp10, + }, [CXT_PINCFG_TOP_SPEAKER] = { .type = HDA_FIXUP_PINS, .v.pins = (const struct hda_pintbl[]) { @@ -1119,6 +1130,7 @@ static const struct hda_quirk cxt5066_fixups[] = { SND_PCI_QUIRK(0x17aa, 0x21da, "Lenovo X220", CXT_PINCFG_LENOVO_TP410), SND_PCI_QUIRK(0x17aa, 0x21db, "Lenovo X220-tablet", CXT_PINCFG_LENOVO_TP410), SND_PCI_QUIRK(0x17aa, 0x38af, "Lenovo IdeaPad Z560", CXT_FIXUP_MUTE_LED_EAPD), + SND_PCI_QUIRK(0x17aa, 0x38b6, "Lenovo IdeaPad Slim 5 16AKP10", CXT_PINCFG_LENOVO_IDEAPAD_SLIM5_16AKP10), SND_PCI_QUIRK(0x17aa, 0x3905, "Lenovo G50-30", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x17aa, 0x390b, "Lenovo G50-80", CXT_FIXUP_STEREO_DMIC), SND_PCI_QUIRK(0x17aa, 0x3975, "Lenovo U300s", CXT_FIXUP_STEREO_DMIC), diff --git a/sound/hda/codecs/hdmi/hdmi.c b/sound/hda/codecs/hdmi/hdmi.c index 1036d08d1431a..f427d04176e92 100644 --- a/sound/hda/codecs/hdmi/hdmi.c +++ b/sound/hda/codecs/hdmi/hdmi.c @@ -1688,6 +1688,9 @@ int snd_hda_hdmi_generic_pcm_prepare(struct hda_pcm_stream *hinfo, per_pin->channels = substream->runtime->channels; per_pin->setup = true; + if (spec->ops.prepare) + spec->ops.prepare(codec, per_pin); + if (get_wcaps(codec, cvt_nid) & AC_WCAP_STRIPE) { stripe = snd_hdac_get_stream_stripe_ctl(&codec->bus->core, substream); diff --git a/sound/hda/codecs/hdmi/hdmi_local.h b/sound/hda/codecs/hdmi/hdmi_local.h index 548241ad3fa99..dce9a335cff28 100644 --- a/sound/hda/codecs/hdmi/hdmi_local.h +++ b/sound/hda/codecs/hdmi/hdmi_local.h @@ -74,6 +74,15 @@ struct hdmi_ops { hda_nid_t pin_nid, int dev_id, u32 stream_tag, int format); + /* + * Optional hook invoked at the beginning of the PCM prepare + * sequence, before the audio infoframe and stream format are + * (re)programmed. Used to disable keep-alive / silent stream so + * that the format change is not done while keep-alive is active. + */ + void (*prepare)(struct hda_codec *codec, + struct hdmi_spec_per_pin *per_pin); + void (*pin_cvt_fixup)(struct hda_codec *codec, struct hdmi_spec_per_pin *per_pin, hda_nid_t cvt_nid); diff --git a/sound/hda/codecs/hdmi/intelhdmi.c b/sound/hda/codecs/hdmi/intelhdmi.c index 23237d5274302..3a942343bc989 100644 --- a/sound/hda/codecs/hdmi/intelhdmi.c +++ b/sound/hda/codecs/hdmi/intelhdmi.c @@ -418,6 +418,28 @@ static void intel_not_share_assigned_cvt_nid(struct hda_codec *codec, intel_not_share_assigned_cvt(codec, pin_nid, dev_id, mux_idx); } +/* + * prepare ops override for HSW+ + * + * Disable keep-alive before the converter format and audio infoframe are + * reprogrammed by the PCM prepare sequence. Changing the audio format (e.g. + * the channel count when switching to multichannel PCM) while a keep-alive + * stream is active is not safe, so release keep-alive here, early in the + * sequence. It is re-enabled once the new stream has been set up, in + * i915_hsw_setup_stream(). + */ +static void i915_hsw_prepare(struct hda_codec *codec, + struct hdmi_spec_per_pin *per_pin) +{ + struct hdmi_spec *spec = codec->spec; + + if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin->silent_stream) { + silent_stream_set_kae(codec, per_pin, false); + /* wait for pending transfers in codec to clear */ + usleep_range(100, 200); + } +} + /* setup_stream ops override for HSW+ */ static int i915_hsw_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid, hda_nid_t pin_nid, int dev_id, u32 stream_tag, @@ -435,15 +457,16 @@ static int i915_hsw_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid, haswell_verify_D0(codec, cvt_nid, pin_nid); - if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin && per_pin->silent_stream) { - silent_stream_set_kae(codec, per_pin, false); - /* wait for pending transfers in codec to clear */ - usleep_range(100, 200); - } - res = snd_hda_hdmi_setup_stream(codec, cvt_nid, pin_nid, dev_id, stream_tag, format); + /* + * Keep-alive was disabled in i915_hsw_prepare(), re-enable it now. + * The pin lookup above resolves to the same per_pin that prepare + * used (pin_nid comes from that per_pin), so this stays balanced; a + * NULL per_pin only occurs on a lookup failure that also implies no + * active keep-alive stream to restore. + */ if (spec->silent_stream_type == SILENT_STREAM_KAE && per_pin && per_pin->silent_stream) { usleep_range(100, 200); silent_stream_set_kae(codec, per_pin, true); @@ -607,6 +630,7 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid, codec->depop_delay = 0; codec->auto_runtime_pm = 1; + spec->ops.prepare = i915_hsw_prepare; spec->ops.setup_stream = i915_hsw_setup_stream; spec->ops.pin_cvt_fixup = i915_pin_cvt_fixup; spec->ops.silent_stream = i915_set_silent_stream; diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 01483eb545cda..aab2456bacaf6 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -3742,6 +3742,7 @@ enum { ALC269_FIXUP_DMIC_THINKPAD_ACPI, ALC269VB_FIXUP_INFINIX_ZERO_BOOK_13, ALC269VC_FIXUP_INFINIX_Y4_MAX, + ALC269VC_FIXUP_LUNNEN_GROUND_14, ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO, ALC255_FIXUP_ACER_MIC_NO_PRESENCE, ALC255_FIXUP_ASUS_MIC_NO_PRESENCE, @@ -3781,6 +3782,7 @@ enum { ALC275_FIXUP_DELL_XPS, ALC293_FIXUP_LENOVO_SPK_NOISE, ALC233_FIXUP_LENOVO_LINE2_MIC_HOTKEY, + ALC233_FIXUP_WUJIE_SPEAKERS, ALC233_FIXUP_LENOVO_L2MH_LOW_ENLED, ALC255_FIXUP_DELL_SPK_NOISE, ALC225_FIXUP_DISABLE_MIC_VREF, @@ -3821,6 +3823,7 @@ enum { ALC256_FIXUP_HUAWEI_MACH_WX9_PINS, ALC298_FIXUP_HUAWEI_MBX_STEREO, ALC295_FIXUP_HP_X360, + ALC295_FIXUP_HP_PAVILION_X360, ALC221_FIXUP_HP_HEADSET_MIC, ALC285_FIXUP_LENOVO_HEADPHONE_NOISE, ALC295_FIXUP_HP_AUTO_MUTE, @@ -4031,6 +4034,13 @@ static void alc287_fixup_lenovo_yoga_book_9i(struct hda_codec *codec, } static const struct hda_fixup alc269_fixups[] = { + [ALC233_FIXUP_WUJIE_SPEAKERS] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x1b, 0x90170150 }, /* internal speaker */ + { } + }, + }, [ALC269_FIXUP_GPIO2] = { .type = HDA_FIXUP_FUNC, .v.func = alc_fixup_gpio2, @@ -4183,6 +4193,13 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_LIMIT_INT_MIC_BOOST }, + [ALC269VC_FIXUP_LUNNEN_GROUND_14] = { + .type = HDA_FIXUP_PINS, + .v.pins = (const struct hda_pintbl[]) { + { 0x1b, 0x90170150 }, /* internal speaker */ + { } + }, + }, [ALC269VB_FIXUP_CHUWI_COREBOOK_XPRO] = { .type = HDA_FIXUP_PINS, .v.pins = (const struct hda_pintbl[]) { @@ -5119,6 +5136,19 @@ static const struct hda_fixup alc269_fixups[] = { .chained = true, .chain_id = ALC269_FIXUP_HP_MUTE_LED_MIC3 }, + [ALC295_FIXUP_HP_PAVILION_X360] = { + .type = HDA_FIXUP_VERBS, + .v.verbs = (const struct hda_verb[]) { + /* force amp gain and processing state */ + { 0x20, AC_VERB_SET_COEF_INDEX, 0x07 }, + { 0x20, AC_VERB_SET_PROC_COEF, 0x7770 }, + { 0x20, AC_VERB_SET_COEF_INDEX, 0x0d }, + { 0x20, AC_VERB_SET_PROC_COEF, 0x3000 }, + {} + }, + .chained = true, + .chain_id = ALC295_FIXUP_HP_X360 + }, [ALC221_FIXUP_HP_HEADSET_MIC] = { .type = HDA_FIXUP_PINS, .v.pins = (const struct hda_pintbl[]) { @@ -6713,6 +6743,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x82c0, "HP G3 mini premium", ALC221_FIXUP_HP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x103c, 0x83b9, "HP Spectre x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x841c, "HP Pavilion 15-CK0xx", ALC269_FIXUP_HP_MUTE_LED_MIC3), + HDA_CODEC_QUIRK(0x103c, 0x8486, "HP Pavilion x360", ALC295_FIXUP_HP_PAVILION_X360), SND_PCI_QUIRK(0x103c, 0x8497, "HP Envy x360", ALC269_FIXUP_HP_MUTE_LED_MIC3), SND_PCI_QUIRK(0x103c, 0x84a6, "HP 250 G7 Notebook PC", ALC269_FIXUP_HP_LINE1_MIC1_LED), SND_PCI_QUIRK(0x103c, 0x84ae, "HP 15-db0403ng", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), @@ -6731,6 +6762,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x869d, "HP", ALC236_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x86c1, "HP Laptop 15-da3001TU", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x86c7, "HP Envy AiO 32", ALC274_FIXUP_HP_ENVY_GPIO), + SND_PCI_QUIRK(0x103c, 0x86c8, "HP Laptop 14s-dr1xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x86e7, "HP Spectre x360 15-eb0xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), SND_PCI_QUIRK(0x103c, 0x863e, "HP Spectre x360 15-df1xxx", ALC285_FIXUP_HP_SPECTRE_X360_DF1), SND_PCI_QUIRK(0x103c, 0x86e8, "HP Spectre x360 15-eb0xxx", ALC285_FIXUP_HP_SPECTRE_X360_EB1), @@ -6799,6 +6831,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x88d1, "HP Pavilion 15-eh1xxx (mainboard 88D1)", ALC245_FIXUP_HP_MUTE_LED_V1_COEFBIT), SND_PCI_QUIRK(0x103c, 0x88dd, "HP Pavilion 15z-ec200", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x88eb, "HP Victus 16-e0xxx", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x88ed, "HP Victus 16-e0xxx (MB 88ED)", + ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x88ee, "HP Victus 16-e0xxx (MB 88EE)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8902, "HP OMEN 16", ALC285_FIXUP_HP_MUTE_LED), SND_PCI_QUIRK(0x103c, 0x890e, "HP 255 G8 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8919, "HP Pavilion Aero Laptop 13-be0xxx", ALC287_FIXUP_HP_GPIO_LED), @@ -6831,7 +6866,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x89d3, "HP EliteBook 645 G9 (MB 89D2)", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x89da, "HP Spectre x360 14t-ea100", ALC245_FIXUP_HP_SPECTRE_X360_EU0XXX), SND_PCI_QUIRK(0x103c, 0x89e7, "HP Elite x2 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8a05, "HP Dragonfly Folio G3 2-in-1", ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8a06, "HP Dragonfly Folio G3 2-in-1", ALC245_FIXUP_CS35L41_SPI_4_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8a0f, "HP Pavilion 14-ec1xxx", ALC287_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8a1b, "HP 255 15.6 inch G9 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a1f, "HP Laptop 14s-dr5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a20, "HP Laptop 15s-fq5xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2), SND_PCI_QUIRK(0x103c, 0x8a25, "HP Victus 16-d1xxx (MB 8A25)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), @@ -6848,6 +6886,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), + SND_PCI_QUIRK(0x103c, 0x8a50, "HP Victus 15-fa0xxx (MB 8A50)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT), SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4), SND_PCI_QUIRK(0x103c, 0x8a74, "HP ProBook 440 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8a75, "HP ProBook 450 G8 Notebook PC", ALC236_FIXUP_HP_GPIO_LED), @@ -6856,6 +6895,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8aa3, "HP ProBook 450 G9 (MB 8AA1)", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8aa8, "HP EliteBook 640 G9 (MB 8AA6)", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8aab, "HP EliteBook 650 G9 (MB 8AA9)", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8ab8, "HP EliteBook 830 G8 Notebook PC (MB 8AB8)", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8ab9, "HP EliteBook 840 G8 (MB 8AB8)", ALC285_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8abb, "HP ZBook Firefly 14 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8ad1, "HP EliteBook 840 14 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED), @@ -7004,6 +7044,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x103c, 0x8dfc, "HP EliteBook 645 G12", ALC236_FIXUP_HP_GPIO_LED), SND_PCI_QUIRK(0x103c, 0x8dfd, "HP EliteBook 6 G1a 16", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8dfe, "HP EliteBook 665 G12", ALC236_FIXUP_HP_GPIO_LED), + SND_PCI_QUIRK(0x103c, 0x8e0d, "HP EliteBook 6 G1a 14 (AD3Q9ET#UUG)", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF), SND_PCI_QUIRK(0x103c, 0x8e11, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e12, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x103c, 0x8e13, "HP Trekker", ALC287_FIXUP_CS35L41_I2C_2), @@ -7107,6 +7148,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1043, 0x18f1, "Asus FX505DT", ALC256_FIXUP_ASUS_HEADSET_MIC), SND_PCI_QUIRK(0x1043, 0x194e, "ASUS UX563FD", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x1970, "ASUS UX550VE", ALC289_FIXUP_ASUS_GA401), + SND_PCI_QUIRK(0x1043, 0x197e, "ASUS VivoBook X509DAP", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), SND_PCI_QUIRK(0x1043, 0x1982, "ASUS B1400CEPE", ALC256_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x19ce, "ASUS B9450FA", ALC294_FIXUP_ASUS_HPE), SND_PCI_QUIRK(0x1043, 0x19e1, "ASUS UX581LV", ALC295_FIXUP_ASUS_MIC_NO_PRESENCE), @@ -7232,6 +7274,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x144d, 0xc1a6, "Samsung Galaxy Book Pro 360 (NP930QBD)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc740, "Samsung Ativ book 8 (NP870Z5G)", ALC269_FIXUP_ATIV_BOOK_8), SND_PCI_QUIRK(0x144d, 0xc812, "Samsung Notebook Pen S (NT950SBE-X58)", ALC298_FIXUP_SAMSUNG_AMP), + HDA_CODEC_QUIRK(0x144d, 0xc824, "Samsung ELECTRONICS (750XBE/730XBE)", ALC298_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc830, "Samsung Galaxy Book Ion (NT950XCJ-X716A)", ALC298_FIXUP_SAMSUNG_AMP), SND_PCI_QUIRK(0x144d, 0xc832, "Samsung Galaxy Book Flex Alpha (NP730QCJ)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), SND_PCI_QUIRK(0x144d, 0xc876, "Samsung 730QED (NP730QED-KA2US)", ALC256_FIXUP_SAMSUNG_HEADPHONE_VERY_QUIET), @@ -7426,18 +7469,24 @@ static const struct hda_quirk alc269_fixup_tbl[] = { HDA_CODEC_QUIRK(0x17aa, 0x38cf, "Lenovo Yoga Pro 7 14IMH9", ALC287_FIXUP_YOGA9_14IMH9_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3847, "Legion 7 16ACHG6", ALC287_FIXUP_LEGION_16ACHG6), SND_PCI_QUIRK(0x17aa, 0x384a, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), + /* Yoga Pro 7 14IRH8 shares PCI SSID 17aa:3852 with Yoga 7 14ITL5; + * use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x38b1, "Lenovo Yoga Pro 7 14IRH8", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3852, "Lenovo Yoga 7 14ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3853, "Lenovo Yoga 7 15ITL5", ALC287_FIXUP_YOGA7_14ITL_SPEAKERS), SND_PCI_QUIRK(0x17aa, 0x3855, "Legion 7 16ITHG6", ALC287_FIXUP_LEGION_16ITHG6), SND_PCI_QUIRK(0x17aa, 0x3865, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3866, "Lenovo 13X", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3869, "Lenovo Yoga7 14IAL7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + HDA_CODEC_QUIRK(0x17aa, 0x386a, "Lenovo Yoga 7 16IAP7", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x386e, "Legion Y9000X 2022 IAH7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x386e, "Yoga Pro 7 14ARP8", ALC285_FIXUP_SPEAKER2_TO_DAC1), HDA_CODEC_QUIRK(0x17aa, 0x38a8, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ HDA_CODEC_QUIRK(0x17aa, 0x38a7, "Legion Pro 7 16ARX8H", ALC287_FIXUP_TAS2781_I2C), /* this must match before PCI SSID 17aa:386f below */ SND_PCI_QUIRK(0x17aa, 0x386f, "Legion Pro 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3870, "Lenovo Yoga 7 14ARB7", ALC287_FIXUP_YOGA7_14ARB7_I2C), + SND_PCI_QUIRK(0x17aa, 0x3874, "Legion 7i 16IAX7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3877, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x3878, "Lenovo Legion 7 Slim 16ARHA7", ALC287_FIXUP_CS35L41_I2C_2), SND_PCI_QUIRK(0x17aa, 0x387d, "Yoga S780-16 pro Quad AAC", ALC287_FIXUP_TAS2781_I2C), @@ -7454,6 +7503,10 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x38a8, "Y780P AMD VECO dual", ALC287_FIXUP_TAS2781_I2C), SND_PCI_QUIRK(0x17aa, 0x38a9, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), SND_PCI_QUIRK(0x17aa, 0x38ab, "Thinkbook 16P", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), + /* Lenovo Yoga Slim 7 14AKP10 shares PCI SSID 17aa:38b4 with Legion Slim 7 + * 16IRH8; use codec SSID to distinguish them + */ + HDA_CODEC_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x38b4, "Legion Slim 7 16IRH8", ALC287_FIXUP_CS35L41_I2C_2), HDA_CODEC_QUIRK(0x17aa, 0x391c, "Lenovo Yoga 7 2-in-1 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), HDA_CODEC_QUIRK(0x17aa, 0x391d, "Lenovo Yoga 7 2-in-1 16AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), @@ -7487,8 +7540,8 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x17aa, 0x3902, "Lenovo E50-80", ALC269_FIXUP_DMIC_THINKPAD_ACPI), SND_PCI_QUIRK(0x17aa, 0x390d, "Lenovo Yoga Pro 7 14ASP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3911, "Lenovo Yoga Pro 7 14IAH10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), + SND_PCI_QUIRK(0x17aa, 0x3912, "Lenovo Xiaoxin 14 GT", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x3913, "Lenovo 145", ALC236_FIXUP_LENOVO_INV_DMIC), - SND_PCI_QUIRK(0x17aa, 0x391a, "Lenovo Yoga Slim 7 14AKP10", ALC287_FIXUP_YOGA9_14IAP7_BASS_SPK_PIN), SND_PCI_QUIRK(0x17aa, 0x391f, "Yoga S990-16 pro Quad YC Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3920, "Yoga S990-16 pro Quad VECO Quad", ALC287_FIXUP_TXNW2781_I2C), SND_PCI_QUIRK(0x17aa, 0x3929, "Thinkbook 13x Gen 5", ALC287_FIXUP_MG_RTKC_CSAMP_CS35L41_I2C_THINKPAD), @@ -7551,7 +7604,9 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1d05, 0x300f, "TongFang X6AR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3019, "TongFang X6FR5xxY", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d05, 0x3031, "TongFang X6AR55xU", ALC2XX_FIXUP_HEADSET_MIC), - SND_PCI_QUIRK(0x1d05, 0x3034, "TongFang X6xx45xU", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x3033, "TongFang X6SP45xU", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x3034, "TongFang X6KK45xU", ALC2XX_FIXUP_HEADSET_MIC), + SND_PCI_QUIRK(0x1d05, 0x30ba, "TongFang XxAF5xxx", ALC2XX_FIXUP_HEADSET_MIC), SND_PCI_QUIRK(0x1d17, 0x3288, "Haier Boyue G42", ALC269VC_FIXUP_ACER_VCOPPERBOX_PINS), SND_PCI_QUIRK(0x1d72, 0x1602, "RedmiBook", ALC255_FIXUP_XIAOMI_HEADSET_MIC), SND_PCI_QUIRK(0x1d72, 0x1701, "XiaomiNotebook Pro", ALC298_FIXUP_DELL1_MIC_NO_PRESENCE), @@ -7573,11 +7628,13 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x2782, 0x1705, "MEDION E15433", ALC269VC_FIXUP_INFINIX_Y4_MAX), SND_PCI_QUIRK(0x2782, 0x1707, "Vaio VJFE-ADL", ALC298_FIXUP_SPK_VOLUME), SND_PCI_QUIRK(0x2782, 0x4900, "MEDION E15443", ALC233_FIXUP_MEDION_MTL_SPK), + SND_PCI_QUIRK(0x2782, 0xa212, "Lunnen Ground 14", ALC269VC_FIXUP_LUNNEN_GROUND_14), SND_PCI_QUIRK(0x7017, 0x2014, "Star Labs StarFighter", ALC233_FIXUP_STARLABS_STARFIGHTER), SND_PCI_QUIRK(0x8086, 0x2074, "Intel NUC 8", ALC233_FIXUP_INTEL_NUC8_DMIC), SND_PCI_QUIRK(0x8086, 0x2080, "Intel NUC 8 Rugged", ALC256_FIXUP_INTEL_NUC8_RUGGED), SND_PCI_QUIRK(0x8086, 0x2081, "Intel NUC 10", ALC256_FIXUP_INTEL_NUC10), SND_PCI_QUIRK(0x8086, 0x3038, "Intel NUC 13", ALC295_FIXUP_CHROME_BOOK), + SND_PCI_QUIRK(0xc011, 0x1d05, "MECHREVO WUJIE Series", ALC233_FIXUP_WUJIE_SPEAKERS), SND_PCI_QUIRK(0xf111, 0x0001, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x0006, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), SND_PCI_QUIRK(0xf111, 0x0009, "Framework Laptop", ALC295_FIXUP_FRAMEWORK_LAPTOP_MIC_NO_PRESENCE), diff --git a/sound/hda/codecs/realtek/alc662.c b/sound/hda/codecs/realtek/alc662.c index 5073165d1f3cf..5678063ac0571 100644 --- a/sound/hda/codecs/realtek/alc662.c +++ b/sound/hda/codecs/realtek/alc662.c @@ -835,6 +835,7 @@ static const struct hda_quirk alc662_fixup_tbl[] = { SND_PCI_QUIRK(0x1b35, 0x1234, "CZC ET26", ALC662_FIXUP_CZC_ET26), SND_PCI_QUIRK(0x1b35, 0x2206, "CZC P10T", ALC662_FIXUP_CZC_P10T), SND_PCI_QUIRK(0x1c6c, 0x1239, "Compaq N14JP6-V2", ALC897_FIXUP_HP_HSMIC_VERB), + SND_PCI_QUIRK(0x1e63, 0x6d9a, "F+ FLAPTOP r", ALC897_FIXUP_HP_HSMIC_VERB), #if 0 /* Below is a quirk table taken from the old code. diff --git a/sound/hda/codecs/realtek/alc882.c b/sound/hda/codecs/realtek/alc882.c index 529fecd5baa0a..fd466b6985f05 100644 --- a/sound/hda/codecs/realtek/alc882.c +++ b/sound/hda/codecs/realtek/alc882.c @@ -61,6 +61,7 @@ enum { ALC887_FIXUP_ASUS_HMIC, ALCS1200A_FIXUP_MIC_VREF, ALC888VD_FIXUP_MIC_100VREF, + ALC898_FIXUP_CLEVO_P775TM1, }; static void alc889_fixup_coef(struct hda_codec *codec, @@ -236,6 +237,19 @@ static void alc1220_fixup_clevo_pb51ed(struct hda_codec *codec, alc_fixup_headset_mode_no_hp_mic(codec, fix, action); } +/* On Clevo P775TM1, VREF of pin 0x1b enables the external headphone amp */ +static void alc898_fixup_clevo_p775tm1(struct hda_codec *codec, + const struct hda_fixup *fix, int action) +{ + struct alc_spec *spec = codec->spec; + + if (action != HDA_FIXUP_ACT_PRE_PROBE) + return; + + snd_hda_set_pin_ctl_cache(codec, 0x1b, PIN_VREF80); + spec->gen.keep_vref_in_automute = 1; +} + static void alc887_asus_hp_automute_hook(struct hda_codec *codec, struct hda_jack_callback *jack) { @@ -560,6 +574,12 @@ static const struct hda_fixup alc882_fixups[] = { {} } }, + [ALC898_FIXUP_CLEVO_P775TM1] = { + .type = HDA_FIXUP_FUNC, + .v.func = alc898_fixup_clevo_p775tm1, + .chained = true, + .chain_id = ALC882_FIXUP_EAPD, + }, }; static const struct hda_quirk alc882_fixup_tbl[] = { @@ -664,6 +684,7 @@ static const struct hda_quirk alc882_fixup_tbl[] = { SND_PCI_QUIRK(0x1558, 0x67f1, "Clevo PC70H[PRS]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), SND_PCI_QUIRK(0x1558, 0x67f5, "Clevo PD70PN[NRT]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), SND_PCI_QUIRK(0x1558, 0x70d1, "Clevo PC70[ER][CDF]", ALC1220_FIXUP_CLEVO_PB51ED_PINS), + SND_PCI_QUIRK(0x1558, 0x7709, "Clevo P775TM1", ALC898_FIXUP_CLEVO_P775TM1), SND_PCI_QUIRK(0x1558, 0x7714, "Clevo X170SM", ALC1220_FIXUP_CLEVO_PB51ED_PINS), SND_PCI_QUIRK(0x1558, 0x7715, "Clevo X170KM-G", ALC1220_FIXUP_CLEVO_PB51ED), SND_PCI_QUIRK(0x1558, 0x9501, "Clevo P950HR", ALC1220_FIXUP_CLEVO_P950), @@ -719,6 +740,7 @@ static const struct hda_model_fixup alc882_fixup_models[] = { {.id = ALC1220_FIXUP_GB_DUAL_CODECS, .name = "dual-codecs"}, {.id = ALC1220_FIXUP_GB_X570, .name = "gb-x570"}, {.id = ALC1220_FIXUP_CLEVO_P950, .name = "clevo-p950"}, + {.id = ALC898_FIXUP_CLEVO_P775TM1, .name = "clevo-p775tm1"}, {} }; diff --git a/sound/hda/codecs/side-codecs/Kconfig b/sound/hda/codecs/side-codecs/Kconfig index cbf1847896bc9..96c7e39b0715c 100644 --- a/sound/hda/codecs/side-codecs/Kconfig +++ b/sound/hda/codecs/side-codecs/Kconfig @@ -27,6 +27,7 @@ config SND_HDA_SCODEC_CS35L41_I2C depends on ACPI depends on EFI depends on SND_SOC + imply SERIAL_MULTI_INSTANTIATE select SND_SOC_CS35L41_LIB select SND_HDA_SCODEC_CS35L41 select SND_SOC_CS_AMP_LIB @@ -43,6 +44,7 @@ config SND_HDA_SCODEC_CS35L41_SPI depends on ACPI depends on EFI depends on SND_SOC + imply SERIAL_MULTI_INSTANTIATE select SND_SOC_CS35L41_LIB select SND_HDA_SCODEC_CS35L41 select SND_SOC_CS_AMP_LIB diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c index 438a19bd979bf..de2fd6471f059 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c @@ -1437,10 +1437,19 @@ static int cs35l41_get_acpi_mute_state(struct cs35l41_hda *cs35l41, acpi_handle guid_parse(CS35L41_UUID, &guid); if (cs35l41_dsm_supported(handle, CS35L41_DSM_GET_MUTE)) { - ret = acpi_evaluate_dsm(handle, &guid, 0, CS35L41_DSM_GET_MUTE, NULL); + ret = acpi_evaluate_dsm_typed(handle, &guid, 0, + CS35L41_DSM_GET_MUTE, NULL, + ACPI_TYPE_BUFFER); + if (!ret) return -EINVAL; + if (!ret->buffer.length || !ret->buffer.pointer) { + ACPI_FREE(ret); + return -EINVAL; + } + mute = *ret->buffer.pointer; + ACPI_FREE(ret); dev_dbg(cs35l41->dev, "CS35L41_DSM_GET_MUTE: %d\n", mute); } diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c index 16d5ea77192f0..dd8875b111ade 100644 --- a/sound/hda/codecs/side-codecs/cs35l41_hda_property.c +++ b/sound/hda/codecs/side-codecs/cs35l41_hda_property.c @@ -123,6 +123,7 @@ static const struct cs35l41_config cs35l41_config_table[] = { { "17AA3866", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA386E", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, 2, -1, 0, 0, 0 }, { "17AA386F", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, + { "17AA3874", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA3877", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA3878", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, -1, -1, 0, 0, 0 }, { "17AA38A9", 2, EXTERNAL, { CS35L41_LEFT, CS35L41_RIGHT, 0, 0 }, 0, 2, -1, 0, 0, 0 }, @@ -548,6 +549,7 @@ static const struct cs35l41_prop_model cs35l41_prop_model_table[] = { { "CSC3551", "17AA3866", generic_dsd_config }, { "CSC3551", "17AA386E", generic_dsd_config }, { "CSC3551", "17AA386F", generic_dsd_config }, + { "CSC3551", "17AA3874", generic_dsd_config }, { "CSC3551", "17AA3877", generic_dsd_config }, { "CSC3551", "17AA3878", generic_dsd_config }, { "CSC3551", "17AA38A9", generic_dsd_config }, diff --git a/sound/hda/codecs/side-codecs/cs35l56_hda.c b/sound/hda/codecs/side-codecs/cs35l56_hda.c index 1d25fe01066ee..baf286bf7ec83 100644 --- a/sound/hda/codecs/side-codecs/cs35l56_hda.c +++ b/sound/hda/codecs/side-codecs/cs35l56_hda.c @@ -516,20 +516,6 @@ static void cs35l56_hda_request_firmware_files(struct cs35l56_hda *cs35l56, NULL, "bin"); return; } - - /* - * Check for system-specific bin files without wmfw before - * falling back to generic firmware - */ - if (amp_name) - cs35l56_hda_request_firmware_file(cs35l56, coeff_firmware, coeff_filename, - base_name, system_name, amp_name, "bin"); - if (!*coeff_firmware) - cs35l56_hda_request_firmware_file(cs35l56, coeff_firmware, coeff_filename, - base_name, system_name, NULL, "bin"); - - if (*coeff_firmware) - return; } ret = cs35l56_hda_request_firmware_file(cs35l56, wmfw_firmware, wmfw_filename, @@ -615,13 +601,15 @@ static void cs35l56_hda_fw_load(struct cs35l56_hda *cs35l56) &wmfw_firmware, &wmfw_filename, &coeff_firmware, &coeff_filename); - /* - * If the BIOS didn't patch the firmware a bin file is mandatory to - * enable the ASP· - */ - if (!coeff_firmware && firmware_missing) { - dev_err(cs35l56->base.dev, ".bin file required but not found\n"); - goto err_fw_release; + /* If the BIOS didn't patch the firmware a wmfw and bin file are mandatory */ + if (firmware_missing) { + if (!wmfw_firmware) { + dev_err(cs35l56->base.dev, ".%s file required but not found\n", "wmfw"); + goto err_fw_release; + } else if (!coeff_firmware) { + dev_err(cs35l56->base.dev, ".%s file required but not found\n", "bin"); + goto err_fw_release; + } } mutex_lock(&cs35l56->base.irq_lock); diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c index ab2a2472d7bdc..314bad6029b00 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c @@ -347,6 +347,8 @@ static int tas2781_read_acpi(struct tas2781_hda *tas_hda, strscpy(p->dev_name, hid, sizeof(p->dev_name)); physdev = get_device(acpi_get_first_physical_node(adev)); acpi_dev_put(adev); + if (!physdev) + return -ENODEV; property = "ti,dev-index"; ret = device_property_count_u32(physdev, property); @@ -389,7 +391,6 @@ static int tas2781_read_acpi(struct tas2781_hda *tas_hda, err: dev_err(p->dev, "read acpi error, ret: %d\n", ret); put_device(physdev); - acpi_dev_put(adev); return ret; } diff --git a/sound/hda/common/codec.c b/sound/hda/common/codec.c index c6d44168c7f9d..164906e2f00f4 100644 --- a/sound/hda/common/codec.c +++ b/sound/hda/common/codec.c @@ -2277,6 +2277,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, int idx = kcontrol->private_value; struct hda_spdif_out *spdif; hda_nid_t nid; + unsigned int old_status; unsigned short val; int change; @@ -2285,6 +2286,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, guard(mutex)(&codec->spdif_mutex); spdif = snd_array_elem(&codec->spdif_out, idx); nid = spdif->nid; + old_status = spdif->status; spdif->status = ucontrol->value.iec958.status[0] | ((unsigned int)ucontrol->value.iec958.status[1] << 8) | ((unsigned int)ucontrol->value.iec958.status[2] << 16) | @@ -2295,7 +2297,7 @@ static int snd_hda_spdif_default_put(struct snd_kcontrol *kcontrol, spdif->ctls = val; if (change && nid != (u16)-1) set_dig_out_convert(codec, nid, val & 0xff, (val >> 8) & 0xff); - return change; + return change || spdif->status != old_status; } #define snd_hda_spdif_out_switch_info snd_ctl_boolean_mono_info diff --git a/sound/hda/common/proc.c b/sound/hda/common/proc.c index 5f3f61519ba64..2d26b4305bcc9 100644 --- a/sound/hda/common/proc.c +++ b/sound/hda/common/proc.c @@ -624,7 +624,7 @@ static void print_conn_list(struct snd_info_buffer *buffer, /* Get Cache connections info */ cache_len = snd_hda_get_conn_list(codec, nid, &list); if (cache_len >= 0 && (cache_len != conn_len || - memcmp(list, conn, conn_len) != 0)) { + memcmp(list, conn, conn_len * sizeof(*conn)) != 0)) { snd_iprintf(buffer, " In-driver Connection: %d\n", cache_len); if (cache_len > 0) { snd_iprintf(buffer, " "); diff --git a/sound/hda/core/device.c b/sound/hda/core/device.c index 160c8d0453b0e..ebaa106d0f018 100644 --- a/sound/hda/core/device.c +++ b/sound/hda/core/device.c @@ -404,6 +404,7 @@ static void setup_fg_nodes(struct hdac_device *codec) */ int snd_hdac_refresh_widgets(struct hdac_device *codec) { + hda_nid_t fg = codec->afg ? codec->afg : codec->mfg; hda_nid_t start_nid; int nums, err = 0; @@ -412,10 +413,10 @@ int snd_hdac_refresh_widgets(struct hdac_device *codec) * widgets array. */ guard(mutex)(&codec->widget_lock); - nums = snd_hdac_get_sub_nodes(codec, codec->afg, &start_nid); + nums = snd_hdac_get_sub_nodes(codec, fg, &start_nid); if (!start_nid || nums <= 0 || nums >= 0xff) { dev_err(&codec->dev, "cannot read sub nodes for FG 0x%02x\n", - codec->afg); + fg); return -EINVAL; } diff --git a/sound/hda/core/ext/stream.c b/sound/hda/core/ext/stream.c index b4759198e51db..f6d80f95333bb 100644 --- a/sound/hda/core/ext/stream.c +++ b/sound/hda/core/ext/stream.c @@ -211,8 +211,8 @@ void snd_hdac_ext_stream_reset(struct hdac_ext_stream *hext_stream) break; udelay(3); } while (--timeout); - val &= ~AZX_PPLCCTL_STRST; - writel(val, hext_stream->pplc_addr + AZX_REG_PPLCCTL); + snd_hdac_updatel(hext_stream->pplc_addr, AZX_REG_PPLCCTL, + AZX_PPLCCTL_STRST, 0); udelay(3); timeout = 50; diff --git a/sound/isa/es18xx.c b/sound/isa/es18xx.c index 1da7b400a17b6..2f454f7797ce2 100644 --- a/sound/isa/es18xx.c +++ b/sound/isa/es18xx.c @@ -1762,6 +1762,8 @@ static int snd_es18xx_mixer(struct snd_card *card) for (idx = 0; idx < ARRAY_SIZE(snd_es18xx_base_controls); idx++) { struct snd_kcontrol *kctl; kctl = snd_ctl_new1(&snd_es18xx_base_controls[idx], chip); + if (!kctl) + return -ENOMEM; if (chip->caps & ES18XX_HWV) { switch (idx) { case 0: @@ -1823,6 +1825,8 @@ static int snd_es18xx_mixer(struct snd_card *card) for (idx = 0; idx < ARRAY_SIZE(snd_es18xx_hw_volume_controls); idx++) { struct snd_kcontrol *kctl; kctl = snd_ctl_new1(&snd_es18xx_hw_volume_controls[idx], chip); + if (!kctl) + return -ENOMEM; if (idx == 0) chip->hw_volume = kctl; else diff --git a/sound/parisc/harmony.c b/sound/parisc/harmony.c index 4b5a54da25fb1..88e9b8dc257e3 100644 --- a/sound/parisc/harmony.c +++ b/sound/parisc/harmony.c @@ -868,6 +868,9 @@ snd_harmony_create(struct snd_card *card, goto free_and_ret; } + spin_lock_init(&h->mixer_lock); + spin_lock_init(&h->lock); + err = request_irq(padev->irq, snd_harmony_interrupt, 0, "harmony", h); if (err) { @@ -877,9 +880,6 @@ snd_harmony_create(struct snd_card *card, } h->irq = padev->irq; - spin_lock_init(&h->mixer_lock); - spin_lock_init(&h->lock); - err = snd_device_new(card, SNDRV_DEV_LOWLEVEL, h, &ops); if (err < 0) goto free_and_ret; diff --git a/sound/pci/asihpi/hpi6000.c b/sound/pci/asihpi/hpi6000.c index b08578c93c6a2..57a730be26745 100644 --- a/sound/pci/asihpi/hpi6000.c +++ b/sound/pci/asihpi/hpi6000.c @@ -537,6 +537,11 @@ static short create_adapter_obj(struct hpi_adapter_obj *pao, hr1.size = sizeof(hr1); error = hpi6000_message_response_sequence(pao, 0, &hm, &hr0); + if (error) { + HPI_DEBUG_LOG(ERROR, "message transport error %d\n", + error); + return error; + } if (hr0.error) { HPI_DEBUG_LOG(DEBUG, "message error %d\n", hr0.error); return hr0.error; diff --git a/sound/pci/ctxfi/cthw20k2.c b/sound/pci/ctxfi/cthw20k2.c index 55af8ef298383..4d970f25a2c6c 100644 --- a/sound/pci/ctxfi/cthw20k2.c +++ b/sound/pci/ctxfi/cthw20k2.c @@ -993,6 +993,7 @@ static int daio_mgr_dao_init(void *blk, unsigned int idx, unsigned int conf) /* S/PDIF output */ switch ((conf & 0xf)) { case 1: + case 9: set_field(&ctl->txctl[idx], ATXCTL_NUC, 0); break; case 2: diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c index 4cec56769c0f6..e20f6bfcbdb87 100644 --- a/sound/pci/ice1712/ice1712.c +++ b/sound/pci/ice1712/ice1712.c @@ -2523,7 +2523,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) { static int dev; - struct snd_card *card; + struct snd_card *card __free(snd_card_unref) = NULL; struct snd_ice1712 *ice; int pcm_dev = 0, err; const struct snd_ice1712_card_info * const *tbl, *c; @@ -2535,8 +2535,8 @@ static int snd_ice1712_probe(struct pci_dev *pci, return -ENOENT; } - err = snd_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, - sizeof(*ice), &card); + err = snd_devm_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE, + sizeof(*ice), &card); if (err < 0) return err; ice = card->private_data; @@ -2640,6 +2640,7 @@ static int snd_ice1712_probe(struct pci_dev *pci, if (err < 0) return err; pci_set_drvdata(pci, card); + card = NULL; /* probe succeeded, don't release as error */ dev++; return 0; } diff --git a/sound/pci/ice1712/ice1724.c b/sound/pci/ice1712/ice1724.c index 7efb0bbcea550..851e7afb58142 100644 --- a/sound/pci/ice1712/ice1724.c +++ b/sound/pci/ice1712/ice1724.c @@ -730,13 +730,22 @@ static int snd_vt1724_pcm_hw_params(struct snd_pcm_substream *substream, static int snd_vt1724_pcm_hw_free(struct snd_pcm_substream *substream) { struct snd_ice1712 *ice = snd_pcm_substream_chip(substream); + bool released = false; int i; - guard(mutex)(&ice->open_mutex); - /* unmark surround channels */ - for (i = 0; i < 3; i++) - if (ice->pcm_reserved[i] == substream) + scoped_guard(mutex, &ice->open_mutex) { + /* unmark surround channels */ + for (i = 0; i < 3; i++) { + if (ice->pcm_reserved[i] != substream) + continue; ice->pcm_reserved[i] = NULL; + released = true; + } + } + + if (released && ice->pcm_ds) + wake_up(&ice->pcm_ds->open_wait); + return 0; } @@ -1364,7 +1373,7 @@ static int snd_vt1724_playback_indep_open(struct snd_pcm_substream *substream) scoped_guard(mutex, &ice->open_mutex) { /* already used by PDMA0? */ if (ice->pcm_reserved[substream->number]) - return -EBUSY; /* FIXME: should handle blocking mode properly */ + return -EAGAIN; } runtime->private_data = (void *)&vt1724_playback_dma_regs[substream->number]; ice->playback_con_substream_ds[substream->number] = substream; diff --git a/sound/pci/lx6464es/lx6464es.c b/sound/pci/lx6464es/lx6464es.c index 96df00db51d53..3b40ec071379e 100644 --- a/sound/pci/lx6464es/lx6464es.c +++ b/sound/pci/lx6464es/lx6464es.c @@ -402,11 +402,8 @@ static void lx_trigger_start(struct lx6464es *chip, struct lx_stream *lx_stream) int err; - const u32 channels = substream->runtime->channels; - const u32 bytes_per_frame = channels * 3; - const u32 period_size = substream->runtime->period_size; const u32 periods = substream->runtime->periods; - const u32 period_bytes = period_size * bytes_per_frame; + const u32 period_bytes = snd_pcm_lib_period_bytes(substream); dma_addr_t buf = substream->dma_buffer.addr; int i; diff --git a/sound/pci/lx6464es/lx_core.c b/sound/pci/lx6464es/lx_core.c index 6f0843cfb3be4..9909f10678716 100644 --- a/sound/pci/lx6464es/lx_core.c +++ b/sound/pci/lx6464es/lx_core.c @@ -969,10 +969,7 @@ static int lx_interrupt_request_new_buffer(struct lx6464es *chip, const unsigned int is_capture = lx_stream->is_capture; int err; - const u32 channels = substream->runtime->channels; - const u32 bytes_per_frame = channels * 3; - const u32 period_size = substream->runtime->period_size; - const u32 period_bytes = period_size * bytes_per_frame; + const u32 period_bytes = snd_pcm_lib_period_bytes(substream); const u32 pos = lx_stream->frame_pos; const u32 next_pos = ((pos+1) == substream->runtime->periods) ? 0 : pos + 1; diff --git a/sound/pci/pcxhr/pcxhr.c b/sound/pci/pcxhr/pcxhr.c index 83066d08367e6..7b37b2aa76a62 100644 --- a/sound/pci/pcxhr/pcxhr.c +++ b/sound/pci/pcxhr/pcxhr.c @@ -1532,6 +1532,13 @@ static int pcxhr_probe(struct pci_dev *pci, mgr->pci = pci; mgr->irq = -1; + /* ISR lock */ + mutex_init(&mgr->lock); + mutex_init(&mgr->msg_lock); + + /* init setup mutex*/ + mutex_init(&mgr->setup_mutex); + if (request_threaded_irq(pci->irq, pcxhr_interrupt, pcxhr_threaded_irq, IRQF_SHARED, KBUILD_MODNAME, mgr)) { @@ -1545,13 +1552,6 @@ static int pcxhr_probe(struct pci_dev *pci, "Digigram at 0x%lx & 0x%lx, 0x%lx irq %i", mgr->port[0], mgr->port[1], mgr->port[2], mgr->irq); - /* ISR lock */ - mutex_init(&mgr->lock); - mutex_init(&mgr->msg_lock); - - /* init setup mutex*/ - mutex_init(&mgr->setup_mutex); - mgr->prmh = kmalloc(sizeof(*mgr->prmh) + sizeof(u32) * (PCXHR_SIZE_MAX_LONG_STATUS - PCXHR_SIZE_MAX_STATUS), diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c index 6ce2cd88cda6c..605a3a0053d92 100644 --- a/sound/pci/via82xx_modem.c +++ b/sound/pci/via82xx_modem.c @@ -575,24 +575,18 @@ static inline unsigned int calc_linear_pos(struct via82xx_modem *chip, viadev->bufsize2, viadev->idx_table[idx].offset, viadev->idx_table[idx].size, count); #endif - if (count && size < count) { + if (! count) + /* bogus count 0 on the DMA boundary? */ + res = viadev->idx_table[idx].offset; + else + /* count register returns full size + * when end of buffer is reached + */ + res = viadev->idx_table[idx].offset + size; + if (check_invalid_pos(viadev, res)) { dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr, using last valid pointer\n"); + "invalid via82xx_cur_ptr (2), using last valid pointer\n"); res = viadev->lastpos; - } else { - if (! count) - /* bogus count 0 on the DMA boundary? */ - res = viadev->idx_table[idx].offset; - else - /* count register returns full size - * when end of buffer is reached - */ - res = viadev->idx_table[idx].offset + size; - if (check_invalid_pos(viadev, res)) { - dev_dbg(chip->card->dev, - "invalid via82xx_cur_ptr (2), using last valid pointer\n"); - res = viadev->lastpos; - } } } viadev->lastpos = res; /* remember the last position */ diff --git a/sound/soc/amd/ps/pci-ps.c b/sound/soc/amd/ps/pci-ps.c index c62299b292041..cf06d4ae299b5 100644 --- a/sound/soc/amd/ps/pci-ps.c +++ b/sound/soc/amd/ps/pci-ps.c @@ -248,7 +248,7 @@ static irqreturn_t acp63_irq_handler(int irq, void *dev_id) if (sdw_dma_irq_flag) return IRQ_WAKE_THREAD; - if (irq_flag | wake_irq_flag) + if (irq_flag || wake_irq_flag) return IRQ_HANDLED; else return IRQ_NONE; @@ -595,7 +595,7 @@ static int snd_acp63_probe(struct pci_dev *pci, return -ENODEV; } - ret = pci_request_regions(pci, "AMD ACP6.2 audio"); + ret = pci_request_regions(pci, "AMD ACP6.3 audio"); if (ret < 0) { dev_err(&pci->dev, "pci_request_regions failed\n"); goto disable_pci; @@ -683,8 +683,37 @@ static int snd_acp_runtime_resume(struct device *dev) return acp_hw_runtime_resume(dev); } +static void acp_disable_msi_on_resume(struct pci_dev *pdev) +{ + u16 control; + + if (!pdev->msi_cap) + return; + + pci_read_config_word(pdev, pdev->msi_cap + PCI_MSI_FLAGS, &control); + if (control & PCI_MSI_FLAGS_ENABLE) { + dev_warn(&pdev->dev, + "ACP: MSI unexpectedly enabled after resume (flags=0x%04x), disabling\n", + control); + control &= ~PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(pdev, pdev->msi_cap + PCI_MSI_FLAGS, control); + } +} + static int snd_acp_resume(struct device *dev) { + struct pci_dev *pdev = to_pci_dev(dev); + + /* + * BIOS/firmware may re-enable MSI in PCI config space during + * system resume even though this driver only uses legacy INTx + * interrupts. If MSI is left enabled with stale address/data + * registers, the device will write interrupts to a bogus address + * causing IOMMU IO_PAGE_FAULT and interrupt delivery failure. + * Explicitly clear the MSI Enable bit before reinitializing + * the ACP hardware. + */ + acp_disable_msi_on_resume(pdev); return acp_hw_resume(dev); } diff --git a/sound/soc/amd/renoir/acp3x-pdm-dma.c b/sound/soc/amd/renoir/acp3x-pdm-dma.c index a560d06097d5e..ecb4b79520135 100644 --- a/sound/soc/amd/renoir/acp3x-pdm-dma.c +++ b/sound/soc/amd/renoir/acp3x-pdm-dma.c @@ -104,7 +104,7 @@ static void disable_pdm_interrupts(void __iomem *acp_base) u32 ext_int_ctrl; ext_int_ctrl = rn_readl(acp_base + ACP_EXTERNAL_INTR_CNTL); - ext_int_ctrl |= ~PDM_DMA_INTR_MASK; + ext_int_ctrl &= ~PDM_DMA_INTR_MASK; rn_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL); } diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c index d9f145b634f35..8abebf17bac5e 100644 --- a/sound/soc/amd/yc/acp6x-mach.c +++ b/sound/soc/amd/yc/acp6x-mach.c @@ -500,6 +500,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_PRODUCT_NAME, "Bravo 17 D7VF"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Alienware"), + DMI_MATCH(DMI_PRODUCT_NAME, "Alienware m15 R7 AMD"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -668,6 +675,20 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "8E35"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_BOARD_NAME, "8F06"), + } + }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "Victus by HP Laptop 16-e1xxx"), + } + }, { .driver_data = &acp6x_card, .matches = { @@ -745,6 +766,13 @@ static const struct dmi_system_id yc_acp_quirk_table[] = { DMI_MATCH(DMI_BOARD_NAME, "BM1403CDA"), } }, + { + .driver_data = &acp6x_card, + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "Micro-Star International Co., Ltd."), + DMI_MATCH(DMI_PRODUCT_NAME, "Thin A15 B7UC"), + } + }, { .driver_data = &acp6x_card, .matches = { diff --git a/sound/soc/amd/yc/acp6x-pdm-dma.c b/sound/soc/amd/yc/acp6x-pdm-dma.c index ac758b90f441d..167cd792d33da 100644 --- a/sound/soc/amd/yc/acp6x-pdm-dma.c +++ b/sound/soc/amd/yc/acp6x-pdm-dma.c @@ -275,9 +275,11 @@ static int acp6x_pdm_dma_close(struct snd_soc_component *component, struct snd_pcm_substream *substream) { struct pdm_dev_data *adata = dev_get_drvdata(component->dev); + struct snd_pcm_runtime *runtime = substream->runtime; acp6x_disable_pdm_interrupts(adata->acp6x_base); adata->capture_stream = NULL; + kfree(runtime->private_data); return 0; } diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c index c4dcb2b545912..17b572ad8b99e 100644 --- a/sound/soc/apple/mca.c +++ b/sound/soc/apple/mca.c @@ -210,10 +210,10 @@ static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd, SERDES_STATUS_EN | SERDES_STATUS_RST, SERDES_STATUS_RST); /* - * Experiments suggest that it takes at most ~1 us - * for the bit to clear, so wait 2 us for good measure. + * The SERDES cluster needs a bit of time to reset itself + * and settle before we start poking it. This is... slow... */ - udelay(2); + udelay(25); WARN_ON(readl_relaxed(cl->base + serdes_unit + REG_SERDES_STATUS) & SERDES_STATUS_RST); mca_modify(cl, serdes_conf, SERDES_CONF_SYNC_SEL, diff --git a/sound/soc/bcm/bcm2835-i2s.c b/sound/soc/bcm/bcm2835-i2s.c index cbbf6723ba208..8be2e282837fb 100644 --- a/sound/soc/bcm/bcm2835-i2s.c +++ b/sound/soc/bcm/bcm2835-i2s.c @@ -149,8 +149,8 @@ static void bcm2835_i2s_stop_clock(struct bcm2835_i2s_dev *dev) dev->clk_prepared = false; } -static void bcm2835_i2s_clear_fifos(struct bcm2835_i2s_dev *dev, - bool tx, bool rx) +static int bcm2835_i2s_clear_fifos(struct bcm2835_i2s_dev *dev, + bool tx, bool rx) { int timeout = 1000; uint32_t syncval; @@ -187,8 +187,9 @@ static void bcm2835_i2s_clear_fifos(struct bcm2835_i2s_dev *dev, /* Wait for 2 PCM clock cycles */ /* - * Toggle the SYNC flag. After 2 PCM clock cycles it can be read back - * FIXME: This does not seem to work for slave mode! + * Toggle the SYNC flag. After 2 PCM clock cycles it can be read back. + * The reset will timeout in clock consumer mode without an external + * clock. */ regmap_read(dev->i2s_regmap, BCM2835_I2S_CS_A_REG, &syncval); syncval &= BCM2835_I2S_SYNC; @@ -204,7 +205,8 @@ static void bcm2835_i2s_clear_fifos(struct bcm2835_i2s_dev *dev, } if (!timeout) - dev_err(dev->dev, "I2S SYNC error!\n"); + dev_err(dev->dev, + "FIFO clear timed out: no PCM clock (clock consumer with the provider not running?)\n"); /* Stop clock if it was not running before */ if (!clk_was_prepared) @@ -213,6 +215,14 @@ static void bcm2835_i2s_clear_fifos(struct bcm2835_i2s_dev *dev, /* Restore I2S state */ regmap_update_bits(dev->i2s_regmap, BCM2835_I2S_CS_A_REG, BCM2835_I2S_RXON | BCM2835_I2S_TXON, i2s_active_state); + + /* + * Without the SYNC round trip the FIFOs were not actually cleared: + * starting DMA on top of a stale FIFO makes RX free-run (DREQ stuck + * high, the same dead word re-read at bus speed) and the state then + * survives further stream cycles. Fail the stream instead. + */ + return timeout ? 0 : -EIO; } static int bcm2835_i2s_set_dai_fmt(struct snd_soc_dai *dai, @@ -588,6 +598,11 @@ static int bcm2835_i2s_hw_params(struct snd_pcm_substream *substream, | BCM2835_I2S_RX(0x20), 0xffffffff); /* Clear FIFOs */ + /* + * May time out benignly here: on clock-consumer links the provider's + * clocks are often not running yet at hw_params time; the clear that + * must succeed is the one in .prepare. + */ bcm2835_i2s_clear_fifos(dev, true, true); dev_dbg(dev->dev, @@ -636,7 +651,13 @@ static int bcm2835_i2s_prepare(struct snd_pcm_substream *substream, bcm2835_i2s_clear_fifos(dev, true, false); else if (substream->stream == SNDRV_PCM_STREAM_CAPTURE && (cs_reg & BCM2835_I2S_RXD)) - bcm2835_i2s_clear_fifos(dev, false, true); + /* + * Stale RX data with no working clock cannot be cleared: DMA + * would free-run over the dead FIFO (DREQ stuck high) and the + * poisoned state survives stream cycles. Fail this capture + * open instead of recording garbage. + */ + return bcm2835_i2s_clear_fifos(dev, false, true); return 0; } diff --git a/sound/soc/bcm/bcm63xx-i2s-whistler.c b/sound/soc/bcm/bcm63xx-i2s-whistler.c index c47ed1e6ea2b6..14d111fe29d83 100644 --- a/sound/soc/bcm/bcm63xx-i2s-whistler.c +++ b/sound/soc/bcm/bcm63xx-i2s-whistler.c @@ -285,6 +285,7 @@ static const struct of_device_id snd_soc_bcm_audio_match[] = { {.compatible = "brcm,bcm63xx-i2s"}, { } }; +MODULE_DEVICE_TABLE(of, snd_soc_bcm_audio_match); #endif static struct platform_driver bcm63xx_i2s_driver = { diff --git a/sound/soc/bcm/iqaudio-codec.c b/sound/soc/bcm/iqaudio-codec.c index 02ea79029efc9..81770b068bc94 100644 --- a/sound/soc/bcm/iqaudio-codec.c +++ b/sound/soc/bcm/iqaudio-codec.c @@ -59,20 +59,15 @@ static int snd_rpi_iqaudio_pll_control(struct snd_soc_dapm_widget *w, return ret; } -static int snd_rpi_iqaudio_post_dapm_event(struct snd_soc_dapm_widget *w, - struct snd_kcontrol *kcontrol, - int event) +static int snd_rpi_iqaudio_mic_bias_event(struct snd_soc_dapm_widget *w, + struct snd_kcontrol *kcontrol, + int event) { - switch (event) { - case SND_SOC_DAPM_POST_PMU: - /* Delay for mic bias ramp */ - msleep(1000); - break; - default: - break; - } - - return 0; + if (event == SND_SOC_DAPM_POST_PMU) + /* Delay for mic bias ramp */ + msleep(1000); + + return 0; } static const struct snd_kcontrol_new dapm_controls[] = { @@ -90,7 +85,9 @@ static const struct snd_soc_dapm_widget dapm_widgets[] = { SND_SOC_DAPM_SUPPLY("PLL Control", SND_SOC_NOPM, 0, 0, snd_rpi_iqaudio_pll_control, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD), - SND_SOC_DAPM_POST("Post Power Up Event", snd_rpi_iqaudio_post_dapm_event), + SND_SOC_DAPM_SUPPLY("Mic Bias Ramp", SND_SOC_NOPM, 0, 0, + snd_rpi_iqaudio_mic_bias_event, + SND_SOC_DAPM_POST_PMU), }; static const struct snd_soc_dapm_route audio_map[] = { @@ -105,8 +102,10 @@ static const struct snd_soc_dapm_route audio_map[] = { /* Assume Mic1 is linked to Headset and Mic2 to on-board mic */ {"MIC1", NULL, "MIC Jack"}, {"MIC Jack", NULL, "PLL Control"}, + {"MIC Jack", NULL, "Mic Bias Ramp"}, {"MIC2", NULL, "Onboard MIC"}, {"Onboard MIC", NULL, "PLL Control"}, + {"Onboard MIC", NULL, "Mic Bias Ramp"}, }; /* machine stream operations */ diff --git a/sound/soc/codecs/ab8500-codec.c b/sound/soc/codecs/ab8500-codec.c index 32d9cdbc8c310..a0f5663157c38 100644 --- a/sound/soc/codecs/ab8500-codec.c +++ b/sound/soc/codecs/ab8500-codec.c @@ -303,7 +303,7 @@ static const struct snd_kcontrol_new dapm_anc_in_select[] = { /* ANC - Enable/Disable */ static const struct snd_kcontrol_new dapm_anc_enable[] = { SOC_DAPM_SINGLE("Switch", AB8500_ANCCONF1, - AB8500_ANCCONF1_ENANC, 0, 0), + AB8500_ANCCONF1_ENANC, 1, 0), }; /* ANC to Earpiece - Mute */ @@ -385,12 +385,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { /* DA/AD */ - SND_SOC_DAPM_INPUT("ADC Input"), - SND_SOC_DAPM_ADC("ADC", "ab8500_0c", SND_SOC_NOPM, 0, 0), - - SND_SOC_DAPM_DAC("DAC", NULL, SND_SOC_NOPM, 0, 0), - SND_SOC_DAPM_OUTPUT("DAC Output"), - SND_SOC_DAPM_AIF_IN("DA_IN1", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_IN("DA_IN2", NULL, 0, SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_IN("DA_IN3", NULL, 0, SND_SOC_NOPM, 0, 0), @@ -582,9 +576,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD3 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD3 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD34, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD34 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD34, 0, NULL, 0), /* Mic 2 */ @@ -643,9 +636,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD12 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD12, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD12 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD12, 0, NULL, 0), /* HD Capture path */ @@ -659,12 +651,8 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD6 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD57 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD5768, 0, - NULL, 0), - SND_SOC_DAPM_MIXER("AD68 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD5768, 0, - NULL, 0), + SND_SOC_DAPM_SUPPLY("AD5768 Enable", AB8500_ADPATHENA, + AB8500_ADPATHENA_ENAD5768, 0, NULL, 0), /* Digital Microphone path */ @@ -696,10 +684,6 @@ static const struct snd_soc_dapm_widget ab8500_dapm_widgets[] = { SND_SOC_DAPM_MIXER("AD4 Channel Volume", SND_SOC_NOPM, 0, 0, NULL, 0), - SND_SOC_DAPM_MIXER("AD4 Enable", - AB8500_ADPATHENA, AB8500_ADPATHENA_ENAD34, - 0, NULL, 0), - /* Acoustical Noise Cancellation path */ SND_SOC_DAPM_INPUT("ANC Configure Input"), @@ -747,24 +731,17 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"Main Supply", NULL, "Audio Power"}, {"Main Supply", NULL, "Audio Analog Power"}, - {"DAC", NULL, "ab8500_0p"}, - {"DAC", NULL, "Main Supply"}, - {"ADC", NULL, "ab8500_0c"}, - {"ADC", NULL, "Main Supply"}, - /* ANC Configure */ {"ANC Configure Input", NULL, "Main Supply"}, {"ANC Configure Output", NULL, "ANC Configure Input"}, - /* AD/DA */ - {"ADC", NULL, "ADC Input"}, - {"DAC Output", NULL, "DAC"}, - /* Powerup charge pump if DA1/2 is in use */ {"DA_IN1", NULL, "ab8500_0p"}, + {"DA_IN1", NULL, "Main Supply"}, {"DA_IN1", NULL, "Charge Pump"}, {"DA_IN2", NULL, "ab8500_0p"}, + {"DA_IN2", NULL, "Main Supply"}, {"DA_IN2", NULL, "Charge Pump"}, /* Headset path */ @@ -799,8 +776,10 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* HF or LineOut path */ {"DA_IN3", NULL, "ab8500_0p"}, + {"DA_IN3", NULL, "Main Supply"}, {"DA3 Channel Volume", NULL, "DA_IN3"}, {"DA_IN4", NULL, "ab8500_0p"}, + {"DA_IN4", NULL, "Main Supply"}, {"DA4 Channel Volume", NULL, "DA_IN4"}, {"Speaker Left Source", "Audio Path", "DA3 Channel Volume"}, @@ -858,8 +837,10 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* Vibrator path */ {"DA_IN5", NULL, "ab8500_0p"}, + {"DA_IN5", NULL, "Main Supply"}, {"DA5 Channel Volume", NULL, "DA_IN5"}, {"DA_IN6", NULL, "ab8500_0p"}, + {"DA_IN6", NULL, "Main Supply"}, {"DA6 Channel Volume", NULL, "DA_IN6"}, {"VIB1 DAC", NULL, "DA5 Channel Volume"}, @@ -901,13 +882,15 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD1 Channel Volume", NULL, "AD1 Source Select"}, {"AD2 Channel Volume", NULL, "AD2 Source Select"}, - {"AD12 Enable", NULL, "AD1 Channel Volume"}, - {"AD12 Enable", NULL, "AD2 Channel Volume"}, + {"AD1 Channel Volume", NULL, "AD12 Enable"}, + {"AD2 Channel Volume", NULL, "AD12 Enable"}, - {"AD_OUT1", NULL, "ab8500_0c"}, - {"AD_OUT1", NULL, "AD12 Enable"}, - {"AD_OUT2", NULL, "ab8500_0c"}, - {"AD_OUT2", NULL, "AD12 Enable"}, + {"ab8500_0c", NULL, "AD_OUT1"}, + {"AD_OUT1", NULL, "Main Supply"}, + {"AD_OUT1", NULL, "AD1 Channel Volume"}, + {"ab8500_0c", NULL, "AD_OUT2"}, + {"AD_OUT2", NULL, "Main Supply"}, + {"AD_OUT2", NULL, "AD2 Channel Volume"}, /* Mic 1 */ @@ -924,11 +907,11 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD3 Source Select", "Mic 1", "MIC1 ADC"}, {"AD3 Channel Volume", NULL, "AD3 Source Select"}, + {"AD3 Channel Volume", NULL, "AD34 Enable"}, - {"AD3 Enable", NULL, "AD3 Channel Volume"}, - - {"AD_OUT3", NULL, "ab8500_0c"}, - {"AD_OUT3", NULL, "AD3 Enable"}, + {"ab8500_0c", NULL, "AD_OUT3"}, + {"AD_OUT3", NULL, "Main Supply"}, + {"AD_OUT3", NULL, "AD3 Channel Volume"}, /* HD Capture path */ @@ -937,14 +920,15 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"AD5 Channel Volume", NULL, "AD5 Source Select"}, {"AD6 Channel Volume", NULL, "AD6 Source Select"}, + {"AD5 Channel Volume", NULL, "AD5768 Enable"}, + {"AD6 Channel Volume", NULL, "AD5768 Enable"}, - {"AD57 Enable", NULL, "AD5 Channel Volume"}, - {"AD68 Enable", NULL, "AD6 Channel Volume"}, - - {"AD_OUT57", NULL, "ab8500_0c"}, - {"AD_OUT57", NULL, "AD57 Enable"}, - {"AD_OUT68", NULL, "ab8500_0c"}, - {"AD_OUT68", NULL, "AD68 Enable"}, + {"ab8500_0c", NULL, "AD_OUT57"}, + {"AD_OUT57", NULL, "Main Supply"}, + {"AD_OUT57", NULL, "AD5 Channel Volume"}, + {"ab8500_0c", NULL, "AD_OUT68"}, + {"AD_OUT68", NULL, "Main Supply"}, + {"AD_OUT68", NULL, "AD6 Channel Volume"}, /* Digital Microphone path */ @@ -955,17 +939,25 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { {"DMic 5", NULL, "V-DMIC"}, {"DMic 6", NULL, "V-DMIC"}, - {"AD1 Source Select", NULL, "DMic 1"}, - {"AD2 Source Select", NULL, "DMic 2"}, - {"AD3 Source Select", NULL, "DMic 3"}, - {"AD5 Source Select", NULL, "DMic 5"}, - {"AD6 Source Select", NULL, "DMic 6"}, + {"DMIC1", NULL, "DMic 1"}, + {"DMIC2", NULL, "DMic 2"}, + {"DMIC3", NULL, "DMic 3"}, + {"DMIC4", NULL, "DMic 4"}, + {"DMIC5", NULL, "DMic 5"}, + {"DMIC6", NULL, "DMic 6"}, + + {"AD1 Source Select", "DMic 1", "DMIC1"}, + {"AD2 Source Select", "DMic 2", "DMIC2"}, + {"AD3 Source Select", "DMic 3", "DMIC3"}, + {"AD5 Source Select", "DMic 5", "DMIC5"}, + {"AD6 Source Select", "DMic 6", "DMIC6"}, - {"AD4 Channel Volume", NULL, "DMic 4"}, - {"AD4 Enable", NULL, "AD4 Channel Volume"}, + {"AD4 Channel Volume", NULL, "DMIC4"}, + {"AD4 Channel Volume", NULL, "AD34 Enable"}, - {"AD_OUT4", NULL, "ab8500_0c"}, - {"AD_OUT4", NULL, "AD4 Enable"}, + {"ab8500_0c", NULL, "AD_OUT4"}, + {"AD_OUT4", NULL, "Main Supply"}, + {"AD_OUT4", NULL, "AD4 Channel Volume"}, /* LineIn Bypass path */ @@ -990,13 +982,13 @@ static const struct snd_soc_dapm_route ab8500_dapm_routes[] = { /* Sidetone Filter path */ - {"Sidetone Left Source", "LineIn Left", "AD12 Enable"}, - {"Sidetone Left Source", "LineIn Right", "AD12 Enable"}, - {"Sidetone Left Source", "Mic 1", "AD3 Enable"}, + {"Sidetone Left Source", "LineIn Left", "AD1 Channel Volume"}, + {"Sidetone Left Source", "LineIn Right", "AD2 Channel Volume"}, + {"Sidetone Left Source", "Mic 1", "AD3 Channel Volume"}, {"Sidetone Left Source", "Headset Left", "DA_IN1"}, - {"Sidetone Right Source", "LineIn Right", "AD12 Enable"}, - {"Sidetone Right Source", "Mic 1", "AD3 Enable"}, - {"Sidetone Right Source", "DMic 4", "AD4 Enable"}, + {"Sidetone Right Source", "LineIn Right", "AD2 Channel Volume"}, + {"Sidetone Right Source", "Mic 1", "AD3 Channel Volume"}, + {"Sidetone Right Source", "DMic 4", "AD4 Channel Volume"}, {"Sidetone Right Source", "Headset Right", "DA_IN2"}, {"STFIR1 Control", NULL, "Sidetone Left Source"}, @@ -1917,18 +1909,18 @@ enum ab8500_filter { static int ab8500_audio_init_audioblock(struct snd_soc_component *component) { int status; + u8 mask = AB8500_STW4500CTRL3_CLK32KOUT2DIS | + AB8500_STW4500CTRL3_RESETAUDN; dev_dbg(component->dev, "%s: Enter.\n", __func__); - /* Reset audio-registers and disable 32kHz-clock output 2 */ - status = ab8500_sysctrl_write(AB8500_STW4500CTRL3, - AB8500_STW4500CTRL3_CLK32KOUT2DIS | - AB8500_STW4500CTRL3_RESETAUDN, - AB8500_STW4500CTRL3_RESETAUDN); + /* Reset the audio registers and disable the unused 32 kHz output. */ + status = ab8500_sysctrl_write(AB8500_STW4500CTRL3, mask, + AB8500_STW4500CTRL3_CLK32KOUT2DIS); if (status < 0) return status; - return 0; + return ab8500_sysctrl_write(AB8500_STW4500CTRL3, mask, mask); } static int ab8500_audio_setup_mics(struct snd_soc_component *component, @@ -2024,149 +2016,91 @@ static int ab8500_audio_set_ear_cmv(struct snd_soc_component *component, return 0; } -static int ab8500_audio_set_bit_delay(struct snd_soc_dai *dai, - unsigned int delay) -{ - unsigned int mask, val; - struct snd_soc_component *component = dai->component; - - mask = BIT(AB8500_DIGIFCONF2_IF0DEL); - val = 0; - - switch (delay) { - case 0: - break; - case 1: - val |= BIT(AB8500_DIGIFCONF2_IF0DEL); - break; - default: - dev_err(dai->component->dev, - "%s: ERROR: Unsupported bit-delay (0x%x)!\n", - __func__, delay); - return -EINVAL; - } - - dev_dbg(dai->component->dev, "%s: IF0 Bit-delay: %d bits.\n", - __func__, delay); - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); - - return 0; -} - -/* Gates clocking according format mask */ -static int ab8500_codec_set_dai_clock_gate(struct snd_soc_component *component, - unsigned int fmt) -{ - unsigned int mask; - unsigned int val; - - mask = BIT(AB8500_DIGIFCONF1_ENMASTGEN) | - BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); - - val = BIT(AB8500_DIGIFCONF1_ENMASTGEN); - - switch (fmt & SND_SOC_DAIFMT_CLOCK_MASK) { - case SND_SOC_DAIFMT_CONT: /* continuous clock */ - dev_dbg(component->dev, "%s: IF0 Clock is continuous.\n", - __func__); - val |= BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); - break; - case SND_SOC_DAIFMT_GATED: /* clock is gated */ - dev_dbg(component->dev, "%s: IF0 Clock is gated.\n", - __func__); - break; - default: - dev_err(component->dev, - "%s: ERROR: Unsupported clock mask (0x%x)!\n", - __func__, fmt & SND_SOC_DAIFMT_CLOCK_MASK); - return -EINVAL; - } - - snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, mask, val); - - return 0; -} - static int ab8500_codec_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) { - unsigned int mask; - unsigned int val; struct snd_soc_component *component = dai->component; - int status; + unsigned int conf1_mask, conf1_val = 0; + unsigned int conf2_mask, conf2_val = 0; + unsigned int conf3_mask, conf3_val = 0; + bool provider = false; + int ret; dev_dbg(component->dev, "%s: Enter (fmt = 0x%x)\n", __func__, fmt); - mask = BIT(AB8500_DIGIFCONF3_IF1DATOIF0AD) | + conf3_mask = BIT(AB8500_DIGIFCONF3_IF1DATOIF0AD) | BIT(AB8500_DIGIFCONF3_IF1CLKTOIF0CLK) | BIT(AB8500_DIGIFCONF3_IF0BFIFOEN) | BIT(AB8500_DIGIFCONF3_IF0MASTER); - val = 0; switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { case SND_SOC_DAIFMT_CBP_CFP: - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Master-mode: AB8500 provider.\n", __func__); - val |= BIT(AB8500_DIGIFCONF3_IF0MASTER); + conf3_val |= BIT(AB8500_DIGIFCONF3_IF0MASTER); + provider = true; break; case SND_SOC_DAIFMT_CBC_CFC: - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Master-mode: AB8500 consumer.\n", __func__); break; case SND_SOC_DAIFMT_CBC_CFP: case SND_SOC_DAIFMT_CBP_CFC: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: The device is either a provider or a consumer.\n", __func__); fallthrough; default: - dev_err(dai->component->dev, - "%s: ERROR: Unsupporter clocking mask 0x%x\n", + dev_err(component->dev, + "%s: ERROR: Unsupported clocking mask 0x%x\n", __func__, fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF3, mask, val); - - /* Set clock gating */ - status = ab8500_codec_set_dai_clock_gate(component, fmt); - if (status) { - dev_err(dai->component->dev, - "%s: ERROR: Failed to set clock gate (%d).\n", - __func__, status); - return status; + conf1_mask = BIT(AB8500_DIGIFCONF1_ENMASTGEN) | + BIT(AB8500_DIGIFCONF1_ENFSBITCLK0); + switch (fmt & SND_SOC_DAIFMT_CLOCK_MASK) { + case SND_SOC_DAIFMT_CONT: + if (provider) + conf1_val = conf1_mask; + break; + case SND_SOC_DAIFMT_GATED: + if (provider) + conf1_val = BIT(AB8500_DIGIFCONF1_ENMASTGEN); + break; + default: + dev_err(component->dev, "%s: Unsupported clock mask 0x%x\n", + __func__, fmt & SND_SOC_DAIFMT_CLOCK_MASK); + return -EINVAL; } - /* Setting data transfer format */ - - mask = BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | - BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | - BIT(AB8500_DIGIFCONF2_FSYNC0P) | - BIT(AB8500_DIGIFCONF2_BITCLK0P); - val = 0; + conf2_mask = BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | + BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | + BIT(AB8500_DIGIFCONF2_IF0DEL) | + BIT(AB8500_DIGIFCONF2_FSYNC0P) | + BIT(AB8500_DIGIFCONF2_BITCLK0P); switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: /* I2S mode */ - dev_dbg(dai->component->dev, "%s: IF0 Protocol: I2S\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT1); - ab8500_audio_set_bit_delay(dai, 0); + dev_dbg(component->dev, "%s: IF0 Protocol: I2S\n", __func__); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT1) | + BIT(AB8500_DIGIFCONF2_IF0DEL); break; case SND_SOC_DAIFMT_DSP_A: /* L data MSB after FRM LRC */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Protocol: DSP A (TDM)\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); - ab8500_audio_set_bit_delay(dai, 1); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0) | + BIT(AB8500_DIGIFCONF2_IF0DEL); break; case SND_SOC_DAIFMT_DSP_B: /* L data MSB during FRM LRC */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0 Protocol: DSP B (TDM)\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); - ab8500_audio_set_bit_delay(dai, 0); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0FORMAT0); break; default: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: Unsupported format (0x%x)!\n", __func__, fmt & SND_SOC_DAIFMT_FORMAT_MASK); return -EINVAL; @@ -2174,39 +2108,50 @@ static int ab8500_codec_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) switch (fmt & SND_SOC_DAIFMT_INV_MASK) { case SND_SOC_DAIFMT_NB_NF: /* normal bit clock + frame */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Normal bit clock, normal frame\n", __func__); break; case SND_SOC_DAIFMT_NB_IF: /* normal BCLK + inv FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Normal bit clock, inverted frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); break; case SND_SOC_DAIFMT_IB_NF: /* invert BCLK + nor FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Inverted bit clock, normal frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); break; case SND_SOC_DAIFMT_IB_IF: /* invert BCLK + FRM */ - dev_dbg(dai->component->dev, + dev_dbg(component->dev, "%s: IF0: Inverted bit clock, inverted frame\n", __func__); - val |= BIT(AB8500_DIGIFCONF2_FSYNC0P); - val |= BIT(AB8500_DIGIFCONF2_BITCLK0P); + conf2_val |= BIT(AB8500_DIGIFCONF2_FSYNC0P) | + BIT(AB8500_DIGIFCONF2_BITCLK0P); break; default: - dev_err(dai->component->dev, + dev_err(component->dev, "%s: ERROR: Unsupported INV mask 0x%x\n", __func__, fmt & SND_SOC_DAIFMT_INV_MASK); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF3, + conf3_mask, conf3_val); + if (ret < 0) + return ret; - return 0; + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, + conf1_mask, conf1_val); + if (ret < 0) + return ret; + + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, + conf2_mask, conf2_val); + + return ret < 0 ? ret : 0; } static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, @@ -2214,23 +2159,27 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, int slots, int slot_width) { struct snd_soc_component *component = dai->component; - unsigned int val, mask, slot, slots_active; + unsigned int active_mask, clock_ratio, slot, value, ad_out, reg; + unsigned int tx_active, rx_active; + unsigned int conf1_val, conf2_val; + unsigned int mask; + int channel, ret; mask = BIT(AB8500_DIGIFCONF2_IF0WL0) | BIT(AB8500_DIGIFCONF2_IF0WL1); - val = 0; + conf2_val = 0; switch (slot_width) { case 16: break; case 20: - val |= BIT(AB8500_DIGIFCONF2_IF0WL0); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL0); break; case 24: - val |= BIT(AB8500_DIGIFCONF2_IF0WL1); + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL1); break; case 32: - val |= BIT(AB8500_DIGIFCONF2_IF0WL1) | + conf2_val |= BIT(AB8500_DIGIFCONF2_IF0WL1) | BIT(AB8500_DIGIFCONF2_IF0WL0); break; default: @@ -2239,27 +2188,11 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, return -EINVAL; } - dev_dbg(dai->component->dev, "%s: IF0 slot-width: %d bits.\n", - __func__, slot_width); - snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, mask, val); - - /* Setup TDM clocking according to slot count */ - dev_dbg(dai->component->dev, "%s: Slots, total: %d\n", __func__, slots); - mask = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | - BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); switch (slots) { case 2: - val = AB8500_MASK_NONE; - break; case 4: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0); - break; case 8: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); - break; case 16: - val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | - BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; default: dev_err(dai->component->dev, @@ -2267,92 +2200,134 @@ static int ab8500_codec_set_dai_tdm_slot(struct snd_soc_dai *dai, __func__, slots); return -EINVAL; } - snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, mask, val); - - /* Setup TDM DA according to active tx slots */ - - if (tx_mask & ~0xff) - return -EINVAL; - - mask = AB8500_DASLOTCONFX_SLTODAX_MASK; - tx_mask = tx_mask << AB8500_DA_DATA0_OFFSET; - slots_active = hweight32(tx_mask); - - dev_dbg(dai->component->dev, "%s: Slots, active, TX: %d\n", __func__, - slots_active); - switch (slots_active) { - case 0: + clock_ratio = slots * slot_width; + switch (clock_ratio) { + case 32: + conf1_val = 0; break; - case 1: - slot = ffs(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF1, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF3, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF2, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF4, mask, slot); + case 64: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0); break; - case 2: - slot = ffs(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF1, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF3, mask, slot); - slot = fls(tx_mask); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF2, mask, slot); - snd_soc_component_update_bits(component, AB8500_DASLOTCONF4, mask, slot); + case 128: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; - case 8: - dev_dbg(dai->component->dev, - "%s: In 8-channel mode DA-from-slot mapping is set manually.", - __func__); + case 256: + conf1_val = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | + BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); break; default: - dev_err(dai->component->dev, - "%s: Unsupported number of active TX-slots (%d)!\n", - __func__, slots_active); + dev_err(component->dev, "%s: Unsupported BCLK ratio (%u)!\n", + __func__, clock_ratio); return -EINVAL; } - /* Setup TDM AD according to active RX-slots */ + active_mask = GENMASK(min(slots, 8) - 1, 0); + if ((tx_mask | rx_mask) & ~active_mask) { + dev_err(component->dev, "%s: Slot mask exceeds slot count\n", + __func__); + return -EINVAL; + } - if (rx_mask & ~0xff) + tx_active = hweight32(tx_mask); + rx_active = hweight32(rx_mask); + if (tx_active != 0 && tx_active != 1 && tx_active != 2 && + tx_active != 8) { + dev_err(component->dev, "%s: Unsupported active TX slots (%u)!\n", + __func__, tx_active); return -EINVAL; + } + if (rx_active != 0 && rx_active != 1 && rx_active != 2 && + rx_active != 8) { + dev_err(component->dev, "%s: Unsupported active RX slots (%u)!\n", + __func__, rx_active); + return -EINVAL; + } - rx_mask = rx_mask << AB8500_AD_DATA0_OFFSET; - slots_active = hweight32(rx_mask); + dev_dbg(component->dev, + "%s: %d slots of %d bits, TX active: %u, RX active: %u\n", + __func__, slots, slot_width, tx_active, rx_active); - dev_dbg(dai->component->dev, "%s: Slots, active, RX: %d\n", __func__, - slots_active); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF2, + mask, conf2_val); + if (ret < 0) + return ret; - switch (slots_active) { - case 0: - break; - case 1: - slot = ffs(rx_mask); - snd_soc_component_update_bits(component, AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, slot)); - break; - case 2: - slot = ffs(rx_mask); - snd_soc_component_update_bits(component, - AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, slot)); - slot = fls(rx_mask); - snd_soc_component_update_bits(component, - AB8500_ADSLOTSEL(slot), - AB8500_MASK_SLOT(slot), - AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT2, slot)); - break; - case 8: - dev_dbg(dai->component->dev, - "%s: In 8-channel mode AD-to-slot mapping is set manually.", - __func__); - break; - default: - dev_err(dai->component->dev, - "%s: Unsupported number of active RX-slots (%d)!\n", - __func__, slots_active); - return -EINVAL; + mask = BIT(AB8500_DIGIFCONF1_IF0BITCLKOS0) | + BIT(AB8500_DIGIFCONF1_IF0BITCLKOS1); + ret = snd_soc_component_update_bits(component, AB8500_DIGIFCONF1, + mask, conf1_val); + if (ret < 0) + return ret; + + mask = AB8500_DASLOTCONFX_SLTODAX_MASK; + if (tx_active == 1 || tx_active == 2) { + slot = __ffs(tx_mask) + AB8500_DA_DATA0_OFFSET; + reg = AB8500_DASLOTCONF1; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + reg = AB8500_DASLOTCONF3; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + + if (tx_active == 2) + slot = __fls(tx_mask) + AB8500_DA_DATA0_OFFSET; + reg = AB8500_DASLOTCONF2; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + reg = AB8500_DASLOTCONF4; + ret = snd_soc_component_update_bits(component, reg, mask, slot); + if (ret < 0) + return ret; + } else if (tx_active == 8) { + channel = 0; + for (slot = 0; slot < 8; slot++) { + if (!(tx_mask & BIT(slot))) + continue; + reg = AB8500_DASLOTCONF1 + channel++; + value = slot + AB8500_DA_DATA0_OFFSET; + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } + } + + if (rx_active == 1 || rx_active == 2) { + slot = __ffs(rx_mask) + AB8500_AD_DATA0_OFFSET; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT3, + slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + + if (rx_active == 2) { + slot = __fls(rx_mask) + AB8500_AD_DATA0_OFFSET; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(AB8500_AD_OUT2, + slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } + } else if (rx_active == 8) { + channel = 0; + for (slot = 0; slot < 8; slot++) { + if (!(rx_mask & BIT(slot))) + continue; + ad_out = AB8500_AD_OUT1 + channel++; + value = AB8500_ADSLOTSELX_AD_OUT_TO_SLOT(ad_out, slot); + reg = AB8500_ADSLOTSEL(slot); + mask = AB8500_MASK_SLOT(slot); + ret = snd_soc_component_update_bits(component, reg, mask, value); + if (ret < 0) + return ret; + } } return 0; @@ -2461,6 +2436,13 @@ static int ab8500_codec_probe(struct snd_soc_component *component) ab8500_codec_of_probe(dev, np, &codec_pdata); + status = ab8500_audio_init_audioblock(component); + if (status < 0) { + dev_err(dev, "%s: failed to init audio-block (%d)!\n", + __func__, status); + return status; + } + status = ab8500_audio_setup_mics(component, &codec_pdata.amics); if (status < 0) { pr_err("%s: Failed to setup mics (%d)!\n", __func__, status); @@ -2473,13 +2455,6 @@ static int ab8500_codec_probe(struct snd_soc_component *component) return status; } - status = ab8500_audio_init_audioblock(component); - if (status < 0) { - dev_err(dev, "%s: failed to init audio-block (%d)!\n", - __func__, status); - return status; - } - /* Override HW-defaults */ snd_soc_component_write(component, AB8500_ANACONF5, BIT(AB8500_ANACONF5_HSAUTOEN)); diff --git a/sound/soc/codecs/aw88261.c b/sound/soc/codecs/aw88261.c index 3270319c32a1f..c0d62992701f6 100644 --- a/sound/soc/codecs/aw88261.c +++ b/sound/soc/codecs/aw88261.c @@ -150,7 +150,7 @@ static int aw88261_dev_get_iis_status(struct aw_device *aw_dev) if (ret) return ret; if ((reg_val & AW88261_BIT_PLL_CHECK) != AW88261_BIT_PLL_CHECK) { - dev_err(aw_dev->dev, "check pll lock fail,reg_val:0x%04x", reg_val); + dev_dbg(aw_dev->dev, "check pll lock fail,reg_val:0x%04x", reg_val); return -EINVAL; } @@ -164,7 +164,7 @@ static int aw88261_dev_check_mode1_pll(struct aw_device *aw_dev) for (i = 0; i < AW88261_DEV_SYSST_CHECK_MAX; i++) { ret = aw88261_dev_get_iis_status(aw_dev); if (ret) { - dev_err(aw_dev->dev, "mode1 iis signal check error"); + dev_dbg(aw_dev->dev, "mode1 iis signal check error"); usleep_range(AW88261_2000_US, AW88261_2000_US + 10); } else { return ret; @@ -253,10 +253,10 @@ static int aw88261_dev_check_sysst(struct aw_device *aw_dev) return ret; check_val = reg_val & (~AW88261_BIT_SYSST_CHECK_MASK) - & AW88261_BIT_SYSST_CHECK; - if (check_val != AW88261_BIT_SYSST_CHECK) { - dev_err(aw_dev->dev, "check sysst fail, reg_val=0x%04x, check:0x%x", - reg_val, AW88261_BIT_SYSST_CHECK); + & AW88261_BIT_PLL_CHECK; + if (check_val != AW88261_BIT_PLL_CHECK) { + dev_dbg(aw_dev->dev, "check sysst fail, reg_val=0x%04x, check:0x%x", + reg_val, AW88261_BIT_PLL_CHECK); usleep_range(AW88261_2000_US, AW88261_2000_US + 10); } else { return 0; @@ -549,7 +549,7 @@ static int aw88261_dev_start(struct aw88261 *aw88261) int ret; if (aw_dev->status == AW88261_DEV_PW_ON) { - dev_info(aw_dev->dev, "already power on"); + dev_dbg(aw_dev->dev, "already power on"); return 0; } @@ -559,7 +559,7 @@ static int aw88261_dev_start(struct aw88261 *aw88261) ret = aw88261_dev_check_syspll(aw_dev); if (ret) { - dev_err(aw_dev->dev, "pll check failed cannot start"); + dev_dbg(aw_dev->dev, "pll check failed"); goto pll_check_fail; } @@ -570,7 +570,7 @@ static int aw88261_dev_start(struct aw88261 *aw88261) /* check i2s status */ ret = aw88261_dev_check_sysst(aw_dev); if (ret) { - dev_err(aw_dev->dev, "sysst check failed"); + dev_dbg(aw_dev->dev, "sysst check failed"); goto sysst_check_fail; } @@ -671,18 +671,22 @@ static void aw88261_start_pa(struct aw88261 *aw88261) for (i = 0; i < AW88261_START_RETRIES; i++) { ret = aw88261_reg_update(aw88261, aw88261->phase_sync); if (ret) { - dev_err(aw88261->aw_pa->dev, "fw update failed, cnt:%d\n", i); + dev_dbg(aw88261->aw_pa->dev, + "aw88261_reg_update failed, cnt:%d, ret:%d\n", i, ret); continue; } ret = aw88261_dev_start(aw88261); if (ret) { - dev_err(aw88261->aw_pa->dev, "aw88261 device start failed. retry = %d", i); + dev_dbg(aw88261->aw_pa->dev, + "aw88261_dev_start failed, cnt:%d, ret:%d\n", i, ret); continue; } else { - dev_info(aw88261->aw_pa->dev, "start success\n"); + dev_dbg(aw88261->aw_pa->dev, "start success\n"); break; } } + if (ret != 0) + dev_err(aw88261->aw_pa->dev, "start failure (%d)\n", ret); } static void aw88261_startup_work(struct work_struct *work) @@ -1198,7 +1202,7 @@ static int aw88261_init(struct aw88261 **aw88261, struct i2c_client *i2c, struct return ret; } if (chip_id != AW88261_CHIP_ID) { - dev_err(&i2c->dev, "unsupported device"); + dev_err(&i2c->dev, "unsupported device id = %x", chip_id); return -ENXIO; } diff --git a/sound/soc/codecs/aw88261.h b/sound/soc/codecs/aw88261.h index 734d0f93ced9f..7863501086746 100644 --- a/sound/soc/codecs/aw88261.h +++ b/sound/soc/codecs/aw88261.h @@ -181,12 +181,6 @@ AW88261_OTHS_OT_VALUE | \ AW88261_PLLS_LOCKED_VALUE)) -#define AW88261_BIT_SYSST_CHECK \ - (AW88261_BSTS_FINISHED_VALUE | \ - AW88261_SWS_SWITCHING_VALUE | \ - AW88261_CLKS_STABLE_VALUE | \ - AW88261_PLLS_LOCKED_VALUE) - #define AW88261_ULS_HMUTE_START_BIT (14) #define AW88261_ULS_HMUTE_BITS_LEN (1) #define AW88261_ULS_HMUTE_MASK \ diff --git a/sound/soc/codecs/bt-sco.c b/sound/soc/codecs/bt-sco.c index 3afcef2dfa352..c0bf45b76cb8c 100644 --- a/sound/soc/codecs/bt-sco.c +++ b/sound/soc/codecs/bt-sco.c @@ -17,11 +17,17 @@ static const struct snd_soc_dapm_widget bt_sco_widgets[] = { SND_SOC_NOPM, 0, 0), SND_SOC_DAPM_AIF_OUT("BT_SCO_TX", "Capture", 0, SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_IN("BT_SCO_RX_WB", "WB Playback", 0, + SND_SOC_NOPM, 0, 0), + SND_SOC_DAPM_AIF_OUT("BT_SCO_TX_WB", "WB Capture", 0, + SND_SOC_NOPM, 0, 0), }; static const struct snd_soc_dapm_route bt_sco_routes[] = { { "BT_SCO_TX", NULL, "RX" }, { "TX", NULL, "BT_SCO_RX" }, + { "BT_SCO_TX_WB", NULL, "RX" }, + { "TX", NULL, "BT_SCO_RX_WB" }, }; static struct snd_soc_dai_driver bt_sco_dai[] = { @@ -45,14 +51,14 @@ static struct snd_soc_dai_driver bt_sco_dai[] = { { .name = "bt-sco-pcm-wb", .playback = { - .stream_name = "Playback", + .stream_name = "WB Playback", .channels_min = 1, .channels_max = 1, .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000, .formats = SNDRV_PCM_FMTBIT_S16_LE, }, .capture = { - .stream_name = "Capture", + .stream_name = "WB Capture", .channels_min = 1, .channels_max = 1, .rates = SNDRV_PCM_RATE_8000 | SNDRV_PCM_RATE_16000, diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c index c927592f90c9f..af9dad199084a 100644 --- a/sound/soc/codecs/cs35l33.c +++ b/sound/soc/codecs/cs35l33.c @@ -40,6 +40,7 @@ struct cs35l33_private { struct regmap *regmap; struct gpio_desc *reset_gpio; bool amp_cal; + bool irq_requested; int mclk_int; struct regulator_bulk_data core_supplies[2]; int num_core_supplies; @@ -881,6 +882,9 @@ static int cs35l33_runtime_resume(struct device *dev) goto err; } + if (cs35l33->irq_requested) + enable_irq(to_i2c_client(dev)->irq); + return 0; err: @@ -900,6 +904,10 @@ static int cs35l33_runtime_suspend(struct device *dev) /* redo the calibration in next power up */ cs35l33->amp_cal = false; + /* Drain and block the threaded IRQ before cache_only/power-off. */ + if (cs35l33->irq_requested) + disable_irq(to_i2c_client(dev)->irq); + regcache_cache_only(cs35l33->regmap, true); regcache_mark_dirty(cs35l33->regmap); regulator_bulk_disable(cs35l33->num_core_supplies, @@ -1154,10 +1162,12 @@ static int cs35l33_i2c_probe(struct i2c_client *i2c_client) } ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL, - cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "cs35l33", cs35l33); + cs35l33_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "cs35l33", cs35l33); if (ret != 0) dev_warn(&i2c_client->dev, "Failed to request IRQ: %d\n", ret); + else + cs35l33->irq_requested = true; /* We could issue !RST or skip it based on AMP topology */ cs35l33->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c index a5a8075598ffb..35bc38132f236 100644 --- a/sound/soc/codecs/cs35l34.c +++ b/sound/soc/codecs/cs35l34.c @@ -45,6 +45,7 @@ struct cs35l34_private { int num_core_supplies; int mclk_int; bool tdm_mode; + bool irq_requested; struct gpio_desc *reset_gpio; /* Active-low reset GPIO */ }; @@ -1032,10 +1033,12 @@ static int cs35l34_i2c_probe(struct i2c_client *i2c_client) } ret = devm_request_threaded_irq(&i2c_client->dev, i2c_client->irq, NULL, - cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, - "cs35l34", cs35l34); + cs35l34_irq_thread, IRQF_ONESHOT | IRQF_TRIGGER_LOW, + "cs35l34", cs35l34); if (ret != 0) dev_err(&i2c_client->dev, "Failed to request IRQ: %d\n", ret); + else + cs35l34->irq_requested = true; cs35l34->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev, "reset", GPIOD_OUT_LOW); @@ -1140,6 +1143,9 @@ static int cs35l34_runtime_resume(struct device *dev) dev_err(dev, "Failed to restore register cache\n"); goto err; } + + if (cs35l34->irq_requested) + enable_irq(to_i2c_client(dev)->irq); return 0; err: regcache_cache_only(cs35l34->regmap, true); @@ -1153,6 +1159,10 @@ static int cs35l34_runtime_suspend(struct device *dev) { struct cs35l34_private *cs35l34 = dev_get_drvdata(dev); + /* Drain and block the threaded IRQ before cache_only/power-off. */ + if (cs35l34->irq_requested) + disable_irq(to_i2c_client(dev)->irq); + regcache_cache_only(cs35l34->regmap, true); regcache_mark_dirty(cs35l34->regmap); diff --git a/sound/soc/codecs/cs35l41-lib.c b/sound/soc/codecs/cs35l41-lib.c index 1702f26049d37..b0fa80705be7d 100644 --- a/sound/soc/codecs/cs35l41-lib.c +++ b/sound/soc/codecs/cs35l41-lib.c @@ -25,9 +25,9 @@ static const struct reg_default cs35l41_reg[] = { { CS35L41_GPIO_PAD_CONTROL, 0x00000000 }, { CS35L41_GLOBAL_CLK_CTRL, 0x00000003 }, { CS35L41_TST_FS_MON0, 0x00020016 }, + { CS35L41_BSTCVRT_PEAK_CUR, 0x0000004A }, { CS35L41_BSTCVRT_COEFF, 0x00002424 }, { CS35L41_BSTCVRT_SLOPE_LBST, 0x00007500 }, - { CS35L41_BSTCVRT_PEAK_CUR, 0x0000004A }, { CS35L41_SP_ENABLES, 0x00000000 }, { CS35L41_SP_RATE_CTRL, 0x00000028 }, { CS35L41_SP_FORMAT, 0x18180200 }, diff --git a/sound/soc/codecs/cs35l45-tables.c b/sound/soc/codecs/cs35l45-tables.c index d2ecc7b3f6193..764dbaa350422 100644 --- a/sound/soc/codecs/cs35l45-tables.c +++ b/sound/soc/codecs/cs35l45-tables.c @@ -66,22 +66,6 @@ static const struct reg_default cs35l45_defaults[] = { { CS35L45_ASPTX3_INPUT, 0x00000020 }, { CS35L45_ASPTX4_INPUT, 0x00000028 }, { CS35L45_ASPTX5_INPUT, 0x00000048 }, - { CS35L45_DSP1_RX1_RATE, 0x00000001 }, - { CS35L45_DSP1_RX2_RATE, 0x00000001 }, - { CS35L45_DSP1_RX3_RATE, 0x00000001 }, - { CS35L45_DSP1_RX4_RATE, 0x00000001 }, - { CS35L45_DSP1_RX5_RATE, 0x00000001 }, - { CS35L45_DSP1_RX6_RATE, 0x00000001 }, - { CS35L45_DSP1_RX7_RATE, 0x00000001 }, - { CS35L45_DSP1_RX8_RATE, 0x00000001 }, - { CS35L45_DSP1_TX1_RATE, 0x00000001 }, - { CS35L45_DSP1_TX2_RATE, 0x00000001 }, - { CS35L45_DSP1_TX3_RATE, 0x00000001 }, - { CS35L45_DSP1_TX4_RATE, 0x00000001 }, - { CS35L45_DSP1_TX5_RATE, 0x00000001 }, - { CS35L45_DSP1_TX6_RATE, 0x00000001 }, - { CS35L45_DSP1_TX7_RATE, 0x00000001 }, - { CS35L45_DSP1_TX8_RATE, 0x00000001 }, { CS35L45_DSP1RX1_INPUT, 0x00000008 }, { CS35L45_DSP1RX2_INPUT, 0x00000009 }, { CS35L45_DSP1RX3_INPUT, 0x00000018 }, @@ -114,6 +98,22 @@ static const struct reg_default cs35l45_defaults[] = { { CS35L45_GPIO1_CTRL1, 0x81000001 }, { CS35L45_GPIO2_CTRL1, 0x81000001 }, { CS35L45_GPIO3_CTRL1, 0x81000001 }, + { CS35L45_DSP1_RX1_RATE, 0x00000001 }, + { CS35L45_DSP1_RX2_RATE, 0x00000001 }, + { CS35L45_DSP1_RX3_RATE, 0x00000001 }, + { CS35L45_DSP1_RX4_RATE, 0x00000001 }, + { CS35L45_DSP1_RX5_RATE, 0x00000001 }, + { CS35L45_DSP1_RX6_RATE, 0x00000001 }, + { CS35L45_DSP1_RX7_RATE, 0x00000001 }, + { CS35L45_DSP1_RX8_RATE, 0x00000001 }, + { CS35L45_DSP1_TX1_RATE, 0x00000001 }, + { CS35L45_DSP1_TX2_RATE, 0x00000001 }, + { CS35L45_DSP1_TX3_RATE, 0x00000001 }, + { CS35L45_DSP1_TX4_RATE, 0x00000001 }, + { CS35L45_DSP1_TX5_RATE, 0x00000001 }, + { CS35L45_DSP1_TX6_RATE, 0x00000001 }, + { CS35L45_DSP1_TX7_RATE, 0x00000001 }, + { CS35L45_DSP1_TX8_RATE, 0x00000001 }, }; static bool cs35l45_readable_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/cs35l56-i2c.c b/sound/soc/codecs/cs35l56-i2c.c index 0492ddc4102d8..8259714d8b269 100644 --- a/sound/soc/codecs/cs35l56-i2c.c +++ b/sound/soc/codecs/cs35l56-i2c.c @@ -55,9 +55,7 @@ static int cs35l56_i2c_probe(struct i2c_client *client) if (ret != 0) return ret; - ret = cs35l56_init(cs35l56); - if (ret == 0) - ret = cs35l56_irq_request(&cs35l56->base, client->irq); + ret = cs35l56_irq_request(&cs35l56->base, client->irq); if (ret < 0) cs35l56_remove(cs35l56); diff --git a/sound/soc/codecs/cs35l56-spi.c b/sound/soc/codecs/cs35l56-spi.c index 9bc9b7c98390d..b1eb924a5b6cc 100644 --- a/sound/soc/codecs/cs35l56-spi.c +++ b/sound/soc/codecs/cs35l56-spi.c @@ -44,9 +44,7 @@ static int cs35l56_spi_probe(struct spi_device *spi) if (ret != 0) return ret; - ret = cs35l56_init(cs35l56); - if (ret == 0) - ret = cs35l56_irq_request(&cs35l56->base, spi->irq); + ret = cs35l56_irq_request(&cs35l56->base, spi->irq); if (ret < 0) cs35l56_remove(cs35l56); diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c index 7b57ef2336ea8..e3334b2d84532 100644 --- a/sound/soc/codecs/cs35l56.c +++ b/sound/soc/codecs/cs35l56.c @@ -1437,9 +1437,19 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56) goto err; } - ret = devm_snd_soc_register_component(cs35l56->base.dev, - &soc_component_dev_cs35l56, - cs35l56_dai, ARRAY_SIZE(cs35l56_dai)); + /* + * On SoundWire the cs35l56_init() cannot be run until after the + * device has been enumerated by the SoundWire core. + */ + if (!cs35l56->sdw_peripheral) { + ret = cs35l56_init(cs35l56); + if (ret) + goto err_remove_wm_adsp; + } + + ret = snd_soc_register_component(cs35l56->base.dev, + &soc_component_dev_cs35l56, + cs35l56_dai, ARRAY_SIZE(cs35l56_dai)); if (ret < 0) { dev_err_probe(cs35l56->base.dev, ret, "Register codec failed\n"); goto err_remove_wm_adsp; @@ -1451,6 +1461,11 @@ int cs35l56_common_probe(struct cs35l56_private *cs35l56) wm_adsp2_remove(&cs35l56->dsp); err: + if (pm_runtime_enabled(cs35l56->base.dev)) { + pm_runtime_dont_use_autosuspend(cs35l56->base.dev); + pm_runtime_disable(cs35l56->base.dev); + } + gpiod_set_value_cansleep(cs35l56->base.reset_gpio, 0); regulator_bulk_disable(ARRAY_SIZE(cs35l56->supplies), cs35l56->supplies); @@ -1533,7 +1548,7 @@ int cs35l56_init(struct cs35l56_private *cs35l56) return dev_err_probe(cs35l56->base.dev, ret, "Failed to write ASP1_CONTROL3\n"); cs35l56->base.init_done = true; - complete(&cs35l56->init_completion); + complete_all(&cs35l56->init_completion); return 0; } @@ -1541,6 +1556,8 @@ EXPORT_SYMBOL_NS_GPL(cs35l56_init, "SND_SOC_CS35L56_CORE"); void cs35l56_remove(struct cs35l56_private *cs35l56) { + snd_soc_unregister_component(cs35l56->base.dev); + cs35l56->base.init_done = false; /* diff --git a/sound/soc/codecs/cs4265.c b/sound/soc/codecs/cs4265.c index 3f759c13d6d12..0bb1aa0c6cd06 100644 --- a/sound/soc/codecs/cs4265.c +++ b/sound/soc/codecs/cs4265.c @@ -46,11 +46,11 @@ static const struct reg_default cs4265_reg_defaults[] = { { CS4265_DAC_CHA_VOL, 0x00 }, { CS4265_DAC_CHB_VOL, 0x00 }, { CS4265_DAC_CTL2, 0xC0 }, - { CS4265_SPDIF_CTL1, 0x00 }, - { CS4265_SPDIF_CTL2, 0x00 }, { CS4265_INT_MASK, 0x00 }, { CS4265_STATUS_MODE_MSB, 0x00 }, { CS4265_STATUS_MODE_LSB, 0x00 }, + { CS4265_SPDIF_CTL1, 0x00 }, + { CS4265_SPDIF_CTL2, 0x00 }, }; static bool cs4265_readable_register(struct device *dev, unsigned int reg) diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c index ecba6c7952384..36fb67059e065 100644 --- a/sound/soc/codecs/cs42l43-jack.c +++ b/sound/soc/codecs/cs42l43-jack.c @@ -310,6 +310,7 @@ irqreturn_t cs42l43_bias_detect_clamp(int irq, void *data) #define CS42L43_JACK_ABSENT 0x0 #define CS42L43_JACK_OPTICAL (SND_JACK_MECHANICAL | SND_JACK_AVOUT) +#define CS42L43_JACK_MICROPHONE (SND_JACK_MECHANICAL | SND_JACK_MICROPHONE) #define CS42L43_JACK_HEADPHONE (SND_JACK_MECHANICAL | SND_JACK_HEADPHONE) #define CS42L43_JACK_HEADSET (SND_JACK_MECHANICAL | SND_JACK_HEADSET) #define CS42L43_JACK_LINEOUT (SND_JACK_MECHANICAL | SND_JACK_LINEOUT) @@ -871,7 +872,7 @@ static const struct cs42l43_jack_override_mode { .hsdet_mode = CS42L43_JACK_3_POLE_SWITCHES, .mic_ctrl = (0x3 << CS42L43_JACK_STEREO_CONFIG_SHIFT) | CS42L43_HS1_BIAS_EN_MASK | CS42L43_HS2_BIAS_EN_MASK, - .report = CS42L43_JACK_LINEIN, + .report = CS42L43_JACK_MICROPHONE, }, [CS42L43_JACK_RAW_OPTICAL] = { .hsdet_mode = CS42L43_JACK_3_POLE_SWITCHES, diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c index afd8edf10fdc8..b4d93b9404bb7 100644 --- a/sound/soc/codecs/hdac_hda.c +++ b/sound/soc/codecs/hdac_hda.c @@ -644,10 +644,8 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev) &hdac_hda_codec, hdac_hda_dais, ARRAY_SIZE(hdac_hda_dais)); - if (ret < 0) { + if (ret < 0) dev_err(&hdev->dev, "%s: failed to register HDA codec %d\n", __func__, ret); - return ret; - } snd_hdac_ext_bus_link_put(hdev->bus, hlink); diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c index e1933f733af10..fcf2797dd31bb 100644 --- a/sound/soc/codecs/hdmi-codec.c +++ b/sound/soc/codecs/hdmi-codec.c @@ -362,6 +362,10 @@ static int hdmi_codec_get_ch_alloc_table_idx(struct hdmi_codec_priv *hcp, unsigned long spk_mask; const struct hdmi_codec_cea_spk_alloc *cap = hdmi_codec_channel_alloc; + /* Index 0 is CA 0x00, which is valid for any sink */ + if (channels <= 2) + return 0; + spk_alloc = drm_eld_get_spk_alloc(hcp->eld); spk_mask = hdmi_codec_spk_mask_from_alloc(spk_alloc); diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c index c7d4dc553e6aa..8cf8a9b687961 100644 --- a/sound/soc/codecs/lpass-tx-macro.c +++ b/sound/soc/codecs/lpass-tx-macro.c @@ -1075,7 +1075,7 @@ static int tx_macro_dec_mode_get(struct snd_kcontrol *kcontrol, struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; int path = e->shift_l; - ucontrol->value.integer.value[0] = tx->dec_mode[path]; + ucontrol->value.enumerated.item[0] = tx->dec_mode[path]; return 0; } @@ -1084,7 +1084,7 @@ static int tx_macro_dec_mode_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); - int value = ucontrol->value.integer.value[0]; + int value = ucontrol->value.enumerated.item[0]; struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; int path = e->shift_l; struct tx_macro *tx = snd_soc_component_get_drvdata(component); diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c index 38faa9074ca3e..8350b02b575fb 100644 --- a/sound/soc/codecs/lpass-wsa-macro.c +++ b/sound/soc/codecs/lpass-wsa-macro.c @@ -2167,7 +2167,7 @@ static int wsa_macro_ear_spkr_pa_gain_get(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); struct wsa_macro *wsa = snd_soc_component_get_drvdata(component); - ucontrol->value.integer.value[0] = wsa->ear_spkr_gain; + ucontrol->value.enumerated.item[0] = wsa->ear_spkr_gain; return 0; } @@ -2178,7 +2178,7 @@ static int wsa_macro_ear_spkr_pa_gain_put(struct snd_kcontrol *kcontrol, struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); struct wsa_macro *wsa = snd_soc_component_get_drvdata(component); - wsa->ear_spkr_gain = ucontrol->value.integer.value[0]; + wsa->ear_spkr_gain = ucontrol->value.enumerated.item[0]; return 0; } @@ -2192,7 +2192,7 @@ static int wsa_macro_rx_mux_get(struct snd_kcontrol *kcontrol, snd_soc_dapm_to_component(widget->dapm); struct wsa_macro *wsa = snd_soc_component_get_drvdata(component); - ucontrol->value.integer.value[0] = + ucontrol->value.enumerated.item[0] = wsa->rx_port_value[widget->shift]; return 0; } @@ -2206,7 +2206,7 @@ static int wsa_macro_rx_mux_put(struct snd_kcontrol *kcontrol, snd_soc_dapm_to_component(widget->dapm); struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; struct snd_soc_dapm_update *update = NULL; - u32 rx_port_value = ucontrol->value.integer.value[0]; + u32 rx_port_value = ucontrol->value.enumerated.item[0]; u32 bit_input; u32 aif_rst; unsigned int dai_id; diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c index 5aff5a459a433..830caf62d0fa8 100644 --- a/sound/soc/codecs/max98090.c +++ b/sound/soc/codecs/max98090.c @@ -2390,8 +2390,9 @@ static int max98090_probe(struct snd_soc_component *component) dev_dbg(component->dev, "max98090_probe\n"); max98090->mclk = devm_clk_get(component->dev, "mclk"); - if (PTR_ERR(max98090->mclk) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (IS_ERR(max98090->mclk)) + if (PTR_ERR(max98090->mclk) == -EPROBE_DEFER) + return -EPROBE_DEFER; max98090->component = component; diff --git a/sound/soc/codecs/max98095.c b/sound/soc/codecs/max98095.c index cfb63fe69267b..ad2587d2df819 100644 --- a/sound/soc/codecs/max98095.c +++ b/sound/soc/codecs/max98095.c @@ -1984,8 +1984,9 @@ static int max98095_probe(struct snd_soc_component *component) int ret = 0; max98095->mclk = devm_clk_get(component->dev, "mclk"); - if (PTR_ERR(max98095->mclk) == -EPROBE_DEFER) - return -EPROBE_DEFER; + if (IS_ERR(max98095->mclk)) + if (PTR_ERR(max98095->mclk) == -EPROBE_DEFER) + return -EPROBE_DEFER; /* reset the codec, the DSP core, and disable all interrupts */ max98095_reset(component); diff --git a/sound/soc/codecs/mt6351.c b/sound/soc/codecs/mt6351.c index 2a5e963fb2b57..401cb0b48c972 100644 --- a/sound/soc/codecs/mt6351.c +++ b/sound/soc/codecs/mt6351.c @@ -1479,6 +1479,7 @@ static const struct of_device_id mt6351_of_match[] = { {.compatible = "mediatek,mt6351-sound",}, {} }; +MODULE_DEVICE_TABLE(of, mt6351_of_match); static struct platform_driver mt6351_codec_driver = { .driver = { diff --git a/sound/soc/codecs/nau8821.c b/sound/soc/codecs/nau8821.c index dfb9630bffe29..53a38eb4eb0e1 100644 --- a/sound/soc/codecs/nau8821.c +++ b/sound/soc/codecs/nau8821.c @@ -1269,6 +1269,14 @@ static int nau8821_component_probe(struct snd_soc_component *component) return 0; } +static void nau8821_component_remove(struct snd_soc_component *component) +{ + struct nau8821 *nau8821 = snd_soc_component_get_drvdata(component); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); +}; + /** * nau8821_calc_fll_param - Calculate FLL parameters. * @fll_in: external clock provided to codec. @@ -1603,6 +1611,10 @@ static int __maybe_unused nau8821_suspend(struct snd_soc_component *component) if (nau8821->irq) disable_irq(nau8821->irq); + + if (nau8821->jdet_active) + cancel_delayed_work_sync(&nau8821->jdet_work); + snd_soc_component_force_bias_level(component, SND_SOC_BIAS_OFF); /* Power down codec power; don't support button wakeup */ snd_soc_component_disable_pin(component, "MICBIAS"); @@ -1627,6 +1639,7 @@ static int __maybe_unused nau8821_resume(struct snd_soc_component *component) static const struct snd_soc_component_driver nau8821_component_driver = { .probe = nau8821_component_probe, + .remove = nau8821_component_remove, .set_sysclk = nau8821_set_sysclk, .set_pll = nau8821_set_fll, .set_bias_level = nau8821_set_bias_level, diff --git a/sound/soc/codecs/pcm3168a.c b/sound/soc/codecs/pcm3168a.c index 0363869aed63f..9ccb2c7d904fb 100644 --- a/sound/soc/codecs/pcm3168a.c +++ b/sound/soc/codecs/pcm3168a.c @@ -834,15 +834,6 @@ int pcm3168a_probe(struct device *dev, struct regmap *regmap) } EXPORT_SYMBOL_GPL(pcm3168a_probe); -static void pcm3168a_disable(struct device *dev) -{ - struct pcm3168a_priv *pcm3168a = dev_get_drvdata(dev); - - regulator_bulk_disable(ARRAY_SIZE(pcm3168a->supplies), - pcm3168a->supplies); - clk_disable_unprepare(pcm3168a->scki); -} - void pcm3168a_remove(struct device *dev) { struct pcm3168a_priv *pcm3168a = dev_get_drvdata(dev); @@ -854,10 +845,12 @@ void pcm3168a_remove(struct device *dev) * The asserted level of GPIO_ACTIVE_LOW is LOW. */ gpiod_set_value_cansleep(pcm3168a->gpio_rst, 1); + pm_runtime_disable(dev); -#ifndef CONFIG_PM - pcm3168a_disable(dev); -#endif + if (!pm_runtime_status_suspended(dev)) { + regulator_bulk_disable(ARRAY_SIZE(pcm3168a->supplies), pcm3168a->supplies); + clk_disable_unprepare(pcm3168a->scki); + } } EXPORT_SYMBOL_GPL(pcm3168a_remove); @@ -912,13 +905,15 @@ static int pcm3168a_rt_suspend(struct device *dev) regcache_cache_only(pcm3168a->regmap, true); - pcm3168a_disable(dev); + regulator_bulk_disable(ARRAY_SIZE(pcm3168a->supplies), pcm3168a->supplies); + clk_disable_unprepare(pcm3168a->scki); return 0; } EXPORT_GPL_DEV_PM_OPS(pcm3168a_pm_ops) = { RUNTIME_PM_OPS(pcm3168a_rt_suspend, pcm3168a_rt_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) }; MODULE_DESCRIPTION("PCM3168A codec driver"); diff --git a/sound/soc/codecs/rk3328_codec.c b/sound/soc/codecs/rk3328_codec.c index 9697aefc6e030..5871b5a819757 100644 --- a/sound/soc/codecs/rk3328_codec.c +++ b/sound/soc/codecs/rk3328_codec.c @@ -425,7 +425,6 @@ static int rk3328_platform_probe(struct platform_device *pdev) struct rk3328_codec_priv *rk3328; struct regmap *grf; void __iomem *base; - int ret = 0; rk3328 = devm_kzalloc(&pdev->dev, sizeof(*rk3328), GFP_KERNEL); if (!rk3328) @@ -441,14 +440,13 @@ static int rk3328_platform_probe(struct platform_device *pdev) regmap_write(grf, RK3328_GRF_SOC_CON2, (BIT(14) << 16 | BIT(14))); - ret = of_property_read_u32(rk3328_np, "spk-depop-time-ms", - &rk3328->spk_depop_time); - if (ret < 0) { + if (of_property_read_u32(rk3328_np, "spk-depop-time-ms", + &rk3328->spk_depop_time)) { dev_info(&pdev->dev, "spk_depop_time use default value.\n"); rk3328->spk_depop_time = 200; } - rk3328->mute = gpiod_get_optional(&pdev->dev, "mute", GPIOD_OUT_HIGH); + rk3328->mute = devm_gpiod_get_optional(&pdev->dev, "mute", GPIOD_OUT_HIGH); if (IS_ERR(rk3328->mute)) return PTR_ERR(rk3328->mute); /* @@ -461,57 +459,31 @@ static int rk3328_platform_probe(struct platform_device *pdev) regmap_write(grf, RK3328_GRF_SOC_CON10, BIT(17) | BIT(1)); } - rk3328->mclk = devm_clk_get(&pdev->dev, "mclk"); + rk3328->mclk = devm_clk_get_enabled(&pdev->dev, "mclk"); if (IS_ERR(rk3328->mclk)) return PTR_ERR(rk3328->mclk); - ret = clk_prepare_enable(rk3328->mclk); - if (ret) - return ret; clk_set_rate(rk3328->mclk, INITIAL_FREQ); - rk3328->pclk = devm_clk_get(&pdev->dev, "pclk"); - if (IS_ERR(rk3328->pclk)) { - dev_err(&pdev->dev, "can't get acodec pclk\n"); - ret = PTR_ERR(rk3328->pclk); - goto err_unprepare_mclk; - } - - ret = clk_prepare_enable(rk3328->pclk); - if (ret < 0) { - dev_err(&pdev->dev, "failed to enable acodec pclk\n"); - goto err_unprepare_mclk; - } + rk3328->pclk = devm_clk_get_enabled(&pdev->dev, "pclk"); + if (IS_ERR(rk3328->pclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(rk3328->pclk), + "failed to get or enable acodec pclk\n"); base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(base)) { - ret = PTR_ERR(base); - goto err_unprepare_pclk; - } + if (IS_ERR(base)) + return PTR_ERR(base); rk3328->regmap = devm_regmap_init_mmio(&pdev->dev, base, &rk3328_codec_regmap_config); - if (IS_ERR(rk3328->regmap)) { - ret = PTR_ERR(rk3328->regmap); - goto err_unprepare_pclk; - } + if (IS_ERR(rk3328->regmap)) + return PTR_ERR(rk3328->regmap); platform_set_drvdata(pdev, rk3328); - ret = devm_snd_soc_register_component(&pdev->dev, &soc_codec_rk3328, + return devm_snd_soc_register_component(&pdev->dev, &soc_codec_rk3328, rk3328_dai, ARRAY_SIZE(rk3328_dai)); - if (ret) - goto err_unprepare_pclk; - - return 0; - -err_unprepare_pclk: - clk_disable_unprepare(rk3328->pclk); - -err_unprepare_mclk: - clk_disable_unprepare(rk3328->mclk); - return ret; } static const struct of_device_id rk3328_codec_of_match[] __maybe_unused = { diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 29a403526cd9e..428cc6850dc5e 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -3497,6 +3497,10 @@ static int rt5645_probe(struct snd_soc_component *component) if (!rt5645->eq_param) return -ENOMEM; + /* no machine driver to call rt5645_set_jack_detect(), so detect here */ + if (!rt5645->pdata.jd_mode && rt5645->gpiod_hp_det) + rt5645_irq(0, rt5645); + return 0; } diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c index 44543c0da1772..f7bd793e3e672 100644 --- a/sound/soc/codecs/rt700-sdw.c +++ b/sound/soc/codecs/rt700-sdw.c @@ -459,10 +459,8 @@ static int rt700_sdw_remove(struct sdw_slave *slave) { struct rt700_priv *rt700 = dev_get_drvdata(&slave->dev); - if (rt700->hw_init) { - cancel_delayed_work_sync(&rt700->jack_detect_work); - cancel_delayed_work_sync(&rt700->jack_btn_check_work); - } + cancel_delayed_work_sync(&rt700->jack_detect_work); + cancel_delayed_work_sync(&rt700->jack_btn_check_work); pm_runtime_disable(&slave->dev); diff --git a/sound/soc/codecs/rt712-sdca.c b/sound/soc/codecs/rt712-sdca.c index 0ebaae426e73b..d8b40663fa92f 100644 --- a/sound/soc/codecs/rt712-sdca.c +++ b/sound/soc/codecs/rt712-sdca.c @@ -1849,6 +1849,15 @@ static void rt712_sdca_vb_io_init(struct rt712_sdca_priv *rt712) } } +static void rt712_sdca_reset(struct rt712_sdca_priv *rt712) +{ + rt712_sdca_index_update_bits(rt712, RT712_VENDOR_REG, + RT712_PARA_VERB_CTL, RT712_HIDDEN_REG_SW_RESET, + RT712_HIDDEN_REG_SW_RESET); + rt712_sdca_index_update_bits(rt712, RT712_VENDOR_HDA_CTL, + RT712_HDA_LEGACY_RESET_CTL, 0x1, 0x1); +} + int rt712_sdca_io_init(struct device *dev, struct sdw_slave *slave) { struct rt712_sdca_priv *rt712 = dev_get_drvdata(dev); @@ -1876,6 +1885,8 @@ int rt712_sdca_io_init(struct device *dev, struct sdw_slave *slave) pm_runtime_get_noresume(&slave->dev); + rt712_sdca_reset(rt712); + rt712_sdca_index_read(rt712, RT712_VENDOR_REG, RT712_JD_PRODUCT_NUM, &val); rt712->hw_id = (val & 0xf000) >> 12; rt712->version_id = (val & 0x0f00) >> 8; diff --git a/sound/soc/codecs/tas2562.c b/sound/soc/codecs/tas2562.c index 8e00dcc09d0c2..118984e30d7b9 100644 --- a/sound/soc/codecs/tas2562.c +++ b/sound/soc/codecs/tas2562.c @@ -32,15 +32,16 @@ static const unsigned int float_vol_db_lookup[] = { 0x00000d43, 0x000010b2, 0x00001505, 0x00001a67, 0x00002151, 0x000029f1, 0x000034cd, 0x00004279, 0x000053af, 0x0000695b, -0x0000695b, 0x0000a6fa, 0x0000d236, 0x000108a4, 0x00014d2a, +0x000084a3, 0x0000a6fa, 0x0000d236, 0x000108a4, 0x00014d2a, 0x0001a36e, 0x00021008, 0x000298c0, 0x000344df, 0x00041d8f, 0x00052e5a, 0x000685c8, 0x00083621, 0x000a566d, 0x000d03a7, 0x0010624d, 0x0014a050, 0x0019f786, 0x0020b0bc, 0x0029279d, 0x0033cf8d, 0x004139d3, 0x00521d50, 0x00676044, 0x0082248a, 0x00a3d70a, 0x00ce4328, 0x0103ab3d, 0x0146e75d, 0x019b8c27, 0x02061b89, 0x028c423f, 0x03352529, 0x0409c2b0, 0x05156d68, -0x080e9f96, 0x0a24b062, 0x0cc509ab, 0x10137987, 0x143d1362, -0x197a967f, 0x2013739e, 0x28619ae9, 0x32d64617, 0x40000000 +0x06666666, 0x080e9f96, 0x0a24b062, 0x0cc509ab, 0x10137987, +0x143d1362, 0x197a967f, 0x2013739e, 0x28619ae9, 0x32d64617, +0x40000000 }; struct tas2562_data { @@ -471,24 +472,36 @@ static int tas2562_volume_control_put(struct snd_kcontrol *kcontrol, { struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol); struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component); - int ret; + int ret, index; u32 reg_val; - reg_val = float_vol_db_lookup[ucontrol->value.integer.value[0]/2]; - ret = snd_soc_component_write(component, TAS2562_DVC_CFG4, - (reg_val & 0xff)); - if (ret) - return ret; - ret = snd_soc_component_write(component, TAS2562_DVC_CFG3, - ((reg_val >> 8) & 0xff)); + index = ucontrol->value.integer.value[0] / 2; + if (index < 0 || index >= ARRAY_SIZE(float_vol_db_lookup)) + return -EINVAL; + + reg_val = float_vol_db_lookup[index]; + + /* + * The device applies the 32-bit coefficient to the playback path on + * the write to DVC_CFG4 (the LSB, book 0 page 2 reg 0x0F), so the + * bytes must be written MSB first and DVC_CFG4 last. Writing CFG4 + * first latches a mix of the previous coefficient's upper bytes and + * the new LSB instead of the requested value. + */ + ret = snd_soc_component_write(component, TAS2562_DVC_CFG1, + ((reg_val >> 24) & 0xff)); if (ret) return ret; ret = snd_soc_component_write(component, TAS2562_DVC_CFG2, ((reg_val >> 16) & 0xff)); if (ret) return ret; - ret = snd_soc_component_write(component, TAS2562_DVC_CFG1, - ((reg_val >> 24) & 0xff)); + ret = snd_soc_component_write(component, TAS2562_DVC_CFG3, + ((reg_val >> 8) & 0xff)); + if (ret) + return ret; + ret = snd_soc_component_write(component, TAS2562_DVC_CFG4, + (reg_val & 0xff)); if (ret) return ret; @@ -675,11 +688,12 @@ static int tas2562_parse_dt(struct tas2562_data *tas2562) if (tas2562->sdz_gpio == NULL) { tas2562->sdz_gpio = devm_gpiod_get_optional(dev, "shut-down", GPIOD_OUT_HIGH); - if (IS_ERR(tas2562->sdz_gpio)) + if (IS_ERR(tas2562->sdz_gpio)) { if (PTR_ERR(tas2562->sdz_gpio) == -EPROBE_DEFER) return -EPROBE_DEFER; - tas2562->sdz_gpio = NULL; + tas2562->sdz_gpio = NULL; + } } if (tas2562->model_id == TAS2110) diff --git a/sound/soc/codecs/tas2781-fmwlib.c b/sound/soc/codecs/tas2781-fmwlib.c index 78fd0a5dc6f22..2f6522f76df9b 100644 --- a/sound/soc/codecs/tas2781-fmwlib.c +++ b/sound/soc/codecs/tas2781-fmwlib.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1082,13 +1083,42 @@ static int tasdevice_load_block_kernel( return 0; } +static int tasdevice_fw_strnlen(const struct firmware *fmw, int offset) +{ + const u8 *start; + const u8 *nul; + size_t remaining; + size_t len; + + if (offset < 0 || offset >= fmw->size) + return -EINVAL; + + start = fmw->data + offset; + remaining = fmw->size - offset; + nul = memchr(start, '\0', remaining); + if (!nul) + return -EINVAL; + + len = nul - start; + if (len > INT_MAX) + return -EOVERFLOW; + + return len; +} + static int fw_parse_variable_hdr(struct tasdevice_priv *tas_priv, struct tasdevice_dspfw_hdr *fw_hdr, const struct firmware *fmw, int offset) { const unsigned char *buf = fmw->data; - int len = strlen((char *)&buf[offset]); + int len; + len = tasdevice_fw_strnlen(fmw, offset); + if (len < 0) { + dev_err(tas_priv->dev, "%s: Description error\n", __func__); + offset = len; + goto out; + } len++; if (offset + len + 8 > fmw->size) { @@ -1220,7 +1250,12 @@ static int fw_parse_data(struct tasdevice_fw *tas_fmw, memcpy(img_data->name, &data[offset], 64); offset += 64; - n = strlen((char *)&data[offset]); + n = tasdevice_fw_strnlen(fmw, offset); + if (n < 0) { + dev_err(tas_fmw->dev, "%s: Description error\n", __func__); + offset = n; + goto out; + } n++; if (offset + n + 2 > fmw->size) { dev_err(tas_fmw->dev, "%s: Description error\n", __func__); @@ -1293,7 +1328,12 @@ static int fw_parse_program_data(struct tasdevice_priv *tas_priv, } offset += 64; - n = strlen((char *)&buf[offset]); + n = tasdevice_fw_strnlen(fmw, offset); + if (n < 0) { + dev_err(tas_priv->dev, "Description err\n"); + offset = n; + goto out; + } /* skip '\0' and 5 unused bytes */ n += 6; if (offset + n > fmw->size) { @@ -1356,7 +1396,12 @@ static int fw_parse_configuration_data( memcpy(config->name, &data[offset], 64); offset += 64; - n = strlen((char *)&data[offset]); + n = tasdevice_fw_strnlen(fmw, offset); + if (n < 0) { + dev_err(tas_priv->dev, "Description err\n"); + offset = n; + goto out; + } n += 15; if (offset + n > fmw->size) { dev_err(tas_priv->dev, "Description err\n"); @@ -2038,7 +2083,8 @@ static int fw_parse_calibration_data(struct tasdevice_priv *tas_priv, { struct tasdevice_calibration *calibration; unsigned char *data = (unsigned char *)fmw->data; - unsigned int i, n; + unsigned int i; + int n; if (offset + 2 > fmw->size) { dev_err(tas_priv->dev, "%s: Calibrations error\n", __func__); @@ -2070,7 +2116,12 @@ static int fw_parse_calibration_data(struct tasdevice_priv *tas_priv, calibration = &(tas_fmw->calibrations[i]); offset += 64; - n = strlen((char *)&data[offset]); + n = tasdevice_fw_strnlen(fmw, offset); + if (n < 0) { + dev_err(tas_priv->dev, "Description err\n"); + offset = n; + goto out; + } /* skip '\0' and 2 unused bytes */ n += 3; if (offset + n > fmw->size) { diff --git a/sound/soc/codecs/tas2781-i2c.c b/sound/soc/codecs/tas2781-i2c.c index a3b4d2c3b4789..58f8ae016dac0 100644 --- a/sound/soc/codecs/tas2781-i2c.c +++ b/sound/soc/codecs/tas2781-i2c.c @@ -1303,8 +1303,8 @@ static void cali_reg_update(struct bulk_reg_val *p, t->sin_gn[2]); break; case TAS2781_PRM_SINEGAIN2_REG: - reg = TASDEVICE_REG(t->sin_gn[0], t->sin_gn[1], - t->sin_gn[2]); + reg = TASDEVICE_REG(t->sin_gn2[0], t->sin_gn2[1], + t->sin_gn2[2]); break; default: reg = 0; @@ -1533,8 +1533,8 @@ static ssize_t acoustic_ctl_write(struct file *file, if (src[0] > max_pkg_len && src[0] != count) { dev_err(priv->dev, "pkg(%u), max(%u), count(%u) mismatch.\n", src[0], max_pkg_len, (unsigned int)count); - ret = 0; - goto exit; + kfree(src); + return 0; } switch (src[1]) { @@ -1548,14 +1548,14 @@ static ssize_t acoustic_ctl_write(struct file *file, break; default: dev_err(priv->dev, "%s Wrong code %02x.\n", __func__, src[1]); - ret = 0; - goto exit; + kfree(src); + return 0; } if (len < 1) { dev_err(priv->dev, "pkg fmt invalid %02x.\n", len); - ret = 0; - goto exit; + kfree(src); + return 0; } for (j = 0; j < priv->ndev; j++) @@ -1565,8 +1565,8 @@ static ssize_t acoustic_ctl_write(struct file *file, } if (j >= priv->ndev) { dev_err(priv->dev, "no such device 0x%02x.\n", src[2]); - ret = 0; - goto exit; + kfree(src); + return 0; } reg = TASDEVICE_REG(src[3], src[4], src[5]); @@ -1597,7 +1597,7 @@ static ssize_t acoustic_ctl_write(struct file *file, dev_err(priv->dev, "i2c communication error.\n"); else ret = count; -exit: + kfree(src); return ret; } diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c index e273b80d033e1..b7f7b025efd37 100644 --- a/sound/soc/codecs/tas2783-sdw.c +++ b/sound/soc/codecs/tas2783-sdw.c @@ -1175,24 +1175,20 @@ static s32 tas2783_sdca_dev_resume(struct device *dev) { struct sdw_slave *slave = dev_to_sdw_dev(dev); struct tas2783_prv *tas_dev = dev_get_drvdata(dev); - unsigned long t; + int ret; - if (!slave->unattach_request) - goto regmap_sync; + ret = sdw_slave_wait_for_init(slave, TAS2783_PROBE_TIMEOUT); + if (ret) + return ret; - t = wait_for_completion_timeout(&slave->initialization_complete, - msecs_to_jiffies(TAS2783_PROBE_TIMEOUT)); - if (!t) { - dev_err(&slave->dev, "resume: initialization timed out\n"); - sdw_show_ping_status(slave->bus, true); - return -ETIMEDOUT; + regcache_cache_only(tas_dev->regmap, false); + ret = regcache_sync(tas_dev->regmap); + if (ret) { + regcache_cache_only(tas_dev->regmap, true); + regcache_mark_dirty(tas_dev->regmap); + return ret; } - slave->unattach_request = 0; - -regmap_sync: - regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); return 0; } @@ -1259,9 +1255,23 @@ static s32 tas_update_status(struct sdw_slave *slave, if (tas_dev->hw_init || tas_dev->status != SDW_SLAVE_ATTACHED) return 0; - /* updated the cache data to device */ regcache_cache_only(tas_dev->regmap, false); - regcache_sync(tas_dev->regmap); + + /* + * The device is attaching uninitialized: either this is the first + * attach, or it lost power (and with it all register and DSP state) + * while the controller was power-gated during system suspend. The + * cache still holds the pre-suspend values, and tas_io_init() below + * resets the device via TAS2783_SW_RESET anyway, so syncing it back + * is both useless and harmful: later read-modify-write updates would + * compare against stale data and skip the hardware write. + * + * Drop the cache instead, so that subsequent accesses see the real + * hardware state. Syncing after the reset is not an option either: + * the cache accepts registers for which tas2783_sdca_mbq_size() + * returns 0, and writing those back fails with -EINVAL. + */ + regcache_drop_region(tas_dev->regmap, 0, UINT_MAX); /* perform I/O transfers required for Slave initialization */ return tas_io_init(&slave->dev, slave); diff --git a/sound/soc/dwc/dwc-i2s.c b/sound/soc/dwc/dwc-i2s.c index 0df1bfc52f4ed..cffa63bba1cb3 100644 --- a/sound/soc/dwc/dwc-i2s.c +++ b/sound/soc/dwc/dwc-i2s.c @@ -813,7 +813,7 @@ static int dw_configure_dai_by_dt(struct dw_i2s_dev *dev, u32 idx2; int ret; - ret = dw_configure_dai(dev, dw_i2s_dai, SNDRV_PCM_RATE_8000_384000); + ret = dw_configure_dai(dev, dw_i2s_dai, SNDRV_PCM_RATE_8000_768000); if (ret < 0) return ret; diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index 71113886e494b..6cf5047eb9551 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -48,6 +48,9 @@ * @mclk_id: MCLK (or main clock) id for set_sysclk() * @fll_id: FLL (or secordary clock) id for set_sysclk() * @pll_id: PLL id for set_pll() + * @pll_ratio_s24: PLL output ratio for S24_LE format (PLL_freq = sample_rate × ratio) + * Default is 384, but some codecs (e.g., WM8904) require lower values + * to stay within PLL frequency limits */ struct codec_priv { struct clk *mclk; @@ -56,6 +59,7 @@ struct codec_priv { u32 mclk_id; int fll_id; int pll_id; + int pll_ratio_s24; }; /** @@ -222,7 +226,7 @@ static int fsl_asoc_card_hw_params(struct snd_pcm_substream *substream, if (codec_priv->pll_id >= 0 && codec_priv->fll_id >= 0) { if (priv->sample_format == SNDRV_PCM_FORMAT_S24_LE) - pll_out = priv->sample_rate * 384; + pll_out = priv->sample_rate * codec_priv->pll_ratio_s24; else pll_out = priv->sample_rate * 256; @@ -651,8 +655,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) cpu_pdev = of_find_device_by_node(cpu_np); if (!cpu_pdev) { - dev_err(&pdev->dev, "failed to find CPU DAI device\n"); - ret = -EINVAL; + ret = dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "failed to find CPU DAI device\n"); goto fail; } @@ -742,6 +746,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) for (codec_idx = 0; codec_idx < 2; codec_idx++) { priv->codec_priv[codec_idx].fll_id = -1; priv->codec_priv[codec_idx].pll_id = -1; + priv->codec_priv[codec_idx].pll_ratio_s24 = 384; } /* Diversify the card configurations */ @@ -835,6 +840,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) priv->codec_priv[0].mclk_id = WM8904_FLL_MCLK; priv->codec_priv[0].fll_id = WM8904_CLK_FLL; priv->codec_priv[0].pll_id = WM8904_FLL_MCLK; + priv->codec_priv[0].pll_ratio_s24 = 192; priv->dai_fmt |= SND_SOC_DAIFMT_CBP_CFP; } else if (of_device_is_compatible(np, "fsl,imx-audio-spdif")) { ret = fsl_asoc_card_spdif_init(codec_np, cpu_np, codec_dai_name, priv); diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c index 92fb16f7be456..40279becb8d96 100644 --- a/sound/soc/fsl/fsl_asrc.c +++ b/sound/soc/fsl/fsl_asrc.c @@ -1388,7 +1388,7 @@ static int fsl_asrc_probe(struct platform_device *pdev) ret = fsl_asrc_m2m_init(asrc); if (ret) { dev_err(&pdev->dev, "failed to init m2m device %d\n", ret); - return ret; + goto err_pm_get_sync; } return 0; diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c index d9b0bd61755d5..c9a7a41ef388c 100644 --- a/sound/soc/fsl/fsl_audmix.c +++ b/sound/soc/fsl/fsl_audmix.c @@ -454,6 +454,9 @@ static const struct of_device_id fsl_audmix_ids[] = { }; MODULE_DEVICE_TABLE(of, fsl_audmix_ids); +static int fsl_audmix_runtime_resume(struct device *dev); +static int fsl_audmix_runtime_suspend(struct device *dev); + static int fsl_audmix_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -485,13 +488,25 @@ static int fsl_audmix_probe(struct platform_device *pdev) spin_lock_init(&priv->lock); platform_set_drvdata(pdev, priv); pm_runtime_enable(dev); + if (!pm_runtime_enabled(dev)) { + ret = fsl_audmix_runtime_resume(dev); + if (ret) + goto err_disable_pm; + } + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto err_pm_get_sync; + + /* To enable regmap cache only when runtime PM enabled */ + pm_runtime_put(dev); ret = devm_snd_soc_register_component(dev, &fsl_audmix_component, fsl_audmix_dai, ARRAY_SIZE(fsl_audmix_dai)); if (ret) { dev_err(dev, "failed to register ASoC DAI\n"); - goto err_disable_pm; + goto err_pm_get_sync; } /* @@ -503,12 +518,15 @@ static int fsl_audmix_probe(struct platform_device *pdev) if (IS_ERR(priv->pdev)) { ret = PTR_ERR(priv->pdev); dev_err(dev, "failed to register platform: %d\n", ret); - goto err_disable_pm; + goto err_pm_get_sync; } } return 0; +err_pm_get_sync: + if (!pm_runtime_status_suspended(dev)) + fsl_audmix_runtime_suspend(dev); err_disable_pm: pm_runtime_disable(dev); return ret; @@ -519,6 +537,8 @@ static void fsl_audmix_remove(struct platform_device *pdev) struct fsl_audmix *priv = dev_get_drvdata(&pdev->dev); pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + fsl_audmix_runtime_suspend(&pdev->dev); if (priv->pdev) platform_device_unregister(priv->pdev); diff --git a/sound/soc/fsl/fsl_easrc.c b/sound/soc/fsl/fsl_easrc.c index 59cdf18731a0a..fc0364fdaf897 100644 --- a/sound/soc/fsl/fsl_easrc.c +++ b/sound/soc/fsl/fsl_easrc.c @@ -2055,7 +2055,7 @@ static int fsl_easrc_m2m_calc_out_len(struct fsl_asrc_pair *pair, int input_buff /* right shift 12 bit to make ratio in 32bit space */ val2 = (u64)in_samples << (frac_bits - 12); val1 = val1 >> 12; - do_div(val2, val1); + val2 = div64_u64(val2, val1); out_samples = val2; out_length = out_samples * out_width * channels; @@ -2266,7 +2266,7 @@ static int fsl_easrc_probe(struct platform_device *pdev) ret = fsl_asrc_m2m_init(easrc); if (ret) { dev_err(&pdev->dev, "failed to init m2m device %d\n", ret); - return ret; + goto err_pm_disable; } return 0; diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c index 09deb880bd37d..c5ecf62b55361 100644 --- a/sound/soc/fsl/fsl_micfil.c +++ b/sound/soc/fsl/fsl_micfil.c @@ -808,12 +808,17 @@ static int fsl_micfil_reparent_rootclk(struct fsl_micfil *micfil, unsigned int s /* Get root clock */ clk = micfil->mclk; - /* Disable clock first, for it was enabled by pm_runtime */ + /* Reparent root clock to the PLL matching this sample rate */ fsl_asoc_reparent_pll_clocks(dev, clk, micfil->pll8k_clk, micfil->pll11k_clk, ratio); - ret = clk_prepare_enable(clk); - if (ret) - return ret; + + /* Enable only once; hw_params can be called multiple times */ + if (!micfil->mclk_flag) { + ret = clk_prepare_enable(clk); + if (ret) + return ret; + micfil->mclk_flag = true; + } return 0; } @@ -846,8 +851,6 @@ static int fsl_micfil_hw_params(struct snd_pcm_substream *substream, if (ret) return ret; - micfil->mclk_flag = true; - /* floor(K * CLKDIV) */ switch (micfil->quality) { case QUALITY_HIGH: @@ -923,8 +926,10 @@ static int fsl_micfil_hw_free(struct snd_pcm_substream *substream, { struct fsl_micfil *micfil = snd_soc_dai_get_drvdata(dai); - clk_disable_unprepare(micfil->mclk); - micfil->mclk_flag = false; + if (micfil->mclk_flag) { + clk_disable_unprepare(micfil->mclk); + micfil->mclk_flag = false; + } return 0; } diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c index d48f72e5d49b6..03107feb9ecaa 100644 --- a/sound/soc/fsl/fsl_sai.c +++ b/sound/soc/fsl/fsl_sai.c @@ -757,6 +757,8 @@ static int fsl_sai_hw_free(struct snd_pcm_substream *substream, struct fsl_sai *sai = snd_soc_dai_get_drvdata(cpu_dai); bool tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; unsigned int ofs = sai->soc_data->reg_offset; + int adir = tx ? RX : TX; + int dir = tx ? TX : RX; /* Clear xMR to avoid channel swap with mclk_with_tere enabled case */ regmap_write(sai->regmap, FSL_SAI_xMR(tx), 0); @@ -764,10 +766,29 @@ static int fsl_sai_hw_free(struct snd_pcm_substream *substream, regmap_update_bits(sai->regmap, FSL_SAI_xCR3(tx, ofs), FSL_SAI_CR3_TRCE_MASK, 0); - if (!sai->is_consumer_mode[tx] && - sai->mclk_streams & BIT(substream->stream)) { - clk_disable_unprepare(sai->mclk_clk[sai->mclk_id[tx]]); - sai->mclk_streams &= ~BIT(substream->stream); + if (!sai->is_consumer_mode[tx]) { + bool adir_active = !!(sai->mclk_streams & BIT(!substream->stream)); + /* + * If opposite stream provides clocks for synchronous mode and + * it is inactive, Clear BYP and BCI + */ + if (fsl_sai_dir_is_synced(sai, adir) && !adir_active) + regmap_update_bits(sai->regmap, FSL_SAI_xCR2(!tx, ofs), + FSL_SAI_CR2_BCI | FSL_SAI_CR2_BYP, 0); + /* + * Clear BYP and BCI of current stream if either of: + * 1. current stream doesn't provide clocks for synchronous mode + * 2. current stream provides clocks for synchronous mode but no + * more stream is active. + */ + if (!fsl_sai_dir_is_synced(sai, dir) || !adir_active) + regmap_update_bits(sai->regmap, FSL_SAI_xCR2(tx, ofs), + FSL_SAI_CR2_BCI | FSL_SAI_CR2_BYP, 0); + + if (sai->mclk_streams & BIT(substream->stream)) { + clk_disable_unprepare(sai->mclk_clk[sai->mclk_id[tx]]); + sai->mclk_streams &= ~BIT(substream->stream); + } } return 0; diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c index a4518fefad690..43438af1e1c68 100644 --- a/sound/soc/fsl/imx-card.c +++ b/sound/soc/fsl/imx-card.c @@ -496,11 +496,15 @@ static void imx_aif_shutdown(struct snd_pcm_substream *substream) struct snd_soc_dai *codec_dai; int i; - for_each_rtd_cpu_dais(rtd, i, cpu_dai) - snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + for_each_rtd_cpu_dais(rtd, i, cpu_dai) { + if (!snd_soc_dai_active(cpu_dai)) + snd_soc_dai_set_sysclk(cpu_dai, 0, 0, SND_SOC_CLOCK_OUT); + } - for_each_rtd_codec_dais(rtd, i, codec_dai) - snd_soc_dai_set_sysclk(codec_dai, 0, 0, SND_SOC_CLOCK_IN); + for_each_rtd_codec_dais(rtd, i, codec_dai) { + if (!snd_soc_dai_active(codec_dai)) + snd_soc_dai_set_sysclk(codec_dai, 0, 0, SND_SOC_CLOCK_IN); + } } static const struct snd_soc_ops imx_aif_ops = { diff --git a/sound/soc/fsl/mpc5200_psc_i2s.c b/sound/soc/fsl/mpc5200_psc_i2s.c index 9ad44eeed6ad8..7831136f4f129 100644 --- a/sound/soc/fsl/mpc5200_psc_i2s.c +++ b/sound/soc/fsl/mpc5200_psc_i2s.c @@ -170,6 +170,7 @@ static int psc_i2s_of_probe(struct platform_device *op) psc_i2s_dai, ARRAY_SIZE(psc_i2s_dai)); if (rc != 0) { pr_err("Failed to register DAI\n"); + mpc5200_audio_dma_destroy(op); return rc; } diff --git a/sound/soc/intel/atom/sst/sst_pci.c b/sound/soc/intel/atom/sst/sst_pci.c index 22ae2d22f121a..1a53c993b57fc 100644 --- a/sound/soc/intel/atom/sst/sst_pci.c +++ b/sound/soc/intel/atom/sst/sst_pci.c @@ -130,13 +130,15 @@ static int intel_sst_probe(struct pci_dev *pci, sst_drv_ctx->pci = pci_dev_get(pci); ret = sst_platform_get_resources(sst_drv_ctx); if (ret < 0) - goto do_free_drv_ctx; + goto do_put_pci; pci_set_drvdata(pci, sst_drv_ctx); sst_configure_runtime_pm(sst_drv_ctx); return ret; +do_put_pci: + pci_dev_put(sst_drv_ctx->pci); do_free_drv_ctx: sst_context_cleanup(sst_drv_ctx); dev_err(sst_drv_ctx->dev, "Probe failed with %d\n", ret); @@ -165,6 +167,7 @@ static const struct pci_device_id intel_sst_ids[] = { { PCI_DEVICE_DATA(INTEL, SST_TNG, 0) }, { 0, } }; +MODULE_DEVICE_TABLE(pci, intel_sst_ids); static struct pci_driver sst_driver = { .name = SST_DRV_NAME, diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index 6e0e65584c7f8..061d97603abbf 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -473,8 +473,13 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) } snd_hdac_bus_parse_capabilities(bus); - if (bus->mlcap) - snd_hdac_ext_bus_get_ml_capabilities(bus); + if (bus->mlcap) { + ret = snd_hdac_ext_bus_get_ml_capabilities(bus); + if (ret < 0) { + dev_err(dev, "failed to get ml capabilities: %d\n", ret); + goto err_ml_cap; + } + } if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64))) dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); @@ -516,6 +521,8 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) snd_hdac_bus_free_stream_pages(bus); snd_hdac_ext_stream_free_all(bus); err_init_streams: + snd_hdac_ext_link_free_all(bus); +err_ml_cap: iounmap(adev->dsp_ba); err_remap_bar4: iounmap(bus->remap_addr); diff --git a/sound/soc/intel/avs/debugfs.c b/sound/soc/intel/avs/debugfs.c index 3534de46f9e45..d793071a22d48 100644 --- a/sound/soc/intel/avs/debugfs.c +++ b/sound/soc/intel/avs/debugfs.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -238,15 +239,20 @@ static int strace_open(struct inode *inode, struct file *file) if (!try_module_get(adev->dev->driver->owner)) return -ENODEV; - if (kfifo_initialized(&adev->trace_fifo)) - return -EBUSY; + if (kfifo_initialized(&adev->trace_fifo)) { + ret = -EBUSY; + goto err; + } ret = kfifo_alloc(&adev->trace_fifo, PAGE_SIZE, GFP_KERNEL); if (ret < 0) - return ret; + goto err; file->private_data = adev; return 0; +err: + module_put(adev->dev->driver->owner); + return ret; } static int strace_release(struct inode *inode, struct file *file) diff --git a/sound/soc/intel/avs/path.c b/sound/soc/intel/avs/path.c index 0c66c8bb2f89d..d42ccf8b645ce 100644 --- a/sound/soc/intel/avs/path.c +++ b/sound/soc/intel/avs/path.c @@ -210,9 +210,11 @@ int avs_path_set_constraint(struct avs_dev *adev, struct avs_tplg_path_template continue; } - blob = avs_nhlt_config_or_default(adev, module_template); - if (IS_ERR(blob)) - continue; + if (!module_template->nhlt_config) { + blob = avs_nhlt_config_or_default(adev, module_template); + if (IS_ERR(blob)) + continue; + } rlist[i] = path_template->fe_fmt->sampling_freq; clist[i] = path_template->fe_fmt->num_channels; @@ -382,7 +384,10 @@ static int avs_fill_gtw_config(struct avs_dev *adev, struct avs_copier_gtw_cfg * struct acpi_nhlt_config *blob; size_t gtw_size; - blob = avs_nhlt_config_or_default(adev, t); + if (t->nhlt_config) + blob = t->nhlt_config->blob; + else + blob = avs_nhlt_config_or_default(adev, t); if (IS_ERR(blob)) return PTR_ERR(blob); @@ -833,15 +838,10 @@ static int avs_path_module_type_create(struct avs_dev *adev, struct avs_path_mod static int avs_path_module_send_init_configs(struct avs_dev *adev, struct avs_path_module *mod) { - struct avs_soc_component *acomp; - - acomp = to_avs_soc_component(mod->template->owner->owner->owner->owner->comp); - - u32 num_ids = mod->template->num_config_ids; - u32 *ids = mod->template->config_ids; + struct avs_tplg_module *template = mod->template; - for (int i = 0; i < num_ids; i++) { - struct avs_tplg_init_config *config = &acomp->tplg->init_configs[ids[i]]; + for (int i = 0; i < template->num_init_configs; i++) { + struct avs_tplg_init_config *config = template->init_configs[i]; size_t len = config->length; void *data = config->data; u32 param = config->param; diff --git a/sound/soc/intel/avs/pcm.c b/sound/soc/intel/avs/pcm.c index 80c001120cdd9..a2c05a4ced72b 100644 --- a/sound/soc/intel/avs/pcm.c +++ b/sound/soc/intel/avs/pcm.c @@ -6,6 +6,7 @@ // Amadeusz Slawinski // +#include #include #include #include @@ -987,13 +988,25 @@ static int avs_component_load_libraries(struct avs_soc_component *acomp) return ret; } +static int avs_request_topology(struct snd_soc_component *component, const char *name, + const struct firmware **fw) +{ + char *fullname __free(kfree) = NULL; + + fullname = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, name); + if (!fullname) + return -ENOMEM; + + return request_firmware(fw, fullname, component->dev); +} + static int avs_component_probe(struct snd_soc_component *component) { struct snd_soc_card *card = component->card; struct snd_soc_acpi_mach *mach; struct avs_soc_component *acomp; + const struct firmware *fw; struct avs_dev *adev; - char *filename; int ret; dev_dbg(card->dev, "probing %s card %s\n", component->name, card->name); @@ -1009,13 +1022,7 @@ static int avs_component_probe(struct snd_soc_component *component) goto finalize; /* Load specified topology and create debugfs for it. */ - filename = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, - mach->tplg_filename); - if (!filename) - return -ENOMEM; - - ret = avs_load_topology(component, filename); - kfree(filename); + ret = avs_request_topology(component, mach->tplg_filename, &fw); if (ret == -ENOENT && !strncmp(mach->tplg_filename, "hda-", 4)) { unsigned int vendor_id; @@ -1030,18 +1037,17 @@ static int avs_component_probe(struct snd_soc_component *component) "hda-generic-tplg.bin"); if (!mach->tplg_filename) return -ENOMEM; - filename = kasprintf(GFP_KERNEL, "%s/%s", component->driver->topology_name_prefix, - mach->tplg_filename); - if (!filename) - return -ENOMEM; dev_info(card->dev, "trying to load fallback topology %s\n", mach->tplg_filename); - ret = avs_load_topology(component, filename); - kfree(filename); + ret = avs_request_topology(component, mach->tplg_filename, &fw); } if (ret < 0) return ret; + ret = snd_soc_tplg_component_load(component, &avs_tplg_ops, fw); + if (ret) + return ret; + ret = avs_component_load_libraries(acomp); if (ret < 0) { dev_err(card->dev, "libraries loading failed: %d\n", ret); diff --git a/sound/soc/intel/avs/topology.c b/sound/soc/intel/avs/topology.c index dfe8cf5053818..c9837d78e6234 100644 --- a/sound/soc/intel/avs/topology.c +++ b/sound/soc/intel/avs/topology.c @@ -350,6 +350,8 @@ AVS_DEFINE_PTR_PARSER(modcfg_base, struct avs_tplg_modcfg_base, modcfgs_base); AVS_DEFINE_PTR_PARSER(modcfg_ext, struct avs_tplg_modcfg_ext, modcfgs_ext); AVS_DEFINE_PTR_PARSER(pplcfg, struct avs_tplg_pplcfg, pplcfgs); AVS_DEFINE_PTR_PARSER(binding, struct avs_tplg_binding, bindings); +AVS_DEFINE_PTR_PARSER(init_config, struct avs_tplg_init_config, init_configs); +AVS_DEFINE_PTR_PARSER(nhlt_config, struct avs_tplg_nhlt_config, nhlt_configs); static int parse_audio_format_bitfield(struct snd_soc_component *comp, void *elem, void *object, u32 offset) @@ -420,6 +422,22 @@ static int parse_link_formatted_string(struct snd_soc_component *comp, void *ele return 0; } +static int avs_parse_nhlt_config_size(struct snd_soc_component *comp, void *elem, void *object, + u32 offset) +{ + struct snd_soc_tplg_vendor_value_elem *tuple = elem; + struct acpi_nhlt_config **blob = (struct acpi_nhlt_config **)((u8 *)object + offset); + u32 size; + + size = le32_to_cpu(tuple->value); + *blob = devm_kzalloc(comp->card->dev, struct_size(*blob, capabilities, size), GFP_KERNEL); + if (!*blob) + return -ENOMEM; + + (*blob)->capabilities_size = size; + return 0; +} + static int parse_dictionary_header(struct snd_soc_component *comp, struct snd_soc_tplg_vendor_array *tuples, @@ -1181,9 +1199,15 @@ static const struct avs_tplg_token_parser module_parsers[] = { { .token = AVS_TKN_MOD_INIT_CONFIG_NUM_IDS_U32, .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, - .offset = offsetof(struct avs_tplg_module, num_config_ids), + .offset = offsetof(struct avs_tplg_module, num_init_configs), .parse = avs_parse_byte_token, }, + { + .token = AVS_TKN_MOD_NHLT_CONFIG_ID_U32, + .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, + .offset = offsetof(struct avs_tplg_module, nhlt_config), + .parse = avs_parse_nhlt_config_ptr, + }, }; static const struct avs_tplg_token_parser init_config_parsers[] = { @@ -1191,10 +1215,32 @@ static const struct avs_tplg_token_parser init_config_parsers[] = { .token = AVS_TKN_MOD_INIT_CONFIG_ID_U32, .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, .offset = 0, - .parse = avs_parse_word_token, + .parse = avs_parse_init_config_ptr, }, }; +static int avs_tplg_module_init_configs(struct snd_soc_component *comp, + struct avs_tplg_module *module, + struct snd_soc_tplg_vendor_array *tuples, u32 block_size) +{ + struct avs_tplg_init_config **cfgs; + int ret; + + if (!module->num_init_configs) + return -EINVAL; + + cfgs = devm_kcalloc(comp->card->dev, module->num_init_configs, sizeof(*cfgs), GFP_KERNEL); + if (!cfgs) + return -ENOMEM; + + ret = parse_dictionary_entries(comp, tuples, block_size, cfgs, module->num_init_configs, + sizeof(*cfgs), AVS_TKN_MOD_INIT_CONFIG_ID_U32, + init_config_parsers, ARRAY_SIZE(init_config_parsers)); + if (!ret) + module->init_configs = cfgs; + return ret; +} + static struct avs_tplg_module * avs_tplg_module_create(struct snd_soc_component *comp, struct avs_tplg_pipeline *owner, struct snd_soc_tplg_vendor_array *tuples, u32 block_size) @@ -1221,27 +1267,11 @@ avs_tplg_module_create(struct snd_soc_component *comp, struct avs_tplg_pipeline block_size -= esize; /* Parse trailing config ids if any. */ if (block_size) { - u32 num_config_ids = module->num_config_ids; - u32 *config_ids; - - if (!num_config_ids) - return ERR_PTR(-EINVAL); - - config_ids = devm_kcalloc(comp->card->dev, num_config_ids, sizeof(*config_ids), - GFP_KERNEL); - if (!config_ids) - return ERR_PTR(-ENOMEM); - tuples = avs_tplg_vendor_array_at(tuples, esize); - ret = parse_dictionary_entries(comp, tuples, block_size, - config_ids, num_config_ids, sizeof(*config_ids), - AVS_TKN_MOD_INIT_CONFIG_ID_U32, - init_config_parsers, - ARRAY_SIZE(init_config_parsers)); + + ret = avs_tplg_module_init_configs(comp, module, tuples, block_size); if (ret) return ERR_PTR(ret); - - module->config_ids = config_ids; } module->owner = owner; @@ -1651,12 +1681,14 @@ static const struct avs_tplg_token_parser mod_init_config_parsers[] = { static int avs_tplg_parse_initial_configs(struct snd_soc_component *comp, struct snd_soc_tplg_vendor_array *tuples, - u32 block_size) + u32 block_size, u32 *offset) { struct avs_soc_component *acomp = to_avs_soc_component(comp); struct avs_tplg *tplg = acomp->tplg; int ret, i; + *offset = 0; + /* Parse tuple section telling how many init configs there are. */ ret = parse_dictionary_header(comp, tuples, (void **)&tplg->init_configs, &tplg->num_init_configs, @@ -1666,6 +1698,7 @@ static int avs_tplg_parse_initial_configs(struct snd_soc_component *comp, return ret; block_size -= le32_to_cpu(tuples->size); + *offset += le32_to_cpu(tuples->size); /* With header parsed, move on to parsing entries. */ tuples = avs_tplg_vendor_array_next(tuples); @@ -1681,6 +1714,7 @@ static int avs_tplg_parse_initial_configs(struct snd_soc_component *comp, */ tmp = avs_tplg_vendor_array_next(tuples); esize = le32_to_cpu(tuples->size) + le32_to_cpu(tmp->size); + *offset += esize; ret = parse_dictionary_entries(comp, tuples, esize, config, 1, sizeof(*config), AVS_TKN_INIT_CONFIG_ID_U32, @@ -1692,6 +1726,7 @@ static int avs_tplg_parse_initial_configs(struct snd_soc_component *comp, /* handle raw data section */ init_config_data = (void *)tuples + esize; esize = config->length; + *offset += esize; config->data = devm_kmemdup(comp->card->dev, init_config_data, esize, GFP_KERNEL); if (!config->data) @@ -1704,6 +1739,70 @@ static int avs_tplg_parse_initial_configs(struct snd_soc_component *comp, return 0; } +static const struct avs_tplg_token_parser mod_nhlt_config_parsers[] = { + { + .token = AVS_TKN_NHLT_CONFIG_ID_U32, + .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, + .offset = offsetof(struct avs_tplg_nhlt_config, id), + .parse = avs_parse_word_token, + }, + { + .token = AVS_TKN_NHLT_CONFIG_SIZE_U32, + .type = SND_SOC_TPLG_TUPLE_TYPE_WORD, + .offset = offsetof(struct avs_tplg_nhlt_config, blob), + .parse = avs_parse_nhlt_config_size, + }, +}; + +static int avs_tplg_parse_nhlt_configs(struct snd_soc_component *comp, + struct snd_soc_tplg_vendor_array *tuples, + u32 block_size) +{ + struct avs_soc_component *acomp = to_avs_soc_component(comp); + struct avs_tplg *tplg = acomp->tplg; + int ret, i; + + /* Parse the header section to know how many entries there are. */ + ret = parse_dictionary_header(comp, tuples, (void **)&tplg->nhlt_configs, + &tplg->num_nhlt_configs, + sizeof(*tplg->nhlt_configs), + AVS_TKN_MANIFEST_NUM_NHLT_CONFIGS_U32); + if (ret) + return ret; + + block_size -= le32_to_cpu(tuples->size); + /* With the header parsed, move on to parsing entries. */ + tuples = avs_tplg_vendor_array_next(tuples); + + for (i = 0; i < tplg->num_nhlt_configs && block_size > 0; i++) { + struct avs_tplg_nhlt_config *config; + u32 esize; + + config = &tplg->nhlt_configs[i]; + esize = le32_to_cpu(tuples->size); + + ret = parse_dictionary_entries(comp, tuples, esize, config, 1, sizeof(*config), + AVS_TKN_NHLT_CONFIG_ID_U32, + mod_nhlt_config_parsers, + ARRAY_SIZE(mod_nhlt_config_parsers)); + if (ret) + return ret; + /* With tuples parsed, the blob shall be allocated. */ + if (!config->blob) + return -EINVAL; + + /* Consume the raw data and move to the next entry. */ + memcpy(config->blob->capabilities, (u8 *)tuples + esize, + config->blob->capabilities_size); + esize += config->blob->capabilities_size; + + block_size -= esize; + tuples = avs_tplg_vendor_array_at(tuples, esize); + } + + return 0; +} + static int avs_route_load(struct snd_soc_component *comp, int index, struct snd_soc_dapm_route *route) { @@ -2008,11 +2107,26 @@ static int avs_manifest(struct snd_soc_component *comp, int index, tuples = avs_tplg_vendor_array_at(tuples, offset); /* Initial configs dictionary. */ - ret = avs_tplg_parse_initial_configs(comp, tuples, remaining); + ret = avs_tplg_parse_initial_configs(comp, tuples, remaining, &offset); if (ret < 0) return ret; - return 0; + remaining -= offset; + tuples = avs_tplg_vendor_array_at(tuples, offset); + + ret = avs_tplg_vendor_array_lookup(tuples, remaining, + AVS_TKN_MANIFEST_NUM_NHLT_CONFIGS_U32, &offset); + if (ret == -ENOENT) + return 0; + if (ret) { + dev_err(comp->dev, "NHLT config lookup failed: %d\n", ret); + return ret; + } + + tuples = avs_tplg_vendor_array_at(tuples, offset); + + /* NHLT configs dictionary. */ + return avs_tplg_parse_nhlt_configs(comp, tuples, remaining); } enum { @@ -2087,7 +2201,7 @@ avs_control_load(struct snd_soc_component *comp, int index, struct snd_kcontrol_ return 0; } -static const struct snd_soc_tplg_ops avs_tplg_ops = { +const struct snd_soc_tplg_ops avs_tplg_ops = { .io_ops = avs_control_ops, .io_ops_count = ARRAY_SIZE(avs_control_ops), .control_load = avs_control_load, diff --git a/sound/soc/intel/avs/topology.h b/sound/soc/intel/avs/topology.h index 1e83fccf2ea21..189984ce7b51e 100644 --- a/sound/soc/intel/avs/topology.h +++ b/sound/soc/intel/avs/topology.h @@ -37,6 +37,8 @@ struct avs_tplg { u32 num_condpath_tmpls; struct avs_tplg_init_config *init_configs; u32 num_init_configs; + struct avs_tplg_nhlt_config *nhlt_configs; + u32 num_nhlt_configs; struct list_head path_tmpl_list; }; @@ -175,6 +177,11 @@ struct avs_tplg_init_config { void *data; }; +struct avs_tplg_nhlt_config { + u32 id; + struct acpi_nhlt_config *blob; +}; + struct avs_tplg_path { u32 id; @@ -214,14 +221,16 @@ struct avs_tplg_module { u8 domain; struct avs_tplg_modcfg_ext *cfg_ext; u32 ctl_id; - u32 num_config_ids; - u32 *config_ids; + u32 num_init_configs; + struct avs_tplg_init_config **init_configs; + struct avs_tplg_nhlt_config *nhlt_config; struct avs_tplg_pipeline *owner; /* Pipeline modules management. */ struct list_head node; }; +extern const struct snd_soc_tplg_ops avs_tplg_ops; struct avs_tplg *avs_tplg_new(struct snd_soc_component *comp); int avs_load_topology(struct snd_soc_component *comp, const char *filename); diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index 465bf5fafecf7..ffa6e2ad73f75 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -805,6 +805,8 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { }; static const struct snd_pci_quirk sof_sdw_ssid_quirk_table[] = { + SND_PCI_QUIRK(0x1028, 0x0e53, "Dell XPS WCL", SOC_SDW_SIDECAR_AMPS), + SND_PCI_QUIRK(0x1028, 0x0e54, "Dell XPS PTL", SOC_SDW_SIDECAR_AMPS), SND_PCI_QUIRK(0x1043, 0x1e13, "ASUS Zenbook S14", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x1043, 0x1f43, "ASUS Zenbook S16", SOC_SDW_CODEC_MIC), SND_PCI_QUIRK(0x17aa, 0x2347, "Lenovo P16", SOC_SDW_CODEC_MIC), @@ -865,10 +867,16 @@ static int create_sdw_dailink(struct snd_soc_card *card, } } + /* + * The dai_type is used to select function topologies. Since the topology stream name + * and DAI link name use partial matching, unconditionally appending the dai_type provides + * necessary selection metadata without breaking existing topologies. Although + * ctx->append_dai_type is not checked here, we overwrite it to ensure consistency in case + * it is referenced elsewhere. + */ + ctx->append_dai_type = true; for_each_pcm_streams(stream) { static const char * const sdw_stream_name[] = { - "SDW%d-Playback", - "SDW%d-Capture", "SDW%d-Playback-%s", "SDW%d-Capture-%s", }; @@ -896,15 +904,10 @@ static int create_sdw_dailink(struct snd_soc_card *card, } /* create stream name according to first link id */ - if (ctx->append_dai_type) - name = devm_kasprintf(dev, GFP_KERNEL, - sdw_stream_name[stream + 2], - ffs(sof_end->link_mask) - 1, - type_strings[sof_end->dai_info->dai_type]); - else - name = devm_kasprintf(dev, GFP_KERNEL, - sdw_stream_name[stream], - ffs(sof_end->link_mask) - 1); + name = devm_kasprintf(dev, GFP_KERNEL, + sdw_stream_name[stream], + ffs(sof_end->link_mask) - 1, + type_strings[sof_end->dai_info->dai_type]); if (!name) return -ENOMEM; diff --git a/sound/soc/intel/catpt/ipc.c b/sound/soc/intel/catpt/ipc.c index d26863249097f..952c26e5d0e6c 100644 --- a/sound/soc/intel/catpt/ipc.c +++ b/sound/soc/intel/catpt/ipc.c @@ -205,6 +205,7 @@ static void catpt_dsp_process_response(struct catpt_dev *cdev, u32 header) memcpy_fromio(&config, cdev->lpe_ba + off, sizeof(config)); trace_catpt_ipc_payload((u8 *)&config, sizeof(config)); + dev_dbg(cdev->dev, "FW READY 0x%08x\n", header); catpt_ipc_arm(ipc, &config); complete(&cdev->fw_ready); return; @@ -215,6 +216,13 @@ static void catpt_dsp_process_response(struct catpt_dev *cdev, u32 header) dev_err(cdev->dev, "ADSP device coredump received\n"); ipc->ready = false; catpt_coredump(cdev); + + if (catpt_readl_dram(cdev, COREDUMP) == CATPT_COREDUMP_REQUEST) { + dev_dbg(cdev->dev, "releasing firmware from the coredump state\n"); + catpt_writel_dram(cdev, COREDUMP, CATPT_COREDUMP_RELEASE); + } + + complete(&cdev->fw_ready); /* TODO: attempt recovery */ break; diff --git a/sound/soc/intel/catpt/loader.c b/sound/soc/intel/catpt/loader.c index 696d84314eeb5..680efad5f458d 100644 --- a/sound/soc/intel/catpt/loader.c +++ b/sound/soc/intel/catpt/loader.c @@ -626,6 +626,9 @@ int catpt_boot_firmware(struct catpt_dev *cdev, bool restore) if (!ret) { dev_err(cdev->dev, "firmware ready timeout\n"); return -ETIMEDOUT; + /* Wake up does not mean FW is ready, an exception could occur. */ + } else if (!cdev->ipc.ready) { + return -EREMOTEIO; } /* update sram pg & clock once done booting */ diff --git a/sound/soc/intel/catpt/registers.h b/sound/soc/intel/catpt/registers.h index 6c1ad28c6d692..64bd534a76ff4 100644 --- a/sound/soc/intel/catpt/registers.h +++ b/sound/soc/intel/catpt/registers.h @@ -124,6 +124,11 @@ #define CATPT_SSCR2_DEFAULT 0x0 #define CATPT_SSPSP2_DEFAULT 0x0 +/* Coredump register and its states */ +#define CATPT_DRAM_COREDUMP 0x1F4 +#define CATPT_COREDUMP_REQUEST UINT_MAX +#define CATPT_COREDUMP_RELEASE 0 + /* Physically the same block, access address differs between host and dsp */ #define CATPT_DSP_DRAM_OFFSET 0x400000 #define catpt_to_host_offset(offset) ((offset) & ~(CATPT_DSP_DRAM_OFFSET)) @@ -137,6 +142,8 @@ /* registry I/O helpers */ +#define catpt_dram_addr(cdev) \ + ((cdev)->lpe_ba + (cdev)->spec->host_dram_offset) #define catpt_shim_addr(cdev) \ ((cdev)->lpe_ba + (cdev)->spec->host_shim_offset) #define catpt_dma_addr(cdev, dma) \ @@ -151,6 +158,11 @@ #define catpt_writel_ssp(cdev, ssp, reg, val) \ writel(val, catpt_ssp_addr(cdev, ssp) + (reg)) +#define catpt_readl_dram(cdev, reg) \ + readl(catpt_dram_addr(cdev) + CATPT_DRAM_##reg) +#define catpt_writel_dram(cdev, reg, val) \ + writel(val, catpt_dram_addr(cdev) + CATPT_DRAM_##reg) + #define catpt_readl_shim(cdev, reg) \ readl(catpt_shim_addr(cdev) + CATPT_SHIM_##reg) #define catpt_writel_shim(cdev, reg, val) \ diff --git a/sound/soc/intel/common/sof-function-topology-lib.c b/sound/soc/intel/common/sof-function-topology-lib.c index 3cc81dcf047e3..7940280bf4480 100644 --- a/sound/soc/intel/common/sof-function-topology-lib.c +++ b/sound/soc/intel/common/sof-function-topology-lib.c @@ -30,7 +30,12 @@ enum tplg_device_id { int sof_sdw_get_tplg_files(struct snd_soc_card *card, const struct snd_soc_acpi_mach *mach, const char *prefix, const char ***tplg_files) { - struct snd_soc_acpi_mach_params mach_params = mach->mach_params; + struct snd_soc_acpi_mach *card_mach = dev_get_platdata(card->dev); + /* + * Use the acpi mach from the machine driver because the machine driver + * may change the dmic_num based on the machine driver quirk. + */ + struct snd_soc_acpi_mach_params mach_params = card_mach->mach_params; struct snd_soc_dai_link *dai_link; const struct firmware *fw; char platform[SOF_INTEL_PLATFORM_NAME_MAX]; diff --git a/sound/soc/loongson/loongson_card.c b/sound/soc/loongson/loongson_card.c index 7910d5d9ac4ff..ea895fe6b5e97 100644 --- a/sound/soc/loongson/loongson_card.c +++ b/sound/soc/loongson/loongson_card.c @@ -91,7 +91,7 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) const char *codec_dai_name; struct acpi_device *adev; struct device *phy_dev; - int i; + int i, ret; /* fixup platform name based on reference node */ adev = loongson_card_acpi_find_device(card, "cpu"); @@ -108,7 +108,9 @@ static int loongson_card_parse_acpi(struct loongson_card_data *data) return -ENOENT; snprintf(codec_name, sizeof(codec_name), "i2c-%s", acpi_dev_name(adev)); - device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + ret = device_property_read_string(card->dev, "codec-dai-name", &codec_dai_name); + if (ret) + return ret; for (i = 0; i < card->num_links; i++) { loongson_dai_links[i].platforms->name = dev_name(phy_dev); diff --git a/sound/soc/mediatek/mt8365/mt8365-afe-pcm.c b/sound/soc/mediatek/mt8365/mt8365-afe-pcm.c index d48252cd96ac4..bf92c8922c236 100644 --- a/sound/soc/mediatek/mt8365/mt8365-afe-pcm.c +++ b/sound/soc/mediatek/mt8365/mt8365-afe-pcm.c @@ -1975,10 +1975,15 @@ static int mt8365_afe_suspend(struct device *dev) mt8365_afe_enable_main_clk(afe); - if (!afe->reg_back_up) + if (!afe->reg_back_up) { afe->reg_back_up = devm_kcalloc(dev, afe->reg_back_up_list_num, sizeof(unsigned int), GFP_KERNEL); + if (!afe->reg_back_up) { + mt8365_afe_disable_main_clk(afe); + return -ENOMEM; + } + } for (i = 0; i < afe->reg_back_up_list_num; i++) regmap_read(regmap, afe->reg_back_up_list[i], @@ -2012,11 +2017,15 @@ static int mt8365_afe_resume(struct device *dev) static int mt8365_afe_dev_runtime_suspend(struct device *dev) { struct mtk_base_afe *afe = dev_get_drvdata(dev); + int ret; if (pm_runtime_status_suspended(dev) || afe->suspended) return 0; - mt8365_afe_suspend(dev); + ret = mt8365_afe_suspend(dev); + if (ret) + return ret; + afe->suspended = true; return 0; } diff --git a/sound/soc/meson/aiu-acodec-ctrl.c b/sound/soc/meson/aiu-acodec-ctrl.c index 7b04b97f7b41c..c4cf97d2c82f5 100644 --- a/sound/soc/meson/aiu-acodec-ctrl.c +++ b/sound/soc/meson/aiu-acodec-ctrl.c @@ -38,6 +38,9 @@ static int aiu_acodec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int mux, changed; + if (ucontrol->value.enumerated.item[0] >= e->items) + return -EINVAL; + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); changed = snd_soc_component_test_bits(component, e->reg, CTRL_DIN_LRCLK_SRC, diff --git a/sound/soc/meson/aiu-codec-ctrl.c b/sound/soc/meson/aiu-codec-ctrl.c index ee0ef6301010c..fe776de739881 100644 --- a/sound/soc/meson/aiu-codec-ctrl.c +++ b/sound/soc/meson/aiu-codec-ctrl.c @@ -30,6 +30,9 @@ static int aiu_codec_ctrl_mux_put_enum(struct snd_kcontrol *kcontrol, struct soc_enum *e = (struct soc_enum *)kcontrol->private_value; unsigned int mux, changed; + if (ucontrol->value.enumerated.item[0] >= e->items) + return -EINVAL; + mux = snd_soc_enum_item_to_val(e, ucontrol->value.enumerated.item[0]); changed = snd_soc_component_test_bits(component, e->reg, CTRL_DATA_SEL, diff --git a/sound/soc/meson/aiu-fifo-spdif.c b/sound/soc/meson/aiu-fifo-spdif.c index e0e00ec026dcc..a9861c5d66374 100644 --- a/sound/soc/meson/aiu-fifo-spdif.c +++ b/sound/soc/meson/aiu-fifo-spdif.c @@ -24,6 +24,7 @@ #define AIU_MEM_IEC958_CONTROL_MODE_16BIT BIT(7) #define AIU_MEM_IEC958_CONTROL_MODE_LINEAR BIT(8) #define AIU_MEM_IEC958_BUF_CNTL_INIT BIT(0) +#define AIU_RST_SOFT_958_FAST BIT(2) #define AIU_FIFO_SPDIF_BLOCK 8 @@ -68,11 +69,15 @@ static int fifo_spdif_trigger(struct snd_pcm_substream *substream, int cmd, case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + snd_soc_component_write(component, AIU_RST_SOFT, + AIU_RST_SOFT_958_FAST); fifo_spdif_dcu_enable(component, true); break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: case SNDRV_PCM_TRIGGER_STOP: + snd_soc_component_write(component, AIU_RST_SOFT, + AIU_RST_SOFT_958_FAST); fifo_spdif_dcu_enable(component, false); break; default: diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c index cdb759b466ad4..8617a4661a339 100644 --- a/sound/soc/meson/meson-card-utils.c +++ b/sound/soc/meson/meson-card-utils.c @@ -50,25 +50,20 @@ int meson_card_reallocate_links(struct snd_soc_card *card, num_links * sizeof(*priv->card.dai_link), GFP_KERNEL | __GFP_ZERO); if (!links) - goto err_links; + return -ENOMEM; + + priv->card.dai_link = links; + priv->card.num_links = num_links; ldata = krealloc(priv->link_data, num_links * sizeof(*priv->link_data), GFP_KERNEL | __GFP_ZERO); + /* meson_card_clean_references() will free the links on this error path */ if (!ldata) - goto err_ldata; + return -ENOMEM; - priv->card.dai_link = links; priv->link_data = ldata; - priv->card.num_links = num_links; return 0; - -err_ldata: - kfree(links); -err_links: - dev_err(priv->card.dev, "failed to allocate links\n"); - return -ENOMEM; - } EXPORT_SYMBOL_GPL(meson_card_reallocate_links); diff --git a/sound/soc/pxa/pxa-ssp.c b/sound/soc/pxa/pxa-ssp.c index b8a3cb8b75978..ba8cae8e85ef5 100644 --- a/sound/soc/pxa/pxa-ssp.c +++ b/sound/soc/pxa/pxa-ssp.c @@ -769,13 +769,10 @@ static int pxa_ssp_probe(struct snd_soc_dai *dai) goto err_priv; } - priv->extclk = devm_clk_get(dev, "extclk"); + priv->extclk = devm_clk_get_optional(dev, "extclk"); if (IS_ERR(priv->extclk)) { ret = PTR_ERR(priv->extclk); - if (ret == -EPROBE_DEFER) - goto err_priv; - - priv->extclk = NULL; + goto err_priv; } } else { priv->ssp = pxa_ssp_request(dai->id + 1, "SoC audio"); diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index 6326649ba31b4..9122876e00a2a 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -203,10 +203,7 @@ int q6apm_graph_media_format_shmem(struct q6apm_graph *graph, if (!module) return -ENODEV; - audioreach_set_media_format(graph, module, cfg); - - return 0; - + return audioreach_set_media_format(graph, module, cfg); } EXPORT_SYMBOL_GPL(q6apm_graph_media_format_shmem); @@ -374,6 +371,7 @@ int q6apm_graph_media_format_pcm(struct q6apm_graph *graph, struct audioreach_mo struct audioreach_sub_graph *sgs; struct audioreach_container *container; struct audioreach_module *module; + int ret; list_for_each_entry(sgs, &info->sg_list, node) { list_for_each_entry(container, &sgs->container_list, node) { @@ -382,7 +380,9 @@ int q6apm_graph_media_format_pcm(struct q6apm_graph *graph, struct audioreach_mo (module->module_id == MODULE_ID_RD_SHARED_MEM_EP)) continue; - audioreach_set_media_format(graph, module, cfg); + ret = audioreach_set_media_format(graph, module, cfg); + if (ret) + return ret; } } } @@ -700,14 +700,17 @@ EXPORT_SYMBOL_GPL(q6apm_graph_prepare); int q6apm_graph_start(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; - int ret = 0; + int ret; - if (ar_graph->start_count == 0) + if (ar_graph->start_count == 0) { ret = audioreach_graph_mgmt_cmd(ar_graph, APM_CMD_GRAPH_START); + if (ret) + return ret; + } ar_graph->start_count++; - return ret; + return 0; } EXPORT_SYMBOL_GPL(q6apm_graph_start); @@ -715,6 +718,9 @@ int q6apm_graph_stop(struct q6apm_graph *graph) { struct audioreach_graph *ar_graph = graph->ar_graph; + if (ar_graph->start_count == 0) + return 0; + if (--ar_graph->start_count > 0) return 0; diff --git a/sound/soc/rockchip/rockchip_pdm.c b/sound/soc/rockchip/rockchip_pdm.c index c69cdd6f24994..115e90d3bbfe0 100644 --- a/sound/soc/rockchip/rockchip_pdm.c +++ b/sound/soc/rockchip/rockchip_pdm.c @@ -321,6 +321,7 @@ static int rockchip_pdm_set_fmt(struct snd_soc_dai *cpu_dai, { struct rk_pdm_dev *pdm = to_info(cpu_dai); unsigned int mask = 0, val = 0; + int ret; mask = PDM_CKP_MSK; switch (fmt & SND_SOC_DAIFMT_INV_MASK) { @@ -334,7 +335,10 @@ static int rockchip_pdm_set_fmt(struct snd_soc_dai *cpu_dai, return -EINVAL; } - pm_runtime_get_sync(cpu_dai->dev); + ret = pm_runtime_resume_and_get(cpu_dai->dev); + if (ret) + return ret; + regmap_update_bits(pdm->regmap, PDM_CLK_CTRL, mask, val); pm_runtime_put(cpu_dai->dev); @@ -422,16 +426,16 @@ static int rockchip_pdm_runtime_resume(struct device *dev) struct rk_pdm_dev *pdm = dev_get_drvdata(dev); int ret; - ret = clk_prepare_enable(pdm->clk); + ret = clk_prepare_enable(pdm->hclk); if (ret) { - dev_err(pdm->dev, "clock enable failed %d\n", ret); + dev_err(pdm->dev, "hclock enable failed %d\n", ret); return ret; } - ret = clk_prepare_enable(pdm->hclk); + ret = clk_prepare_enable(pdm->clk); if (ret) { - clk_disable_unprepare(pdm->clk); - dev_err(pdm->dev, "hclock enable failed %d\n", ret); + clk_disable_unprepare(pdm->hclk); + dev_err(pdm->dev, "clock enable failed %d\n", ret); return ret; } diff --git a/sound/soc/rockchip/rockchip_spdif.c b/sound/soc/rockchip/rockchip_spdif.c index d365168934dc6..7a15f680de2fe 100644 --- a/sound/soc/rockchip/rockchip_spdif.c +++ b/sound/soc/rockchip/rockchip_spdif.c @@ -97,6 +97,7 @@ static int rk_spdif_runtime_resume(struct device *dev) ret = regcache_sync(spdif->regmap); if (ret) { + regcache_cache_only(spdif->regmap, true); clk_disable_unprepare(spdif->mclk); clk_disable_unprepare(spdif->hclk); } diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c index 3723329b266dd..61191a6033eb1 100644 --- a/sound/soc/samsung/aries_wm8994.c +++ b/sound/soc/samsung/aries_wm8994.c @@ -658,6 +658,7 @@ static int aries_audio_probe(struct platform_device *pdev) goto out; } + of_node_get(aries_dai[0].cpus->of_node); aries_dai[0].platforms->of_node = aries_dai[0].cpus->of_node; /* Set CPU of_node for BT DAI */ diff --git a/sound/soc/sdca/sdca_functions.c b/sound/soc/sdca/sdca_functions.c index 4417278e39bb1..e0cae47641ba5 100644 --- a/sound/soc/sdca/sdca_functions.c +++ b/sound/soc/sdca/sdca_functions.c @@ -849,6 +849,8 @@ static int find_sdca_control_range(struct device *dev, return 0; else if (num_range < 0) return num_range; + else if (num_range < 2 * sizeof(*limits)) + return -EINVAL; range_list = devm_kcalloc(dev, num_range, sizeof(*range_list), GFP_KERNEL); if (!range_list) diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c index 51fb09d56c5a1..f640bbe319fae 100644 --- a/sound/soc/soc-dapm.c +++ b/sound/soc/soc-dapm.c @@ -3549,7 +3549,7 @@ int snd_soc_dapm_put_enum_double(struct snd_kcontrol *kcontrol, val = snd_soc_enum_item_to_val(e, item[0]) << e->shift_l; mask = e->mask << e->shift_l; if (e->shift_l != e->shift_r) { - if (item[1] > e->items) + if (item[1] >= e->items) return -EINVAL; val |= snd_soc_enum_item_to_val(e, item[1]) << e->shift_r; mask |= e->mask << e->shift_r; diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c index 44b60324eaa2e..3e4f96e79021a 100644 --- a/sound/soc/soc-topology.c +++ b/sound/soc/soc-topology.c @@ -1320,9 +1320,24 @@ static int soc_tplg_dapm_complete(struct soc_tplg *tplg) return ret; } +static int soc_tplg_check_name(const char *name) +{ + if (strnlen(name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN) == + SNDRV_CTL_ELEM_ID_NAME_MAXLEN) + return -EINVAL; + + return 0; +} + static int set_stream_info(struct soc_tplg *tplg, struct snd_soc_pcm_stream *stream, struct snd_soc_tplg_stream_caps *caps) { + int ret; + + ret = soc_tplg_check_name(caps->name); + if (ret) + return ret; + stream->stream_name = devm_kstrdup(tplg->dev, caps->name, GFP_KERNEL); if (!stream->stream_name) return -ENOMEM; @@ -1375,7 +1390,11 @@ static int soc_tplg_dai_create(struct soc_tplg *tplg, if (dai_drv == NULL) return -ENOMEM; - if (strlen(pcm->dai_name)) { + ret = soc_tplg_check_name(pcm->dai_name); + if (ret) + goto err; + + if (pcm->dai_name[0]) { dai_drv->name = devm_kstrdup(tplg->dev, pcm->dai_name, GFP_KERNEL); if (!dai_drv->name) { ret = -ENOMEM; @@ -1481,7 +1500,11 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg, if (tplg->ops) link->dobj.unload = tplg->ops->link_unload; - if (strlen(pcm->pcm_name)) { + ret = soc_tplg_check_name(pcm->pcm_name); + if (ret) + goto err; + + if (pcm->pcm_name[0]) { link->name = devm_kstrdup(tplg->dev, pcm->pcm_name, GFP_KERNEL); link->stream_name = devm_kstrdup(tplg->dev, pcm->pcm_name, GFP_KERNEL); if (!link->name || !link->stream_name) { @@ -1491,7 +1514,11 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg, } link->id = le32_to_cpu(pcm->pcm_id); - if (strlen(pcm->dai_name)) { + ret = soc_tplg_check_name(pcm->dai_name); + if (ret) + goto err; + + if (pcm->dai_name[0]) { link->cpus->dai_name = devm_kstrdup(tplg->dev, pcm->dai_name, GFP_KERNEL); if (!link->cpus->dai_name) { ret = -ENOMEM; @@ -1843,6 +1870,10 @@ static int soc_tplg_dai_config(struct soc_tplg *tplg, memset(&dai_component, 0, sizeof(dai_component)); + ret = soc_tplg_check_name(d->dai_name); + if (ret) + return ret; + dai_component.dai_name = d->dai_name; dai = snd_soc_find_dai(&dai_component); if (!dai) { diff --git a/sound/soc/sof/ipc4-pcm.c b/sound/soc/sof/ipc4-pcm.c index 6d81969e181c8..0c1d229c7bcbd 100644 --- a/sound/soc/sof/ipc4-pcm.c +++ b/sound/soc/sof/ipc4-pcm.c @@ -528,7 +528,19 @@ static int sof_ipc4_trigger_pipelines(struct snd_soc_component *component, ret = sof_ipc4_set_multi_pipeline_state(sdev, SOF_IPC4_PIPE_PAUSED, trigger_list); if (ret < 0) { spcm_err(spcm, substream->stream, "failed to pause all pipelines\n"); - goto free; + /* + * workaround: if the firmware is crashed or the IPC timed out + * while setting the pipeline state we must ignore the error + * code and proceed to set adjust the local pipeline states. + * + * If the firmware is crashed we will not send IPC messages + * and we are going to see errors printed, but the state of the + * widgets will be correct for the next boot. + */ + if (sdev->fw_state != SOF_FW_CRASHED && ret != -ETIMEDOUT) + goto free; + + ret = 0; } /* update PAUSED state for all pipelines just triggered */ @@ -560,14 +572,15 @@ static int sof_ipc4_trigger_pipelines(struct snd_soc_component *component, "failed to set final state %d for all pipelines\n", state); /* - * workaround: if the firmware is crashed while setting the - * pipelines to reset state we must ignore the error code and - * reset it to 0. - * Since the firmware is crashed we will not send IPC messages + * workaround: if the firmware is crashed or the IPC timed out + * while setting the pipeline state we must ignore the error + * code and proceed to set adjust the local pipeline states. + * + * If the firmware is crashed we will not send IPC messages * and we are going to see errors printed, but the state of the * widgets will be correct for the next boot. */ - if (sdev->fw_state != SOF_FW_CRASHED || state != SOF_IPC4_PIPE_RESET) + if (sdev->fw_state != SOF_FW_CRASHED && ret != -ETIMEDOUT) goto free; ret = 0; diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c index d74567d6afc41..5701e4d1ac130 100644 --- a/sound/soc/sof/ipc4-topology.c +++ b/sound/soc/sof/ipc4-topology.c @@ -3007,6 +3007,15 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget ipc_size = ipc4_copier->ipc_config_size; ipc_data = ipc4_copier->ipc_config_data; + /* + * Refresh copier_data in ipc_config_data for host copiers. + * The node_id may have been updated by host_config after + * ipc_prepare, e.g. when host stream tags change after a + * suspend/resume cycle. + */ + if (swidget->id != snd_soc_dapm_buffer) + memcpy(ipc_data, &ipc4_copier->data, sizeof(ipc4_copier->data)); + msg = &ipc4_copier->msg; break; } @@ -3015,6 +3024,9 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget { struct snd_sof_dai *dai = swidget->private; struct sof_ipc4_copier *ipc4_copier = dai->private; + struct sof_ipc4_copier_data *copier_data; + u32 gtw_cfg_config_length; + u32 tlv_size; pipeline = pipe_widget->private; if (pipeline->use_chain_dma) @@ -3023,6 +3035,27 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget ipc_size = ipc4_copier->ipc_config_size; ipc_data = ipc4_copier->ipc_config_data; + /* + * Refresh copier_data and dma_config_tlv in ipc_config_data. + * These may have been updated after ipc_prepare, e.g. when + * link DMA stream tags change after a suspend/resume cycle. + * + * copier_data->gtw_cfg.config_length does not include the + * TLV size (it was restored after sof_ipc4_prepare_copier_module), + * so temporarily inflate it to match the ipc_config_data layout. + */ + copier_data = &ipc4_copier->data; + gtw_cfg_config_length = copier_data->gtw_cfg.config_length * 4; + tlv_size = ipc_size - sizeof(*copier_data) - gtw_cfg_config_length; + + copier_data->gtw_cfg.config_length += tlv_size / 4; + memcpy(ipc_data, copier_data, sizeof(*copier_data)); + copier_data->gtw_cfg.config_length = gtw_cfg_config_length / 4; + + if (tlv_size) + memcpy(ipc_data + sizeof(*copier_data) + gtw_cfg_config_length, + &ipc4_copier->dma_config_tlv, tlv_size); + msg = &ipc4_copier->msg; break; } diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c index a9664b4cf43f9..ef7d2f4b31c81 100644 --- a/sound/soc/sof/sof-audio.c +++ b/sound/soc/sof/sof-audio.c @@ -136,7 +136,6 @@ static int sof_widget_setup_unlocked(struct snd_sof_dev *sdev, { const struct sof_ipc_tplg_ops *tplg_ops = sof_ipc_get_ops(sdev, tplg); struct snd_sof_pipeline *spipe = swidget->spipe; - bool use_count_decremented = false; int ret; int i; @@ -215,9 +214,10 @@ static int sof_widget_setup_unlocked(struct snd_sof_dev *sdev, return 0; widget_free: - /* widget use_count will be decremented by sof_widget_free() */ + /* widget use_count and core_put handled by sof_widget_free() */ sof_widget_free_unlocked(sdev, swidget); - use_count_decremented = true; + return ret; + pipe_widget_free: if (swidget->id != snd_soc_dapm_scheduler) { sof_widget_free_unlocked(sdev, swidget->spipe->pipe_widget); @@ -232,8 +232,7 @@ static int sof_widget_setup_unlocked(struct snd_sof_dev *sdev, } } use_count_dec: - if (!use_count_decremented) - swidget->use_count--; + swidget->use_count--; return ret; } diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index d6134df874ee5..5ddfa2cc51d18 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -839,6 +839,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, struct snd_soc_tplg_mixer_control *mc = container_of(hdr, struct snd_soc_tplg_mixer_control, hdr); int tlv[SOF_TLV_ITEMS]; + u32 min, max; unsigned int mask; int ret; @@ -846,6 +847,11 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, if (le32_to_cpu(mc->num_channels) > SND_SOC_TPLG_MAX_CHAN) return -EINVAL; + min = le32_to_cpu(mc->min); + max = le32_to_cpu(mc->max); + if (min > max || max >= INT_MAX) + return -EINVAL; + /* * If control has more than 2 channels we need to override the info. This is because even if * ASoC layer has defined topology's max channel count to SND_SOC_TPLG_MAX_CHAN = 8, the @@ -856,12 +862,12 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, kc->info = snd_sof_volume_info; scontrol->comp_id = sdev->next_comp_id; - scontrol->min_volume_step = le32_to_cpu(mc->min); - scontrol->max_volume_step = le32_to_cpu(mc->max); + scontrol->min_volume_step = min; + scontrol->max_volume_step = max; scontrol->num_channels = le32_to_cpu(mc->num_channels); - scontrol->max = le32_to_cpu(mc->max); - if (le32_to_cpu(mc->max) == 1) + scontrol->max = max; + if (max == 1) goto skip; /* extract tlv data */ @@ -871,7 +877,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, } /* set up volume table */ - ret = set_up_volume_table(scontrol, tlv, le32_to_cpu(mc->max) + 1); + ret = set_up_volume_table(scontrol, tlv, max + 1); if (ret < 0) { dev_err(scomp->dev, "error: setting up volume table\n"); return ret; @@ -904,7 +910,7 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, return 0; err: - if (le32_to_cpu(mc->max) > 1) + if (max > 1) kfree(scontrol->volume_table); return ret; diff --git a/sound/soc/sprd/sprd-pcm-compress.c b/sound/soc/sprd/sprd-pcm-compress.c index 4b6ebfa5b0339..c3f18c53d4bce 100644 --- a/sound/soc/sprd/sprd-pcm-compress.c +++ b/sound/soc/sprd/sprd-pcm-compress.c @@ -17,7 +17,7 @@ /* Default values if userspace does not set */ #define SPRD_COMPR_MIN_FRAGMENT_SIZE SZ_8K -#define SPRD_COMPR_MAX_FRAGMENT_SIZE SZ_128K +#define SPRD_COMPR_MAX_FRAGMENT_SIZE SZ_32K #define SPRD_COMPR_MIN_NUM_FRAGMENTS 4 #define SPRD_COMPR_MAX_NUM_FRAGMENTS 64 @@ -271,6 +271,19 @@ static int sprd_platform_compr_set_params(struct snd_soc_component *component, struct sprd_compr_params compr_params = { }; int ret; + /* + * The stage 0 IRAM buffer and the stage 1 DDR buffer are allocated + * with fixed sizes at open time, so the requested fragment size and + * fragments must fit into them, otherwise sprd_platform_compr_copy() + * would overflow the buffers. Note the compress core only checks the + * fragment size and fragments against an u32 overflow, not against + * the buffer sizes advertised by get_caps. + */ + if (params->buffer.fragment_size > SPRD_COMPR_IRAM_BUF_SIZE || + (u64)params->buffer.fragment_size * params->buffer.fragments > + SPRD_COMPR_AREA_BUF_SIZE) + return -EINVAL; + /* * Configure the DMA engine 2-stage transfer mode. Channel 1 set as the * destination channel, and channel 0 set as the source channel, that diff --git a/sound/soc/sti/uniperif_reader.c b/sound/soc/sti/uniperif_reader.c index 05ea2b794eb92..4058e542bce0c 100644 --- a/sound/soc/sti/uniperif_reader.c +++ b/sound/soc/sti/uniperif_reader.c @@ -422,6 +422,8 @@ int uni_reader_init(struct platform_device *pdev, else reader->hw = &uni_reader_pcm_hw; + spin_lock_init(&reader->irq_lock); + ret = devm_request_irq(&pdev->dev, reader->irq, uni_reader_irq_handler, IRQF_SHARED, dev_name(&pdev->dev), reader); @@ -430,8 +432,6 @@ int uni_reader_init(struct platform_device *pdev, return -EBUSY; } - spin_lock_init(&reader->irq_lock); - return 0; } EXPORT_SYMBOL_GPL(uni_reader_init); diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c index 93733ff2e32a0..3b92318e18ba6 100644 --- a/sound/soc/sunxi/sun4i-codec.c +++ b/sound/soc/sunxi/sun4i-codec.c @@ -2236,6 +2236,7 @@ static const struct sun4i_codec_quirks sun50i_h616_codec_quirks = { .reg_dac_fifoc = REG_FIELD(SUN50I_H616_CODEC_DAC_FIFOC, 0, 31), .reg_dac_txdata = SUN8I_H3_CODEC_DAC_TXDATA, .has_reset = true, + .playback_only = true, .dma_max_burst = SUN4I_DMA_MAX_BURST, }; diff --git a/sound/soc/ti/omap3pandora.c b/sound/soc/ti/omap3pandora.c index 808fb6765c05f..2ddd14e8f6b6c 100644 --- a/sound/soc/ti/omap3pandora.c +++ b/sound/soc/ti/omap3pandora.c @@ -11,12 +11,12 @@ #include #include #include +#include #include #include #include -#include #include #include "omap-mcbsp.h" @@ -223,7 +223,8 @@ static int __init omap3pandora_soc_init(void) { int ret; - if (!machine_is_omap3_pandora()) + if (!of_machine_is_compatible("openpandora,omap3-pandora-600mhz") && + !of_machine_is_compatible("openpandora,omap3-pandora-1ghz")) return -ENODEV; pr_info("OMAP3 Pandora SoC init\n"); diff --git a/sound/soc/ux500/ux500_msp_dai.c b/sound/soc/ux500/ux500_msp_dai.c index 7798957c6504d..37c48cc70394e 100644 --- a/sound/soc/ux500/ux500_msp_dai.c +++ b/sound/soc/ux500/ux500_msp_dai.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include @@ -34,8 +34,10 @@ static int setup_pcm_multichan(struct snd_soc_dai *dai, if (drvdata->slots > 1) { msp_config->multichannel_configured = 1; - multi->tx_multichannel_enable = true; - multi->rx_multichannel_enable = true; + multi->tx_multichannel_enable = + msp_config->direction & MSP_DIR_TX; + multi->rx_multichannel_enable = + msp_config->direction & MSP_DIR_RX; multi->rx_comparison_enable_mode = MSP_COMPARISON_DISABLED; multi->tx_channel_0_enable = drvdata->tx_mask; @@ -57,72 +59,21 @@ static int setup_pcm_multichan(struct snd_soc_dai *dai, return 0; } -static int setup_frameper(struct snd_soc_dai *dai, unsigned int rate, - struct msp_protdesc *prot_desc) +static void setup_frameper(struct snd_soc_dai *dai, + struct msp_protdesc *prot_desc) { struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); - switch (drvdata->slots) { - case 1: - switch (rate) { - case 8000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_8_KHZ; - break; - - case 16000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_16_KHZ; - break; - - case 44100: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_44_1_KHZ; - break; - - case 48000: - prot_desc->frame_period = - FRAME_PER_SINGLE_SLOT_48_KHZ; - break; - - default: - dev_err(dai->dev, - "%s: Error: Unsupported sample-rate (freq = %d)!\n", - __func__, rate); - return -EINVAL; - } - break; - - case 2: - prot_desc->frame_period = FRAME_PER_2_SLOTS; - break; - - case 8: - prot_desc->frame_period = FRAME_PER_8_SLOTS; - break; - - case 16: - prot_desc->frame_period = FRAME_PER_16_SLOTS; - break; - default: - dev_err(dai->dev, - "%s: Error: Unsupported slot-count (slots = %d)!\n", - __func__, drvdata->slots); - return -EINVAL; - } - - prot_desc->clocks_per_frame = - prot_desc->frame_period+1; + prot_desc->clocks_per_frame = drvdata->slots * drvdata->slot_width; + prot_desc->frame_period = prot_desc->clocks_per_frame - 1; dev_dbg(dai->dev, "%s: Clocks per frame: %u\n", __func__, prot_desc->clocks_per_frame); - - return 0; } -static int setup_pcm_framing(struct snd_soc_dai *dai, unsigned int rate, - struct msp_protdesc *prot_desc) +static int setup_pcm_framing(struct snd_soc_dai *dai, + struct msp_protdesc *prot_desc) { struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); @@ -163,7 +114,9 @@ static int setup_pcm_framing(struct snd_soc_dai *dai, unsigned int rate, prot_desc->tx_elem_len_2 = MSP_ELEM_LEN_16; prot_desc->rx_elem_len_2 = MSP_ELEM_LEN_16; - return setup_frameper(dai, rate, prot_desc); + setup_frameper(dai, prot_desc); + + return 0; } static int setup_clocking(struct snd_soc_dai *dai, @@ -177,7 +130,16 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_NB_IF: msp_config->tx_fsync_pol ^= 1 << TFSPOL_SHIFT; msp_config->rx_fsync_pol ^= 1 << RFSPOL_SHIFT; + break; + + case SND_SOC_DAIFMT_IB_NF: + msp_config->bclk_inverted = true; + break; + case SND_SOC_DAIFMT_IB_IF: + msp_config->bclk_inverted = true; + msp_config->tx_fsync_pol ^= 1 << TFSPOL_SHIFT; + msp_config->rx_fsync_pol ^= 1 << RFSPOL_SHIFT; break; default: @@ -192,6 +154,7 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_BC_FC: dev_dbg(dai->dev, "%s: Codec is master.\n", __func__); + msp_config->clock_provider = false; msp_config->iodelay = 0x20; msp_config->rx_fsync_sel = 0; msp_config->tx_fsync_sel = 1 << TFSSEL_SHIFT; @@ -204,6 +167,7 @@ static int setup_clocking(struct snd_soc_dai *dai, case SND_SOC_DAIFMT_BP_FP: dev_dbg(dai->dev, "%s: Codec is slave.\n", __func__); + msp_config->clock_provider = true; msp_config->tx_clk_sel = TX_CLK_SEL_SRG; msp_config->tx_fsync_sel = TX_SYNC_SRG_PROG; msp_config->rx_clk_sel = RX_CLK_SEL_SRG; @@ -362,7 +326,7 @@ static int setup_msp_config(struct snd_pcm_substream *substream, if (ret < 0) return ret; - ret = setup_pcm_framing(dai, runtime->rate, prot_desc); + ret = setup_pcm_framing(dai, prot_desc); if (ret < 0) return ret; @@ -424,21 +388,21 @@ static void ux500_msp_dai_shutdown(struct snd_pcm_substream *substream, int ret; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); bool is_playback = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK); + unsigned int configured = is_playback ? PLAYBACK_CONFIGURED : + CAPTURE_CONFIGURED; + unsigned int dir = is_playback ? MSP_DIR_TX : MSP_DIR_RX; dev_dbg(dai->dev, "%s: MSP %d (%s): Enter.\n", __func__, dai->id, snd_pcm_stream_str(substream)); - if (drvdata->vape_opp_constraint == 1) { - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500_msp_i2s", 50); - drvdata->vape_opp_constraint = 0; - } - - if (ux500_msp_i2s_close(drvdata->msp, - is_playback ? MSP_DIR_TX : MSP_DIR_RX)) { - dev_err(dai->dev, - "%s: Error: MSP %d (%s): Unable to close i2s.\n", - __func__, dai->id, snd_pcm_stream_str(substream)); + if (drvdata->configured & configured) { + if (ux500_msp_i2s_close(drvdata->msp, dir)) { + dev_err(dai->dev, + "%s: Error: MSP %d (%s): Unable to close i2s.\n", + __func__, dai->id, + snd_pcm_stream_str(substream)); + } + drvdata->configured &= ~configured; } /* Disable and unprepare clocks */ @@ -456,15 +420,23 @@ static void ux500_msp_dai_shutdown(struct snd_pcm_substream *substream, static int ux500_msp_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { - int ret = 0; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); struct snd_pcm_runtime *runtime = substream->runtime; struct ux500_msp_config msp_config; + bool is_playback = substream->stream == SNDRV_PCM_STREAM_PLAYBACK; + unsigned int configured = is_playback ? PLAYBACK_CONFIGURED : + CAPTURE_CONFIGURED; + int ret; dev_dbg(dai->dev, "%s: MSP %d (%s): Enter (rate = %d).\n", __func__, dai->id, snd_pcm_stream_str(substream), runtime->rate); - setup_msp_config(substream, dai, &msp_config); + if (drvdata->configured & configured) + return 0; + + ret = setup_msp_config(substream, dai, &msp_config); + if (ret) + return ret; ret = ux500_msp_i2s_open(drvdata->msp, &msp_config); if (ret < 0) { @@ -473,22 +445,9 @@ static int ux500_msp_dai_prepare(struct snd_pcm_substream *substream, return ret; } - /* Set OPP-level */ - if ((drvdata->fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) && - (drvdata->msp->f_bitclk > 19200000)) { - /* If the bit-clock is higher than 19.2MHz, Vape should be - * run in 100% OPP. Only when bit-clock is used (MSP master) - */ - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500-msp-i2s", 100); - drvdata->vape_opp_constraint = 1; - } else { - prcmu_qos_update_requirement(PRCMU_QOS_APE_OPP, - "ux500-msp-i2s", 50); - drvdata->vape_opp_constraint = 0; - } + drvdata->configured |= configured; - return ret; + return 0; } static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, @@ -496,7 +455,6 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { unsigned int mask, slots_active; - struct snd_pcm_runtime *runtime = substream->runtime; struct ux500_msp_i2s_drvdata *drvdata = dev_get_drvdata(dai->dev); dev_dbg(dai->dev, "%s: MSP %d (%s): Enter.\n", @@ -504,9 +462,8 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, switch (drvdata->fmt & SND_SOC_DAIFMT_FORMAT_MASK) { case SND_SOC_DAIFMT_I2S: - snd_pcm_hw_constraint_minmax(runtime, - SNDRV_PCM_HW_PARAM_CHANNELS, - 1, 2); + if (params_channels(params) < 1 || params_channels(params) > 2) + return -EINVAL; break; case SND_SOC_DAIFMT_DSP_B: @@ -518,9 +475,8 @@ static int ux500_msp_dai_hw_params(struct snd_pcm_substream *substream, slots_active = hweight32(mask); dev_dbg(dai->dev, "TDM-slots active: %d", slots_active); - snd_pcm_hw_constraint_single(runtime, - SNDRV_PCM_HW_PARAM_CHANNELS, - slots_active); + if (!slots_active || params_channels(params) != slots_active) + return -EINVAL; break; default: @@ -553,20 +509,21 @@ static int ux500_msp_dai_set_dai_fmt(struct snd_soc_dai *dai, default: dev_err(dai->dev, "%s: Error: Unsupported protocol/master (fmt = 0x%x)!\n", - __func__, drvdata->fmt); + __func__, fmt); return -EINVAL; } switch (fmt & SND_SOC_DAIFMT_INV_MASK) { case SND_SOC_DAIFMT_NB_NF: case SND_SOC_DAIFMT_NB_IF: + case SND_SOC_DAIFMT_IB_NF: case SND_SOC_DAIFMT_IB_IF: break; default: dev_err(dai->dev, "%s: Error: Unsupported inversion (fmt = 0x%x)!\n", - __func__, drvdata->fmt); + __func__, fmt); return -EINVAL; } @@ -600,17 +557,23 @@ static int ux500_msp_dai_set_tdm_slot(struct snd_soc_dai *dai, __func__, slots); return -EINVAL; } - drvdata->slots = slots; - if (!(slot_width == 16)) { + if (slot_width != 16) { dev_err(dai->dev, "%s: Error: Unsupported slot-width (%d)!\n", __func__, slot_width); return -EINVAL; } - drvdata->slot_width = slot_width; - drvdata->tx_mask = tx_mask & cap; - drvdata->rx_mask = rx_mask & cap; + if ((tx_mask | rx_mask) & ~cap) { + dev_err(dai->dev, "%s: Slot mask exceeds %d slots\n", + __func__, slots); + return -EINVAL; + } + + drvdata->slots = slots; + drvdata->slot_width = slot_width; + drvdata->tx_mask = tx_mask; + drvdata->rx_mask = rx_mask; return 0; } @@ -716,6 +679,7 @@ static const struct snd_soc_component_driver ux500_msp_component = { static int ux500_msp_drv_probe(struct platform_device *pdev) { struct ux500_msp_i2s_drvdata *drvdata; + struct reset_control *reset; int ret = 0; drvdata = devm_kzalloc(&pdev->dev, @@ -729,7 +693,6 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) drvdata->tx_mask = 0x01; drvdata->rx_mask = 0x01; drvdata->slot_width = 16; - drvdata->master_clk = MSP_INPUT_FREQ_APB; drvdata->reg_vape = devm_regulator_get(&pdev->dev, "v-ape"); if (IS_ERR(drvdata->reg_vape)) { @@ -739,8 +702,6 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) __func__, ret); return ret; } - prcmu_qos_add_requirement(PRCMU_QOS_APE_OPP, (char *)pdev->name, 50); - drvdata->pclk = devm_clk_get(&pdev->dev, "apb_pclk"); if (IS_ERR(drvdata->pclk)) { ret = PTR_ERR(drvdata->pclk); @@ -758,9 +719,19 @@ static int ux500_msp_drv_probe(struct platform_device *pdev) __func__, ret); return ret; } + drvdata->master_clk = clk_get_rate(drvdata->clk); + if (!drvdata->master_clk) { + dev_err(&pdev->dev, "MSP clock has no rate\n"); + return -EINVAL; + } + + reset = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(reset), + "Failed to deassert MSP reset\n"); ret = ux500_msp_i2s_init_msp(pdev, &drvdata->msp); - if (!drvdata->msp) { + if (ret) { dev_err(&pdev->dev, "%s: ERROR: Failed to init MSP-struct (%d)!", __func__, ret); @@ -799,8 +770,6 @@ static void ux500_msp_drv_remove(struct platform_device *pdev) snd_soc_unregister_component(&pdev->dev); - prcmu_qos_remove_requirement(PRCMU_QOS_APE_OPP, "ux500_msp_i2s"); - ux500_msp_i2s_cleanup_msp(pdev, drvdata->msp); } diff --git a/sound/soc/ux500/ux500_msp_dai.h b/sound/soc/ux500/ux500_msp_dai.h index 30bf708381961..aae582030d959 100644 --- a/sound/soc/ux500/ux500_msp_dai.h +++ b/sound/soc/ux500/ux500_msp_dai.h @@ -22,17 +22,6 @@ #define UX500_I2S_FORMATS (SNDRV_PCM_FMTBIT_S16_LE) -#define FRAME_PER_SINGLE_SLOT_8_KHZ 31 -#define FRAME_PER_SINGLE_SLOT_16_KHZ 124 -#define FRAME_PER_SINGLE_SLOT_44_1_KHZ 63 -#define FRAME_PER_SINGLE_SLOT_48_KHZ 49 -#define FRAME_PER_2_SLOTS 31 -#define FRAME_PER_8_SLOTS 138 -#define FRAME_PER_16_SLOTS 277 - -#define UX500_MSP_INTERNAL_CLOCK_FREQ 40000000 -#define UX500_MSP1_INTERNAL_CLOCK_FREQ UX500_MSP_INTERNAL_CLOCK_FREQ - #define UX500_MSP_MIN_CHANNELS 1 #define UX500_MSP_MAX_CHANNELS 8 @@ -47,6 +36,7 @@ struct ux500_msp_i2s_drvdata { struct ux500_msp *msp; struct regulator *reg_vape; unsigned int fmt; + unsigned int configured; unsigned int tx_mask; unsigned int rx_mask; int slots; @@ -57,8 +47,6 @@ struct ux500_msp_i2s_drvdata { struct clk *clk; struct clk *pclk; - /* Regulators */ - int vape_opp_constraint; }; int ux500_msp_dai_set_data_delay(struct snd_soc_dai *dai, int delay); diff --git a/sound/soc/ux500/ux500_msp_i2s.c b/sound/soc/ux500/ux500_msp_i2s.c index fbfeefa418ca7..683b485fb5708 100644 --- a/sound/soc/ux500/ux500_msp_i2s.c +++ b/sound/soc/ux500/ux500_msp_i2s.c @@ -201,10 +201,12 @@ static int configure_protocol(struct ux500_msp *msp, /* The code below should not be separated. */ temp_reg = readl(msp->registers + MSP_GCR) & ~TX_CLK_POL_RISING; - temp_reg |= MSP_TX_CLKPOL_BIT(~protdesc->tx_clk_pol); + temp_reg |= MSP_TX_CLKPOL_BIT(!protdesc->tx_clk_pol ^ + config->bclk_inverted); writel(temp_reg, msp->registers + MSP_GCR); temp_reg = readl(msp->registers + MSP_GCR) & ~RX_CLK_POL_RISING; - temp_reg |= MSP_RX_CLKPOL_BIT(protdesc->rx_clk_pol); + temp_reg |= MSP_RX_CLKPOL_BIT(protdesc->rx_clk_pol ^ + config->bclk_inverted); writel(temp_reg, msp->registers + MSP_GCR); return 0; @@ -212,35 +214,20 @@ static int configure_protocol(struct ux500_msp *msp, static int setup_bitclk(struct ux500_msp *msp, struct ux500_msp_config *config) { + struct msp_protdesc *protdesc; + u64 desired_bitclk; + unsigned int bitclk; u32 reg_val_GCR; - u32 frame_per = 0; - u32 sck_div = 0; - u32 frame_width = 0; - u32 temp_reg = 0; - struct msp_protdesc *protdesc = NULL; + u32 sck_div; + u32 temp_reg; reg_val_GCR = readl(msp->registers + MSP_GCR); writel(reg_val_GCR & ~SRG_ENABLE, msp->registers + MSP_GCR); - if (config->default_protdesc) - protdesc = - (struct msp_protdesc *)&prot_descs[config->protocol]; - else - protdesc = (struct msp_protdesc *)&config->protdesc; - switch (config->protocol) { case MSP_PCM_PROTOCOL: case MSP_PCM_COMPAND_PROTOCOL: - frame_width = protdesc->frame_width; - sck_div = config->f_inputclk / (config->frame_freq * - (protdesc->clocks_per_frame)); - frame_per = protdesc->frame_period; - break; case MSP_I2S_PROTOCOL: - frame_width = protdesc->frame_width; - sck_div = config->f_inputclk / (config->frame_freq * - (protdesc->clocks_per_frame)); - frame_per = protdesc->frame_period; break; default: dev_err(msp->dev, "%s: ERROR: Unknown protocol (%d)!\n", @@ -249,12 +236,35 @@ static int setup_bitclk(struct ux500_msp *msp, struct ux500_msp_config *config) return -EINVAL; } + if (config->default_protdesc) + protdesc = (struct msp_protdesc *)&prot_descs[config->protocol]; + else + protdesc = &config->protdesc; + + if (!config->frame_freq || !protdesc->clocks_per_frame) + return -EINVAL; + + desired_bitclk = (u64)config->frame_freq * protdesc->clocks_per_frame; + if (desired_bitclk > config->f_inputclk) + return -EINVAL; + bitclk = desired_bitclk; + if (config->f_inputclk % bitclk) { + dev_err(msp->dev, + "Input clock %u cannot generate bit clock %u\n", + config->f_inputclk, bitclk); + return -EINVAL; + } + + sck_div = config->f_inputclk / bitclk; + if (!sck_div || sck_div > SCK_DIV_MASK + 1) + return -EINVAL; + temp_reg = (sck_div - 1) & SCK_DIV_MASK; - temp_reg |= FRAME_WIDTH_BITS(frame_width); - temp_reg |= FRAME_PERIOD_BITS(frame_per); + temp_reg |= FRAME_WIDTH_BITS(protdesc->frame_width); + temp_reg |= FRAME_PERIOD_BITS(protdesc->frame_period); writel(temp_reg, msp->registers + MSP_SRG); - msp->f_bitclk = (config->f_inputclk)/(sck_div + 1); + msp->f_bitclk = config->f_inputclk / sck_div; /* Enable bit-clock */ udelay(100); @@ -344,20 +354,27 @@ static int configure_multichannel(struct ux500_msp *msp, return 0; } -static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config) +static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config, + bool first) { - int status = 0; - u32 reg_val_DMACR, reg_val_GCR; + int status; + u32 reg_val_DMACR; /* Configure msp with protocol dependent settings */ - configure_protocol(msp, config); - setup_bitclk(msp, config); + status = configure_protocol(msp, config); + if (status) + return status; + + if (first && config->clock_provider) { + status = setup_bitclk(msp, config); + if (status) + return status; + } + if (config->multichannel_configured == 1) { status = configure_multichannel(msp, config); if (status) - dev_warn(msp->dev, - "%s: WARN: configure_multichannel failed (%d)!\n", - __func__, status); + return status; } reg_val_DMACR = readl(msp->registers + MSP_DMACR); @@ -369,11 +386,7 @@ static int enable_msp(struct ux500_msp *msp, struct ux500_msp_config *config) writel(config->iodelay, msp->registers + MSP_IODLY); - /* Enable frame generation logic */ - reg_val_GCR = readl(msp->registers + MSP_GCR); - writel(reg_val_GCR | FRAME_GEN_ENABLE, msp->registers + MSP_GCR); - - return status; + return 0; } static void flush_fifo_rx(struct ux500_msp *msp) @@ -411,12 +424,37 @@ static void flush_fifo_tx(struct ux500_msp *msp) writel(reg_val_GCR, msp->registers + MSP_GCR); } +static bool ux500_msp_config_compatible(struct ux500_msp *msp, + struct ux500_msp_config *config) +{ + struct ux500_msp_config *active = &msp->config; + + return active->f_inputclk == config->f_inputclk && + active->tx_clk_sel == config->tx_clk_sel && + active->rx_clk_sel == config->rx_clk_sel && + active->srg_clk_sel == config->srg_clk_sel && + active->rx_fsync_pol == config->rx_fsync_pol && + active->tx_fsync_pol == config->tx_fsync_pol && + active->rx_fsync_sel == config->rx_fsync_sel && + active->tx_fsync_sel == config->tx_fsync_sel && + active->default_protdesc == config->default_protdesc && + active->protocol == config->protocol && + active->frame_freq == config->frame_freq && + active->data_size == config->data_size && + active->def_elem_len == config->def_elem_len && + active->clock_provider == config->clock_provider && + active->bclk_inverted == config->bclk_inverted && + !memcmp(&active->protdesc, &config->protdesc, + sizeof(active->protdesc)); +} + int ux500_msp_i2s_open(struct ux500_msp *msp, struct ux500_msp_config *config) { u32 old_reg, new_reg, mask; int res; unsigned int tx_sel, rx_sel, tx_busy, rx_busy; + bool first; if (in_interrupt()) { dev_err(msp->dev, @@ -444,40 +482,68 @@ int ux500_msp_i2s_open(struct ux500_msp *msp, return -EBUSY; } - msp->dir_busy |= (tx_sel ? MSP_DIR_TX : 0) | (rx_sel ? MSP_DIR_RX : 0); - - /* First do the global config register */ - mask = RX_CLK_SEL_MASK | TX_CLK_SEL_MASK | RX_FSYNC_MASK | - TX_FSYNC_MASK | RX_SYNC_SEL_MASK | TX_SYNC_SEL_MASK | - RX_FIFO_ENABLE_MASK | TX_FIFO_ENABLE_MASK | SRG_CLK_SEL_MASK | - LOOPBACK_MASK | TX_EXTRA_DELAY_MASK; - - new_reg = (config->tx_clk_sel | config->rx_clk_sel | - config->rx_fsync_pol | config->tx_fsync_pol | - config->rx_fsync_sel | config->tx_fsync_sel | - config->rx_fifo_config | config->tx_fifo_config | - config->srg_clk_sel | config->loopback_enable | - config->tx_data_enable); + first = !msp->dir_busy; + if (!first && !ux500_msp_config_compatible(msp, config)) { + dev_err(msp->dev, "%s: Incompatible duplex configuration\n", + __func__); + return -EBUSY; + } - old_reg = readl(msp->registers + MSP_GCR); - old_reg &= ~mask; - new_reg |= old_reg; - writel(new_reg, msp->registers + MSP_GCR); + if (first) { + /* First do the global config register */ + mask = RX_CLK_SEL_MASK | TX_CLK_SEL_MASK | RX_FSYNC_MASK | + TX_FSYNC_MASK | RX_SYNC_SEL_MASK | TX_SYNC_SEL_MASK | + RX_FIFO_ENABLE_MASK | TX_FIFO_ENABLE_MASK | + SRG_CLK_SEL_MASK | LOOPBACK_MASK | TX_EXTRA_DELAY_MASK; + + new_reg = config->tx_clk_sel | config->rx_clk_sel | + config->rx_fsync_pol | config->tx_fsync_pol | + config->rx_fsync_sel | config->tx_fsync_sel | + config->rx_fifo_config | config->tx_fifo_config | + config->srg_clk_sel | config->loopback_enable | + config->tx_data_enable; + + old_reg = readl(msp->registers + MSP_GCR); + old_reg &= ~mask; + new_reg |= old_reg; + writel(new_reg, msp->registers + MSP_GCR); + writel(MSP_WMRK_TX_4_ELEMENTS | MSP_WMRK_RX_4_ELEMENTS, + msp->registers + MSP_WMRK); + } - res = enable_msp(msp, config); + res = enable_msp(msp, config, first); if (res < 0) { dev_err(msp->dev, "%s: ERROR: enable_msp failed (%d)!\n", __func__, res); - return -EBUSY; + if (tx_sel) + writel(0, msp->registers + MSP_TCF); + if (rx_sel) + writel(0, msp->registers + MSP_RCF); + if (first) { + writel(0, msp->registers + MSP_GCR); + writel(0, msp->registers + MSP_DMACR); + writel(0, msp->registers + MSP_SRG); + writel(0, msp->registers + MSP_MCR); + } + return res; + } + + msp->dir_busy |= config->direction; + if (first) { + msp->config = *config; + msp->clock_provider = config->clock_provider; } if (config->loopback_enable & 0x80) msp->loopback_enable = 1; /* Flush FIFOs */ - flush_fifo_tx(msp); - flush_fifo_rx(msp); + if (tx_sel) + flush_fifo_tx(msp); + if (rx_sel) + flush_fifo_rx(msp); - msp->msp_state = MSP_STATE_CONFIGURED; + if (!msp->dir_running) + msp->msp_state = MSP_STATE_CONFIGURED; return 0; } @@ -494,7 +560,6 @@ static void disable_msp_rx(struct ux500_msp *msp) ~(RX_SERVICE_INT | RX_OVERRUN_ERROR_INT), msp->registers + MSP_IMSC); - msp->dir_busy &= ~MSP_DIR_RX; } static void disable_msp_tx(struct ux500_msp *msp) @@ -510,7 +575,6 @@ static void disable_msp_tx(struct ux500_msp *msp) ~(TX_SERVICE_INT | TX_UNDERRUN_ERR_INT), msp->registers + MSP_IMSC); - msp->dir_busy &= ~MSP_DIR_TX; } static int disable_msp(struct ux500_msp *msp, unsigned int dir) @@ -520,7 +584,7 @@ static int disable_msp(struct ux500_msp *msp, unsigned int dir) reg_val_GCR = readl(msp->registers + MSP_GCR); disable_tx = dir & MSP_DIR_TX; - disable_rx = dir & MSP_DIR_TX; + disable_rx = dir & MSP_DIR_RX; if (disable_tx && disable_rx) { reg_val_GCR = readl(msp->registers + MSP_GCR); writel(reg_val_GCR | LOOPBACK_MASK, @@ -553,7 +617,15 @@ static int disable_msp(struct ux500_msp *msp, unsigned int dir) int ux500_msp_i2s_trigger(struct ux500_msp *msp, int cmd, int direction) { - u32 reg_val_GCR, enable_bit; + u32 reg_val_DMACR, reg_val_GCR, dma_enable_bit, enable_bit; + unsigned int dir; + + if (direction == SNDRV_PCM_STREAM_PLAYBACK) + dir = MSP_DIR_TX; + else if (direction == SNDRV_PCM_STREAM_CAPTURE) + dir = MSP_DIR_RX; + else + return -EINVAL; if (msp->msp_state == MSP_STATE_IDLE) { dev_err(msp->dev, "%s: ERROR: MSP is not configured!\n", @@ -565,21 +637,44 @@ int ux500_msp_i2s_trigger(struct ux500_msp *msp, int cmd, int direction) case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: - if (direction == SNDRV_PCM_STREAM_PLAYBACK) + if (direction == SNDRV_PCM_STREAM_PLAYBACK) { enable_bit = TX_ENABLE; - else + dma_enable_bit = TX_DMA_ENABLE; + } else { enable_bit = RX_ENABLE; + dma_enable_bit = RX_DMA_ENABLE; + } + if (!(msp->dir_busy & dir)) + return -EINVAL; + reg_val_DMACR = readl(msp->registers + MSP_DMACR); + writel(reg_val_DMACR | dma_enable_bit, + msp->registers + MSP_DMACR); reg_val_GCR = readl(msp->registers + MSP_GCR); + if (msp->clock_provider) + enable_bit |= FRAME_GEN_ENABLE; writel(reg_val_GCR | enable_bit, msp->registers + MSP_GCR); + msp->dir_running |= dir; + msp->msp_state = MSP_STATE_RUNNING; break; case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - if (direction == SNDRV_PCM_STREAM_PLAYBACK) + if (!(msp->dir_busy & dir)) + return -EINVAL; + if (direction == SNDRV_PCM_STREAM_PLAYBACK) { disable_msp_tx(msp); - else + msp->dir_running &= ~MSP_DIR_TX; + } else { disable_msp_rx(msp); + msp->dir_running &= ~MSP_DIR_RX; + } + if (!msp->dir_running) { + reg_val_GCR = readl(msp->registers + MSP_GCR); + writel(reg_val_GCR & ~FRAME_GEN_ENABLE, + msp->registers + MSP_GCR); + msp->msp_state = MSP_STATE_CONFIGURED; + } break; default: return -EINVAL; @@ -594,7 +689,18 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) dev_dbg(msp->dev, "%s: Enter (dir = 0x%01x).\n", __func__, dir); + if (!dir || dir & ~(MSP_DIR_TX | MSP_DIR_RX) || + (msp->dir_busy & dir) != dir) + return -EINVAL; + status = disable_msp(msp, dir); + msp->dir_busy &= ~dir; + msp->dir_running &= ~dir; + if (msp->dir_busy && !msp->dir_running) { + writel(readl(msp->registers + MSP_GCR) & ~FRAME_GEN_ENABLE, + msp->registers + MSP_GCR); + msp->msp_state = MSP_STATE_CONFIGURED; + } if (msp->dir_busy == 0) { /* disable sample rate and frame generators */ msp->msp_state = MSP_STATE_IDLE; @@ -618,6 +724,8 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) writel(0, msp->registers + MSP_RCE1); writel(0, msp->registers + MSP_RCE2); writel(0, msp->registers + MSP_RCE3); + memset(&msp->config, 0, sizeof(msp->config)); + msp->clock_provider = false; } return status; @@ -627,7 +735,7 @@ int ux500_msp_i2s_close(struct ux500_msp *msp, unsigned int dir) int ux500_msp_i2s_init_msp(struct platform_device *pdev, struct ux500_msp **msp_p) { - struct resource *res = NULL; + struct resource *res; struct ux500_msp *msp; *msp_p = devm_kzalloc(&pdev->dev, sizeof(struct ux500_msp), GFP_KERNEL); @@ -637,20 +745,10 @@ int ux500_msp_i2s_init_msp(struct platform_device *pdev, msp->dev = &pdev->dev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res == NULL) { - dev_err(&pdev->dev, "%s: ERROR: Unable to get resource!\n", - __func__); - return -ENOMEM; - } - + msp->registers = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(msp->registers)) + return PTR_ERR(msp->registers); msp->tx_rx_addr = res->start + MSP_DR; - msp->registers = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (msp->registers == NULL) { - dev_err(&pdev->dev, "%s: ERROR: ioremap failed!\n", __func__); - return -ENOMEM; - } msp->msp_state = MSP_STATE_IDLE; msp->loopback_enable = 0; diff --git a/sound/soc/ux500/ux500_msp_i2s.h b/sound/soc/ux500/ux500_msp_i2s.h index 69d4ebc409fc1..2bf2699bdc49f 100644 --- a/sound/soc/ux500/ux500_msp_i2s.h +++ b/sound/soc/ux500/ux500_msp_i2s.h @@ -12,8 +12,6 @@ #include -#define MSP_INPUT_FREQ_APB 48000000 - /*** Stereo mode. Used for APB data accesses as 16 bits accesses (mono), * 32 bits accesses (stereo). ***/ @@ -64,6 +62,7 @@ enum msp_direction { #define MSP_SRG 0x10 #define MSP_FLR 0x14 #define MSP_DMACR 0x18 +#define MSP_WMRK 0x1c #define MSP_IMSC 0x20 #define MSP_RIS 0x24 @@ -230,6 +229,10 @@ enum msp_direction { #define RDMAE_SHIFT 0 #define TDMAE_SHIFT 1 +/* FIFO watermark register */ +#define MSP_WMRK_RX_4_ELEMENTS BIT(0) +#define MSP_WMRK_TX_4_ELEMENTS BIT(3) + /* Interrupt Register */ #define RX_SERVICE_INT BIT(0) #define RX_OVERRUN_ERROR_INT BIT(1) @@ -460,6 +463,8 @@ struct ux500_msp_config { enum msp_data_size data_size; unsigned int def_elem_len; unsigned int iodelay; + bool clock_provider; + bool bclk_inverted; }; struct ux500_msp { @@ -470,8 +475,11 @@ struct ux500_msp { enum msp_state msp_state; int def_elem_len; unsigned int dir_busy; + unsigned int dir_running; int loopback_enable; unsigned int f_bitclk; + bool clock_provider; + struct ux500_msp_config config; }; int ux500_msp_i2s_init_msp(struct platform_device *pdev, diff --git a/sound/soc/xilinx/xlnx_formatter_pcm.c b/sound/soc/xilinx/xlnx_formatter_pcm.c index 17ef053094694..4d90e96a677c9 100644 --- a/sound/soc/xilinx/xlnx_formatter_pcm.c +++ b/sound/soc/xilinx/xlnx_formatter_pcm.c @@ -281,8 +281,7 @@ static irqreturn_t xlnx_mm2s_irq_handler(int irq, void *arg) { u32 val; void __iomem *reg; - struct device *dev = arg; - struct xlnx_pcm_drv_data *adata = dev_get_drvdata(dev); + struct xlnx_pcm_drv_data *adata = arg; reg = adata->mmio + XLNX_MM2S_OFFSET + XLNX_AUD_STS; val = readl(reg); @@ -300,8 +299,7 @@ static irqreturn_t xlnx_s2mm_irq_handler(int irq, void *arg) { u32 val; void __iomem *reg; - struct device *dev = arg; - struct xlnx_pcm_drv_data *adata = dev_get_drvdata(dev); + struct xlnx_pcm_drv_data *adata = arg; reg = adata->mmio + XLNX_S2MM_OFFSET + XLNX_AUD_STS; val = readl(reg); @@ -386,7 +384,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on period bytes\n"); - return err; + goto error; } /* Resize the buffer bytes as divisible by 64 */ @@ -396,7 +394,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err) { dev_err(component->dev, "Unable to set constraint on buffer bytes\n"); - return err; + goto error; } /* Set periods as integer multiple */ @@ -405,7 +403,7 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, if (err < 0) { dev_err(component->dev, "Unable to set constraint on periods to be integer\n"); - return err; + goto error; } /* enable DMA IOC irq */ @@ -414,6 +412,14 @@ static int xlnx_formatter_pcm_open(struct snd_soc_component *component, writel(val, stream_data->mmio + XLNX_AUD_CTRL); return 0; + +error: + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) + adata->play_stream = NULL; + else + adata->capture_stream = NULL; + kfree(stream_data); + return err; } static int xlnx_formatter_pcm_close(struct snd_soc_component *component, @@ -637,7 +643,7 @@ static int xlnx_formatter_pcm_probe(struct platform_device *pdev) } ret = devm_request_irq(dev, aud_drv_data->mm2s_irq, xlnx_mm2s_irq_handler, 0, - "xlnx_formatter_pcm_mm2s_irq", dev); + "xlnx_formatter_pcm_mm2s_irq", aud_drv_data); if (ret) { dev_err(dev, "xlnx audio mm2s irq request failed\n"); goto clk_err; @@ -664,7 +670,7 @@ static int xlnx_formatter_pcm_probe(struct platform_device *pdev) ret = devm_request_irq(dev, aud_drv_data->s2mm_irq, xlnx_s2mm_irq_handler, 0, "xlnx_formatter_pcm_s2mm_irq", - dev); + aud_drv_data); if (ret) { dev_err(dev, "xlnx audio s2mm irq request failed\n"); goto clk_err; diff --git a/sound/usb/6fire/chip.c b/sound/usb/6fire/chip.c index 874f6cd503ca5..e6cfe0d7ab0be 100644 --- a/sound/usb/6fire/chip.c +++ b/sound/usb/6fire/chip.c @@ -156,6 +156,10 @@ static int usb6fire_chip_probe(struct usb_interface *intf, return 0; destroy_chip: + chip->shutdown = true; + if (card) + snd_card_disconnect(card); + usb6fire_chip_abort(chip); snd_card_free(card); return ret; } diff --git a/sound/usb/6fire/comm.c b/sound/usb/6fire/comm.c index 49629d4bb327a..bcfb34db37d95 100644 --- a/sound/usb/6fire/comm.c +++ b/sound/usb/6fire/comm.c @@ -36,11 +36,14 @@ static void usb6fire_comm_receiver_handler(struct urb *urb) struct midi_runtime *midi_rt = rt->chip->midi; if (!urb->status) { - if (rt->receiver_buffer[0] == 0x10) /* midi in event */ + u8 len = rt->receiver_buffer[1]; + + if (rt->receiver_buffer[0] == 0x10 && /* midi in event */ + len <= COMM_RECEIVER_BUFSIZE - 2 && + urb->actual_length >= len + 2) if (midi_rt) midi_rt->in_received(midi_rt, - rt->receiver_buffer + 2, - rt->receiver_buffer[1]); + rt->receiver_buffer + 2, len); } if (!rt->chip->shutdown) { diff --git a/sound/usb/bcd2000/bcd2000.c b/sound/usb/bcd2000/bcd2000.c index bebb48cb9abc8..c5c542d17ccc1 100644 --- a/sound/usb/bcd2000/bcd2000.c +++ b/sound/usb/bcd2000/bcd2000.c @@ -134,6 +134,9 @@ static void bcd2000_midi_send(struct bcd2000 *bcd2k) if (!midi_out_substream) return; + if (!bcd2k->midi_out_urb) + return; + /* copy command prefix bytes */ memcpy(bcd2k->midi_out_buf, device_cmd_prefix, sizeof(device_cmd_prefix)); @@ -178,7 +181,7 @@ static int bcd2000_midi_output_close(struct snd_rawmidi_substream *substream) { struct bcd2000 *bcd2k = substream->rmidi->private_data; - if (bcd2k->midi_out_active) { + if (bcd2k->midi_out_active && bcd2k->midi_out_urb) { usb_kill_urb(bcd2k->midi_out_urb); bcd2k->midi_out_active = 0; } @@ -348,11 +351,13 @@ static int bcd2000_init_midi(struct bcd2000 *bcd2k) static void bcd2000_free_usb_related_resources(struct bcd2000 *bcd2k, struct usb_interface *interface) { - usb_kill_urb(bcd2k->midi_out_urb); - usb_kill_urb(bcd2k->midi_in_urb); + usb_poison_urb(bcd2k->midi_out_urb); + usb_poison_urb(bcd2k->midi_in_urb); usb_free_urb(bcd2k->midi_out_urb); usb_free_urb(bcd2k->midi_in_urb); + bcd2k->midi_out_urb = NULL; + bcd2k->midi_in_urb = NULL; if (bcd2k->intf) { usb_set_intfdata(bcd2k->intf, NULL); diff --git a/sound/usb/caiaq/audio.c b/sound/usb/caiaq/audio.c index 95d425dd9d703..29f6cd9ee9415 100644 --- a/sound/usb/caiaq/audio.c +++ b/sound/usb/caiaq/audio.c @@ -829,16 +829,13 @@ int snd_usb_caiaq_audio_init(struct snd_usb_caiaqdev *cdev) cdev->data_urbs_in = alloc_urbs(cdev, SNDRV_PCM_STREAM_CAPTURE, &ret); if (ret < 0) { - kfree(cdev->data_cb_info); - free_urbs(cdev->data_urbs_in); + snd_usb_caiaq_audio_free(cdev); return ret; } cdev->data_urbs_out = alloc_urbs(cdev, SNDRV_PCM_STREAM_PLAYBACK, &ret); if (ret < 0) { - kfree(cdev->data_cb_info); - free_urbs(cdev->data_urbs_in); - free_urbs(cdev->data_urbs_out); + snd_usb_caiaq_audio_free(cdev); return ret; } @@ -859,6 +856,9 @@ void snd_usb_caiaq_audio_free(struct snd_usb_caiaqdev *cdev) dev_dbg(dev, "%s(%p)\n", __func__, cdev); free_urbs(cdev->data_urbs_in); + cdev->data_urbs_in = NULL; free_urbs(cdev->data_urbs_out); + cdev->data_urbs_out = NULL; kfree(cdev->data_cb_info); + cdev->data_cb_info = NULL; } diff --git a/sound/usb/caiaq/device.c b/sound/usb/caiaq/device.c index b20aae0caf60a..3d821fde45820 100644 --- a/sound/usb/caiaq/device.c +++ b/sound/usb/caiaq/device.c @@ -134,14 +134,22 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) struct device *dev = &urb->dev->dev; struct snd_usb_caiaqdev *cdev = urb->context; unsigned char *buf = urb->transfer_buffer; + unsigned int payload_len; + unsigned int copy_len; if (urb->status || !cdev) { dev_warn(dev, "received EP1 urb->status = %i\n", urb->status); return; } + if (urb->actual_length < 1) + return; + + payload_len = urb->actual_length - 1; switch(buf[0]) { case EP1_CMD_GET_DEVICE_INFO: + if (payload_len < sizeof(struct caiaq_device_spec)) + break; memcpy(&cdev->spec, buf+1, sizeof(struct caiaq_device_spec)); cdev->spec.fw_version = le16_to_cpu(cdev->spec.fw_version); dev_dbg(dev, "device spec (firmware %d): audio: %d in, %d out, " @@ -157,18 +165,21 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) wake_up(&cdev->ep1_wait_queue); break; case EP1_CMD_AUDIO_PARAMS: + if (payload_len < 1) + break; cdev->audio_parm_answer = buf[1]; wake_up(&cdev->ep1_wait_queue); break; case EP1_CMD_MIDI_READ: + if (urb->actual_length < 3 || urb->actual_length - 3 < buf[2]) + break; snd_usb_caiaq_midi_handle_input(cdev, buf[1], buf + 3, buf[2]); break; case EP1_CMD_READ_IO: if (cdev->chip.usb_id == USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_AUDIO8DJ)) { - if (urb->actual_length > sizeof(cdev->control_state)) - urb->actual_length = sizeof(cdev->control_state); - memcpy(cdev->control_state, buf + 1, urb->actual_length); + copy_len = min_t(unsigned int, payload_len, sizeof(cdev->control_state)); + memcpy(cdev->control_state, buf + 1, copy_len); wake_up(&cdev->ep1_wait_queue); break; } @@ -181,8 +192,8 @@ static void usb_ep1_command_reply_dispatch (struct urb* urb) break; } - cdev->ep1_in_urb.actual_length = 0; - ret = usb_submit_urb(&cdev->ep1_in_urb, GFP_ATOMIC); + cdev->ep1_in_urb->actual_length = 0; + ret = usb_submit_urb(cdev->ep1_in_urb, GFP_ATOMIC); if (ret < 0) dev_err(dev, "unable to submit urb. OOM!?\n"); } @@ -397,6 +408,10 @@ static void card_free(struct snd_card *card) #endif snd_usb_caiaq_audio_free(cdev); usb_put_dev(cdev->chip.dev); + usb_free_urb(cdev->ep1_in_urb); + cdev->ep1_in_urb = NULL; + usb_free_urb(cdev->midi_out_urb); + cdev->midi_out_urb = NULL; } static int create_card(struct usb_device *usb_dev, @@ -446,22 +461,30 @@ static int init_card(struct snd_usb_caiaqdev *cdev) return -EIO; } - usb_init_urb(&cdev->ep1_in_urb); - usb_init_urb(&cdev->midi_out_urb); + cdev->ep1_in_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!cdev->ep1_in_urb) + return -ENOMEM; + + cdev->midi_out_urb = usb_alloc_urb(0, GFP_KERNEL); + if (!cdev->midi_out_urb) { + usb_free_urb(cdev->ep1_in_urb); + cdev->ep1_in_urb = NULL; + return -ENOMEM; + } - usb_fill_bulk_urb(&cdev->ep1_in_urb, usb_dev, + usb_fill_bulk_urb(cdev->ep1_in_urb, usb_dev, usb_rcvbulkpipe(usb_dev, 0x1), cdev->ep1_in_buf, EP1_BUFSIZE, usb_ep1_command_reply_dispatch, cdev); - usb_fill_bulk_urb(&cdev->midi_out_urb, usb_dev, + usb_fill_bulk_urb(cdev->midi_out_urb, usb_dev, usb_sndbulkpipe(usb_dev, 0x1), cdev->midi_out_buf, EP1_BUFSIZE, snd_usb_caiaq_midi_output_done, cdev); /* sanity checks of EPs before actually submitting */ - if (usb_urb_ep_type_check(&cdev->ep1_in_urb) || - usb_urb_ep_type_check(&cdev->midi_out_urb)) { + if (usb_urb_ep_type_check(cdev->ep1_in_urb) || + usb_urb_ep_type_check(cdev->midi_out_urb)) { dev_err(dev, "invalid EPs\n"); return -EINVAL; } @@ -469,7 +492,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) init_waitqueue_head(&cdev->ep1_wait_queue); init_waitqueue_head(&cdev->prepare_wait_queue); - if (usb_submit_urb(&cdev->ep1_in_urb, GFP_KERNEL) != 0) + if (usb_submit_urb(cdev->ep1_in_urb, GFP_KERNEL) != 0) return -EIO; err = snd_usb_caiaq_send_command(cdev, EP1_CMD_GET_DEVICE_INFO, NULL, 0); @@ -519,7 +542,7 @@ static int init_card(struct snd_usb_caiaqdev *cdev) return 0; err_kill_urb: - usb_kill_urb(&cdev->ep1_in_urb); + usb_kill_urb(cdev->ep1_in_urb); return err; } @@ -565,8 +588,8 @@ static void snd_disconnect(struct usb_interface *intf) #endif snd_usb_caiaq_audio_disconnect(cdev); - usb_kill_urb(&cdev->ep1_in_urb); - usb_kill_urb(&cdev->midi_out_urb); + usb_kill_urb(cdev->ep1_in_urb); + usb_kill_urb(cdev->midi_out_urb); snd_card_free_when_closed(card); } diff --git a/sound/usb/caiaq/device.h b/sound/usb/caiaq/device.h index 743eb0387b5fb..1c6f34693fa81 100644 --- a/sound/usb/caiaq/device.h +++ b/sound/usb/caiaq/device.h @@ -60,8 +60,8 @@ struct snd_usb_caiaq_cb_info; struct snd_usb_caiaqdev { struct snd_usb_audio chip; - struct urb ep1_in_urb; - struct urb midi_out_urb; + struct urb *ep1_in_urb; + struct urb *midi_out_urb; struct urb **data_urbs_in; struct urb **data_urbs_out; struct snd_usb_caiaq_cb_info *data_cb_info; diff --git a/sound/usb/caiaq/input.c b/sound/usb/caiaq/input.c index 2db4d1332df1c..c12eeb9710002 100644 --- a/sound/usb/caiaq/input.c +++ b/sound/usb/caiaq/input.c @@ -203,6 +203,8 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev, switch (cdev->chip.usb_id) { case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL2): + if (len < 6) + return; snd_caiaq_input_report_abs(cdev, ABS_X, buf, 2); snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 0); snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 1); @@ -210,11 +212,15 @@ static void snd_caiaq_input_read_analog(struct snd_usb_caiaqdev *cdev, case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_RIGKONTROL3): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER): case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_KORECONTROLLER2): + if (len < 6) + return; snd_caiaq_input_report_abs(cdev, ABS_X, buf, 0); snd_caiaq_input_report_abs(cdev, ABS_Y, buf, 1); snd_caiaq_input_report_abs(cdev, ABS_Z, buf, 2); break; case USB_ID(USB_VID_NATIVEINSTRUMENTS, USB_PID_TRAKTORKONTROLX1): + if (len < 16) + return; snd_caiaq_input_report_abs(cdev, ABS_HAT0X, buf, 4); snd_caiaq_input_report_abs(cdev, ABS_HAT0Y, buf, 2); snd_caiaq_input_report_abs(cdev, ABS_HAT1X, buf, 6); diff --git a/sound/usb/caiaq/midi.c b/sound/usb/caiaq/midi.c index c656d01624321..18529484c8dcc 100644 --- a/sound/usb/caiaq/midi.c +++ b/sound/usb/caiaq/midi.c @@ -43,7 +43,7 @@ static int snd_usb_caiaq_midi_output_close(struct snd_rawmidi_substream *substre { struct snd_usb_caiaqdev *cdev = substream->rmidi->private_data; if (cdev->midi_out_active) { - usb_kill_urb(&cdev->midi_out_urb); + usb_kill_urb(cdev->midi_out_urb); cdev->midi_out_active = 0; } return 0; @@ -64,9 +64,9 @@ static void snd_usb_caiaq_midi_send(struct snd_usb_caiaqdev *cdev, return; cdev->midi_out_buf[2] = len; - cdev->midi_out_urb.transfer_buffer_length = len+3; + cdev->midi_out_urb->transfer_buffer_length = len+3; - ret = usb_submit_urb(&cdev->midi_out_urb, GFP_ATOMIC); + ret = usb_submit_urb(cdev->midi_out_urb, GFP_ATOMIC); if (ret < 0) dev_err(dev, "snd_usb_caiaq_midi_send(%p): usb_submit_urb() failed," diff --git a/sound/usb/card.c b/sound/usb/card.c index 970ad247a5075..4e8cb31c93931 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -1284,8 +1284,11 @@ static int usb_audio_resume(struct usb_interface *intf) list_for_each_entry(as, &chip->pcm_list, list) { err = snd_usb_pcm_resume(as); - if (err < 0) - goto err_out; + if (err < 0) { + if (!chip->system_suspend) + goto err_out; + goto out; + } } /* @@ -1294,8 +1297,11 @@ static int usb_audio_resume(struct usb_interface *intf) */ list_for_each_entry(mixer, &chip->mixer_list, list) { err = snd_usb_mixer_resume(mixer); - if (err < 0) - goto err_out; + if (err < 0) { + if (!chip->system_suspend) + goto err_out; + goto out; + } } list_for_each(p, &chip->midi_list) { diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 15490a87b460e..bdf0e724d4187 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -385,13 +385,15 @@ static int prepare_inbound_urb(struct snd_usb_endpoint *ep, case SND_USB_ENDPOINT_TYPE_DATA: offs = 0; for (i = 0; i < urb_ctx->packets; i++) { + if (offs + ep->curpacksize > urb_ctx->buffer_size) + break; urb->iso_frame_desc[i].offset = offs; urb->iso_frame_desc[i].length = ep->curpacksize; offs += ep->curpacksize; } urb->transfer_buffer_length = offs; - urb->number_of_packets = urb_ctx->packets; + urb->number_of_packets = i; break; case SND_USB_ENDPOINT_TYPE_SYNC: @@ -1164,10 +1166,12 @@ static int data_ep_set_params(struct snd_usb_endpoint *ep) << (16 - ep->datainterval); } - if (ep->fill_max) + if (ep->fill_max) { ep->curpacksize = ep->maxpacksize; - else + maxsize = ep->curpacksize; + } else { ep->curpacksize = maxsize; + } if (snd_usb_get_speed(chip->dev) != USB_SPEED_FULL) { packs_per_ms = 8 >> ep->datainterval; @@ -1237,10 +1241,10 @@ static int data_ep_set_params(struct snd_usb_endpoint *ep) u->index = i; u->ep = ep; u->packets = urb_packs; - u->buffer_size = maxsize * u->packets; if (fmt->fmt_type == UAC_FORMAT_TYPE_II) u->packets++; /* for transfer delimiter */ + u->buffer_size = maxsize * u->packets; u->urb = usb_alloc_urb(u->packets, GFP_KERNEL); if (!u->urb) goto out_of_memory; @@ -1804,11 +1808,13 @@ static void snd_usb_handle_sync_urb(struct snd_usb_endpoint *ep, out_packet->packets = in_ctx->packets; for (i = 0; i < in_ctx->packets; i++) { - if (urb->iso_frame_desc[i].status == 0) - out_packet->packet_size[i] = + if (urb->iso_frame_desc[i].status == 0) { + unsigned int frames = urb->iso_frame_desc[i].actual_length / sender->stride; - else + out_packet->packet_size[i] = min(frames, ep->maxframesize); + } else { out_packet->packet_size[i] = 0; + } } spin_unlock_irqrestore(&ep->lock, flags); diff --git a/sound/usb/fcp.c b/sound/usb/fcp.c index 6eab5cd0cd358..a17bf0028a8ad 100644 --- a/sound/usb/fcp.c +++ b/sound/usb/fcp.c @@ -82,6 +82,7 @@ struct fcp_data { struct mutex mutex; /* serialise access to the device */ struct completion cmd_done; /* wait for command completion */ struct file *file; /* hwdep file */ + struct urb *urb; /* FCP notification endpoint */ struct fcp_notify notify; @@ -129,6 +130,10 @@ struct fcp_data { #define FCP_SEGMENT_APP_GOLD 0 +#define FCP_MAX_METER_MAP_SIZE \ + (sizeof_field(struct snd_ctl_elem_value, value.integer.value) / \ + sizeof(long)) + /* Forward declarations */ static int fcp_init(struct usb_mixer_interface *mixer, void *step0_resp, void *step2_resp); @@ -182,21 +187,21 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, { struct fcp_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct fcp_usb_packet *req __free(kfree) = NULL; - struct fcp_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; - if (!mixer->urb) + if (!private->urb) return -ENODEV; + struct fcp_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct fcp_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -300,16 +305,17 @@ static int fcp_usb(struct usb_mixer_interface *mixer, u32 opcode, static int fcp_reinit(struct usb_mixer_interface *mixer) { struct fcp_data *private = mixer->private_data; - void *step0_resp __free(kfree) = NULL; - void *step2_resp __free(kfree) = NULL; - if (mixer->urb) + if (private->urb) return 0; - step0_resp = kmalloc(private->step0_resp_size, GFP_KERNEL); + void *step0_resp __free(kfree) = + kmalloc(private->step0_resp_size, GFP_KERNEL); if (!step0_resp) return -ENOMEM; - step2_resp = kmalloc(private->step2_resp_size, GFP_KERNEL); + + void *step2_resp __free(kfree) = + kmalloc(private->step2_resp_size, GFP_KERNEL); if (!step2_resp) return -ENOMEM; @@ -409,6 +415,9 @@ static int fcp_meter_ctl_get(struct snd_kcontrol *kctl, if (err < 0) return err; + if (WARN_ON_ONCE(elem->channels > FCP_MAX_METER_MAP_SIZE)) + return -EINVAL; + /* copy & translate from resp[] using meter_level_map[] */ for (i = 0; i < elem->channels; i++) { int idx = private->meter_level_map[i]; @@ -464,7 +473,6 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, struct fcp_init init; struct usb_device *dev = mixer->chip->dev; struct fcp_data *private = mixer->private_data; - void *resp __free(kfree) = NULL; void *step2_resp; int err, buf_size; @@ -485,7 +493,8 @@ static int fcp_ioctl_init(struct usb_mixer_interface *mixer, /* Allocate response buffer */ buf_size = init.step0_resp_size + init.step2_resp_size; - resp = kmalloc(buf_size, GFP_KERNEL); + void *resp __free(kfree) = + kzalloc(buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -619,7 +628,6 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, { struct fcp_meter_map map; struct fcp_data *private = mixer->private_data; - s16 *tmp_map __free(kfree) = NULL; int err; if (copy_from_user(&map, arg, sizeof(map))) @@ -636,12 +644,14 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, } /* Validate the map size */ - if (map.map_size < 1 || map.map_size > 255 || + if (map.map_size < 1 || + map.map_size > FCP_MAX_METER_MAP_SIZE || map.meter_slots < 1 || map.meter_slots > 255) return -EINVAL; /* Allocate and copy the map data */ - tmp_map = memdup_array_user(arg->map, map.map_size, sizeof(s16)); + s16 *tmp_map __free(kfree) = + memdup_array_user(arg->map, map.map_size, sizeof(s16)); if (IS_ERR(tmp_map)) return PTR_ERR(tmp_map); @@ -651,17 +661,16 @@ static int fcp_ioctl_set_meter_map(struct usb_mixer_interface *mixer, /* If the control doesn't exist, create it */ if (!private->meter_ctl) { - s16 *new_map __free(kfree) = NULL; - __le32 *meter_levels __free(kfree) = NULL; - /* Allocate buffer for the map */ - new_map = kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); + s16 *new_map __free(kfree) = + kmalloc_array(map.map_size, sizeof(s16), GFP_KERNEL); if (!new_map) return -ENOMEM; /* Allocate buffer for reading meter levels */ - meter_levels = kmalloc_array(map.meter_slots, sizeof(__le32), - GFP_KERNEL); + __le32 *meter_levels __free(kfree) = + kmalloc_array(map.meter_slots, sizeof(__le32), + GFP_KERNEL); if (!meter_levels) return -ENOMEM; @@ -893,13 +902,15 @@ static int fcp_hwdep_init(struct usb_mixer_interface *mixer) static void fcp_cleanup_urb(struct usb_mixer_interface *mixer) { - if (!mixer->urb) + struct fcp_data *private = mixer->private_data; + + if (!private->urb) return; - usb_kill_urb(mixer->urb); - kfree(mixer->urb->transfer_buffer); - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_kill_urb(private->urb); + kfree(private->urb->transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; } static void fcp_private_free(struct usb_mixer_interface *mixer) @@ -970,37 +981,37 @@ static int fcp_init_notify(struct usb_mixer_interface *mixer) int err; /* Already set up */ - if (mixer->urb) + if (private->urb) return 0; if (usb_pipe_type_check(dev, pipe)) return -EINVAL; - mixer->urb = usb_alloc_urb(0, GFP_KERNEL); - if (!mixer->urb) + private->urb = usb_alloc_urb(0, GFP_KERNEL); + if (!private->urb) return -ENOMEM; transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); if (!transfer_buffer) { - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_free_urb(private->urb); + private->urb = NULL; return -ENOMEM; } - usb_fill_int_urb(mixer->urb, dev, pipe, + usb_fill_int_urb(private->urb, dev, pipe, transfer_buffer, private->wMaxPacketSize, fcp_notify, mixer, private->bInterval); - init_completion(&private->cmd_done); + reinit_completion(&private->cmd_done); - err = usb_submit_urb(mixer->urb, GFP_KERNEL); + err = usb_submit_urb(private->urb, GFP_KERNEL); if (err) { usb_audio_err(mixer->chip, "%s: usb_submit_urb failed: %d\n", __func__, err); kfree(transfer_buffer); - usb_free_urb(mixer->urb); - mixer->urb = NULL; + usb_free_urb(private->urb); + private->urb = NULL; } return err; @@ -1022,6 +1033,8 @@ static int fcp_init(struct usb_mixer_interface *mixer, step0_resp, private->step0_resp_size); if (err < 0) return err; + if (err != private->step0_resp_size) + return -EIO; err = fcp_init_notify(mixer); if (err < 0) @@ -1051,6 +1064,7 @@ static int fcp_init_private(struct usb_mixer_interface *mixer) return -ENOMEM; mutex_init(&private->mutex); + init_completion(&private->cmd_done); init_waitqueue_head(&private->notify.queue); spin_lock_init(&private->notify.lock); diff --git a/sound/usb/midi.c b/sound/usb/midi.c index e09761b6c91f3..787956252e800 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -793,6 +793,8 @@ static void snd_usbmidi_akai_output(struct snd_usb_midi_out_endpoint *ep, msg = urb->transfer_buffer + urb->transfer_buffer_length; buf_end = ep->max_transfer - MAX_AKAI_SYSEX_LEN - 1; + if (buf_end <= 0) + return; /* only try adding more data when there's space for at least 1 SysEx */ while (urb->transfer_buffer_length < buf_end) { @@ -869,6 +871,8 @@ static void snd_usbmidi_novation_output(struct snd_usb_midi_out_endpoint *ep, if (!ep->ports[0].active) return; + if (ep->max_transfer < 3) + return; transfer_buffer = urb->transfer_buffer; count = snd_rawmidi_transmit(ep->ports[0].substream, &transfer_buffer[2], @@ -963,6 +967,8 @@ static void snd_usbmidi_us122l_output(struct snd_usb_midi_out_endpoint *ep, default: count = 2; } + if (ep->max_transfer < count) + return; count = snd_rawmidi_transmit(ep->ports[0].substream, urb->transfer_buffer, count); diff --git a/sound/usb/midi2.c b/sound/usb/midi2.c index 50b04e5a190b6..0f3c70ec4a77e 100644 --- a/sound/usb/midi2.c +++ b/sound/usb/midi2.c @@ -329,7 +329,7 @@ ump_to_endpoint(struct snd_ump_endpoint *ump, int dir) { struct snd_usb_midi2_ump *rmidi = ump->private_data; - return rmidi->eps[dir]; + return rmidi ? rmidi->eps[dir] : NULL; } /* ump open callback */ @@ -677,6 +677,14 @@ static int parse_midi_2_0_endpoints(struct snd_usb_midi2_interface *umidi) return 0; } +static void free_ump_private_data(struct snd_ump_endpoint *ump) +{ + struct snd_usb_midi2_ump *rmidi = ump->private_data; + + if (rmidi) + rmidi->ump = NULL; +} + static void free_all_midi2_umps(struct snd_usb_midi2_interface *umidi) { struct snd_usb_midi2_ump *rmidi; @@ -685,6 +693,8 @@ static void free_all_midi2_umps(struct snd_usb_midi2_interface *umidi) rmidi = list_first_entry(&umidi->rawmidi_list, struct snd_usb_midi2_ump, list); list_del(&rmidi->list); + if (rmidi->ump) + rmidi->ump->private_data = NULL; kfree(rmidi); } } @@ -725,6 +735,7 @@ static int create_midi2_ump(struct snd_usb_midi2_interface *umidi, ump->private_data = rmidi; ump->ops = &snd_usb_midi_v2_ump_ops; + ump->private_free = free_ump_private_data; rmidi->eps[STR_IN] = ep_in; rmidi->eps[STR_OUT] = ep_out; diff --git a/sound/usb/mixer.c b/sound/usb/mixer.c index 0765250f3a56d..72bd94b44ce34 100644 --- a/sound/usb/mixer.c +++ b/sound/usb/mixer.c @@ -1469,7 +1469,10 @@ static int mixer_ctl_feature_put(struct snd_kcontrol *kcontrol, return -EINVAL; val = get_abs_value(cval, val); if (oval != val) { - snd_usb_set_cur_mix_value(cval, c + 1, cnt, val); + err = snd_usb_set_cur_mix_value(cval, c + 1, + cnt, val); + if (err < 0) + return filter_error(cval, err); changed = 1; } cnt++; @@ -1484,7 +1487,9 @@ static int mixer_ctl_feature_put(struct snd_kcontrol *kcontrol, return -EINVAL; val = get_abs_value(cval, val); if (val != oval) { - snd_usb_set_cur_mix_value(cval, 0, 0, val); + err = snd_usb_set_cur_mix_value(cval, 0, 0, val); + if (err < 0) + return filter_error(cval, err); changed = 1; } } @@ -2350,7 +2355,9 @@ static int mixer_ctl_procunit_put(struct snd_kcontrol *kcontrol, return -EINVAL; val = get_abs_value(cval, val); if (val != oval) { - set_cur_ctl_value(cval, cval->control << 8, val); + err = set_cur_ctl_value(cval, cval->control << 8, val); + if (err < 0) + return filter_error(cval, err); return 1; } return 0; @@ -2714,7 +2721,9 @@ static int mixer_ctl_selector_put(struct snd_kcontrol *kcontrol, return -EINVAL; val = get_abs_value(cval, val); if (val != oval) { - set_cur_ctl_value(cval, cval->control << 8, val); + err = set_cur_ctl_value(cval, cval->control << 8, val); + if (err < 0) + return filter_error(cval, err); return 1; } return 0; @@ -3743,6 +3752,12 @@ int snd_usb_mixer_resume(struct usb_mixer_interface *mixer) struct usb_mixer_elem_list *list; int id, err; + if (mixer->private_resume) { + err = mixer->private_resume(mixer); + if (err < 0) + return err; + } + /* restore cached mixer values */ for (id = 0; id < MAX_ID_ELEMS; id++) { for_each_mixer_elem(list, mixer, id) { diff --git a/sound/usb/mixer.h b/sound/usb/mixer.h index 167fbfcf01ace..efaccdd923385 100644 --- a/sound/usb/mixer.h +++ b/sound/usb/mixer.h @@ -18,6 +18,7 @@ struct usb_mixer_interface { struct usb_host_interface *hostif; struct list_head list; unsigned int ignore_ctl_error; + /* UAC2 status interrupt endpoint; owned by mixer.c */ struct urb *urb; /* array[MAX_ID_ELEMS], indexed by unit id */ struct usb_mixer_elem_list **id_elems; @@ -42,6 +43,7 @@ struct usb_mixer_interface { void *private_data; void (*private_free)(struct usb_mixer_interface *mixer); void (*private_suspend)(struct usb_mixer_interface *mixer); + int (*private_resume)(struct usb_mixer_interface *mixer); }; #define MAX_CHANNELS 16 /* max logical channels */ diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c index faac7df1fbcf0..ce27fc871f513 100644 --- a/sound/usb/mixer_maps.c +++ b/sound/usb/mixer_maps.c @@ -344,6 +344,16 @@ static const struct usbmix_name_map bose_soundlink_map[] = { { 0 } /* terminator */ }; +/* + * Razer Barracuda X 2.4: Firmware reports cval->min = -16800 in 1/256 dB units + * (-65.62 dB), which stock ALSA misinterprets as a -168 dB floor + */ +static const struct usbmix_dB_map razer_barracuda_x_2_4_dB = {-6562, 0}; +static const struct usbmix_name_map razer_barracuda_x_2_4_map[] = { + { 2, NULL, .dB = &razer_barracuda_x_2_4_dB }, + { 0 } /* terminator */ +}; + /* Sennheiser Communications Headset [PC 8], the dB value is reported as -6 negative maximum */ static const struct usbmix_dB_map sennheiser_pc8_dB = {-9500, 0}; static const struct usbmix_name_map sennheiser_pc8_map[] = { @@ -628,6 +638,16 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .id = USB_ID(0x1b1c, 0x0a42), .map = corsair_virtuoso_map, }, + { + /* Corsair Virtuoso (wired mode, later revision) */ + .id = USB_ID(0x1b1c, 0x0a43), + .map = corsair_virtuoso_map, + }, + { + /* Corsair Virtuoso (wireless mode, later revision) */ + .id = USB_ID(0x1b1c, 0x0a44), + .map = corsair_virtuoso_map, + }, { /* Corsair HS80 RGB Wireless (wired mode) */ .id = USB_ID(0x1b1c, 0x0a6a), @@ -689,6 +709,11 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .id = USB_ID(0x17aa, 0x1046), .map = lenovo_p620_rear_map, }, + { + /* Razer Barracuda X 2.4 */ + .id = USB_ID(0x1532, 0x0552), + .map = razer_barracuda_x_2_4_map, + }, { /* Sennheiser Communications Headset [PC 8] */ .id = USB_ID(0x1395, 0x0025), diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index 02d9ede674670..ce057bbb2e116 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -3352,7 +3352,7 @@ static int snd_rme_digiface_read_status(struct snd_kcontrol *kcontrol, u32 statu struct usb_mixer_elem_list *list = snd_kcontrol_chip(kcontrol); struct snd_usb_audio *chip = list->mixer->chip; struct usb_device *dev = chip->dev; - __le32 buf[4]; + __le32 buf[4] = {}; int err; err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), diff --git a/sound/usb/mixer_scarlett2.c b/sound/usb/mixer_scarlett2.c index 6ac2fad37f04a..133e735dd3e93 100644 --- a/sound/usb/mixer_scarlett2.c +++ b/sound/usb/mixer_scarlett2.c @@ -1292,6 +1292,7 @@ struct scarlett2_data { struct usb_mixer_interface *mixer; struct mutex usb_mutex; /* prevent sending concurrent USB requests */ struct completion cmd_done; + struct urb *urb; /* notification endpoint */ struct mutex data_mutex; /* lock access to this data */ u8 running; u8 hwdep_in_use; @@ -2498,18 +2499,18 @@ static int scarlett2_usb( { struct scarlett2_data *private = mixer->private_data; struct usb_device *dev = mixer->chip->dev; - struct scarlett2_usb_packet *req __free(kfree) = NULL; - struct scarlett2_usb_packet *resp __free(kfree) = NULL; - size_t req_buf_size = struct_size(req, data, req_size); - size_t resp_buf_size = struct_size(resp, data, resp_size); int retries = 0; const int max_retries = 5; int err; + struct scarlett2_usb_packet *req __free(kfree) = NULL; + size_t req_buf_size = struct_size(req, data, req_size); req = kmalloc(req_buf_size, GFP_KERNEL); if (!req) return -ENOMEM; + struct scarlett2_usb_packet *resp __free(kfree) = NULL; + size_t resp_buf_size = struct_size(resp, data, resp_size); resp = kmalloc(resp_buf_size, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -4064,9 +4065,9 @@ static int scarlett2_input_select_ctl_info( struct scarlett2_data *private = mixer->private_data; int inputs = private->info->gain_input_count; int i, err; - char **values __free(kfree) = NULL; + char **values __free(kfree) = + kcalloc(inputs, sizeof(char *), GFP_KERNEL); - values = kcalloc(inputs, sizeof(char *), GFP_KERNEL); if (!values) return -ENOMEM; @@ -8313,13 +8314,70 @@ static void scarlett2_notify(struct urb *urb) } } -/*** Cleanup/Suspend Callbacks ***/ +/*** Notification URB and Cleanup/Suspend Callbacks ***/ + +/* Submit a URB to receive notifications from the device */ +static int scarlett2_init_notify(struct usb_mixer_interface *mixer) +{ + struct usb_device *dev = mixer->chip->dev; + struct scarlett2_data *private = mixer->private_data; + unsigned int pipe = usb_rcvintpipe(dev, private->bEndpointAddress); + void *transfer_buffer; + int err; + + /* Already set up */ + if (private->urb) + return 0; + + if (usb_pipe_type_check(dev, pipe)) + return -EINVAL; + + private->urb = usb_alloc_urb(0, GFP_KERNEL); + if (!private->urb) + return -ENOMEM; + + transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); + if (!transfer_buffer) { + usb_free_urb(private->urb); + private->urb = NULL; + return -ENOMEM; + } + + usb_fill_int_urb(private->urb, dev, pipe, + transfer_buffer, private->wMaxPacketSize, + scarlett2_notify, mixer, private->bInterval); + + reinit_completion(&private->cmd_done); + + err = usb_submit_urb(private->urb, GFP_KERNEL); + if (err) { + kfree(transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; + } + + return err; +} + +static void scarlett2_cleanup_urb(struct usb_mixer_interface *mixer) +{ + struct scarlett2_data *private = mixer->private_data; + + if (!private->urb) + return; + + usb_kill_urb(private->urb); + kfree(private->urb->transfer_buffer); + usb_free_urb(private->urb); + private->urb = NULL; +} static void scarlett2_private_free(struct usb_mixer_interface *mixer) { struct scarlett2_data *private = mixer->private_data; cancel_delayed_work_sync(&private->work); + scarlett2_cleanup_urb(mixer); kfree(private); mixer->private_data = NULL; } @@ -8330,6 +8388,8 @@ static void scarlett2_private_suspend(struct usb_mixer_interface *mixer) if (cancel_delayed_work_sync(&private->work)) scarlett2_config_save(private->mixer); + + scarlett2_cleanup_urb(mixer); } /*** Initialisation ***/ @@ -8449,11 +8509,13 @@ static int scarlett2_init_private(struct usb_mixer_interface *mixer, mutex_init(&private->usb_mutex); mutex_init(&private->data_mutex); + init_completion(&private->cmd_done); INIT_DELAYED_WORK(&private->work, scarlett2_config_save_work); mixer->private_data = private; mixer->private_free = scarlett2_private_free; mixer->private_suspend = scarlett2_private_suspend; + mixer->private_resume = scarlett2_init_notify; private->info = entry->info; @@ -8470,40 +8532,6 @@ static int scarlett2_init_private(struct usb_mixer_interface *mixer, return scarlett2_find_fc_interface(mixer->chip->dev, private); } -/* Submit a URB to receive notifications from the device */ -static int scarlett2_init_notify(struct usb_mixer_interface *mixer) -{ - struct usb_device *dev = mixer->chip->dev; - struct scarlett2_data *private = mixer->private_data; - unsigned int pipe = usb_rcvintpipe(dev, private->bEndpointAddress); - void *transfer_buffer; - - if (mixer->urb) { - usb_audio_err(mixer->chip, - "%s: mixer urb already in use!\n", __func__); - return 0; - } - - if (usb_pipe_type_check(dev, pipe)) - return -EINVAL; - - mixer->urb = usb_alloc_urb(0, GFP_KERNEL); - if (!mixer->urb) - return -ENOMEM; - - transfer_buffer = kmalloc(private->wMaxPacketSize, GFP_KERNEL); - if (!transfer_buffer) - return -ENOMEM; - - usb_fill_int_urb(mixer->urb, dev, pipe, - transfer_buffer, private->wMaxPacketSize, - scarlett2_notify, mixer, private->bInterval); - - init_completion(&private->cmd_done); - - return usb_submit_urb(mixer->urb, GFP_KERNEL); -} - /* Cargo cult proprietary initialisation sequence */ static int scarlett2_usb_init(struct usb_mixer_interface *mixer) { @@ -9265,8 +9293,6 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, __le32 len; } __packed req; - u8 *resp __free(kfree) = NULL; - /* Flash segment must first be selected */ if (private->flash_write_state != SCARLETT2_FLASH_WRITE_STATE_SELECTED) return -EINVAL; @@ -9304,7 +9330,8 @@ static long scarlett2_hwdep_read(struct snd_hwdep *hw, req.offset = cpu_to_le32(*offset); req.len = cpu_to_le32(count); - resp = kzalloc(count, GFP_KERNEL); + u8 *resp __free(kfree) = + kzalloc(count, GFP_KERNEL); if (!resp) return -ENOMEM; @@ -9452,7 +9479,6 @@ static ssize_t scarlett2_devmap_read( loff_t pos) { struct usb_mixer_interface *mixer = entry->private_data; - u8 *resp_buf __free(kfree) = NULL; const size_t block_size = SCARLETT2_DEVMAP_BLOCK_SIZE; size_t copied = 0; @@ -9462,7 +9488,8 @@ static ssize_t scarlett2_devmap_read( if (pos + count > entry->size) count = entry->size - pos; - resp_buf = kmalloc(block_size, GFP_KERNEL); + u8 *resp_buf __free(kfree) = + kmalloc(block_size, GFP_KERNEL); if (!resp_buf) return -ENOMEM; diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index eafc0d73cca1f..d783dae1850a0 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -390,6 +390,20 @@ YAMAHA_DEVICE(0x105d, NULL), } } }, +{ + USB_DEVICE(0x0499, 0x150d), + QUIRK_DRIVER_INFO { + /* .vendor_name = "Yamaha", */ + /* .product_name = "CDS3000", */ + QUIRK_DATA_COMPOSITE { + { QUIRK_DATA_STANDARD_AUDIO(1) }, + { QUIRK_DATA_STANDARD_AUDIO(2) }, + { QUIRK_DATA_MIDI_YAMAHA(3) }, + { QUIRK_DATA_IGNORE(4) }, + QUIRK_COMPOSITE_END + } + } +}, { USB_DEVICE(0x0499, 0x1718), QUIRK_DRIVER_INFO { @@ -2131,6 +2145,14 @@ YAMAHA_DEVICE(0x7010, "UB99"), } } }, +{ + USB_DEVICE(0x1235, 0x001e), + QUIRK_DRIVER_INFO { + /* .vendor_name = "Novation", */ + /* .product_name = "Mininova", */ + QUIRK_DATA_RAW_BYTES(0) + } +}, { USB_DEVICE_VENDOR_SPEC(0x1235, 0x4661), QUIRK_DRIVER_INFO { diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index b993037acdffe..bcc9333f59acb 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -555,7 +555,6 @@ int snd_usb_create_quirk(struct snd_usb_audio *chip, static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interface *intf) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; if (le16_to_cpu(get_cfg_desc(config)->wTotalLength) == EXTIGY_FIRMWARE_SIZE_OLD || @@ -566,8 +565,8 @@ static int snd_usb_extigy_boot_quirk(struct usb_device *dev, struct usb_interfac 0x10, 0x43, 0x0001, 0x000a, NULL, 0); if (err < 0) dev_dbg(&dev->dev, "error sending boot message: %d\n", err); - - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; err = usb_get_descriptor(dev, USB_DT_DEVICE, 0, @@ -910,7 +909,6 @@ static void mbox2_setup_48_24_magic(struct usb_device *dev) static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; u8 bootresponse[0x12]; int fwsize; @@ -945,7 +943,8 @@ static int snd_usb_mbox2_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; @@ -1267,7 +1266,6 @@ static void mbox3_setup_defaults(struct usb_device *dev) static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) { struct usb_host_config *config = dev->actconfig; - struct usb_device_descriptor *new_device_descriptor __free(kfree) = NULL; int err; int descriptor_size; @@ -1280,7 +1278,8 @@ static int snd_usb_mbox3_boot_quirk(struct usb_device *dev) dev_dbg(&dev->dev, "MBOX3: device initialised!\n"); - new_device_descriptor = kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); + struct usb_device_descriptor *new_device_descriptor __free(kfree) = + kmalloc(sizeof(*new_device_descriptor), GFP_KERNEL); if (!new_device_descriptor) return -ENOMEM; @@ -2274,6 +2273,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M | QUIRK_FLAG_MIXER_PLAYBACK_MIN_MUTE), DEVICE_FLG(0x0ecb, 0x205c, /* JBL Quantum610 Wireless */ QUIRK_FLAG_FIXED_RATE), + DEVICE_FLG(0x0ecb, 0x2125, /* JBL Quantum650 Wireless */ + QUIRK_FLAG_FIXED_RATE), DEVICE_FLG(0x0ecb, 0x2069, /* JBL Quantum810 Wireless */ QUIRK_FLAG_FIXED_RATE), DEVICE_FLG(0x0fd9, 0x0008, /* Hauppauge HVR-950Q */ @@ -2336,6 +2337,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), DEVICE_FLG(0x1bcf, 0x2283, /* NexiGo N930AF FHD Webcam */ QUIRK_FLAG_GET_SAMPLE_RATE | QUIRK_FLAG_MIC_RES_16), + DEVICE_FLG(0x1ff7, 0x0f81, /* SC13A Webcam */ + QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x2040, 0x7200, /* Hauppauge HVR-950Q */ QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x2040, 0x7201, /* Hauppauge HVR-950Q-MXL */ @@ -2380,6 +2383,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_DSD_RAW), DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x2772, 0x0502, /* Musical Fidelity M6s DAC */ + 0), /* for avoiding QUIRK_FLAG_DSD_RAW with vendor match */ DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */ QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x2a70, 0x1881, /* OnePlus Technology (Shenzhen) Co., Ltd. BE02T */ diff --git a/sound/usb/usx2y/us122l.c b/sound/usb/usx2y/us122l.c index 011ea96e97799..aaeae099b5f2e 100644 --- a/sound/usb/usx2y/us122l.c +++ b/sound/usb/usx2y/us122l.c @@ -180,8 +180,11 @@ static int usb_stream_hwdep_mmap(struct snd_hwdep *hw, guard(mutex)(&us122l->mutex); s = us122l->sk.s; read = offset < s->read_size; - if (read && area->vm_flags & VM_WRITE) - return -EPERM; + if (read) { + if (area->vm_flags & VM_WRITE) + return -EPERM; + vm_flags_clear(area, VM_MAYWRITE); + } /* if userspace tries to mmap beyond end of our buffer, fail */ if (size > PAGE_ALIGN(read ? s->read_size : s->write_size)) { dev_warn(hw->card->dev, "%s: size %lu > %u\n", __func__, diff --git a/sound/usb/usx2y/us144mkii_capture.c b/sound/usb/usx2y/us144mkii_capture.c index 6a45d2b6130ca..5a3236217e095 100644 --- a/sound/usb/usx2y/us144mkii_capture.c +++ b/sound/usb/usx2y/us144mkii_capture.c @@ -302,6 +302,7 @@ void capture_urb_complete(struct urb *urb) } usb_get_urb(urb); + usb_anchor_urb(urb, &tascam->capture_anchor); ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err_ratelimited(tascam->card->dev, diff --git a/sound/usb/usx2y/usX2Yhwdep.c b/sound/usb/usx2y/usX2Yhwdep.c index 4d79251848261..7f153a2fcdd81 100644 --- a/sound/usb/usx2y/usX2Yhwdep.c +++ b/sound/usb/usx2y/usX2Yhwdep.c @@ -25,6 +25,8 @@ static vm_fault_t snd_us428ctls_vm_fault(struct vm_fault *vmf) void *vaddr; offset = vmf->pgoff << PAGE_SHIFT; + if (offset >= US428_SHAREDMEM_PAGES) + return VM_FAULT_SIGBUS; vaddr = (char *)((struct usx2ydev *)vmf->vma->vm_private_data)->us428ctls_sharedmem + offset; page = virt_to_page(vaddr); get_page(page); diff --git a/sound/usb/usx2y/usbusx2y.c b/sound/usb/usx2y/usbusx2y.c index f34e78910200a..108f9bddf9aa3 100644 --- a/sound/usb/usx2y/usbusx2y.c +++ b/sound/usb/usx2y/usbusx2y.c @@ -180,7 +180,7 @@ static void i_usx2y_in04_int(struct urb *urb) struct usx2ydev *usx2y = urb->context; struct us428ctls_sharedmem *us428ctls = usx2y->us428ctls_sharedmem; struct us428_p4out *p4out; - int i, j, n, diff, send; + int i, j, n, diff, send, len; usx2y->in04_int_calls++; @@ -189,6 +189,9 @@ static void i_usx2y_in04_int(struct urb *urb) return; } + if (urb->actual_length < USX2Y_IN04_SIZE) + goto resubmit; + if (us428ctls) { diff = -1; if (us428ctls->ctl_snapshot_last == -2) { @@ -196,7 +199,7 @@ static void i_usx2y_in04_int(struct urb *urb) memcpy(usx2y->in04_last, usx2y->in04_buf, sizeof(usx2y->in04_last)); us428ctls->ctl_snapshot_last = -1; } else { - for (i = 0; i < 21; i++) { + for (i = 0; i < USX2Y_IN04_SIZE; i++) { if (usx2y->in04_last[i] != ((char *)usx2y->in04_buf)[i]) { if (diff < 0) diff = i; @@ -222,30 +225,38 @@ static void i_usx2y_in04_int(struct urb *urb) } while (!err && usx2y->us04->submitted < usx2y->us04->len); } } else { - if (us428ctls && us428ctls->p4out_last >= 0 && us428ctls->p4out_last < N_US428_P4OUT_BUFS) { - if (us428ctls->p4out_last != us428ctls->p4out_sent) { - send = us428ctls->p4out_sent + 1; - if (send >= N_US428_P4OUT_BUFS) - send = 0; - for (j = 0; j < URBS_ASYNC_SEQ && !err; ++j) { - if (!usx2y->as04.urb[j]->status) { - p4out = us428ctls->p4out + send; // FIXME if more than 1 p4out is new, 1 gets lost. - usb_fill_bulk_urb(usx2y->as04.urb[j], usx2y->dev, - usb_sndbulkpipe(usx2y->dev, 0x04), &p4out->val.vol, - p4out->type == ELT_LIGHT ? sizeof(struct us428_lights) : 5, - i_usx2y_out04_int, usx2y); - err = usb_submit_urb(usx2y->as04.urb[j], GFP_ATOMIC); + while (us428ctls && + us428ctls->p4out_last >= 0 && + us428ctls->p4out_last < N_US428_P4OUT_BUFS && + us428ctls->p4out_last != us428ctls->p4out_sent) { + for (j = 0; j < URBS_ASYNC_SEQ && !err; ++j) { + if (!usx2y->as04.urb[j]->status) { + send = us428ctls->p4out_sent + 1; + if (send >= N_US428_P4OUT_BUFS) + send = 0; + + p4out = us428ctls->p4out + send; + len = p4out->type == ELT_LIGHT ? + sizeof(struct us428_lights) : 5; + memcpy(usx2y->as04.urb[j]->transfer_buffer, + &p4out->val.vol, len); + usx2y->as04.urb[j]->transfer_buffer_length = len; + err = usb_submit_urb(usx2y->as04.urb[j], GFP_ATOMIC); + if (!err) us428ctls->p4out_sent = send; - break; - } + + break; } } + if (j >= URBS_ASYNC_SEQ || err) + break; } } if (err) dev_err(&urb->dev->dev, "in04_int() usb_submit_urb err=%i\n", err); +resubmit: urb->dev = usx2y->dev; usb_submit_urb(urb, GFP_ATOMIC); } @@ -298,7 +309,7 @@ int usx2y_in04_init(struct usx2ydev *usx2y) goto error; } - usx2y->in04_buf = kmalloc(21, GFP_KERNEL); + usx2y->in04_buf = kzalloc(USX2Y_IN04_SIZE, GFP_KERNEL); if (!usx2y->in04_buf) { err = -ENOMEM; goto error; @@ -306,7 +317,7 @@ int usx2y_in04_init(struct usx2ydev *usx2y) init_waitqueue_head(&usx2y->in04_wait_queue); usb_fill_int_urb(usx2y->in04_urb, usx2y->dev, usb_rcvintpipe(usx2y->dev, 0x4), - usx2y->in04_buf, 21, + usx2y->in04_buf, USX2Y_IN04_SIZE, i_usx2y_in04_int, usx2y, 10); if (usb_urb_ep_type_check(usx2y->in04_urb)) { diff --git a/sound/usb/usx2y/usbusx2y.h b/sound/usb/usx2y/usbusx2y.h index 6a76d04bf1c7d..7b6deed17d4b1 100644 --- a/sound/usb/usx2y/usbusx2y.h +++ b/sound/usb/usx2y/usbusx2y.h @@ -5,6 +5,8 @@ #include "../midi.h" #include "usbus428ctldefs.h" +#define USX2Y_IN04_SIZE sizeof(struct us428_ctls) + #define NRURBS 2 /* Default value used for nr of packs per urb. @@ -55,7 +57,7 @@ struct usx2ydev { int stride; struct urb *in04_urb; void *in04_buf; - char in04_last[24]; + char in04_last[USX2Y_IN04_SIZE]; unsigned int in04_int_calls; struct snd_usx2y_urb_seq *us04; wait_queue_head_t in04_wait_queue; diff --git a/sound/usb/usx2y/usx2yhwdeppcm.c b/sound/usb/usx2y/usx2yhwdeppcm.c index 7c90214485d97..f1d5b4b797ad0 100644 --- a/sound/usb/usx2y/usx2yhwdeppcm.c +++ b/sound/usb/usx2y/usx2yhwdeppcm.c @@ -672,6 +672,8 @@ static vm_fault_t snd_usx2y_hwdep_pcm_vm_fault(struct vm_fault *vmf) void *vaddr; offset = vmf->pgoff << PAGE_SHIFT; + if (offset >= USX2Y_HWDEP_PCM_PAGES) + return VM_FAULT_SIGBUS; vaddr = (char *)((struct usx2ydev *)vmf->vma->vm_private_data)->hwdep_pcm_shm + offset; vmf->page = virt_to_page(vaddr); get_page(vmf->page); diff --git a/tools/bootconfig/main.c b/tools/bootconfig/main.c index ddabde20585f2..bf455757aab93 100644 --- a/tools/bootconfig/main.c +++ b/tools/bootconfig/main.c @@ -140,6 +140,9 @@ static int load_xbc_fd(int fd, char **buf, int size) { int ret; + if (size < 0 || size > XBC_DATA_MAX) + return -EINVAL; + *buf = malloc(size + 1); if (!*buf) return -ENOMEM; @@ -168,6 +171,13 @@ static int load_xbc_file(const char *path, char **buf) return ret; } + if (stat.st_size > XBC_DATA_MAX) { + pr_err("%s size is too big\n", path); + ret = -E2BIG; + close(fd); + return ret; + } + ret = load_xbc_fd(fd, buf, stat.st_size); close(fd); @@ -218,7 +228,8 @@ static int load_xbc_from_initrd(int fd, char **buf) csum = le32toh(csum); /* Wrong size error */ - if (stat.st_size < size + BOOTCONFIG_FOOTER_SIZE) { + if (size > XBC_DATA_MAX || + size > stat.st_size - BOOTCONFIG_FOOTER_SIZE) { pr_err("bootconfig size is too big\n"); return -E2BIG; } diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index fd43e5ea63f38..0081f5bf73b72 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -47,7 +47,8 @@ $(LIBBPF_INTERNAL_HDRS): $(LIBBPF_HDRS_DIR)/%.h: $(BPF_DIR)/%.h | $(LIBBPF_HDRS_ $(LIBBPF_BOOTSTRAP): $(wildcard $(BPF_DIR)/*.[ch] $(BPF_DIR)/Makefile) | $(LIBBPF_BOOTSTRAP_OUTPUT) $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \ DESTDIR=$(LIBBPF_BOOTSTRAP_DESTDIR:/=) prefix= \ - ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD="$(HOSTLD)" AR="$(HOSTAR)" $@ install_headers + ARCH= CROSS_COMPILE= CC="$(HOSTCC)" LD="$(HOSTLD)" AR="$(HOSTAR)" \ + CFLAGS="$(LIBBPF_BOOTSTRAP_CFLAGS)" EXTRA_CFLAGS= $@ install_headers $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS): $(LIBBPF_BOOTSTRAP_HDRS_DIR)/%.h: $(BPF_DIR)/%.h | $(LIBBPF_BOOTSTRAP_HDRS_DIR) $(call QUIET_INSTALL, $@) @@ -92,6 +93,9 @@ HOST_CFLAGS := $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),\ $(subst $(CLANG_CROSS_FLAGS),,$(CFLAGS))) HOST_LDFLAGS := $(LDFLAGS) +# Remove warnings for libbpf bootstrap build +LIBBPF_BOOTSTRAP_CFLAGS := $(filter-out -W -Wall -Wextra -Wformat%,$(HOST_CFLAGS)) + INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index c9de44a45778b..941d73b737de3 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -659,8 +659,6 @@ static int do_show_subset(int argc, char **argv) show_map_close_json(fds[i], &info); else show_map_close_plain(fds[i], &info); - - close(fds[i]); } if (json_output && nb_fds > 1) jsonw_end_array(json_wtr); /* root array */ @@ -790,6 +788,12 @@ static int maps_have_btf(int *fds, int nb_fds) static struct btf *btf_vmlinux; +static void free_btf_vmlinux(void) +{ + btf__free(btf_vmlinux); + btf_vmlinux = NULL; +} + static int get_map_kv_btf(const struct bpf_map_info *info, struct btf **btf) { int err = 0; @@ -889,7 +893,6 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, exit_free: free(key); free(value); - close(fd); free_map_kv_btf(btf); return err; @@ -938,6 +941,7 @@ static int do_dump(int argc, char **argv) for (i = 0; i < nb_fds; i++) { if (bpf_map_get_info_by_fd(fds[i], &info, &len)) { p_err("can't get map info: %s", strerror(errno)); + err = -1; break; } err = map_dump(fds[i], &info, wtr, nb_fds > 1); @@ -958,7 +962,7 @@ static int do_dump(int argc, char **argv) close(fds[i]); exit_free: free(fds); - btf__free(btf_vmlinux); + free_btf_vmlinux(); return err; } @@ -1049,7 +1053,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, btf_wtr = get_btf_writer(); if (!btf_wtr) { p_info("failed to create json writer for btf. falling back to plain output"); - btf__free(btf); + free_map_kv_btf(btf); btf = NULL; print_entry_plain(info, key, value); } else { @@ -1065,7 +1069,7 @@ static void print_key_value(struct bpf_map_info *info, void *key, } else { print_entry_plain(info, key, value); } - btf__free(btf); + free_map_kv_btf(btf); } static int do_lookup(int argc, char **argv) diff --git a/tools/bpf/bpftool/sign.c b/tools/bpf/bpftool/sign.c index f9b742f4bb104..1257dba8ef2fd 100644 --- a/tools/bpf/bpftool/sign.c +++ b/tools/bpf/bpftool/sign.c @@ -175,8 +175,11 @@ int bpftool_prog_sign(struct bpf_load_and_run_opts *opts) goto cleanup; } - EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, - &opts->excl_prog_hash_sz, EVP_sha256(), NULL); + if (EVP_Digest(opts->insns, opts->insns_sz, opts->excl_prog_hash, + &opts->excl_prog_hash_sz, EVP_sha256(), NULL) != 1) { + err = -EIO; + goto cleanup; + } bd_out = BIO_new(BIO_s_mem()); if (!bd_out) { diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index aa43dead249cb..835e5e561f7fc 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -643,6 +643,10 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); btf__free(btf_vmlinux); + btf_vmlinux = NULL; + map_info_type = NULL; + map_info_alloc_len = 0; + map_info_type_id = 0; return err; } diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index 32bbe29fe5f6c..a0133d52aa8a0 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -119,9 +119,7 @@ FEATURE_TESTS_EXTRA := \ libbfd-liberty-z \ libopencsd \ libperl \ - cxx \ llvm \ - clang \ libbpf \ libpfm4 \ libdebuginfod \ diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index 95646290cb89c..c972304c32117 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -58,7 +58,6 @@ FILES= \ test-libbpf.bin \ test-get_cpuid.bin \ test-sdt.bin \ - test-cxx.bin \ test-gettid.bin \ test-jvmti.bin \ test-jvmti-cmlr.bin \ @@ -66,7 +65,6 @@ FILES= \ test-sched_getcpu.bin \ test-setns.bin \ test-libopencsd.bin \ - test-clang.bin \ test-llvm.bin \ test-llvm-perf.bin \ test-libaio.bin \ @@ -330,9 +328,6 @@ $(OUTPUT)test-libbpf.bin: $(OUTPUT)test-sdt.bin: $(BUILD) -$(OUTPUT)test-cxx.bin: - $(BUILDXX) -std=gnu++11 - $(OUTPUT)test-gettid.bin: $(BUILD) @@ -358,15 +353,6 @@ $(OUTPUT)test-llvm-perf.bin: $(shell $(LLVM_CONFIG) --system-libs) \ > $(@:.bin=.make.output) 2>&1 -$(OUTPUT)test-clang.bin: - $(BUILDXX) -std=gnu++17 \ - -I$(shell $(LLVM_CONFIG) --includedir) \ - -L$(shell $(LLVM_CONFIG) --libdir) \ - -Wl,--start-group -lclang-cpp -Wl,--end-group \ - $(shell $(LLVM_CONFIG) --libs Core option) \ - $(shell $(LLVM_CONFIG) --system-libs) \ - > $(@:.bin=.make.output) 2>&1 - -include $(OUTPUT)*.d $(OUTPUT)test-libaio.bin: diff --git a/tools/build/feature/test-bpf.c b/tools/build/feature/test-bpf.c index e7a405f83af6e..89d59674f39b9 100644 --- a/tools/build/feature/test-bpf.c +++ b/tools/build/feature/test-bpf.c @@ -20,6 +20,8 @@ # define __NR_bpf 6319 # elif defined(__mips__) && defined(_ABI64) # define __NR_bpf 5315 +# elif defined(__loongarch__) +# define __NR_bpf 280 # else # error __NR_bpf not defined. libbpf does not support your arch. # endif diff --git a/tools/include/linux/compiler.h b/tools/include/linux/compiler.h index f40bd2b04c298..f2f54b0381680 100644 --- a/tools/include/linux/compiler.h +++ b/tools/include/linux/compiler.h @@ -119,7 +119,7 @@ #define __read_mostly #ifndef __attribute_const__ -# define __attribute_const__ +# define __attribute_const__ __attribute__ ((__const__)) #endif #ifndef __maybe_unused diff --git a/tools/include/nolibc/arch-powerpc.h b/tools/include/nolibc/arch-powerpc.h index 204564bbcd328..c7d7a72c16dc6 100644 --- a/tools/include/nolibc/arch-powerpc.h +++ b/tools/include/nolibc/arch-powerpc.h @@ -23,7 +23,7 @@ */ #define _NOLIBC_SYSCALL_CLOBBERLIST \ - "memory", "cr0", "r12", "r11", "r10", "r9" + "memory", "cr0", "ctr", "xer", "r12", "r11", "r10", "r9" #define my_syscall0(num) \ ({ \ diff --git a/tools/include/nolibc/arch-sparc.h b/tools/include/nolibc/arch-sparc.h index ca420d843e254..a9386a4808a75 100644 --- a/tools/include/nolibc/arch-sparc.h +++ b/tools/include/nolibc/arch-sparc.h @@ -45,7 +45,7 @@ \ __asm__ volatile ( \ _NOLIBC_SYSCALL \ - : "+r"(_arg1) \ + : "=r"(_arg1) \ : "r"(_num) \ : "memory", "cc" \ ); \ diff --git a/tools/include/nolibc/crt.h b/tools/include/nolibc/crt.h index 961cfe777c356..d4e726ed80765 100644 --- a/tools/include/nolibc/crt.h +++ b/tools/include/nolibc/crt.h @@ -71,7 +71,7 @@ void _start_c(long *sp) /* find _auxv */ for (auxv = (void *)envp; *auxv++;) - ; + __asm__(""); _auxv = auxv; for (ctor_func = __preinit_array_start; ctor_func < __preinit_array_end; ctor_func++) diff --git a/tools/include/uapi/linux/if_xdp.h b/tools/include/uapi/linux/if_xdp.h index 23a0627814687..50d67df78911f 100644 --- a/tools/include/uapi/linux/if_xdp.h +++ b/tools/include/uapi/linux/if_xdp.h @@ -149,6 +149,7 @@ struct xsk_tx_metadata { __u16 csum_start; /* Offset from csum_start where checksum should be stored. */ __u16 csum_offset; + __u32 reserved; /* XDP_TXMD_FLAGS_LAUNCH_TIME */ /* Launch time in nanosecond against the PTP HW Clock */ diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 339b197972374..1e8f01bfc1bfd 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -59,6 +59,8 @@ # define __NR_bpf 6319 # elif defined(__mips__) && defined(_ABI64) # define __NR_bpf 5315 +# elif defined(__loongarch__) +# define __NR_bpf 280 # else # error __NR_bpf not defined. libbpf does not support your arch. # endif diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index d41defa1936d3..4e2d25e024af4 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -63,6 +63,7 @@ static int realloc_insn_buf(struct bpf_gen *gen, __u32 size) gen->error = -ENOMEM; free(gen->insn_start); gen->insn_start = NULL; + gen->insn_cur = NULL; return -ENOMEM; } gen->insn_start = insn_start; @@ -86,6 +87,7 @@ static int realloc_data_buf(struct bpf_gen *gen, __u32 size) gen->error = -ENOMEM; free(gen->data_start); gen->data_start = NULL; + gen->data_cur = NULL; return -ENOMEM; } gen->data_start = data_start; diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 3baa6025ecba0..94ee5f52b8661 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -12148,13 +12148,14 @@ static const char *arch_specific_lib_paths(void) /* Get full path to program/shared library. */ static int resolve_full_path(const char *file, char *result, size_t result_sz) { - const char *search_paths[3] = {}; + const char *search_paths[4] = {}; int i, perm; if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { search_paths[0] = getenv("LD_LIBRARY_PATH"); search_paths[1] = "/usr/lib64:/usr/lib"; search_paths[2] = arch_specific_lib_paths(); + search_paths[3] = "/lib64:/lib"; perm = R_OK; } else { search_paths[0] = getenv("PATH"); diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 index d1405c7c5354a..c7ae506076bb4 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/declaration/enum.j2 @@ -1,4 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} - bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr); bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, {{ name }} value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 index a07586cbee179..446266ad6d17d 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close.j2 @@ -1,3 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} }; + typedef enum {{ name }} {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 index 2c18948bddf75..cfeee2287e68a 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/enum/definition/close_be.j2 @@ -1,3 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} }; + typedef __be32 {{ name }}; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 index cf65b71eaef39..7ddc2bf3edacc 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 index 1d477c2d197aa..5bf00070ae959 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 index cf65b71eaef39..7ddc2bf3edacc 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/string.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length string) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 index 1d477c2d197aa..5bf00070ae959 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/variable_length_opaque.j2 @@ -2,7 +2,9 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} +{% if maxsize != "0" %} if (value->{{ name }}.len > {{ maxsize }}) return false; +{% endif %} if (xdr_stream_encode_opaque(xdr, value->{{ name }}.data, value->{{ name }}.len) < 0) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 new file mode 100644 index 0000000000000..816291184e8c5 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 index 01d716d0099e9..19ee759d70c6a 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/definition/close.j2 @@ -1,8 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} } u; }; -{%- if name in public_apis %} - -bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); -bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *ptr); -{%- endif -%} diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 75b6592afcc29..141ff11f70a4b 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -216,7 +216,8 @@ static bool is_rust_noreturn(const struct symbol *func) * as well as changes to the source code itself between versions (since * these come from the Rust standard library). */ - return str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || + return str_ends_with(func->name, "_4core3num22from_ascii_radix_panic") || + str_ends_with(func->name, "_4core5sliceSp15copy_from_slice17len_mismatch_fail") || str_ends_with(func->name, "_4core6option13expect_failed") || str_ends_with(func->name, "_4core6option13unwrap_failed") || str_ends_with(func->name, "_4core6result13unwrap_failed") || @@ -228,6 +229,7 @@ static bool is_rust_noreturn(const struct symbol *func) str_ends_with(func->name, "_4core9panicking18panic_nounwind_fmt") || str_ends_with(func->name, "_4core9panicking19assert_failed_inner") || str_ends_with(func->name, "_4core9panicking30panic_null_pointer_dereference") || + str_ends_with(func->name, "_4core9panicking32panic_null_reference_constructed") || str_ends_with(func->name, "_4core9panicking36panic_misaligned_pointer_dereference") || str_ends_with(func->name, "_7___rustc17rust_begin_unwind") || strstr(func->name, "_4core9panicking13assert_failed") || diff --git a/tools/perf/builtin-ftrace.c b/tools/perf/builtin-ftrace.c index 4cc33452d79b6..d8118256243e2 100644 --- a/tools/perf/builtin-ftrace.c +++ b/tools/perf/builtin-ftrace.c @@ -71,18 +71,11 @@ static void ftrace__workload_exec_failed_signal(int signo __maybe_unused, static bool check_ftrace_capable(void) { - bool used_root; - - if (perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_SYS_ADMIN, &used_root)) + if (perf_cap__capable(CAP_PERFMON) || + perf_cap__capable(CAP_SYS_ADMIN)) return true; - pr_err("ftrace only works for %s!\n", - used_root ? "root" - : "users with the CAP_PERFMON or CAP_SYS_ADMIN capability" - ); + pr_err("ftrace only works for users with the CAP_PERFMON or CAP_SYS_ADMIN capability!\n"); return false; } diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c index c82b9720c9296..87cff5f5a99c1 100644 --- a/tools/perf/builtin-record.c +++ b/tools/perf/builtin-record.c @@ -63,6 +63,7 @@ #include #include #include +#include #ifndef HAVE_GETTID #include #endif @@ -652,27 +653,14 @@ static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size) struct record *rec = to; if (record__comp_enabled(rec)) { - struct perf_record_compressed2 *event = map->data; - size_t padding = 0; - u8 pad[8] = {0}; ssize_t compressed = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size); if (compressed < 0) return (int)compressed; - bf = event; thread->samples++; - - /* - * The record from `zstd_compress` is not 8 bytes aligned, which would cause asan - * error. We make it aligned here. - */ - event->data_size = compressed - sizeof(struct perf_record_compressed2); - event->header.size = PERF_ALIGN(compressed, sizeof(u64)); - padding = event->header.size - compressed; - return record__write(rec, map, bf, compressed) || - record__write(rec, map, &pad, padding); + return record__write(rec, map, map->data, compressed); } thread->samples++; @@ -1550,18 +1538,36 @@ static void record__adjust_affinity(struct record *rec, struct mmap *map) } } -static size_t process_comp_header(void *record, size_t increment) +/* + * Called once with data_size == 0 to start a record, then once with + * data_size == compressed payload size to finalize and 8-byte-pad it + * (unaligned records trip ASan in the reader). + * Returns the bytes written, or -1 if it won't fit. + */ +static ssize_t process_comp_header(void *record, size_t dst_size, + size_t data_size) { struct perf_record_compressed2 *event = record; size_t size = sizeof(*event); - if (increment) { - event->header.size += increment; - return increment; + if (data_size) { + size_t padding; + + event->data_size = data_size; + event->header.size = PERF_ALIGN(size + data_size, sizeof(u64)); + padding = event->header.size - size - data_size; + if (padding > dst_size) + return -1; + memset(record + size + data_size, 0, padding); + return padding; } + if (size > dst_size) + return -1; + event->header.type = PERF_RECORD_COMPRESSED2; event->header.size = size; + event->data_size = 0; return size; } @@ -1570,7 +1576,12 @@ static ssize_t zstd_compress(struct perf_session *session, struct mmap *map, void *dst, size_t dst_size, void *src, size_t src_size) { ssize_t compressed; - size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed2) - 1; + /* + * Reserve space so per-record PERF_ALIGN() padding keeps header.size + * within u16. + */ + size_t max_record_size = PERF_SAMPLE_MAX_SIZE + - sizeof(struct perf_record_compressed2) - sizeof(u64); struct zstd_data *zstd_data = &session->zstd_data; if (map && map->file) @@ -2702,7 +2713,7 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) trigger_error(&auxtrace_snapshot_trigger); trigger_error(&switch_output_trigger); err = -1; - goto out_child; + goto out_child_no_flush; } if (auxtrace_record__snapshot_started) { @@ -2849,6 +2860,10 @@ static int __cmd_record(struct record *rec, int argc, const char **argv) out_child: record__stop_threads(rec); record__mmap_read_all(rec, true); + goto out_free_threads; +out_child_no_flush: + /* mmap read already failed — retrying would just fail again */ + record__stop_threads(rec); out_free_threads: record__free_thread_data(rec); evlist__finalize_ctlfd(rec->evlist); diff --git a/tools/perf/builtin-stat.c b/tools/perf/builtin-stat.c index 1202ceb425f07..3fc05180b157c 100644 --- a/tools/perf/builtin-stat.c +++ b/tools/perf/builtin-stat.c @@ -2637,7 +2637,8 @@ int cmd_stat(int argc, const char **argv) output = fopen(output_name, mode); if (!output) { perror("failed to create output file"); - return -1; + status = -1; + goto out; } if (!stat_config.json_output) { clock_gettime(CLOCK_REALTIME, &tm); @@ -2648,7 +2649,8 @@ int cmd_stat(int argc, const char **argv) output = fdopen(output_fd, mode); if (!output) { perror("Failed opening logfd"); - return -errno; + status = -errno; + goto out; } } @@ -2657,7 +2659,8 @@ int cmd_stat(int argc, const char **argv) parse_options_usage(stat_usage, stat_options, "o", 1); parse_options_usage(NULL, stat_options, "log-fd", 0); parse_options_usage(NULL, stat_options, "interval-clear", 0); - return -1; + status = -1; + goto out; } stat_config.output = output; diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json index 9204bfb1d69e0..50d38434f8d3d 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/floating-point.json @@ -179,6 +179,30 @@ "BriefDescription": "Retired scalar floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_ops_retired_by_type.scalar_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point MOV ops.", + "UMask": "0x0a" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_shuffle", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", + "UMask": "0x0b" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point bfloat ops.", + "UMask": "0x0c" + }, + { + "EventName": "fp_ops_retired_by_type.scalar_logical", + "EventCode": "0x0a", + "BriefDescription": "Retired scalar floating-point logical ops.", + "UMask": "0x0d" + }, { "EventName": "fp_ops_retired_by_type.scalar_other", "EventCode": "0x0a", @@ -245,12 +269,24 @@ "BriefDescription": "Retired vector floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_ops_retired_by_type.vector_mov", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_ops_retired_by_type.vector_shuffle", "EventCode": "0x0a", "BriefDescription": "Retired vector floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_ops_retired_by_type.vector_bfloat", + "EventCode": "0x0a", + "BriefDescription": "Retired vector floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_ops_retired_by_type.vector_logical", "EventCode": "0x0a", @@ -278,7 +314,7 @@ { "EventName": "sse_avx_ops_retired.mmx_add", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer add.", + "BriefDescription": "Retired MMX integer add ops.", "UMask": "0x01" }, { @@ -299,16 +335,34 @@ "BriefDescription": "Retired MMX integer multiply-accumulate ops.", "UMask": "0x04" }, + { + "EventName": "sse_avx_ops_retired.mmx_aes", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer AES ops.", + "UMask": "0x05" + }, + { + "EventName": "sse_avx_ops_retired.mmx_sha", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer SHA ops.", + "UMask": "0x06" + }, { "EventName": "sse_avx_ops_retired.mmx_cmp", "EventCode": "0x0b", "BriefDescription": "Retired MMX integer compare ops.", "UMask": "0x07" }, + { + "EventName": "sse_avx_ops_retired.mmx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "sse_avx_ops_retired.mmx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer shift ops.", + "BriefDescription": "Retired MMX integer shift or rotate ops.", "UMask": "0x09" }, { @@ -323,10 +377,17 @@ "BriefDescription": "Retired MMX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "sse_avx_ops_retired.mmx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired MMX integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "sse_avx_ops_retired.mmx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired MMX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.mmx_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -389,16 +450,23 @@ "BriefDescription": "Retired SSE and AVX integer compare ops.", "UMask": "0x70" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_cvt", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "sse_avx_ops_retired.sse_avx_clm", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_cvt", + "Deprecated": "1", "UMask": "0x80" }, { "EventName": "sse_avx_ops_retired.sse_avx_shift", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer shift ops.", + "BriefDescription": "Retired SSE and AVX integer shift or rotate ops.", "UMask": "0x90" }, { @@ -413,10 +481,17 @@ "BriefDescription": "Retired SSE and AVX integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "sse_avx_ops_retired.sse_avx_vnni", + "EventCode": "0x0b", + "BriefDescription": "Retired SSE and AVX integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "sse_avx_ops_retired.sse_avx_pack", "EventCode": "0x0b", - "BriefDescription": "Retired SSE and AVX integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event sse_avx_ops_retired.sse_avx_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { @@ -497,12 +572,24 @@ "BriefDescription": "Retired 128-bit packed floating-point blend ops.", "UMask": "0x09" }, + { + "EventName": "fp_pack_ops_retired.fp128_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point MOV ops.", + "UMask": "0x0a" + }, { "EventName": "fp_pack_ops_retired.fp128_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 128-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "fp_pack_ops_retired.fp128_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 128-bit packed floating-point bfloat ops.", + "UMask": "0x0c" + }, { "EventName": "fp_pack_ops_retired.fp128_logical", "EventCode": "0x0c", @@ -575,12 +662,24 @@ "BriefDescription": "Retired 256-bit packed floating-point blend ops.", "UMask": "0x90" }, + { + "EventName": "fp_pack_ops_retired.fp256_mov", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point MOV ops.", + "UMask": "0xa0" + }, { "EventName": "fp_pack_ops_retired.fp256_shuffle", "EventCode": "0x0c", "BriefDescription": "Retired 256-bit packed floating-point shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "fp_pack_ops_retired.fp256_bfloat", + "EventCode": "0x0c", + "BriefDescription": "Retired 256-bit packed floating-point bfloat ops.", + "UMask": "0xc0" + }, { "EventName": "fp_pack_ops_retired.fp256_logical", "EventCode": "0x0c", @@ -647,16 +746,23 @@ "BriefDescription": "Retired 128-bit packed integer compare ops.", "UMask": "0x07" }, + { + "EventName": "packed_int_op_type.int128_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer convert or pack ops.", + "UMask": "0x08" + }, { "EventName": "packed_int_op_type.int128_clm", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer CLM ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_cvt", + "Deprecated": "1", "UMask": "0x08" }, { "EventName": "packed_int_op_type.int128_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer shift ops.", + "BriefDescription": "Retired 128-bit packed integer shift or rotate ops.", "UMask": "0x09" }, { @@ -671,10 +777,17 @@ "BriefDescription": "Retired 128-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0x0b" }, + { + "EventName": "packed_int_op_type.int128_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 128-bit packed integer VNNI ops.", + "UMask": "0x0c" + }, { "EventName": "packed_int_op_type.int128_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 128-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int128_vnni", + "Deprecated": "1", "UMask": "0x0c" }, { @@ -719,16 +832,34 @@ "BriefDescription": "Retired 256-bit packed integer multiply-accumulate ops.", "UMask": "0x40" }, + { + "EventName": "packed_int_op_type.int256_aes", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer AES ops.", + "UMask": "0x50" + }, + { + "EventName": "packed_int_op_type.int256_sha", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer SHA ops.", + "UMask": "0x60" + }, { "EventName": "packed_int_op_type.int256_cmp", "EventCode": "0x0d", "BriefDescription": "Retired 256-bit packed integer compare ops.", "UMask": "0x70" }, + { + "EventName": "packed_int_op_type.int256_cvt", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer convert or pack ops.", + "UMask": "0x80" + }, { "EventName": "packed_int_op_type.int256_shift", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer shift ops.", + "BriefDescription": "Retired 256-bit packed integer shift or rotate ops.", "UMask": "0x90" }, { @@ -743,10 +874,17 @@ "BriefDescription": "Retired 256-bit packed integer shuffle ops (may include instructions not necessarily thought of as including shuffles e.g. horizontal add, dot product, and certain MOV instructions).", "UMask": "0xb0" }, + { + "EventName": "packed_int_op_type.int256_vnni", + "EventCode": "0x0d", + "BriefDescription": "Retired 256-bit packed integer VNNI ops.", + "UMask": "0xc0" + }, { "EventName": "packed_int_op_type.int256_pack", "EventCode": "0x0d", - "BriefDescription": "Retired 256-bit packed integer pack ops.", + "BriefDescription": "This event is deprecated. Refer to new event packed_int_op_type.int256_vnni", + "Deprecated": "1", "UMask": "0xc0" }, { diff --git a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json index 06bbaea159259..b1994539ece82 100644 --- a/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json +++ b/tools/perf/pmu-events/arch/x86/amdzen5/load-store.json @@ -8,9 +8,15 @@ { "EventName": "ls_locks.bus_lock", "EventCode": "0x25", - "BriefDescription": "Retired Lock instructions which caused a bus lock.", + "BriefDescription": "Retired lock instructions which caused a bus lock.", "UMask": "0x01" }, + { + "EventName": "ls_locks.all", + "EventCode": "0x25", + "BriefDescription": "Retired lock instructions of all types.", + "UMask": "0x1f" + }, { "EventName": "ls_ret_cl_flush", "EventCode": "0x26", diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py index 168c044dd7cc3..59426ba68de55 100755 --- a/tools/perf/pmu-events/jevents.py +++ b/tools/perf/pmu-events/jevents.py @@ -559,13 +559,14 @@ def fix_none(s: Optional[str]) -> str: def print_pending_metrics() -> None: """Optionally close metrics table.""" - def metric_cmp_key(j: JsonEvent) -> Tuple[bool, str, str]: + def metric_cmp_key(j: JsonEvent) -> Tuple[str, str, str, str]: def fix_none(s: Optional[str]) -> str: if s is None: return '' return s - return (j.desc is not None, fix_none(j.pmu), fix_none(j.metric_name)) + return (fix_none(j.pmu), fix_none(j.metric_name), j.metric_expr.ToPerfJson(), + fix_none(j.desc)) global _pending_metrics if not _pending_metrics: diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py index 92acd89ed97aa..ac322891c4f1b 100644 --- a/tools/perf/pmu-events/metric.py +++ b/tools/perf/pmu-events/metric.py @@ -445,7 +445,11 @@ def __init__(self, def __lt__(self, other): """Sort order.""" - return self.name < other.name + if self.name != other.name: + return self.name < other.name + if not self.expr.Equals(other.expr): + return self.expr.ToPerfJson() < other.expr.ToPerfJson() + return self.description < other.description def AddToMetricGroup(self, group): """Callback used when being added to a MetricGroup.""" diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c index 0d2fb7a4ae5bd..a4ee849ad8646 100644 --- a/tools/perf/tests/builtin-test.c +++ b/tools/perf/tests/builtin-test.c @@ -10,34 +10,43 @@ #ifdef HAVE_BACKTRACE_SUPPORT #include #endif -#include -#include #include -#include #include -#include +#include + #include -#include +#include "util/term.h" +#include +#include +#include +#include +#include #include +#include +#include +#include +#include + +#include +#include +#include + #include "builtin.h" +#include "color.h" #include "config.h" +#include "debug.h" #include "hist.h" #include "intlist.h" -#include "tests.h" -#include "debug.h" -#include "color.h" -#include -#include #include "string2.h" #include "symbol.h" +#include "tests-scripts.h" +#include "tests.h" #include "util/rlimit.h" #include "util/strbuf.h" -#include -#include -#include -#include +#include "util/term.h" -#include "tests-scripts.h" +static const char *junit_filename; +static struct strbuf junit_xml_buf = STRBUF_INIT; /* * Command line option to not fork the test running in the same process and @@ -48,6 +57,8 @@ static bool dont_fork; static bool sequential; /* Number of times each test is run. */ static unsigned int runs_per_test = 1; +/* Number of lines to include in failure snippet. */ +static unsigned int failure_snippet_lines = 10; const char *dso_to_test; const char *test_objdump_path = "objdump"; @@ -295,6 +306,11 @@ struct child_test { struct test_suite *test; int suite_num; int test_case_num; + struct strbuf err_output; + int result; + bool done; + struct timespec start_time; + struct timespec end_time; }; static jmp_buf run_test_jmp_buf; @@ -334,7 +350,7 @@ static int run_test_child(struct child_process *process) for (size_t i = 0; i < ARRAY_SIZE(signals); i++) signal(signals[i], child_test_sig_handler); - pr_debug("--- start ---\n"); + pr_debug("---- start ----\n"); pr_debug("test child forked, pid %d\n", getpid()); err = test_function(child->test, child->test_case_num)(child->test, child->test_case_num); pr_debug("---- end(%d) ----\n", err); @@ -349,42 +365,362 @@ static int run_test_child(struct child_process *process) #define TEST_RUNNING -3 +static struct pollfd *global_pfds; +static size_t *global_pfd_indices; +static unsigned int summary_tests_passed; +static unsigned int summary_subtests_passed; +static unsigned int summary_tests_skipped; +static unsigned int summary_tests_failed; +static struct strbuf summary_failed_tests_buf = STRBUF_INIT; + +static int strbuf_addstr_safe(struct strbuf *sb, const char *s); +static int __printf(2, 3) strbuf_addf_safe(struct strbuf *sb, const char *fmt, ...); + +static char *xml_escape(const char *str) +{ + struct strbuf buf = STRBUF_INIT; + const char *p; + char *res; + + if (!str) + return strdup(""); + + for (p = str; *p; p++) { + if (*p == '&') + strbuf_addstr(&buf, "&"); + else if (*p == '<') + strbuf_addstr(&buf, "<"); + else if (*p == '>') + strbuf_addstr(&buf, ">"); + else if (*p == '"') + strbuf_addstr(&buf, """); + else if ((unsigned char)*p >= 32 || *p == '\n' || *p == '\t') + strbuf_addch(&buf, *p); + } + res = strbuf_detach(&buf, NULL); + return res ? res : strdup(""); +} + +static int get_term_width(void) +{ + struct winsize ws; + int cols = 80; + int term_width; + + /* + * If output is redirected to a file or piped, we don't need to wrap + * or truncate at all. Use a massive virtually infinite terminal width + * so descriptions are printed in full. + */ + if (!isatty(fileno(debug_file()))) + return 10000; + + get_term_dimensions(&ws); + if (ws.ws_col > 0) + cols = ws.ws_col; + + /* + * Limit description width to fit on a single line. We subtract 35 + * columns of headroom to allocate space for: + * - The suite index prefix: e.g. " 10.100:" (8 characters) plus 1 space separator. + * - The trailing colon (1 character) and space before status (1 character). + * - The longest status results: e.g. "Skip (some metrics failed)" (26 characters) + * or "Running (XX active)" (20 characters). + * + * A minimum description width of 10 is enforced to ensure names are + * legible even on very narrow consoles. + */ + term_width = cols - 35; + if (term_width < 10) + term_width = 10; + + return term_width; +} + +static int get_max_desc_width(int width) +{ + int term_width = get_term_width(); + + return width > term_width ? term_width : width; +} + static int print_test_result(struct test_suite *t, int curr_suite, int curr_test_case, - int result, int width, int running) + int result, int width, int running, + const char *err_output, double elapsed) { + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + if (test_suite__num_test_cases(t) > 1) { - int subw = width > 2 ? width - 2 : width; + char prefix[32]; + int len = snprintf(prefix, sizeof(prefix), "%3d.%1d:", + curr_suite + 1, curr_test_case + 1); + int pad = len >= 4 ? pad_width + 4 - len : pad_width; + int trunc = len >= 4 ? term_width + 4 - len : term_width; - pr_info("%3d.%1d: %-*s:", curr_suite + 1, curr_test_case + 1, subw, + pr_info("%s %-*.*s:", prefix, pad, trunc, + test_description(t, curr_test_case)); + } else { + pr_info("%3d: %-*.*s:", curr_suite + 1, pad_width, term_width, test_description(t, curr_test_case)); - } else - pr_info("%3d: %-*s:", curr_suite + 1, width, test_description(t, curr_test_case)); + } switch (result) { case TEST_RUNNING: - color_fprintf(stderr, PERF_COLOR_YELLOW, " Running (%d active)\n", running); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Running (%d active)\n", running); break; case TEST_OK: + if (test_suite__num_test_cases(t) > 1) + summary_subtests_passed++; + else + summary_tests_passed++; pr_info(" Ok\n"); break; case TEST_SKIP: { const char *reason = skip_reason(t, curr_test_case); + summary_tests_skipped++; if (reason) - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (%s)\n", reason); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip (%s)\n", reason); else - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n"); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, " Skip\n"); } break; case TEST_FAIL: default: - color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n"); + summary_tests_failed++; + if (test_suite__num_test_cases(t) > 1) + strbuf_addf_safe(&summary_failed_tests_buf, " %3d.%1d: %s\n", + curr_suite + 1, curr_test_case + 1, + test_description(t, curr_test_case)); + else + strbuf_addf_safe(&summary_failed_tests_buf, " %3d: %s\n", + curr_suite + 1, + test_description(t, curr_test_case)); + color_fprintf(debug_file(), PERF_COLOR_RED, " FAILED!\n"); break; } + if (junit_filename && result != TEST_RUNNING) { + const char *classname = t->desc; + const char *testname = test_description(t, curr_test_case); + char *escaped_err = xml_escape(err_output); + char *escaped_class = xml_escape(classname); + char *escaped_test = xml_escape(testname); + + strbuf_addf(&junit_xml_buf, + " \n", + escaped_class, escaped_test, elapsed); + if (result != TEST_OK && result != TEST_SKIP) { + strbuf_addf(&junit_xml_buf, + " \n%s\n \n", + escaped_err); + } else if (result == TEST_SKIP) { + const char *reason = skip_reason(t, curr_test_case); + char *escaped_reason = xml_escape(reason ? reason : "Skip"); + + strbuf_addf(&junit_xml_buf, " \n", + escaped_reason); + free(escaped_reason); + } + strbuf_addstr(&junit_xml_buf, " \n"); + free(escaped_err); + free(escaped_class); + free(escaped_test); + } + return 0; } +static const char * const fail_keywords[] = { + "error", "fail", "segv", "abort", + "signal", "fatal", "panic", "corrupt", NULL +}; + +static const char *find_next_keyword(const char *str, size_t max_len, size_t *kw_len) +{ + const char *best = NULL; + size_t best_len = 0; + int k; + + for (k = 0; fail_keywords[k]; k++) { + const char *s = str; + size_t len = strlen(fail_keywords[k]); + + while ((size_t)(s - str) + len <= max_len) { + size_t i; + + if (best && s >= best) + break; + + for (i = 0; i < len; i++) { + if (tolower(s[i]) != fail_keywords[k][i]) + break; + } + if (i == len) { + if (!best || s < best) { + best = s; + best_len = len; + } + break; + } + s++; + } + } + if (best) { + *kw_len = best_len; + return best; + } + return NULL; +} + +static void print_line_highlighted(FILE *fp, const char *line, size_t len) +{ + const char *s = line; + + while (len > 0) { + size_t kw_len = 0; + const char *match = find_next_keyword(s, len, &kw_len); + + if (!match) { + fwrite(s, 1, len, fp); + break; + } + if (match > s) + fwrite(s, 1, match - s, fp); + if (perf_use_color_default) + fprintf(fp, "%s", PERF_COLOR_RED); + fwrite(match, 1, kw_len, fp); + if (perf_use_color_default) + fprintf(fp, "%s", PERF_COLOR_RESET); + + len -= (match + kw_len) - s; + s = match + kw_len; + } +} + + +static void print_test_failure_snippet(FILE *fp, const char *buf) +{ + size_t num_lines = 0; + size_t max_lines = 128; + const char **lines = calloc(max_lines, sizeof(const char *)); + size_t *line_lens = calloc(max_lines, sizeof(size_t)); + const char *s = buf; + size_t i; + unsigned int picked_count = 0; + bool *pick; + int last_printed = -1; + + if (!lines || !line_lens) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + + while (*s) { + const char *eol = strchr(s, '\n'); + size_t len; + + if (eol) + len = eol - s + 1; + else + len = strlen(s); + + if (num_lines == max_lines) { + const char **new_lines; + size_t *new_lens; + + max_lines *= 2; + new_lines = realloc(lines, max_lines * sizeof(const char *)); + if (!new_lines) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + lines = new_lines; + + new_lens = realloc(line_lens, max_lines * sizeof(size_t)); + if (!new_lens) { + free(lines); free(line_lens); + fprintf(fp, "%s", buf); + return; + } + line_lens = new_lens; + } + lines[num_lines] = s; + line_lens[num_lines] = len; + num_lines++; + s += len; + } + + if (num_lines <= failure_snippet_lines) { + for (i = 0; i < num_lines; i++) + print_line_highlighted(fp, lines[i], line_lens[i]); + free(lines); free(line_lens); + return; + } + + pick = calloc(num_lines, sizeof(bool)); + if (!pick) { + for (i = 0; i < num_lines; i++) + print_line_highlighted(fp, lines[i], line_lens[i]); + free(lines); free(line_lens); + return; + } + + /* Pass 0: Always pick the very first line */ + if (num_lines > 0 && picked_count < failure_snippet_lines) { + pick[0] = true; + picked_count++; + } + + /* Pass 1: Pick lines with failure keywords from start (Highest Priority) */ + for (i = 0; i < num_lines && picked_count < failure_snippet_lines; i++) { + size_t dummy; + + if (find_next_keyword(lines[i], line_lens[i], &dummy)) { + if (!pick[i]) { + pick[i] = true; + picked_count++; + } + /* Prioritize getting the immediate next line for context */ + if (i + 1 < num_lines && !pick[i + 1] && + picked_count < failure_snippet_lines) { + pick[i + 1] = true; + picked_count++; + } + } + } + + /* Pass 2: Fill remaining quota from the end backwards */ + i = num_lines; + while (i > 0 && picked_count < failure_snippet_lines) { + i--; + if (!pick[i]) { + pick[i] = true; + picked_count++; + } + } + + for (i = 0; i < num_lines; i++) { + if (!pick[i]) + continue; + if (last_printed != -1 && (int)i > last_printed + 1) { + if (perf_use_color_default) + fprintf(fp, "%s...%s\n", PERF_COLOR_BLUE, PERF_COLOR_RESET); + else + fprintf(fp, "...\n"); + } + print_line_highlighted(fp, lines[i], line_lens[i]); + last_printed = i; + } + + free(pick); + free(lines); + free(line_lens); +} + static void finish_test(struct child_test **child_tests, int running_test, int child_test_num, int width) { @@ -395,6 +731,9 @@ static void finish_test(struct child_test **child_tests, int running_test, int c struct strbuf err_output = STRBUF_INIT; int last_running = -1; int ret; + struct timespec end_time; + double elapsed; + width = get_max_desc_width(width); if (child_test == NULL) { /* Test wasn't started. */ @@ -409,13 +748,14 @@ static void finish_test(struct child_test **child_tests, int running_test, int c * sub test names. */ if (test_suite__num_test_cases(t) > 1 && curr_test_case == 0) - pr_info("%3d: %-*s:\n", curr_suite + 1, width, test_description(t, -1)); + pr_info("%3d: %-*.*s:\n", curr_suite + 1, width, width, + test_description(t, -1)); /* * Busy loop reading from the child's stdout/stderr that are set to be * non-blocking until EOF. */ - if (err > 0) + if (err >= 0) fcntl(err, F_SETFL, O_NONBLOCK); if (verbose > 1) { if (test_suite__num_test_cases(t) > 1) @@ -448,7 +788,7 @@ static void finish_test(struct child_test **child_tests, int running_test, int c fprintf(debug_file(), PERF_COLOR_DELETE_LINE); } print_test_result(t, curr_suite, curr_test_case, TEST_RUNNING, - width, running); + width, running, NULL, 0.0); last_running = running; } } @@ -469,29 +809,338 @@ static void finish_test(struct child_test **child_tests, int running_test, int c if (len > 0) { err_done = false; buf[len] = '\0'; - strbuf_addstr(&err_output, buf); + strbuf_addstr_safe(&err_output, buf); } } } if (err_done) err_done = check_if_command_finished(&child_test->process); } + /* Drain any remaining data from the pipe. */ + if (err >= 0) { + char buf[512]; + ssize_t len; + + while ((len = read(err, buf, sizeof(buf) - 1)) > 0) { + buf[len] = '\0'; + strbuf_addstr_safe(&err_output, buf); + } + } if (perf_use_color_default && last_running != -1) { /* Erase "Running (.. active)" line printed before poll/sleep. */ fprintf(debug_file(), PERF_COLOR_DELETE_LINE); } /* Clean up child process. */ ret = finish_command(&child_test->process); - if (verbose > 1 || (verbose == 1 && ret == TEST_FAIL)) + child_test->process.pid = 0; + if (child_test->err_output.len > 0) { + struct strbuf merged = STRBUF_INIT; + + if (child_test->err_output.buf) + strbuf_addstr_safe(&merged, child_test->err_output.buf); + if (err_output.buf) + strbuf_addstr_safe(&merged, err_output.buf); + strbuf_release(&err_output); + err_output = merged; + } + if (verbose > 1) fprintf(stderr, "%s", err_output.buf); + else if (verbose == 1 && ret == TEST_FAIL) + print_test_failure_snippet(stderr, err_output.buf); + clock_gettime(CLOCK_MONOTONIC, &end_time); + elapsed = (end_time.tv_sec - child_test->start_time.tv_sec) + + (end_time.tv_nsec - child_test->start_time.tv_nsec) / 1000000000.0; + + print_test_result(t, curr_suite, curr_test_case, ret, width, /*running=*/0, + err_output.buf, elapsed); strbuf_release(&err_output); - print_test_result(t, curr_suite, curr_test_case, ret, width, /*running=*/0); + strbuf_release(&child_test->err_output); if (err > 0) close(err); zfree(&child_tests[running_test]); } +static int strbuf_addstr_safe(struct strbuf *sb, const char *s) +{ + sigset_t set, oldset; + int ret; + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + pthread_sigmask(SIG_BLOCK, &set, &oldset); + ret = strbuf_addstr(sb, s); + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return ret; +} + +static int __printf(2, 3) strbuf_addf_safe(struct strbuf *sb, const char *fmt, ...) +{ + char buf[1024]; + va_list ap; + int len; + sigset_t set, oldset; + int ret; + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigprocmask(SIG_BLOCK, &set, &oldset); + + va_start(ap, fmt); + len = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + + if (len < 0) { + sigprocmask(SIG_SETMASK, &oldset, NULL); + return len; + } + if ((size_t)len >= sizeof(buf)) { + char *dynamic_buf = malloc(len + 1); + + if (!dynamic_buf) { + sigprocmask(SIG_SETMASK, &oldset, NULL); + return -ENOMEM; + } + va_start(ap, fmt); + vsnprintf(dynamic_buf, len + 1, fmt, ap); + va_end(ap); + ret = strbuf_addstr(sb, dynamic_buf); + free(dynamic_buf); + } else { + ret = strbuf_addstr(sb, buf); + } + + sigprocmask(SIG_SETMASK, &oldset, NULL); + return ret; +} + +static void drain_child_process_err(struct child_test *child) +{ + char buf[512]; + ssize_t len; + + while ((len = read(child->process.err, buf, sizeof(buf) - 1)) > 0) { + buf[len] = '\0'; + strbuf_addstr_safe(&child->err_output, buf); + } +} + +static void handle_child_pipe_activity(struct child_test *child, short revents) +{ + if (!revents) + return; + + drain_child_process_err(child); + /* + * If the child closed its end of the pipe (EOF) or encountered + * an error, close the file descriptor immediately and set it + * to -1. This removes it from the pfds array for subsequent + * iterations, preventing a tight CPU busy-loop while waiting + * for the process itself to exit. + */ + if (revents & (POLLHUP | POLLERR | POLLNVAL)) { + close(child->process.err); + child->process.err = -1; + } +} + +static int finish_tests_parallel(struct child_test **child_tests, size_t num_tests, int width) +{ + size_t next_to_print = 0; + struct pollfd *pfds; + size_t *pfd_indices; + size_t num_pfds = 0; + int last_running = -1; + size_t i; + int last_suite_printed = -1; + sigset_t set, oldset; + + width = get_max_desc_width(width); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + + pthread_sigmask(SIG_BLOCK, &set, &oldset); + global_pfds = calloc(num_tests, sizeof(*pfds)); + global_pfd_indices = calloc(num_tests, sizeof(*pfd_indices)); + pfds = global_pfds; + pfd_indices = global_pfd_indices; + if (!pfds || !pfd_indices) { + free(pfds); + free(pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return -ENOMEM; + } + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + + for (i = 0; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child) + continue; + strbuf_init(&child->err_output, 0); + if (child->process.err >= 0) + fcntl(child->process.err, F_SETFL, O_NONBLOCK); + } + + while (next_to_print < num_tests) { + size_t running_count = 0; + size_t p; + + while (next_to_print < num_tests && + (!child_tests[next_to_print] || child_tests[next_to_print]->done)) + next_to_print++; + + if (next_to_print >= num_tests) + break; + + num_pfds = 0; + + for (i = next_to_print; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child || child->done) + continue; + + if (!check_if_command_finished(&child->process)) + running_count++; + + if (child->process.err >= 0) { + pfds[num_pfds].fd = child->process.err; + pfds[num_pfds].events = POLLIN | POLLERR | POLLHUP | POLLNVAL; + pfd_indices[num_pfds] = i; + num_pfds++; + } + } + + if (perf_use_color_default && running_count != (size_t)last_running) { + struct child_test *next_child = child_tests[next_to_print]; + + if (last_running != -1) + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); + + if (next_child) { + if (test_suite__num_test_cases(next_child->test) > 1 && + last_suite_printed != next_child->suite_num) { + pr_info("%3d: %-*.*s:\n", + next_child->suite_num + 1, + width, width, + test_description( + next_child->test, -1)); + last_suite_printed = next_child->suite_num; + } + print_test_result(next_child->test, next_child->suite_num, + next_child->test_case_num, TEST_RUNNING, width, + running_count, NULL, 0.0); + } + last_running = running_count; + } + + if (num_pfds == 0) { + if (running_count > 0) + usleep(10 * 1000); + } else { + int pret = poll(pfds, num_pfds, 100); + + if (pret > 0) { + for (p = 0; p < num_pfds; p++) { + size_t idx = pfd_indices[p]; + + handle_child_pipe_activity(child_tests[idx], + pfds[p].revents); + } + } + } + + for (i = next_to_print; i < num_tests; i++) { + struct child_test *child = child_tests[i]; + + if (!child || child->done) + continue; + + if (check_if_command_finished(&child->process)) { + if (child->process.err >= 0) { + drain_child_process_err(child); + close(child->process.err); + child->process.err = -1; + } + child->result = finish_command(&child->process); + child->process.pid = 0; + clock_gettime(CLOCK_MONOTONIC, &child->end_time); + child->done = true; + } + } + + while (next_to_print < num_tests) { + struct child_test *child = child_tests[next_to_print]; + double elapsed; + + if (!child) { + next_to_print++; + continue; + } + if (!child->done) + break; + + if (perf_use_color_default && last_running != -1) { + fprintf(debug_file(), PERF_COLOR_DELETE_LINE); + last_running = -1; + } + + if (test_suite__num_test_cases(child->test) > 1 && + last_suite_printed != child->suite_num) { + pr_info("%3d: %-*.*s:\n", child->suite_num + 1, + width, width, + test_description(child->test, -1)); + last_suite_printed = child->suite_num; + } + + if (verbose > 1) { + if (test_suite__num_test_cases(child->test) > 1) { + pr_info("%3d.%1d: %s:\n", child->suite_num + 1, + child->test_case_num + 1, + test_description(child->test, + child->test_case_num)); + } else { + pr_info("%3d: %s:\n", child->suite_num + 1, + test_description(child->test, -1)); + } + } + + if (verbose > 1) + fprintf(stderr, "%s", child->err_output.buf); + else if (verbose == 1 && child->result == TEST_FAIL) + print_test_failure_snippet(stderr, child->err_output.buf); + + elapsed = (child->end_time.tv_sec - child->start_time.tv_sec) + + (child->end_time.tv_nsec - + child->start_time.tv_nsec) / 1000000000.0; + + print_test_result(child->test, child->suite_num, child->test_case_num, + child->result, width, 0, child->err_output.buf, elapsed); + pthread_sigmask(SIG_BLOCK, &set, &oldset); + strbuf_release(&child->err_output); + child_tests[next_to_print] = NULL; + zfree(&child); + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + next_to_print++; + } + } + + pthread_sigmask(SIG_BLOCK, &set, &oldset); + free(global_pfds); + free(global_pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; + pthread_sigmask(SIG_SETMASK, &oldset, NULL); + return 0; +} + static int start_test(struct test_suite *test, int curr_suite, int curr_test_case, struct child_test **child, int width, int pass) { @@ -500,11 +1149,18 @@ static int start_test(struct test_suite *test, int curr_suite, int curr_test_cas *child = NULL; if (dont_fork) { if (pass == 1) { + struct timespec start_time, end_time; + double elapsed; + + clock_gettime(CLOCK_MONOTONIC, &start_time); pr_debug("--- start ---\n"); err = test_function(test, curr_test_case)(test, curr_test_case); pr_debug("---- end ----\n"); + clock_gettime(CLOCK_MONOTONIC, &end_time); + elapsed = (end_time.tv_sec - start_time.tv_sec) + + (end_time.tv_nsec - start_time.tv_nsec) / 1000000000.0; print_test_result(test, curr_suite, curr_test_case, err, width, - /*running=*/0); + /*running=*/0, NULL, elapsed); } return 0; } @@ -525,13 +1181,14 @@ static int start_test(struct test_suite *test, int curr_suite, int curr_test_cas (*child)->test_case_num = curr_test_case; (*child)->process.pid = -1; (*child)->process.no_stdin = 1; + (*child)->process.in = -1; + (*child)->process.out = -1; + (*child)->process.err = -1; if (verbose <= 0) { (*child)->process.no_stdout = 1; (*child)->process.no_stderr = 1; } else { (*child)->process.stdout_to_stderr = 1; - (*child)->process.out = -1; - (*child)->process.err = -1; } (*child)->process.no_exec_cmd = run_test_child; if (sequential || pass == 2) { @@ -555,6 +1212,58 @@ static void cmd_test_sig_handler(int sig) siglongjmp(cmd_test_jmp_buf, sig); } +static void print_tests_summary(void) +{ + pr_info("\n=== Test Summary ===\n"); + pr_info("Passed main tests : %u\n", summary_tests_passed); + pr_info("Passed subtests : %u\n", summary_subtests_passed); + pr_info("Skipped tests : %u\n", summary_tests_skipped); + if (summary_tests_failed > 0) { + color_fprintf(debug_file(), PERF_COLOR_RED, "Failed tests : %u\n", + summary_tests_failed); + pr_info("List of failed tests:\n"); + pr_info("%s", summary_failed_tests_buf.buf); + } else { + color_fprintf(debug_file(), PERF_COLOR_GREEN, "Failed tests : 0\n"); + } + + if (junit_filename) { + int fd; + FILE *fp; + + fd = open(junit_filename, O_CREAT | O_TRUNC | O_WRONLY | O_NOFOLLOW, 0644); + if (fd >= 0) { + fp = fdopen(fd, "w"); + if (fp) { + unsigned int total = summary_tests_passed + + summary_subtests_passed + + summary_tests_skipped + + summary_tests_failed; + fprintf(fp, "\n"); + fprintf(fp, "\n"); + fprintf(fp, + " \n", + total, summary_tests_failed, + summary_tests_skipped); + fprintf(fp, "%s", junit_xml_buf.buf); + fprintf(fp, " \n"); + fprintf(fp, "\n"); + fclose(fp); + pr_info("Wrote junit XML output to %s\n", junit_filename); + } else { + close(fd); + pr_err("Failed to associate stream with fd for %s: %s\n", + junit_filename, strerror(errno)); + } + } else { + pr_err("Failed to open %s for writing junit XML output: %s\n", + junit_filename, strerror(errno)); + } + } + strbuf_release(&junit_xml_buf); + strbuf_release(&summary_failed_tests_buf); +} + static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], struct intlist *skiplist) { @@ -632,9 +1341,36 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], } if (intlist__find(skiplist, curr_suite + 1)) { - pr_info("%3d: %-*s:", curr_suite + 1, width, - test_description(*t, -1)); - color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip (user override)\n"); + if (pass == 1) { + int pad_width = get_max_desc_width(width); + int term_width = get_term_width(); + + pr_info("%3d: %-*.*s:", curr_suite + 1, + pad_width, term_width, + test_description(*t, -1)); + color_fprintf(debug_file(), PERF_COLOR_YELLOW, + " Skip (user override)\n"); + summary_tests_skipped++; + if (junit_filename) { + char *escaped_class = + xml_escape((const char *) + test_description(*t, -1)); + char *escaped_test = xml_escape("override"); + char *escaped_reason = + xml_escape("user override"); + + strbuf_addf(&junit_xml_buf, + " \n", + escaped_class, escaped_test); + strbuf_addf(&junit_xml_buf, + " \n", + escaped_reason); + strbuf_addstr(&junit_xml_buf, " \n"); + free(escaped_reason); + free(escaped_test); + free(escaped_class); + } + } continue; } @@ -654,8 +1390,9 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], } if (!sequential) { /* Parallel mode starts tests but doesn't finish them. Do that now. */ - for (size_t x = 0; x < num_tests; x++) - finish_test(child_tests, x, num_tests, width); + err = finish_tests_parallel(child_tests, num_tests, width); + if (err) + goto err_out; } } err_out: @@ -666,6 +1403,11 @@ static int __cmd_test(struct test_suite **suites, int argc, const char *argv[], for (size_t x = 0; x < num_tests; x++) finish_test(child_tests, x, num_tests, width); } + print_tests_summary(); + free(global_pfds); + free(global_pfd_indices); + global_pfds = NULL; + global_pfd_indices = NULL; free(child_tests); return err; } @@ -748,10 +1490,21 @@ static struct test_suite **build_suites(void) for (size_t i = 0, j = 0; i < ARRAY_SIZE(suites); i++, j = 0) \ while ((suite = suites[i][j++]) != NULL) - for_each_suite(t) + for_each_suite(t) { + if (t->setup) { + int ret = t->setup(t); + + if (ret < 0) { + errno = -ret; + return NULL; + } + } num_suites++; + } result = calloc(num_suites + 1, sizeof(struct test_suite *)); + if (!result) + return NULL; for (int pass = 1; pass <= 2; pass++) { for_each_suite(t) { @@ -796,6 +1549,10 @@ int cmd_test(int argc, const char **argv) OPT_STRING(0, "dso", &dso_to_test, "dso", "dso to test"), OPT_STRING(0, "objdump", &test_objdump_path, "path", "objdump binary to use for disassembly and annotations"), + OPT_UINTEGER(0, "failure-snippet-lines", &failure_snippet_lines, + "Number of lines to include in failure snippet, default 10"), + OPT_STRING_OPTARG('j', "junit", &junit_filename, "file", + "Generate junit XML output, default test.xml", "test.xml"), OPT_END() }; const char * const test_subcommands[] = { "list", NULL }; @@ -814,6 +1571,8 @@ int cmd_test(int argc, const char **argv) argc = parse_options_subcommand(argc, argv, test_options, test_subcommands, test_usage, 0); if (argc >= 1 && !strcmp(argv[0], "list")) { suites = build_suites(); + if (!suites) + return errno ? -errno : -ENOMEM; ret = perf_test__list(stdout, suites, argc - 1, argv + 1); free(suites); return ret; @@ -846,7 +1605,14 @@ int cmd_test(int argc, const char **argv) rlimit__bump_memlock(); suites = build_suites(); + if (!suites) { + int err = errno; + + intlist__delete(skiplist); + return err ? -err : -ENOMEM; + } ret = __cmd_test(suites, argc, argv, skiplist); free(suites); + intlist__delete(skiplist); return ret; } diff --git a/tools/perf/tests/shell/kvm.sh b/tools/perf/tests/shell/kvm.sh new file mode 100755 index 0000000000000..5244c3ddfb1cf --- /dev/null +++ b/tools/perf/tests/shell/kvm.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# perf kvm tests +# SPDX-License-Identifier: GPL-2.0 + +set -e + +err=0 +perfdata=$(mktemp /tmp/__perf_kvm_test.perf.data.XXXXX) +qemu_pid_file=$(mktemp /tmp/__perf_kvm_test.qemu.pid.XXXXX) + +cleanup() { + rm -f "${perfdata}" + if [ -f "${qemu_pid_file}" ]; then + if [ -s "${qemu_pid_file}" ]; then + qemu_pid=$(cat "${qemu_pid_file}") + if [ -n "${qemu_pid}" ]; then + kill "${qemu_pid}" 2>/dev/null || true + fi + fi + rm -f "${qemu_pid_file}" + fi + trap - EXIT TERM INT +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT TERM INT + +skip() { + echo "Skip: $1" + cleanup + exit 2 +} + +test_kvm_stat() { + echo "Testing perf kvm stat" + + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm events for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + if ! perf kvm stat record -p "${qemu_pid}" -o "${perfdata}" \ + sleep ${duration} >/dev/null 2>&1; then + echo "perf kvm stat record failed, retrying..." + continue + fi + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" stat report 2>&1 | grep -q "VM-EXIT"; then + success=true + break + fi + echo "No VM-EXIT events found, retrying..." + done + + if [ "$success" = false ]; then + echo "Failed to find VM-EXIT in report" + perf kvm -i "${perfdata}" stat report 2>&1 || true + err=1 + return + fi + + echo "perf kvm stat test [Success]" +} + +test_kvm_record_report() { + echo "Testing perf kvm record/report" + + local duration + local success=false + for duration in 1 2 4 8; do + echo "Recording kvm profile for pid ${qemu_pid} (duration ${duration}s)..." + rm -f "${perfdata}" "${perfdata}".old + + perf kvm --host record -p "${qemu_pid}" -o "${perfdata}" \ + -e cpu-clock sleep ${duration} + + if [ -e "${perfdata}" ] && \ + perf kvm -i "${perfdata}" report --stdio 2>&1 | grep -q "Event count"; then + success=true + break + fi + echo "No samples or report failed, retrying..." + done + + if [ "$success" = false ]; then + echo "Failed to report kvm profile" + perf kvm -i "${perfdata}" report --stdio 2>&1 || true + err=1 + return + fi + + echo "perf kvm record/report test [Success]" +} + +test_kvm_buildid_list() { + echo "Testing perf kvm buildid-list" + + # We reuse the perf.data from the previous record test + if ! perf kvm --host -i "${perfdata}" buildid-list 2>&1 | grep -q "."; then + echo "Failed to list buildids" + perf kvm --host -i "${perfdata}" buildid-list 2>&1 + err=1 + return + fi + + echo "perf kvm buildid-list test [Success]" +} + +setup_qemu() { + # Find qemu + if [ "$(uname -m)" = "x86_64" ]; then + qemu="qemu-system-x86_64" + elif [ "$(uname -m)" = "aarch64" ]; then + qemu="qemu-system-aarch64" + elif [ "$(uname -m)" = "s390x" ]; then + qemu="qemu-system-s390x" + elif [ "$(uname -m)" = "ppc64le" ]; then + qemu="qemu-system-ppc64" + else + qemu="qemu-system-$(uname -m)" + fi + + if ! which -s "$qemu"; then + skip "$qemu not found" + fi + + if [ ! -r /dev/kvm ] || [ ! -w /dev/kvm ]; then + skip "/dev/kvm not accessible" + fi + + if ! perf kvm stat record -o /dev/null -a sleep 0.01 >/dev/null 2>&1; then + skip "No permission to record kvm events" + fi + + echo "Starting $qemu..." + # Start qemu in background, detached, with pidfile + # We use -display none -daemonize and a monitor to keep it alive/controllable if needed + # We don't need a real kernel, just KVM active. + if ! $qemu -enable-kvm -display none -daemonize -pidfile "${qemu_pid_file}" -monitor none; then + echo "Failed to start qemu" + err=1 + return + fi + + # Wait a bit for qemu to start + sleep 1 + qemu_pid=$(cat "${qemu_pid_file}") + + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "Qemu process failed to stay alive" + err=1 + return + fi +} + +setup_qemu +if [ $err -eq 0 ]; then + test_kvm_stat + test_kvm_record_report + test_kvm_buildid_list +fi + +cleanup +exit $err diff --git a/tools/perf/tests/shell/lib/perf_metric_validation.py b/tools/perf/tests/shell/lib/perf_metric_validation.py index dea8ef1977bf6..3d52f94f22b91 100644 --- a/tools/perf/tests/shell/lib/perf_metric_validation.py +++ b/tools/perf/tests/shell/lib/perf_metric_validation.py @@ -383,10 +383,13 @@ def _run_perf(self, metric, workload: str): wl = workload.split() command.extend(wl) print(" ".join(command)) - cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8') - data = [x+'}' for x in cmd.stderr.split('}\n') if x] - if data[0][0] != '{': - data[0] = data[0][data[0].find('{'):] + cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') + lines = cmd.stderr.splitlines() + cmd.stdout.splitlines() + data = [] + for line in lines: + line = line.strip() + if line.startswith('{') and line.endswith('}'): + data.append(line) return data def collect_perf(self, workload: str): diff --git a/tools/perf/tests/shell/lib/perf_record.sh b/tools/perf/tests/shell/lib/perf_record.sh new file mode 100644 index 0000000000000..2b9e11b66dc7a --- /dev/null +++ b/tools/perf/tests/shell/lib/perf_record.sh @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: GPL-2.0 + +PERF_RECORD_LOGS=() + +perf_record_with_retry() { + local perfdata="$1" + local check_cmd="$2" + local testprog_base="$3" + shift 3 + + local logfile + logfile=$(mktemp /tmp/__perf_record_retry.XXXXXX) + PERF_RECORD_LOGS+=("$logfile") + + # Save the e flag state and disable it + local save_e + if [[ $- == *e* ]]; then + save_e="set -e" + else + save_e="set +e" + fi + set +e + + local duration + local first_run=true + local ret=1 + local cmd_prefix="perf record" + if [ -n "${PERF_RECORD_CMD}" ]; then + cmd_prefix="${PERF_RECORD_CMD}" + fi + + for duration in 0.01 0.1 0.3 1.0 2.0; do + rm -f "${perfdata}".old + ${cmd_prefix} "$@" -o "${perfdata}" ${testprog_base} ${duration} > "$logfile" 2>&1 + local record_exit=$? + + if [ "$first_run" = true ] && [ $record_exit -ne 0 ]; then + ret=2 + break + fi + first_run=false + + if [ -e "${perfdata}" ] && eval "${check_cmd}"; then + ret=0 + break + fi + done + + eval "$save_e" + return $ret +} + +perf_record_cleanup() { + for logfile in "${PERF_RECORD_LOGS[@]}"; do + rm -f "$logfile" + done + PERF_RECORD_LOGS=() +} diff --git a/tools/perf/tests/shell/pipe_test.sh b/tools/perf/tests/shell/pipe_test.sh index e459aa99a9515..ce68d850c9838 100755 --- a/tools/perf/tests/shell/pipe_test.sh +++ b/tools/perf/tests/shell/pipe_test.sh @@ -12,8 +12,8 @@ skip_test_missing_symbol ${sym} data=$(mktemp /tmp/perf.data.XXXXXX) data2=$(mktemp /tmp/perf.data2.XXXXXX) -prog="perf test -w noploop" -[ "$(uname -m)" = "s390x" ] && prog="$prog 3" +prog="perf test -w noploop 0.1" +[ "$(uname -m)" = "s390x" ] && prog="perf test -w noploop 3" err=0 set -e diff --git a/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh new file mode 100755 index 0000000000000..c05ace8214ca3 --- /dev/null +++ b/tools/perf/tests/shell/record+zstd_comp_decomp_multi_record.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Zstd perf.data compression/decompression of multi-record data +# SPDX-License-Identifier: GPL-2.0 + +perfdata=$(mktemp /tmp/__perf_test.perf.data.XXXXX) +recout=$(mktemp /tmp/__perf_test.zstd.rec.XXXXX) +injout=$(mktemp /tmp/__perf_test.zstd.inj.XXXXX) +perf_tool=perf + +cleanup() { + rm -f "${perfdata}" "${perfdata}".old "${perfdata}".decomp "${recout}" "${injout}" +} +trap cleanup EXIT TERM INT + +skip_if_no_z_record() { + $perf_tool record -h 2>&1 | grep -q -- '-z, --compression-level' +} + +collect_z_record() { + echo "Collecting compressed record file:" + [ "$(uname -m)" != s390x ] && gflag='-g' + $perf_tool record -o "${perfdata}" $gflag -z -F max -m 32M --per-thread -- \ + $perf_tool test -w thloop 5 1 \ + >/dev/null 2>"${recout}" +} + +check_record() { + echo "Checking record did not fail to write data:" + if grep -q "failed to write perf data" "${recout}"; then + cat "${recout}" + return 1 + fi +} + +check_decompress() { + echo "Checking compressed file decompresses cleanly:" + if ! $perf_tool inject -i "${perfdata}" -o "${perfdata}".decomp 2>"${injout}"; then + cat "${injout}" + return 1 + fi + if grep -Eqi "decompress|corrupt|failed to process type" "${injout}"; then + cat "${injout}" + return 1 + fi +} + +skip_if_no_z_record || exit 2 +collect_z_record +check_record || exit 1 + +# Need >1 record, else the multi-record path wasn't exercised. +# Skip rather than pass/fail spuriously. +nr=$($perf_tool report -i "${perfdata}" --stats 2>/dev/null | + awk '/COMPRESSED2 events:/ { print $3 }') +if [ -z "${nr}" ] || [ "${nr}" -lt 2 ]; then + echo "less than two compressed records (${nr:-0}), skipping" + exit 2 +fi +echo "Produced ${nr} compressed records" + +check_decompress +err=$? +exit $err diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh index 0f5841c479e75..fc78ca52075c4 100755 --- a/tools/perf/tests/shell/record.sh +++ b/tools/perf/tests/shell/record.sh @@ -1,10 +1,13 @@ #!/bin/bash -# perf record tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record tests set -e shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + + # shellcheck source=lib/waiting.sh . "${shelldir}"/lib/waiting.sh @@ -39,6 +42,7 @@ cleanup() { rm -f "${perfdata}" rm -f "${perfdata}".old rm -f "${script_output}" + perf_record_cleanup trap - EXIT TERM INT } @@ -50,22 +54,20 @@ trap_cleanup() { } trap trap_cleanup EXIT TERM INT +check_per_thread() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_per_thread() { echo "Basic --per-thread mode test" - if ! perf record -o /dev/null --quiet ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_per_thread" "perf test -w thloop" \ + --per-thread || ret=$? + if [ $ret -eq 2 ]; then echo "Per-thread record [Skipped event not supported]" return - fi - if ! perf record --per-thread -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Per-thread record [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Per-thread record [Failed missing output]" + elif [ $ret -eq 1 ]; then + echo "Per-thread record [Failed record or missing output]" err=1 return fi @@ -96,6 +98,10 @@ test_per_thread() { echo "Basic --per-thread mode test [Success]" } +check_register_capture() { + perf script -F ip,sym,iregs -i "${perfdata}" 2>/dev/null | grep -q "DI:" +} + test_register_capture() { echo "Register capture test" if ! perf list pmu | grep -q 'br_inst_retired.near_call' @@ -108,11 +114,12 @@ test_register_capture() { echo "Register capture test [Skipped missing registers]" return fi - if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call \ - -c 1000 --per-thread ${testprog} 2> /dev/null \ - | perf script -F ip,sym,iregs -i - 2> /dev/null \ - | grep -q "DI:" - then + + local ret=0 + perf_record_with_retry "${perfdata}" "check_register_capture" "perf test -w thloop" \ + --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call -c 1000 --per-thread || ret=$? + + if [ $ret -ne 0 ]; then echo "Register capture test [Failed missing output]" err=1 return @@ -120,65 +127,66 @@ test_register_capture() { echo "Register capture test [Success]" } +check_system_wide() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_system_wide() { echo "Basic --system-wide mode test" - if ! perf record -aB --synth=no -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no || ret=$? + if [ $ret -eq 2 ]; then echo "System-wide record [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "System-wide record [Failed missing output]" err=1 return fi - if ! perf record -aB --synth=no -e cpu-clock,cs --threads=cpu \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "System-wide record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "System-wide record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_system_wide" "perf test -w thloop" \ + -aB --synth=no -e cpu-clock,cs --threads=cpu || ret=$? + if [ $ret -ne 0 ]; then + echo "System-wide record [Failed record --threads option or missing output]" err=1 return fi echo "Basic --system-wide mode test [Success]" } +check_workload() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_workload() { echo "Basic target workload test" - if ! perf record -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record or missing output]" err=1 return fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed missing output]" - err=1 - return - fi - if ! perf record -e cpu-clock,cs --threads=package \ - -o "${perfdata}" ${testprog} 2> /dev/null - then - echo "Workload record [Failed record --threads option]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then - echo "Workload record [Failed --threads missing output]" + + ret=0 + perf_record_with_retry "${perfdata}" "check_workload" "perf test -w thloop" \ + -e cpu-clock,cs --threads=package || ret=$? + if [ $ret -ne 0 ]; then + echo "Workload record [Failed record --threads option or missing output]" err=1 return fi echo "Basic target workload test [Success]" } +check_branch_counter() { + perf report -i "${perfdata}" -D -q 2>/dev/null | grep -q "$br_cntr_output" && \ + perf script -i "${perfdata}" -F +brstackinsn,+brcntr 2>/dev/null | \ + grep -q "$br_cntr_script_output" +} + test_branch_counter() { echo "Branch counter test" # Check if the branch counter feature is supported @@ -190,67 +198,60 @@ test_branch_counter() { return fi done - if ! perf record -o "${perfdata}" -e "{branches:p,instructions}" -j any,counter ${testprog} 2> /dev/null - then - echo "Branch counter record test [Failed record]" - err=1 - return - fi - if ! perf report -i "${perfdata}" -D -q | grep -q "$br_cntr_output" - then - echo "Branch counter report test [Failed missing output]" - err=1 - return - fi - if ! perf script -i "${perfdata}" -F +brstackinsn,+brcntr | grep -q "$br_cntr_script_output" - then - echo " Branch counter script test [Failed missing output]" + local ret=0 + perf_record_with_retry "${perfdata}" "check_branch_counter" "perf test -w thloop" \ + -e "{branches:p,instructions}" -j any,counter || ret=$? + if [ $ret -ne 0 ]; then + echo "Branch counter test [Failed record or missing output]" err=1 return fi echo "Branch counter test [Success]" } +check_cgroup() { + perf report -i "${perfdata}" -D 2>/dev/null | grep -q "CGROUP" && \ + perf script -i "${perfdata}" -F cgroup 2>/dev/null | grep -q -v "unknown" +} + test_cgroup() { echo "Cgroup sampling test" - if ! perf record -aB --synth=cgroup --all-cgroups -o "${perfdata}" ${testprog} 2> /dev/null - then + local ret=0 + perf_record_with_retry "${perfdata}" "check_cgroup" "perf test -w thloop" \ + -aB --synth=cgroup --all-cgroups || ret=$? + if [ $ret -eq 2 ]; then echo "Cgroup sampling [Skipped not supported]" return - fi - if ! perf report -i "${perfdata}" -D | grep -q "CGROUP" - then + elif [ $ret -eq 1 ]; then echo "Cgroup sampling [Failed missing output]" err=1 return fi - if ! perf script -i "${perfdata}" -F cgroup | grep -q -v "unknown" - then - echo "Cgroup sampling [Failed cannot resolve cgroup names]" - err=1 - return - fi echo "Cgroup sampling test [Success]" } +check_uid() { + perf report -i "${perfdata}" -q | grep -q "${testsym}" +} + test_uid() { echo "Uid sampling test" - if ! perf record -aB --synth=no --uid "$(id -u)" -o "${perfdata}" ${testprog} \ - > "${script_output}" 2>&1 - then - if grep -q "libbpf.*EPERM" "${script_output}" + local ret=0 + perf_record_with_retry "${perfdata}" "check_uid" "perf test -w thloop" \ + -aB --synth=no --uid "$(id -u)" || ret=$? + if [ $ret -eq 2 ]; then + local logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + if grep -q -E "libbpf.*EPERM|Access to performance monitoring" "$logfile" || \ + grep -q -E "Permission denied|Failure to open any events" "$logfile" then echo "Uid sampling [Skipped permissions]" return else echo "Uid sampling [Failed to record]" err=1 - # cat "${script_output}" return fi - fi - if ! perf report -i "${perfdata}" -q | grep -q "${testsym}" - then + elif [ $ret -eq 1 ]; then echo "Uid sampling [Failed missing output]" err=1 return diff --git a/tools/perf/tests/shell/record_lbr.sh b/tools/perf/tests/shell/record_lbr.sh index 78a02e90ece1e..8d51afeb437ba 100755 --- a/tools/perf/tests/shell/record_lbr.sh +++ b/tools/perf/tests/shell/record_lbr.sh @@ -1,9 +1,12 @@ #!/bin/bash -# perf record LBR tests (exclusive) # SPDX-License-Identifier: GPL-2.0 +# perf record LBR tests set -e +shelldir=$(dirname "$0") +. "${shelldir}"/lib/perf_record.sh + ParanoidAndNotRoot() { [ "$(id -u)" != 0 ] && [ "$(cat /proc/sys/kernel/perf_event_paranoid)" -gt $1 ] } @@ -22,6 +25,7 @@ cleanup() { rm -rf "${perfdata}" rm -rf "${perfdata}".old rm -rf "${perfdata}".txt + perf_record_cleanup trap - EXIT TERM INT } @@ -34,22 +38,28 @@ trap_cleanup() { trap trap_cleanup EXIT TERM INT +check_lbr_callgraph() { + perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt 2>&1 +} + lbr_callgraph_test() { test="LBR callgraph" echo "$test" - if ! perf record -e cycles --call-graph lbr -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_callgraph" "perf test -w thloop" \ + -e cycles --call-graph lbr + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" if [ $err -eq 0 ] then err=2 fi return - fi - - if ! perf report --stitch-lbr -i "${perfdata}" > "${perfdata}".txt - then + elif [ $ret -eq 1 ]; then cat "${perfdata}".txt echo "$test [Failed in perf report]" err=1 @@ -59,6 +69,12 @@ lbr_callgraph_test() { echo "$test [Success]" } +check_lbr_samples() { + local out + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + [ "$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true)" -gt 0 ] +} + lbr_test() { local branch_flags=$1 local test="LBR $2 test" @@ -70,25 +86,27 @@ lbr_test() { local r echo "$test" - if ! perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop - then + set +e + perf_record_with_retry "${perfdata}" "check_lbr_samples" "perf test -w thloop" \ + -e cycles $branch_flags + local ret=$? + set -e + + if [ $ret -eq 2 ]; then echo "$test [Failed support missing]" - perf record -e cycles $branch_flags -o "${perfdata}" perf test -w thloop || true if [ $err -eq 0 ] then err=2 fi return - fi - - out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') - sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) - if [ $sam_nr -eq 0 ] - then + elif [ $ret -eq 1 ]; then echo "$test [Failed no samples captured]" err=1 return fi + + out=$(perf report -D -i "${perfdata}" 2> /dev/null | grep -A1 'PERF_RECORD_SAMPLE') + sam_nr=$(echo "$out" | grep -c 'PERF_RECORD_SAMPLE' || true) echo "$test: $sam_nr samples" bs_nr=$(echo "$out" | grep -c 'branch stack: nr:' || true) diff --git a/tools/perf/tests/shell/stat_all_metrics.sh b/tools/perf/tests/shell/stat_all_metrics.sh index 6fa585a1e34c9..feeb34c6fa6df 100755 --- a/tools/perf/tests/shell/stat_all_metrics.sh +++ b/tools/perf/tests/shell/stat_all_metrics.sh @@ -12,93 +12,137 @@ system_wide_flag="-a" if ParanoidAndNotRoot 0 then system_wide_flag="" - test_prog="perf test -w noploop" + test_prog="perf test -w noploop 0.01" fi -err=0 +check_metric() { + local output="$1" + local status="$2" + local metric="$3" + + if [[ $status -ne 0 || ! "$output" =~ ${metric:0:50} ]]; then + return 1 + fi + + if [[ "$output" =~ "" || "$output" =~ "" ]]; then + return 1 + fi + + return 0 +} + +skip=0 +err=3 for m in $(perf list --raw-dump metrics); do echo "Testing $m" result=$(perf stat -M "$m" $system_wide_flag -- $test_prog 2>&1) result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. + + if check_metric "$result" $result_err "$m"; then + if [[ "$err" -ne 1 ]] + then + err=0 + fi continue fi - if [[ "$result" =~ "Cannot resolve IDs for" ]] + + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" || \ + "$result" =~ "in per-thread mode, enable system wide" || \ + "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" || \ + "$result" =~ "FP_ARITH" || \ + "$result" =~ "AMX" || \ + "$result" =~ "PMM" ]] then - echo "Metric contains missing events" - echo $result - err=1 # Fail - continue - elif [[ "$result" =~ \ - "Access to performance monitoring and observability operations is limited" ]] + true + else + result=$(perf stat -M "$m" $system_wide_flag -- perf test -w noploop 0.1 2>&1) + result_err=$? + + if check_metric "$result" $result_err "$m"; then + if [[ "$err" -ne 1 ]] + then + err=0 + fi + continue + fi + fi + + # If retry also failed, determine if we skip, ignore, or fail + if [[ "$result" =~ "Access to performance monitoring and observability operations is limited" ]] then - echo "Permission failure" + echo "[Skipped $m] Permission failure" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue elif [[ "$result" =~ "in per-thread mode, enable system wide" ]] then - echo "Permissions - need system wide mode" + echo "[Skipped $m] Permissions - need system wide mode" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue - elif [[ "$result" =~ "" ]] + elif [[ "$result" =~ "" || \ + "$result" =~ "Cannot resolve IDs for" || \ + "$result" =~ "No supported events found" ]] then - echo "Not supported events" + if [[ $(perf list --raw-dump $m) == "Default"* ]] + then + echo "[Ignored $m] failed but as a Default metric this can be expected" + echo $result + continue + fi + echo "[Skipped $m] Not supported events" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue elif [[ "$result" =~ "" ]] then - echo "Not counted events" + echo "[Skipped $m] Not counted events" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue elif [[ "$result" =~ "FP_ARITH" || "$result" =~ "AMX" ]] then - echo "FP issues" + echo "[Skipped $m] FP issues" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue elif [[ "$result" =~ "PMM" ]] then - echo "Optane memory issues" + echo "[Skipped $m] Optane memory issues" echo $result if [[ $err -eq 0 ]] then - err=2 # Skip + skip=1 fi continue fi - # Failed, possibly the workload was too small so retry with something longer. - result=$(perf stat -M "$m" $system_wide_flag -- perf bench internals synthesize 2>&1) - result_err=$? - if [[ $result_err -eq 0 && "$result" =~ ${m:0:50} ]] - then - # No error result and metric shown. - continue - fi - echo "Metric '$m' has non-zero error '$result_err' or not printed in:" + echo "[Failed $m] has non-zero error '$result_err' or not printed/counted in:" echo "$result" err=1 done +# return SKIP only if no success returned +if [[ "$err" -eq 3 && "$skip" -eq 1 ]] +then + err=2 +fi + exit "$err" diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/tests/shell/stat_bpf_counters.sh index f43e28a136d3c..11de77ee38ad4 100755 --- a/tools/perf/tests/shell/stat_bpf_counters.sh +++ b/tools/perf/tests/shell/stat_bpf_counters.sh @@ -4,21 +4,26 @@ set -e -workload="perf test -w sqrtloop" +# Get the first allowed CPU +CPU=$(taskset -c -p $$ | awk -F': ' '{print $2}' | awk -F'[,-]' '{print $1}') +if [ -z "$CPU" ]; then + CPU=0 +fi +workload=(taskset -c "$CPU" awk 'BEGIN { for (i=0; i<10000000; i++) sum+=i }') -# check whether $2 is within +/- 20% of $1 +# check whether $2 is within +/- 15% of $1 compare_number() { first_num=$1 second_num=$2 - # upper bound is first_num * 120% - upper=$(expr $first_num + $first_num / 5 ) - # lower bound is first_num * 80% - lower=$(expr $first_num - $first_num / 5 ) + # upper bound is first_num * 115% + upper=$(expr $first_num + $first_num / 20 \* 3 ) + # lower bound is first_num * 85% + lower=$(expr $first_num - $first_num / 20 \* 3 ) if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then - echo "The difference between $first_num and $second_num are greater than 20%." + echo "The difference between $first_num and $second_num are greater than 15%." exit 1 fi } @@ -41,8 +46,15 @@ check_counts() test_bpf_counters() { printf "Testing --bpf-counters " - base_instructions=$(perf stat --no-big-num -e instructions -- $workload 2>&1 | awk '/instructions/ {print $1}') - bpf_instructions=$(perf stat --no-big-num --bpf-counters -e instructions -- $workload 2>&1 | awk '/instructions/ {print $1}') + base_instructions=$(perf stat --no-big-num -e instructions:u -- "${workload[@]}" 2>&1 | \ + awk -v i=0 -v c=0 '/instructions/ { \ + if ($1 != " 0) printf "%.0f", c; else print "&1 | \ + awk -v i=0 -v c=0 '/instructions/ { \ + if ($1 != " 0) printf "%.0f", c; else print "&1) - base_instructions=$(echo "$stat_output"| awk '/base_instructions/ {print $1}') - bpf_instructions=$(echo "$stat_output"| awk '/bpf_instructions/ {print $1}') + stat_output=$(perf stat --no-big-num \ + -e instructions/name=base_instructions/u,instructions/name=bpf_instructions/bu \ + -- "${workload[@]}" 2>&1) + base_instructions=$(echo "$stat_output"| \ + awk -v i=0 -v c=0 '/base_instructions/ { \ + if ($1 != " 0) printf "%.0f", c; else print " 0) printf "%.0f", c; else print "/dev/null 2>&1; then + echo "Skipping: no permission to record system-wide events (-a)" + exit 2 +fi + + pythonvalidator=$(dirname $0)/lib/perf_metric_validation.py rulefile=$(dirname $0)/lib/perf_metric_validation_rules.json tmpdir=$(mktemp -d /tmp/__perf_test.program.XXXXX) diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh index 85233d435be63..71550e0b37baa 100755 --- a/tools/perf/tests/shell/test_brstack.sh +++ b/tools/perf/tests/shell/test_brstack.sh @@ -38,9 +38,13 @@ is_arm64() { [ "$(uname -m)" = "aarch64" ]; } +has_kaslr_bug() { + [ "$(uname -m)" != "aarch64" ]; +} + check_branches() { if ! tr -s ' ' '\n' < "$TMPDIR/perf.script" | grep -E -m1 -q "$1"; then - echo "Branches missing $1" + echo "ERROR: Branches missing $1" err=1 fi } @@ -48,6 +52,8 @@ check_branches() { test_user_branches() { echo "Testing user branch stack sampling" + start_err=$err + err=0 perf record -o "$TMPDIR/perf.data" --branch-filter any,save_type,u -- ${TESTPROG} > "$TMPDIR/record.txt" 2>&1 perf script -i "$TMPDIR/perf.data" --fields brstacksym > "$TMPDIR/perf.script" @@ -73,59 +79,105 @@ test_user_branches() { perf script -i "$TMPDIR/perf.data" --fields brstack | \ tr ' ' '\n' > "$TMPDIR/perf.script" - # There should be no kernel addresses with the u option, in either - # source or target addresses. - if grep -E -m1 "0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then - echo "ERROR: Kernel address found in user mode" + # There should be no kernel addresses in the target with the u option. + local regex="0x[89a-f][0-9a-f]{15}" + if has_kaslr_bug; then + # If the system has a kaslr bug that may leak kernel addresses + # in the source of something like an ERET/SYSRET. Make the regex + # more specific and just check the target address is in user + # code. + regex="^0x[0-9a-f]{0,16}/0x[89a-f][0-9a-f]{15}/" + fi + if grep -q -E -m1 "$regex" $TMPDIR/perf.script; then + echo "Testing user branch stack sampling [Failed kernel address found in user mode]" err=1 fi # some branch types are still not being tested: # IND COND_CALL COND_RET SYSRET SERROR NO_TX + if [ $err -eq 0 ]; then + echo "Testing user branch stack sampling [Passed]" + err=$start_err + else + echo "Testing user branch stack sampling [Failed]" + fi } test_trap_eret_branches() { echo "Testing trap & eret branches" + if ! is_arm64; then - echo "skip: not arm64" - else + echo "Testing trap & eret branches [Skipped not arm64]" + return + fi + start_err=$err + local ret=1 + for loops in 1000 10000 100000; do + err=0 perf record -o $TMPDIR/perf.data --branch-filter any,save_type,u,k -- \ - perf test -w traploop 1000 + perf test -w traploop $loops > "$TMPDIR/record.txt" 2>&1 perf script -i $TMPDIR/perf.data --fields brstacksym | \ tr ' ' '\n' > $TMPDIR/perf.script # BRBINF.TYPE == TRAP are mapped to PERF_BR_IRQ by the BRBE driver check_branches "^trap_bench\+[^ ]+/[^ ]/IRQ/" check_branches "^[^ ]+/trap_bench\+[^ ]+/ERET/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then + echo "Testing trap & eret branches [Passed]" + err=$start_err + else + echo "Testing trap & eret branches [Failed]" + err=1 fi } test_kernel_branches() { - echo "Testing that k option only includes kernel source addresses" + echo "Testing kernel branch sampling" - if ! perf record --branch-filter any,k -o- -- true > /dev/null; then - echo "skip: not enough privileges" - else + if ! perf record --branch-filter any,k -o- -- true > "$TMPDIR/record.txt" 2>&1; then + echo "Testing that k option [Skipped not enough privileges]" + return + fi + start_err=$err + local ret=1 + for loops in 1000 10000 100000; do + err=0 perf record -o $TMPDIR/perf.data --branch-filter any,k -- \ - perf bench syscall basic --loop 1000 + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 perf script -i $TMPDIR/perf.data --fields brstack | \ tr ' ' '\n' > $TMPDIR/perf.script # Example of branch entries: # "0xffffffff93bda241/0xffffffff93bda20f/M/-/-/..." - # Source addresses come first and target address can be either - # userspace or kernel even with k option, as long as the source - # is in kernel. + # Source addresses come first in user or kernel code. Next is the target + # address that must be in the kernel. - #Look for source addresses with top bit set - if ! grep -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then - echo "ERROR: Kernel branches missing" + # Look for source addresses with top bit set + if ! grep -q -E -m1 "^0x[89a-f][0-9a-f]{15}" $TMPDIR/perf.script; then err=1 fi - # Look for no source addresses without top bit set - if grep -E -m1 "^0x[0-7][0-9a-f]{0,15}" $TMPDIR/perf.script; then - echo "ERROR: User branches found with kernel filter" + # Look for no target addresses without top bit set + if grep -q -E -m1 "^0x[0-9a-f]{0,16}/0x[0-7][0-9a-f]{1,15}/" \ + $TMPDIR/perf.script; then err=1 fi + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then + echo "Testing kernel branch sampling [Passed]" + err=$start_err + else + echo "Testing kernel branch sampling [Failed]" + err=1 fi } @@ -136,14 +188,15 @@ test_filter() { test_filter_expect=$2 echo "Testing branch stack filtering permutation ($test_filter_filter,$test_filter_expect)" - perf record -o "$TMPDIR/perf.data" --branch-filter "$test_filter_filter,save_type,u" -- ${TESTPROG} > "$TMPDIR/record.txt" 2>&1 + perf record -o "$TMPDIR/perf.data" --branch-filter "$test_filter_filter,save_type,u" -- \ + ${TESTPROG} > "$TMPDIR/record.txt" 2>&1 perf script -i "$TMPDIR/perf.data" --fields brstack > "$TMPDIR/perf.script" # fail if we find any branch type that doesn't match any of the expected ones # also consider UNKNOWN branch types (-) if [ ! -s "$TMPDIR/perf.script" ] then - echo "Empty script output" + echo "Testing branch stack filtering [Failed empty script output]" err=1 return fi @@ -154,26 +207,44 @@ test_filter() { > "$TMPDIR/perf.script-filtered" || true if [ -s "$TMPDIR/perf.script-filtered" ] then - echo "Unexpected branch filter in script output" + echo "Testing branch stack filtering [Failed unexpected branch filter]" cat "$TMPDIR/perf.script" err=1 return fi + echo "Testing branch stack filtering [Passed]" } test_syscall() { echo "Testing syscalls" # skip if perf doesn't have enough privileges - if ! perf record --branch-filter any,k -o- -- true > /dev/null; then - echo "skip: not enough privileges" - else + if ! perf record --branch-filter any,k -o- -- true > "$TMPDIR/record.txt" 2>&1; then + echo "Testing syscalls [Skipped: not enough privileges]" + return + fi + start_err=$err + local ret=1 + for loops in 8000 30000 100000; do + err=0 perf record -o $TMPDIR/perf.data --branch-filter \ - any_call,save_type,u,k -c 10000 -- \ - perf bench syscall basic --loop 1000 + any_call,save_type,u,k -c 10007 -- \ + perf bench syscall basic --loop $loops > "$TMPDIR/record.txt" 2>&1 perf script -i $TMPDIR/perf.data --fields brstacksym | \ tr ' ' '\n' > $TMPDIR/perf.script check_branches "getppid[^ ]*/SYSCALL/" + if [ $err -eq 0 ]; then + ret=0 + break + fi + done + + if [ $ret -eq 0 ]; then + echo "Testing syscalls [Passed]" + err=$start_err + else + echo "Testing syscalls [Failed]" + err=1 fi } set -e diff --git a/tools/perf/tests/shell/trace_record_replay.sh b/tools/perf/tests/shell/trace_record_replay.sh index 88d30a03dcecb..38fcafcdfb91c 100755 --- a/tools/perf/tests/shell/trace_record_replay.sh +++ b/tools/perf/tests/shell/trace_record_replay.sh @@ -6,16 +6,46 @@ # shellcheck source=lib/probe.sh . "$(dirname $0)"/lib/probe.sh +# shellcheck source=lib/perf_record.sh +. "$(dirname $0)"/lib/perf_record.sh skip_if_no_perf_trace || exit 2 [ "$(id -u)" = 0 ] || exit 2 file=$(mktemp /tmp/temporary_file.XXXXX) +err=0 -perf trace record -o ${file} sleep 1 || exit 1 -if ! perf trace -i ${file} 2>&1 | grep nanosleep; then - echo "Failed: cannot find *nanosleep syscall" +cleanup() { + rm -f ${file} + perf_record_cleanup + trap - EXIT INT TERM +} + +trap_cleanup() { + echo "Unexpected signal in ${FUNCNAME[1]}" + cleanup + exit 1 +} +trap trap_cleanup EXIT INT TERM + +check_nanosleep() { + perf trace -i "${file}" 2>&1 | grep -q nanosleep +} + +PERF_RECORD_CMD="perf trace record" perf_record_with_retry "${file}" "check_nanosleep" "sleep" +err=$? + +if [ $err -ne 0 ]; then + if [ $err -eq 2 ]; then + logfile="${PERF_RECORD_LOGS[${#PERF_RECORD_LOGS[@]}-1]}" + echo "perf trace record failed. Log output:" + cat "$logfile" + else + echo "Failed: cannot find *nanosleep syscall" + fi + cleanup exit 1 fi -rm -f ${file} +cleanup +exit 0 diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h index 33de16dde737f..c79f3ab93cd42 100644 --- a/tools/perf/tests/tests.h +++ b/tools/perf/tests/tests.h @@ -38,12 +38,14 @@ struct test_case { const char *skip_reason; test_fnptr run_case; bool exclusive; + void *priv; }; struct test_suite { const char *desc; struct test_case *test_cases; void *priv; + int (*setup)(struct test_suite *suite); }; #define DECLARE_SUITE(name) \ diff --git a/tools/perf/ui/browsers/hists.c b/tools/perf/ui/browsers/hists.c index 487c0b08c0038..9c371b262b3b8 100644 --- a/tools/perf/ui/browsers/hists.c +++ b/tools/perf/ui/browsers/hists.c @@ -3064,15 +3064,15 @@ static int evsel__hists_browse(struct evsel *evsel, int nr_events, const char *h browser->min_pcnt = min_pcnt; hist_browser__update_nr_entries(browser); + memset(options, 0, sizeof(options)); + memset(actions, 0, sizeof(actions)); + browser->pstack = pstack__new(3); if (browser->pstack == NULL) goto out; ui_helpline__push(helpline); - memset(options, 0, sizeof(options)); - memset(actions, 0, sizeof(actions)); - if (symbol_conf.col_width_list_str) perf_hpp__set_user_width(symbol_conf.col_width_list_str); diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 71be979f50771..cc99b06e59802 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -1543,6 +1543,10 @@ static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + /* nr_cpu is used as a divisor below */ + if (*nr_cpu <= 0) + return NULL; + metadata = calloc(*nr_cpu, sizeof(*metadata)); if (!metadata) return NULL; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 1539c1dc823c2..1749359aa9a20 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -235,8 +235,12 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, { unsigned int nr_queues = queues->nr_queues; struct auxtrace_queue *queue_array; + struct auxtrace_queue *old_array = queues->queue_array; unsigned int i; + if (!new_nr_queues) + return -EINVAL; + if (!nr_queues) nr_queues = AUXTRACE_INIT_NR_QUEUES; @@ -251,16 +255,17 @@ static int auxtrace_queues__grow(struct auxtrace_queues *queues, return -ENOMEM; for (i = 0; i < queues->nr_queues; i++) { - list_splice_tail(&queues->queue_array[i].head, + list_splice_tail(&old_array[i].head, &queue_array[i].head); - queue_array[i].tid = queues->queue_array[i].tid; - queue_array[i].cpu = queues->queue_array[i].cpu; - queue_array[i].set = queues->queue_array[i].set; - queue_array[i].priv = queues->queue_array[i].priv; + queue_array[i].tid = old_array[i].tid; + queue_array[i].cpu = old_array[i].cpu; + queue_array[i].set = old_array[i].set; + queue_array[i].priv = old_array[i].priv; } queues->nr_queues = nr_queues; queues->queue_array = queue_array; + free(old_array); return 0; } diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c index 2294336f6e60b..195516356f62b 100644 --- a/tools/perf/util/bpf-event.c +++ b/tools/perf/util/bpf-event.c @@ -969,7 +969,8 @@ void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, { struct bpf_prog_info *info = &info_linear->info; __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | - (1UL << PERF_BPIL_JITED_FUNC_LENS); + (1UL << PERF_BPIL_JITED_FUNC_LENS) | + (1UL << PERF_BPIL_PROG_TAGS); __u32 *prog_lens; __u64 *prog_addrs; char name[KSYM_NAME_LEN]; diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index 1a2e7b388d57d..bcd81084e3420 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -629,24 +629,20 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term static bool check_bpf_filter_capable(void) { - bool used_root; + int fd; - if (perf_cap__capable(CAP_BPF, &used_root)) + if (perf_cap__capable(CAP_BPF)) return true; - if (!used_root) { - /* Check if root already pinned the filter programs and maps */ - int fd = get_pinned_fd("filters"); - - if (fd >= 0) { - close(fd); - return true; - } + /* Check if root already pinned the filter programs and maps */ + fd = get_pinned_fd("filters"); + if (fd >= 0) { + close(fd); + return true; } - pr_err("Error: BPF filter only works for %s!\n" - "\tPlease run 'perf record --setup-filter pin' as root first.\n", - used_root ? "root" : "users with the CAP_BPF capability"); + pr_err("Error: BPF filter only works for users with the CAP_BPF capability!\n" + "\tPlease run 'perf record --setup-filter pin' as root first.\n"); return false; } diff --git a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c index cb86e261b4de0..e0a844927c5ce 100644 --- a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c +++ b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c @@ -429,15 +429,88 @@ static bool pid_filter__has(struct pids_filtered *pids, pid_t pid) return bpf_map_lookup_elem(pids, &pid) != NULL; } +u64 ZERO = 0; + +/* + * Determine what type of argument and how many bytes to read from user space, using the + * value in the beauty_map. This is the relation of parameter type and its corresponding + * value in the beauty map, and how many bytes we read eventually: + * + * string: 1 -> size of string + * struct: size of struct -> size of struct + * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) + */ +static inline int augment_arg(struct syscall_enter_args *args, int i, + unsigned int *beauty_map, + struct beauty_payload_enter *payload, u64 offset) +{ + int index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + struct augmented_arg *payload_offset; + s64 aug_size, size; + bool augmented; + void *arg; + + arg = (void *)args->args[i]; + augmented = false; + size = beauty_map[i]; + aug_size = size; /* size of the augmented data read from user space */ + + if (size == 0 || arg == NULL) + return 0; + + /* bounds check for the verifier */ + if (offset > sizeof(payload->aug_args) - sizeof(payload->aug_args[0])) + return -1; + barrier_var(offset); + payload_offset = (struct augmented_arg *)((void *)&payload->aug_args + offset); + + if (size == 1) { /* string */ + aug_size = bpf_probe_read_user_str(payload_offset->value, value_size, arg); + /* minimum of 0 to pass the verifier */ + if (aug_size < 0) + aug_size = 0; + + augmented = true; + } else if (size > 0 && size <= value_size) { /* struct */ + if (!bpf_probe_read_user(payload_offset->value, size, arg)) + augmented = true; + } else if ((int)size < 0 && size >= -6) { /* buffer */ + index = -(size + 1); + barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. + index &= 7; // Satisfy the bounds checking with the verifier in some kernels. + aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; + + if (aug_size > 0) { + if (!bpf_probe_read_user(payload_offset->value, aug_size, arg)) + augmented = true; + } + } + + /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ + if (aug_size > value_size) + aug_size = value_size; + + /* write data to payload */ + if (augmented) { + int written = offsetof(struct augmented_arg, value) + aug_size; + + if (written < 0 || written > sizeof(struct augmented_arg)) + return -1; + + payload_offset->size = aug_size; + return written; + } + + return 0; +} + static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) { - bool augmented, do_output = false; - int zero = 0, index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + bool do_output = false; + int i, zero = 0, written; u64 output = 0; /* has to be u64, otherwise it won't pass the verifier */ - s64 aug_size, size; unsigned int nr, *beauty_map; struct beauty_payload_enter *payload; - void *arg, *payload_offset; /* fall back to do predefined tail call */ if (args == NULL) @@ -449,7 +522,6 @@ static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) /* set up payload for output */ payload = bpf_map_lookup_elem(&beauty_payload_enter_map, &zero); - payload_offset = (void *)&payload->aug_args; if (beauty_map == NULL || payload == NULL) return 1; @@ -457,61 +529,30 @@ static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) /* copy the sys_enter header, which has the syscall_nr */ __builtin_memcpy(&payload->args, args, sizeof(struct syscall_enter_args)); - /* - * Determine what type of argument and how many bytes to read from user space, using the - * value in the beauty_map. This is the relation of parameter type and its corresponding - * value in the beauty map, and how many bytes we read eventually: - * - * string: 1 -> size of string - * struct: size of struct -> size of struct - * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) - */ - for (int i = 0; i < 6; i++) { - arg = (void *)args->args[i]; - augmented = false; - size = beauty_map[i]; - aug_size = size; /* size of the augmented data read from user space */ - - if (size == 0 || arg == NULL) - continue; - - if (size == 1) { /* string */ - aug_size = bpf_probe_read_user_str(((struct augmented_arg *)payload_offset)->value, value_size, arg); - /* minimum of 0 to pass the verifier */ - if (aug_size < 0) - aug_size = 0; - - augmented = true; - } else if (size > 0 && size <= value_size) { /* struct */ - if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, size, arg)) - augmented = true; - } else if ((int)size < 0 && size >= -6) { /* buffer */ - index = -(size + 1); - barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. - index &= 7; // Satisfy the bounds checking with the verifier in some kernels. - aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; - - if (aug_size > 0) { - if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, aug_size, arg)) - augmented = true; + if (bpf_ksym_exists(bpf_iter_num_new)) { + bpf_for(i, 0, 6) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) + return 1; + if (written > 0) { + output += written; + /* + * guide the verifier to forget range of `output`, which + * helps to prove convergence of the loop + */ + output += ZERO; + do_output = true; } } - - /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ - if (aug_size > value_size) - aug_size = value_size; - - /* write data to payload */ - if (augmented) { - int written = offsetof(struct augmented_arg, value) + aug_size; - - if (written < 0 || written > sizeof(struct augmented_arg)) + } else { + for (i = 0; i < 6; i++) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) return 1; - - ((struct augmented_arg *)payload_offset)->size = aug_size; - output += written; - payload_offset += written; - do_output = true; + if (written > 0) { + output += written; + do_output = true; + } } } diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c index ac6d1d9a523d9..272bd8255ff12 100644 --- a/tools/perf/util/cap.c +++ b/tools/perf/util/cap.c @@ -12,7 +12,7 @@ #define MAX_LINUX_CAPABILITY_U32S _LINUX_CAPABILITY_U32S_3 -bool perf_cap__capable(int cap, bool *used_root) +bool perf_cap__capable(int cap) { struct __user_cap_header_struct header = { .version = _LINUX_CAPABILITY_VERSION_3, @@ -21,7 +21,6 @@ bool perf_cap__capable(int cap, bool *used_root) struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {}; __u32 cap_val; - *used_root = false; while (syscall(SYS_capget, &header, &data[0]) == -1) { /* Retry, first attempt has set the header.version correctly. */ if (errno == EINVAL && header.version != _LINUX_CAPABILITY_VERSION_3 && @@ -29,7 +28,6 @@ bool perf_cap__capable(int cap, bool *used_root) continue; pr_debug2("capget syscall failed (%m) fall back on root check\n"); - *used_root = true; return geteuid() == 0; } diff --git a/tools/perf/util/cap.h b/tools/perf/util/cap.h index c1b8ac033ccc5..bf09fb20c7793 100644 --- a/tools/perf/util/cap.h +++ b/tools/perf/util/cap.h @@ -18,7 +18,6 @@ #define CAP_BPF 39 #endif -/* Query if a capability is supported, used_root is set if the fallback root check was used. */ -bool perf_cap__capable(int cap, bool *used_root); +bool perf_cap__capable(int cap); #endif /* __PERF_CAP_H */ diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c index 2c7feab61b7bf..1e2b697c86fc5 100644 --- a/tools/perf/util/capstone.c +++ b/tools/perf/util/capstone.c @@ -156,6 +156,7 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, for (i = 0; i < insn->detail->x86.op_count; i++) { cs_x86_op *op = &insn->detail->x86.operands[i]; u64 orig_addr; + struct map *found_map = NULL; if (op->type != X86_OP_MEM) continue; @@ -171,19 +172,22 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, if (dso__kernel(map__dso(map))) { /* * The kernel maps can be split into sections, let's - * find the map first and the search the symbol. + * find the map first and then search the symbol. */ - map = maps__find(map__kmaps(map), addr); - if (map == NULL) + found_map = maps__find(map__kmaps(map), addr); + if (found_map == NULL) continue; + map = found_map; } /* convert it to map-relative address for search */ addr = map__map_ip(map, addr); sym = map__find_symbol(map, addr); - if (sym == NULL) + if (sym == NULL) { + map__put(found_map); continue; + } if (addr == sym->start) { scnprintf(buf, len, "\t# %"PRIx64" <%s>", @@ -192,6 +196,7 @@ static void print_capstone_detail(cs_insn *insn, char *buf, size_t len, scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">", orig_addr, sym->name, addr - sym->start); } + map__put(found_map); break; } } diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h index 6cfecfca16f24..ec6c38129e248 100644 --- a/tools/perf/util/compress.h +++ b/tools/perf/util/compress.h @@ -54,7 +54,8 @@ int zstd_fini(struct zstd_data *data); ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)); + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)); size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, void *dst, size_t dst_size); @@ -75,7 +76,8 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused, void *dst __maybe_unused, size_t dst_size __maybe_unused, void *src __maybe_unused, size_t src_size __maybe_unused, size_t max_record_size __maybe_unused, - size_t process_header(void *record, size_t increment) __maybe_unused) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size) __maybe_unused) { return 0; } diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 520670348311e..15f99c2c2775a 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -645,6 +645,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, queue->tid); tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, queue->tid); + if (!tidq->frontend_thread || !tidq->decode_thread) + goto out; tidq->packet = zalloc(sizeof(struct cs_etm_packet)); if (!tidq->packet) @@ -679,6 +681,8 @@ static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, zfree(&tidq->prev_packet); zfree(&tidq->packet); out: + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); return rc; } @@ -1505,8 +1509,7 @@ cs_etm__get_trace(struct cs_etm_queue *etmq) etmq->buf_used = 0; etmq->buf_len = aux_buffer->size; etmq->buf = aux_buffer->data; - - return etmq->buf_len; + return 0; } /* @@ -2062,6 +2065,45 @@ static int cs_etm__end_block(struct cs_etm_queue *etmq, return 0; } + +static int cs_etm__flush_stack_cb(struct thread *thread, + void *data __maybe_unused) +{ + thread_stack__flush(thread); + return 0; +} + +static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) +{ + struct machine *machine; + + machine = machines__find(&etmq->etm->session->machines, pid); + if (machine) + machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); +} + +static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + if (!etmq->etm->synth_opts.last_branch) + return; + + switch (pid_fmt) { + case CS_ETM_PIDFMT_CTXTID2: + /* Clear the guest stack if virtualization is supported */ + cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); + fallthrough; + case CS_ETM_PIDFMT_CTXTID: + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + + } +} + /* * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue * if need be. @@ -2073,20 +2115,33 @@ static int cs_etm__get_data_block(struct cs_etm_queue *etmq) { int ret; - if (!etmq->buf_len) { - ret = cs_etm__get_trace(etmq); - if (ret <= 0) - return ret; - /* - * We cannot assume consecutive blocks in the data file - * are contiguous, reset the decoder to force re-sync. - */ - ret = cs_etm_decoder__reset(etmq->decoder); - if (ret) - return ret; - } + /* The current block is not finished */ + if (etmq->buf_len) + return 1; + + ret = cs_etm__get_trace(etmq); + if (ret < 0) + return ret; - return etmq->buf_len; + /* No more buffer to read */ + if (!etmq->buf_len) + return 0; + + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = cs_etm_decoder__reset(etmq->decoder); + if (ret) + return ret; + + /* + * Since the decoder is reset, this causes a global trace + * discontinuity. Flush all thread stacks. + */ + cs_etm__flush_all_stack(etmq); + + return 1; } static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c index 9dc1e184cf3c9..9c96d84df900a 100644 --- a/tools/perf/util/data-convert-json.c +++ b/tools/perf/util/data-convert-json.c @@ -242,6 +242,7 @@ static int process_sample_event(const struct perf_tool *tool, trace_seq_init(&s); tep_print_field(&s, sample->raw_data, fields[i]); output_json_key_string(out, true, 3, fields[i]->name, s.buffer); + trace_seq_destroy(&s); i++; } diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index 6fa104752de2a..70daf4fd200e3 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -394,7 +394,9 @@ int dso__decompress_kmodule_path(struct dso *dso, const char *name, { int fd = decompress_kmodule(dso, name, pathname, len); - close(fd); + /* decompress_kmodule() returns -1 on failure, don't close(-1) */ + if (fd >= 0) + close(fd); return fd >= 0 ? 0 : -1; } @@ -581,9 +583,18 @@ static char *dso__get_filename(struct dso *dso, const char *root_dir, goto out; if (!is_regular_file(name)) { + struct stat st; char *new_name; - if (errno != ENOENT || dso__nsinfo(dso) == NULL) + /* + * errno only reflects the failure reason when stat() itself + * failed: a successful stat() on a non-regular file (e.g. a + * directory) leaves a stale errno, which a previous failed + * iteration of the try_to_open_dso() fallback loop may have + * set to ENOENT. + */ + if (stat(name, &st) == 0 || errno != ENOENT || + dso__nsinfo(dso) == NULL) goto out; new_name = dso__filename_with_chroot(dso, name); @@ -639,10 +650,13 @@ static int __open_dso(struct dso *dso, struct machine *machine) mutex_lock(dso__lock(dso)); name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); - if (name) + if (name) { fd = do_open(name); - else + } else { + if (errno == 0) + errno = ENOENT; fd = -errno; + } if (decomp) unlink(name); @@ -999,7 +1013,20 @@ static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, u64 size, bool out) { u64 cache_offset = offset - cache->offset; - u64 cache_size = min(cache->size - cache_offset, size); + u64 cache_size; + + /* + * The RB tree matches using DSO__DATA_CACHE_SIZE, but a short + * pread may leave cache->size smaller. For a regular file a + * short pread only happens at end-of-file, so an offset past + * the valid data is EOF: return 0, matching what a direct + * pread() at that offset would return, and cached_io() then + * stops its read loop. + */ + if (cache_offset >= cache->size) + return 0; + + cache_size = min(cache->size - cache_offset, size); if (out) memcpy(data, cache->data + cache_offset, cache_size); @@ -1023,7 +1050,7 @@ static ssize_t file_read(struct dso *dso, struct machine *machine, if (dso__data(dso)->fd < 0) { dso__data(dso)->status = DSO_DATA_STATUS_ERROR; - ret = -errno; + ret = dso__data(dso)->fd; goto out; } @@ -1145,8 +1172,8 @@ static int file_size(struct dso *dso, struct machine *machine) try_to_open_dso(dso, machine); if (dso__data(dso)->fd < 0) { - ret = -errno; dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = dso__data(dso)->fd; goto out; } @@ -1936,7 +1963,12 @@ const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; return NULL; } - assert(len <= info_linear->info.jited_prog_len); + if (len > info_linear->info.jited_prog_len) { + pr_debug("BPF symbol length %zu exceeds jited_prog_len %u\n", + len, info_linear->info.jited_prog_len); + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } *out_buf_len = len; return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); #else diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h index 54e470dd07305..b374dfabf49cc 100644 --- a/tools/perf/util/dso.h +++ b/tools/perf/util/dso.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "build-id.h" #include "debuginfo.h" #include "mutex.h" @@ -20,6 +21,40 @@ struct perf_env; #define DSO__NAME_KALLSYMS "[kernel.kallsyms]" #define DSO__NAME_KCORE "[kernel.kcore]" +#define DSO__NAME_GUEST_KALLSYMS "[guest.kernel.kallsyms]" +#define DSO__NAME_GUEST_KALLSYMS_PID_PREFIX "[guest.kernel.kallsyms." + +/* + * Validate names of the form "[guest.kernel.kallsyms.]", where + * is the PID of the guest VM and varies per guest, so it + * cannot be matched with strcmp() against a fixed string. + * + * Every character after the fixed prefix must be a decimal digit, + * with ']' immediately terminating the digit run and nothing + * following it. This rules out '/', "..", or any other character + * being smuggled into the name. + */ +static inline bool is_guest_kallsyms_pid_name(const char *name) +{ + const size_t prefix_len = sizeof(DSO__NAME_GUEST_KALLSYMS_PID_PREFIX) - 1; + size_t digits; + + if (strncmp(name, DSO__NAME_GUEST_KALLSYMS_PID_PREFIX, prefix_len) != 0) + return false; + + digits = strspn(name + prefix_len, "0123456789"); + if (digits == 0) + return false; + + /* ']' must terminate the digit run, with nothing trailing it */ + if (name[prefix_len + digits] != ']') + return false; + + if (name[prefix_len + digits + 1] != '\0') + return false; + + return true; +} /** * enum dso_binary_type - The kind of DSO generally associated with a memory @@ -894,8 +929,28 @@ static inline bool dso__is_kcore(const struct dso *dso) static inline bool dso__is_kallsyms(const struct dso *dso) { enum dso_binary_type bt = dso__binary_type(dso); + const char *name; + + if (bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS) + return true; + + if (bt != DSO_BINARY_TYPE__NOT_FOUND) + return false; + + if (!dso__kernel(dso)) + return false; + + name = dso__long_name(dso); + if (!name) + return false; + + if (!strcmp(name, DSO__NAME_KALLSYMS)) + return true; + + if (!strcmp(name, DSO__NAME_GUEST_KALLSYMS)) + return true; - return bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS; + return is_guest_kallsyms_pid_name(name); } bool dso__is_object_file(const struct dso *dso); diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 22e43a57dc95f..e01103443b23f 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -646,7 +646,7 @@ struct tep_event *evsel__tp_format(struct evsel *evsel) tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); if (IS_ERR(tp_format)) { - int err = -PTR_ERR(evsel->tp_format); + int err = -PTR_ERR(tp_format); errno = err; pr_err("Error getting tracepoint format '%s': %m\n", diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c index a17c423a526dd..5db44556fc4f5 100644 --- a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include "../color.h" @@ -73,29 +75,20 @@ static const char * const hisi_ptt_4dw_pkt_field_name[] = { [HISI_PTT_4DW_HEAD3] = "Header DW3", }; -union hisi_ptt_4dw { - struct { - uint32_t format : 2; - uint32_t type : 5; - uint32_t t9 : 1; - uint32_t t8 : 1; - uint32_t th : 1; - uint32_t so : 1; - uint32_t len : 10; - uint32_t time : 11; - }; - uint32_t value; -}; - static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) { const char *color = PERF_COLOR_BLUE; + uint8_t byte; + uint32_t dw; int i; + dw = get_unaligned_le32(buf + pos); printf("."); color_fprintf(stdout, color, " %08x: ", pos); - for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) - color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) color_fprintf(stdout, color, " "); color_fprintf(stdout, color, " %s\n", desc); @@ -122,22 +115,30 @@ static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos) static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) { const char *color = PERF_COLOR_BLUE; - union hisi_ptt_4dw dw0; + uint8_t byte; + uint32_t dw; int i; - dw0.value = *(uint32_t *)(buf + pos); + dw = get_unaligned_le32(buf + pos); printf("."); color_fprintf(stdout, color, " %08x: ", pos); - for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) - color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) color_fprintf(stdout, color, " "); color_fprintf(stdout, color, " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", - "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9, - "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length", - dw0.len, "Time", dw0.time); + "Format", FIELD_GET(HISI_PTT_HEAD0_4DW_FORMAT, dw), + "Type", FIELD_GET(HISI_PTT_HEAD0_4DW_TYPE, dw), + "T9", FIELD_GET(HISI_PTT_HEAD0_4DW_T9, dw), + "T8", FIELD_GET(HISI_PTT_HEAD0_4DW_T8, dw), + "TH", FIELD_GET(HISI_PTT_HEAD0_4DW_TH, dw), + "SO", FIELD_GET(HISI_PTT_HEAD0_4DW_SO, dw), + "Length", FIELD_GET(HISI_PTT_HEAD0_4DW_LEN, dw), + "Time", FIELD_GET(HISI_PTT_HEAD0_4DW_TIME, dw)); } static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos) diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h index e78f1b5bc836e..333eee02b19d2 100644 --- a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -9,12 +9,24 @@ #include #include +#include +#include #define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) #define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) #define HISI_PTT_MAX_SPACE_LEN 10 #define HISI_PTT_FIELD_LENTH 4 +/* Header DW0 fields for 4DW format */ +#define HISI_PTT_HEAD0_4DW_TIME GENMASK_U32(10, 0) +#define HISI_PTT_HEAD0_4DW_LEN GENMASK_U32(20, 11) +#define HISI_PTT_HEAD0_4DW_SO BIT_U32(21) +#define HISI_PTT_HEAD0_4DW_TH BIT_U32(22) +#define HISI_PTT_HEAD0_4DW_T8 BIT_U32(23) +#define HISI_PTT_HEAD0_4DW_T9 BIT_U32(24) +#define HISI_PTT_HEAD0_4DW_TYPE GENMASK_U32(29, 25) +#define HISI_PTT_HEAD0_4DW_FORMAT GENMASK_U32(31, 30) + enum hisi_ptt_pkt_type { HISI_PTT_4DW_PKT, HISI_PTT_8DW_PKT, diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h index c64005278687c..a4f244a046866 100644 --- a/tools/perf/util/hist.h +++ b/tools/perf/util/hist.h @@ -709,6 +709,8 @@ struct block_hist { struct hist_entry he; }; +#define NO_ADDR 0 + #ifdef HAVE_SLANG_SUPPORT #include "../ui/keysyms.h" void attr_to_script(char *buf, struct perf_event_attr *attr); @@ -746,14 +748,16 @@ int evlist__tui_browse_hists(struct evlist *evlist __maybe_unused, static inline int __hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, struct map_symbol *ms __maybe_unused, struct evsel *evsel __maybe_unused, - struct hist_browser_timer *hbt __maybe_unused) + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) { return 0; } static inline int hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, struct evsel *evsel __maybe_unused, - struct hist_browser_timer *hbt __maybe_unused) + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) { return 0; } diff --git a/tools/perf/util/intel-bts.c b/tools/perf/util/intel-bts.c index 3625c62247502..9f5f378cf7743 100644 --- a/tools/perf/util/intel-bts.c +++ b/tools/perf/util/intel-bts.c @@ -831,7 +831,7 @@ int intel_bts_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_BTS_SNAPSHOT_MODE; + size_t min_sz = sizeof(u64) * (INTEL_BTS_SNAPSHOT_MODE + 1); struct intel_bts *bts; int err; diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index 9b1011fe48267..fd636f5639cab 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -4411,7 +4411,7 @@ int intel_pt_process_auxtrace_info(union perf_event *event, struct perf_session *session) { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; - size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS; + size_t min_sz = sizeof(u64) * (INTEL_PT_PER_CPU_MMAPS + 1); struct intel_pt *pt; void *info_end; __u64 *info; diff --git a/tools/perf/util/libbfd.c b/tools/perf/util/libbfd.c index 63ea3fb53e77d..6df72889e5075 100644 --- a/tools/perf/util/libbfd.c +++ b/tools/perf/util/libbfd.c @@ -15,6 +15,7 @@ #ifdef HAVE_LIBBPF_SUPPORT #include #include +#include #endif #include #include @@ -510,7 +511,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, char tpath[PATH_MAX]; size_t buf_size; int nr_skip = 0; - char *buf; + char *buf = NULL; bfd *bfdf; int ret; FILE *s; @@ -552,6 +553,11 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, info_linear = info_node->info_linear; sub_id = dso__bpf_prog(dso)->sub_id; + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } info.buffer = (void *)(uintptr_t)(info_linear->info.jited_prog_insns); info.buffer_length = info_linear->info.jited_prog_len; @@ -581,6 +587,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (disassemble == NULL) abort(); + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) { + ret = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + goto out; + } + fflush(s); do { const struct bpf_line_info *linfo = NULL; @@ -609,7 +621,7 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, if (!annotate_opts.hide_src_code && srcline) { args->offset = -1; - args->line = strdup(srcline); + args->line = (char *)srcline; args->line_nr = 0; args->fileloc = NULL; args->ms->sym = sym; @@ -634,9 +646,12 @@ int symbol__disassemble_bpf_libbfd(struct symbol *sym __maybe_unused, ret = 0; out: - free(prog_linfo); + bpf_prog_linfo__free(prog_linfo); btf__free(btf); - fclose(s); + if (s) { + fclose(s); + free(buf); + } bfd_close(bfdf); return ret; #else diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 9331b4cc19f18..cb138d1bdd7b9 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -327,16 +327,22 @@ struct machine *machines__findnew(struct machines *machines, pid_t pid) if ((pid != HOST_KERNEL_ID) && (pid != DEFAULT_GUEST_KERNEL_ID) && (symbol_conf.guestmount)) { - snprintf(path, sizeof(path), "%s/%d", symbol_conf.guestmount, pid); + if (snprintf(path, sizeof(path), "%s/%d", + symbol_conf.guestmount, pid) >= (int)sizeof(path)) { + pr_err("Guest path too long for pid %d\n", pid); + machine = NULL; + goto out; + } if (access(path, R_OK)) { static struct strlist *seen; if (!seen) seen = strlist__new(NULL, NULL); - if (!strlist__has_entry(seen, path)) { + if (!seen || !strlist__has_entry(seen, path)) { pr_err("Can't access file %s\n", path); - strlist__add(seen, path); + if (seen) + strlist__add(seen, path); } machine = NULL; goto out; @@ -1229,27 +1235,35 @@ int machines__create_guest_kernel_maps(struct machines *machines) for (i = 0; i < items; i++) { if (!isdigit(namelist[i]->d_name[0])) { /* Filter out . and .. */ + free(namelist[i]); continue; } + errno = 0; pid = (pid_t)strtol(namelist[i]->d_name, &endp, 10); if ((*endp != '\0') || (endp == namelist[i]->d_name) || (errno == ERANGE)) { pr_debug("invalid directory (%s). Skipping.\n", namelist[i]->d_name); + free(namelist[i]); + continue; + } + if (snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", + symbol_conf.guestmount, + namelist[i]->d_name) >= (int)sizeof(path)) { + pr_debug("Guest kallsyms path too long for %s. Skipping.\n", + namelist[i]->d_name); + free(namelist[i]); continue; } - snprintf(path, sizeof(path), "%s/%s/proc/kallsyms", - symbol_conf.guestmount, - namelist[i]->d_name); - ret = access(path, R_OK); - if (ret) { + if (access(path, R_OK)) { pr_debug("Can't access file %s\n", path); - goto failure; + free(namelist[i]); + continue; } machines__create_kernel_maps(machines, pid); + free(namelist[i]); } -failure: free(namelist); } @@ -1390,8 +1404,10 @@ static int maps__set_modules_path_dir(struct maps *maps, char *path, size_t path return -1; } /* Bounds check, should never happen. */ - if (root_len >= path_size) - return -1; + if (root_len >= path_size) { + ret = -1; + goto out; + } path[root_len++] = '/'; while ((dent = io_dir__readdir(&iod)) != NULL) { if (io_dir__is_dir(&iod, dent)) { @@ -1878,7 +1894,8 @@ int machine__process_fork_event(struct machine *machine, union perf_event *event * (fork) event that would have removed the thread was lost. Assume the * latter case and continue on as best we can. */ - if (thread__pid(parent) != (pid_t)event->fork.ppid) { + if (parent != NULL && + thread__pid(parent) != (pid_t)event->fork.ppid) { dump_printf("removing erroneous parent thread %d/%d\n", thread__pid(parent), thread__tid(parent)); machine__remove_thread(machine, parent); diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 238c3e605bfe1..bf98a01c4222d 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -1614,8 +1614,10 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, new_expr->metric_expr = old_expr->metric_expr; new_expr->metric_threshold = old_expr->metric_threshold; new_expr->metric_name = strdup(old_expr->metric_name); - if (!new_expr->metric_name) + if (!new_expr->metric_name) { + free(new_expr); return -ENOMEM; + } new_expr->metric_unit = old_expr->metric_unit; new_expr->runtime = old_expr->runtime; @@ -1627,6 +1629,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_refs); new_expr->metric_refs = calloc(nr + 1, alloc_size); if (!new_expr->metric_refs) { + zfree(&new_expr->metric_name); free(new_expr); return -ENOMEM; } @@ -1643,6 +1646,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, alloc_size = sizeof(*new_expr->metric_events); new_expr->metric_events = calloc(nr + 1, alloc_size); if (!new_expr->metric_events) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_refs); free(new_expr); return -ENOMEM; @@ -1653,6 +1657,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, evsel = old_expr->metric_events[idx]; evsel = evlist__find_evsel(evlist, evsel->core.idx); if (evsel == NULL) { + zfree(&new_expr->metric_name); zfree(&new_expr->metric_events); zfree(&new_expr->metric_refs); free(new_expr); diff --git a/tools/perf/util/powerpc-vpadtl.c b/tools/perf/util/powerpc-vpadtl.c index 39a3fb3f13301..55b9b6e070746 100644 --- a/tools/perf/util/powerpc-vpadtl.c +++ b/tools/perf/util/powerpc-vpadtl.c @@ -192,7 +192,7 @@ static int powerpc_vpadtl_sample(struct powerpc_vpadtl_entry *record, sample.cpumode = PERF_RECORD_MISC_KERNEL; sample.time = save; sample.raw_data = record; - sample.raw_size = sizeof(record); + sample.raw_size = sizeof(*record); event.sample.header.type = PERF_RECORD_SAMPLE; event.sample.header.misc = sample.cpumode; event.sample.header.size = sizeof(struct perf_event_header); diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index 779fe1280a567..a65a4d9f8cc83 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -872,11 +872,11 @@ static void pyrf_counts_values__delete(struct pyrf_counts_values *pcounts_values 0, help } static PyMemberDef pyrf_counts_values_members[] = { - counts_values_member_def(val, T_ULONG, "Value of event"), - counts_values_member_def(ena, T_ULONG, "Time for which enabled"), - counts_values_member_def(run, T_ULONG, "Time for which running"), - counts_values_member_def(id, T_ULONG, "Unique ID for an event"), - counts_values_member_def(lost, T_ULONG, "Num of lost samples"), + counts_values_member_def(val, T_ULONGLONG, "Value of event"), + counts_values_member_def(ena, T_ULONGLONG, "Time for which enabled"), + counts_values_member_def(run, T_ULONGLONG, "Time for which running"), + counts_values_member_def(id, T_ULONGLONG, "Unique ID for an event"), + counts_values_member_def(lost, T_ULONGLONG, "Num of lost samples"), { .name = NULL, }, }; @@ -886,8 +886,15 @@ static PyObject *pyrf_counts_values_get_values(struct pyrf_counts_values *self, if (!vals) return NULL; - for (int i = 0; i < 5; i++) - PyList_SetItem(vals, i, PyLong_FromLong(self->values.values[i])); + for (int i = 0; i < 5; i++) { + PyObject *val = PyLong_FromUnsignedLongLong(self->values.values[i]); + + if (!val) { + Py_DECREF(vals); + return NULL; + } + PyList_SetItem(vals, i, val); + } return vals; } @@ -898,19 +905,34 @@ static int pyrf_counts_values_set_values(struct pyrf_counts_values *self, PyObje Py_ssize_t size; PyObject *item = NULL; + if (list == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot delete attribute"); + return -1; + } + if (!PyList_Check(list)) { PyErr_SetString(PyExc_TypeError, "Value assigned must be a list"); return -1; } size = PyList_Size(list); + if (size != 5) { + PyErr_SetString(PyExc_ValueError, "List must have exactly 5 entries"); + return -1; + } + for (Py_ssize_t i = 0; i < size; i++) { + unsigned long long val; + item = PyList_GetItem(list, i); if (!PyLong_Check(item)) { PyErr_SetString(PyExc_TypeError, "List members should be numbers"); return -1; } - self->values.values[i] = PyLong_AsLong(item); + val = PyLong_AsUnsignedLongLong(item); + if (val == (unsigned long long)-1 && PyErr_Occurred()) + return -1; + self->values.values[i] = val; } return 0; @@ -1074,11 +1096,21 @@ static PyObject *pyrf_evsel__open(struct pyrf_evsel *pevsel, &pcpus, &pthreads, &group, &inherit)) return NULL; - if (pthreads != NULL) + if (pthreads != NULL && pthreads != Py_None) { + if (!PyObject_TypeCheck(pthreads, &pyrf_thread_map__type)) { + PyErr_SetString(PyExc_TypeError, "threads must be a thread_map"); + return NULL; + } threads = ((struct pyrf_thread_map *)pthreads)->threads; + } - if (pcpus != NULL) + if (pcpus != NULL && pcpus != Py_None) { + if (!PyObject_TypeCheck(pcpus, &pyrf_cpu_map__type)) { + PyErr_SetString(PyExc_TypeError, "cpus must be a cpu_map"); + return NULL; + } cpus = ((struct pyrf_cpu_map *)pcpus)->cpus; + } evsel->core.attr.inherit = inherit; /* @@ -2034,8 +2066,10 @@ static PyObject *pyrf__parse_events(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) return NULL; - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; parse_events_error__init(&err); evlist__init(&evlist, cpus, threads); @@ -2062,8 +2096,10 @@ static PyObject *pyrf__parse_metrics(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "s|OO", &input, &pcpus, &pthreads)) return NULL; - threads = pthreads ? ((struct pyrf_thread_map *)pthreads)->threads : NULL; - cpus = pcpus ? ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; + threads = (pthreads && pthreads != Py_None) ? + ((struct pyrf_thread_map *)pthreads)->threads : NULL; + cpus = (pcpus && pcpus != Py_None) ? + ((struct pyrf_cpu_map *)pcpus)->cpus : NULL; evlist__init(&evlist, cpus, threads); ret = metricgroup__parse_groups(&evlist, /*pmu=*/"all", input, @@ -2123,6 +2159,8 @@ static int pyrf__metrics_cb(const struct pmu_metric *pm, Py_XDECREF(dict); return -ENOMEM; } + Py_DECREF(key); + Py_DECREF(value); if (!add_to_dict(dict, "MetricName", pm->metric_name) || !add_to_dict(dict, "PMU", pm->pmu) || diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c index af0df841d640c..c1e8f5b371777 100644 --- a/tools/perf/util/symbol.c +++ b/tools/perf/util/symbol.c @@ -1829,7 +1829,16 @@ int dso__load(struct dso *dso, struct map *map) if (next_slot) { ss_pos++; - if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) + /* + * The binary type is used to find the file containing + * the executed instructions, so prefer the types that + * refer to the actual object over debug-only files such + * as DSO_BINARY_TYPE__DEBUGLINK. + */ + if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND || + symtab_type == DSO_BINARY_TYPE__BUILD_ID_CACHE || + (symtab_type == DSO_BINARY_TYPE__SYSTEM_PATH_DSO && + dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE)) dso__set_binary_type(dso, symtab_type); if (syms_ss && runtime_ss) @@ -2326,8 +2335,7 @@ static bool symbol__read_kptr_restrict(void) { bool value = false; FILE *fp = fopen("/proc/sys/kernel/kptr_restrict", "r"); - bool used_root; - bool cap_syslog = perf_cap__capable(CAP_SYSLOG, &used_root); + bool cap_syslog = perf_cap__capable(CAP_SYSLOG); if (fp != NULL) { char line[8]; diff --git a/tools/perf/util/synthetic-events.c b/tools/perf/util/synthetic-events.c index c5b0ba775098f..ef0586cbc103b 100644 --- a/tools/perf/util/synthetic-events.c +++ b/tools/perf/util/synthetic-events.c @@ -1058,7 +1058,7 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, else thread_nr = nr_threads_synthesize; - if (thread_nr <= 1) { + if (thread_nr <= 1 || n <= 1) { err = __perf_event__synthesize_threads(tool, process, machine, needs_mmap, mmap_data, @@ -1104,8 +1104,8 @@ int perf_event__synthesize_threads(const struct perf_tool *tool, } err = 0; out_join: - for (i = 0; i < thread_nr; i++) - pthread_join(synthesize_threads[i], NULL); + for (j = 0; j < i; j++) + pthread_join(synthesize_threads[j], NULL); free(args); free_threads: free(synthesize_threads); diff --git a/tools/perf/util/thread-stack.c b/tools/perf/util/thread-stack.c index c6a0a27b12c2a..47d5922efdee3 100644 --- a/tools/perf/util/thread-stack.c +++ b/tools/perf/util/thread-stack.c @@ -642,7 +642,7 @@ void thread_stack__br_sample(struct thread *thread, int cpu, sz -= nr; be = &dst->entries[nr]; nr = min(ts->br_stack_pos, sz); - memcpy(be, &src->entries[0], bsz * ts->br_stack_pos); + memcpy(be, &src->entries[0], bsz * nr); } } diff --git a/tools/perf/util/trace-event-read.c b/tools/perf/util/trace-event-read.c index ecbbb93f01853..52ed496d92c3b 100644 --- a/tools/perf/util/trace-event-read.c +++ b/tools/perf/util/trace-event-read.c @@ -25,18 +25,18 @@ static int input_fd; static ssize_t trace_data_size; static bool repipe; -static int __do_read(int fd, void *buf, int size) +static ssize_t __do_read(int fd, void *buf, size_t size) { - int rsize = size; + size_t rsize = size; while (size) { - int ret = read(fd, buf, size); + ssize_t ret = read(fd, buf, size); if (ret <= 0) return -1; if (repipe) { - int retw = write(STDOUT_FILENO, buf, ret); + ssize_t retw = write(STDOUT_FILENO, buf, ret); if (retw <= 0 || retw != ret) { pr_debug("repiping input file"); @@ -51,13 +51,13 @@ static int __do_read(int fd, void *buf, int size) return rsize; } -static int do_read(void *data, int size) +static ssize_t do_read(void *data, size_t size) { - int r; + ssize_t r; r = __do_read(input_fd, data, size); if (r <= 0) { - pr_debug("reading input file (size expected=%d received=%d)", + pr_debug("reading input file (size expected=%zu received=%zd)", size, r); return -1; } @@ -68,10 +68,10 @@ static int do_read(void *data, int size) } /* If it fails, the next read will report it */ -static void skip(int size) +static void skip(size_t size) { char buf[BUFSIZ]; - int r; + size_t r; while (size) { r = size > BUFSIZ ? BUFSIZ : size; @@ -127,6 +127,11 @@ static char *read_string(void) } } + if (size >= (int)sizeof(buf) - 1) { + pr_debug("string too long (max %zu bytes)", sizeof(buf) - 1); + goto out; + } + buf[size++] = c; if (!c) @@ -197,7 +202,7 @@ static int read_header_files(struct tep_handle *pevent) unsigned long long size; char *header_page; char buf[BUFSIZ]; - int ret = 0; + ssize_t ret = 0; if (do_read(buf, 12) < 0) return -1; @@ -245,7 +250,7 @@ static int read_header_files(struct tep_handle *pevent) static int read_ftrace_file(struct tep_handle *pevent, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -271,7 +276,7 @@ static int read_ftrace_file(struct tep_handle *pevent, unsigned long long size) static int read_event_file(struct tep_handle *pevent, char *sys, unsigned long long size) { - int ret; + ssize_t ret; char *buf; buf = malloc(size); @@ -317,7 +322,7 @@ static int read_event_files(struct tep_handle *pevent) int systems; int count; int i,x; - int ret; + ssize_t ret; systems = read4(pevent); @@ -345,7 +350,7 @@ static int read_saved_cmdline(struct tep_handle *pevent) { unsigned long long size; char *buf; - int ret; + ssize_t ret; /* it can have 0 size */ size = read8(pevent); diff --git a/tools/perf/util/util.c b/tools/perf/util/util.c index 0f031eb80b4c5..452c691991ae1 100644 --- a/tools/perf/util/util.c +++ b/tools/perf/util/util.c @@ -331,15 +331,9 @@ int perf_event_paranoid(void) bool perf_event_paranoid_check(int max_level) { - bool used_root; - - if (perf_cap__capable(CAP_SYS_ADMIN, &used_root)) - return true; - - if (!used_root && perf_cap__capable(CAP_PERFMON, &used_root)) - return true; - - return perf_event_paranoid() <= max_level; + return perf_cap__capable(CAP_SYS_ADMIN) || + perf_cap__capable(CAP_PERFMON) || + perf_event_paranoid() <= max_level; } int perf_tip(char **strp, const char *dirpath) diff --git a/tools/perf/util/zstd.c b/tools/perf/util/zstd.c index 57027e0ac7b65..d17726b8fb50a 100644 --- a/tools/perf/util/zstd.c +++ b/tools/perf/util/zstd.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include #include "util/compress.h" #include "util/debug.h" @@ -30,9 +31,11 @@ int zstd_fini(struct zstd_data *data) ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, void *src, size_t src_size, size_t max_record_size, - size_t process_header(void *record, size_t increment)) + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)) { - size_t ret, size, compressed = 0; + size_t ret, compressed = 0; + ssize_t size; ZSTD_inBuffer input = { src, src_size, 0 }; ZSTD_outBuffer output; void *record; @@ -54,7 +57,10 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ while (input.pos < input.size) { record = dst; - size = process_header(record, 0); + size = process_header(record, dst_size, 0); + /* Output buffer full — cannot fit even the record header */ + if (size < 0) + goto reset; compressed += size; dst += size; dst_size -= size; @@ -65,17 +71,37 @@ ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_ if (ZSTD_isError(ret)) { pr_err("failed to compress %ld bytes: %s\n", (long)src_size, ZSTD_getErrorName(ret)); - memcpy(dst, src, src_size); - return src_size; + goto reset; } - size = output.pos; - size = process_header(record, size); + compressed += output.pos; + dst += output.pos; + dst_size -= output.pos; + /* + * No progress: ZSTD couldn't emit any bytes into the + * remaining output buffer. Calling process_header + * with output.pos=0 would re-trigger header initialization, + * double-subtracting the header size from dst_size and + * underflowing the unsigned counter. + */ + if (output.pos == 0) + goto reset; + size = process_header(record, dst_size, output.pos); + if (size < 0) + goto reset; compressed += size; dst += size; dst_size -= size; } return compressed; + +reset: + /* Reset so the context is usable if the caller retries */ + ret = ZSTD_initCStream(data->cstream, data->comp_level); + if (ZSTD_isError(ret)) + pr_err("failed to reset compression context: %s\n", + ZSTD_getErrorName(ret)); + return -1; } size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h index dd9144624dc99..3ffd14e9c948f 100644 --- a/tools/sched_ext/include/scx/compat.bpf.h +++ b/tools/sched_ext/include/scx/compat.bpf.h @@ -15,121 +15,17 @@ __ret; \ }) -/* v6.12: 819513666966 ("sched_ext: Add cgroup support") */ -#define __COMPAT_scx_bpf_task_cgroup(p) \ - (bpf_ksym_exists(scx_bpf_task_cgroup) ? \ - scx_bpf_task_cgroup((p)) : NULL) - /* - * v6.13: The verb `dispatch` was too overloaded and confusing. kfuncs are - * renamed to unload the verb. - * - * Build error is triggered if old names are used. New binaries work with both - * new and old names. The compat macros will be removed on v6.15 release. + * v6.15: 950ad93df2fc ("bpf: add kfunc for populating cpumask bits") * - * scx_bpf_dispatch_from_dsq() and friends were added during v6.12 by - * 4c30f5ce4f7a ("sched_ext: Implement scx_bpf_dispatch[_vtime]_from_dsq()"). - * Preserve __COMPAT macros until v6.15. + * Compat macro will be dropped on v6.19 release. */ -void scx_bpf_dispatch___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym __weak; -void scx_bpf_dispatch_vtime___compat(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym __weak; -bool scx_bpf_consume___compat(u64 dsq_id) __ksym __weak; -void scx_bpf_dispatch_from_dsq_set_slice___compat(struct bpf_iter_scx_dsq *it__iter, u64 slice) __ksym __weak; -void scx_bpf_dispatch_from_dsq_set_vtime___compat(struct bpf_iter_scx_dsq *it__iter, u64 vtime) __ksym __weak; -bool scx_bpf_dispatch_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; -bool scx_bpf_dispatch_vtime_from_dsq___compat(struct bpf_iter_scx_dsq *it__iter, struct task_struct *p, u64 dsq_id, u64 enq_flags) __ksym __weak; -int bpf_cpumask_populate(struct cpumask *dst, void *src, size_t src__sz) __ksym __weak; - -#define scx_bpf_dsq_insert(p, dsq_id, slice, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_insert) ? \ - scx_bpf_dsq_insert((p), (dsq_id), (slice), (enq_flags)) : \ - scx_bpf_dispatch___compat((p), (dsq_id), (slice), (enq_flags))) - -#define scx_bpf_dsq_insert_vtime(p, dsq_id, slice, vtime, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_insert_vtime) ? \ - scx_bpf_dsq_insert_vtime((p), (dsq_id), (slice), (vtime), (enq_flags)) : \ - scx_bpf_dispatch_vtime___compat((p), (dsq_id), (slice), (vtime), (enq_flags))) - -#define scx_bpf_dsq_move_to_local(dsq_id) \ - (bpf_ksym_exists(scx_bpf_dsq_move_to_local) ? \ - scx_bpf_dsq_move_to_local((dsq_id)) : \ - scx_bpf_consume___compat((dsq_id))) - -#define __COMPAT_scx_bpf_dsq_move_set_slice(it__iter, slice) \ - (bpf_ksym_exists(scx_bpf_dsq_move_set_slice) ? \ - scx_bpf_dsq_move_set_slice((it__iter), (slice)) : \ - (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_slice___compat) ? \ - scx_bpf_dispatch_from_dsq_set_slice___compat((it__iter), (slice)) : \ - (void)0)) - -#define __COMPAT_scx_bpf_dsq_move_set_vtime(it__iter, vtime) \ - (bpf_ksym_exists(scx_bpf_dsq_move_set_vtime) ? \ - scx_bpf_dsq_move_set_vtime((it__iter), (vtime)) : \ - (bpf_ksym_exists(scx_bpf_dispatch_from_dsq_set_vtime___compat) ? \ - scx_bpf_dispatch_from_dsq_set_vtime___compat((it__iter), (vtime)) : \ - (void) 0)) - -#define __COMPAT_scx_bpf_dsq_move(it__iter, p, dsq_id, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_move) ? \ - scx_bpf_dsq_move((it__iter), (p), (dsq_id), (enq_flags)) : \ - (bpf_ksym_exists(scx_bpf_dispatch_from_dsq___compat) ? \ - scx_bpf_dispatch_from_dsq___compat((it__iter), (p), (dsq_id), (enq_flags)) : \ - false)) - -#define __COMPAT_scx_bpf_dsq_move_vtime(it__iter, p, dsq_id, enq_flags) \ - (bpf_ksym_exists(scx_bpf_dsq_move_vtime) ? \ - scx_bpf_dsq_move_vtime((it__iter), (p), (dsq_id), (enq_flags)) : \ - (bpf_ksym_exists(scx_bpf_dispatch_vtime_from_dsq___compat) ? \ - scx_bpf_dispatch_vtime_from_dsq___compat((it__iter), (p), (dsq_id), (enq_flags)) : \ - false)) +int bpf_cpumask_populate(struct bpf_cpumask *dst, void *src, size_t src__sz) __ksym __weak; #define __COMPAT_bpf_cpumask_populate(cpumask, src, size__sz) \ (bpf_ksym_exists(bpf_cpumask_populate) ? \ (bpf_cpumask_populate(cpumask, src, size__sz)) : -EOPNOTSUPP) -#define scx_bpf_dispatch(p, dsq_id, slice, enq_flags) \ - _Static_assert(false, "scx_bpf_dispatch() renamed to scx_bpf_dsq_insert()") - -#define scx_bpf_dispatch_vtime(p, dsq_id, slice, vtime, enq_flags) \ - _Static_assert(false, "scx_bpf_dispatch_vtime() renamed to scx_bpf_dsq_insert_vtime()") - -#define scx_bpf_consume(dsq_id) ({ \ - _Static_assert(false, "scx_bpf_consume() renamed to scx_bpf_dsq_move_to_local()"); \ - false; \ -}) - -#define scx_bpf_dispatch_from_dsq_set_slice(it__iter, slice) \ - _Static_assert(false, "scx_bpf_dispatch_from_dsq_set_slice() renamed to scx_bpf_dsq_move_set_slice()") - -#define scx_bpf_dispatch_from_dsq_set_vtime(it__iter, vtime) \ - _Static_assert(false, "scx_bpf_dispatch_from_dsq_set_vtime() renamed to scx_bpf_dsq_move_set_vtime()") - -#define scx_bpf_dispatch_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ - _Static_assert(false, "scx_bpf_dispatch_from_dsq() renamed to scx_bpf_dsq_move()"); \ - false; \ -}) - -#define scx_bpf_dispatch_vtime_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ - _Static_assert(false, "scx_bpf_dispatch_vtime_from_dsq() renamed to scx_bpf_dsq_move_vtime()"); \ - false; \ -}) - -#define __COMPAT_scx_bpf_dispatch_from_dsq_set_slice(it__iter, slice) \ - _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq_set_slice() renamed to __COMPAT_scx_bpf_dsq_move_set_slice()") - -#define __COMPAT_scx_bpf_dispatch_from_dsq_set_vtime(it__iter, vtime) \ - _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq_set_vtime() renamed to __COMPAT_scx_bpf_dsq_move_set_vtime()") - -#define __COMPAT_scx_bpf_dispatch_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ - _Static_assert(false, "__COMPAT_scx_bpf_dispatch_from_dsq() renamed to __COMPAT_scx_bpf_dsq_move()"); \ - false; \ -}) - -#define __COMPAT_scx_bpf_dispatch_vtime_from_dsq(it__iter, p, dsq_id, enq_flags) ({ \ - _Static_assert(false, "__COMPAT_scx_bpf_dispatch_vtime_from_dsq() renamed to __COMPAT_scx_bpf_dsq_move_vtime()"); \ - false; \ -}) - /** * __COMPAT_is_enq_cpu_selected - Test if SCX_ENQ_CPU_SELECTED is on * in a compatible way. We will preserve this __COMPAT helper until v6.16. diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c index 55df8b7988657..4f0eee5d8f26b 100644 --- a/tools/sched_ext/scx_central.bpf.c +++ b/tools/sched_ext/scx_central.bpf.c @@ -256,6 +256,7 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) u64 now = scx_bpf_now(); u64 nr_to_kick = nr_queued; s32 i, curr_cpu; + int ret; curr_cpu = bpf_get_smp_processor_id(); if (timer_pinned && (curr_cpu != central_cpu)) { @@ -289,7 +290,10 @@ static int central_timerfn(void *map, int *key, struct bpf_timer *timer) scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT); } - bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN); + ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, + timer_pinned ? BPF_F_TIMER_CPU_PIN : 0); + if (ret) + scx_bpf_error("bpf_timer_start failed (%d)", ret); __sync_fetch_and_add(&nr_timers, 1); return 0; } diff --git a/tools/sched_ext/scx_flatcg.bpf.c b/tools/sched_ext/scx_flatcg.bpf.c index 2c720e3ecad59..d1fc6cb8206c0 100644 --- a/tools/sched_ext/scx_flatcg.bpf.c +++ b/tools/sched_ext/scx_flatcg.bpf.c @@ -256,7 +256,7 @@ static void cgrp_cap_budget(struct cgv_node *cgv_node, struct fcg_cgrp_ctx *cgc) * and thus can't be updated and repositioned. Instead, we collect the * vtime deltas separately and apply it asynchronously here. */ - delta = __sync_fetch_and_sub(&cgc->cvtime_delta, cgc->cvtime_delta); + delta = __sync_fetch_and_and(&cgc->cvtime_delta, 0); cvtime = cgv_node->cvtime + delta; /* @@ -382,7 +382,7 @@ void BPF_STRUCT_OPS(fcg_enqueue, struct task_struct *p, u64 enq_flags) return; } - cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgrp = scx_bpf_task_cgroup(p); cgc = find_cgrp_ctx(cgrp); if (!cgc) goto out_release; @@ -508,7 +508,7 @@ void BPF_STRUCT_OPS(fcg_runnable, struct task_struct *p, u64 enq_flags) { struct cgroup *cgrp; - cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgrp = scx_bpf_task_cgroup(p); update_active_weight_sums(cgrp, true); bpf_cgroup_release(cgrp); } @@ -521,7 +521,7 @@ void BPF_STRUCT_OPS(fcg_running, struct task_struct *p) if (fifo_sched) return; - cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgrp = scx_bpf_task_cgroup(p); cgc = find_cgrp_ctx(cgrp); if (cgc) { /* @@ -551,9 +551,11 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) * too much, determine the execution time by taking explicit timestamps * instead of depending on @p->scx.slice. */ - if (!fifo_sched) - p->scx.dsq_vtime += - (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; + if (!fifo_sched) { + u64 delta = scale_by_task_weight_inverse(p, SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); + } taskc = bpf_task_storage_get(&task_ctx, p, 0, 0); if (!taskc) { @@ -564,11 +566,12 @@ void BPF_STRUCT_OPS(fcg_stopping, struct task_struct *p, bool runnable) if (!taskc->bypassed_at) return; - cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgrp = scx_bpf_task_cgroup(p); cgc = find_cgrp_ctx(cgrp); if (cgc) { __sync_fetch_and_add(&cgc->cvtime_delta, - p->se.sum_exec_runtime - taskc->bypassed_at); + (p->se.sum_exec_runtime - taskc->bypassed_at) * + FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); taskc->bypassed_at = 0; } bpf_cgroup_release(cgrp); @@ -578,7 +581,7 @@ void BPF_STRUCT_OPS(fcg_quiescent, struct task_struct *p, u64 deq_flags) { struct cgroup *cgrp; - cgrp = __COMPAT_scx_bpf_task_cgroup(p); + cgrp = scx_bpf_task_cgroup(p); update_active_weight_sums(cgrp, false); bpf_cgroup_release(cgrp); } @@ -766,10 +769,18 @@ void BPF_STRUCT_OPS(fcg_dispatch, s32 cpu, struct task_struct *prev) * cgroup to execute but the latter needs to be done in a loop * and we can't keep the lock held. Oh well... */ + s64 delta = now - cpuc->cur_at - cgrp_slice_ns; + bpf_spin_lock(&cgv_tree_lock); - __sync_fetch_and_add(&cgc->cvtime_delta, - (cpuc->cur_at + cgrp_slice_ns - now) * - FCG_HWEIGHT_ONE / (cgc->hweight ?: 1)); + /* keep the dividends positive, BPF division is unsigned */ + if (delta >= 0) + __sync_fetch_and_add(&cgc->cvtime_delta, + (u64)delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); + else + __sync_fetch_and_sub(&cgc->cvtime_delta, + (u64)-delta * FCG_HWEIGHT_ONE / + (cgc->hweight ?: 1)); bpf_spin_unlock(&cgv_tree_lock); } else { stat_inc(FCG_STAT_CNS_GONE); @@ -822,7 +833,7 @@ s32 BPF_STRUCT_OPS(fcg_init_task, struct task_struct *p, if (!(cgc = find_cgrp_ctx(args->cgroup))) return -ENOENT; - p->scx.dsq_vtime = cgc->tvtime_now; + scx_bpf_task_set_dsq_vtime(p, cgc->tvtime_now); return 0; } @@ -921,8 +932,8 @@ void BPF_STRUCT_OPS(fcg_cgroup_move, struct task_struct *p, if (!(from_cgc = find_cgrp_ctx(from)) || !(to_cgc = find_cgrp_ctx(to))) return; - delta = time_delta(p->scx.dsq_vtime, from_cgc->tvtime_now); - p->scx.dsq_vtime = to_cgc->tvtime_now + delta; + delta = (s64)(p->scx.dsq_vtime - from_cgc->tvtime_now); + scx_bpf_task_set_dsq_vtime(p, to_cgc->tvtime_now + delta); } s32 BPF_STRUCT_OPS_SLEEPABLE(fcg_init) diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c index 3072b593f8981..19984b52323f5 100644 --- a/tools/sched_ext/scx_qmap.bpf.c +++ b/tools/sched_ext/scx_qmap.bpf.c @@ -301,7 +301,7 @@ static void update_core_sched_head_seq(struct task_struct *p) * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor * difference only when dsp_batch is larger than 1. * - * scx_bpf_dispatch[_vtime]_from_dsq() are allowed both from ops.dispatch() and + * scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and * non-rq-lock holding BPF programs. As demonstration, this function is called * from qmap_dispatch() and monitor_timerfn(). */ @@ -320,12 +320,9 @@ static bool dispatch_highpri(bool from_timer) if (tctx->highpri) { /* exercise the set_*() and vtime interface too */ - __COMPAT_scx_bpf_dsq_move_set_slice( - BPF_FOR_EACH_ITER, slice_ns * 2); - __COMPAT_scx_bpf_dsq_move_set_vtime( - BPF_FOR_EACH_ITER, highpri_seq++); - __COMPAT_scx_bpf_dsq_move_vtime( - BPF_FOR_EACH_ITER, p, HIGHPRI_DSQ, 0); + scx_bpf_dsq_move_set_slice(BPF_FOR_EACH_ITER, slice_ns * 2); + scx_bpf_dsq_move_set_vtime(BPF_FOR_EACH_ITER, highpri_seq++); + scx_bpf_dsq_move_vtime(BPF_FOR_EACH_ITER, p, HIGHPRI_DSQ, 0); } } @@ -342,9 +339,8 @@ static bool dispatch_highpri(bool from_timer) else cpu = scx_bpf_pick_any_cpu(p->cpus_ptr, 0); - if (__COMPAT_scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, - SCX_DSQ_LOCAL_ON | cpu, - SCX_ENQ_PREEMPT)) { + if (scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | cpu, + SCX_ENQ_PREEMPT)) { if (cpu == this_cpu) { dispatched = true; __sync_fetch_and_add(&nr_expedited_local, 1); @@ -530,7 +526,7 @@ static s64 task_qdist(struct task_struct *p) bool BPF_STRUCT_OPS(qmap_core_sched_before, struct task_struct *a, struct task_struct *b) { - return task_qdist(a) > task_qdist(b); + return task_qdist(a) < task_qdist(b); } void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args) diff --git a/tools/sched_ext/scx_simple.bpf.c b/tools/sched_ext/scx_simple.bpf.c index e6de99dba7db6..ab2a85316de39 100644 --- a/tools/sched_ext/scx_simple.bpf.c +++ b/tools/sched_ext/scx_simple.bpf.c @@ -121,12 +121,14 @@ void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable) * too much, determine the execution time by taking explicit timestamps * instead of depending on @p->scx.slice. */ - p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; + u64 delta = scale_by_task_weight_inverse(p, SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); } void BPF_STRUCT_OPS(simple_enable, struct task_struct *p) { - p->scx.dsq_vtime = vtime_now; + scx_bpf_task_set_dsq_vtime(p, vtime_now); } s32 BPF_STRUCT_OPS_SLEEPABLE(simple_init) diff --git a/tools/scripts/Makefile.include b/tools/scripts/Makefile.include index ded48263dd5e0..c5bb7bf42f430 100644 --- a/tools/scripts/Makefile.include +++ b/tools/scripts/Makefile.include @@ -59,10 +59,18 @@ LLVM_SUFFIX := $(LLVM) endif $(call allow-override,CC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,CLANG,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) +$(call allow-override,HOSTCC,$(LLVM_PREFIX)clang$(LLVM_SUFFIX)) $(call allow-override,AR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) +$(call allow-override,HOSTAR,$(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX)) $(call allow-override,LD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) +$(call allow-override,HOSTLD,$(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX)) $(call allow-override,CXX,$(LLVM_PREFIX)clang++$(LLVM_SUFFIX)) $(call allow-override,STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLVM_STRIP,$(LLVM_PREFIX)llvm-strip$(LLVM_SUFFIX)) +$(call allow-override,LLC,$(LLVM_PREFIX)llc$(LLVM_SUFFIX)) +$(call allow-override,LLVM_CONFIG,$(LLVM_PREFIX)llvm-config$(LLVM_SUFFIX)) +$(call allow-override,LLVM_OBJCOPY,$(LLVM_PREFIX)llvm-objcopy$(LLVM_SUFFIX)) else # Allow setting various cross-compile vars or setting CROSS_COMPILE as a prefix. $(call allow-override,CC,$(CROSS_COMPILE)gcc) @@ -70,26 +78,21 @@ $(call allow-override,AR,$(CROSS_COMPILE)ar) $(call allow-override,LD,$(CROSS_COMPILE)ld) $(call allow-override,CXX,$(CROSS_COMPILE)g++) $(call allow-override,STRIP,$(CROSS_COMPILE)strip) -endif - -CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) -ifneq ($(LLVM),) -HOSTAR ?= $(LLVM_PREFIX)llvm-ar$(LLVM_SUFFIX) -HOSTCC ?= $(LLVM_PREFIX)clang$(LLVM_SUFFIX) -HOSTLD ?= $(LLVM_PREFIX)ld.lld$(LLVM_SUFFIX) -else -HOSTAR ?= ar -HOSTCC ?= gcc -HOSTLD ?= ld +# Host versions aren't prefixed +$(call allow-override,HOSTAR,ar) +$(call allow-override,HOSTCC,gcc) +$(call allow-override,HOSTLD,ld) + +# Some tools still require Clang, LLC and/or LLVM utils +$(call allow-override,CLANG,clang) +$(call allow-override,LLC,llc) +$(call allow-override,LLVM_CONFIG,llvm-config) +$(call allow-override,LLVM_OBJCOPY,llvm-objcopy) +$(call allow-override,LLVM_STRIP,llvm-strip) endif -# Some tools require Clang, LLC and/or LLVM utils -CLANG ?= clang -LLC ?= llc -LLVM_CONFIG ?= llvm-config -LLVM_OBJCOPY ?= llvm-objcopy -LLVM_STRIP ?= llvm-strip +CC_NO_CLANG := $(shell $(CC) -dM -E -x c /dev/null | grep -Fq "__clang__"; echo $$?) # Some tools require bpftool SYSTEM_BPFTOOL ?= bpftool diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 0d5ce4b74b9f7..950335b3c128f 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -58,12 +58,13 @@ cxl_core-y += $(CXL_CORE_SRC)/pci.o cxl_core-y += $(CXL_CORE_SRC)/hdm.o cxl_core-y += $(CXL_CORE_SRC)/pmu.o cxl_core-y += $(CXL_CORE_SRC)/cdat.o -cxl_core-y += $(CXL_CORE_SRC)/ras.o cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o cxl_core-$(CONFIG_CXL_MCE) += $(CXL_CORE_SRC)/mce.o cxl_core-$(CONFIG_CXL_FEATURES) += $(CXL_CORE_SRC)/features.o cxl_core-$(CONFIG_CXL_EDAC_MEM_FEATURES) += $(CXL_CORE_SRC)/edac.o +cxl_core-$(CONFIG_CXL_RAS) += $(CXL_CORE_SRC)/ras.o +cxl_core-$(CONFIG_CXL_RAS) += $(CXL_CORE_SRC)/ras_rch.o cxl_core-y += config_check.o cxl_core-y += cxl_core_test.o cxl_core-y += cxl_core_exports.o diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py index cd99c1956331d..f3392cb3b7af7 100755 --- a/tools/testing/kunit/kunit.py +++ b/tools/testing/kunit/kunit.py @@ -123,7 +123,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) lines.pop() # Filter out any extraneous non-test output that might have gotten mixed in. - return [l for l in output if re.match(r'^[^\s.]+\.[^\s.]+$', l)] + return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] def _list_tests_attr(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> Iterable[str]: args = ['kunit.action=list_attr'] diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py index 2998e1bc088b2..2869fcb199ff7 100644 --- a/tools/testing/kunit/kunit_kernel.py +++ b/tools/testing/kunit/kunit_kernel.py @@ -16,7 +16,7 @@ import signal import sys import threading -from typing import Iterator, List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple, Any from types import FrameType import kunit_config @@ -265,6 +265,7 @@ def __init__( if kconfig_add: kconfig = kunit_config.parse_from_string('\n'.join(kconfig_add)) self._kconfig.merge_in_entries(kconfig) + self._process : Optional[subprocess.Popen[Any]] = None def arch(self) -> str: return self._arch @@ -345,6 +346,12 @@ def build_kernel(self, jobs: int, build_dir: str, make_options: Optional[List[st return False return self.validate_config(build_dir) + def _restore_terminal_if_tty(self) -> None: + # stty requires a controlling terminal; skip headless runs. + if sys.stdin is None or not sys.stdin.isatty(): + return + subprocess.call(['stty', 'sane']) + def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_glob: str='', filter: str='', filter_action: Optional[str]=None, timeout: Optional[int]=None) -> Iterator[str]: # Copy to avoid mutating the caller-supplied list. exec_tests() reuses # the same args across repeated run_kernel() calls (e.g. --run_isolated), @@ -358,36 +365,45 @@ def run_kernel(self, args: Optional[List[str]]=None, build_dir: str='', filter_g args.append('kunit.filter_action=' + filter_action) args.append('kunit.enable=1') - process = self._ops.start(args, build_dir) - assert process.stdout is not None # tell mypy it's set + self._process = self._ops.start(args, build_dir) + assert self._process is not None # tell mypy it's set + assert self._process.stdout is not None # tell mypy it's set # Enforce the timeout in a background thread. def _wait_proc() -> None: try: - process.wait(timeout=timeout) + if self._process: + self._process.wait(timeout=timeout) except Exception as e: print(e) - process.terminate() - process.wait() + if self._process: + self._process.terminate() + self._process.wait() waiter = threading.Thread(target=_wait_proc) waiter.start() output = open(get_outfile_path(build_dir), 'w') try: # Tee the output to the file and to our caller in real time. - for line in process.stdout: + for line in self._process.stdout: output.write(line) yield line # This runs even if our caller doesn't consume every line. finally: # Flush any leftover output to the file - output.write(process.stdout.read()) + if self._process: + if self._process.stdout: + output.write(self._process.stdout.read()) + self._process.stdout.close() + self._process = None output.close() - process.stdout.close() waiter.join() - subprocess.call(['stty', 'sane']) + self._restore_terminal_if_tty() def signal_handler(self, unused_sig: int, unused_frame: Optional[FrameType]) -> None: logging.error('Build interruption occurred. Cleaning console.') - subprocess.call(['stty', 'sane']) + if self._process: + self._process.terminate() + self._process.wait() + self._restore_terminal_if_tty() diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py index ed45bac1548d9..a8eb223e57e10 100755 --- a/tools/testing/kunit/kunit_tool_test.py +++ b/tools/testing/kunit/kunit_tool_test.py @@ -515,6 +515,48 @@ def fake_start(start_args, unused_build_dir): self.assertIn('kunit.filter_glob=suite.test1', start_calls[0]) self.assertIn('kunit.filter_glob=suite.test2', start_calls[1]) + def test_run_kernel_skips_terminal_reset_without_tty(self): + def fake_start(unused_args, unused_build_dir): + return subprocess.Popen(['printf', 'KTAP version 1\n'], + text=True, stdout=subprocess.PIPE) + + non_tty_stdin = mock.Mock() + non_tty_stdin.isatty.return_value = False + + with tempfile.TemporaryDirectory('') as build_dir: + tree = kunit_kernel.LinuxSourceTree(build_dir, kunitconfig_paths=[os.devnull]) + with mock.patch.object(tree._ops, 'start', side_effect=fake_start), \ + mock.patch.object(kunit_kernel.sys, 'stdin', non_tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call: + for _ in tree.run_kernel(build_dir=build_dir): + pass + + mock_call.assert_not_called() + + def test_signal_handler_skips_terminal_reset_without_tty(self): + non_tty_stdin = mock.Mock() + non_tty_stdin.isatty.return_value = False + tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull]) + + with mock.patch.object(kunit_kernel.sys, 'stdin', non_tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call, \ + mock.patch.object(kunit_kernel.logging, 'error') as mock_error: + tree.signal_handler(signal.SIGINT, None) + mock_error.assert_called_once() + mock_call.assert_not_called() + + def test_signal_handler_resets_terminal_with_tty(self): + tty_stdin = mock.Mock() + tty_stdin.isatty.return_value = True + tree = kunit_kernel.LinuxSourceTree('', kunitconfig_paths=[os.devnull]) + + with mock.patch.object(kunit_kernel.sys, 'stdin', tty_stdin), \ + mock.patch.object(kunit_kernel.subprocess, 'call') as mock_call, \ + mock.patch.object(kunit_kernel.logging, 'error') as mock_error: + tree.signal_handler(signal.SIGINT, None) + mock_error.assert_called_once() + mock_call.assert_called_once_with(['stty', 'sane']) + def test_build_reconfig_no_config(self): with tempfile.TemporaryDirectory('') as build_dir: with open(kunit_kernel.get_kunitconfig_path(build_dir), 'w') as f: @@ -854,6 +896,18 @@ def test_list_tests(self): self.linux_source_mock.run_kernel.assert_called_once_with( args=['kunit.action=list'], build_dir='.kunit', filter_glob='suite*', filter='', filter_action=None, timeout=300) + def test_list_tests_with_prefix(self): + want = ['suite.test1', 'suite.test2', 'suite2.test1'] + self.linux_source_mock.run_kernel.return_value = [ + '[ 0.100000] TAP version 14', + '[ 0.200000] suite.test1', + '[ 0.200000] suite.test2', + '[ 0.300000] suite2.test1'] + + got = kunit._list_tests(self.linux_source_mock, + kunit.KunitExecRequest(None, None, None, False, False, '.kunit', 300, 'suite*', '', None, None, 'suite', False, False, False)) + self.assertEqual(got, want) + @mock.patch.object(kunit, '_list_tests') def test_run_isolated_by_suite(self, mock_tests): mock_tests.return_value = ['suite.test1', 'suite.test2', 'suite2.test1'] diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c index e113dafa5c246..2b9278475e6de 100644 --- a/tools/testing/selftests/alsa/mixer-test.c +++ b/tools/testing/selftests/alsa/mixer-test.c @@ -84,6 +84,7 @@ static void find_controls(void) if (err < 0) { ksft_print_msg("Failed to get hctl for card %d: %s\n", card, snd_strerror(err)); + free(card_data); goto next_card; } @@ -318,8 +319,8 @@ static bool ctl_value_index_valid(struct ctl_data *ctl, /* Only check step size if there is one and we're in bounds */ if (snd_ctl_elem_info_get_step(ctl->info) && - (int_val - snd_ctl_elem_info_get_min(ctl->info) % - snd_ctl_elem_info_get_step(ctl->info))) { + (int_val - snd_ctl_elem_info_get_min(ctl->info)) % + snd_ctl_elem_info_get_step(ctl->info)) { ksft_print_msg("%s.%d value %ld invalid for step %ld minimum %ld\n", ctl->name, index, int_val, snd_ctl_elem_info_get_step(ctl->info), diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index a85c19e9524e1..c3fcd47e33968 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 80e072f221cde..7ef7835389e76 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -298,15 +298,20 @@ function irritator_handler add x0, x0, #1 str x0, [x2, #ucontext_regs + 8 * 23] +#ifndef SSVE // Corrupt some random Z-regs movi v0.8b, #1 movi v9.16b, #2 movi v31.8b, #3 // And P0 ptrue p0.d -#ifndef SSVE // And FFR wrffr p15.b +#else + // Enter and exit streaming mode, will reset all of the V, Z, P + // and FFR registers that the system has. + smstart_sm + smstop #endif ret diff --git a/tools/testing/selftests/arm64/mte/check_buffer_fill.c b/tools/testing/selftests/arm64/mte/check_buffer_fill.c index ff4e075033493..039b1d7d85663 100644 --- a/tools/testing/selftests/arm64/mte/check_buffer_fill.c +++ b/tools/testing/selftests/arm64/mte/check_buffer_fill.c @@ -406,6 +406,8 @@ int main(int argc, char *argv[]) size_t page_size = getpagesize(); int item = ARRAY_SIZE(sizes); + ksft_print_header(); + sizes[item - 3] = page_size - 1; sizes[item - 2] = page_size; sizes[item - 1] = page_size + 1; diff --git a/tools/testing/selftests/arm64/mte/check_child_memory.c b/tools/testing/selftests/arm64/mte/check_child_memory.c index 5e97ee792e4d2..e6a8acca2a941 100644 --- a/tools/testing/selftests/arm64/mte/check_child_memory.c +++ b/tools/testing/selftests/arm64/mte/check_child_memory.c @@ -146,6 +146,8 @@ int main(int argc, char *argv[]) int err; int item = ARRAY_SIZE(sizes); + ksft_print_header(); + page_size = getpagesize(); if (!page_size) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c index 325bca0de0f6e..d23f154d3288c 100644 --- a/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c +++ b/tools/testing/selftests/arm64/mte/check_gcr_el1_cswitch.c @@ -131,6 +131,7 @@ int main(int argc, char *argv[]) if (err) return err; + ksft_print_header(); ksft_set_plan(1); evaluate_test(mte_gcr_fork_test(), diff --git a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c index aad1234c7e0fe..23e4a7a9950c1 100644 --- a/tools/testing/selftests/arm64/mte/check_hugetlb_options.c +++ b/tools/testing/selftests/arm64/mte/check_hugetlb_options.c @@ -230,6 +230,8 @@ int main(int argc, char *argv[]) void *map_ptr; unsigned long map_size; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_ksm_options.c b/tools/testing/selftests/arm64/mte/check_ksm_options.c index 0cf5faef17248..4855b737d5507 100644 --- a/tools/testing/selftests/arm64/mte/check_ksm_options.c +++ b/tools/testing/selftests/arm64/mte/check_ksm_options.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,20 @@ static size_t page_sz; static unsigned long ksm_sysfs[5]; +static bool has_merge_across_nodes; + +static bool merge_across_nodes_available(void) +{ + const char *path = PATH_KSM "merge_across_nodes"; + + if (!access(path, R_OK | W_OK)) + return true; + if (errno == ENOENT) + return false; + + ksft_exit_skip("Unable to read and write %s: %s\n", path, + strerror(errno)); +} static unsigned long read_sysfs(char *str) { @@ -56,8 +71,10 @@ static void write_sysfs(char *str, unsigned long val) static void mte_ksm_setup(void) { - ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); - write_sysfs(PATH_KSM "merge_across_nodes", 1); + if (has_merge_across_nodes) { + ksm_sysfs[0] = read_sysfs(PATH_KSM "merge_across_nodes"); + write_sysfs(PATH_KSM "merge_across_nodes", 1); + } ksm_sysfs[1] = read_sysfs(PATH_KSM "sleep_millisecs"); write_sysfs(PATH_KSM "sleep_millisecs", 0); ksm_sysfs[2] = read_sysfs(PATH_KSM "run"); @@ -70,7 +87,8 @@ static void mte_ksm_setup(void) static void mte_ksm_restore(void) { - write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); + if (has_merge_across_nodes) + write_sysfs(PATH_KSM "merge_across_nodes", ksm_sysfs[0]); write_sysfs(PATH_KSM "sleep_millisecs", ksm_sysfs[1]); write_sysfs(PATH_KSM "run", ksm_sysfs[2]); write_sysfs(PATH_KSM "max_page_sharing", ksm_sysfs[3]); @@ -132,9 +150,16 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; + + if (geteuid() != 0) + ksft_exit_skip("Please run the test as root\n"); + + has_merge_across_nodes = merge_across_nodes_available(); page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/arm64/mte/check_mmap_options.c b/tools/testing/selftests/arm64/mte/check_mmap_options.c index c100af3012cbb..492f2cd41f43b 100644 --- a/tools/testing/selftests/arm64/mte/check_mmap_options.c +++ b/tools/testing/selftests/arm64/mte/check_mmap_options.c @@ -945,6 +945,8 @@ int main(int argc, char *argv[]) }, }; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_prctl.c b/tools/testing/selftests/arm64/mte/check_prctl.c index f7f320defa7b9..d16a91117eef9 100644 --- a/tools/testing/selftests/arm64/mte/check_prctl.c +++ b/tools/testing/selftests/arm64/mte/check_prctl.c @@ -119,7 +119,7 @@ int main(void) int i; ksft_print_header(); - ksft_set_plan(ARRAY_SIZE(mte_modes)); + ksft_set_plan(ARRAY_SIZE(mte_modes) + 1); check_basic_read(); for (i = 0; i < ARRAY_SIZE(mte_modes); i++) diff --git a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c index 4b764f2a81857..6b4fa6705d7c1 100644 --- a/tools/testing/selftests/arm64/mte/check_tags_inclusion.c +++ b/tools/testing/selftests/arm64/mte/check_tags_inclusion.c @@ -175,6 +175,8 @@ int main(int argc, char *argv[]) { int err; + ksft_print_header(); + err = mte_default_setup(); if (err) return err; diff --git a/tools/testing/selftests/arm64/mte/check_user_mem.c b/tools/testing/selftests/arm64/mte/check_user_mem.c index fb7936c4e0978..af343aa617323 100644 --- a/tools/testing/selftests/arm64/mte/check_user_mem.c +++ b/tools/testing/selftests/arm64/mte/check_user_mem.c @@ -201,6 +201,8 @@ int main(int argc, char *argv[]) int tag_offsets[] = {page_sz, MT_GRANULE_SIZE}; char test_name[TEST_NAME_MAX]; + ksft_print_header(); + page_sz = getpagesize(); if (!page_sz) { ksft_print_msg("ERR: Unable to get page size\n"); diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 776fbe3cb8f9b..a36afb460e264 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -107,12 +107,12 @@ Docker container and local rootfs image. The overall steps are as follows: tools/testing/selftests/bpf/vmtest.sh \ -l -- \ ./test_progs -d \ - \"$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ + "$(cat tools/testing/selftests/bpf/DENYLIST.riscv64 \ | cut -d'#' -f1 \ | sed -e 's/^[[:space:]]*//' \ -e 's/[[:space:]]*$//' \ | tr -s '\n' ',' \ - )\" + )" Link: https://github.com/pulehui/riscv-bpf-vmtest.git [0] Link: https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh [1] diff --git a/tools/testing/selftests/bpf/network_helpers.c b/tools/testing/selftests/bpf/network_helpers.c index cdf7b66414442..2067e12034ba0 100644 --- a/tools/testing/selftests/bpf/network_helpers.c +++ b/tools/testing/selftests/bpf/network_helpers.c @@ -97,7 +97,7 @@ int settimeo(int fd, int timeout_ms) int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t addrlen, const struct network_helper_opts *opts) { - int fd; + int on = 1, fd; if (!opts) opts = &default_opts; @@ -111,6 +111,12 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a if (settimeo(fd, opts->timeout_ms)) goto error_close; + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM && + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on))) { + log_err("Failed to enable SO_REUSEADDR"); + goto error_close; + } + if (opts->post_socket_cb && opts->post_socket_cb(fd, opts->cb_opts)) { log_err("Failed to call post_socket_cb"); @@ -122,7 +128,7 @@ int start_server_addr(int type, const struct sockaddr_storage *addr, socklen_t a goto error_close; } - if (type == SOCK_STREAM) { + if ((type & SOCK_TYPE_MASK) == SOCK_STREAM) { if (listen(fd, opts->backlog ? MAX(opts->backlog, 0) : 1) < 0) { log_err("Failed to listed on socket"); goto error_close; diff --git a/tools/testing/selftests/bpf/network_helpers.h b/tools/testing/selftests/bpf/network_helpers.h index ef208eefd571a..5ea989bc40098 100644 --- a/tools/testing/selftests/bpf/network_helpers.h +++ b/tools/testing/selftests/bpf/network_helpers.h @@ -25,6 +25,11 @@ typedef __u16 __sum16; #define VIP_NUM 5 #define MAGIC_BYTES 123 +/* include/linux/net.h */ +#ifndef SOCK_TYPE_MASK +#define SOCK_TYPE_MASK 0xf +#endif + struct network_helper_opts { int timeout_ms; int proto; diff --git a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c index 56b5baef35c8c..602ce30f1720c 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_map_resize.c +++ b/tools/testing/selftests/bpf/prog_tests/global_map_resize.c @@ -23,6 +23,7 @@ static void global_map_resize_bss_subtest(void) struct bpf_map *map; const __u32 desired_sz = sizeof(skel->bss->sum) + sysconf(_SC_PAGE_SIZE) * 2; size_t array_len, actual_sz, new_sz; + int *array; skel = test_global_map_resize__open(); if (!ASSERT_OK_PTR(skel, "test_global_map_resize__open")) @@ -58,10 +59,13 @@ static void global_map_resize_bss_subtest(void) goto teardown; /* fill the newly resized array with ones, - * skipping the first element which was previously set + * skipping the first element which was previously set; + * access through a plain pointer to avoid -Warray-bounds + * since the array was resized beyond its declared length. */ + array = skel->bss->array; for (int i = 1; i < array_len; i++) - skel->bss->array[i] = 1; + array[i] = 1; /* set global const values before loading */ skel->rodata->pid = getpid(); diff --git a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c index b6391af5f6f96..39e8a3b8b6afb 100644 --- a/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/prog_tests/lwt_ip_encap.c @@ -3,6 +3,7 @@ #include "network_helpers.h" #include "test_progs.h" +#include "test_lwt_ip_encap.skel.h" #define BPF_FILE "test_lwt_ip_encap.bpf.o" @@ -32,6 +33,9 @@ #define IP6_ADDR_8 "fb08::1" #define IP6_ADDR_GRE "fb10::1" +#define IP4_ADDR_VXLAN "172.16.17.100" +#define IP6_ADDR_VXLAN "fb11::1" + #define IP6_ADDR_SRC IP6_ADDR_1 #define IP6_ADDR_DST IP6_ADDR_4 @@ -406,7 +410,8 @@ static int test_gso_fix(const char *ns1, const char *ns3, int family) static int check_ping_ok(const char *ns1) { SYS(fail, "ip netns exec %s ping -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP4_ADDR_DST); - SYS(fail, "ip netns exec %s ping6 -c 1 -W1 -I veth1 %s > /dev/null", ns1, IP6_ADDR_DST); + SYS(fail, "ip netns exec %s %s -c 1 -W1 -I veth1 %s > /dev/null", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); return 0; fail: return -1; @@ -420,7 +425,8 @@ static int check_ping_fails(const char *ns1) if (!ret) return -1; - ret = SYS_NOFAIL("ip netns exec %s ping6 -c 1 -W1 -I veth1 %s", ns1, IP6_ADDR_DST); + ret = SYS_NOFAIL("ip netns exec %s %s -c 1 -W1 -I veth1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); if (!ret) return -1; @@ -538,3 +544,145 @@ void test_lwt_ip_encap_ipv4(void) if (test__start_subtest("ingress")) lwt_ip_encap(IPV4_ENCAP, INGRESS, ""); } + +/* + * VxLAN Setup/topology: + * + * NS1 (IP*_ADDR_1) NS2 NS3 (IP*_ADDR_4) + * [ping src] + * | top route + * veth1 (LWT encap) <<-- veth2 veth3 <<-- veth4 (ping dst) + * | ^ + * (bottom route) | (inner pkt) + * v bottom route | + * veth5 -->> veth6 veth7 -->> veth8 (vxlan decap) + * (IP*_ADDR_VXLAN) + * + * Add the VxLAN endpoint addresses to NS3's veth8, create standard + * VxLAN decap devices bound to those addresses, and install routes so + * NS1/NS2 can reach the endpoints via the bottom route. NS2 here is to + * make sure the LWT-encap VxLAN packets are routed to NS3 correctly. + */ +static int setup_vxlan_routes(const char *ns3, const char *ns1, const char *ns2) +{ + struct nstoken *nstoken; + + nstoken = open_netns(ns3); + if (!ASSERT_OK_PTR(nstoken, "open ns3 for vxlan")) + return -1; + + SYS(fail_close, "ip a add %s/32 dev veth8", IP4_ADDR_VXLAN); + SYS(fail_close, "ip -6 a add %s/128 dev veth8", IP6_ADDR_VXLAN); + /* + * Standard VxLAN devices to decap the encapsulated packets. The inner + * Ethernet frame uses a broadcast dst MAC so the IP stack accepts it + * without ARP or FDB configuration. + */ + SYS(fail_close, "ip link add vxlan4 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning noudpcsum", + IP4_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan4 up"); + SYS(fail_close, "ip link add vxlan6 type vxlan id 1 dstport 4789 local %s dev veth8 nolearning udp6zerocsumrx", + IP6_ADDR_VXLAN); + SYS(fail_close, "ip link set vxlan6 up"); + close_netns(nstoken); + + SYS(fail, "ip -n %s route add %s/32 dev veth5 via %s", + ns1, IP4_ADDR_VXLAN, IP4_ADDR_6); + SYS(fail, "ip -n %s route add %s/32 dev veth7 via %s", + ns2, IP4_ADDR_VXLAN, IP4_ADDR_8); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth5 via %s", + ns1, IP6_ADDR_VXLAN, IP6_ADDR_6); + SYS(fail, "ip -n %s -6 route add %s/128 dev veth7 via %s", + ns2, IP6_ADDR_VXLAN, IP6_ADDR_8); + return 0; + +fail_close: + close_netns(nstoken); +fail: + return -1; +} + +static void lwt_ip_encap_vxlan(bool ipv4_encap) +{ + char ns1[NETNS_NAME_SIZE] = NETNS_BASE "-1-"; + char ns2[NETNS_NAME_SIZE] = NETNS_BASE "-2-"; + char ns3[NETNS_NAME_SIZE] = NETNS_BASE "-3-"; + const char *sec = ipv4_encap ? "encap_vxlan" : "encap_vxlan6"; + int expected_offset = ipv4_encap ? (int)sizeof(struct iphdr) + : (int)sizeof(struct ipv6hdr); + struct test_lwt_ip_encap *skel = NULL; + int thdr_offset, err; + + if (!ASSERT_OK(create_ns(ns1, NETNS_NAME_SIZE), "create ns1")) + goto out; + if (!ASSERT_OK(create_ns(ns2, NETNS_NAME_SIZE), "create ns2")) + goto out; + if (!ASSERT_OK(create_ns(ns3, NETNS_NAME_SIZE), "create ns3")) + goto out; + + if (!ASSERT_OK(setup_network(ns1, ns2, ns3, ""), "setup network")) + goto out; + + if (!ASSERT_OK(setup_vxlan_routes(ns3, ns1, ns2), "setup vxlan routes")) + goto out; + + skel = test_lwt_ip_encap__open(); + if (!ASSERT_OK_PTR(skel, "test_lwt_ip_encap__open")) + goto out; + + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_gre6, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan, false); + bpf_program__set_autoload(skel->progs.bpf_lwt_encap_vxlan6, false); + bpf_program__set_autoload(skel->progs.fexit_lwt_push_ip_encap, true); + skel->rodata->tgt_ip_version = ipv4_encap ? 4 : 6; + + err = test_lwt_ip_encap__load(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__load")) + goto out; + + err = test_lwt_ip_encap__attach(skel); + if (!ASSERT_OK(err, "test_lwt_ip_encap__attach")) + goto out; + + /* Remove the direct NS2->DST route so packets must go via LWT encap. */ + SYS(out, "ip -n %s route del %s/32 dev veth3", ns2, IP4_ADDR_DST); + SYS(out, "ip -n %s -6 route del %s/128 dev veth3", ns2, IP6_ADDR_DST); + + if (ipv4_encap) + SYS(out, "ip -n %s route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP4_ADDR_DST, BPF_FILE, sec); + else + SYS(out, "ip -n %s -6 route add %s encap bpf xmit obj %s sec %s dev veth1", + ns1, IP6_ADDR_DST, BPF_FILE, sec); + + skel->bss->fexit_triggered = false; + + if (ipv4_encap) + SYS(out, "ip netns exec %s ping -c 1 -W1 %s", ns1, IP4_ADDR_DST); + else + SYS(out, "ip netns exec %s %s -c 1 -W1 %s", ns1, + ping_command(AF_INET6), IP6_ADDR_DST); + + if (!ASSERT_TRUE(skel->bss->fexit_triggered, "fexit_triggered")) + goto out; + + thdr_offset = (int)skel->bss->transport_hdr - (int)skel->bss->network_hdr; + ASSERT_EQ(thdr_offset, expected_offset, "transport_hdr offset"); + +out: + test_lwt_ip_encap__destroy(skel); + SYS_NOFAIL("ip netns del %s", ns1); + SYS_NOFAIL("ip netns del %s", ns2); + SYS_NOFAIL("ip netns del %s", ns3); +} + +void test_lwt_ip_encap_vxlan_ipv4(void) +{ + lwt_ip_encap_vxlan(IPV4_ENCAP); +} + +void test_lwt_ip_encap_vxlan_ipv6(void) +{ + lwt_ip_encap_vxlan(IPV6_ENCAP); +} diff --git a/tools/testing/selftests/bpf/prog_tests/mptcp.c b/tools/testing/selftests/bpf/prog_tests/mptcp.c index 8fade8bdc4516..32dfc1c511af6 100644 --- a/tools/testing/selftests/bpf/prog_tests/mptcp.c +++ b/tools/testing/selftests/bpf/prog_tests/mptcp.c @@ -264,7 +264,7 @@ static int verify_mptcpify(int server_fd, int client_fd) return err; } -static int run_mptcpify(int cgroup_fd) +static int run_mptcpify(int cgroup_fd, int type) { int server_fd, client_fd, err = 0; struct mptcpify *mptcpify_skel; @@ -280,7 +280,7 @@ static int run_mptcpify(int cgroup_fd) goto out; /* without MPTCP */ - server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0); + server_fd = start_server(AF_INET, type, NULL, 0, 0); if (!ASSERT_GE(server_fd, 0, "start_server")) { err = -EIO; goto out; @@ -317,7 +317,14 @@ static void test_mptcpify(void) if (!ASSERT_OK_PTR(netns, "netns_new")) goto fail; - ASSERT_OK(run_mptcpify(cgroup_fd), "run_mptcpify"); + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM), "run_mptcpify"); + /* userspace sets flags such as SOCK_CLOEXEC together with the type; + * the BPF prog must still upgrade the socket to MPTCP. See + * update_socket_protocol() in net/socket.c, which runs before the + * type is masked with SOCK_TYPE_MASK. + */ + ASSERT_OK(run_mptcpify(cgroup_fd, SOCK_STREAM | SOCK_CLOEXEC), + "run_mptcpify_cloexec"); fail: netns_free(netns); diff --git a/tools/testing/selftests/bpf/prog_tests/sha256.c b/tools/testing/selftests/bpf/prog_tests/sha256.c index 604a0b1423d55..5edbc6194b071 100644 --- a/tools/testing/selftests/bpf/prog_tests/sha256.c +++ b/tools/testing/selftests/bpf/prog_tests/sha256.c @@ -25,10 +25,10 @@ void test_sha256(void) size_t i; data = malloc(MAX_LEN); - if (!ASSERT_OK_PTR(data, "malloc")) + if (!ASSERT_NEQ(data, NULL, "malloc")) goto out; digests = malloc((MAX_LEN + 1) * SHA256_DIGEST_LENGTH); - if (!ASSERT_OK_PTR(digests, "malloc")) + if (!ASSERT_NEQ(digests, NULL, "malloc")) goto out; /* Generate MAX_LEN bytes of "random" data deterministically. */ diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c index 1e3e4392dcca0..54fd84203dc2b 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c @@ -786,7 +786,7 @@ static void test_sockmap_many_socket(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -855,7 +855,7 @@ static void test_sockmap_many_maps(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); @@ -926,7 +926,7 @@ static void test_sockmap_same_sock(void) return; } - udp = xsocket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK, 0); + udp = socket_loopback(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK); if (udp < 0) { close(dgram); close(tcp); diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c index f1bdccc7e4e79..0a744f786e107 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c +++ b/tools/testing/selftests/bpf/prog_tests/sockmap_listen.c @@ -53,8 +53,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, int family, int sotype, int mapfd) { u32 key = 0; - u64 value; int err, s; + u64 value; s = xsocket(family, sotype, 0); if (s == -1) @@ -63,11 +63,8 @@ static void test_insert_opened(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (sotype == SOCK_STREAM) { - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); - } else if (err) - FAIL_ERRNO("map_update: expected success"); + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); xclose(s); } @@ -77,8 +74,8 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, struct sockaddr_storage addr; socklen_t len = 0; u32 key = 0; - u64 value; int err, s; + u64 value; init_addr_loopback(family, &addr, &len); @@ -93,8 +90,12 @@ static void test_insert_bound(struct test_sockmap_listen *skel __always_unused, errno = 0; value = s; err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST); - if (!err || errno != EOPNOTSUPP) - FAIL_ERRNO("map_update: expected EOPNOTSUPP"); + if (sotype == SOCK_STREAM) { + ASSERT_ERR(err, "map_update"); + ASSERT_EQ(errno, EOPNOTSUPP, "errno"); + } else { + ASSERT_OK(err, "map_update"); + } close: xclose(s); } @@ -1289,7 +1290,7 @@ static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map, /* insert */ TEST(test_insert_invalid), TEST(test_insert_opened), - TEST(test_insert_bound, SOCK_STREAM), + TEST(test_insert_bound), TEST(test_insert), /* delete */ TEST(test_delete_after_insert), diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c index 164640db3a29c..b1e509b231cd9 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp4.c @@ -99,13 +99,13 @@ static int dump_tcp_sock(struct seq_file *seq, struct tcp_sock *tp, icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) { timer_active = 1; - timer_expires = icsk->icsk_retransmit_timer.expires; + timer_expires = sp->tcp_retransmit_timer.expires; } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) { timer_active = 4; - timer_expires = icsk->icsk_retransmit_timer.expires; - } else if (timer_pending(&sp->sk_timer)) { + timer_expires = sp->tcp_retransmit_timer.expires; + } else if (timer_pending(&icsk->icsk_keepalive_timer)) { timer_active = 2; - timer_expires = sp->sk_timer.expires; + timer_expires = icsk->icsk_keepalive_timer.expires; } else { timer_active = 0; timer_expires = bpf_jiffies64(); diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c index 591c703f5032f..dbc7166aee91f 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_tcp6.c @@ -99,13 +99,13 @@ static int dump_tcp6_sock(struct seq_file *seq, struct tcp6_sock *tp, icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) { timer_active = 1; - timer_expires = icsk->icsk_retransmit_timer.expires; + timer_expires = sp->tcp_retransmit_timer.expires; } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) { timer_active = 4; - timer_expires = icsk->icsk_retransmit_timer.expires; - } else if (timer_pending(&sp->sk_timer)) { + timer_expires = sp->tcp_retransmit_timer.expires; + } else if (timer_pending(&icsk->icsk_keepalive_timer)) { timer_active = 2; - timer_expires = sp->sk_timer.expires; + timer_expires = icsk->icsk_keepalive_timer.expires; } else { timer_active = 0; timer_expires = bpf_jiffies64(); diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index 17db400f0e0d9..a972e174f741b 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -8,6 +8,9 @@ #define AF_INET 2 #define AF_INET6 10 +/* include/linux/net.h */ +#define SOCK_TYPE_MASK 0xf + #define SOL_SOCKET 1 #define SO_REUSEADDR 2 #define SO_SNDBUF 7 diff --git a/tools/testing/selftests/bpf/progs/mptcpify.c b/tools/testing/selftests/bpf/progs/mptcpify.c index cbdc730c3a471..e3f8cb54dbe97 100644 --- a/tools/testing/selftests/bpf/progs/mptcpify.c +++ b/tools/testing/selftests/bpf/progs/mptcpify.c @@ -15,7 +15,7 @@ int BPF_PROG(mptcpify, int family, int type, int protocol) return protocol; if ((family == AF_INET || family == AF_INET6) && - type == SOCK_STREAM && + (type & SOCK_TYPE_MASK) == SOCK_STREAM && (!protocol || protocol == IPPROTO_TCP)) { return IPPROTO_MPTCP; } diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c index f7b330ddd007e..b42e6e149e26c 100644 --- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c +++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c @@ -171,8 +171,8 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32, ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; @@ -274,8 +274,8 @@ nf_ct_opts_new_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple * ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def, sizeof(opts_def)); if (ct) { - __u16 sport = bpf_get_prandom_u32(); - __u16 dport = bpf_get_prandom_u32(); + __u16 sport = bpf_get_prandom_u32() % 65535 + 1; + __u16 dport = bpf_get_prandom_u32() % 65535 + 1; union nf_inet_addr saddr = {}; union nf_inet_addr daddr = {}; struct nf_conn *ct_ins; diff --git a/tools/testing/selftests/bpf/progs/test_global_func3.c b/tools/testing/selftests/bpf/progs/test_global_func3.c index 142b682d3c2f0..974fd8c195618 100644 --- a/tools/testing/selftests/bpf/progs/test_global_func3.c +++ b/tools/testing/selftests/bpf/progs/test_global_func3.c @@ -5,56 +5,56 @@ #include #include "bpf_misc.h" -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f1(struct __sk_buff *skb) { return skb->len; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f2(int val, struct __sk_buff *skb) { return f1(skb) + val; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f3(int val, struct __sk_buff *skb, int var) { return f2(var, skb) + val; } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f4(struct __sk_buff *skb) { return f3(1, skb, 2); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f5(struct __sk_buff *skb) { return f4(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f6(struct __sk_buff *skb) { return f5(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f7(struct __sk_buff *skb) { return f6(skb); } -__attribute__ ((noinline)) +static __attribute__ ((noinline)) int f8(struct __sk_buff *skb) { return f7(skb); } SEC("tc") -__failure __msg("the call stack of 8 frames") +__failure __msg("the call stack of 9 frames") int global_func3(struct __sk_buff *skb) { return f8(skb); diff --git a/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c b/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c index d6cb986e7533f..4a934fccf8f5b 100644 --- a/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c +++ b/tools/testing/selftests/bpf/progs/test_lwt_ip_encap.c @@ -1,11 +1,9 @@ // SPDX-License-Identifier: GPL-2.0 -#include +#include "vmlinux.h" #include -#include -#include -#include #include #include +#include struct grehdr { __be16 flags; @@ -64,13 +62,13 @@ int bpf_lwt_encap_gre6(struct __sk_buff *skb) hdr.ip6hdr.nexthdr = 47; /* IPPROTO_GRE */ hdr.ip6hdr.hop_limit = 0x40; /* fb01::1 */ - hdr.ip6hdr.saddr.s6_addr[0] = 0xfb; - hdr.ip6hdr.saddr.s6_addr[1] = 1; - hdr.ip6hdr.saddr.s6_addr[15] = 1; + hdr.ip6hdr.saddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.saddr.in6_u.u6_addr8[1] = 1; + hdr.ip6hdr.saddr.in6_u.u6_addr8[15] = 1; /* fb10::1 */ - hdr.ip6hdr.daddr.s6_addr[0] = 0xfb; - hdr.ip6hdr.daddr.s6_addr[1] = 0x10; - hdr.ip6hdr.daddr.s6_addr[15] = 1; + hdr.ip6hdr.daddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.daddr.in6_u.u6_addr8[1] = 0x10; + hdr.ip6hdr.daddr.in6_u.u6_addr8[15] = 1; hdr.greh.protocol = skb->protocol; @@ -82,4 +80,141 @@ int bpf_lwt_encap_gre6(struct __sk_buff *skb) return BPF_LWT_REROUTE; } +#define VXLAN_PORT 4789 +#define VXLAN_FLAGS 0x08000000 +#define VXLAN_VNI 1 + +#define ETH_ALEN 6 /* Octets in one ethernet addr */ +#define ETH_P_IP 0x0800 /* Internet Protocol packet */ +#define ETH_P_IPV6 0x86DD /* IPv6 over bluebook */ + +static const __u8 bcast[ETH_ALEN] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +static const __u8 srcmac[ETH_ALEN] = { + 0x02, 0x00, 0x00, 0x00, 0x00, 0x01, +}; + +SEC("encap_vxlan") +int bpf_lwt_encap_vxlan(struct __sk_buff *skb) +{ + struct encap_hdr { + struct iphdr iph; + struct udphdr udph; + struct vxlanhdr vxh; + struct ethhdr eth; + } __attribute__((__packed__)) hdr; + int err; + + memset(&hdr, 0, sizeof(hdr)); + + hdr.iph.ihl = 5; + hdr.iph.version = 4; + hdr.iph.ttl = 0x40; + hdr.iph.protocol = 17; /* IPPROTO_UDP */ + hdr.iph.tot_len = bpf_htons(skb->len + sizeof(hdr)); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + hdr.iph.saddr = 0x640510ac; /* 172.16.5.100 */ + hdr.iph.daddr = 0x641110ac; /* 172.16.17.100 */ +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + hdr.iph.saddr = 0xac100564; /* 172.16.5.100 */ + hdr.iph.daddr = 0xac101164; /* 172.16.17.100 */ +#else +#error "Fix your compiler's __BYTE_ORDER__?!" +#endif + + hdr.udph.source = bpf_htons(VXLAN_PORT); + hdr.udph.dest = bpf_htons(VXLAN_PORT); + hdr.udph.len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + + hdr.vxh.vx_flags = bpf_htonl(VXLAN_FLAGS); + hdr.vxh.vx_vni = bpf_htonl(VXLAN_VNI << 8); + + __builtin_memcpy(hdr.eth.h_dest, bcast, ETH_ALEN); + __builtin_memcpy(hdr.eth.h_source, srcmac, ETH_ALEN); + hdr.eth.h_proto = bpf_htons(ETH_P_IP); + + err = bpf_lwt_push_encap(skb, BPF_LWT_ENCAP_IP, &hdr, sizeof(hdr)); + if (err) + return BPF_DROP; + + return BPF_LWT_REROUTE; +} + +SEC("encap_vxlan6") +int bpf_lwt_encap_vxlan6(struct __sk_buff *skb) +{ + struct encap_hdr { + struct ipv6hdr ip6hdr; + struct udphdr udph; + struct vxlanhdr vxh; + struct ethhdr eth; + } __attribute__((__packed__)) hdr; + int err; + + memset(&hdr, 0, sizeof(hdr)); + + hdr.ip6hdr.version = 6; + hdr.ip6hdr.nexthdr = 17; /* IPPROTO_UDP */ + hdr.ip6hdr.hop_limit = 0x40; + hdr.ip6hdr.payload_len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + /* fb05::1 */ + hdr.ip6hdr.saddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.saddr.in6_u.u6_addr8[1] = 0x05; + hdr.ip6hdr.saddr.in6_u.u6_addr8[15] = 1; + /* fb11::1 */ + hdr.ip6hdr.daddr.in6_u.u6_addr8[0] = 0xfb; + hdr.ip6hdr.daddr.in6_u.u6_addr8[1] = 0x11; + hdr.ip6hdr.daddr.in6_u.u6_addr8[15] = 1; + + hdr.udph.source = bpf_htons(VXLAN_PORT); + hdr.udph.dest = bpf_htons(VXLAN_PORT); + hdr.udph.len = bpf_htons(skb->len + sizeof(hdr.udph) + sizeof(hdr.vxh) + + sizeof(hdr.eth)); + + hdr.vxh.vx_flags = bpf_htonl(VXLAN_FLAGS); + hdr.vxh.vx_vni = bpf_htonl(VXLAN_VNI << 8); + + __builtin_memcpy(hdr.eth.h_dest, bcast, ETH_ALEN); + __builtin_memcpy(hdr.eth.h_source, srcmac, ETH_ALEN); + hdr.eth.h_proto = bpf_htons(ETH_P_IPV6); + + err = bpf_lwt_push_encap(skb, BPF_LWT_ENCAP_IP, &hdr, sizeof(hdr)); + if (err) + return BPF_DROP; + + return BPF_LWT_REROUTE; +} + +volatile const int tgt_ip_version; + +__u16 transport_hdr = 0; +__u16 network_hdr = 0; +bool fexit_triggered = false; + +SEC("?fexit/bpf_lwt_push_ip_encap") +int BPF_PROG(fexit_lwt_push_ip_encap, struct sk_buff *skb, void *hdr, u32 len, bool ingress, + int retval) +{ + struct iphdr *iph; + + if (retval || fexit_triggered) + return 0; + + iph = (typeof(iph)) (skb->head + skb->network_header); + if (iph->version != tgt_ip_version) + return 0; + + if ((iph->version == 4 && iph->protocol == 17 /* IPPROTO_UDP */) || + (iph->version == 6 && ((struct ipv6hdr *)iph)->nexthdr == 17 /* IPPROTO_UDP */)) { + fexit_triggered = true; + transport_hdr = skb->transport_header; + network_hdr = skb->network_header; + } + return 0; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c index c925ba9a2e74c..09e81b99eecb4 100644 --- a/tools/testing/selftests/bpf/progs/verifier_ld_ind.c +++ b/tools/testing/selftests/bpf/progs/verifier_ld_ind.c @@ -107,4 +107,146 @@ __naked void ind_check_calling_conv_r7(void) : __clobber_all); } +/* + * ld_{abs,ind} subprog that always sets r0=1 on the success path. + * bpf_gen_ld_abs() emits a hidden exit with r0=0 when the load helper + * fails. The verifier must model this failure return so that callers + * account for r0=0 as a possible return value. + */ +__naked __noinline __used +static int ldabs_subprog(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[ld_abs];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +__naked __noinline __used +static int ldind_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: subprog early exit on ld_abs failure") +__failure __msg("R9 !read_ok") +__naked void ld_abs_subprog_early_exit(void) +{ + asm volatile ( + "call ldabs_subprog;" + "if r0 != 0 goto l_exit_%=;" + "r0 = r9;" + "l_exit_%=:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: subprog early exit on ld_ind failure") +__failure __msg("R9 !read_ok") +__naked void ld_ind_subprog_early_exit(void) +{ + asm volatile ( + "call ldind_subprog;" + "if r0 != 0 goto l_exit_%=;" + "r0 = r9;" + "l_exit_%=:" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_abs: subprog with both paths safe") +__success +__naked void ld_abs_subprog_both_paths_safe(void) +{ + asm volatile ( + "call ldabs_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +__description("ld_ind: subprog with both paths safe") +__success +__naked void ld_ind_subprog_both_paths_safe(void) +{ + asm volatile ( + "call ldind_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * ld_{abs,ind} in subprogs require scalar (int) return type in BTF. + * A test with void return must be rejected. + */ +__naked __noinline __used +static void ldabs_void_subprog(void) +{ + asm volatile ( + "r6 = r1;" + ".8byte %[ld_abs];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_abs, BPF_LD_ABS(BPF_W, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_abs: reject void return subprog") +__failure __msg("LD_ABS is only allowed in functions that return 'int'") +__naked void ld_abs_void_subprog_reject(void) +{ + asm volatile ( + "call ldabs_void_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +__naked __noinline __used +static void ldind_void_subprog(void) +{ + asm volatile ( + "r6 = r1;" + "r7 = 0;" + ".8byte %[ld_ind];" + "r0 = 1;" + "exit;" + : + : __imm_insn(ld_ind, BPF_LD_IND(BPF_W, BPF_REG_7, 0)) + : __clobber_all); +} + +SEC("socket") +__description("ld_ind: reject void return subprog") +__failure __msg("LD_ABS is only allowed in functions that return 'int'") +__naked void ld_ind_void_subprog_reject(void) +{ + asm volatile ( + "call ldind_void_subprog;" + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c index e2767d27d8aaf..1661936598703 100644 --- a/tools/testing/selftests/bpf/progs/verifier_map_ptr.c +++ b/tools/testing/selftests/bpf/progs/verifier_map_ptr.c @@ -70,17 +70,45 @@ __naked void bpf_map_ptr_write_rejected(void) : __clobber_all); } -/* The first element of struct bpf_map is a SHA256 hash of 32 bytes, accessing - * into this array is valid. The opts field is now at offset 33. +/* + * struct bpf_map starts with the SHA256 hash sha[32] at offset 0 (a readable + * byte array), the u32 excl field at offset 32, and the ops pointer at offset + * 40. Reading a u32 at offset 41 reaches into the middle of the ops pointer, + * i.e. a partial pointer access, which is rejected. */ SEC("socket") __description("bpf_map_ptr: read non-existent field rejected") __failure -__msg("cannot access ptr member ops with moff 32 in struct bpf_map with off 33 size 4") +__msg("cannot access ptr member ops with moff 40 in struct bpf_map with off 41 size 4") __failure_unpriv __msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") __flag(BPF_F_ANY_ALIGNMENT) __naked void read_non_existent_field_rejected(void) +{ + asm volatile (" \ + r6 = 0; \ + r1 = %[map_array_48b] ll; \ + r6 = *(u32*)(r1 + 41); \ + r0 = 1; \ + exit; \ +" : + : __imm_addr(map_array_48b) + : __clobber_all); +} + +/* + * The u32 excl field spans offsets 32..35 (mend 36). Reading a u32 at offset + * 33 starts inside excl but extends past its end, which the verifier rejects + * as an out-of-bounds scalar access. + */ +SEC("socket") +__description("bpf_map_ptr: read beyond excl field rejected") +__failure +__msg("access beyond the end of member excl (mend:36) in struct bpf_map with off 33 size 4") +__failure_unpriv +__msg_unpriv("access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN") +__flag(BPF_F_ANY_ALIGNMENT) +__naked void read_beyond_excl_field_rejected(void) { asm volatile (" \ r6 = 0; \ @@ -103,7 +131,7 @@ __naked void ptr_read_ops_field_accepted(void) asm volatile (" \ r6 = 0; \ r1 = %[map_array_48b] ll; \ - r6 = *(u64*)(r1 + 0); \ + r6 = *(u64*)(r1 + 40); \ r0 = 1; \ exit; \ " : diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index 3fae9ce46ca9b..cea33b94d06f8 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -748,16 +748,15 @@ static void test_sockmap(unsigned int tasks, void *data) goto out_sockmap; } - /* Test update with unsupported UDP socket */ + /* Test update with unsupported unbound UDP socket */ udp = socket(AF_INET, SOCK_DGRAM, 0); - i = 0; - err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY); - if (err) { - printf("Failed socket update SOCK_DGRAM '%i:%i'\n", - i, udp); + CHECK(udp < 0, "socket(AF_INET, SOCK_DGRAM)", "errno:%d\n", errno); + err = bpf_map_update_elem(fd, &(int){0}, &udp, BPF_ANY); + close(udp); + if (!err) { + printf("Unexpectedly succeeded unbound UDP update '0:%i'\n", udp); goto out_sockmap; } - close(udp); /* Test update without programs */ for (i = 0; i < 6; i++) { diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c index 02a85dda30e64..eda3fcf72095f 100644 --- a/tools/testing/selftests/bpf/test_progs.c +++ b/tools/testing/selftests/bpf/test_progs.c @@ -550,18 +550,19 @@ bool test__start_subtest(const char *subtest_name) struct test_state *state = env.test_state; struct subtest_state *subtest_state; size_t sub_state_size = sizeof(*subtest_state); + void *tmp; if (env.subtest_state) test__end_subtest(); state->subtest_num++; - state->subtest_states = - realloc(state->subtest_states, - state->subtest_num * sub_state_size); - if (!state->subtest_states) { + tmp = realloc(state->subtest_states, state->subtest_num * sub_state_size); + if (!tmp) { + state->subtest_num--; fprintf(stderr, "Not enough memory to allocate subtest result\n"); return false; } + state->subtest_states = tmp; subtest_state = &state->subtest_states[state->subtest_num - 1]; @@ -1690,7 +1691,7 @@ static void server_main(void) data[i].worker_id = i; data[i].sock_fd = env.worker_socks[i]; rc = pthread_create(&dispatcher_threads[i], NULL, dispatch_thread, &data[i]); - if (rc < 0) { + if (rc) { perror("Failed to launch dispatcher thread"); exit(EXIT_ERR_SETUP_INFRA); } diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c index 76568db7a6642..f8b57f3c3f236 100644 --- a/tools/testing/selftests/bpf/test_sockmap.c +++ b/tools/testing/selftests/bpf/test_sockmap.c @@ -514,7 +514,8 @@ static int msg_alloc_iov(struct msghdr *msg, return 0; unwind_iov: for (i--; i >= 0 ; i--) - free(msg->msg_iov[i].iov_base); + free(iov[i].iov_base); + free(iov); return -ENOMEM; } diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c index 1be1e353d40a7..df4aea8bbe82c 100644 --- a/tools/testing/selftests/bpf/veristat.c +++ b/tools/testing/selftests/bpf/veristat.c @@ -513,6 +513,40 @@ static bool is_bpf_obj_file(const char *path) { return err == 0; } +/* Exact filter match */ +static bool name_filter_matches(struct filter *f, const char *filename, const char *prog_name) +{ + if (f->any_glob) + return glob_matches(filename, f->any_glob) || + (prog_name && glob_matches(prog_name, f->any_glob)); + if (f->file_glob && f->prog_glob) + return prog_name && + glob_matches(filename, f->file_glob) && + glob_matches(prog_name, f->prog_glob); + if (f->file_glob) + return glob_matches(filename, f->file_glob); + if (f->prog_glob) + return prog_name && glob_matches(prog_name, f->prog_glob); + return false; +} + +/* Check if the filter does not outright reject the file name */ +static bool name_filter_may_match(struct filter *f, const char *filename) +{ + if (f->file_glob) + return glob_matches(filename, f->file_glob); + /* + * If we don't know program name yet, any_glob filter + * has to assume that current BPF object file might be + * relevant; we'll check again later on after opening + * BPF object file, at which point program name will + * be known finally. + */ + if (f->any_glob || f->prog_glob) + return true; + return false; +} + static bool should_process_file_prog(const char *filename, const char *prog_name) { struct filter *f; @@ -520,16 +554,7 @@ static bool should_process_file_prog(const char *filename, const char *prog_name for (i = 0; i < env.deny_filter_cnt; i++) { f = &env.deny_filters[i]; - if (f->kind != FILTER_NAME) - continue; - - if (f->any_glob && glob_matches(filename, f->any_glob)) - return false; - if (f->any_glob && prog_name && glob_matches(prog_name, f->any_glob)) - return false; - if (f->file_glob && glob_matches(filename, f->file_glob)) - return false; - if (f->prog_glob && prog_name && glob_matches(prog_name, f->prog_glob)) + if (f->kind == FILTER_NAME && name_filter_matches(f, filename, prog_name)) return false; } @@ -539,24 +564,15 @@ static bool should_process_file_prog(const char *filename, const char *prog_name continue; allow_cnt++; - if (f->any_glob) { - if (glob_matches(filename, f->any_glob)) - return true; - /* If we don't know program name yet, any_glob filter - * has to assume that current BPF object file might be - * relevant; we'll check again later on after opening - * BPF object file, at which point program name will - * be known finally. - */ - if (!prog_name || glob_matches(prog_name, f->any_glob)) - return true; - } else { - if (f->file_glob && !glob_matches(filename, f->file_glob)) - continue; - if (f->prog_glob && prog_name && !glob_matches(prog_name, f->prog_glob)) - continue; + if (prog_name && name_filter_matches(f, filename, prog_name)) + return true; + /* + * If there is no prog_name and the file name is not blocked by + * the filter, allow to open the file. Afterwards there would be + * a second refining query with prog_name set. + */ + if (!prog_name && name_filter_may_match(f, filename)) return true; - } } /* if there are no file/prog name allow filters, allow all progs, @@ -702,6 +718,12 @@ static int append_filter(struct filter **filters, int *cnt, const char *str) } } + if ((!f->any_glob && !f->file_glob && !f->prog_glob) || + (f->any_glob && strcmp(f->any_glob, "") == 0)) { + fprintf(stderr, "Invalid filter: '%s'\n", str); + return -EINVAL; + } + *cnt += 1; return 0; } diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh index 9ca8022853933..6a3d026d76bd6 100755 --- a/tools/testing/selftests/bpf/vmtest.sh +++ b/tools/testing/selftests/bpf/vmtest.sh @@ -428,8 +428,17 @@ main() if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" - else - command="$@" + elif [[ $# -gt 0 ]]; then + # Quote each argument so the command survives into the guest: the + # host expands ${command} into the generated init script, which + # the guest bash then parses as shell source. Without the %q + # escapes an argument with a space or a glob (e.g. -a 'verifier_*') + # is re-split and expanded against /root/bpf there. + # + # Skip this when there is no command: printf '%q ' would still + # apply the format once and emit '', which is not the empty + # command that -s (debug shell) expects. + command=$(printf '%q ' "$@") fi local kconfig_file="${OUTPUT_DIR}/latest.config" diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index a17256d9f88a8..592e533cd4d7b 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -488,6 +488,7 @@ write_cpu_online() CPU=${1%=*} VAL=${1#*=} CPUFILE=//sys/devices/system/cpu/cpu${CPU}/online + echo $VAL > $CPUFILE || return 1 if [[ $VAL -eq 0 ]] then OFFLINE_CPUS="$OFFLINE_CPUS $CPU" @@ -497,7 +498,6 @@ write_cpu_online() sort | uniq -u) } fi - echo $VAL > $CPUFILE pause 0.05 } @@ -565,7 +565,8 @@ set_ctrl_state() eval $COMM $REDIRECT ;; O*) VAL=${CMD#?} - write_cpu_online $VAL + COMM="write_cpu_online $VAL" + eval $COMM $REDIRECT ;; T*) COMM="echo 0 > $TFILE" eval $COMM $REDIRECT diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c index 63b3c9aad399a..b362eb9f18409 100644 --- a/tools/testing/selftests/cgroup/test_kmem.c +++ b/tools/testing/selftests/cgroup/test_kmem.c @@ -144,7 +144,7 @@ static int cg_run_in_subcgroups(const char *parent, return -1; } - if (cg_run(child, fn, NULL)) { + if (cg_run(child, fn, arg)) { cg_destroy(child); free(child); return -1; diff --git a/tools/testing/selftests/clone3/clone3_set_tid.c b/tools/testing/selftests/clone3/clone3_set_tid.c index bfb0da2b4fdd1..4debe0756b671 100644 --- a/tools/testing/selftests/clone3/clone3_set_tid.c +++ b/tools/testing/selftests/clone3/clone3_set_tid.c @@ -141,7 +141,7 @@ int main(int argc, char *argv[]) { FILE *f; char buf; - char *line; + char *line = NULL; int status; int ret = -1; size_t len = 0; diff --git a/tools/testing/selftests/drivers/net/settings b/tools/testing/selftests/drivers/net/settings new file mode 100644 index 0000000000000..eef533824a3c1 --- /dev/null +++ b/tools/testing/selftests/drivers/net/settings @@ -0,0 +1 @@ +timeout=360 diff --git a/tools/testing/selftests/drivers/ntsync/config b/tools/testing/selftests/drivers/ntsync/config index 60539c826d062..0aa68de147af1 100644 --- a/tools/testing/selftests/drivers/ntsync/config +++ b/tools/testing/selftests/drivers/ntsync/config @@ -1 +1 @@ -CONFIG_WINESYNC=y +CONFIG_NTSYNC=y diff --git a/tools/testing/selftests/filesystems/fclog.c b/tools/testing/selftests/filesystems/fclog.c index 912a8b755c3b6..3e6fb609ef56a 100644 --- a/tools/testing/selftests/filesystems/fclog.c +++ b/tools/testing/selftests/filesystems/fclog.c @@ -6,10 +6,8 @@ #include #include +#include #include -#include -#include -#include #include #include diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_eprobe.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_eprobe.tc index c300eb0202620..e2322693d0c32 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_eprobe.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_eprobe.tc @@ -1,16 +1,16 @@ #!/bin/sh # SPDX-License-Identifier: GPL-2.0 # description: Generic dynamic event - add/remove eprobe events -# requires: dynamic_events events/syscalls/sys_enter_openat ". []":README +# requires: dynamic_events events/syscalls/sys_enter_chdir ". []":README echo 0 > events/enable clear_dynamic_events SYSTEM="syscalls" -EVENT="sys_enter_openat" +EVENT="sys_enter_chdir" FIELD="filename" -EPROBE="eprobe_open" +EPROBE="eprobe_chdir" OPTIONS="file=+0(\$filename):ustring" echo "e:$EPROBE $SYSTEM/$EVENT $OPTIONS" >> dynamic_events @@ -18,20 +18,14 @@ grep -q "$EPROBE" dynamic_events test -d events/eprobes/$EPROBE echo 1 > events/eprobes/$EPROBE/enable -ls +cd /sys/kernel/tracing echo 0 > events/eprobes/$EPROBE/enable -content=`grep '^ *ls-' trace | grep 'file='` -nocontent=`grep '^ *ls-' trace | grep 'file=' | grep -v -e '"/' -e '"."' -e '(fault)' ` || true - +content=`grep -e 'file="/sys/kernel/tracing"\|(fault)' trace` if [ -z "$content" ]; then exit_fail fi -if [ ! -z "$nocontent" ]; then - exit_fail -fi - echo "-:$EPROBE" >> dynamic_events ! grep -q "$EPROBE" dynamic_events diff --git a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_uprobe.tc b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_uprobe.tc index f2048c2445261..19430bd5864c1 100644 --- a/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_uprobe.tc +++ b/tools/testing/selftests/ftrace/test.d/dynevent/add_remove_uprobe.tc @@ -12,9 +12,32 @@ echo 0 > events/enable echo > dynamic_events REALBIN=`readlink -f /bin/sh` -ENTRYPOINT=`readelf -h ${REALBIN} | grep Entry | sed -e 's/[^0]*//'` -echo "p:myevent ${REALBIN}:${ENTRYPOINT}" >> uprobe_events +# Get the entry point virtual address from ELF header +ENTRY=`readelf -hW ${REALBIN} | grep "Entry point" | awk '{print $NF}'` + +# Convert virtual address to file offset: find the LOAD segment containing +# the entry point, then compute file_offset = e_entry - p_vaddr + p_offset. +# For PIE binaries this is a no-op (vaddr == file offset), but for non-PIE +# executables the virtual address is much larger than the file size and +# must be converted, otherwise uprobe_register() rejects it with -EINVAL. +ENTRY_DEC=$(printf '%d' "$ENTRY") +OFFSET=$ENTRY +while IFS= read -r line; do + set -- $line + [ "$1" = "LOAD" ] || continue + VA_DEC=$(printf '%d' "$3") + OFF_DEC=$(printf '%d' "$2") + FSZ_DEC=$(printf '%d' "$5") + if [ "$ENTRY_DEC" -ge "$VA_DEC" ] && [ "$ENTRY_DEC" -lt "$((VA_DEC + FSZ_DEC))" ]; then + OFFSET=$(printf '0x%x' "$((ENTRY_DEC - VA_DEC + OFF_DEC))") + break + fi +done << EOF +$(readelf -lW ${REALBIN} | grep LOAD) +EOF + +echo "p:myevent ${REALBIN}:${OFFSET}" >> uprobe_events grep -q myevent uprobe_events test -d events/uprobes/myevent diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 6928915a643b1..be7f0ecab7704 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -76,7 +76,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) memset(s, c, n); } -#define KSELFTEST_PRIO_TEST_F 20000 +#define KSELFTEST_PRIO_TEST 20000 #define KSELFTEST_PRIO_XFAIL 20001 #define TEST_TIMEOUT_DEFAULT 30 @@ -194,7 +194,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) .fixture = &_fixture_global, \ .termsig = _signal, \ .timeout = TEST_TIMEOUT_DEFAULT, }; \ - static void __attribute__((constructor)) _register_##test_name(void) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) _register_##test_name(void) \ { \ __register_test(&_##test_name##_object); \ } \ @@ -238,7 +238,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) FIXTURE_VARIANT(fixture_name); \ static struct __fixture_metadata _##fixture_name##_fixture_object = \ { .name = #fixture_name, }; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \ _register_##fixture_name##_data(void) \ { \ __register_fixture(&_##fixture_name##_fixture_object); \ @@ -364,7 +364,7 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) _##fixture_name##_##variant_name##_object = \ { .name = #variant_name, \ .data = &_##fixture_name##_##variant_name##_variant}; \ - static void __attribute__((constructor)) \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \ _register_##fixture_name##_##variant_name(void) \ { \ __register_fixture_variant(&_##fixture_name##_fixture_object, \ @@ -467,8 +467,8 @@ static inline void __kselftest_memset_safe(void *s, int c, size_t n) !__atomic_test_and_set(_metadata->no_teardown, __ATOMIC_RELAXED)) \ fixture_name##_teardown(_metadata, self, variant); \ } \ - static struct __test_metadata *_##fixture_name##_##test_name##_object; \ - static void __attribute__((constructor(KSELFTEST_PRIO_TEST_F))) \ + static struct __test_metadata *_##fixture_name##_##test_name##_object __maybe_unused; \ + static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) \ _register_##fixture_name##_##test_name(void) \ { \ struct __test_metadata *object = mmap(NULL, sizeof(*object), \ @@ -1315,7 +1315,7 @@ static int test_harness_run(int argc, char **argv) return KSFT_FAIL; } -static void __attribute__((constructor)) __constructor_order_first(void) +static void __attribute__((constructor(KSELFTEST_PRIO_TEST))) __constructor_order_first(void) { __constructor_order_forward = true; } diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt index cff124c1eddd3..665a13a37363c 100644 --- a/tools/testing/selftests/lkdtm/tests.txt +++ b/tools/testing/selftests/lkdtm/tests.txt @@ -74,7 +74,7 @@ USERCOPY_STACK_FRAME_TO USERCOPY_STACK_FRAME_FROM USERCOPY_STACK_BEYOND USERCOPY_KERNEL -STACKLEAK_ERASING OK: the rest of the thread stack is properly erased +KSTACK_ERASE OK: the rest of the thread stack is properly erased CFI_FORWARD_PROTO CFI_BACKWARD call trace:|ok: control flow unchanged FORTIFY_STRSCPY detected buffer overflow diff --git a/tools/testing/selftests/lsm/common.c b/tools/testing/selftests/lsm/common.c index 9ad258912646c..927dce4f04cb2 100644 --- a/tools/testing/selftests/lsm/common.c +++ b/tools/testing/selftests/lsm/common.c @@ -76,7 +76,7 @@ int attr_lsm_count(void) return 0; if (read_sysfs_lsms(names, sysconf(_SC_PAGESIZE))) - return 0; + goto out; if (strstr(names, "selinux")) count++; @@ -85,5 +85,7 @@ int attr_lsm_count(void) if (strstr(names, "apparmor")) count++; +out: + free(names); return count; } diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 0df61422467da..af8db738423d4 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -1699,8 +1699,13 @@ static void run_with_tmpfile(non_anon_test_fn fn, const char *desc) /* File consists of a single page filled with zeroes. */ if (fallocate(fd, 0, 0, pagesize)) { - ksft_perror("fallocate() failed"); - log_test_result(KSFT_FAIL); + if (errno == EOPNOTSUPP) { + ksft_print_msg("fallocate() not supported by filesystem\n"); + log_test_result(KSFT_SKIP); + } else { + ksft_perror("fallocate() failed"); + log_test_result(KSFT_FAIL); + } goto close; } diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index b77462b5c240b..e763295b5b2ae 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -175,12 +175,12 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_ void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); if (!map_ptr) { - perror("mmap"); + ksft_perror("mmap"); return NULL; } memset(map_ptr, data, map_size); if (mprotect(map_ptr, map_size, prot)) { - perror("mprotect"); + ksft_perror("mprotect"); munmap(map_ptr, map_size); return NULL; } @@ -201,11 +201,11 @@ static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) return 1; if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return 1; } if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { - printf("Scan time limit exceeded\n"); + ksft_print_msg("Scan time limit exceeded\n"); return 1; } } @@ -218,12 +218,12 @@ static int ksm_merge_pages(int merge_type, void *addr, size_t size, { if (merge_type == KSM_MERGE_MADVISE) { if (madvise(addr, size, MADV_MERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } } else if (merge_type == KSM_MERGE_PRCTL) { if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { - perror("prctl"); + ksft_perror("prctl"); return 1; } } @@ -242,7 +242,7 @@ static int ksm_unmerge_pages(void *addr, size_t size, struct timespec start_time, int timeout) { if (madvise(addr, size, MADV_UNMERGEABLE)) { - perror("madvise"); + ksft_perror("madvise"); return 1; } return 0; @@ -288,8 +288,8 @@ static bool assert_ksm_pages_count(long dupl_page_count) static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) { if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes)) || ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || @@ -304,8 +304,8 @@ static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) static int ksm_restore(struct ksm_sysfs *ksm_sysfs) { if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || @@ -324,7 +324,7 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -338,7 +338,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, /* verify that the right number of pages are merged */ if (assert_ksm_pages_count(page_count)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); if (merge_type == KSM_MERGE_PRCTL) prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); @@ -346,7 +345,6 @@ static int check_ksm_merge(int merge_type, int mapping, int prot, } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -358,7 +356,7 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, int page_count = 2; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -380,13 +378,11 @@ static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, /* check that unmerging was successful and 0 pages are currently merged */ if (assert_ksm_pages_count(0)) { - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; } err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -398,7 +394,7 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long struct timespec start_time; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } @@ -425,12 +421,10 @@ static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) goto err_out; - printf("OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -446,9 +440,9 @@ static int get_next_mem_node(int node) mem_node = i % (max_node + 1); node_size = numa_node_size(mem_node, NULL); if (node_size > 0) - break; + return mem_node; } - return mem_node; + return -ENODEV; } static int get_first_mem_node(void) @@ -461,31 +455,33 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo { void *numa1_map_ptr, *numa2_map_ptr; struct timespec start_time; + int first_node, second_node; int page_count = 2; - int first_node; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } if (numa_available() < 0) { - perror("NUMA support not enabled"); + ksft_print_msg("NUMA support not enabled\n"); return KSFT_SKIP; } - if (numa_num_configured_nodes() <= 1) { - printf("At least 2 NUMA nodes must be available\n"); + first_node = get_first_mem_node(); + second_node = get_next_mem_node(first_node); + + if (second_node < 0) { + ksft_print_msg("At least 2 NUMA nodes with memory must be available\n"); return KSFT_SKIP; } if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) return KSFT_FAIL; /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ - first_node = get_first_mem_node(); numa1_map_ptr = numa_alloc_onnode(page_size, first_node); - numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); + numa2_map_ptr = numa_alloc_onnode(page_size, second_node); if (!numa1_map_ptr || !numa2_map_ptr) { - perror("numa_alloc_onnode"); + ksft_perror("numa_alloc_onnode"); return KSFT_FAIL; } @@ -510,13 +506,11 @@ static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeo numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("OK\n"); return KSFT_PASS; err_out: numa_free(numa1_map_ptr, page_size); numa_free(numa2_map_ptr, page_size); - printf("Not OK\n"); return KSFT_FAIL; } @@ -529,7 +523,7 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, int pagemap_fd, n_normal_pages, n_huge_pages; if (!thp_is_enabled()) { - printf("Transparent Hugepages not available\n"); + ksft_print_msg("Transparent Hugepages not available\n"); return KSFT_SKIP; } @@ -559,36 +553,35 @@ static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, else n_huge_pages++; } - printf("Number of normal pages: %d\n", n_normal_pages); - printf("Number of huge pages: %d\n", n_huge_pages); + ksft_print_msg("Number of normal pages: %d\n", n_normal_pages); + ksft_print_msg("Number of huge pages: %d\n", n_huge_pages); memset(map_ptr, '*', len); if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr_orig, len + HPAGE_SIZE); return KSFT_FAIL; } @@ -606,30 +599,29 @@ static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, si return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -646,37 +638,36 @@ static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, if (!map_ptr) return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n", map_size / MB); - printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n", map_size / MB); + ksft_print_msg("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, scan_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", (map_size / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", (map_size / MB) / ((double)scan_time_ns / NSEC_PER_SEC)); munmap(map_ptr, map_size); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, map_size); return KSFT_FAIL; } @@ -695,24 +686,24 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size return KSFT_FAIL; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); return KSFT_FAIL; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); - printf("Not merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Total size: %lu MiB\n\n", (page_size * page_count) / MB); + ksft_print_msg("Not merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); /* Create 2000 pairs of duplicate pages */ @@ -724,30 +715,29 @@ static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size goto err_out; if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } for (size_t i = 0; i < page_count - 1; i = i + 2) memset(map_ptr + page_size * i, '-', 1); if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { - perror("clock_gettime"); + ksft_perror("clock_gettime"); goto err_out; } cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + (end_time.tv_nsec - start_time.tv_nsec); - printf("Merged pages:\n"); - printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, + ksft_print_msg("Merged pages:\n"); + ksft_print_msg("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, cow_time_ns % NSEC_PER_SEC); - printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / + ksft_print_msg("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / ((double)cow_time_ns / NSEC_PER_SEC)); munmap(map_ptr, page_size * page_count); return KSFT_PASS; err_out: - printf("Not OK\n"); munmap(map_ptr, page_size * page_count); return KSFT_FAIL; } @@ -765,6 +755,10 @@ int main(int argc, char *argv[]) bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; long size_MB = 0; + const char *test_descr = "KSM merging"; + + ksft_print_header(); + ksft_set_plan(1); while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { switch (opt) { @@ -773,17 +767,13 @@ int main(int argc, char *argv[]) break; case 'p': page_count = atol(optarg); - if (page_count <= 0) { - printf("The number of pages must be greater than 0\n"); - return KSFT_FAIL; - } + if (page_count <= 0) + ksft_exit_fail_msg("The number of pages must be greater than 0\n"); break; case 'l': ksm_scan_limit_sec = atoi(optarg); - if (ksm_scan_limit_sec <= 0) { - printf("Timeout value must be greater than 0\n"); - return KSFT_FAIL; - } + if (ksm_scan_limit_sec <= 0) + ksft_exit_fail_msg("Timeout value must be greater than 0\n"); break; case 'h': print_help(); @@ -805,19 +795,15 @@ int main(int argc, char *argv[]) break; case 's': size_MB = atoi(optarg); - if (size_MB <= 0) { - printf("Size must be greater than 0\n"); - return KSFT_FAIL; - } + if (size_MB <= 0) + ksft_exit_fail_msg("Size must be greater than 0\n"); break; case 't': { int tmp = atoi(optarg); - if (tmp < 0 || tmp > KSM_MERGE_LAST) { - printf("Invalid merge type\n"); - return KSFT_FAIL; - } + if (tmp < 0 || tmp > KSM_MERGE_LAST) + ksft_exit_fail_msg("Invalid merge type\n"); merge_type = tmp; } break; @@ -845,82 +831,80 @@ int main(int argc, char *argv[]) test_name = KSM_COW_TIME; break; default: - return KSFT_FAIL; + ksft_exit_fail_msg("Unknown option\n"); } } if (prot == 0) prot = str_to_prot(KSM_PROT_STR_DEFAULT); - if (access(KSM_SYSFS_PATH, F_OK)) { - printf("Config KSM not enabled\n"); - return KSFT_SKIP; - } + if (access(KSM_SYSFS_PATH, F_OK)) + ksft_exit_skip("Config KSM not enabled\n"); - if (ksm_save_def(&ksm_sysfs_old)) { - printf("Cannot save default tunables\n"); - return KSFT_FAIL; - } + if (ksm_save_def(&ksm_sysfs_old)) + ksft_exit_fail_msg("Cannot save default tunables\n"); if (ksm_write_sysfs(KSM_FP("run"), 2) || ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || - numa_available() ? 0 : - ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || + (numa_available() ? 0 : + ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1)) || ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) - return KSFT_FAIL; + ksft_exit_fail_msg("Cannot set up KSM tunables\n"); switch (test_name) { case CHECK_KSM_MERGE: + test_descr = "KSM merging"; ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_UNMERGE: + test_descr = "KSM unmerging"; ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; case CHECK_KSM_ZERO_PAGE_MERGE: + test_descr = "KSM zero page merging"; ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, ksm_scan_limit_sec, use_zero_pages, page_size); break; case CHECK_KSM_NUMA_MERGE: + test_descr = "KSM NUMA merging"; ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, merge_across_nodes, page_size); break; case KSM_MERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time"; ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_MERGE_TIME_HUGE_PAGES: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM merge time with huge pages"; ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_UNMERGE_TIME: - if (size_MB == 0) { - printf("Option '-s' is required.\n"); - return KSFT_FAIL; - } + if (size_MB == 0) + ksft_exit_fail_msg("Option '-s' is required\n"); + test_descr = "KSM unmerge time"; ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, size_MB); break; case KSM_COW_TIME: + test_descr = "KSM COW time"; ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, ksm_scan_limit_sec, page_size); break; } - if (ksm_restore(&ksm_sysfs_old)) { - printf("Cannot restore default tunables\n"); - return KSFT_FAIL; - } + if (ksm_restore(&ksm_sysfs_old)) + ksft_print_msg("Cannot restore default tunables\n"); + + ksft_test_result_report(ret, "%s\n", test_descr); - return ret; + ksft_finished(); } diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index 9a0597310a765..06cffbc3fd77f 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -57,33 +57,6 @@ static void test_file_apis(int fd) pass("file IO is blocked as expected\n"); } -static void test_mlock_limit(int fd) -{ - size_t len; - char *mem; - - len = mlock_limit_cur; - if (len % page_size != 0) - len = (len/page_size) * page_size; - - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem == MAP_FAILED) { - fail("unable to mmap secret memory\n"); - return; - } - munmap(mem, len); - - len = mlock_limit_max * 2; - mem = mmap(NULL, len, prot, mode, fd, 0); - if (mem != MAP_FAILED) { - fail("unexpected mlock limit violation\n"); - munmap(mem, len); - return; - } - - pass("mlock limit is respected\n"); -} - static void test_vmsplice(int fd, const char *desc) { ssize_t transferred; @@ -297,7 +270,7 @@ static void prepare(void) strerror(errno)); } -#define NUM_TESTS 6 +#define NUM_TESTS 5 int main(int argc, char *argv[]) { @@ -319,7 +292,6 @@ int main(int argc, char *argv[]) if (ftruncate(fd, page_size)) ksft_exit_fail_msg("ftruncate failed: %s\n", strerror(errno)); - test_mlock_limit(fd); test_file_apis(fd); /* * We have to run the first vmsplice test before any secretmem page was diff --git a/tools/testing/selftests/mm/mlock-random-test.c b/tools/testing/selftests/mm/mlock-random-test.c index b8d7e966f44c6..aa720c52c6509 100644 --- a/tools/testing/selftests/mm/mlock-random-test.c +++ b/tools/testing/selftests/mm/mlock-random-test.c @@ -84,7 +84,7 @@ int get_proc_locked_vm_size(void) int get_proc_page_size(unsigned long addr) { FILE *smaps; - char *line; + char *line = NULL; unsigned long mmupage_size = 0; size_t size; diff --git a/tools/testing/selftests/mm/run_vmtests.sh b/tools/testing/selftests/mm/run_vmtests.sh index 1edb2d785d936..0a6bc35190a1a 100755 --- a/tools/testing/selftests/mm/run_vmtests.sh +++ b/tools/testing/selftests/mm/run_vmtests.sh @@ -370,7 +370,7 @@ CATEGORY="compaction" run_test ./compaction_test if command -v sudo &> /dev/null && sudo -u nobody ls ./on-fault-limit >/dev/null; then - CATEGORY="mlock" run_test sudo -u nobody ./on-fault-limit + CATEGORY="mlock" run_test sudo -u nobody sh -c ./on-fault-limit else echo "# SKIP ./on-fault-limit" fi diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore index 8f9850a71f542..e2e2063399675 100644 --- a/tools/testing/selftests/net/.gitignore +++ b/tools/testing/selftests/net/.gitignore @@ -4,7 +4,6 @@ bind_timewait bind_wildcard busy_poller cmsg_sender -diag_uid epoll_busy_poll fin_ack_lat gro @@ -18,7 +17,6 @@ ipv6_flowlabel ipv6_flowlabel_mgr ipv6_fragmentation log.txt -msg_oob msg_zerocopy netlink-dumps nettest @@ -35,9 +33,6 @@ reuseport_bpf_numa reuseport_dualstack rxtimestamp sctp_hello -scm_inq -scm_pidfd -scm_rights sk_bind_sendto_listen sk_connect_zero_addr sk_so_peek_off @@ -45,7 +40,6 @@ skf_net_off socket so_incoming_cpu so_netns_cookie -so_peek_off so_txtime so_rcv_listener stress_reuseport_listen @@ -65,4 +59,3 @@ txtimestamp udpgso udpgso_bench_rx udpgso_bench_tx -unix_connect diff --git a/tools/testing/selftests/net/af_unix/.gitignore b/tools/testing/selftests/net/af_unix/.gitignore new file mode 100644 index 0000000000000..9731766441036 --- /dev/null +++ b/tools/testing/selftests/net/af_unix/.gitignore @@ -0,0 +1,9 @@ +diag_uid +msg_oob +scm_inq +scm_pidfd +scm_rights +so_peek_off +unix_connect +unix_connreset +unix_listen diff --git a/tools/testing/selftests/net/af_unix/Makefile b/tools/testing/selftests/net/af_unix/Makefile index 2889403e35468..57d159803a3ab 100644 --- a/tools/testing/selftests/net/af_unix/Makefile +++ b/tools/testing/selftests/net/af_unix/Makefile @@ -13,6 +13,8 @@ TEST_GEN_PROGS := \ scm_rights \ so_peek_off \ unix_connect \ + unix_connreset \ + unix_listen \ # end of TEST_GEN_PROGS include ../../lib.mk diff --git a/tools/testing/selftests/net/af_unix/config b/tools/testing/selftests/net/af_unix/config index b5429c15a53c7..41dbb03c747eb 100644 --- a/tools/testing/selftests/net/af_unix/config +++ b/tools/testing/selftests/net/af_unix/config @@ -1,3 +1,4 @@ CONFIG_AF_UNIX_OOB=y CONFIG_UNIX=y CONFIG_UNIX_DIAG=m +CONFIG_USER_NS=y diff --git a/tools/testing/selftests/net/af_unix/unix_connreset.c b/tools/testing/selftests/net/af_unix/unix_connreset.c new file mode 100644 index 0000000000000..bffef2b54bfd1 --- /dev/null +++ b/tools/testing/selftests/net/af_unix/unix_connreset.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Selftest for AF_UNIX socket close and ECONNRESET behaviour. + * + * This test verifies: + * 1. SOCK_STREAM returns EOF when the peer closes normally. + * 2. SOCK_STREAM returns ECONNRESET if peer closes with unread data. + * 3. SOCK_SEQPACKET returns EOF when the peer closes normally. + * 4. SOCK_SEQPACKET returns ECONNRESET if the peer closes with unread data. + * 5. SOCK_DGRAM does not return ECONNRESET when the peer closes. + * + * These tests document the intended Linux behaviour. + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "../../kselftest_harness.h" + +#define SOCK_PATH "/tmp/af_unix_connreset.sock" + +static void remove_socket_file(void) +{ + unlink(SOCK_PATH); +} + +FIXTURE(unix_sock) +{ + int server; + int client; + int child; +}; + +FIXTURE_VARIANT(unix_sock) +{ + int socket_type; + const char *name; +}; + +FIXTURE_VARIANT_ADD(unix_sock, stream) { + .socket_type = SOCK_STREAM, + .name = "SOCK_STREAM", +}; + +FIXTURE_VARIANT_ADD(unix_sock, dgram) { + .socket_type = SOCK_DGRAM, + .name = "SOCK_DGRAM", +}; + +FIXTURE_VARIANT_ADD(unix_sock, seqpacket) { + .socket_type = SOCK_SEQPACKET, + .name = "SOCK_SEQPACKET", +}; + +FIXTURE_SETUP(unix_sock) +{ + struct sockaddr_un addr = {}; + int err; + + addr.sun_family = AF_UNIX; + strcpy(addr.sun_path, SOCK_PATH); + remove_socket_file(); + + self->server = socket(AF_UNIX, variant->socket_type, 0); + ASSERT_LT(-1, self->server); + + err = bind(self->server, (struct sockaddr *)&addr, sizeof(addr)); + ASSERT_EQ(0, err); + + if (variant->socket_type == SOCK_STREAM || + variant->socket_type == SOCK_SEQPACKET) { + err = listen(self->server, 1); + ASSERT_EQ(0, err); + } + + self->client = socket(AF_UNIX, variant->socket_type | SOCK_NONBLOCK, 0); + ASSERT_LT(-1, self->client); + + err = connect(self->client, (struct sockaddr *)&addr, sizeof(addr)); + ASSERT_EQ(0, err); +} + +FIXTURE_TEARDOWN(unix_sock) +{ + if (variant->socket_type == SOCK_STREAM || + variant->socket_type == SOCK_SEQPACKET) + close(self->child); + + close(self->client); + close(self->server); + remove_socket_file(); +} + +/* Test 1: peer closes normally */ +TEST_F(unix_sock, eof) +{ + char buf[16] = {}; + ssize_t n; + + if (variant->socket_type == SOCK_STREAM || + variant->socket_type == SOCK_SEQPACKET) { + self->child = accept(self->server, NULL, NULL); + ASSERT_LT(-1, self->child); + + close(self->child); + } else { + close(self->server); + } + + n = recv(self->client, buf, sizeof(buf), 0); + + if (variant->socket_type == SOCK_STREAM || + variant->socket_type == SOCK_SEQPACKET) { + ASSERT_EQ(0, n); + } else { + ASSERT_EQ(-1, n); + ASSERT_EQ(EAGAIN, errno); + } +} + +/* Test 2: peer closes with unread data */ +TEST_F(unix_sock, reset_unread_behavior) +{ + char buf[16] = {}; + ssize_t n; + + /* Send data that will remain unread */ + send(self->client, "hello", 5, 0); + + if (variant->socket_type == SOCK_DGRAM) { + /* No real connection, just close the server */ + close(self->server); + } else { + self->child = accept(self->server, NULL, NULL); + ASSERT_LT(-1, self->child); + + /* Peer closes before client reads */ + close(self->child); + } + + n = recv(self->client, buf, sizeof(buf), 0); + ASSERT_EQ(-1, n); + + if (variant->socket_type == SOCK_STREAM || + variant->socket_type == SOCK_SEQPACKET) { + ASSERT_EQ(ECONNRESET, errno); + } else { + ASSERT_EQ(EAGAIN, errno); + } +} + +/* Test 3: closing unaccepted (embryo) server socket should reset client. */ +TEST_F(unix_sock, reset_closed_embryo) +{ + char buf[16] = {}; + ssize_t n; + + if (variant->socket_type == SOCK_DGRAM) + SKIP(return, "This test only applies to SOCK_STREAM and SOCK_SEQPACKET"); + + /* Close server without accept()ing */ + close(self->server); + + n = recv(self->client, buf, sizeof(buf), 0); + + ASSERT_EQ(-1, n); + ASSERT_EQ(ECONNRESET, errno); +} + +TEST_HARNESS_MAIN + diff --git a/tools/testing/selftests/net/af_unix/unix_listen.c b/tools/testing/selftests/net/af_unix/unix_listen.c new file mode 100644 index 0000000000000..416fa3e5bfe9b --- /dev/null +++ b/tools/testing/selftests/net/af_unix/unix_listen.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Tests for the state checks in AF_UNIX listen(). + * + * The central case is a regression test: listen() on a bound socket that + * is already connected (i.e. not in TCP_CLOSE or TCP_LISTEN state) must + * fail with EINVAL. A prior change accidentally let it return success + * without doing anything, because a helper called in between reset the + * error code to 0. The neighbouring checks (unbound, already listening) + * are tested too so they cannot silently regress the same way. + * + * Every case runs for both listenable socket types (SOCK_STREAM and + * SOCK_SEQPACKET) and both pathname and abstract addresses. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include + +#include +#include + +#include "kselftest_harness.h" + +#define SK_NAME "unix_listen_sk" +#define SRV_NAME "unix_listen_srv" + +FIXTURE(unix_listen) +{ + int sk; /* socket under test */ + int server; /* a listening peer, when a test needs one */ + struct sockaddr_un addr, srv_addr; + socklen_t addrlen, srv_addrlen; +}; + +FIXTURE_VARIANT(unix_listen) +{ + int type; + int abstract; +}; + +FIXTURE_VARIANT_ADD(unix_listen, stream_pathname) +{ + .type = SOCK_STREAM, + .abstract = 0, +}; + +FIXTURE_VARIANT_ADD(unix_listen, stream_abstract) +{ + .type = SOCK_STREAM, + .abstract = 1, +}; + +FIXTURE_VARIANT_ADD(unix_listen, seqpacket_pathname) +{ + .type = SOCK_SEQPACKET, + .abstract = 0, +}; + +FIXTURE_VARIANT_ADD(unix_listen, seqpacket_abstract) +{ + .type = SOCK_SEQPACKET, + .abstract = 1, +}; + +/* Fill @addr with a pathname or abstract address named @name. */ +static socklen_t unix_set_addr(struct sockaddr_un *addr, const char *name, + int abstract) +{ + size_t len = strlen(name); + + memset(addr, 0, sizeof(*addr)); + addr->sun_family = AF_UNIX; + /* An abstract address leads with a NUL and has no filesystem entry. */ + memcpy(addr->sun_path + (abstract ? 1 : 0), name, len); + + return offsetof(struct sockaddr_un, sun_path) + len + 1; +} + +FIXTURE_SETUP(unix_listen) +{ + self->sk = -1; + self->server = -1; + self->addrlen = unix_set_addr(&self->addr, SK_NAME, variant->abstract); + self->srv_addrlen = unix_set_addr(&self->srv_addr, SRV_NAME, + variant->abstract); +} + +FIXTURE_TEARDOWN(unix_listen) +{ + if (self->sk >= 0) + close(self->sk); + if (self->server >= 0) + close(self->server); + + /* Pathname sockets leave a filesystem entry behind; abstract ones do not. */ + if (!variant->abstract) { + remove(SK_NAME); + remove(SRV_NAME); + } +} + +/* A bound socket in TCP_CLOSE is the normal, allowed case. */ +TEST_F(unix_listen, bound_is_ok) +{ + int err; + + self->sk = socket(AF_UNIX, variant->type, 0); + ASSERT_LE(0, self->sk); + + err = bind(self->sk, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(0, err); + + err = listen(self->sk, 8); + EXPECT_EQ(0, err); +} + +/* Listening again on an already-listening socket (TCP_LISTEN) is allowed. */ +TEST_F(unix_listen, relisten_is_ok) +{ + int err; + + self->sk = socket(AF_UNIX, variant->type, 0); + ASSERT_LE(0, self->sk); + + err = bind(self->sk, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(0, err); + + err = listen(self->sk, 8); + ASSERT_EQ(0, err); + + err = listen(self->sk, 16); + EXPECT_EQ(0, err); +} + +/* listen() on an unbound socket fails: there is nothing to listen on. */ +TEST_F(unix_listen, unbound_is_einval) +{ + int err; + + self->sk = socket(AF_UNIX, variant->type, 0); + ASSERT_LE(0, self->sk); + + err = listen(self->sk, 8); + EXPECT_EQ(-1, err); + EXPECT_EQ(EINVAL, errno); +} + +/* + * The regression: a bound socket that has already been connected is not in + * TCP_CLOSE or TCP_LISTEN, so listen() must reject it with EINVAL rather + * than quietly succeeding. + */ +TEST_F(unix_listen, connected_is_einval) +{ + int err; + + self->server = socket(AF_UNIX, variant->type, 0); + ASSERT_LE(0, self->server); + + err = bind(self->server, (struct sockaddr *)&self->srv_addr, + self->srv_addrlen); + ASSERT_EQ(0, err); + + err = listen(self->server, 8); + ASSERT_EQ(0, err); + + self->sk = socket(AF_UNIX, variant->type, 0); + ASSERT_LE(0, self->sk); + + /* Bind first so the unbound check does not mask the state check. */ + err = bind(self->sk, (struct sockaddr *)&self->addr, self->addrlen); + ASSERT_EQ(0, err); + + err = connect(self->sk, (struct sockaddr *)&self->srv_addr, + self->srv_addrlen); + ASSERT_EQ(0, err); + + err = listen(self->sk, 8); + EXPECT_EQ(-1, err); + EXPECT_EQ(EINVAL, errno); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/net/fib_nexthops.sh b/tools/testing/selftests/net/fib_nexthops.sh index 2b0a90581e2f1..7e450fb9d65b0 100755 --- a/tools/testing/selftests/net/fib_nexthops.sh +++ b/tools/testing/selftests/net/fib_nexthops.sh @@ -522,6 +522,20 @@ ipv6_fdb_grp_fcnal() run_cmd "$BRIDGE fdb add 02:02:00:00:00:14 dev vx10 nhid 61 self" log_test $? 255 "Fdb mac add with nexthop" + # fdb entries with a nexthop group cannot be aged out + run_cmd "$BRIDGE fdb add 02:02:00:00:00:15 dev vx10 nhid 102 self static" + log_test $? 0 "Fdb mac add with nexthop group and static state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:16 dev vx10 nhid 102 self dynamic" + log_test $? 255 "Fdb mac add with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:17 dev vx10 nhid 102 self" + run_cmd "$BRIDGE fdb replace 02:02:00:00:00:17 dev vx10 dst 2001:db8:91::11 self dynamic" + log_test $? 255 "Fdb mac replace with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb append 02:02:00:00:00:17 dev vx10 dst 2001:db8:91::11 self dynamic" + log_test $? 255 "Fdb mac append with nexthop group and dynamic state" + run_cmd "$IP -6 ro add 2001:db8:101::1/128 nhid 66" log_test $? 2 "Route add with fdb nexthop" @@ -622,6 +636,20 @@ ipv4_fdb_grp_fcnal() run_cmd "$BRIDGE fdb add 02:02:00:00:00:14 dev vx10 nhid 12 self" log_test $? 255 "Fdb mac add with nexthop" + # fdb entries with a nexthop group cannot be aged out + run_cmd "$BRIDGE fdb add 02:02:00:00:00:15 dev vx10 nhid 102 self static" + log_test $? 0 "Fdb mac add with nexthop group and static state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:16 dev vx10 nhid 102 self dynamic" + log_test $? 255 "Fdb mac add with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb add 02:02:00:00:00:17 dev vx10 nhid 102 self" + run_cmd "$BRIDGE fdb replace 02:02:00:00:00:17 dev vx10 dst 10.0.0.3 self dynamic" + log_test $? 255 "Fdb mac replace with nexthop group and dynamic state" + + run_cmd "$BRIDGE fdb append 02:02:00:00:00:17 dev vx10 dst 10.0.0.3 self dynamic" + log_test $? 255 "Fdb mac append with nexthop group and dynamic state" + run_cmd "$IP ro add 172.16.0.0/22 nhid 16" log_test $? 2 "Route add with fdb nexthop" diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index fc7e22b503d3b..b1628e6131a6a 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -379,6 +379,9 @@ static int sock_connect_mptcp(const char * const remoteaddr, hints.ai_family = pf; + /* Keep the resolved address alive for the whole execution: it is + * used again when reconnecting, and will be released at exit time. + */ xgetaddrinfo(remoteaddr, port, &hints, &addr); for (a = addr; a; a = a->ai_next) { sock = socket(a->ai_family, a->ai_socktype, proto); @@ -422,7 +425,6 @@ static int sock_connect_mptcp(const char * const remoteaddr, } } - freeaddrinfo(addr); if (sock != -1) SOCK_TEST_TCPULP(sock, proto); return sock; diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh index 11b603742d8f4..c3e158bf347df 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_join.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh @@ -565,7 +565,7 @@ check_transfer() mv "$tmpfile" "$out" tmpfile="" fi - cmp -l "$in" "$out" | while read -r i a b; do + while read -r i a b; do local sum=$((0${a} + 0${b})) if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then fail_test "$what does not match (in, out):" @@ -576,7 +576,7 @@ check_transfer() else print_info "$what has inverted byte at ${i}" fi - done + done < <(cmp -l "$in" "$out") return 0 } diff --git a/tools/testing/selftests/net/openvswitch/config b/tools/testing/selftests/net/openvswitch/config new file mode 100644 index 0000000000000..c659749cd086c --- /dev/null +++ b/tools/testing/selftests/net/openvswitch/config @@ -0,0 +1,16 @@ +CONFIG_GENEVE=m +CONFIG_INET_DIAG=y +CONFIG_IPV6=y +CONFIG_NETFILTER=y +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_DEMUX=m +CONFIG_NF_CONNTRACK=m +CONFIG_NF_CONNTRACK_OVS=y +CONFIG_OPENVSWITCH=m +CONFIG_OPENVSWITCH_GENEVE=m +CONFIG_OPENVSWITCH_GRE=m +CONFIG_OPENVSWITCH_VXLAN=m +CONFIG_PSAMPLE=m +CONFIG_VETH=y +CONFIG_VLAN_8021Q=y +CONFIG_VXLAN=m diff --git a/tools/testing/selftests/net/ovpn/config b/tools/testing/selftests/net/ovpn/config index 42699740936db..c0a21a43018a2 100644 --- a/tools/testing/selftests/net/ovpn/config +++ b/tools/testing/selftests/net/ovpn/config @@ -4,7 +4,9 @@ CONFIG_CRYPTO_CHACHA20POLY1305=y CONFIG_CRYPTO_GCM=y CONFIG_DST_CACHE=y CONFIG_INET=y +CONFIG_IPV6=y CONFIG_NET=y CONFIG_NET_UDP_TUNNEL=y CONFIG_OVPN=m CONFIG_STREAM_PARSER=y +CONFIG_VETH=y diff --git a/tools/testing/selftests/net/ovpn/ovpn-cli.c b/tools/testing/selftests/net/ovpn/ovpn-cli.c index 0a5226196a2e7..44418118f6ea7 100644 --- a/tools/testing/selftests/net/ovpn/ovpn-cli.c +++ b/tools/testing/selftests/net/ovpn/ovpn-cli.c @@ -1746,7 +1746,7 @@ static int ovpn_parse_remote(struct ovpn_ctx *ovpn, const char *host, const char *service, const char *vpnip) { int ret; - struct addrinfo *result; + struct addrinfo *result = NULL; struct addrinfo hints = { .ai_family = ovpn->sa_family, .ai_socktype = SOCK_DGRAM, @@ -1770,6 +1770,8 @@ static int ovpn_parse_remote(struct ovpn_ctx *ovpn, const char *host, } memcpy(&ovpn->remote, result->ai_addr, result->ai_addrlen); + freeaddrinfo(result); + result = NULL; } if (vpnip) { diff --git a/tools/testing/selftests/net/ovpn/settings b/tools/testing/selftests/net/ovpn/settings new file mode 100644 index 0000000000000..ba4d85f74cd6b --- /dev/null +++ b/tools/testing/selftests/net/ovpn/settings @@ -0,0 +1 @@ +timeout=90 diff --git a/tools/testing/selftests/net/pmtu.sh b/tools/testing/selftests/net/pmtu.sh index a3323c21f0014..c7cd271714ef7 100755 --- a/tools/testing/selftests/net/pmtu.sh +++ b/tools/testing/selftests/net/pmtu.sh @@ -1457,7 +1457,7 @@ test_pmtu_ipvX_over_bridged_vxlanY_or_geneveY_exception() { mtu "${ns_b}" ${type}_b $((${ll_mtu} + 1000)) run_cmd ${ns_c} ${ping} -q -M want -i 0.1 -c 10 -s $((${ll_mtu} + 500)) ${dst} || return 1 - run_cmd ${ns_a} ${ping} -q -M want -i 0.1 -w 1 -s $((${ll_mtu} + 500)) ${dst} || return 1 + run_cmd ${ns_a} ${ping} -q -M want -i 0.1 -c 10 -s $((${ll_mtu} + 500)) ${dst} || return 1 # Check that exceptions were created pmtu="$(route_get_dst_pmtu_from_exception "${ns_c}" ${dst})" diff --git a/tools/testing/selftests/net/test_vxlan_mdb.sh b/tools/testing/selftests/net/test_vxlan_mdb.sh index 58da5de99ac45..f9600aabd4a29 100755 --- a/tools/testing/selftests/net/test_vxlan_mdb.sh +++ b/tools/testing/selftests/net/test_vxlan_mdb.sh @@ -685,6 +685,9 @@ star_g_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $grp dst $vtep_ip src_vni 10010" log_test $? 255 "Invalid source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent filter_mode exclude source_list $all_zeros_grp dst $vtep_ip src_vni 10010" + log_test $? 255 "All-zeros source in source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp permanent source_list $src1 dst $vtep_ip src_vni 10010" log_test $? 255 "Source list without filter mode" } @@ -784,6 +787,9 @@ sg_common() run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $grp permanent dst $vtep_ip src_vni 10010" log_test $? 255 "(S, G) with an invalid source list" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $grp src $all_zeros_grp permanent dst $vtep_ip src_vni 10010" + log_test $? 255 "(S, G) with an all-zeros source" + run_cmd "bridge -n $ns1 mdb add dev vx0 port vx0 grp $all_zeros_grp src $src permanent dst $vtep_ip src_vni 10010" log_test $? 255 "All-zeros group with source" } diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index 6dc9b5da738c8..d9f8a70e90897 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -1759,6 +1759,63 @@ TEST_F(tls, recv_efault) } #define TLS_RECORD_TYPE_HANDSHAKE 0x16 + +TEST_F(tls_basic, recvmsg_nopad_retry_iov) +{ + char payload[32]; + char first_iov[sizeof(payload)]; + char later_iov[sizeof(payload) * 2]; + char expected_later_iov[sizeof(later_iov)]; + char cbuf[CMSG_SPACE(sizeof(char))]; + struct tls_crypto_info_keys tls13; + struct iovec iov[] = { + { .iov_base = first_iov, .iov_len = sizeof(first_iov) }, + { .iov_base = later_iov, .iov_len = sizeof(later_iov) }, + }; + struct msghdr msg = { + .msg_iov = iov, + .msg_iovlen = ARRAY_SIZE(iov), + .msg_control = cbuf, + .msg_controllen = sizeof(cbuf), + }; + int one = 1; + int ret; + int i; + + if (self->notls) + SKIP(return, "no TLS support"); + + tls_crypto_info_init(TLS_1_3_VERSION, TLS_CIPHER_AES_GCM_128, + &tls13, 0); + + ret = setsockopt(self->fd, SOL_TLS, TLS_TX, &tls13, tls13.len); + ASSERT_EQ(ret, 0); + + ret = setsockopt(self->cfd, SOL_TLS, TLS_RX, &tls13, tls13.len); + ASSERT_EQ(ret, 0); + + ret = setsockopt(self->cfd, SOL_TLS, TLS_RX_EXPECT_NO_PAD, + &one, sizeof(one)); + ASSERT_EQ(ret, 0); + + for (i = 0; i < sizeof(payload); i++) + payload[i] = 0x40 + i; + memset(first_iov, 0xa5, sizeof(first_iov)); + memset(later_iov, 0x5a, sizeof(later_iov)); + memset(expected_later_iov, 0x5a, sizeof(expected_later_iov)); + + /* A control record forces optimistic TLS 1.3 RX to retry. */ + ret = tls_send_cmsg(self->fd, TLS_RECORD_TYPE_HANDSHAKE, + payload, sizeof(payload), 0); + ASSERT_EQ(ret, sizeof(payload)); + + ret = recvmsg(self->cfd, &msg, 0); + ASSERT_EQ(ret, sizeof(payload)); + EXPECT_EQ(memcmp(first_iov, payload, sizeof(payload)), 0); + EXPECT_EQ(memcmp(later_iov, expected_later_iov, + sizeof(later_iov)), 0); +} + /* key_update, length 1, update_not_requested */ static const char key_update_msg[] = "\x18\x00\x00\x01\x00"; static void tls_send_keyupdate(struct __test_metadata *_metadata, int fd) diff --git a/tools/testing/selftests/powerpc/tm/tm.h b/tools/testing/selftests/powerpc/tm/tm.h index c03c6e7788767..6024ce4ba6ffd 100644 --- a/tools/testing/selftests/powerpc/tm/tm.h +++ b/tools/testing/selftests/powerpc/tm/tm.h @@ -105,8 +105,12 @@ static inline bool failure_is_nesting(void) static inline int tcheck(void) { long cr; - asm volatile ("tcheck 0" : "=r"(cr) : : "cr0"); - return (cr >> 28) & 4; + asm volatile("tcheck 0;" + "mfcr %0;" + : "=r"(cr) + : + : "cr0"); + return (cr >> 28) & 0xf; } static inline bool tcheck_doomed(void) diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index a546475db5502..7de87221bf27e 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -340,7 +340,8 @@ static bool query_addr_at(int maps_fd, void *addr, static inline bool split_vma(FIXTURE_DATA(proc_maps_race) *self) { - return mmap(self->mod_info->addr, self->page_size, self->mod_info->prot | PROT_EXEC, + /* PROT_NONE differs from both readable neighbors. */ + return mmap(self->mod_info->addr, self->page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED; } diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c index 52500597f9514..f0fff6991d461 100644 --- a/tools/testing/selftests/proc/proc-pidns.c +++ b/tools/testing/selftests/proc/proc-pidns.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include diff --git a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h index d3133587d9968..5a29d6bec51f4 100644 --- a/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h +++ b/tools/testing/selftests/rseq/rseq-x86-thread-pointer.h @@ -8,13 +8,11 @@ #ifndef _RSEQ_X86_THREAD_POINTER #define _RSEQ_X86_THREAD_POINTER -#include - #ifdef __cplusplus extern "C" { #endif -#if __GNUC_PREREQ (11, 1) +#if __GNUC__ > 11 || (__GNUC__ == 11 && __GNUC_MINOR__ >= 1) static inline void *rseq_thread_pointer(void) { return __builtin_thread_pointer(); diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c index 6f4c3f5a1c5d9..7ef9de7b27ebf 100644 --- a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c @@ -14,18 +14,16 @@ s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* - * If we dispatch to a bogus DSQ that will fall back to the - * builtin global DSQ, we fail gracefully. - */ - scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } - - return prev_cpu; + /* + * If we dispatch to a bogus DSQ that will fall back to the + * builtin global DSQ, we fail gracefully. + */ + scx_bpf_dsq_insert_vtime(p, 0xcafef00d, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); + return cpu; } void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c index e4a55027778fd..82dca4cdc0a6c 100644 --- a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c +++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c @@ -14,15 +14,14 @@ s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0); + if (cpu < 0) + cpu = prev_cpu; - if (cpu >= 0) { - /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ - scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, - p->scx.dsq_vtime, 0); - return cpu; - } + /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */ + scx_bpf_dsq_insert_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, + p->scx.dsq_vtime, 0); - return prev_cpu; + return cpu; } void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei) diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c index b987611789d16..01b17092d5c8f 100644 --- a/tools/testing/selftests/sched_ext/exit.c +++ b/tools/testing/selftests/sched_ext/exit.c @@ -31,6 +31,7 @@ static enum scx_test_status run(void *ctx) continue; skel = exit__open(); + SCX_FAIL_IF(!skel, "Failed to open"); SCX_ENUM_INIT(skel); skel->rodata->exit_point = tc; SCX_FAIL_IF(exit__load(skel), "Failed to load skel"); diff --git a/tools/testing/selftests/sched_ext/numa.bpf.c b/tools/testing/selftests/sched_ext/numa.bpf.c index a79d86ed54a1b..0ebb6170331bf 100644 --- a/tools/testing/selftests/sched_ext/numa.bpf.c +++ b/tools/testing/selftests/sched_ext/numa.bpf.c @@ -34,7 +34,8 @@ static bool is_cpu_idle(s32 cpu, int node) s32 BPF_STRUCT_OPS(numa_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags) { - int node = __COMPAT_scx_bpf_cpu_node(scx_bpf_task_cpu(p)); + s32 task_cpu = scx_bpf_task_cpu(p); + int node = __COMPAT_scx_bpf_cpu_node(task_cpu); s32 cpu; /* @@ -48,6 +49,16 @@ s32 BPF_STRUCT_OPS(numa_select_cpu, cpu = __COMPAT_scx_bpf_pick_any_cpu_node(p->cpus_ptr, node, __COMPAT_SCX_PICK_IDLE_IN_NODE); + /* + * @task_cpu may be outside of p->cpus_ptr if @p's affinity + * changed while it was sleeping. This means it's possible for + * p->cpus_ptr to not include any CPUs from @node. + * If we failed to find a cpu in @node, check if @task_cpu + * is outside of p->cpus_ptr and just return @prev_cpu if it is. + */ + if (cpu < 0 && !bpf_cpumask_test_cpu(task_cpu, p->cpus_ptr)) + return prev_cpu; + if (is_cpu_idle(cpu, node)) scx_bpf_error("CPU %d should be marked as busy", cpu); diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c index 05974820ca69d..1129ec2aaddc7 100644 --- a/tools/testing/selftests/sched_ext/prog_run.c +++ b/tools/testing/selftests/sched_ext/prog_run.c @@ -28,7 +28,8 @@ static enum scx_test_status setup(void **ctx) static enum scx_test_status run(void *ctx) { struct prog_run *skel = ctx; - struct bpf_link *link; + struct bpf_link *link = NULL; + enum scx_test_status status = SCX_TEST_PASS; int prog_fd, err = 0; prog_fd = bpf_program__fd(skel->progs.prog_run_syscall); @@ -42,23 +43,40 @@ static enum scx_test_status run(void *ctx) link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops); if (!link) { SCX_ERR("Failed to attach scheduler"); - close(prog_fd); - return SCX_TEST_FAIL; + status = SCX_TEST_FAIL; + goto out; } err = bpf_prog_test_run_opts(prog_fd, &topts); - SCX_EQ(err, 0); + if (err) { + SCX_ERR("BPF_PROG_RUN failed (%d)", err); + status = SCX_TEST_FAIL; + goto out; + } /* Assumes uei.kind is written last */ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE)) sched_yield(); - SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF)); - SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef); + if (skel->data->uei.kind != EXIT_KIND(SCX_EXIT_UNREG_BPF)) { + SCX_ERR("Unexpected exit kind: %llu", + (unsigned long long)skel->data->uei.kind); + status = SCX_TEST_FAIL; + goto out; + } + if (skel->data->uei.exit_code != 0xdeadbeef) { + SCX_ERR("Unexpected exit code: %lld", + (long long)skel->data->uei.exit_code); + status = SCX_TEST_FAIL; + goto out; + } + +out: close(prog_fd); - bpf_link__destroy(link); + if (link) + bpf_link__destroy(link); - return SCX_TEST_PASS; + return status; } static void cleanup(void *ctx) diff --git a/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c index bfcb96cd4954b..aad8de51723e9 100644 --- a/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c +++ b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c @@ -66,12 +66,14 @@ void BPF_STRUCT_OPS(select_cpu_vtime_running, struct task_struct *p) void BPF_STRUCT_OPS(select_cpu_vtime_stopping, struct task_struct *p, bool runnable) { - p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight; + u64 delta = scale_by_task_weight_inverse(p, SCX_SLICE_DFL - p->scx.slice); + + scx_bpf_task_set_dsq_vtime(p, p->scx.dsq_vtime + delta); } void BPF_STRUCT_OPS(select_cpu_vtime_enable, struct task_struct *p) { - p->scx.dsq_vtime = vtime_now; + scx_bpf_task_set_dsq_vtime(p, vtime_now); } s32 BPF_STRUCT_OPS_SLEEPABLE(select_cpu_vtime_init) diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 874f17763536b..19c9afa0be719 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -5172,7 +5172,8 @@ FIXTURE_SETUP(UPROBE) ASSERT_GE(bit, 0); } - offset = get_uprobe_offset(variant->uretprobe ? probed_uretprobe : probed_uprobe); + offset = get_uprobe_offset(variant->uretprobe ? (void *)probed_uretprobe + : (void *)probed_uprobe); ASSERT_GE(offset, 0); if (variant->uretprobe) diff --git a/tools/testing/selftests/timers/leap-a-day.c b/tools/testing/selftests/timers/leap-a-day.c index 04004a7c0934f..e677e31a22110 100644 --- a/tools/testing/selftests/timers/leap-a-day.c +++ b/tools/testing/selftests/timers/leap-a-day.c @@ -9,16 +9,19 @@ * kernel's leap-second behavior, as well as how well applications * handle the leap-second discontinuity. * - * Usage: leap-a-day [-s] [-i ] + * Usage: leap-a-day [-w] [-i ] [-t] * * Options: - * -s: Each iteration, set the date to 10 seconds before midnight GMT. - * This speeds up the number of leapsecond transitions tested, - * but because it calls settimeofday frequently, advancing the - * time by 24 hours every ~16 seconds, it may cause application - * disruption. + * -w: Only set the leap-second flag and wait for the leap second + * each iteration, instead of advancing the time. By default the + * date is set to 10 seconds before midnight GMT, which speeds up + * the number of leapsecond transitions tested, but because it + * calls settimeofday frequently, advancing the time by 24 hours + * every ~16 seconds, it may cause application disruption. * - * -i: Number of iterations to run (default: infinite) + * -i: Number of iterations to run (-1 = infinite, default: 10) + * + * -t: Print TAI time. * * Other notes: Disabling NTP prior to running this is advised, as the two * may conflict in their commands to the kernel. @@ -186,7 +189,7 @@ int main(int argc, char **argv) int opt; /* Process arguments */ - while ((opt = getopt(argc, argv, "sti:")) != -1) { + while ((opt = getopt(argc, argv, "wti:")) != -1) { switch (opt) { case 'w': printf("Only setting leap-flag, not changing time. It could take up to a day for leap to trigger.\n"); diff --git a/tools/testing/selftests/vDSO/vgetrandom-chacha.S b/tools/testing/selftests/vDSO/vgetrandom-chacha.S index a4a82e1c28a90..8c3cbf4dfd6a1 100644 --- a/tools/testing/selftests/vDSO/vgetrandom-chacha.S +++ b/tools/testing/selftests/vDSO/vgetrandom-chacha.S @@ -14,7 +14,7 @@ #elif defined(__riscv) && __riscv_xlen == 64 #include "../../../../arch/riscv/kernel/vdso/vgetrandom-chacha.S" #elif defined(__s390x__) -#include "../../../../arch/s390/kernel/vdso64/vgetrandom-chacha.S" +#include "../../../../arch/s390/kernel/vdso/vgetrandom-chacha.S" #elif defined(__x86_64__) #include "../../../../arch/x86/entry/vdso/vgetrandom-chacha.S" #endif diff --git a/tools/testing/selftests/zram/zram_lib.sh b/tools/testing/selftests/zram/zram_lib.sh index 21ec1966de76c..0d44d83888f9d 100755 --- a/tools/testing/selftests/zram/zram_lib.sh +++ b/tools/testing/selftests/zram/zram_lib.sh @@ -37,7 +37,7 @@ kernel_gte() if [ $kernel_major -gt $major ]; then return 0 - elif [[ $kernel_major -eq $major && $kernel_minor -ge $minor ]]; then + elif [ $kernel_major -eq $major ] && [ $kernel_minor -ge $minor ]; then return 0 fi diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index bec634ac68cbb..c16115a06b55b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -5630,7 +5630,7 @@ static int kvm_offline_cpu(unsigned int cpu) return 0; } -static void kvm_shutdown(void) +static void kvm_shutdown(void *data) { /* * Disable hardware virtualization and set kvm_rebooting to indicate @@ -5648,7 +5648,7 @@ static void kvm_shutdown(void) on_each_cpu(kvm_disable_virtualization_cpu, NULL, 1); } -static int kvm_suspend(void) +static int kvm_suspend(void *data) { /* * Secondary CPUs and CPU hotplug are disabled across the suspend/resume @@ -5665,7 +5665,7 @@ static int kvm_suspend(void) return 0; } -static void kvm_resume(void) +static void kvm_resume(void *data) { lockdep_assert_not_held(&kvm_usage_lock); lockdep_assert_irqs_disabled(); @@ -5673,12 +5673,16 @@ static void kvm_resume(void) WARN_ON_ONCE(kvm_enable_virtualization_cpu()); } -static struct syscore_ops kvm_syscore_ops = { +static const struct syscore_ops kvm_syscore_ops = { .suspend = kvm_suspend, .resume = kvm_resume, .shutdown = kvm_shutdown, }; +static struct syscore kvm_syscore = { + .ops = &kvm_syscore_ops, +}; + int kvm_enable_virtualization(void) { int r; @@ -5695,7 +5699,7 @@ int kvm_enable_virtualization(void) if (r) goto err_cpuhp; - register_syscore_ops(&kvm_syscore_ops); + register_syscore(&kvm_syscore); /* * Undo virtualization enabling and bail if the system is going down. @@ -5717,7 +5721,7 @@ int kvm_enable_virtualization(void) return 0; err_rebooting: - unregister_syscore_ops(&kvm_syscore_ops); + unregister_syscore(&kvm_syscore); cpuhp_remove_state(CPUHP_AP_KVM_ONLINE); err_cpuhp: kvm_arch_disable_virtualization(); @@ -5733,7 +5737,7 @@ void kvm_disable_virtualization(void) if (--kvm_usage_count) return; - unregister_syscore_ops(&kvm_syscore_ops); + unregister_syscore(&kvm_syscore); cpuhp_remove_state(CPUHP_AP_KVM_ONLINE); kvm_arch_disable_virtualization(); }